diff --git a/.gitattributes b/.gitattributes index eeee266b079387f4c3da4ded7c6ea997591ddb88..ff36415f832168132661732ecbb41466e7d966c4 100644 --- a/.gitattributes +++ b/.gitattributes @@ -2,11 +2,5 @@ *.wav filter=lfs diff=lfs merge=lfs -text assets/user.png filter=lfs diff=lfs merge=lfs -text assets/assistant.png filter=lfs diff=lfs merge=lfs -text -speakers/闫雨婷_prompt.wav filter=lfs diff=lfs merge=lfs -text -speakers/闫雨婷RAP_prompt.wav filter=lfs diff=lfs merge=lfs -text -speakers/闫雨婷VOCAL_prompt.wav filter=lfs diff=lfs merge=lfs -text -speakers/Tingting_prompt.wav filter=lfs diff=lfs merge=lfs -text -speakers/TingtingRAP_prompt.wav filter=lfs diff=lfs merge=lfs -text -speakers/TingtingVOCAL_prompt.wav filter=lfs diff=lfs merge=lfs -text assets/yuewen.jpeg filter=lfs diff=lfs merge=lfs -text assets/request_rap_zh.wav filter=lfs diff=lfs merge=lfs -text diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 194246b02193f10593987aa6cd656f1cbde85a27..0000000000000000000000000000000000000000 --- a/Dockerfile +++ /dev/null @@ -1,46 +0,0 @@ -FROM nvidia/cuda:12.1.0-base-ubuntu20.04 - -ENV TZ=Asia/Shanghai -RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime \ - && echo $TZ > /etc/timezone - -RUN apt-get update \ - && apt-get install -y build-essential \ - && apt-get install -y wget \ - && apt-get install -y software-properties-common curl zip unzip git-lfs awscli libssl-dev openssh-server vim \ - && apt-get install -y net-tools iputils-ping iproute2 - -RUN apt-get install --reinstall ca-certificates && update-ca-certificates - -RUN add-apt-repository -y 'ppa:deadsnakes/ppa' && apt update -RUN apt install python3.10 python3.10-dev python3.10-distutils python3.10-venv -y \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* - -RUN wget -qO- https://bootstrap.pypa.io/get-pip.py | python3.10 -RUN ln -s /usr/bin/python3.10 /usr/bin/python -RUN pip uninstall -y Pillow && pip install pillow - -# https://huggingface.co/docs/hub/spaces-sdks-docker#permissions -RUN useradd -m -u 1000 user -USER user - -ENV HOME="/home/user" \ - PATH="/home/user/.local/bin:${PATH}" - -RUN python3.10 -m pip install pipx -RUN pipx install poetry - -RUN poetry --version || { echo 'Poetry installation check failed' ; exit 1; } - -WORKDIR /workspace - -COPY --chown=user requirements.txt . -RUN pip install -r requirements.txt - -COPY --chown=user . . - -RUN pip install gradio -RUN pip install openai -RUN chmod +x start_app.sh -CMD ["./start_app.sh", "/tmp/hf_model"] diff --git a/app.py b/app.py index c4b88347cd021c8d72f855cfbddb5640b1112c44..0bb296f84804557cd194fa675fa4f7b58558153f 100644 --- a/app.py +++ b/app.py @@ -4,15 +4,13 @@ import gradio as gr import time from pathlib import Path -from tokenizer import StepAudioTokenizer -from tts import StepAudioTTS -from yuewen_api import call_audiochat, call_asr +from yuewen_api import call_audiochat, call_asr, call_tts CACHE_DIR = "/tmp/gradio/" -CACHE_CLEAN_AGE = 864000 +CACHE_CLEAN_AGE = 86400 CHINESE_PROMPT_CONTENT = """你是一个为对话而设计的人工智能模型，目前无法连接到互联网。 -当你需要唱歌或说唱时，请以（RAP）开头。当你需要快速说话时，请以（快速）开头。当你需要慢速说话时，请以（慢速）开头。 +当你需要唱歌时，请以（哼唱）开头。当你需要rap或说唱时，请以（RAP）开头。当你需要快速说话时，请以（快速）开头。当你需要慢速说话时，请以（慢速）开头。 现在，你需要倾听用户的语音内容，并以礼貌、简洁、口语化的文本进行回复。你需要尽量用户的语种进行回复。""" ENGLISH_PROMPT_CONTENT = """You are an AI designed for conversation, currently unable to connect to the internet. @@ -89,20 +87,15 @@ def add_message(chatbot, history, mic, text): return chatbot, history, None -def save_tmp_audio(audio, sr): +def get_tmp_audio_path(): import tempfile - import torchaudio - with tempfile.NamedTemporaryFile( - dir=CACHE_DIR, delete=False, suffix=".wav" - ) as temp_audio: - temp_audio_path = temp_audio.name - torchaudio.save(temp_audio_path, audio, sr) + temp_audio = tempfile.NamedTemporaryFile(dir=CACHE_DIR, delete=False, suffix=".mp3") return temp_audio.name -def predict(chatbot, history, tts_model, user_prompt, enable_asr): +def predict(chatbot, history, user_prompt, enable_asr): """Generate a response from the model.""" start_time = time.time() try: @@ -126,8 +119,8 @@ def predict(chatbot, history, tts_model, user_prompt, enable_asr): text = call_audiochat(messages) print(f"predict {text=}") - audio, sr = tts_model(text, "Tingting") - audio_path = save_tmp_audio(audio, sr) + audio_path = get_tmp_audio_path() + call_tts(text, audio_path) print(f"save_tmp_audio {audio_path=}") chatbot.append({"role": "assistant", "content": text}) chatbot.append({"role": "assistant", "content": {"path": audio_path}}) @@ -142,17 +135,15 @@ def predict(chatbot, history, tts_model, user_prompt, enable_asr): return chatbot, history -def _launch_demo(args, tts_model): - with gr.Blocks(delete_cache=(86400, CACHE_CLEAN_AGE)) as demo: +def _launch_demo(args): + with gr.Blocks(delete_cache=(3600, CACHE_CLEAN_AGE)) as demo: # 保存 chat 历史，不需要每次再重新拼格式 history = gr.State([]) gr.Markdown("""
Step Audio Chat
""") gr.Markdown( """This preview demonstrates core functionalities. To unlock the cormplete real-time voice conversation system with end-to-end encryption and advanced features, download the [Yuewen APP](https://m.yuewen.cn/call-app) with the link or via QR Code.""" ) - with gr.Accordion( - label="Click to view the QR code ", open=False - ): + with gr.Accordion(label="Click to view the QR code ", open=False): gr.Image( value="assets/yuewen.jpeg", interactive=False, @@ -161,7 +152,8 @@ def _launch_demo(args, tts_model): show_fullscreen_button=False, ) with gr.Accordion( - label="The performance of English prompts is not as stable as that of Chinese prompts. You can click here to change sys prompt.", open=False + label="The performance of English prompts is not as stable as that of Chinese prompts. You can click here to change sys prompt.", + open=False, ): prompt_choice = gr.Radio( choices=list(PROMPT_TEMPLATE.keys()), @@ -222,7 +214,7 @@ def _launch_demo(args, tts_model): print(f"update_examples error") return chatbot, history else: - chatbot, history = predict(chatbot, history, tts_model, user_prompt, enable_asr) + chatbot, history = predict(chatbot, history, user_prompt, enable_asr) print(f"update_examples done") return chatbot, history @@ -230,7 +222,13 @@ def _launch_demo(args, tts_model): gr.Examples( fn=update_examples, examples=CHAT_EXAMPLES, - inputs=[example_comment, example_text, example_audio, user_prompt, enable_asr], + inputs=[ + example_comment, + example_text, + example_audio, + user_prompt, + enable_asr, + ], outputs=[chatbot, history], run_on_click=True, ) @@ -241,7 +239,7 @@ def _launch_demo(args, tts_model): gr.Warning(error) return chatbot, history, None, None else: - chatbot, history = predict(chatbot, history, tts_model, user_prompt, enable_asr) + chatbot, history = predict(chatbot, history, user_prompt, enable_asr) return chatbot, history, None, None gen_btn.click( @@ -266,7 +264,7 @@ def _launch_demo(args, tts_model): while history and history[-1]["role"] == "assistant": print(f"discard {history[-1]}") history.pop() - return predict(chatbot, history, tts_model, user_prompt, enable_asr) + return predict(chatbot, history, user_prompt, enable_asr) regen_btn.click( regenerate, @@ -295,10 +293,4 @@ if __name__ == "__main__": "--server-name", type=str, default="0.0.0.0", help="Demo server name." ) args = parser.parse_args() - tokenizer = StepAudioTokenizer( - os.path.join(args.model_path, "Step-Audio-Tokenizer") - ) - tts_model = StepAudioTTS( - os.path.join(args.model_path, "Step-Audio-TTS-3B"), tokenizer - ) - _launch_demo(args, tts_model) + _launch_demo(args) diff --git a/cosyvoice/__init__.py b/cosyvoice/__init__.py deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/cosyvoice/cli/__init__.py b/cosyvoice/cli/__init__.py deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/cosyvoice/cli/cosyvoice.py b/cosyvoice/cli/cosyvoice.py deleted file mode 100644 index b8c0b4ba2a45c8d5bdeca5279fdaaac53d89b544..0000000000000000000000000000000000000000 --- a/cosyvoice/cli/cosyvoice.py +++ /dev/null @@ -1,68 +0,0 @@ -# Copyright (c) 2024 Alibaba Inc (authors: Xiang Lyu) -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import os -import uuid -import time -from tqdm import tqdm -import torch -import torchaudio -from hyperpyyaml import load_hyperpyyaml -from cosyvoice.cli.frontend import CosyVoiceFrontEnd -from cosyvoice.cli.model import CosyVoiceModel - - -class CosyVoice: - - def __init__( - self, - model_dir, - ): - self.model_dir = model_dir - with open("{}/cosyvoice.yaml".format(model_dir), "r") as f: - configs = load_hyperpyyaml(f) - self.frontend = CosyVoiceFrontEnd( - configs["feat_extractor"], - "{}/campplus.onnx".format(model_dir), - "{}/speech_tokenizer_v1.onnx".format(model_dir), - ) - self.model = CosyVoiceModel(configs["flow"], configs["hift"]) - self.model.load( - "{}/flow.pt".format(model_dir), - "{}/hift.pt".format(model_dir), - ) - self.model.flow = self.model.flow.to(torch.bfloat16) - del configs - - def token_to_wav_offline( - self, - speech_token, - speech_feat, - speech_feat_len, - prompt_token, - prompt_token_len, - embedding, - ): - tts_mel = self.model.flow.inference( - token=speech_token.to(self.model.device), - token_len=torch.tensor([speech_token.size(1)], dtype=torch.int32).to( - self.model.device - ), - prompt_token=prompt_token.to(self.model.device), - prompt_token_len=prompt_token_len.to(self.model.device), - prompt_feat=speech_feat.to(self.model.device), - prompt_feat_len=speech_feat_len.to(self.model.device), - embedding=embedding.to(self.model.device), - ) - tts_speech = self.model.hift.inference(mel=tts_mel.float())[0].cpu() - return tts_speech diff --git a/cosyvoice/cli/frontend.py b/cosyvoice/cli/frontend.py deleted file mode 100644 index d968551816eaa87f32547222470afdf2c64f4d68..0000000000000000000000000000000000000000 --- a/cosyvoice/cli/frontend.py +++ /dev/null @@ -1,106 +0,0 @@ -# Copyright (c) 2024 Alibaba Inc (authors: Xiang Lyu) -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import onnxruntime -import torch -import numpy as np -import whisper -from typing import Callable -import torchaudio.compliance.kaldi as kaldi - - -class CosyVoiceFrontEnd: - - def __init__( - self, - feat_extractor: Callable, - campplus_model: str, - speech_tokenizer_model: str, - ): - self.feat_extractor = feat_extractor - self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") - option = onnxruntime.SessionOptions() - option.graph_optimization_level = ( - onnxruntime.GraphOptimizationLevel.ORT_ENABLE_ALL - ) - option.intra_op_num_threads = 1 - self.campplus_session = onnxruntime.InferenceSession( - campplus_model, sess_options=option, providers=["CPUExecutionProvider"] - ) - self.speech_tokenizer_session = onnxruntime.InferenceSession( - speech_tokenizer_model, - sess_options=option, - providers=[ - ( - "CUDAExecutionProvider" - if torch.cuda.is_available() - else "CPUExecutionProvider" - ) - ], - ) - - def _extract_speech_token(self, speech): - assert ( - speech.shape[1] / 16000 <= 30 - ), "do not support extract speech token for audio longer than 30s" - feat = whisper.log_mel_spectrogram(speech, n_mels=128) - speech_token = ( - self.speech_tokenizer_session.run( - None, - { - self.speech_tokenizer_session.get_inputs()[0] - .name: feat.detach() - .cpu() - .numpy(), - self.speech_tokenizer_session.get_inputs()[1].name: np.array( - [feat.shape[2]], dtype=np.int32 - ), - }, - )[0] - .flatten() - .tolist() - ) - speech_token = torch.tensor([speech_token], dtype=torch.int32).to(self.device) - speech_token_len = torch.tensor([speech_token.shape[1]], dtype=torch.int32).to( - self.device - ) - return speech_token, speech_token_len - - def _extract_spk_embedding(self, speech): - feat = kaldi.fbank(speech, num_mel_bins=80, dither=0, sample_frequency=16000) - feat = feat - feat.mean(dim=0, keepdim=True) - embedding = ( - self.campplus_session.run( - None, - { - self.campplus_session.get_inputs()[0] - .name: feat.unsqueeze(dim=0) - .cpu() - .numpy() - }, - )[0] - .flatten() - .tolist() - ) - embedding = torch.tensor([embedding]).to(self.device) - return embedding - - def _extract_speech_feat(self, speech): - speech_feat = ( - self.feat_extractor(speech).squeeze(dim=0).transpose(0, 1).to(self.device) - ) - speech_feat = speech_feat.unsqueeze(dim=0) - speech_feat_len = torch.tensor([speech_feat.shape[1]], dtype=torch.int32).to( - self.device - ) - return speech_feat, speech_feat_len diff --git a/cosyvoice/cli/model.py b/cosyvoice/cli/model.py deleted file mode 100644 index b284d9e869b06d9d083431446295b56d2023076d..0000000000000000000000000000000000000000 --- a/cosyvoice/cli/model.py +++ /dev/null @@ -1,32 +0,0 @@ -# Copyright (c) 2024 Alibaba Inc (authors: Xiang Lyu) -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import torch - - -class CosyVoiceModel: - - def __init__( - self, - flow: torch.nn.Module, - hift: torch.nn.Module, - ): - self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") - self.flow = flow - self.hift = hift - - def load(self, flow_model, hift_model): - self.flow.load_state_dict(torch.load(flow_model, map_location=self.device)) - self.flow.to(self.device).eval() - self.hift.load_state_dict(torch.load(hift_model, map_location=self.device)) - self.hift.to(self.device).eval() diff --git a/cosyvoice/flow/decoder.py b/cosyvoice/flow/decoder.py deleted file mode 100644 index 9fa6ba6dab254c8d8cff5f32003fbb202491a422..0000000000000000000000000000000000000000 --- a/cosyvoice/flow/decoder.py +++ /dev/null @@ -1,238 +0,0 @@ -# Copyright (c) 2024 Alibaba Inc (authors: Xiang Lyu, Zhihao Du) -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import torch -import torch.nn as nn -from einops import pack, rearrange, repeat -from cosyvoice.matcha.decoder import ( - SinusoidalPosEmb, - Block1D, - ResnetBlock1D, - Downsample1D, - TimestepEmbedding, - Upsample1D, -) -from cosyvoice.matcha.transformer import BasicTransformerBlock - - -class ConditionalDecoder(nn.Module): - def __init__( - self, - in_channels, - out_channels, - channels=(256, 256), - dropout=0.05, - attention_head_dim=64, - n_blocks=1, - num_mid_blocks=2, - num_heads=4, - act_fn="snake", - ): - """ - This decoder requires an input with the same shape of the target. So, if your text content - is shorter or longer than the outputs, please re-sampling it before feeding to the decoder. - """ - super().__init__() - channels = tuple(channels) - self.in_channels = in_channels - self.out_channels = out_channels - - self.time_embeddings = SinusoidalPosEmb(in_channels) - time_embed_dim = channels[0] * 4 - self.time_mlp = TimestepEmbedding( - in_channels=in_channels, - time_embed_dim=time_embed_dim, - act_fn="silu", - ) - self.down_blocks = nn.ModuleList([]) - self.mid_blocks = nn.ModuleList([]) - self.up_blocks = nn.ModuleList([]) - - output_channel = in_channels - for i in range(len(channels)): # pylint: disable=consider-using-enumerate - input_channel = output_channel - output_channel = channels[i] - is_last = i == len(channels) - 1 - resnet = ResnetBlock1D( - dim=input_channel, dim_out=output_channel, time_emb_dim=time_embed_dim - ) - transformer_blocks = nn.ModuleList( - [ - BasicTransformerBlock( - dim=output_channel, - num_attention_heads=num_heads, - attention_head_dim=attention_head_dim, - dropout=dropout, - activation_fn=act_fn, - ) - for _ in range(n_blocks) - ] - ) - downsample = ( - Downsample1D(output_channel) - if not is_last - else nn.Conv1d(output_channel, output_channel, 3, padding=1) - ) - self.down_blocks.append( - nn.ModuleList([resnet, transformer_blocks, downsample]) - ) - - for _ in range(num_mid_blocks): - input_channel = channels[-1] - out_channels = channels[-1] - resnet = ResnetBlock1D( - dim=input_channel, dim_out=output_channel, time_emb_dim=time_embed_dim - ) - - transformer_blocks = nn.ModuleList( - [ - BasicTransformerBlock( - dim=output_channel, - num_attention_heads=num_heads, - attention_head_dim=attention_head_dim, - dropout=dropout, - activation_fn=act_fn, - ) - for _ in range(n_blocks) - ] - ) - - self.mid_blocks.append(nn.ModuleList([resnet, transformer_blocks])) - - channels = channels[::-1] + (channels[0],) - for i in range(len(channels) - 1): - input_channel = channels[i] * 2 - output_channel = channels[i + 1] - is_last = i == len(channels) - 2 - resnet = ResnetBlock1D( - dim=input_channel, - dim_out=output_channel, - time_emb_dim=time_embed_dim, - ) - transformer_blocks = nn.ModuleList( - [ - BasicTransformerBlock( - dim=output_channel, - num_attention_heads=num_heads, - attention_head_dim=attention_head_dim, - dropout=dropout, - activation_fn=act_fn, - ) - for _ in range(n_blocks) - ] - ) - upsample = ( - Upsample1D(output_channel, use_conv_transpose=True) - if not is_last - else nn.Conv1d(output_channel, output_channel, 3, padding=1) - ) - self.up_blocks.append(nn.ModuleList([resnet, transformer_blocks, upsample])) - self.final_block = Block1D(channels[-1], channels[-1]) - self.final_proj = nn.Conv1d(channels[-1], self.out_channels, 1) - self.initialize_weights() - - def initialize_weights(self): - for m in self.modules(): - if isinstance(m, nn.Conv1d): - nn.init.kaiming_normal_(m.weight, nonlinearity="relu") - if m.bias is not None: - nn.init.constant_(m.bias, 0) - elif isinstance(m, nn.GroupNorm): - nn.init.constant_(m.weight, 1) - nn.init.constant_(m.bias, 0) - elif isinstance(m, nn.Linear): - nn.init.kaiming_normal_(m.weight, nonlinearity="relu") - if m.bias is not None: - nn.init.constant_(m.bias, 0) - - def forward(self, x, mask, mu, t, spks=None, cond=None): - """Forward pass of the UNet1DConditional model. - - Args: - x (torch.Tensor): shape (batch_size, in_channels, time) - mask (_type_): shape (batch_size, 1, time) - t (_type_): shape (batch_size) - spks (_type_, optional): shape: (batch_size, condition_channels). Defaults to None. - cond (_type_, optional): placeholder for future use. Defaults to None. - - Raises: - ValueError: _description_ - ValueError: _description_ - - Returns: - _type_: _description_ - """ - - t = self.time_embeddings(t).to(t.dtype) - t = self.time_mlp(t) - - x = pack([x, mu], "b * t")[0] - - if spks is not None: - spks = repeat(spks, "b c -> b c t", t=x.shape[-1]) - x = pack([x, spks], "b * t")[0] - if cond is not None: - x = pack([x, cond], "b * t")[0] - - hiddens = [] - masks = [mask] - for resnet, transformer_blocks, downsample in self.down_blocks: - mask_down = masks[-1] - x = resnet( - x.to(torch.bfloat16), mask_down.to(torch.bfloat16), t.to(torch.bfloat16) - ) - x = rearrange(x, "b c t -> b t c").contiguous() - # attn_mask = torch.matmul(mask_down.transpose(1, 2).contiguous(), mask_down) - for transformer_block in transformer_blocks: - x = transformer_block( - hidden_states=x, - # attention_mask=attn_mask, - timestep=t, - ) - x = rearrange(x, "b t c -> b c t").contiguous() - hiddens.append(x) # Save hidden states for skip connections - x = downsample(x * mask_down) - masks.append(mask_down[:, :, ::2]) - masks = masks[:-1] - mask_mid = masks[-1] - - for resnet, transformer_blocks in self.mid_blocks: - x = resnet(x, mask_mid, t) - x = rearrange(x, "b c t -> b t c").contiguous() - # attn_mask = torch.matmul(mask_mid.transpose(1, 2).contiguous(), mask_mid) - for transformer_block in transformer_blocks: - x = transformer_block( - hidden_states=x, - # attention_mask=attn_mask, - timestep=t, - ) - x = rearrange(x, "b t c -> b c t").contiguous() - - for resnet, transformer_blocks, upsample in self.up_blocks: - mask_up = masks.pop() - skip = hiddens.pop() - x = pack([x[:, :, : skip.shape[-1]], skip], "b * t")[0] - x = resnet(x, mask_up, t) - x = rearrange(x, "b c t -> b t c").contiguous() - # attn_mask = torch.matmul(mask_up.transpose(1, 2).contiguous(), mask_up) - for transformer_block in transformer_blocks: - x = transformer_block( - hidden_states=x, - # attention_mask=attn_mask, - timestep=t, - ) - x = rearrange(x, "b t c -> b c t").contiguous() - x = upsample(x * mask_up) - x = self.final_block(x, mask_up) - output = self.final_proj(x * mask_up) - return output * mask diff --git a/cosyvoice/flow/flow.py b/cosyvoice/flow/flow.py deleted file mode 100644 index ab9a8124ef86d542965aa4d2a1b8f97e6bd27e43..0000000000000000000000000000000000000000 --- a/cosyvoice/flow/flow.py +++ /dev/null @@ -1,196 +0,0 @@ -# Copyright (c) 2024 Alibaba Inc (authors: Xiang Lyu, Zhihao Du) -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import logging -import random -from typing import Dict, Optional -import torch -import torch.nn as nn -from torch.nn import functional as F -from omegaconf import DictConfig -from cosyvoice.utils.mask import make_pad_mask -import time - - -class MaskedDiffWithXvec(torch.nn.Module): - def __init__( - self, - input_size: int = 512, - output_size: int = 80, - spk_embed_dim: int = 192, - output_type: str = "mel", - vocab_size: int = 4096, - input_frame_rate: int = 50, - only_mask_loss: bool = True, - encoder: torch.nn.Module = None, - length_regulator: torch.nn.Module = None, - decoder: torch.nn.Module = None, - decoder_conf: Dict = { - "in_channels": 240, - "out_channel": 80, - "spk_emb_dim": 80, - "n_spks": 1, - "cfm_params": DictConfig( - { - "sigma_min": 1e-06, - "solver": "euler", - "t_scheduler": "cosine", - "training_cfg_rate": 0.2, - "inference_cfg_rate": 0.7, - "reg_loss_type": "l1", - } - ), - "decoder_params": { - "channels": [256, 256], - "dropout": 0.0, - "attention_head_dim": 64, - "n_blocks": 4, - "num_mid_blocks": 12, - "num_heads": 8, - "act_fn": "gelu", - }, - }, - mel_feat_conf: Dict = { - "n_fft": 1024, - "num_mels": 80, - "sampling_rate": 22050, - "hop_size": 256, - "win_size": 1024, - "fmin": 0, - "fmax": 8000, - }, - ): - super().__init__() - self.input_size = input_size - self.output_size = output_size - self.decoder_conf = decoder_conf - self.mel_feat_conf = mel_feat_conf - self.vocab_size = vocab_size - self.output_type = output_type - self.input_frame_rate = input_frame_rate - logging.info(f"input frame rate={self.input_frame_rate}") - self.input_embedding = nn.Embedding(vocab_size, input_size) - self.spk_embed_affine_layer = torch.nn.Linear(spk_embed_dim, output_size) - self.encoder = encoder - self.encoder_proj = torch.nn.Linear(self.encoder.output_size(), output_size) - self.decoder = decoder - self.length_regulator = length_regulator - self.only_mask_loss = only_mask_loss - - def forward( - self, - batch: dict, - device: torch.device, - ) -> Dict[str, Optional[torch.Tensor]]: - token = batch["speech_token"].to(device) - token_len = batch["speech_token_len"].to(device) - feat = batch["speech_feat"].to(device) - feat_len = batch["speech_feat_len"].to(device) - embedding = batch["embedding"].to(device) - - # xvec projection - embedding = F.normalize(embedding, dim=1) - embedding = self.spk_embed_affine_layer(embedding) - - # concat text and prompt_text - mask = (~make_pad_mask(token_len)).float().unsqueeze(-1).to(device) - token = self.input_embedding(torch.clamp(token, min=0)) * mask - - # text encode - h, h_lengths = self.encoder(token, token_len) - h = self.encoder_proj(h) - h, h_lengths = self.length_regulator(h, feat_len) - - # get conditions - conds = torch.zeros(feat.shape, device=token.device) - for i, j in enumerate(feat_len): - if random.random() < 0.5: - continue - index = random.randint(0, int(0.3 * j)) - conds[i, :index] = feat[i, :index] - conds = conds.transpose(1, 2) - - mask = (~make_pad_mask(feat_len)).to(h) - feat = F.interpolate( - feat.unsqueeze(dim=1), size=h.shape[1:], mode="nearest" - ).squeeze(dim=1) - loss, _ = self.decoder.compute_loss( - feat.transpose(1, 2).contiguous(), - mask.unsqueeze(1), - h.transpose(1, 2).contiguous(), - embedding, - cond=conds, - ) - return {"loss": loss} - - @torch.inference_mode() - def inference( - self, - token, - token_len, - prompt_token, - prompt_token_len, - prompt_feat, - prompt_feat_len, - embedding, - ): - assert token.shape[0] == 1 - # xvec projection - embedding = F.normalize(embedding, dim=1) - embedding = self.spk_embed_affine_layer(embedding) - - # concat text and prompt_text - token_len1, token_len2 = prompt_token.shape[1], token.shape[1] - # text encode - token, token_len = ( - torch.concat([prompt_token, token], dim=1), - prompt_token_len + token_len, - ) - token = self.input_embedding(torch.clamp(token, min=0)) - h, _ = self.encoder.inference(token, token_len) - h = self.encoder_proj(h) - mel_len1, mel_len2 = prompt_feat.shape[1], int( - token_len2 - / self.input_frame_rate - * self.mel_feat_conf["sampling_rate"] - / self.mel_feat_conf["hop_size"] - ) - - h, _ = self.length_regulator.inference( - h[:, :token_len1], - h[:, token_len1:], - mel_len1, - mel_len2, - ) - - # get conditions - conds = torch.zeros( - [1, mel_len1 + mel_len2, self.output_size], device=token.device - ) - conds[:, :mel_len1] = prompt_feat - conds = conds.transpose(1, 2) - - # mask = (~make_pad_mask(torch.tensor([mel_len1 + mel_len2]))).to(h) - mask = torch.ones( - [1, mel_len1 + mel_len2], device=h.device, dtype=torch.bfloat16 - ) - feat = self.decoder( - mu=h.transpose(1, 2).contiguous(), - mask=mask.unsqueeze(1), - spks=embedding, - cond=conds, - n_timesteps=10, - ) - feat = feat[:, :, mel_len1:] - assert feat.shape[2] == mel_len2 - return feat diff --git a/cosyvoice/flow/flow_matching.py b/cosyvoice/flow/flow_matching.py deleted file mode 100644 index d29673fcb4f21e43e65359df93ca119f707ca27b..0000000000000000000000000000000000000000 --- a/cosyvoice/flow/flow_matching.py +++ /dev/null @@ -1,315 +0,0 @@ -# Copyright (c) 2024 Alibaba Inc (authors: Xiang Lyu, Zhihao Du) -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import time -import torch -import torch.nn.functional as F -from cosyvoice.matcha.flow_matching import BASECFM - - -class ConditionalCFM(BASECFM): - def __init__( - self, - in_channels, - cfm_params, - n_spks=1, - spk_emb_dim=64, - estimator: torch.nn.Module = None, - ): - super().__init__( - n_feats=in_channels, - cfm_params=cfm_params, - n_spks=n_spks, - spk_emb_dim=spk_emb_dim, - ) - self.t_scheduler = cfm_params.t_scheduler - self.training_cfg_rate = cfm_params.training_cfg_rate - self.inference_cfg_rate = cfm_params.inference_cfg_rate - in_channels = in_channels + (spk_emb_dim if n_spks > 0 else 0) - # Just change the architecture of the estimator here - self.estimator = estimator - self.inference_graphs = {} - self.inference_buffers = {} - # self.capture_inference() - - @torch.inference_mode() - def forward( - self, - mu, - mask, - n_timesteps, - temperature=1.0, - spks=None, - cond=None, - ): - """Forward diffusion - - Args: - mu (torch.Tensor): output of encoder - shape: (batch_size, n_feats, mel_timesteps) - mask (torch.Tensor): output_mask - shape: (batch_size, 1, mel_timesteps) - n_timesteps (int): number of diffusion steps - temperature (float, optional): temperature for scaling noise. Defaults to 1.0. - spks (torch.Tensor, optional): speaker ids. Defaults to None. - shape: (batch_size, spk_emb_dim) - cond: Not used but kept for future purposes - - Returns: - sample: generated mel-spectrogram - shape: (batch_size, n_feats, mel_timesteps) - """ - z = torch.randn_like(mu) * temperature - t_span = torch.linspace(0, 1, n_timesteps + 1, device=mu.device, dtype=mu.dtype) - if self.t_scheduler == "cosine": - t_span = 1 - torch.cos(t_span * 0.5 * torch.pi) - return self.solve_euler( - z, t_span=t_span, mu=mu, mask=mask, spks=spks, cond=cond - ) - - @torch.inference_mode() - def capture_inference(self, seq_len_to_capture=list(range(128, 512, 8))): - start_time = time.time() - print( - f"capture_inference for ConditionalCFM solve euler, seq_len_to_capture: {seq_len_to_capture}" - ) - for seq_len in seq_len_to_capture: - static_z = torch.randn( - 1, 80, seq_len, device=torch.device("cuda"), dtype=torch.bfloat16 - ) - static_t_span = torch.linspace( - 0, 1, 11, device=torch.device("cuda"), dtype=torch.bfloat16 - ) # only capture at 10 steps - static_mu = torch.randn( - 1, 80, seq_len, device=torch.device("cuda"), dtype=torch.bfloat16 - ) - static_mask = torch.ones( - 1, 1, seq_len, device=torch.device("cuda"), dtype=torch.bfloat16 - ) - static_spks = torch.randn( - 1, 80, device=torch.device("cuda"), dtype=torch.bfloat16 - ) - static_cond = torch.randn( - 1, 80, seq_len, device=torch.device("cuda"), dtype=torch.float32 - ) - static_out = torch.randn( - 1, 80, seq_len, device=torch.device("cuda"), dtype=torch.bfloat16 - ) - - self._solve_euler_impl( - static_z, - t_span=static_t_span, - mu=static_mu, - mask=static_mask, - spks=static_spks, - cond=static_cond, - ) - torch.cuda.synchronize() - - g = torch.cuda.CUDAGraph() - with torch.cuda.graph(g): - static_out = self._solve_euler_impl( - static_z, - t_span=static_t_span, - mu=static_mu, - mask=static_mask, - spks=static_spks, - cond=static_cond, - ) - - self.inference_buffers[seq_len] = { - "z": static_z, - "t_span": static_t_span, - "mu": static_mu, - "mask": static_mask, - "spks": static_spks, - "cond": static_cond, - "out": static_out, - } - self.inference_graphs[seq_len] = g - end_time = time.time() - print( - f"capture_inference for ConditionalCFM solve euler, time elapsed: {end_time - start_time}" - ) - - def solve_euler(self, x, t_span, mu, mask, spks, cond): - if hasattr(self, "inference_graphs") and len(self.inference_graphs) > 0: - curr_seq_len = x.shape[2] - - available_lengths = sorted(list(self.inference_graphs.keys())) - - if curr_seq_len <= max(available_lengths): - target_len = min(available_lengths, key=lambda x: abs(x - curr_seq_len)) - if target_len == curr_seq_len: - padded_x = x - padded_mu = mu - padded_mask = mask - if cond is not None: - padded_cond = cond - else: - padded_x = torch.randn( - (x.shape[0], x.shape[1], target_len), - dtype=x.dtype, - device=x.device, - ) - padded_x[:, :, :curr_seq_len] = x - - padded_mu = torch.randn( - (mu.shape[0], mu.shape[1], target_len), - dtype=mu.dtype, - device=mu.device, - ) - padded_mu[:, :, :curr_seq_len] = mu - - # FIXME(ys): uses zeros and maskgroupnorm - padded_mask = torch.ones( - (mask.shape[0], mask.shape[1], target_len), - dtype=mask.dtype, - device=mask.device, - ) - - if cond is not None: - padded_cond = torch.randn( - (cond.shape[0], cond.shape[1], target_len), - dtype=cond.dtype, - device=cond.device, - ) - padded_cond[:, :, :curr_seq_len] = cond - - buffer = self.inference_buffers[target_len] - buffer["z"].copy_(padded_x) - buffer["t_span"].copy_(t_span) - buffer["mu"].copy_(padded_mu) - buffer["mask"].copy_(padded_mask) - buffer["spks"].copy_(spks) - if cond is not None: - buffer["cond"].copy_(padded_cond) - - self.inference_graphs[target_len].replay() - - output = buffer["out"][:, :, :curr_seq_len] - return output - - return self._solve_euler_impl(x, t_span, mu, mask, spks, cond) - - def _solve_euler_impl(self, x, t_span, mu, mask, spks, cond): - """ - Fixed euler solver for ODEs. - Args: - x (torch.Tensor): random noise - t_span (torch.Tensor): n_timesteps interpolated - shape: (n_timesteps + 1,) - mu (torch.Tensor): output of encoder - shape: (batch_size, n_feats, mel_timesteps) - mask (torch.Tensor): output_mask - shape: (batch_size, 1, mel_timesteps) - spks (torch.Tensor, optional): speaker ids. Defaults to None. - shape: (batch_size, spk_emb_dim) - cond: Not used but kept for future purposes - """ - t, _, dt = t_span[0], t_span[-1], t_span[1] - t_span[0] - t = t.unsqueeze(dim=0) - - # I am storing this because I can later plot it by putting a debugger here and saving it to a file - # Or in future might add like a return_all_steps flag - sol = [] - - for step in range(1, len(t_span)): - if self.inference_cfg_rate > 0: - x_double = torch.cat([x, x], dim=0) - mask_double = torch.cat([mask, mask], dim=0) - mu_double = torch.cat([mu, torch.zeros_like(mu)], dim=0) - t_double = torch.cat([t, t], dim=0) - spks_double = ( - torch.cat([spks, torch.zeros_like(spks)], dim=0) - if spks is not None - else None - ) - cond_double = torch.cat([cond, torch.zeros_like(cond)], dim=0) - - dphi_dt_double = self.forward_estimator( - x_double, mask_double, mu_double, t_double, spks_double, cond_double - ) - - dphi_dt, cfg_dphi_dt = torch.chunk(dphi_dt_double, 2, dim=0) - dphi_dt = ( - 1.0 + self.inference_cfg_rate - ) * dphi_dt - self.inference_cfg_rate * cfg_dphi_dt - else: - dphi_dt = self.forward_estimator(x, mask, mu, t, spks, cond) - - x = x + dt * dphi_dt - t = t + dt - sol.append(x) - if step < len(t_span) - 1: - dt = t_span[step + 1] - t - - return sol[-1] - - def forward_estimator(self, x, mask, mu, t, spks, cond): - if isinstance(self.estimator, torch.nn.Module): - return self.estimator.forward(x, mask, mu, t, spks, cond) - else: - ort_inputs = { - "x": x.cpu().numpy(), - "mask": mask.cpu().numpy(), - "mu": mu.cpu().numpy(), - "t": t.cpu().numpy(), - "spks": spks.cpu().numpy(), - "cond": cond.cpu().numpy(), - } - output = self.estimator.run(None, ort_inputs)[0] - return torch.tensor(output, dtype=x.dtype, device=x.device) - - def compute_loss(self, x1, mask, mu, spks=None, cond=None): - """Computes diffusion loss - - Args: - x1 (torch.Tensor): Target - shape: (batch_size, n_feats, mel_timesteps) - mask (torch.Tensor): target mask - shape: (batch_size, 1, mel_timesteps) - mu (torch.Tensor): output of encoder - shape: (batch_size, n_feats, mel_timesteps) - spks (torch.Tensor, optional): speaker embedding. Defaults to None. - shape: (batch_size, spk_emb_dim) - - Returns: - loss: conditional flow matching loss - y: conditional flow - shape: (batch_size, n_feats, mel_timesteps) - """ - b, _, t = mu.shape - - # random timestep - t = torch.rand([b, 1, 1], device=mu.device, dtype=mu.dtype) - if self.t_scheduler == "cosine": - t = 1 - torch.cos(t * 0.5 * torch.pi) - # sample noise p(x_0) - z = torch.randn_like(x1) - - y = (1 - (1 - self.sigma_min) * t) * z + t * x1 - u = x1 - (1 - self.sigma_min) * z - - # during training, we randomly drop condition to trade off mode coverage and sample fidelity - if self.training_cfg_rate > 0: - cfg_mask = torch.rand(b, device=x1.device) > self.training_cfg_rate - mu = mu * cfg_mask.view(-1, 1, 1) - spks = spks * cfg_mask.view(-1, 1) - cond = cond * cfg_mask.view(-1, 1, 1) - - pred = self.estimator(y, mask, mu, t.squeeze(), spks, cond) - loss = F.mse_loss(pred * mask, u * mask, reduction="sum") / ( - torch.sum(mask) * u.shape[1] - ) - return loss, y diff --git a/cosyvoice/flow/length_regulator.py b/cosyvoice/flow/length_regulator.py deleted file mode 100644 index 19c07a0163327492332d1e5de12fa36bf99921b9..0000000000000000000000000000000000000000 --- a/cosyvoice/flow/length_regulator.py +++ /dev/null @@ -1,65 +0,0 @@ -# Copyright (c) 2024 Alibaba Inc (authors: Xiang Lyu, Zhihao Du) -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from typing import Tuple -import torch.nn as nn -import torch -from torch.nn import functional as F -from cosyvoice.utils.mask import make_pad_mask - - -class InterpolateRegulator(nn.Module): - def __init__( - self, - channels: int, - sampling_ratios: Tuple, - out_channels: int = None, - groups: int = 1, - ): - super().__init__() - self.sampling_ratios = sampling_ratios - out_channels = out_channels or channels - model = nn.ModuleList([]) - if len(sampling_ratios) > 0: - for _ in sampling_ratios: - module = nn.Conv1d(channels, channels, 3, 1, 1) - norm = nn.GroupNorm(groups, channels) - act = nn.Mish() - model.extend([module, norm, act]) - model.append(nn.Conv1d(channels, out_channels, 1, 1)) - self.model = nn.Sequential(*model) - - def forward(self, x, ylens=None): - # x in (B, T, D) - mask = (~make_pad_mask(ylens)).to(x).unsqueeze(-1) - x = F.interpolate( - x.transpose(1, 2).contiguous(), size=ylens.max(), mode="linear" - ) - out = self.model(x).transpose(1, 2).contiguous() - olens = ylens - return out * mask, olens - - def inference(self, x1, x2, mel_len1, mel_len2): - # x in (B, T, D) - x2 = F.interpolate( - x2.transpose(1, 2).contiguous(), size=mel_len2, mode="linear" - ) - if x1.shape[1] != 0: - x1 = F.interpolate( - x1.transpose(1, 2).contiguous(), size=mel_len1, mode="linear" - ) - x = torch.concat([x1, x2], dim=2) - else: - x = x2 - out = self.model(x).transpose(1, 2).contiguous() - return out, mel_len1 + mel_len2 diff --git a/cosyvoice/hifigan/f0_predictor.py b/cosyvoice/hifigan/f0_predictor.py deleted file mode 100644 index f465fcbecf9606e8627571aa4592289da50fec26..0000000000000000000000000000000000000000 --- a/cosyvoice/hifigan/f0_predictor.py +++ /dev/null @@ -1,55 +0,0 @@ -# Copyright (c) 2024 Alibaba Inc (authors: Xiang Lyu, Kai Hu) -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import torch -import torch.nn as nn -from torch.nn.utils import weight_norm - - -class ConvRNNF0Predictor(nn.Module): - def __init__( - self, num_class: int = 1, in_channels: int = 80, cond_channels: int = 512 - ): - super().__init__() - - self.num_class = num_class - self.condnet = nn.Sequential( - weight_norm( - nn.Conv1d(in_channels, cond_channels, kernel_size=3, padding=1) - ), - nn.ELU(), - weight_norm( - nn.Conv1d(cond_channels, cond_channels, kernel_size=3, padding=1) - ), - nn.ELU(), - weight_norm( - nn.Conv1d(cond_channels, cond_channels, kernel_size=3, padding=1) - ), - nn.ELU(), - weight_norm( - nn.Conv1d(cond_channels, cond_channels, kernel_size=3, padding=1) - ), - nn.ELU(), - weight_norm( - nn.Conv1d(cond_channels, cond_channels, kernel_size=3, padding=1) - ), - nn.ELU(), - ) - self.classifier = nn.Linear( - in_features=cond_channels, out_features=self.num_class - ) - - def forward(self, x: torch.Tensor) -> torch.Tensor: - x = self.condnet(x) - x = x.transpose(1, 2) - return torch.abs(self.classifier(x).squeeze(-1)) diff --git a/cosyvoice/hifigan/generator.py b/cosyvoice/hifigan/generator.py deleted file mode 100644 index 4d02c03da59775503b85998127b295289d284287..0000000000000000000000000000000000000000 --- a/cosyvoice/hifigan/generator.py +++ /dev/null @@ -1,566 +0,0 @@ -# Copyright (c) 2024 Alibaba Inc (authors: Xiang Lyu, Kai Hu) -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""HIFI-GAN""" - -import typing as tp -import time -import numpy as np -from scipy.signal import get_window -import torch -import torch.nn as nn -import torch.nn.functional as F -from torch.nn import Conv1d -from torch.nn import ConvTranspose1d -from torch.nn.utils import remove_weight_norm -from torch.nn.utils import weight_norm -from torch.distributions.uniform import Uniform - -from cosyvoice.transformer.activation import Snake -from cosyvoice.utils.common import get_padding -from cosyvoice.utils.common import init_weights - - -"""hifigan based generator implementation. - -This code is modified from https://github.com/jik876/hifi-gan - ,https://github.com/kan-bayashi/ParallelWaveGAN and - https://github.com/NVIDIA/BigVGAN - -""" - - -class ResBlock(torch.nn.Module): - """Residual block module in HiFiGAN/BigVGAN.""" - - def __init__( - self, - channels: int = 512, - kernel_size: int = 3, - dilations: tp.List[int] = [1, 3, 5], - ): - super(ResBlock, self).__init__() - self.convs1 = nn.ModuleList() - self.convs2 = nn.ModuleList() - - for dilation in dilations: - self.convs1.append( - weight_norm( - Conv1d( - channels, - channels, - kernel_size, - 1, - dilation=dilation, - padding=get_padding(kernel_size, dilation), - ) - ) - ) - self.convs2.append( - weight_norm( - Conv1d( - channels, - channels, - kernel_size, - 1, - dilation=1, - padding=get_padding(kernel_size, 1), - ) - ) - ) - self.convs1.apply(init_weights) - self.convs2.apply(init_weights) - self.activations1 = nn.ModuleList( - [Snake(channels, alpha_logscale=False) for _ in range(len(self.convs1))] - ) - self.activations2 = nn.ModuleList( - [Snake(channels, alpha_logscale=False) for _ in range(len(self.convs2))] - ) - - def forward(self, x: torch.Tensor) -> torch.Tensor: - for idx in range(len(self.convs1)): - xt = self.activations1[idx](x) - xt = self.convs1[idx](xt) - xt = self.activations2[idx](xt) - xt = self.convs2[idx](xt) - x = xt + x - return x - - def remove_weight_norm(self): - for idx in range(len(self.convs1)): - remove_weight_norm(self.convs1[idx]) - remove_weight_norm(self.convs2[idx]) - - -class SineGen(torch.nn.Module): - """Definition of sine generator - SineGen(samp_rate, harmonic_num = 0, - sine_amp = 0.1, noise_std = 0.003, - voiced_threshold = 0, - flag_for_pulse=False) - samp_rate: sampling rate in Hz - harmonic_num: number of harmonic overtones (default 0) - sine_amp: amplitude of sine-wavefrom (default 0.1) - noise_std: std of Gaussian noise (default 0.003) - voiced_thoreshold: F0 threshold for U/V classification (default 0) - flag_for_pulse: this SinGen is used inside PulseGen (default False) - Note: when flag_for_pulse is True, the first time step of a voiced - segment is always sin(np.pi) or cos(0) - """ - - def __init__( - self, - samp_rate, - harmonic_num=0, - sine_amp=0.1, - noise_std=0.003, - voiced_threshold=0, - ): - super(SineGen, self).__init__() - self.sine_amp = sine_amp - self.noise_std = noise_std - self.harmonic_num = harmonic_num - self.sampling_rate = samp_rate - self.voiced_threshold = voiced_threshold - - def _f02uv(self, f0): - # generate uv signal - uv = (f0 > self.voiced_threshold).type(torch.float32) - return uv - - @torch.no_grad() - def forward(self, f0): - """ - :param f0: [B, 1, sample_len], Hz - :return: [B, 1, sample_len] - """ - - F_mat = torch.zeros((f0.size(0), self.harmonic_num + 1, f0.size(-1))).to( - f0.device - ) - for i in range(self.harmonic_num + 1): - F_mat[:, i : i + 1, :] = f0 * (i + 1) / self.sampling_rate - - theta_mat = 2 * np.pi * (torch.cumsum(F_mat, dim=-1) % 1) - u_dist = Uniform(low=-np.pi, high=np.pi) - phase_vec = u_dist.sample( - sample_shape=(f0.size(0), self.harmonic_num + 1, 1) - ).to(F_mat.device) - phase_vec[:, 0, :] = 0 - - # generate sine waveforms - sine_waves = self.sine_amp * torch.sin(theta_mat + phase_vec) - - # generate uv signal - uv = self._f02uv(f0) - - # noise: for unvoiced should be similar to sine_amp - # std = self.sine_amp/3 -> max value ~ self.sine_amp - # . for voiced regions is self.noise_std - noise_amp = uv * self.noise_std + (1 - uv) * self.sine_amp / 3 - noise = noise_amp * torch.randn_like(sine_waves) - - # first: set the unvoiced part to 0 by uv - # then: additive noise - sine_waves = sine_waves * uv + noise - return sine_waves, uv, noise - - -class SourceModuleHnNSF(torch.nn.Module): - """SourceModule for hn-nsf - SourceModule(sampling_rate, harmonic_num=0, sine_amp=0.1, - add_noise_std=0.003, voiced_threshod=0) - sampling_rate: sampling_rate in Hz - harmonic_num: number of harmonic above F0 (default: 0) - sine_amp: amplitude of sine source signal (default: 0.1) - add_noise_std: std of additive Gaussian noise (default: 0.003) - note that amplitude of noise in unvoiced is decided - by sine_amp - voiced_threshold: threhold to set U/V given F0 (default: 0) - Sine_source, noise_source = SourceModuleHnNSF(F0_sampled) - F0_sampled (batchsize, length, 1) - Sine_source (batchsize, length, 1) - noise_source (batchsize, length 1) - uv (batchsize, length, 1) - """ - - def __init__( - self, - sampling_rate, - upsample_scale, - harmonic_num=0, - sine_amp=0.1, - add_noise_std=0.003, - voiced_threshod=0, - ): - super(SourceModuleHnNSF, self).__init__() - - self.sine_amp = sine_amp - self.noise_std = add_noise_std - - # to produce sine waveforms - self.l_sin_gen = SineGen( - sampling_rate, harmonic_num, sine_amp, add_noise_std, voiced_threshod - ) - - # to merge source harmonics into a single excitation - self.l_linear = torch.nn.Linear(harmonic_num + 1, 1) - self.l_tanh = torch.nn.Tanh() - - def forward(self, x): - """ - Sine_source, noise_source = SourceModuleHnNSF(F0_sampled) - F0_sampled (batchsize, length, 1) - Sine_source (batchsize, length, 1) - noise_source (batchsize, length 1) - """ - # source for harmonic branch - with torch.no_grad(): - sine_wavs, uv, _ = self.l_sin_gen(x.transpose(1, 2)) - sine_wavs = sine_wavs.transpose(1, 2) - uv = uv.transpose(1, 2) - sine_merge = self.l_tanh(self.l_linear(sine_wavs)) - - # source for noise branch, in the same shape as uv - noise = torch.randn_like(uv) * self.sine_amp / 3 - return sine_merge, noise, uv - - -class HiFTGenerator(nn.Module): - """ - HiFTNet Generator: Neural Source Filter + ISTFTNet - https://arxiv.org/abs/2309.09493 - """ - - def __init__( - self, - in_channels: int = 80, - base_channels: int = 512, - nb_harmonics: int = 8, - sampling_rate: int = 22050, - nsf_alpha: float = 0.1, - nsf_sigma: float = 0.003, - nsf_voiced_threshold: float = 10, - upsample_rates: tp.List[int] = [8, 8], - upsample_kernel_sizes: tp.List[int] = [16, 16], - istft_params: tp.Dict[str, int] = {"n_fft": 16, "hop_len": 4}, - resblock_kernel_sizes: tp.List[int] = [3, 7, 11], - resblock_dilation_sizes: tp.List[tp.List[int]] = [ - [1, 3, 5], - [1, 3, 5], - [1, 3, 5], - ], - source_resblock_kernel_sizes: tp.List[int] = [7, 11], - source_resblock_dilation_sizes: tp.List[tp.List[int]] = [[1, 3, 5], [1, 3, 5]], - lrelu_slope: float = 0.1, - audio_limit: float = 0.99, - f0_predictor: torch.nn.Module = None, - ): - super(HiFTGenerator, self).__init__() - - self.out_channels = 1 - self.nb_harmonics = nb_harmonics - self.sampling_rate = sampling_rate - self.istft_params = istft_params - self.lrelu_slope = lrelu_slope - self.audio_limit = audio_limit - - self.num_kernels = len(resblock_kernel_sizes) - self.num_upsamples = len(upsample_rates) - self.upsample_rates = upsample_rates - self.m_source = SourceModuleHnNSF( - sampling_rate=sampling_rate, - upsample_scale=np.prod(upsample_rates) * istft_params["hop_len"], - harmonic_num=nb_harmonics, - sine_amp=nsf_alpha, - add_noise_std=nsf_sigma, - voiced_threshod=nsf_voiced_threshold, - ) - self.f0_upsamp = torch.nn.Upsample( - scale_factor=np.prod(upsample_rates) * istft_params["hop_len"] - ) - - self.conv_pre = weight_norm(Conv1d(in_channels, base_channels, 7, 1, padding=3)) - - # Up - self.ups = nn.ModuleList() - for i, (u, k) in enumerate(zip(upsample_rates, upsample_kernel_sizes)): - self.ups.append( - weight_norm( - ConvTranspose1d( - base_channels // (2**i), - base_channels // (2 ** (i + 1)), - k, - u, - padding=(k - u) // 2, - ) - ) - ) - - # Down - self.source_downs = nn.ModuleList() - self.source_resblocks = nn.ModuleList() - downsample_rates = [1] + upsample_rates[::-1][:-1] - downsample_cum_rates = np.cumprod(downsample_rates) - for i, (u, k, d) in enumerate( - zip( - downsample_cum_rates[::-1], - source_resblock_kernel_sizes, - source_resblock_dilation_sizes, - ) - ): - if u == 1: - self.source_downs.append( - Conv1d( - istft_params["n_fft"] + 2, base_channels // (2 ** (i + 1)), 1, 1 - ) - ) - else: - self.source_downs.append( - Conv1d( - istft_params["n_fft"] + 2, - base_channels // (2 ** (i + 1)), - u * 2, - u, - padding=(u // 2), - ) - ) - - self.source_resblocks.append( - ResBlock(base_channels // (2 ** (i + 1)), k, d) - ) - - self.resblocks = nn.ModuleList() - for i in range(len(self.ups)): - ch = base_channels // (2 ** (i + 1)) - for _, (k, d) in enumerate( - zip(resblock_kernel_sizes, resblock_dilation_sizes) - ): - self.resblocks.append(ResBlock(ch, k, d)) - - self.conv_post = weight_norm( - Conv1d(ch, istft_params["n_fft"] + 2, 7, 1, padding=3) - ) - self.ups.apply(init_weights) - self.conv_post.apply(init_weights) - self.reflection_pad = nn.ReflectionPad1d((1, 0)) - self.stft_window = torch.from_numpy( - get_window("hann", istft_params["n_fft"], fftbins=True).astype(np.float32) - ).cuda() - self.f0_predictor = f0_predictor - self.inference_buffers = {} - self.inference_graphs = {} - - def _f02source(self, f0: torch.Tensor) -> torch.Tensor: - f0 = self.f0_upsamp(f0[:, None]).transpose(1, 2) # bs,n,t - - har_source, _, _ = self.m_source(f0) - return har_source.transpose(1, 2) - - def _stft(self, x): - spec = torch.stft( - x, - self.istft_params["n_fft"], - self.istft_params["hop_len"], - self.istft_params["n_fft"], - window=self.stft_window, - return_complex=True, - ) - spec = torch.view_as_real(spec) # [B, F, TT, 2] - return spec[..., 0], spec[..., 1] - - def _istft(self, magnitude, phase): - magnitude = torch.clip(magnitude, max=1e2) - real = magnitude * torch.cos(phase) - img = magnitude * torch.sin(phase) - inverse_transform = torch.istft( - torch.complex(real, img), - self.istft_params["n_fft"], - self.istft_params["hop_len"], - self.istft_params["n_fft"], - window=self.stft_window, - ) - return inverse_transform - - def forward( - self, x: torch.Tensor, cache_source: torch.Tensor = torch.zeros(1, 1, 0) - ) -> torch.Tensor: - f0 = self.f0_predictor(x) - s = self._f02source(f0) - - # use cache_source to avoid glitch - if cache_source.shape[2] != 0: - s[:, :, : cache_source.shape[2]] = cache_source - - s_stft_real, s_stft_imag = self._stft(s.squeeze(1)) - s_stft = torch.cat([s_stft_real, s_stft_imag], dim=1) - - x = self.conv_pre(x) - for i in range(self.num_upsamples): - x = F.leaky_relu(x, self.lrelu_slope) - x = self.ups[i](x) - - if i == self.num_upsamples - 1: - x = self.reflection_pad(x) - - # fusion - si = self.source_downs[i](s_stft) - si = self.source_resblocks[i](si) - x = x + si - - xs = None - for j in range(self.num_kernels): - if xs is None: - xs = self.resblocks[i * self.num_kernels + j](x) - else: - xs += self.resblocks[i * self.num_kernels + j](x) - x = xs / self.num_kernels - - x = F.leaky_relu(x) - x = self.conv_post(x) - magnitude = torch.exp(x[:, : self.istft_params["n_fft"] // 2 + 1, :]) - phase = torch.sin( - x[:, self.istft_params["n_fft"] // 2 + 1 :, :] - ) # actually, sin is redundancy - - x = self._istft(magnitude, phase) - x = torch.clamp(x, -self.audio_limit, self.audio_limit) - return x, s - - def remove_weight_norm(self): - print("Removing weight norm...") - for l in self.ups: - remove_weight_norm(l) - for l in self.resblocks: - l.remove_weight_norm() - remove_weight_norm(self.conv_pre) - remove_weight_norm(self.conv_post) - self.source_module.remove_weight_norm() - for l in self.source_downs: - remove_weight_norm(l) - for l in self.source_resblocks: - l.remove_weight_norm() - - @torch.inference_mode() - def _inference_impl(self, mel: torch.Tensor, s_stft: torch.Tensor) -> torch.Tensor: - x = self.conv_pre(mel) - for i in range(self.num_upsamples): - x = F.leaky_relu(x, self.lrelu_slope) - x = self.ups[i](x) - - if i == self.num_upsamples - 1: - x = self.reflection_pad(x) - - # fusion - si = self.source_downs[i](s_stft) - si = self.source_resblocks[i](si) - x = x + si - - xs = None - for j in range(self.num_kernels): - if xs is None: - xs = self.resblocks[i * self.num_kernels + j](x) - else: - xs += self.resblocks[i * self.num_kernels + j](x) - x = xs / self.num_kernels - - x = F.leaky_relu(x) - x = self.conv_post(x) - magnitude = torch.exp(x[:, : self.istft_params["n_fft"] // 2 + 1, :]) - phase = torch.sin( - x[:, self.istft_params["n_fft"] // 2 + 1 :, :] - ) # actually, sin is redundancy - # print(f"mel: {mel.shape}, magnitude: {magnitude.shape}, phase: {phase.shape}") - return magnitude, phase - - @torch.inference_mode() - def inference( - self, mel: torch.Tensor, cache_source: torch.Tensor = torch.zeros(1, 1, 0) - ) -> torch.Tensor: - curr_seq_len = mel.shape[2] - f0 = self.f0_predictor(mel) - s = self._f02source(f0) - s_stft_real, s_stft_imag = self._stft(s.squeeze(1)) - s_stft = torch.cat([s_stft_real, s_stft_imag], dim=1) - - target_len = None - for seq_len in sorted(self.inference_buffers.keys()): - if curr_seq_len <= seq_len: - target_len = seq_len - break - - if target_len is not None: - buffer = self.inference_buffers[target_len] - - if curr_seq_len < target_len: - padded_mel = torch.zeros_like(buffer["mel"]) - padded_mel[:, :, :curr_seq_len] = mel - buffer["mel"].copy_(padded_mel) - padded_s_stft = torch.zeros_like(buffer["s_stft"]) - cur_s_stft_len = s_stft.shape[2] - padded_s_stft[:, :, :cur_s_stft_len] = s_stft - buffer["s_stft"].copy_(padded_s_stft) - - else: - buffer["mel"].copy_(mel) - buffer["s_stft"].copy_(s_stft) - cur_s_stft_len = s_stft.shape[2] - - self.inference_graphs[target_len].replay() - - magnitude, phase = ( - buffer["magnitude"][:, :, :cur_s_stft_len], - buffer["phase"][:, :, :cur_s_stft_len], - ) - else: - magnitude, phase = self._inference_impl(mel=mel, s_stft=s_stft) - - x = self._istft(magnitude, phase) - x = torch.clamp(x, -self.audio_limit, self.audio_limit) - return x, s - - @torch.inference_mode() - def capture_inference(self, seq_len_to_capture=[64, 128, 256, 512, 1024]): - start_time = time.time() - print( - f"capture inference for HiFTGenerator with seq_len_to_capture: {seq_len_to_capture}" - ) - for seq_len in seq_len_to_capture: - mel = torch.randn( - 1, 80, seq_len, device=torch.device("cuda"), dtype=torch.float32 - ) - f0 = self.f0_predictor(mel) - s = self._f02source(f0) - s_stft_real, s_stft_imag = self._stft(s.squeeze(1)) - s_stft = torch.cat([s_stft_real, s_stft_imag], dim=1) - - magnitude, phase = self._inference_impl(mel=mel, s_stft=s_stft) - torch.cuda.synchronize() - - g = torch.cuda.CUDAGraph() - with torch.cuda.graph(g): - magnitude, phase = self._inference_impl(mel=mel, s_stft=s_stft) - inference_buffer = { - "mel": mel, - "s_stft": s_stft, - "magnitude": magnitude, - "phase": phase, - } - self.inference_buffers[seq_len] = inference_buffer - self.inference_graphs[seq_len] = g - - end_time = time.time() - print( - f"capture inference for HiFTGenerator with seq_len_to_capture: {seq_len_to_capture} takes {end_time - start_time} seconds" - ) diff --git a/cosyvoice/matcha/audio.py b/cosyvoice/matcha/audio.py deleted file mode 100644 index 0a9b8db2a96e5c06ce04681ca373477dfa69fea2..0000000000000000000000000000000000000000 --- a/cosyvoice/matcha/audio.py +++ /dev/null @@ -1,90 +0,0 @@ -import numpy as np -import torch -import torch.utils.data -from librosa.filters import mel as librosa_mel_fn -from scipy.io.wavfile import read - -MAX_WAV_VALUE = 32768.0 - - -def load_wav(full_path): - sampling_rate, data = read(full_path) - return data, sampling_rate - - -def dynamic_range_compression(x, C=1, clip_val=1e-5): - return np.log(np.clip(x, a_min=clip_val, a_max=None) * C) - - -def dynamic_range_decompression(x, C=1): - return np.exp(x) / C - - -def dynamic_range_compression_torch(x, C=1, clip_val=1e-5): - return torch.log(torch.clamp(x, min=clip_val) * C) - - -def dynamic_range_decompression_torch(x, C=1): - return torch.exp(x) / C - - -def spectral_normalize_torch(magnitudes): - output = dynamic_range_compression_torch(magnitudes) - return output - - -def spectral_de_normalize_torch(magnitudes): - output = dynamic_range_decompression_torch(magnitudes) - return output - - -mel_basis = {} -hann_window = {} - - -def mel_spectrogram( - y, n_fft, num_mels, sampling_rate, hop_size, win_size, fmin, fmax, center=False -): - if torch.min(y) < -1.0: - print("min value is ", torch.min(y)) - if torch.max(y) > 1.0: - print("max value is ", torch.max(y)) - - global mel_basis, hann_window # pylint: disable=global-statement - if f"{str(fmax)}_{str(y.device)}" not in mel_basis: - mel = librosa_mel_fn( - sr=sampling_rate, n_fft=n_fft, n_mels=num_mels, fmin=fmin, fmax=fmax - ) - mel_basis[str(fmax) + "_" + str(y.device)] = ( - torch.from_numpy(mel).float().to(y.device) - ) - hann_window[str(y.device)] = torch.hann_window(win_size).to(y.device) - - y = torch.nn.functional.pad( - y.unsqueeze(1), - (int((n_fft - hop_size) / 2), int((n_fft - hop_size) / 2)), - mode="reflect", - ) - y = y.squeeze(1) - - spec = torch.view_as_real( - torch.stft( - y, - n_fft, - hop_length=hop_size, - win_length=win_size, - window=hann_window[str(y.device)], - center=center, - pad_mode="reflect", - normalized=False, - onesided=True, - return_complex=True, - ) - ) - - spec = torch.sqrt(spec.pow(2).sum(-1) + (1e-9)) - - spec = torch.matmul(mel_basis[str(fmax) + "_" + str(y.device)], spec) - spec = spectral_normalize_torch(spec) - - return spec diff --git a/cosyvoice/matcha/decoder.py b/cosyvoice/matcha/decoder.py deleted file mode 100644 index d571dcca22a404a1272940cf1480820b58a10549..0000000000000000000000000000000000000000 --- a/cosyvoice/matcha/decoder.py +++ /dev/null @@ -1,511 +0,0 @@ -import math -from typing import Optional - -import torch -import torch.nn as nn -import torch.nn.functional as F -from conformer import ConformerBlock -from diffusers.models.activations import get_activation -from einops import pack, rearrange, repeat - -from cosyvoice.matcha.transformer import BasicTransformerBlock - - -class SinusoidalPosEmb(torch.nn.Module): - def __init__(self, dim): - super().__init__() - self.dim = dim - assert self.dim % 2 == 0, "SinusoidalPosEmb requires dim to be even" - - def forward(self, x, scale=1000): - if x.ndim < 1: - x = x.unsqueeze(0) - device = x.device - half_dim = self.dim // 2 - emb = math.log(10000) / (half_dim - 1) - emb = torch.exp(torch.arange(half_dim, device=device).float() * -emb) - emb = scale * x.unsqueeze(1) * emb.unsqueeze(0) - emb = torch.cat((emb.sin(), emb.cos()), dim=-1) - return emb - - -class MaskedGroupNorm(nn.GroupNorm): - """ - Masked verstion of the Group normalization. - - Based on: https://github.com/ptrblck/pytorch_misc/blob/20e8ea93bd458b88f921a87e2d4001a4eb753a02/batch_norm_manual.py - - Receives a N-dim tensor of sequence lengths per batch element - along with the regular input for masking. - - Check pytorch's GroupNorm implementation for argument details. - """ - - def __init__(self, num_groups, num_channels, eps=1e-5, affine=True): - super(MaskedGroupNorm, self).__init__(num_groups, num_channels, eps, affine) - - def forward(self, inp, mask=None): - assert ( - inp.shape[1] % self.num_groups == 0 - ), "Feature size not divisible by groups" - - # 计算有效长度 - seq_lengths = mask.sum(-1, keepdim=True) # [batch_size, 1] - - # 将输入reshape为groups - features_per_group = inp.shape[1] // self.num_groups - inp_r = inp.reshape( - inp.shape[0], self.num_groups, features_per_group, inp.shape[-1] - ) - mask_r = mask.unsqueeze(1) # [batch_size, 1, 1, length] - - # 计算masked mean和variance - masked_inp = inp_r * mask_r - n = seq_lengths * features_per_group # 每组的有效元素数量 - mean = masked_inp.sum([2, 3], keepdim=True) / (n.view(-1, 1, 1, 1) + 1e-5) - var = ((masked_inp - mean * mask_r) ** 2).sum([2, 3], keepdim=True) / ( - n.view(-1, 1, 1, 1) + 1e-5 - ) - - # 标准化 - inp_r = (inp_r - mean) / (torch.sqrt(var + self.eps)) - out = inp_r.reshape(inp.shape[0], self.num_channels, inp.shape[-1]) - - # 应用仿射变换 - if self.affine: - out = out * self.weight[None, :, None] + self.bias[None, :, None] - - return out - - -class Block1D(torch.nn.Module): - def __init__(self, dim, dim_out, groups=8): - super().__init__() - self.block = torch.nn.Sequential( - torch.nn.Conv1d(dim, dim_out, 3, padding=1), - torch.nn.GroupNorm(groups, dim_out), - # MaskedGroupNorm(groups, dim_out), - nn.Mish(), - ) - - def forward(self, x, mask): - output = self.block(x * mask) - return output * mask - return x * mask - - -class ResnetBlock1D(torch.nn.Module): - def __init__(self, dim, dim_out, time_emb_dim, groups=8): - super().__init__() - self.mlp = torch.nn.Sequential( - nn.Mish(), torch.nn.Linear(time_emb_dim, dim_out) - ) - - self.block1 = Block1D(dim, dim_out, groups=groups) - self.block2 = Block1D(dim_out, dim_out, groups=groups) - - self.res_conv = torch.nn.Conv1d(dim, dim_out, 1) - - def forward(self, x, mask, time_emb): - h = self.block1(x, mask) - h += self.mlp(time_emb).unsqueeze(-1) - h = self.block2(h, mask) - output = h + self.res_conv(x * mask) - return output - - -class Downsample1D(nn.Module): - def __init__(self, dim): - super().__init__() - self.conv = torch.nn.Conv1d(dim, dim, 3, 2, 1) - - def forward(self, x): - return self.conv(x) - - -class TimestepEmbedding(nn.Module): - def __init__( - self, - in_channels: int, - time_embed_dim: int, - act_fn: str = "silu", - out_dim: int = None, - post_act_fn: Optional[str] = None, - cond_proj_dim=None, - ): - super().__init__() - - self.linear_1 = nn.Linear(in_channels, time_embed_dim) - - if cond_proj_dim is not None: - self.cond_proj = nn.Linear(cond_proj_dim, in_channels, bias=False) - else: - self.cond_proj = None - - self.act = get_activation(act_fn) - - if out_dim is not None: - time_embed_dim_out = out_dim - else: - time_embed_dim_out = time_embed_dim - self.linear_2 = nn.Linear(time_embed_dim, time_embed_dim_out) - - if post_act_fn is None: - self.post_act = None - else: - self.post_act = get_activation(post_act_fn) - - def forward(self, sample, condition=None): - if condition is not None: - sample = sample + self.cond_proj(condition) - sample = self.linear_1(sample) - - if self.act is not None: - sample = self.act(sample) - - sample = self.linear_2(sample) - - if self.post_act is not None: - sample = self.post_act(sample) - return sample - - -class Upsample1D(nn.Module): - """A 1D upsampling layer with an optional convolution. - - Parameters: - channels (`int`): - number of channels in the inputs and outputs. - use_conv (`bool`, default `False`): - option to use a convolution. - use_conv_transpose (`bool`, default `False`): - option to use a convolution transpose. - out_channels (`int`, optional): - number of output channels. Defaults to `channels`. - """ - - def __init__( - self, - channels, - use_conv=False, - use_conv_transpose=True, - out_channels=None, - name="conv", - ): - super().__init__() - self.channels = channels - self.out_channels = out_channels or channels - self.use_conv = use_conv - self.use_conv_transpose = use_conv_transpose - self.name = name - - self.conv = None - if use_conv_transpose: - self.conv = nn.ConvTranspose1d(channels, self.out_channels, 4, 2, 1) - elif use_conv: - self.conv = nn.Conv1d(self.channels, self.out_channels, 3, padding=1) - - def forward(self, inputs): - assert inputs.shape[1] == self.channels - if self.use_conv_transpose: - return self.conv(inputs) - - outputs = F.interpolate(inputs, scale_factor=2.0, mode="nearest") - - if self.use_conv: - outputs = self.conv(outputs) - - return outputs - - -class ConformerWrapper(ConformerBlock): - def __init__( # pylint: disable=useless-super-delegation - self, - *, - dim, - dim_head=64, - heads=8, - ff_mult=4, - conv_expansion_factor=2, - conv_kernel_size=31, - attn_dropout=0, - ff_dropout=0, - conv_dropout=0, - conv_causal=False, - ): - super().__init__( - dim=dim, - dim_head=dim_head, - heads=heads, - ff_mult=ff_mult, - conv_expansion_factor=conv_expansion_factor, - conv_kernel_size=conv_kernel_size, - attn_dropout=attn_dropout, - ff_dropout=ff_dropout, - conv_dropout=conv_dropout, - conv_causal=conv_causal, - ) - - def forward( - self, - hidden_states, - attention_mask, - encoder_hidden_states=None, - encoder_attention_mask=None, - timestep=None, - ): - return super().forward(x=hidden_states, mask=attention_mask.bool()) - - -class Decoder(nn.Module): - def __init__( - self, - in_channels, - out_channels, - channels=(256, 256), - dropout=0.05, - attention_head_dim=64, - n_blocks=1, - num_mid_blocks=2, - num_heads=4, - act_fn="snake", - down_block_type="transformer", - mid_block_type="transformer", - up_block_type="transformer", - ): - super().__init__() - channels = tuple(channels) - self.in_channels = in_channels - self.out_channels = out_channels - - self.time_embeddings = SinusoidalPosEmb(in_channels) - time_embed_dim = channels[0] * 4 - self.time_mlp = TimestepEmbedding( - in_channels=in_channels, - time_embed_dim=time_embed_dim, - act_fn="silu", - ) - - self.down_blocks = nn.ModuleList([]) - self.mid_blocks = nn.ModuleList([]) - self.up_blocks = nn.ModuleList([]) - - output_channel = in_channels - for i in range(len(channels)): # pylint: disable=consider-using-enumerate - input_channel = output_channel - output_channel = channels[i] - is_last = i == len(channels) - 1 - resnet = ResnetBlock1D( - dim=input_channel, dim_out=output_channel, time_emb_dim=time_embed_dim - ) - transformer_blocks = nn.ModuleList( - [ - self.get_block( - down_block_type, - output_channel, - attention_head_dim, - num_heads, - dropout, - act_fn, - ) - for _ in range(n_blocks) - ] - ) - downsample = ( - Downsample1D(output_channel) - if not is_last - else nn.Conv1d(output_channel, output_channel, 3, padding=1) - ) - - self.down_blocks.append( - nn.ModuleList([resnet, transformer_blocks, downsample]) - ) - - for i in range(num_mid_blocks): - input_channel = channels[-1] - out_channels = channels[-1] - - resnet = ResnetBlock1D( - dim=input_channel, dim_out=output_channel, time_emb_dim=time_embed_dim - ) - - transformer_blocks = nn.ModuleList( - [ - self.get_block( - mid_block_type, - output_channel, - attention_head_dim, - num_heads, - dropout, - act_fn, - ) - for _ in range(n_blocks) - ] - ) - - self.mid_blocks.append(nn.ModuleList([resnet, transformer_blocks])) - - channels = channels[::-1] + (channels[0],) - for i in range(len(channels) - 1): - input_channel = channels[i] - output_channel = channels[i + 1] - is_last = i == len(channels) - 2 - - resnet = ResnetBlock1D( - dim=2 * input_channel, - dim_out=output_channel, - time_emb_dim=time_embed_dim, - ) - transformer_blocks = nn.ModuleList( - [ - self.get_block( - up_block_type, - output_channel, - attention_head_dim, - num_heads, - dropout, - act_fn, - ) - for _ in range(n_blocks) - ] - ) - upsample = ( - Upsample1D(output_channel, use_conv_transpose=True) - if not is_last - else nn.Conv1d(output_channel, output_channel, 3, padding=1) - ) - - self.up_blocks.append(nn.ModuleList([resnet, transformer_blocks, upsample])) - - self.final_block = Block1D(channels[-1], channels[-1]) - self.final_proj = nn.Conv1d(channels[-1], self.out_channels, 1) - - self.initialize_weights() - # nn.init.normal_(self.final_proj.weight) - - @staticmethod - def get_block(block_type, dim, attention_head_dim, num_heads, dropout, act_fn): - if block_type == "conformer": - block = ConformerWrapper( - dim=dim, - dim_head=attention_head_dim, - heads=num_heads, - ff_mult=1, - conv_expansion_factor=2, - ff_dropout=dropout, - attn_dropout=dropout, - conv_dropout=dropout, - conv_kernel_size=31, - ) - elif block_type == "transformer": - block = BasicTransformerBlock( - dim=dim, - num_attention_heads=num_heads, - attention_head_dim=attention_head_dim, - dropout=dropout, - activation_fn=act_fn, - ) - else: - raise ValueError(f"Unknown block type {block_type}") - - return block - - def initialize_weights(self): - for m in self.modules(): - if isinstance(m, nn.Conv1d): - nn.init.kaiming_normal_(m.weight, nonlinearity="relu") - - if m.bias is not None: - nn.init.constant_(m.bias, 0) - - elif isinstance(m, nn.GroupNorm): - nn.init.constant_(m.weight, 1) - nn.init.constant_(m.bias, 0) - - elif isinstance(m, nn.Linear): - nn.init.kaiming_normal_(m.weight, nonlinearity="relu") - - if m.bias is not None: - nn.init.constant_(m.bias, 0) - - def forward(self, x, mask, mu, t, spks=None, cond=None): - """Forward pass of the UNet1DConditional model. - - Args: - x (torch.Tensor): shape (batch_size, in_channels, time) - mask (_type_): shape (batch_size, 1, time) - t (_type_): shape (batch_size) - spks (_type_, optional): shape: (batch_size, condition_channels). Defaults to None. - cond (_type_, optional): placeholder for future use. Defaults to None. - - Raises: - ValueError: _description_ - ValueError: _description_ - - Returns: - _type_: _description_ - """ - - t = self.time_embeddings(t) - t = self.time_mlp(t) - - x = pack([x, mu], "b * t")[0] - - if spks is not None: - spks = repeat(spks, "b c -> b c t", t=x.shape[-1]) - x = pack([x, spks], "b * t")[0] - - hiddens = [] - masks = [mask] - for resnet, transformer_blocks, downsample in self.down_blocks: - mask_down = masks[-1] - x = resnet(x, mask_down, t) - x = rearrange(x, "b c t -> b t c") - mask_down = rearrange(mask_down, "b 1 t -> b t") - for transformer_block in transformer_blocks: - x = transformer_block( - hidden_states=x, - attention_mask=mask_down, - timestep=t, - ) - x = rearrange(x, "b t c -> b c t") - mask_down = rearrange(mask_down, "b t -> b 1 t") - hiddens.append(x) # Save hidden states for skip connections - x = downsample(x * mask_down) - masks.append(mask_down[:, :, ::2]) - - masks = masks[:-1] - mask_mid = masks[-1] - - for resnet, transformer_blocks in self.mid_blocks: - x = resnet(x, mask_mid, t) - x = rearrange(x, "b c t -> b t c") - mask_mid = rearrange(mask_mid, "b 1 t -> b t") - for transformer_block in transformer_blocks: - x = transformer_block( - hidden_states=x, - attention_mask=mask_mid, - timestep=t, - ) - x = rearrange(x, "b t c -> b c t") - mask_mid = rearrange(mask_mid, "b t -> b 1 t") - - for resnet, transformer_blocks, upsample in self.up_blocks: - mask_up = masks.pop() - x = resnet(pack([x, hiddens.pop()], "b * t")[0], mask_up, t) - x = rearrange(x, "b c t -> b t c") - mask_up = rearrange(mask_up, "b 1 t -> b t") - for transformer_block in transformer_blocks: - x = transformer_block( - hidden_states=x, - attention_mask=mask_up, - timestep=t, - ) - x = rearrange(x, "b t c -> b c t") - mask_up = rearrange(mask_up, "b t -> b 1 t") - x = upsample(x * mask_up) - - x = self.final_block(x, mask_up) - output = self.final_proj(x * mask_up) - - return output * mask diff --git a/cosyvoice/matcha/flow_matching.py b/cosyvoice/matcha/flow_matching.py deleted file mode 100644 index 678153c2eb345aa66ff5fbe643025bd2332315f8..0000000000000000000000000000000000000000 --- a/cosyvoice/matcha/flow_matching.py +++ /dev/null @@ -1,141 +0,0 @@ -from abc import ABC - -import torch -import torch.nn.functional as F - -from cosyvoice.matcha.decoder import Decoder - - -class BASECFM(torch.nn.Module, ABC): - def __init__( - self, - n_feats, - cfm_params, - n_spks=1, - spk_emb_dim=128, - ): - super().__init__() - self.n_feats = n_feats - self.n_spks = n_spks - self.spk_emb_dim = spk_emb_dim - self.solver = cfm_params.solver - if hasattr(cfm_params, "sigma_min"): - self.sigma_min = cfm_params.sigma_min - else: - self.sigma_min = 1e-4 - - self.estimator = None - - @torch.inference_mode() - def forward(self, mu, mask, n_timesteps, temperature=1.0, spks=None, cond=None): - """Forward diffusion - - Args: - mu (torch.Tensor): output of encoder - shape: (batch_size, n_feats, mel_timesteps) - mask (torch.Tensor): output_mask - shape: (batch_size, 1, mel_timesteps) - n_timesteps (int): number of diffusion steps - temperature (float, optional): temperature for scaling noise. Defaults to 1.0. - spks (torch.Tensor, optional): speaker ids. Defaults to None. - shape: (batch_size, spk_emb_dim) - cond: Not used but kept for future purposes - - Returns: - sample: generated mel-spectrogram - shape: (batch_size, n_feats, mel_timesteps) - """ - z = torch.randn_like(mu) * temperature - t_span = torch.linspace(0, 1, n_timesteps + 1, device=mu.device) - return self.solve_euler( - z, t_span=t_span, mu=mu, mask=mask, spks=spks, cond=cond - ) - - def solve_euler(self, x, t_span, mu, mask, spks, cond): - """ - Fixed euler solver for ODEs. - Args: - x (torch.Tensor): random noise - t_span (torch.Tensor): n_timesteps interpolated - shape: (n_timesteps + 1,) - mu (torch.Tensor): output of encoder - shape: (batch_size, n_feats, mel_timesteps) - mask (torch.Tensor): output_mask - shape: (batch_size, 1, mel_timesteps) - spks (torch.Tensor, optional): speaker ids. Defaults to None. - shape: (batch_size, spk_emb_dim) - cond: Not used but kept for future purposes - """ - t, _, dt = t_span[0], t_span[-1], t_span[1] - t_span[0] - - # I am storing this because I can later plot it by putting a debugger here and saving it to a file - # Or in future might add like a return_all_steps flag - sol = [] - - for step in range(1, len(t_span)): - dphi_dt = self.estimator(x, mask, mu, t, spks, cond) - - x = x + dt * dphi_dt - t = t + dt - sol.append(x) - if step < len(t_span) - 1: - dt = t_span[step + 1] - t - - return sol[-1] - - def compute_loss(self, x1, mask, mu, spks=None, cond=None): - """Computes diffusion loss - - Args: - x1 (torch.Tensor): Target - shape: (batch_size, n_feats, mel_timesteps) - mask (torch.Tensor): target mask - shape: (batch_size, 1, mel_timesteps) - mu (torch.Tensor): output of encoder - shape: (batch_size, n_feats, mel_timesteps) - spks (torch.Tensor, optional): speaker embedding. Defaults to None. - shape: (batch_size, spk_emb_dim) - - Returns: - loss: conditional flow matching loss - y: conditional flow - shape: (batch_size, n_feats, mel_timesteps) - """ - b, _, t = mu.shape - - # random timestep - t = torch.rand([b, 1, 1], device=mu.device, dtype=mu.dtype) - # sample noise p(x_0) - z = torch.randn_like(x1) - - y = (1 - (1 - self.sigma_min) * t) * z + t * x1 - u = x1 - (1 - self.sigma_min) * z - - loss = F.mse_loss( - self.estimator(y, mask, mu, t.squeeze(), spks), u, reduction="sum" - ) / (torch.sum(mask) * u.shape[1]) - return loss, y - - -class CFM(BASECFM): - def __init__( - self, - in_channels, - out_channel, - cfm_params, - decoder_params, - n_spks=1, - spk_emb_dim=64, - ): - super().__init__( - n_feats=in_channels, - cfm_params=cfm_params, - n_spks=n_spks, - spk_emb_dim=spk_emb_dim, - ) - - in_channels = in_channels + (spk_emb_dim if n_spks > 1 else 0) - # Just change the architecture of the estimator here - self.estimator = Decoder( - in_channels=in_channels, out_channels=out_channel, **decoder_params - ) diff --git a/cosyvoice/matcha/transformer.py b/cosyvoice/matcha/transformer.py deleted file mode 100644 index 12712dc79590d7eafa95f745d0be95c3b13bb0a4..0000000000000000000000000000000000000000 --- a/cosyvoice/matcha/transformer.py +++ /dev/null @@ -1,443 +0,0 @@ -from typing import Any, Dict, Optional - -import torch -import torch.nn as nn -from diffusers.models.attention import ( - GEGLU, - GELU, - AdaLayerNorm, - AdaLayerNormZero, - ApproximateGELU, -) -from diffusers.models.attention_processor import Attention -from diffusers.models.lora import LoRACompatibleLinear -from diffusers.utils.torch_utils import maybe_allow_in_graph - - -class SnakeBeta(nn.Module): - """ - A modified Snake function which uses separate parameters for the magnitude of the periodic components - Shape: - - Input: (B, C, T) - - Output: (B, C, T), same shape as the input - Parameters: - - alpha - trainable parameter that controls frequency - - beta - trainable parameter that controls magnitude - References: - - This activation function is a modified version based on this paper by Liu Ziyin, Tilman Hartwig, Masahito Ueda: - https://arxiv.org/abs/2006.08195 - Examples: - >>> a1 = snakebeta(256) - >>> x = torch.randn(256) - >>> x = a1(x) - """ - - def __init__( - self, - in_features, - out_features, - alpha=1.0, - alpha_trainable=True, - alpha_logscale=True, - ): - """ - Initialization. - INPUT: - - in_features: shape of the input - - alpha - trainable parameter that controls frequency - - beta - trainable parameter that controls magnitude - alpha is initialized to 1 by default, higher values = higher-frequency. - beta is initialized to 1 by default, higher values = higher-magnitude. - alpha will be trained along with the rest of your model. - """ - super().__init__() - self.in_features = ( - out_features if isinstance(out_features, list) else [out_features] - ) - self.proj = LoRACompatibleLinear(in_features, out_features) - - # initialize alpha - self.alpha_logscale = alpha_logscale - if self.alpha_logscale: # log scale alphas initialized to zeros - self.alpha = nn.Parameter(torch.zeros(self.in_features) * alpha) - self.beta = nn.Parameter(torch.zeros(self.in_features) * alpha) - else: # linear scale alphas initialized to ones - self.alpha = nn.Parameter(torch.ones(self.in_features) * alpha) - self.beta = nn.Parameter(torch.ones(self.in_features) * alpha) - - self.alpha.requires_grad = alpha_trainable - self.beta.requires_grad = alpha_trainable - - self.no_div_by_zero = 0.000000001 - - def forward(self, x): - """ - Forward pass of the function. - Applies the function to the input elementwise. - SnakeBeta ∶= x + 1/b * sin^2 (xa) - """ - x = self.proj(x) - if self.alpha_logscale: - alpha = torch.exp(self.alpha) - beta = torch.exp(self.beta) - else: - alpha = self.alpha - beta = self.beta - - x = x + (1.0 / (beta + self.no_div_by_zero)) * torch.pow( - torch.sin(x * alpha), 2 - ) - - return x - - -class FeedForward(nn.Module): - r""" - A feed-forward layer. - - Parameters: - dim (`int`): The number of channels in the input. - dim_out (`int`, *optional*): The number of channels in the output. If not given, defaults to `dim`. - mult (`int`, *optional*, defaults to 4): The multiplier to use for the hidden dimension. - dropout (`float`, *optional*, defaults to 0.0): The dropout probability to use. - activation_fn (`str`, *optional*, defaults to `"geglu"`): Activation function to be used in feed-forward. - final_dropout (`bool` *optional*, defaults to False): Apply a final dropout. - """ - - def __init__( - self, - dim: int, - dim_out: Optional[int] = None, - mult: int = 4, - dropout: float = 0.0, - activation_fn: str = "geglu", - final_dropout: bool = False, - ): - super().__init__() - inner_dim = int(dim * mult) - dim_out = dim_out if dim_out is not None else dim - - if activation_fn == "gelu": - act_fn = GELU(dim, inner_dim) - if activation_fn == "gelu-approximate": - act_fn = GELU(dim, inner_dim, approximate="tanh") - elif activation_fn == "geglu": - act_fn = GEGLU(dim, inner_dim) - elif activation_fn == "geglu-approximate": - act_fn = ApproximateGELU(dim, inner_dim) - elif activation_fn == "snakebeta": - act_fn = SnakeBeta(dim, inner_dim) - - self.net = nn.ModuleList([]) - # project in - self.net.append(act_fn) - # project dropout - self.net.append(nn.Dropout(dropout)) - # project out - self.net.append(LoRACompatibleLinear(inner_dim, dim_out)) - # FF as used in Vision Transformer, MLP-Mixer, etc. have a final dropout - if final_dropout: - self.net.append(nn.Dropout(dropout)) - - def forward(self, hidden_states): - for module in self.net: - hidden_states = module(hidden_states) - return hidden_states - - -@maybe_allow_in_graph -class BasicTransformerBlock(nn.Module): - r""" - A basic Transformer block. - - Parameters: - dim (`int`): The number of channels in the input and output. - num_attention_heads (`int`): The number of heads to use for multi-head attention. - attention_head_dim (`int`): The number of channels in each head. - dropout (`float`, *optional*, defaults to 0.0): The dropout probability to use. - cross_attention_dim (`int`, *optional*): The size of the encoder_hidden_states vector for cross attention. - only_cross_attention (`bool`, *optional*): - Whether to use only cross-attention layers. In this case two cross attention layers are used. - double_self_attention (`bool`, *optional*): - Whether to use two self-attention layers. In this case no cross attention layers are used. - activation_fn (`str`, *optional*, defaults to `"geglu"`): Activation function to be used in feed-forward. - num_embeds_ada_norm (: - obj: `int`, *optional*): The number of diffusion steps used during training. See `Transformer2DModel`. - attention_bias (: - obj: `bool`, *optional*, defaults to `False`): Configure if the attentions should contain a bias parameter. - """ - - def __init__( - self, - dim: int, - num_attention_heads: int, - attention_head_dim: int, - dropout=0.0, - cross_attention_dim: Optional[int] = None, - activation_fn: str = "geglu", - num_embeds_ada_norm: Optional[int] = None, - attention_bias: bool = False, - only_cross_attention: bool = False, - double_self_attention: bool = False, - upcast_attention: bool = False, - norm_elementwise_affine: bool = True, - norm_type: str = "layer_norm", - final_dropout: bool = False, - ): - super().__init__() - self.only_cross_attention = only_cross_attention - - self.use_ada_layer_norm_zero = ( - num_embeds_ada_norm is not None - ) and norm_type == "ada_norm_zero" - self.use_ada_layer_norm = ( - num_embeds_ada_norm is not None - ) and norm_type == "ada_norm" - - if norm_type in ("ada_norm", "ada_norm_zero") and num_embeds_ada_norm is None: - raise ValueError( - f"`norm_type` is set to {norm_type}, but `num_embeds_ada_norm` is not defined. Please make sure to" - f" define `num_embeds_ada_norm` if setting `norm_type` to {norm_type}." - ) - - # Define 3 blocks. Each block has its own normalization layer. - # 1. Self-Attn - if self.use_ada_layer_norm: - self.norm1 = AdaLayerNorm(dim, num_embeds_ada_norm) - elif self.use_ada_layer_norm_zero: - self.norm1 = AdaLayerNormZero(dim, num_embeds_ada_norm) - else: - self.norm1 = nn.LayerNorm(dim, elementwise_affine=norm_elementwise_affine) - self.attn1 = Attention( - query_dim=dim, - heads=num_attention_heads, - dim_head=attention_head_dim, - dropout=dropout, - bias=attention_bias, - cross_attention_dim=cross_attention_dim if only_cross_attention else None, - upcast_attention=upcast_attention, - ) - - # 2. Cross-Attn - if cross_attention_dim is not None or double_self_attention: - # We currently only use AdaLayerNormZero for self attention where there will only be one attention block. - # I.e. the number of returned modulation chunks from AdaLayerZero would not make sense if returned during - # the second cross attention block. - self.norm2 = ( - AdaLayerNorm(dim, num_embeds_ada_norm) - if self.use_ada_layer_norm - else nn.LayerNorm(dim, elementwise_affine=norm_elementwise_affine) - ) - self.attn2 = Attention( - query_dim=dim, - cross_attention_dim=( - cross_attention_dim if not double_self_attention else None - ), - heads=num_attention_heads, - dim_head=attention_head_dim, - dropout=dropout, - bias=attention_bias, - upcast_attention=upcast_attention, - # scale_qk=False, # uncomment this to not to use flash attention - ) # is self-attn if encoder_hidden_states is none - else: - self.norm2 = None - self.attn2 = None - - # 3. Feed-forward - self.norm3 = nn.LayerNorm(dim, elementwise_affine=norm_elementwise_affine) - self.ff = FeedForward( - dim, - dropout=dropout, - activation_fn=activation_fn, - final_dropout=final_dropout, - ) - - # let chunk size default to None - self._chunk_size = None - self._chunk_dim = 0 - - def set_chunk_feed_forward(self, chunk_size: Optional[int], dim: int): - # Sets chunk feed-forward - self._chunk_size = chunk_size - self._chunk_dim = dim - - def forward_native( - self, - hidden_states: torch.FloatTensor, - attention_mask: Optional[torch.FloatTensor] = None, - encoder_hidden_states: Optional[torch.FloatTensor] = None, - encoder_attention_mask: Optional[torch.FloatTensor] = None, - timestep: Optional[torch.LongTensor] = None, - cross_attention_kwargs: Dict[str, Any] = None, - class_labels: Optional[torch.LongTensor] = None, - ): - # Notice that normalization is always applied before the real computation in the following blocks. - # 1. Self-Attention - if self.use_ada_layer_norm: - norm_hidden_states = self.norm1(hidden_states, timestep) - elif self.use_ada_layer_norm_zero: - norm_hidden_states, gate_msa, shift_mlp, scale_mlp, gate_mlp = self.norm1( - hidden_states, timestep, class_labels, hidden_dtype=hidden_states.dtype - ) - else: - norm_hidden_states = self.norm1(hidden_states) - - cross_attention_kwargs = ( - cross_attention_kwargs if cross_attention_kwargs is not None else {} - ) - - attn_output = self.attn1( - norm_hidden_states, - encoder_hidden_states=( - encoder_hidden_states if self.only_cross_attention else None - ), - attention_mask=( - encoder_attention_mask if self.only_cross_attention else attention_mask - ), - **cross_attention_kwargs, - ) - if self.use_ada_layer_norm_zero: - attn_output = gate_msa.unsqueeze(1) * attn_output - hidden_states = attn_output + hidden_states - - # 2. Cross-Attention - if self.attn2 is not None: - norm_hidden_states = ( - self.norm2(hidden_states, timestep) - if self.use_ada_layer_norm - else self.norm2(hidden_states) - ) - - attn_output = self.attn2( - norm_hidden_states, - encoder_hidden_states=encoder_hidden_states, - attention_mask=encoder_attention_mask, - **cross_attention_kwargs, - ) - hidden_states = attn_output + hidden_states - - # 3. Feed-forward - norm_hidden_states = self.norm3(hidden_states) - - if self.use_ada_layer_norm_zero: - norm_hidden_states = ( - norm_hidden_states * (1 + scale_mlp[:, None]) + shift_mlp[:, None] - ) - - if self._chunk_size is not None: - # "feed_forward_chunk_size" can be used to save memory - if norm_hidden_states.shape[self._chunk_dim] % self._chunk_size != 0: - raise ValueError( - f"`hidden_states` dimension to be chunked: {norm_hidden_states.shape[self._chunk_dim]} has to be divisible by chunk size: {self._chunk_size}. Make sure to set an appropriate `chunk_size` when calling `unet.enable_forward_chunking`." - ) - - num_chunks = norm_hidden_states.shape[self._chunk_dim] // self._chunk_size - ff_output = torch.cat( - [ - self.ff(hid_slice) - for hid_slice in norm_hidden_states.chunk( - num_chunks, dim=self._chunk_dim - ) - ], - dim=self._chunk_dim, - ) - else: - ff_output = self.ff(norm_hidden_states) - - if self.use_ada_layer_norm_zero: - ff_output = gate_mlp.unsqueeze(1) * ff_output - - hidden_states = ff_output + hidden_states - - return hidden_states - - def forward( - self, - hidden_states: torch.FloatTensor, - attention_mask: Optional[torch.FloatTensor] = None, - encoder_hidden_states: Optional[torch.FloatTensor] = None, - encoder_attention_mask: Optional[torch.FloatTensor] = None, - timestep: Optional[torch.LongTensor] = None, - cross_attention_kwargs: Dict[str, Any] = None, - class_labels: Optional[torch.LongTensor] = None, - ): - # Notice that normalization is always applied before the real computation in the following blocks. - # 1. Self-Attention - if self.use_ada_layer_norm: - norm_hidden_states = self.norm1(hidden_states, timestep) - elif self.use_ada_layer_norm_zero: - norm_hidden_states, gate_msa, shift_mlp, scale_mlp, gate_mlp = self.norm1( - hidden_states, timestep, class_labels, hidden_dtype=hidden_states.dtype - ) - else: - norm_hidden_states = self.norm1(hidden_states) - - cross_attention_kwargs = ( - cross_attention_kwargs if cross_attention_kwargs is not None else {} - ) - - attn_output = self.attn1( - norm_hidden_states, - encoder_hidden_states=( - encoder_hidden_states if self.only_cross_attention else None - ), - attention_mask=( - encoder_attention_mask if self.only_cross_attention else attention_mask - ), - **cross_attention_kwargs, - ) - if self.use_ada_layer_norm_zero: - attn_output = gate_msa.unsqueeze(1) * attn_output - hidden_states = attn_output + hidden_states - - # 2. Cross-Attention - if self.attn2 is not None: - norm_hidden_states = ( - self.norm2(hidden_states, timestep) - if self.use_ada_layer_norm - else self.norm2(hidden_states) - ) - - attn_output = self.attn2( - norm_hidden_states, - encoder_hidden_states=encoder_hidden_states, - attention_mask=encoder_attention_mask, - **cross_attention_kwargs, - ) - hidden_states = attn_output + hidden_states - - # 3. Feed-forward - norm_hidden_states = self.norm3(hidden_states) - - if self.use_ada_layer_norm_zero: - norm_hidden_states = ( - norm_hidden_states * (1 + scale_mlp[:, None]) + shift_mlp[:, None] - ) - - if self._chunk_size is not None: - # "feed_forward_chunk_size" can be used to save memory - if norm_hidden_states.shape[self._chunk_dim] % self._chunk_size != 0: - raise ValueError( - f"`hidden_states` dimension to be chunked: {norm_hidden_states.shape[self._chunk_dim]} has to be divisible by chunk size: {self._chunk_size}. Make sure to set an appropriate `chunk_size` when calling `unet.enable_forward_chunking`." - ) - - num_chunks = norm_hidden_states.shape[self._chunk_dim] // self._chunk_size - ff_output = torch.cat( - [ - self.ff(hid_slice) - for hid_slice in norm_hidden_states.chunk( - num_chunks, dim=self._chunk_dim - ) - ], - dim=self._chunk_dim, - ) - else: - ff_output = self.ff(norm_hidden_states) - - if self.use_ada_layer_norm_zero: - ff_output = gate_mlp.unsqueeze(1) * ff_output - - hidden_states = ff_output + hidden_states - - return hidden_states diff --git a/cosyvoice/transformer/__init__.py b/cosyvoice/transformer/__init__.py deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/cosyvoice/transformer/activation.py b/cosyvoice/transformer/activation.py deleted file mode 100644 index 507fff4ba2b72af3aaff87cc85f98cf3612434d7..0000000000000000000000000000000000000000 --- a/cosyvoice/transformer/activation.py +++ /dev/null @@ -1,87 +0,0 @@ -# Copyright (c) 2020 Johns Hopkins University (Shinji Watanabe) -# 2020 Northwestern Polytechnical University (Pengcheng Guo) -# 2020 Mobvoi Inc (Binbin Zhang) -# 2024 Alibaba Inc (Xiang Lyu) -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Swish() activation function for Conformer.""" - -import torch -from torch import nn, sin, pow -from torch.nn import Parameter - - -class Swish(torch.nn.Module): - """Construct an Swish object.""" - - def forward(self, x: torch.Tensor) -> torch.Tensor: - """Return Swish activation function.""" - return x * torch.sigmoid(x) - - -# Implementation adapted from https://github.com/EdwardDixon/snake under the MIT license. -# LICENSE is in incl_licenses directory. -class Snake(nn.Module): - """ - Implementation of a sine-based periodic activation function - Shape: - - Input: (B, C, T) - - Output: (B, C, T), same shape as the input - Parameters: - - alpha - trainable parameter - References: - - This activation function is from this paper by Liu Ziyin, Tilman Hartwig, Masahito Ueda: - https://arxiv.org/abs/2006.08195 - Examples: - >>> a1 = snake(256) - >>> x = torch.randn(256) - >>> x = a1(x) - """ - - def __init__( - self, in_features, alpha=1.0, alpha_trainable=True, alpha_logscale=False - ): - """ - Initialization. - INPUT: - - in_features: shape of the input - - alpha: trainable parameter - alpha is initialized to 1 by default, higher values = higher-frequency. - alpha will be trained along with the rest of your model. - """ - super(Snake, self).__init__() - self.in_features = in_features - - # initialize alpha - self.alpha_logscale = alpha_logscale - if self.alpha_logscale: # log scale alphas initialized to zeros - self.alpha = Parameter(torch.zeros(in_features) * alpha) - else: # linear scale alphas initialized to ones - self.alpha = Parameter(torch.ones(in_features) * alpha) - - self.alpha.requires_grad = alpha_trainable - - self.no_div_by_zero = 0.000000001 - - def forward(self, x): - """ - Forward pass of the function. - Applies the function to the input elementwise. - Snake ∶= x + 1/a * sin^2 (xa) - """ - alpha = self.alpha.unsqueeze(0).unsqueeze(-1) # line up with x to [B, C, T] - if self.alpha_logscale: - alpha = torch.exp(alpha) - x = x + (1.0 / (alpha + self.no_div_by_zero)) * pow(sin(x * alpha), 2) - - return x diff --git a/cosyvoice/transformer/attention.py b/cosyvoice/transformer/attention.py deleted file mode 100644 index 4b2a17b2471d5486d38ad36312ff2f41f70ed5c1..0000000000000000000000000000000000000000 --- a/cosyvoice/transformer/attention.py +++ /dev/null @@ -1,322 +0,0 @@ -# Copyright (c) 2019 Shigeki Karita -# 2020 Mobvoi Inc (Binbin Zhang) -# 2022 Xingchen Song (sxc19@mails.tsinghua.edu.cn) -# 2024 Alibaba Inc (Xiang Lyu) -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Multi-Head Attention layer definition.""" - -import math -from typing import Tuple - -import torch -from torch import nn - - -class MultiHeadedAttention(nn.Module): - """Multi-Head Attention layer. - - Args: - n_head (int): The number of heads. - n_feat (int): The number of features. - dropout_rate (float): Dropout rate. - - """ - - def __init__( - self, n_head: int, n_feat: int, dropout_rate: float, key_bias: bool = True - ): - """Construct an MultiHeadedAttention object.""" - super().__init__() - assert n_feat % n_head == 0 - # We assume d_v always equals d_k - self.d_k = n_feat // n_head - self.h = n_head - self.linear_q = nn.Linear(n_feat, n_feat) - self.linear_k = nn.Linear(n_feat, n_feat, bias=key_bias) - self.linear_v = nn.Linear(n_feat, n_feat) - self.linear_out = nn.Linear(n_feat, n_feat) - self.dropout = nn.Dropout(p=dropout_rate) - - def forward_qkv( - self, query: torch.Tensor, key: torch.Tensor, value: torch.Tensor - ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: - """Transform query, key and value. - - Args: - query (torch.Tensor): Query tensor (#batch, time1, size). - key (torch.Tensor): Key tensor (#batch, time2, size). - value (torch.Tensor): Value tensor (#batch, time2, size). - - Returns: - torch.Tensor: Transformed query tensor, size - (#batch, n_head, time1, d_k). - torch.Tensor: Transformed key tensor, size - (#batch, n_head, time2, d_k). - torch.Tensor: Transformed value tensor, size - (#batch, n_head, time2, d_k). - - """ - n_batch = query.size(0) - q = self.linear_q(query).view(n_batch, -1, self.h, self.d_k) - k = self.linear_k(key).view(n_batch, -1, self.h, self.d_k) - v = self.linear_v(value).view(n_batch, -1, self.h, self.d_k) - q = q.transpose(1, 2) # (batch, head, time1, d_k) - k = k.transpose(1, 2) # (batch, head, time2, d_k) - v = v.transpose(1, 2) # (batch, head, time2, d_k) - - return q, k, v - - def forward_attention( - self, - value: torch.Tensor, - scores: torch.Tensor, - mask: torch.Tensor = torch.ones((0, 0, 0), dtype=torch.bool), - ) -> torch.Tensor: - """Compute attention context vector. - - Args: - value (torch.Tensor): Transformed value, size - (#batch, n_head, time2, d_k). - scores (torch.Tensor): Attention score, size - (#batch, n_head, time1, time2). - mask (torch.Tensor): Mask, size (#batch, 1, time2) or - (#batch, time1, time2), (0, 0, 0) means fake mask. - - Returns: - torch.Tensor: Transformed value (#batch, time1, d_model) - weighted by the attention score (#batch, time1, time2). - - """ - n_batch = value.size(0) - # NOTE(xcsong): When will `if mask.size(2) > 0` be True? - # 1. onnx(16/4) [WHY? Because we feed real cache & real mask for the - # 1st chunk to ease the onnx export.] - # 2. pytorch training - if mask.size(2) > 0: # time2 > 0 - mask = mask.unsqueeze(1).eq(0) # (batch, 1, *, time2) - # For last chunk, time2 might be larger than scores.size(-1) - mask = mask[:, :, :, : scores.size(-1)] # (batch, 1, *, time2) - scores = scores.masked_fill(mask, -float("inf")) - attn = torch.softmax(scores, dim=-1).masked_fill( - mask, 0.0 - ) # (batch, head, time1, time2) - # NOTE(xcsong): When will `if mask.size(2) > 0` be False? - # 1. onnx(16/-1, -1/-1, 16/0) - # 2. jit (16/-1, -1/-1, 16/0, 16/4) - else: - attn = torch.softmax(scores, dim=-1) # (batch, head, time1, time2) - - p_attn = self.dropout(attn) - x = torch.matmul(p_attn, value) # (batch, head, time1, d_k) - x = ( - x.transpose(1, 2).contiguous().view(n_batch, -1, self.h * self.d_k) - ) # (batch, time1, d_model) - - return self.linear_out(x) # (batch, time1, d_model) - - def forward( - self, - query: torch.Tensor, - key: torch.Tensor, - value: torch.Tensor, - mask: torch.Tensor = torch.ones((0, 0, 0), dtype=torch.bool), - pos_emb: torch.Tensor = torch.empty(0), - cache: torch.Tensor = torch.zeros((0, 0, 0, 0)), - ) -> Tuple[torch.Tensor, torch.Tensor]: - """Compute scaled dot product attention. - - Args: - query (torch.Tensor): Query tensor (#batch, time1, size). - key (torch.Tensor): Key tensor (#batch, time2, size). - value (torch.Tensor): Value tensor (#batch, time2, size). - mask (torch.Tensor): Mask tensor (#batch, 1, time2) or - (#batch, time1, time2). - 1.When applying cross attention between decoder and encoder, - the batch padding mask for input is in (#batch, 1, T) shape. - 2.When applying self attention of encoder, - the mask is in (#batch, T, T) shape. - 3.When applying self attention of decoder, - the mask is in (#batch, L, L) shape. - 4.If the different position in decoder see different block - of the encoder, such as Mocha, the passed in mask could be - in (#batch, L, T) shape. But there is no such case in current - CosyVoice. - cache (torch.Tensor): Cache tensor (1, head, cache_t, d_k * 2), - where `cache_t == chunk_size * num_decoding_left_chunks` - and `head * d_k == size` - - - Returns: - torch.Tensor: Output tensor (#batch, time1, d_model). - torch.Tensor: Cache tensor (1, head, cache_t + time1, d_k * 2) - where `cache_t == chunk_size * num_decoding_left_chunks` - and `head * d_k == size` - - """ - q, k, v = self.forward_qkv(query, key, value) - - # NOTE(xcsong): - # when export onnx model, for 1st chunk, we feed - # cache(1, head, 0, d_k * 2) (16/-1, -1/-1, 16/0 mode) - # or cache(1, head, real_cache_t, d_k * 2) (16/4 mode). - # In all modes, `if cache.size(0) > 0` will alwayse be `True` - # and we will always do splitting and - # concatnation(this will simplify onnx export). Note that - # it's OK to concat & split zero-shaped tensors(see code below). - # when export jit model, for 1st chunk, we always feed - # cache(0, 0, 0, 0) since jit supports dynamic if-branch. - # >>> a = torch.ones((1, 2, 0, 4)) - # >>> b = torch.ones((1, 2, 3, 4)) - # >>> c = torch.cat((a, b), dim=2) - # >>> torch.equal(b, c) # True - # >>> d = torch.split(a, 2, dim=-1) - # >>> torch.equal(d[0], d[1]) # True - if cache.size(0) > 0: - key_cache, value_cache = torch.split(cache, cache.size(-1) // 2, dim=-1) - k = torch.cat([key_cache, k], dim=2) - v = torch.cat([value_cache, v], dim=2) - # NOTE(xcsong): We do cache slicing in encoder.forward_chunk, since it's - # non-trivial to calculate `next_cache_start` here. - new_cache = torch.cat((k, v), dim=-1) - - scores = torch.matmul(q, k.transpose(-2, -1)) / math.sqrt(self.d_k) - return self.forward_attention(v, scores, mask), new_cache - - -class RelPositionMultiHeadedAttention(MultiHeadedAttention): - """Multi-Head Attention layer with relative position encoding. - Paper: https://arxiv.org/abs/1901.02860 - Args: - n_head (int): The number of heads. - n_feat (int): The number of features. - dropout_rate (float): Dropout rate. - """ - - def __init__( - self, n_head: int, n_feat: int, dropout_rate: float, key_bias: bool = True - ): - """Construct an RelPositionMultiHeadedAttention object.""" - super().__init__(n_head, n_feat, dropout_rate, key_bias) - # linear transformation for positional encoding - self.linear_pos = nn.Linear(n_feat, n_feat, bias=False) - # these two learnable bias are used in matrix c and matrix d - # as described in https://arxiv.org/abs/1901.02860 Section 3.3 - self.pos_bias_u = nn.Parameter(torch.Tensor(self.h, self.d_k)) - self.pos_bias_v = nn.Parameter(torch.Tensor(self.h, self.d_k)) - torch.nn.init.xavier_uniform_(self.pos_bias_u) - torch.nn.init.xavier_uniform_(self.pos_bias_v) - - def rel_shift(self, x: torch.Tensor) -> torch.Tensor: - """Compute relative positional encoding. - - Args: - x (torch.Tensor): Input tensor (batch, head, time1, 2*time1-1). - time1 means the length of query vector. - - Returns: - torch.Tensor: Output tensor. - - """ - zero_pad = torch.zeros( - (x.size()[0], x.size()[1], x.size()[2], 1), device=x.device, dtype=x.dtype - ) - x_padded = torch.cat([zero_pad, x], dim=-1) - - x_padded = x_padded.view(x.size()[0], x.size()[1], x.size(3) + 1, x.size(2)) - x = x_padded[:, :, 1:].view_as(x)[ - :, :, :, : x.size(-1) // 2 + 1 - ] # only keep the positions from 0 to time2 - return x - - def forward( - self, - query: torch.Tensor, - key: torch.Tensor, - value: torch.Tensor, - mask: torch.Tensor = torch.ones((0, 0, 0), dtype=torch.bool), - pos_emb: torch.Tensor = torch.empty(0), - cache: torch.Tensor = torch.zeros((0, 0, 0, 0)), - ) -> Tuple[torch.Tensor, torch.Tensor]: - """Compute 'Scaled Dot Product Attention' with rel. positional encoding. - Args: - query (torch.Tensor): Query tensor (#batch, time1, size). - key (torch.Tensor): Key tensor (#batch, time2, size). - value (torch.Tensor): Value tensor (#batch, time2, size). - mask (torch.Tensor): Mask tensor (#batch, 1, time2) or - (#batch, time1, time2), (0, 0, 0) means fake mask. - pos_emb (torch.Tensor): Positional embedding tensor - (#batch, time2, size). - cache (torch.Tensor): Cache tensor (1, head, cache_t, d_k * 2), - where `cache_t == chunk_size * num_decoding_left_chunks` - and `head * d_k == size` - Returns: - torch.Tensor: Output tensor (#batch, time1, d_model). - torch.Tensor: Cache tensor (1, head, cache_t + time1, d_k * 2) - where `cache_t == chunk_size * num_decoding_left_chunks` - and `head * d_k == size` - """ - q, k, v = self.forward_qkv(query, key, value) - q = q.transpose(1, 2) # (batch, time1, head, d_k) - - # NOTE(xcsong): - # when export onnx model, for 1st chunk, we feed - # cache(1, head, 0, d_k * 2) (16/-1, -1/-1, 16/0 mode) - # or cache(1, head, real_cache_t, d_k * 2) (16/4 mode). - # In all modes, `if cache.size(0) > 0` will alwayse be `True` - # and we will always do splitting and - # concatnation(this will simplify onnx export). Note that - # it's OK to concat & split zero-shaped tensors(see code below). - # when export jit model, for 1st chunk, we always feed - # cache(0, 0, 0, 0) since jit supports dynamic if-branch. - # >>> a = torch.ones((1, 2, 0, 4)) - # >>> b = torch.ones((1, 2, 3, 4)) - # >>> c = torch.cat((a, b), dim=2) - # >>> torch.equal(b, c) # True - # >>> d = torch.split(a, 2, dim=-1) - # >>> torch.equal(d[0], d[1]) # True - if cache.size(0) > 0: - key_cache, value_cache = torch.split(cache, cache.size(-1) // 2, dim=-1) - k = torch.cat([key_cache, k], dim=2) - v = torch.cat([value_cache, v], dim=2) - # NOTE(xcsong): We do cache slicing in encoder.forward_chunk, since it's - # non-trivial to calculate `next_cache_start` here. - new_cache = torch.cat((k, v), dim=-1) - - n_batch_pos = pos_emb.size(0) - p = self.linear_pos(pos_emb).view(n_batch_pos, -1, self.h, self.d_k) - p = p.transpose(1, 2) # (batch, head, time1, d_k) - - # (batch, head, time1, d_k) - q_with_bias_u = (q + self.pos_bias_u).transpose(1, 2) - # (batch, head, time1, d_k) - q_with_bias_v = (q + self.pos_bias_v).transpose(1, 2) - - # compute attention score - # first compute matrix a and matrix c - # as described in https://arxiv.org/abs/1901.02860 Section 3.3 - # (batch, head, time1, time2) - matrix_ac = torch.matmul(q_with_bias_u, k.transpose(-2, -1)) - - # compute matrix b and matrix d - # (batch, head, time1, time2) - matrix_bd = torch.matmul(q_with_bias_v, p.transpose(-2, -1)) - # NOTE(Xiang Lyu): Keep rel_shift since espnet rel_pos_emb is used - if matrix_ac.shape != matrix_bd.shape: - matrix_bd = self.rel_shift(matrix_bd) - - scores = (matrix_ac + matrix_bd) / math.sqrt( - self.d_k - ) # (batch, head, time1, time2) - - return self.forward_attention(v, scores, mask), new_cache diff --git a/cosyvoice/transformer/convolution.py b/cosyvoice/transformer/convolution.py deleted file mode 100644 index ef3dfc2a88146db06ea69e897d6aab9c28895bdb..0000000000000000000000000000000000000000 --- a/cosyvoice/transformer/convolution.py +++ /dev/null @@ -1,147 +0,0 @@ -# Copyright (c) 2020 Mobvoi Inc. (authors: Binbin Zhang, Di Wu) -# 2024 Alibaba Inc (Xiang Lyu) -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# Modified from ESPnet(https://github.com/espnet/espnet) -"""ConvolutionModule definition.""" - -from typing import Tuple - -import torch -from torch import nn - - -class ConvolutionModule(nn.Module): - """ConvolutionModule in Conformer model.""" - - def __init__( - self, - channels: int, - kernel_size: int = 15, - activation: nn.Module = nn.ReLU(), - norm: str = "batch_norm", - causal: bool = False, - bias: bool = True, - ): - """Construct an ConvolutionModule object. - Args: - channels (int): The number of channels of conv layers. - kernel_size (int): Kernel size of conv layers. - causal (int): Whether use causal convolution or not - """ - super().__init__() - - self.pointwise_conv1 = nn.Conv1d( - channels, - 2 * channels, - kernel_size=1, - stride=1, - padding=0, - bias=bias, - ) - # self.lorder is used to distinguish if it's a causal convolution, - # if self.lorder > 0: it's a causal convolution, the input will be - # padded with self.lorder frames on the left in forward. - # else: it's a symmetrical convolution - if causal: - padding = 0 - self.lorder = kernel_size - 1 - else: - # kernel_size should be an odd number for none causal convolution - assert (kernel_size - 1) % 2 == 0 - padding = (kernel_size - 1) // 2 - self.lorder = 0 - self.depthwise_conv = nn.Conv1d( - channels, - channels, - kernel_size, - stride=1, - padding=padding, - groups=channels, - bias=bias, - ) - - assert norm in ["batch_norm", "layer_norm"] - if norm == "batch_norm": - self.use_layer_norm = False - self.norm = nn.BatchNorm1d(channels) - else: - self.use_layer_norm = True - self.norm = nn.LayerNorm(channels) - - self.pointwise_conv2 = nn.Conv1d( - channels, - channels, - kernel_size=1, - stride=1, - padding=0, - bias=bias, - ) - self.activation = activation - - def forward( - self, - x: torch.Tensor, - mask_pad: torch.Tensor = torch.ones((0, 0, 0), dtype=torch.bool), - cache: torch.Tensor = torch.zeros((0, 0, 0)), - ) -> Tuple[torch.Tensor, torch.Tensor]: - """Compute convolution module. - Args: - x (torch.Tensor): Input tensor (#batch, time, channels). - mask_pad (torch.Tensor): used for batch padding (#batch, 1, time), - (0, 0, 0) means fake mask. - cache (torch.Tensor): left context cache, it is only - used in causal convolution (#batch, channels, cache_t), - (0, 0, 0) meas fake cache. - Returns: - torch.Tensor: Output tensor (#batch, time, channels). - """ - # exchange the temporal dimension and the feature dimension - x = x.transpose(1, 2) # (#batch, channels, time) - - # mask batch padding - if mask_pad.size(2) > 0: # time > 0 - x.masked_fill_(~mask_pad, 0.0) - - if self.lorder > 0: - if cache.size(2) == 0: # cache_t == 0 - x = nn.functional.pad(x, (self.lorder, 0), "constant", 0.0) - else: - assert cache.size(0) == x.size(0) # equal batch - assert cache.size(1) == x.size(1) # equal channel - x = torch.cat((cache, x), dim=2) - assert x.size(2) > self.lorder - new_cache = x[:, :, -self.lorder :] - else: - # It's better we just return None if no cache is required, - # However, for JIT export, here we just fake one tensor instead of - # None. - new_cache = torch.zeros((0, 0, 0), dtype=x.dtype, device=x.device) - - # GLU mechanism - x = self.pointwise_conv1(x) # (batch, 2*channel, dim) - x = nn.functional.glu(x, dim=1) # (batch, channel, dim) - - # 1D Depthwise Conv - x = self.depthwise_conv(x) - if self.use_layer_norm: - x = x.transpose(1, 2) - x = self.activation(self.norm(x)) - if self.use_layer_norm: - x = x.transpose(1, 2) - x = self.pointwise_conv2(x) - # mask batch padding - if mask_pad.size(2) > 0: # time > 0 - x.masked_fill_(~mask_pad, 0.0) - - return x.transpose(1, 2), new_cache diff --git a/cosyvoice/transformer/decoder.py b/cosyvoice/transformer/decoder.py deleted file mode 100644 index db1e30ce3ded1724828a50d12f798ee41df27ef0..0000000000000000000000000000000000000000 --- a/cosyvoice/transformer/decoder.py +++ /dev/null @@ -1,418 +0,0 @@ -# Copyright (c) 2021 Mobvoi Inc. (authors: Binbin Zhang, Di Wu) -# 2024 Alibaba Inc (Xiang Lyu) -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# Modified from ESPnet(https://github.com/espnet/espnet) -"""Decoder definition.""" -from typing import Tuple, List, Optional - -import torch -import torch.utils.checkpoint as ckpt -import logging - -from cosyvoice.transformer.decoder_layer import DecoderLayer -from cosyvoice.transformer.positionwise_feed_forward import ( - PositionwiseFeedForward, -) -from cosyvoice.utils.class_utils import ( - COSYVOICE_EMB_CLASSES, - COSYVOICE_ATTENTION_CLASSES, - COSYVOICE_ACTIVATION_CLASSES, -) -from cosyvoice.utils.mask import subsequent_mask, make_pad_mask - - -class TransformerDecoder(torch.nn.Module): - """Base class of Transfomer decoder module. - Args: - vocab_size: output dim - encoder_output_size: dimension of attention - attention_heads: the number of heads of multi head attention - linear_units: the hidden units number of position-wise feedforward - num_blocks: the number of decoder blocks - dropout_rate: dropout rate - self_attention_dropout_rate: dropout rate for attention - input_layer: input layer type - use_output_layer: whether to use output layer - pos_enc_class: PositionalEncoding or ScaledPositionalEncoding - normalize_before: - True: use layer_norm before each sub-block of a layer. - False: use layer_norm after each sub-block of a layer. - src_attention: if false, encoder-decoder cross attention is not - applied, such as CIF model - key_bias: whether use bias in attention.linear_k, False for whisper models. - gradient_checkpointing: rerunning a forward-pass segment for each - checkpointed segment during backward. - tie_word_embedding: Tie or clone module weights depending of whether we are - using TorchScript or not - """ - - def __init__( - self, - vocab_size: int, - encoder_output_size: int, - attention_heads: int = 4, - linear_units: int = 2048, - num_blocks: int = 6, - dropout_rate: float = 0.1, - positional_dropout_rate: float = 0.1, - self_attention_dropout_rate: float = 0.0, - src_attention_dropout_rate: float = 0.0, - input_layer: str = "embed", - use_output_layer: bool = True, - normalize_before: bool = True, - src_attention: bool = True, - key_bias: bool = True, - activation_type: str = "relu", - gradient_checkpointing: bool = False, - tie_word_embedding: bool = False, - ): - super().__init__() - attention_dim = encoder_output_size - activation = COSYVOICE_ACTIVATION_CLASSES[activation_type]() - - self.embed = torch.nn.Sequential( - ( - torch.nn.Identity() - if input_layer == "no_pos" - else torch.nn.Embedding(vocab_size, attention_dim) - ), - COSYVOICE_EMB_CLASSES[input_layer](attention_dim, positional_dropout_rate), - ) - - self.normalize_before = normalize_before - self.after_norm = torch.nn.LayerNorm(attention_dim, eps=1e-5) - self.use_output_layer = use_output_layer - if use_output_layer: - self.output_layer = torch.nn.Linear(attention_dim, vocab_size) - else: - self.output_layer = torch.nn.Identity() - self.num_blocks = num_blocks - self.decoders = torch.nn.ModuleList( - [ - DecoderLayer( - attention_dim, - COSYVOICE_ATTENTION_CLASSES["selfattn"]( - attention_heads, - attention_dim, - self_attention_dropout_rate, - key_bias, - ), - ( - COSYVOICE_ATTENTION_CLASSES["selfattn"]( - attention_heads, - attention_dim, - src_attention_dropout_rate, - key_bias, - ) - if src_attention - else None - ), - PositionwiseFeedForward( - attention_dim, linear_units, dropout_rate, activation - ), - dropout_rate, - normalize_before, - ) - for _ in range(self.num_blocks) - ] - ) - - self.gradient_checkpointing = gradient_checkpointing - self.tie_word_embedding = tie_word_embedding - - def forward( - self, - memory: torch.Tensor, - memory_mask: torch.Tensor, - ys_in_pad: torch.Tensor, - ys_in_lens: torch.Tensor, - r_ys_in_pad: torch.Tensor = torch.empty(0), - reverse_weight: float = 0.0, - ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: - """Forward decoder. - Args: - memory: encoded memory, float32 (batch, maxlen_in, feat) - memory_mask: encoder memory mask, (batch, 1, maxlen_in) - ys_in_pad: padded input token ids, int64 (batch, maxlen_out) - ys_in_lens: input lengths of this batch (batch) - r_ys_in_pad: not used in transformer decoder, in order to unify api - with bidirectional decoder - reverse_weight: not used in transformer decoder, in order to unify - api with bidirectional decode - Returns: - (tuple): tuple containing: - x: decoded token score before softmax (batch, maxlen_out, - vocab_size) if use_output_layer is True, - torch.tensor(0.0), in order to unify api with bidirectional decoder - olens: (batch, ) - NOTE(xcsong): - We pass the `__call__` method of the modules instead of `forward` to the - checkpointing API because `__call__` attaches all the hooks of the module. - https://discuss.pytorch.org/t/any-different-between-model-input-and-model-forward-input/3690/2 - """ - tgt = ys_in_pad - maxlen = tgt.size(1) - # tgt_mask: (B, 1, L) - tgt_mask = ~make_pad_mask(ys_in_lens, maxlen).unsqueeze(1) - tgt_mask = tgt_mask.to(tgt.device) - # m: (1, L, L) - m = subsequent_mask(tgt_mask.size(-1), device=tgt_mask.device).unsqueeze(0) - # tgt_mask: (B, L, L) - tgt_mask = tgt_mask & m - x, _ = self.embed(tgt) - if self.gradient_checkpointing and self.training: - x = self.forward_layers_checkpointed(x, tgt_mask, memory, memory_mask) - else: - x = self.forward_layers(x, tgt_mask, memory, memory_mask) - if self.normalize_before: - x = self.after_norm(x) - if self.use_output_layer: - x = self.output_layer(x) - olens = tgt_mask.sum(1) - return x, torch.tensor(0.0), olens - - def forward_layers( - self, - x: torch.Tensor, - tgt_mask: torch.Tensor, - memory: torch.Tensor, - memory_mask: torch.Tensor, - ) -> torch.Tensor: - for layer in self.decoders: - x, tgt_mask, memory, memory_mask = layer(x, tgt_mask, memory, memory_mask) - return x - - @torch.jit.unused - def forward_layers_checkpointed( - self, - x: torch.Tensor, - tgt_mask: torch.Tensor, - memory: torch.Tensor, - memory_mask: torch.Tensor, - ) -> torch.Tensor: - for layer in self.decoders: - x, tgt_mask, memory, memory_mask = ckpt.checkpoint( - layer.__call__, x, tgt_mask, memory, memory_mask - ) - return x - - def forward_one_step( - self, - memory: torch.Tensor, - memory_mask: torch.Tensor, - tgt: torch.Tensor, - tgt_mask: torch.Tensor, - cache: Optional[List[torch.Tensor]] = None, - ) -> Tuple[torch.Tensor, List[torch.Tensor]]: - """Forward one step. - This is only used for decoding. - Args: - memory: encoded memory, float32 (batch, maxlen_in, feat) - memory_mask: encoded memory mask, (batch, 1, maxlen_in) - tgt: input token ids, int64 (batch, maxlen_out) - tgt_mask: input token mask, (batch, maxlen_out) - dtype=torch.uint8 in PyTorch 1.2- - dtype=torch.bool in PyTorch 1.2+ (include 1.2) - cache: cached output list of (batch, max_time_out-1, size) - Returns: - y, cache: NN output value and cache per `self.decoders`. - y.shape` is (batch, maxlen_out, token) - """ - x, _ = self.embed(tgt) - new_cache = [] - for i, decoder in enumerate(self.decoders): - if cache is None: - c = None - else: - c = cache[i] - x, tgt_mask, memory, memory_mask = decoder( - x, tgt_mask, memory, memory_mask, cache=c - ) - new_cache.append(x) - if self.normalize_before: - y = self.after_norm(x[:, -1]) - else: - y = x[:, -1] - if self.use_output_layer: - y = torch.log_softmax(self.output_layer(y), dim=-1) - return y, new_cache - - def tie_or_clone_weights(self, jit_mode: bool = True): - """Tie or clone module weights (between word_emb and output_layer) - depending of whether we are using TorchScript or not""" - if not self.use_output_layer: - return - if jit_mode: - logging.info("clone emb.weight to output.weight") - self.output_layer.weight = torch.nn.Parameter(self.embed[0].weight.clone()) - else: - logging.info("tie emb.weight with output.weight") - self.output_layer.weight = self.embed[0].weight - - if getattr(self.output_layer, "bias", None) is not None: - self.output_layer.bias.data = torch.nn.functional.pad( - self.output_layer.bias.data, - ( - 0, - self.output_layer.weight.shape[0] - self.output_layer.bias.shape[0], - ), - "constant", - 0, - ) - - -class BiTransformerDecoder(torch.nn.Module): - """Base class of Transfomer decoder module. - Args: - vocab_size: output dim - encoder_output_size: dimension of attention - attention_heads: the number of heads of multi head attention - linear_units: the hidden units number of position-wise feedforward - num_blocks: the number of decoder blocks - r_num_blocks: the number of right to left decoder blocks - dropout_rate: dropout rate - self_attention_dropout_rate: dropout rate for attention - input_layer: input layer type - use_output_layer: whether to use output layer - pos_enc_class: PositionalEncoding or ScaledPositionalEncoding - normalize_before: - True: use layer_norm before each sub-block of a layer. - False: use layer_norm after each sub-block of a layer. - key_bias: whether use bias in attention.linear_k, False for whisper models. - """ - - def __init__( - self, - vocab_size: int, - encoder_output_size: int, - attention_heads: int = 4, - linear_units: int = 2048, - num_blocks: int = 6, - r_num_blocks: int = 0, - dropout_rate: float = 0.1, - positional_dropout_rate: float = 0.1, - self_attention_dropout_rate: float = 0.0, - src_attention_dropout_rate: float = 0.0, - input_layer: str = "embed", - use_output_layer: bool = True, - normalize_before: bool = True, - key_bias: bool = True, - gradient_checkpointing: bool = False, - tie_word_embedding: bool = False, - ): - - super().__init__() - self.tie_word_embedding = tie_word_embedding - self.left_decoder = TransformerDecoder( - vocab_size, - encoder_output_size, - attention_heads, - linear_units, - num_blocks, - dropout_rate, - positional_dropout_rate, - self_attention_dropout_rate, - src_attention_dropout_rate, - input_layer, - use_output_layer, - normalize_before, - key_bias=key_bias, - gradient_checkpointing=gradient_checkpointing, - tie_word_embedding=tie_word_embedding, - ) - - self.right_decoder = TransformerDecoder( - vocab_size, - encoder_output_size, - attention_heads, - linear_units, - r_num_blocks, - dropout_rate, - positional_dropout_rate, - self_attention_dropout_rate, - src_attention_dropout_rate, - input_layer, - use_output_layer, - normalize_before, - key_bias=key_bias, - gradient_checkpointing=gradient_checkpointing, - tie_word_embedding=tie_word_embedding, - ) - - def forward( - self, - memory: torch.Tensor, - memory_mask: torch.Tensor, - ys_in_pad: torch.Tensor, - ys_in_lens: torch.Tensor, - r_ys_in_pad: torch.Tensor, - reverse_weight: float = 0.0, - ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: - """Forward decoder. - Args: - memory: encoded memory, float32 (batch, maxlen_in, feat) - memory_mask: encoder memory mask, (batch, 1, maxlen_in) - ys_in_pad: padded input token ids, int64 (batch, maxlen_out) - ys_in_lens: input lengths of this batch (batch) - r_ys_in_pad: padded input token ids, int64 (batch, maxlen_out), - used for right to left decoder - reverse_weight: used for right to left decoder - Returns: - (tuple): tuple containing: - x: decoded token score before softmax (batch, maxlen_out, - vocab_size) if use_output_layer is True, - r_x: x: decoded token score (right to left decoder) - before softmax (batch, maxlen_out, vocab_size) - if use_output_layer is True, - olens: (batch, ) - """ - l_x, _, olens = self.left_decoder(memory, memory_mask, ys_in_pad, ys_in_lens) - r_x = torch.tensor(0.0) - if reverse_weight > 0.0: - r_x, _, olens = self.right_decoder( - memory, memory_mask, r_ys_in_pad, ys_in_lens - ) - return l_x, r_x, olens - - def forward_one_step( - self, - memory: torch.Tensor, - memory_mask: torch.Tensor, - tgt: torch.Tensor, - tgt_mask: torch.Tensor, - cache: Optional[List[torch.Tensor]] = None, - ) -> Tuple[torch.Tensor, List[torch.Tensor]]: - """Forward one step. - This is only used for decoding. - Args: - memory: encoded memory, float32 (batch, maxlen_in, feat) - memory_mask: encoded memory mask, (batch, 1, maxlen_in) - tgt: input token ids, int64 (batch, maxlen_out) - tgt_mask: input token mask, (batch, maxlen_out) - dtype=torch.uint8 in PyTorch 1.2- - dtype=torch.bool in PyTorch 1.2+ (include 1.2) - cache: cached output list of (batch, max_time_out-1, size) - Returns: - y, cache: NN output value and cache per `self.decoders`. - y.shape` is (batch, maxlen_out, token) - """ - return self.left_decoder.forward_one_step( - memory, memory_mask, tgt, tgt_mask, cache - ) - - def tie_or_clone_weights(self, jit_mode: bool = True): - """Tie or clone module weights (between word_emb and output_layer) - depending of whether we are using TorchScript or not""" - self.left_decoder.tie_or_clone_weights(jit_mode) - self.right_decoder.tie_or_clone_weights(jit_mode) diff --git a/cosyvoice/transformer/decoder_layer.py b/cosyvoice/transformer/decoder_layer.py deleted file mode 100644 index 6f1420732591b96ba2f09335e96dee99eae71003..0000000000000000000000000000000000000000 --- a/cosyvoice/transformer/decoder_layer.py +++ /dev/null @@ -1,132 +0,0 @@ -# Copyright (c) 2019 Shigeki Karita -# 2020 Mobvoi Inc (Binbin Zhang) -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Decoder self-attention layer definition.""" -from typing import Optional, Tuple - -import torch -from torch import nn - - -class DecoderLayer(nn.Module): - """Single decoder layer module. - - Args: - size (int): Input dimension. - self_attn (torch.nn.Module): Self-attention module instance. - `MultiHeadedAttention` instance can be used as the argument. - src_attn (torch.nn.Module): Inter-attention module instance. - `MultiHeadedAttention` instance can be used as the argument. - If `None` is passed, Inter-attention is not used, such as - CIF, GPT, and other decoder only model. - feed_forward (torch.nn.Module): Feed-forward module instance. - `PositionwiseFeedForward` instance can be used as the argument. - dropout_rate (float): Dropout rate. - normalize_before (bool): - True: use layer_norm before each sub-block. - False: to use layer_norm after each sub-block. - """ - - def __init__( - self, - size: int, - self_attn: nn.Module, - src_attn: Optional[nn.Module], - feed_forward: nn.Module, - dropout_rate: float, - normalize_before: bool = True, - ): - """Construct an DecoderLayer object.""" - super().__init__() - self.size = size - self.self_attn = self_attn - self.src_attn = src_attn - self.feed_forward = feed_forward - self.norm1 = nn.LayerNorm(size, eps=1e-5) - self.norm2 = nn.LayerNorm(size, eps=1e-5) - self.norm3 = nn.LayerNorm(size, eps=1e-5) - self.dropout = nn.Dropout(dropout_rate) - self.normalize_before = normalize_before - - def forward( - self, - tgt: torch.Tensor, - tgt_mask: torch.Tensor, - memory: torch.Tensor, - memory_mask: torch.Tensor, - cache: Optional[torch.Tensor] = None, - ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: - """Compute decoded features. - - Args: - tgt (torch.Tensor): Input tensor (#batch, maxlen_out, size). - tgt_mask (torch.Tensor): Mask for input tensor - (#batch, maxlen_out). - memory (torch.Tensor): Encoded memory - (#batch, maxlen_in, size). - memory_mask (torch.Tensor): Encoded memory mask - (#batch, maxlen_in). - cache (torch.Tensor): cached tensors. - (#batch, maxlen_out - 1, size). - - Returns: - torch.Tensor: Output tensor (#batch, maxlen_out, size). - torch.Tensor: Mask for output tensor (#batch, maxlen_out). - torch.Tensor: Encoded memory (#batch, maxlen_in, size). - torch.Tensor: Encoded memory mask (#batch, maxlen_in). - - """ - residual = tgt - if self.normalize_before: - tgt = self.norm1(tgt) - - if cache is None: - tgt_q = tgt - tgt_q_mask = tgt_mask - else: - # compute only the last frame query keeping dim: max_time_out -> 1 - assert cache.shape == ( - tgt.shape[0], - tgt.shape[1] - 1, - self.size, - ), "{cache.shape} == {(tgt.shape[0], tgt.shape[1] - 1, self.size)}" - tgt_q = tgt[:, -1:, :] - residual = residual[:, -1:, :] - tgt_q_mask = tgt_mask[:, -1:, :] - - x = residual + self.dropout(self.self_attn(tgt_q, tgt, tgt, tgt_q_mask)[0]) - if not self.normalize_before: - x = self.norm1(x) - - if self.src_attn is not None: - residual = x - if self.normalize_before: - x = self.norm2(x) - x = residual + self.dropout( - self.src_attn(x, memory, memory, memory_mask)[0] - ) - if not self.normalize_before: - x = self.norm2(x) - - residual = x - if self.normalize_before: - x = self.norm3(x) - x = residual + self.dropout(self.feed_forward(x)) - if not self.normalize_before: - x = self.norm3(x) - - if cache is not None: - x = torch.cat([cache, x], dim=1) - - return x, tgt_mask, memory, memory_mask diff --git a/cosyvoice/transformer/embedding.py b/cosyvoice/transformer/embedding.py deleted file mode 100644 index 7d4f6e1af263da2b9144648208e3f2a659220d16..0000000000000000000000000000000000000000 --- a/cosyvoice/transformer/embedding.py +++ /dev/null @@ -1,293 +0,0 @@ -# Copyright (c) 2020 Mobvoi Inc. (authors: Binbin Zhang, Di Wu) -# 2024 Alibaba Inc (Xiang Lyu) -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# Modified from ESPnet(https://github.com/espnet/espnet) -"""Positonal Encoding Module.""" - -import math -from typing import Tuple, Union - -import torch -import torch.nn.functional as F -import numpy as np - - -class PositionalEncoding(torch.nn.Module): - """Positional encoding. - - :param int d_model: embedding dim - :param float dropout_rate: dropout rate - :param int max_len: maximum input length - - PE(pos, 2i) = sin(pos/(10000^(2i/dmodel))) - PE(pos, 2i+1) = cos(pos/(10000^(2i/dmodel))) - """ - - def __init__( - self, - d_model: int, - dropout_rate: float, - max_len: int = 5000, - reverse: bool = False, - ): - """Construct an PositionalEncoding object.""" - super().__init__() - self.d_model = d_model - self.xscale = math.sqrt(self.d_model) - self.dropout = torch.nn.Dropout(p=dropout_rate) - self.max_len = max_len - - self.pe = torch.zeros(self.max_len, self.d_model) - position = torch.arange(0, self.max_len, dtype=torch.float32).unsqueeze(1) - div_term = torch.exp( - torch.arange(0, self.d_model, 2, dtype=torch.float32) - * -(math.log(10000.0) / self.d_model) - ) - self.pe[:, 0::2] = torch.sin(position * div_term) - self.pe[:, 1::2] = torch.cos(position * div_term) - self.pe = self.pe.unsqueeze(0) - - def forward( - self, x: torch.Tensor, offset: Union[int, torch.Tensor] = 0 - ) -> Tuple[torch.Tensor, torch.Tensor]: - """Add positional encoding. - - Args: - x (torch.Tensor): Input. Its shape is (batch, time, ...) - offset (int, torch.tensor): position offset - - Returns: - torch.Tensor: Encoded tensor. Its shape is (batch, time, ...) - torch.Tensor: for compatibility to RelPositionalEncoding - """ - - self.pe = self.pe.to(x.device) - pos_emb = self.position_encoding(offset, x.size(1), False) - x = x * self.xscale + pos_emb - return self.dropout(x), self.dropout(pos_emb) - - def position_encoding( - self, offset: Union[int, torch.Tensor], size: int, apply_dropout: bool = True - ) -> torch.Tensor: - """For getting encoding in a streaming fashion - - Attention!!!!! - we apply dropout only once at the whole utterance level in a none - streaming way, but will call this function several times with - increasing input size in a streaming scenario, so the dropout will - be applied several times. - - Args: - offset (int or torch.tensor): start offset - size (int): required size of position encoding - - Returns: - torch.Tensor: Corresponding encoding - """ - # How to subscript a Union type: - # https://github.com/pytorch/pytorch/issues/69434 - if isinstance(offset, int): - assert offset + size <= self.max_len - pos_emb = self.pe[:, offset : offset + size] - elif isinstance(offset, torch.Tensor) and offset.dim() == 0: # scalar - assert offset + size <= self.max_len - pos_emb = self.pe[:, offset : offset + size] - else: # for batched streaming decoding on GPU - assert torch.max(offset) + size <= self.max_len - index = offset.unsqueeze(1) + torch.arange(0, size).to( - offset.device - ) # B X T - flag = index > 0 - # remove negative offset - index = index * flag - pos_emb = F.embedding(index, self.pe[0]) # B X T X d_model - - if apply_dropout: - pos_emb = self.dropout(pos_emb) - return pos_emb - - -class RelPositionalEncoding(PositionalEncoding): - """Relative positional encoding module. - See : Appendix B in https://arxiv.org/abs/1901.02860 - Args: - d_model (int): Embedding dimension. - dropout_rate (float): Dropout rate. - max_len (int): Maximum input length. - """ - - def __init__(self, d_model: int, dropout_rate: float, max_len: int = 5000): - """Initialize class.""" - super().__init__(d_model, dropout_rate, max_len, reverse=True) - - def forward( - self, x: torch.Tensor, offset: Union[int, torch.Tensor] = 0 - ) -> Tuple[torch.Tensor, torch.Tensor]: - """Compute positional encoding. - Args: - x (torch.Tensor): Input tensor (batch, time, `*`). - Returns: - torch.Tensor: Encoded tensor (batch, time, `*`). - torch.Tensor: Positional embedding tensor (1, time, `*`). - """ - self.pe = self.pe.to(x.device) - x = x * self.xscale - pos_emb = self.position_encoding(offset, x.size(1), False) - return self.dropout(x), self.dropout(pos_emb) - - -class WhisperPositionalEncoding(PositionalEncoding): - """Sinusoids position encoding used in openai-whisper.encoder""" - - def __init__(self, d_model: int, dropout_rate: float, max_len: int = 1500): - super().__init__(d_model, dropout_rate, max_len) - self.xscale = 1.0 - log_timescale_increment = np.log(10000) / (d_model // 2 - 1) - inv_timescales = torch.exp( - -log_timescale_increment * torch.arange(d_model // 2) - ) - scaled_time = ( - torch.arange(max_len)[:, np.newaxis] * inv_timescales[np.newaxis, :] - ) - pe = torch.cat([torch.sin(scaled_time), torch.cos(scaled_time)], dim=1) - delattr(self, "pe") - self.register_buffer("pe", pe.unsqueeze(0)) - - -class LearnablePositionalEncoding(PositionalEncoding): - """Learnable position encoding used in openai-whisper.decoder""" - - def __init__(self, d_model: int, dropout_rate: float, max_len: int = 448): - super().__init__(d_model, dropout_rate, max_len) - # NOTE(xcsong): overwrite self.pe & self.xscale - self.pe = torch.nn.Parameter(torch.empty(1, max_len, d_model)) - self.xscale = 1.0 - - -class NoPositionalEncoding(torch.nn.Module): - """No position encoding""" - - def __init__(self, d_model: int, dropout_rate: float): - super().__init__() - self.d_model = d_model - self.dropout = torch.nn.Dropout(p=dropout_rate) - - def forward( - self, x: torch.Tensor, offset: Union[int, torch.Tensor] = 0 - ) -> Tuple[torch.Tensor, torch.Tensor]: - """Just return zero vector for interface compatibility""" - pos_emb = torch.zeros(1, x.size(1), self.d_model).to(x.device) - return self.dropout(x), pos_emb - - def position_encoding( - self, offset: Union[int, torch.Tensor], size: int - ) -> torch.Tensor: - return torch.zeros(1, size, self.d_model) - - -class EspnetRelPositionalEncoding(torch.nn.Module): - """Relative positional encoding module (new implementation). - - Details can be found in https://github.com/espnet/espnet/pull/2816. - - See : Appendix B in https://arxiv.org/abs/1901.02860 - - Args: - d_model (int): Embedding dimension. - dropout_rate (float): Dropout rate. - max_len (int): Maximum input length. - - """ - - def __init__(self, d_model: int, dropout_rate: float, max_len: int = 5000): - """Construct an PositionalEncoding object.""" - super(EspnetRelPositionalEncoding, self).__init__() - self.d_model = d_model - self.xscale = math.sqrt(self.d_model) - self.dropout = torch.nn.Dropout(p=dropout_rate) - self.pe = None - self.extend_pe(torch.tensor(0.0).expand(1, max_len)) - - def extend_pe(self, x: torch.Tensor): - """Reset the positional encodings.""" - if self.pe is not None: - # self.pe contains both positive and negative parts - # the length of self.pe is 2 * input_len - 1 - if self.pe.size(1) >= x.size(1) * 2 - 1: - if self.pe.dtype != x.dtype or self.pe.device != x.device: - self.pe = self.pe.to(dtype=x.dtype, device=x.device) - return - # Suppose `i` means to the position of query vecotr and `j` means the - # position of key vector. We use position relative positions when keys - # are to the left (i>j) and negative relative positions otherwise (i Tuple[torch.Tensor, torch.Tensor]: - """Add positional encoding. - - Args: - x (torch.Tensor): Input tensor (batch, time, `*`). - - Returns: - torch.Tensor: Encoded tensor (batch, time, `*`). - - """ - self.extend_pe(x) - x = x * self.xscale - pos_emb = self.position_encoding(size=x.size(1), offset=offset) - return self.dropout(x), self.dropout(pos_emb) - - def position_encoding( - self, offset: Union[int, torch.Tensor], size: int - ) -> torch.Tensor: - """For getting encoding in a streaming fashion - - Attention!!!!! - we apply dropout only once at the whole utterance level in a none - streaming way, but will call this function several times with - increasing input size in a streaming scenario, so the dropout will - be applied several times. - - Args: - offset (int or torch.tensor): start offset - size (int): required size of position encoding - - Returns: - torch.Tensor: Corresponding encoding - """ - pos_emb = self.pe[ - :, - self.pe.size(1) // 2 - size + 1 : self.pe.size(1) // 2 + size, - ] - return pos_emb diff --git a/cosyvoice/transformer/encoder.py b/cosyvoice/transformer/encoder.py deleted file mode 100644 index 76d00cff324bb4c7e0da1de9ad3d4e8417205fe6..0000000000000000000000000000000000000000 --- a/cosyvoice/transformer/encoder.py +++ /dev/null @@ -1,633 +0,0 @@ -# Copyright (c) 2021 Mobvoi Inc (Binbin Zhang, Di Wu) -# 2022 Xingchen Song (sxc19@mails.tsinghua.edu.cn) -# 2024 Alibaba Inc (Xiang Lyu) -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# Modified from ESPnet(https://github.com/espnet/espnet) -"""Encoder definition.""" -from typing import Tuple -import time - -import torch -import torch.utils.checkpoint as ckpt -import torch.nn.functional as F - -from cosyvoice.transformer.convolution import ConvolutionModule -from cosyvoice.transformer.encoder_layer import ( - TransformerEncoderLayer, -) -from cosyvoice.transformer.encoder_layer import ( - ConformerEncoderLayer, -) -from cosyvoice.transformer.positionwise_feed_forward import ( - PositionwiseFeedForward, -) -from cosyvoice.utils.class_utils import ( - COSYVOICE_EMB_CLASSES, - COSYVOICE_SUBSAMPLE_CLASSES, - COSYVOICE_ATTENTION_CLASSES, - COSYVOICE_ACTIVATION_CLASSES, -) -from cosyvoice.utils.mask import make_pad_mask -from cosyvoice.utils.mask import add_optional_chunk_mask - - -class BaseEncoder(torch.nn.Module): - - def __init__( - self, - input_size: int, - output_size: int = 256, - attention_heads: int = 4, - linear_units: int = 2048, - num_blocks: int = 6, - dropout_rate: float = 0.1, - positional_dropout_rate: float = 0.1, - attention_dropout_rate: float = 0.0, - input_layer: str = "conv2d", - pos_enc_layer_type: str = "abs_pos", - normalize_before: bool = True, - static_chunk_size: int = 0, - use_dynamic_chunk: bool = False, - global_cmvn: torch.nn.Module = None, - use_dynamic_left_chunk: bool = False, - gradient_checkpointing: bool = False, - ): - """ - Args: - input_size (int): input dim - output_size (int): dimension of attention - attention_heads (int): the number of heads of multi head attention - linear_units (int): the hidden units number of position-wise feed - forward - num_blocks (int): the number of decoder blocks - dropout_rate (float): dropout rate - attention_dropout_rate (float): dropout rate in attention - positional_dropout_rate (float): dropout rate after adding - positional encoding - input_layer (str): input layer type. - optional [linear, conv2d, conv2d6, conv2d8] - pos_enc_layer_type (str): Encoder positional encoding layer type. - opitonal [abs_pos, scaled_abs_pos, rel_pos, no_pos] - normalize_before (bool): - True: use layer_norm before each sub-block of a layer. - False: use layer_norm after each sub-block of a layer. - static_chunk_size (int): chunk size for static chunk training and - decoding - use_dynamic_chunk (bool): whether use dynamic chunk size for - training or not, You can only use fixed chunk(chunk_size > 0) - or dyanmic chunk size(use_dynamic_chunk = True) - global_cmvn (Optional[torch.nn.Module]): Optional GlobalCMVN module - use_dynamic_left_chunk (bool): whether use dynamic left chunk in - dynamic chunk training - key_bias: whether use bias in attention.linear_k, False for whisper models. - gradient_checkpointing: rerunning a forward-pass segment for each - checkpointed segment during backward. - """ - super().__init__() - self._output_size = output_size - - self.global_cmvn = global_cmvn - self.embed = COSYVOICE_SUBSAMPLE_CLASSES[input_layer]( - input_size, - output_size, - dropout_rate, - COSYVOICE_EMB_CLASSES[pos_enc_layer_type]( - output_size, positional_dropout_rate - ), - ) - - self.normalize_before = normalize_before - self.after_norm = torch.nn.LayerNorm(output_size, eps=1e-5) - self.static_chunk_size = static_chunk_size - self.use_dynamic_chunk = use_dynamic_chunk - self.use_dynamic_left_chunk = use_dynamic_left_chunk - self.gradient_checkpointing = gradient_checkpointing - - def output_size(self) -> int: - return self._output_size - - def forward( - self, - xs: torch.Tensor, - xs_lens: torch.Tensor, - decoding_chunk_size: int = 0, - num_decoding_left_chunks: int = -1, - ) -> Tuple[torch.Tensor, torch.Tensor]: - """Embed positions in tensor. - - Args: - xs: padded input tensor (B, T, D) - xs_lens: input length (B) - decoding_chunk_size: decoding chunk size for dynamic chunk - 0: default for training, use random dynamic chunk. - <0: for decoding, use full chunk. - >0: for decoding, use fixed chunk size as set. - num_decoding_left_chunks: number of left chunks, this is for decoding, - the chunk size is decoding_chunk_size. - >=0: use num_decoding_left_chunks - <0: use all left chunks - Returns: - encoder output tensor xs, and subsampled masks - xs: padded output tensor (B, T' ~= T/subsample_rate, D) - masks: torch.Tensor batch padding mask after subsample - (B, 1, T' ~= T/subsample_rate) - NOTE(xcsong): - We pass the `__call__` method of the modules instead of `forward` to the - checkpointing API because `__call__` attaches all the hooks of the module. - https://discuss.pytorch.org/t/any-different-between-model-input-and-model-forward-input/3690/2 - """ - T = xs.size(1) - masks = ~make_pad_mask(xs_lens, T).unsqueeze(1) # (B, 1, T) - if self.global_cmvn is not None: - xs = self.global_cmvn(xs) - xs, pos_emb, masks = self.embed(xs, masks) - mask_pad = masks # (B, 1, T/subsample_rate) - chunk_masks = add_optional_chunk_mask( - xs, - masks, - self.use_dynamic_chunk, - self.use_dynamic_left_chunk, - decoding_chunk_size, - self.static_chunk_size, - num_decoding_left_chunks, - ) - print(f"chunk_masks shape: {chunk_masks.shape}") - if self.gradient_checkpointing and self.training: - xs = self.forward_layers_checkpointed(xs, chunk_masks, pos_emb, mask_pad) - else: - xs = self.forward_layers(xs, chunk_masks, pos_emb, mask_pad) - if self.normalize_before: - xs = self.after_norm(xs) - # Here we assume the mask is not changed in encoder layers, so just - # return the masks before encoder layers, and the masks will be used - # for cross attention with decoder later - return xs, masks - - def forward_layers( - self, - xs: torch.Tensor, - chunk_masks: torch.Tensor, - pos_emb: torch.Tensor, - mask_pad: torch.Tensor, - ) -> torch.Tensor: - for layer in self.encoders: - xs, chunk_masks, _, _ = layer(xs, chunk_masks, pos_emb, mask_pad) - return xs - - @torch.jit.unused - def forward_layers_checkpointed( - self, - xs: torch.Tensor, - chunk_masks: torch.Tensor, - pos_emb: torch.Tensor, - mask_pad: torch.Tensor, - ) -> torch.Tensor: - for layer in self.encoders: - xs, chunk_masks, _, _ = ckpt.checkpoint( - layer.__call__, xs, chunk_masks, pos_emb, mask_pad - ) - return xs - - @torch.jit.export - def forward_chunk( - self, - xs: torch.Tensor, - offset: int, - required_cache_size: int, - att_cache: torch.Tensor = torch.zeros(0, 0, 0, 0), - cnn_cache: torch.Tensor = torch.zeros(0, 0, 0, 0), - att_mask: torch.Tensor = torch.ones((0, 0, 0), dtype=torch.bool), - ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: - """ Forward just one chunk - - Args: - xs (torch.Tensor): chunk input, with shape (b=1, time, mel-dim), - where `time == (chunk_size - 1) * subsample_rate + \ - subsample.right_context + 1` - offset (int): current offset in encoder output time stamp - required_cache_size (int): cache size required for next chunk - compuation - >=0: actual cache size - <0: means all history cache is required - att_cache (torch.Tensor): cache tensor for KEY & VALUE in - transformer/conformer attention, with shape - (elayers, head, cache_t1, d_k * 2), where - `head * d_k == hidden-dim` and - `cache_t1 == chunk_size * num_decoding_left_chunks`. - cnn_cache (torch.Tensor): cache tensor for cnn_module in conformer, - (elayers, b=1, hidden-dim, cache_t2), where - `cache_t2 == cnn.lorder - 1` - - Returns: - torch.Tensor: output of current input xs, - with shape (b=1, chunk_size, hidden-dim). - torch.Tensor: new attention cache required for next chunk, with - dynamic shape (elayers, head, ?, d_k * 2) - depending on required_cache_size. - torch.Tensor: new conformer cnn cache required for next chunk, with - same shape as the original cnn_cache. - - """ - assert xs.size(0) == 1 - # tmp_masks is just for interface compatibility - tmp_masks = torch.ones(1, xs.size(1), device=xs.device, dtype=torch.bool) - tmp_masks = tmp_masks.unsqueeze(1) - if self.global_cmvn is not None: - xs = self.global_cmvn(xs) - # NOTE(xcsong): Before embed, shape(xs) is (b=1, time, mel-dim) - xs, pos_emb, _ = self.embed(xs, tmp_masks, offset) - # NOTE(xcsong): After embed, shape(xs) is (b=1, chunk_size, hidden-dim) - elayers, cache_t1 = att_cache.size(0), att_cache.size(2) - chunk_size = xs.size(1) - attention_key_size = cache_t1 + chunk_size - pos_emb = self.embed.position_encoding( - offset=offset - cache_t1, size=attention_key_size - ) - if required_cache_size < 0: - next_cache_start = 0 - elif required_cache_size == 0: - next_cache_start = attention_key_size - else: - next_cache_start = max(attention_key_size - required_cache_size, 0) - r_att_cache = [] - r_cnn_cache = [] - for i, layer in enumerate(self.encoders): - # NOTE(xcsong): Before layer.forward - # shape(att_cache[i:i + 1]) is (1, head, cache_t1, d_k * 2), - # shape(cnn_cache[i]) is (b=1, hidden-dim, cache_t2) - xs, _, new_att_cache, new_cnn_cache = layer( - xs, - att_mask, - pos_emb, - att_cache=att_cache[i : i + 1] if elayers > 0 else att_cache, - cnn_cache=cnn_cache[i] if cnn_cache.size(0) > 0 else cnn_cache, - ) - # NOTE(xcsong): After layer.forward - # shape(new_att_cache) is (1, head, attention_key_size, d_k * 2), - # shape(new_cnn_cache) is (b=1, hidden-dim, cache_t2) - r_att_cache.append(new_att_cache[:, :, next_cache_start:, :]) - r_cnn_cache.append(new_cnn_cache.unsqueeze(0)) - if self.normalize_before: - xs = self.after_norm(xs) - - # NOTE(xcsong): shape(r_att_cache) is (elayers, head, ?, d_k * 2), - # ? may be larger than cache_t1, it depends on required_cache_size - r_att_cache = torch.cat(r_att_cache, dim=0) - # NOTE(xcsong): shape(r_cnn_cache) is (e, b=1, hidden-dim, cache_t2) - r_cnn_cache = torch.cat(r_cnn_cache, dim=0) - - return (xs, r_att_cache, r_cnn_cache) - - @torch.jit.unused - def forward_chunk_by_chunk( - self, - xs: torch.Tensor, - decoding_chunk_size: int, - num_decoding_left_chunks: int = -1, - ) -> Tuple[torch.Tensor, torch.Tensor]: - """Forward input chunk by chunk with chunk_size like a streaming - fashion - - Here we should pay special attention to computation cache in the - streaming style forward chunk by chunk. Three things should be taken - into account for computation in the current network: - 1. transformer/conformer encoder layers output cache - 2. convolution in conformer - 3. convolution in subsampling - - However, we don't implement subsampling cache for: - 1. We can control subsampling module to output the right result by - overlapping input instead of cache left context, even though it - wastes some computation, but subsampling only takes a very - small fraction of computation in the whole model. - 2. Typically, there are several covolution layers with subsampling - in subsampling module, it is tricky and complicated to do cache - with different convolution layers with different subsampling - rate. - 3. Currently, nn.Sequential is used to stack all the convolution - layers in subsampling, we need to rewrite it to make it work - with cache, which is not preferred. - Args: - xs (torch.Tensor): (1, max_len, dim) - chunk_size (int): decoding chunk size - """ - assert decoding_chunk_size > 0 - # The model is trained by static or dynamic chunk - assert self.static_chunk_size > 0 or self.use_dynamic_chunk - subsampling = self.embed.subsampling_rate - context = self.embed.right_context + 1 # Add current frame - stride = subsampling * decoding_chunk_size - decoding_window = (decoding_chunk_size - 1) * subsampling + context - num_frames = xs.size(1) - att_cache: torch.Tensor = torch.zeros((0, 0, 0, 0), device=xs.device) - cnn_cache: torch.Tensor = torch.zeros((0, 0, 0, 0), device=xs.device) - outputs = [] - offset = 0 - required_cache_size = decoding_chunk_size * num_decoding_left_chunks - - # Feed forward overlap input step by step - for cur in range(0, num_frames - context + 1, stride): - end = min(cur + decoding_window, num_frames) - chunk_xs = xs[:, cur:end, :] - (y, att_cache, cnn_cache) = self.forward_chunk( - chunk_xs, offset, required_cache_size, att_cache, cnn_cache - ) - outputs.append(y) - offset += y.size(1) - ys = torch.cat(outputs, 1) - masks = torch.ones((1, 1, ys.size(1)), device=ys.device, dtype=torch.bool) - return ys, masks - - -class TransformerEncoder(BaseEncoder): - """Transformer encoder module.""" - - def __init__( - self, - input_size: int, - output_size: int = 256, - attention_heads: int = 4, - linear_units: int = 2048, - num_blocks: int = 6, - dropout_rate: float = 0.1, - positional_dropout_rate: float = 0.1, - attention_dropout_rate: float = 0.0, - input_layer: str = "conv2d", - pos_enc_layer_type: str = "abs_pos", - normalize_before: bool = True, - static_chunk_size: int = 0, - use_dynamic_chunk: bool = False, - global_cmvn: torch.nn.Module = None, - use_dynamic_left_chunk: bool = False, - key_bias: bool = True, - selfattention_layer_type: str = "selfattn", - activation_type: str = "relu", - gradient_checkpointing: bool = False, - ): - """Construct TransformerEncoder - - See Encoder for the meaning of each parameter. - """ - super().__init__( - input_size, - output_size, - attention_heads, - linear_units, - num_blocks, - dropout_rate, - positional_dropout_rate, - attention_dropout_rate, - input_layer, - pos_enc_layer_type, - normalize_before, - static_chunk_size, - use_dynamic_chunk, - global_cmvn, - use_dynamic_left_chunk, - gradient_checkpointing, - ) - activation = COSYVOICE_ACTIVATION_CLASSES[activation_type]() - self.encoders = torch.nn.ModuleList( - [ - TransformerEncoderLayer( - output_size, - COSYVOICE_ATTENTION_CLASSES[selfattention_layer_type]( - attention_heads, output_size, attention_dropout_rate, key_bias - ), - PositionwiseFeedForward( - output_size, linear_units, dropout_rate, activation - ), - dropout_rate, - normalize_before, - ) - for _ in range(num_blocks) - ] - ) - - -class ConformerEncoder(BaseEncoder): - """Conformer encoder module.""" - - def __init__( - self, - input_size: int, - output_size: int = 256, - attention_heads: int = 4, - linear_units: int = 2048, - num_blocks: int = 6, - dropout_rate: float = 0.1, - positional_dropout_rate: float = 0.1, - attention_dropout_rate: float = 0.0, - input_layer: str = "conv2d", - pos_enc_layer_type: str = "rel_pos", - normalize_before: bool = True, - static_chunk_size: int = 0, - use_dynamic_chunk: bool = False, - global_cmvn: torch.nn.Module = None, - use_dynamic_left_chunk: bool = False, - positionwise_conv_kernel_size: int = 1, - macaron_style: bool = True, - selfattention_layer_type: str = "rel_selfattn", - activation_type: str = "swish", - use_cnn_module: bool = True, - cnn_module_kernel: int = 15, - causal: bool = False, - cnn_module_norm: str = "batch_norm", - key_bias: bool = True, - gradient_checkpointing: bool = False, - ): - """Construct ConformerEncoder - - Args: - input_size to use_dynamic_chunk, see in BaseEncoder - positionwise_conv_kernel_size (int): Kernel size of positionwise - conv1d layer. - macaron_style (bool): Whether to use macaron style for - positionwise layer. - selfattention_layer_type (str): Encoder attention layer type, - the parameter has no effect now, it's just for configure - compatibility. - activation_type (str): Encoder activation function type. - use_cnn_module (bool): Whether to use convolution module. - cnn_module_kernel (int): Kernel size of convolution module. - causal (bool): whether to use causal convolution or not. - key_bias: whether use bias in attention.linear_k, False for whisper models. - """ - super().__init__( - input_size, - output_size, - attention_heads, - linear_units, - num_blocks, - dropout_rate, - positional_dropout_rate, - attention_dropout_rate, - input_layer, - pos_enc_layer_type, - normalize_before, - static_chunk_size, - use_dynamic_chunk, - global_cmvn, - use_dynamic_left_chunk, - gradient_checkpointing, - ) - activation = COSYVOICE_ACTIVATION_CLASSES[activation_type]() - - # self-attention module definition - encoder_selfattn_layer_args = ( - attention_heads, - output_size, - attention_dropout_rate, - key_bias, - ) - # feed-forward module definition - positionwise_layer_args = ( - output_size, - linear_units, - dropout_rate, - activation, - ) - # convolution module definition - convolution_layer_args = ( - output_size, - cnn_module_kernel, - activation, - cnn_module_norm, - causal, - ) - - self.encoders = torch.nn.ModuleList( - [ - ConformerEncoderLayer( - output_size, - COSYVOICE_ATTENTION_CLASSES[selfattention_layer_type]( - *encoder_selfattn_layer_args - ), - PositionwiseFeedForward(*positionwise_layer_args), - ( - PositionwiseFeedForward(*positionwise_layer_args) - if macaron_style - else None - ), - ( - ConvolutionModule(*convolution_layer_args) - if use_cnn_module - else None - ), - dropout_rate, - normalize_before, - ) - for _ in range(num_blocks) - ] - ) - self.inference_buffers = {} - self.inference_graphs = {} - - @torch.inference_mode() - def capture_inference(self, seq_len_to_capture=[128, 256, 512, 1024]): - device = next(self.parameters()).device - start_time = time.time() - print( - f"Start capture_inference for ConformerEncoder, seq_len_to_capture: {seq_len_to_capture}" - ) - - for seq_len in seq_len_to_capture: - xs = torch.randn( - 1, seq_len, self._output_size, device=device, dtype=torch.bfloat16 - ) - xs_lens = torch.tensor([seq_len], device=device, dtype=torch.int32) - decoding_chunk_size = 0 - num_decoding_left_chunks = -1 - - T = xs.size(1) - masks = ~make_pad_mask(xs_lens, T).unsqueeze(1) # (B, 1, T) - if self.global_cmvn is not None: - xs = self.global_cmvn(xs) - xs, pos_emb, masks = self.embed(xs, masks) - mask_pad = masks # (B, 1, T/subsample_rate) - chunk_masks = add_optional_chunk_mask( - xs, - masks, - self.use_dynamic_chunk, - self.use_dynamic_left_chunk, - decoding_chunk_size, - self.static_chunk_size, - num_decoding_left_chunks, - ) - - g = torch.cuda.CUDAGraph() - with torch.cuda.graph(g): - out = self.forward_layers(xs, chunk_masks, pos_emb, mask_pad) - - self.inference_graphs[seq_len] = g - self.inference_buffers[seq_len] = { - "xs": xs, - "chunk_masks": chunk_masks, - "pos_emb": pos_emb, - "mask_pad": mask_pad, - "out": out, - } - end_time = time.time() - print( - f"Finish capture_inference for ConformerEncoder, time elapsed: {end_time - start_time}" - ) - - @torch.inference_mode() - def inference(self, xs: torch.Tensor, xs_lens: torch.Tensor): - curr_seq_len = xs.shape[1] - target_len = None - - for seq_len in sorted(self.inference_graphs.keys()): - if seq_len >= curr_seq_len: - target_len = seq_len - break - - if target_len is not None: - xs = F.pad(xs, (0, 0, 0, target_len - curr_seq_len), "constant", 0) - - decoding_chunk_size = 0 - num_decoding_left_chunks = -1 - - T = xs.size(1) - masks = ~make_pad_mask(xs_lens, T).unsqueeze(1) # (B, 1, T) - if self.global_cmvn is not None: - xs = self.global_cmvn(xs) - xs, pos_emb, masks = self.embed(xs, masks) - mask_pad = masks # (B, 1, T/subsample_rate) - chunk_masks = add_optional_chunk_mask( - xs, - masks, - self.use_dynamic_chunk, - self.use_dynamic_left_chunk, - decoding_chunk_size, - self.static_chunk_size, - num_decoding_left_chunks, - ) - - if target_len is not None: - buffer = self.inference_buffers[target_len] - buffer["xs"].copy_(xs) - buffer["chunk_masks"].copy_(chunk_masks) - buffer["pos_emb"].copy_(pos_emb) - buffer["mask_pad"].copy_(mask_pad) - - self.inference_graphs[target_len].replay() - - out = buffer["out"][:, :curr_seq_len, :] - else: - out = self.forward_layers(xs, chunk_masks, pos_emb, mask_pad) - - if self.normalize_before: - out = self.after_norm(out) - return out, masks diff --git a/cosyvoice/transformer/encoder_layer.py b/cosyvoice/transformer/encoder_layer.py deleted file mode 100644 index 59045afa6ff78e3973737e47b6734989d2b823b9..0000000000000000000000000000000000000000 --- a/cosyvoice/transformer/encoder_layer.py +++ /dev/null @@ -1,237 +0,0 @@ -# Copyright (c) 2021 Mobvoi Inc (Binbin Zhang, Di Wu) -# 2022 Xingchen Song (sxc19@mails.tsinghua.edu.cn) -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# Modified from ESPnet(https://github.com/espnet/espnet) -"""Encoder self-attention layer definition.""" - -from typing import Optional, Tuple - -import torch -from torch import nn - - -class TransformerEncoderLayer(nn.Module): - """Encoder layer module. - - Args: - size (int): Input dimension. - self_attn (torch.nn.Module): Self-attention module instance. - `MultiHeadedAttention` or `RelPositionMultiHeadedAttention` - instance can be used as the argument. - feed_forward (torch.nn.Module): Feed-forward module instance. - `PositionwiseFeedForward`, instance can be used as the argument. - dropout_rate (float): Dropout rate. - normalize_before (bool): - True: use layer_norm before each sub-block. - False: to use layer_norm after each sub-block. - """ - - def __init__( - self, - size: int, - self_attn: torch.nn.Module, - feed_forward: torch.nn.Module, - dropout_rate: float, - normalize_before: bool = True, - ): - """Construct an EncoderLayer object.""" - super().__init__() - self.self_attn = self_attn - self.feed_forward = feed_forward - self.norm1 = nn.LayerNorm(size, eps=1e-5) - self.norm2 = nn.LayerNorm(size, eps=1e-5) - self.dropout = nn.Dropout(dropout_rate) - self.size = size - self.normalize_before = normalize_before - - def forward( - self, - x: torch.Tensor, - mask: torch.Tensor, - pos_emb: torch.Tensor, - mask_pad: torch.Tensor = torch.ones((0, 0, 0), dtype=torch.bool), - att_cache: torch.Tensor = torch.zeros((0, 0, 0, 0)), - cnn_cache: torch.Tensor = torch.zeros((0, 0, 0, 0)), - ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: - """Compute encoded features. - - Args: - x (torch.Tensor): (#batch, time, size) - mask (torch.Tensor): Mask tensor for the input (#batch, time，time), - (0, 0, 0) means fake mask. - pos_emb (torch.Tensor): just for interface compatibility - to ConformerEncoderLayer - mask_pad (torch.Tensor): does not used in transformer layer, - just for unified api with conformer. - att_cache (torch.Tensor): Cache tensor of the KEY & VALUE - (#batch=1, head, cache_t1, d_k * 2), head * d_k == size. - cnn_cache (torch.Tensor): Convolution cache in conformer layer - (#batch=1, size, cache_t2), not used here, it's for interface - compatibility to ConformerEncoderLayer. - Returns: - torch.Tensor: Output tensor (#batch, time, size). - torch.Tensor: Mask tensor (#batch, time, time). - torch.Tensor: att_cache tensor, - (#batch=1, head, cache_t1 + time, d_k * 2). - torch.Tensor: cnn_cahce tensor (#batch=1, size, cache_t2). - - """ - residual = x - if self.normalize_before: - x = self.norm1(x) - x_att, new_att_cache = self.self_attn( - x, x, x, mask, pos_emb=pos_emb, cache=att_cache - ) - x = residual + self.dropout(x_att) - if not self.normalize_before: - x = self.norm1(x) - - residual = x - if self.normalize_before: - x = self.norm2(x) - x = residual + self.dropout(self.feed_forward(x)) - if not self.normalize_before: - x = self.norm2(x) - - fake_cnn_cache = torch.zeros((0, 0, 0), dtype=x.dtype, device=x.device) - return x, mask, new_att_cache, fake_cnn_cache - - -class ConformerEncoderLayer(nn.Module): - """Encoder layer module. - Args: - size (int): Input dimension. - self_attn (torch.nn.Module): Self-attention module instance. - `MultiHeadedAttention` or `RelPositionMultiHeadedAttention` - instance can be used as the argument. - feed_forward (torch.nn.Module): Feed-forward module instance. - `PositionwiseFeedForward` instance can be used as the argument. - feed_forward_macaron (torch.nn.Module): Additional feed-forward module - instance. - `PositionwiseFeedForward` instance can be used as the argument. - conv_module (torch.nn.Module): Convolution module instance. - `ConvlutionModule` instance can be used as the argument. - dropout_rate (float): Dropout rate. - normalize_before (bool): - True: use layer_norm before each sub-block. - False: use layer_norm after each sub-block. - """ - - def __init__( - self, - size: int, - self_attn: torch.nn.Module, - feed_forward: Optional[nn.Module] = None, - feed_forward_macaron: Optional[nn.Module] = None, - conv_module: Optional[nn.Module] = None, - dropout_rate: float = 0.1, - normalize_before: bool = True, - ): - """Construct an EncoderLayer object.""" - super().__init__() - self.self_attn = self_attn - self.feed_forward = feed_forward - self.feed_forward_macaron = feed_forward_macaron - self.conv_module = conv_module - self.norm_ff = nn.LayerNorm(size, eps=1e-5) # for the FNN module - self.norm_mha = nn.LayerNorm(size, eps=1e-5) # for the MHA module - if feed_forward_macaron is not None: - self.norm_ff_macaron = nn.LayerNorm(size, eps=1e-5) - self.ff_scale = 0.5 - else: - self.ff_scale = 1.0 - if self.conv_module is not None: - self.norm_conv = nn.LayerNorm(size, eps=1e-5) # for the CNN module - self.norm_final = nn.LayerNorm( - size, eps=1e-5 - ) # for the final output of the block - self.dropout = nn.Dropout(dropout_rate) - self.size = size - self.normalize_before = normalize_before - - def forward( - self, - x: torch.Tensor, - mask: torch.Tensor, - pos_emb: torch.Tensor, - mask_pad: torch.Tensor = torch.ones((0, 0, 0), dtype=torch.bool), - att_cache: torch.Tensor = torch.zeros((0, 0, 0, 0)), - cnn_cache: torch.Tensor = torch.zeros((0, 0, 0, 0)), - ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: - """Compute encoded features. - - Args: - x (torch.Tensor): (#batch, time, size) - mask (torch.Tensor): Mask tensor for the input (#batch, time，time), - (0, 0, 0) means fake mask. - pos_emb (torch.Tensor): positional encoding, must not be None - for ConformerEncoderLayer. - mask_pad (torch.Tensor): batch padding mask used for conv module. - (#batch, 1，time), (0, 0, 0) means fake mask. - att_cache (torch.Tensor): Cache tensor of the KEY & VALUE - (#batch=1, head, cache_t1, d_k * 2), head * d_k == size. - cnn_cache (torch.Tensor): Convolution cache in conformer layer - (#batch=1, size, cache_t2) - Returns: - torch.Tensor: Output tensor (#batch, time, size). - torch.Tensor: Mask tensor (#batch, time, time). - torch.Tensor: att_cache tensor, - (#batch=1, head, cache_t1 + time, d_k * 2). - torch.Tensor: cnn_cahce tensor (#batch, size, cache_t2). - """ - - # whether to use macaron style - if self.feed_forward_macaron is not None: - residual = x - if self.normalize_before: - x = self.norm_ff_macaron(x) - x = residual + self.ff_scale * self.dropout(self.feed_forward_macaron(x)) - if not self.normalize_before: - x = self.norm_ff_macaron(x) - - # multi-headed self-attention module - residual = x - if self.normalize_before: - x = self.norm_mha(x) - x_att, new_att_cache = self.self_attn(x, x, x, mask, pos_emb, att_cache) - x = residual + self.dropout(x_att) - if not self.normalize_before: - x = self.norm_mha(x) - - # convolution module - # Fake new cnn cache here, and then change it in conv_module - new_cnn_cache = torch.zeros((0, 0, 0), dtype=x.dtype, device=x.device) - if self.conv_module is not None: - residual = x - if self.normalize_before: - x = self.norm_conv(x) - x, new_cnn_cache = self.conv_module(x, mask_pad, cnn_cache) - x = residual + self.dropout(x) - - if not self.normalize_before: - x = self.norm_conv(x) - - # feed forward module - residual = x - if self.normalize_before: - x = self.norm_ff(x) - - x = residual + self.ff_scale * self.dropout(self.feed_forward(x)) - if not self.normalize_before: - x = self.norm_ff(x) - - if self.conv_module is not None: - x = self.norm_final(x) - - return x, mask, new_att_cache, new_cnn_cache diff --git a/cosyvoice/transformer/label_smoothing_loss.py b/cosyvoice/transformer/label_smoothing_loss.py deleted file mode 100644 index 47574a25fd4369132bfc444cae1b0e6b85a89933..0000000000000000000000000000000000000000 --- a/cosyvoice/transformer/label_smoothing_loss.py +++ /dev/null @@ -1,98 +0,0 @@ -# Copyright (c) 2019 Shigeki Karita -# 2020 Mobvoi Inc (Binbin Zhang) -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Label smoothing module.""" - -import torch -from torch import nn - - -class LabelSmoothingLoss(nn.Module): - """Label-smoothing loss. - - In a standard CE loss, the label's data distribution is: - [0,1,2] -> - [ - [1.0, 0.0, 0.0], - [0.0, 1.0, 0.0], - [0.0, 0.0, 1.0], - ] - - In the smoothing version CE Loss,some probabilities - are taken from the true label prob (1.0) and are divided - among other labels. - - e.g. - smoothing=0.1 - [0,1,2] -> - [ - [0.9, 0.05, 0.05], - [0.05, 0.9, 0.05], - [0.05, 0.05, 0.9], - ] - - Args: - size (int): the number of class - padding_idx (int): padding class id which will be ignored for loss - smoothing (float): smoothing rate (0.0 means the conventional CE) - normalize_length (bool): - normalize loss by sequence length if True - normalize loss by batch size if False - """ - - def __init__( - self, - size: int, - padding_idx: int, - smoothing: float, - normalize_length: bool = False, - ): - """Construct an LabelSmoothingLoss object.""" - super(LabelSmoothingLoss, self).__init__() - self.criterion = nn.KLDivLoss(reduction="none") - self.padding_idx = padding_idx - self.confidence = 1.0 - smoothing - self.smoothing = smoothing - self.size = size - self.normalize_length = normalize_length - - def forward(self, x: torch.Tensor, target: torch.Tensor) -> torch.Tensor: - """Compute loss between x and target. - - The model outputs and data labels tensors are flatten to - (batch*seqlen, class) shape and a mask is applied to the - padding part which should not be calculated for loss. - - Args: - x (torch.Tensor): prediction (batch, seqlen, class) - target (torch.Tensor): - target signal masked with self.padding_id (batch, seqlen) - Returns: - loss (torch.Tensor) : The KL loss, scalar float value - """ - assert x.size(2) == self.size - batch_size = x.size(0) - x = x.view(-1, self.size) - target = target.view(-1) - # use zeros_like instead of torch.no_grad() for true_dist, - # since no_grad() can not be exported by JIT - true_dist = torch.zeros_like(x) - true_dist.fill_(self.smoothing / (self.size - 1)) - ignore = target == self.padding_idx # (B,) - total = len(target) - ignore.sum().item() - target = target.masked_fill(ignore, 0) # avoid -1 index - true_dist.scatter_(1, target.unsqueeze(1), self.confidence) - kl = self.criterion(torch.log_softmax(x, dim=1), true_dist) - denom = total if self.normalize_length else batch_size - return kl.masked_fill(ignore.unsqueeze(1), 0).sum() / denom diff --git a/cosyvoice/transformer/positionwise_feed_forward.py b/cosyvoice/transformer/positionwise_feed_forward.py deleted file mode 100644 index 3e60fb55fd9149745821f505c98a6fb53cfc3d9f..0000000000000000000000000000000000000000 --- a/cosyvoice/transformer/positionwise_feed_forward.py +++ /dev/null @@ -1,116 +0,0 @@ -# Copyright (c) 2019 Shigeki Karita -# 2020 Mobvoi Inc (Binbin Zhang) -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Positionwise feed forward layer definition.""" - -import torch - - -class PositionwiseFeedForward(torch.nn.Module): - """Positionwise feed forward layer. - - FeedForward are appied on each position of the sequence. - The output dim is same with the input dim. - - Args: - idim (int): Input dimenstion. - hidden_units (int): The number of hidden units. - dropout_rate (float): Dropout rate. - activation (torch.nn.Module): Activation function - """ - - def __init__( - self, - idim: int, - hidden_units: int, - dropout_rate: float, - activation: torch.nn.Module = torch.nn.ReLU(), - ): - """Construct a PositionwiseFeedForward object.""" - super(PositionwiseFeedForward, self).__init__() - self.w_1 = torch.nn.Linear(idim, hidden_units) - self.activation = activation - self.dropout = torch.nn.Dropout(dropout_rate) - self.w_2 = torch.nn.Linear(hidden_units, idim) - - def forward(self, xs: torch.Tensor) -> torch.Tensor: - """Forward function. - - Args: - xs: input tensor (B, L, D) - Returns: - output tensor, (B, L, D) - """ - return self.w_2(self.dropout(self.activation(self.w_1(xs)))) - - -class MoEFFNLayer(torch.nn.Module): - """ - Mixture of expert with Positionwise feed forward layer - See also figure 1 in https://arxiv.org/pdf/2305.15663.pdf - The output dim is same with the input dim. - - Modified from https://github.com/Lightning-AI/lit-gpt/pull/823 - https://github.com/mistralai/mistral-src/blob/b46d6/moe_one_file_ref.py#L203-L219 - Args: - n_expert: number of expert. - n_expert_per_token: The actual number of experts used for each frame - idim (int): Input dimenstion. - hidden_units (int): The number of hidden units. - dropout_rate (float): Dropout rate. - activation (torch.nn.Module): Activation function - """ - - def __init__( - self, - n_expert: int, - n_expert_per_token: int, - idim: int, - hidden_units: int, - dropout_rate: float, - activation: torch.nn.Module = torch.nn.ReLU(), - ): - super(MoEFFNLayer, self).__init__() - self.gate = torch.nn.Linear(idim, n_expert, bias=False) - self.experts = torch.nn.ModuleList( - PositionwiseFeedForward(idim, hidden_units, dropout_rate, activation) - for _ in range(n_expert) - ) - self.n_expert_per_token = n_expert_per_token - - def forward(self, xs: torch.Tensor) -> torch.Tensor: - """Foward function. - Args: - xs: input tensor (B, L, D) - Returns: - output tensor, (B, L, D) - - """ - B, L, D = xs.size() # batch size, sequence length, embedding dimension (idim) - xs = xs.view(-1, D) # (B*L, D) - router = self.gate(xs) # (B*L, n_expert) - logits, indices = torch.topk( - router, self.n_expert_per_token - ) # probs:(B*L, n_expert), indices: (B*L, n_expert) - weights = torch.nn.functional.softmax(logits, dim=1, dtype=torch.float).to( - dtype=xs.dtype - ) # (B*L, n_expert_per_token) - output = torch.zeros_like(xs) # (B*L, D) - for i, expert in enumerate(self.experts): - mask = indices == i - batch_idx, ith_expert = torch.where(mask) - output[batch_idx] += weights[batch_idx, ith_expert, None] * expert( - xs[batch_idx] - ) - return output.view(B, L, D) diff --git a/cosyvoice/transformer/subsampling.py b/cosyvoice/transformer/subsampling.py deleted file mode 100644 index 01052c78c05e066c2305382b0e4aee8597254c00..0000000000000000000000000000000000000000 --- a/cosyvoice/transformer/subsampling.py +++ /dev/null @@ -1,391 +0,0 @@ -# Copyright (c) 2021 Mobvoi Inc (Binbin Zhang, Di Wu) -# 2024 Alibaba Inc (Xiang Lyu) -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# Modified from ESPnet(https://github.com/espnet/espnet) -"""Subsampling layer definition.""" - -from typing import Tuple, Union - -import torch - - -class BaseSubsampling(torch.nn.Module): - - def __init__(self): - super().__init__() - self.right_context = 0 - self.subsampling_rate = 1 - - def position_encoding( - self, offset: Union[int, torch.Tensor], size: int - ) -> torch.Tensor: - return self.pos_enc.position_encoding(offset, size) - - -class EmbedinigNoSubsampling(BaseSubsampling): - """Embedding input without subsampling""" - - def __init__( - self, idim: int, odim: int, dropout_rate: float, pos_enc_class: torch.nn.Module - ): - super().__init__() - self.embed = torch.nn.Embedding(idim, odim) - self.pos_enc = pos_enc_class - - def forward( - self, - x: torch.Tensor, - x_mask: torch.Tensor, - offset: Union[int, torch.Tensor] = 0, - ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: - """Input x. - - Args: - x (torch.Tensor): Input tensor (#batch, time, idim). - x_mask (torch.Tensor): Input mask (#batch, 1, time). - - Returns: - torch.Tensor: linear input tensor (#batch, time', odim), - where time' = time . - torch.Tensor: linear input mask (#batch, 1, time'), - where time' = time . - - """ - x = self.embed(x) - x, pos_emb = self.pos_enc(x, offset) - return x, pos_emb, x_mask - - -class LinearNoSubsampling(BaseSubsampling): - """Linear transform the input without subsampling - - Args: - idim (int): Input dimension. - odim (int): Output dimension. - dropout_rate (float): Dropout rate. - - """ - - def __init__( - self, idim: int, odim: int, dropout_rate: float, pos_enc_class: torch.nn.Module - ): - """Construct an linear object.""" - super().__init__() - self.out = torch.nn.Sequential( - torch.nn.Linear(idim, odim), - torch.nn.LayerNorm(odim, eps=1e-5), - torch.nn.Dropout(dropout_rate), - ) - self.pos_enc = pos_enc_class - self.right_context = 0 - self.subsampling_rate = 1 - - def forward( - self, - x: torch.Tensor, - x_mask: torch.Tensor, - offset: Union[int, torch.Tensor] = 0, - ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: - """Input x. - - Args: - x (torch.Tensor): Input tensor (#batch, time, idim). - x_mask (torch.Tensor): Input mask (#batch, 1, time). - - Returns: - torch.Tensor: linear input tensor (#batch, time', odim), - where time' = time . - torch.Tensor: linear input mask (#batch, 1, time'), - where time' = time . - - """ - x = self.out(x) - x, pos_emb = self.pos_enc(x, offset) - return x, pos_emb, x_mask - - -class Conv1dSubsampling2(BaseSubsampling): - """Convolutional 1D subsampling (to 1/2 length). - It is designed for Whisper, ref: - https://github.com/openai/whisper/blob/main/whisper/model.py - - Args: - idim (int): Input dimension. - odim (int): Output dimension. - dropout_rate (float): Dropout rate. - - """ - - def __init__( - self, idim: int, odim: int, dropout_rate: float, pos_enc_class: torch.nn.Module - ): - """Construct an Conv1dSubsampling2 object.""" - super().__init__() - self.conv = torch.nn.Sequential( - torch.nn.Conv1d(idim, odim, kernel_size=3, padding=1), - torch.nn.GELU(), - torch.nn.Conv1d(odim, odim, kernel_size=3, stride=2, padding=1), - torch.nn.GELU(), - ) - self.pos_enc = pos_enc_class - # The right context for every conv layer is computed by: - # (kernel_size - 1) * frame_rate_of_this_layer - self.subsampling_rate = 2 - # 4 = (3 - 1) * 1 + (3 - 1) * 1 - self.right_context = 4 - - def forward( - self, - x: torch.Tensor, - x_mask: torch.Tensor, - offset: Union[int, torch.Tensor] = 0, - ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: - """Subsample x. - - Args: - x (torch.Tensor): Input tensor (#batch, time, idim). - x_mask (torch.Tensor): Input mask (#batch, 1, time). - - Returns: - torch.Tensor: Subsampled tensor (#batch, time', odim), - where time' = time // 2. - torch.Tensor: Subsampled mask (#batch, 1, time'), - where time' = time // 2. - torch.Tensor: positional encoding - - """ - time = x.size(1) - x = x.transpose(1, 2) # (b, f, t) - x = self.conv(x) - x = x.transpose(1, 2) # (b, t, f) - x, pos_emb = self.pos_enc(x, offset) - return x, pos_emb, x_mask[:, :, (time + 1) % 2 :: 2] - - -class Conv2dSubsampling4(BaseSubsampling): - """Convolutional 2D subsampling (to 1/4 length). - - Args: - idim (int): Input dimension. - odim (int): Output dimension. - dropout_rate (float): Dropout rate. - - """ - - def __init__( - self, idim: int, odim: int, dropout_rate: float, pos_enc_class: torch.nn.Module - ): - """Construct an Conv2dSubsampling4 object.""" - super().__init__() - self.conv = torch.nn.Sequential( - torch.nn.Conv2d(1, odim, 3, 2), - torch.nn.ReLU(), - torch.nn.Conv2d(odim, odim, 3, 2), - torch.nn.ReLU(), - ) - self.out = torch.nn.Sequential( - torch.nn.Linear(odim * (((idim - 1) // 2 - 1) // 2), odim) - ) - self.pos_enc = pos_enc_class - # The right context for every conv layer is computed by: - # (kernel_size - 1) * frame_rate_of_this_layer - self.subsampling_rate = 4 - # 6 = (3 - 1) * 1 + (3 - 1) * 2 - self.right_context = 6 - - def forward( - self, - x: torch.Tensor, - x_mask: torch.Tensor, - offset: Union[int, torch.Tensor] = 0, - ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: - """Subsample x. - - Args: - x (torch.Tensor): Input tensor (#batch, time, idim). - x_mask (torch.Tensor): Input mask (#batch, 1, time). - - Returns: - torch.Tensor: Subsampled tensor (#batch, time', odim), - where time' = time // 4. - torch.Tensor: Subsampled mask (#batch, 1, time'), - where time' = time // 4. - torch.Tensor: positional encoding - - """ - x = x.unsqueeze(1) # (b, c=1, t, f) - x = self.conv(x) - b, c, t, f = x.size() - x = self.out(x.transpose(1, 2).contiguous().view(b, t, c * f)) - x, pos_emb = self.pos_enc(x, offset) - return x, pos_emb, x_mask[:, :, 2::2][:, :, 2::2] - - -class Conv2dSubsampling6(BaseSubsampling): - """Convolutional 2D subsampling (to 1/6 length). - Args: - idim (int): Input dimension. - odim (int): Output dimension. - dropout_rate (float): Dropout rate. - pos_enc (torch.nn.Module): Custom position encoding layer. - """ - - def __init__( - self, idim: int, odim: int, dropout_rate: float, pos_enc_class: torch.nn.Module - ): - """Construct an Conv2dSubsampling6 object.""" - super().__init__() - self.conv = torch.nn.Sequential( - torch.nn.Conv2d(1, odim, 3, 2), - torch.nn.ReLU(), - torch.nn.Conv2d(odim, odim, 5, 3), - torch.nn.ReLU(), - ) - self.linear = torch.nn.Linear(odim * (((idim - 1) // 2 - 2) // 3), odim) - self.pos_enc = pos_enc_class - # 10 = (3 - 1) * 1 + (5 - 1) * 2 - self.subsampling_rate = 6 - self.right_context = 10 - - def forward( - self, - x: torch.Tensor, - x_mask: torch.Tensor, - offset: Union[int, torch.Tensor] = 0, - ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: - """Subsample x. - Args: - x (torch.Tensor): Input tensor (#batch, time, idim). - x_mask (torch.Tensor): Input mask (#batch, 1, time). - - Returns: - torch.Tensor: Subsampled tensor (#batch, time', odim), - where time' = time // 6. - torch.Tensor: Subsampled mask (#batch, 1, time'), - where time' = time // 6. - torch.Tensor: positional encoding - """ - x = x.unsqueeze(1) # (b, c, t, f) - x = self.conv(x) - b, c, t, f = x.size() - x = self.linear(x.transpose(1, 2).contiguous().view(b, t, c * f)) - x, pos_emb = self.pos_enc(x, offset) - return x, pos_emb, x_mask[:, :, 2::2][:, :, 4::3] - - -class Conv2dSubsampling8(BaseSubsampling): - """Convolutional 2D subsampling (to 1/8 length). - - Args: - idim (int): Input dimension. - odim (int): Output dimension. - dropout_rate (float): Dropout rate. - - """ - - def __init__( - self, idim: int, odim: int, dropout_rate: float, pos_enc_class: torch.nn.Module - ): - """Construct an Conv2dSubsampling8 object.""" - super().__init__() - self.conv = torch.nn.Sequential( - torch.nn.Conv2d(1, odim, 3, 2), - torch.nn.ReLU(), - torch.nn.Conv2d(odim, odim, 3, 2), - torch.nn.ReLU(), - torch.nn.Conv2d(odim, odim, 3, 2), - torch.nn.ReLU(), - ) - self.linear = torch.nn.Linear( - odim * ((((idim - 1) // 2 - 1) // 2 - 1) // 2), odim - ) - self.pos_enc = pos_enc_class - self.subsampling_rate = 8 - # 14 = (3 - 1) * 1 + (3 - 1) * 2 + (3 - 1) * 4 - self.right_context = 14 - - def forward( - self, - x: torch.Tensor, - x_mask: torch.Tensor, - offset: Union[int, torch.Tensor] = 0, - ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: - """Subsample x. - - Args: - x (torch.Tensor): Input tensor (#batch, time, idim). - x_mask (torch.Tensor): Input mask (#batch, 1, time). - - Returns: - torch.Tensor: Subsampled tensor (#batch, time', odim), - where time' = time // 8. - torch.Tensor: Subsampled mask (#batch, 1, time'), - where time' = time // 8. - torch.Tensor: positional encoding - """ - x = x.unsqueeze(1) # (b, c, t, f) - x = self.conv(x) - b, c, t, f = x.size() - x = self.linear(x.transpose(1, 2).contiguous().view(b, t, c * f)) - x, pos_emb = self.pos_enc(x, offset) - return x, pos_emb, x_mask[:, :, 2::2][:, :, 2::2][:, :, 2::2] - - -class LegacyLinearNoSubsampling(BaseSubsampling): - """Linear transform the input without subsampling - - Args: - idim (int): Input dimension. - odim (int): Output dimension. - dropout_rate (float): Dropout rate. - - """ - - def __init__( - self, idim: int, odim: int, dropout_rate: float, pos_enc_class: torch.nn.Module - ): - """Construct an linear object.""" - super().__init__() - self.out = torch.nn.Sequential( - torch.nn.Linear(idim, odim), - torch.nn.LayerNorm(odim, eps=1e-5), - torch.nn.Dropout(dropout_rate), - torch.nn.ReLU(), - ) - self.pos_enc = pos_enc_class - self.right_context = 0 - self.subsampling_rate = 1 - - def forward( - self, - x: torch.Tensor, - x_mask: torch.Tensor, - offset: Union[int, torch.Tensor] = 0, - ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: - """Input x. - - Args: - x (torch.Tensor): Input tensor (#batch, time, idim). - x_mask (torch.Tensor): Input mask (#batch, 1, time). - - Returns: - torch.Tensor: linear input tensor (#batch, time', odim), - where time' = time . - torch.Tensor: linear input mask (#batch, 1, time'), - where time' = time . - - """ - x = self.out(x) - x, pos_emb = self.pos_enc(x, offset) - return x, pos_emb, x_mask diff --git a/cosyvoice/utils/__init__.py b/cosyvoice/utils/__init__.py deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/cosyvoice/utils/audio.py b/cosyvoice/utils/audio.py deleted file mode 100644 index c837df56e74d52c7ef64a33a49851fb521352630..0000000000000000000000000000000000000000 --- a/cosyvoice/utils/audio.py +++ /dev/null @@ -1,90 +0,0 @@ -import numpy as np -import torch -import torch.utils.data -from librosa.filters import mel as librosa_mel_fn -from scipy.io.wavfile import read - -MAX_WAV_VALUE = 32768.0 - - -def load_wav(full_path): - sampling_rate, data = read(full_path) - return data, sampling_rate - - -def dynamic_range_compression(x, C=1, clip_val=1e-5): - return np.log(np.clip(x, a_min=clip_val, a_max=None) * C) - - -def dynamic_range_decompression(x, C=1): - return np.exp(x) / C - - -def dynamic_range_compression_torch(x, C=1, clip_val=1e-5): - return torch.log(torch.clamp(x, min=clip_val) * C) - - -def dynamic_range_decompression_torch(x, C=1): - return torch.exp(x) / C - - -def spectral_normalize_torch(magnitudes): - output = dynamic_range_compression_torch(magnitudes) - return output - - -def spectral_de_normalize_torch(magnitudes): - output = dynamic_range_decompression_torch(magnitudes) - return output - - -mel_basis = {} -hann_window = {} - - -def mel_spectrogram( - y, n_fft, num_mels, sampling_rate, hop_size, win_size, fmin, fmax, center=False -): - # if torch.min(y) < -1.0: - # print("min value is ", torch.min(y)) - # if torch.max(y) > 1.0: - # print("max value is ", torch.max(y)) - - global mel_basis, hann_window # pylint: disable=global-statement - if f"{str(fmax)}_{str(y.device)}" not in mel_basis: - mel = librosa_mel_fn( - sr=sampling_rate, n_fft=n_fft, n_mels=num_mels, fmin=fmin, fmax=fmax - ) - mel_basis[str(fmax) + "_" + str(y.device)] = ( - torch.from_numpy(mel).float().to(y.device) - ) - hann_window[str(y.device)] = torch.hann_window(win_size).to(y.device) - - y = torch.nn.functional.pad( - y.unsqueeze(1), - (int((n_fft - hop_size) / 2), int((n_fft - hop_size) / 2)), - mode="reflect", - ) - y = y.squeeze(1) - - spec = torch.view_as_real( - torch.stft( - y, - n_fft, - hop_length=hop_size, - win_length=win_size, - window=hann_window[str(y.device)], - center=center, - pad_mode="reflect", - normalized=False, - onesided=True, - return_complex=True, - ) - ) - - spec = torch.sqrt(spec.pow(2).sum(-1) + (1e-9)) - - spec = torch.matmul(mel_basis[str(fmax) + "_" + str(y.device)], spec) - spec = spectral_normalize_torch(spec) - - return spec diff --git a/cosyvoice/utils/class_utils.py b/cosyvoice/utils/class_utils.py deleted file mode 100644 index 52d7a959a9097fc21fb53f5ac43e4c1d15d953c3..0000000000000000000000000000000000000000 --- a/cosyvoice/utils/class_utils.py +++ /dev/null @@ -1,78 +0,0 @@ -# Copyright [2023-11-28] -# 2024 Alibaba Inc (authors: Xiang Lyu) -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import torch - -from cosyvoice.transformer.activation import Swish -from cosyvoice.transformer.subsampling import ( - LinearNoSubsampling, - EmbedinigNoSubsampling, - Conv1dSubsampling2, - Conv2dSubsampling4, - Conv2dSubsampling6, - Conv2dSubsampling8, -) -from cosyvoice.transformer.embedding import ( - PositionalEncoding, - RelPositionalEncoding, - WhisperPositionalEncoding, - LearnablePositionalEncoding, - NoPositionalEncoding, -) -from cosyvoice.transformer.attention import ( - MultiHeadedAttention, - RelPositionMultiHeadedAttention, -) -from cosyvoice.transformer.embedding import ( - EspnetRelPositionalEncoding, -) -from cosyvoice.transformer.subsampling import ( - LegacyLinearNoSubsampling, -) - - -COSYVOICE_ACTIVATION_CLASSES = { - "hardtanh": torch.nn.Hardtanh, - "tanh": torch.nn.Tanh, - "relu": torch.nn.ReLU, - "selu": torch.nn.SELU, - "swish": getattr(torch.nn, "SiLU", Swish), - "gelu": torch.nn.GELU, -} - -COSYVOICE_SUBSAMPLE_CLASSES = { - "linear": LinearNoSubsampling, - "linear_legacy": LegacyLinearNoSubsampling, - "embed": EmbedinigNoSubsampling, - "conv1d2": Conv1dSubsampling2, - "conv2d": Conv2dSubsampling4, - "conv2d6": Conv2dSubsampling6, - "conv2d8": Conv2dSubsampling8, - "paraformer_dummy": torch.nn.Identity, -} - -COSYVOICE_EMB_CLASSES = { - "embed": PositionalEncoding, - "abs_pos": PositionalEncoding, - "rel_pos": RelPositionalEncoding, - "rel_pos_espnet": EspnetRelPositionalEncoding, - "no_pos": NoPositionalEncoding, - "abs_pos_whisper": WhisperPositionalEncoding, - "embed_learnable_pe": LearnablePositionalEncoding, -} - -COSYVOICE_ATTENTION_CLASSES = { - "selfattn": MultiHeadedAttention, - "rel_selfattn": RelPositionMultiHeadedAttention, -} diff --git a/cosyvoice/utils/common.py b/cosyvoice/utils/common.py deleted file mode 100644 index e9611b6aa8ea5a12f50357d9b78af0e5e969ab85..0000000000000000000000000000000000000000 --- a/cosyvoice/utils/common.py +++ /dev/null @@ -1,169 +0,0 @@ -# Copyright (c) 2020 Mobvoi Inc (Binbin Zhang) -# 2024 Alibaba Inc (authors: Xiang Lyu) -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# Modified from ESPnet(https://github.com/espnet/espnet) -"""Unility functions for Transformer.""" - -import random -from typing import List - -import numpy as np -import torch - -IGNORE_ID = -1 - - -def pad_list(xs: List[torch.Tensor], pad_value: int): - """Perform padding for the list of tensors. - - Args: - xs (List): List of Tensors [(T_1, `*`), (T_2, `*`), ..., (T_B, `*`)]. - pad_value (float): Value for padding. - - Returns: - Tensor: Padded tensor (B, Tmax, `*`). - - Examples: - >>> x = [torch.ones(4), torch.ones(2), torch.ones(1)] - >>> x - [tensor([1., 1., 1., 1.]), tensor([1., 1.]), tensor([1.])] - >>> pad_list(x, 0) - tensor([[1., 1., 1., 1.], - [1., 1., 0., 0.], - [1., 0., 0., 0.]]) - - """ - max_len = max([len(item) for item in xs]) - batchs = len(xs) - ndim = xs[0].ndim - if ndim == 1: - pad_res = torch.zeros(batchs, max_len, dtype=xs[0].dtype, device=xs[0].device) - elif ndim == 2: - pad_res = torch.zeros( - batchs, max_len, xs[0].shape[1], dtype=xs[0].dtype, device=xs[0].device - ) - elif ndim == 3: - pad_res = torch.zeros( - batchs, - max_len, - xs[0].shape[1], - xs[0].shape[2], - dtype=xs[0].dtype, - device=xs[0].device, - ) - else: - raise ValueError(f"Unsupported ndim: {ndim}") - pad_res.fill_(pad_value) - for i in range(batchs): - pad_res[i, : len(xs[i])] = xs[i] - return pad_res - - -def th_accuracy( - pad_outputs: torch.Tensor, pad_targets: torch.Tensor, ignore_label: int -) -> torch.Tensor: - """Calculate accuracy. - - Args: - pad_outputs (Tensor): Prediction tensors (B * Lmax, D). - pad_targets (LongTensor): Target label tensors (B, Lmax). - ignore_label (int): Ignore label id. - - Returns: - torch.Tensor: Accuracy value (0.0 - 1.0). - - """ - pad_pred = pad_outputs.view( - pad_targets.size(0), pad_targets.size(1), pad_outputs.size(1) - ).argmax(2) - mask = pad_targets != ignore_label - numerator = torch.sum( - pad_pred.masked_select(mask) == pad_targets.masked_select(mask) - ) - denominator = torch.sum(mask) - return (numerator / denominator).detach() - - -def get_padding(kernel_size, dilation=1): - return int((kernel_size * dilation - dilation) / 2) - - -def init_weights(m, mean=0.0, std=0.01): - classname = m.__class__.__name__ - if classname.find("Conv") != -1: - m.weight.data.normal_(mean, std) - - -# Repetition Aware Sampling in VALL-E 2 -def ras_sampling( - weighted_scores, - decoded_tokens, - sampling, - top_p=0.8, - top_k=25, - win_size=10, - tau_r=0.1, -): - top_ids = nucleus_sampling(weighted_scores, top_p=top_p, top_k=top_k) - rep_num = ( - (torch.tensor(decoded_tokens[-win_size:]).to(weighted_scores.device) == top_ids) - .sum() - .item() - ) - if rep_num >= win_size * tau_r: - top_ids = random_sampling(weighted_scores, decoded_tokens, sampling) - return top_ids - - -def nucleus_sampling(weighted_scores, top_p=0.8, top_k=25): - prob, indices = [], [] - cum_prob = 0.0 - sorted_value, sorted_idx = weighted_scores.softmax(dim=0).sort( - descending=True, stable=True - ) - for i in range(len(sorted_idx)): - # sampling both top-p and numbers. - if cum_prob < top_p and len(prob) < top_k: - cum_prob += sorted_value[i] - prob.append(sorted_value[i]) - indices.append(sorted_idx[i]) - else: - break - prob = torch.tensor(prob).to(weighted_scores) - indices = torch.tensor(indices, dtype=torch.long).to(weighted_scores.device) - top_ids = indices[prob.multinomial(1, replacement=True)] - return top_ids - - -def random_sampling(weighted_scores, decoded_tokens, sampling): - top_ids = weighted_scores.softmax(dim=0).multinomial(1, replacement=True) - return top_ids - - -def fade_in_out(fade_in_mel, fade_out_mel, window): - device = fade_in_mel.device - fade_in_mel, fade_out_mel = fade_in_mel.cpu(), fade_out_mel.cpu() - mel_overlap_len = int(window.shape[0] / 2) - fade_in_mel[..., :mel_overlap_len] = ( - fade_in_mel[..., :mel_overlap_len] * window[:mel_overlap_len] - + fade_out_mel[..., -mel_overlap_len:] * window[mel_overlap_len:] - ) - return fade_in_mel.to(device) - - -def set_all_random_seed(seed): - random.seed(seed) - np.random.seed(seed) - torch.manual_seed(seed) - torch.cuda.manual_seed_all(seed) diff --git a/cosyvoice/utils/executor.py b/cosyvoice/utils/executor.py deleted file mode 100644 index d79e9350f27804e972c5e642b58dc6a450cadb7c..0000000000000000000000000000000000000000 --- a/cosyvoice/utils/executor.py +++ /dev/null @@ -1,151 +0,0 @@ -# Copyright (c) 2020 Mobvoi Inc (Binbin Zhang) -# 2024 Alibaba Inc (authors: Xiang Lyu) -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import logging -from contextlib import nullcontext -import os - -import torch -import torch.distributed as dist - -from cosyvoice.utils.train_utils import ( - update_parameter_and_lr, - log_per_step, - log_per_save, - batch_forward, - batch_backward, - save_model, - cosyvoice_join, -) - - -class Executor: - - def __init__(self): - self.step = 0 - self.epoch = 0 - self.rank = int(os.environ.get("RANK", 0)) - self.device = torch.device("cuda:{}".format(self.rank)) - - def train_one_epoc( - self, - model, - optimizer, - scheduler, - train_data_loader, - cv_data_loader, - writer, - info_dict, - group_join, - ): - """Train one epoch""" - - lr = optimizer.param_groups[0]["lr"] - logging.info( - "Epoch {} TRAIN info lr {} rank {}".format(self.epoch, lr, self.rank) - ) - logging.info( - "using accumulate grad, new batch size is {} times" - " larger than before".format(info_dict["accum_grad"]) - ) - # A context manager to be used in conjunction with an instance of - # torch.nn.parallel.DistributedDataParallel to be able to train - # with uneven inputs across participating processes. - model.train() - model_context = ( - model.join if info_dict["train_engine"] == "torch_ddp" else nullcontext - ) - with model_context(): - for batch_idx, batch_dict in enumerate(train_data_loader): - info_dict["tag"] = "TRAIN" - info_dict["step"] = self.step - info_dict["epoch"] = self.epoch - info_dict["batch_idx"] = batch_idx - if cosyvoice_join(group_join, info_dict): - break - - # Disable gradient synchronizations across DDP processes. - # Within this context, gradients will be accumulated on module - # variables, which will later be synchronized. - if ( - info_dict["train_engine"] == "torch_ddp" - and (batch_idx + 1) % info_dict["accum_grad"] != 0 - ): - context = model.no_sync - # Used for single gpu training and DDP gradient synchronization - # processes. - else: - context = nullcontext - - with context(): - info_dict = batch_forward(model, batch_dict, info_dict) - info_dict = batch_backward(model, info_dict) - - info_dict = update_parameter_and_lr( - model, optimizer, scheduler, info_dict - ) - log_per_step(writer, info_dict) - # NOTE specify save_per_step in cosyvoice.yaml if you want to enable step save - if ( - info_dict["save_per_step"] > 0 - and (self.step + 1) % info_dict["save_per_step"] == 0 - and (batch_idx + 1) % info_dict["accum_grad"] == 0 - ): - dist.barrier() - self.cv( - model, cv_data_loader, writer, info_dict, on_batch_end=False - ) - model.train() - if (batch_idx + 1) % info_dict["accum_grad"] == 0: - self.step += 1 - dist.barrier() - self.cv(model, cv_data_loader, writer, info_dict, on_batch_end=True) - - @torch.inference_mode() - def cv(self, model, cv_data_loader, writer, info_dict, on_batch_end=True): - """Cross validation on""" - logging.info( - "Epoch {} Step {} on_batch_end {} CV rank {}".format( - self.epoch, self.step + 1, on_batch_end, self.rank - ) - ) - model.eval() - total_num_utts, total_loss_dict = 0, {} # avoid division by 0 - for batch_idx, batch_dict in enumerate(cv_data_loader): - info_dict["tag"] = "CV" - info_dict["step"] = self.step - info_dict["epoch"] = self.epoch - info_dict["batch_idx"] = batch_idx - - num_utts = len(batch_dict["utts"]) - total_num_utts += num_utts - - info_dict = batch_forward(model, batch_dict, info_dict) - - for k, v in info_dict["loss_dict"].items(): - if k not in total_loss_dict: - total_loss_dict[k] = [] - total_loss_dict[k].append(v.item() * num_utts) - log_per_step(None, info_dict) - for k, v in total_loss_dict.items(): - total_loss_dict[k] = sum(v) / total_num_utts - info_dict["loss_dict"] = total_loss_dict - log_per_save(writer, info_dict) - model_name = ( - "epoch_{}_whole".format(self.epoch) - if on_batch_end - else "epoch_{}_step_{}".format(self.epoch, self.step + 1) - ) - save_model(model, model_name, info_dict) diff --git a/cosyvoice/utils/file_utils.py b/cosyvoice/utils/file_utils.py deleted file mode 100644 index 40cfa24528c99cae008e0d9b0e9515c4abbcb7db..0000000000000000000000000000000000000000 --- a/cosyvoice/utils/file_utils.py +++ /dev/null @@ -1,49 +0,0 @@ -# Copyright (c) 2021 Mobvoi Inc. (authors: Binbin Zhang) -# 2024 Alibaba Inc (authors: Xiang Lyu) -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import json -import torchaudio -import logging - -logging.getLogger("matplotlib").setLevel(logging.WARNING) -logging.basicConfig(level=logging.DEBUG, format="%(asctime)s %(levelname)s %(message)s") - - -def read_lists(list_file): - lists = [] - with open(list_file, "r", encoding="utf8") as fin: - for line in fin: - lists.append(line.strip()) - return lists - - -def read_json_lists(list_file): - lists = read_lists(list_file) - results = {} - for fn in lists: - with open(fn, "r", encoding="utf8") as fin: - results.update(json.load(fin)) - return results - - -def load_wav(wav, target_sr): - speech, sample_rate = torchaudio.load(wav) - speech = speech.mean(dim=0, keepdim=True) - if sample_rate != target_sr: - # assert sample_rate > target_sr, 'wav sample rate {} must be greater than {}'.format(sample_rate, target_sr) - speech = torchaudio.transforms.Resample( - orig_freq=sample_rate, new_freq=target_sr - )(speech) - return speech diff --git a/cosyvoice/utils/frontend_utils.py b/cosyvoice/utils/frontend_utils.py deleted file mode 100644 index 9e536f23a280fccd80b34494f2654cc035e1b74a..0000000000000000000000000000000000000000 --- a/cosyvoice/utils/frontend_utils.py +++ /dev/null @@ -1,142 +0,0 @@ -# Copyright (c) 2024 Alibaba Inc (authors: Xiang Lyu, Zhihao Du) -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import re - -chinese_char_pattern = re.compile(r"[\u4e00-\u9fff]+") - - -# whether contain chinese character -def contains_chinese(text): - return bool(chinese_char_pattern.search(text)) - - -# replace special symbol -def replace_corner_mark(text): - text = text.replace("²", "平方") - text = text.replace("³", "立方") - return text - - -# remove meaningless symbol -def remove_bracket(text): - text = text.replace("（", "").replace("）", "") - text = text.replace("【", "").replace("】", "") - text = text.replace("`", "").replace("`", "") - text = text.replace("——", " ") - return text - - -# spell Arabic numerals -def spell_out_number(text: str, inflect_parser): - new_text = [] - st = None - for i, c in enumerate(text): - if not c.isdigit(): - if st is not None: - num_str = inflect_parser.number_to_words(text[st:i]) - new_text.append(num_str) - st = None - new_text.append(c) - else: - if st is None: - st = i - if st is not None and st < len(text): - num_str = inflect_parser.number_to_words(text[st:]) - new_text.append(num_str) - return "".join(new_text) - - -# split paragrah logic： -# 1. per sentence max len token_max_n, min len token_min_n, merge if last sentence len less than merge_len -# 2. cal sentence len according to lang -# 3. split sentence according to puncatation -def split_paragraph( - text: str, - tokenize, - lang="zh", - token_max_n=80, - token_min_n=60, - merge_len=20, - comma_split=False, -): - def calc_utt_length(_text: str): - if lang == "zh": - return len(_text) - else: - return len(tokenize(_text)) - - def should_merge(_text: str): - if lang == "zh": - return len(_text) < merge_len - else: - return len(tokenize(_text)) < merge_len - - if lang == "zh": - pounc = ["。", "？", "！", "；", "：", "、", ".", "?", "!", ";"] - else: - pounc = [".", "?", "!", ";", ":"] - if comma_split: - pounc.extend(["，", ","]) - - if text[-1] not in pounc: - if lang == "zh": - text += "。" - else: - text += "." - - st = 0 - utts = [] - for i, c in enumerate(text): - if c in pounc: - if len(text[st:i]) > 0: - utts.append(text[st:i] + c) - if i + 1 < len(text) and text[i + 1] in ['"', "”"]: - tmp = utts.pop(-1) - utts.append(tmp + text[i + 1]) - st = i + 2 - else: - st = i + 1 - - final_utts = [] - cur_utt = "" - for utt in utts: - if ( - calc_utt_length(cur_utt + utt) > token_max_n - and calc_utt_length(cur_utt) > token_min_n - ): - final_utts.append(cur_utt) - cur_utt = "" - cur_utt = cur_utt + utt - if len(cur_utt) > 0: - if should_merge(cur_utt) and len(final_utts) != 0: - final_utts[-1] = final_utts[-1] + cur_utt - else: - final_utts.append(cur_utt) - - return final_utts - - -# remove blank between chinese character -def replace_blank(text: str): - out_str = [] - for i, c in enumerate(text): - if c == " ": - if (text[i + 1].isascii() and text[i + 1] != " ") and ( - text[i - 1].isascii() and text[i - 1] != " " - ): - out_str.append(c) - else: - out_str.append(c) - return "".join(out_str) diff --git a/cosyvoice/utils/mask.py b/cosyvoice/utils/mask.py deleted file mode 100644 index be8bba4779acd4491088c690b8ac9665f92a5283..0000000000000000000000000000000000000000 --- a/cosyvoice/utils/mask.py +++ /dev/null @@ -1,226 +0,0 @@ -# Copyright (c) 2019 Shigeki Karita -# 2020 Mobvoi Inc (Binbin Zhang) -# 2024 Alibaba Inc (authors: Xiang Lyu) -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import torch - -''' -def subsequent_mask( - size: int, - device: torch.device = torch.device("cpu"), -) -> torch.Tensor: - """Create mask for subsequent steps (size, size). - - This mask is used only in decoder which works in an auto-regressive mode. - This means the current step could only do attention with its left steps. - - In encoder, fully attention is used when streaming is not necessary and - the sequence is not long. In this case, no attention mask is needed. - - When streaming is need, chunk-based attention is used in encoder. See - subsequent_chunk_mask for the chunk-based attention mask. - - Args: - size (int): size of mask - str device (str): "cpu" or "cuda" or torch.Tensor.device - dtype (torch.device): result dtype - - Returns: - torch.Tensor: mask - - Examples: - >>> subsequent_mask(3) - [[1, 0, 0], - [1, 1, 0], - [1, 1, 1]] - """ - ret = torch.ones(size, size, device=device, dtype=torch.bool) - return torch.tril(ret) -''' - - -def subsequent_mask( - size: int, - device: torch.device = torch.device("cpu"), -) -> torch.Tensor: - """Create mask for subsequent steps (size, size). - - This mask is used only in decoder which works in an auto-regressive mode. - This means the current step could only do attention with its left steps. - - In encoder, fully attention is used when streaming is not necessary and - the sequence is not long. In this case, no attention mask is needed. - - When streaming is need, chunk-based attention is used in encoder. See - subsequent_chunk_mask for the chunk-based attention mask. - - Args: - size (int): size of mask - str device (str): "cpu" or "cuda" or torch.Tensor.device - dtype (torch.device): result dtype - - Returns: - torch.Tensor: mask - - Examples: - >>> subsequent_mask(3) - [[1, 0, 0], - [1, 1, 0], - [1, 1, 1]] - """ - arange = torch.arange(size, device=device) - mask = arange.expand(size, size) - arange = arange.unsqueeze(-1) - mask = mask <= arange - return mask - - -def subsequent_chunk_mask( - size: int, - chunk_size: int, - num_left_chunks: int = -1, - device: torch.device = torch.device("cpu"), -) -> torch.Tensor: - """Create mask for subsequent steps (size, size) with chunk size, - this is for streaming encoder - - Args: - size (int): size of mask - chunk_size (int): size of chunk - num_left_chunks (int): number of left chunks - <0: use full chunk - >=0: use num_left_chunks - device (torch.device): "cpu" or "cuda" or torch.Tensor.device - - Returns: - torch.Tensor: mask - - Examples: - >>> subsequent_chunk_mask(4, 2) - [[1, 1, 0, 0], - [1, 1, 0, 0], - [1, 1, 1, 1], - [1, 1, 1, 1]] - """ - ret = torch.zeros(size, size, device=device, dtype=torch.bool) - for i in range(size): - if num_left_chunks < 0: - start = 0 - else: - start = max((i // chunk_size - num_left_chunks) * chunk_size, 0) - ending = min((i // chunk_size + 1) * chunk_size, size) - ret[i, start:ending] = True - return ret - - -def add_optional_chunk_mask( - xs: torch.Tensor, - masks: torch.Tensor, - use_dynamic_chunk: bool, - use_dynamic_left_chunk: bool, - decoding_chunk_size: int, - static_chunk_size: int, - num_decoding_left_chunks: int, - enable_full_context: bool = True, -): - """Apply optional mask for encoder. - - Args: - xs (torch.Tensor): padded input, (B, L, D), L for max length - mask (torch.Tensor): mask for xs, (B, 1, L) - use_dynamic_chunk (bool): whether to use dynamic chunk or not - use_dynamic_left_chunk (bool): whether to use dynamic left chunk for - training. - decoding_chunk_size (int): decoding chunk size for dynamic chunk, it's - 0: default for training, use random dynamic chunk. - <0: for decoding, use full chunk. - >0: for decoding, use fixed chunk size as set. - static_chunk_size (int): chunk size for static chunk training/decoding - if it's greater than 0, if use_dynamic_chunk is true, - this parameter will be ignored - num_decoding_left_chunks: number of left chunks, this is for decoding, - the chunk size is decoding_chunk_size. - >=0: use num_decoding_left_chunks - <0: use all left chunks - enable_full_context (bool): - True: chunk size is either [1, 25] or full context(max_len) - False: chunk size ~ U[1, 25] - - Returns: - torch.Tensor: chunk mask of the input xs. - """ - # Whether to use chunk mask or not - if use_dynamic_chunk: - max_len = xs.size(1) - if decoding_chunk_size < 0: - chunk_size = max_len - num_left_chunks = -1 - elif decoding_chunk_size > 0: - chunk_size = decoding_chunk_size - num_left_chunks = num_decoding_left_chunks - else: - # chunk size is either [1, 25] or full context(max_len). - # Since we use 4 times subsampling and allow up to 1s(100 frames) - # delay, the maximum frame is 100 / 4 = 25. - chunk_size = torch.randint(1, max_len, (1,)).item() - num_left_chunks = -1 - if chunk_size > max_len // 2 and enable_full_context: - chunk_size = max_len - else: - chunk_size = chunk_size % 25 + 1 - if use_dynamic_left_chunk: - max_left_chunks = (max_len - 1) // chunk_size - num_left_chunks = torch.randint(0, max_left_chunks, (1,)).item() - chunk_masks = subsequent_chunk_mask( - xs.size(1), chunk_size, num_left_chunks, xs.device - ) # (L, L) - chunk_masks = chunk_masks.unsqueeze(0) # (1, L, L) - chunk_masks = masks & chunk_masks # (B, L, L) - elif static_chunk_size > 0: - num_left_chunks = num_decoding_left_chunks - chunk_masks = subsequent_chunk_mask( - xs.size(1), static_chunk_size, num_left_chunks, xs.device - ) # (L, L) - chunk_masks = chunk_masks.unsqueeze(0) # (1, L, L) - chunk_masks = masks & chunk_masks # (B, L, L) - else: - chunk_masks = masks - return chunk_masks - - -def make_pad_mask(lengths: torch.Tensor, max_len: int = 0) -> torch.Tensor: - """Make mask tensor containing indices of padded part. - - See description of make_non_pad_mask. - - Args: - lengths (torch.Tensor): Batch of lengths (B,). - Returns: - torch.Tensor: Mask tensor containing indices of padded part. - - Examples: - >>> lengths = [5, 3, 2] - >>> make_pad_mask(lengths) - masks = [[0, 0, 0, 0 ,0], - [0, 0, 0, 1, 1], - [0, 0, 1, 1, 1]] - """ - batch_size = lengths.size(0) - max_len = max_len if max_len > 0 else lengths.max().item() - seq_range = torch.arange(0, max_len, dtype=torch.int64, device=lengths.device) - seq_range_expand = seq_range.unsqueeze(0).expand(batch_size, max_len) - seq_length_expand = lengths.unsqueeze(-1) - mask = seq_range_expand >= seq_length_expand - return mask diff --git a/cosyvoice/utils/scheduler.py b/cosyvoice/utils/scheduler.py deleted file mode 100644 index 8235d212bcdfc59eacd0b111e14bb0d87831d3ae..0000000000000000000000000000000000000000 --- a/cosyvoice/utils/scheduler.py +++ /dev/null @@ -1,761 +0,0 @@ -# Copyright (c) 2020 Mobvoi Inc (Binbin Zhang) -# 2022 Ximalaya Inc (Yuguang Yang) -# 2024 Alibaba Inc (authors: Xiang Lyu) -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# Modified from ESPnet(https://github.com/espnet/espnet) -# NeMo(https://github.com/NVIDIA/NeMo) - -from typing import Union - -import math -import warnings -import torch -from torch.optim.lr_scheduler import _LRScheduler - - -class WarmupLR(_LRScheduler): - """The WarmupLR scheduler - - This scheduler is almost same as NoamLR Scheduler except for following - difference: - - NoamLR: - lr = optimizer.lr * model_size ** -0.5 - * min(step ** -0.5, step * warmup_step ** -1.5) - WarmupLR: - lr = optimizer.lr * warmup_step ** 0.5 - * min(step ** -0.5, step * warmup_step ** -1.5) - - Note that the maximum lr equals to optimizer.lr in this scheduler. - - """ - - def __init__( - self, - optimizer: torch.optim.Optimizer, - warmup_steps: Union[int, float] = 25000, - last_epoch: int = -1, - ): - self.warmup_steps = warmup_steps - - # __init__() must be invoked before setting field - # because step() is also invoked in __init__() - super().__init__(optimizer, last_epoch) - - def __repr__(self): - return f"{self.__class__.__name__}(warmup_steps={self.warmup_steps})" - - def get_lr(self): - step_num = self.last_epoch + 1 - if self.warmup_steps == 0: - return [lr * step_num**-0.5 for lr in self.base_lrs] - else: - return [ - lr - * self.warmup_steps**0.5 - * min(step_num**-0.5, step_num * self.warmup_steps**-1.5) - for lr in self.base_lrs - ] - - def set_step(self, step: int): - self.last_epoch = step - - -class WarmupPolicy(_LRScheduler): - """Adds warmup kwargs and warmup logic to lr policy. - All arguments should be passed as kwargs for clarity, - Args: - warmup_steps: Number of training steps in warmup stage - warmup_ratio: Ratio of warmup steps to total steps - max_steps: Total number of steps while training or `None` for - infinite training - """ - - def __init__( - self, - optimizer, - *, - warmup_steps=None, - warmup_ratio=None, - max_steps=None, - min_lr=0.0, - last_epoch=-1, - ): - assert not ( - warmup_steps is not None and warmup_ratio is not None - ), "Either use particular number of step or ratio" - assert ( - warmup_ratio is None or max_steps is not None - ), "If there is a ratio, there should be a total steps" - - # It is necessary to assign all attributes *before* __init__, - # as class is wrapped by an inner class. - self.max_steps = max_steps - if warmup_steps is not None: - self.warmup_steps = warmup_steps - elif warmup_ratio is not None: - self.warmup_steps = int(warmup_ratio * max_steps) - else: - self.warmup_steps = 0 - - self.min_lr = min_lr - super().__init__(optimizer, last_epoch) - - def get_lr(self): - if not self._get_lr_called_within_step: - warnings.warn( - "To get the last learning rate computed " - "by the scheduler, please use `get_last_lr()`.", - UserWarning, - stacklevel=2, - ) - - step = self.last_epoch - - if step <= self.warmup_steps and self.warmup_steps > 0: - return self._get_warmup_lr(step) - - if step > self.max_steps: - return [self.min_lr for _ in self.base_lrs] - - return self._get_lr(step) - - def _get_warmup_lr(self, step): - lr_val = (step + 1) / (self.warmup_steps + 1) - return [initial_lr * lr_val for initial_lr in self.base_lrs] - - def _get_lr(self, step): - """Simple const lr policy""" - return self.base_lrs - - -class SquareRootConstantPolicy(_LRScheduler): - """Adds warmup kwargs and warmup logic to lr policy. - All arguments should be passed as kwargs for clarity, - Args: - warmup_steps: Number of training steps in warmup stage - warmup_ratio: Ratio of warmup steps to total steps - max_steps: Total number of steps while training or `None` for - infinite training - """ - - def __init__( - self, - optimizer, - *, - constant_steps=None, - constant_ratio=None, - max_steps=None, - min_lr=0.0, - last_epoch=-1, - ): - assert not ( - constant_steps is not None and constant_ratio is not None - ), "Either use particular number of step or ratio" - assert ( - constant_ratio is None or max_steps is not None - ), "If there is a ratio, there should be a total steps" - - # It is necessary to assign all attributes *before* __init__, - # as class is wrapped by an inner class. - self.max_steps = max_steps - if constant_steps is not None: - self.constant_steps = constant_steps - elif constant_ratio is not None: - self.constant_steps = int(constant_ratio * max_steps) - else: - self.constant_steps = 0 - - self.constant_lr = 1 / (constant_steps**0.5) - self.min_lr = min_lr - super().__init__(optimizer, last_epoch) - - def get_lr(self): - if not self._get_lr_called_within_step: - warnings.warn( - "To get the last learning rate computed " - "by the scheduler, please use `get_last_lr()`.", - UserWarning, - stacklevel=2, - ) - - step = self.last_epoch - - if step <= self.constant_steps: - return [self.constant_lr for _ in self.base_lrs] - - if step > self.max_steps: - return [self.min_lr for _ in self.base_lrs] - - return self._get_lr(step) - - def _get_lr(self, step): - """Simple const lr policy""" - return self.base_lrs - - -class WarmupHoldPolicy(WarmupPolicy): - """Variant of WarmupPolicy which maintains high - learning rate for a defined number of steps. - All arguments should be passed as kwargs for clarity, - Args: - warmup_steps: Number of training steps in warmup stage - warmup_ratio: Ratio of warmup steps to total steps - hold_steps: Number of training steps to - hold the learning rate after warm up - hold_ratio: Ratio of hold steps to total steps - max_steps: Total number of steps while training or `None` for - infinite training - """ - - def __init__( - self, - optimizer, - *, - warmup_steps=None, - warmup_ratio=None, - hold_steps=None, - hold_ratio=None, - max_steps=None, - min_lr=0.0, - last_epoch=-1, - ): - assert not ( - hold_steps is not None and hold_ratio is not None - ), "Either use particular number of step or ratio" - assert ( - hold_ratio is None or max_steps is not None - ), "If there is a ratio, there should be a total steps" - - self.min_lr = min_lr - self._last_warmup_lr = 0.0 - - # Necessary to duplicate as class attributes are hidden in inner class - self.max_steps = max_steps - if warmup_steps is not None: - self.warmup_steps = warmup_steps - elif warmup_ratio is not None: - self.warmup_steps = int(warmup_ratio * max_steps) - else: - self.warmup_steps = 0 - - if hold_steps is not None: - self.hold_steps = hold_steps + self.warmup_steps - elif hold_ratio is not None: - self.hold_steps = int(hold_ratio * max_steps) + self.warmup_steps - else: - self.hold_steps = 0 - - super().__init__( - optimizer, - warmup_steps=warmup_steps, - warmup_ratio=warmup_ratio, - max_steps=max_steps, - last_epoch=last_epoch, - min_lr=min_lr, - ) - - def get_lr(self): - if not self._get_lr_called_within_step: - warnings.warn( - "To get the last learning rate computed by the scheduler," - " " - "please use `get_last_lr()`.", - UserWarning, - stacklevel=2, - ) - - step = self.last_epoch - - # Warmup phase - if step <= self.warmup_steps and self.warmup_steps > 0: - return self._get_warmup_lr(step) - - # Hold phase - if (step >= self.warmup_steps) and (step < self.hold_steps): - return self.base_lrs - - if step > self.max_steps: - return [self.min_lr for _ in self.base_lrs] - - return self._get_lr(step) - - -class WarmupAnnealHoldPolicy(_LRScheduler): - """Adds warmup kwargs and warmup logic to lr policy. - All arguments should be passed as kwargs for clarity, - Args: - warmup_steps: Number of training steps in warmup stage - warmup_ratio: Ratio of warmup steps to total steps - max_steps: Total number of steps while training or `None` for - infinite training - min_lr: Minimum lr to hold the learning rate after decay at. - constant_steps: Number of steps to keep lr constant at. - constant_ratio: Ratio of steps to keep lr constant. - """ - - def __init__( - self, - optimizer, - *, - warmup_steps=None, - warmup_ratio=None, - constant_steps=None, - constant_ratio=None, - max_steps=None, - min_lr=0.0, - last_epoch=-1, - ): - assert not ( - warmup_steps is not None and warmup_ratio is not None - ), "Either use particular number of step or ratio" - assert not ( - constant_steps is not None and constant_ratio is not None - ), "Either use constant_steps or constant_ratio" - assert ( - warmup_ratio is None or max_steps is not None - ), "If there is a ratio, there should be a total steps" - - # It is necessary to assign all attributes *before* __init__, - # as class is wrapped by an inner class. - self.max_steps = max_steps - - if warmup_steps is not None: - self.warmup_steps = warmup_steps - elif warmup_ratio is not None: - self.warmup_steps = int(warmup_ratio * max_steps) - else: - self.warmup_steps = 0 - - if constant_steps is not None: - self.constant_steps = constant_steps - elif constant_ratio is not None: - self.constant_steps = int(constant_ratio * max_steps) - else: - self.constant_steps = 0 - - self.decay_steps = max_steps - (self.constant_steps + self.warmup_steps) - - self.min_lr = min_lr - super().__init__(optimizer, last_epoch) - - def get_lr(self): - if not self._get_lr_called_within_step: - warnings.warn( - "To get the last learning rate computed " - "by the scheduler, please use `get_last_lr()`.", - UserWarning, - stacklevel=2, - ) - - step = self.last_epoch - - # Warmup steps - if self.warmup_steps > 0 and step <= self.warmup_steps: - return self._get_warmup_lr(step) - - # Constant steps after warmup and decay - if ( - self.constant_steps > 0 - and (self.warmup_steps + self.decay_steps) < step <= self.max_steps - ): - return self._get_constant_lr(step) - - # Min lr after max steps of updates - if step > self.max_steps: - return [self.min_lr for _ in self.base_lrs] - - return self._get_lr(step) - - def _get_warmup_lr(self, step): - lr_val = (step + 1) / (self.warmup_steps + 1) - return [initial_lr * lr_val for initial_lr in self.base_lrs] - - def _get_constant_lr(self, step): - return [self.min_lr for _ in self.base_lrs] - - def _get_lr(self, step): - """Simple const lr policy""" - return self.base_lrs - - -def _squareroot_annealing(initial_lr, step, max_steps, min_lr): - mult = ((max_steps - step) / max_steps) ** 0.5 - out_lr = initial_lr * mult - out_lr = max(out_lr, min_lr) - return out_lr - - -def _square_annealing(initial_lr, step, max_steps, min_lr): - mult = ((max_steps - step) / max_steps) ** 2 - out_lr = initial_lr * mult - out_lr = max(out_lr, min_lr) - return out_lr - - -def _cosine_annealing(initial_lr, step, max_steps, min_lr): - mult = 0.5 * (1 + math.cos(math.pi * step / max_steps)) - out_lr = (initial_lr - min_lr) * mult + min_lr - return out_lr - - -def _linear_warmup_with_cosine_annealing( - max_lr, warmup_steps, step, decay_steps, min_lr -): - assert max_lr > min_lr - # Use linear warmup for the initial part. - if warmup_steps > 0 and step <= warmup_steps: - return max_lr * float(step) / float(warmup_steps) - - # For any steps larger than `decay_steps`, use `min_lr`. - if step > warmup_steps + decay_steps: - return min_lr - - # If we are done with the warmup period, use the decay style. - num_steps_ = step - warmup_steps - decay_steps_ = decay_steps - decay_ratio = float(num_steps_) / float(decay_steps_) - assert decay_ratio >= 0.0 - assert decay_ratio <= 1.0 - delta_lr = max_lr - min_lr - - coeff = 0.5 * (math.cos(math.pi * decay_ratio) + 1.0) - - return min_lr + coeff * delta_lr - - -def _poly_decay(initial_lr, step, decay_steps, power, min_lr, cycle): - if cycle: - multiplier = 1.0 if step == 0 else math.ceil(step / decay_steps) - decay_steps *= multiplier - else: - step = min(step, decay_steps) - p = step / decay_steps - lr = (initial_lr - min_lr) * math.pow(1.0 - p, power) - lr += min_lr - return lr - - -def _noam_hold_annealing( - initial_lr, step, warmup_steps, hold_steps, decay_rate, min_lr -): - # hold_steps = total number of steps - # to hold the LR, not the warmup + hold steps. - T_warmup_decay = max(1, warmup_steps**decay_rate) - T_hold_decay = max(1, (step - hold_steps) ** decay_rate) - lr = (initial_lr * T_warmup_decay) / T_hold_decay - lr = max(lr, min_lr) - return lr - - -class SquareAnnealing(WarmupPolicy): - - def __init__(self, optimizer, *, max_steps, min_lr=1e-5, last_epoch=-1, **kwargs): - super().__init__( - optimizer=optimizer, - max_steps=max_steps, - last_epoch=last_epoch, - min_lr=min_lr, - **kwargs, - ) - - def _get_lr(self, step): - new_lrs = [ - _square_annealing( - initial_lr=initial_lr, - step=step - self.warmup_steps, - max_steps=self.max_steps - self.warmup_steps, - min_lr=self.min_lr, - ) - for initial_lr in self.base_lrs - ] - return new_lrs - - -class SquareRootAnnealing(WarmupPolicy): - - def __init__(self, optimizer, *, max_steps, min_lr=0, last_epoch=-1, **kwargs): - super().__init__( - optimizer=optimizer, - max_steps=max_steps, - last_epoch=last_epoch, - min_lr=min_lr, - **kwargs, - ) - - def _get_lr(self, step): - new_lrs = [ - _squareroot_annealing( - initial_lr=initial_lr, - step=step, - max_steps=self.max_steps, - min_lr=self.min_lr, - ) - for initial_lr in self.base_lrs - ] - return new_lrs - - -class CosineAnnealing(WarmupAnnealHoldPolicy): - - def __init__(self, optimizer, *, max_steps, min_lr=0, last_epoch=-1, **kwargs): - super().__init__( - optimizer=optimizer, - max_steps=max_steps, - last_epoch=last_epoch, - min_lr=min_lr, - **kwargs, - ) - - def _get_lr(self, step): - for initial_lr in self.base_lrs: - if initial_lr < self.min_lr: - raise ValueError( - f"{self} received an initial learning rate " - f"that was lower than the minimum learning rate." - ) - - if self.constant_steps is None or self.constant_steps == 0: - new_lrs = [ - _cosine_annealing( - initial_lr=initial_lr, - step=step - self.warmup_steps, - max_steps=self.max_steps - self.warmup_steps, - min_lr=self.min_lr, - ) - for initial_lr in self.base_lrs - ] - else: - new_lrs = self._get_linear_warmup_with_cosine_annealing_lr(step) - return new_lrs - - def _get_warmup_lr(self, step): - if self.constant_steps is None or self.constant_steps == 0: - return super()._get_warmup_lr(step) - else: - # Use linear warmup for the initial part. - return self._get_linear_warmup_with_cosine_annealing_lr(step) - - def _get_constant_lr(self, step): - # Only called when `constant_steps` > 0. - return self._get_linear_warmup_with_cosine_annealing_lr(step) - - def _get_linear_warmup_with_cosine_annealing_lr(self, step): - # Cosine Schedule for Megatron LM, - # slightly different warmup schedule + constant LR at the end. - new_lrs = [ - _linear_warmup_with_cosine_annealing( - max_lr=self.base_lrs[0], - warmup_steps=self.warmup_steps, - step=step, - decay_steps=self.decay_steps, - min_lr=self.min_lr, - ) - for _ in self.base_lrs - ] - return new_lrs - - -class NoamAnnealing(_LRScheduler): - - def __init__( - self, - optimizer, - *, - d_model, - warmup_steps=None, - warmup_ratio=None, - max_steps=None, - min_lr=0.0, - last_epoch=-1, - ): - self._normalize = d_model ** (-0.5) - assert not ( - warmup_steps is not None and warmup_ratio is not None - ), "Either use particular number of step or ratio" - assert ( - warmup_ratio is None or max_steps is not None - ), "If there is a ratio, there should be a total steps" - - # It is necessary to assign all attributes *before* __init__, - # as class is wrapped by an inner class. - self.max_steps = max_steps - if warmup_steps is not None: - self.warmup_steps = warmup_steps - elif warmup_ratio is not None: - self.warmup_steps = int(warmup_ratio * max_steps) - else: - self.warmup_steps = 0 - - self.min_lr = min_lr - super().__init__(optimizer, last_epoch) - - def get_lr(self): - if not self._get_lr_called_within_step: - warnings.warn( - "To get the last learning rate computed " - "by the scheduler, please use `get_last_lr()`.", - UserWarning, - stacklevel=2, - ) - - step = max(1, self.last_epoch) - - for initial_lr in self.base_lrs: - if initial_lr < self.min_lr: - raise ValueError( - f"{self} received an initial learning rate " - f"that was lower than the minimum learning rate." - ) - - new_lrs = [ - self._noam_annealing(initial_lr=initial_lr, step=step) - for initial_lr in self.base_lrs - ] - return new_lrs - - def _noam_annealing(self, initial_lr, step): - if self.warmup_steps > 0: - mult = self._normalize * min( - step ** (-0.5), step * (self.warmup_steps ** (-1.5)) - ) - else: - mult = self._normalize * step ** (-0.5) - - out_lr = initial_lr * mult - if step > self.warmup_steps: - out_lr = max(out_lr, self.min_lr) - return out_lr - - -class NoamHoldAnnealing(WarmupHoldPolicy): - - def __init__( - self, - optimizer, - *, - max_steps, - decay_rate=0.5, - min_lr=0.0, - last_epoch=-1, - **kwargs, - ): - """ - From Nemo: - Implementation of the Noam Hold Annealing policy - from the SqueezeFormer paper. - - Unlike NoamAnnealing, the peak learning rate - can be explicitly set for this scheduler. - The schedule first performs linear warmup, - then holds the peak LR, then decays with some schedule for - the remainder of the steps. - Therefore the min-lr is still dependent - on the hyper parameters selected. - - It's schedule is determined by three factors- - - Warmup Steps: Initial stage, where linear warmup - occurs uptil the peak LR is reached. Unlike NoamAnnealing, - the peak LR is explicitly stated here instead of a scaling factor. - - Hold Steps: Intermediate stage, where the peak LR - is maintained for some number of steps. In this region, - the high peak LR allows the model to converge faster - if training is stable. However the high LR - may also cause instability during training. - Should usually be a significant fraction of training - steps (around 30-40% of the entire training steps). - - Decay Steps: Final stage, where the LR rapidly decays - with some scaling rate (set by decay rate). - To attain Noam decay, use 0.5, - for Squeezeformer recommended decay, use 1.0. - The fast decay after prolonged high LR during - hold phase allows for rapid convergence. - - References: - - [Squeezeformer: - An Efficient Transformer for Automatic Speech Recognition] - (https://arxiv.org/abs/2206.00888) - - Args: - optimizer: Pytorch compatible Optimizer object. - warmup_steps: Number of training steps in warmup stage - warmup_ratio: Ratio of warmup steps to total steps - hold_steps: Number of training steps to - hold the learning rate after warm up - hold_ratio: Ratio of hold steps to total steps - max_steps: Total number of steps while training or `None` for - infinite training - decay_rate: Float value describing the polynomial decay - after the hold period. Default value - of 0.5 corresponds to Noam decay. - min_lr: Minimum learning rate. - """ - self.decay_rate = decay_rate - super().__init__( - optimizer=optimizer, - max_steps=max_steps, - last_epoch=last_epoch, - min_lr=min_lr, - **kwargs, - ) - - def _get_lr(self, step): - if self.warmup_steps is None or self.warmup_steps == 0: - raise ValueError("Noam scheduler cannot be used without warmup steps") - - if self.hold_steps > 0: - hold_steps = self.hold_steps - self.warmup_steps - else: - hold_steps = 0 - - new_lrs = [ - _noam_hold_annealing( - initial_lr, - step=step, - warmup_steps=self.warmup_steps, - hold_steps=hold_steps, - decay_rate=self.decay_rate, - min_lr=self.min_lr, - ) - for initial_lr in self.base_lrs - ] - return new_lrs - - def set_step(self, step: int): - self.last_epoch = step - - -class ConstantLR(_LRScheduler): - """The ConstantLR scheduler - - This scheduler keeps a constant lr - - """ - - def __init__( - self, - optimizer: torch.optim.Optimizer, - ): - # __init__() must be invoked before setting field - # because step() is also invoked in __init__() - super().__init__(optimizer) - - def get_lr(self): - return self.base_lrs - - def set_step(self, step: int): - self.last_epoch = step diff --git a/cosyvoice/utils/train_utils.py b/cosyvoice/utils/train_utils.py deleted file mode 100644 index fe9150c67ecc692266bcc31c6c5a35bf9bc84dac..0000000000000000000000000000000000000000 --- a/cosyvoice/utils/train_utils.py +++ /dev/null @@ -1,350 +0,0 @@ -# Copyright (c) 2021 Mobvoi Inc. (authors: Binbin Zhang) -# 2023 Horizon Inc. (authors: Xingchen Song) -# 2024 Alibaba Inc (authors: Xiang Lyu) -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from contextlib import nullcontext -import logging -import os -import torch -import json -import re -import datetime -import yaml - -import deepspeed -import torch.optim as optim -import torch.distributed as dist - -from torch.utils.tensorboard import SummaryWriter -from torch.utils.data import DataLoader -from torch.nn.utils import clip_grad_norm_ - -from deepspeed.runtime.zero.stage_1_and_2 import ( - estimate_zero2_model_states_mem_needs_all_live, -) - -from cosyvoice.dataset.dataset import Dataset -from cosyvoice.utils.scheduler import ( - WarmupLR, - NoamHoldAnnealing, - ConstantLR, -) - - -def init_distributed(args): - world_size = int(os.environ.get("WORLD_SIZE", 1)) - local_rank = int(os.environ.get("LOCAL_RANK", 0)) - rank = int(os.environ.get("RANK", 0)) - logging.info( - "training on multiple gpus, this gpu {}".format(local_rank) - + ", rank {}, world_size {}".format(rank, world_size) - ) - if args.train_engine == "torch_ddp": - torch.cuda.set_device(local_rank) - dist.init_process_group(args.dist_backend) - else: - deepspeed.init_distributed(dist_backend=args.dist_backend) - return world_size, local_rank, rank - - -def init_dataset_and_dataloader(args, configs): - train_dataset = Dataset( - args.train_data, - data_pipeline=configs["data_pipeline"], - mode="train", - shuffle=True, - partition=True, - ) - cv_dataset = Dataset( - args.cv_data, - data_pipeline=configs["data_pipeline"], - mode="train", - shuffle=False, - partition=False, - ) - - # do not use persistent_workers=True, as whisper tokenizer opens tiktoken file each time when the for loop starts - train_data_loader = DataLoader( - train_dataset, - batch_size=None, - pin_memory=args.pin_memory, - num_workers=args.num_workers, - prefetch_factor=args.prefetch, - ) - cv_data_loader = DataLoader( - cv_dataset, - batch_size=None, - pin_memory=args.pin_memory, - num_workers=args.num_workers, - prefetch_factor=args.prefetch, - ) - return train_dataset, cv_dataset, train_data_loader, cv_data_loader - - -def check_modify_and_save_config(args, configs): - if args.train_engine == "torch_ddp": - configs["train_conf"]["dtype"] = "fp32" - else: - with open(args.deepspeed_config, "r") as fin: - ds_configs = json.load(fin) - if "fp16" in ds_configs and ds_configs["fp16"]["enabled"]: - configs["train_conf"]["dtype"] = "fp16" - elif "bf16" in ds_configs and ds_configs["bf16"]["enabled"]: - configs["train_conf"]["dtype"] = "bf16" - else: - configs["train_conf"]["dtype"] = "fp32" - assert ds_configs["train_micro_batch_size_per_gpu"] == 1 - # if use deepspeed, override ddp config - configs["train_conf"]["save_per_step"] = int( - configs["train_conf"]["save_per_step"] - * configs["train_conf"]["accum_grad"] - / ds_configs["gradient_accumulation_steps"] - ) - configs["train_conf"]["accum_grad"] = ds_configs["gradient_accumulation_steps"] - configs["train_conf"]["grad_clip"] = ds_configs["gradient_clipping"] - configs["train_conf"]["log_interval"] = ds_configs["steps_per_print"] - return configs - - -def wrap_cuda_model(args, model): - local_world_size = int(os.environ.get("LOCAL_WORLD_SIZE", 1)) - world_size = int(os.environ.get("WORLD_SIZE", 1)) - if args.train_engine == "torch_ddp": # native pytorch ddp - assert torch.cuda.is_available() - model.cuda() - model = torch.nn.parallel.DistributedDataParallel( - model, find_unused_parameters=True - ) - else: - if int(os.environ.get("RANK", 0)) == 0: - logging.info("Estimating model states memory needs (zero2)...") - estimate_zero2_model_states_mem_needs_all_live( - model, - num_gpus_per_node=local_world_size, - num_nodes=world_size // local_world_size, - ) - return model - - -def init_optimizer_and_scheduler(args, configs, model): - if configs["train_conf"]["optim"] == "adam": - optimizer = optim.Adam( - model.parameters(), **configs["train_conf"]["optim_conf"] - ) - elif configs["train_conf"]["optim"] == "adamw": - optimizer = optim.AdamW( - model.parameters(), **configs["train_conf"]["optim_conf"] - ) - else: - raise ValueError("unknown optimizer: " + configs["train_conf"]) - - if configs["train_conf"]["scheduler"] == "warmuplr": - scheduler_type = WarmupLR - scheduler = WarmupLR(optimizer, **configs["train_conf"]["scheduler_conf"]) - elif configs["train_conf"]["scheduler"] == "NoamHoldAnnealing": - scheduler_type = NoamHoldAnnealing - scheduler = NoamHoldAnnealing( - optimizer, **configs["train_conf"]["scheduler_conf"] - ) - elif configs["train_conf"]["scheduler"] == "constantlr": - scheduler_type = ConstantLR - scheduler = ConstantLR(optimizer) - else: - raise ValueError("unknown scheduler: " + configs["train_conf"]) - - # use deepspeed optimizer for speedup - if args.train_engine == "deepspeed": - - def scheduler(opt): - return scheduler_type(opt, **configs["train_conf"]["scheduler_conf"]) - - model, optimizer, _, scheduler = deepspeed.initialize( - args=args, - model=model, - optimizer=None, - lr_scheduler=scheduler, - model_parameters=model.parameters(), - ) - - return model, optimizer, scheduler - - -def init_summarywriter(args): - writer = None - if int(os.environ.get("RANK", 0)) == 0: - os.makedirs(args.model_dir, exist_ok=True) - writer = SummaryWriter(args.tensorboard_dir) - return writer - - -def save_model(model, model_name, info_dict): - rank = int(os.environ.get("RANK", 0)) - model_dir = info_dict["model_dir"] - save_model_path = os.path.join(model_dir, "{}.pt".format(model_name)) - - if info_dict["train_engine"] == "torch_ddp": - if rank == 0: - torch.save(model.module.state_dict(), save_model_path) - else: - with torch.no_grad(): - model.save_checkpoint( - save_dir=model_dir, tag=model_name, client_state=info_dict - ) - if rank == 0: - info_path = re.sub(".pt$", ".yaml", save_model_path) - info_dict["save_time"] = datetime.datetime.now().strftime("%d/%m/%Y %H:%M:%S") - with open(info_path, "w") as fout: - data = yaml.dump(info_dict) - fout.write(data) - logging.info( - "[Rank {}] Checkpoint: save to checkpoint {}".format(rank, save_model_path) - ) - - -def cosyvoice_join(group_join, info_dict): - world_size = int(os.environ.get("WORLD_SIZE", 1)) - local_rank = int(os.environ.get("LOCAL_RANK", 0)) - rank = int(os.environ.get("RANK", 0)) - - if info_dict["batch_idx"] != 0: - # we try to join all rank in both ddp and deepspeed mode, in case different rank has different lr - try: - dist.monitored_barrier( - group=group_join, timeout=group_join.options._timeout - ) - return False - except RuntimeError as e: - logging.info( - "Detected uneven workload distribution: {}\n".format(e) - + "Break current worker to manually join all workers, " - + "world_size {}, current rank {}, current local_rank {}\n".format( - world_size, rank, local_rank - ) - ) - return True - else: - return False - - -def batch_forward(model, batch, info_dict): - device = int(os.environ.get("LOCAL_RANK", 0)) - - dtype = info_dict["dtype"] - if dtype == "fp16": - dtype = torch.float16 - elif dtype == "bf16": - dtype = torch.bfloat16 - else: # fp32 - dtype = torch.float32 - - if info_dict["train_engine"] == "torch_ddp": - autocast = nullcontext() - else: - autocast = torch.cuda.amp.autocast( - enabled=True, dtype=dtype, cache_enabled=False - ) - - with autocast: - info_dict["loss_dict"] = model(batch, device) - return info_dict - - -def batch_backward(model, info_dict): - if info_dict["train_engine"] == "deepspeed": - scaled_loss = model.backward(info_dict["loss_dict"]["loss"]) - else: - scaled_loss = info_dict["loss_dict"]["loss"] / info_dict["accum_grad"] - scaled_loss.backward() - - info_dict["loss_dict"]["loss"] = scaled_loss - return info_dict - - -def update_parameter_and_lr(model, optimizer, scheduler, info_dict): - grad_norm = 0.0 - if info_dict["train_engine"] == "deepspeed": - info_dict["is_gradient_accumulation_boundary"] = ( - model.is_gradient_accumulation_boundary() - ) - model.step() - grad_norm = model.get_global_grad_norm() - elif (info_dict["batch_idx"] + 1) % info_dict["accum_grad"] == 0: - grad_norm = clip_grad_norm_(model.parameters(), info_dict["grad_clip"]) - if torch.isfinite(grad_norm): - optimizer.step() - optimizer.zero_grad() - scheduler.step() - info_dict["lr"] = optimizer.param_groups[0]["lr"] - info_dict["grad_norm"] = grad_norm - return info_dict - - -def log_per_step(writer, info_dict): - tag = info_dict["tag"] - epoch = info_dict.get("epoch", 0) - step = info_dict["step"] - batch_idx = info_dict["batch_idx"] - loss_dict = info_dict["loss_dict"] - rank = int(os.environ.get("RANK", 0)) - - # only rank 0 write to tensorboard to avoid multi-process write - if writer is not None: - if ( - info_dict["train_engine"] == "deepspeed" - and info_dict["is_gradient_accumulation_boundary"] is True - ) or ( - info_dict["train_engine"] == "torch_ddp" - and (info_dict["batch_idx"] + 1) % info_dict["accum_grad"] == 0 - ): - for k in ["epoch", "lr", "grad_norm"]: - writer.add_scalar("{}/{}".format(tag, k), info_dict[k], step + 1) - for k, v in loss_dict.items(): - writer.add_scalar("{}/{}".format(tag, k), v, step + 1) - - # TRAIN & CV, Shell log (stdout) - if (info_dict["batch_idx"] + 1) % info_dict["log_interval"] == 0: - log_str = "{} Batch {}/{} ".format(tag, epoch, batch_idx + 1) - for name, value in loss_dict.items(): - log_str += "{} {:.6f} ".format(name, value) - if tag == "TRAIN": - log_str += "lr {:.8f} grad_norm {:.6f}".format( - info_dict["lr"], info_dict["grad_norm"] - ) - log_str += " rank {}".format(rank) - logging.debug(log_str) - - -def log_per_save(writer, info_dict): - tag = info_dict["tag"] - epoch = info_dict["epoch"] - step = info_dict["step"] - loss_dict = info_dict["loss_dict"] - lr = info_dict["lr"] - rank = int(os.environ.get("RANK", 0)) - logging.info( - "Epoch {} Step {} CV info lr {} {} rank {}".format( - epoch, - step + 1, - lr, - rank, - " ".join(["{}_{}".format(k, v) for k, v in loss_dict.items()]), - ) - ) - - if writer is not None: - for k in ["epoch", "lr"]: - writer.add_scalar("{}/{}".format(tag, k), info_dict[k], step + 1) - for k, v in loss_dict.items(): - writer.add_scalar("{}/{}".format(tag, k), v, step + 1) diff --git a/funasr_detach/__init__.py b/funasr_detach/__init__.py deleted file mode 100644 index 1a85dbca3c32b291b206a49e60559069c8c4dadc..0000000000000000000000000000000000000000 --- a/funasr_detach/__init__.py +++ /dev/null @@ -1,38 +0,0 @@ -"""Initialize funasr package.""" - -import os -import pkgutil -import importlib - -dirname = os.path.dirname(__file__) -version_file = os.path.join(dirname, "version.txt") -with open(version_file, "r") as f: - __version__ = f.read().strip() - - -import importlib -import pkgutil - - -def import_submodules(package, recursive=True): - if isinstance(package, str): - package = importlib.import_module(package) - results = {} - for loader, name, is_pkg in pkgutil.walk_packages( - package.__path__, package.__name__ + "." - ): - try: - results[name] = importlib.import_module(name) - except Exception as e: - # 如果想要看到导入错误的具体信息，可以取消注释下面的行 - # print(f"Failed to import {name}: {e}") - pass - if recursive and is_pkg: - results.update(import_submodules(name)) - return results - - -import_submodules(__name__) - -from funasr_detach.auto.auto_model import AutoModel -from funasr_detach.auto.auto_frontend import AutoFrontend diff --git a/funasr_detach/auto/__init__.py b/funasr_detach/auto/__init__.py deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/funasr_detach/auto/auto_frontend.py b/funasr_detach/auto/auto_frontend.py deleted file mode 100644 index 2bba442373cb0609df00b7d3bda6db3a338b9723..0000000000000000000000000000000000000000 --- a/funasr_detach/auto/auto_frontend.py +++ /dev/null @@ -1,90 +0,0 @@ -import time -import logging -from tqdm import tqdm - -from funasr_detach.register import tables -from funasr_detach.download.download_from_hub import download_model -from funasr_detach.utils.load_utils import load_audio_text_image_video, extract_fbank -from funasr_detach.auto.auto_model import prepare_data_iterator -from funasr_detach.auto.auto_model import prepare_data_iterator - - -class AutoFrontend: - def __init__(self, **kwargs): - assert "model" in kwargs - if "model_conf" not in kwargs: - logging.info( - "download models from model hub: {}".format( - kwargs.get("model_hub", "ms") - ) - ) - kwargs = download_model(**kwargs) - - # build frontend - frontend = kwargs.get("frontend", None) - if frontend is not None: - frontend_class = tables.frontend_classes.get(frontend) - frontend = frontend_class(**kwargs["frontend_conf"]) - - self.frontend = frontend - if "frontend" in kwargs: - del kwargs["frontend"] - self.kwargs = kwargs - - def __call__(self, input, input_len=None, kwargs=None, **cfg): - - kwargs = self.kwargs if kwargs is None else kwargs - kwargs.update(cfg) - - key_list, data_list = prepare_data_iterator(input, input_len=input_len) - batch_size = kwargs.get("batch_size", 1) - device = kwargs.get("device", "cpu") - if device == "cpu": - batch_size = 1 - - meta_data = {} - - result_list = [] - num_samples = len(data_list) - pbar = tqdm(colour="blue", total=num_samples + 1, dynamic_ncols=True) - - time0 = time.perf_counter() - for beg_idx in range(0, num_samples, batch_size): - end_idx = min(num_samples, beg_idx + batch_size) - data_batch = data_list[beg_idx:end_idx] - key_batch = key_list[beg_idx:end_idx] - - # extract fbank feats - time1 = time.perf_counter() - audio_sample_list = load_audio_text_image_video( - data_batch, fs=self.frontend.fs, audio_fs=kwargs.get("fs", 16000) - ) - time2 = time.perf_counter() - meta_data["load_data"] = f"{time2 - time1:0.3f}" - speech, speech_lengths = extract_fbank( - audio_sample_list, - data_type=kwargs.get("data_type", "sound"), - frontend=self.frontend, - **kwargs, - ) - time3 = time.perf_counter() - meta_data["extract_feat"] = f"{time3 - time2:0.3f}" - meta_data["batch_data_time"] = ( - speech_lengths.sum().item() - * self.frontend.frame_shift - * self.frontend.lfr_n - / 1000 - ) - - speech.to(device=device), speech_lengths.to(device=device) - batch = {"input": speech, "input_len": speech_lengths, "key": key_batch} - result_list.append(batch) - - pbar.update(1) - description = f"{meta_data}, " - pbar.set_description(description) - - time_end = time.perf_counter() - pbar.set_description(f"time escaped total: {time_end - time0:0.3f}") - - return result_list diff --git a/funasr_detach/auto/auto_model.py b/funasr_detach/auto/auto_model.py deleted file mode 100644 index 9315da8bfff1716377e9400e104c667fce743cfb..0000000000000000000000000000000000000000 --- a/funasr_detach/auto/auto_model.py +++ /dev/null @@ -1,573 +0,0 @@ -import json -import time -import copy -import torch -import random -import string -import logging -import os.path -import numpy as np -from tqdm import tqdm - -from funasr_detach.register import tables -from funasr_detach.utils.load_utils import load_bytes -from funasr_detach.download.file import download_from_url -from funasr_detach.download.download_from_hub import download_model -from funasr_detach.utils.vad_utils import slice_padding_audio_samples -from funasr_detach.train_utils.set_all_random_seed import set_all_random_seed -from funasr_detach.train_utils.load_pretrained_model import load_pretrained_model -from funasr_detach.utils.load_utils import load_audio_text_image_video -from funasr_detach.utils.timestamp_tools import timestamp_sentence -from funasr_detach.models.campplus.utils import sv_chunk, postprocess, distribute_spk - -try: - from funasr_detach.models.campplus.cluster_backend import ClusterBackend -except: - print("If you want to use the speaker diarization, please `pip install hdbscan`") - - -def prepare_data_iterator(data_in, input_len=None, data_type=None, key=None): - """ - - :param input: - :param input_len: - :param data_type: - :param frontend: - :return: - """ - data_list = [] - key_list = [] - filelist = [".scp", ".txt", ".json", ".jsonl"] - - chars = string.ascii_letters + string.digits - if isinstance(data_in, str) and data_in.startswith("http"): # url - data_in = download_from_url(data_in) - if isinstance(data_in, str) and os.path.exists( - data_in - ): # wav_path; filelist: wav.scp, file.jsonl;text.txt; - _, file_extension = os.path.splitext(data_in) - file_extension = file_extension.lower() - if file_extension in filelist: # filelist: wav.scp, file.jsonl;text.txt; - with open(data_in, encoding="utf-8") as fin: - for line in fin: - key = "rand_key_" + "".join(random.choice(chars) for _ in range(13)) - if data_in.endswith( - ".jsonl" - ): # file.jsonl: json.dumps({"source": data}) - lines = json.loads(line.strip()) - data = lines["source"] - key = data["key"] if "key" in data else key - else: # filelist, wav.scp, text.txt: id \t data or data - lines = line.strip().split(maxsplit=1) - data = lines[1] if len(lines) > 1 else lines[0] - key = lines[0] if len(lines) > 1 else key - - data_list.append(data) - key_list.append(key) - else: - key = "rand_key_" + "".join(random.choice(chars) for _ in range(13)) - data_list = [data_in] - key_list = [key] - elif isinstance(data_in, (list, tuple)): - if data_type is not None and isinstance( - data_type, (list, tuple) - ): # mutiple inputs - data_list_tmp = [] - for data_in_i, data_type_i in zip(data_in, data_type): - key_list, data_list_i = prepare_data_iterator( - data_in=data_in_i, data_type=data_type_i - ) - data_list_tmp.append(data_list_i) - data_list = [] - for item in zip(*data_list_tmp): - data_list.append(item) - else: - # [audio sample point, fbank, text] - data_list = data_in - key_list = [ - "rand_key_" + "".join(random.choice(chars) for _ in range(13)) - for _ in range(len(data_in)) - ] - else: # raw text; audio sample point, fbank; bytes - if isinstance(data_in, bytes): # audio bytes - data_in = load_bytes(data_in) - if key is None: - key = "rand_key_" + "".join(random.choice(chars) for _ in range(13)) - data_list = [data_in] - key_list = [key] - - return key_list, data_list - - -class AutoModel: - - def __init__(self, **kwargs): - if not kwargs.get("disable_log", False): - tables.print() - - model, kwargs = self.build_model(**kwargs) - - # if vad_model is not None, build vad model else None - vad_model = kwargs.get("vad_model", None) - vad_kwargs = kwargs.get("vad_model_revision", None) - if vad_model is not None: - logging.info("Building VAD model.") - vad_kwargs = { - "model": vad_model, - "model_revision": vad_kwargs, - "device": kwargs["device"], - } - vad_model, vad_kwargs = self.build_model(**vad_kwargs) - - # if punc_model is not None, build punc model else None - punc_model = kwargs.get("punc_model", None) - punc_kwargs = kwargs.get("punc_model_revision", None) - if punc_model is not None: - logging.info("Building punc model.") - punc_kwargs = { - "model": punc_model, - "model_revision": punc_kwargs, - "device": kwargs["device"], - } - punc_model, punc_kwargs = self.build_model(**punc_kwargs) - - # if spk_model is not None, build spk model else None - spk_model = kwargs.get("spk_model", None) - spk_kwargs = kwargs.get("spk_model_revision", None) - if spk_model is not None: - logging.info("Building SPK model.") - spk_kwargs = { - "model": spk_model, - "model_revision": spk_kwargs, - "device": kwargs["device"], - } - spk_model, spk_kwargs = self.build_model(**spk_kwargs) - self.cb_model = ClusterBackend().to(kwargs["device"]) - spk_mode = kwargs.get("spk_mode", "punc_segment") - if spk_mode not in ["default", "vad_segment", "punc_segment"]: - logging.error( - "spk_mode should be one of default, vad_segment and punc_segment." - ) - self.spk_mode = spk_mode - - self.kwargs = kwargs - self.model = model - self.vad_model = vad_model - self.vad_kwargs = vad_kwargs - self.punc_model = punc_model - self.punc_kwargs = punc_kwargs - self.spk_model = spk_model - self.spk_kwargs = spk_kwargs - self.model_path = kwargs.get("model_path") - - def build_model(self, **kwargs): - assert "model" in kwargs - if "model_conf" not in kwargs: - logging.info( - "download models from model hub: {}".format( - kwargs.get("model_hub", "ms") - ) - ) - kwargs = download_model(**kwargs) - - set_all_random_seed(kwargs.get("seed", 0)) - - device = kwargs.get("device", "cuda") - if not torch.cuda.is_available() or kwargs.get("ngpu", 1) == 0: - device = "cpu" - kwargs["batch_size"] = 1 - kwargs["device"] = device - - if kwargs.get("ncpu", None): - torch.set_num_threads(kwargs.get("ncpu")) - - # build tokenizer - tokenizer = kwargs.get("tokenizer", None) - if tokenizer is not None: - tokenizer_class = tables.tokenizer_classes.get(tokenizer) - tokenizer = tokenizer_class(**kwargs["tokenizer_conf"]) - kwargs["tokenizer"] = tokenizer - kwargs["token_list"] = tokenizer.token_list - vocab_size = len(tokenizer.token_list) - else: - vocab_size = -1 - - # build frontend - frontend = kwargs.get("frontend", None) - if frontend is not None: - frontend_class = tables.frontend_classes.get(frontend) - frontend = frontend_class(**kwargs["frontend_conf"]) - kwargs["frontend"] = frontend - kwargs["input_size"] = frontend.output_size() - - # build model - model_class = tables.model_classes.get(kwargs["model"]) - model = model_class(**kwargs, **kwargs["model_conf"], vocab_size=vocab_size) - - model.to(device) - - # init_param - init_param = kwargs.get("init_param", None) - if init_param is not None: - logging.info(f"Loading pretrained params from {init_param}") - load_pretrained_model( - model=model, - path=init_param, - ignore_init_mismatch=kwargs.get("ignore_init_mismatch", False), - oss_bucket=kwargs.get("oss_bucket", None), - scope_map=kwargs.get("scope_map", None), - excludes=kwargs.get("excludes", None), - ) - - return model, kwargs - - def __call__(self, *args, **cfg): - kwargs = self.kwargs - kwargs.update(cfg) - res = self.model(*args, kwargs) - return res - - def generate(self, input, input_len=None, **cfg): - if self.vad_model is None: - return self.inference(input, input_len=input_len, **cfg) - - else: - return self.inference_with_vad(input, input_len=input_len, **cfg) - - def inference( - self, input, input_len=None, model=None, kwargs=None, key=None, **cfg - ): - kwargs = self.kwargs if kwargs is None else kwargs - kwargs.update(cfg) - model = self.model if model is None else model - model = model.cuda() - model.eval() - - batch_size = kwargs.get("batch_size", 1) - # if kwargs.get("device", "cpu") == "cpu": - # batch_size = 1 - - key_list, data_list = prepare_data_iterator( - input, input_len=input_len, data_type=kwargs.get("data_type", None), key=key - ) - - speed_stats = {} - asr_result_list = [] - num_samples = len(data_list) - disable_pbar = kwargs.get("disable_pbar", False) - pbar = ( - tqdm(colour="blue", total=num_samples, dynamic_ncols=True) - if not disable_pbar - else None - ) - time_speech_total = 0.0 - time_escape_total = 0.0 - for beg_idx in range(0, num_samples, batch_size): - end_idx = min(num_samples, beg_idx + batch_size) - data_batch = data_list[beg_idx:end_idx] - key_batch = key_list[beg_idx:end_idx] - batch = {"data_in": data_batch, "key": key_batch} - if (end_idx - beg_idx) == 1 and kwargs.get( - "data_type", None - ) == "fbank": # fbank - batch["data_in"] = data_batch[0] - batch["data_lengths"] = input_len - - time1 = time.perf_counter() - with torch.no_grad(): - results, meta_data = model.inference(**batch, **kwargs) - time2 = time.perf_counter() - - asr_result_list.extend(results) - - # batch_data_time = time_per_frame_s * data_batch_i["speech_lengths"].sum().item() - batch_data_time = meta_data.get("batch_data_time", -1) - time_escape = time2 - time1 - speed_stats["load_data"] = meta_data.get("load_data", 0.0) - speed_stats["extract_feat"] = meta_data.get("extract_feat", 0.0) - speed_stats["forward"] = f"{time_escape:0.3f}" - speed_stats["batch_size"] = f"{len(results)}" - speed_stats["time_cost"] = f"{(time_escape)}" - speed_stats["rtf"] = f"{(time_escape) / batch_data_time:0.3f}" - description = f"{speed_stats}, " - if pbar: - pbar.update(1) - pbar.set_description(description) - time_speech_total += batch_data_time - time_escape_total += time_escape - - if pbar: - # pbar.update(1) - pbar.set_description(f"rtf_avg: {time_escape_total/time_speech_total:0.3f}") - torch.cuda.empty_cache() - return asr_result_list - - def inference_with_vad(self, input, input_len=None, **cfg): - - # step.1: compute the vad model - self.vad_kwargs.update(cfg) - beg_vad = time.time() - res = self.inference( - input, - input_len=input_len, - model=self.vad_model, - kwargs=self.vad_kwargs, - **cfg, - ) - end_vad = time.time() - print(f"time cost vad: {end_vad - beg_vad:0.3f}") - - # step.2 compute asr model - model = self.model - kwargs = self.kwargs - kwargs.update(cfg) - batch_size = int(kwargs.get("batch_size_s", 300)) * 1000 - batch_size_threshold_ms = int(kwargs.get("batch_size_threshold_s", 60)) * 1000 - kwargs["batch_size"] = batch_size - - key_list, data_list = prepare_data_iterator( - input, input_len=input_len, data_type=kwargs.get("data_type", None) - ) - results_ret_list = [] - time_speech_total_all_samples = 1e-6 - - beg_total = time.time() - pbar_total = tqdm(colour="red", total=len(res), dynamic_ncols=True) - for i in range(len(res)): - key = res[i]["key"] - vadsegments = res[i]["value"] - input_i = data_list[i] - speech = load_audio_text_image_video( - input_i, fs=kwargs["frontend"].fs, audio_fs=kwargs.get("fs", 16000) - ) - speech_lengths = len(speech) - n = len(vadsegments) - data_with_index = [(vadsegments[i], i) for i in range(n)] - sorted_data = sorted(data_with_index, key=lambda x: x[0][1] - x[0][0]) - results_sorted = [] - - if not len(sorted_data): - logging.info("decoding, utt: {}, empty speech".format(key)) - continue - - if len(sorted_data) > 0 and len(sorted_data[0]) > 0: - batch_size = max( - batch_size, sorted_data[0][0][1] - sorted_data[0][0][0] - ) - - batch_size_ms_cum = 0 - beg_idx = 0 - beg_asr_total = time.time() - time_speech_total_per_sample = speech_lengths / 16000 - time_speech_total_all_samples += time_speech_total_per_sample - - all_segments = [] - for j, _ in enumerate(range(0, n)): - # pbar_sample.update(1) - batch_size_ms_cum += sorted_data[j][0][1] - sorted_data[j][0][0] - if ( - j < n - 1 - and ( - batch_size_ms_cum - + sorted_data[j + 1][0][1] - - sorted_data[j + 1][0][0] - ) - < batch_size - and (sorted_data[j + 1][0][1] - sorted_data[j + 1][0][0]) - < batch_size_threshold_ms - ): - continue - batch_size_ms_cum = 0 - end_idx = j + 1 - speech_j, speech_lengths_j = slice_padding_audio_samples( - speech, speech_lengths, sorted_data[beg_idx:end_idx] - ) - results = self.inference( - speech_j, - input_len=None, - model=model, - kwargs=kwargs, - disable_pbar=True, - **cfg, - ) - if self.spk_model is not None: - # compose vad segments: [[start_time_sec, end_time_sec, speech], [...]] - for _b in range(len(speech_j)): - vad_segments = [ - [ - sorted_data[beg_idx:end_idx][_b][0][0] / 1000.0, - sorted_data[beg_idx:end_idx][_b][0][1] / 1000.0, - np.array(speech_j[_b]), - ] - ] - segments = sv_chunk(vad_segments) - all_segments.extend(segments) - speech_b = [i[2] for i in segments] - spk_res = self.inference( - speech_b, - input_len=None, - model=self.spk_model, - kwargs=kwargs, - disable_pbar=True, - **cfg, - ) - results[_b]["spk_embedding"] = spk_res[0]["spk_embedding"] - beg_idx = end_idx - if len(results) < 1: - continue - results_sorted.extend(results) - - restored_data = [0] * n - for j in range(n): - index = sorted_data[j][1] - restored_data[index] = results_sorted[j] - result = {} - - # results combine for texts, timestamps, speaker embeddings and others - # TODO: rewrite for clean code - for j in range(n): - for k, v in restored_data[j].items(): - if k.startswith("timestamp"): - if k not in result: - result[k] = [] - for t in restored_data[j][k]: - t[0] += vadsegments[j][0] - t[1] += vadsegments[j][0] - result[k].extend(restored_data[j][k]) - elif k == "spk_embedding": - if k not in result: - result[k] = restored_data[j][k] - else: - result[k] = torch.cat( - [result[k], restored_data[j][k]], dim=0 - ) - elif "text" in k: - if k not in result: - result[k] = restored_data[j][k] - else: - result[k] += " " + restored_data[j][k] - else: - if k not in result: - result[k] = restored_data[j][k] - else: - result[k] += restored_data[j][k] - - return_raw_text = kwargs.get("return_raw_text", False) - # step.3 compute punc model - if self.punc_model is not None: - self.punc_kwargs.update(cfg) - punc_res = self.inference( - result["text"], - model=self.punc_model, - kwargs=self.punc_kwargs, - disable_pbar=True, - **cfg, - ) - raw_text = copy.copy(result["text"]) - if return_raw_text: - result["raw_text"] = raw_text - result["text"] = punc_res[0]["text"] - else: - raw_text = None - - # speaker embedding cluster after resorted - if self.spk_model is not None and kwargs.get("return_spk_res", True): - if raw_text is None: - logging.error("Missing punc_model, which is required by spk_model.") - all_segments = sorted(all_segments, key=lambda x: x[0]) - spk_embedding = result["spk_embedding"] - labels = self.cb_model( - spk_embedding.cpu(), oracle_num=kwargs.get("preset_spk_num", None) - ) - # del result['spk_embedding'] - sv_output = postprocess(all_segments, None, labels, spk_embedding.cpu()) - if self.spk_mode == "vad_segment": # recover sentence_list - sentence_list = [] - for res, vadsegment in zip(restored_data, vadsegments): - if "timestamp" not in res: - logging.error( - "Only 'iic/speech_paraformer-large-vad-punc_asr_nat-zh-cn-16k-common-vocab8404-pytorch' \ - and 'iic/speech_seaco_paraformer_large_asr_nat-zh-cn-16k-common-vocab8404-pytorch'\ - can predict timestamp, and speaker diarization relies on timestamps." - ) - sentence_list.append( - { - "start": vadsegment[0], - "end": vadsegment[1], - "sentence": res["text"], - "timestamp": res["timestamp"], - } - ) - elif self.spk_mode == "punc_segment": - if "timestamp" not in result: - logging.error( - "Only 'iic/speech_paraformer-large-vad-punc_asr_nat-zh-cn-16k-common-vocab8404-pytorch' \ - and 'iic/speech_seaco_paraformer_large_asr_nat-zh-cn-16k-common-vocab8404-pytorch'\ - can predict timestamp, and speaker diarization relies on timestamps." - ) - sentence_list = timestamp_sentence( - punc_res[0]["punc_array"], - result["timestamp"], - raw_text, - return_raw_text=return_raw_text, - ) - distribute_spk(sentence_list, sv_output) - result["sentence_info"] = sentence_list - elif kwargs.get("sentence_timestamp", False): - sentence_list = timestamp_sentence( - punc_res[0]["punc_array"], - result["timestamp"], - raw_text, - return_raw_text=return_raw_text, - ) - result["sentence_info"] = sentence_list - if "spk_embedding" in result: - del result["spk_embedding"] - - result["key"] = key - results_ret_list.append(result) - end_asr_total = time.time() - time_escape_total_per_sample = end_asr_total - beg_asr_total - pbar_total.update(1) - pbar_total.set_description( - f"rtf_avg: {time_escape_total_per_sample / time_speech_total_per_sample:0.3f}, " - f"time_speech: {time_speech_total_per_sample: 0.3f}, " - f"time_escape: {time_escape_total_per_sample:0.3f}" - ) - - return results_ret_list - - def infer_encoder( - self, input, input_len=None, model=None, kwargs=None, key=None, **cfg - ): - kwargs = self.kwargs if kwargs is None else kwargs - kwargs.update(cfg) - model = self.model if model is None else model - model = model.cuda() - model.eval() - - batch_size = kwargs.get("batch_size", 1) - - key_list, data_list = prepare_data_iterator( - input, input_len=input_len, data_type=kwargs.get("data_type", None), key=key - ) - - asr_result_list = [] - num_samples = len(data_list) - for beg_idx in range(0, num_samples, batch_size): - end_idx = min(num_samples, beg_idx + batch_size) - data_batch = data_list[beg_idx:end_idx] - key_batch = key_list[beg_idx:end_idx] - batch = {"data_in": data_batch, "key": key_batch} - if (end_idx - beg_idx) == 1 and kwargs.get( - "data_type", None - ) == "fbank": # fbank - batch["data_in"] = data_batch[0] - batch["data_lengths"] = input_len - - with torch.no_grad(): - results, meta_data, cache = model.infer_encoder(**batch, **kwargs) - asr_result_list.extend(results) - - torch.cuda.empty_cache() - return asr_result_list, cache diff --git a/funasr_detach/auto/auto_tokenizer.py b/funasr_detach/auto/auto_tokenizer.py deleted file mode 100644 index 59ff8893d9ccadd6971b9a9d982116ae82e09255..0000000000000000000000000000000000000000 --- a/funasr_detach/auto/auto_tokenizer.py +++ /dev/null @@ -1,7 +0,0 @@ -class AutoTokenizer: - """ - Undo - """ - - def __init__(self): - pass diff --git a/funasr_detach/bin/__init__.py b/funasr_detach/bin/__init__.py deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/funasr_detach/bin/compute_audio_cmvn.py b/funasr_detach/bin/compute_audio_cmvn.py deleted file mode 100644 index 3459c71bfe8f63568f86fbf9b22a12bcd502d16a..0000000000000000000000000000000000000000 --- a/funasr_detach/bin/compute_audio_cmvn.py +++ /dev/null @@ -1,152 +0,0 @@ -import os -import json -import numpy as np -import torch -import hydra -import logging -from omegaconf import DictConfig, OmegaConf - -from funasr_detach.register import tables -from funasr_detach.download.download_from_hub import download_model -from funasr_detach.train_utils.set_all_random_seed import set_all_random_seed - - -@hydra.main(config_name=None, version_base=None) -def main_hydra(kwargs: DictConfig): - if kwargs.get("debug", False): - import pdb - - pdb.set_trace() - - assert "model" in kwargs - if "model_conf" not in kwargs: - logging.info( - "download models from model hub: {}".format(kwargs.get("model_hub", "ms")) - ) - kwargs = download_model(is_training=kwargs.get("is_training", True), **kwargs) - - main(**kwargs) - - -def main(**kwargs): - print(kwargs) - # set random seed - tables.print() - set_all_random_seed(kwargs.get("seed", 0)) - torch.backends.cudnn.enabled = kwargs.get( - "cudnn_enabled", torch.backends.cudnn.enabled - ) - torch.backends.cudnn.benchmark = kwargs.get( - "cudnn_benchmark", torch.backends.cudnn.benchmark - ) - torch.backends.cudnn.deterministic = kwargs.get("cudnn_deterministic", True) - - tokenizer = kwargs.get("tokenizer", None) - - # build frontend if frontend is none None - frontend = kwargs.get("frontend", None) - if frontend is not None: - frontend_class = tables.frontend_classes.get(frontend) - frontend = frontend_class(**kwargs["frontend_conf"]) - kwargs["frontend"] = frontend - kwargs["input_size"] = frontend.output_size() - - # dataset - dataset_class = tables.dataset_classes.get(kwargs.get("dataset", "AudioDataset")) - dataset_train = dataset_class( - kwargs.get("train_data_set_list"), - frontend=frontend, - tokenizer=None, - is_training=False, - **kwargs.get("dataset_conf") - ) - - # dataloader - batch_sampler = kwargs["dataset_conf"].get( - "batch_sampler", "DynamicBatchLocalShuffleSampler" - ) - batch_sampler_train = None - if batch_sampler is not None: - batch_sampler_class = tables.batch_sampler_classes.get(batch_sampler) - dataset_conf = kwargs.get("dataset_conf") - dataset_conf["batch_type"] = "example" - dataset_conf["batch_size"] = 1 - batch_sampler_train = batch_sampler_class( - dataset_train, is_training=False, **dataset_conf - ) - - dataloader_train = torch.utils.data.DataLoader( - dataset_train, - collate_fn=dataset_train.collator, - batch_sampler=batch_sampler_train, - num_workers=int(kwargs.get("dataset_conf").get("num_workers", 4)), - pin_memory=True, - ) - - iter_stop = int(kwargs.get("scale", 1.0) * len(dataloader_train)) - - total_frames = 0 - for batch_idx, batch in enumerate(dataloader_train): - if batch_idx >= iter_stop: - break - - fbank = batch["speech"].numpy()[0, :, :] - if total_frames == 0: - mean_stats = np.sum(fbank, axis=0) - var_stats = np.sum(np.square(fbank), axis=0) - else: - mean_stats += np.sum(fbank, axis=0) - var_stats += np.sum(np.square(fbank), axis=0) - total_frames += fbank.shape[0] - - cmvn_info = { - "mean_stats": list(mean_stats.tolist()), - "var_stats": list(var_stats.tolist()), - "total_frames": total_frames, - } - cmvn_file = kwargs.get("cmvn_file", "cmvn.json") - # import pdb;pdb.set_trace() - with open(cmvn_file, "w") as fout: - fout.write(json.dumps(cmvn_info)) - - mean = -1.0 * mean_stats / total_frames - var = 1.0 / np.sqrt(var_stats / total_frames - mean * mean) - dims = mean.shape[0] - am_mvn = os.path.dirname(cmvn_file) + "/am.mvn" - with open(am_mvn, "w") as fout: - fout.write( - "" - + "\n" - + " " - + str(dims) - + " " - + str(dims) - + "\n" - + "[ 0 ]" - + "\n" - + " " - + str(dims) - + " " - + str(dims) - + "\n" - ) - mean_str = ( - str(list(mean)).replace(",", "").replace("[", "[ ").replace("]", " ]") - ) - fout.write(" 0 " + mean_str + "\n") - fout.write(" " + str(dims) + " " + str(dims) + "\n") - var_str = str(list(var)).replace(",", "").replace("[", "[ ").replace("]", " ]") - fout.write(" 0 " + var_str + "\n") - fout.write("" + "\n") - - -""" -python funasr/bin/compute_audio_cmvn.py \ ---config-path "/Users/zhifu/funasr1.0/examples/aishell/paraformer/conf" \ ---config-name "train_asr_paraformer_conformer_12e_6d_2048_256.yaml" \ -++train_data_set_list="/Users/zhifu/funasr1.0/data/list/audio_datasets.jsonl" \ -++cmvn_file="/Users/zhifu/funasr1.0/data/list/cmvn.json" \ -++dataset_conf.num_workers=0 -""" -if __name__ == "__main__": - main_hydra() diff --git a/funasr_detach/bin/inference.py b/funasr_detach/bin/inference.py deleted file mode 100644 index f53ae31024380cdcf731af0d78a0d46a53d3d03d..0000000000000000000000000000000000000000 --- a/funasr_detach/bin/inference.py +++ /dev/null @@ -1,33 +0,0 @@ -import hydra -import logging -from omegaconf import DictConfig, OmegaConf, ListConfig - -from funasr_detach.auto.auto_model import AutoModel - - -@hydra.main(config_name=None, version_base=None) -def main_hydra(cfg: DictConfig): - def to_plain_list(cfg_item): - if isinstance(cfg_item, ListConfig): - return OmegaConf.to_container(cfg_item, resolve=True) - elif isinstance(cfg_item, DictConfig): - return {k: to_plain_list(v) for k, v in cfg_item.items()} - else: - return cfg_item - - kwargs = to_plain_list(cfg) - log_level = getattr(logging, kwargs.get("log_level", "INFO").upper()) - - logging.basicConfig(level=log_level) - - if kwargs.get("debug", False): - import pdb - - pdb.set_trace() - model = AutoModel(**kwargs) - res = model.generate(input=kwargs["input"]) - print(res) - - -if __name__ == "__main__": - main_hydra() diff --git a/funasr_detach/bin/tokenize_text.py b/funasr_detach/bin/tokenize_text.py deleted file mode 100755 index cad9892a4fcbea40413d5effe78c74e01020bcae..0000000000000000000000000000000000000000 --- a/funasr_detach/bin/tokenize_text.py +++ /dev/null @@ -1,281 +0,0 @@ -#!/usr/bin/env python3 -import argparse -from collections import Counter -import logging -from pathlib import Path -import sys -from typing import List -from typing import Optional - - -from funasr_detach.utils.cli_utils import get_commandline_args -from funasr_detach.tokenizer.build_tokenizer import build_tokenizer -from funasr_detach.tokenizer.cleaner import TextCleaner -from funasr_detach.tokenizer.phoneme_tokenizer import g2p_classes -from funasr_detach.utils.types import str2bool -from funasr_detach.utils.types import str_or_none - - -def field2slice(field: Optional[str]) -> slice: - """Convert field string to slice - - Note that field string accepts 1-based integer. - - Examples: - >>> field2slice("1-") - slice(0, None, None) - >>> field2slice("1-3") - slice(0, 3, None) - >>> field2slice("-3") - slice(None, 3, None) - """ - field = field.strip() - try: - if "-" in field: - # e.g. "2-" or "2-5" or "-7" - s1, s2 = field.split("-", maxsplit=1) - if s1.strip() == "": - s1 = None - else: - s1 = int(s1) - if s1 == 0: - raise ValueError("1-based string") - if s2.strip() == "": - s2 = None - else: - s2 = int(s2) - else: - # e.g. "2" - s1 = int(field) - s2 = s1 + 1 - if s1 == 0: - raise ValueError("must be 1 or more value") - except ValueError: - raise RuntimeError(f"Format error: e.g. '2-', '2-5', or '-5': {field}") - - if s1 is None: - slic = slice(None, s2) - else: - # -1 because of 1-based integer following "cut" command - # e.g "1-3" -> slice(0, 3) - slic = slice(s1 - 1, s2) - return slic - - -def tokenize( - input: str, - output: str, - field: Optional[str], - delimiter: Optional[str], - token_type: str, - space_symbol: str, - non_linguistic_symbols: Optional[str], - bpemodel: Optional[str], - log_level: str, - write_vocabulary: bool, - vocabulary_size: int, - remove_non_linguistic_symbols: bool, - cutoff: int, - add_symbol: List[str], - cleaner: Optional[str], - g2p: Optional[str], -): - - logging.basicConfig( - level=log_level, - format="%(asctime)s (%(module)s:%(lineno)d) %(levelname)s: %(message)s", - ) - if input == "-": - fin = sys.stdin - else: - fin = Path(input).open("r", encoding="utf-8") - if output == "-": - fout = sys.stdout - else: - p = Path(output) - p.parent.mkdir(parents=True, exist_ok=True) - fout = p.open("w", encoding="utf-8") - - cleaner = TextCleaner(cleaner) - tokenizer = build_tokenizer( - token_type=token_type, - bpemodel=bpemodel, - delimiter=delimiter, - space_symbol=space_symbol, - non_linguistic_symbols=non_linguistic_symbols, - remove_non_linguistic_symbols=remove_non_linguistic_symbols, - g2p_type=g2p, - ) - - counter = Counter() - if field is not None: - field = field2slice(field) - - for line in fin: - line = line.rstrip() - if field is not None: - # e.g. field="2-" - # uttidA hello world!! -> hello world!! - tokens = line.split(delimiter) - tokens = tokens[field] - if delimiter is None: - line = " ".join(tokens) - else: - line = delimiter.join(tokens) - - line = cleaner(line) - tokens = tokenizer.text2tokens(line) - if not write_vocabulary: - fout.write(" ".join(tokens) + "\n") - else: - for t in tokens: - counter[t] += 1 - - if not write_vocabulary: - return - - ## FIXME - ## del duplicate add_symbols in counter - for symbol_and_id in add_symbol: - # e.g symbol=":0" - try: - symbol, idx = symbol_and_id.split(":") - except ValueError: - raise RuntimeError(f"Format error: e.g. ':0': {symbol_and_id}") - symbol = symbol.strip() - if symbol in counter: - del counter[symbol] - - # ======= write_vocabulary mode from here ======= - # Sort by the number of occurrences in descending order - # and filter lower frequency words than cutoff value - words_and_counts = list( - filter(lambda x: x[1] > cutoff, sorted(counter.items(), key=lambda x: -x[1])) - ) - # Restrict the vocabulary size - if vocabulary_size > 0: - if vocabulary_size < len(add_symbol): - raise RuntimeError(f"vocabulary_size is too small: {vocabulary_size}") - words_and_counts = words_and_counts[: vocabulary_size - len(add_symbol)] - - # Parse the values of --add_symbol - for symbol_and_id in add_symbol: - # e.g symbol=":0" - try: - symbol, idx = symbol_and_id.split(":") - idx = int(idx) - except ValueError: - raise RuntimeError(f"Format error: e.g. ':0': {symbol_and_id}") - symbol = symbol.strip() - - # e.g. idx=0 -> append as the first symbol - # e.g. idx=-1 -> append as the last symbol - if idx < 0: - idx = len(words_and_counts) + 1 + idx - words_and_counts.insert(idx, (symbol, None)) - - # Write words - for w, c in words_and_counts: - fout.write(w + "\n") - - # Logging - total_count = sum(counter.values()) - invocab_count = sum(c for w, c in words_and_counts if c is not None) - logging.info(f"OOV rate = {(total_count - invocab_count) / total_count * 100} %") - - -def get_parser() -> argparse.ArgumentParser: - parser = argparse.ArgumentParser( - description="Tokenize texts", - formatter_class=argparse.ArgumentDefaultsHelpFormatter, - ) - parser.add_argument( - "--log_level", - type=lambda x: x.upper(), - default="INFO", - choices=("CRITICAL", "ERROR", "WARNING", "INFO", "DEBUG", "NOTSET"), - help="The verbose level of logging", - ) - - parser.add_argument( - "--input", "-i", required=True, help="Input text. - indicates sys.stdin" - ) - parser.add_argument( - "--output", "-o", required=True, help="Output text. - indicates sys.stdout" - ) - parser.add_argument( - "--field", - "-f", - help="The target columns of the input text as 1-based integer. e.g 2-", - ) - parser.add_argument( - "--token_type", - "-t", - default="char", - choices=["char", "bpe", "word", "phn"], - help="Token type", - ) - parser.add_argument("--delimiter", "-d", default=None, help="The delimiter") - parser.add_argument("--space_symbol", default="", help="The space symbol") - parser.add_argument("--bpemodel", default=None, help="The bpemodel file path") - parser.add_argument( - "--non_linguistic_symbols", - type=str_or_none, - help="non_linguistic_symbols file path", - ) - parser.add_argument( - "--remove_non_linguistic_symbols", - type=str2bool, - default=False, - help="Remove non-language-symbols from tokens", - ) - parser.add_argument( - "--cleaner", - type=str_or_none, - choices=[None, "tacotron", "jaconv", "vietnamese", "korean_cleaner"], - default=None, - help="Apply text cleaning", - ) - parser.add_argument( - "--g2p", - type=str_or_none, - choices=g2p_classes, - default=None, - help="Specify g2p method if --token_type=phn", - ) - - group = parser.add_argument_group("write_vocabulary mode related") - group.add_argument( - "--write_vocabulary", - type=str2bool, - default=False, - help="Write tokens list instead of tokenized text per line", - ) - group.add_argument("--vocabulary_size", type=int, default=0, help="Vocabulary size") - group.add_argument( - "--cutoff", - default=0, - type=int, - help="cut-off frequency used for write-vocabulary mode", - ) - group.add_argument( - "--add_symbol", - type=str, - default=[], - action="append", - help="Append symbol e.g. --add_symbol ':0' --add_symbol ':1'", - ) - - return parser - - -def main(cmd=None): - print(get_commandline_args(), file=sys.stderr) - parser = get_parser() - args = parser.parse_args(cmd) - kwargs = vars(args) - tokenize(**kwargs) - - -if __name__ == "__main__": - main() diff --git a/funasr_detach/bin/train.py b/funasr_detach/bin/train.py deleted file mode 100644 index cceeb781433914b28af5341c3be932eb8c32cef6..0000000000000000000000000000000000000000 --- a/funasr_detach/bin/train.py +++ /dev/null @@ -1,227 +0,0 @@ -#!/usr/bin/env python3 -# -*- encoding: utf-8 -*- - -import os -import sys -import torch -import hydra -import logging -import argparse -from io import BytesIO -import torch.distributed as dist -from collections.abc import Sequence -from omegaconf import DictConfig, OmegaConf -from torch.nn.parallel import DistributedDataParallel as DDP -from torch.distributed.fsdp import FullyShardedDataParallel as FSDP - -from funasr_detach.register import tables -from funasr_detach.optimizers import optim_classes -from funasr_detach.train_utils.trainer import Trainer -from funasr_detach.schedulers import scheduler_classes -from funasr_detach.train_utils.initialize import initialize -from funasr_detach.download.download_from_hub import download_model -from funasr_detach.models.lora.utils import mark_only_lora_as_trainable -from funasr_detach.train_utils.set_all_random_seed import set_all_random_seed -from funasr_detach.train_utils.load_pretrained_model import load_pretrained_model - -# from funasr_detach.tokenizer.build_tokenizer import build_tokenizer -# from funasr_detach.tokenizer.token_id_converter import TokenIDConverter -# from funasr_detach.tokenizer.funtoken import build_tokenizer - - -@hydra.main(config_name=None, version_base=None) -def main_hydra(kwargs: DictConfig): - if kwargs.get("debug", False): - import pdb - - pdb.set_trace() - - assert "model" in kwargs - if "model_conf" not in kwargs: - logging.info( - "download models from model hub: {}".format(kwargs.get("model_hub", "ms")) - ) - kwargs = download_model(is_training=kwargs.get("is_training", True), **kwargs) - - main(**kwargs) - - -def main(**kwargs): - print(kwargs) - - # set random seed - set_all_random_seed(kwargs.get("seed", 0)) - torch.backends.cudnn.enabled = kwargs.get( - "cudnn_enabled", torch.backends.cudnn.enabled - ) - torch.backends.cudnn.benchmark = kwargs.get( - "cudnn_benchmark", torch.backends.cudnn.benchmark - ) - torch.backends.cudnn.deterministic = kwargs.get("cudnn_deterministic", True) - - local_rank = int(os.environ.get("LOCAL_RANK", 0)) - if local_rank == 0: - tables.print() - # Check if we are using DDP or FSDP - use_ddp = "WORLD_SIZE" in os.environ and int(os.environ["WORLD_SIZE"]) > 1 - use_fsdp = kwargs.get("use_fsdp", None) - if use_ddp or use_fsdp: - dist.init_process_group( - backend=kwargs.get("backend", "nccl"), init_method="env://" - ) - torch.cuda.set_device(local_rank) - - # save config.yaml - if ( - (use_ddp or use_fsdp) - and dist.get_rank() == 0 - or not (use_ddp or use_fsdp) - and local_rank == 0 - ): - os.makedirs(kwargs.get("output_dir", "./"), exist_ok=True) - yaml_file = os.path.join(kwargs.get("output_dir", "./"), "config.yaml") - OmegaConf.save(config=kwargs, f=yaml_file) - logging.info("config.yaml is saved to: %s", yaml_file) - - tokenizer = kwargs.get("tokenizer", None) - if tokenizer is not None: - tokenizer_class = tables.tokenizer_classes.get(tokenizer) - tokenizer = tokenizer_class(**kwargs["tokenizer_conf"]) - kwargs["tokenizer"] = tokenizer - - # build frontend if frontend is none None - frontend = kwargs.get("frontend", None) - if frontend is not None: - frontend_class = tables.frontend_classes.get(frontend) - frontend = frontend_class(**kwargs["frontend_conf"]) - kwargs["frontend"] = frontend - kwargs["input_size"] = frontend.output_size() - - # build model - model_class = tables.model_classes.get(kwargs["model"]) - model = model_class( - **kwargs, **kwargs["model_conf"], vocab_size=len(tokenizer.token_list) - ) - - # init_param - init_param = kwargs.get("init_param", None) - if init_param is not None: - if not isinstance(init_param, (list, tuple)): - init_param = (init_param,) - logging.info("init_param is not None: %s", init_param) - for p in init_param: - logging.info(f"Loading pretrained params from {p}") - load_pretrained_model( - model=model, - path=p, - ignore_init_mismatch=kwargs.get("ignore_init_mismatch", True), - oss_bucket=kwargs.get("oss_bucket", None), - scope_map=kwargs.get("scope_map", None), - excludes=kwargs.get("excludes", None), - ) - else: - initialize(model, kwargs.get("init", "kaiming_normal")) - - # freeze_param - freeze_param = kwargs.get("freeze_param", None) - if freeze_param is not None: - freeze_param = eval(freeze_param) - if isinstance(freeze_param, Sequence): - freeze_param = (freeze_param,) - logging.info("freeze_param is not None: %s", freeze_param) - for t in freeze_param: - for k, p in model.named_parameters(): - if k.startswith(t + ".") or k == t: - logging.info(f"Setting {k}.requires_grad = False") - p.requires_grad = False - - if use_ddp: - model = model.cuda(local_rank) - model = DDP( - model, - device_ids=[local_rank], - find_unused_parameters=kwargs.get("train_conf", {}).get( - "find_unused_parameters", False - ), - ) - elif use_fsdp: - model = FSDP(model).cuda(local_rank) - else: - model = model.to(device=kwargs.get("device", "cuda")) - - # optim - optim = kwargs.get("optim", "adam") - assert optim in optim_classes - optim_class = optim_classes.get(optim) - optim = optim_class(model.parameters(), **kwargs.get("optim_conf")) - - # scheduler - scheduler = kwargs.get("scheduler", "warmuplr") - assert scheduler in scheduler_classes - scheduler_class = scheduler_classes.get(scheduler) - scheduler = scheduler_class(optim, **kwargs.get("scheduler_conf")) - - # dataset - dataset_class = tables.dataset_classes.get(kwargs.get("dataset", "AudioDataset")) - dataset_tr = dataset_class( - kwargs.get("train_data_set_list"), - frontend=frontend, - tokenizer=tokenizer, - is_training=True, - **kwargs.get("dataset_conf"), - ) - dataset_val = dataset_class( - kwargs.get("valid_data_set_list"), - frontend=frontend, - tokenizer=tokenizer, - is_training=False, - **kwargs.get("dataset_conf"), - ) - - # dataloader - batch_sampler = kwargs["dataset_conf"].get( - "batch_sampler", "DynamicBatchLocalShuffleSampler" - ) - batch_sampler_val = None - if batch_sampler is not None: - batch_sampler_class = tables.batch_sampler_classes.get(batch_sampler) - batch_sampler = batch_sampler_class(dataset_tr, **kwargs.get("dataset_conf")) - batch_sampler_val = batch_sampler_class( - dataset_val, is_training=False, **kwargs.get("dataset_conf") - ) - dataloader_tr = torch.utils.data.DataLoader( - dataset_tr, - collate_fn=dataset_tr.collator, - batch_sampler=batch_sampler, - num_workers=kwargs.get("dataset_conf").get("num_workers", 4), - pin_memory=True, - ) - - dataloader_val = torch.utils.data.DataLoader( - dataset_val, - collate_fn=dataset_val.collator, - batch_sampler=batch_sampler_val, - num_workers=kwargs.get("dataset_conf").get("num_workers", 4), - pin_memory=True, - ) - trainer = Trainer( - model=model, - optim=optim, - scheduler=scheduler, - dataloader_train=dataloader_tr, - dataloader_val=dataloader_val, - local_rank=local_rank, - use_ddp=use_ddp, - use_fsdp=use_fsdp, - output_dir=kwargs.get("output_dir", "./exp"), - resume=kwargs.get("resume", True), - **kwargs.get("train_conf"), - ) - trainer.run() - - if use_ddp or use_fsdp: - torch.distributed.destroy_process_group() - - -if __name__ == "__main__": - main_hydra() diff --git a/funasr_detach/datasets/__init__.py b/funasr_detach/datasets/__init__.py deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/funasr_detach/datasets/audio_datasets/__init__.py b/funasr_detach/datasets/audio_datasets/__init__.py deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/funasr_detach/datasets/audio_datasets/datasets.py b/funasr_detach/datasets/audio_datasets/datasets.py deleted file mode 100644 index a3f5c6522c4a81d52d32bbf1e51ebc6f52b7ca09..0000000000000000000000000000000000000000 --- a/funasr_detach/datasets/audio_datasets/datasets.py +++ /dev/null @@ -1,112 +0,0 @@ -import torch - -from funasr_detach.register import tables -from funasr_detach.utils.load_utils import extract_fbank, load_audio_text_image_video - - -@tables.register("dataset_classes", "AudioDataset") -class AudioDataset(torch.utils.data.Dataset): - """ - AudioDataset - """ - - def __init__( - self, - path, - index_ds: str = None, - frontend=None, - tokenizer=None, - int_pad_value: int = -1, - float_pad_value: float = 0.0, - **kwargs - ): - super().__init__() - index_ds_class = tables.index_ds_classes.get(index_ds) - self.index_ds = index_ds_class(path, **kwargs) - preprocessor_speech = kwargs.get("preprocessor_speech", None) - if preprocessor_speech: - preprocessor_speech_class = tables.preprocessor_classes.get( - preprocessor_speech - ) - preprocessor_speech = preprocessor_speech_class( - **kwargs.get("preprocessor_speech_conf") - ) - self.preprocessor_speech = preprocessor_speech - preprocessor_text = kwargs.get("preprocessor_text", None) - if preprocessor_text: - preprocessor_text_class = tables.preprocessor_classes.get(preprocessor_text) - preprocessor_text = preprocessor_text_class( - **kwargs.get("preprocessor_text_conf") - ) - self.preprocessor_text = preprocessor_text - - self.frontend = frontend - self.fs = 16000 if frontend is None else frontend.fs - self.data_type = "sound" - self.tokenizer = tokenizer - - self.int_pad_value = int_pad_value - self.float_pad_value = float_pad_value - - def get_source_len(self, index): - item = self.index_ds[index] - return self.index_ds.get_source_len(item) - - def get_target_len(self, index): - item = self.index_ds[index] - return self.index_ds.get_target_len(item) - - def __len__(self): - return len(self.index_ds) - - def __getitem__(self, index): - item = self.index_ds[index] - # import pdb; - # pdb.set_trace() - source = item["source"] - data_src = load_audio_text_image_video(source, fs=self.fs) - if self.preprocessor_speech: - data_src = self.preprocessor_speech(data_src, fs=self.fs) - speech, speech_lengths = extract_fbank( - data_src, data_type=self.data_type, frontend=self.frontend, is_final=True - ) # speech: [b, T, d] - - target = item["target"] - if self.preprocessor_text: - target = self.preprocessor_text(target) - if self.tokenizer: - ids = self.tokenizer.encode(target) - text = torch.tensor(ids, dtype=torch.int64) - else: - ids = target - text = ids - ids_lengths = len(ids) - text_lengths = torch.tensor([ids_lengths], dtype=torch.int32) - - return { - "speech": speech[0, :, :], - "speech_lengths": speech_lengths, - "text": text, - "text_lengths": text_lengths, - } - - def collator(self, samples: list = None): - outputs = {} - for sample in samples: - for key in sample.keys(): - if key not in outputs: - outputs[key] = [] - outputs[key].append(sample[key]) - - for key, data_list in outputs.items(): - if isinstance(data_list[0], torch.Tensor): - if data_list[0].dtype == torch.int64: - - pad_value = self.int_pad_value - else: - pad_value = self.float_pad_value - - outputs[key] = torch.nn.utils.rnn.pad_sequence( - data_list, batch_first=True, padding_value=pad_value - ) - return outputs diff --git a/funasr_detach/datasets/audio_datasets/index_ds.py b/funasr_detach/datasets/audio_datasets/index_ds.py deleted file mode 100644 index 90b606c887f4960357f3b7ea68f7eca9f1055f96..0000000000000000000000000000000000000000 --- a/funasr_detach/datasets/audio_datasets/index_ds.py +++ /dev/null @@ -1,150 +0,0 @@ -import os -import json -import torch -import logging -import concurrent.futures -import librosa -import torch.distributed as dist - -from funasr_detach.register import tables - - -@tables.register("index_ds_classes", "IndexDSJsonlRankSplit") -class IndexDSJsonlRankSplit(torch.utils.data.Dataset): - - def __init__(self, path): - super().__init__() - - contents = [] - with open(path, encoding="utf-8") as fin: - for line in fin: - data = json.loads(line.strip()) - if "text" in data: # for sft - self.contents.append(data["text"]) - if "source" in data: # for speech lab pretrain - prompt = data["prompt"] - source = data["source"] - target = data["target"] - source_len = data["source_len"] - target_len = data["target_len"] - - contents.append( - { - "source": source, - "prompt": prompt, - "target": target, - "source_len": source_len, - "target_len": target_len, - } - ) - - self.contents = [] - total_num = len(contents) - try: - rank = dist.get_rank() - world_size = dist.get_world_size() - except: - rank = 0 - world_size = 1 - logging.warning("distributed is not initialized, only single shard") - num_per_rank = total_num // world_size - - # rank = 0 - # import ipdb; ipdb.set_trace() - self.contents = contents[rank * num_per_rank : (rank + 1) * num_per_rank] - - logging.info( - "in rank: {}, num of samplers: {}, total_num of samplers across ranks: {}".format( - rank, len(self.contents), len(contents) - ) - ) - - def __len__(self): - return len(self.contents) - - def __getitem__(self, index): - try: - data = self.contents[index] - except: - print(index) - return data - - def get_source_len(self, data_dict): - return data_dict["source_len"] - - def get_target_len(self, data_dict): - - return data_dict["target_len"] if "target_len" in data_dict else 0 - - -@tables.register("index_ds_classes", "IndexDSJsonl") -@tables.register("index_ds_classes", "IndexDSJsonlRankFull") -class IndexDSJsonlRankFull(torch.utils.data.Dataset): - - def __init__(self, path: str, **kwargs): - super().__init__() - - if isinstance(path, (list, tuple)): # wav.scp, text.txt/text.trans - from funasr_detach.datasets.audio_datasets.scp2jsonl import ( - gen_jsonl_from_wav_text_list, - ) - - jsonl_outdir = os.path.dirname(path[0]) - jsonl_name = ( - "datalist_train.jsonl" - if kwargs.get("is_training", True) - else "datalist_val.jsonl" - ) - jsonl_file_out = os.path.join(jsonl_outdir, jsonl_name) - if not os.path.exists(jsonl_file_out): - print(f"datalist is: {path}, generate jsonl from it") - gen_jsonl_from_wav_text_list( - path, jsonl_file_out=jsonl_file_out, **kwargs - ) - path = jsonl_file_out - - contents = [] - with open(path, encoding="utf-8") as fin: - for line in fin: - data = json.loads(line.strip()) - if "text" in data: # for sft - self.contents.append(data["text"]) - if "source" in data: # for speech lab pretrain - prompt = data.get("prompt", "") - source = data["source"] - target = data["target"] - source_len = data.get("source_len", 1) - target_len = data.get("target_len", 0) - - contents.append( - { - "source": source, - "prompt": prompt, - "target": target, - "source_len": source_len, - "target_len": target_len, - } - ) - - self.contents = contents - - logging.info( - "total_num of samplers across ranks: {}".format(len(self.contents)) - ) - - def __len__(self): - return len(self.contents) - - def __getitem__(self, index): - try: - data = self.contents[index] - except: - print(index) - return data - - def get_source_len(self, data_dict): - return data_dict.get("source_len", 1) - - def get_target_len(self, data_dict): - - return data_dict.get("target_len", 0) diff --git a/funasr_detach/datasets/audio_datasets/preprocessor.py b/funasr_detach/datasets/audio_datasets/preprocessor.py deleted file mode 100644 index a1c4c8322eb6405bb6d1489dd01cd89df9ca9ef6..0000000000000000000000000000000000000000 --- a/funasr_detach/datasets/audio_datasets/preprocessor.py +++ /dev/null @@ -1,55 +0,0 @@ -import os -import json -import torch -import logging -import concurrent.futures -import librosa -import torch.distributed as dist -from typing import Collection -import torch -import torchaudio -from torch import nn -import random -import re -from funasr_detach.tokenizer.cleaner import TextCleaner -from funasr_detach.register import tables - - -@tables.register("preprocessor_classes", "SpeechPreprocessSpeedPerturb") -class SpeechPreprocessSpeedPerturb(nn.Module): - def __init__(self, speed_perturb: list = None, **kwargs): - super().__init__() - self.speed_perturb = speed_perturb - - def forward(self, waveform, fs, **kwargs): - if self.speed_perturb is None: - return waveform - speed = random.choice(self.speed_perturb) - if speed != 1.0: - if not isinstance(waveform, torch.Tensor): - waveform = torch.tensor(waveform) - waveform, _ = torchaudio.sox_effects.apply_effects_tensor( - waveform.view(1, -1), fs, [["speed", str(speed)], ["rate", str(fs)]] - ) - waveform = waveform.view(-1) - - return waveform - - -@tables.register("preprocessor_classes", "TextPreprocessSegDict") -class TextPreprocessSegDict(nn.Module): - def __init__( - self, - seg_dict: str = None, - text_cleaner: Collection[str] = None, - split_with_space: bool = False, - **kwargs - ): - super().__init__() - - self.text_cleaner = TextCleaner(text_cleaner) - - def forward(self, text, **kwargs): - text = self.text_cleaner(text) - - return text diff --git a/funasr_detach/datasets/audio_datasets/samplers.py b/funasr_detach/datasets/audio_datasets/samplers.py deleted file mode 100644 index a4b80936b170325d190678180ff9f1b32d9dd3d5..0000000000000000000000000000000000000000 --- a/funasr_detach/datasets/audio_datasets/samplers.py +++ /dev/null @@ -1,306 +0,0 @@ -import torch -import numpy as np -import logging -import torch.distributed as dist - -from funasr_detach.register import tables - - -@tables.register("batch_sampler_classes", "DynamicBatchLocalShuffleSampler") -class BatchSampler(torch.utils.data.BatchSampler): - - def __init__( - self, - dataset, - batch_type: str = "example", - batch_size: int = 100, - buffer_size: int = 30, - drop_last: bool = False, - shuffle: bool = True, - is_training: bool = True, - **kwargs - ): - - self.drop_last = drop_last - self.pre_idx = -1 - self.dataset = dataset - self.total_samples = len(dataset) - self.batch_type = batch_type - self.batch_size = int(batch_size) - self.buffer_size = buffer_size - self.max_token_length = kwargs.get("max_token_length", 5000) - self.shuffle_idx = np.arange(self.total_samples) - self.shuffle = shuffle and is_training - self.length_scale_source = kwargs.get("length_scale_source", 1.0) - - def __len__(self): - return (self.total_samples - 1) // self.batch_size + 1 - - def set_epoch(self, epoch): - np.random.seed(epoch) - - def __iter__(self): - - if self.shuffle: - np.random.shuffle(self.shuffle_idx) - - batch = [] - max_token = 0 - num_sample = 0 - - iter_num = (self.total_samples - 1) // self.buffer_size + 1 - # print("iter_num: ", iter_num) - for iter in range(self.pre_idx + 1, iter_num): - datalen_with_index = [] - for i in range(self.buffer_size): - idx = iter * self.buffer_size + i - if idx >= self.total_samples: - continue - - idx_map = self.shuffle_idx[idx] - # prompt = self.dataset.indexed_dataset[idx_map]["prompt"] - target_len = ( - self.dataset.get_target_len(idx_map) - if self.batch_type == "length" - else 0.0 - ) - source_len = ( - self.dataset.get_source_len(idx_map) / self.length_scale_source - ) - sample_len_cur = source_len + target_len - - datalen_with_index.append([idx, sample_len_cur]) - - datalen_with_index_sort = sorted(datalen_with_index, key=lambda x: x[1]) - for item in datalen_with_index_sort: - idx, sample_len_cur_raw = item - if sample_len_cur_raw > self.max_token_length: - continue - - max_token_cur = max(max_token, sample_len_cur_raw) - max_token_padding = 1 + num_sample - if self.batch_type != "example": - max_token_padding *= max_token_cur - if max_token_padding <= self.batch_size: - batch.append(idx) - max_token = max_token_cur - num_sample += 1 - else: - yield batch - batch = [idx] - max_token = sample_len_cur_raw - num_sample = 1 - - -@tables.register("batch_sampler_classes", "BatchSampler") -@tables.register("batch_sampler_classes", "RankFullLocalShuffleBatchSampler") -class RankFullLocalShuffleBatchSampler(torch.utils.data.BatchSampler): - - def __init__( - self, - dataset, - batch_type: str = "example", - batch_size: int = 100, - buffer_size: int = 30, - drop_last: bool = True, - shuffle: bool = True, - is_training: bool = True, - **kwargs - ): - - self.drop_last = drop_last - self.pre_idx = -1 - self.dataset = dataset - self.total_samples = len(dataset) - self.batch_type = batch_type - self.batch_size = int(batch_size) - self.buffer_size = buffer_size - self.max_token_length = kwargs.get("max_token_length", 1500) - self.shuffle_idx = np.arange(self.total_samples) - self.shuffle = shuffle and is_training - self.length_scale_source = kwargs.get("length_scale_source", 1.0) - - try: - rank = dist.get_rank() - world_size = dist.get_world_size() - except: - rank = 0 - world_size = 1 - self.rank = rank - self.world_size = world_size - - def __len__(self): - return (self.total_samples - 1) // (self.batch_size * self.world_size) + 1 - - def set_epoch(self, epoch): - np.random.seed(epoch) - - def __iter__(self): - - batch_size_total = self.batch_size * self.world_size - - if self.shuffle: - np.random.shuffle(self.shuffle_idx) - - batch = [] - max_token = 0 - num_sample = 0 - - iter_num = (self.total_samples - 1) // self.buffer_size + 1 - # print("iter_num: ", iter_num) - for iter in range(self.pre_idx + 1, iter_num): - # if iter == iter_num -1 and self.drop_last: - # continue - datalen_with_index = [] - for i in range(self.buffer_size): - idx = iter * self.buffer_size + i - if idx >= self.total_samples: - continue - - idx_map = self.shuffle_idx[idx] - # prompt = self.dataset.indexed_dataset[idx_map]["prompt"] - - source_len = ( - self.dataset.get_source_len(idx_map) / self.length_scale_source - ) - target_len = ( - self.dataset.get_target_len(idx_map) - if self.batch_type == "length" - else 0.0 - ) - sample_len_cur = source_len + target_len - - datalen_with_index.append([idx, sample_len_cur]) - - datalen_with_index_sort = sorted(datalen_with_index, key=lambda x: x[1]) - for item in datalen_with_index_sort: - idx, sample_len_cur_raw = item - if sample_len_cur_raw > self.max_token_length: - continue - - max_token_cur = max(max_token, sample_len_cur_raw) - max_token_padding = 1 + num_sample - # if self.batch_type != 'example': - # max_token_padding *= max_token_cur - if max_token_padding <= batch_size_total: - batch.append(idx) - max_token = max_token_cur - num_sample += 1 - else: - batch_rank = batch[ - self.rank * self.batch_size : (self.rank + 1) * self.batch_size - ] - yield batch_rank - batch = [idx] - max_token = sample_len_cur_raw - num_sample = 1 - - -@tables.register("batch_sampler_classes", "RankFullLocalShuffleDynamicBatchSampler") -class RankFullLocalShuffleDynamicBatchSampler(torch.utils.data.BatchSampler): - - def __init__( - self, - dataset, - batch_type: str = "example", - batch_size: int = 100, - buffer_size: int = 30, - drop_last: bool = True, - shuffle: bool = True, - is_training: bool = True, - **kwargs - ): - - self.drop_last = drop_last - self.pre_idx = -1 - self.dataset = dataset - self.total_samples = len(dataset) - self.batch_type = batch_type - self.batch_size = int(batch_size) - self.buffer_size = buffer_size - self.max_token_length = kwargs.get("max_token_length", 1500) - self.shuffle_idx = np.arange(self.total_samples) - self.shuffle = shuffle and is_training - self.length_scale_source = kwargs.get("length_scale_source", 1.0) - - try: - rank = dist.get_rank() - world_size = dist.get_world_size() - except: - rank = 0 - world_size = 1 - self.rank = rank - self.world_size = world_size - - def __len__(self): - return (self.total_samples - 1) // (self.batch_size * self.world_size) + 1 - - def set_epoch(self, epoch): - np.random.seed(epoch) - - def __iter__(self): - - batch_size_total = self.batch_size * self.world_size - if self.shuffle: - np.random.shuffle(self.shuffle_idx) - - batch_list_all_rank = [] - batch_list_cur = [] - max_token = 0 - num_sample = 0 - - iter_num = (self.total_samples - 1) // self.buffer_size + 1 - # print("iter_num: ", iter_num) - for iter in range(self.pre_idx + 1, iter_num): - # if iter == iter_num - 1 and self.drop_last: - # continue - datalen_with_index = [] - for i in range(self.buffer_size): - idx = iter * self.buffer_size + i - if idx >= self.total_samples: - continue - - idx_map = self.shuffle_idx[idx] - # prompt = self.dataset.indexed_dataset[idx_map]["prompt"] - - source_len = ( - self.dataset.get_source_len(idx_map) / self.length_scale_source - ) - target_len = ( - self.dataset.get_target_len(idx_map) - if self.batch_type == "length" - else 0.0 - ) - sample_len_cur = source_len + target_len - - datalen_with_index.append([idx, sample_len_cur]) - - datalen_with_index_sort = sorted(datalen_with_index, key=lambda x: x[1]) - for ii, item in enumerate(datalen_with_index_sort): - is_last_batch = iter == iter_num - 1 and ii == len( - datalen_with_index_sort - ) - idx, sample_len_cur_raw = item - if sample_len_cur_raw > self.max_token_length: - continue - - max_token_cur = max(max_token, sample_len_cur_raw) - max_token_padding = 1 + num_sample - - if self.batch_type != "example": - max_token_padding *= max_token_cur - if len(batch_list_all_rank) < self.world_size: - - if max_token_padding <= self.batch_size: - batch_list_cur.append(idx) - max_token = max_token_cur - num_sample += 1 - else: - batch_list_all_rank.append(batch_list_cur) - batch_list_cur = [] - else: - batch_rank = batch_list_all_rank[self.rank] - yield batch_rank - batch_list_all_rank = [idx] - max_token = sample_len_cur_raw - num_sample = 1 diff --git a/funasr_detach/datasets/audio_datasets/scp2jsonl.py b/funasr_detach/datasets/audio_datasets/scp2jsonl.py deleted file mode 100644 index 275593749020ac79ee8c297a7c0118d9ab5f7c83..0000000000000000000000000000000000000000 --- a/funasr_detach/datasets/audio_datasets/scp2jsonl.py +++ /dev/null @@ -1,116 +0,0 @@ -import os -import json -import torch -import logging -import hydra -from omegaconf import DictConfig, OmegaConf -import concurrent.futures -import librosa -import torch.distributed as dist - - -def gen_jsonl_from_wav_text_list( - path, data_type_list=("source", "target"), jsonl_file_out: str = None, **kwargs -): - try: - rank = dist.get_rank() - world_size = dist.get_world_size() - except: - rank = 0 - world_size = 1 - - cpu_cores = os.cpu_count() or 1 - print(f"convert wav.scp text to jsonl, ncpu: {cpu_cores}") - if rank == 0: - json_dict = {} - for data_type, data_file in zip(data_type_list, path): - json_dict[data_type] = {} - with open(data_file, "r") as f: - - data_file_lists = f.readlines() - lines_for_each_th = (len(data_file_lists) - 1) // cpu_cores + 1 - task_num = cpu_cores if len(data_file_lists) > cpu_cores else 1 - with concurrent.futures.ThreadPoolExecutor( - max_workers=cpu_cores - ) as executor: - - futures = [ - executor.submit( - parse_context_length, - data_file_lists[ - i * lines_for_each_th : (i + 1) * lines_for_each_th - ], - data_type, - ) - for i in range(task_num) - ] - - for future in concurrent.futures.as_completed(futures): - - json_dict[data_type].update(future.result()) - # print(json_dict) - - with open(jsonl_file_out, "w") as f: - for key in json_dict[data_type_list[0]].keys(): - jsonl_line = {"key": key} - for data_file in data_type_list: - jsonl_line.update(json_dict[data_file][key]) - jsonl_line = json.dumps(jsonl_line, ensure_ascii=False) - f.write(jsonl_line + "\n") - f.flush() - - else: - pass - - if world_size > 1: - dist.barrier() - - -def parse_context_length(data_list: list, data_type: str): - - res = {} - for i, line in enumerate(data_list): - key, line = line.strip().split(maxsplit=1) - line = line.strip() - if os.path.exists(line): - waveform, _ = librosa.load(line, sr=16000) - sample_num = len(waveform) - context_len = int(sample_num // 16000 * 1000 / 10) - else: - context_len = len(line.split()) if " " in line else len(line) - res[key] = {data_type: line, f"{data_type}_len": context_len} - return res - - -@hydra.main(config_name=None, version_base=None) -def main_hydra(cfg: DictConfig): - - kwargs = OmegaConf.to_container(cfg, resolve=True) - - scp_file_list = kwargs.get( - "scp_file_list", - ( - "/Users/zhifu/funasr1.0/test_local/wav.scp", - "/Users/zhifu/funasr1.0/test_local/text.txt", - ), - ) - if isinstance(scp_file_list, str): - scp_file_list = eval(scp_file_list) - data_type_list = kwargs.get("data_type_list", ("source", "target")) - jsonl_file_out = kwargs.get( - "jsonl_file_out", "/Users/zhifu/funasr1.0/test_local/audio_datasets.jsonl" - ) - gen_jsonl_from_wav_text_list( - scp_file_list, data_type_list=data_type_list, jsonl_file_out=jsonl_file_out - ) - - -""" -python -m funasr_detach.datasets.audio_datasets.scp2jsonl \ -++scp_file_list='["/Users/zhifu/funasr1.0/test_local/wav.scp", "/Users/zhifu/funasr1.0/test_local/text.txt"]' \ -++data_type_list='["source", "target"]' \ -++jsonl_file_out=/Users/zhifu/funasr1.0/test_local/audio_datasets.jsonl -""" - -if __name__ == "__main__": - main_hydra() diff --git a/funasr_detach/download/__init__.py b/funasr_detach/download/__init__.py deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/funasr_detach/download/download_dataset_from_hub.py b/funasr_detach/download/download_dataset_from_hub.py deleted file mode 100644 index 6834137bc849c6b6f710d5b23168ebceb9a898ed..0000000000000000000000000000000000000000 --- a/funasr_detach/download/download_dataset_from_hub.py +++ /dev/null @@ -1,19 +0,0 @@ -def download_dataset(): - pass - - -def download_dataset_from_ms(**kwargs): - from modelscope.msdatasets import MsDataset - - dataset_name = kwargs.get( - "dataset_name", "speech_asr/speech_asr_aishell1_trainsets" - ) - subset_name = kwargs.get("subset_name", "default") - split = kwargs.get("split", "train") - data_dump_dir = kwargs.get("data_dump_dir", None) - ds = MsDataset.load( - dataset_name=dataset_name, - subset_name=subset_name, - split=split, - cache_dir=data_dump_dir, - ) diff --git a/funasr_detach/download/download_from_hub.py b/funasr_detach/download/download_from_hub.py deleted file mode 100644 index 6cfca6f75becbe5625cd444e4be0e08bcac4e31e..0000000000000000000000000000000000000000 --- a/funasr_detach/download/download_from_hub.py +++ /dev/null @@ -1,122 +0,0 @@ -import os -import json -from omegaconf import OmegaConf - -from funasr_detach.download.name_maps_from_hub import name_maps_ms, name_maps_hf - - -def download_model(**kwargs): - model_hub = kwargs.get("model_hub", "ms") - if model_hub == "ms": - kwargs = download_from_ms(**kwargs) - - return kwargs - - -def download_from_ms(**kwargs): - model_or_path = kwargs.get("model") - if model_or_path in name_maps_ms: - model_or_path = name_maps_ms[model_or_path] - model_revision = kwargs.get("model_revision") - if not os.path.exists(model_or_path): - model_or_path = get_or_download_model_dir( - model_or_path, - model_revision, - is_training=kwargs.get("is_training"), - check_latest=kwargs.get("kwargs", True), - ) - kwargs["model_path"] = model_or_path - - if os.path.exists(os.path.join(model_or_path, "configuration.json")): - with open( - os.path.join(model_or_path, "configuration.json"), "r", encoding="utf-8" - ) as f: - conf_json = json.load(f) - cfg = {} - add_file_root_path(model_or_path, conf_json["file_path_metas"], cfg) - cfg.update(kwargs) - config = OmegaConf.load(cfg["config"]) - kwargs = OmegaConf.merge(config, cfg) - kwargs["model"] = config["model"] - elif os.path.exists(os.path.join(model_or_path, "config.yaml")) and os.path.exists( - os.path.join(model_or_path, "model.pt") - ): - config = OmegaConf.load(os.path.join(model_or_path, "config.yaml")) - kwargs = OmegaConf.merge(config, kwargs) - init_param = os.path.join(model_or_path, "model.pb") - kwargs["init_param"] = init_param - if os.path.exists(os.path.join(model_or_path, "tokens.txt")): - kwargs["tokenizer_conf"]["token_list"] = os.path.join( - model_or_path, "tokens.txt" - ) - if os.path.exists(os.path.join(model_or_path, "tokens.json")): - kwargs["tokenizer_conf"]["token_list"] = os.path.join( - model_or_path, "tokens.json" - ) - if os.path.exists(os.path.join(model_or_path, "seg_dict")): - kwargs["tokenizer_conf"]["seg_dict"] = os.path.join( - model_or_path, "seg_dict" - ) - if os.path.exists(os.path.join(model_or_path, "bpe.model")): - kwargs["tokenizer_conf"]["bpemodel"] = os.path.join( - model_or_path, "bpe.model" - ) - kwargs["model"] = config["model"] - if os.path.exists(os.path.join(model_or_path, "am.mvn")): - kwargs["frontend_conf"]["cmvn_file"] = os.path.join(model_or_path, "am.mvn") - if os.path.exists(os.path.join(model_or_path, "jieba_usr_dict")): - kwargs["jieba_usr_dict"] = os.path.join(model_or_path, "jieba_usr_dict") - return OmegaConf.to_container(kwargs, resolve=True) - - -def add_file_root_path(model_or_path: str, file_path_metas: dict, cfg={}): - - if isinstance(file_path_metas, dict): - for k, v in file_path_metas.items(): - if isinstance(v, str): - p = os.path.join(model_or_path, v) - if os.path.exists(p): - cfg[k] = p - elif isinstance(v, dict): - if k not in cfg: - cfg[k] = {} - add_file_root_path(model_or_path, v, cfg[k]) - - return cfg - - -def get_or_download_model_dir( - model, - model_revision=None, - is_training=False, - check_latest=True, -): - """Get local model directory or download model if necessary. - - Args: - model (str): model id or path to local model directory. - model_revision (str, optional): model version number. - :param is_training: - """ - from modelscope.hub.check_model import check_local_model_is_latest - from modelscope.hub.snapshot_download import snapshot_download - - from modelscope.utils.constant import Invoke, ThirdParty - - key = Invoke.LOCAL_TRAINER if is_training else Invoke.PIPELINE - - if os.path.exists(model) and check_latest: - model_cache_dir = model if os.path.isdir(model) else os.path.dirname(model) - try: - check_local_model_is_latest( - model_cache_dir, user_agent={Invoke.KEY: key, ThirdParty.KEY: "funasr"} - ) - except: - print("could not check the latest version") - else: - model_cache_dir = snapshot_download( - model, - revision=model_revision, - user_agent={Invoke.KEY: key, ThirdParty.KEY: "funasr"}, - ) - return model_cache_dir diff --git a/funasr_detach/download/file.py b/funasr_detach/download/file.py deleted file mode 100644 index ebb1a3dc10b7ef9385694e892c676847812adaab..0000000000000000000000000000000000000000 --- a/funasr_detach/download/file.py +++ /dev/null @@ -1,335 +0,0 @@ -# Copyright (c) Alibaba, Inc. and its affiliates. - -import contextlib -import os -import tempfile -from abc import ABCMeta, abstractmethod -from pathlib import Path -from typing import Generator, Union - -import requests -from urllib.parse import urlparse - - -def download_from_url(url): - result = urlparse(url) - file_path = None - if result.scheme is not None and len(result.scheme) > 0: - storage = HTTPStorage() - # bytes - data = storage.read(url) - work_dir = tempfile.TemporaryDirectory().name - if not os.path.exists(work_dir): - os.makedirs(work_dir) - file_path = os.path.join(work_dir, os.path.basename(url)) - with open(file_path, "wb") as fb: - fb.write(data) - assert file_path is not None, f"failed to download: {url}" - return file_path - - -class Storage(metaclass=ABCMeta): - """Abstract class of storage. - - All backends need to implement two apis: ``read()`` and ``read_text()``. - ``read()`` reads the file as a byte stream and ``read_text()`` reads - the file as texts. - """ - - @abstractmethod - def read(self, filepath: str): - pass - - @abstractmethod - def read_text(self, filepath: str): - pass - - @abstractmethod - def write(self, obj: bytes, filepath: Union[str, Path]) -> None: - pass - - @abstractmethod - def write_text( - self, obj: str, filepath: Union[str, Path], encoding: str = "utf-8" - ) -> None: - pass - - -class LocalStorage(Storage): - """Local hard disk storage""" - - def read(self, filepath: Union[str, Path]) -> bytes: - """Read data from a given ``filepath`` with 'rb' mode. - - Args: - filepath (str or Path): Path to read data. - - Returns: - bytes: Expected bytes object. - """ - with open(filepath, "rb") as f: - content = f.read() - return content - - def read_text(self, filepath: Union[str, Path], encoding: str = "utf-8") -> str: - """Read data from a given ``filepath`` with 'r' mode. - - Args: - filepath (str or Path): Path to read data. - encoding (str): The encoding format used to open the ``filepath``. - Default: 'utf-8'. - - Returns: - str: Expected text reading from ``filepath``. - """ - with open(filepath, "r", encoding=encoding) as f: - value_buf = f.read() - return value_buf - - def write(self, obj: bytes, filepath: Union[str, Path]) -> None: - """Write data to a given ``filepath`` with 'wb' mode. - - Note: - ``write`` will create a directory if the directory of ``filepath`` - does not exist. - - Args: - obj (bytes): Data to be written. - filepath (str or Path): Path to write data. - """ - dirname = os.path.dirname(filepath) - if dirname and not os.path.exists(dirname): - os.makedirs(dirname, exist_ok=True) - - with open(filepath, "wb") as f: - f.write(obj) - - def write_text( - self, obj: str, filepath: Union[str, Path], encoding: str = "utf-8" - ) -> None: - """Write data to a given ``filepath`` with 'w' mode. - - Note: - ``write_text`` will create a directory if the directory of - ``filepath`` does not exist. - - Args: - obj (str): Data to be written. - filepath (str or Path): Path to write data. - encoding (str): The encoding format used to open the ``filepath``. - Default: 'utf-8'. - """ - dirname = os.path.dirname(filepath) - if dirname and not os.path.exists(dirname): - os.makedirs(dirname, exist_ok=True) - - with open(filepath, "w", encoding=encoding) as f: - f.write(obj) - - @contextlib.contextmanager - def as_local_path( - self, filepath: Union[str, Path] - ) -> Generator[Union[str, Path], None, None]: - """Only for unified API and do nothing.""" - yield filepath - - -class HTTPStorage(Storage): - """HTTP and HTTPS storage.""" - - def read(self, url): - # TODO @wenmeng.zwm add progress bar if file is too large - r = requests.get(url) - r.raise_for_status() - return r.content - - def read_text(self, url): - r = requests.get(url) - r.raise_for_status() - return r.text - - @contextlib.contextmanager - def as_local_path(self, filepath: str) -> Generator[Union[str, Path], None, None]: - """Download a file from ``filepath``. - - ``as_local_path`` is decorated by :meth:`contextlib.contextmanager`. It - can be called with ``with`` statement, and when exists from the - ``with`` statement, the temporary path will be released. - - Args: - filepath (str): Download a file from ``filepath``. - - Examples: - >>> storage = HTTPStorage() - >>> # After existing from the ``with`` clause, - >>> # the path will be removed - >>> with storage.get_local_path('http://path/to/file') as path: - ... # do something here - """ - try: - f = tempfile.NamedTemporaryFile(delete=False) - f.write(self.read(filepath)) - f.close() - yield f.name - finally: - os.remove(f.name) - - def write(self, obj: bytes, url: Union[str, Path]) -> None: - raise NotImplementedError("write is not supported by HTTP Storage") - - def write_text( - self, obj: str, url: Union[str, Path], encoding: str = "utf-8" - ) -> None: - raise NotImplementedError("write_text is not supported by HTTP Storage") - - -class OSSStorage(Storage): - """OSS storage.""" - - def __init__(self, oss_config_file=None): - # read from config file or env var - raise NotImplementedError("OSSStorage.__init__ to be implemented in the future") - - def read(self, filepath): - raise NotImplementedError("OSSStorage.read to be implemented in the future") - - def read_text(self, filepath, encoding="utf-8"): - raise NotImplementedError( - "OSSStorage.read_text to be implemented in the future" - ) - - @contextlib.contextmanager - def as_local_path(self, filepath: str) -> Generator[Union[str, Path], None, None]: - """Download a file from ``filepath``. - - ``as_local_path`` is decorated by :meth:`contextlib.contextmanager`. It - can be called with ``with`` statement, and when exists from the - ``with`` statement, the temporary path will be released. - - Args: - filepath (str): Download a file from ``filepath``. - - Examples: - >>> storage = OSSStorage() - >>> # After existing from the ``with`` clause, - >>> # the path will be removed - >>> with storage.get_local_path('http://path/to/file') as path: - ... # do something here - """ - try: - f = tempfile.NamedTemporaryFile(delete=False) - f.write(self.read(filepath)) - f.close() - yield f.name - finally: - os.remove(f.name) - - def write(self, obj: bytes, filepath: Union[str, Path]) -> None: - raise NotImplementedError("OSSStorage.write to be implemented in the future") - - def write_text( - self, obj: str, filepath: Union[str, Path], encoding: str = "utf-8" - ) -> None: - raise NotImplementedError( - "OSSStorage.write_text to be implemented in the future" - ) - - -G_STORAGES = {} - - -class File(object): - _prefix_to_storage: dict = { - "oss": OSSStorage, - "http": HTTPStorage, - "https": HTTPStorage, - "local": LocalStorage, - } - - @staticmethod - def _get_storage(uri): - assert isinstance(uri, str), f"uri should be str type, but got {type(uri)}" - - if "://" not in uri: - # local path - storage_type = "local" - else: - prefix, _ = uri.split("://") - storage_type = prefix - - assert storage_type in File._prefix_to_storage, ( - f"Unsupported uri {uri}, valid prefixs: " - f"{list(File._prefix_to_storage.keys())}" - ) - - if storage_type not in G_STORAGES: - G_STORAGES[storage_type] = File._prefix_to_storage[storage_type]() - - return G_STORAGES[storage_type] - - @staticmethod - def read(uri: str) -> bytes: - """Read data from a given ``filepath`` with 'rb' mode. - - Args: - filepath (str or Path): Path to read data. - - Returns: - bytes: Expected bytes object. - """ - storage = File._get_storage(uri) - return storage.read(uri) - - @staticmethod - def read_text(uri: Union[str, Path], encoding: str = "utf-8") -> str: - """Read data from a given ``filepath`` with 'r' mode. - - Args: - filepath (str or Path): Path to read data. - encoding (str): The encoding format used to open the ``filepath``. - Default: 'utf-8'. - - Returns: - str: Expected text reading from ``filepath``. - """ - storage = File._get_storage(uri) - return storage.read_text(uri) - - @staticmethod - def write(obj: bytes, uri: Union[str, Path]) -> None: - """Write data to a given ``filepath`` with 'wb' mode. - - Note: - ``write`` will create a directory if the directory of ``filepath`` - does not exist. - - Args: - obj (bytes): Data to be written. - filepath (str or Path): Path to write data. - """ - storage = File._get_storage(uri) - return storage.write(obj, uri) - - @staticmethod - def write_text(obj: str, uri: str, encoding: str = "utf-8") -> None: - """Write data to a given ``filepath`` with 'w' mode. - - Note: - ``write_text`` will create a directory if the directory of - ``filepath`` does not exist. - - Args: - obj (str): Data to be written. - filepath (str or Path): Path to write data. - encoding (str): The encoding format used to open the ``filepath``. - Default: 'utf-8'. - """ - storage = File._get_storage(uri) - return storage.write_text(obj, uri) - - @contextlib.contextmanager - def as_local_path(uri: str) -> Generator[Union[str, Path], None, None]: - """Only for unified API and do nothing.""" - storage = File._get_storage(uri) - with storage.as_local_path(uri) as local_path: - yield local_path diff --git a/funasr_detach/download/name_maps_from_hub.py b/funasr_detach/download/name_maps_from_hub.py deleted file mode 100644 index 695d247374ef6e2bdacb78adadbdeadf451f93ca..0000000000000000000000000000000000000000 --- a/funasr_detach/download/name_maps_from_hub.py +++ /dev/null @@ -1,13 +0,0 @@ -name_maps_ms = { - "paraformer-zh": "damo/speech_seaco_paraformer_large_asr_nat-zh-cn-16k-common-vocab8404-pytorch", - "paraformer-en": "damo/speech_paraformer-large-vad-punc_asr_nat-en-16k-common-vocab10020", - "paraformer-en-spk": "damo/speech_paraformer-large-vad-punc_asr_nat-en-16k-common-vocab10020", - "paraformer-zh-streaming": "damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-online", - "fsmn-vad": "damo/speech_fsmn_vad_zh-cn-16k-common-pytorch", - "ct-punc": "damo/punc_ct-transformer_cn-en-common-vocab471067-large", - "ct-punc-c": "damo/punc_ct-transformer_zh-cn-common-vocab272727-pytorch", - "fa-zh": "damo/speech_timestamp_prediction-v1-16k-offline", - "cam++": "damo/speech_campplus_sv_zh-cn_16k-common", -} - -name_maps_hf = {} diff --git a/funasr_detach/download/runtime_sdk_download_tool.py b/funasr_detach/download/runtime_sdk_download_tool.py deleted file mode 100644 index bc7e8e465aed348d25cbdd015095271fb7434cd3..0000000000000000000000000000000000000000 --- a/funasr_detach/download/runtime_sdk_download_tool.py +++ /dev/null @@ -1,60 +0,0 @@ -import os -import argparse -from pathlib import Path - -from funasr_detach.utils.types import str2bool - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument("--model-name", type=str, required=True) - parser.add_argument("--export-dir", type=str, required=True) - parser.add_argument( - "--export", type=str2bool, default=True, help="whether to export model" - ) - parser.add_argument("--type", type=str, default="onnx", help='["onnx", "torch"]') - parser.add_argument("--device", type=str, default="cpu", help='["cpu", "cuda"]') - parser.add_argument( - "--quantize", type=str2bool, default=False, help="export quantized model" - ) - parser.add_argument( - "--fallback-num", type=int, default=0, help="amp fallback number" - ) - parser.add_argument("--audio_in", type=str, default=None, help='["wav", "wav.scp"]') - parser.add_argument( - "--model_revision", type=str, default=None, help="model_revision" - ) - parser.add_argument("--calib_num", type=int, default=200, help="calib max num") - args = parser.parse_args() - - model_dir = args.model_name - if not Path(args.model_name).exists(): - from modelscope.hub.snapshot_download import snapshot_download - - try: - model_dir = snapshot_download( - args.model_name, cache_dir=args.export_dir, revision=args.model_revision - ) - except: - raise "model_dir must be model_name in modelscope or local path downloaded from modelscope, but is {}".format( - model_dir - ) - if args.export: - model_file = os.path.join(model_dir, "model.onnx") - if args.quantize: - model_file = os.path.join(model_dir, "model_quant.onnx") - if not os.path.exists(model_file): - print(".onnx is not exist, begin to export onnx") - from funasr_detach.bin.export_model import ModelExport - - export_model = ModelExport( - cache_dir=args.export_dir, - onnx=True, - device="cpu", - quant=args.quantize, - ) - export_model.export(model_dir) - - -if __name__ == "__main__": - main() diff --git a/funasr_detach/frontends/__init__.py b/funasr_detach/frontends/__init__.py deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/funasr_detach/frontends/default.py b/funasr_detach/frontends/default.py deleted file mode 100644 index 80ba539b1d06d8d3fd50bdc021b7ad7e1dd7bc82..0000000000000000000000000000000000000000 --- a/funasr_detach/frontends/default.py +++ /dev/null @@ -1,347 +0,0 @@ -import copy -from typing import Optional -from typing import Tuple -from typing import Union -import logging -import humanfriendly -import numpy as np -import torch -import torch.nn as nn - -try: - from torch_complex.tensor import ComplexTensor -except: - print("Please install torch_complex firstly") - -from funasr_detach.frontends.utils.log_mel import LogMel -from funasr_detach.frontends.utils.stft import Stft -from funasr_detach.frontends.utils.frontend import Frontend -from funasr_detach.models.transformer.utils.nets_utils import make_pad_mask - - -class DefaultFrontend(nn.Module): - """Conventional frontend structure for ASR. - Stft -> WPE -> MVDR-Beamformer -> Power-spec -> Mel-Fbank -> CMVN - """ - - def __init__( - self, - fs: Union[int, str] = 16000, - n_fft: int = 512, - win_length: int = None, - hop_length: int = 128, - window: Optional[str] = "hann", - center: bool = True, - normalized: bool = False, - onesided: bool = True, - n_mels: int = 80, - fmin: int = None, - fmax: int = None, - htk: bool = False, - frontend_conf: Optional[dict] = None, - apply_stft: bool = True, - use_channel: int = None, - ): - super().__init__() - if isinstance(fs, str): - fs = humanfriendly.parse_size(fs) - - # Deepcopy (In general, dict shouldn't be used as default arg) - frontend_conf = copy.deepcopy(frontend_conf) - self.hop_length = hop_length - - if apply_stft: - self.stft = Stft( - n_fft=n_fft, - win_length=win_length, - hop_length=hop_length, - center=center, - window=window, - normalized=normalized, - onesided=onesided, - ) - else: - self.stft = None - self.apply_stft = apply_stft - - if frontend_conf is not None: - self.frontend = Frontend(idim=n_fft // 2 + 1, **frontend_conf) - else: - self.frontend = None - - self.logmel = LogMel( - fs=fs, - n_fft=n_fft, - n_mels=n_mels, - fmin=fmin, - fmax=fmax, - htk=htk, - ) - self.n_mels = n_mels - self.use_channel = use_channel - self.frontend_type = "default" - - def output_size(self) -> int: - return self.n_mels - - def forward( - self, input: torch.Tensor, input_lengths: torch.Tensor - ) -> Tuple[torch.Tensor, torch.Tensor]: - # 1. Domain-conversion: e.g. Stft: time -> time-freq - if self.stft is not None: - input_stft, feats_lens = self._compute_stft(input, input_lengths) - else: - input_stft = ComplexTensor(input[..., 0], input[..., 1]) - feats_lens = input_lengths - # 2. [Option] Speech enhancement - if self.frontend is not None: - assert isinstance(input_stft, ComplexTensor), type(input_stft) - # input_stft: (Batch, Length, [Channel], Freq) - input_stft, _, mask = self.frontend(input_stft, feats_lens) - - # 3. [Multi channel case]: Select a channel - if input_stft.dim() == 4: - # h: (B, T, C, F) -> h: (B, T, F) - if self.training: - if self.use_channel is not None: - input_stft = input_stft[:, :, self.use_channel, :] - else: - # Select 1ch randomly - ch = np.random.randint(input_stft.size(2)) - input_stft = input_stft[:, :, ch, :] - else: - # Use the first channel - input_stft = input_stft[:, :, 0, :] - - # 4. STFT -> Power spectrum - # h: ComplexTensor(B, T, F) -> torch.Tensor(B, T, F) - input_power = input_stft.real**2 + input_stft.imag**2 - - # 5. Feature transform e.g. Stft -> Log-Mel-Fbank - # input_power: (Batch, [Channel,] Length, Freq) - # -> input_feats: (Batch, Length, Dim) - input_feats, _ = self.logmel(input_power, feats_lens) - - return input_feats, feats_lens - - def _compute_stft( - self, input: torch.Tensor, input_lengths: torch.Tensor - ) -> torch.Tensor: - input_stft, feats_lens = self.stft(input, input_lengths) - - assert input_stft.dim() >= 4, input_stft.shape - # "2" refers to the real/imag parts of Complex - assert input_stft.shape[-1] == 2, input_stft.shape - - # Change torch.Tensor to ComplexTensor - # input_stft: (..., F, 2) -> (..., F) - input_stft = ComplexTensor(input_stft[..., 0], input_stft[..., 1]) - return input_stft, feats_lens - - -class MultiChannelFrontend(nn.Module): - """Conventional frontend structure for ASR. - Stft -> WPE -> MVDR-Beamformer -> Power-spec -> Mel-Fbank -> CMVN - """ - - def __init__( - self, - fs: Union[int, str] = 16000, - n_fft: int = 512, - win_length: int = None, - hop_length: int = None, - frame_length: int = None, - frame_shift: int = None, - window: Optional[str] = "hann", - center: bool = True, - normalized: bool = False, - onesided: bool = True, - n_mels: int = 80, - fmin: int = None, - fmax: int = None, - htk: bool = False, - frontend_conf: Optional[dict] = None, - apply_stft: bool = True, - use_channel: int = None, - lfr_m: int = 1, - lfr_n: int = 1, - cmvn_file: str = None, - mc: bool = True, - ): - super().__init__() - if isinstance(fs, str): - fs = humanfriendly.parse_size(fs) - - # Deepcopy (In general, dict shouldn't be used as default arg) - frontend_conf = copy.deepcopy(frontend_conf) - if win_length is None and hop_length is None: - self.win_length = frame_length * 16 - self.hop_length = frame_shift * 16 - elif frame_length is None and frame_shift is None: - self.win_length = self.win_length - self.hop_length = self.hop_length - else: - logging.error( - "Only one of (win_length, hop_length) and (frame_length, frame_shift)" - "can be set." - ) - exit(1) - - if apply_stft: - self.stft = Stft( - n_fft=n_fft, - win_length=self.win_length, - hop_length=self.hop_length, - center=center, - window=window, - normalized=normalized, - onesided=onesided, - ) - else: - self.stft = None - self.apply_stft = apply_stft - - if frontend_conf is not None: - self.frontend = Frontend(idim=n_fft // 2 + 1, **frontend_conf) - else: - self.frontend = None - - self.logmel = LogMel( - fs=fs, - n_fft=n_fft, - n_mels=n_mels, - fmin=fmin, - fmax=fmax, - htk=htk, - ) - self.n_mels = n_mels - self.use_channel = use_channel - self.mc = mc - if not self.mc: - if self.use_channel is not None: - logging.info("use the channel %d" % (self.use_channel)) - else: - logging.info("random select channel") - self.cmvn_file = cmvn_file - if self.cmvn_file is not None: - mean, std = self._load_cmvn(self.cmvn_file) - self.register_buffer("mean", torch.from_numpy(mean)) - self.register_buffer("std", torch.from_numpy(std)) - self.frontend_type = "multichannelfrontend" - - def output_size(self) -> int: - return self.n_mels - - def forward( - self, input: torch.Tensor, input_lengths: torch.Tensor - ) -> Tuple[torch.Tensor, torch.Tensor]: - # 1. Domain-conversion: e.g. Stft: time -> time-freq - # import pdb;pdb.set_trace() - if self.stft is not None: - input_stft, feats_lens = self._compute_stft(input, input_lengths) - else: - input_stft = ComplexTensor(input[..., 0], input[..., 1]) - feats_lens = input_lengths - # 2. [Option] Speech enhancement - if self.frontend is not None: - assert isinstance(input_stft, ComplexTensor), type(input_stft) - # input_stft: (Batch, Length, [Channel], Freq) - input_stft, _, mask = self.frontend(input_stft, feats_lens) - - # 3. [Multi channel case]: Select a channel(sa_asr) - if input_stft.dim() == 4 and not self.mc: - # h: (B, T, C, F) -> h: (B, T, F) - if self.training: - if self.use_channel is not None: - input_stft = input_stft[:, :, self.use_channel, :] - - else: - # Select 1ch randomly - ch = np.random.randint(input_stft.size(2)) - input_stft = input_stft[:, :, ch, :] - else: - # Use the first channel - input_stft = input_stft[:, :, 0, :] - - # 4. STFT -> Power spectrum - # h: ComplexTensor(B, T, F) -> torch.Tensor(B, T, F) - input_power = input_stft.real**2 + input_stft.imag**2 - - # 5. Feature transform e.g. Stft -> Log-Mel-Fbank - # input_power: (Batch, [Channel,] Length, Freq) - # -> input_feats: (Batch, Length, Dim) - input_feats, _ = self.logmel(input_power, feats_lens) - if self.mc: - # MFCCA - if input_feats.dim() == 4: - bt = input_feats.size(0) - channel_size = input_feats.size(2) - input_feats = ( - input_feats.transpose(1, 2) - .reshape(bt * channel_size, -1, 80) - .contiguous() - ) - feats_lens = feats_lens.repeat(1, channel_size).squeeze() - else: - channel_size = 1 - return input_feats, feats_lens, channel_size - else: - # 6. Apply CMVN - if self.cmvn_file is not None: - if feats_lens is None: - feats_lens = input_feats.new_full( - [input_feats.size(0)], input_feats.size(1) - ) - self.mean = self.mean.to(input_feats.device, input_feats.dtype) - self.std = self.std.to(input_feats.device, input_feats.dtype) - mask = make_pad_mask(feats_lens, input_feats, 1) - - if input_feats.requires_grad: - input_feats = input_feats + self.mean - else: - input_feats += self.mean - if input_feats.requires_grad: - input_feats = input_feats.masked_fill(mask, 0.0) - else: - input_feats.masked_fill_(mask, 0.0) - - input_feats *= self.std - - return input_feats, feats_lens - - def _compute_stft( - self, input: torch.Tensor, input_lengths: torch.Tensor - ) -> torch.Tensor: - input_stft, feats_lens = self.stft(input, input_lengths) - - assert input_stft.dim() >= 4, input_stft.shape - # "2" refers to the real/imag parts of Complex - assert input_stft.shape[-1] == 2, input_stft.shape - - # Change torch.Tensor to ComplexTensor - # input_stft: (..., F, 2) -> (..., F) - input_stft = ComplexTensor(input_stft[..., 0], input_stft[..., 1]) - return input_stft, feats_lens - - def _load_cmvn(self, cmvn_file): - with open(cmvn_file, "r", encoding="utf-8") as f: - lines = f.readlines() - means_list = [] - vars_list = [] - for i in range(len(lines)): - line_item = lines[i].split() - if line_item[0] == "": - line_item = lines[i + 1].split() - if line_item[0] == "": - add_shift_line = line_item[3 : (len(line_item) - 1)] - means_list = list(add_shift_line) - continue - elif line_item[0] == "": - line_item = lines[i + 1].split() - if line_item[0] == "": - rescale_line = line_item[3 : (len(line_item) - 1)] - vars_list = list(rescale_line) - continue - means = np.array(means_list).astype(np.float) - vars = np.array(vars_list).astype(np.float) - return means, vars diff --git a/funasr_detach/frontends/eend_ola_feature.py b/funasr_detach/frontends/eend_ola_feature.py deleted file mode 100644 index 13438175867c9795cc349200dae754b9282c81b9..0000000000000000000000000000000000000000 --- a/funasr_detach/frontends/eend_ola_feature.py +++ /dev/null @@ -1,49 +0,0 @@ -# Copyright 2019 Hitachi, Ltd. (author: Yusuke Fujita) -# Licensed under the MIT license. -# -# This module is for computing audio features - -import librosa -import numpy as np - - -def transform(Y, dtype=np.float32): - Y = np.abs(Y) - n_fft = 2 * (Y.shape[1] - 1) - sr = 8000 - n_mels = 23 - mel_basis = librosa.filters.mel(sr, n_fft, n_mels) - Y = np.dot(Y**2, mel_basis.T) - Y = np.log10(np.maximum(Y, 1e-10)) - mean = np.mean(Y, axis=0) - Y = Y - mean - return Y.astype(dtype) - - -def subsample(Y, T, subsampling=1): - Y_ss = Y[::subsampling] - T_ss = T[::subsampling] - return Y_ss, T_ss - - -def splice(Y, context_size=0): - Y_pad = np.pad(Y, [(context_size, context_size), (0, 0)], "constant") - Y_spliced = np.lib.stride_tricks.as_strided( - np.ascontiguousarray(Y_pad), - (Y.shape[0], Y.shape[1] * (2 * context_size + 1)), - (Y.itemsize * Y.shape[1], Y.itemsize), - writeable=False, - ) - return Y_spliced - - -def stft(data, frame_size=1024, frame_shift=256): - fft_size = 1 << (frame_size - 1).bit_length() - if len(data) % frame_shift == 0: - return librosa.stft( - data, n_fft=fft_size, win_length=frame_size, hop_length=frame_shift - ).T[:-1] - else: - return librosa.stft( - data, n_fft=fft_size, win_length=frame_size, hop_length=frame_shift - ).T diff --git a/funasr_detach/frontends/fused.py b/funasr_detach/frontends/fused.py deleted file mode 100644 index cd4eeb10af985154d1bd192a8ad7b92ed3d4a383..0000000000000000000000000000000000000000 --- a/funasr_detach/frontends/fused.py +++ /dev/null @@ -1,144 +0,0 @@ -from funasr_detach.frontends.default import DefaultFrontend -from funasr_detach.frontends.s3prl import S3prlFrontend -import numpy as np -import torch -import torch.nn as nn -from typing import Tuple - - -class FusedFrontends(nn.Module): - def __init__( - self, frontends=None, align_method="linear_projection", proj_dim=100, fs=16000 - ): - - super().__init__() - self.align_method = ( - align_method # fusing method : linear_projection only for now - ) - self.proj_dim = proj_dim # dim of the projection done on each frontend - self.frontends = [] # list of the frontends to combine - - for i, frontend in enumerate(frontends): - frontend_type = frontend["frontend_type"] - if frontend_type == "default": - n_mels, fs, n_fft, win_length, hop_length = ( - frontend.get("n_mels", 80), - fs, - frontend.get("n_fft", 512), - frontend.get("win_length"), - frontend.get("hop_length", 128), - ) - window, center, normalized, onesided = ( - frontend.get("window", "hann"), - frontend.get("center", True), - frontend.get("normalized", False), - frontend.get("onesided", True), - ) - fmin, fmax, htk, apply_stft = ( - frontend.get("fmin", None), - frontend.get("fmax", None), - frontend.get("htk", False), - frontend.get("apply_stft", True), - ) - - self.frontends.append( - DefaultFrontend( - n_mels=n_mels, - n_fft=n_fft, - fs=fs, - win_length=win_length, - hop_length=hop_length, - window=window, - center=center, - normalized=normalized, - onesided=onesided, - fmin=fmin, - fmax=fmax, - htk=htk, - apply_stft=apply_stft, - ) - ) - elif frontend_type == "s3prl": - frontend_conf, download_dir, multilayer_feature = ( - frontend.get("frontend_conf"), - frontend.get("download_dir"), - frontend.get("multilayer_feature"), - ) - self.frontends.append( - S3prlFrontend( - fs=fs, - frontend_conf=frontend_conf, - download_dir=download_dir, - multilayer_feature=multilayer_feature, - ) - ) - - else: - raise NotImplementedError # frontends are only default or s3prl - - self.frontends = torch.nn.ModuleList(self.frontends) - - self.gcd = np.gcd.reduce([frontend.hop_length for frontend in self.frontends]) - self.factors = [frontend.hop_length // self.gcd for frontend in self.frontends] - if torch.cuda.is_available(): - dev = "cuda" - else: - dev = "cpu" - if self.align_method == "linear_projection": - self.projection_layers = [ - torch.nn.Linear( - in_features=frontend.output_size(), - out_features=self.factors[i] * self.proj_dim, - ) - for i, frontend in enumerate(self.frontends) - ] - self.projection_layers = torch.nn.ModuleList(self.projection_layers) - self.projection_layers = self.projection_layers.to(torch.device(dev)) - - def output_size(self) -> int: - return len(self.frontends) * self.proj_dim - - def forward( - self, input: torch.Tensor, input_lengths: torch.Tensor - ) -> Tuple[torch.Tensor, torch.Tensor]: - - # step 0 : get all frontends features - self.feats = [] - for frontend in self.frontends: - with torch.no_grad(): - input_feats, feats_lens = frontend.forward(input, input_lengths) - self.feats.append([input_feats, feats_lens]) - - if ( - self.align_method == "linear_projection" - ): # TODO(Dan): to add other align methods - - # first step : projections - self.feats_proj = [] - for i, frontend in enumerate(self.frontends): - input_feats = self.feats[i][0] - self.feats_proj.append(self.projection_layers[i](input_feats)) - - # 2nd step : reshape - self.feats_reshaped = [] - for i, frontend in enumerate(self.frontends): - input_feats_proj = self.feats_proj[i] - bs, nf, dim = input_feats_proj.shape - input_feats_reshaped = torch.reshape( - input_feats_proj, (bs, nf * self.factors[i], dim // self.factors[i]) - ) - self.feats_reshaped.append(input_feats_reshaped) - - # 3rd step : drop the few last frames - m = min([x.shape[1] for x in self.feats_reshaped]) - self.feats_final = [x[:, :m, :] for x in self.feats_reshaped] - - input_feats = torch.cat( - self.feats_final, dim=-1 - ) # change the input size of the preencoder : proj_dim * n_frontends - feats_lens = torch.ones_like(self.feats[0][1]) * (m) - - else: - raise NotImplementedError - - return input_feats, feats_lens diff --git a/funasr_detach/frontends/s3prl.py b/funasr_detach/frontends/s3prl.py deleted file mode 100644 index f4c759fd740245bc695bca72fa196aba8e49c007..0000000000000000000000000000000000000000 --- a/funasr_detach/frontends/s3prl.py +++ /dev/null @@ -1,139 +0,0 @@ -import copy -import logging -import os -from argparse import Namespace -from typing import Optional -from typing import Tuple -from typing import Union - -import humanfriendly -import torch -import torch.nn as nn - -from funasr_detach.frontends.utils.frontend import Frontend -from funasr_detach.models.transformer.utils.nets_utils import pad_list - - -def base_s3prl_setup(args): - args.upstream_feature_selection = getattr(args, "upstream_feature_selection", None) - args.upstream_model_config = getattr(args, "upstream_model_config", None) - args.upstream_refresh = getattr(args, "upstream_refresh", False) - args.upstream_ckpt = getattr(args, "upstream_ckpt", None) - args.init_ckpt = getattr(args, "init_ckpt", None) - args.verbose = getattr(args, "verbose", False) - args.tile_factor = getattr(args, "tile_factor", 1) - return args - - -class S3prlFrontend(nn.Module): - """Speech Pretrained Representation frontend structure for ASR.""" - - def __init__( - self, - fs: Union[int, str] = 16000, - frontend_conf: Optional[dict] = None, - download_dir: str = None, - multilayer_feature: bool = False, - ): - super().__init__() - if isinstance(fs, str): - fs = humanfriendly.parse_size(fs) - - if download_dir is not None: - torch.hub.set_dir(download_dir) - - self.multilayer_feature = multilayer_feature - self.upstream, self.featurizer = self._get_upstream(frontend_conf) - self.pretrained_params = copy.deepcopy(self.upstream.state_dict()) - self.output_dim = self.featurizer.output_dim - self.frontend_type = "s3prl" - self.hop_length = self.upstream.get_downsample_rates("key") - - def _get_upstream(self, frontend_conf): - """Get S3PRL upstream model.""" - s3prl_args = base_s3prl_setup( - Namespace(**frontend_conf, device="cpu"), - ) - self.args = s3prl_args - - s3prl_path = None - python_path_list = os.environ.get("PYTHONPATH", "(None)").split(":") - for p in python_path_list: - if p.endswith("s3prl"): - s3prl_path = p - break - assert s3prl_path is not None - - s3prl_upstream = torch.hub.load( - s3prl_path, - s3prl_args.upstream, - ckpt=s3prl_args.upstream_ckpt, - model_config=s3prl_args.upstream_model_config, - refresh=s3prl_args.upstream_refresh, - source="local", - ).to("cpu") - - if getattr( - s3prl_upstream, "model", None - ) is not None and s3prl_upstream.model.__class__.__name__ in [ - "Wav2Vec2Model", - "HubertModel", - ]: - s3prl_upstream.model.encoder.layerdrop = 0.0 - - from s3prl.upstream.interfaces import Featurizer - - if self.multilayer_feature is None: - feature_selection = "last_hidden_state" - else: - feature_selection = "hidden_states" - s3prl_featurizer = Featurizer( - upstream=s3prl_upstream, - feature_selection=feature_selection, - upstream_device="cpu", - ) - - return s3prl_upstream, s3prl_featurizer - - def _tile_representations(self, feature): - """Tile up the representations by `tile_factor`. - Input - sequence of representations - shape: (batch_size, seq_len, feature_dim) - Output - sequence of tiled representations - shape: (batch_size, seq_len * factor, feature_dim) - """ - assert ( - len(feature.shape) == 3 - ), "Input argument `feature` has invalid shape: {}".format(feature.shape) - tiled_feature = feature.repeat(1, 1, self.args.tile_factor) - tiled_feature = tiled_feature.reshape( - feature.size(0), feature.size(1) * self.args.tile_factor, feature.size(2) - ) - return tiled_feature - - def output_size(self) -> int: - return self.output_dim - - def forward( - self, input: torch.Tensor, input_lengths: torch.Tensor - ) -> Tuple[torch.Tensor, torch.Tensor]: - wavs = [wav[: input_lengths[i]] for i, wav in enumerate(input)] - self.upstream.eval() - with torch.no_grad(): - feats = self.upstream(wavs) - feats = self.featurizer(wavs, feats) - - if self.args.tile_factor != 1: - feats = self._tile_representations(feats) - - input_feats = pad_list(feats, 0.0) - feats_lens = torch.tensor([f.shape[0] for f in feats], dtype=torch.long) - - # Saving CUDA Memory - del feats - - return input_feats, feats_lens - - def reload_pretrained_parameters(self): - self.upstream.load_state_dict(self.pretrained_params) - logging.info("Pretrained S3PRL frontend model parameters reloaded!") diff --git a/funasr_detach/frontends/utils/__init__.py b/funasr_detach/frontends/utils/__init__.py deleted file mode 100644 index b7f177368e62a5578b8706300e101f831a3972ac..0000000000000000000000000000000000000000 --- a/funasr_detach/frontends/utils/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Initialize sub package.""" diff --git a/funasr_detach/frontends/utils/beamformer.py b/funasr_detach/frontends/utils/beamformer.py deleted file mode 100644 index f3eccee4cf98b164f8eb9802bde3741ac23dc9dc..0000000000000000000000000000000000000000 --- a/funasr_detach/frontends/utils/beamformer.py +++ /dev/null @@ -1,84 +0,0 @@ -import torch -from torch_complex import functional as FC -from torch_complex.tensor import ComplexTensor - - -def get_power_spectral_density_matrix( - xs: ComplexTensor, mask: torch.Tensor, normalization=True, eps: float = 1e-15 -) -> ComplexTensor: - """Return cross-channel power spectral density (PSD) matrix - - Args: - xs (ComplexTensor): (..., F, C, T) - mask (torch.Tensor): (..., F, C, T) - normalization (bool): - eps (float): - Returns - psd (ComplexTensor): (..., F, C, C) - - """ - # outer product: (..., C_1, T) x (..., C_2, T) -> (..., T, C, C_2) - psd_Y = FC.einsum("...ct,...et->...tce", [xs, xs.conj()]) - - # Averaging mask along C: (..., C, T) -> (..., T) - mask = mask.mean(dim=-2) - - # Normalized mask along T: (..., T) - if normalization: - # If assuming the tensor is padded with zero, the summation along - # the time axis is same regardless of the padding length. - mask = mask / (mask.sum(dim=-1, keepdim=True) + eps) - - # psd: (..., T, C, C) - psd = psd_Y * mask[..., None, None] - # (..., T, C, C) -> (..., C, C) - psd = psd.sum(dim=-3) - - return psd - - -def get_mvdr_vector( - psd_s: ComplexTensor, - psd_n: ComplexTensor, - reference_vector: torch.Tensor, - eps: float = 1e-15, -) -> ComplexTensor: - """Return the MVDR(Minimum Variance Distortionless Response) vector: - - h = (Npsd^-1 @ Spsd) / (Tr(Npsd^-1 @ Spsd)) @ u - - Reference: - On optimal frequency-domain multichannel linear filtering - for noise reduction; M. Souden et al., 2010; - https://ieeexplore.ieee.org/document/5089420 - - Args: - psd_s (ComplexTensor): (..., F, C, C) - psd_n (ComplexTensor): (..., F, C, C) - reference_vector (torch.Tensor): (..., C) - eps (float): - Returns: - beamform_vector (ComplexTensor)r: (..., F, C) - """ - # Add eps - C = psd_n.size(-1) - eye = torch.eye(C, dtype=psd_n.dtype, device=psd_n.device) - shape = [1 for _ in range(psd_n.dim() - 2)] + [C, C] - eye = eye.view(*shape) - psd_n += eps * eye - - # numerator: (..., C_1, C_2) x (..., C_2, C_3) -> (..., C_1, C_3) - numerator = FC.einsum("...ec,...cd->...ed", [psd_n.inverse(), psd_s]) - # ws: (..., C, C) / (...,) -> (..., C, C) - ws = numerator / (FC.trace(numerator)[..., None, None] + eps) - # h: (..., F, C_1, C_2) x (..., C_2) -> (..., F, C_1) - beamform_vector = FC.einsum("...fec,...c->...fe", [ws, reference_vector]) - return beamform_vector - - -def apply_beamforming_vector( - beamform_vector: ComplexTensor, mix: ComplexTensor -) -> ComplexTensor: - # (..., C) x (..., C, T) -> (..., T) - es = FC.einsum("...c,...ct->...t", [beamform_vector.conj(), mix]) - return es diff --git a/funasr_detach/frontends/utils/complex_utils.py b/funasr_detach/frontends/utils/complex_utils.py deleted file mode 100644 index 3fcd494c7d7611b1e3ef837c78b0001e3d6c5554..0000000000000000000000000000000000000000 --- a/funasr_detach/frontends/utils/complex_utils.py +++ /dev/null @@ -1,194 +0,0 @@ -"""Beamformer module.""" - -from distutils.version import LooseVersion -from typing import Sequence -from typing import Tuple -from typing import Union - -import torch - -try: - from torch_complex import functional as FC - from torch_complex.tensor import ComplexTensor -except: - print("Please install torch_complex firstly") - - -EPS = torch.finfo(torch.double).eps -is_torch_1_8_plus = LooseVersion(torch.__version__) >= LooseVersion("1.8.0") -is_torch_1_9_plus = LooseVersion(torch.__version__) >= LooseVersion("1.9.0") - - -def new_complex_like( - ref: Union[torch.Tensor, ComplexTensor], - real_imag: Tuple[torch.Tensor, torch.Tensor], -): - if isinstance(ref, ComplexTensor): - return ComplexTensor(*real_imag) - elif is_torch_complex_tensor(ref): - return torch.complex(*real_imag) - else: - raise ValueError( - "Please update your PyTorch version to 1.9+ for complex support." - ) - - -def is_torch_complex_tensor(c): - return ( - not isinstance(c, ComplexTensor) and is_torch_1_9_plus and torch.is_complex(c) - ) - - -def is_complex(c): - return isinstance(c, ComplexTensor) or is_torch_complex_tensor(c) - - -def to_double(c): - if not isinstance(c, ComplexTensor) and is_torch_1_9_plus and torch.is_complex(c): - return c.to(dtype=torch.complex128) - else: - return c.double() - - -def to_float(c): - if not isinstance(c, ComplexTensor) and is_torch_1_9_plus and torch.is_complex(c): - return c.to(dtype=torch.complex64) - else: - return c.float() - - -def cat(seq: Sequence[Union[ComplexTensor, torch.Tensor]], *args, **kwargs): - if not isinstance(seq, (list, tuple)): - raise TypeError( - "cat(): argument 'tensors' (position 1) must be tuple of Tensors, " - "not Tensor" - ) - if isinstance(seq[0], ComplexTensor): - return FC.cat(seq, *args, **kwargs) - else: - return torch.cat(seq, *args, **kwargs) - - -def complex_norm( - c: Union[torch.Tensor, ComplexTensor], dim=-1, keepdim=False -) -> torch.Tensor: - if not is_complex(c): - raise TypeError("Input is not a complex tensor.") - if is_torch_complex_tensor(c): - return torch.norm(c, dim=dim, keepdim=keepdim) - else: - return torch.sqrt((c.real**2 + c.imag**2).sum(dim=dim, keepdim=keepdim) + EPS) - - -def einsum(equation, *operands): - # NOTE: Do not mix ComplexTensor and torch.complex in the input! - # NOTE (wangyou): Until PyTorch 1.9.0, torch.einsum does not support - # mixed input with complex and real tensors. - if len(operands) == 1: - if isinstance(operands[0], (tuple, list)): - operands = operands[0] - complex_module = FC if isinstance(operands[0], ComplexTensor) else torch - return complex_module.einsum(equation, *operands) - elif len(operands) != 2: - op0 = operands[0] - same_type = all(op.dtype == op0.dtype for op in operands[1:]) - if same_type: - _einsum = FC.einsum if isinstance(op0, ComplexTensor) else torch.einsum - return _einsum(equation, *operands) - else: - raise ValueError("0 or More than 2 operands are not supported.") - a, b = operands - if isinstance(a, ComplexTensor) or isinstance(b, ComplexTensor): - return FC.einsum(equation, a, b) - elif is_torch_1_9_plus and (torch.is_complex(a) or torch.is_complex(b)): - if not torch.is_complex(a): - o_real = torch.einsum(equation, a, b.real) - o_imag = torch.einsum(equation, a, b.imag) - return torch.complex(o_real, o_imag) - elif not torch.is_complex(b): - o_real = torch.einsum(equation, a.real, b) - o_imag = torch.einsum(equation, a.imag, b) - return torch.complex(o_real, o_imag) - else: - return torch.einsum(equation, a, b) - else: - return torch.einsum(equation, a, b) - - -def inverse( - c: Union[torch.Tensor, ComplexTensor], -) -> Union[torch.Tensor, ComplexTensor]: - if isinstance(c, ComplexTensor): - return c.inverse2() - else: - return c.inverse() - - -def matmul( - a: Union[torch.Tensor, ComplexTensor], b: Union[torch.Tensor, ComplexTensor] -) -> Union[torch.Tensor, ComplexTensor]: - # NOTE: Do not mix ComplexTensor and torch.complex in the input! - # NOTE (wangyou): Until PyTorch 1.9.0, torch.matmul does not support - # multiplication between complex and real tensors. - if isinstance(a, ComplexTensor) or isinstance(b, ComplexTensor): - return FC.matmul(a, b) - elif is_torch_1_9_plus and (torch.is_complex(a) or torch.is_complex(b)): - if not torch.is_complex(a): - o_real = torch.matmul(a, b.real) - o_imag = torch.matmul(a, b.imag) - return torch.complex(o_real, o_imag) - elif not torch.is_complex(b): - o_real = torch.matmul(a.real, b) - o_imag = torch.matmul(a.imag, b) - return torch.complex(o_real, o_imag) - else: - return torch.matmul(a, b) - else: - return torch.matmul(a, b) - - -def trace(a: Union[torch.Tensor, ComplexTensor]): - # NOTE (wangyou): until PyTorch 1.9.0, torch.trace does not - # support bacth processing. Use FC.trace() as fallback. - return FC.trace(a) - - -def reverse(a: Union[torch.Tensor, ComplexTensor], dim=0): - if isinstance(a, ComplexTensor): - return FC.reverse(a, dim=dim) - else: - return torch.flip(a, dims=(dim,)) - - -def solve(b: Union[torch.Tensor, ComplexTensor], a: Union[torch.Tensor, ComplexTensor]): - """Solve the linear equation ax = b.""" - # NOTE: Do not mix ComplexTensor and torch.complex in the input! - # NOTE (wangyou): Until PyTorch 1.9.0, torch.solve does not support - # mixed input with complex and real tensors. - if isinstance(a, ComplexTensor) or isinstance(b, ComplexTensor): - if isinstance(a, ComplexTensor) and isinstance(b, ComplexTensor): - return FC.solve(b, a, return_LU=False) - else: - return matmul(inverse(a), b) - elif is_torch_1_9_plus and (torch.is_complex(a) or torch.is_complex(b)): - if torch.is_complex(a) and torch.is_complex(b): - return torch.linalg.solve(a, b) - else: - return matmul(inverse(a), b) - else: - if is_torch_1_8_plus: - return torch.linalg.solve(a, b) - else: - return torch.solve(b, a)[0] - - -def stack(seq: Sequence[Union[ComplexTensor, torch.Tensor]], *args, **kwargs): - if not isinstance(seq, (list, tuple)): - raise TypeError( - "stack(): argument 'tensors' (position 1) must be tuple of Tensors, " - "not Tensor" - ) - if isinstance(seq[0], ComplexTensor): - return FC.stack(seq, *args, **kwargs) - else: - return torch.stack(seq, *args, **kwargs) diff --git a/funasr_detach/frontends/utils/dnn_beamformer.py b/funasr_detach/frontends/utils/dnn_beamformer.py deleted file mode 100644 index 135926971e1bc8e9d7d3c4950cc847d37738583f..0000000000000000000000000000000000000000 --- a/funasr_detach/frontends/utils/dnn_beamformer.py +++ /dev/null @@ -1,173 +0,0 @@ -"""DNN beamformer module.""" - -from typing import Tuple - -import torch -from torch.nn import functional as F - -from funasr_detach.frontends.utils.beamformer import apply_beamforming_vector -from funasr_detach.frontends.utils.beamformer import get_mvdr_vector -from funasr_detach.frontends.utils.beamformer import ( - get_power_spectral_density_matrix, # noqa: H301 -) -from funasr_detach.frontends.utils.mask_estimator import MaskEstimator -from torch_complex.tensor import ComplexTensor - - -class DNN_Beamformer(torch.nn.Module): - """DNN mask based Beamformer - - Citation: - Multichannel End-to-end Speech Recognition; T. Ochiai et al., 2017; - https://arxiv.org/abs/1703.04783 - - """ - - def __init__( - self, - bidim, - btype="blstmp", - blayers=3, - bunits=300, - bprojs=320, - bnmask=2, - dropout_rate=0.0, - badim=320, - ref_channel: int = -1, - beamformer_type="mvdr", - ): - super().__init__() - self.mask = MaskEstimator( - btype, bidim, blayers, bunits, bprojs, dropout_rate, nmask=bnmask - ) - self.ref = AttentionReference(bidim, badim) - self.ref_channel = ref_channel - - self.nmask = bnmask - - if beamformer_type != "mvdr": - raise ValueError( - "Not supporting beamformer_type={}".format(beamformer_type) - ) - self.beamformer_type = beamformer_type - - def forward( - self, data: ComplexTensor, ilens: torch.LongTensor - ) -> Tuple[ComplexTensor, torch.LongTensor, ComplexTensor]: - """The forward function - - Notation: - B: Batch - C: Channel - T: Time or Sequence length - F: Freq - - Args: - data (ComplexTensor): (B, T, C, F) - ilens (torch.Tensor): (B,) - Returns: - enhanced (ComplexTensor): (B, T, F) - ilens (torch.Tensor): (B,) - - """ - - def apply_beamforming(data, ilens, psd_speech, psd_noise): - # u: (B, C) - if self.ref_channel < 0: - u, _ = self.ref(psd_speech, ilens) - else: - # (optional) Create onehot vector for fixed reference microphone - u = torch.zeros( - *(data.size()[:-3] + (data.size(-2),)), device=data.device - ) - u[..., self.ref_channel].fill_(1) - - ws = get_mvdr_vector(psd_speech, psd_noise, u) - enhanced = apply_beamforming_vector(ws, data) - - return enhanced, ws - - # data (B, T, C, F) -> (B, F, C, T) - data = data.permute(0, 3, 2, 1) - - # mask: (B, F, C, T) - masks, _ = self.mask(data, ilens) - assert self.nmask == len(masks) - - if self.nmask == 2: # (mask_speech, mask_noise) - mask_speech, mask_noise = masks - - psd_speech = get_power_spectral_density_matrix(data, mask_speech) - psd_noise = get_power_spectral_density_matrix(data, mask_noise) - - enhanced, ws = apply_beamforming(data, ilens, psd_speech, psd_noise) - - # (..., F, T) -> (..., T, F) - enhanced = enhanced.transpose(-1, -2) - mask_speech = mask_speech.transpose(-1, -3) - else: # multi-speaker case: (mask_speech1, ..., mask_noise) - mask_speech = list(masks[:-1]) - mask_noise = masks[-1] - - psd_speeches = [ - get_power_spectral_density_matrix(data, mask) for mask in mask_speech - ] - psd_noise = get_power_spectral_density_matrix(data, mask_noise) - - enhanced = [] - ws = [] - for i in range(self.nmask - 1): - psd_speech = psd_speeches.pop(i) - # treat all other speakers' psd_speech as noises - enh, w = apply_beamforming( - data, ilens, psd_speech, sum(psd_speeches) + psd_noise - ) - psd_speeches.insert(i, psd_speech) - - # (..., F, T) -> (..., T, F) - enh = enh.transpose(-1, -2) - mask_speech[i] = mask_speech[i].transpose(-1, -3) - - enhanced.append(enh) - ws.append(w) - - return enhanced, ilens, mask_speech - - -class AttentionReference(torch.nn.Module): - def __init__(self, bidim, att_dim): - super().__init__() - self.mlp_psd = torch.nn.Linear(bidim, att_dim) - self.gvec = torch.nn.Linear(att_dim, 1) - - def forward( - self, psd_in: ComplexTensor, ilens: torch.LongTensor, scaling: float = 2.0 - ) -> Tuple[torch.Tensor, torch.LongTensor]: - """The forward function - - Args: - psd_in (ComplexTensor): (B, F, C, C) - ilens (torch.Tensor): (B,) - scaling (float): - Returns: - u (torch.Tensor): (B, C) - ilens (torch.Tensor): (B,) - """ - B, _, C = psd_in.size()[:3] - assert psd_in.size(2) == psd_in.size(3), psd_in.size() - # psd_in: (B, F, C, C) - psd = psd_in.masked_fill( - torch.eye(C, dtype=torch.bool, device=psd_in.device), 0 - ) - # psd: (B, F, C, C) -> (B, C, F) - psd = (psd.sum(dim=-1) / (C - 1)).transpose(-1, -2) - - # Calculate amplitude - psd_feat = (psd.real**2 + psd.imag**2) ** 0.5 - - # (B, C, F) -> (B, C, F2) - mlp_psd = self.mlp_psd(psd_feat) - # (B, C, F2) -> (B, C, 1) -> (B, C) - e = self.gvec(torch.tanh(mlp_psd)).squeeze(-1) - u = F.softmax(scaling * e, dim=-1) - return u, ilens diff --git a/funasr_detach/frontends/utils/dnn_wpe.py b/funasr_detach/frontends/utils/dnn_wpe.py deleted file mode 100644 index 33d1ea93624af97b3255b4dffecf3e9ece969c41..0000000000000000000000000000000000000000 --- a/funasr_detach/frontends/utils/dnn_wpe.py +++ /dev/null @@ -1,93 +0,0 @@ -from typing import Tuple - -from pytorch_wpe import wpe_one_iteration -import torch -from torch_complex.tensor import ComplexTensor - -from funasr_detach.frontends.utils.mask_estimator import MaskEstimator -from funasr_detach.models.transformer.utils.nets_utils import make_pad_mask - - -class DNN_WPE(torch.nn.Module): - def __init__( - self, - wtype: str = "blstmp", - widim: int = 257, - wlayers: int = 3, - wunits: int = 300, - wprojs: int = 320, - dropout_rate: float = 0.0, - taps: int = 5, - delay: int = 3, - use_dnn_mask: bool = True, - iterations: int = 1, - normalization: bool = False, - ): - super().__init__() - self.iterations = iterations - self.taps = taps - self.delay = delay - - self.normalization = normalization - self.use_dnn_mask = use_dnn_mask - - self.inverse_power = True - - if self.use_dnn_mask: - self.mask_est = MaskEstimator( - wtype, widim, wlayers, wunits, wprojs, dropout_rate, nmask=1 - ) - - def forward( - self, data: ComplexTensor, ilens: torch.LongTensor - ) -> Tuple[ComplexTensor, torch.LongTensor, ComplexTensor]: - """The forward function - - Notation: - B: Batch - C: Channel - T: Time or Sequence length - F: Freq or Some dimension of the feature vector - - Args: - data: (B, C, T, F) - ilens: (B,) - Returns: - data: (B, C, T, F) - ilens: (B,) - """ - # (B, T, C, F) -> (B, F, C, T) - enhanced = data = data.permute(0, 3, 2, 1) - mask = None - - for i in range(self.iterations): - # Calculate power: (..., C, T) - power = enhanced.real**2 + enhanced.imag**2 - if i == 0 and self.use_dnn_mask: - # mask: (B, F, C, T) - (mask,), _ = self.mask_est(enhanced, ilens) - if self.normalization: - # Normalize along T - mask = mask / mask.sum(dim=-1)[..., None] - # (..., C, T) * (..., C, T) -> (..., C, T) - power = power * mask - - # Averaging along the channel axis: (..., C, T) -> (..., T) - power = power.mean(dim=-2) - - # enhanced: (..., C, T) -> (..., C, T) - enhanced = wpe_one_iteration( - data.contiguous(), - power, - taps=self.taps, - delay=self.delay, - inverse_power=self.inverse_power, - ) - - enhanced.masked_fill_(make_pad_mask(ilens, enhanced.real), 0) - - # (B, F, C, T) -> (B, T, C, F) - enhanced = enhanced.permute(0, 3, 2, 1) - if mask is not None: - mask = mask.transpose(-1, -3) - return enhanced, ilens, mask diff --git a/funasr_detach/frontends/utils/feature_transform.py b/funasr_detach/frontends/utils/feature_transform.py deleted file mode 100644 index f4e04042ab646b2e66a29bfb3827d67c43e9ae5a..0000000000000000000000000000000000000000 --- a/funasr_detach/frontends/utils/feature_transform.py +++ /dev/null @@ -1,263 +0,0 @@ -from typing import List -from typing import Tuple -from typing import Union - -import librosa -import numpy as np -import torch -from torch_complex.tensor import ComplexTensor - -from funasr_detach.models.transformer.utils.nets_utils import make_pad_mask - - -class FeatureTransform(torch.nn.Module): - def __init__( - self, - # Mel options, - fs: int = 16000, - n_fft: int = 512, - n_mels: int = 80, - fmin: float = 0.0, - fmax: float = None, - # Normalization - stats_file: str = None, - apply_uttmvn: bool = True, - uttmvn_norm_means: bool = True, - uttmvn_norm_vars: bool = False, - ): - super().__init__() - self.apply_uttmvn = apply_uttmvn - - self.logmel = LogMel(fs=fs, n_fft=n_fft, n_mels=n_mels, fmin=fmin, fmax=fmax) - self.stats_file = stats_file - if stats_file is not None: - self.global_mvn = GlobalMVN(stats_file) - else: - self.global_mvn = None - - if self.apply_uttmvn is not None: - self.uttmvn = UtteranceMVN( - norm_means=uttmvn_norm_means, norm_vars=uttmvn_norm_vars - ) - else: - self.uttmvn = None - - def forward( - self, x: ComplexTensor, ilens: Union[torch.LongTensor, np.ndarray, List[int]] - ) -> Tuple[torch.Tensor, torch.LongTensor]: - # (B, T, F) or (B, T, C, F) - if x.dim() not in (3, 4): - raise ValueError(f"Input dim must be 3 or 4: {x.dim()}") - if not torch.is_tensor(ilens): - ilens = torch.from_numpy(np.asarray(ilens)).to(x.device) - - if x.dim() == 4: - # h: (B, T, C, F) -> h: (B, T, F) - if self.training: - # Select 1ch randomly - ch = np.random.randint(x.size(2)) - h = x[:, :, ch, :] - else: - # Use the first channel - h = x[:, :, 0, :] - else: - h = x - - # h: ComplexTensor(B, T, F) -> torch.Tensor(B, T, F) - h = h.real**2 + h.imag**2 - - h, _ = self.logmel(h, ilens) - if self.stats_file is not None: - h, _ = self.global_mvn(h, ilens) - if self.apply_uttmvn: - h, _ = self.uttmvn(h, ilens) - - return h, ilens - - -class LogMel(torch.nn.Module): - """Convert STFT to fbank feats - - The arguments is same as librosa.filters.mel - - Args: - fs: number > 0 [scalar] sampling rate of the incoming signal - n_fft: int > 0 [scalar] number of FFT components - n_mels: int > 0 [scalar] number of Mel bands to generate - fmin: float >= 0 [scalar] lowest frequency (in Hz) - fmax: float >= 0 [scalar] highest frequency (in Hz). - If `None`, use `fmax = fs / 2.0` - htk: use HTK formula instead of Slaney - norm: {None, 1, np.inf} [scalar] - if 1, divide the triangular mel weights by the width of the mel band - (area normalization). Otherwise, leave all the triangles aiming for - a peak value of 1.0 - - """ - - def __init__( - self, - fs: int = 16000, - n_fft: int = 512, - n_mels: int = 80, - fmin: float = 0.0, - fmax: float = None, - htk: bool = False, - norm=1, - ): - super().__init__() - - _mel_options = dict( - sr=fs, n_fft=n_fft, n_mels=n_mels, fmin=fmin, fmax=fmax, htk=htk, norm=norm - ) - self.mel_options = _mel_options - - # Note(kamo): The mel matrix of librosa is different from kaldi. - melmat = librosa.filters.mel(**_mel_options) - # melmat: (D2, D1) -> (D1, D2) - self.register_buffer("melmat", torch.from_numpy(melmat.T).float()) - - def extra_repr(self): - return ", ".join(f"{k}={v}" for k, v in self.mel_options.items()) - - def forward( - self, feat: torch.Tensor, ilens: torch.LongTensor - ) -> Tuple[torch.Tensor, torch.LongTensor]: - # feat: (B, T, D1) x melmat: (D1, D2) -> mel_feat: (B, T, D2) - mel_feat = torch.matmul(feat, self.melmat) - - logmel_feat = (mel_feat + 1e-20).log() - # Zero padding - logmel_feat = logmel_feat.masked_fill(make_pad_mask(ilens, logmel_feat, 1), 0.0) - return logmel_feat, ilens - - -class GlobalMVN(torch.nn.Module): - """Apply global mean and variance normalization - - Args: - stats_file(str): npy file of 1-dim array or text file. - From the _first element to - the {(len(array) - 1) / 2}th element are treated as - the sum of features, - and the rest excluding the last elements are - treated as the sum of the square value of features, - and the last elements eqauls to the number of samples. - std_floor(float): - """ - - def __init__( - self, - stats_file: str, - norm_means: bool = True, - norm_vars: bool = True, - eps: float = 1.0e-20, - ): - super().__init__() - self.norm_means = norm_means - self.norm_vars = norm_vars - - self.stats_file = stats_file - stats = np.load(stats_file) - - stats = stats.astype(float) - assert (len(stats) - 1) % 2 == 0, stats.shape - - count = stats.flatten()[-1] - mean = stats[: (len(stats) - 1) // 2] / count - var = stats[(len(stats) - 1) // 2 : -1] / count - mean * mean - std = np.maximum(np.sqrt(var), eps) - - self.register_buffer("bias", torch.from_numpy(-mean.astype(np.float32))) - self.register_buffer("scale", torch.from_numpy(1 / std.astype(np.float32))) - - def extra_repr(self): - return ( - f"stats_file={self.stats_file}, " - f"norm_means={self.norm_means}, norm_vars={self.norm_vars}" - ) - - def forward( - self, x: torch.Tensor, ilens: torch.LongTensor - ) -> Tuple[torch.Tensor, torch.LongTensor]: - # feat: (B, T, D) - if self.norm_means: - x += self.bias.type_as(x) - x.masked_fill(make_pad_mask(ilens, x, 1), 0.0) - - if self.norm_vars: - x *= self.scale.type_as(x) - return x, ilens - - -class UtteranceMVN(torch.nn.Module): - def __init__( - self, norm_means: bool = True, norm_vars: bool = False, eps: float = 1.0e-20 - ): - super().__init__() - self.norm_means = norm_means - self.norm_vars = norm_vars - self.eps = eps - - def extra_repr(self): - return f"norm_means={self.norm_means}, norm_vars={self.norm_vars}" - - def forward( - self, x: torch.Tensor, ilens: torch.LongTensor - ) -> Tuple[torch.Tensor, torch.LongTensor]: - return utterance_mvn( - x, ilens, norm_means=self.norm_means, norm_vars=self.norm_vars, eps=self.eps - ) - - -def utterance_mvn( - x: torch.Tensor, - ilens: torch.LongTensor, - norm_means: bool = True, - norm_vars: bool = False, - eps: float = 1.0e-20, -) -> Tuple[torch.Tensor, torch.LongTensor]: - """Apply utterance mean and variance normalization - - Args: - x: (B, T, D), assumed zero padded - ilens: (B, T, D) - norm_means: - norm_vars: - eps: - - """ - ilens_ = ilens.type_as(x) - # mean: (B, D) - mean = x.sum(dim=1) / ilens_[:, None] - - if norm_means: - x -= mean[:, None, :] - x_ = x - else: - x_ = x - mean[:, None, :] - - # Zero padding - x_.masked_fill(make_pad_mask(ilens, x_, 1), 0.0) - if norm_vars: - var = x_.pow(2).sum(dim=1) / ilens_[:, None] - var = torch.clamp(var, min=eps) - x /= var.sqrt()[:, None, :] - x_ = x - return x_, ilens - - -def feature_transform_for(args, n_fft): - return FeatureTransform( - # Mel options, - fs=args.fbank_fs, - n_fft=n_fft, - n_mels=args.n_mels, - fmin=args.fbank_fmin, - fmax=args.fbank_fmax, - # Normalization - stats_file=args.stats_file, - apply_uttmvn=args.apply_uttmvn, - uttmvn_norm_means=args.uttmvn_norm_means, - uttmvn_norm_vars=args.uttmvn_norm_vars, - ) diff --git a/funasr_detach/frontends/utils/frontend.py b/funasr_detach/frontends/utils/frontend.py deleted file mode 100644 index bd6d730343892a5b98bee32e0b378bada145aa7a..0000000000000000000000000000000000000000 --- a/funasr_detach/frontends/utils/frontend.py +++ /dev/null @@ -1,151 +0,0 @@ -from typing import List -from typing import Optional -from typing import Tuple -from typing import Union - -import numpy -import torch -import torch.nn as nn -from torch_complex.tensor import ComplexTensor - -from funasr_detach.frontends.utils.dnn_beamformer import DNN_Beamformer -from funasr_detach.frontends.utils.dnn_wpe import DNN_WPE - - -class Frontend(nn.Module): - def __init__( - self, - idim: int, - # WPE options - use_wpe: bool = False, - wtype: str = "blstmp", - wlayers: int = 3, - wunits: int = 300, - wprojs: int = 320, - wdropout_rate: float = 0.0, - taps: int = 5, - delay: int = 3, - use_dnn_mask_for_wpe: bool = True, - # Beamformer options - use_beamformer: bool = False, - btype: str = "blstmp", - blayers: int = 3, - bunits: int = 300, - bprojs: int = 320, - bnmask: int = 2, - badim: int = 320, - ref_channel: int = -1, - bdropout_rate=0.0, - ): - super().__init__() - - self.use_beamformer = use_beamformer - self.use_wpe = use_wpe - self.use_dnn_mask_for_wpe = use_dnn_mask_for_wpe - # use frontend for all the data, - # e.g. in the case of multi-speaker speech separation - self.use_frontend_for_all = bnmask > 2 - - if self.use_wpe: - if self.use_dnn_mask_for_wpe: - # Use DNN for power estimation - # (Not observed significant gains) - iterations = 1 - else: - # Performing as conventional WPE, without DNN Estimator - iterations = 2 - - self.wpe = DNN_WPE( - wtype=wtype, - widim=idim, - wunits=wunits, - wprojs=wprojs, - wlayers=wlayers, - taps=taps, - delay=delay, - dropout_rate=wdropout_rate, - iterations=iterations, - use_dnn_mask=use_dnn_mask_for_wpe, - ) - else: - self.wpe = None - - if self.use_beamformer: - self.beamformer = DNN_Beamformer( - btype=btype, - bidim=idim, - bunits=bunits, - bprojs=bprojs, - blayers=blayers, - bnmask=bnmask, - dropout_rate=bdropout_rate, - badim=badim, - ref_channel=ref_channel, - ) - else: - self.beamformer = None - - def forward( - self, x: ComplexTensor, ilens: Union[torch.LongTensor, numpy.ndarray, List[int]] - ) -> Tuple[ComplexTensor, torch.LongTensor, Optional[ComplexTensor]]: - assert len(x) == len(ilens), (len(x), len(ilens)) - # (B, T, F) or (B, T, C, F) - if x.dim() not in (3, 4): - raise ValueError(f"Input dim must be 3 or 4: {x.dim()}") - if not torch.is_tensor(ilens): - ilens = torch.from_numpy(numpy.asarray(ilens)).to(x.device) - - mask = None - h = x - if h.dim() == 4: - if self.training: - choices = [(False, False)] if not self.use_frontend_for_all else [] - if self.use_wpe: - choices.append((True, False)) - - if self.use_beamformer: - choices.append((False, True)) - - use_wpe, use_beamformer = choices[numpy.random.randint(len(choices))] - - else: - use_wpe = self.use_wpe - use_beamformer = self.use_beamformer - - # 1. WPE - if use_wpe: - # h: (B, T, C, F) -> h: (B, T, C, F) - h, ilens, mask = self.wpe(h, ilens) - - # 2. Beamformer - if use_beamformer: - # h: (B, T, C, F) -> h: (B, T, F) - h, ilens, mask = self.beamformer(h, ilens) - - return h, ilens, mask - - -def frontend_for(args, idim): - return Frontend( - idim=idim, - # WPE options - use_wpe=args.use_wpe, - wtype=args.wtype, - wlayers=args.wlayers, - wunits=args.wunits, - wprojs=args.wprojs, - wdropout_rate=args.wdropout_rate, - taps=args.wpe_taps, - delay=args.wpe_delay, - use_dnn_mask_for_wpe=args.use_dnn_mask_for_wpe, - # Beamformer options - use_beamformer=args.use_beamformer, - btype=args.btype, - blayers=args.blayers, - bunits=args.bunits, - bprojs=args.bprojs, - bnmask=args.bnmask, - badim=args.badim, - ref_channel=args.ref_channel, - bdropout_rate=args.bdropout_rate, - ) diff --git a/funasr_detach/frontends/utils/log_mel.py b/funasr_detach/frontends/utils/log_mel.py deleted file mode 100644 index cbec82be1a6bb7627447dd2ae076f22f87c16f80..0000000000000000000000000000000000000000 --- a/funasr_detach/frontends/utils/log_mel.py +++ /dev/null @@ -1,83 +0,0 @@ -import librosa -import torch -from typing import Tuple - -from funasr_detach.models.transformer.utils.nets_utils import make_pad_mask - - -class LogMel(torch.nn.Module): - """Convert STFT to fbank feats - - The arguments is same as librosa.filters.mel - - Args: - fs: number > 0 [scalar] sampling rate of the incoming signal - n_fft: int > 0 [scalar] number of FFT components - n_mels: int > 0 [scalar] number of Mel bands to generate - fmin: float >= 0 [scalar] lowest frequency (in Hz) - fmax: float >= 0 [scalar] highest frequency (in Hz). - If `None`, use `fmax = fs / 2.0` - htk: use HTK formula instead of Slaney - """ - - def __init__( - self, - fs: int = 16000, - n_fft: int = 512, - n_mels: int = 80, - fmin: float = None, - fmax: float = None, - htk: bool = False, - log_base: float = None, - ): - super().__init__() - - fmin = 0 if fmin is None else fmin - fmax = fs / 2 if fmax is None else fmax - _mel_options = dict( - sr=fs, - n_fft=n_fft, - n_mels=n_mels, - fmin=fmin, - fmax=fmax, - htk=htk, - ) - self.mel_options = _mel_options - self.log_base = log_base - - # Note(kamo): The mel matrix of librosa is different from kaldi. - melmat = librosa.filters.mel(**_mel_options) - # melmat: (D2, D1) -> (D1, D2) - self.register_buffer("melmat", torch.from_numpy(melmat.T).float()) - - def extra_repr(self): - return ", ".join(f"{k}={v}" for k, v in self.mel_options.items()) - - def forward( - self, - feat: torch.Tensor, - ilens: torch.Tensor = None, - ) -> Tuple[torch.Tensor, torch.Tensor]: - # feat: (B, T, D1) x melmat: (D1, D2) -> mel_feat: (B, T, D2) - mel_feat = torch.matmul(feat, self.melmat) - mel_feat = torch.clamp(mel_feat, min=1e-10) - - if self.log_base is None: - logmel_feat = mel_feat.log() - elif self.log_base == 2.0: - logmel_feat = mel_feat.log2() - elif self.log_base == 10.0: - logmel_feat = mel_feat.log10() - else: - logmel_feat = mel_feat.log() / torch.log(self.log_base) - - # Zero padding - if ilens is not None: - logmel_feat = logmel_feat.masked_fill( - make_pad_mask(ilens, logmel_feat, 1), 0.0 - ) - else: - ilens = feat.new_full( - [feat.size(0)], fill_value=feat.size(1), dtype=torch.long - ) - return logmel_feat, ilens diff --git a/funasr_detach/frontends/utils/mask_estimator.py b/funasr_detach/frontends/utils/mask_estimator.py deleted file mode 100644 index 3e4d8d396ded057ec3af6b93837938276b2a8a31..0000000000000000000000000000000000000000 --- a/funasr_detach/frontends/utils/mask_estimator.py +++ /dev/null @@ -1,77 +0,0 @@ -from typing import Tuple - -import numpy as np -import torch -from torch.nn import functional as F -from torch_complex.tensor import ComplexTensor - -from funasr_detach.models.transformer.utils.nets_utils import make_pad_mask -from funasr_detach.models.language_model.rnn.encoders import RNN -from funasr_detach.models.language_model.rnn.encoders import RNNP - - -class MaskEstimator(torch.nn.Module): - def __init__(self, type, idim, layers, units, projs, dropout, nmask=1): - super().__init__() - subsample = np.ones(layers + 1, dtype=np.int32) - - typ = type.lstrip("vgg").rstrip("p") - if type[-1] == "p": - self.brnn = RNNP(idim, layers, units, projs, subsample, dropout, typ=typ) - else: - self.brnn = RNN(idim, layers, units, projs, dropout, typ=typ) - - self.type = type - self.nmask = nmask - self.linears = torch.nn.ModuleList( - [torch.nn.Linear(projs, idim) for _ in range(nmask)] - ) - - def forward( - self, xs: ComplexTensor, ilens: torch.LongTensor - ) -> Tuple[Tuple[torch.Tensor, ...], torch.LongTensor]: - """The forward function - - Args: - xs: (B, F, C, T) - ilens: (B,) - Returns: - hs (torch.Tensor): The hidden vector (B, F, C, T) - masks: A tuple of the masks. (B, F, C, T) - ilens: (B,) - """ - assert xs.size(0) == ilens.size(0), (xs.size(0), ilens.size(0)) - _, _, C, input_length = xs.size() - # (B, F, C, T) -> (B, C, T, F) - xs = xs.permute(0, 2, 3, 1) - - # Calculate amplitude: (B, C, T, F) -> (B, C, T, F) - xs = (xs.real**2 + xs.imag**2) ** 0.5 - # xs: (B, C, T, F) -> xs: (B * C, T, F) - xs = xs.contiguous().view(-1, xs.size(-2), xs.size(-1)) - # ilens: (B,) -> ilens_: (B * C) - ilens_ = ilens[:, None].expand(-1, C).contiguous().view(-1) - - # xs: (B * C, T, F) -> xs: (B * C, T, D) - xs, _, _ = self.brnn(xs, ilens_) - # xs: (B * C, T, D) -> xs: (B, C, T, D) - xs = xs.view(-1, C, xs.size(-2), xs.size(-1)) - - masks = [] - for linear in self.linears: - # xs: (B, C, T, D) -> mask:(B, C, T, F) - mask = linear(xs) - - mask = torch.sigmoid(mask) - # Zero padding - mask.masked_fill(make_pad_mask(ilens, mask, length_dim=2), 0) - - # (B, C, T, F) -> (B, F, C, T) - mask = mask.permute(0, 3, 1, 2) - - # Take cares of multi gpu cases: If input_length > max(ilens) - if mask.size(-1) < input_length: - mask = F.pad(mask, [0, input_length - mask.size(-1)], value=0) - masks.append(mask) - - return tuple(masks), ilens diff --git a/funasr_detach/frontends/utils/stft.py b/funasr_detach/frontends/utils/stft.py deleted file mode 100644 index 99d5c5e84ba891c5c8c0c3bb8ffa97538007709c..0000000000000000000000000000000000000000 --- a/funasr_detach/frontends/utils/stft.py +++ /dev/null @@ -1,239 +0,0 @@ -from distutils.version import LooseVersion -from typing import Optional -from typing import Tuple -from typing import Union - -import torch - -try: - from torch_complex.tensor import ComplexTensor -except: - print("Please install torch_complex firstly") -from funasr_detach.models.transformer.utils.nets_utils import make_pad_mask -from funasr_detach.frontends.utils.complex_utils import is_complex - -import librosa -import numpy as np - -is_torch_1_9_plus = LooseVersion(torch.__version__) >= LooseVersion("1.9.0") - - -is_torch_1_7_plus = LooseVersion(torch.__version__) >= LooseVersion("1.7") - - -class Stft(torch.nn.Module): - def __init__( - self, - n_fft: int = 512, - win_length: int = None, - hop_length: int = 128, - window: Optional[str] = "hann", - center: bool = True, - normalized: bool = False, - onesided: bool = True, - ): - super().__init__() - self.n_fft = n_fft - if win_length is None: - self.win_length = n_fft - else: - self.win_length = win_length - self.hop_length = hop_length - self.center = center - self.normalized = normalized - self.onesided = onesided - if window is not None and not hasattr(torch, f"{window}_window"): - if window.lower() != "povey": - raise ValueError(f"{window} window is not implemented") - self.window = window - - def extra_repr(self): - return ( - f"n_fft={self.n_fft}, " - f"win_length={self.win_length}, " - f"hop_length={self.hop_length}, " - f"center={self.center}, " - f"normalized={self.normalized}, " - f"onesided={self.onesided}" - ) - - def forward( - self, input: torch.Tensor, ilens: torch.Tensor = None - ) -> Tuple[torch.Tensor, Optional[torch.Tensor]]: - """STFT forward function. - - Args: - input: (Batch, Nsamples) or (Batch, Nsample, Channels) - ilens: (Batch) - Returns: - output: (Batch, Frames, Freq, 2) or (Batch, Frames, Channels, Freq, 2) - - """ - bs = input.size(0) - if input.dim() == 3: - multi_channel = True - # input: (Batch, Nsample, Channels) -> (Batch * Channels, Nsample) - input = input.transpose(1, 2).reshape(-1, input.size(1)) - else: - multi_channel = False - - # NOTE(kamo): - # The default behaviour of torch.stft is compatible with librosa.stft - # about padding and scaling. - # Note that it's different from scipy.signal.stft - - # output: (Batch, Freq, Frames, 2=real_imag) - # or (Batch, Channel, Freq, Frames, 2=real_imag) - if self.window is not None: - if self.window.lower() == "povey": - window = torch.hann_window( - self.win_length, - periodic=False, - device=input.device, - dtype=input.dtype, - ).pow(0.85) - else: - window_func = getattr(torch, f"{self.window}_window") - window = window_func( - self.win_length, dtype=input.dtype, device=input.device - ) - else: - window = None - - # For the compatibility of ARM devices, which do not support - # torch.stft() due to the lake of MKL. - if input.is_cuda or torch.backends.mkl.is_available(): - stft_kwargs = dict( - n_fft=self.n_fft, - win_length=self.win_length, - hop_length=self.hop_length, - center=self.center, - window=window, - normalized=self.normalized, - onesided=self.onesided, - ) - if is_torch_1_7_plus: - stft_kwargs["return_complex"] = False - output = torch.stft(input, **stft_kwargs) - else: - if self.training: - raise NotImplementedError( - "stft is implemented with librosa on this device, which does not " - "support the training mode." - ) - - # use stft_kwargs to flexibly control different PyTorch versions' kwargs - stft_kwargs = dict( - n_fft=self.n_fft, - win_length=self.win_length, - hop_length=self.hop_length, - center=self.center, - window=window, - ) - - if window is not None: - # pad the given window to n_fft - n_pad_left = (self.n_fft - window.shape[0]) // 2 - n_pad_right = self.n_fft - window.shape[0] - n_pad_left - stft_kwargs["window"] = torch.cat( - [torch.zeros(n_pad_left), window, torch.zeros(n_pad_right)], 0 - ).numpy() - else: - win_length = ( - self.win_length if self.win_length is not None else self.n_fft - ) - stft_kwargs["window"] = torch.ones(win_length) - - output = [] - # iterate over istances in a batch - for i, instance in enumerate(input): - stft = librosa.stft(input[i].numpy(), **stft_kwargs) - output.append(torch.tensor(np.stack([stft.real, stft.imag], -1))) - output = torch.stack(output, 0) - if not self.onesided: - len_conj = self.n_fft - output.shape[1] - conj = output[:, 1 : 1 + len_conj].flip(1) - conj[:, :, :, -1].data *= -1 - output = torch.cat([output, conj], 1) - if self.normalized: - output = output * (stft_kwargs["window"].shape[0] ** (-0.5)) - - # output: (Batch, Freq, Frames, 2=real_imag) - # -> (Batch, Frames, Freq, 2=real_imag) - output = output.transpose(1, 2) - if multi_channel: - # output: (Batch * Channel, Frames, Freq, 2=real_imag) - # -> (Batch, Frame, Channel, Freq, 2=real_imag) - output = output.view(bs, -1, output.size(1), output.size(2), 2).transpose( - 1, 2 - ) - - if ilens is not None: - if self.center: - pad = self.n_fft // 2 - ilens = ilens + 2 * pad - - olens = (ilens - self.n_fft) // self.hop_length + 1 - output.masked_fill_(make_pad_mask(olens, output, 1), 0.0) - else: - olens = None - - return output, olens - - def inverse( - self, input: Union[torch.Tensor, ComplexTensor], ilens: torch.Tensor = None - ) -> Tuple[torch.Tensor, Optional[torch.Tensor]]: - """Inverse STFT. - - Args: - input: Tensor(batch, T, F, 2) or ComplexTensor(batch, T, F) - ilens: (batch,) - Returns: - wavs: (batch, samples) - ilens: (batch,) - """ - if LooseVersion(torch.__version__) >= LooseVersion("1.6.0"): - istft = torch.functional.istft - else: - try: - import torchaudio - except ImportError: - raise ImportError( - "Please install torchaudio>=0.3.0 or use torch>=1.6.0" - ) - - if not hasattr(torchaudio.functional, "istft"): - raise ImportError( - "Please install torchaudio>=0.3.0 or use torch>=1.6.0" - ) - istft = torchaudio.functional.istft - - if self.window is not None: - window_func = getattr(torch, f"{self.window}_window") - if is_complex(input): - datatype = input.real.dtype - else: - datatype = input.dtype - window = window_func(self.win_length, dtype=datatype, device=input.device) - else: - window = None - - if is_complex(input): - input = torch.stack([input.real, input.imag], dim=-1) - elif input.shape[-1] != 2: - raise TypeError("Invalid input type") - input = input.transpose(1, 2) - - wavs = istft( - input, - n_fft=self.n_fft, - hop_length=self.hop_length, - win_length=self.win_length, - window=window, - center=self.center, - normalized=self.normalized, - onesided=self.onesided, - length=ilens.max() if ilens is not None else ilens, - ) - - return wavs, ilens diff --git a/funasr_detach/frontends/wav_frontend.py b/funasr_detach/frontends/wav_frontend.py deleted file mode 100644 index 36210348b45dd878c11ea5d13b5d01073c49c807..0000000000000000000000000000000000000000 --- a/funasr_detach/frontends/wav_frontend.py +++ /dev/null @@ -1,556 +0,0 @@ -# Copyright (c) Alibaba, Inc. and its affiliates. -# Part of the implementation is borrowed from espnet/espnet. -from typing import Tuple -import copy -import numpy as np -import torch -import torch.nn as nn -import torchaudio.compliance.kaldi as kaldi -from torch.nn.utils.rnn import pad_sequence - -import funasr_detach.frontends.eend_ola_feature as eend_ola_feature -from funasr_detach.register import tables - - -def load_cmvn(cmvn_file): - with open(cmvn_file, "r", encoding="utf-8") as f: - lines = f.readlines() - means_list = [] - vars_list = [] - for i in range(len(lines)): - line_item = lines[i].split() - if line_item[0] == "": - line_item = lines[i + 1].split() - if line_item[0] == "": - add_shift_line = line_item[3 : (len(line_item) - 1)] - means_list = list(add_shift_line) - continue - elif line_item[0] == "": - line_item = lines[i + 1].split() - if line_item[0] == "": - rescale_line = line_item[3 : (len(line_item) - 1)] - vars_list = list(rescale_line) - continue - means = np.array(means_list).astype(np.float32) - vars = np.array(vars_list).astype(np.float32) - cmvn = np.array([means, vars]) - cmvn = torch.as_tensor(cmvn, dtype=torch.float32) - return cmvn - - -def apply_cmvn(inputs, cmvn): # noqa - """ - Apply CMVN with mvn data - """ - - device = inputs.device - dtype = inputs.dtype - frame, dim = inputs.shape - - means = cmvn[0:1, :dim] - vars = cmvn[1:2, :dim] - inputs += means.to(device) - inputs *= vars.to(device) - - return inputs.type(torch.float32) - - -def apply_lfr(inputs, lfr_m, lfr_n): - LFR_inputs = [] - T = inputs.shape[0] - T_lfr = int(np.ceil(T / lfr_n)) - left_padding = inputs[0].repeat((lfr_m - 1) // 2, 1) - inputs = torch.vstack((left_padding, inputs)) - T = T + (lfr_m - 1) // 2 - for i in range(T_lfr): - if lfr_m <= T - i * lfr_n: - LFR_inputs.append((inputs[i * lfr_n : i * lfr_n + lfr_m]).view(1, -1)) - else: # process last LFR frame - num_padding = lfr_m - (T - i * lfr_n) - frame = (inputs[i * lfr_n :]).view(-1) - for _ in range(num_padding): - frame = torch.hstack((frame, inputs[-1])) - LFR_inputs.append(frame) - LFR_outputs = torch.vstack(LFR_inputs) - return LFR_outputs.type(torch.float32) - - -@tables.register("frontend_classes", "WavFrontend") -class WavFrontend(nn.Module): - """Conventional frontend structure for ASR.""" - - def __init__( - self, - cmvn_file: str = None, - fs: int = 16000, - window: str = "hamming", - n_mels: int = 80, - frame_length: int = 25, - frame_shift: int = 10, - filter_length_min: int = -1, - filter_length_max: int = -1, - lfr_m: int = 1, - lfr_n: int = 1, - dither: float = 1.0, - snip_edges: bool = True, - upsacle_samples: bool = True, - **kwargs, - ): - super().__init__() - self.fs = fs - self.window = window - self.n_mels = n_mels - self.frame_length = frame_length - self.frame_shift = frame_shift - self.filter_length_min = filter_length_min - self.filter_length_max = filter_length_max - self.lfr_m = lfr_m - self.lfr_n = lfr_n - self.cmvn_file = cmvn_file - self.dither = dither - self.snip_edges = snip_edges - self.upsacle_samples = upsacle_samples - self.cmvn = None if self.cmvn_file is None else load_cmvn(self.cmvn_file) - - def output_size(self) -> int: - return self.n_mels * self.lfr_m - - def forward( - self, - input: torch.Tensor, - input_lengths, - **kwargs, - ) -> Tuple[torch.Tensor, torch.Tensor]: - batch_size = input.size(0) - feats = [] - feats_lens = [] - for i in range(batch_size): - waveform_length = input_lengths[i] - waveform = input[i][:waveform_length] - if self.upsacle_samples: - waveform = waveform * (1 << 15) - waveform = waveform.unsqueeze(0) - mat = kaldi.fbank( - waveform, - num_mel_bins=self.n_mels, - frame_length=self.frame_length, - frame_shift=self.frame_shift, - dither=self.dither, - energy_floor=0.0, - window_type=self.window, - sample_frequency=self.fs, - snip_edges=self.snip_edges, - ) - - if self.lfr_m != 1 or self.lfr_n != 1: - mat = apply_lfr(mat, self.lfr_m, self.lfr_n) - if self.cmvn is not None: - mat = apply_cmvn(mat, self.cmvn) - feat_length = mat.size(0) - feats.append(mat) - feats_lens.append(feat_length) - - feats_lens = torch.as_tensor(feats_lens) - if batch_size == 1: - feats_pad = feats[0][None, :, :] - else: - feats_pad = pad_sequence(feats, batch_first=True, padding_value=0.0) - return feats_pad, feats_lens - - def forward_fbank( - self, input: torch.Tensor, input_lengths: torch.Tensor - ) -> Tuple[torch.Tensor, torch.Tensor]: - batch_size = input.size(0) - feats = [] - feats_lens = [] - for i in range(batch_size): - waveform_length = input_lengths[i] - waveform = input[i][:waveform_length] - waveform = waveform * (1 << 15) - waveform = waveform.unsqueeze(0) - mat = kaldi.fbank( - waveform, - num_mel_bins=self.n_mels, - frame_length=self.frame_length, - frame_shift=self.frame_shift, - dither=self.dither, - energy_floor=0.0, - window_type=self.window, - sample_frequency=self.fs, - ) - - feat_length = mat.size(0) - feats.append(mat) - feats_lens.append(feat_length) - - feats_lens = torch.as_tensor(feats_lens) - feats_pad = pad_sequence(feats, batch_first=True, padding_value=0.0) - return feats_pad, feats_lens - - def forward_lfr_cmvn( - self, input: torch.Tensor, input_lengths: torch.Tensor - ) -> Tuple[torch.Tensor, torch.Tensor]: - batch_size = input.size(0) - feats = [] - feats_lens = [] - for i in range(batch_size): - mat = input[i, : input_lengths[i], :] - if self.lfr_m != 1 or self.lfr_n != 1: - mat = apply_lfr(mat, self.lfr_m, self.lfr_n) - if self.cmvn is not None: - mat = apply_cmvn(mat, self.cmvn) - feat_length = mat.size(0) - feats.append(mat) - feats_lens.append(feat_length) - - feats_lens = torch.as_tensor(feats_lens) - feats_pad = pad_sequence(feats, batch_first=True, padding_value=0.0) - return feats_pad, feats_lens - - -@tables.register("frontend_classes", "WavFrontendOnline") -class WavFrontendOnline(nn.Module): - """Conventional frontend structure for streaming ASR/VAD.""" - - def __init__( - self, - cmvn_file: str = None, - fs: int = 16000, - window: str = "hamming", - n_mels: int = 80, - frame_length: int = 25, - frame_shift: int = 10, - filter_length_min: int = -1, - filter_length_max: int = -1, - lfr_m: int = 1, - lfr_n: int = 1, - dither: float = 1.0, - snip_edges: bool = True, - upsacle_samples: bool = True, - **kwargs, - ): - super().__init__() - self.fs = fs - self.window = window - self.n_mels = n_mels - self.frame_length = frame_length - self.frame_shift = frame_shift - self.frame_sample_length = int(self.frame_length * self.fs / 1000) - self.frame_shift_sample_length = int(self.frame_shift * self.fs / 1000) - self.filter_length_min = filter_length_min - self.filter_length_max = filter_length_max - self.lfr_m = lfr_m - self.lfr_n = lfr_n - self.cmvn_file = cmvn_file - self.dither = dither - self.snip_edges = snip_edges - self.upsacle_samples = upsacle_samples - # self.waveforms = None - # self.reserve_waveforms = None - # self.fbanks = None - # self.fbanks_lens = None - self.cmvn = None if self.cmvn_file is None else load_cmvn(self.cmvn_file) - # self.input_cache = None - # self.lfr_splice_cache = [] - - def output_size(self) -> int: - return self.n_mels * self.lfr_m - - @staticmethod - def apply_cmvn(inputs: torch.Tensor, cmvn: torch.Tensor) -> torch.Tensor: - """ - Apply CMVN with mvn data - """ - - device = inputs.device - dtype = inputs.dtype - frame, dim = inputs.shape - - means = np.tile(cmvn[0:1, :dim], (frame, 1)) - vars = np.tile(cmvn[1:2, :dim], (frame, 1)) - inputs += torch.from_numpy(means).type(dtype).to(device) - inputs *= torch.from_numpy(vars).type(dtype).to(device) - - return inputs.type(torch.float32) - - @staticmethod - def apply_lfr( - inputs: torch.Tensor, lfr_m: int, lfr_n: int, is_final: bool = False - ) -> Tuple[torch.Tensor, torch.Tensor, int]: - """ - Apply lfr with data - """ - - LFR_inputs = [] - # inputs = torch.vstack((inputs_lfr_cache, inputs)) - T = inputs.shape[0] # include the right context - T_lfr = int( - np.ceil((T - (lfr_m - 1) // 2) / lfr_n) - ) # minus the right context: (lfr_m - 1) // 2 - splice_idx = T_lfr - for i in range(T_lfr): - if lfr_m <= T - i * lfr_n: - LFR_inputs.append((inputs[i * lfr_n : i * lfr_n + lfr_m]).view(1, -1)) - else: # process last LFR frame - if is_final: - num_padding = lfr_m - (T - i * lfr_n) - frame = (inputs[i * lfr_n :]).view(-1) - for _ in range(num_padding): - frame = torch.hstack((frame, inputs[-1])) - LFR_inputs.append(frame) - else: - # update splice_idx and break the circle - splice_idx = i - break - splice_idx = min(T - 1, splice_idx * lfr_n) - lfr_splice_cache = inputs[splice_idx:, :] - LFR_outputs = torch.vstack(LFR_inputs) - return LFR_outputs.type(torch.float32), lfr_splice_cache, splice_idx - - @staticmethod - def compute_frame_num( - sample_length: int, frame_sample_length: int, frame_shift_sample_length: int - ) -> int: - frame_num = int( - (sample_length - frame_sample_length) / frame_shift_sample_length + 1 - ) - return ( - frame_num if frame_num >= 1 and sample_length >= frame_sample_length else 0 - ) - - def forward_fbank( - self, - input: torch.Tensor, - input_lengths: torch.Tensor, - cache: dict = {}, - **kwargs, - ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: - batch_size = input.size(0) - assert batch_size == 1 - input = torch.cat((cache["input_cache"], input), dim=1) - frame_num = self.compute_frame_num( - input.shape[-1], self.frame_sample_length, self.frame_shift_sample_length - ) - # update self.in_cache - cache["input_cache"] = input[ - :, -(input.shape[-1] - frame_num * self.frame_shift_sample_length) : - ] - waveforms = torch.empty(0) - feats_pad = torch.empty(0) - feats_lens = torch.empty(0) - if frame_num: - waveforms = [] - feats = [] - feats_lens = [] - for i in range(batch_size): - waveform = input[i].cuda() - # we need accurate wave samples that used for fbank extracting - waveforms.append( - waveform[ - : ( - (frame_num - 1) * self.frame_shift_sample_length - + self.frame_sample_length - ) - ] - ) - waveform = waveform * (1 << 15) - waveform = waveform.unsqueeze(0) - mat = kaldi.fbank( - waveform, - num_mel_bins=self.n_mels, - frame_length=self.frame_length, - frame_shift=self.frame_shift, - dither=self.dither, - energy_floor=0.0, - window_type=self.window, - sample_frequency=self.fs, - ) - - feat_length = mat.size(0) - feats.append(mat) - feats_lens.append(feat_length) - - waveforms = torch.stack(waveforms) - feats_lens = torch.as_tensor(feats_lens) - feats_pad = pad_sequence(feats, batch_first=True, padding_value=0.0) - cache["fbanks"] = feats_pad - cache["fbanks_lens"] = copy.deepcopy(feats_lens) - return waveforms, feats_pad, feats_lens - - def forward_lfr_cmvn( - self, - input: torch.Tensor, - input_lengths: torch.Tensor, - is_final: bool = False, - cache: dict = {}, - **kwargs, - ): - batch_size = input.size(0) - feats = [] - feats_lens = [] - lfr_splice_frame_idxs = [] - for i in range(batch_size): - mat = input[i, : input_lengths[i], :] - if self.lfr_m != 1 or self.lfr_n != 1: - # update self.lfr_splice_cache in self.apply_lfr - # mat, self.lfr_splice_cache[i], lfr_splice_frame_idx = self.apply_lfr(mat, self.lfr_m, self.lfr_n, self.lfr_splice_cache[i], - mat, cache["lfr_splice_cache"][i], lfr_splice_frame_idx = ( - self.apply_lfr(mat, self.lfr_m, self.lfr_n, is_final) - ) - if self.cmvn_file is not None: - mat = self.apply_cmvn(mat, self.cmvn) - feat_length = mat.size(0) - feats.append(mat) - feats_lens.append(feat_length) - lfr_splice_frame_idxs.append(lfr_splice_frame_idx) - feats_lens = torch.as_tensor(feats_lens) - feats_pad = pad_sequence(feats, batch_first=True, padding_value=0.0) - lfr_splice_frame_idxs = torch.as_tensor(lfr_splice_frame_idxs) - return feats_pad, feats_lens, lfr_splice_frame_idxs - - def forward(self, input: torch.Tensor, input_lengths: torch.Tensor, **kwargs): - is_final = kwargs.get("is_final", False) - cache = kwargs.get("cache", {}) - if len(cache) == 0: - self.init_cache(cache) - - batch_size = input.shape[0] - assert ( - batch_size == 1 - ), "we support to extract feature online only when the batch size is equal to 1 now" - - waveforms, feats, feats_lengths = self.forward_fbank( - input, input_lengths, cache=cache - ) # input shape: B T D - - if feats.shape[0]: - - cache["waveforms"] = torch.cat( - (cache["reserve_waveforms"], waveforms.cpu()), dim=1 - ) - - if not cache["lfr_splice_cache"]: # 初始化splice_cache - for i in range(batch_size): - cache["lfr_splice_cache"].append( - feats[i][0, :].unsqueeze(dim=0).repeat((self.lfr_m - 1) // 2, 1) - ) - # need the number of the input frames + self.lfr_splice_cache[0].shape[0] is greater than self.lfr_m - if feats_lengths[0] + cache["lfr_splice_cache"][0].shape[0] >= self.lfr_m: - lfr_splice_cache_tensor = torch.stack( - cache["lfr_splice_cache"] - ) # B T D - feats = torch.cat((lfr_splice_cache_tensor, feats), dim=1) - - feats_lengths += lfr_splice_cache_tensor[0].shape[0] - frame_from_waveforms = int( - (cache["waveforms"].shape[1] - self.frame_sample_length) - / self.frame_shift_sample_length - + 1 - ) - minus_frame = ( - (self.lfr_m - 1) // 2 - if cache["reserve_waveforms"].numel() == 0 - else 0 - ) - feats, feats_lengths, lfr_splice_frame_idxs = self.forward_lfr_cmvn( - feats, feats_lengths, is_final, cache=cache - ) - if self.lfr_m == 1: - cache["reserve_waveforms"] = torch.empty(0) - else: - reserve_frame_idx = lfr_splice_frame_idxs[0] - minus_frame - # print('reserve_frame_idx: ' + str(reserve_frame_idx)) - # print('frame_frame: ' + str(frame_from_waveforms)) - cache["reserve_waveforms"] = cache["waveforms"][ - :, - reserve_frame_idx - * self.frame_shift_sample_length : frame_from_waveforms - * self.frame_shift_sample_length, - ] - sample_length = ( - frame_from_waveforms - 1 - ) * self.frame_shift_sample_length + self.frame_sample_length - cache["waveforms"] = cache["waveforms"][:, :sample_length] - else: - # update self.reserve_waveforms and self.lfr_splice_cache - cache["reserve_waveforms"] = cache["waveforms"][ - :, : -(self.frame_sample_length - self.frame_shift_sample_length) - ] - for i in range(batch_size): - cache["lfr_splice_cache"][i] = torch.cat( - (cache["lfr_splice_cache"][i], feats[i]), dim=0 - ) - return torch.empty(0), feats_lengths - else: - if is_final: - cache["waveforms"] = ( - waveforms - if cache["reserve_waveforms"].numel() == 0 - else cache["reserve_waveforms"] - ) - feats = torch.stack(cache["lfr_splice_cache"]) - feats_lengths = ( - torch.zeros(batch_size, dtype=torch.int) + feats.shape[1] - ) - feats, feats_lengths, _ = self.forward_lfr_cmvn( - feats, feats_lengths, is_final, cache=cache - ) - # if is_final: - # self.init_cache(cache) - return feats, feats_lengths - - def init_cache(self, cache: dict = {}): - cache["reserve_waveforms"] = torch.empty(0) - cache["input_cache"] = torch.empty(0) - cache["lfr_splice_cache"] = [] - cache["waveforms"] = None - cache["fbanks"] = None - cache["fbanks_lens"] = None - return cache - - -class WavFrontendMel23(nn.Module): - """Conventional frontend structure for ASR.""" - - def __init__( - self, - fs: int = 16000, - frame_length: int = 25, - frame_shift: int = 10, - lfr_m: int = 1, - lfr_n: int = 1, - **kwargs, - ): - super().__init__() - self.fs = fs - self.frame_length = frame_length - self.frame_shift = frame_shift - self.lfr_m = lfr_m - self.lfr_n = lfr_n - self.n_mels = 23 - - def output_size(self) -> int: - return self.n_mels * (2 * self.lfr_m + 1) - - def forward( - self, input: torch.Tensor, input_lengths: torch.Tensor - ) -> Tuple[torch.Tensor, torch.Tensor]: - batch_size = input.size(0) - feats = [] - feats_lens = [] - for i in range(batch_size): - waveform_length = input_lengths[i] - waveform = input[i][:waveform_length] - waveform = waveform.numpy() - mat = eend_ola_feature.stft(waveform, self.frame_length, self.frame_shift) - mat = eend_ola_feature.transform(mat) - mat = eend_ola_feature.splice(mat, context_size=self.lfr_m) - mat = mat[:: self.lfr_n] - mat = torch.from_numpy(mat) - feat_length = mat.size(0) - feats.append(mat) - feats_lens.append(feat_length) - - feats_lens = torch.as_tensor(feats_lens) - feats_pad = pad_sequence(feats, batch_first=True, padding_value=0.0) - return feats_pad, feats_lens diff --git a/funasr_detach/frontends/windowing.py b/funasr_detach/frontends/windowing.py deleted file mode 100644 index 84ec8e200e2cbae7e6c4f50e33e3ba933b03195f..0000000000000000000000000000000000000000 --- a/funasr_detach/frontends/windowing.py +++ /dev/null @@ -1,74 +0,0 @@ -#!/usr/bin/env python3 -# 2020, Technische Universität München; Ludwig Kürzinger -# Apache 2.0 (http://www.apache.org/licenses/LICENSE-2.0) - -"""Sliding Window for raw audio input data.""" - -import torch -import torch.nn as nn -from typing import Tuple - - -class SlidingWindow(nn.Module): - """Sliding Window. - Provides a sliding window over a batched continuous raw audio tensor. - Optionally, provides padding (Currently not implemented). - Combine this module with a pre-encoder compatible with raw audio data, - for example Sinc convolutions. - Known issues: - Output length is calculated incorrectly if audio shorter than win_length. - WARNING: trailing values are discarded - padding not implemented yet. - There is currently no additional window function applied to input values. - """ - - def __init__( - self, - win_length: int = 400, - hop_length: int = 160, - channels: int = 1, - padding: int = None, - fs=None, - ): - """Initialize. - Args: - win_length: Length of frame. - hop_length: Relative starting point of next frame. - channels: Number of input channels. - padding: Padding (placeholder, currently not implemented). - fs: Sampling rate (placeholder for compatibility, not used). - """ - super().__init__() - self.fs = fs - self.win_length = win_length - self.hop_length = hop_length - self.channels = channels - self.padding = padding - - def forward( - self, input: torch.Tensor, input_lengths: torch.Tensor - ) -> Tuple[torch.Tensor, torch.Tensor]: - """Apply a sliding window on the input. - Args: - input: Input (B, T, C*D) or (B, T*C*D), with D=C=1. - input_lengths: Input lengths within batch. - Returns: - Tensor: Output with dimensions (B, T, C, D), with D=win_length. - Tensor: Output lengths within batch. - """ - input_size = input.size() - B = input_size[0] - T = input_size[1] - C = self.channels - D = self.win_length - # (B, T, C) --> (T, B, C) - continuous = input.view(B, T, C).permute(1, 0, 2) - windowed = continuous.unfold(0, D, self.hop_length) - # (T, B, C, D) --> (B, T, C, D) - output = windowed.permute(1, 0, 2, 3).contiguous() - # After unfold(), windowed lengths change: - output_lengths = (input_lengths - self.win_length) // self.hop_length + 1 - return output, output_lengths - - def output_size(self) -> int: - """Return output length of feature dimension D, i.e. the window length.""" - return self.win_length diff --git a/funasr_detach/losses/__init__.py b/funasr_detach/losses/__init__.py deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/funasr_detach/losses/label_smoothing_loss.py b/funasr_detach/losses/label_smoothing_loss.py deleted file mode 100644 index d96a29905192e55d9abfdae86d2f75b7f3d0d02f..0000000000000000000000000000000000000000 --- a/funasr_detach/losses/label_smoothing_loss.py +++ /dev/null @@ -1,125 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- - -# Copyright 2019 Shigeki Karita -# Apache 2.0 (http://www.apache.org/licenses/LICENSE-2.0) - -"""Label smoothing module.""" - -import torch -from torch import nn -from funasr_detach.models.transformer.utils.nets_utils import make_pad_mask - - -class LabelSmoothingLoss(nn.Module): - """Label-smoothing loss. - - :param int size: the number of class - :param int padding_idx: ignored class id - :param float smoothing: smoothing rate (0.0 means the conventional CE) - :param bool normalize_length: normalize loss by sequence length if True - :param torch.nn.Module criterion: loss function to be smoothed - """ - - def __init__( - self, - size, - padding_idx, - smoothing, - normalize_length=False, - criterion=nn.KLDivLoss(reduction="none"), - ): - """Construct an LabelSmoothingLoss object.""" - super(LabelSmoothingLoss, self).__init__() - self.criterion = criterion - self.padding_idx = padding_idx - self.confidence = 1.0 - smoothing - self.smoothing = smoothing - self.size = size - self.true_dist = None - self.normalize_length = normalize_length - - def forward(self, x, target): - """Compute loss between x and target. - - :param torch.Tensor x: prediction (batch, seqlen, class) - :param torch.Tensor target: - target signal masked with self.padding_id (batch, seqlen) - :return: scalar float value - :rtype torch.Tensor - """ - assert x.size(2) == self.size - batch_size = x.size(0) - x = x.view(-1, self.size) - target = target.view(-1) - with torch.no_grad(): - true_dist = x.clone() - true_dist.fill_(self.smoothing / (self.size - 1)) - ignore = target == self.padding_idx # (B,) - total = len(target) - ignore.sum().item() - target = target.masked_fill(ignore, 0) # avoid -1 index - true_dist.scatter_(1, target.unsqueeze(1), self.confidence) - kl = self.criterion(torch.log_softmax(x, dim=1), true_dist) - denom = total if self.normalize_length else batch_size - return kl.masked_fill(ignore.unsqueeze(1), 0).sum() / denom - - -class SequenceBinaryCrossEntropy(nn.Module): - def __init__( - self, normalize_length=False, criterion=nn.BCEWithLogitsLoss(reduction="none") - ): - super().__init__() - self.normalize_length = normalize_length - self.criterion = criterion - - def forward(self, pred, label, lengths): - pad_mask = make_pad_mask(lengths, maxlen=pred.shape[1]).to(pred.device) - loss = self.criterion(pred, label) - denom = (~pad_mask).sum() if self.normalize_length else pred.shape[0] - return loss.masked_fill(pad_mask.unsqueeze(-1), 0).sum() / denom - - -class NllLoss(nn.Module): - """Nll loss. - - :param int size: the number of class - :param int padding_idx: ignored class id - :param bool normalize_length: normalize loss by sequence length if True - :param torch.nn.Module criterion: loss function - """ - - def __init__( - self, - size, - padding_idx, - normalize_length=False, - criterion=nn.NLLLoss(reduction="none"), - ): - """Construct an NllLoss object.""" - super(NllLoss, self).__init__() - self.criterion = criterion - self.padding_idx = padding_idx - self.size = size - self.true_dist = None - self.normalize_length = normalize_length - - def forward(self, x, target): - """Compute loss between x and target. - - :param torch.Tensor x: prediction (batch, seqlen, class) - :param torch.Tensor target: - target signal masked with self.padding_id (batch, seqlen) - :return: scalar float value - :rtype torch.Tensor - """ - assert x.size(2) == self.size - batch_size = x.size(0) - x = x.view(-1, self.size) - target = target.view(-1) - with torch.no_grad(): - ignore = target == self.padding_idx # (B,) - total = len(target) - ignore.sum().item() - target = target.masked_fill(ignore, 0) # avoid -1 index - kl = self.criterion(x, target) - denom = total if self.normalize_length else batch_size - return kl.masked_fill(ignore, 0).sum() / denom diff --git a/funasr_detach/metrics/__init__.py b/funasr_detach/metrics/__init__.py deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/funasr_detach/metrics/common.py b/funasr_detach/metrics/common.py deleted file mode 100644 index 92f90796a3a230b3bfc47ebe8d9292fae37a1b9c..0000000000000000000000000000000000000000 --- a/funasr_detach/metrics/common.py +++ /dev/null @@ -1,249 +0,0 @@ -#!/usr/bin/env python3 -# encoding: utf-8 - -# Copyright 2017 Johns Hopkins University (Shinji Watanabe) -# Apache 2.0 (http://www.apache.org/licenses/LICENSE-2.0) - -"""Common functions for ASR.""" - -import json -import logging -import sys - -from itertools import groupby -import numpy as np -import six - - -def end_detect(ended_hyps, i, M=3, D_end=np.log(1 * np.exp(-10))): - """End detection. - - described in Eq. (50) of S. Watanabe et al - "Hybrid CTC/Attention Architecture for End-to-End Speech Recognition" - - :param ended_hyps: - :param i: - :param M: - :param D_end: - :return: - """ - if len(ended_hyps) == 0: - return False - count = 0 - best_hyp = sorted(ended_hyps, key=lambda x: x["score"], reverse=True)[0] - for m in six.moves.range(M): - # get ended_hyps with their length is i - m - hyp_length = i - m - hyps_same_length = [x for x in ended_hyps if len(x["yseq"]) == hyp_length] - if len(hyps_same_length) > 0: - best_hyp_same_length = sorted( - hyps_same_length, key=lambda x: x["score"], reverse=True - )[0] - if best_hyp_same_length["score"] - best_hyp["score"] < D_end: - count += 1 - - if count == M: - return True - else: - return False - - -# TODO(takaaki-hori): add different smoothing methods -def label_smoothing_dist(odim, lsm_type, transcript=None, blank=0): - """Obtain label distribution for loss smoothing. - - :param odim: - :param lsm_type: - :param blank: - :param transcript: - :return: - """ - if transcript is not None: - with open(transcript, "rb") as f: - trans_json = json.load(f)["utts"] - - if lsm_type == "unigram": - assert transcript is not None, ( - "transcript is required for %s label smoothing" % lsm_type - ) - labelcount = np.zeros(odim) - for k, v in trans_json.items(): - ids = np.array([int(n) for n in v["output"][0]["tokenid"].split()]) - # to avoid an error when there is no text in an uttrance - if len(ids) > 0: - labelcount[ids] += 1 - labelcount[odim - 1] = len(transcript) # count - labelcount[labelcount == 0] = 1 # flooring - labelcount[blank] = 0 # remove counts for blank - labeldist = labelcount.astype(np.float32) / np.sum(labelcount) - else: - logging.error("Error: unexpected label smoothing type: %s" % lsm_type) - sys.exit() - - return labeldist - - -def get_vgg2l_odim(idim, in_channel=3, out_channel=128): - """Return the output size of the VGG frontend. - - :param in_channel: input channel size - :param out_channel: output channel size - :return: output size - :rtype int - """ - idim = idim / in_channel - idim = np.ceil(np.array(idim, dtype=np.float32) / 2) # 1st max pooling - idim = np.ceil(np.array(idim, dtype=np.float32) / 2) # 2nd max pooling - return int(idim) * out_channel # numer of channels - - -class ErrorCalculator(object): - """Calculate CER and WER for E2E_ASR and CTC models during training. - - :param y_hats: numpy array with predicted text - :param y_pads: numpy array with true (target) text - :param char_list: - :param sym_space: - :param sym_blank: - :return: - """ - - def __init__( - self, char_list, sym_space, sym_blank, report_cer=False, report_wer=False - ): - """Construct an ErrorCalculator object.""" - super(ErrorCalculator, self).__init__() - - self.report_cer = report_cer - self.report_wer = report_wer - - self.char_list = char_list - self.space = sym_space - self.blank = sym_blank - self.idx_blank = self.char_list.index(self.blank) - if self.space in self.char_list: - self.idx_space = self.char_list.index(self.space) - else: - self.idx_space = None - - def __call__(self, ys_hat, ys_pad, is_ctc=False): - """Calculate sentence-level WER/CER score. - - :param torch.Tensor ys_hat: prediction (batch, seqlen) - :param torch.Tensor ys_pad: reference (batch, seqlen) - :param bool is_ctc: calculate CER score for CTC - :return: sentence-level WER score - :rtype float - :return: sentence-level CER score - :rtype float - """ - cer, wer = None, None - if is_ctc: - return self.calculate_cer_ctc(ys_hat, ys_pad) - elif not self.report_cer and not self.report_wer: - return cer, wer - - seqs_hat, seqs_true = self.convert_to_char(ys_hat, ys_pad) - if self.report_cer: - cer = self.calculate_cer(seqs_hat, seqs_true) - - if self.report_wer: - wer = self.calculate_wer(seqs_hat, seqs_true) - return cer, wer - - def calculate_cer_ctc(self, ys_hat, ys_pad): - """Calculate sentence-level CER score for CTC. - - :param torch.Tensor ys_hat: prediction (batch, seqlen) - :param torch.Tensor ys_pad: reference (batch, seqlen) - :return: average sentence-level CER score - :rtype float - """ - import editdistance - - cers, char_ref_lens = [], [] - for i, y in enumerate(ys_hat): - y_hat = [x[0] for x in groupby(y)] - y_true = ys_pad[i] - seq_hat, seq_true = [], [] - for idx in y_hat: - idx = int(idx) - if idx != -1 and idx != self.idx_blank and idx != self.idx_space: - seq_hat.append(self.char_list[int(idx)]) - - for idx in y_true: - idx = int(idx) - if idx != -1 and idx != self.idx_blank and idx != self.idx_space: - seq_true.append(self.char_list[int(idx)]) - - hyp_chars = "".join(seq_hat) - ref_chars = "".join(seq_true) - if len(ref_chars) > 0: - cers.append(editdistance.eval(hyp_chars, ref_chars)) - char_ref_lens.append(len(ref_chars)) - - cer_ctc = float(sum(cers)) / sum(char_ref_lens) if cers else None - return cer_ctc - - def convert_to_char(self, ys_hat, ys_pad): - """Convert index to character. - - :param torch.Tensor seqs_hat: prediction (batch, seqlen) - :param torch.Tensor seqs_true: reference (batch, seqlen) - :return: token list of prediction - :rtype list - :return: token list of reference - :rtype list - """ - seqs_hat, seqs_true = [], [] - for i, y_hat in enumerate(ys_hat): - y_true = ys_pad[i] - eos_true = np.where(y_true == -1)[0] - ymax = eos_true[0] if len(eos_true) > 0 else len(y_true) - # NOTE: padding index (-1) in y_true is used to pad y_hat - seq_hat = [self.char_list[int(idx)] for idx in y_hat[:ymax]] - seq_true = [self.char_list[int(idx)] for idx in y_true if int(idx) != -1] - seq_hat_text = "".join(seq_hat).replace(self.space, " ") - seq_hat_text = seq_hat_text.replace(self.blank, "") - seq_true_text = "".join(seq_true).replace(self.space, " ") - seqs_hat.append(seq_hat_text) - seqs_true.append(seq_true_text) - return seqs_hat, seqs_true - - def calculate_cer(self, seqs_hat, seqs_true): - """Calculate sentence-level CER score. - - :param list seqs_hat: prediction - :param list seqs_true: reference - :return: average sentence-level CER score - :rtype float - """ - import editdistance - - char_eds, char_ref_lens = [], [] - for i, seq_hat_text in enumerate(seqs_hat): - seq_true_text = seqs_true[i] - hyp_chars = seq_hat_text.replace(" ", "") - ref_chars = seq_true_text.replace(" ", "") - char_eds.append(editdistance.eval(hyp_chars, ref_chars)) - char_ref_lens.append(len(ref_chars)) - return float(sum(char_eds)) / sum(char_ref_lens) - - def calculate_wer(self, seqs_hat, seqs_true): - """Calculate sentence-level WER score. - - :param list seqs_hat: prediction - :param list seqs_true: reference - :return: average sentence-level WER score - :rtype float - """ - import editdistance - - word_eds, word_ref_lens = [], [] - for i, seq_hat_text in enumerate(seqs_hat): - seq_true_text = seqs_true[i] - hyp_words = seq_hat_text.split() - ref_words = seq_true_text.split() - word_eds.append(editdistance.eval(hyp_words, ref_words)) - word_ref_lens.append(len(ref_words)) - return float(sum(word_eds)) / sum(word_ref_lens) diff --git a/funasr_detach/metrics/compute_acc.py b/funasr_detach/metrics/compute_acc.py deleted file mode 100644 index 25c3591203d1358e9d56eadf19d6cce23d8739c0..0000000000000000000000000000000000000000 --- a/funasr_detach/metrics/compute_acc.py +++ /dev/null @@ -1,24 +0,0 @@ -import torch - - -def th_accuracy(pad_outputs, pad_targets, ignore_label): - """Calculate accuracy. - - Args: - pad_outputs (Tensor): Prediction tensors (B * Lmax, D). - pad_targets (LongTensor): Target label tensors (B, Lmax, D). - ignore_label (int): Ignore label id. - - Returns: - float: Accuracy value (0.0 - 1.0). - - """ - pad_pred = pad_outputs.view( - pad_targets.size(0), pad_targets.size(1), pad_outputs.size(1) - ).argmax(2) - mask = pad_targets != ignore_label - numerator = torch.sum( - pad_pred.masked_select(mask) == pad_targets.masked_select(mask) - ) - denominator = torch.sum(mask) - return float(numerator) / float(denominator) diff --git a/funasr_detach/metrics/compute_eer.py b/funasr_detach/metrics/compute_eer.py deleted file mode 100644 index 5180b16694f629c532e13c94912f7e056dd8f0bf..0000000000000000000000000000000000000000 --- a/funasr_detach/metrics/compute_eer.py +++ /dev/null @@ -1,59 +0,0 @@ -import numpy as np -from sklearn.metrics import roc_curve -import argparse - - -def _compute_eer(label, pred, positive_label=1): - """ - Python compute equal error rate (eer) - ONLY tested on binary classification - - :param label: ground-truth label, should be a 1-d list or np.array, each element represents the ground-truth label of one sample - :param pred: model prediction, should be a 1-d list or np.array, each element represents the model prediction of one sample - :param positive_label: the class that is viewed as positive class when computing EER - :return: equal error rate (EER) - """ - - # all fpr, tpr, fnr, fnr, threshold are lists (in the format of np.array) - fpr, tpr, threshold = roc_curve(label, pred, pos_label=positive_label) - fnr = 1 - tpr - - # the threshold of fnr == fpr - eer_threshold = threshold[np.nanargmin(np.absolute((fnr - fpr)))] - - # theoretically eer from fpr and eer from fnr should be identical but they can be slightly differ in reality - eer_1 = fpr[np.nanargmin(np.absolute((fnr - fpr)))] - eer_2 = fnr[np.nanargmin(np.absolute((fnr - fpr)))] - - # return the mean of eer from fpr and from fnr - eer = (eer_1 + eer_2) / 2 - return eer, eer_threshold - - -def compute_eer(trials_path, scores_path): - labels = [] - for one_line in open(trials_path, "r"): - labels.append(one_line.strip().rsplit(" ", 1)[-1] == "target") - labels = np.array(labels, dtype=int) - - scores = [] - for one_line in open(scores_path, "r"): - scores.append(float(one_line.strip().rsplit(" ", 1)[-1])) - scores = np.array(scores, dtype=float) - - eer, threshold = _compute_eer(labels, scores) - return eer, threshold - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument("trials", help="trial list") - parser.add_argument("scores", help="score file, normalized to [0, 1]") - args = parser.parse_args() - - eer, threshold = compute_eer(args.trials, args.scores) - print("EER is {:.4f} at threshold {:.4f}".format(eer * 100.0, threshold)) - - -if __name__ == "__main__": - main() diff --git a/funasr_detach/metrics/compute_min_dcf.py b/funasr_detach/metrics/compute_min_dcf.py deleted file mode 100644 index a3da8d40a12f6b2a5fd7933083262769a218d6fe..0000000000000000000000000000000000000000 --- a/funasr_detach/metrics/compute_min_dcf.py +++ /dev/null @@ -1,186 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2018 David Snyder -# Apache 2.0 - -# This script computes the minimum detection cost function, which is a common -# error metric used in speaker recognition. Compared to equal error-rate, -# which assigns equal weight to false negatives and false positives, this -# error-rate is usually used to assess performance in settings where achieving -# a low false positive rate is more important than achieving a low false -# negative rate. See the NIST 2016 Speaker Recognition Evaluation Plan at -# https://www.nist.gov/sites/default/files/documents/2016/10/07/sre16_eval_plan_v1.3.pdf -# for more details about the metric. -from __future__ import print_function -from operator import itemgetter -import sys, argparse, os - - -def GetArgs(): - parser = argparse.ArgumentParser( - description="Compute the minimum " - "detection cost function along with the threshold at which it occurs. " - "Usage: sid/compute_min_dcf.py [options...] " - " " - "E.g., sid/compute_min_dcf.py --p-target 0.01 --c-miss 1 --c-fa 1 " - "exp/scores/trials data/test/trials", - formatter_class=argparse.ArgumentDefaultsHelpFormatter, - ) - parser.add_argument( - "--p-target", - type=float, - dest="p_target", - default=0.01, - help="The prior probability of the target speaker in a trial.", - ) - parser.add_argument( - "--c-miss", - type=float, - dest="c_miss", - default=1, - help="Cost of a missed detection. This is usually not changed.", - ) - parser.add_argument( - "--c-fa", - type=float, - dest="c_fa", - default=1, - help="Cost of a spurious detection. This is usually not changed.", - ) - parser.add_argument( - "scores_filename", - help="Input scores file, with columns of the form " " ", - ) - parser.add_argument( - "trials_filename", - help="Input trials file, with columns of the form " - " ", - ) - sys.stderr.write(" ".join(sys.argv) + "\n") - args = parser.parse_args() - args = CheckArgs(args) - return args - - -def CheckArgs(args): - if args.c_fa <= 0: - raise Exception("--c-fa must be greater than 0") - if args.c_miss <= 0: - raise Exception("--c-miss must be greater than 0") - if args.p_target <= 0 or args.p_target >= 1: - raise Exception("--p-target must be greater than 0 and less than 1") - return args - - -# Creates a list of false-negative rates, a list of false-positive rates -# and a list of decision thresholds that give those error-rates. -def ComputeErrorRates(scores, labels): - - # Sort the scores from smallest to largest, and also get the corresponding - # indexes of the sorted scores. We will treat the sorted scores as the - # thresholds at which the the error-rates are evaluated. - sorted_indexes, thresholds = zip( - *sorted( - [(index, threshold) for index, threshold in enumerate(scores)], - key=itemgetter(1), - ) - ) - labels = [labels[i] for i in sorted_indexes] - fns = [] - tns = [] - - # At the end of this loop, fns[i] is the number of errors made by - # incorrectly rejecting scores less than thresholds[i]. And, tns[i] - # is the total number of times that we have correctly rejected scores - # less than thresholds[i]. - for i in range(0, len(labels)): - if i == 0: - fns.append(labels[i]) - tns.append(1 - labels[i]) - else: - fns.append(fns[i - 1] + labels[i]) - tns.append(tns[i - 1] + 1 - labels[i]) - positives = sum(labels) - negatives = len(labels) - positives - - # Now divide the false negatives by the total number of - # positives to obtain the false negative rates across - # all thresholds - fnrs = [fn / float(positives) for fn in fns] - - # Divide the true negatives by the total number of - # negatives to get the true negative rate. Subtract these - # quantities from 1 to get the false positive rates. - fprs = [1 - tn / float(negatives) for tn in tns] - return fnrs, fprs, thresholds - - -# Computes the minimum of the detection cost function. The comments refer to -# equations in Section 3 of the NIST 2016 Speaker Recognition Evaluation Plan. -def ComputeMinDcf(fnrs, fprs, thresholds, p_target, c_miss, c_fa): - min_c_det = float("inf") - min_c_det_threshold = thresholds[0] - for i in range(0, len(fnrs)): - # See Equation (2). it is a weighted sum of false negative - # and false positive errors. - c_det = c_miss * fnrs[i] * p_target + c_fa * fprs[i] * (1 - p_target) - if c_det < min_c_det: - min_c_det = c_det - min_c_det_threshold = thresholds[i] - # See Equations (3) and (4). Now we normalize the cost. - c_def = min(c_miss * p_target, c_fa * (1 - p_target)) - min_dcf = min_c_det / c_def - return min_dcf, min_c_det_threshold - - -def compute_min_dcf(scores_filename, trials_filename, c_miss=1, c_fa=1, p_target=0.01): - scores_file = open(scores_filename, "r").readlines() - trials_file = open(trials_filename, "r").readlines() - c_miss = c_miss - c_fa = c_fa - p_target = p_target - - scores = [] - labels = [] - - trials = {} - for line in trials_file: - utt1, utt2, target = line.rstrip().split() - trial = utt1 + " " + utt2 - trials[trial] = target - - for line in scores_file: - utt1, utt2, score = line.rstrip().split() - trial = utt1 + " " + utt2 - if trial in trials: - scores.append(float(score)) - if trials[trial] == "target": - labels.append(1) - else: - labels.append(0) - else: - raise Exception( - "Missing entry for " + utt1 + " and " + utt2 + " " + scores_filename - ) - - fnrs, fprs, thresholds = ComputeErrorRates(scores, labels) - mindcf, threshold = ComputeMinDcf(fnrs, fprs, thresholds, p_target, c_miss, c_fa) - return mindcf, threshold - - -def main(): - args = GetArgs() - mindcf, threshold = compute_min_dcf( - args.scores_filename, - args.trials_filename, - args.c_miss, - args.c_fa, - args.p_target, - ) - sys.stdout.write( - "minDCF is {0:.4f} at threshold {1:.4f} (p-target={2}, c-miss={3}, " - "c-fa={4})\n".format(mindcf, threshold, args.p_target, args.c_miss, args.c_fa) - ) - - -if __name__ == "__main__": - main() diff --git a/funasr_detach/metrics/compute_wer.py b/funasr_detach/metrics/compute_wer.py deleted file mode 100755 index 9f86be25faa54bde1e5da12483fbe00fc3ff8827..0000000000000000000000000000000000000000 --- a/funasr_detach/metrics/compute_wer.py +++ /dev/null @@ -1,203 +0,0 @@ -import os -import numpy as np -import sys - - -def compute_wer(ref_file, hyp_file, cer_detail_file): - rst = { - "Wrd": 0, - "Corr": 0, - "Ins": 0, - "Del": 0, - "Sub": 0, - "Snt": 0, - "Err": 0.0, - "S.Err": 0.0, - "wrong_words": 0, - "wrong_sentences": 0, - } - - hyp_dict = {} - ref_dict = {} - with open(hyp_file, "r") as hyp_reader: - for line in hyp_reader: - key = line.strip().split()[0] - value = line.strip().split()[1:] - hyp_dict[key] = value - with open(ref_file, "r") as ref_reader: - for line in ref_reader: - key = line.strip().split()[0] - value = line.strip().split()[1:] - ref_dict[key] = value - - cer_detail_writer = open(cer_detail_file, "w") - for hyp_key in hyp_dict: - if hyp_key in ref_dict: - out_item = compute_wer_by_line(hyp_dict[hyp_key], ref_dict[hyp_key]) - rst["Wrd"] += out_item["nwords"] - rst["Corr"] += out_item["cor"] - rst["wrong_words"] += out_item["wrong"] - rst["Ins"] += out_item["ins"] - rst["Del"] += out_item["del"] - rst["Sub"] += out_item["sub"] - rst["Snt"] += 1 - if out_item["wrong"] > 0: - rst["wrong_sentences"] += 1 - cer_detail_writer.write(hyp_key + print_cer_detail(out_item) + "\n") - cer_detail_writer.write( - "ref:" - + "\t" - + " ".join(list(map(lambda x: x.lower(), ref_dict[hyp_key]))) - + "\n" - ) - cer_detail_writer.write( - "hyp:" - + "\t" - + " ".join(list(map(lambda x: x.lower(), hyp_dict[hyp_key]))) - + "\n" - ) - - if rst["Wrd"] > 0: - rst["Err"] = round(rst["wrong_words"] * 100 / rst["Wrd"], 2) - if rst["Snt"] > 0: - rst["S.Err"] = round(rst["wrong_sentences"] * 100 / rst["Snt"], 2) - - cer_detail_writer.write("\n") - cer_detail_writer.write( - "%WER " - + str(rst["Err"]) - + " [ " - + str(rst["wrong_words"]) - + " / " - + str(rst["Wrd"]) - + ", " - + str(rst["Ins"]) - + " ins, " - + str(rst["Del"]) - + " del, " - + str(rst["Sub"]) - + " sub ]" - + "\n" - ) - cer_detail_writer.write( - "%SER " - + str(rst["S.Err"]) - + " [ " - + str(rst["wrong_sentences"]) - + " / " - + str(rst["Snt"]) - + " ]" - + "\n" - ) - cer_detail_writer.write( - "Scored " - + str(len(hyp_dict)) - + " sentences, " - + str(len(hyp_dict) - rst["Snt"]) - + " not present in hyp." - + "\n" - ) - - -def compute_wer_by_line(hyp, ref): - hyp = list(map(lambda x: x.lower(), hyp)) - ref = list(map(lambda x: x.lower(), ref)) - - len_hyp = len(hyp) - len_ref = len(ref) - - cost_matrix = np.zeros((len_hyp + 1, len_ref + 1), dtype=np.int16) - - ops_matrix = np.zeros((len_hyp + 1, len_ref + 1), dtype=np.int8) - - for i in range(len_hyp + 1): - cost_matrix[i][0] = i - for j in range(len_ref + 1): - cost_matrix[0][j] = j - - for i in range(1, len_hyp + 1): - for j in range(1, len_ref + 1): - if hyp[i - 1] == ref[j - 1]: - cost_matrix[i][j] = cost_matrix[i - 1][j - 1] - else: - substitution = cost_matrix[i - 1][j - 1] + 1 - insertion = cost_matrix[i - 1][j] + 1 - deletion = cost_matrix[i][j - 1] + 1 - - compare_val = [substitution, insertion, deletion] - - min_val = min(compare_val) - operation_idx = compare_val.index(min_val) + 1 - cost_matrix[i][j] = min_val - ops_matrix[i][j] = operation_idx - - match_idx = [] - i = len_hyp - j = len_ref - rst = {"nwords": len_ref, "cor": 0, "wrong": 0, "ins": 0, "del": 0, "sub": 0} - while i >= 0 or j >= 0: - i_idx = max(0, i) - j_idx = max(0, j) - - if ops_matrix[i_idx][j_idx] == 0: # correct - if i - 1 >= 0 and j - 1 >= 0: - match_idx.append((j - 1, i - 1)) - rst["cor"] += 1 - - i -= 1 - j -= 1 - - elif ops_matrix[i_idx][j_idx] == 2: # insert - i -= 1 - rst["ins"] += 1 - - elif ops_matrix[i_idx][j_idx] == 3: # delete - j -= 1 - rst["del"] += 1 - - elif ops_matrix[i_idx][j_idx] == 1: # substitute - i -= 1 - j -= 1 - rst["sub"] += 1 - - if i < 0 and j >= 0: - rst["del"] += 1 - elif j < 0 and i >= 0: - rst["ins"] += 1 - - match_idx.reverse() - wrong_cnt = cost_matrix[len_hyp][len_ref] - rst["wrong"] = wrong_cnt - - return rst - - -def print_cer_detail(rst): - return ( - "(" - + "nwords=" - + str(rst["nwords"]) - + ",cor=" - + str(rst["cor"]) - + ",ins=" - + str(rst["ins"]) - + ",del=" - + str(rst["del"]) - + ",sub=" - + str(rst["sub"]) - + ") corr:" - + "{:.2%}".format(rst["cor"] / rst["nwords"]) - + ",cer:" - + "{:.2%}".format(rst["wrong"] / rst["nwords"]) - ) - - -if __name__ == "__main__": - if len(sys.argv) != 4: - print("usage : python compute-wer.py test.ref test.hyp test.wer") - sys.exit(0) - - ref_file = sys.argv[1] - hyp_file = sys.argv[2] - cer_detail_file = sys.argv[3] - compute_wer(ref_file, hyp_file, cer_detail_file) diff --git a/funasr_detach/models/__init__.py b/funasr_detach/models/__init__.py deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/funasr_detach/models/bat/__init__.py b/funasr_detach/models/bat/__init__.py deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/funasr_detach/models/bat/model.py b/funasr_detach/models/bat/model.py deleted file mode 100644 index 2755fc2fccb4d0ed00f8e50976185a76f22084cb..0000000000000000000000000000000000000000 --- a/funasr_detach/models/bat/model.py +++ /dev/null @@ -1,37 +0,0 @@ -#!/usr/bin/env python3 -# -*- encoding: utf-8 -*- -# Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved. -# MIT License (https://opensource.org/licenses/MIT) - -import time -import torch -import logging -from contextlib import contextmanager -from typing import Dict, Optional, Tuple -from distutils.version import LooseVersion - -from funasr_detach.register import tables -from funasr_detach.utils import postprocess_utils -from funasr_detach.utils.datadir_writer import DatadirWriter -from funasr_detach.models.transducer.model import Transducer -from funasr_detach.train_utils.device_funcs import force_gatherable -from funasr_detach.models.transformer.scorers.ctc import CTCPrefixScorer -from funasr_detach.losses.label_smoothing_loss import LabelSmoothingLoss -from funasr_detach.models.transformer.scorers.length_bonus import LengthBonus -from funasr_detach.models.transformer.utils.nets_utils import get_transducer_task_io -from funasr_detach.utils.load_utils import load_audio_text_image_video, extract_fbank -from funasr_detach.models.transducer.beam_search_transducer import BeamSearchTransducer - - -if LooseVersion(torch.__version__) >= LooseVersion("1.6.0"): - from torch.cuda.amp import autocast -else: - # Nothing to do if torch<1.6.0 - @contextmanager - def autocast(enabled=True): - yield - - -@tables.register("model_classes", "BAT") # TODO: BAT training -class BAT(Transducer): - pass diff --git a/funasr_detach/models/bicif_paraformer/__init__.py b/funasr_detach/models/bicif_paraformer/__init__.py deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/funasr_detach/models/bicif_paraformer/cif_predictor.py b/funasr_detach/models/bicif_paraformer/cif_predictor.py deleted file mode 100644 index 3b0a229769e27c9002c34766c663bdcbb36c8f69..0000000000000000000000000000000000000000 --- a/funasr_detach/models/bicif_paraformer/cif_predictor.py +++ /dev/null @@ -1,417 +0,0 @@ -#!/usr/bin/env python3 -# -*- encoding: utf-8 -*- -# Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved. -# MIT License (https://opensource.org/licenses/MIT) - -import torch - -from funasr_detach.register import tables -from funasr_detach.models.transformer.utils.nets_utils import make_pad_mask - - -class mae_loss(torch.nn.Module): - - def __init__(self, normalize_length=False): - super(mae_loss, self).__init__() - self.normalize_length = normalize_length - self.criterion = torch.nn.L1Loss(reduction="sum") - - def forward(self, token_length, pre_token_length): - loss_token_normalizer = token_length.size(0) - if self.normalize_length: - loss_token_normalizer = token_length.sum().type(torch.float32) - loss = self.criterion(token_length, pre_token_length) - loss = loss / loss_token_normalizer - return loss - - -def cif(hidden, alphas, threshold): - batch_size, len_time, hidden_size = hidden.size() - - # loop varss - integrate = torch.zeros([batch_size], device=hidden.device) - frame = torch.zeros([batch_size, hidden_size], device=hidden.device) - # intermediate vars along time - list_fires = [] - list_frames = [] - - for t in range(len_time): - alpha = alphas[:, t] - distribution_completion = ( - torch.ones([batch_size], device=hidden.device) - integrate - ) - - integrate += alpha - list_fires.append(integrate) - - fire_place = integrate >= threshold - integrate = torch.where( - fire_place, - integrate - torch.ones([batch_size], device=hidden.device), - integrate, - ) - cur = torch.where(fire_place, distribution_completion, alpha) - remainds = alpha - cur - - frame += cur[:, None] * hidden[:, t, :] - list_frames.append(frame) - frame = torch.where( - fire_place[:, None].repeat(1, hidden_size), - remainds[:, None] * hidden[:, t, :], - frame, - ) - - fires = torch.stack(list_fires, 1) - frames = torch.stack(list_frames, 1) - list_ls = [] - len_labels = torch.round(alphas.sum(-1)).int() - max_label_len = len_labels.max() - for b in range(batch_size): - fire = fires[b, :] - l = torch.index_select( - frames[b, :, :], 0, torch.nonzero(fire >= threshold).squeeze() - ) - pad_l = torch.zeros( - [max_label_len - l.size(0), hidden_size], device=hidden.device - ) - list_ls.append(torch.cat([l, pad_l], 0)) - return torch.stack(list_ls, 0), fires - - -def cif_wo_hidden(alphas, threshold): - batch_size, len_time = alphas.size() - - # loop varss - integrate = torch.zeros([batch_size], device=alphas.device) - # intermediate vars along time - list_fires = [] - - for t in range(len_time): - alpha = alphas[:, t] - - integrate += alpha - list_fires.append(integrate) - - fire_place = integrate >= threshold - integrate = torch.where( - fire_place, - integrate - torch.ones([batch_size], device=alphas.device) * threshold, - integrate, - ) - - fires = torch.stack(list_fires, 1) - return fires - - -@tables.register("predictor_classes", "CifPredictorV3") -class CifPredictorV3(torch.nn.Module): - def __init__( - self, - idim, - l_order, - r_order, - threshold=1.0, - dropout=0.1, - smooth_factor=1.0, - noise_threshold=0, - tail_threshold=0.0, - tf2torch_tensor_name_prefix_torch="predictor", - tf2torch_tensor_name_prefix_tf="seq2seq/cif", - smooth_factor2=1.0, - noise_threshold2=0, - upsample_times=5, - upsample_type="cnn", - use_cif1_cnn=True, - tail_mask=True, - ): - super(CifPredictorV3, self).__init__() - - self.pad = torch.nn.ConstantPad1d((l_order, r_order), 0) - self.cif_conv1d = torch.nn.Conv1d(idim, idim, l_order + r_order + 1) - self.cif_output = torch.nn.Linear(idim, 1) - self.dropout = torch.nn.Dropout(p=dropout) - self.threshold = threshold - self.smooth_factor = smooth_factor - self.noise_threshold = noise_threshold - self.tail_threshold = tail_threshold - self.tf2torch_tensor_name_prefix_torch = tf2torch_tensor_name_prefix_torch - self.tf2torch_tensor_name_prefix_tf = tf2torch_tensor_name_prefix_tf - - self.upsample_times = upsample_times - self.upsample_type = upsample_type - self.use_cif1_cnn = use_cif1_cnn - if self.upsample_type == "cnn": - self.upsample_cnn = torch.nn.ConvTranspose1d( - idim, idim, self.upsample_times, self.upsample_times - ) - self.cif_output2 = torch.nn.Linear(idim, 1) - elif self.upsample_type == "cnn_blstm": - self.upsample_cnn = torch.nn.ConvTranspose1d( - idim, idim, self.upsample_times, self.upsample_times - ) - self.blstm = torch.nn.LSTM( - idim, - idim, - 1, - bias=True, - batch_first=True, - dropout=0.0, - bidirectional=True, - ) - self.cif_output2 = torch.nn.Linear(idim * 2, 1) - elif self.upsample_type == "cnn_attn": - self.upsample_cnn = torch.nn.ConvTranspose1d( - idim, idim, self.upsample_times, self.upsample_times - ) - from funasr_detach.models.transformer.encoder import ( - EncoderLayer as TransformerEncoderLayer, - ) - from funasr_detach.models.transformer.attention import MultiHeadedAttention - from funasr_detach.models.transformer.positionwise_feed_forward import ( - PositionwiseFeedForward, - ) - - positionwise_layer_args = ( - idim, - idim * 2, - 0.1, - ) - self.self_attn = TransformerEncoderLayer( - idim, - MultiHeadedAttention(4, idim, 0.1), - PositionwiseFeedForward(*positionwise_layer_args), - 0.1, - True, # normalize_before, - False, # concat_after, - ) - self.cif_output2 = torch.nn.Linear(idim, 1) - self.smooth_factor2 = smooth_factor2 - self.noise_threshold2 = noise_threshold2 - - def forward( - self, - hidden, - target_label=None, - mask=None, - ignore_id=-1, - mask_chunk_predictor=None, - target_label_length=None, - ): - h = hidden - context = h.transpose(1, 2) - queries = self.pad(context) - output = torch.relu(self.cif_conv1d(queries)) - - # alphas2 is an extra head for timestamp prediction - if not self.use_cif1_cnn: - _output = context - else: - _output = output - if self.upsample_type == "cnn": - output2 = self.upsample_cnn(_output) - output2 = output2.transpose(1, 2) - elif self.upsample_type == "cnn_blstm": - output2 = self.upsample_cnn(_output) - output2 = output2.transpose(1, 2) - output2, (_, _) = self.blstm(output2) - elif self.upsample_type == "cnn_attn": - output2 = self.upsample_cnn(_output) - output2 = output2.transpose(1, 2) - output2, _ = self.self_attn(output2, mask) - # import pdb; pdb.set_trace() - alphas2 = torch.sigmoid(self.cif_output2(output2)) - alphas2 = torch.nn.functional.relu( - alphas2 * self.smooth_factor2 - self.noise_threshold2 - ) - # repeat the mask in T demension to match the upsampled length - if mask is not None: - mask2 = ( - mask.repeat(1, self.upsample_times, 1) - .transpose(-1, -2) - .reshape(alphas2.shape[0], -1) - ) - mask2 = mask2.unsqueeze(-1) - alphas2 = alphas2 * mask2 - alphas2 = alphas2.squeeze(-1) - token_num2 = alphas2.sum(-1) - - output = output.transpose(1, 2) - - output = self.cif_output(output) - alphas = torch.sigmoid(output) - alphas = torch.nn.functional.relu( - alphas * self.smooth_factor - self.noise_threshold - ) - if mask is not None: - mask = mask.transpose(-1, -2).float() - alphas = alphas * mask - if mask_chunk_predictor is not None: - alphas = alphas * mask_chunk_predictor - alphas = alphas.squeeze(-1) - mask = mask.squeeze(-1) - if target_label_length is not None: - target_length = target_label_length - elif target_label is not None: - target_length = (target_label != ignore_id).float().sum(-1) - else: - target_length = None - token_num = alphas.sum(-1) - - if target_length is not None: - alphas *= (target_length / token_num)[:, None].repeat(1, alphas.size(1)) - elif self.tail_threshold > 0.0: - hidden, alphas, token_num = self.tail_process_fn( - hidden, alphas, token_num, mask=mask - ) - - acoustic_embeds, cif_peak = cif(hidden, alphas, self.threshold) - if target_length is None and self.tail_threshold > 0.0: - token_num_int = torch.max(token_num).type(torch.int32).item() - acoustic_embeds = acoustic_embeds[:, :token_num_int, :] - return acoustic_embeds, token_num, alphas, cif_peak, token_num2 - - def get_upsample_timestamp(self, hidden, mask=None, token_num=None): - h = hidden - b = hidden.shape[0] - context = h.transpose(1, 2) - queries = self.pad(context) - output = torch.relu(self.cif_conv1d(queries)) - - # alphas2 is an extra head for timestamp prediction - if not self.use_cif1_cnn: - _output = context - else: - _output = output - if self.upsample_type == "cnn": - output2 = self.upsample_cnn(_output) - output2 = output2.transpose(1, 2) - elif self.upsample_type == "cnn_blstm": - output2 = self.upsample_cnn(_output) - output2 = output2.transpose(1, 2) - output2, (_, _) = self.blstm(output2) - elif self.upsample_type == "cnn_attn": - output2 = self.upsample_cnn(_output) - output2 = output2.transpose(1, 2) - output2, _ = self.self_attn(output2, mask) - alphas2 = torch.sigmoid(self.cif_output2(output2)) - alphas2 = torch.nn.functional.relu( - alphas2 * self.smooth_factor2 - self.noise_threshold2 - ) - # repeat the mask in T demension to match the upsampled length - if mask is not None: - mask2 = ( - mask.repeat(1, self.upsample_times, 1) - .transpose(-1, -2) - .reshape(alphas2.shape[0], -1) - ) - mask2 = mask2.unsqueeze(-1) - alphas2 = alphas2 * mask2 - alphas2 = alphas2.squeeze(-1) - _token_num = alphas2.sum(-1) - if token_num is not None: - alphas2 *= (token_num / _token_num)[:, None].repeat(1, alphas2.size(1)) - # re-downsample - ds_alphas = alphas2.reshape(b, -1, self.upsample_times).sum(-1) - ds_cif_peak = cif_wo_hidden(ds_alphas, self.threshold - 1e-4) - # upsampled alphas and cif_peak - us_alphas = alphas2 - us_cif_peak = cif_wo_hidden(us_alphas, self.threshold - 1e-4) - return ds_alphas, ds_cif_peak, us_alphas, us_cif_peak - - def tail_process_fn(self, hidden, alphas, token_num=None, mask=None): - b, t, d = hidden.size() - tail_threshold = self.tail_threshold - if mask is not None: - zeros_t = torch.zeros((b, 1), dtype=torch.float32, device=alphas.device) - ones_t = torch.ones_like(zeros_t) - mask_1 = torch.cat([mask, zeros_t], dim=1) - mask_2 = torch.cat([ones_t, mask], dim=1) - mask = mask_2 - mask_1 - tail_threshold = mask * tail_threshold - alphas = torch.cat([alphas, zeros_t], dim=1) - alphas = torch.add(alphas, tail_threshold) - else: - tail_threshold = torch.tensor([tail_threshold], dtype=alphas.dtype).to( - alphas.device - ) - tail_threshold = torch.reshape(tail_threshold, (1, 1)) - alphas = torch.cat([alphas, tail_threshold], dim=1) - zeros = torch.zeros((b, 1, d), dtype=hidden.dtype).to(hidden.device) - hidden = torch.cat([hidden, zeros], dim=1) - token_num = alphas.sum(dim=-1) - token_num_floor = torch.floor(token_num) - - return hidden, alphas, token_num_floor - - def gen_frame_alignments( - self, alphas: torch.Tensor = None, encoder_sequence_length: torch.Tensor = None - ): - batch_size, maximum_length = alphas.size() - int_type = torch.int32 - - is_training = self.training - if is_training: - token_num = torch.round(torch.sum(alphas, dim=1)).type(int_type) - else: - token_num = torch.floor(torch.sum(alphas, dim=1)).type(int_type) - - max_token_num = torch.max(token_num).item() - - alphas_cumsum = torch.cumsum(alphas, dim=1) - alphas_cumsum = torch.floor(alphas_cumsum).type(int_type) - alphas_cumsum = alphas_cumsum[:, None, :].repeat(1, max_token_num, 1) - - index = torch.ones([batch_size, max_token_num], dtype=int_type) - index = torch.cumsum(index, dim=1) - index = index[:, :, None].repeat(1, 1, maximum_length).to(alphas_cumsum.device) - - index_div = torch.floor(torch.true_divide(alphas_cumsum, index)).type(int_type) - index_div_bool_zeros = index_div.eq(0) - index_div_bool_zeros_count = torch.sum(index_div_bool_zeros, dim=-1) + 1 - index_div_bool_zeros_count = torch.clamp( - index_div_bool_zeros_count, 0, encoder_sequence_length.max() - ) - token_num_mask = (~make_pad_mask(token_num, maxlen=max_token_num)).to( - token_num.device - ) - index_div_bool_zeros_count *= token_num_mask - - index_div_bool_zeros_count_tile = index_div_bool_zeros_count[:, :, None].repeat( - 1, 1, maximum_length - ) - ones = torch.ones_like(index_div_bool_zeros_count_tile) - zeros = torch.zeros_like(index_div_bool_zeros_count_tile) - ones = torch.cumsum(ones, dim=2) - cond = index_div_bool_zeros_count_tile == ones - index_div_bool_zeros_count_tile = torch.where(cond, zeros, ones) - - index_div_bool_zeros_count_tile_bool = index_div_bool_zeros_count_tile.type( - torch.bool - ) - index_div_bool_zeros_count_tile = 1 - index_div_bool_zeros_count_tile_bool.type( - int_type - ) - index_div_bool_zeros_count_tile_out = torch.sum( - index_div_bool_zeros_count_tile, dim=1 - ) - index_div_bool_zeros_count_tile_out = index_div_bool_zeros_count_tile_out.type( - int_type - ) - predictor_mask = ( - ( - ~make_pad_mask( - encoder_sequence_length, maxlen=encoder_sequence_length.max() - ) - ) - .type(int_type) - .to(encoder_sequence_length.device) - ) - index_div_bool_zeros_count_tile_out = ( - index_div_bool_zeros_count_tile_out * predictor_mask - ) - - predictor_alignments = index_div_bool_zeros_count_tile_out - predictor_alignments_length = predictor_alignments.sum(-1).type( - encoder_sequence_length.dtype - ) - return predictor_alignments.detach(), predictor_alignments_length.detach() diff --git a/funasr_detach/models/bicif_paraformer/model.py b/funasr_detach/models/bicif_paraformer/model.py deleted file mode 100644 index cbf078c2896ac16c13fd402f0573ee09f22ab351..0000000000000000000000000000000000000000 --- a/funasr_detach/models/bicif_paraformer/model.py +++ /dev/null @@ -1,395 +0,0 @@ -#!/usr/bin/env python3 -# -*- encoding: utf-8 -*- -# Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved. -# MIT License (https://opensource.org/licenses/MIT) - -import copy -import time -import torch -import logging -from contextlib import contextmanager -from distutils.version import LooseVersion -from typing import Dict, List, Optional, Tuple - -from funasr_detach.register import tables -from funasr_detach.models.ctc.ctc import CTC -from funasr_detach.utils import postprocess_utils -from funasr_detach.metrics.compute_acc import th_accuracy -from funasr_detach.utils.datadir_writer import DatadirWriter -from funasr_detach.models.paraformer.model import Paraformer -from funasr_detach.models.paraformer.search import Hypothesis -from funasr_detach.train_utils.device_funcs import force_gatherable -from funasr_detach.models.transformer.utils.add_sos_eos import add_sos_eos -from funasr_detach.utils.timestamp_tools import ts_prediction_lfr6_standard -from funasr_detach.models.transformer.utils.nets_utils import make_pad_mask, pad_list -from funasr_detach.utils.load_utils import load_audio_text_image_video, extract_fbank - - -if LooseVersion(torch.__version__) >= LooseVersion("1.6.0"): - from torch.cuda.amp import autocast -else: - # Nothing to do if torch<1.6.0 - @contextmanager - def autocast(enabled=True): - yield - - -@tables.register("model_classes", "BiCifParaformer") -class BiCifParaformer(Paraformer): - """ - Author: Speech Lab of DAMO Academy, Alibaba Group - Paper1: FunASR: A Fundamental End-to-End Speech Recognition Toolkit - https://arxiv.org/abs/2305.11013 - Paper2: Achieving timestamp prediction while recognizing with non-autoregressive end-to-end ASR model - https://arxiv.org/abs/2301.12343 - """ - - def __init__( - self, - *args, - **kwargs, - ): - super().__init__(*args, **kwargs) - - def _calc_pre2_loss( - self, - encoder_out: torch.Tensor, - encoder_out_lens: torch.Tensor, - ys_pad: torch.Tensor, - ys_pad_lens: torch.Tensor, - ): - encoder_out_mask = ( - ~make_pad_mask(encoder_out_lens, maxlen=encoder_out.size(1))[:, None, :] - ).to(encoder_out.device) - if self.predictor_bias == 1: - _, ys_pad = add_sos_eos(ys_pad, self.sos, self.eos, self.ignore_id) - ys_pad_lens = ys_pad_lens + self.predictor_bias - _, _, _, _, pre_token_length2 = self.predictor( - encoder_out, ys_pad, encoder_out_mask, ignore_id=self.ignore_id - ) - - # loss_pre = self.criterion_pre(ys_pad_lens.type_as(pre_token_length), pre_token_length) - loss_pre2 = self.criterion_pre( - ys_pad_lens.type_as(pre_token_length2), pre_token_length2 - ) - - return loss_pre2 - - def _calc_att_loss( - self, - encoder_out: torch.Tensor, - encoder_out_lens: torch.Tensor, - ys_pad: torch.Tensor, - ys_pad_lens: torch.Tensor, - ): - encoder_out_mask = ( - ~make_pad_mask(encoder_out_lens, maxlen=encoder_out.size(1))[:, None, :] - ).to(encoder_out.device) - if self.predictor_bias == 1: - _, ys_pad = add_sos_eos(ys_pad, self.sos, self.eos, self.ignore_id) - ys_pad_lens = ys_pad_lens + self.predictor_bias - pre_acoustic_embeds, pre_token_length, _, pre_peak_index, _ = self.predictor( - encoder_out, ys_pad, encoder_out_mask, ignore_id=self.ignore_id - ) - - # 0. sampler - decoder_out_1st = None - if self.sampling_ratio > 0.0: - sematic_embeds, decoder_out_1st = self.sampler( - encoder_out, encoder_out_lens, ys_pad, ys_pad_lens, pre_acoustic_embeds - ) - else: - sematic_embeds = pre_acoustic_embeds - - # 1. Forward decoder - decoder_outs = self.decoder( - encoder_out, encoder_out_lens, sematic_embeds, ys_pad_lens - ) - decoder_out, _ = decoder_outs[0], decoder_outs[1] - - if decoder_out_1st is None: - decoder_out_1st = decoder_out - # 2. Compute attention loss - loss_att = self.criterion_att(decoder_out, ys_pad) - acc_att = th_accuracy( - decoder_out_1st.view(-1, self.vocab_size), - ys_pad, - ignore_label=self.ignore_id, - ) - loss_pre = self.criterion_pre( - ys_pad_lens.type_as(pre_token_length), pre_token_length - ) - - # Compute cer/wer using attention-decoder - if self.training or self.error_calculator is None: - cer_att, wer_att = None, None - else: - ys_hat = decoder_out_1st.argmax(dim=-1) - cer_att, wer_att = self.error_calculator(ys_hat.cpu(), ys_pad.cpu()) - - return loss_att, acc_att, cer_att, wer_att, loss_pre - - def calc_predictor(self, encoder_out, encoder_out_lens): - encoder_out_mask = ( - ~make_pad_mask(encoder_out_lens, maxlen=encoder_out.size(1))[:, None, :] - ).to(encoder_out.device) - ( - pre_acoustic_embeds, - pre_token_length, - alphas, - pre_peak_index, - pre_token_length2, - ) = self.predictor( - encoder_out, None, encoder_out_mask, ignore_id=self.ignore_id - ) - return pre_acoustic_embeds, pre_token_length, alphas, pre_peak_index - - def calc_predictor_timestamp(self, encoder_out, encoder_out_lens, token_num): - encoder_out_mask = ( - ~make_pad_mask(encoder_out_lens, maxlen=encoder_out.size(1))[:, None, :] - ).to(encoder_out.device) - ds_alphas, ds_cif_peak, us_alphas, us_peaks = ( - self.predictor.get_upsample_timestamp( - encoder_out, encoder_out_mask, token_num - ) - ) - return ds_alphas, ds_cif_peak, us_alphas, us_peaks - - def forward( - self, - speech: torch.Tensor, - speech_lengths: torch.Tensor, - text: torch.Tensor, - text_lengths: torch.Tensor, - **kwargs, - ) -> Tuple[torch.Tensor, Dict[str, torch.Tensor], torch.Tensor]: - """Frontend + Encoder + Decoder + Calc loss - Args: - speech: (Batch, Length, ...) - speech_lengths: (Batch, ) - text: (Batch, Length) - text_lengths: (Batch,) - """ - if len(text_lengths.size()) > 1: - text_lengths = text_lengths[:, 0] - if len(speech_lengths.size()) > 1: - speech_lengths = speech_lengths[:, 0] - - batch_size = speech.shape[0] - - # Encoder - encoder_out, encoder_out_lens = self.encode(speech, speech_lengths) - - loss_ctc, cer_ctc = None, None - loss_pre = None - stats = dict() - - # decoder: CTC branch - if self.ctc_weight != 0.0: - loss_ctc, cer_ctc = self._calc_ctc_loss( - encoder_out, encoder_out_lens, text, text_lengths - ) - - # Collect CTC branch stats - stats["loss_ctc"] = loss_ctc.detach() if loss_ctc is not None else None - stats["cer_ctc"] = cer_ctc - - # decoder: Attention decoder branch - loss_att, acc_att, cer_att, wer_att, loss_pre = self._calc_att_loss( - encoder_out, encoder_out_lens, text, text_lengths - ) - - loss_pre2 = self._calc_pre2_loss( - encoder_out, encoder_out_lens, text, text_lengths - ) - - # 3. CTC-Att loss definition - if self.ctc_weight == 0.0: - loss = ( - loss_att - + loss_pre * self.predictor_weight - + loss_pre2 * self.predictor_weight * 0.5 - ) - else: - loss = ( - self.ctc_weight * loss_ctc - + (1 - self.ctc_weight) * loss_att - + loss_pre * self.predictor_weight - + loss_pre2 * self.predictor_weight * 0.5 - ) - - # Collect Attn branch stats - stats["loss_att"] = loss_att.detach() if loss_att is not None else None - stats["acc"] = acc_att - stats["cer"] = cer_att - stats["wer"] = wer_att - stats["loss_pre"] = loss_pre.detach().cpu() if loss_pre is not None else None - stats["loss_pre2"] = loss_pre2.detach().cpu() - - stats["loss"] = torch.clone(loss.detach()) - - # force_gatherable: to-device and to-tensor if scalar for DataParallel - if self.length_normalized_loss: - batch_size = int((text_lengths + self.predictor_bias).sum()) - - loss, stats, weight = force_gatherable((loss, stats, batch_size), loss.device) - return loss, stats, weight - - def inference( - self, - data_in, - data_lengths=None, - key: list = None, - tokenizer=None, - frontend=None, - **kwargs, - ): - - # init beamsearch - is_use_ctc = ( - kwargs.get("decoding_ctc_weight", 0.0) > 0.00001 and self.ctc != None - ) - is_use_lm = ( - kwargs.get("lm_weight", 0.0) > 0.00001 - and kwargs.get("lm_file", None) is not None - ) - if self.beam_search is None and (is_use_lm or is_use_ctc): - logging.info("enable beam_search") - self.init_beam_search(**kwargs) - self.nbest = kwargs.get("nbest", 1) - - meta_data = {} - # if isinstance(data_in, torch.Tensor): # fbank - # speech, speech_lengths = data_in, data_lengths - # if len(speech.shape) < 3: - # speech = speech[None, :, :] - # if speech_lengths is None: - # speech_lengths = speech.shape[1] - # else: - # extract fbank feats - time1 = time.perf_counter() - audio_sample_list = load_audio_text_image_video( - data_in, fs=frontend.fs, audio_fs=kwargs.get("fs", 16000) - ) - time2 = time.perf_counter() - meta_data["load_data"] = f"{time2 - time1:0.3f}" - speech, speech_lengths = extract_fbank( - audio_sample_list, - data_type=kwargs.get("data_type", "sound"), - frontend=frontend, - ) - time3 = time.perf_counter() - meta_data["extract_feat"] = f"{time3 - time2:0.3f}" - meta_data["batch_data_time"] = ( - speech_lengths.sum().item() * frontend.frame_shift * frontend.lfr_n / 1000 - ) - - speech = speech.to(device=kwargs["device"]) - speech_lengths = speech_lengths.to(device=kwargs["device"]) - - # Encoder - encoder_out, encoder_out_lens = self.encode(speech, speech_lengths) - if isinstance(encoder_out, tuple): - encoder_out = encoder_out[0] - - # predictor - predictor_outs = self.calc_predictor(encoder_out, encoder_out_lens) - pre_acoustic_embeds, pre_token_length, alphas, pre_peak_index = ( - predictor_outs[0], - predictor_outs[1], - predictor_outs[2], - predictor_outs[3], - ) - pre_token_length = pre_token_length.round().long() - if torch.max(pre_token_length) < 1: - return [] - decoder_outs = self.cal_decoder_with_predictor( - encoder_out, encoder_out_lens, pre_acoustic_embeds, pre_token_length - ) - decoder_out, ys_pad_lens = decoder_outs[0], decoder_outs[1] - - # BiCifParaformer, test no bias cif2 - _, _, us_alphas, us_peaks = self.calc_predictor_timestamp( - encoder_out, encoder_out_lens, pre_token_length - ) - - results = [] - b, n, d = decoder_out.size() - for i in range(b): - x = encoder_out[i, : encoder_out_lens[i], :] - am_scores = decoder_out[i, : pre_token_length[i], :] - if self.beam_search is not None: - nbest_hyps = self.beam_search( - x=x, - am_scores=am_scores, - maxlenratio=kwargs.get("maxlenratio", 0.0), - minlenratio=kwargs.get("minlenratio", 0.0), - ) - - nbest_hyps = nbest_hyps[: self.nbest] - else: - - yseq = am_scores.argmax(dim=-1) - score = am_scores.max(dim=-1)[0] - score = torch.sum(score, dim=-1) - # pad with mask tokens to ensure compatibility with sos/eos tokens - yseq = torch.tensor( - [self.sos] + yseq.tolist() + [self.eos], device=yseq.device - ) - nbest_hyps = [Hypothesis(yseq=yseq, score=score)] - for nbest_idx, hyp in enumerate(nbest_hyps): - ibest_writer = None - if kwargs.get("output_dir") is not None: - if not hasattr(self, "writer"): - self.writer = DatadirWriter(kwargs.get("output_dir")) - ibest_writer = self.writer[f"{nbest_idx+1}best_recog"] - - # remove sos/eos and get results - last_pos = -1 - if isinstance(hyp.yseq, list): - token_int = hyp.yseq[1:last_pos] - else: - token_int = hyp.yseq[1:last_pos].tolist() - - # remove blank symbol id, which is assumed to be 0 - token_int = list( - filter( - lambda x: x != self.eos - and x != self.sos - and x != self.blank_id, - token_int, - ) - ) - - if tokenizer is not None: - # Change integer-ids to tokens - token = tokenizer.ids2tokens(token_int) - text = tokenizer.tokens2text(token) - - _, timestamp = ts_prediction_lfr6_standard( - us_alphas[i][: encoder_out_lens[i] * 3], - us_peaks[i][: encoder_out_lens[i] * 3], - copy.copy(token), - vad_offset=kwargs.get("begin_time", 0), - ) - - text_postprocessed, time_stamp_postprocessed, word_lists = ( - postprocess_utils.sentence_postprocess(token, timestamp) - ) - - result_i = { - "key": key[i], - "text": text_postprocessed, - "timestamp": time_stamp_postprocessed, - } - - if ibest_writer is not None: - ibest_writer["token"][key[i]] = " ".join(token) - # ibest_writer["text"][key[i]] = text - ibest_writer["timestamp"][key[i]] = time_stamp_postprocessed - ibest_writer["text"][key[i]] = text_postprocessed - else: - result_i = {"key": key[i], "token_int": token_int} - results.append(result_i) - - return results, meta_data diff --git a/funasr_detach/models/bicif_paraformer/template.yaml b/funasr_detach/models/bicif_paraformer/template.yaml deleted file mode 100644 index d2b0e0a48472ae4ea2c3cfb27f2acd5ffcbb0b29..0000000000000000000000000000000000000000 --- a/funasr_detach/models/bicif_paraformer/template.yaml +++ /dev/null @@ -1,134 +0,0 @@ -# This is an example that demonstrates how to configure a model file. -# You can modify the configuration according to your own requirements. - -# to print the register_table: -# from funasr.register import tables -# tables.print() - -# network architecture -#model: funasr.models.paraformer.model:Paraformer -model: BiCifParaformer -model_conf: - ctc_weight: 0.0 - lsm_weight: 0.1 - length_normalized_loss: true - predictor_weight: 1.0 - predictor_bias: 1 - sampling_ratio: 0.75 - -# encoder -encoder: SANMEncoder -encoder_conf: - output_size: 512 - attention_heads: 4 - linear_units: 2048 - num_blocks: 50 - dropout_rate: 0.1 - positional_dropout_rate: 0.1 - attention_dropout_rate: 0.1 - input_layer: pe - pos_enc_class: SinusoidalPositionEncoder - normalize_before: true - kernel_size: 11 - sanm_shfit: 0 - selfattention_layer_type: sanm - -# decoder -decoder: ParaformerSANMDecoder -decoder_conf: - attention_heads: 4 - linear_units: 2048 - num_blocks: 16 - dropout_rate: 0.1 - positional_dropout_rate: 0.1 - self_attention_dropout_rate: 0.1 - src_attention_dropout_rate: 0.1 - att_layer_num: 16 - kernel_size: 11 - sanm_shfit: 0 - -predictor: CifPredictorV3 -predictor_conf: - idim: 512 - threshold: 1.0 - l_order: 1 - r_order: 1 - tail_threshold: 0.45 - smooth_factor2: 0.25 - noise_threshold2: 0.01 - upsample_times: 3 - use_cif1_cnn: false - upsample_type: cnn_blstm - -# frontend related -frontend: WavFrontend -frontend_conf: - fs: 16000 - window: hamming - n_mels: 80 - frame_length: 25 - frame_shift: 10 - lfr_m: 7 - lfr_n: 6 - -specaug: SpecAugLFR -specaug_conf: - apply_time_warp: false - time_warp_window: 5 - time_warp_mode: bicubic - apply_freq_mask: true - freq_mask_width_range: - - 0 - - 30 - lfr_rate: 6 - num_freq_mask: 1 - apply_time_mask: true - time_mask_width_range: - - 0 - - 12 - num_time_mask: 1 - -train_conf: - accum_grad: 1 - grad_clip: 5 - max_epoch: 150 - val_scheduler_criterion: - - valid - - acc - best_model_criterion: - - - valid - - acc - - max - keep_nbest_models: 10 - log_interval: 50 - -optim: adam -optim_conf: - lr: 0.0005 -scheduler: warmuplr -scheduler_conf: - warmup_steps: 30000 - -dataset: AudioDataset -dataset_conf: - index_ds: IndexDSJsonl - batch_sampler: DynamicBatchLocalShuffleSampler - batch_type: example # example or length - batch_size: 1 # if batch_type is example, batch_size is the numbers of samples; if length, batch_size is source_token_len+target_token_len; - max_token_length: 2048 # filter samples if source_token_len+target_token_len > max_token_length, - buffer_size: 500 - shuffle: True - num_workers: 0 - -tokenizer: CharTokenizer -tokenizer_conf: - unk_symbol: - split_with_space: true - - -ctc_conf: - dropout_rate: 0.0 - ctc_type: builtin - reduce: true - ignore_nan_grad: true -normalize: null diff --git a/funasr_detach/models/branchformer/__init__.py b/funasr_detach/models/branchformer/__init__.py deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/funasr_detach/models/branchformer/cgmlp.py b/funasr_detach/models/branchformer/cgmlp.py deleted file mode 100644 index 03f2d45c9d371411e98693f48d634d8151c65bb9..0000000000000000000000000000000000000000 --- a/funasr_detach/models/branchformer/cgmlp.py +++ /dev/null @@ -1,124 +0,0 @@ -"""MLP with convolutional gating (cgMLP) definition. - -References: - https://openreview.net/forum?id=RA-zVvZLYIy - https://arxiv.org/abs/2105.08050 - -""" - -import torch - -from funasr_detach.models.transformer.utils.nets_utils import get_activation -from funasr_detach.models.transformer.layer_norm import LayerNorm - - -class ConvolutionalSpatialGatingUnit(torch.nn.Module): - """Convolutional Spatial Gating Unit (CSGU).""" - - def __init__( - self, - size: int, - kernel_size: int, - dropout_rate: float, - use_linear_after_conv: bool, - gate_activation: str, - ): - super().__init__() - - n_channels = size // 2 # split input channels - self.norm = LayerNorm(n_channels) - self.conv = torch.nn.Conv1d( - n_channels, - n_channels, - kernel_size, - 1, - (kernel_size - 1) // 2, - groups=n_channels, - ) - if use_linear_after_conv: - self.linear = torch.nn.Linear(n_channels, n_channels) - else: - self.linear = None - - if gate_activation == "identity": - self.act = torch.nn.Identity() - else: - self.act = get_activation(gate_activation) - - self.dropout = torch.nn.Dropout(dropout_rate) - - def espnet_initialization_fn(self): - torch.nn.init.normal_(self.conv.weight, std=1e-6) - torch.nn.init.ones_(self.conv.bias) - if self.linear is not None: - torch.nn.init.normal_(self.linear.weight, std=1e-6) - torch.nn.init.ones_(self.linear.bias) - - def forward(self, x, gate_add=None): - """Forward method - - Args: - x (torch.Tensor): (N, T, D) - gate_add (torch.Tensor): (N, T, D/2) - - Returns: - out (torch.Tensor): (N, T, D/2) - """ - - x_r, x_g = x.chunk(2, dim=-1) - - x_g = self.norm(x_g) # (N, T, D/2) - x_g = self.conv(x_g.transpose(1, 2)).transpose(1, 2) # (N, T, D/2) - if self.linear is not None: - x_g = self.linear(x_g) - - if gate_add is not None: - x_g = x_g + gate_add - - x_g = self.act(x_g) - out = x_r * x_g # (N, T, D/2) - out = self.dropout(out) - return out - - -class ConvolutionalGatingMLP(torch.nn.Module): - """Convolutional Gating MLP (cgMLP).""" - - def __init__( - self, - size: int, - linear_units: int, - kernel_size: int, - dropout_rate: float, - use_linear_after_conv: bool, - gate_activation: str, - ): - super().__init__() - - self.channel_proj1 = torch.nn.Sequential( - torch.nn.Linear(size, linear_units), torch.nn.GELU() - ) - self.csgu = ConvolutionalSpatialGatingUnit( - size=linear_units, - kernel_size=kernel_size, - dropout_rate=dropout_rate, - use_linear_after_conv=use_linear_after_conv, - gate_activation=gate_activation, - ) - self.channel_proj2 = torch.nn.Linear(linear_units // 2, size) - - def forward(self, x, mask): - if isinstance(x, tuple): - xs_pad, pos_emb = x - else: - xs_pad, pos_emb = x, None - - xs_pad = self.channel_proj1(xs_pad) # size -> linear_units - xs_pad = self.csgu(xs_pad) # linear_units -> linear_units/2 - xs_pad = self.channel_proj2(xs_pad) # linear_units/2 -> size - - if pos_emb is not None: - out = (xs_pad, pos_emb) - else: - out = xs_pad - return out diff --git a/funasr_detach/models/branchformer/encoder.py b/funasr_detach/models/branchformer/encoder.py deleted file mode 100644 index 48afd292bb6e372a899da6e00212239f7abb16b9..0000000000000000000000000000000000000000 --- a/funasr_detach/models/branchformer/encoder.py +++ /dev/null @@ -1,550 +0,0 @@ -# Copyright 2022 Yifan Peng (Carnegie Mellon University) -# Apache 2.0 (http://www.apache.org/licenses/LICENSE-2.0) - -"""Branchformer encoder definition. - -Reference: - Yifan Peng, Siddharth Dalmia, Ian Lane, and Shinji Watanabe, - “Branchformer: Parallel MLP-Attention Architectures to Capture - Local and Global Context for Speech Recognition and Understanding,” - in Proceedings of ICML, 2022. - -""" - -import logging -from typing import List, Optional, Tuple, Union - -import numpy -import torch -import torch.nn as nn - -from funasr_detach.models.branchformer.cgmlp import ConvolutionalGatingMLP -from funasr_detach.models.branchformer.fastformer import FastSelfAttention -from funasr_detach.models.transformer.utils.nets_utils import make_pad_mask -from funasr_detach.models.transformer.attention import ( # noqa: H301 - LegacyRelPositionMultiHeadedAttention, - MultiHeadedAttention, - RelPositionMultiHeadedAttention, -) -from funasr_detach.models.transformer.embedding import ( # noqa: H301 - LegacyRelPositionalEncoding, - PositionalEncoding, - RelPositionalEncoding, - ScaledPositionalEncoding, -) -from funasr_detach.models.transformer.layer_norm import LayerNorm -from funasr_detach.models.transformer.utils.repeat import repeat -from funasr_detach.models.transformer.utils.subsampling import ( - Conv2dSubsampling, - Conv2dSubsampling2, - Conv2dSubsampling6, - Conv2dSubsampling8, - TooShortUttError, - check_short_utt, -) - -from funasr_detach.register import tables - - -class BranchformerEncoderLayer(torch.nn.Module): - """Branchformer encoder layer module. - - Args: - size (int): model dimension - attn: standard self-attention or efficient attention, optional - cgmlp: ConvolutionalGatingMLP, optional - dropout_rate (float): dropout probability - merge_method (str): concat, learned_ave, fixed_ave - cgmlp_weight (float): weight of the cgmlp branch, between 0 and 1, - used if merge_method is fixed_ave - attn_branch_drop_rate (float): probability of dropping the attn branch, - used if merge_method is learned_ave - stochastic_depth_rate (float): stochastic depth probability - """ - - def __init__( - self, - size: int, - attn: Optional[torch.nn.Module], - cgmlp: Optional[torch.nn.Module], - dropout_rate: float, - merge_method: str, - cgmlp_weight: float = 0.5, - attn_branch_drop_rate: float = 0.0, - stochastic_depth_rate: float = 0.0, - ): - super().__init__() - assert (attn is not None) or ( - cgmlp is not None - ), "At least one branch should be valid" - - self.size = size - self.attn = attn - self.cgmlp = cgmlp - self.merge_method = merge_method - self.cgmlp_weight = cgmlp_weight - self.attn_branch_drop_rate = attn_branch_drop_rate - self.stochastic_depth_rate = stochastic_depth_rate - self.use_two_branches = (attn is not None) and (cgmlp is not None) - - if attn is not None: - self.norm_mha = LayerNorm(size) # for the MHA module - if cgmlp is not None: - self.norm_mlp = LayerNorm(size) # for the MLP module - self.norm_final = LayerNorm(size) # for the final output of the block - - self.dropout = torch.nn.Dropout(dropout_rate) - - if self.use_two_branches: - if merge_method == "concat": - self.merge_proj = torch.nn.Linear(size + size, size) - - elif merge_method == "learned_ave": - # attention-based pooling for two branches - self.pooling_proj1 = torch.nn.Linear(size, 1) - self.pooling_proj2 = torch.nn.Linear(size, 1) - - # linear projections for calculating merging weights - self.weight_proj1 = torch.nn.Linear(size, 1) - self.weight_proj2 = torch.nn.Linear(size, 1) - - # linear projection after weighted average - self.merge_proj = torch.nn.Linear(size, size) - - elif merge_method == "fixed_ave": - assert ( - 0.0 <= cgmlp_weight <= 1.0 - ), "cgmlp weight should be between 0.0 and 1.0" - - # remove the other branch if only one branch is used - if cgmlp_weight == 0.0: - self.use_two_branches = False - self.cgmlp = None - self.norm_mlp = None - elif cgmlp_weight == 1.0: - self.use_two_branches = False - self.attn = None - self.norm_mha = None - - # linear projection after weighted average - self.merge_proj = torch.nn.Linear(size, size) - - else: - raise ValueError(f"unknown merge method: {merge_method}") - - else: - self.merge_proj = torch.nn.Identity() - - def forward(self, x_input, mask, cache=None): - """Compute encoded features. - - Args: - x_input (Union[Tuple, torch.Tensor]): Input tensor w/ or w/o pos emb. - - w/ pos emb: Tuple of tensors [(#batch, time, size), (1, time, size)]. - - w/o pos emb: Tensor (#batch, time, size). - mask (torch.Tensor): Mask tensor for the input (#batch, 1, time). - cache (torch.Tensor): Cache tensor of the input (#batch, time - 1, size). - - Returns: - torch.Tensor: Output tensor (#batch, time, size). - torch.Tensor: Mask tensor (#batch, time). - """ - - if cache is not None: - raise NotImplementedError("cache is not None, which is not tested") - - if isinstance(x_input, tuple): - x, pos_emb = x_input[0], x_input[1] - else: - x, pos_emb = x_input, None - - skip_layer = False - # with stochastic depth, residual connection `x + f(x)` becomes - # `x <- x + 1 / (1 - p) * f(x)` at training time. - stoch_layer_coeff = 1.0 - if self.training and self.stochastic_depth_rate > 0: - skip_layer = torch.rand(1).item() < self.stochastic_depth_rate - stoch_layer_coeff = 1.0 / (1 - self.stochastic_depth_rate) - - if skip_layer: - if cache is not None: - x = torch.cat([cache, x], dim=1) - if pos_emb is not None: - return (x, pos_emb), mask - return x, mask - - # Two branches - x1 = x - x2 = x - - # Branch 1: multi-headed attention module - if self.attn is not None: - x1 = self.norm_mha(x1) - - if isinstance(self.attn, FastSelfAttention): - x_att = self.attn(x1, mask) - else: - if pos_emb is not None: - x_att = self.attn(x1, x1, x1, pos_emb, mask) - else: - x_att = self.attn(x1, x1, x1, mask) - - x1 = self.dropout(x_att) - - # Branch 2: convolutional gating mlp - if self.cgmlp is not None: - x2 = self.norm_mlp(x2) - - if pos_emb is not None: - x2 = (x2, pos_emb) - x2 = self.cgmlp(x2, mask) - if isinstance(x2, tuple): - x2 = x2[0] - - x2 = self.dropout(x2) - - # Merge two branches - if self.use_two_branches: - if self.merge_method == "concat": - x = x + stoch_layer_coeff * self.dropout( - self.merge_proj(torch.cat([x1, x2], dim=-1)) - ) - elif self.merge_method == "learned_ave": - if ( - self.training - and self.attn_branch_drop_rate > 0 - and torch.rand(1).item() < self.attn_branch_drop_rate - ): - # Drop the attn branch - w1, w2 = 0.0, 1.0 - else: - # branch1 - score1 = ( - self.pooling_proj1(x1).transpose(1, 2) / self.size**0.5 - ) # (batch, 1, time) - if mask is not None: - min_value = float( - numpy.finfo( - torch.tensor(0, dtype=score1.dtype).numpy().dtype - ).min - ) - score1 = score1.masked_fill(mask.eq(0), min_value) - score1 = torch.softmax(score1, dim=-1).masked_fill( - mask.eq(0), 0.0 - ) - else: - score1 = torch.softmax(score1, dim=-1) - pooled1 = torch.matmul(score1, x1).squeeze(1) # (batch, size) - weight1 = self.weight_proj1(pooled1) # (batch, 1) - - # branch2 - score2 = ( - self.pooling_proj2(x2).transpose(1, 2) / self.size**0.5 - ) # (batch, 1, time) - if mask is not None: - min_value = float( - numpy.finfo( - torch.tensor(0, dtype=score2.dtype).numpy().dtype - ).min - ) - score2 = score2.masked_fill(mask.eq(0), min_value) - score2 = torch.softmax(score2, dim=-1).masked_fill( - mask.eq(0), 0.0 - ) - else: - score2 = torch.softmax(score2, dim=-1) - pooled2 = torch.matmul(score2, x2).squeeze(1) # (batch, size) - weight2 = self.weight_proj2(pooled2) # (batch, 1) - - # normalize weights of two branches - merge_weights = torch.softmax( - torch.cat([weight1, weight2], dim=-1), dim=-1 - ) # (batch, 2) - merge_weights = merge_weights.unsqueeze(-1).unsqueeze( - -1 - ) # (batch, 2, 1, 1) - w1, w2 = merge_weights[:, 0], merge_weights[:, 1] # (batch, 1, 1) - - x = x + stoch_layer_coeff * self.dropout( - self.merge_proj(w1 * x1 + w2 * x2) - ) - elif self.merge_method == "fixed_ave": - x = x + stoch_layer_coeff * self.dropout( - self.merge_proj( - (1.0 - self.cgmlp_weight) * x1 + self.cgmlp_weight * x2 - ) - ) - else: - raise RuntimeError(f"unknown merge method: {self.merge_method}") - else: - if self.attn is None: - x = x + stoch_layer_coeff * self.dropout(self.merge_proj(x2)) - elif self.cgmlp is None: - x = x + stoch_layer_coeff * self.dropout(self.merge_proj(x1)) - else: - # This should not happen - raise RuntimeError("Both branches are not None, which is unexpected.") - - x = self.norm_final(x) - - if pos_emb is not None: - return (x, pos_emb), mask - - return x, mask - - -@tables.register("encoder_classes", "BranchformerEncoder") -class BranchformerEncoder(nn.Module): - """Branchformer encoder module.""" - - def __init__( - self, - input_size: int, - output_size: int = 256, - use_attn: bool = True, - attention_heads: int = 4, - attention_layer_type: str = "rel_selfattn", - pos_enc_layer_type: str = "rel_pos", - rel_pos_type: str = "latest", - use_cgmlp: bool = True, - cgmlp_linear_units: int = 2048, - cgmlp_conv_kernel: int = 31, - use_linear_after_conv: bool = False, - gate_activation: str = "identity", - merge_method: str = "concat", - cgmlp_weight: Union[float, List[float]] = 0.5, - attn_branch_drop_rate: Union[float, List[float]] = 0.0, - num_blocks: int = 12, - dropout_rate: float = 0.1, - positional_dropout_rate: float = 0.1, - attention_dropout_rate: float = 0.0, - input_layer: Optional[str] = "conv2d", - zero_triu: bool = False, - padding_idx: int = -1, - stochastic_depth_rate: Union[float, List[float]] = 0.0, - ): - super().__init__() - self._output_size = output_size - - if rel_pos_type == "legacy": - if pos_enc_layer_type == "rel_pos": - pos_enc_layer_type = "legacy_rel_pos" - if attention_layer_type == "rel_selfattn": - attention_layer_type = "legacy_rel_selfattn" - elif rel_pos_type == "latest": - assert attention_layer_type != "legacy_rel_selfattn" - assert pos_enc_layer_type != "legacy_rel_pos" - else: - raise ValueError("unknown rel_pos_type: " + rel_pos_type) - - if pos_enc_layer_type == "abs_pos": - pos_enc_class = PositionalEncoding - elif pos_enc_layer_type == "scaled_abs_pos": - pos_enc_class = ScaledPositionalEncoding - elif pos_enc_layer_type == "rel_pos": - assert attention_layer_type == "rel_selfattn" - pos_enc_class = RelPositionalEncoding - elif pos_enc_layer_type == "legacy_rel_pos": - assert attention_layer_type == "legacy_rel_selfattn" - pos_enc_class = LegacyRelPositionalEncoding - logging.warning( - "Using legacy_rel_pos and it will be deprecated in the future." - ) - else: - raise ValueError("unknown pos_enc_layer: " + pos_enc_layer_type) - - if input_layer == "linear": - self.embed = torch.nn.Sequential( - torch.nn.Linear(input_size, output_size), - torch.nn.LayerNorm(output_size), - torch.nn.Dropout(dropout_rate), - pos_enc_class(output_size, positional_dropout_rate), - ) - elif input_layer == "conv2d": - self.embed = Conv2dSubsampling( - input_size, - output_size, - dropout_rate, - pos_enc_class(output_size, positional_dropout_rate), - ) - elif input_layer == "conv2d2": - self.embed = Conv2dSubsampling2( - input_size, - output_size, - dropout_rate, - pos_enc_class(output_size, positional_dropout_rate), - ) - elif input_layer == "conv2d6": - self.embed = Conv2dSubsampling6( - input_size, - output_size, - dropout_rate, - pos_enc_class(output_size, positional_dropout_rate), - ) - elif input_layer == "conv2d8": - self.embed = Conv2dSubsampling8( - input_size, - output_size, - dropout_rate, - pos_enc_class(output_size, positional_dropout_rate), - ) - elif input_layer == "embed": - self.embed = torch.nn.Sequential( - torch.nn.Embedding(input_size, output_size, padding_idx=padding_idx), - pos_enc_class(output_size, positional_dropout_rate), - ) - elif isinstance(input_layer, torch.nn.Module): - self.embed = torch.nn.Sequential( - input_layer, - pos_enc_class(output_size, positional_dropout_rate), - ) - elif input_layer is None: - if input_size == output_size: - self.embed = None - else: - self.embed = torch.nn.Linear(input_size, output_size) - else: - raise ValueError("unknown input_layer: " + input_layer) - - if attention_layer_type == "selfattn": - encoder_selfattn_layer = MultiHeadedAttention - encoder_selfattn_layer_args = ( - attention_heads, - output_size, - attention_dropout_rate, - ) - elif attention_layer_type == "legacy_rel_selfattn": - assert pos_enc_layer_type == "legacy_rel_pos" - encoder_selfattn_layer = LegacyRelPositionMultiHeadedAttention - encoder_selfattn_layer_args = ( - attention_heads, - output_size, - attention_dropout_rate, - ) - logging.warning( - "Using legacy_rel_selfattn and it will be deprecated in the future." - ) - elif attention_layer_type == "rel_selfattn": - assert pos_enc_layer_type == "rel_pos" - encoder_selfattn_layer = RelPositionMultiHeadedAttention - encoder_selfattn_layer_args = ( - attention_heads, - output_size, - attention_dropout_rate, - zero_triu, - ) - elif attention_layer_type == "fast_selfattn": - assert pos_enc_layer_type in ["abs_pos", "scaled_abs_pos"] - encoder_selfattn_layer = FastSelfAttention - encoder_selfattn_layer_args = ( - output_size, - attention_heads, - attention_dropout_rate, - ) - else: - raise ValueError("unknown encoder_attn_layer: " + attention_layer_type) - - cgmlp_layer = ConvolutionalGatingMLP - cgmlp_layer_args = ( - output_size, - cgmlp_linear_units, - cgmlp_conv_kernel, - dropout_rate, - use_linear_after_conv, - gate_activation, - ) - - if isinstance(stochastic_depth_rate, float): - stochastic_depth_rate = [stochastic_depth_rate] * num_blocks - if len(stochastic_depth_rate) != num_blocks: - raise ValueError( - f"Length of stochastic_depth_rate ({len(stochastic_depth_rate)}) " - f"should be equal to num_blocks ({num_blocks})" - ) - - if isinstance(cgmlp_weight, float): - cgmlp_weight = [cgmlp_weight] * num_blocks - if len(cgmlp_weight) != num_blocks: - raise ValueError( - f"Length of cgmlp_weight ({len(cgmlp_weight)}) should be equal to " - f"num_blocks ({num_blocks})" - ) - - if isinstance(attn_branch_drop_rate, float): - attn_branch_drop_rate = [attn_branch_drop_rate] * num_blocks - if len(attn_branch_drop_rate) != num_blocks: - raise ValueError( - f"Length of attn_branch_drop_rate ({len(attn_branch_drop_rate)}) " - f"should be equal to num_blocks ({num_blocks})" - ) - - self.encoders = repeat( - num_blocks, - lambda lnum: BranchformerEncoderLayer( - output_size, - ( - encoder_selfattn_layer(*encoder_selfattn_layer_args) - if use_attn - else None - ), - cgmlp_layer(*cgmlp_layer_args) if use_cgmlp else None, - dropout_rate, - merge_method, - cgmlp_weight[lnum], - attn_branch_drop_rate[lnum], - stochastic_depth_rate[lnum], - ), - ) - self.after_norm = LayerNorm(output_size) - - def output_size(self) -> int: - return self._output_size - - def forward( - self, - xs_pad: torch.Tensor, - ilens: torch.Tensor, - prev_states: torch.Tensor = None, - ) -> Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]: - """Calculate forward propagation. - - Args: - xs_pad (torch.Tensor): Input tensor (#batch, L, input_size). - ilens (torch.Tensor): Input length (#batch). - prev_states (torch.Tensor): Not to be used now. - - Returns: - torch.Tensor: Output tensor (#batch, L, output_size). - torch.Tensor: Output length (#batch). - torch.Tensor: Not to be used now. - - """ - - masks = (~make_pad_mask(ilens)[:, None, :]).to(xs_pad.device) - - if ( - isinstance(self.embed, Conv2dSubsampling) - or isinstance(self.embed, Conv2dSubsampling2) - or isinstance(self.embed, Conv2dSubsampling6) - or isinstance(self.embed, Conv2dSubsampling8) - ): - short_status, limit_size = check_short_utt(self.embed, xs_pad.size(1)) - if short_status: - raise TooShortUttError( - f"has {xs_pad.size(1)} frames and is too short for subsampling " - + f"(it needs more than {limit_size} frames), return empty results", - xs_pad.size(1), - limit_size, - ) - xs_pad, masks = self.embed(xs_pad, masks) - elif self.embed is not None: - xs_pad = self.embed(xs_pad) - - xs_pad, masks = self.encoders(xs_pad, masks) - - if isinstance(xs_pad, tuple): - xs_pad = xs_pad[0] - - xs_pad = self.after_norm(xs_pad) - olens = masks.squeeze(1).sum(1) - return xs_pad, olens, None diff --git a/funasr_detach/models/branchformer/fastformer.py b/funasr_detach/models/branchformer/fastformer.py deleted file mode 100644 index 24ca9475c15c3ef0d4a9611905b23b958472f083..0000000000000000000000000000000000000000 --- a/funasr_detach/models/branchformer/fastformer.py +++ /dev/null @@ -1,153 +0,0 @@ -"""Fastformer attention definition. - -Reference: - Wu et al., "Fastformer: Additive Attention Can Be All You Need" - https://arxiv.org/abs/2108.09084 - https://github.com/wuch15/Fastformer - -""" - -import numpy -import torch - - -class FastSelfAttention(torch.nn.Module): - """Fast self-attention used in Fastformer.""" - - def __init__( - self, - size, - attention_heads, - dropout_rate, - ): - super().__init__() - if size % attention_heads != 0: - raise ValueError( - f"Hidden size ({size}) is not an integer multiple " - f"of attention heads ({attention_heads})" - ) - self.attention_head_size = size // attention_heads - self.num_attention_heads = attention_heads - - self.query = torch.nn.Linear(size, size) - self.query_att = torch.nn.Linear(size, attention_heads) - self.key = torch.nn.Linear(size, size) - self.key_att = torch.nn.Linear(size, attention_heads) - self.transform = torch.nn.Linear(size, size) - self.dropout = torch.nn.Dropout(dropout_rate) - - def espnet_initialization_fn(self): - self.apply(self.init_weights) - - def init_weights(self, module): - if isinstance(module, torch.nn.Linear): - module.weight.data.normal_(mean=0.0, std=0.02) - if isinstance(module, torch.nn.Linear) and module.bias is not None: - module.bias.data.zero_() - - def transpose_for_scores(self, x): - """Reshape and transpose to compute scores. - - Args: - x: (batch, time, size = n_heads * attn_dim) - - Returns: - (batch, n_heads, time, attn_dim) - """ - - new_x_shape = x.shape[:-1] + ( - self.num_attention_heads, - self.attention_head_size, - ) - return x.reshape(*new_x_shape).transpose(1, 2) - - def forward(self, xs_pad, mask): - """Forward method. - - Args: - xs_pad: (batch, time, size = n_heads * attn_dim) - mask: (batch, 1, time), nonpadding is 1, padding is 0 - - Returns: - torch.Tensor: (batch, time, size) - """ - - batch_size, seq_len, _ = xs_pad.shape - mixed_query_layer = self.query(xs_pad) # (batch, time, size) - mixed_key_layer = self.key(xs_pad) # (batch, time, size) - - if mask is not None: - mask = mask.eq(0) # padding is 1, nonpadding is 0 - - # (batch, n_heads, time) - query_for_score = ( - self.query_att(mixed_query_layer).transpose(1, 2) - / self.attention_head_size**0.5 - ) - if mask is not None: - min_value = float( - numpy.finfo( - torch.tensor(0, dtype=query_for_score.dtype).numpy().dtype - ).min - ) - query_for_score = query_for_score.masked_fill(mask, min_value) - query_weight = torch.softmax(query_for_score, dim=-1).masked_fill(mask, 0.0) - else: - query_weight = torch.softmax(query_for_score, dim=-1) - - query_weight = query_weight.unsqueeze(2) # (batch, n_heads, 1, time) - query_layer = self.transpose_for_scores( - mixed_query_layer - ) # (batch, n_heads, time, attn_dim) - - pooled_query = ( - torch.matmul(query_weight, query_layer) - .transpose(1, 2) - .reshape(-1, 1, self.num_attention_heads * self.attention_head_size) - ) # (batch, 1, size = n_heads * attn_dim) - pooled_query = self.dropout(pooled_query) - pooled_query_repeat = pooled_query.repeat(1, seq_len, 1) # (batch, time, size) - - mixed_query_key_layer = ( - mixed_key_layer * pooled_query_repeat - ) # (batch, time, size) - - # (batch, n_heads, time) - query_key_score = ( - self.key_att(mixed_query_key_layer) / self.attention_head_size**0.5 - ).transpose(1, 2) - if mask is not None: - min_value = float( - numpy.finfo( - torch.tensor(0, dtype=query_key_score.dtype).numpy().dtype - ).min - ) - query_key_score = query_key_score.masked_fill(mask, min_value) - query_key_weight = torch.softmax(query_key_score, dim=-1).masked_fill( - mask, 0.0 - ) - else: - query_key_weight = torch.softmax(query_key_score, dim=-1) - - query_key_weight = query_key_weight.unsqueeze(2) # (batch, n_heads, 1, time) - key_layer = self.transpose_for_scores( - mixed_query_key_layer - ) # (batch, n_heads, time, attn_dim) - pooled_key = torch.matmul( - query_key_weight, key_layer - ) # (batch, n_heads, 1, attn_dim) - pooled_key = self.dropout(pooled_key) - - # NOTE: value = query, due to param sharing - weighted_value = (pooled_key * query_layer).transpose( - 1, 2 - ) # (batch, time, n_heads, attn_dim) - weighted_value = weighted_value.reshape( - weighted_value.shape[:-2] - + (self.num_attention_heads * self.attention_head_size,) - ) # (batch, time, size) - weighted_value = ( - self.dropout(self.transform(weighted_value)) + mixed_query_layer - ) - - return weighted_value diff --git a/funasr_detach/models/branchformer/model.py b/funasr_detach/models/branchformer/model.py deleted file mode 100644 index 8e031cf7d490ee33661c5d596e198b96178647da..0000000000000000000000000000000000000000 --- a/funasr_detach/models/branchformer/model.py +++ /dev/null @@ -1,17 +0,0 @@ -import logging - -from funasr_detach.models.transformer.model import Transformer -from funasr_detach.register import tables - - -@tables.register("model_classes", "Branchformer") -class Branchformer(Transformer): - """CTC-attention hybrid Encoder-Decoder model""" - - def __init__( - self, - *args, - **kwargs, - ): - - super().__init__(*args, **kwargs) diff --git a/funasr_detach/models/branchformer/template.yaml b/funasr_detach/models/branchformer/template.yaml deleted file mode 100644 index 09b1e0f4f8c43fe0deecd43ebab69c1443be09b1..0000000000000000000000000000000000000000 --- a/funasr_detach/models/branchformer/template.yaml +++ /dev/null @@ -1,116 +0,0 @@ -# This is an example that demonstrates how to configure a model file. -# You can modify the configuration according to your own requirements. - -# to print the register_table: -# from funasr.register import tables -# tables.print() - -# network architecture -model: Branchformer -model_conf: - ctc_weight: 0.3 - lsm_weight: 0.1 # label smoothing option - length_normalized_loss: false - -# encoder -encoder: BranchformerEncoder -encoder_conf: - output_size: 256 - use_attn: true - attention_heads: 4 - attention_layer_type: rel_selfattn - pos_enc_layer_type: rel_pos - rel_pos_type: latest - use_cgmlp: true - cgmlp_linear_units: 2048 - cgmlp_conv_kernel: 31 - use_linear_after_conv: false - gate_activation: identity - merge_method: concat - cgmlp_weight: 0.5 # used only if merge_method is "fixed_ave" - attn_branch_drop_rate: 0.0 # used only if merge_method is "learned_ave" - num_blocks: 24 - dropout_rate: 0.1 - positional_dropout_rate: 0.1 - attention_dropout_rate: 0.1 - input_layer: conv2d - stochastic_depth_rate: 0.0 - -# decoder -decoder: TransformerDecoder -decoder_conf: - attention_heads: 4 - linear_units: 2048 - num_blocks: 6 - dropout_rate: 0.1 - positional_dropout_rate: 0.1 - self_attention_dropout_rate: 0. - src_attention_dropout_rate: 0. - - -# frontend related -frontend: WavFrontend -frontend_conf: - fs: 16000 - window: hamming - n_mels: 80 - frame_length: 25 - frame_shift: 10 - dither: 0.0 - lfr_m: 1 - lfr_n: 1 - -specaug: SpecAug -specaug_conf: - apply_time_warp: true - time_warp_window: 5 - time_warp_mode: bicubic - apply_freq_mask: true - freq_mask_width_range: - - 0 - - 30 - num_freq_mask: 2 - apply_time_mask: true - time_mask_width_range: - - 0 - - 40 - num_time_mask: 2 - -train_conf: - accum_grad: 1 - grad_clip: 5 - max_epoch: 150 - keep_nbest_models: 10 - log_interval: 50 - -optim: adam -optim_conf: - lr: 0.001 - weight_decay: 0.000001 -scheduler: warmuplr -scheduler_conf: - warmup_steps: 35000 - -dataset: AudioDataset -dataset_conf: - index_ds: IndexDSJsonl - batch_sampler: DynamicBatchLocalShuffleSampler - batch_type: example # example or length - batch_size: 1 # if batch_type is example, batch_size is the numbers of samples; if length, batch_size is source_token_len+target_token_len; - max_token_length: 2048 # filter samples if source_token_len+target_token_len > max_token_length, - buffer_size: 500 - shuffle: True - num_workers: 4 - -tokenizer: CharTokenizer -tokenizer_conf: - unk_symbol: - split_with_space: true - - -ctc_conf: - dropout_rate: 0.0 - ctc_type: builtin - reduce: true - ignore_nan_grad: true -normalize: null diff --git a/funasr_detach/models/campplus/__init__.py b/funasr_detach/models/campplus/__init__.py deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/funasr_detach/models/campplus/cluster_backend.py b/funasr_detach/models/campplus/cluster_backend.py deleted file mode 100644 index 2506126a04aa1028022b93df961459a2050fe2dc..0000000000000000000000000000000000000000 --- a/funasr_detach/models/campplus/cluster_backend.py +++ /dev/null @@ -1,200 +0,0 @@ -#!/usr/bin/env python3 -# -*- encoding: utf-8 -*- -# Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved. -# MIT License (https://opensource.org/licenses/MIT) -# Modified from 3D-Speaker (https://github.com/alibaba-damo-academy/3D-Speaker) - -import scipy -import torch -import sklearn -import hdbscan -import numpy as np - -from sklearn.cluster._kmeans import k_means - - -class SpectralCluster: - r"""A spectral clustering mehtod using unnormalized Laplacian of affinity matrix. - This implementation is adapted from https://github.com/speechbrain/speechbrain. - """ - - def __init__(self, min_num_spks=1, max_num_spks=15, pval=0.022): - self.min_num_spks = min_num_spks - self.max_num_spks = max_num_spks - self.pval = pval - - def __call__(self, X, oracle_num=None): - # Similarity matrix computation - sim_mat = self.get_sim_mat(X) - - # Refining similarity matrix with pval - prunned_sim_mat = self.p_pruning(sim_mat) - - # Symmetrization - sym_prund_sim_mat = 0.5 * (prunned_sim_mat + prunned_sim_mat.T) - - # Laplacian calculation - laplacian = self.get_laplacian(sym_prund_sim_mat) - - # Get Spectral Embeddings - emb, num_of_spk = self.get_spec_embs(laplacian, oracle_num) - - # Perform clustering - labels = self.cluster_embs(emb, num_of_spk) - - return labels - - def get_sim_mat(self, X): - # Cosine similarities - M = sklearn.metrics.pairwise.cosine_similarity(X, X) - return M - - def p_pruning(self, A): - if A.shape[0] * self.pval < 6: - pval = 6.0 / A.shape[0] - else: - pval = self.pval - - n_elems = int((1 - pval) * A.shape[0]) - - # For each row in a affinity matrix - for i in range(A.shape[0]): - low_indexes = np.argsort(A[i, :]) - low_indexes = low_indexes[0:n_elems] - - # Replace smaller similarity values by 0s - A[i, low_indexes] = 0 - return A - - def get_laplacian(self, M): - M[np.diag_indices(M.shape[0])] = 0 - D = np.sum(np.abs(M), axis=1) - D = np.diag(D) - L = D - M - return L - - def get_spec_embs(self, L, k_oracle=None): - lambdas, eig_vecs = scipy.linalg.eigh(L) - - if k_oracle is not None: - num_of_spk = k_oracle - else: - lambda_gap_list = self.getEigenGaps( - lambdas[self.min_num_spks - 1 : self.max_num_spks + 1] - ) - num_of_spk = np.argmax(lambda_gap_list) + self.min_num_spks - - emb = eig_vecs[:, :num_of_spk] - return emb, num_of_spk - - def cluster_embs(self, emb, k): - _, labels, _ = k_means(emb, k) - return labels - - def getEigenGaps(self, eig_vals): - eig_vals_gap_list = [] - for i in range(len(eig_vals) - 1): - gap = float(eig_vals[i + 1]) - float(eig_vals[i]) - eig_vals_gap_list.append(gap) - return eig_vals_gap_list - - -class UmapHdbscan: - r""" - Reference: - - Siqi Zheng, Hongbin Suo. Reformulating Speaker Diarization as Community Detection With - Emphasis On Topological Structure. ICASSP2022 - """ - - def __init__( - self, - n_neighbors=20, - n_components=60, - min_samples=10, - min_cluster_size=10, - metric="cosine", - ): - self.n_neighbors = n_neighbors - self.n_components = n_components - self.min_samples = min_samples - self.min_cluster_size = min_cluster_size - self.metric = metric - - def __call__(self, X): - import umap.umap_ as umap - - umap_X = umap.UMAP( - n_neighbors=self.n_neighbors, - min_dist=0.0, - n_components=min(self.n_components, X.shape[0] - 2), - metric=self.metric, - ).fit_transform(X) - labels = hdbscan.HDBSCAN( - min_samples=self.min_samples, - min_cluster_size=self.min_cluster_size, - allow_single_cluster=True, - ).fit_predict(umap_X) - return labels - - -class ClusterBackend(torch.nn.Module): - r"""Perfom clustering for input embeddings and output the labels. - Args: - model_dir: A model dir. - model_config: The model config. - """ - - def __init__(self): - super().__init__() - self.model_config = {"merge_thr": 0.78} - # self.other_config = kwargs - - self.spectral_cluster = SpectralCluster() - self.umap_hdbscan_cluster = UmapHdbscan() - - def forward(self, X, **params): - # clustering and return the labels - k = params["oracle_num"] if "oracle_num" in params else None - assert ( - len(X.shape) == 2 - ), "modelscope error: the shape of input should be [N, C]" - if X.shape[0] < 20: - return np.zeros(X.shape[0], dtype="int") - if X.shape[0] < 2048 or k is not None: - # unexpected corner case - labels = self.spectral_cluster(X, k) - else: - labels = self.umap_hdbscan_cluster(X) - - if k is None and "merge_thr" in self.model_config: - labels = self.merge_by_cos(labels, X, self.model_config["merge_thr"]) - - return labels - - def merge_by_cos(self, labels, embs, cos_thr): - # merge the similar speakers by cosine similarity - assert cos_thr > 0 and cos_thr <= 1 - while True: - spk_num = labels.max() + 1 - if spk_num == 1: - break - spk_center = [] - for i in range(spk_num): - spk_emb = embs[labels == i].mean(0) - spk_center.append(spk_emb) - assert len(spk_center) > 0 - spk_center = np.stack(spk_center, axis=0) - norm_spk_center = spk_center / np.linalg.norm( - spk_center, axis=1, keepdims=True - ) - affinity = np.matmul(norm_spk_center, norm_spk_center.T) - affinity = np.triu(affinity, 1) - spks = np.unravel_index(np.argmax(affinity), affinity.shape) - if affinity[spks] < cos_thr: - break - for i in range(len(labels)): - if labels[i] == spks[1]: - labels[i] = spks[0] - elif labels[i] > spks[1]: - labels[i] -= 1 - return labels diff --git a/funasr_detach/models/campplus/components.py b/funasr_detach/models/campplus/components.py deleted file mode 100644 index 1f0daa13ff558db1e543b1b50669acf1977a443c..0000000000000000000000000000000000000000 --- a/funasr_detach/models/campplus/components.py +++ /dev/null @@ -1,310 +0,0 @@ -#!/usr/bin/env python3 -# -*- encoding: utf-8 -*- -# Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved. -# MIT License (https://opensource.org/licenses/MIT) -# Modified from 3D-Speaker (https://github.com/alibaba-damo-academy/3D-Speaker) - -import torch -import torch.nn.functional as F -import torch.utils.checkpoint as cp - - -class BasicResBlock(torch.nn.Module): - expansion = 1 - - def __init__(self, in_planes, planes, stride=1): - super(BasicResBlock, self).__init__() - self.conv1 = torch.nn.Conv2d( - in_planes, planes, kernel_size=3, stride=(stride, 1), padding=1, bias=False - ) - self.bn1 = torch.nn.BatchNorm2d(planes) - self.conv2 = torch.nn.Conv2d( - planes, planes, kernel_size=3, stride=1, padding=1, bias=False - ) - self.bn2 = torch.nn.BatchNorm2d(planes) - - self.shortcut = torch.nn.Sequential() - if stride != 1 or in_planes != self.expansion * planes: - self.shortcut = torch.nn.Sequential( - torch.nn.Conv2d( - in_planes, - self.expansion * planes, - kernel_size=1, - stride=(stride, 1), - bias=False, - ), - torch.nn.BatchNorm2d(self.expansion * planes), - ) - - def forward(self, x): - out = F.relu(self.bn1(self.conv1(x))) - out = self.bn2(self.conv2(out)) - out += self.shortcut(x) - out = F.relu(out) - return out - - -class FCM(torch.nn.Module): - def __init__( - self, block=BasicResBlock, num_blocks=[2, 2], m_channels=32, feat_dim=80 - ): - super(FCM, self).__init__() - self.in_planes = m_channels - self.conv1 = torch.nn.Conv2d( - 1, m_channels, kernel_size=3, stride=1, padding=1, bias=False - ) - self.bn1 = torch.nn.BatchNorm2d(m_channels) - - self.layer1 = self._make_layer(block, m_channels, num_blocks[0], stride=2) - self.layer2 = self._make_layer(block, m_channels, num_blocks[0], stride=2) - - self.conv2 = torch.nn.Conv2d( - m_channels, m_channels, kernel_size=3, stride=(2, 1), padding=1, bias=False - ) - self.bn2 = torch.nn.BatchNorm2d(m_channels) - self.out_channels = m_channels * (feat_dim // 8) - - def _make_layer(self, block, planes, num_blocks, stride): - strides = [stride] + [1] * (num_blocks - 1) - layers = [] - for stride in strides: - layers.append(block(self.in_planes, planes, stride)) - self.in_planes = planes * block.expansion - return torch.nn.Sequential(*layers) - - def forward(self, x): - x = x.unsqueeze(1) - out = F.relu(self.bn1(self.conv1(x))) - out = self.layer1(out) - out = self.layer2(out) - out = F.relu(self.bn2(self.conv2(out))) - - shape = out.shape - out = out.reshape(shape[0], shape[1] * shape[2], shape[3]) - return out - - -def get_nonlinear(config_str, channels): - nonlinear = torch.nn.Sequential() - for name in config_str.split("-"): - if name == "relu": - nonlinear.add_module("relu", torch.nn.ReLU(inplace=True)) - elif name == "prelu": - nonlinear.add_module("prelu", torch.nn.PReLU(channels)) - elif name == "batchnorm": - nonlinear.add_module("batchnorm", torch.nn.BatchNorm1d(channels)) - elif name == "batchnorm_": - nonlinear.add_module( - "batchnorm", torch.nn.BatchNorm1d(channels, affine=False) - ) - else: - raise ValueError("Unexpected module ({}).".format(name)) - return nonlinear - - -def statistics_pooling(x, dim=-1, keepdim=False, unbiased=True, eps=1e-2): - mean = x.mean(dim=dim) - std = x.std(dim=dim, unbiased=unbiased) - stats = torch.cat([mean, std], dim=-1) - if keepdim: - stats = stats.unsqueeze(dim=dim) - return stats - - -class StatsPool(torch.nn.Module): - def forward(self, x): - return statistics_pooling(x) - - -class TDNNLayer(torch.nn.Module): - def __init__( - self, - in_channels, - out_channels, - kernel_size, - stride=1, - padding=0, - dilation=1, - bias=False, - config_str="batchnorm-relu", - ): - super(TDNNLayer, self).__init__() - if padding < 0: - assert ( - kernel_size % 2 == 1 - ), "Expect equal paddings, but got even kernel size ({})".format( - kernel_size - ) - padding = (kernel_size - 1) // 2 * dilation - self.linear = torch.nn.Conv1d( - in_channels, - out_channels, - kernel_size, - stride=stride, - padding=padding, - dilation=dilation, - bias=bias, - ) - self.nonlinear = get_nonlinear(config_str, out_channels) - - def forward(self, x): - x = self.linear(x) - x = self.nonlinear(x) - return x - - -class CAMLayer(torch.nn.Module): - def __init__( - self, - bn_channels, - out_channels, - kernel_size, - stride, - padding, - dilation, - bias, - reduction=2, - ): - super(CAMLayer, self).__init__() - self.linear_local = torch.nn.Conv1d( - bn_channels, - out_channels, - kernel_size, - stride=stride, - padding=padding, - dilation=dilation, - bias=bias, - ) - self.linear1 = torch.nn.Conv1d(bn_channels, bn_channels // reduction, 1) - self.relu = torch.nn.ReLU(inplace=True) - self.linear2 = torch.nn.Conv1d(bn_channels // reduction, out_channels, 1) - self.sigmoid = torch.nn.Sigmoid() - - def forward(self, x): - y = self.linear_local(x) - context = x.mean(-1, keepdim=True) + self.seg_pooling(x) - context = self.relu(self.linear1(context)) - m = self.sigmoid(self.linear2(context)) - return y * m - - def seg_pooling(self, x, seg_len=100, stype="avg"): - if stype == "avg": - seg = F.avg_pool1d(x, kernel_size=seg_len, stride=seg_len, ceil_mode=True) - elif stype == "max": - seg = F.max_pool1d(x, kernel_size=seg_len, stride=seg_len, ceil_mode=True) - else: - raise ValueError("Wrong segment pooling type.") - shape = seg.shape - seg = seg.unsqueeze(-1).expand(*shape, seg_len).reshape(*shape[:-1], -1) - seg = seg[..., : x.shape[-1]] - return seg - - -class CAMDenseTDNNLayer(torch.nn.Module): - def __init__( - self, - in_channels, - out_channels, - bn_channels, - kernel_size, - stride=1, - dilation=1, - bias=False, - config_str="batchnorm-relu", - memory_efficient=False, - ): - super(CAMDenseTDNNLayer, self).__init__() - assert ( - kernel_size % 2 == 1 - ), "Expect equal paddings, but got even kernel size ({})".format(kernel_size) - padding = (kernel_size - 1) // 2 * dilation - self.memory_efficient = memory_efficient - self.nonlinear1 = get_nonlinear(config_str, in_channels) - self.linear1 = torch.nn.Conv1d(in_channels, bn_channels, 1, bias=False) - self.nonlinear2 = get_nonlinear(config_str, bn_channels) - self.cam_layer = CAMLayer( - bn_channels, - out_channels, - kernel_size, - stride=stride, - padding=padding, - dilation=dilation, - bias=bias, - ) - - def bn_function(self, x): - return self.linear1(self.nonlinear1(x)) - - def forward(self, x): - if self.training and self.memory_efficient: - x = cp.checkpoint(self.bn_function, x) - else: - x = self.bn_function(x) - x = self.cam_layer(self.nonlinear2(x)) - return x - - -class CAMDenseTDNNBlock(torch.nn.ModuleList): - def __init__( - self, - num_layers, - in_channels, - out_channels, - bn_channels, - kernel_size, - stride=1, - dilation=1, - bias=False, - config_str="batchnorm-relu", - memory_efficient=False, - ): - super(CAMDenseTDNNBlock, self).__init__() - for i in range(num_layers): - layer = CAMDenseTDNNLayer( - in_channels=in_channels + i * out_channels, - out_channels=out_channels, - bn_channels=bn_channels, - kernel_size=kernel_size, - stride=stride, - dilation=dilation, - bias=bias, - config_str=config_str, - memory_efficient=memory_efficient, - ) - self.add_module("tdnnd%d" % (i + 1), layer) - - def forward(self, x): - for layer in self: - x = torch.cat([x, layer(x)], dim=1) - return x - - -class TransitLayer(torch.nn.Module): - def __init__( - self, in_channels, out_channels, bias=True, config_str="batchnorm-relu" - ): - super(TransitLayer, self).__init__() - self.nonlinear = get_nonlinear(config_str, in_channels) - self.linear = torch.nn.Conv1d(in_channels, out_channels, 1, bias=bias) - - def forward(self, x): - x = self.nonlinear(x) - x = self.linear(x) - return x - - -class DenseLayer(torch.nn.Module): - def __init__( - self, in_channels, out_channels, bias=False, config_str="batchnorm-relu" - ): - super(DenseLayer, self).__init__() - self.linear = torch.nn.Conv1d(in_channels, out_channels, 1, bias=bias) - self.nonlinear = get_nonlinear(config_str, out_channels) - - def forward(self, x): - if len(x.shape) == 2: - x = self.linear(x.unsqueeze(dim=-1)).squeeze(dim=-1) - else: - x = self.linear(x) - x = self.nonlinear(x) - return x diff --git a/funasr_detach/models/campplus/model.py b/funasr_detach/models/campplus/model.py deleted file mode 100644 index bd226e96f643f7ac01fe9476b7891a69dc5fd54f..0000000000000000000000000000000000000000 --- a/funasr_detach/models/campplus/model.py +++ /dev/null @@ -1,149 +0,0 @@ -#!/usr/bin/env python3 -# -*- encoding: utf-8 -*- -# Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved. -# MIT License (https://opensource.org/licenses/MIT) -# Modified from 3D-Speaker (https://github.com/alibaba-damo-academy/3D-Speaker) - -import time -import torch -import numpy as np -from collections import OrderedDict -from contextlib import contextmanager -from distutils.version import LooseVersion - -from funasr_detach.register import tables -from funasr_detach.models.campplus.utils import extract_feature -from funasr_detach.utils.load_utils import load_audio_text_image_video -from funasr_detach.models.campplus.components import ( - DenseLayer, - StatsPool, - TDNNLayer, - CAMDenseTDNNBlock, - TransitLayer, - get_nonlinear, - FCM, -) - - -if LooseVersion(torch.__version__) >= LooseVersion("1.6.0"): - from torch.cuda.amp import autocast -else: - # Nothing to do if torch<1.6.0 - @contextmanager - def autocast(enabled=True): - yield - - -@tables.register("model_classes", "CAMPPlus") -class CAMPPlus(torch.nn.Module): - def __init__( - self, - feat_dim=80, - embedding_size=192, - growth_rate=32, - bn_size=4, - init_channels=128, - config_str="batchnorm-relu", - memory_efficient=True, - output_level="segment", - **kwargs, - ): - super().__init__() - - self.head = FCM(feat_dim=feat_dim) - channels = self.head.out_channels - self.output_level = output_level - - self.xvector = torch.nn.Sequential( - OrderedDict( - [ - ( - "tdnn", - TDNNLayer( - channels, - init_channels, - 5, - stride=2, - dilation=1, - padding=-1, - config_str=config_str, - ), - ), - ] - ) - ) - channels = init_channels - for i, (num_layers, kernel_size, dilation) in enumerate( - zip((12, 24, 16), (3, 3, 3), (1, 2, 2)) - ): - block = CAMDenseTDNNBlock( - num_layers=num_layers, - in_channels=channels, - out_channels=growth_rate, - bn_channels=bn_size * growth_rate, - kernel_size=kernel_size, - dilation=dilation, - config_str=config_str, - memory_efficient=memory_efficient, - ) - self.xvector.add_module("block%d" % (i + 1), block) - channels = channels + num_layers * growth_rate - self.xvector.add_module( - "transit%d" % (i + 1), - TransitLayer( - channels, channels // 2, bias=False, config_str=config_str - ), - ) - channels //= 2 - - self.xvector.add_module("out_nonlinear", get_nonlinear(config_str, channels)) - - if self.output_level == "segment": - self.xvector.add_module("stats", StatsPool()) - self.xvector.add_module( - "dense", - DenseLayer(channels * 2, embedding_size, config_str="batchnorm_"), - ) - else: - assert ( - self.output_level == "frame" - ), "`output_level` should be set to 'segment' or 'frame'. " - - for m in self.modules(): - if isinstance(m, (torch.nn.Conv1d, torch.nn.Linear)): - torch.nn.init.kaiming_normal_(m.weight.data) - if m.bias is not None: - torch.nn.init.zeros_(m.bias) - - def forward(self, x): - x = x.permute(0, 2, 1) # (B,T,F) => (B,F,T) - x = self.head(x) - x = self.xvector(x) - if self.output_level == "frame": - x = x.transpose(1, 2) - return x - - def inference( - self, - data_in, - data_lengths=None, - key: list = None, - tokenizer=None, - frontend=None, - **kwargs, - ): - # extract fbank feats - meta_data = {} - time1 = time.perf_counter() - audio_sample_list = load_audio_text_image_video( - data_in, fs=16000, audio_fs=kwargs.get("fs", 16000), data_type="sound" - ) - time2 = time.perf_counter() - meta_data["load_data"] = f"{time2 - time1:0.3f}" - speech, speech_lengths, speech_times = extract_feature(audio_sample_list) - speech = speech.to(device=kwargs["device"]) - time3 = time.perf_counter() - meta_data["extract_feat"] = f"{time3 - time2:0.3f}" - meta_data["batch_data_time"] = np.array(speech_times).sum().item() / 16000.0 - results = [{"spk_embedding": self.forward(speech.to(torch.float32))}] - return results, meta_data diff --git a/funasr_detach/models/campplus/template.yaml b/funasr_detach/models/campplus/template.yaml deleted file mode 100644 index 38dcfde3e1828f468446eac35df33a8bb8b95b6a..0000000000000000000000000000000000000000 --- a/funasr_detach/models/campplus/template.yaml +++ /dev/null @@ -1,23 +0,0 @@ -# This is an example that demonstrates how to configure a model file. -# You can modify the configuration according to your own requirements. - -# to print the register_table: -# from funasr.register import tables -# tables.print() - -# network architecture -model: CAMPPlus -model_conf: - feat_dim: 80 - embedding_size: 192 - growth_rate: 32 - bn_size: 4 - init_channels: 128 - config_str: 'batchnorm-relu' - memory_efficient: True - output_level: 'segment' - -# frontend related -frontend: WavFrontend -frontend_conf: - fs: 16000 diff --git a/funasr_detach/models/campplus/utils.py b/funasr_detach/models/campplus/utils.py deleted file mode 100644 index 14f97dceb11410684d92cee669f8264889b3ade5..0000000000000000000000000000000000000000 --- a/funasr_detach/models/campplus/utils.py +++ /dev/null @@ -1,520 +0,0 @@ -#!/usr/bin/env python3 -# -*- encoding: utf-8 -*- -# Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved. -# MIT License (https://opensource.org/licenses/MIT) -# Modified from 3D-Speaker (https://github.com/alibaba-damo-academy/3D-Speaker) - -import io -import os -import torch -import requests -import tempfile -import contextlib -import numpy as np -import librosa as sf -from typing import Union -from pathlib import Path -from typing import Generator, Union -from abc import ABCMeta, abstractmethod -import torchaudio.compliance.kaldi as Kaldi - -from funasr_detach.models.transformer.utils.nets_utils import pad_list - - -def check_audio_list(audio: list): - audio_dur = 0 - for i in range(len(audio)): - seg = audio[i] - assert seg[1] >= seg[0], "modelscope error: Wrong time stamps." - assert isinstance(seg[2], np.ndarray), "modelscope error: Wrong data type." - assert ( - int(seg[1] * 16000) - int(seg[0] * 16000) == seg[2].shape[0] - ), "modelscope error: audio data in list is inconsistent with time length." - if i > 0: - assert seg[0] >= audio[i - 1][1], "modelscope error: Wrong time stamps." - audio_dur += seg[1] - seg[0] - return audio_dur - # assert audio_dur > 5, 'modelscope error: The effective audio duration is too short.' - - -def sv_preprocess(inputs: Union[np.ndarray, list]): - output = [] - for i in range(len(inputs)): - if isinstance(inputs[i], str): - file_bytes = File.read(inputs[i]) - data, fs = sf.load(io.BytesIO(file_bytes), dtype="float32") - if len(data.shape) == 2: - data = data[:, 0] - data = torch.from_numpy(data).unsqueeze(0) - data = data.squeeze(0) - elif isinstance(inputs[i], np.ndarray): - assert ( - len(inputs[i].shape) == 1 - ), "modelscope error: Input array should be [N, T]" - data = inputs[i] - if data.dtype in ["int16", "int32", "int64"]: - data = (data / (1 << 15)).astype("float32") - else: - data = data.astype("float32") - data = torch.from_numpy(data) - else: - raise ValueError( - "modelscope error: The input type is restricted to audio address and nump array." - ) - output.append(data) - return output - - -def sv_chunk(vad_segments: list, fs=16000) -> list: - config = { - "seg_dur": 1.5, - "seg_shift": 0.75, - } - - def seg_chunk(seg_data): - seg_st = seg_data[0] - data = seg_data[2] - chunk_len = int(config["seg_dur"] * fs) - chunk_shift = int(config["seg_shift"] * fs) - last_chunk_ed = 0 - seg_res = [] - for chunk_st in range(0, data.shape[0], chunk_shift): - chunk_ed = min(chunk_st + chunk_len, data.shape[0]) - if chunk_ed <= last_chunk_ed: - break - last_chunk_ed = chunk_ed - chunk_st = max(0, chunk_ed - chunk_len) - chunk_data = data[chunk_st:chunk_ed] - if chunk_data.shape[0] < chunk_len: - chunk_data = np.pad( - chunk_data, (0, chunk_len - chunk_data.shape[0]), "constant" - ) - seg_res.append([chunk_st / fs + seg_st, chunk_ed / fs + seg_st, chunk_data]) - return seg_res - - segs = [] - for i, s in enumerate(vad_segments): - segs.extend(seg_chunk(s)) - - return segs - - -def extract_feature(audio): - features = [] - feature_times = [] - feature_lengths = [] - for au in audio: - feature = Kaldi.fbank(au.unsqueeze(0), num_mel_bins=80) - feature = feature - feature.mean(dim=0, keepdim=True) - features.append(feature) - feature_times.append(au.shape[0]) - feature_lengths.append(feature.shape[0]) - # padding for batch inference - features_padded = pad_list(features, pad_value=0) - # features = torch.cat(features) - return features_padded, feature_lengths, feature_times - - -def postprocess( - segments: list, vad_segments: list, labels: np.ndarray, embeddings: np.ndarray -) -> list: - assert len(segments) == len(labels) - labels = correct_labels(labels) - distribute_res = [] - for i in range(len(segments)): - distribute_res.append([segments[i][0], segments[i][1], labels[i]]) - # merge the same speakers chronologically - distribute_res = merge_seque(distribute_res) - - # accquire speaker center - spk_embs = [] - for i in range(labels.max() + 1): - spk_emb = embeddings[labels == i].mean(0) - spk_embs.append(spk_emb) - spk_embs = np.stack(spk_embs) - - def is_overlapped(t1, t2): - if t1 > t2 + 1e-4: - return True - return False - - # distribute the overlap region - for i in range(1, len(distribute_res)): - if is_overlapped(distribute_res[i - 1][1], distribute_res[i][0]): - p = (distribute_res[i][0] + distribute_res[i - 1][1]) / 2 - distribute_res[i][0] = p - distribute_res[i - 1][1] = p - - # smooth the result - distribute_res = smooth(distribute_res) - - return distribute_res - - -def correct_labels(labels): - labels_id = 0 - id2id = {} - new_labels = [] - for i in labels: - if i not in id2id: - id2id[i] = labels_id - labels_id += 1 - new_labels.append(id2id[i]) - return np.array(new_labels) - - -def merge_seque(distribute_res): - res = [distribute_res[0]] - for i in range(1, len(distribute_res)): - if distribute_res[i][2] != res[-1][2] or distribute_res[i][0] > res[-1][1]: - res.append(distribute_res[i]) - else: - res[-1][1] = distribute_res[i][1] - return res - - -def smooth(res, mindur=1): - # short segments are assigned to nearest speakers. - for i in range(len(res)): - res[i][0] = round(res[i][0], 2) - res[i][1] = round(res[i][1], 2) - if res[i][1] - res[i][0] < mindur: - if i == 0: - res[i][2] = res[i + 1][2] - elif i == len(res) - 1: - res[i][2] = res[i - 1][2] - elif res[i][0] - res[i - 1][1] <= res[i + 1][0] - res[i][1]: - res[i][2] = res[i - 1][2] - else: - res[i][2] = res[i + 1][2] - # merge the speakers - res = merge_seque(res) - - return res - - -def distribute_spk(sentence_list, sd_time_list): - sd_sentence_list = [] - for d in sentence_list: - sentence_start = d["start"] - sentence_end = d["end"] - sentence_spk = 0 - max_overlap = 0 - for sd_time in sd_time_list: - spk_st, spk_ed, spk = sd_time - spk_st = spk_st * 1000 - spk_ed = spk_ed * 1000 - overlap = max(min(sentence_end, spk_ed) - max(sentence_start, spk_st), 0) - if overlap > max_overlap: - max_overlap = overlap - sentence_spk = spk - d["spk"] = int(sentence_spk) - sd_sentence_list.append(d) - return sd_sentence_list - - -class Storage(metaclass=ABCMeta): - """Abstract class of storage. - - All backends need to implement two apis: ``read()`` and ``read_text()``. - ``read()`` reads the file as a byte stream and ``read_text()`` reads - the file as texts. - """ - - @abstractmethod - def read(self, filepath: str): - pass - - @abstractmethod - def read_text(self, filepath: str): - pass - - @abstractmethod - def write(self, obj: bytes, filepath: Union[str, Path]) -> None: - pass - - @abstractmethod - def write_text( - self, obj: str, filepath: Union[str, Path], encoding: str = "utf-8" - ) -> None: - pass - - -class LocalStorage(Storage): - """Local hard disk storage""" - - def read(self, filepath: Union[str, Path]) -> bytes: - """Read data from a given ``filepath`` with 'rb' mode. - - Args: - filepath (str or Path): Path to read data. - - Returns: - bytes: Expected bytes object. - """ - with open(filepath, "rb") as f: - content = f.read() - return content - - def read_text(self, filepath: Union[str, Path], encoding: str = "utf-8") -> str: - """Read data from a given ``filepath`` with 'r' mode. - - Args: - filepath (str or Path): Path to read data. - encoding (str): The encoding format used to open the ``filepath``. - Default: 'utf-8'. - - Returns: - str: Expected text reading from ``filepath``. - """ - with open(filepath, "r", encoding=encoding) as f: - value_buf = f.read() - return value_buf - - def write(self, obj: bytes, filepath: Union[str, Path]) -> None: - """Write data to a given ``filepath`` with 'wb' mode. - - Note: - ``write`` will create a directory if the directory of ``filepath`` - does not exist. - - Args: - obj (bytes): Data to be written. - filepath (str or Path): Path to write data. - """ - dirname = os.path.dirname(filepath) - if dirname and not os.path.exists(dirname): - os.makedirs(dirname, exist_ok=True) - - with open(filepath, "wb") as f: - f.write(obj) - - def write_text( - self, obj: str, filepath: Union[str, Path], encoding: str = "utf-8" - ) -> None: - """Write data to a given ``filepath`` with 'w' mode. - - Note: - ``write_text`` will create a directory if the directory of - ``filepath`` does not exist. - - Args: - obj (str): Data to be written. - filepath (str or Path): Path to write data. - encoding (str): The encoding format used to open the ``filepath``. - Default: 'utf-8'. - """ - dirname = os.path.dirname(filepath) - if dirname and not os.path.exists(dirname): - os.makedirs(dirname, exist_ok=True) - - with open(filepath, "w", encoding=encoding) as f: - f.write(obj) - - @contextlib.contextmanager - def as_local_path( - self, filepath: Union[str, Path] - ) -> Generator[Union[str, Path], None, None]: - """Only for unified API and do nothing.""" - yield filepath - - -class HTTPStorage(Storage): - """HTTP and HTTPS storage.""" - - def read(self, url): - # TODO @wenmeng.zwm add progress bar if file is too large - r = requests.get(url) - r.raise_for_status() - return r.content - - def read_text(self, url): - r = requests.get(url) - r.raise_for_status() - return r.text - - @contextlib.contextmanager - def as_local_path(self, filepath: str) -> Generator[Union[str, Path], None, None]: - """Download a file from ``filepath``. - - ``as_local_path`` is decorated by :meth:`contextlib.contextmanager`. It - can be called with ``with`` statement, and when exists from the - ``with`` statement, the temporary path will be released. - - Args: - filepath (str): Download a file from ``filepath``. - - Examples: - >>> storage = HTTPStorage() - >>> # After existing from the ``with`` clause, - >>> # the path will be removed - >>> with storage.get_local_path('http://path/to/file') as path: - ... # do something here - """ - try: - f = tempfile.NamedTemporaryFile(delete=False) - f.write(self.read(filepath)) - f.close() - yield f.name - finally: - os.remove(f.name) - - def write(self, obj: bytes, url: Union[str, Path]) -> None: - raise NotImplementedError("write is not supported by HTTP Storage") - - def write_text( - self, obj: str, url: Union[str, Path], encoding: str = "utf-8" - ) -> None: - raise NotImplementedError("write_text is not supported by HTTP Storage") - - -class OSSStorage(Storage): - """OSS storage.""" - - def __init__(self, oss_config_file=None): - # read from config file or env var - raise NotImplementedError("OSSStorage.__init__ to be implemented in the future") - - def read(self, filepath): - raise NotImplementedError("OSSStorage.read to be implemented in the future") - - def read_text(self, filepath, encoding="utf-8"): - raise NotImplementedError( - "OSSStorage.read_text to be implemented in the future" - ) - - @contextlib.contextmanager - def as_local_path(self, filepath: str) -> Generator[Union[str, Path], None, None]: - """Download a file from ``filepath``. - - ``as_local_path`` is decorated by :meth:`contextlib.contextmanager`. It - can be called with ``with`` statement, and when exists from the - ``with`` statement, the temporary path will be released. - - Args: - filepath (str): Download a file from ``filepath``. - - Examples: - >>> storage = OSSStorage() - >>> # After existing from the ``with`` clause, - >>> # the path will be removed - >>> with storage.get_local_path('http://path/to/file') as path: - ... # do something here - """ - try: - f = tempfile.NamedTemporaryFile(delete=False) - f.write(self.read(filepath)) - f.close() - yield f.name - finally: - os.remove(f.name) - - def write(self, obj: bytes, filepath: Union[str, Path]) -> None: - raise NotImplementedError("OSSStorage.write to be implemented in the future") - - def write_text( - self, obj: str, filepath: Union[str, Path], encoding: str = "utf-8" - ) -> None: - raise NotImplementedError( - "OSSStorage.write_text to be implemented in the future" - ) - - -G_STORAGES = {} - - -class File(object): - _prefix_to_storage: dict = { - "oss": OSSStorage, - "http": HTTPStorage, - "https": HTTPStorage, - "local": LocalStorage, - } - - @staticmethod - def _get_storage(uri): - assert isinstance(uri, str), f"uri should be str type, but got {type(uri)}" - - if "://" not in uri: - # local path - storage_type = "local" - else: - prefix, _ = uri.split("://") - storage_type = prefix - - assert storage_type in File._prefix_to_storage, ( - f"Unsupported uri {uri}, valid prefixs: " - f"{list(File._prefix_to_storage.keys())}" - ) - - if storage_type not in G_STORAGES: - G_STORAGES[storage_type] = File._prefix_to_storage[storage_type]() - - return G_STORAGES[storage_type] - - @staticmethod - def read(uri: str) -> bytes: - """Read data from a given ``filepath`` with 'rb' mode. - - Args: - filepath (str or Path): Path to read data. - - Returns: - bytes: Expected bytes object. - """ - storage = File._get_storage(uri) - return storage.read(uri) - - @staticmethod - def read_text(uri: Union[str, Path], encoding: str = "utf-8") -> str: - """Read data from a given ``filepath`` with 'r' mode. - - Args: - filepath (str or Path): Path to read data. - encoding (str): The encoding format used to open the ``filepath``. - Default: 'utf-8'. - - Returns: - str: Expected text reading from ``filepath``. - """ - storage = File._get_storage(uri) - return storage.read_text(uri) - - @staticmethod - def write(obj: bytes, uri: Union[str, Path]) -> None: - """Write data to a given ``filepath`` with 'wb' mode. - - Note: - ``write`` will create a directory if the directory of ``filepath`` - does not exist. - - Args: - obj (bytes): Data to be written. - filepath (str or Path): Path to write data. - """ - storage = File._get_storage(uri) - return storage.write(obj, uri) - - @staticmethod - def write_text(obj: str, uri: str, encoding: str = "utf-8") -> None: - """Write data to a given ``filepath`` with 'w' mode. - - Note: - ``write_text`` will create a directory if the directory of - ``filepath`` does not exist. - - Args: - obj (str): Data to be written. - filepath (str or Path): Path to write data. - encoding (str): The encoding format used to open the ``filepath``. - Default: 'utf-8'. - """ - storage = File._get_storage(uri) - return storage.write_text(obj, uri) - - @contextlib.contextmanager - def as_local_path(uri: str) -> Generator[Union[str, Path], None, None]: - """Only for unified API and do nothing.""" - storage = File._get_storage(uri) - with storage.as_local_path(uri) as local_path: - yield local_path diff --git a/funasr_detach/models/conformer/__init__.py b/funasr_detach/models/conformer/__init__.py deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/funasr_detach/models/conformer/encoder.py b/funasr_detach/models/conformer/encoder.py deleted file mode 100644 index c931996c68279a5768f14e26028b24ae5a6e8b22..0000000000000000000000000000000000000000 --- a/funasr_detach/models/conformer/encoder.py +++ /dev/null @@ -1,1281 +0,0 @@ -# Copyright 2020 Tomoki Hayashi -# Apache 2.0 (http://www.apache.org/licenses/LICENSE-2.0) - -"""Conformer encoder definition.""" - -import logging -from typing import Union, Dict, List, Tuple, Optional - -import torch -from torch import nn - -from funasr_detach.models.ctc.ctc import CTC -from funasr_detach.models.transformer.attention import ( - MultiHeadedAttention, # noqa: H301 - RelPositionMultiHeadedAttention, # noqa: H301 - LegacyRelPositionMultiHeadedAttention, # noqa: H301 - RelPositionMultiHeadedAttentionChunk, -) -from funasr_detach.models.transformer.embedding import ( - PositionalEncoding, # noqa: H301 - ScaledPositionalEncoding, # noqa: H301 - RelPositionalEncoding, # noqa: H301 - LegacyRelPositionalEncoding, # noqa: H301 - StreamingRelPositionalEncoding, -) -from funasr_detach.models.transformer.layer_norm import LayerNorm -from funasr_detach.models.transformer.utils.multi_layer_conv import Conv1dLinear -from funasr_detach.models.transformer.utils.multi_layer_conv import MultiLayeredConv1d -from funasr_detach.models.transformer.utils.nets_utils import get_activation -from funasr_detach.models.transformer.utils.nets_utils import make_pad_mask -from funasr_detach.models.transformer.utils.nets_utils import ( - TooShortUttError, - check_short_utt, - make_chunk_mask, - make_source_mask, -) -from funasr_detach.models.transformer.positionwise_feed_forward import ( - PositionwiseFeedForward, # noqa: H301 -) -from funasr_detach.models.transformer.utils.repeat import repeat, MultiBlocks -from funasr_detach.models.transformer.utils.subsampling import Conv2dSubsampling -from funasr_detach.models.transformer.utils.subsampling import Conv2dSubsampling2 -from funasr_detach.models.transformer.utils.subsampling import Conv2dSubsampling6 -from funasr_detach.models.transformer.utils.subsampling import Conv2dSubsampling8 -from funasr_detach.models.transformer.utils.subsampling import TooShortUttError -from funasr_detach.models.transformer.utils.subsampling import check_short_utt -from funasr_detach.models.transformer.utils.subsampling import Conv2dSubsamplingPad -from funasr_detach.models.transformer.utils.subsampling import StreamingConvInput -from funasr_detach.register import tables - - -class ConvolutionModule(nn.Module): - """ConvolutionModule in Conformer model. - - Args: - channels (int): The number of channels of conv layers. - kernel_size (int): Kernerl size of conv layers. - - """ - - def __init__(self, channels, kernel_size, activation=nn.ReLU(), bias=True): - """Construct an ConvolutionModule object.""" - super(ConvolutionModule, self).__init__() - # kernerl_size should be a odd number for 'SAME' padding - assert (kernel_size - 1) % 2 == 0 - - self.pointwise_conv1 = nn.Conv1d( - channels, - 2 * channels, - kernel_size=1, - stride=1, - padding=0, - bias=bias, - ) - self.depthwise_conv = nn.Conv1d( - channels, - channels, - kernel_size, - stride=1, - padding=(kernel_size - 1) // 2, - groups=channels, - bias=bias, - ) - self.norm = nn.BatchNorm1d(channels) - self.pointwise_conv2 = nn.Conv1d( - channels, - channels, - kernel_size=1, - stride=1, - padding=0, - bias=bias, - ) - self.activation = activation - - def forward(self, x): - """Compute convolution module. - - Args: - x (torch.Tensor): Input tensor (#batch, time, channels). - - Returns: - torch.Tensor: Output tensor (#batch, time, channels). - - """ - # exchange the temporal dimension and the feature dimension - x = x.transpose(1, 2) - - # GLU mechanism - x = self.pointwise_conv1(x) # (batch, 2*channel, dim) - x = nn.functional.glu(x, dim=1) # (batch, channel, dim) - - # 1D Depthwise Conv - x = self.depthwise_conv(x) - x = self.activation(self.norm(x)) - - x = self.pointwise_conv2(x) - - return x.transpose(1, 2) - - -class EncoderLayer(nn.Module): - """Encoder layer module. - - Args: - size (int): Input dimension. - self_attn (torch.nn.Module): Self-attention module instance. - `MultiHeadedAttention` or `RelPositionMultiHeadedAttention` instance - can be used as the argument. - feed_forward (torch.nn.Module): Feed-forward module instance. - `PositionwiseFeedForward`, `MultiLayeredConv1d`, or `Conv1dLinear` instance - can be used as the argument. - feed_forward_macaron (torch.nn.Module): Additional feed-forward module instance. - `PositionwiseFeedForward`, `MultiLayeredConv1d`, or `Conv1dLinear` instance - can be used as the argument. - conv_module (torch.nn.Module): Convolution module instance. - `ConvlutionModule` instance can be used as the argument. - dropout_rate (float): Dropout rate. - normalize_before (bool): Whether to use layer_norm before the first block. - concat_after (bool): Whether to concat attention layer's input and output. - if True, additional linear will be applied. - i.e. x -> x + linear(concat(x, att(x))) - if False, no additional linear will be applied. i.e. x -> x + att(x) - stochastic_depth_rate (float): Proability to skip this layer. - During training, the layer may skip residual computation and return input - as-is with given probability. - """ - - def __init__( - self, - size, - self_attn, - feed_forward, - feed_forward_macaron, - conv_module, - dropout_rate, - normalize_before=True, - concat_after=False, - stochastic_depth_rate=0.0, - ): - """Construct an EncoderLayer object.""" - super(EncoderLayer, self).__init__() - self.self_attn = self_attn - self.feed_forward = feed_forward - self.feed_forward_macaron = feed_forward_macaron - self.conv_module = conv_module - self.norm_ff = LayerNorm(size) # for the FNN module - self.norm_mha = LayerNorm(size) # for the MHA module - if feed_forward_macaron is not None: - self.norm_ff_macaron = LayerNorm(size) - self.ff_scale = 0.5 - else: - self.ff_scale = 1.0 - if self.conv_module is not None: - self.norm_conv = LayerNorm(size) # for the CNN module - self.norm_final = LayerNorm(size) # for the final output of the block - self.dropout = nn.Dropout(dropout_rate) - self.size = size - self.normalize_before = normalize_before - self.concat_after = concat_after - if self.concat_after: - self.concat_linear = nn.Linear(size + size, size) - self.stochastic_depth_rate = stochastic_depth_rate - - def forward(self, x_input, mask, cache=None): - """Compute encoded features. - - Args: - x_input (Union[Tuple, torch.Tensor]): Input tensor w/ or w/o pos emb. - - w/ pos emb: Tuple of tensors [(#batch, time, size), (1, time, size)]. - - w/o pos emb: Tensor (#batch, time, size). - mask (torch.Tensor): Mask tensor for the input (#batch, time). - cache (torch.Tensor): Cache tensor of the input (#batch, time - 1, size). - - Returns: - torch.Tensor: Output tensor (#batch, time, size). - torch.Tensor: Mask tensor (#batch, time). - - """ - if isinstance(x_input, tuple): - x, pos_emb = x_input[0], x_input[1] - else: - x, pos_emb = x_input, None - - skip_layer = False - # with stochastic depth, residual connection `x + f(x)` becomes - # `x <- x + 1 / (1 - p) * f(x)` at training time. - stoch_layer_coeff = 1.0 - if self.training and self.stochastic_depth_rate > 0: - skip_layer = torch.rand(1).item() < self.stochastic_depth_rate - stoch_layer_coeff = 1.0 / (1 - self.stochastic_depth_rate) - - if skip_layer: - if cache is not None: - x = torch.cat([cache, x], dim=1) - if pos_emb is not None: - return (x, pos_emb), mask - return x, mask - - # whether to use macaron style - if self.feed_forward_macaron is not None: - residual = x - if self.normalize_before: - x = self.norm_ff_macaron(x) - x = residual + stoch_layer_coeff * self.ff_scale * self.dropout( - self.feed_forward_macaron(x) - ) - if not self.normalize_before: - x = self.norm_ff_macaron(x) - - # multi-headed self-attention module - residual = x - if self.normalize_before: - x = self.norm_mha(x) - - if cache is None: - x_q = x - else: - assert cache.shape == (x.shape[0], x.shape[1] - 1, self.size) - x_q = x[:, -1:, :] - residual = residual[:, -1:, :] - mask = None if mask is None else mask[:, -1:, :] - - if pos_emb is not None: - x_att = self.self_attn(x_q, x, x, pos_emb, mask) - else: - x_att = self.self_attn(x_q, x, x, mask) - - if self.concat_after: - x_concat = torch.cat((x, x_att), dim=-1) - x = residual + stoch_layer_coeff * self.concat_linear(x_concat) - else: - x = residual + stoch_layer_coeff * self.dropout(x_att) - if not self.normalize_before: - x = self.norm_mha(x) - - # convolution module - if self.conv_module is not None: - residual = x - if self.normalize_before: - x = self.norm_conv(x) - x = residual + stoch_layer_coeff * self.dropout(self.conv_module(x)) - if not self.normalize_before: - x = self.norm_conv(x) - - # feed forward module - residual = x - if self.normalize_before: - x = self.norm_ff(x) - x = residual + stoch_layer_coeff * self.ff_scale * self.dropout( - self.feed_forward(x) - ) - if not self.normalize_before: - x = self.norm_ff(x) - - if self.conv_module is not None: - x = self.norm_final(x) - - if cache is not None: - x = torch.cat([cache, x], dim=1) - - if pos_emb is not None: - return (x, pos_emb), mask - - return x, mask - - -@tables.register("encoder_classes", "ConformerEncoder") -class ConformerEncoder(nn.Module): - """Conformer encoder module. - - Args: - input_size (int): Input dimension. - output_size (int): Dimension of attention. - attention_heads (int): The number of heads of multi head attention. - linear_units (int): The number of units of position-wise feed forward. - num_blocks (int): The number of decoder blocks. - dropout_rate (float): Dropout rate. - attention_dropout_rate (float): Dropout rate in attention. - positional_dropout_rate (float): Dropout rate after adding positional encoding. - input_layer (Union[str, torch.nn.Module]): Input layer type. - normalize_before (bool): Whether to use layer_norm before the first block. - concat_after (bool): Whether to concat attention layer's input and output. - If True, additional linear will be applied. - i.e. x -> x + linear(concat(x, att(x))) - If False, no additional linear will be applied. i.e. x -> x + att(x) - positionwise_layer_type (str): "linear", "conv1d", or "conv1d-linear". - positionwise_conv_kernel_size (int): Kernel size of positionwise conv1d layer. - rel_pos_type (str): Whether to use the latest relative positional encoding or - the legacy one. The legacy relative positional encoding will be deprecated - in the future. More Details can be found in - https://github.com/espnet/espnet/pull/2816. - encoder_pos_enc_layer_type (str): Encoder positional encoding layer type. - encoder_attn_layer_type (str): Encoder attention layer type. - activation_type (str): Encoder activation function type. - macaron_style (bool): Whether to use macaron style for positionwise layer. - use_cnn_module (bool): Whether to use convolution module. - zero_triu (bool): Whether to zero the upper triangular part of attention matrix. - cnn_module_kernel (int): Kernerl size of convolution module. - padding_idx (int): Padding idx for input_layer=embed. - - """ - - def __init__( - self, - input_size: int, - output_size: int = 256, - attention_heads: int = 4, - linear_units: int = 2048, - num_blocks: int = 6, - dropout_rate: float = 0.1, - positional_dropout_rate: float = 0.1, - attention_dropout_rate: float = 0.0, - input_layer: str = "conv2d", - normalize_before: bool = True, - concat_after: bool = False, - positionwise_layer_type: str = "linear", - positionwise_conv_kernel_size: int = 3, - macaron_style: bool = False, - rel_pos_type: str = "legacy", - pos_enc_layer_type: str = "rel_pos", - selfattention_layer_type: str = "rel_selfattn", - activation_type: str = "swish", - use_cnn_module: bool = True, - zero_triu: bool = False, - cnn_module_kernel: int = 31, - padding_idx: int = -1, - interctc_layer_idx: List[int] = [], - interctc_use_conditioning: bool = False, - stochastic_depth_rate: Union[float, List[float]] = 0.0, - ): - super().__init__() - self._output_size = output_size - - if rel_pos_type == "legacy": - if pos_enc_layer_type == "rel_pos": - pos_enc_layer_type = "legacy_rel_pos" - if selfattention_layer_type == "rel_selfattn": - selfattention_layer_type = "legacy_rel_selfattn" - elif rel_pos_type == "latest": - assert selfattention_layer_type != "legacy_rel_selfattn" - assert pos_enc_layer_type != "legacy_rel_pos" - else: - raise ValueError("unknown rel_pos_type: " + rel_pos_type) - - activation = get_activation(activation_type) - if pos_enc_layer_type == "abs_pos": - pos_enc_class = PositionalEncoding - elif pos_enc_layer_type == "scaled_abs_pos": - pos_enc_class = ScaledPositionalEncoding - elif pos_enc_layer_type == "rel_pos": - assert selfattention_layer_type == "rel_selfattn" - pos_enc_class = RelPositionalEncoding - elif pos_enc_layer_type == "legacy_rel_pos": - assert selfattention_layer_type == "legacy_rel_selfattn" - pos_enc_class = LegacyRelPositionalEncoding - logging.warning( - "Using legacy_rel_pos and it will be deprecated in the future." - ) - else: - raise ValueError("unknown pos_enc_layer: " + pos_enc_layer_type) - - if input_layer == "linear": - self.embed = torch.nn.Sequential( - torch.nn.Linear(input_size, output_size), - torch.nn.LayerNorm(output_size), - torch.nn.Dropout(dropout_rate), - pos_enc_class(output_size, positional_dropout_rate), - ) - elif input_layer == "conv2d": - self.embed = Conv2dSubsampling( - input_size, - output_size, - dropout_rate, - pos_enc_class(output_size, positional_dropout_rate), - ) - elif input_layer == "conv2dpad": - self.embed = Conv2dSubsamplingPad( - input_size, - output_size, - dropout_rate, - pos_enc_class(output_size, positional_dropout_rate), - ) - elif input_layer == "conv2d2": - self.embed = Conv2dSubsampling2( - input_size, - output_size, - dropout_rate, - pos_enc_class(output_size, positional_dropout_rate), - ) - elif input_layer == "conv2d6": - self.embed = Conv2dSubsampling6( - input_size, - output_size, - dropout_rate, - pos_enc_class(output_size, positional_dropout_rate), - ) - elif input_layer == "conv2d8": - self.embed = Conv2dSubsampling8( - input_size, - output_size, - dropout_rate, - pos_enc_class(output_size, positional_dropout_rate), - ) - elif input_layer == "embed": - self.embed = torch.nn.Sequential( - torch.nn.Embedding(input_size, output_size, padding_idx=padding_idx), - pos_enc_class(output_size, positional_dropout_rate), - ) - elif isinstance(input_layer, torch.nn.Module): - self.embed = torch.nn.Sequential( - input_layer, - pos_enc_class(output_size, positional_dropout_rate), - ) - elif input_layer is None: - self.embed = torch.nn.Sequential( - pos_enc_class(output_size, positional_dropout_rate) - ) - else: - raise ValueError("unknown input_layer: " + input_layer) - self.normalize_before = normalize_before - if positionwise_layer_type == "linear": - positionwise_layer = PositionwiseFeedForward - positionwise_layer_args = ( - output_size, - linear_units, - dropout_rate, - activation, - ) - elif positionwise_layer_type == "conv1d": - positionwise_layer = MultiLayeredConv1d - positionwise_layer_args = ( - output_size, - linear_units, - positionwise_conv_kernel_size, - dropout_rate, - ) - elif positionwise_layer_type == "conv1d-linear": - positionwise_layer = Conv1dLinear - positionwise_layer_args = ( - output_size, - linear_units, - positionwise_conv_kernel_size, - dropout_rate, - ) - else: - raise NotImplementedError("Support only linear or conv1d.") - - if selfattention_layer_type == "selfattn": - encoder_selfattn_layer = MultiHeadedAttention - encoder_selfattn_layer_args = ( - attention_heads, - output_size, - attention_dropout_rate, - ) - elif selfattention_layer_type == "legacy_rel_selfattn": - assert pos_enc_layer_type == "legacy_rel_pos" - encoder_selfattn_layer = LegacyRelPositionMultiHeadedAttention - encoder_selfattn_layer_args = ( - attention_heads, - output_size, - attention_dropout_rate, - ) - logging.warning( - "Using legacy_rel_selfattn and it will be deprecated in the future." - ) - elif selfattention_layer_type == "rel_selfattn": - assert pos_enc_layer_type == "rel_pos" - encoder_selfattn_layer = RelPositionMultiHeadedAttention - encoder_selfattn_layer_args = ( - attention_heads, - output_size, - attention_dropout_rate, - zero_triu, - ) - else: - raise ValueError("unknown encoder_attn_layer: " + selfattention_layer_type) - - convolution_layer = ConvolutionModule - convolution_layer_args = (output_size, cnn_module_kernel, activation) - - if isinstance(stochastic_depth_rate, float): - stochastic_depth_rate = [stochastic_depth_rate] * num_blocks - - if len(stochastic_depth_rate) != num_blocks: - raise ValueError( - f"Length of stochastic_depth_rate ({len(stochastic_depth_rate)}) " - f"should be equal to num_blocks ({num_blocks})" - ) - - self.encoders = repeat( - num_blocks, - lambda lnum: EncoderLayer( - output_size, - encoder_selfattn_layer(*encoder_selfattn_layer_args), - positionwise_layer(*positionwise_layer_args), - positionwise_layer(*positionwise_layer_args) if macaron_style else None, - convolution_layer(*convolution_layer_args) if use_cnn_module else None, - dropout_rate, - normalize_before, - concat_after, - stochastic_depth_rate[lnum], - ), - ) - if self.normalize_before: - self.after_norm = LayerNorm(output_size) - - self.interctc_layer_idx = interctc_layer_idx - if len(interctc_layer_idx) > 0: - assert 0 < min(interctc_layer_idx) and max(interctc_layer_idx) < num_blocks - self.interctc_use_conditioning = interctc_use_conditioning - self.conditioning_layer = None - - def output_size(self) -> int: - return self._output_size - - def forward( - self, - xs_pad: torch.Tensor, - ilens: torch.Tensor, - prev_states: torch.Tensor = None, - ctc: CTC = None, - ) -> Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]: - """Calculate forward propagation. - - Args: - xs_pad (torch.Tensor): Input tensor (#batch, L, input_size). - ilens (torch.Tensor): Input length (#batch). - prev_states (torch.Tensor): Not to be used now. - - Returns: - torch.Tensor: Output tensor (#batch, L, output_size). - torch.Tensor: Output length (#batch). - torch.Tensor: Not to be used now. - - """ - masks = (~make_pad_mask(ilens)[:, None, :]).to(xs_pad.device) - - if ( - isinstance(self.embed, Conv2dSubsampling) - or isinstance(self.embed, Conv2dSubsampling2) - or isinstance(self.embed, Conv2dSubsampling6) - or isinstance(self.embed, Conv2dSubsampling8) - or isinstance(self.embed, Conv2dSubsamplingPad) - ): - short_status, limit_size = check_short_utt(self.embed, xs_pad.size(1)) - if short_status: - raise TooShortUttError( - f"has {xs_pad.size(1)} frames and is too short for subsampling " - + f"(it needs more than {limit_size} frames), return empty results", - xs_pad.size(1), - limit_size, - ) - xs_pad, masks = self.embed(xs_pad, masks) - else: - xs_pad = self.embed(xs_pad) - - intermediate_outs = [] - if len(self.interctc_layer_idx) == 0: - xs_pad, masks = self.encoders(xs_pad, masks) - else: - for layer_idx, encoder_layer in enumerate(self.encoders): - xs_pad, masks = encoder_layer(xs_pad, masks) - - if layer_idx + 1 in self.interctc_layer_idx: - encoder_out = xs_pad - if isinstance(encoder_out, tuple): - encoder_out = encoder_out[0] - - # intermediate outputs are also normalized - if self.normalize_before: - encoder_out = self.after_norm(encoder_out) - - intermediate_outs.append((layer_idx + 1, encoder_out)) - - if self.interctc_use_conditioning: - ctc_out = ctc.softmax(encoder_out) - - if isinstance(xs_pad, tuple): - x, pos_emb = xs_pad - x = x + self.conditioning_layer(ctc_out) - xs_pad = (x, pos_emb) - else: - xs_pad = xs_pad + self.conditioning_layer(ctc_out) - - if isinstance(xs_pad, tuple): - xs_pad = xs_pad[0] - if self.normalize_before: - xs_pad = self.after_norm(xs_pad) - - olens = masks.squeeze(1).sum(1) - if len(intermediate_outs) > 0: - return (xs_pad, intermediate_outs), olens, None - return xs_pad, olens, None - - -class CausalConvolution(torch.nn.Module): - """ConformerConvolution module definition. - Args: - channels: The number of channels. - kernel_size: Size of the convolving kernel. - activation: Type of activation function. - norm_args: Normalization module arguments. - causal: Whether to use causal convolution (set to True if streaming). - """ - - def __init__( - self, - channels: int, - kernel_size: int, - activation: torch.nn.Module = torch.nn.ReLU(), - norm_args: Dict = {}, - causal: bool = False, - ) -> None: - """Construct an ConformerConvolution object.""" - super().__init__() - - assert (kernel_size - 1) % 2 == 0 - - self.kernel_size = kernel_size - - self.pointwise_conv1 = torch.nn.Conv1d( - channels, - 2 * channels, - kernel_size=1, - stride=1, - padding=0, - ) - - if causal: - self.lorder = kernel_size - 1 - padding = 0 - else: - self.lorder = 0 - padding = (kernel_size - 1) // 2 - - self.depthwise_conv = torch.nn.Conv1d( - channels, - channels, - kernel_size, - stride=1, - padding=padding, - groups=channels, - ) - self.norm = torch.nn.BatchNorm1d(channels, **norm_args) - self.pointwise_conv2 = torch.nn.Conv1d( - channels, - channels, - kernel_size=1, - stride=1, - padding=0, - ) - - self.activation = activation - - def forward( - self, - x: torch.Tensor, - cache: Optional[torch.Tensor] = None, - right_context: int = 0, - ) -> Tuple[torch.Tensor, torch.Tensor]: - """Compute convolution module. - Args: - x: ConformerConvolution input sequences. (B, T, D_hidden) - cache: ConformerConvolution input cache. (1, conv_kernel, D_hidden) - right_context: Number of frames in right context. - Returns: - x: ConformerConvolution output sequences. (B, T, D_hidden) - cache: ConformerConvolution output cache. (1, conv_kernel, D_hidden) - """ - x = self.pointwise_conv1(x.transpose(1, 2)) - x = torch.nn.functional.glu(x, dim=1) - - if self.lorder > 0: - if cache is None: - x = torch.nn.functional.pad(x, (self.lorder, 0), "constant", 0.0) - else: - x = torch.cat([cache, x], dim=2) - - if right_context > 0: - cache = x[:, :, -(self.lorder + right_context) : -right_context] - else: - cache = x[:, :, -self.lorder :] - - x = self.depthwise_conv(x) - x = self.activation(self.norm(x)) - - x = self.pointwise_conv2(x).transpose(1, 2) - - return x, cache - - -class ChunkEncoderLayer(torch.nn.Module): - """Chunk Conformer module definition. - Args: - block_size: Input/output size. - self_att: Self-attention module instance. - feed_forward: Feed-forward module instance. - feed_forward_macaron: Feed-forward module instance for macaron network. - conv_mod: Convolution module instance. - norm_class: Normalization module class. - norm_args: Normalization module arguments. - dropout_rate: Dropout rate. - """ - - def __init__( - self, - block_size: int, - self_att: torch.nn.Module, - feed_forward: torch.nn.Module, - feed_forward_macaron: torch.nn.Module, - conv_mod: torch.nn.Module, - norm_class: torch.nn.Module = LayerNorm, - norm_args: Dict = {}, - dropout_rate: float = 0.0, - ) -> None: - """Construct a Conformer object.""" - super().__init__() - - self.self_att = self_att - - self.feed_forward = feed_forward - self.feed_forward_macaron = feed_forward_macaron - self.feed_forward_scale = 0.5 - - self.conv_mod = conv_mod - - self.norm_feed_forward = norm_class(block_size, **norm_args) - self.norm_self_att = norm_class(block_size, **norm_args) - - self.norm_macaron = norm_class(block_size, **norm_args) - self.norm_conv = norm_class(block_size, **norm_args) - self.norm_final = norm_class(block_size, **norm_args) - - self.dropout = torch.nn.Dropout(dropout_rate) - - self.block_size = block_size - self.cache = None - - def reset_streaming_cache(self, left_context: int, device: torch.device) -> None: - """Initialize/Reset self-attention and convolution modules cache for streaming. - Args: - left_context: Number of left frames during chunk-by-chunk inference. - device: Device to use for cache tensor. - """ - self.cache = [ - torch.zeros( - (1, left_context, self.block_size), - device=device, - ), - torch.zeros( - ( - 1, - self.block_size, - self.conv_mod.kernel_size - 1, - ), - device=device, - ), - ] - - def forward( - self, - x: torch.Tensor, - pos_enc: torch.Tensor, - mask: torch.Tensor, - chunk_mask: Optional[torch.Tensor] = None, - ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: - """Encode input sequences. - Args: - x: Conformer input sequences. (B, T, D_block) - pos_enc: Positional embedding sequences. (B, 2 * (T - 1), D_block) - mask: Source mask. (B, T) - chunk_mask: Chunk mask. (T_2, T_2) - Returns: - x: Conformer output sequences. (B, T, D_block) - mask: Source mask. (B, T) - pos_enc: Positional embedding sequences. (B, 2 * (T - 1), D_block) - """ - residual = x - - x = self.norm_macaron(x) - x = residual + self.feed_forward_scale * self.dropout( - self.feed_forward_macaron(x) - ) - - residual = x - x = self.norm_self_att(x) - x_q = x - x = residual + self.dropout( - self.self_att( - x_q, - x, - x, - pos_enc, - mask, - chunk_mask=chunk_mask, - ) - ) - - residual = x - - x = self.norm_conv(x) - x, _ = self.conv_mod(x) - x = residual + self.dropout(x) - residual = x - - x = self.norm_feed_forward(x) - x = residual + self.feed_forward_scale * self.dropout(self.feed_forward(x)) - - x = self.norm_final(x) - return x, mask, pos_enc - - def chunk_forward( - self, - x: torch.Tensor, - pos_enc: torch.Tensor, - mask: torch.Tensor, - chunk_size: int = 16, - left_context: int = 0, - right_context: int = 0, - ) -> Tuple[torch.Tensor, torch.Tensor]: - """Encode chunk of input sequence. - Args: - x: Conformer input sequences. (B, T, D_block) - pos_enc: Positional embedding sequences. (B, 2 * (T - 1), D_block) - mask: Source mask. (B, T_2) - left_context: Number of frames in left context. - right_context: Number of frames in right context. - Returns: - x: Conformer output sequences. (B, T, D_block) - pos_enc: Positional embedding sequences. (B, 2 * (T - 1), D_block) - """ - residual = x - - x = self.norm_macaron(x) - x = residual + self.feed_forward_scale * self.feed_forward_macaron(x) - - residual = x - x = self.norm_self_att(x) - if left_context > 0: - key = torch.cat([self.cache[0], x], dim=1) - else: - key = x - val = key - - if right_context > 0: - att_cache = key[:, -(left_context + right_context) : -right_context, :] - else: - att_cache = key[:, -left_context:, :] - x = residual + self.self_att( - x, - key, - val, - pos_enc, - mask, - left_context=left_context, - ) - - residual = x - x = self.norm_conv(x) - x, conv_cache = self.conv_mod( - x, cache=self.cache[1], right_context=right_context - ) - x = residual + x - residual = x - - x = self.norm_feed_forward(x) - x = residual + self.feed_forward_scale * self.feed_forward(x) - - x = self.norm_final(x) - self.cache = [att_cache, conv_cache] - - return x, pos_enc - - -@tables.register("encoder_classes", "ChunkConformerEncoder") -class ConformerChunkEncoder(torch.nn.Module): - """Encoder module definition. - Args: - input_size: Input size. - body_conf: Encoder body configuration. - input_conf: Encoder input configuration. - main_conf: Encoder main configuration. - """ - - def __init__( - self, - input_size: int, - output_size: int = 256, - attention_heads: int = 4, - linear_units: int = 2048, - num_blocks: int = 6, - dropout_rate: float = 0.1, - positional_dropout_rate: float = 0.1, - attention_dropout_rate: float = 0.0, - embed_vgg_like: bool = False, - normalize_before: bool = True, - concat_after: bool = False, - positionwise_layer_type: str = "linear", - positionwise_conv_kernel_size: int = 3, - macaron_style: bool = False, - rel_pos_type: str = "legacy", - pos_enc_layer_type: str = "rel_pos", - selfattention_layer_type: str = "rel_selfattn", - activation_type: str = "swish", - use_cnn_module: bool = True, - zero_triu: bool = False, - norm_type: str = "layer_norm", - cnn_module_kernel: int = 31, - conv_mod_norm_eps: float = 0.00001, - conv_mod_norm_momentum: float = 0.1, - simplified_att_score: bool = False, - dynamic_chunk_training: bool = False, - short_chunk_threshold: float = 0.75, - short_chunk_size: int = 25, - left_chunk_size: int = 0, - time_reduction_factor: int = 1, - unified_model_training: bool = False, - default_chunk_size: int = 16, - jitter_range: int = 4, - subsampling_factor: int = 1, - ) -> None: - """Construct an Encoder object.""" - super().__init__() - - self.embed = StreamingConvInput( - input_size=input_size, - conv_size=output_size, - subsampling_factor=subsampling_factor, - vgg_like=embed_vgg_like, - output_size=output_size, - ) - - self.pos_enc = StreamingRelPositionalEncoding( - output_size, - positional_dropout_rate, - ) - - activation = get_activation(activation_type) - - pos_wise_args = ( - output_size, - linear_units, - positional_dropout_rate, - activation, - ) - - conv_mod_norm_args = { - "eps": conv_mod_norm_eps, - "momentum": conv_mod_norm_momentum, - } - - conv_mod_args = ( - output_size, - cnn_module_kernel, - activation, - conv_mod_norm_args, - dynamic_chunk_training or unified_model_training, - ) - - mult_att_args = ( - attention_heads, - output_size, - attention_dropout_rate, - simplified_att_score, - ) - - fn_modules = [] - for _ in range(num_blocks): - module = lambda: ChunkEncoderLayer( - output_size, - RelPositionMultiHeadedAttentionChunk(*mult_att_args), - PositionwiseFeedForward(*pos_wise_args), - PositionwiseFeedForward(*pos_wise_args), - CausalConvolution(*conv_mod_args), - dropout_rate=dropout_rate, - ) - fn_modules.append(module) - - self.encoders = MultiBlocks( - [fn() for fn in fn_modules], - output_size, - ) - - self._output_size = output_size - - self.dynamic_chunk_training = dynamic_chunk_training - self.short_chunk_threshold = short_chunk_threshold - self.short_chunk_size = short_chunk_size - self.left_chunk_size = left_chunk_size - - self.unified_model_training = unified_model_training - self.default_chunk_size = default_chunk_size - self.jitter_range = jitter_range - - self.time_reduction_factor = time_reduction_factor - - def output_size(self) -> int: - return self._output_size - - def get_encoder_input_raw_size(self, size: int, hop_length: int) -> int: - """Return the corresponding number of sample for a given chunk size, in frames. - Where size is the number of features frames after applying subsampling. - Args: - size: Number of frames after subsampling. - hop_length: Frontend's hop length - Returns: - : Number of raw samples - """ - return self.embed.get_size_before_subsampling(size) * hop_length - - def get_encoder_input_size(self, size: int) -> int: - """Return the corresponding number of sample for a given chunk size, in frames. - Where size is the number of features frames after applying subsampling. - Args: - size: Number of frames after subsampling. - Returns: - : Number of raw samples - """ - return self.embed.get_size_before_subsampling(size) - - def reset_streaming_cache(self, left_context: int, device: torch.device) -> None: - """Initialize/Reset encoder streaming cache. - Args: - left_context: Number of frames in left context. - device: Device ID. - """ - return self.encoders.reset_streaming_cache(left_context, device) - - def forward( - self, - x: torch.Tensor, - x_len: torch.Tensor, - ) -> Tuple[torch.Tensor, torch.Tensor]: - """Encode input sequences. - Args: - x: Encoder input features. (B, T_in, F) - x_len: Encoder input features lengths. (B,) - Returns: - x: Encoder outputs. (B, T_out, D_enc) - x_len: Encoder outputs lenghts. (B,) - """ - short_status, limit_size = check_short_utt( - self.embed.subsampling_factor, x.size(1) - ) - - if short_status: - raise TooShortUttError( - f"has {x.size(1)} frames and is too short for subsampling " - + f"(it needs more than {limit_size} frames), return empty results", - x.size(1), - limit_size, - ) - - mask = make_source_mask(x_len).to(x.device) - - if self.unified_model_training: - if self.training: - chunk_size = ( - self.default_chunk_size - + torch.randint( - -self.jitter_range, self.jitter_range + 1, (1,) - ).item() - ) - else: - chunk_size = self.default_chunk_size - x, mask = self.embed(x, mask, chunk_size) - pos_enc = self.pos_enc(x) - chunk_mask = make_chunk_mask( - x.size(1), - chunk_size, - left_chunk_size=self.left_chunk_size, - device=x.device, - ) - x_utt = self.encoders( - x, - pos_enc, - mask, - chunk_mask=None, - ) - x_chunk = self.encoders( - x, - pos_enc, - mask, - chunk_mask=chunk_mask, - ) - - olens = mask.eq(0).sum(1) - if self.time_reduction_factor > 1: - x_utt = x_utt[:, :: self.time_reduction_factor, :] - x_chunk = x_chunk[:, :: self.time_reduction_factor, :] - olens = torch.floor_divide(olens - 1, self.time_reduction_factor) + 1 - - return x_utt, x_chunk, olens - - elif self.dynamic_chunk_training: - max_len = x.size(1) - if self.training: - chunk_size = torch.randint(1, max_len, (1,)).item() - - if chunk_size > (max_len * self.short_chunk_threshold): - chunk_size = max_len - else: - chunk_size = (chunk_size % self.short_chunk_size) + 1 - else: - chunk_size = self.default_chunk_size - - x, mask = self.embed(x, mask, chunk_size) - pos_enc = self.pos_enc(x) - - chunk_mask = make_chunk_mask( - x.size(1), - chunk_size, - left_chunk_size=self.left_chunk_size, - device=x.device, - ) - else: - x, mask = self.embed(x, mask, None) - pos_enc = self.pos_enc(x) - chunk_mask = None - x = self.encoders( - x, - pos_enc, - mask, - chunk_mask=chunk_mask, - ) - - olens = mask.eq(0).sum(1) - if self.time_reduction_factor > 1: - x = x[:, :: self.time_reduction_factor, :] - olens = torch.floor_divide(olens - 1, self.time_reduction_factor) + 1 - - return x, olens, None - - def full_utt_forward( - self, - x: torch.Tensor, - x_len: torch.Tensor, - ) -> Tuple[torch.Tensor, torch.Tensor]: - """Encode input sequences. - Args: - x: Encoder input features. (B, T_in, F) - x_len: Encoder input features lengths. (B,) - Returns: - x: Encoder outputs. (B, T_out, D_enc) - x_len: Encoder outputs lenghts. (B,) - """ - short_status, limit_size = check_short_utt( - self.embed.subsampling_factor, x.size(1) - ) - - if short_status: - raise TooShortUttError( - f"has {x.size(1)} frames and is too short for subsampling " - + f"(it needs more than {limit_size} frames), return empty results", - x.size(1), - limit_size, - ) - - mask = make_source_mask(x_len).to(x.device) - x, mask = self.embed(x, mask, None) - pos_enc = self.pos_enc(x) - x_utt = self.encoders( - x, - pos_enc, - mask, - chunk_mask=None, - ) - - if self.time_reduction_factor > 1: - x_utt = x_utt[:, :: self.time_reduction_factor, :] - return x_utt - - def simu_chunk_forward( - self, - x: torch.Tensor, - x_len: torch.Tensor, - chunk_size: int = 16, - left_context: int = 32, - right_context: int = 0, - ) -> torch.Tensor: - short_status, limit_size = check_short_utt( - self.embed.subsampling_factor, x.size(1) - ) - - if short_status: - raise TooShortUttError( - f"has {x.size(1)} frames and is too short for subsampling " - + f"(it needs more than {limit_size} frames), return empty results", - x.size(1), - limit_size, - ) - - mask = make_source_mask(x_len) - - x, mask = self.embed(x, mask, chunk_size) - pos_enc = self.pos_enc(x) - chunk_mask = make_chunk_mask( - x.size(1), - chunk_size, - left_chunk_size=self.left_chunk_size, - device=x.device, - ) - - x = self.encoders( - x, - pos_enc, - mask, - chunk_mask=chunk_mask, - ) - olens = mask.eq(0).sum(1) - if self.time_reduction_factor > 1: - x = x[:, :: self.time_reduction_factor, :] - - return x - - def chunk_forward( - self, - x: torch.Tensor, - x_len: torch.Tensor, - processed_frames: torch.tensor, - chunk_size: int = 16, - left_context: int = 32, - right_context: int = 0, - ) -> torch.Tensor: - """Encode input sequences as chunks. - Args: - x: Encoder input features. (1, T_in, F) - x_len: Encoder input features lengths. (1,) - processed_frames: Number of frames already seen. - left_context: Number of frames in left context. - right_context: Number of frames in right context. - Returns: - x: Encoder outputs. (B, T_out, D_enc) - """ - mask = make_source_mask(x_len) - x, mask = self.embed(x, mask, None) - - if left_context > 0: - processed_mask = ( - torch.arange(left_context, device=x.device) - .view(1, left_context) - .flip(1) - ) - processed_mask = processed_mask >= processed_frames - mask = torch.cat([processed_mask, mask], dim=1) - pos_enc = self.pos_enc(x, left_context=left_context) - x = self.encoders.chunk_forward( - x, - pos_enc, - mask, - chunk_size=chunk_size, - left_context=left_context, - right_context=right_context, - ) - - if right_context > 0: - x = x[:, 0:-right_context, :] - - if self.time_reduction_factor > 1: - x = x[:, :: self.time_reduction_factor, :] - return x diff --git a/funasr_detach/models/conformer/model.py b/funasr_detach/models/conformer/model.py deleted file mode 100644 index 2240b568e11de8b3ec299f4b5ca0a35586491d51..0000000000000000000000000000000000000000 --- a/funasr_detach/models/conformer/model.py +++ /dev/null @@ -1,19 +0,0 @@ -import logging - -import torch - -from funasr_detach.models.transformer.model import Transformer -from funasr_detach.register import tables - - -@tables.register("model_classes", "Conformer") -class Conformer(Transformer): - """CTC-attention hybrid Encoder-Decoder model""" - - def __init__( - self, - *args, - **kwargs, - ): - - super().__init__(*args, **kwargs) diff --git a/funasr_detach/models/conformer/template.yaml b/funasr_detach/models/conformer/template.yaml deleted file mode 100644 index f646acc9d99faafc59d9808fd5d05c64a254cd28..0000000000000000000000000000000000000000 --- a/funasr_detach/models/conformer/template.yaml +++ /dev/null @@ -1,117 +0,0 @@ -# This is an example that demonstrates how to configure a model file. -# You can modify the configuration according to your own requirements. - -# to print the register_table: -# from funasr.register import tables -# tables.print() - -# network architecture -model: Conformer -model_conf: - ctc_weight: 0.3 - lsm_weight: 0.1 # label smoothing option - length_normalized_loss: false - -# encoder -encoder: ConformerEncoder -encoder_conf: - output_size: 256 - attention_heads: 4 - linear_units: 2048 - num_blocks: 12 - dropout_rate: 0.1 - positional_dropout_rate: 0.1 - attention_dropout_rate: 0.0 - input_layer: conv2d - normalize_before: true - pos_enc_layer_type: rel_pos - selfattention_layer_type: rel_selfattn - activation_type: swish - macaron_style: true - use_cnn_module: true - cnn_module_kernel: 15 - -# decoder -decoder: TransformerDecoder -decoder_conf: - attention_heads: 4 - linear_units: 2048 - num_blocks: 6 - dropout_rate: 0.1 - positional_dropout_rate: 0.1 - self_attention_dropout_rate: 0.0 - src_attention_dropout_rate: 0.0 - - -# frontend related -frontend: WavFrontend -frontend_conf: - fs: 16000 - window: hamming - n_mels: 80 - frame_length: 25 - frame_shift: 10 - dither: 0.0 - lfr_m: 1 - lfr_n: 1 - -specaug: SpecAug -specaug_conf: - apply_time_warp: true - time_warp_window: 5 - time_warp_mode: bicubic - apply_freq_mask: true - freq_mask_width_range: - - 0 - - 30 - num_freq_mask: 2 - apply_time_mask: true - time_mask_width_range: - - 0 - - 40 - num_time_mask: 2 - -train_conf: - accum_grad: 1 - grad_clip: 5 - max_epoch: 150 - val_scheduler_criterion: - - valid - - acc - best_model_criterion: - - - valid - - acc - - max - keep_nbest_models: 10 - log_interval: 50 - -optim: adam -optim_conf: - lr: 0.0005 -scheduler: warmuplr -scheduler_conf: - warmup_steps: 30000 - -dataset: AudioDataset -dataset_conf: - index_ds: IndexDSJsonl - batch_sampler: DynamicBatchLocalShuffleSampler - batch_type: example # example or length - batch_size: 1 # if batch_type is example, batch_size is the numbers of samples; if length, batch_size is source_token_len+target_token_len; - max_token_length: 2048 # filter samples if source_token_len+target_token_len > max_token_length, - buffer_size: 500 - shuffle: True - num_workers: 0 - -tokenizer: CharTokenizer -tokenizer_conf: - unk_symbol: - split_with_space: true - - -ctc_conf: - dropout_rate: 0.0 - ctc_type: builtin - reduce: true - ignore_nan_grad: true -normalize: null diff --git a/funasr_detach/models/contextual_paraformer/__init__.py b/funasr_detach/models/contextual_paraformer/__init__.py deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/funasr_detach/models/contextual_paraformer/decoder.py b/funasr_detach/models/contextual_paraformer/decoder.py deleted file mode 100644 index 2e1eb8d67452ab023cfaba22bacdf04f91bba9ec..0000000000000000000000000000000000000000 --- a/funasr_detach/models/contextual_paraformer/decoder.py +++ /dev/null @@ -1,1023 +0,0 @@ -#!/usr/bin/env python3 -# -*- encoding: utf-8 -*- -# Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved. -# MIT License (https://opensource.org/licenses/MIT) - -import torch -import logging -import numpy as np -from typing import Tuple - -from funasr_detach.register import tables -from funasr_detach.models.scama import utils as myutils -from funasr_detach.models.transformer.utils.repeat import repeat -from funasr_detach.models.transformer.layer_norm import LayerNorm -from funasr_detach.models.transformer.embedding import PositionalEncoding -from funasr_detach.models.paraformer.decoder import ( - DecoderLayerSANM, - ParaformerSANMDecoder, -) -from funasr_detach.models.sanm.positionwise_feed_forward import ( - PositionwiseFeedForwardDecoderSANM, -) -from funasr_detach.models.sanm.attention import ( - MultiHeadedAttentionSANMDecoder, - MultiHeadedAttentionCrossAtt, -) - - -class ContextualDecoderLayer(torch.nn.Module): - def __init__( - self, - size, - self_attn, - src_attn, - feed_forward, - dropout_rate, - normalize_before=True, - concat_after=False, - ): - """Construct an DecoderLayer object.""" - super(ContextualDecoderLayer, self).__init__() - self.size = size - self.self_attn = self_attn - self.src_attn = src_attn - self.feed_forward = feed_forward - self.norm1 = LayerNorm(size) - if self_attn is not None: - self.norm2 = LayerNorm(size) - if src_attn is not None: - self.norm3 = LayerNorm(size) - self.dropout = torch.nn.Dropout(dropout_rate) - self.normalize_before = normalize_before - self.concat_after = concat_after - if self.concat_after: - self.concat_linear1 = torch.nn.Linear(size + size, size) - self.concat_linear2 = torch.nn.Linear(size + size, size) - - def forward( - self, - tgt, - tgt_mask, - memory, - memory_mask, - cache=None, - ): - # tgt = self.dropout(tgt) - if isinstance(tgt, Tuple): - tgt, _ = tgt - residual = tgt - if self.normalize_before: - tgt = self.norm1(tgt) - tgt = self.feed_forward(tgt) - - x = tgt - if self.normalize_before: - tgt = self.norm2(tgt) - if self.training: - cache = None - x, cache = self.self_attn(tgt, tgt_mask, cache=cache) - x = residual + self.dropout(x) - x_self_attn = x - - residual = x - if self.normalize_before: - x = self.norm3(x) - x = self.src_attn(x, memory, memory_mask) - x_src_attn = x - - x = residual + self.dropout(x) - return x, tgt_mask, x_self_attn, x_src_attn - - -class ContextualBiasDecoder(torch.nn.Module): - def __init__( - self, - size, - src_attn, - dropout_rate, - normalize_before=True, - ): - """Construct an DecoderLayer object.""" - super(ContextualBiasDecoder, self).__init__() - self.size = size - self.src_attn = src_attn - if src_attn is not None: - self.norm3 = LayerNorm(size) - self.dropout = torch.nn.Dropout(dropout_rate) - self.normalize_before = normalize_before - - def forward(self, tgt, tgt_mask, memory, memory_mask=None, cache=None): - x = tgt - if self.src_attn is not None: - if self.normalize_before: - x = self.norm3(x) - x = self.dropout(self.src_attn(x, memory, memory_mask)) - return x, tgt_mask, memory, memory_mask, cache - - -@tables.register("decoder_classes", "ContextualParaformerDecoder") -class ContextualParaformerDecoder(ParaformerSANMDecoder): - """ - Author: Speech Lab of DAMO Academy, Alibaba Group - Paraformer: Fast and Accurate Parallel Transformer for Non-autoregressive End-to-End Speech Recognition - https://arxiv.org/abs/2006.01713 - """ - - def __init__( - self, - vocab_size: int, - encoder_output_size: int, - attention_heads: int = 4, - linear_units: int = 2048, - num_blocks: int = 6, - dropout_rate: float = 0.1, - positional_dropout_rate: float = 0.1, - self_attention_dropout_rate: float = 0.0, - src_attention_dropout_rate: float = 0.0, - input_layer: str = "embed", - use_output_layer: bool = True, - pos_enc_class=PositionalEncoding, - normalize_before: bool = True, - concat_after: bool = False, - att_layer_num: int = 6, - kernel_size: int = 21, - sanm_shfit: int = 0, - ): - super().__init__( - vocab_size=vocab_size, - encoder_output_size=encoder_output_size, - dropout_rate=dropout_rate, - positional_dropout_rate=positional_dropout_rate, - input_layer=input_layer, - use_output_layer=use_output_layer, - pos_enc_class=pos_enc_class, - normalize_before=normalize_before, - ) - - attention_dim = encoder_output_size - if input_layer == "none": - self.embed = None - if input_layer == "embed": - self.embed = torch.nn.Sequential( - torch.nn.Embedding(vocab_size, attention_dim), - # pos_enc_class(attention_dim, positional_dropout_rate), - ) - elif input_layer == "linear": - self.embed = torch.nn.Sequential( - torch.nn.Linear(vocab_size, attention_dim), - torch.nn.LayerNorm(attention_dim), - torch.nn.Dropout(dropout_rate), - torch.nn.ReLU(), - pos_enc_class(attention_dim, positional_dropout_rate), - ) - else: - raise ValueError(f"only 'embed' or 'linear' is supported: {input_layer}") - - self.normalize_before = normalize_before - if self.normalize_before: - self.after_norm = LayerNorm(attention_dim) - if use_output_layer: - self.output_layer = torch.nn.Linear(attention_dim, vocab_size) - else: - self.output_layer = None - - self.att_layer_num = att_layer_num - self.num_blocks = num_blocks - if sanm_shfit is None: - sanm_shfit = (kernel_size - 1) // 2 - self.decoders = repeat( - att_layer_num - 1, - lambda lnum: DecoderLayerSANM( - attention_dim, - MultiHeadedAttentionSANMDecoder( - attention_dim, - self_attention_dropout_rate, - kernel_size, - sanm_shfit=sanm_shfit, - ), - MultiHeadedAttentionCrossAtt( - attention_heads, attention_dim, src_attention_dropout_rate - ), - PositionwiseFeedForwardDecoderSANM( - attention_dim, linear_units, dropout_rate - ), - dropout_rate, - normalize_before, - concat_after, - ), - ) - self.dropout = torch.nn.Dropout(dropout_rate) - self.bias_decoder = ContextualBiasDecoder( - size=attention_dim, - src_attn=MultiHeadedAttentionCrossAtt( - attention_heads, attention_dim, src_attention_dropout_rate - ), - dropout_rate=dropout_rate, - normalize_before=True, - ) - self.bias_output = torch.nn.Conv1d( - attention_dim * 2, attention_dim, 1, bias=False - ) - self.last_decoder = ContextualDecoderLayer( - attention_dim, - MultiHeadedAttentionSANMDecoder( - attention_dim, - self_attention_dropout_rate, - kernel_size, - sanm_shfit=sanm_shfit, - ), - MultiHeadedAttentionCrossAtt( - attention_heads, attention_dim, src_attention_dropout_rate - ), - PositionwiseFeedForwardDecoderSANM( - attention_dim, linear_units, dropout_rate - ), - dropout_rate, - normalize_before, - concat_after, - ) - if num_blocks - att_layer_num <= 0: - self.decoders2 = None - else: - self.decoders2 = repeat( - num_blocks - att_layer_num, - lambda lnum: DecoderLayerSANM( - attention_dim, - MultiHeadedAttentionSANMDecoder( - attention_dim, - self_attention_dropout_rate, - kernel_size, - sanm_shfit=0, - ), - None, - PositionwiseFeedForwardDecoderSANM( - attention_dim, linear_units, dropout_rate - ), - dropout_rate, - normalize_before, - concat_after, - ), - ) - - self.decoders3 = repeat( - 1, - lambda lnum: DecoderLayerSANM( - attention_dim, - None, - None, - PositionwiseFeedForwardDecoderSANM( - attention_dim, linear_units, dropout_rate - ), - dropout_rate, - normalize_before, - concat_after, - ), - ) - - def forward( - self, - hs_pad: torch.Tensor, - hlens: torch.Tensor, - ys_in_pad: torch.Tensor, - ys_in_lens: torch.Tensor, - contextual_info: torch.Tensor, - clas_scale: float = 1.0, - return_hidden: bool = False, - ) -> Tuple[torch.Tensor, torch.Tensor]: - """Forward decoder. - - Args: - hs_pad: encoded memory, float32 (batch, maxlen_in, feat) - hlens: (batch) - ys_in_pad: - input token ids, int64 (batch, maxlen_out) - if input_layer == "embed" - input tensor (batch, maxlen_out, #mels) in the other cases - ys_in_lens: (batch) - Returns: - (tuple): tuple containing: - - x: decoded token score before softmax (batch, maxlen_out, token) - if use_output_layer is True, - olens: (batch, ) - """ - tgt = ys_in_pad - tgt_mask = myutils.sequence_mask(ys_in_lens, device=tgt.device)[:, :, None] - - memory = hs_pad - memory_mask = myutils.sequence_mask(hlens, device=memory.device)[:, None, :] - - x = tgt - x, tgt_mask, memory, memory_mask, _ = self.decoders( - x, tgt_mask, memory, memory_mask - ) - _, _, x_self_attn, x_src_attn = self.last_decoder( - x, tgt_mask, memory, memory_mask - ) - - # contextual paraformer related - contextual_length = ( - torch.Tensor([contextual_info.shape[1]]).int().repeat(hs_pad.shape[0]) - ) - contextual_mask = myutils.sequence_mask( - contextual_length, device=memory.device - )[:, None, :] - cx, tgt_mask, _, _, _ = self.bias_decoder( - x_self_attn, tgt_mask, contextual_info, memory_mask=contextual_mask - ) - - if self.bias_output is not None: - x = torch.cat([x_src_attn, cx * clas_scale], dim=2) - x = self.bias_output(x.transpose(1, 2)).transpose(1, 2) # 2D -> D - x = x_self_attn + self.dropout(x) - - if self.decoders2 is not None: - x, tgt_mask, memory, memory_mask, _ = self.decoders2( - x, tgt_mask, memory, memory_mask - ) - - x, tgt_mask, memory, memory_mask, _ = self.decoders3( - x, tgt_mask, memory, memory_mask - ) - if self.normalize_before: - x = self.after_norm(x) - olens = tgt_mask.sum(1) - if self.output_layer is not None and return_hidden is False: - x = self.output_layer(x) - return x, olens - - def gen_tf2torch_map_dict(self): - - tensor_name_prefix_torch = self.tf2torch_tensor_name_prefix_torch - tensor_name_prefix_tf = self.tf2torch_tensor_name_prefix_tf - map_dict_local = { - ## decoder - # ffn - "{}.decoders.layeridx.norm1.weight".format(tensor_name_prefix_torch): { - "name": "{}/decoder_fsmn_layer_layeridx/decoder_ffn/LayerNorm/gamma".format( - tensor_name_prefix_tf - ), - "squeeze": None, - "transpose": None, - }, # (256,),(256,) - "{}.decoders.layeridx.norm1.bias".format(tensor_name_prefix_torch): { - "name": "{}/decoder_fsmn_layer_layeridx/decoder_ffn/LayerNorm/beta".format( - tensor_name_prefix_tf - ), - "squeeze": None, - "transpose": None, - }, # (256,),(256,) - "{}.decoders.layeridx.feed_forward.w_1.weight".format( - tensor_name_prefix_torch - ): { - "name": "{}/decoder_fsmn_layer_layeridx/decoder_ffn/conv1d/kernel".format( - tensor_name_prefix_tf - ), - "squeeze": 0, - "transpose": (1, 0), - }, # (1024,256),(1,256,1024) - "{}.decoders.layeridx.feed_forward.w_1.bias".format( - tensor_name_prefix_torch - ): { - "name": "{}/decoder_fsmn_layer_layeridx/decoder_ffn/conv1d/bias".format( - tensor_name_prefix_tf - ), - "squeeze": None, - "transpose": None, - }, # (1024,),(1024,) - "{}.decoders.layeridx.feed_forward.norm.weight".format( - tensor_name_prefix_torch - ): { - "name": "{}/decoder_fsmn_layer_layeridx/decoder_ffn/LayerNorm_1/gamma".format( - tensor_name_prefix_tf - ), - "squeeze": None, - "transpose": None, - }, # (1024,),(1024,) - "{}.decoders.layeridx.feed_forward.norm.bias".format( - tensor_name_prefix_torch - ): { - "name": "{}/decoder_fsmn_layer_layeridx/decoder_ffn/LayerNorm_1/beta".format( - tensor_name_prefix_tf - ), - "squeeze": None, - "transpose": None, - }, # (1024,),(1024,) - "{}.decoders.layeridx.feed_forward.w_2.weight".format( - tensor_name_prefix_torch - ): { - "name": "{}/decoder_fsmn_layer_layeridx/decoder_ffn/conv1d_1/kernel".format( - tensor_name_prefix_tf - ), - "squeeze": 0, - "transpose": (1, 0), - }, # (256,1024),(1,1024,256) - # fsmn - "{}.decoders.layeridx.norm2.weight".format(tensor_name_prefix_torch): { - "name": "{}/decoder_fsmn_layer_layeridx/decoder_memory_block/LayerNorm/gamma".format( - tensor_name_prefix_tf - ), - "squeeze": None, - "transpose": None, - }, # (256,),(256,) - "{}.decoders.layeridx.norm2.bias".format(tensor_name_prefix_torch): { - "name": "{}/decoder_fsmn_layer_layeridx/decoder_memory_block/LayerNorm/beta".format( - tensor_name_prefix_tf - ), - "squeeze": None, - "transpose": None, - }, # (256,),(256,) - "{}.decoders.layeridx.self_attn.fsmn_block.weight".format( - tensor_name_prefix_torch - ): { - "name": "{}/decoder_fsmn_layer_layeridx/decoder_memory_block/depth_conv_w".format( - tensor_name_prefix_tf - ), - "squeeze": 0, - "transpose": (1, 2, 0), - }, # (256,1,31),(1,31,256,1) - # src att - "{}.decoders.layeridx.norm3.weight".format(tensor_name_prefix_torch): { - "name": "{}/decoder_fsmn_layer_layeridx/multi_head/LayerNorm/gamma".format( - tensor_name_prefix_tf - ), - "squeeze": None, - "transpose": None, - }, # (256,),(256,) - "{}.decoders.layeridx.norm3.bias".format(tensor_name_prefix_torch): { - "name": "{}/decoder_fsmn_layer_layeridx/multi_head/LayerNorm/beta".format( - tensor_name_prefix_tf - ), - "squeeze": None, - "transpose": None, - }, # (256,),(256,) - "{}.decoders.layeridx.src_attn.linear_q.weight".format( - tensor_name_prefix_torch - ): { - "name": "{}/decoder_fsmn_layer_layeridx/multi_head/conv1d/kernel".format( - tensor_name_prefix_tf - ), - "squeeze": 0, - "transpose": (1, 0), - }, # (256,256),(1,256,256) - "{}.decoders.layeridx.src_attn.linear_q.bias".format( - tensor_name_prefix_torch - ): { - "name": "{}/decoder_fsmn_layer_layeridx/multi_head/conv1d/bias".format( - tensor_name_prefix_tf - ), - "squeeze": None, - "transpose": None, - }, # (256,),(256,) - "{}.decoders.layeridx.src_attn.linear_k_v.weight".format( - tensor_name_prefix_torch - ): { - "name": "{}/decoder_fsmn_layer_layeridx/multi_head/conv1d_1/kernel".format( - tensor_name_prefix_tf - ), - "squeeze": 0, - "transpose": (1, 0), - }, # (1024,256),(1,256,1024) - "{}.decoders.layeridx.src_attn.linear_k_v.bias".format( - tensor_name_prefix_torch - ): { - "name": "{}/decoder_fsmn_layer_layeridx/multi_head/conv1d_1/bias".format( - tensor_name_prefix_tf - ), - "squeeze": None, - "transpose": None, - }, # (1024,),(1024,) - "{}.decoders.layeridx.src_attn.linear_out.weight".format( - tensor_name_prefix_torch - ): { - "name": "{}/decoder_fsmn_layer_layeridx/multi_head/conv1d_2/kernel".format( - tensor_name_prefix_tf - ), - "squeeze": 0, - "transpose": (1, 0), - }, # (256,256),(1,256,256) - "{}.decoders.layeridx.src_attn.linear_out.bias".format( - tensor_name_prefix_torch - ): { - "name": "{}/decoder_fsmn_layer_layeridx/multi_head/conv1d_2/bias".format( - tensor_name_prefix_tf - ), - "squeeze": None, - "transpose": None, - }, # (256,),(256,) - # dnn - "{}.decoders3.layeridx.norm1.weight".format(tensor_name_prefix_torch): { - "name": "{}/decoder_dnn_layer_layeridx/LayerNorm/gamma".format( - tensor_name_prefix_tf - ), - "squeeze": None, - "transpose": None, - }, # (256,),(256,) - "{}.decoders3.layeridx.norm1.bias".format(tensor_name_prefix_torch): { - "name": "{}/decoder_dnn_layer_layeridx/LayerNorm/beta".format( - tensor_name_prefix_tf - ), - "squeeze": None, - "transpose": None, - }, # (256,),(256,) - "{}.decoders3.layeridx.feed_forward.w_1.weight".format( - tensor_name_prefix_torch - ): { - "name": "{}/decoder_dnn_layer_layeridx/conv1d/kernel".format( - tensor_name_prefix_tf - ), - "squeeze": 0, - "transpose": (1, 0), - }, # (1024,256),(1,256,1024) - "{}.decoders3.layeridx.feed_forward.w_1.bias".format( - tensor_name_prefix_torch - ): { - "name": "{}/decoder_dnn_layer_layeridx/conv1d/bias".format( - tensor_name_prefix_tf - ), - "squeeze": None, - "transpose": None, - }, # (1024,),(1024,) - "{}.decoders3.layeridx.feed_forward.norm.weight".format( - tensor_name_prefix_torch - ): { - "name": "{}/decoder_dnn_layer_layeridx/LayerNorm_1/gamma".format( - tensor_name_prefix_tf - ), - "squeeze": None, - "transpose": None, - }, # (1024,),(1024,) - "{}.decoders3.layeridx.feed_forward.norm.bias".format( - tensor_name_prefix_torch - ): { - "name": "{}/decoder_dnn_layer_layeridx/LayerNorm_1/beta".format( - tensor_name_prefix_tf - ), - "squeeze": None, - "transpose": None, - }, # (1024,),(1024,) - "{}.decoders3.layeridx.feed_forward.w_2.weight".format( - tensor_name_prefix_torch - ): { - "name": "{}/decoder_dnn_layer_layeridx/conv1d_1/kernel".format( - tensor_name_prefix_tf - ), - "squeeze": 0, - "transpose": (1, 0), - }, # (256,1024),(1,1024,256) - # embed_concat_ffn - "{}.embed_concat_ffn.layeridx.norm1.weight".format( - tensor_name_prefix_torch - ): { - "name": "{}/cif_concat/LayerNorm/gamma".format(tensor_name_prefix_tf), - "squeeze": None, - "transpose": None, - }, # (256,),(256,) - "{}.embed_concat_ffn.layeridx.norm1.bias".format( - tensor_name_prefix_torch - ): { - "name": "{}/cif_concat/LayerNorm/beta".format(tensor_name_prefix_tf), - "squeeze": None, - "transpose": None, - }, # (256,),(256,) - "{}.embed_concat_ffn.layeridx.feed_forward.w_1.weight".format( - tensor_name_prefix_torch - ): { - "name": "{}/cif_concat/conv1d/kernel".format(tensor_name_prefix_tf), - "squeeze": 0, - "transpose": (1, 0), - }, # (1024,256),(1,256,1024) - "{}.embed_concat_ffn.layeridx.feed_forward.w_1.bias".format( - tensor_name_prefix_torch - ): { - "name": "{}/cif_concat/conv1d/bias".format(tensor_name_prefix_tf), - "squeeze": None, - "transpose": None, - }, # (1024,),(1024,) - "{}.embed_concat_ffn.layeridx.feed_forward.norm.weight".format( - tensor_name_prefix_torch - ): { - "name": "{}/cif_concat/LayerNorm_1/gamma".format(tensor_name_prefix_tf), - "squeeze": None, - "transpose": None, - }, # (1024,),(1024,) - "{}.embed_concat_ffn.layeridx.feed_forward.norm.bias".format( - tensor_name_prefix_torch - ): { - "name": "{}/cif_concat/LayerNorm_1/beta".format(tensor_name_prefix_tf), - "squeeze": None, - "transpose": None, - }, # (1024,),(1024,) - "{}.embed_concat_ffn.layeridx.feed_forward.w_2.weight".format( - tensor_name_prefix_torch - ): { - "name": "{}/cif_concat/conv1d_1/kernel".format(tensor_name_prefix_tf), - "squeeze": 0, - "transpose": (1, 0), - }, # (256,1024),(1,1024,256) - # out norm - "{}.after_norm.weight".format(tensor_name_prefix_torch): { - "name": "{}/LayerNorm/gamma".format(tensor_name_prefix_tf), - "squeeze": None, - "transpose": None, - }, # (256,),(256,) - "{}.after_norm.bias".format(tensor_name_prefix_torch): { - "name": "{}/LayerNorm/beta".format(tensor_name_prefix_tf), - "squeeze": None, - "transpose": None, - }, # (256,),(256,) - # in embed - "{}.embed.0.weight".format(tensor_name_prefix_torch): { - "name": "{}/w_embs".format(tensor_name_prefix_tf), - "squeeze": None, - "transpose": None, - }, # (4235,256),(4235,256) - # out layer - "{}.output_layer.weight".format(tensor_name_prefix_torch): { - "name": [ - "{}/dense/kernel".format(tensor_name_prefix_tf), - "{}/w_embs".format(tensor_name_prefix_tf), - ], - "squeeze": [None, None], - "transpose": [(1, 0), None], - }, # (4235,256),(256,4235) - "{}.output_layer.bias".format(tensor_name_prefix_torch): { - "name": [ - "{}/dense/bias".format(tensor_name_prefix_tf), - ( - "seq2seq/2bias" - if tensor_name_prefix_tf == "seq2seq/decoder/inputter_1" - else "seq2seq/bias" - ), - ], - "squeeze": [None, None], - "transpose": [None, None], - }, # (4235,),(4235,) - ## clas decoder - # src att - "{}.bias_decoder.norm3.weight".format(tensor_name_prefix_torch): { - "name": "{}/decoder_fsmn_layer_15/multi_head_1/LayerNorm/gamma".format( - tensor_name_prefix_tf - ), - "squeeze": None, - "transpose": None, - }, # (256,),(256,) - "{}.bias_decoder.norm3.bias".format(tensor_name_prefix_torch): { - "name": "{}/decoder_fsmn_layer_15/multi_head_1/LayerNorm/beta".format( - tensor_name_prefix_tf - ), - "squeeze": None, - "transpose": None, - }, # (256,),(256,) - "{}.bias_decoder.src_attn.linear_q.weight".format( - tensor_name_prefix_torch - ): { - "name": "{}/decoder_fsmn_layer_15/multi_head_1/conv1d/kernel".format( - tensor_name_prefix_tf - ), - "squeeze": 0, - "transpose": (1, 0), - }, # (256,256),(1,256,256) - "{}.bias_decoder.src_attn.linear_q.bias".format(tensor_name_prefix_torch): { - "name": "{}/decoder_fsmn_layer_15/multi_head_1/conv1d/bias".format( - tensor_name_prefix_tf - ), - "squeeze": None, - "transpose": None, - }, # (256,),(256,) - "{}.bias_decoder.src_attn.linear_k_v.weight".format( - tensor_name_prefix_torch - ): { - "name": "{}/decoder_fsmn_layer_15/multi_head_1/conv1d_1/kernel".format( - tensor_name_prefix_tf - ), - "squeeze": 0, - "transpose": (1, 0), - }, # (1024,256),(1,256,1024) - "{}.bias_decoder.src_attn.linear_k_v.bias".format( - tensor_name_prefix_torch - ): { - "name": "{}/decoder_fsmn_layer_15/multi_head_1/conv1d_1/bias".format( - tensor_name_prefix_tf - ), - "squeeze": None, - "transpose": None, - }, # (1024,),(1024,) - "{}.bias_decoder.src_attn.linear_out.weight".format( - tensor_name_prefix_torch - ): { - "name": "{}/decoder_fsmn_layer_15/multi_head_1/conv1d_2/kernel".format( - tensor_name_prefix_tf - ), - "squeeze": 0, - "transpose": (1, 0), - }, # (256,256),(1,256,256) - "{}.bias_decoder.src_attn.linear_out.bias".format( - tensor_name_prefix_torch - ): { - "name": "{}/decoder_fsmn_layer_15/multi_head_1/conv1d_2/bias".format( - tensor_name_prefix_tf - ), - "squeeze": None, - "transpose": None, - }, # (256,),(256,) - # dnn - "{}.bias_output.weight".format(tensor_name_prefix_torch): { - "name": "{}/decoder_fsmn_layer_15/conv1d/kernel".format( - tensor_name_prefix_tf - ), - "squeeze": None, - "transpose": (2, 1, 0), - }, # (1024,256),(1,256,1024) - } - return map_dict_local - - def convert_tf2torch( - self, - var_dict_tf, - var_dict_torch, - ): - map_dict = self.gen_tf2torch_map_dict() - var_dict_torch_update = dict() - decoder_layeridx_sets = set() - for name in sorted(var_dict_torch.keys(), reverse=False): - names = name.split(".") - if names[0] == self.tf2torch_tensor_name_prefix_torch: - if names[1] == "decoders": - layeridx = int(names[2]) - name_q = name.replace(".{}.".format(layeridx), ".layeridx.") - layeridx_bias = 0 - layeridx += layeridx_bias - decoder_layeridx_sets.add(layeridx) - if name_q in map_dict.keys(): - name_v = map_dict[name_q]["name"] - name_tf = name_v.replace("layeridx", "{}".format(layeridx)) - data_tf = var_dict_tf[name_tf] - if map_dict[name_q]["squeeze"] is not None: - data_tf = np.squeeze( - data_tf, axis=map_dict[name_q]["squeeze"] - ) - if map_dict[name_q]["transpose"] is not None: - data_tf = np.transpose( - data_tf, map_dict[name_q]["transpose"] - ) - data_tf = ( - torch.from_numpy(data_tf).type(torch.float32).to("cpu") - ) - assert ( - var_dict_torch[name].size() == data_tf.size() - ), "{}, {}, {} != {}".format( - name, name_tf, var_dict_torch[name].size(), data_tf.size() - ) - var_dict_torch_update[name] = data_tf - logging.info( - "torch tensor: {}, {}, loading from tf tensor: {}, {}".format( - name, data_tf.size(), name_v, var_dict_tf[name_tf].shape - ) - ) - elif names[1] == "last_decoder": - layeridx = 15 - name_q = name.replace("last_decoder", "decoders.layeridx") - layeridx_bias = 0 - layeridx += layeridx_bias - decoder_layeridx_sets.add(layeridx) - if name_q in map_dict.keys(): - name_v = map_dict[name_q]["name"] - name_tf = name_v.replace("layeridx", "{}".format(layeridx)) - data_tf = var_dict_tf[name_tf] - if map_dict[name_q]["squeeze"] is not None: - data_tf = np.squeeze( - data_tf, axis=map_dict[name_q]["squeeze"] - ) - if map_dict[name_q]["transpose"] is not None: - data_tf = np.transpose( - data_tf, map_dict[name_q]["transpose"] - ) - data_tf = ( - torch.from_numpy(data_tf).type(torch.float32).to("cpu") - ) - assert ( - var_dict_torch[name].size() == data_tf.size() - ), "{}, {}, {} != {}".format( - name, name_tf, var_dict_torch[name].size(), data_tf.size() - ) - var_dict_torch_update[name] = data_tf - logging.info( - "torch tensor: {}, {}, loading from tf tensor: {}, {}".format( - name, data_tf.size(), name_v, var_dict_tf[name_tf].shape - ) - ) - - elif names[1] == "decoders2": - layeridx = int(names[2]) - name_q = name.replace(".{}.".format(layeridx), ".layeridx.") - name_q = name_q.replace("decoders2", "decoders") - layeridx_bias = len(decoder_layeridx_sets) - - layeridx += layeridx_bias - if "decoders." in name: - decoder_layeridx_sets.add(layeridx) - if name_q in map_dict.keys(): - name_v = map_dict[name_q]["name"] - name_tf = name_v.replace("layeridx", "{}".format(layeridx)) - data_tf = var_dict_tf[name_tf] - if map_dict[name_q]["squeeze"] is not None: - data_tf = np.squeeze( - data_tf, axis=map_dict[name_q]["squeeze"] - ) - if map_dict[name_q]["transpose"] is not None: - data_tf = np.transpose( - data_tf, map_dict[name_q]["transpose"] - ) - data_tf = ( - torch.from_numpy(data_tf).type(torch.float32).to("cpu") - ) - assert ( - var_dict_torch[name].size() == data_tf.size() - ), "{}, {}, {} != {}".format( - name, name_tf, var_dict_torch[name].size(), data_tf.size() - ) - var_dict_torch_update[name] = data_tf - logging.info( - "torch tensor: {}, {}, loading from tf tensor: {}, {}".format( - name, data_tf.size(), name_v, var_dict_tf[name_tf].shape - ) - ) - - elif names[1] == "decoders3": - layeridx = int(names[2]) - name_q = name.replace(".{}.".format(layeridx), ".layeridx.") - - layeridx_bias = 0 - layeridx += layeridx_bias - if "decoders." in name: - decoder_layeridx_sets.add(layeridx) - if name_q in map_dict.keys(): - name_v = map_dict[name_q]["name"] - name_tf = name_v.replace("layeridx", "{}".format(layeridx)) - data_tf = var_dict_tf[name_tf] - if map_dict[name_q]["squeeze"] is not None: - data_tf = np.squeeze( - data_tf, axis=map_dict[name_q]["squeeze"] - ) - if map_dict[name_q]["transpose"] is not None: - data_tf = np.transpose( - data_tf, map_dict[name_q]["transpose"] - ) - data_tf = ( - torch.from_numpy(data_tf).type(torch.float32).to("cpu") - ) - assert ( - var_dict_torch[name].size() == data_tf.size() - ), "{}, {}, {} != {}".format( - name, name_tf, var_dict_torch[name].size(), data_tf.size() - ) - var_dict_torch_update[name] = data_tf - logging.info( - "torch tensor: {}, {}, loading from tf tensor: {}, {}".format( - name, data_tf.size(), name_v, var_dict_tf[name_tf].shape - ) - ) - elif names[1] == "bias_decoder": - name_q = name - - if name_q in map_dict.keys(): - name_v = map_dict[name_q]["name"] - name_tf = name_v - data_tf = var_dict_tf[name_tf] - if map_dict[name_q]["squeeze"] is not None: - data_tf = np.squeeze( - data_tf, axis=map_dict[name_q]["squeeze"] - ) - if map_dict[name_q]["transpose"] is not None: - data_tf = np.transpose( - data_tf, map_dict[name_q]["transpose"] - ) - data_tf = ( - torch.from_numpy(data_tf).type(torch.float32).to("cpu") - ) - assert ( - var_dict_torch[name].size() == data_tf.size() - ), "{}, {}, {} != {}".format( - name, name_tf, var_dict_torch[name].size(), data_tf.size() - ) - var_dict_torch_update[name] = data_tf - logging.info( - "torch tensor: {}, {}, loading from tf tensor: {}, {}".format( - name, data_tf.size(), name_v, var_dict_tf[name_tf].shape - ) - ) - - elif ( - names[1] == "embed" - or names[1] == "output_layer" - or names[1] == "bias_output" - ): - name_tf = map_dict[name]["name"] - if isinstance(name_tf, list): - idx_list = 0 - if name_tf[idx_list] in var_dict_tf.keys(): - pass - else: - idx_list = 1 - data_tf = var_dict_tf[name_tf[idx_list]] - if map_dict[name]["squeeze"][idx_list] is not None: - data_tf = np.squeeze( - data_tf, axis=map_dict[name]["squeeze"][idx_list] - ) - if map_dict[name]["transpose"][idx_list] is not None: - data_tf = np.transpose( - data_tf, map_dict[name]["transpose"][idx_list] - ) - data_tf = ( - torch.from_numpy(data_tf).type(torch.float32).to("cpu") - ) - assert ( - var_dict_torch[name].size() == data_tf.size() - ), "{}, {}, {} != {}".format( - name, name_tf, var_dict_torch[name].size(), data_tf.size() - ) - var_dict_torch_update[name] = data_tf - logging.info( - "torch tensor: {}, {}, loading from tf tensor: {}, {}".format( - name, - data_tf.size(), - name_tf[idx_list], - var_dict_tf[name_tf[idx_list]].shape, - ) - ) - - else: - data_tf = var_dict_tf[name_tf] - if map_dict[name]["squeeze"] is not None: - data_tf = np.squeeze( - data_tf, axis=map_dict[name]["squeeze"] - ) - if map_dict[name]["transpose"] is not None: - data_tf = np.transpose(data_tf, map_dict[name]["transpose"]) - data_tf = ( - torch.from_numpy(data_tf).type(torch.float32).to("cpu") - ) - assert ( - var_dict_torch[name].size() == data_tf.size() - ), "{}, {}, {} != {}".format( - name, name_tf, var_dict_torch[name].size(), data_tf.size() - ) - var_dict_torch_update[name] = data_tf - logging.info( - "torch tensor: {}, {}, loading from tf tensor: {}, {}".format( - name, - data_tf.size(), - name_tf, - var_dict_tf[name_tf].shape, - ) - ) - - elif names[1] == "after_norm": - name_tf = map_dict[name]["name"] - data_tf = var_dict_tf[name_tf] - data_tf = torch.from_numpy(data_tf).type(torch.float32).to("cpu") - var_dict_torch_update[name] = data_tf - logging.info( - "torch tensor: {}, {}, loading from tf tensor: {}, {}".format( - name, data_tf.size(), name_tf, var_dict_tf[name_tf].shape - ) - ) - - elif names[1] == "embed_concat_ffn": - layeridx = int(names[2]) - name_q = name.replace(".{}.".format(layeridx), ".layeridx.") - - layeridx_bias = 0 - layeridx += layeridx_bias - if "decoders." in name: - decoder_layeridx_sets.add(layeridx) - if name_q in map_dict.keys(): - name_v = map_dict[name_q]["name"] - name_tf = name_v.replace("layeridx", "{}".format(layeridx)) - data_tf = var_dict_tf[name_tf] - if map_dict[name_q]["squeeze"] is not None: - data_tf = np.squeeze( - data_tf, axis=map_dict[name_q]["squeeze"] - ) - if map_dict[name_q]["transpose"] is not None: - data_tf = np.transpose( - data_tf, map_dict[name_q]["transpose"] - ) - data_tf = ( - torch.from_numpy(data_tf).type(torch.float32).to("cpu") - ) - assert ( - var_dict_torch[name].size() == data_tf.size() - ), "{}, {}, {} != {}".format( - name, name_tf, var_dict_torch[name].size(), data_tf.size() - ) - var_dict_torch_update[name] = data_tf - logging.info( - "torch tensor: {}, {}, loading from tf tensor: {}, {}".format( - name, data_tf.size(), name_v, var_dict_tf[name_tf].shape - ) - ) - - return var_dict_torch_update diff --git a/funasr_detach/models/contextual_paraformer/model.py b/funasr_detach/models/contextual_paraformer/model.py deleted file mode 100644 index a6879b7d9924031899da04ac9054014145c59fb4..0000000000000000000000000000000000000000 --- a/funasr_detach/models/contextual_paraformer/model.py +++ /dev/null @@ -1,634 +0,0 @@ -#!/usr/bin/env python3 -# -*- encoding: utf-8 -*- -# Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved. -# MIT License (https://opensource.org/licenses/MIT) - -import os -import re -import time -import torch -import codecs -import logging -import tempfile -import requests -import numpy as np -from typing import Dict, Tuple -from contextlib import contextmanager -from distutils.version import LooseVersion - -from funasr_detach.register import tables -from funasr_detach.losses.label_smoothing_loss import ( - LabelSmoothingLoss, # noqa: H301 -) -from funasr_detach.utils import postprocess_utils -from funasr_detach.metrics.compute_acc import th_accuracy -from funasr_detach.models.paraformer.model import Paraformer -from funasr_detach.utils.datadir_writer import DatadirWriter -from funasr_detach.models.paraformer.search import Hypothesis -from funasr_detach.train_utils.device_funcs import force_gatherable -from funasr_detach.models.transformer.utils.add_sos_eos import add_sos_eos -from funasr_detach.models.transformer.utils.nets_utils import make_pad_mask, pad_list -from funasr_detach.utils.load_utils import load_audio_text_image_video, extract_fbank - - -if LooseVersion(torch.__version__) >= LooseVersion("1.6.0"): - from torch.cuda.amp import autocast -else: - # Nothing to do if torch<1.6.0 - @contextmanager - def autocast(enabled=True): - yield - - -@tables.register("model_classes", "ContextualParaformer") -class ContextualParaformer(Paraformer): - """ - Author: Speech Lab of DAMO Academy, Alibaba Group - FunASR: A Fundamental End-to-End Speech Recognition Toolkit - https://arxiv.org/abs/2305.11013 - """ - - def __init__( - self, - *args, - **kwargs, - ): - super().__init__(*args, **kwargs) - - self.target_buffer_length = kwargs.get("target_buffer_length", -1) - inner_dim = kwargs.get("inner_dim", 256) - bias_encoder_type = kwargs.get("bias_encoder_type", "lstm") - use_decoder_embedding = kwargs.get("use_decoder_embedding", False) - crit_attn_weight = kwargs.get("crit_attn_weight", 0.0) - crit_attn_smooth = kwargs.get("crit_attn_smooth", 0.0) - bias_encoder_dropout_rate = kwargs.get("bias_encoder_dropout_rate", 0.0) - - if bias_encoder_type == "lstm": - self.bias_encoder = torch.nn.LSTM( - inner_dim, - inner_dim, - 1, - batch_first=True, - dropout=bias_encoder_dropout_rate, - ) - self.bias_embed = torch.nn.Embedding(self.vocab_size, inner_dim) - elif bias_encoder_type == "mean": - self.bias_embed = torch.nn.Embedding(self.vocab_size, inner_dim) - else: - logging.error("Unsupport bias encoder type: {}".format(bias_encoder_type)) - - if self.target_buffer_length > 0: - self.hotword_buffer = None - self.length_record = [] - self.current_buffer_length = 0 - self.use_decoder_embedding = use_decoder_embedding - self.crit_attn_weight = crit_attn_weight - if self.crit_attn_weight > 0: - self.attn_loss = torch.nn.L1Loss() - self.crit_attn_smooth = crit_attn_smooth - - def forward( - self, - speech: torch.Tensor, - speech_lengths: torch.Tensor, - text: torch.Tensor, - text_lengths: torch.Tensor, - **kwargs, - ) -> Tuple[torch.Tensor, Dict[str, torch.Tensor], torch.Tensor]: - """Frontend + Encoder + Decoder + Calc loss - - Args: - speech: (Batch, Length, ...) - speech_lengths: (Batch, ) - text: (Batch, Length) - text_lengths: (Batch,) - """ - if len(text_lengths.size()) > 1: - text_lengths = text_lengths[:, 0] - if len(speech_lengths.size()) > 1: - speech_lengths = speech_lengths[:, 0] - - batch_size = speech.shape[0] - - hotword_pad = kwargs.get("hotword_pad") - hotword_lengths = kwargs.get("hotword_lengths") - dha_pad = kwargs.get("dha_pad") - - # 1. Encoder - encoder_out, encoder_out_lens = self.encode(speech, speech_lengths) - - loss_ctc, cer_ctc = None, None - - stats = dict() - - # 1. CTC branch - if self.ctc_weight != 0.0: - loss_ctc, cer_ctc = self._calc_ctc_loss( - encoder_out, encoder_out_lens, text, text_lengths - ) - - # Collect CTC branch stats - stats["loss_ctc"] = loss_ctc.detach() if loss_ctc is not None else None - stats["cer_ctc"] = cer_ctc - - # 2b. Attention decoder branch - loss_att, acc_att, cer_att, wer_att, loss_pre, loss_ideal = ( - self._calc_att_clas_loss( - encoder_out, - encoder_out_lens, - text, - text_lengths, - hotword_pad, - hotword_lengths, - ) - ) - - # 3. CTC-Att loss definition - if self.ctc_weight == 0.0: - loss = loss_att + loss_pre * self.predictor_weight - else: - loss = ( - self.ctc_weight * loss_ctc - + (1 - self.ctc_weight) * loss_att - + loss_pre * self.predictor_weight - ) - - if loss_ideal is not None: - loss = loss + loss_ideal * self.crit_attn_weight - stats["loss_ideal"] = loss_ideal.detach().cpu() - - # Collect Attn branch stats - stats["loss_att"] = loss_att.detach() if loss_att is not None else None - stats["acc"] = acc_att - stats["cer"] = cer_att - stats["wer"] = wer_att - stats["loss_pre"] = loss_pre.detach().cpu() if loss_pre is not None else None - - stats["loss"] = torch.clone(loss.detach()) - # force_gatherable: to-device and to-tensor if scalar for DataParallel - if self.length_normalized_loss: - batch_size = int((text_lengths + self.predictor_bias).sum()) - - loss, stats, weight = force_gatherable((loss, stats, batch_size), loss.device) - return loss, stats, weight - - def _calc_att_clas_loss( - self, - encoder_out: torch.Tensor, - encoder_out_lens: torch.Tensor, - ys_pad: torch.Tensor, - ys_pad_lens: torch.Tensor, - hotword_pad: torch.Tensor, - hotword_lengths: torch.Tensor, - ): - encoder_out_mask = ( - ~make_pad_mask(encoder_out_lens, maxlen=encoder_out.size(1))[:, None, :] - ).to(encoder_out.device) - if self.predictor_bias == 1: - _, ys_pad = add_sos_eos(ys_pad, self.sos, self.eos, self.ignore_id) - ys_pad_lens = ys_pad_lens + self.predictor_bias - pre_acoustic_embeds, pre_token_length, _, _ = self.predictor( - encoder_out, ys_pad, encoder_out_mask, ignore_id=self.ignore_id - ) - - # -1. bias encoder - if self.use_decoder_embedding: - hw_embed = self.decoder.embed(hotword_pad) - else: - hw_embed = self.bias_embed(hotword_pad) - hw_embed, (_, _) = self.bias_encoder(hw_embed) - _ind = np.arange(0, hotword_pad.shape[0]).tolist() - selected = hw_embed[ - _ind, [i - 1 for i in hotword_lengths.detach().cpu().tolist()] - ] - contextual_info = ( - selected.squeeze(0).repeat(ys_pad.shape[0], 1, 1).to(ys_pad.device) - ) - - # 0. sampler - decoder_out_1st = None - if self.sampling_ratio > 0.0: - if self.step_cur < 2: - logging.info( - "enable sampler in paraformer, sampling_ratio: {}".format( - self.sampling_ratio - ) - ) - sematic_embeds, decoder_out_1st = self.sampler( - encoder_out, - encoder_out_lens, - ys_pad, - ys_pad_lens, - pre_acoustic_embeds, - contextual_info, - ) - else: - if self.step_cur < 2: - logging.info( - "disable sampler in paraformer, sampling_ratio: {}".format( - self.sampling_ratio - ) - ) - sematic_embeds = pre_acoustic_embeds - - # 1. Forward decoder - decoder_outs = self.decoder( - encoder_out, - encoder_out_lens, - sematic_embeds, - ys_pad_lens, - contextual_info=contextual_info, - ) - decoder_out, _ = decoder_outs[0], decoder_outs[1] - """ - if self.crit_attn_weight > 0 and attn.shape[-1] > 1: - ideal_attn = ideal_attn + self.crit_attn_smooth / (self.crit_attn_smooth + 1.0) - attn_non_blank = attn[:,:,:,:-1] - ideal_attn_non_blank = ideal_attn[:,:,:-1] - loss_ideal = self.attn_loss(attn_non_blank.max(1)[0], ideal_attn_non_blank.to(attn.device)) - else: - loss_ideal = None - """ - loss_ideal = None - - if decoder_out_1st is None: - decoder_out_1st = decoder_out - # 2. Compute attention loss - loss_att = self.criterion_att(decoder_out, ys_pad) - acc_att = th_accuracy( - decoder_out_1st.view(-1, self.vocab_size), - ys_pad, - ignore_label=self.ignore_id, - ) - loss_pre = self.criterion_pre( - ys_pad_lens.type_as(pre_token_length), pre_token_length - ) - - # Compute cer/wer using attention-decoder - if self.training or self.error_calculator is None: - cer_att, wer_att = None, None - else: - ys_hat = decoder_out_1st.argmax(dim=-1) - cer_att, wer_att = self.error_calculator(ys_hat.cpu(), ys_pad.cpu()) - - return loss_att, acc_att, cer_att, wer_att, loss_pre, loss_ideal - - def sampler( - self, - encoder_out, - encoder_out_lens, - ys_pad, - ys_pad_lens, - pre_acoustic_embeds, - contextual_info, - ): - tgt_mask = ( - ~make_pad_mask(ys_pad_lens, maxlen=ys_pad_lens.max())[:, :, None] - ).to(ys_pad.device) - ys_pad = ys_pad * tgt_mask[:, :, 0] - if self.share_embedding: - ys_pad_embed = self.decoder.output_layer.weight[ys_pad] - else: - ys_pad_embed = self.decoder.embed(ys_pad) - with torch.no_grad(): - decoder_outs = self.decoder( - encoder_out, - encoder_out_lens, - pre_acoustic_embeds, - ys_pad_lens, - contextual_info=contextual_info, - ) - decoder_out, _ = decoder_outs[0], decoder_outs[1] - pred_tokens = decoder_out.argmax(-1) - nonpad_positions = ys_pad.ne(self.ignore_id) - seq_lens = (nonpad_positions).sum(1) - same_num = ((pred_tokens == ys_pad) & nonpad_positions).sum(1) - input_mask = torch.ones_like(nonpad_positions) - bsz, seq_len = ys_pad.size() - for li in range(bsz): - target_num = ( - ((seq_lens[li] - same_num[li].sum()).float()) * self.sampling_ratio - ).long() - if target_num > 0: - input_mask[li].scatter_( - dim=0, - index=torch.randperm(seq_lens[li])[:target_num].to( - pre_acoustic_embeds.device - ), - value=0, - ) - input_mask = input_mask.eq(1) - input_mask = input_mask.masked_fill(~nonpad_positions, False) - input_mask_expand_dim = input_mask.unsqueeze(2).to( - pre_acoustic_embeds.device - ) - - sematic_embeds = pre_acoustic_embeds.masked_fill( - ~input_mask_expand_dim, 0 - ) + ys_pad_embed.masked_fill(input_mask_expand_dim, 0) - return sematic_embeds * tgt_mask, decoder_out * tgt_mask - - def cal_decoder_with_predictor( - self, - encoder_out, - encoder_out_lens, - sematic_embeds, - ys_pad_lens, - hw_list=None, - clas_scale=1.0, - ): - if hw_list is None: - hw_list = [ - torch.Tensor([1]).long().to(encoder_out.device) - ] # empty hotword list - hw_list_pad = pad_list(hw_list, 0) - if self.use_decoder_embedding: - hw_embed = self.decoder.embed(hw_list_pad) - else: - hw_embed = self.bias_embed(hw_list_pad) - hw_embed, (h_n, _) = self.bias_encoder(hw_embed) - hw_embed = h_n.repeat(encoder_out.shape[0], 1, 1) - else: - hw_lengths = [len(i) for i in hw_list] - hw_list_pad = pad_list([torch.Tensor(i).long() for i in hw_list], 0).to( - encoder_out.device - ) - if self.use_decoder_embedding: - hw_embed = self.decoder.embed(hw_list_pad) - else: - hw_embed = self.bias_embed(hw_list_pad) - hw_embed = torch.nn.utils.rnn.pack_padded_sequence( - hw_embed, hw_lengths, batch_first=True, enforce_sorted=False - ) - _, (h_n, _) = self.bias_encoder(hw_embed) - hw_embed = h_n.repeat(encoder_out.shape[0], 1, 1) - - decoder_outs = self.decoder( - encoder_out, - encoder_out_lens, - sematic_embeds, - ys_pad_lens, - contextual_info=hw_embed, - clas_scale=clas_scale, - ) - decoder_out = decoder_outs[0] - decoder_out = torch.log_softmax(decoder_out, dim=-1) - return decoder_out, ys_pad_lens - - def inference( - self, - data_in, - data_lengths=None, - key: list = None, - tokenizer=None, - frontend=None, - **kwargs, - ): - # init beamsearch - is_use_ctc = ( - kwargs.get("decoding_ctc_weight", 0.0) > 0.00001 and self.ctc != None - ) - is_use_lm = ( - kwargs.get("lm_weight", 0.0) > 0.00001 - and kwargs.get("lm_file", None) is not None - ) - if self.beam_search is None and (is_use_lm or is_use_ctc): - logging.info("enable beam_search") - self.init_beam_search(**kwargs) - self.nbest = kwargs.get("nbest", 1) - - meta_data = {} - - # extract fbank feats - time1 = time.perf_counter() - audio_sample_list = load_audio_text_image_video( - data_in, fs=frontend.fs, audio_fs=kwargs.get("fs", 16000) - ) - time2 = time.perf_counter() - meta_data["load_data"] = f"{time2 - time1:0.3f}" - speech, speech_lengths = extract_fbank( - audio_sample_list, - data_type=kwargs.get("data_type", "sound"), - frontend=frontend, - ) - time3 = time.perf_counter() - meta_data["extract_feat"] = f"{time3 - time2:0.3f}" - meta_data["batch_data_time"] = ( - speech_lengths.sum().item() * frontend.frame_shift * frontend.lfr_n / 1000 - ) - - speech = speech.to(device=kwargs["device"]) - speech_lengths = speech_lengths.to(device=kwargs["device"]) - - # hotword - self.hotword_list = self.generate_hotwords_list( - kwargs.get("hotword", None), tokenizer=tokenizer, frontend=frontend - ) - - # Encoder - encoder_out, encoder_out_lens = self.encode(speech, speech_lengths) - if isinstance(encoder_out, tuple): - encoder_out = encoder_out[0] - - # predictor - predictor_outs = self.calc_predictor(encoder_out, encoder_out_lens) - pre_acoustic_embeds, pre_token_length, alphas, pre_peak_index = ( - predictor_outs[0], - predictor_outs[1], - predictor_outs[2], - predictor_outs[3], - ) - pre_token_length = pre_token_length.round().long() - if torch.max(pre_token_length) < 1: - return [] - - decoder_outs = self.cal_decoder_with_predictor( - encoder_out, - encoder_out_lens, - pre_acoustic_embeds, - pre_token_length, - hw_list=self.hotword_list, - clas_scale=kwargs.get("clas_scale", 1.0), - ) - decoder_out, ys_pad_lens = decoder_outs[0], decoder_outs[1] - - results = [] - b, n, d = decoder_out.size() - for i in range(b): - x = encoder_out[i, : encoder_out_lens[i], :] - am_scores = decoder_out[i, : pre_token_length[i], :] - if self.beam_search is not None: - nbest_hyps = self.beam_search( - x=x, - am_scores=am_scores, - maxlenratio=kwargs.get("maxlenratio", 0.0), - minlenratio=kwargs.get("minlenratio", 0.0), - ) - - nbest_hyps = nbest_hyps[: self.nbest] - else: - - yseq = am_scores.argmax(dim=-1) - score = am_scores.max(dim=-1)[0] - score = torch.sum(score, dim=-1) - # pad with mask tokens to ensure compatibility with sos/eos tokens - yseq = torch.tensor( - [self.sos] + yseq.tolist() + [self.eos], device=yseq.device - ) - nbest_hyps = [Hypothesis(yseq=yseq, score=score)] - for nbest_idx, hyp in enumerate(nbest_hyps): - ibest_writer = None - if kwargs.get("output_dir") is not None: - if not hasattr(self, "writer"): - self.writer = DatadirWriter(kwargs.get("output_dir")) - ibest_writer = self.writer[f"{nbest_idx + 1}best_recog"] - - # remove sos/eos and get results - last_pos = -1 - if isinstance(hyp.yseq, list): - token_int = hyp.yseq[1:last_pos] - else: - token_int = hyp.yseq[1:last_pos].tolist() - - # remove blank symbol id, which is assumed to be 0 - token_int = list( - filter( - lambda x: x != self.eos - and x != self.sos - and x != self.blank_id, - token_int, - ) - ) - - if tokenizer is not None: - # Change integer-ids to tokens - token = tokenizer.ids2tokens(token_int) - text = tokenizer.tokens2text(token) - - text_postprocessed, _ = postprocess_utils.sentence_postprocess( - token - ) - result_i = {"key": key[i], "text": text_postprocessed} - - if ibest_writer is not None: - ibest_writer["token"][key[i]] = " ".join(token) - ibest_writer["text"][key[i]] = text - ibest_writer["text_postprocessed"][key[i]] = text_postprocessed - else: - result_i = {"key": key[i], "token_int": token_int} - results.append(result_i) - - return results, meta_data - - def generate_hotwords_list( - self, hotword_list_or_file, tokenizer=None, frontend=None - ): - def load_seg_dict(seg_dict_file): - seg_dict = {} - assert isinstance(seg_dict_file, str) - with open(seg_dict_file, "r", encoding="utf8") as f: - lines = f.readlines() - for line in lines: - s = line.strip().split() - key = s[0] - value = s[1:] - seg_dict[key] = " ".join(value) - return seg_dict - - def seg_tokenize(txt, seg_dict): - pattern = re.compile(r"^[\u4E00-\u9FA50-9]+$") - out_txt = "" - for word in txt: - word = word.lower() - if word in seg_dict: - out_txt += seg_dict[word] + " " - else: - if pattern.match(word): - for char in word: - if char in seg_dict: - out_txt += seg_dict[char] + " " - else: - out_txt += "" + " " - else: - out_txt += "" + " " - return out_txt.strip().split() - - seg_dict = None - if frontend.cmvn_file is not None: - model_dir = os.path.dirname(frontend.cmvn_file) - seg_dict_file = os.path.join(model_dir, "seg_dict") - if os.path.exists(seg_dict_file): - seg_dict = load_seg_dict(seg_dict_file) - else: - seg_dict = None - # for None - if hotword_list_or_file is None: - hotword_list = None - # for local txt inputs - elif os.path.exists(hotword_list_or_file) and hotword_list_or_file.endswith( - ".txt" - ): - logging.info("Attempting to parse hotwords from local txt...") - hotword_list = [] - hotword_str_list = [] - with codecs.open(hotword_list_or_file, "r") as fin: - for line in fin.readlines(): - hw = line.strip() - hw_list = hw.split() - if seg_dict is not None: - hw_list = seg_tokenize(hw_list, seg_dict) - hotword_str_list.append(hw) - hotword_list.append(tokenizer.tokens2ids(hw_list)) - hotword_list.append([self.sos]) - hotword_str_list.append("") - logging.info( - "Initialized hotword list from file: {}, hotword list: {}.".format( - hotword_list_or_file, hotword_str_list - ) - ) - # for url, download and generate txt - elif hotword_list_or_file.startswith("http"): - logging.info("Attempting to parse hotwords from url...") - work_dir = tempfile.TemporaryDirectory().name - if not os.path.exists(work_dir): - os.makedirs(work_dir) - text_file_path = os.path.join( - work_dir, os.path.basename(hotword_list_or_file) - ) - local_file = requests.get(hotword_list_or_file) - open(text_file_path, "wb").write(local_file.content) - hotword_list_or_file = text_file_path - hotword_list = [] - hotword_str_list = [] - with codecs.open(hotword_list_or_file, "r") as fin: - for line in fin.readlines(): - hw = line.strip() - hw_list = hw.split() - if seg_dict is not None: - hw_list = seg_tokenize(hw_list, seg_dict) - hotword_str_list.append(hw) - hotword_list.append(tokenizer.tokens2ids(hw_list)) - hotword_list.append([self.sos]) - hotword_str_list.append("") - logging.info( - "Initialized hotword list from file: {}, hotword list: {}.".format( - hotword_list_or_file, hotword_str_list - ) - ) - # for text str input - elif not hotword_list_or_file.endswith(".txt"): - logging.info("Attempting to parse hotwords as str...") - hotword_list = [] - hotword_str_list = [] - for hw in hotword_list_or_file.strip().split(): - hotword_str_list.append(hw) - hw_list = hw.strip().split() - if seg_dict is not None: - hw_list = seg_tokenize(hw_list, seg_dict) - hotword_list.append(tokenizer.tokens2ids(hw_list)) - hotword_list.append([self.sos]) - hotword_str_list.append("") - logging.info("Hotword list: {}.".format(hotword_str_list)) - else: - hotword_list = None - return hotword_list diff --git a/funasr_detach/models/contextual_paraformer/template.yaml b/funasr_detach/models/contextual_paraformer/template.yaml deleted file mode 100644 index fbd15cea9ab3cf5416197eeabd83f785a2936d2e..0000000000000000000000000000000000000000 --- a/funasr_detach/models/contextual_paraformer/template.yaml +++ /dev/null @@ -1,129 +0,0 @@ -# This is an example that demonstrates how to configure a model file. -# You can modify the configuration according to your own requirements. - -# to print the register_table: -# from funasr.register import tables -# tables.print() - -# network architecture -model: ContextualParaformer -model_conf: - ctc_weight: 0.0 - lsm_weight: 0.1 - length_normalized_loss: true - predictor_weight: 1.0 - predictor_bias: 1 - sampling_ratio: 0.75 - inner_dim: 512 - -# encoder -encoder: SANMEncoder -encoder_conf: - output_size: 512 - attention_heads: 4 - linear_units: 2048 - num_blocks: 50 - dropout_rate: 0.1 - positional_dropout_rate: 0.1 - attention_dropout_rate: 0.1 - input_layer: pe - pos_enc_class: SinusoidalPositionEncoder - normalize_before: true - kernel_size: 11 - sanm_shfit: 0 - selfattention_layer_type: sanm - - -# decoder -decoder: ContextualParaformerDecoder -decoder_conf: - attention_heads: 4 - linear_units: 2048 - num_blocks: 16 - dropout_rate: 0.1 - positional_dropout_rate: 0.1 - self_attention_dropout_rate: 0.1 - src_attention_dropout_rate: 0.1 - att_layer_num: 16 - kernel_size: 11 - sanm_shfit: 0 - -predictor: CifPredictorV2 -predictor_conf: - idim: 512 - threshold: 1.0 - l_order: 1 - r_order: 1 - tail_threshold: 0.45 - -# frontend related -frontend: WavFrontend -frontend_conf: - fs: 16000 - window: hamming - n_mels: 80 - frame_length: 25 - frame_shift: 10 - lfr_m: 7 - lfr_n: 6 - -specaug: SpecAugLFR -specaug_conf: - apply_time_warp: false - time_warp_window: 5 - time_warp_mode: bicubic - apply_freq_mask: true - freq_mask_width_range: - - 0 - - 30 - lfr_rate: 6 - num_freq_mask: 1 - apply_time_mask: true - time_mask_width_range: - - 0 - - 12 - num_time_mask: 1 - -train_conf: - accum_grad: 1 - grad_clip: 5 - max_epoch: 150 - val_scheduler_criterion: - - valid - - acc - best_model_criterion: - - - valid - - acc - - max - keep_nbest_models: 10 - log_interval: 50 - -optim: adam -optim_conf: - lr: 0.0005 -scheduler: warmuplr -scheduler_conf: - warmup_steps: 30000 - -dataset: AudioDataset -dataset_conf: - index_ds: IndexDSJsonl - batch_sampler: DynamicBatchLocalShuffleSampler - batch_type: example # example or length - batch_size: 1 # if batch_type is example, batch_size is the numbers of samples; if length, batch_size is source_token_len+target_token_len; - max_token_length: 2048 # filter samples if source_token_len+target_token_len > max_token_length, - buffer_size: 500 - shuffle: True - num_workers: 0 - -tokenizer: CharTokenizer -tokenizer_conf: - unk_symbol: - split_with_space: true - -ctc_conf: - dropout_rate: 0.0 - ctc_type: builtin - reduce: true - ignore_nan_grad: true -normalize: null \ No newline at end of file diff --git a/funasr_detach/models/ct_transformer/__init__.py b/funasr_detach/models/ct_transformer/__init__.py deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/funasr_detach/models/ct_transformer/model.py b/funasr_detach/models/ct_transformer/model.py deleted file mode 100644 index 4cbc6d597d305770dc469ad91b12dd66c11fce6e..0000000000000000000000000000000000000000 --- a/funasr_detach/models/ct_transformer/model.py +++ /dev/null @@ -1,425 +0,0 @@ -#!/usr/bin/env python3 -# -*- encoding: utf-8 -*- -# Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved. -# MIT License (https://opensource.org/licenses/MIT) - -import torch -import numpy as np -import torch.nn.functional as F -from contextlib import contextmanager -from distutils.version import LooseVersion -from typing import Any, List, Tuple, Optional - -from funasr_detach.register import tables -from funasr_detach.train_utils.device_funcs import to_device -from funasr_detach.train_utils.device_funcs import force_gatherable -from funasr_detach.utils.load_utils import load_audio_text_image_video -from funasr_detach.models.transformer.utils.nets_utils import make_pad_mask -from funasr_detach.models.ct_transformer.utils import ( - split_to_mini_sentence, - split_words, -) - -import jieba as jieba - -load_jieba = False - - -if LooseVersion(torch.__version__) >= LooseVersion("1.6.0"): - from torch.cuda.amp import autocast -else: - # Nothing to do if torch<1.6.0 - @contextmanager - def autocast(enabled=True): - yield - - -@tables.register("model_classes", "CTTransformer") -class CTTransformer(torch.nn.Module): - """ - Author: Speech Lab of DAMO Academy, Alibaba Group - CT-Transformer: Controllable time-delay transformer for real-time punctuation prediction and disfluency detection - https://arxiv.org/pdf/2003.01309.pdf - """ - - def __init__( - self, - encoder: str = None, - encoder_conf: dict = None, - vocab_size: int = -1, - punc_list: list = None, - punc_weight: list = None, - embed_unit: int = 128, - att_unit: int = 256, - dropout_rate: float = 0.5, - ignore_id: int = -1, - sos: int = 1, - eos: int = 2, - sentence_end_id: int = 3, - **kwargs, - ): - super().__init__() - - punc_size = len(punc_list) - if punc_weight is None: - punc_weight = [1] * punc_size - - self.embed = torch.nn.Embedding(vocab_size, embed_unit) - encoder_class = tables.encoder_classes.get(encoder) - encoder = encoder_class(**encoder_conf) - - self.decoder = torch.nn.Linear(att_unit, punc_size) - self.encoder = encoder - self.punc_list = punc_list - self.punc_weight = punc_weight - self.ignore_id = ignore_id - self.sos = sos - self.eos = eos - self.sentence_end_id = sentence_end_id - - def punc_forward(self, text: torch.Tensor, text_lengths: torch.Tensor, **kwargs): - """Compute loss value from buffer sequences. - - Args: - input (torch.Tensor): Input ids. (batch, len) - hidden (torch.Tensor): Target ids. (batch, len) - - """ - x = self.embed(text) - # mask = self._target_mask(input) - h, _, _ = self.encoder(x, text_lengths) - y = self.decoder(h) - return y, None - - def with_vad(self): - return False - - def score( - self, y: torch.Tensor, state: Any, x: torch.Tensor - ) -> Tuple[torch.Tensor, Any]: - """Score new token. - - Args: - y (torch.Tensor): 1D torch.int64 prefix tokens. - state: Scorer state for prefix tokens - x (torch.Tensor): encoder feature that generates ys. - - Returns: - tuple[torch.Tensor, Any]: Tuple of - torch.float32 scores for next token (vocab_size) - and next state for ys - - """ - y = y.unsqueeze(0) - h, _, cache = self.encoder.forward_one_step( - self.embed(y), self._target_mask(y), cache=state - ) - h = self.decoder(h[:, -1]) - logp = h.log_softmax(dim=-1).squeeze(0) - return logp, cache - - def batch_score( - self, ys: torch.Tensor, states: List[Any], xs: torch.Tensor - ) -> Tuple[torch.Tensor, List[Any]]: - """Score new token batch. - - Args: - ys (torch.Tensor): torch.int64 prefix tokens (n_batch, ylen). - states (List[Any]): Scorer states for prefix tokens. - xs (torch.Tensor): - The encoder feature that generates ys (n_batch, xlen, n_feat). - - Returns: - tuple[torch.Tensor, List[Any]]: Tuple of - batchfied scores for next token with shape of `(n_batch, vocab_size)` - and next state list for ys. - - """ - # merge states - n_batch = len(ys) - n_layers = len(self.encoder.encoders) - if states[0] is None: - batch_state = None - else: - # transpose state of [batch, layer] into [layer, batch] - batch_state = [ - torch.stack([states[b][i] for b in range(n_batch)]) - for i in range(n_layers) - ] - - # batch decoding - h, _, states = self.encoder.forward_one_step( - self.embed(ys), self._target_mask(ys), cache=batch_state - ) - h = self.decoder(h[:, -1]) - logp = h.log_softmax(dim=-1) - - # transpose state of [layer, batch] into [batch, layer] - state_list = [[states[i][b] for i in range(n_layers)] for b in range(n_batch)] - return logp, state_list - - def nll( - self, - text: torch.Tensor, - punc: torch.Tensor, - text_lengths: torch.Tensor, - punc_lengths: torch.Tensor, - max_length: Optional[int] = None, - vad_indexes: Optional[torch.Tensor] = None, - vad_indexes_lengths: Optional[torch.Tensor] = None, - ) -> Tuple[torch.Tensor, torch.Tensor]: - """Compute negative log likelihood(nll) - - Normally, this function is called in batchify_nll. - Args: - text: (Batch, Length) - punc: (Batch, Length) - text_lengths: (Batch,) - max_lengths: int - """ - batch_size = text.size(0) - # For data parallel - if max_length is None: - text = text[:, : text_lengths.max()] - punc = punc[:, : text_lengths.max()] - else: - text = text[:, :max_length] - punc = punc[:, :max_length] - - if self.with_vad(): - # Should be VadRealtimeTransformer - assert vad_indexes is not None - y, _ = self.punc_forward(text, text_lengths, vad_indexes) - else: - # Should be TargetDelayTransformer, - y, _ = self.punc_forward(text, text_lengths) - - # Calc negative log likelihood - # nll: (BxL,) - if self.training == False: - _, indices = y.view(-1, y.shape[-1]).topk(1, dim=1) - from sklearn.metrics import f1_score - - f1_score = f1_score( - punc.view(-1).detach().cpu().numpy(), - indices.squeeze(-1).detach().cpu().numpy(), - average="micro", - ) - nll = torch.Tensor([f1_score]).repeat(text_lengths.sum()) - return nll, text_lengths - else: - self.punc_weight = self.punc_weight.to(punc.device) - nll = F.cross_entropy( - y.view(-1, y.shape[-1]), - punc.view(-1), - self.punc_weight, - reduction="none", - ignore_index=self.ignore_id, - ) - # nll: (BxL,) -> (BxL,) - if max_length is None: - nll.masked_fill_(make_pad_mask(text_lengths).to(nll.device).view(-1), 0.0) - else: - nll.masked_fill_( - make_pad_mask(text_lengths, maxlen=max_length + 1) - .to(nll.device) - .view(-1), - 0.0, - ) - # nll: (BxL,) -> (B, L) - nll = nll.view(batch_size, -1) - return nll, text_lengths - - def forward( - self, - text: torch.Tensor, - punc: torch.Tensor, - text_lengths: torch.Tensor, - punc_lengths: torch.Tensor, - vad_indexes: Optional[torch.Tensor] = None, - vad_indexes_lengths: Optional[torch.Tensor] = None, - ): - nll, y_lengths = self.nll( - text, punc, text_lengths, punc_lengths, vad_indexes=vad_indexes - ) - ntokens = y_lengths.sum() - loss = nll.sum() / ntokens - stats = dict(loss=loss.detach()) - - # force_gatherable: to-device and to-tensor if scalar for DataParallel - loss, stats, weight = force_gatherable((loss, stats, ntokens), loss.device) - return loss, stats, weight - - def inference( - self, - data_in, - data_lengths=None, - key: list = None, - tokenizer=None, - frontend=None, - **kwargs, - ): - assert len(data_in) == 1 - text = load_audio_text_image_video( - data_in, data_type=kwargs.get("kwargs", "text") - )[0] - vad_indexes = kwargs.get("vad_indexes", None) - # text = data_in[0] - # text_lengths = data_lengths[0] if data_lengths is not None else None - split_size = kwargs.get("split_size", 20) - - jieba_usr_dict = kwargs.get("jieba_usr_dict", None) - global load_jieba - if load_jieba: - jieba_usr_dict = jieba - kwargs["jieba_usr_dict"] = "jieba_usr_dict" - else: - if jieba_usr_dict and isinstance(jieba_usr_dict, str): - # import jieba - jieba.load_userdict(jieba_usr_dict) - jieba_usr_dict = jieba - kwargs["jieba_usr_dict"] = "jieba_usr_dict" - load_jieba = True - tokens = split_words(text, jieba_usr_dict=jieba_usr_dict) - tokens_int = tokenizer.encode(tokens) - - mini_sentences = split_to_mini_sentence(tokens, split_size) - mini_sentences_id = split_to_mini_sentence(tokens_int, split_size) - assert len(mini_sentences) == len(mini_sentences_id) - cache_sent = [] - cache_sent_id = torch.from_numpy(np.array([], dtype="int32")) - new_mini_sentence = "" - new_mini_sentence_punc = [] - cache_pop_trigger_limit = 200 - results = [] - meta_data = {} - punc_array = None - for mini_sentence_i in range(len(mini_sentences)): - mini_sentence = mini_sentences[mini_sentence_i] - mini_sentence_id = mini_sentences_id[mini_sentence_i] - mini_sentence = cache_sent + mini_sentence - mini_sentence_id = np.concatenate((cache_sent_id, mini_sentence_id), axis=0) - data = { - "text": torch.unsqueeze(torch.from_numpy(mini_sentence_id), 0), - "text_lengths": torch.from_numpy( - np.array([len(mini_sentence_id)], dtype="int32") - ), - } - data = to_device(data, kwargs["device"]) - # y, _ = self.wrapped_model(**data) - y, _ = self.punc_forward(**data) - _, indices = y.view(-1, y.shape[-1]).topk(1, dim=1) - punctuations = indices - if indices.size()[0] != 1: - punctuations = torch.squeeze(indices) - assert punctuations.size()[0] == len(mini_sentence) - - # Search for the last Period/QuestionMark as cache - if mini_sentence_i < len(mini_sentences) - 1: - sentenceEnd = -1 - last_comma_index = -1 - for i in range(len(punctuations) - 2, 1, -1): - if ( - self.punc_list[punctuations[i]] == "。" - or self.punc_list[punctuations[i]] == "？" - ): - sentenceEnd = i - break - if last_comma_index < 0 and self.punc_list[punctuations[i]] == "，": - last_comma_index = i - - if ( - sentenceEnd < 0 - and len(mini_sentence) > cache_pop_trigger_limit - and last_comma_index >= 0 - ): - # The sentence it too long, cut off at a comma. - sentenceEnd = last_comma_index - punctuations[sentenceEnd] = self.sentence_end_id - cache_sent = mini_sentence[sentenceEnd + 1 :] - cache_sent_id = mini_sentence_id[sentenceEnd + 1 :] - mini_sentence = mini_sentence[0 : sentenceEnd + 1] - punctuations = punctuations[0 : sentenceEnd + 1] - - # if len(punctuations) == 0: - # continue - - punctuations_np = punctuations.cpu().numpy() - new_mini_sentence_punc += [int(x) for x in punctuations_np] - words_with_punc = [] - for i in range(len(mini_sentence)): - if ( - i == 0 - or self.punc_list[punctuations[i - 1]] == "。" - or self.punc_list[punctuations[i - 1]] == "？" - ) and len(mini_sentence[i][0].encode()) == 1: - mini_sentence[i] = mini_sentence[i].capitalize() - if i == 0: - if len(mini_sentence[i][0].encode()) == 1: - mini_sentence[i] = " " + mini_sentence[i] - if i > 0: - if ( - len(mini_sentence[i][0].encode()) == 1 - and len(mini_sentence[i - 1][0].encode()) == 1 - ): - mini_sentence[i] = " " + mini_sentence[i] - words_with_punc.append(mini_sentence[i]) - if self.punc_list[punctuations[i]] != "_": - punc_res = self.punc_list[punctuations[i]] - if len(mini_sentence[i][0].encode()) == 1: - if punc_res == "，": - punc_res = "," - elif punc_res == "。": - punc_res = "." - elif punc_res == "？": - punc_res = "?" - words_with_punc.append(punc_res) - new_mini_sentence += "".join(words_with_punc) - # Add Period for the end of the sentence - new_mini_sentence_out = new_mini_sentence - new_mini_sentence_punc_out = new_mini_sentence_punc - if mini_sentence_i == len(mini_sentences) - 1: - if new_mini_sentence[-1] == "，" or new_mini_sentence[-1] == "、": - new_mini_sentence_out = new_mini_sentence[:-1] + "。" - new_mini_sentence_punc_out = new_mini_sentence_punc[:-1] + [ - self.sentence_end_id - ] - elif new_mini_sentence[-1] == ",": - new_mini_sentence_out = new_mini_sentence[:-1] + "." - new_mini_sentence_punc_out = new_mini_sentence_punc[:-1] + [ - self.sentence_end_id - ] - elif ( - new_mini_sentence[-1] != "。" - and new_mini_sentence[-1] != "？" - and len(new_mini_sentence[-1].encode()) != 1 - ): - new_mini_sentence_out = new_mini_sentence + "。" - new_mini_sentence_punc_out = new_mini_sentence_punc[:-1] + [ - self.sentence_end_id - ] - if len(punctuations): - punctuations[-1] = 2 - elif ( - new_mini_sentence[-1] != "." - and new_mini_sentence[-1] != "?" - and len(new_mini_sentence[-1].encode()) == 1 - ): - new_mini_sentence_out = new_mini_sentence + "." - new_mini_sentence_punc_out = new_mini_sentence_punc[:-1] + [ - self.sentence_end_id - ] - if len(punctuations): - punctuations[-1] = 2 - # keep a punctuations array for punc segment - if punc_array is None: - punc_array = punctuations - else: - punc_array = torch.cat([punc_array, punctuations], dim=0) - result_i = { - "key": key[0], - "text": new_mini_sentence_out, - "punc_array": punc_array, - } - results.append(result_i) - return results, meta_data diff --git a/funasr_detach/models/ct_transformer/template.yaml b/funasr_detach/models/ct_transformer/template.yaml deleted file mode 100644 index 2538e6b1d4e7ef0ef41742214772b536b9440194..0000000000000000000000000000000000000000 --- a/funasr_detach/models/ct_transformer/template.yaml +++ /dev/null @@ -1,53 +0,0 @@ -# This is an example that demonstrates how to configure a model file. -# You can modify the configuration according to your own requirements. - -# to print the register_table: -# from funasr.register import tables -# tables.print() - -model: CTTransformer -model_conf: - ignore_id: 0 - embed_unit: 256 - att_unit: 256 - dropout_rate: 0.1 - punc_list: - - - - _ - - ',' - - 。 - - '?' - - 、 - punc_weight: - - 1.0 - - 1.0 - - 1.0 - - 1.0 - - 1.0 - - 1.0 - sentence_end_id: 3 - -encoder: SANMEncoder -encoder_conf: - input_size: 256 - output_size: 256 - attention_heads: 8 - linear_units: 1024 - num_blocks: 4 - dropout_rate: 0.1 - positional_dropout_rate: 0.1 - attention_dropout_rate: 0.0 - input_layer: pe - pos_enc_class: SinusoidalPositionEncoder - normalize_before: true - kernel_size: 11 - sanm_shfit: 0 - selfattention_layer_type: sanm - padding_idx: 0 - -tokenizer: CharTokenizer -tokenizer_conf: - unk_symbol: - - - diff --git a/funasr_detach/models/ct_transformer/utils.py b/funasr_detach/models/ct_transformer/utils.py deleted file mode 100644 index 588aaa5944f4982ff9e4df124fc979ebd807595a..0000000000000000000000000000000000000000 --- a/funasr_detach/models/ct_transformer/utils.py +++ /dev/null @@ -1,100 +0,0 @@ -#!/usr/bin/env python3 -# -*- encoding: utf-8 -*- -# Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved. -# MIT License (https://opensource.org/licenses/MIT) - -import re - - -def split_to_mini_sentence(words: list, word_limit: int = 20): - assert word_limit > 1 - if len(words) <= word_limit: - return [words] - sentences = [] - length = len(words) - sentence_len = length // word_limit - for i in range(sentence_len): - sentences.append(words[i * word_limit : (i + 1) * word_limit]) - if length % word_limit > 0: - sentences.append(words[sentence_len * word_limit :]) - return sentences - - -def split_words(text: str, jieba_usr_dict=None, **kwargs): - if jieba_usr_dict: - input_list = text.split() - token_list_all = [] - langauge_list = [] - token_list_tmp = [] - language_flag = None - for token in input_list: - if isEnglish(token) and language_flag == "Chinese": - token_list_all.append(token_list_tmp) - langauge_list.append("Chinese") - token_list_tmp = [] - elif not isEnglish(token) and language_flag == "English": - token_list_all.append(token_list_tmp) - langauge_list.append("English") - token_list_tmp = [] - - token_list_tmp.append(token) - - if isEnglish(token): - language_flag = "English" - else: - language_flag = "Chinese" - - if token_list_tmp: - token_list_all.append(token_list_tmp) - langauge_list.append(language_flag) - - result_list = [] - for token_list_tmp, language_flag in zip(token_list_all, langauge_list): - if language_flag == "English": - result_list.extend(token_list_tmp) - else: - seg_list = jieba_usr_dict.cut( - join_chinese_and_english(token_list_tmp), HMM=False - ) - result_list.extend(seg_list) - - return result_list - - else: - words = [] - segs = text.split() - for seg in segs: - # There is no space in seg. - current_word = "" - for c in seg: - if len(c.encode()) == 1: - # This is an ASCII char. - current_word += c - else: - # This is a Chinese char. - if len(current_word) > 0: - words.append(current_word) - current_word = "" - words.append(c) - if len(current_word) > 0: - words.append(current_word) - return words - - -def isEnglish(text: str): - if re.search("^[a-zA-Z']+$", text): - return True - else: - return False - - -def join_chinese_and_english(input_list): - line = "" - for token in input_list: - if isEnglish(token): - line = line + " " + token - else: - line = line + token - - line = line.strip() - return line diff --git a/funasr_detach/models/ct_transformer_streaming/__init__.py b/funasr_detach/models/ct_transformer_streaming/__init__.py deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/funasr_detach/models/ct_transformer_streaming/attention.py b/funasr_detach/models/ct_transformer_streaming/attention.py deleted file mode 100644 index 923d16eeb8f06f1a26cb324d2a4c0fe0b63ad112..0000000000000000000000000000000000000000 --- a/funasr_detach/models/ct_transformer_streaming/attention.py +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env python3 -# -*- encoding: utf-8 -*- -# Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved. -# MIT License (https://opensource.org/licenses/MIT) - -import torch -from funasr_detach.models.sanm.attention import MultiHeadedAttentionSANM - - -class MultiHeadedAttentionSANMwithMask(MultiHeadedAttentionSANM): - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - - def forward(self, x, mask, mask_shfit_chunk=None, mask_att_chunk_encoder=None): - q_h, k_h, v_h, v = self.forward_qkv(x) - fsmn_memory = self.forward_fsmn(v, mask[0], mask_shfit_chunk) - q_h = q_h * self.d_k ** (-0.5) - scores = torch.matmul(q_h, k_h.transpose(-2, -1)) - att_outs = self.forward_attention(v_h, scores, mask[1], mask_att_chunk_encoder) - return att_outs + fsmn_memory diff --git a/funasr_detach/models/ct_transformer_streaming/encoder.py b/funasr_detach/models/ct_transformer_streaming/encoder.py deleted file mode 100644 index 159f1e85d53d13323de65ada2d8dc79322c6a750..0000000000000000000000000000000000000000 --- a/funasr_detach/models/ct_transformer_streaming/encoder.py +++ /dev/null @@ -1,410 +0,0 @@ -#!/usr/bin/env python3 -# -*- encoding: utf-8 -*- -# Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved. -# MIT License (https://opensource.org/licenses/MIT) - -import torch -from typing import List, Optional, Tuple - -from funasr_detach.register import tables -from funasr_detach.models.ctc.ctc import CTC -from funasr_detach.models.transformer.utils.repeat import repeat -from funasr_detach.models.transformer.layer_norm import LayerNorm -from funasr_detach.models.sanm.attention import MultiHeadedAttention -from funasr_detach.models.transformer.utils.nets_utils import make_pad_mask -from funasr_detach.models.transformer.utils.subsampling import check_short_utt -from funasr_detach.models.transformer.utils.subsampling import TooShortUttError -from funasr_detach.models.transformer.embedding import SinusoidalPositionEncoder -from funasr_detach.models.transformer.utils.multi_layer_conv import Conv1dLinear -from funasr_detach.models.transformer.utils.mask import subsequent_mask, vad_mask -from funasr_detach.models.transformer.utils.multi_layer_conv import MultiLayeredConv1d -from funasr_detach.models.transformer.positionwise_feed_forward import ( - PositionwiseFeedForward, -) -from funasr_detach.models.ct_transformer_streaming.attention import ( - MultiHeadedAttentionSANMwithMask, -) -from funasr_detach.models.transformer.utils.subsampling import ( - Conv2dSubsampling, - Conv2dSubsampling2, - Conv2dSubsampling6, - Conv2dSubsampling8, -) - - -class EncoderLayerSANM(torch.nn.Module): - def __init__( - self, - in_size, - size, - self_attn, - feed_forward, - dropout_rate, - normalize_before=True, - concat_after=False, - stochastic_depth_rate=0.0, - ): - """Construct an EncoderLayer object.""" - super(EncoderLayerSANM, self).__init__() - self.self_attn = self_attn - self.feed_forward = feed_forward - self.norm1 = LayerNorm(in_size) - self.norm2 = LayerNorm(size) - self.dropout = torch.nn.Dropout(dropout_rate) - self.in_size = in_size - self.size = size - self.normalize_before = normalize_before - self.concat_after = concat_after - if self.concat_after: - self.concat_linear = torch.nn.Linear(size + size, size) - self.stochastic_depth_rate = stochastic_depth_rate - self.dropout_rate = dropout_rate - - def forward( - self, x, mask, cache=None, mask_shfit_chunk=None, mask_att_chunk_encoder=None - ): - """Compute encoded features. - - Args: - x_input (torch.Tensor): Input tensor (#batch, time, size). - mask (torch.Tensor): Mask tensor for the input (#batch, time). - cache (torch.Tensor): Cache tensor of the input (#batch, time - 1, size). - - Returns: - torch.Tensor: Output tensor (#batch, time, size). - torch.Tensor: Mask tensor (#batch, time). - - """ - skip_layer = False - # with stochastic depth, residual connection `x + f(x)` becomes - # `x <- x + 1 / (1 - p) * f(x)` at training time. - stoch_layer_coeff = 1.0 - if self.training and self.stochastic_depth_rate > 0: - skip_layer = torch.rand(1).item() < self.stochastic_depth_rate - stoch_layer_coeff = 1.0 / (1 - self.stochastic_depth_rate) - - if skip_layer: - if cache is not None: - x = torch.cat([cache, x], dim=1) - return x, mask - - residual = x - if self.normalize_before: - x = self.norm1(x) - - if self.concat_after: - x_concat = torch.cat( - ( - x, - self.self_attn( - x, - mask, - mask_shfit_chunk=mask_shfit_chunk, - mask_att_chunk_encoder=mask_att_chunk_encoder, - ), - ), - dim=-1, - ) - if self.in_size == self.size: - x = residual + stoch_layer_coeff * self.concat_linear(x_concat) - else: - x = stoch_layer_coeff * self.concat_linear(x_concat) - else: - if self.in_size == self.size: - x = residual + stoch_layer_coeff * self.dropout( - self.self_attn( - x, - mask, - mask_shfit_chunk=mask_shfit_chunk, - mask_att_chunk_encoder=mask_att_chunk_encoder, - ) - ) - else: - x = stoch_layer_coeff * self.dropout( - self.self_attn( - x, - mask, - mask_shfit_chunk=mask_shfit_chunk, - mask_att_chunk_encoder=mask_att_chunk_encoder, - ) - ) - if not self.normalize_before: - x = self.norm1(x) - - residual = x - if self.normalize_before: - x = self.norm2(x) - x = residual + stoch_layer_coeff * self.dropout(self.feed_forward(x)) - if not self.normalize_before: - x = self.norm2(x) - - return x, mask, cache, mask_shfit_chunk, mask_att_chunk_encoder - - def forward_chunk(self, x, cache=None, chunk_size=None, look_back=0): - """Compute encoded features. - - Args: - x_input (torch.Tensor): Input tensor (#batch, time, size). - mask (torch.Tensor): Mask tensor for the input (#batch, time). - cache (torch.Tensor): Cache tensor of the input (#batch, time - 1, size). - - Returns: - torch.Tensor: Output tensor (#batch, time, size). - torch.Tensor: Mask tensor (#batch, time). - - """ - - residual = x - if self.normalize_before: - x = self.norm1(x) - - if self.in_size == self.size: - attn, cache = self.self_attn.forward_chunk(x, cache, chunk_size, look_back) - x = residual + attn - else: - x, cache = self.self_attn.forward_chunk(x, cache, chunk_size, look_back) - - if not self.normalize_before: - x = self.norm1(x) - - residual = x - if self.normalize_before: - x = self.norm2(x) - x = residual + self.feed_forward(x) - if not self.normalize_before: - x = self.norm2(x) - - return x, cache - - -@tables.register("encoder_classes", "SANMVadEncoder") -class SANMVadEncoder(torch.nn.Module): - """ - Author: Speech Lab of DAMO Academy, Alibaba Group - - """ - - def __init__( - self, - input_size: int, - output_size: int = 256, - attention_heads: int = 4, - linear_units: int = 2048, - num_blocks: int = 6, - dropout_rate: float = 0.1, - positional_dropout_rate: float = 0.1, - attention_dropout_rate: float = 0.0, - input_layer: Optional[str] = "conv2d", - pos_enc_class=SinusoidalPositionEncoder, - normalize_before: bool = True, - concat_after: bool = False, - positionwise_layer_type: str = "linear", - positionwise_conv_kernel_size: int = 1, - padding_idx: int = -1, - interctc_layer_idx: List[int] = [], - interctc_use_conditioning: bool = False, - kernel_size: int = 11, - sanm_shfit: int = 0, - selfattention_layer_type: str = "sanm", - ): - super().__init__() - self._output_size = output_size - - if input_layer == "linear": - self.embed = torch.nn.Sequential( - torch.nn.Linear(input_size, output_size), - torch.nn.LayerNorm(output_size), - torch.nn.Dropout(dropout_rate), - torch.nn.ReLU(), - pos_enc_class(output_size, positional_dropout_rate), - ) - elif input_layer == "conv2d": - self.embed = Conv2dSubsampling(input_size, output_size, dropout_rate) - elif input_layer == "conv2d2": - self.embed = Conv2dSubsampling2(input_size, output_size, dropout_rate) - elif input_layer == "conv2d6": - self.embed = Conv2dSubsampling6(input_size, output_size, dropout_rate) - elif input_layer == "conv2d8": - self.embed = Conv2dSubsampling8(input_size, output_size, dropout_rate) - elif input_layer == "embed": - self.embed = torch.nn.Sequential( - torch.nn.Embedding(input_size, output_size, padding_idx=padding_idx), - SinusoidalPositionEncoder(), - ) - elif input_layer is None: - if input_size == output_size: - self.embed = None - else: - self.embed = torch.nn.Linear(input_size, output_size) - elif input_layer == "pe": - self.embed = SinusoidalPositionEncoder() - else: - raise ValueError("unknown input_layer: " + input_layer) - self.normalize_before = normalize_before - if positionwise_layer_type == "linear": - positionwise_layer = PositionwiseFeedForward - positionwise_layer_args = ( - output_size, - linear_units, - dropout_rate, - ) - elif positionwise_layer_type == "conv1d": - positionwise_layer = MultiLayeredConv1d - positionwise_layer_args = ( - output_size, - linear_units, - positionwise_conv_kernel_size, - dropout_rate, - ) - elif positionwise_layer_type == "conv1d-linear": - positionwise_layer = Conv1dLinear - positionwise_layer_args = ( - output_size, - linear_units, - positionwise_conv_kernel_size, - dropout_rate, - ) - else: - raise NotImplementedError("Support only linear or conv1d.") - - if selfattention_layer_type == "selfattn": - encoder_selfattn_layer = MultiHeadedAttention - encoder_selfattn_layer_args = ( - attention_heads, - output_size, - attention_dropout_rate, - ) - - elif selfattention_layer_type == "sanm": - self.encoder_selfattn_layer = MultiHeadedAttentionSANMwithMask - encoder_selfattn_layer_args0 = ( - attention_heads, - input_size, - output_size, - attention_dropout_rate, - kernel_size, - sanm_shfit, - ) - - encoder_selfattn_layer_args = ( - attention_heads, - output_size, - output_size, - attention_dropout_rate, - kernel_size, - sanm_shfit, - ) - - self.encoders0 = repeat( - 1, - lambda lnum: EncoderLayerSANM( - input_size, - output_size, - self.encoder_selfattn_layer(*encoder_selfattn_layer_args0), - positionwise_layer(*positionwise_layer_args), - dropout_rate, - normalize_before, - concat_after, - ), - ) - - self.encoders = repeat( - num_blocks - 1, - lambda lnum: EncoderLayerSANM( - output_size, - output_size, - self.encoder_selfattn_layer(*encoder_selfattn_layer_args), - positionwise_layer(*positionwise_layer_args), - dropout_rate, - normalize_before, - concat_after, - ), - ) - if self.normalize_before: - self.after_norm = LayerNorm(output_size) - - self.interctc_layer_idx = interctc_layer_idx - if len(interctc_layer_idx) > 0: - assert 0 < min(interctc_layer_idx) and max(interctc_layer_idx) < num_blocks - self.interctc_use_conditioning = interctc_use_conditioning - self.conditioning_layer = None - self.dropout = torch.nn.Dropout(dropout_rate) - - def output_size(self) -> int: - return self._output_size - - def forward( - self, - xs_pad: torch.Tensor, - ilens: torch.Tensor, - vad_indexes: torch.Tensor, - prev_states: torch.Tensor = None, - ctc: CTC = None, - ) -> Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]: - """Embed positions in tensor. - - Args: - xs_pad: input tensor (B, L, D) - ilens: input length (B) - prev_states: Not to be used now. - Returns: - position embedded tensor and mask - """ - masks = (~make_pad_mask(ilens)[:, None, :]).to(xs_pad.device) - sub_masks = subsequent_mask(masks.size(-1), device=xs_pad.device).unsqueeze(0) - no_future_masks = masks & sub_masks - xs_pad *= self.output_size() ** 0.5 - if self.embed is None: - xs_pad = xs_pad - elif ( - isinstance(self.embed, Conv2dSubsampling) - or isinstance(self.embed, Conv2dSubsampling2) - or isinstance(self.embed, Conv2dSubsampling6) - or isinstance(self.embed, Conv2dSubsampling8) - ): - short_status, limit_size = check_short_utt(self.embed, xs_pad.size(1)) - if short_status: - raise TooShortUttError( - f"has {xs_pad.size(1)} frames and is too short for subsampling " - + f"(it needs more than {limit_size} frames), return empty results", - xs_pad.size(1), - limit_size, - ) - xs_pad, masks = self.embed(xs_pad, masks) - else: - xs_pad = self.embed(xs_pad) - - # xs_pad = self.dropout(xs_pad) - mask_tup0 = [masks, no_future_masks] - encoder_outs = self.encoders0(xs_pad, mask_tup0) - xs_pad, _ = encoder_outs[0], encoder_outs[1] - intermediate_outs = [] - - for layer_idx, encoder_layer in enumerate(self.encoders): - if layer_idx + 1 == len(self.encoders): - # This is last layer. - coner_mask = torch.ones( - masks.size(0), - masks.size(-1), - masks.size(-1), - device=xs_pad.device, - dtype=torch.bool, - ) - for word_index, length in enumerate(ilens): - coner_mask[word_index, :, :] = vad_mask( - masks.size(-1), vad_indexes[word_index], device=xs_pad.device - ) - layer_mask = masks & coner_mask - else: - layer_mask = no_future_masks - mask_tup1 = [masks, layer_mask] - encoder_outs = encoder_layer(xs_pad, mask_tup1) - xs_pad, layer_mask = encoder_outs[0], encoder_outs[1] - - if self.normalize_before: - xs_pad = self.after_norm(xs_pad) - - olens = masks.squeeze(1).sum(1) - if len(intermediate_outs) > 0: - return (xs_pad, intermediate_outs), olens, None - return xs_pad, olens, None diff --git a/funasr_detach/models/ct_transformer_streaming/model.py b/funasr_detach/models/ct_transformer_streaming/model.py deleted file mode 100644 index 7a403c7322342c3d7d951d5a3847b68c9515abd4..0000000000000000000000000000000000000000 --- a/funasr_detach/models/ct_transformer_streaming/model.py +++ /dev/null @@ -1,199 +0,0 @@ -#!/usr/bin/env python3 -# -*- encoding: utf-8 -*- -# Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved. -# MIT License (https://opensource.org/licenses/MIT) - -import torch -import numpy as np -from contextlib import contextmanager -from distutils.version import LooseVersion - -from funasr_detach.register import tables -from funasr_detach.train_utils.device_funcs import to_device -from funasr_detach.models.ct_transformer.model import CTTransformer -from funasr_detach.utils.load_utils import load_audio_text_image_video -from funasr_detach.models.ct_transformer.utils import ( - split_to_mini_sentence, - split_words, -) - - -if LooseVersion(torch.__version__) >= LooseVersion("1.6.0"): - from torch.cuda.amp import autocast -else: - # Nothing to do if torch<1.6.0 - @contextmanager - def autocast(enabled=True): - yield - - -@tables.register("model_classes", "CTTransformerStreaming") -class CTTransformerStreaming(CTTransformer): - """ - Author: Speech Lab of DAMO Academy, Alibaba Group - CT-Transformer: Controllable time-delay transformer for real-time punctuation prediction and disfluency detection - https://arxiv.org/pdf/2003.01309.pdf - """ - - def __init__( - self, - *args, - **kwargs, - ): - super().__init__(*args, **kwargs) - - def punc_forward( - self, - text: torch.Tensor, - text_lengths: torch.Tensor, - vad_indexes: torch.Tensor, - **kwargs, - ): - """Compute loss value from buffer sequences. - - Args: - input (torch.Tensor): Input ids. (batch, len) - hidden (torch.Tensor): Target ids. (batch, len) - - """ - x = self.embed(text) - # mask = self._target_mask(input) - h, _, _ = self.encoder(x, text_lengths, vad_indexes=vad_indexes) - y = self.decoder(h) - return y, None - - def with_vad(self): - return True - - def inference( - self, - data_in, - data_lengths=None, - key: list = None, - tokenizer=None, - frontend=None, - cache: dict = {}, - **kwargs, - ): - assert len(data_in) == 1 - - if len(cache) == 0: - cache["pre_text"] = [] - text = load_audio_text_image_video( - data_in, data_type=kwargs.get("kwargs", "text") - )[0] - text = "".join(cache["pre_text"]) + " " + text - - split_size = kwargs.get("split_size", 20) - - tokens = split_words(text) - tokens_int = tokenizer.encode(tokens) - - mini_sentences = split_to_mini_sentence(tokens, split_size) - mini_sentences_id = split_to_mini_sentence(tokens_int, split_size) - assert len(mini_sentences) == len(mini_sentences_id) - cache_sent = [] - cache_sent_id = torch.from_numpy(np.array([], dtype="int32")) - skip_num = 0 - sentence_punc_list = [] - sentence_words_list = [] - cache_pop_trigger_limit = 200 - results = [] - meta_data = {} - punc_array = None - for mini_sentence_i in range(len(mini_sentences)): - mini_sentence = mini_sentences[mini_sentence_i] - mini_sentence_id = mini_sentences_id[mini_sentence_i] - mini_sentence = cache_sent + mini_sentence - mini_sentence_id = np.concatenate((cache_sent_id, mini_sentence_id), axis=0) - data = { - "text": torch.unsqueeze(torch.from_numpy(mini_sentence_id), 0), - "text_lengths": torch.from_numpy( - np.array([len(mini_sentence_id)], dtype="int32") - ), - "vad_indexes": torch.from_numpy( - np.array([len(cache["pre_text"])], dtype="int32") - ), - } - data = to_device(data, kwargs["device"]) - # y, _ = self.wrapped_model(**data) - y, _ = self.punc_forward(**data) - _, indices = y.view(-1, y.shape[-1]).topk(1, dim=1) - punctuations = indices - if indices.size()[0] != 1: - punctuations = torch.squeeze(indices) - assert punctuations.size()[0] == len(mini_sentence) - - # Search for the last Period/QuestionMark as cache - if mini_sentence_i < len(mini_sentences) - 1: - sentenceEnd = -1 - last_comma_index = -1 - for i in range(len(punctuations) - 2, 1, -1): - if ( - self.punc_list[punctuations[i]] == "。" - or self.punc_list[punctuations[i]] == "？" - ): - sentenceEnd = i - break - if last_comma_index < 0 and self.punc_list[punctuations[i]] == "，": - last_comma_index = i - - if ( - sentenceEnd < 0 - and len(mini_sentence) > cache_pop_trigger_limit - and last_comma_index >= 0 - ): - # The sentence it too long, cut off at a comma. - sentenceEnd = last_comma_index - punctuations[sentenceEnd] = self.sentence_end_id - cache_sent = mini_sentence[sentenceEnd + 1 :] - cache_sent_id = mini_sentence_id[sentenceEnd + 1 :] - mini_sentence = mini_sentence[0 : sentenceEnd + 1] - punctuations = punctuations[0 : sentenceEnd + 1] - - # if len(punctuations) == 0: - # continue - - punctuations_np = punctuations.cpu().numpy() - sentence_punc_list += [self.punc_list[int(x)] for x in punctuations_np] - sentence_words_list += mini_sentence - - assert len(sentence_punc_list) == len(sentence_words_list) - words_with_punc = [] - sentence_punc_list_out = [] - for i in range(0, len(sentence_words_list)): - if i > 0: - if ( - len(sentence_words_list[i][0].encode()) == 1 - and len(sentence_words_list[i - 1][-1].encode()) == 1 - ): - sentence_words_list[i] = " " + sentence_words_list[i] - if skip_num < len(cache["pre_text"]): - skip_num += 1 - else: - words_with_punc.append(sentence_words_list[i]) - if skip_num >= len(cache["pre_text"]): - sentence_punc_list_out.append(sentence_punc_list[i]) - if sentence_punc_list[i] != "_": - words_with_punc.append(sentence_punc_list[i]) - sentence_out = "".join(words_with_punc) - - sentenceEnd = -1 - for i in range(len(sentence_punc_list) - 2, 1, -1): - if sentence_punc_list[i] == "。" or sentence_punc_list[i] == "？": - sentenceEnd = i - break - cache["pre_text"] = sentence_words_list[sentenceEnd + 1 :] - if sentence_out[-1] in self.punc_list: - sentence_out = sentence_out[:-1] - sentence_punc_list_out[-1] = "_" - # keep a punctuations array for punc segment - if punc_array is None: - punc_array = punctuations - else: - punc_array = torch.cat([punc_array, punctuations], dim=0) - - result_i = {"key": key[0], "text": sentence_out, "punc_array": punc_array} - results.append(result_i) - - return results, meta_data diff --git a/funasr_detach/models/ct_transformer_streaming/template.yaml b/funasr_detach/models/ct_transformer_streaming/template.yaml deleted file mode 100644 index 2477ac2bebd84608e3499692ef2ef81e4f88c99d..0000000000000000000000000000000000000000 --- a/funasr_detach/models/ct_transformer_streaming/template.yaml +++ /dev/null @@ -1,50 +0,0 @@ -# This is an example that demonstrates how to configure a model file. -# You can modify the configuration according to your own requirements. - -# to print the register_table: -# from funasr.register import tables -# tables.print() - -model: CTTransformerStreaming -model_conf: - ignore_id: 0 - embed_unit: 256 - att_unit: 256 - dropout_rate: 0.1 - punc_list: - - - - _ - - ， - - 。 - - ？ - - 、 - punc_weight: - - 1.0 - - 1.0 - - 1.0 - - 1.0 - - 1.0 - - 1.0 - sentence_end_id: 3 - -encoder: SANMVadEncoder -encoder_conf: - input_size: 256 - output_size: 256 - attention_heads: 8 - linear_units: 1024 - num_blocks: 3 - dropout_rate: 0.1 - positional_dropout_rate: 0.1 - attention_dropout_rate: 0.0 - input_layer: pe - pos_enc_class: SinusoidalPositionEncoder - normalize_before: true - kernel_size: 11 - sanm_shfit: 5 - selfattention_layer_type: sanm - padding_idx: 0 - -tokenizer: CharTokenizer -tokenizer_conf: - unk_symbol: \ No newline at end of file diff --git a/funasr_detach/models/ctc/__init__.py b/funasr_detach/models/ctc/__init__.py deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/funasr_detach/models/ctc/ctc.py b/funasr_detach/models/ctc/ctc.py deleted file mode 100644 index d3c10fa40956af524a7efece0c1ff2e872046cde..0000000000000000000000000000000000000000 --- a/funasr_detach/models/ctc/ctc.py +++ /dev/null @@ -1,180 +0,0 @@ -import logging - -import torch -import torch.nn.functional as F - - -class CTC(torch.nn.Module): - """CTC module. - - Args: - odim: dimension of outputs - encoder_output_size: number of encoder projection units - dropout_rate: dropout rate (0.0 ~ 1.0) - ctc_type: builtin or warpctc - reduce: reduce the CTC loss into a scalar - """ - - def __init__( - self, - odim: int, - encoder_output_size: int, - dropout_rate: float = 0.0, - ctc_type: str = "builtin", - reduce: bool = True, - ignore_nan_grad: bool = True, - ): - super().__init__() - eprojs = encoder_output_size - self.dropout_rate = dropout_rate - self.ctc_lo = torch.nn.Linear(eprojs, odim) - self.ctc_type = ctc_type - self.ignore_nan_grad = ignore_nan_grad - - if self.ctc_type == "builtin": - self.ctc_loss = torch.nn.CTCLoss(reduction="none") - elif self.ctc_type == "warpctc": - import warpctc_pytorch as warp_ctc - - if ignore_nan_grad: - logging.warning("ignore_nan_grad option is not supported for warp_ctc") - self.ctc_loss = warp_ctc.CTCLoss(size_average=True, reduce=reduce) - else: - raise ValueError( - f'ctc_type must be "builtin" or "warpctc": {self.ctc_type}' - ) - - self.reduce = reduce - - def loss_fn(self, th_pred, th_target, th_ilen, th_olen) -> torch.Tensor: - if self.ctc_type == "builtin": - th_pred = th_pred.log_softmax(2) - loss = self.ctc_loss(th_pred, th_target, th_ilen, th_olen) - - if loss.requires_grad and self.ignore_nan_grad: - # ctc_grad: (L, B, O) - ctc_grad = loss.grad_fn(torch.ones_like(loss)) - ctc_grad = ctc_grad.sum([0, 2]) - indices = torch.isfinite(ctc_grad) - size = indices.long().sum() - if size == 0: - # Return as is - logging.warning( - "All samples in this mini-batch got nan grad." - " Returning nan value instead of CTC loss" - ) - elif size != th_pred.size(1): - logging.warning( - f"{th_pred.size(1) - size}/{th_pred.size(1)}" - " samples got nan grad." - " These were ignored for CTC loss." - ) - - # Create mask for target - target_mask = torch.full( - [th_target.size(0)], - 1, - dtype=torch.bool, - device=th_target.device, - ) - s = 0 - for ind, le in enumerate(th_olen): - if not indices[ind]: - target_mask[s : s + le] = 0 - s += le - - # Calc loss again using maksed data - loss = self.ctc_loss( - th_pred[:, indices, :], - th_target[target_mask], - th_ilen[indices], - th_olen[indices], - ) - else: - size = th_pred.size(1) - - if self.reduce: - # Batch-size average - loss = loss.sum() / size - else: - loss = loss / size - return loss - - elif self.ctc_type == "warpctc": - # warpctc only supports float32 - th_pred = th_pred.to(dtype=torch.float32) - - th_target = th_target.cpu().int() - th_ilen = th_ilen.cpu().int() - th_olen = th_olen.cpu().int() - loss = self.ctc_loss(th_pred, th_target, th_ilen, th_olen) - if self.reduce: - # NOTE: sum() is needed to keep consistency since warpctc - # return as tensor w/ shape (1,) - # but builtin return as tensor w/o shape (scalar). - loss = loss.sum() - return loss - - elif self.ctc_type == "gtnctc": - log_probs = torch.nn.functional.log_softmax(th_pred, dim=2) - return self.ctc_loss(log_probs, th_target, th_ilen, 0, "none") - - else: - raise NotImplementedError - - def forward(self, hs_pad, hlens, ys_pad, ys_lens): - """Calculate CTC loss. - - Args: - hs_pad: batch of padded hidden state sequences (B, Tmax, D) - hlens: batch of lengths of hidden state sequences (B) - ys_pad: batch of padded character id sequence tensor (B, Lmax) - ys_lens: batch of lengths of character sequence (B) - """ - # hs_pad: (B, L, NProj) -> ys_hat: (B, L, Nvocab) - ys_hat = self.ctc_lo(F.dropout(hs_pad, p=self.dropout_rate)) - - if self.ctc_type == "gtnctc": - # gtn expects list form for ys - ys_true = [y[y != -1] for y in ys_pad] # parse padded ys - else: - # ys_hat: (B, L, D) -> (L, B, D) - ys_hat = ys_hat.transpose(0, 1) - # (B, L) -> (BxL,) - ys_true = torch.cat([ys_pad[i, :l] for i, l in enumerate(ys_lens)]) - - loss = self.loss_fn(ys_hat, ys_true, hlens, ys_lens).to( - device=hs_pad.device, dtype=hs_pad.dtype - ) - - return loss - - def softmax(self, hs_pad): - """softmax of frame activations - - Args: - Tensor hs_pad: 3d tensor (B, Tmax, eprojs) - Returns: - torch.Tensor: softmax applied 3d tensor (B, Tmax, odim) - """ - return F.softmax(self.ctc_lo(hs_pad), dim=2) - - def log_softmax(self, hs_pad): - """log_softmax of frame activations - - Args: - Tensor hs_pad: 3d tensor (B, Tmax, eprojs) - Returns: - torch.Tensor: log softmax applied 3d tensor (B, Tmax, odim) - """ - return F.log_softmax(self.ctc_lo(hs_pad), dim=2) - - def argmax(self, hs_pad): - """argmax of frame activations - - Args: - torch.Tensor hs_pad: 3d tensor (B, Tmax, eprojs) - Returns: - torch.Tensor: argmax applied 2d tensor (B, Tmax) - """ - return torch.argmax(self.ctc_lo(hs_pad), dim=2) diff --git a/funasr_detach/models/data2vec/__init__.py b/funasr_detach/models/data2vec/__init__.py deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/funasr_detach/models/data2vec/data2vec.py b/funasr_detach/models/data2vec/data2vec.py deleted file mode 100644 index 3dff699030ca410e2f93b5c2314197af5836ccf4..0000000000000000000000000000000000000000 --- a/funasr_detach/models/data2vec/data2vec.py +++ /dev/null @@ -1,160 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -from contextlib import contextmanager -from distutils.version import LooseVersion -from typing import Dict -from typing import Optional -from typing import Tuple - -import torch -import torch.nn as nn - -# from funasr_detach.layers.abs_normalize import AbsNormalize -# from funasr_detach.models.base_model import FunASRModel -# from funasr_detach.models.encoder.abs_encoder import AbsEncoder -from funasr_detach.frontends.abs_frontend import AbsFrontend - -# from funasr_detach.models.preencoder.abs_preencoder import AbsPreEncoder -# from funasr_detach.models.specaug.abs_specaug import AbsSpecAug -from funasr_detach.train_utils.device_funcs import force_gatherable - -if LooseVersion(torch.__version__) >= LooseVersion("1.6.0"): - from torch.cuda.amp import autocast -else: - # Nothing to do if torch<1.6.0 - @contextmanager - def autocast(enabled=True): - yield - - -class Data2VecPretrainModel(nn.Module): - """Data2Vec Pretrain model""" - - def __init__( - self, - frontend=None, - specaug=None, - normalize=None, - encoder=None, - preencoder=None, - ): - - super().__init__() - - self.frontend = frontend - self.specaug = specaug - self.normalize = normalize - self.preencoder = preencoder - self.encoder = encoder - self.num_updates = 0 - - def forward( - self, - speech: torch.Tensor, - speech_lengths: torch.Tensor, - ) -> Tuple[torch.Tensor, Dict[str, torch.Tensor], torch.Tensor]: - """Frontend + Encoder + Calc loss - Args: - speech: (Batch, Length, ...) - speech_lengths: (Batch, ) - """ - # Check that batch_size is unified - assert speech.shape[0] == speech_lengths.shape[0], ( - speech.shape, - speech_lengths.shape, - ) - - self.encoder.set_num_updates(self.num_updates) - - # 1. Encoder - encoder_out = self.encode(speech, speech_lengths) - - losses = encoder_out["losses"] - loss = sum(losses.values()) - sample_size = encoder_out["sample_size"] - loss = loss.sum() / sample_size - - target_var = float(encoder_out["target_var"]) - pred_var = float(encoder_out["pred_var"]) - ema_decay = float(encoder_out["ema_decay"]) - - stats = dict( - loss=torch.clone(loss.detach()), - target_var=target_var, - pred_var=pred_var, - ema_decay=ema_decay, - ) - - loss, stats, weight = force_gatherable((loss, stats, sample_size), loss.device) - return loss, stats, weight - - def collect_feats( - self, speech: torch.Tensor, speech_lengths: torch.Tensor - ) -> Dict[str, torch.Tensor]: - feats, feats_lengths = self._extract_feats(speech, speech_lengths) - return {"feats": feats, "feats_lengths": feats_lengths} - - def encode( - self, - speech: torch.Tensor, - speech_lengths: torch.Tensor, - ): - """Frontend + Encoder. - Args: - speech: (Batch, Length, ...) - speech_lengths: (Batch, ) - """ - with autocast(False): - # 1. Extract feats - feats, feats_lengths = self._extract_feats(speech, speech_lengths) - - # 2. Data augmentation - if self.specaug is not None and self.training: - feats, feats_lengths = self.specaug(feats, feats_lengths) - - # 3. Normalization for feature: e.g. Global-CMVN, Utterance-CMVN - if self.normalize is not None: - feats, feats_lengths = self.normalize(feats, feats_lengths) - - # Pre-encoder, e.g. used for raw input data - if self.preencoder is not None: - feats, feats_lengths = self.preencoder(feats, feats_lengths) - - # 4. Forward encoder - if min(speech_lengths) == max( - speech_lengths - ): # for clipping, set speech_lengths as None - speech_lengths = None - encoder_out = self.encoder( - feats, speech_lengths, mask=True, features_only=False - ) - - return encoder_out - - def _extract_feats( - self, speech: torch.Tensor, speech_lengths: torch.Tensor - ) -> Tuple[torch.Tensor, torch.Tensor]: - assert speech_lengths.dim() == 1, speech_lengths.shape - - # for data-parallel - speech = speech[:, : speech_lengths.max()] - - if self.frontend is not None: - # Frontend - # e.g. STFT and Feature extract - # data_loader may send time-domain signal in this case - # speech (Batch, NSamples) -> feats: (Batch, NFrames, Dim) - feats, feats_lengths = self.frontend(speech, speech_lengths) - else: - # No frontend and no feature extract - feats, feats_lengths = speech, speech_lengths - return feats, feats_lengths - - def set_num_updates(self, num_updates): - self.num_updates = num_updates - - def get_num_updates(self): - return self.num_updates diff --git a/funasr_detach/models/data2vec/data2vec_encoder.py b/funasr_detach/models/data2vec/data2vec_encoder.py deleted file mode 100644 index 52b4a20ab19b6835c9ded54f369b6e3b808f0814..0000000000000000000000000000000000000000 --- a/funasr_detach/models/data2vec/data2vec_encoder.py +++ /dev/null @@ -1,578 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import logging -import math - -import torch -import torch.distributed as dist -import torch.nn as nn -import torch.nn.functional as F - -from funasr_detach.models.data2vec.data_utils import compute_mask_indices -from funasr_detach.models.data2vec.ema_module import EMAModule -from funasr_detach.models.data2vec.grad_multiply import GradMultiply -from funasr_detach.models.data2vec.wav2vec2 import ( - ConvFeatureExtractionModel, - TransformerEncoder, -) -from funasr_detach.models.transformer.utils.nets_utils import make_pad_mask - - -def get_annealed_rate(start, end, curr_step, total_steps): - r = end - start - pct_remaining = 1 - curr_step / total_steps - return end - r * pct_remaining - - -class Data2VecEncoder(nn.Module): - def __init__( - self, - # for ConvFeatureExtractionModel - input_size: int = None, - extractor_mode: str = None, - conv_feature_layers: str = "[(512,2,2)] + [(512,2,2)]", - # for Transformer Encoder - ## model architecture - layer_type: str = "transformer", - layer_norm_first: bool = False, - encoder_layers: int = 12, - encoder_embed_dim: int = 768, - encoder_ffn_embed_dim: int = 3072, - encoder_attention_heads: int = 12, - activation_fn: str = "gelu", - ## dropouts - dropout: float = 0.1, - attention_dropout: float = 0.1, - activation_dropout: float = 0.0, - encoder_layerdrop: float = 0.0, - dropout_input: float = 0.0, - dropout_features: float = 0.0, - ## grad settings - feature_grad_mult: float = 1.0, - ## masking - mask_prob: float = 0.65, - mask_length: int = 10, - mask_selection: str = "static", - mask_other: int = 0, - no_mask_overlap: bool = False, - mask_min_space: int = 1, - require_same_masks: bool = True, # if set as True, collate_fn should be clipping - mask_dropout: float = 0.0, - ## channel masking - mask_channel_length: int = 10, - mask_channel_prob: float = 0.0, - mask_channel_before: bool = False, - mask_channel_selection: str = "static", - mask_channel_other: int = 0, - no_mask_channel_overlap: bool = False, - mask_channel_min_space: int = 1, - ## positional embeddings - conv_pos: int = 128, - conv_pos_groups: int = 16, - pos_conv_depth: int = 1, - max_positions: int = 100000, - # EMA module - average_top_k_layers: int = 8, - layer_norm_target_layer: bool = False, - instance_norm_target_layer: bool = False, - instance_norm_targets: bool = False, - layer_norm_targets: bool = False, - batch_norm_target_layer: bool = False, - group_norm_target_layer: bool = False, - ema_decay: float = 0.999, - ema_end_decay: float = 0.9999, - ema_anneal_end_step: int = 100000, - ema_transformer_only: bool = True, - ema_layers_only: bool = True, - min_target_var: float = 0.1, - min_pred_var: float = 0.01, - # Loss - loss_beta: float = 0.0, - loss_scale: float = None, - # FP16 optimization - required_seq_len_multiple: int = 2, - ): - super().__init__() - - # ConvFeatureExtractionModel - self.conv_feature_layers = conv_feature_layers - feature_enc_layers = eval(conv_feature_layers) - self.extractor_embed = feature_enc_layers[-1][0] - self.feature_extractor = ConvFeatureExtractionModel( - conv_layers=feature_enc_layers, - dropout=0.0, - mode=extractor_mode, - in_d=input_size, - ) - - # Transformer Encoder - ## model architecture - self.layer_type = layer_type - self.layer_norm_first = layer_norm_first - self.encoder_layers = encoder_layers - self.encoder_embed_dim = encoder_embed_dim - self.encoder_ffn_embed_dim = encoder_ffn_embed_dim - self.encoder_attention_heads = encoder_attention_heads - self.activation_fn = activation_fn - ## dropout - self.dropout = dropout - self.attention_dropout = attention_dropout - self.activation_dropout = activation_dropout - self.encoder_layerdrop = encoder_layerdrop - self.dropout_input = dropout_input - self.dropout_features = dropout_features - ## grad settings - self.feature_grad_mult = feature_grad_mult - ## masking - self.mask_prob = mask_prob - self.mask_length = mask_length - self.mask_selection = mask_selection - self.mask_other = mask_other - self.no_mask_overlap = no_mask_overlap - self.mask_min_space = mask_min_space - self.require_same_masks = ( - require_same_masks # if set as True, collate_fn should be clipping - ) - self.mask_dropout = mask_dropout - ## channel masking - self.mask_channel_length = mask_channel_length - self.mask_channel_prob = mask_channel_prob - self.mask_channel_before = mask_channel_before - self.mask_channel_selection = mask_channel_selection - self.mask_channel_other = mask_channel_other - self.no_mask_channel_overlap = no_mask_channel_overlap - self.mask_channel_min_space = mask_channel_min_space - ## positional embeddings - self.conv_pos = conv_pos - self.conv_pos_groups = conv_pos_groups - self.pos_conv_depth = pos_conv_depth - self.max_positions = max_positions - self.mask_emb = nn.Parameter( - torch.FloatTensor(self.encoder_embed_dim).uniform_() - ) - self.encoder = TransformerEncoder( - dropout=self.dropout, - encoder_embed_dim=self.encoder_embed_dim, - required_seq_len_multiple=required_seq_len_multiple, - pos_conv_depth=self.pos_conv_depth, - conv_pos=self.conv_pos, - conv_pos_groups=self.conv_pos_groups, - # transformer layers - layer_type=self.layer_type, - encoder_layers=self.encoder_layers, - encoder_ffn_embed_dim=self.encoder_ffn_embed_dim, - encoder_attention_heads=self.encoder_attention_heads, - attention_dropout=self.attention_dropout, - activation_dropout=self.activation_dropout, - activation_fn=self.activation_fn, - layer_norm_first=self.layer_norm_first, - encoder_layerdrop=self.encoder_layerdrop, - max_positions=self.max_positions, - ) - ## projections and dropouts - self.post_extract_proj = nn.Linear(self.extractor_embed, self.encoder_embed_dim) - self.dropout_input = nn.Dropout(self.dropout_input) - self.dropout_features = nn.Dropout(self.dropout_features) - self.layer_norm = torch.nn.LayerNorm(self.extractor_embed) - self.final_proj = nn.Linear(self.encoder_embed_dim, self.encoder_embed_dim) - - # EMA module - self.average_top_k_layers = average_top_k_layers - self.layer_norm_target_layer = layer_norm_target_layer - self.instance_norm_target_layer = instance_norm_target_layer - self.instance_norm_targets = instance_norm_targets - self.layer_norm_targets = layer_norm_targets - self.batch_norm_target_layer = batch_norm_target_layer - self.group_norm_target_layer = group_norm_target_layer - self.ema_decay = ema_decay - self.ema_end_decay = ema_end_decay - self.ema_anneal_end_step = ema_anneal_end_step - self.ema_transformer_only = ema_transformer_only - self.ema_layers_only = ema_layers_only - self.min_target_var = min_target_var - self.min_pred_var = min_pred_var - self.ema = None - - # Loss - self.loss_beta = loss_beta - self.loss_scale = loss_scale - - # FP16 optimization - self.required_seq_len_multiple = required_seq_len_multiple - - self.num_updates = 0 - - logging.info("Data2VecEncoder settings: {}".format(self.__dict__)) - - def make_ema_teacher(self): - skip_keys = set() - if self.ema_layers_only: - self.ema_transformer_only = True - for k, _ in self.encoder.pos_conv.named_parameters(): - skip_keys.add(f"pos_conv.{k}") - - self.ema = EMAModule( - self.encoder if self.ema_transformer_only else self, - ema_decay=self.ema_decay, - ema_fp32=True, - skip_keys=skip_keys, - ) - - def set_num_updates(self, num_updates): - if self.ema is None and self.final_proj is not None: - logging.info("Making EMA Teacher") - self.make_ema_teacher() - elif self.training and self.ema is not None: - if self.ema_decay != self.ema_end_decay: - if num_updates >= self.ema_anneal_end_step: - decay = self.ema_end_decay - else: - decay = get_annealed_rate( - self.ema_decay, - self.ema_end_decay, - num_updates, - self.ema_anneal_end_step, - ) - self.ema.set_decay(decay) - if self.ema.get_decay() < 1: - self.ema.step(self.encoder if self.ema_transformer_only else self) - - self.num_updates = num_updates - - def apply_mask( - self, - x, - padding_mask, - mask_indices=None, - mask_channel_indices=None, - ): - B, T, C = x.shape - - if self.mask_channel_prob > 0 and self.mask_channel_before: - mask_channel_indices = compute_mask_indices( - (B, C), - None, - self.mask_channel_prob, - self.mask_channel_length, - self.mask_channel_selection, - self.mask_channel_other, - no_overlap=self.no_mask_channel_overlap, - min_space=self.mask_channel_min_space, - ) - mask_channel_indices = ( - torch.from_numpy(mask_channel_indices) - .to(x.device) - .unsqueeze(1) - .expand(-1, T, -1) - ) - x[mask_channel_indices] = 0 - - if self.mask_prob > 0: - if mask_indices is None: - mask_indices = compute_mask_indices( - (B, T), - padding_mask, - self.mask_prob, - self.mask_length, - self.mask_selection, - self.mask_other, - min_masks=1, - no_overlap=self.no_mask_overlap, - min_space=self.mask_min_space, - require_same_masks=self.require_same_masks, - mask_dropout=self.mask_dropout, - ) - mask_indices = torch.from_numpy(mask_indices).to(x.device) - x[mask_indices] = self.mask_emb - else: - mask_indices = None - - if self.mask_channel_prob > 0 and not self.mask_channel_before: - if mask_channel_indices is None: - mask_channel_indices = compute_mask_indices( - (B, C), - None, - self.mask_channel_prob, - self.mask_channel_length, - self.mask_channel_selection, - self.mask_channel_other, - no_overlap=self.no_mask_channel_overlap, - min_space=self.mask_channel_min_space, - ) - mask_channel_indices = ( - torch.from_numpy(mask_channel_indices) - .to(x.device) - .unsqueeze(1) - .expand(-1, T, -1) - ) - x[mask_channel_indices] = 0 - - return x, mask_indices - - def _get_feat_extract_output_lengths(self, input_lengths: torch.LongTensor): - """ - Computes the output length of the convolutional layers - """ - - def _conv_out_length(input_length, kernel_size, stride): - return torch.floor( - (input_length - kernel_size).to(torch.float32) / stride + 1 - ) - - conv_cfg_list = eval(self.conv_feature_layers) - - for i in range(len(conv_cfg_list)): - input_lengths = _conv_out_length( - input_lengths, conv_cfg_list[i][1], conv_cfg_list[i][2] - ) - - return input_lengths.to(torch.long) - - def forward( - self, - xs_pad, - ilens=None, - mask=False, - features_only=True, - layer=None, - mask_indices=None, - mask_channel_indices=None, - padding_count=None, - ): - # create padding_mask by ilens - if ilens is not None: - padding_mask = make_pad_mask(lengths=ilens).to(xs_pad.device) - else: - padding_mask = None - - features = xs_pad - - if self.feature_grad_mult > 0: - features = self.feature_extractor(features) - if self.feature_grad_mult != 1.0: - features = GradMultiply.apply(features, self.feature_grad_mult) - else: - with torch.no_grad(): - features = self.feature_extractor(features) - - features = features.transpose(1, 2) - - features = self.layer_norm(features) - - orig_padding_mask = padding_mask - - if padding_mask is not None: - input_lengths = (1 - padding_mask.long()).sum(-1) - # apply conv formula to get real output_lengths - output_lengths = self._get_feat_extract_output_lengths(input_lengths) - - padding_mask = torch.zeros( - features.shape[:2], dtype=features.dtype, device=features.device - ) - # these two operations makes sure that all values - # before the output lengths indices are attended to - padding_mask[ - ( - torch.arange(padding_mask.shape[0], device=padding_mask.device), - output_lengths - 1, - ) - ] = 1 - padding_mask = (1 - padding_mask.flip([-1]).cumsum(-1).flip([-1])).bool() - else: - padding_mask = None - - if self.post_extract_proj is not None: - features = self.post_extract_proj(features) - - pre_encoder_features = None - if self.ema_transformer_only: - pre_encoder_features = features.clone() - - features = self.dropout_input(features) - - if mask: - x, mask_indices = self.apply_mask( - features, - padding_mask, - mask_indices=mask_indices, - mask_channel_indices=mask_channel_indices, - ) - else: - x = features - mask_indices = None - - x, layer_results = self.encoder( - x, - padding_mask=padding_mask, - layer=layer, - ) - - if features_only: - encoder_out_lens = (1 - padding_mask.long()).sum(1) - return x, encoder_out_lens, None - - result = { - "losses": {}, - "padding_mask": padding_mask, - "x": x, - } - - with torch.no_grad(): - self.ema.model.eval() - - if self.ema_transformer_only: - y, layer_results = self.ema.model.extract_features( - pre_encoder_features, - padding_mask=padding_mask, - min_layer=self.encoder_layers - self.average_top_k_layers, - ) - y = { - "x": y, - "padding_mask": padding_mask, - "layer_results": layer_results, - } - else: - y = self.ema.model.extract_features( - source=xs_pad, - padding_mask=orig_padding_mask, - mask=False, - ) - - target_layer_results = [l[2] for l in y["layer_results"]] - - permuted = False - if self.instance_norm_target_layer or self.batch_norm_target_layer: - target_layer_results = [ - tl.permute(1, 2, 0) for tl in target_layer_results # TBC -> BCT - ] - permuted = True - - if self.batch_norm_target_layer: - target_layer_results = [ - F.batch_norm( - tl.float(), running_mean=None, running_var=None, training=True - ) - for tl in target_layer_results - ] - - if self.instance_norm_target_layer: - target_layer_results = [ - F.instance_norm(tl.float()) for tl in target_layer_results - ] - - if permuted: - target_layer_results = [ - tl.transpose(1, 2) for tl in target_layer_results # BCT -> BTC - ] - - if self.group_norm_target_layer: - target_layer_results = [ - F.layer_norm(tl.float(), tl.shape[-2:]) - for tl in target_layer_results - ] - - if self.layer_norm_target_layer: - target_layer_results = [ - F.layer_norm(tl.float(), tl.shape[-1:]) - for tl in target_layer_results - ] - - y = sum(target_layer_results) / len(target_layer_results) - - if self.layer_norm_targets: - y = F.layer_norm(y.float(), y.shape[-1:]) - - if self.instance_norm_targets: - y = F.instance_norm(y.float().transpose(1, 2)).transpose(1, 2) - - if not permuted: - y = y.transpose(0, 1) - - y = y[mask_indices] - - x = x[mask_indices] - x = self.final_proj(x) - - sz = x.size(-1) - - if self.loss_beta == 0: - loss = F.mse_loss(x.float(), y.float(), reduction="none").sum(dim=-1) - else: - loss = F.smooth_l1_loss( - x.float(), y.float(), reduction="none", beta=self.loss_beta - ).sum(dim=-1) - - if self.loss_scale is not None: - scale = self.loss_scale - else: - scale = 1 / math.sqrt(sz) - - result["losses"]["regression"] = loss.sum() * scale - - if "sample_size" not in result: - result["sample_size"] = loss.numel() - - with torch.no_grad(): - result["target_var"] = self.compute_var(y) - result["pred_var"] = self.compute_var(x.float()) - - if self.num_updates > 5000 and result["target_var"] < self.min_target_var: - logging.error( - f"target var is {result['target_var'].item()} < {self.min_target_var}, exiting" - ) - raise Exception( - f"target var is {result['target_var'].item()} < {self.min_target_var}, exiting" - ) - if self.num_updates > 5000 and result["pred_var"] < self.min_pred_var: - logging.error( - f"pred var is {result['pred_var'].item()} < {self.min_pred_var}, exiting" - ) - raise Exception( - f"pred var is {result['pred_var'].item()} < {self.min_pred_var}, exiting" - ) - - if self.ema is not None: - result["ema_decay"] = self.ema.get_decay() * 1000 - - return result - - @staticmethod - def compute_var(y): - y = y.view(-1, y.size(-1)) - if dist.is_initialized(): - zc = torch.tensor(y.size(0)).cuda() - zs = y.sum(dim=0) - zss = (y**2).sum(dim=0) - - dist.all_reduce(zc) - dist.all_reduce(zs) - dist.all_reduce(zss) - - var = zss / (zc - 1) - (zs**2) / (zc * (zc - 1)) - return torch.sqrt(var + 1e-6).mean() - else: - return torch.sqrt(y.var(dim=0) + 1e-6).mean() - - def extract_features(self, xs_pad, ilens, mask=False, layer=None): - res = self.forward( - xs_pad, - ilens, - mask=mask, - features_only=True, - layer=layer, - ) - return res - - def remove_pretraining_modules(self, last_layer=None): - self.final_proj = None - self.ema = None - if last_layer is not None: - self.encoder.layers = nn.ModuleList( - l for i, l in enumerate(self.encoder.layers) if i <= last_layer - ) - - def output_size(self) -> int: - return self.encoder_embed_dim diff --git a/funasr_detach/models/data2vec/data_utils.py b/funasr_detach/models/data2vec/data_utils.py deleted file mode 100644 index 69c0bbcebedd6c193a31bb7c6f1f32419f2d6fbf..0000000000000000000000000000000000000000 --- a/funasr_detach/models/data2vec/data_utils.py +++ /dev/null @@ -1,147 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - - -from typing import Optional, Tuple - -import numpy as np -import torch - - -def compute_mask_indices( - shape: Tuple[int, int], - padding_mask: Optional[torch.Tensor], - mask_prob: float, - mask_length: int, - mask_type: str = "static", - mask_other: float = 0.0, - min_masks: int = 0, - no_overlap: bool = False, - min_space: int = 0, - require_same_masks: bool = True, - mask_dropout: float = 0.0, -) -> np.ndarray: - """ - Computes random mask spans for a given shape - - Args: - shape: the the shape for which to compute masks. - should be of size 2 where first element is batch size and 2nd is timesteps - padding_mask: optional padding mask of the same size as shape, which will prevent masking padded elements - mask_prob: probability for each token to be chosen as start of the span to be masked. this will be multiplied by - number of timesteps divided by length of mask span to mask approximately this percentage of all elements. - however due to overlaps, the actual number will be smaller (unless no_overlap is True) - mask_type: how to compute mask lengths - static = fixed size - uniform = sample from uniform distribution [mask_other, mask_length*2] - normal = sample from normal distribution with mean mask_length and stdev mask_other. mask is min 1 element - poisson = sample from possion distribution with lambda = mask length - min_masks: minimum number of masked spans - no_overlap: if false, will switch to an alternative recursive algorithm that prevents spans from overlapping - min_space: only used if no_overlap is True, this is how many elements to keep unmasked between spans - require_same_masks: if true, will randomly drop out masks until same amount of masks remains in each sample - mask_dropout: randomly dropout this percentage of masks in each example - """ - - bsz, all_sz = shape - mask = np.full((bsz, all_sz), False) - - all_num_mask = int( - # add a random number for probabilistic rounding - mask_prob * all_sz / float(mask_length) - + np.random.rand() - ) - - all_num_mask = max(min_masks, all_num_mask) - - mask_idcs = [] - for i in range(bsz): - if padding_mask is not None: - sz = all_sz - padding_mask[i].long().sum().item() - num_mask = int( - # add a random number for probabilistic rounding - mask_prob * sz / float(mask_length) - + np.random.rand() - ) - num_mask = max(min_masks, num_mask) - else: - sz = all_sz - num_mask = all_num_mask - - if mask_type == "static": - lengths = np.full(num_mask, mask_length) - elif mask_type == "uniform": - lengths = np.random.randint(mask_other, mask_length * 2 + 1, size=num_mask) - elif mask_type == "normal": - lengths = np.random.normal(mask_length, mask_other, size=num_mask) - lengths = [max(1, int(round(x))) for x in lengths] - elif mask_type == "poisson": - lengths = np.random.poisson(mask_length, size=num_mask) - lengths = [int(round(x)) for x in lengths] - else: - raise Exception("unknown mask selection " + mask_type) - - if sum(lengths) == 0: - lengths[0] = min(mask_length, sz - 1) - - if no_overlap: - mask_idc = [] - - def arrange(s, e, length, keep_length): - span_start = np.random.randint(s, e - length) - mask_idc.extend(span_start + i for i in range(length)) - - new_parts = [] - if span_start - s - min_space >= keep_length: - new_parts.append((s, span_start - min_space + 1)) - if e - span_start - length - min_space > keep_length: - new_parts.append((span_start + length + min_space, e)) - return new_parts - - parts = [(0, sz)] - min_length = min(lengths) - for length in sorted(lengths, reverse=True): - lens = np.fromiter( - (e - s if e - s >= length + min_space else 0 for s, e in parts), - np.int32, - ) - l_sum = np.sum(lens) - if l_sum == 0: - break - probs = lens / np.sum(lens) - c = np.random.choice(len(parts), p=probs) - s, e = parts.pop(c) - parts.extend(arrange(s, e, length, min_length)) - mask_idc = np.asarray(mask_idc) - else: - min_len = min(lengths) - if sz - min_len <= num_mask: - min_len = sz - num_mask - 1 - - mask_idc = np.random.choice(sz - min_len, num_mask, replace=False) - - mask_idc = np.asarray( - [ - mask_idc[j] + offset - for j in range(len(mask_idc)) - for offset in range(lengths[j]) - ] - ) - - mask_idcs.append(np.unique(mask_idc[mask_idc < sz])) - - min_len = min([len(m) for m in mask_idcs]) - for i, mask_idc in enumerate(mask_idcs): - if len(mask_idc) > min_len and require_same_masks: - mask_idc = np.random.choice(mask_idc, min_len, replace=False) - if mask_dropout > 0: - num_holes = np.rint(len(mask_idc) * mask_dropout).astype(int) - mask_idc = np.random.choice( - mask_idc, len(mask_idc) - num_holes, replace=False - ) - - mask[i, mask_idc] = True - - return mask diff --git a/funasr_detach/models/data2vec/ema_module.py b/funasr_detach/models/data2vec/ema_module.py deleted file mode 100644 index a98da5daf758ecdd578f75d27c7b2c39c5c4a257..0000000000000000000000000000000000000000 --- a/funasr_detach/models/data2vec/ema_module.py +++ /dev/null @@ -1,134 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -""" -Used for EMA tracking a given pytorch module. The user is responsible for calling step() -and setting the appropriate decay -""" - -import copy -import logging - -import torch - - -class EMAModule: - """Exponential Moving Average of Fairseq Models""" - - def __init__( - self, model, ema_decay=0.9999, ema_fp32=False, device=None, skip_keys=None - ): - """ - @param model model to initialize the EMA with - @param config EMAConfig object with configuration like - ema_decay, ema_update_freq, ema_fp32 - @param device If provided, copy EMA to this device (e.g. gpu). - Otherwise EMA is in the same device as the model. - """ - - self.decay = ema_decay - self.ema_fp32 = ema_fp32 - self.model = copy.deepcopy(model) - self.model.requires_grad_(False) - self.skip_keys = skip_keys or set() - self.fp32_params = {} - - if device is not None: - logging.info(f"Copying EMA model to device {device}") - self.model = self.model.to(device=device) - - if self.ema_fp32: - self.build_fp32_params() - - self.update_freq_counter = 0 - - def build_fp32_params(self, state_dict=None): - """ - Store a copy of the EMA params in fp32. - If state dict is passed, the EMA params is copied from - the provided state dict. Otherwise, it is copied from the - current EMA model parameters. - """ - if not self.ema_fp32: - raise RuntimeError( - "build_fp32_params should not be called if ema_fp32=False. " - "Use ema_fp32=True if this is really intended." - ) - - if state_dict is None: - state_dict = self.model.state_dict() - - def _to_float(t): - return t.float() if torch.is_floating_point(t) else t - - for param_key in state_dict: - if param_key in self.fp32_params: - self.fp32_params[param_key].copy_(state_dict[param_key]) - else: - self.fp32_params[param_key] = _to_float(state_dict[param_key]) - - def restore(self, state_dict, build_fp32_params=False): - """Load data from a model spec into EMA model""" - self.model.load_state_dict(state_dict, strict=False) - if build_fp32_params: - self.build_fp32_params(state_dict) - - def set_decay(self, decay): - self.decay = decay - - def get_decay(self): - return self.decay - - def _step_internal(self, new_model): - """One update of the EMA model based on new model weights""" - decay = self.decay - - ema_state_dict = {} - ema_params = self.fp32_params if self.ema_fp32 else self.model.state_dict() - for key, param in new_model.state_dict().items(): - if isinstance(param, dict): - continue - try: - ema_param = ema_params[key] - except KeyError: - ema_param = ( - param.float().clone() if param.ndim == 1 else copy.deepcopy(param) - ) - - if param.shape != ema_param.shape: - raise ValueError( - "incompatible tensor shapes between model param and ema param" - + "{} vs. {}".format(param.shape, ema_param.shape) - ) - - if "version" in key: - # Do not decay a model.version pytorch param - continue - - if key in self.skip_keys or ( - "num_batches_tracked" in key and ema_param.dtype == torch.int64 - ): - ema_param = param.to(dtype=ema_param.dtype).clone() - ema_params[key].copy_(ema_param) - else: - ema_param.mul_(decay) - ema_param.add_(param.to(dtype=ema_param.dtype), alpha=1 - decay) - ema_state_dict[key] = ema_param - self.restore(ema_state_dict, build_fp32_params=False) - - def step(self, new_model): - self._step_internal(new_model) - - def reverse(self, model): - """ - Load the model parameters from EMA model. - Useful for inference or fine-tuning from the EMA model. - """ - d = self.model.state_dict() - if "_ema" in d: - del d["_ema"] - - model.load_state_dict(d, strict=False) - return model diff --git a/funasr_detach/models/data2vec/grad_multiply.py b/funasr_detach/models/data2vec/grad_multiply.py deleted file mode 100644 index 08d15f55dfda9c61a1cf8641ea31424fe1d97f57..0000000000000000000000000000000000000000 --- a/funasr_detach/models/data2vec/grad_multiply.py +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import torch - - -class GradMultiply(torch.autograd.Function): - @staticmethod - def forward(ctx, x, scale): - ctx.scale = scale - res = x.new(x) - return res - - @staticmethod - def backward(ctx, grad): - return grad * ctx.scale, None diff --git a/funasr_detach/models/data2vec/multihead_attention.py b/funasr_detach/models/data2vec/multihead_attention.py deleted file mode 100644 index be7d18aefb10e1a43ba7bc6b9c29442c9423d8ef..0000000000000000000000000000000000000000 --- a/funasr_detach/models/data2vec/multihead_attention.py +++ /dev/null @@ -1,641 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import logging -import math -from typing import Dict, List, Optional, Tuple - -import torch -import torch.nn.functional as F -from torch import Tensor, nn -from torch.nn import Parameter - -from funasr_detach.models.data2vec.quant_noise import quant_noise - - -class FairseqDropout(nn.Module): - def __init__(self, p, module_name=None): - super().__init__() - self.p = p - self.module_name = module_name - self.apply_during_inference = False - - def forward(self, x, inplace: bool = False): - if self.p > 0 and (self.training or self.apply_during_inference): - return F.dropout(x, p=self.p, training=True, inplace=inplace) - else: - return x - - def make_generation_fast_( - self, - name: str, - retain_dropout: bool = False, - retain_dropout_modules: Optional[List[str]] = None, - **kwargs, - ): - if retain_dropout: - if retain_dropout_modules is not None and self.module_name is None: - logging.warning( - "Cannot enable dropout during inference for module {} " - "because module_name was not set".format(name) - ) - elif ( - retain_dropout_modules is None # if None, apply to all modules - or self.module_name in retain_dropout_modules - ): - logging.info( - "Enabling dropout during inference for module: {}".format(name) - ) - self.apply_during_inference = True - else: - logging.info("Disabling dropout for module: {}".format(name)) - - -class MultiheadAttention(nn.Module): - """Multi-headed attention. - - See "Attention Is All You Need" for more details. - """ - - def __init__( - self, - embed_dim, - num_heads, - kdim=None, - vdim=None, - dropout=0.0, - bias=True, - add_bias_kv=False, - add_zero_attn=False, - self_attention=False, - encoder_decoder_attention=False, - q_noise=0.0, - qn_block_size=8, - ): - super().__init__() - self.embed_dim = embed_dim - self.kdim = kdim if kdim is not None else embed_dim - self.vdim = vdim if vdim is not None else embed_dim - self.qkv_same_dim = self.kdim == embed_dim and self.vdim == embed_dim - - self.num_heads = num_heads - self.dropout_module = FairseqDropout( - dropout, module_name=self.__class__.__name__ - ) - - self.head_dim = embed_dim // num_heads - assert ( - self.head_dim * num_heads == self.embed_dim - ), "embed_dim must be divisible by num_heads" - self.scaling = self.head_dim**-0.5 - - self.self_attention = self_attention - self.encoder_decoder_attention = encoder_decoder_attention - - assert not self.self_attention or self.qkv_same_dim, ( - "Self-attention requires query, key and " "value to be of the same size" - ) - - self.k_proj = quant_noise( - nn.Linear(self.kdim, embed_dim, bias=bias), q_noise, qn_block_size - ) - self.v_proj = quant_noise( - nn.Linear(self.vdim, embed_dim, bias=bias), q_noise, qn_block_size - ) - self.q_proj = quant_noise( - nn.Linear(embed_dim, embed_dim, bias=bias), q_noise, qn_block_size - ) - - self.out_proj = quant_noise( - nn.Linear(embed_dim, embed_dim, bias=bias), q_noise, qn_block_size - ) - - if add_bias_kv: - self.bias_k = Parameter(torch.Tensor(1, 1, embed_dim)) - self.bias_v = Parameter(torch.Tensor(1, 1, embed_dim)) - else: - self.bias_k = self.bias_v = None - - self.add_zero_attn = add_zero_attn - - self.reset_parameters() - - self.onnx_trace = False - self.skip_embed_dim_check = False - - def prepare_for_onnx_export_(self): - self.onnx_trace = True - - def reset_parameters(self): - if self.qkv_same_dim: - # Empirically observed the convergence to be much better with - # the scaled initialization - nn.init.xavier_uniform_(self.k_proj.weight, gain=1 / math.sqrt(2)) - nn.init.xavier_uniform_(self.v_proj.weight, gain=1 / math.sqrt(2)) - nn.init.xavier_uniform_(self.q_proj.weight, gain=1 / math.sqrt(2)) - else: - nn.init.xavier_uniform_(self.k_proj.weight) - nn.init.xavier_uniform_(self.v_proj.weight) - nn.init.xavier_uniform_(self.q_proj.weight) - - nn.init.xavier_uniform_(self.out_proj.weight) - if self.out_proj.bias is not None: - nn.init.constant_(self.out_proj.bias, 0.0) - if self.bias_k is not None: - nn.init.xavier_normal_(self.bias_k) - if self.bias_v is not None: - nn.init.xavier_normal_(self.bias_v) - - def _get_reserve_head_index(self, num_heads_to_keep: int): - k_proj_heads_norm = [] - q_proj_heads_norm = [] - v_proj_heads_norm = [] - - for i in range(self.num_heads): - start_idx = i * self.head_dim - end_idx = (i + 1) * self.head_dim - k_proj_heads_norm.append( - torch.sum(torch.abs(self.k_proj.weight[start_idx:end_idx,])).tolist() - + torch.sum(torch.abs(self.k_proj.bias[start_idx:end_idx])).tolist() - ) - q_proj_heads_norm.append( - torch.sum(torch.abs(self.q_proj.weight[start_idx:end_idx,])).tolist() - + torch.sum(torch.abs(self.q_proj.bias[start_idx:end_idx])).tolist() - ) - v_proj_heads_norm.append( - torch.sum(torch.abs(self.v_proj.weight[start_idx:end_idx,])).tolist() - + torch.sum(torch.abs(self.v_proj.bias[start_idx:end_idx])).tolist() - ) - - heads_norm = [] - for i in range(self.num_heads): - heads_norm.append( - k_proj_heads_norm[i] + q_proj_heads_norm[i] + v_proj_heads_norm[i] - ) - - sorted_head_index = sorted( - range(self.num_heads), key=lambda k: heads_norm[k], reverse=True - ) - reserve_head_index = [] - for i in range(num_heads_to_keep): - start = sorted_head_index[i] * self.head_dim - end = (sorted_head_index[i] + 1) * self.head_dim - reserve_head_index.append((start, end)) - return reserve_head_index - - def _adaptive_prune_heads(self, reserve_head_index: List[Tuple[int, int]]): - new_q_weight = [] - new_q_bias = [] - new_k_weight = [] - new_k_bias = [] - new_v_weight = [] - new_v_bias = [] - new_out_proj_weight = [] - - for ele in reserve_head_index: - start_idx, end_idx = ele - new_q_weight.append(self.q_proj.weight[start_idx:end_idx,]) - new_q_bias.append(self.q_proj.bias[start_idx:end_idx]) - - new_k_weight.append(self.k_proj.weight[start_idx:end_idx,]) - - new_k_bias.append(self.k_proj.bias[start_idx:end_idx]) - - new_v_weight.append(self.v_proj.weight[start_idx:end_idx,]) - new_v_bias.append(self.v_proj.bias[start_idx:end_idx]) - - new_out_proj_weight.append(self.out_proj.weight[:, start_idx:end_idx]) - - new_q_weight = torch.cat(new_q_weight).detach() - new_k_weight = torch.cat(new_k_weight).detach() - new_v_weight = torch.cat(new_v_weight).detach() - new_out_proj_weight = torch.cat(new_out_proj_weight, dim=-1).detach() - new_q_weight.requires_grad = True - new_k_weight.requires_grad = True - new_v_weight.requires_grad = True - new_out_proj_weight.requires_grad = True - - new_q_bias = torch.cat(new_q_bias).detach() - new_q_bias.requires_grad = True - - new_k_bias = torch.cat(new_k_bias).detach() - new_k_bias.requires_grad = True - - new_v_bias = torch.cat(new_v_bias).detach() - new_v_bias.requires_grad = True - - self.q_proj.weight = torch.nn.Parameter(new_q_weight) - self.q_proj.bias = torch.nn.Parameter(new_q_bias) - - self.k_proj.weight = torch.nn.Parameter(new_k_weight) - self.k_proj.bias = torch.nn.Parameter(new_k_bias) - - self.v_proj.weight = torch.nn.Parameter(new_v_weight) - self.v_proj.bias = torch.nn.Parameter(new_v_bias) - - self.out_proj.weight = torch.nn.Parameter(new_out_proj_weight) - - self.num_heads = len(reserve_head_index) - self.embed_dim = self.head_dim * self.num_heads - self.q_proj.out_features = self.embed_dim - self.k_proj.out_features = self.embed_dim - self.v_proj.out_features = self.embed_dim - - def _set_skip_embed_dim_check(self): - self.skip_embed_dim_check = True - - def forward( - self, - query, - key: Optional[Tensor], - value: Optional[Tensor], - key_padding_mask: Optional[Tensor] = None, - incremental_state: Optional[Dict[str, Dict[str, Optional[Tensor]]]] = None, - need_weights: bool = True, - static_kv: bool = False, - attn_mask: Optional[Tensor] = None, - before_softmax: bool = False, - need_head_weights: bool = False, - ) -> Tuple[Tensor, Optional[Tensor]]: - """Input shape: Time x Batch x Channel - - Args: - key_padding_mask (ByteTensor, optional): mask to exclude - keys that are pads, of shape `(batch, src_len)`, where - padding elements are indicated by 1s. - need_weights (bool, optional): return the attention weights, - averaged over heads (default: False). - attn_mask (ByteTensor, optional): typically used to - implement causal attention, where the mask prevents the - attention from looking forward in time (default: None). - before_softmax (bool, optional): return the raw attention - weights and values before the attention softmax. - need_head_weights (bool, optional): return the attention - weights for each head. Implies *need_weights*. Default: - return the average attention weights over all heads. - """ - if need_head_weights: - need_weights = True - - is_tpu = query.device.type == "xla" - - tgt_len, bsz, embed_dim = query.size() - src_len = tgt_len - if not self.skip_embed_dim_check: - assert ( - embed_dim == self.embed_dim - ), f"query dim {embed_dim} != {self.embed_dim}" - assert list(query.size()) == [tgt_len, bsz, embed_dim] - if key is not None: - src_len, key_bsz, _ = key.size() - if not torch.jit.is_scripting(): - assert key_bsz == bsz - assert value is not None - assert src_len, bsz == value.shape[:2] - - if ( - not self.onnx_trace - and not is_tpu # don't use PyTorch version on TPUs - and incremental_state is None - and not static_kv - # A workaround for quantization to work. Otherwise JIT compilation - # treats bias in linear module as method. - and not torch.jit.is_scripting() - # The Multihead attention implemented in pytorch forces strong dimension check - # for input embedding dimention and K,Q,V projection dimension. - # Since pruning will break the dimension check and it is not easy to modify the pytorch API, - # it is preferred to bypass the pytorch MHA when we need to skip embed_dim_check - and not self.skip_embed_dim_check - ): - assert key is not None and value is not None - return F.multi_head_attention_forward( - query, - key, - value, - self.embed_dim, - self.num_heads, - torch.empty([0]), - torch.cat((self.q_proj.bias, self.k_proj.bias, self.v_proj.bias)), - self.bias_k, - self.bias_v, - self.add_zero_attn, - self.dropout_module.p, - self.out_proj.weight, - self.out_proj.bias, - self.training or self.dropout_module.apply_during_inference, - key_padding_mask, - need_weights, - attn_mask, - use_separate_proj_weight=True, - q_proj_weight=self.q_proj.weight, - k_proj_weight=self.k_proj.weight, - v_proj_weight=self.v_proj.weight, - ) - - if incremental_state is not None: - saved_state = self._get_input_buffer(incremental_state) - if saved_state is not None and "prev_key" in saved_state: - # previous time steps are cached - no need to recompute - # key and value if they are static - if static_kv: - assert self.encoder_decoder_attention and not self.self_attention - key = value = None - else: - saved_state = None - - if self.self_attention: - q = self.q_proj(query) - k = self.k_proj(query) - v = self.v_proj(query) - elif self.encoder_decoder_attention: - # encoder-decoder attention - q = self.q_proj(query) - if key is None: - assert value is None - k = v = None - else: - k = self.k_proj(key) - v = self.v_proj(key) - - else: - assert key is not None and value is not None - q = self.q_proj(query) - k = self.k_proj(key) - v = self.v_proj(value) - q *= self.scaling - - if self.bias_k is not None: - assert self.bias_v is not None - k = torch.cat([k, self.bias_k.repeat(1, bsz, 1)]) - v = torch.cat([v, self.bias_v.repeat(1, bsz, 1)]) - if attn_mask is not None: - attn_mask = torch.cat( - [attn_mask, attn_mask.new_zeros(attn_mask.size(0), 1)], dim=1 - ) - if key_padding_mask is not None: - key_padding_mask = torch.cat( - [ - key_padding_mask, - key_padding_mask.new_zeros(key_padding_mask.size(0), 1), - ], - dim=1, - ) - - q = ( - q.contiguous() - .view(tgt_len, bsz * self.num_heads, self.head_dim) - .transpose(0, 1) - ) - if k is not None: - k = ( - k.contiguous() - .view(-1, bsz * self.num_heads, self.head_dim) - .transpose(0, 1) - ) - if v is not None: - v = ( - v.contiguous() - .view(-1, bsz * self.num_heads, self.head_dim) - .transpose(0, 1) - ) - - if saved_state is not None: - # saved states are stored with shape (bsz, num_heads, seq_len, head_dim) - if "prev_key" in saved_state: - _prev_key = saved_state["prev_key"] - assert _prev_key is not None - prev_key = _prev_key.view(bsz * self.num_heads, -1, self.head_dim) - if static_kv: - k = prev_key - else: - assert k is not None - k = torch.cat([prev_key, k], dim=1) - src_len = k.size(1) - if "prev_value" in saved_state: - _prev_value = saved_state["prev_value"] - assert _prev_value is not None - prev_value = _prev_value.view(bsz * self.num_heads, -1, self.head_dim) - if static_kv: - v = prev_value - else: - assert v is not None - v = torch.cat([prev_value, v], dim=1) - prev_key_padding_mask: Optional[Tensor] = None - if "prev_key_padding_mask" in saved_state: - prev_key_padding_mask = saved_state["prev_key_padding_mask"] - assert k is not None and v is not None - key_padding_mask = MultiheadAttention._append_prev_key_padding_mask( - key_padding_mask=key_padding_mask, - prev_key_padding_mask=prev_key_padding_mask, - batch_size=bsz, - src_len=k.size(1), - static_kv=static_kv, - ) - - saved_state["prev_key"] = k.view(bsz, self.num_heads, -1, self.head_dim) - saved_state["prev_value"] = v.view(bsz, self.num_heads, -1, self.head_dim) - saved_state["prev_key_padding_mask"] = key_padding_mask - # In this branch incremental_state is never None - assert incremental_state is not None - incremental_state = self._set_input_buffer(incremental_state, saved_state) - assert k is not None - assert k.size(1) == src_len - - # This is part of a workaround to get around fork/join parallelism - # not supporting Optional types. - if key_padding_mask is not None and key_padding_mask.dim() == 0: - key_padding_mask = None - - if key_padding_mask is not None: - assert key_padding_mask.size(0) == bsz - assert key_padding_mask.size(1) == src_len - - if self.add_zero_attn: - assert v is not None - src_len += 1 - k = torch.cat([k, k.new_zeros((k.size(0), 1) + k.size()[2:])], dim=1) - v = torch.cat([v, v.new_zeros((v.size(0), 1) + v.size()[2:])], dim=1) - if attn_mask is not None: - attn_mask = torch.cat( - [attn_mask, attn_mask.new_zeros(attn_mask.size(0), 1)], dim=1 - ) - if key_padding_mask is not None: - key_padding_mask = torch.cat( - [ - key_padding_mask, - torch.zeros(key_padding_mask.size(0), 1).type_as( - key_padding_mask - ), - ], - dim=1, - ) - - attn_weights = torch.bmm(q, k.transpose(1, 2)) - attn_weights = self.apply_sparse_mask(attn_weights, tgt_len, src_len, bsz) - - assert list(attn_weights.size()) == [bsz * self.num_heads, tgt_len, src_len] - - if attn_mask is not None: - attn_mask = attn_mask.unsqueeze(0) - if self.onnx_trace: - attn_mask = attn_mask.repeat(attn_weights.size(0), 1, 1) - attn_weights += attn_mask - - if key_padding_mask is not None: - # don't attend to padding symbols - attn_weights = attn_weights.view(bsz, self.num_heads, tgt_len, src_len) - if not is_tpu: - attn_weights = attn_weights.masked_fill( - key_padding_mask.unsqueeze(1).unsqueeze(2).to(torch.bool), - float("-inf"), - ) - else: - attn_weights = attn_weights.transpose(0, 2) - attn_weights = attn_weights.masked_fill(key_padding_mask, float("-inf")) - attn_weights = attn_weights.transpose(0, 2) - attn_weights = attn_weights.view(bsz * self.num_heads, tgt_len, src_len) - - if before_softmax: - return attn_weights, v - - attn_weights_float = F.softmax(attn_weights, dim=-1, dtype=torch.float32) - attn_weights = attn_weights_float.type_as(attn_weights) - attn_probs = self.dropout_module(attn_weights) - - assert v is not None - attn = torch.bmm(attn_probs, v) - assert list(attn.size()) == [bsz * self.num_heads, tgt_len, self.head_dim] - if self.onnx_trace and attn.size(1) == 1: - # when ONNX tracing a single decoder step (sequence length == 1) - # the transpose is a no-op copy before view, thus unnecessary - attn = attn.contiguous().view(tgt_len, bsz, self.embed_dim) - else: - attn = attn.transpose(0, 1).contiguous().view(tgt_len, bsz, self.embed_dim) - attn = self.out_proj(attn) - attn_weights: Optional[Tensor] = None - if need_weights: - attn_weights = attn_weights_float.view( - bsz, self.num_heads, tgt_len, src_len - ).transpose(1, 0) - if not need_head_weights: - # average attention weights over heads - attn_weights = attn_weights.mean(dim=0) - - return attn, attn_weights - - @staticmethod - def _append_prev_key_padding_mask( - key_padding_mask: Optional[Tensor], - prev_key_padding_mask: Optional[Tensor], - batch_size: int, - src_len: int, - static_kv: bool, - ) -> Optional[Tensor]: - # saved key padding masks have shape (bsz, seq_len) - if prev_key_padding_mask is not None and static_kv: - new_key_padding_mask = prev_key_padding_mask - elif prev_key_padding_mask is not None and key_padding_mask is not None: - new_key_padding_mask = torch.cat( - [prev_key_padding_mask.float(), key_padding_mask.float()], dim=1 - ) - # During incremental decoding, as the padding token enters and - # leaves the frame, there will be a time when prev or current - # is None - elif prev_key_padding_mask is not None: - if src_len > prev_key_padding_mask.size(1): - filler = torch.zeros( - (batch_size, src_len - prev_key_padding_mask.size(1)), - device=prev_key_padding_mask.device, - ) - new_key_padding_mask = torch.cat( - [prev_key_padding_mask.float(), filler.float()], dim=1 - ) - else: - new_key_padding_mask = prev_key_padding_mask.float() - elif key_padding_mask is not None: - if src_len > key_padding_mask.size(1): - filler = torch.zeros( - (batch_size, src_len - key_padding_mask.size(1)), - device=key_padding_mask.device, - ) - new_key_padding_mask = torch.cat( - [filler.float(), key_padding_mask.float()], dim=1 - ) - else: - new_key_padding_mask = key_padding_mask.float() - else: - new_key_padding_mask = prev_key_padding_mask - return new_key_padding_mask - - @torch.jit.export - def reorder_incremental_state( - self, - incremental_state: Dict[str, Dict[str, Optional[Tensor]]], - new_order: Tensor, - ): - """Reorder buffered internal state (for incremental generation).""" - input_buffer = self._get_input_buffer(incremental_state) - if input_buffer is not None: - for k in input_buffer.keys(): - input_buffer_k = input_buffer[k] - if input_buffer_k is not None: - if self.encoder_decoder_attention and input_buffer_k.size( - 0 - ) == new_order.size(0): - break - input_buffer[k] = input_buffer_k.index_select(0, new_order) - incremental_state = self._set_input_buffer(incremental_state, input_buffer) - return incremental_state - - def _get_input_buffer( - self, incremental_state: Optional[Dict[str, Dict[str, Optional[Tensor]]]] - ) -> Dict[str, Optional[Tensor]]: - result = self.get_incremental_state(incremental_state, "attn_state") - if result is not None: - return result - else: - empty_result: Dict[str, Optional[Tensor]] = {} - return empty_result - - def _set_input_buffer( - self, - incremental_state: Dict[str, Dict[str, Optional[Tensor]]], - buffer: Dict[str, Optional[Tensor]], - ): - return self.set_incremental_state(incremental_state, "attn_state", buffer) - - def apply_sparse_mask(self, attn_weights, tgt_len: int, src_len: int, bsz: int): - return attn_weights - - def upgrade_state_dict_named(self, state_dict, name): - prefix = name + "." if name != "" else "" - items_to_add = {} - keys_to_remove = [] - for k in state_dict.keys(): - if k.endswith(prefix + "in_proj_weight"): - # in_proj_weight used to be q + k + v with same dimensions - dim = int(state_dict[k].shape[0] / 3) - items_to_add[prefix + "q_proj.weight"] = state_dict[k][:dim] - items_to_add[prefix + "k_proj.weight"] = state_dict[k][dim : 2 * dim] - items_to_add[prefix + "v_proj.weight"] = state_dict[k][2 * dim :] - - keys_to_remove.append(k) - - k_bias = prefix + "in_proj_bias" - if k_bias in state_dict.keys(): - dim = int(state_dict[k].shape[0] / 3) - items_to_add[prefix + "q_proj.bias"] = state_dict[k_bias][:dim] - items_to_add[prefix + "k_proj.bias"] = state_dict[k_bias][ - dim : 2 * dim - ] - items_to_add[prefix + "v_proj.bias"] = state_dict[k_bias][2 * dim :] - - keys_to_remove.append(prefix + "in_proj_bias") - - for k in keys_to_remove: - del state_dict[k] - - for key, value in items_to_add.items(): - state_dict[key] = value diff --git a/funasr_detach/models/data2vec/quant_noise.py b/funasr_detach/models/data2vec/quant_noise.py deleted file mode 100644 index d777dfbb6c1bf6a9b769dfdaec35d5ef084c8a8b..0000000000000000000000000000000000000000 --- a/funasr_detach/models/data2vec/quant_noise.py +++ /dev/null @@ -1,107 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import torch -import torch.nn as nn - - -def quant_noise(module, p, block_size): - """ - Wraps modules and applies quantization noise to the weights for - subsequent quantization with Iterative Product Quantization as - described in "Training with Quantization Noise for Extreme Model Compression" - - Args: - - module: nn.Module - - p: amount of Quantization Noise - - block_size: size of the blocks for subsequent quantization with iPQ - - Remarks: - - Module weights must have the right sizes wrt the block size - - Only Linear, Embedding and Conv2d modules are supported for the moment - - For more detail on how to quantize by blocks with convolutional weights, - see "And the Bit Goes Down: Revisiting the Quantization of Neural Networks" - - We implement the simplest form of noise here as stated in the paper - which consists in randomly dropping blocks - """ - - # if no quantization noise, don't register hook - if p <= 0: - return module - - # supported modules - assert isinstance(module, (nn.Linear, nn.Embedding, nn.Conv2d)) - - # test whether module.weight has the right sizes wrt block_size - is_conv = module.weight.ndim == 4 - - # 2D matrix - if not is_conv: - assert ( - module.weight.size(1) % block_size == 0 - ), "Input features must be a multiple of block sizes" - - # 4D matrix - else: - # 1x1 convolutions - if module.kernel_size == (1, 1): - assert ( - module.in_channels % block_size == 0 - ), "Input channels must be a multiple of block sizes" - # regular convolutions - else: - k = module.kernel_size[0] * module.kernel_size[1] - assert k % block_size == 0, "Kernel size must be a multiple of block size" - - def _forward_pre_hook(mod, input): - # no noise for evaluation - if mod.training: - if not is_conv: - # gather weight and sizes - weight = mod.weight - in_features = weight.size(1) - out_features = weight.size(0) - - # split weight matrix into blocks and randomly drop selected blocks - mask = torch.zeros( - in_features // block_size * out_features, device=weight.device - ) - mask.bernoulli_(p) - mask = mask.repeat_interleave(block_size, -1).view(-1, in_features) - - else: - # gather weight and sizes - weight = mod.weight - in_channels = mod.in_channels - out_channels = mod.out_channels - - # split weight matrix into blocks and randomly drop selected blocks - if mod.kernel_size == (1, 1): - mask = torch.zeros( - int(in_channels // block_size * out_channels), - device=weight.device, - ) - mask.bernoulli_(p) - mask = mask.repeat_interleave(block_size, -1).view(-1, in_channels) - else: - mask = torch.zeros( - weight.size(0), weight.size(1), device=weight.device - ) - mask.bernoulli_(p) - mask = ( - mask.unsqueeze(2) - .unsqueeze(3) - .repeat(1, 1, mod.kernel_size[0], mod.kernel_size[1]) - ) - - # scale weights and apply mask - mask = mask.to( - torch.bool - ) # x.bool() is not currently supported in TorchScript - s = 1 / (1 - p) - mod.weight.data = s * weight.masked_fill(mask, 0) - - module.register_forward_pre_hook(_forward_pre_hook) - return module diff --git a/funasr_detach/models/data2vec/utils.py b/funasr_detach/models/data2vec/utils.py deleted file mode 100644 index 9b92bfeb5ab3f1c8008089754a9a1b36d6fd8a5f..0000000000000000000000000000000000000000 --- a/funasr_detach/models/data2vec/utils.py +++ /dev/null @@ -1,156 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import math - -import torch -import torch.nn as nn -import torch.nn.functional as F - -from funasr_detach.models.data2vec.multihead_attention import MultiheadAttention - - -class Fp32LayerNorm(nn.LayerNorm): - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - - def forward(self, input): - output = F.layer_norm( - input.float(), - self.normalized_shape, - self.weight.float() if self.weight is not None else None, - self.bias.float() if self.bias is not None else None, - self.eps, - ) - return output.type_as(input) - - -class Fp32GroupNorm(nn.GroupNorm): - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - - def forward(self, input): - output = F.group_norm( - input.float(), - self.num_groups, - self.weight.float() if self.weight is not None else None, - self.bias.float() if self.bias is not None else None, - self.eps, - ) - return output.type_as(input) - - -class TransposeLast(nn.Module): - def __init__(self, deconstruct_idx=None): - super().__init__() - self.deconstruct_idx = deconstruct_idx - - def forward(self, x): - if self.deconstruct_idx is not None: - x = x[self.deconstruct_idx] - return x.transpose(-2, -1) - - -class SamePad(nn.Module): - def __init__(self, kernel_size, causal=False): - super().__init__() - if causal: - self.remove = kernel_size - 1 - else: - self.remove = 1 if kernel_size % 2 == 0 else 0 - - def forward(self, x): - if self.remove > 0: - x = x[:, :, : -self.remove] - return x - - -def pad_to_multiple(x, multiple, dim=-1, value=0): - # Inspired from https://github.com/lucidrains/local-attention/blob/master/local_attention/local_attention.py#L41 - if x is None: - return None, 0 - tsz = x.size(dim) - m = tsz / multiple - remainder = math.ceil(m) * multiple - tsz - if m.is_integer(): - return x, 0 - pad_offset = (0,) * (-1 - dim) * 2 - - return F.pad(x, (*pad_offset, 0, remainder), value=value), remainder - - -def gelu_accurate(x): - if not hasattr(gelu_accurate, "_a"): - gelu_accurate._a = math.sqrt(2 / math.pi) - return ( - 0.5 * x * (1 + torch.tanh(gelu_accurate._a * (x + 0.044715 * torch.pow(x, 3)))) - ) - - -def gelu(x: torch.Tensor) -> torch.Tensor: - return torch.nn.functional.gelu(x.float()).type_as(x) - - -def get_available_activation_fns(): - return [ - "relu", - "gelu", - "gelu_fast", # deprecated - "gelu_accurate", - "tanh", - "linear", - ] - - -def get_activation_fn(activation: str): - """Returns the activation function corresponding to `activation`""" - - if activation == "relu": - return F.relu - elif activation == "gelu": - return gelu - elif activation == "gelu_accurate": - return gelu_accurate - elif activation == "tanh": - return torch.tanh - elif activation == "linear": - return lambda x: x - elif activation == "swish": - return torch.nn.SiLU - else: - raise RuntimeError("--activation-fn {} not supported".format(activation)) - - -def init_bert_params(module): - """ - Initialize the weights specific to the BERT Model. - This overrides the default initializations depending on the specified arguments. - 1. If normal_init_linear_weights is set then weights of linear - layer will be initialized using the normal distribution and - bais will be set to the specified value. - 2. If normal_init_embed_weights is set then weights of embedding - layer will be initialized using the normal distribution. - 3. If normal_init_proj_weights is set then weights of - in_project_weight for MultiHeadAttention initialized using - the normal distribution (to be validated). - """ - - def normal_(data): - # with FSDP, module params will be on CUDA, so we cast them back to CPU - # so that the RNG is consistent with and without FSDP - data.copy_(data.cpu().normal_(mean=0.0, std=0.02).to(data.device)) - - if isinstance(module, nn.Linear): - normal_(module.weight.data) - if module.bias is not None: - module.bias.data.zero_() - if isinstance(module, nn.Embedding): - normal_(module.weight.data) - if module.padding_idx is not None: - module.weight.data[module.padding_idx].zero_() - if isinstance(module, MultiheadAttention): - normal_(module.q_proj.weight.data) - normal_(module.k_proj.weight.data) - normal_(module.v_proj.weight.data) diff --git a/funasr_detach/models/data2vec/wav2vec2.py b/funasr_detach/models/data2vec/wav2vec2.py deleted file mode 100644 index cbda714ce0d1184f2cb1f5b1c16a880342b6c901..0000000000000000000000000000000000000000 --- a/funasr_detach/models/data2vec/wav2vec2.py +++ /dev/null @@ -1,407 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import logging -import math -from typing import List, Tuple - -import numpy as np -import torch -import torch.nn as nn -import torch.nn.functional as F - -from funasr_detach.models.data2vec import utils -from funasr_detach.models.data2vec.multihead_attention import MultiheadAttention - - -class ConvFeatureExtractionModel(nn.Module): - def __init__( - self, - conv_layers: List[Tuple[int, int, int]], - dropout: float = 0.0, - mode: str = "default", - conv_bias: bool = False, - in_d: int = 1, - ): - super().__init__() - - assert mode in {"default", "layer_norm"} - - def block( - n_in, - n_out, - k, - stride, - is_layer_norm=False, - is_group_norm=False, - conv_bias=False, - ): - def make_conv(): - conv = nn.Conv1d(n_in, n_out, k, stride=stride, bias=conv_bias) - nn.init.kaiming_normal_(conv.weight) - return conv - - assert ( - is_layer_norm and is_group_norm - ) == False, "layer norm and group norm are exclusive" - - if is_layer_norm: - return nn.Sequential( - make_conv(), - nn.Dropout(p=dropout), - nn.Sequential( - utils.TransposeLast(), - utils.Fp32LayerNorm(dim, elementwise_affine=True), - utils.TransposeLast(), - ), - nn.GELU(), - ) - elif is_group_norm: - return nn.Sequential( - make_conv(), - nn.Dropout(p=dropout), - utils.Fp32GroupNorm(dim, dim, affine=True), - nn.GELU(), - ) - else: - return nn.Sequential(make_conv(), nn.Dropout(p=dropout), nn.GELU()) - - self.conv_layers = nn.ModuleList() - for i, cl in enumerate(conv_layers): - assert len(cl) == 3, "invalid conv definition: " + str(cl) - (dim, k, stride) = cl - - self.conv_layers.append( - block( - in_d, - dim, - k, - stride, - is_layer_norm=mode == "layer_norm", - is_group_norm=mode == "default" and i == 0, - conv_bias=conv_bias, - ) - ) - in_d = dim - - def forward(self, x): - if len(x.shape) == 2: - x = x.unsqueeze(1) - else: - x = x.transpose(1, 2) - - for conv in self.conv_layers: - x = conv(x) - return x - - -def make_conv_pos(e, k, g): - pos_conv = nn.Conv1d( - e, - e, - kernel_size=k, - padding=k // 2, - groups=g, - ) - dropout = 0 - std = math.sqrt((4 * (1.0 - dropout)) / (k * e)) - nn.init.normal_(pos_conv.weight, mean=0, std=std) - nn.init.constant_(pos_conv.bias, 0) - - pos_conv = nn.utils.weight_norm(pos_conv, name="weight", dim=2) - pos_conv = nn.Sequential(pos_conv, utils.SamePad(k), nn.GELU()) - - return pos_conv - - -class TransformerEncoder(nn.Module): - def build_encoder_layer(self): - if self.layer_type == "transformer": - layer = TransformerSentenceEncoderLayer( - embedding_dim=self.embedding_dim, - ffn_embedding_dim=self.encoder_ffn_embed_dim, - num_attention_heads=self.encoder_attention_heads, - dropout=self.dropout, - attention_dropout=self.attention_dropout, - activation_dropout=self.activation_dropout, - activation_fn=self.activation_fn, - layer_norm_first=self.layer_norm_first, - ) - else: - logging.error("Only transformer is supported for data2vec now") - return layer - - def __init__( - self, - # position - dropout, - encoder_embed_dim, - required_seq_len_multiple, - pos_conv_depth, - conv_pos, - conv_pos_groups, - # transformer layers - layer_type, - encoder_layers, - encoder_ffn_embed_dim, - encoder_attention_heads, - attention_dropout, - activation_dropout, - activation_fn, - layer_norm_first, - encoder_layerdrop, - max_positions, - ): - super().__init__() - - # position - self.dropout = dropout - self.embedding_dim = encoder_embed_dim - self.required_seq_len_multiple = required_seq_len_multiple - if pos_conv_depth > 1: - num_layers = pos_conv_depth - k = max(3, conv_pos // num_layers) - - def make_conv_block(e, k, g, l): - return nn.Sequential( - *[ - nn.Sequential( - nn.Conv1d( - e, - e, - kernel_size=k, - padding=k // 2, - groups=g, - ), - utils.SamePad(k), - utils.TransposeLast(), - torch.nn.LayerNorm(e, elementwise_affine=False), - utils.TransposeLast(), - nn.GELU(), - ) - for _ in range(l) - ] - ) - - self.pos_conv = make_conv_block( - self.embedding_dim, k, conv_pos_groups, num_layers - ) - - else: - self.pos_conv = make_conv_pos( - self.embedding_dim, - conv_pos, - conv_pos_groups, - ) - - # transformer layers - self.layer_type = layer_type - self.encoder_ffn_embed_dim = encoder_ffn_embed_dim - self.encoder_attention_heads = encoder_attention_heads - self.attention_dropout = attention_dropout - self.activation_dropout = activation_dropout - self.activation_fn = activation_fn - self.layer_norm_first = layer_norm_first - self.layerdrop = encoder_layerdrop - self.max_positions = max_positions - self.layers = nn.ModuleList( - [self.build_encoder_layer() for _ in range(encoder_layers)] - ) - self.layer_norm = torch.nn.LayerNorm(self.embedding_dim) - - self.apply(utils.init_bert_params) - - def forward(self, x, padding_mask=None, layer=None): - x, layer_results = self.extract_features(x, padding_mask, layer) - - if self.layer_norm_first and layer is None: - x = self.layer_norm(x) - - return x, layer_results - - def extract_features( - self, - x, - padding_mask=None, - tgt_layer=None, - min_layer=0, - ): - - if padding_mask is not None: - x[padding_mask] = 0 - - x_conv = self.pos_conv(x.transpose(1, 2)) - x_conv = x_conv.transpose(1, 2) - x = x + x_conv - - if not self.layer_norm_first: - x = self.layer_norm(x) - - # pad to the sequence length dimension - x, pad_length = utils.pad_to_multiple( - x, self.required_seq_len_multiple, dim=-2, value=0 - ) - if pad_length > 0 and padding_mask is None: - padding_mask = x.new_zeros((x.size(0), x.size(1)), dtype=torch.bool) - padding_mask[:, -pad_length:] = True - else: - padding_mask, _ = utils.pad_to_multiple( - padding_mask, self.required_seq_len_multiple, dim=-1, value=True - ) - x = F.dropout(x, p=self.dropout, training=self.training) - - # B x T x C -> T x B x C - x = x.transpose(0, 1) - - layer_results = [] - r = None - for i, layer in enumerate(self.layers): - dropout_probability = np.random.random() if self.layerdrop > 0 else 1 - if not self.training or (dropout_probability > self.layerdrop): - x, (z, lr) = layer(x, self_attn_padding_mask=padding_mask) - if i >= min_layer: - layer_results.append((x, z, lr)) - if i == tgt_layer: - r = x - break - - if r is not None: - x = r - - # T x B x C -> B x T x C - x = x.transpose(0, 1) - - # undo paddding - if pad_length > 0: - x = x[:, :-pad_length] - - def undo_pad(a, b, c): - return ( - a[:-pad_length], - b[:-pad_length] if b is not None else b, - c[:-pad_length], - ) - - layer_results = [undo_pad(*u) for u in layer_results] - - return x, layer_results - - def max_positions(self): - """Maximum output length supported by the encoder.""" - return self.max_positions - - def upgrade_state_dict_named(self, state_dict, name): - """Upgrade a (possibly old) state dict for new versions of fairseq.""" - return state_dict - - -class TransformerSentenceEncoderLayer(nn.Module): - """ - Implements a Transformer Encoder Layer used in BERT/XLM style pre-trained - models. - """ - - def __init__( - self, - embedding_dim: int = 768, - ffn_embedding_dim: int = 3072, - num_attention_heads: int = 8, - dropout: float = 0.1, - attention_dropout: float = 0.1, - activation_dropout: float = 0.1, - activation_fn: str = "relu", - layer_norm_first: bool = False, - ) -> None: - - super().__init__() - # Initialize parameters - self.embedding_dim = embedding_dim - self.dropout = dropout - self.activation_dropout = activation_dropout - - # Initialize blocks - self.activation_fn = utils.get_activation_fn(activation_fn) - self.self_attn = MultiheadAttention( - self.embedding_dim, - num_attention_heads, - dropout=attention_dropout, - self_attention=True, - ) - - self.dropout1 = nn.Dropout(dropout) - self.dropout2 = nn.Dropout(self.activation_dropout) - self.dropout3 = nn.Dropout(dropout) - - self.layer_norm_first = layer_norm_first - - # layer norm associated with the self attention layer - self.self_attn_layer_norm = torch.nn.LayerNorm(self.embedding_dim) - self.fc1 = nn.Linear(self.embedding_dim, ffn_embedding_dim) - self.fc2 = nn.Linear(ffn_embedding_dim, self.embedding_dim) - - # layer norm associated with the position wise feed-forward NN - self.final_layer_norm = torch.nn.LayerNorm(self.embedding_dim) - - def forward( - self, - x: torch.Tensor, # (T, B, C) - self_attn_mask: torch.Tensor = None, - self_attn_padding_mask: torch.Tensor = None, - ): - """ - LayerNorm is applied either before or after the self-attention/ffn - modules similar to the original Transformer imlementation. - """ - residual = x - - if self.layer_norm_first: - x = self.self_attn_layer_norm(x) - x, attn = self.self_attn( - query=x, - key=x, - value=x, - key_padding_mask=self_attn_padding_mask, - attn_mask=self_attn_mask, - need_weights=False, - ) - x = self.dropout1(x) - x = residual + x - - residual = x - x = self.final_layer_norm(x) - x = self.activation_fn(self.fc1(x)) - x = self.dropout2(x) - x = self.fc2(x) - - layer_result = x - - x = self.dropout3(x) - x = residual + x - else: - x, attn = self.self_attn( - query=x, - key=x, - value=x, - key_padding_mask=self_attn_padding_mask, - need_weights=False, - ) - - x = self.dropout1(x) - x = residual + x - - x = self.self_attn_layer_norm(x) - - residual = x - x = self.activation_fn(self.fc1(x)) - x = self.dropout2(x) - x = self.fc2(x) - - layer_result = x - - x = self.dropout3(x) - x = residual + x - x = self.final_layer_norm(x) - - return x, (attn, layer_result) diff --git a/funasr_detach/models/e_branchformer/__init__.py b/funasr_detach/models/e_branchformer/__init__.py deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/funasr_detach/models/e_branchformer/encoder.py b/funasr_detach/models/e_branchformer/encoder.py deleted file mode 100644 index 33ab040454ffbb48dc8677456187f7a6a5db57e7..0000000000000000000000000000000000000000 --- a/funasr_detach/models/e_branchformer/encoder.py +++ /dev/null @@ -1,471 +0,0 @@ -# Copyright 2022 Kwangyoun Kim (ASAPP inc.) -# Apache 2.0 (http://www.apache.org/licenses/LICENSE-2.0) - -"""E-Branchformer encoder definition. -Reference: - Kwangyoun Kim, Felix Wu, Yifan Peng, Jing Pan, - Prashant Sridhar, Kyu J. Han, Shinji Watanabe, - "E-Branchformer: Branchformer with Enhanced merging - for speech recognition," in SLT 2022. -""" - -import logging -from typing import List, Optional, Tuple - -import torch -import torch.nn as nn -from funasr_detach.models.ctc.ctc import CTC -from funasr_detach.models.branchformer.cgmlp import ConvolutionalGatingMLP -from funasr_detach.models.branchformer.fastformer import FastSelfAttention -from funasr_detach.models.transformer.utils.nets_utils import ( - get_activation, - make_pad_mask, -) -from funasr_detach.models.transformer.attention import ( # noqa: H301 - LegacyRelPositionMultiHeadedAttention, - MultiHeadedAttention, - RelPositionMultiHeadedAttention, -) -from funasr_detach.models.transformer.embedding import ( # noqa: H301 - LegacyRelPositionalEncoding, - PositionalEncoding, - RelPositionalEncoding, - ScaledPositionalEncoding, -) -from funasr_detach.models.transformer.layer_norm import LayerNorm -from funasr_detach.models.transformer.positionwise_feed_forward import ( - PositionwiseFeedForward, -) -from funasr_detach.models.transformer.utils.repeat import repeat -from funasr_detach.models.transformer.utils.subsampling import ( - Conv2dSubsampling, - Conv2dSubsampling2, - Conv2dSubsampling6, - Conv2dSubsampling8, - TooShortUttError, - check_short_utt, -) -from funasr_detach.register import tables - - -class EBranchformerEncoderLayer(torch.nn.Module): - """E-Branchformer encoder layer module. - - Args: - size (int): model dimension - attn: standard self-attention or efficient attention - cgmlp: ConvolutionalGatingMLP - feed_forward: feed-forward module, optional - feed_forward: macaron-style feed-forward module, optional - dropout_rate (float): dropout probability - merge_conv_kernel (int): kernel size of the depth-wise conv in merge module - """ - - def __init__( - self, - size: int, - attn: torch.nn.Module, - cgmlp: torch.nn.Module, - feed_forward: Optional[torch.nn.Module], - feed_forward_macaron: Optional[torch.nn.Module], - dropout_rate: float, - merge_conv_kernel: int = 3, - ): - super().__init__() - - self.size = size - self.attn = attn - self.cgmlp = cgmlp - - self.feed_forward = feed_forward - self.feed_forward_macaron = feed_forward_macaron - self.ff_scale = 1.0 - if self.feed_forward is not None: - self.norm_ff = LayerNorm(size) - if self.feed_forward_macaron is not None: - self.ff_scale = 0.5 - self.norm_ff_macaron = LayerNorm(size) - - self.norm_mha = LayerNorm(size) # for the MHA module - self.norm_mlp = LayerNorm(size) # for the MLP module - self.norm_final = LayerNorm(size) # for the final output of the block - - self.dropout = torch.nn.Dropout(dropout_rate) - - self.depthwise_conv_fusion = torch.nn.Conv1d( - size + size, - size + size, - kernel_size=merge_conv_kernel, - stride=1, - padding=(merge_conv_kernel - 1) // 2, - groups=size + size, - bias=True, - ) - self.merge_proj = torch.nn.Linear(size + size, size) - - def forward(self, x_input, mask, cache=None): - """Compute encoded features. - - Args: - x_input (Union[Tuple, torch.Tensor]): Input tensor w/ or w/o pos emb. - - w/ pos emb: Tuple of tensors [(#batch, time, size), (1, time, size)]. - - w/o pos emb: Tensor (#batch, time, size). - mask (torch.Tensor): Mask tensor for the input (#batch, 1, time). - cache (torch.Tensor): Cache tensor of the input (#batch, time - 1, size). - Returns: - torch.Tensor: Output tensor (#batch, time, size). - torch.Tensor: Mask tensor (#batch, time). - """ - - if cache is not None: - raise NotImplementedError("cache is not None, which is not tested") - - if isinstance(x_input, tuple): - x, pos_emb = x_input[0], x_input[1] - else: - x, pos_emb = x_input, None - - if self.feed_forward_macaron is not None: - residual = x - x = self.norm_ff_macaron(x) - x = residual + self.ff_scale * self.dropout(self.feed_forward_macaron(x)) - - # Two branches - x1 = x - x2 = x - - # Branch 1: multi-headed attention module - x1 = self.norm_mha(x1) - - if isinstance(self.attn, FastSelfAttention): - x_att = self.attn(x1, mask) - else: - if pos_emb is not None: - x_att = self.attn(x1, x1, x1, pos_emb, mask) - else: - x_att = self.attn(x1, x1, x1, mask) - - x1 = self.dropout(x_att) - - # Branch 2: convolutional gating mlp - x2 = self.norm_mlp(x2) - - if pos_emb is not None: - x2 = (x2, pos_emb) - x2 = self.cgmlp(x2, mask) - if isinstance(x2, tuple): - x2 = x2[0] - - x2 = self.dropout(x2) - - # Merge two branches - x_concat = torch.cat([x1, x2], dim=-1) - x_tmp = x_concat.transpose(1, 2) - x_tmp = self.depthwise_conv_fusion(x_tmp) - x_tmp = x_tmp.transpose(1, 2) - x = x + self.dropout(self.merge_proj(x_concat + x_tmp)) - - if self.feed_forward is not None: - # feed forward module - residual = x - x = self.norm_ff(x) - x = residual + self.ff_scale * self.dropout(self.feed_forward(x)) - - x = self.norm_final(x) - - if pos_emb is not None: - return (x, pos_emb), mask - - return x, mask - - -@tables.register("encoder_classes", "EBranchformerEncoder") -class EBranchformerEncoder(nn.Module): - """E-Branchformer encoder module.""" - - def __init__( - self, - input_size: int, - output_size: int = 256, - attention_heads: int = 4, - attention_layer_type: str = "rel_selfattn", - pos_enc_layer_type: str = "rel_pos", - rel_pos_type: str = "latest", - cgmlp_linear_units: int = 2048, - cgmlp_conv_kernel: int = 31, - use_linear_after_conv: bool = False, - gate_activation: str = "identity", - num_blocks: int = 12, - dropout_rate: float = 0.1, - positional_dropout_rate: float = 0.1, - attention_dropout_rate: float = 0.0, - input_layer: Optional[str] = "conv2d", - zero_triu: bool = False, - padding_idx: int = -1, - layer_drop_rate: float = 0.0, - max_pos_emb_len: int = 5000, - use_ffn: bool = False, - macaron_ffn: bool = False, - ffn_activation_type: str = "swish", - linear_units: int = 2048, - positionwise_layer_type: str = "linear", - merge_conv_kernel: int = 3, - interctc_layer_idx=None, - interctc_use_conditioning: bool = False, - ): - super().__init__() - self._output_size = output_size - - if rel_pos_type == "legacy": - if pos_enc_layer_type == "rel_pos": - pos_enc_layer_type = "legacy_rel_pos" - if attention_layer_type == "rel_selfattn": - attention_layer_type = "legacy_rel_selfattn" - elif rel_pos_type == "latest": - assert attention_layer_type != "legacy_rel_selfattn" - assert pos_enc_layer_type != "legacy_rel_pos" - else: - raise ValueError("unknown rel_pos_type: " + rel_pos_type) - - if pos_enc_layer_type == "abs_pos": - pos_enc_class = PositionalEncoding - elif pos_enc_layer_type == "scaled_abs_pos": - pos_enc_class = ScaledPositionalEncoding - elif pos_enc_layer_type == "rel_pos": - assert attention_layer_type == "rel_selfattn" - pos_enc_class = RelPositionalEncoding - elif pos_enc_layer_type == "legacy_rel_pos": - assert attention_layer_type == "legacy_rel_selfattn" - pos_enc_class = LegacyRelPositionalEncoding - logging.warning( - "Using legacy_rel_pos and it will be deprecated in the future." - ) - else: - raise ValueError("unknown pos_enc_layer: " + pos_enc_layer_type) - - if input_layer == "linear": - self.embed = torch.nn.Sequential( - torch.nn.Linear(input_size, output_size), - torch.nn.LayerNorm(output_size), - torch.nn.Dropout(dropout_rate), - pos_enc_class(output_size, positional_dropout_rate, max_pos_emb_len), - ) - elif input_layer == "conv2d": - self.embed = Conv2dSubsampling( - input_size, - output_size, - dropout_rate, - pos_enc_class(output_size, positional_dropout_rate, max_pos_emb_len), - ) - elif input_layer == "conv2d2": - self.embed = Conv2dSubsampling2( - input_size, - output_size, - dropout_rate, - pos_enc_class(output_size, positional_dropout_rate, max_pos_emb_len), - ) - elif input_layer == "conv2d6": - self.embed = Conv2dSubsampling6( - input_size, - output_size, - dropout_rate, - pos_enc_class(output_size, positional_dropout_rate, max_pos_emb_len), - ) - elif input_layer == "conv2d8": - self.embed = Conv2dSubsampling8( - input_size, - output_size, - dropout_rate, - pos_enc_class(output_size, positional_dropout_rate, max_pos_emb_len), - ) - elif input_layer == "embed": - self.embed = torch.nn.Sequential( - torch.nn.Embedding(input_size, output_size, padding_idx=padding_idx), - pos_enc_class(output_size, positional_dropout_rate, max_pos_emb_len), - ) - elif isinstance(input_layer, torch.nn.Module): - self.embed = torch.nn.Sequential( - input_layer, - pos_enc_class(output_size, positional_dropout_rate, max_pos_emb_len), - ) - elif input_layer is None: - if input_size == output_size: - self.embed = None - else: - self.embed = torch.nn.Linear(input_size, output_size) - else: - raise ValueError("unknown input_layer: " + input_layer) - - activation = get_activation(ffn_activation_type) - if positionwise_layer_type == "linear": - positionwise_layer = PositionwiseFeedForward - positionwise_layer_args = ( - output_size, - linear_units, - dropout_rate, - activation, - ) - elif positionwise_layer_type is None: - logging.warning("no macaron ffn") - else: - raise ValueError("Support only linear.") - - if attention_layer_type == "selfattn": - encoder_selfattn_layer = MultiHeadedAttention - encoder_selfattn_layer_args = ( - attention_heads, - output_size, - attention_dropout_rate, - ) - elif attention_layer_type == "legacy_rel_selfattn": - assert pos_enc_layer_type == "legacy_rel_pos" - encoder_selfattn_layer = LegacyRelPositionMultiHeadedAttention - encoder_selfattn_layer_args = ( - attention_heads, - output_size, - attention_dropout_rate, - ) - logging.warning( - "Using legacy_rel_selfattn and it will be deprecated in the future." - ) - elif attention_layer_type == "rel_selfattn": - assert pos_enc_layer_type == "rel_pos" - encoder_selfattn_layer = RelPositionMultiHeadedAttention - encoder_selfattn_layer_args = ( - attention_heads, - output_size, - attention_dropout_rate, - zero_triu, - ) - elif attention_layer_type == "fast_selfattn": - assert pos_enc_layer_type in ["abs_pos", "scaled_abs_pos"] - encoder_selfattn_layer = FastSelfAttention - encoder_selfattn_layer_args = ( - output_size, - attention_heads, - attention_dropout_rate, - ) - else: - raise ValueError("unknown encoder_attn_layer: " + attention_layer_type) - - cgmlp_layer = ConvolutionalGatingMLP - cgmlp_layer_args = ( - output_size, - cgmlp_linear_units, - cgmlp_conv_kernel, - dropout_rate, - use_linear_after_conv, - gate_activation, - ) - - self.encoders = repeat( - num_blocks, - lambda lnum: EBranchformerEncoderLayer( - output_size, - encoder_selfattn_layer(*encoder_selfattn_layer_args), - cgmlp_layer(*cgmlp_layer_args), - positionwise_layer(*positionwise_layer_args) if use_ffn else None, - ( - positionwise_layer(*positionwise_layer_args) - if use_ffn and macaron_ffn - else None - ), - dropout_rate, - merge_conv_kernel, - ), - layer_drop_rate, - ) - self.after_norm = LayerNorm(output_size) - - if interctc_layer_idx is None: - interctc_layer_idx = [] - self.interctc_layer_idx = interctc_layer_idx - if len(interctc_layer_idx) > 0: - assert 0 < min(interctc_layer_idx) and max(interctc_layer_idx) < num_blocks - self.interctc_use_conditioning = interctc_use_conditioning - self.conditioning_layer = None - - def output_size(self) -> int: - return self._output_size - - def forward( - self, - xs_pad: torch.Tensor, - ilens: torch.Tensor, - prev_states: torch.Tensor = None, - ctc: CTC = None, - max_layer: int = None, - ) -> Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]: - """Calculate forward propagation. - - Args: - xs_pad (torch.Tensor): Input tensor (#batch, L, input_size). - ilens (torch.Tensor): Input length (#batch). - prev_states (torch.Tensor): Not to be used now. - ctc (CTC): Intermediate CTC module. - max_layer (int): Layer depth below which InterCTC is applied. - Returns: - torch.Tensor: Output tensor (#batch, L, output_size). - torch.Tensor: Output length (#batch). - torch.Tensor: Not to be used now. - """ - - masks = (~make_pad_mask(ilens)[:, None, :]).to(xs_pad.device) - - if ( - isinstance(self.embed, Conv2dSubsampling) - or isinstance(self.embed, Conv2dSubsampling2) - or isinstance(self.embed, Conv2dSubsampling6) - or isinstance(self.embed, Conv2dSubsampling8) - ): - short_status, limit_size = check_short_utt(self.embed, xs_pad.size(1)) - if short_status: - raise TooShortUttError( - f"has {xs_pad.size(1)} frames and is too short for subsampling " - + f"(it needs more than {limit_size} frames), return empty results", - xs_pad.size(1), - limit_size, - ) - xs_pad, masks = self.embed(xs_pad, masks) - elif self.embed is not None: - xs_pad = self.embed(xs_pad) - - intermediate_outs = [] - if len(self.interctc_layer_idx) == 0: - if max_layer is not None and 0 <= max_layer < len(self.encoders): - for layer_idx, encoder_layer in enumerate(self.encoders): - xs_pad, masks = encoder_layer(xs_pad, masks) - if layer_idx >= max_layer: - break - else: - xs_pad, masks = self.encoders(xs_pad, masks) - else: - for layer_idx, encoder_layer in enumerate(self.encoders): - xs_pad, masks = encoder_layer(xs_pad, masks) - - if layer_idx + 1 in self.interctc_layer_idx: - encoder_out = xs_pad - - if isinstance(encoder_out, tuple): - encoder_out = encoder_out[0] - - intermediate_outs.append((layer_idx + 1, encoder_out)) - - if self.interctc_use_conditioning: - ctc_out = ctc.softmax(encoder_out) - - if isinstance(xs_pad, tuple): - xs_pad = list(xs_pad) - xs_pad[0] = xs_pad[0] + self.conditioning_layer(ctc_out) - xs_pad = tuple(xs_pad) - else: - xs_pad = xs_pad + self.conditioning_layer(ctc_out) - - if isinstance(xs_pad, tuple): - xs_pad = xs_pad[0] - - xs_pad = self.after_norm(xs_pad) - olens = masks.squeeze(1).sum(1) - if len(intermediate_outs) > 0: - return (xs_pad, intermediate_outs), olens, None - return xs_pad, olens, None diff --git a/funasr_detach/models/e_branchformer/model.py b/funasr_detach/models/e_branchformer/model.py deleted file mode 100644 index c9bcf394ae61a8a7cb619b86fbe3144092ea9185..0000000000000000000000000000000000000000 --- a/funasr_detach/models/e_branchformer/model.py +++ /dev/null @@ -1,17 +0,0 @@ -import logging - -from funasr_detach.models.transformer.model import Transformer -from funasr_detach.register import tables - - -@tables.register("model_classes", "EBranchformer") -class EBranchformer(Transformer): - """CTC-attention hybrid Encoder-Decoder model""" - - def __init__( - self, - *args, - **kwargs, - ): - - super().__init__(*args, **kwargs) diff --git a/funasr_detach/models/e_branchformer/template.yaml b/funasr_detach/models/e_branchformer/template.yaml deleted file mode 100644 index dea580e5254f23f1c8510de6a6e56a1b3be7495b..0000000000000000000000000000000000000000 --- a/funasr_detach/models/e_branchformer/template.yaml +++ /dev/null @@ -1,116 +0,0 @@ -# This is an example that demonstrates how to configure a model file. -# You can modify the configuration according to your own requirements. - -# to print the register_table: -# from funasr.register import tables -# tables.print() - -# network architecture -model: Branchformer -model_conf: - ctc_weight: 0.3 - lsm_weight: 0.1 # label smoothing option - length_normalized_loss: false - -# encoder -encoder: EBranchformerEncoder -encoder_conf: - output_size: 256 - attention_heads: 4 - attention_layer_type: rel_selfattn - pos_enc_layer_type: rel_pos - rel_pos_type: latest - cgmlp_linear_units: 1024 - cgmlp_conv_kernel: 31 - use_linear_after_conv: false - gate_activation: identity - num_blocks: 12 - dropout_rate: 0.1 - positional_dropout_rate: 0.1 - attention_dropout_rate: 0.1 - input_layer: conv2d - layer_drop_rate: 0.0 - linear_units: 1024 - positionwise_layer_type: linear - use_ffn: true - macaron_ffn: true - merge_conv_kernel: 31 - -# decoder -decoder: TransformerDecoder -decoder_conf: - attention_heads: 4 - linear_units: 2048 - num_blocks: 6 - dropout_rate: 0.1 - positional_dropout_rate: 0.1 - self_attention_dropout_rate: 0. - src_attention_dropout_rate: 0. - - -# frontend related -frontend: WavFrontend -frontend_conf: - fs: 16000 - window: hamming - n_mels: 80 - frame_length: 25 - frame_shift: 10 - dither: 0.0 - lfr_m: 1 - lfr_n: 1 - -specaug: SpecAug -specaug_conf: - apply_time_warp: true - time_warp_window: 5 - time_warp_mode: bicubic - apply_freq_mask: true - freq_mask_width_range: - - 0 - - 30 - num_freq_mask: 2 - apply_time_mask: true - time_mask_width_range: - - 0 - - 40 - num_time_mask: 2 - -train_conf: - accum_grad: 1 - grad_clip: 5 - max_epoch: 180 - keep_nbest_models: 10 - log_interval: 50 - -optim: adam -optim_conf: - lr: 0.001 - weight_decay: 0.000001 -scheduler: warmuplr -scheduler_conf: - warmup_steps: 35000 - -dataset: AudioDataset -dataset_conf: - index_ds: IndexDSJsonl - batch_sampler: DynamicBatchLocalShuffleSampler - batch_type: example # example or length - batch_size: 1 # if batch_type is example, batch_size is the numbers of samples; if length, batch_size is source_token_len+target_token_len; - max_token_length: 2048 # filter samples if source_token_len+target_token_len > max_token_length, - buffer_size: 500 - shuffle: True - num_workers: 4 - -tokenizer: CharTokenizer -tokenizer_conf: - unk_symbol: - split_with_space: true - - -ctc_conf: - dropout_rate: 0.0 - ctc_type: builtin - reduce: true - ignore_nan_grad: true -normalize: null diff --git a/funasr_detach/models/eend/__init__.py b/funasr_detach/models/eend/__init__.py deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/funasr_detach/models/eend/e2e_diar_eend_ola.py b/funasr_detach/models/eend/e2e_diar_eend_ola.py deleted file mode 100644 index 2fe7982aece28ea42d1c0f69a9069c2d88be3ed0..0000000000000000000000000000000000000000 --- a/funasr_detach/models/eend/e2e_diar_eend_ola.py +++ /dev/null @@ -1,272 +0,0 @@ -from contextlib import contextmanager -from distutils.version import LooseVersion -from typing import Dict, List, Tuple, Optional - -import numpy as np -import torch -import torch.nn as nn -import torch.nn.functional as F - -from funasr_detach.frontends.wav_frontend import WavFrontendMel23 -from funasr_detach.models.eend.encoder import EENDOLATransformerEncoder -from funasr_detach.models.eend.encoder_decoder_attractor import EncoderDecoderAttractor -from funasr_detach.models.eend.utils.losses import ( - standard_loss, - cal_power_loss, - fast_batch_pit_n_speaker_loss, -) -from funasr_detach.models.eend.utils.power import create_powerlabel -from funasr_detach.models.eend.utils.power import generate_mapping_dict -from funasr_detach.train_utils.device_funcs import force_gatherable - -if LooseVersion(torch.__version__) >= LooseVersion("1.6.0"): - pass -else: - # Nothing to do if torch<1.6.0 - @contextmanager - def autocast(enabled=True): - yield - - -def pad_attractor(att, max_n_speakers): - C, D = att.shape - if C < max_n_speakers: - att = torch.cat( - [att, torch.zeros(max_n_speakers - C, D).to(torch.float32).to(att.device)], - dim=0, - ) - return att - - -def pad_labels(ts, out_size): - for i, t in enumerate(ts): - if t.shape[1] < out_size: - ts[i] = F.pad( - t, (0, out_size - t.shape[1], 0, 0), mode="constant", value=0.0 - ) - return ts - - -def pad_results(ys, out_size): - ys_padded = [] - for i, y in enumerate(ys): - if y.shape[1] < out_size: - ys_padded.append( - torch.cat( - [ - y, - torch.zeros(y.shape[0], out_size - y.shape[1]) - .to(torch.float32) - .to(y.device), - ], - dim=1, - ) - ) - else: - ys_padded.append(y) - return ys_padded - - -class DiarEENDOLAModel(nn.Module): - """EEND-OLA diarization model""" - - def __init__( - self, - frontend: Optional[WavFrontendMel23], - encoder: EENDOLATransformerEncoder, - encoder_decoder_attractor: EncoderDecoderAttractor, - n_units: int = 256, - max_n_speaker: int = 8, - attractor_loss_weight: float = 1.0, - mapping_dict=None, - **kwargs, - ): - super().__init__() - self.frontend = frontend - self.enc = encoder - self.encoder_decoder_attractor = encoder_decoder_attractor - self.attractor_loss_weight = attractor_loss_weight - self.max_n_speaker = max_n_speaker - if mapping_dict is None: - mapping_dict = generate_mapping_dict(max_speaker_num=self.max_n_speaker) - self.mapping_dict = mapping_dict - # PostNet - self.postnet = nn.LSTM(self.max_n_speaker, n_units, 1, batch_first=True) - self.output_layer = nn.Linear(n_units, mapping_dict["oov"] + 1) - - def forward_encoder(self, xs, ilens): - xs = nn.utils.rnn.pad_sequence(xs, batch_first=True, padding_value=-1) - pad_shape = xs.shape - xs_mask = [torch.ones(ilen).to(xs.device) for ilen in ilens] - xs_mask = torch.nn.utils.rnn.pad_sequence( - xs_mask, batch_first=True, padding_value=0 - ).unsqueeze(-2) - emb = self.enc(xs, xs_mask) - emb = torch.split(emb.view(pad_shape[0], pad_shape[1], -1), 1, dim=0) - emb = [e[0][:ilen] for e, ilen in zip(emb, ilens)] - return emb - - def forward_post_net(self, logits, ilens): - maxlen = torch.max(ilens).to(torch.int).item() - logits = nn.utils.rnn.pad_sequence(logits, batch_first=True, padding_value=-1) - logits = nn.utils.rnn.pack_padded_sequence( - logits, ilens.cpu().to(torch.int64), batch_first=True, enforce_sorted=False - ) - outputs, (_, _) = self.postnet(logits) - outputs = nn.utils.rnn.pad_packed_sequence( - outputs, batch_first=True, padding_value=-1, total_length=maxlen - )[0] - outputs = [ - output[: ilens[i].to(torch.int).item()] for i, output in enumerate(outputs) - ] - outputs = [self.output_layer(output) for output in outputs] - return outputs - - def forward( - self, - speech: List[torch.Tensor], - speaker_labels: List[torch.Tensor], - orders: torch.Tensor, - ) -> Tuple[torch.Tensor, Dict[str, torch.Tensor], torch.Tensor]: - - # Check that batch_size is unified - assert len(speech) == len(speaker_labels), (len(speech), len(speaker_labels)) - speech_lengths = torch.tensor([len(sph) for sph in speech]).to(torch.int64) - speaker_labels_lengths = torch.tensor( - [spk.shape[-1] for spk in speaker_labels] - ).to(torch.int64) - batch_size = len(speech) - - # Encoder - encoder_out = self.forward_encoder(speech, speech_lengths) - - # Encoder-decoder attractor - attractor_loss, attractors = self.encoder_decoder_attractor( - [e[order] for e, order in zip(encoder_out, orders)], speaker_labels_lengths - ) - speaker_logits = [ - torch.matmul(e, att.permute(1, 0)) - for e, att in zip(encoder_out, attractors) - ] - - # pit loss - pit_speaker_labels = fast_batch_pit_n_speaker_loss( - speaker_logits, speaker_labels - ) - pit_loss = standard_loss(speaker_logits, pit_speaker_labels) - - # pse loss - with torch.no_grad(): - power_ts = [ - create_powerlabel( - label.cpu().numpy(), self.mapping_dict, self.max_n_speaker - ).to(encoder_out[0].device, non_blocking=True) - for label in pit_speaker_labels - ] - pad_attractors = [pad_attractor(att, self.max_n_speaker) for att in attractors] - pse_speaker_logits = [ - torch.matmul(e, pad_att.permute(1, 0)) - for e, pad_att in zip(encoder_out, pad_attractors) - ] - pse_speaker_logits = self.forward_post_net(pse_speaker_logits, speech_lengths) - pse_loss = cal_power_loss(pse_speaker_logits, power_ts) - - loss = pse_loss + pit_loss + self.attractor_loss_weight * attractor_loss - - stats = dict() - stats["pse_loss"] = pse_loss.detach() - stats["pit_loss"] = pit_loss.detach() - stats["attractor_loss"] = attractor_loss.detach() - stats["batch_size"] = batch_size - - # Collect total loss stats - stats["loss"] = torch.clone(loss.detach()) - - # force_gatherable: to-device and to-tensor if scalar for DataParallel - loss, stats, weight = force_gatherable((loss, stats, batch_size), loss.device) - return loss, stats, weight - - def estimate_sequential( - self, - speech: torch.Tensor, - n_speakers: int = None, - shuffle: bool = True, - threshold: float = 0.5, - **kwargs, - ): - speech_lengths = torch.tensor([len(sph) for sph in speech]).to(torch.int64) - emb = self.forward_encoder(speech, speech_lengths) - if shuffle: - orders = [np.arange(e.shape[0]) for e in emb] - for order in orders: - np.random.shuffle(order) - attractors, probs = self.encoder_decoder_attractor.estimate( - [ - e[torch.from_numpy(order).to(torch.long).to(speech[0].device)] - for e, order in zip(emb, orders) - ] - ) - else: - attractors, probs = self.encoder_decoder_attractor.estimate(emb) - attractors_active = [] - for p, att, e in zip(probs, attractors, emb): - if n_speakers and n_speakers >= 0: - att = att[:n_speakers,] - attractors_active.append(att) - elif threshold is not None: - silence = torch.nonzero(p < threshold)[0] - n_spk = silence[0] if silence.size else None - att = att[:n_spk,] - attractors_active.append(att) - else: - NotImplementedError("n_speakers or threshold has to be given.") - raw_n_speakers = [att.shape[0] for att in attractors_active] - attractors = [ - ( - pad_attractor(att, self.max_n_speaker) - if att.shape[0] <= self.max_n_speaker - else att[: self.max_n_speaker] - ) - for att in attractors_active - ] - ys = [torch.matmul(e, att.permute(1, 0)) for e, att in zip(emb, attractors)] - logits = self.forward_post_net(ys, speech_lengths) - ys = [ - self.recover_y_from_powerlabel(logit, raw_n_speaker) - for logit, raw_n_speaker in zip(logits, raw_n_speakers) - ] - - return ys, emb, attractors, raw_n_speakers - - def recover_y_from_powerlabel(self, logit, n_speaker): - pred = torch.argmax(torch.softmax(logit, dim=-1), dim=-1) - oov_index = torch.where(pred == self.mapping_dict["oov"])[0] - for i in oov_index: - if i > 0: - pred[i] = pred[i - 1] - else: - pred[i] = 0 - pred = [self.inv_mapping_func(i) for i in pred] - decisions = [bin(num)[2:].zfill(self.max_n_speaker)[::-1] for num in pred] - decisions = ( - torch.from_numpy( - np.stack([np.array([int(i) for i in dec]) for dec in decisions], axis=0) - ) - .to(logit.device) - .to(torch.float32) - ) - decisions = decisions[:, :n_speaker] - return decisions - - def inv_mapping_func(self, label): - - if not isinstance(label, int): - label = int(label) - if label in self.mapping_dict["label2dec"].keys(): - num = self.mapping_dict["label2dec"][label] - else: - num = -1 - return num - - def collect_feats(self, **batch: torch.Tensor) -> Dict[str, torch.Tensor]: - pass diff --git a/funasr_detach/models/eend/eend_ola_dataloader.py b/funasr_detach/models/eend/eend_ola_dataloader.py deleted file mode 100644 index 983046751f4c1fb1133aef714172b0aa7ad1f06a..0000000000000000000000000000000000000000 --- a/funasr_detach/models/eend/eend_ola_dataloader.py +++ /dev/null @@ -1,61 +0,0 @@ -import logging - -import kaldiio -import numpy as np -import torch -from torch.utils.data import DataLoader -from torch.utils.data import Dataset - - -def custom_collate(batch): - keys, speech, speaker_labels, orders = zip(*batch) - speech = [torch.from_numpy(np.copy(sph)).to(torch.float32) for sph in speech] - speaker_labels = [ - torch.from_numpy(np.copy(spk)).to(torch.float32) for spk in speaker_labels - ] - orders = [torch.from_numpy(np.copy(o)).to(torch.int64) for o in orders] - batch = dict(speech=speech, speaker_labels=speaker_labels, orders=orders) - - return keys, batch - - -class EENDOLADataset(Dataset): - def __init__( - self, - data_file, - ): - self.data_file = data_file - with open(data_file) as f: - lines = f.readlines() - self.samples = [line.strip().split() for line in lines] - logging.info("total samples: {}".format(len(self.samples))) - - def __len__(self): - return len(self.samples) - - def __getitem__(self, idx): - key, speech_path, speaker_label_path = self.samples[idx] - speech = kaldiio.load_mat(speech_path) - speaker_label = kaldiio.load_mat(speaker_label_path).reshape( - speech.shape[0], -1 - ) - - order = np.arange(speech.shape[0]) - np.random.shuffle(order) - - return key, speech, speaker_label, order - - -class EENDOLADataLoader: - def __init__(self, data_file, batch_size, shuffle=True, num_workers=8): - dataset = EENDOLADataset(data_file) - self.data_loader = DataLoader( - dataset, - batch_size=batch_size, - collate_fn=custom_collate, - shuffle=shuffle, - num_workers=num_workers, - ) - - def build_iter(self, epoch): - return self.data_loader diff --git a/funasr_detach/models/eend/encoder.py b/funasr_detach/models/eend/encoder.py deleted file mode 100644 index 4a8039525c2e1219fad71923d5b87eec416679ca..0000000000000000000000000000000000000000 --- a/funasr_detach/models/eend/encoder.py +++ /dev/null @@ -1,126 +0,0 @@ -import math - -import torch -import torch.nn.functional as F -from torch import nn - - -class MultiHeadSelfAttention(nn.Module): - def __init__(self, n_units, h=8, dropout_rate=0.1): - super().__init__() - self.linearQ = nn.Linear(n_units, n_units) - self.linearK = nn.Linear(n_units, n_units) - self.linearV = nn.Linear(n_units, n_units) - self.linearO = nn.Linear(n_units, n_units) - self.d_k = n_units // h - self.h = h - self.dropout = nn.Dropout(dropout_rate) - - def __call__(self, x, batch_size, x_mask): - q = self.linearQ(x).view(batch_size, -1, self.h, self.d_k) - k = self.linearK(x).view(batch_size, -1, self.h, self.d_k) - v = self.linearV(x).view(batch_size, -1, self.h, self.d_k) - scores = torch.matmul(q.permute(0, 2, 1, 3), k.permute(0, 2, 3, 1)) / math.sqrt( - self.d_k - ) - if x_mask is not None: - x_mask = x_mask.unsqueeze(1) - scores = scores.masked_fill(x_mask == 0, -1e9) - self.att = F.softmax(scores, dim=3) - p_att = self.dropout(self.att) - x = torch.matmul(p_att, v.permute(0, 2, 1, 3)) - x = x.permute(0, 2, 1, 3).contiguous().view(-1, self.h * self.d_k) - return self.linearO(x) - - -class PositionwiseFeedForward(nn.Module): - def __init__(self, n_units, d_units, dropout_rate): - super(PositionwiseFeedForward, self).__init__() - self.linear1 = nn.Linear(n_units, d_units) - self.linear2 = nn.Linear(d_units, n_units) - self.dropout = nn.Dropout(dropout_rate) - - def __call__(self, x): - return self.linear2(self.dropout(F.relu(self.linear1(x)))) - - -class PositionalEncoding(torch.nn.Module): - def __init__(self, d_model, dropout_rate, max_len=5000, reverse=False): - super(PositionalEncoding, self).__init__() - self.d_model = d_model - self.reverse = reverse - self.xscale = math.sqrt(self.d_model) - self.dropout = torch.nn.Dropout(p=dropout_rate) - self.pe = None - self.extend_pe(torch.tensor(0.0).expand(1, max_len)) - - def extend_pe(self, x): - if self.pe is not None: - if self.pe.size(1) >= x.size(1): - if self.pe.dtype != x.dtype or self.pe.device != x.device: - self.pe = self.pe.to(dtype=x.dtype, device=x.device) - return - pe = torch.zeros(x.size(1), self.d_model) - if self.reverse: - position = torch.arange( - x.size(1) - 1, -1, -1.0, dtype=torch.float32 - ).unsqueeze(1) - else: - position = torch.arange(0, x.size(1), dtype=torch.float32).unsqueeze(1) - div_term = torch.exp( - torch.arange(0, self.d_model, 2, dtype=torch.float32) - * -(math.log(10000.0) / self.d_model) - ) - pe[:, 0::2] = torch.sin(position * div_term) - pe[:, 1::2] = torch.cos(position * div_term) - pe = pe.unsqueeze(0) - self.pe = pe.to(device=x.device, dtype=x.dtype) - - def forward(self, x: torch.Tensor): - self.extend_pe(x) - x = x * self.xscale + self.pe[:, : x.size(1)] - return self.dropout(x) - - -class EENDOLATransformerEncoder(nn.Module): - def __init__( - self, - idim: int, - n_layers: int, - n_units: int, - e_units: int = 2048, - h: int = 4, - dropout_rate: float = 0.1, - use_pos_emb: bool = False, - ): - super(EENDOLATransformerEncoder, self).__init__() - self.linear_in = nn.Linear(idim, n_units) - self.lnorm_in = nn.LayerNorm(n_units) - self.n_layers = n_layers - self.dropout = nn.Dropout(dropout_rate) - for i in range(n_layers): - setattr(self, "{}{:d}".format("lnorm1_", i), nn.LayerNorm(n_units)) - setattr( - self, - "{}{:d}".format("self_att_", i), - MultiHeadSelfAttention(n_units, h), - ) - setattr(self, "{}{:d}".format("lnorm2_", i), nn.LayerNorm(n_units)) - setattr( - self, - "{}{:d}".format("ff_", i), - PositionwiseFeedForward(n_units, e_units, dropout_rate), - ) - self.lnorm_out = nn.LayerNorm(n_units) - - def __call__(self, x, x_mask=None): - BT_size = x.shape[0] * x.shape[1] - e = self.linear_in(x.reshape(BT_size, -1)) - for i in range(self.n_layers): - e = getattr(self, "{}{:d}".format("lnorm1_", i))(e) - s = getattr(self, "{}{:d}".format("self_att_", i))(e, x.shape[0], x_mask) - e = e + self.dropout(s) - e = getattr(self, "{}{:d}".format("lnorm2_", i))(e) - s = getattr(self, "{}{:d}".format("ff_", i))(e) - e = e + self.dropout(s) - return self.lnorm_out(e) diff --git a/funasr_detach/models/eend/encoder_decoder_attractor.py b/funasr_detach/models/eend/encoder_decoder_attractor.py deleted file mode 100644 index d6935ebd3bada28425ea0bd1639bd09e5ab976c4..0000000000000000000000000000000000000000 --- a/funasr_detach/models/eend/encoder_decoder_attractor.py +++ /dev/null @@ -1,79 +0,0 @@ -import numpy as np -import torch -import torch.nn.functional as F -from torch import nn - - -class EncoderDecoderAttractor(nn.Module): - - def __init__(self, n_units, encoder_dropout=0.1, decoder_dropout=0.1): - super(EncoderDecoderAttractor, self).__init__() - self.enc0_dropout = nn.Dropout(encoder_dropout) - self.encoder = nn.LSTM( - n_units, n_units, 1, batch_first=True, dropout=encoder_dropout - ) - self.dec0_dropout = nn.Dropout(decoder_dropout) - self.decoder = nn.LSTM( - n_units, n_units, 1, batch_first=True, dropout=decoder_dropout - ) - self.counter = nn.Linear(n_units, 1) - self.n_units = n_units - - def forward_core(self, xs, zeros): - ilens = torch.from_numpy(np.array([x.shape[0] for x in xs])).to(torch.int64) - xs = [self.enc0_dropout(x) for x in xs] - xs = nn.utils.rnn.pad_sequence(xs, batch_first=True, padding_value=-1) - xs = nn.utils.rnn.pack_padded_sequence( - xs, ilens, batch_first=True, enforce_sorted=False - ) - _, (hx, cx) = self.encoder(xs) - zlens = torch.from_numpy(np.array([z.shape[0] for z in zeros])).to(torch.int64) - max_zlen = torch.max(zlens).to(torch.int).item() - zeros = [self.enc0_dropout(z) for z in zeros] - zeros = nn.utils.rnn.pad_sequence(zeros, batch_first=True, padding_value=-1) - zeros = nn.utils.rnn.pack_padded_sequence( - zeros, zlens, batch_first=True, enforce_sorted=False - ) - attractors, (_, _) = self.decoder(zeros, (hx, cx)) - attractors = nn.utils.rnn.pad_packed_sequence( - attractors, batch_first=True, padding_value=-1, total_length=max_zlen - )[0] - attractors = [ - att[: zlens[i].to(torch.int).item()] for i, att in enumerate(attractors) - ] - return attractors - - def forward(self, xs, n_speakers): - zeros = [ - torch.zeros(n_spk + 1, self.n_units).to(torch.float32).to(xs[0].device) - for n_spk in n_speakers - ] - attractors = self.forward_core(xs, zeros) - labels = torch.cat( - [ - torch.from_numpy(np.array([[1] * n_spk + [0]], np.float32)) - for n_spk in n_speakers - ], - dim=1, - ) - labels = labels.to(xs[0].device) - logit = torch.cat( - [ - self.counter(att).view(-1, n_spk + 1) - for att, n_spk in zip(attractors, n_speakers) - ], - dim=1, - ) - loss = F.binary_cross_entropy(torch.sigmoid(logit), labels) - - attractors = [att[slice(0, att.shape[0] - 1)] for att in attractors] - return loss, attractors - - def estimate(self, xs, max_n_speakers=15): - zeros = [ - torch.zeros(max_n_speakers, self.n_units).to(torch.float32).to(xs[0].device) - for _ in xs - ] - attractors = self.forward_core(xs, zeros) - probs = [torch.sigmoid(torch.flatten(self.counter(att))) for att in attractors] - return attractors, probs diff --git a/funasr_detach/models/eend/utils/__init__.py b/funasr_detach/models/eend/utils/__init__.py deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/funasr_detach/models/eend/utils/feature.py b/funasr_detach/models/eend/utils/feature.py deleted file mode 100644 index d3e6cc59e2cb4ce1f8e04127975baedb009284a4..0000000000000000000000000000000000000000 --- a/funasr_detach/models/eend/utils/feature.py +++ /dev/null @@ -1,275 +0,0 @@ -# Copyright 2019 Hitachi, Ltd. (author: Yusuke Fujita) -# Licensed under the MIT license. -# -# This module is for computing audio features - -import numpy as np -import librosa - - -def get_input_dim( - frame_size, - context_size, - transform_type, -): - if transform_type.startswith("logmel23"): - frame_size = 23 - elif transform_type.startswith("logmel"): - frame_size = 40 - else: - fft_size = 1 << (frame_size - 1).bit_length() - frame_size = int(fft_size / 2) + 1 - input_dim = (2 * context_size + 1) * frame_size - return input_dim - - -def transform(Y, transform_type=None, dtype=np.float32): - """Transform STFT feature - - Args: - Y: STFT - (n_frames, n_bins)-shaped np.complex array - transform_type: - None, "log" - dtype: output data type - np.float32 is expected - Returns: - Y (numpy.array): transformed feature - """ - Y = np.abs(Y) - if not transform_type: - pass - elif transform_type == "log": - Y = np.log(np.maximum(Y, 1e-10)) - elif transform_type == "logmel": - n_fft = 2 * (Y.shape[1] - 1) - sr = 16000 - n_mels = 40 - mel_basis = librosa.filters.mel(sr, n_fft, n_mels) - Y = np.dot(Y**2, mel_basis.T) - Y = np.log10(np.maximum(Y, 1e-10)) - elif transform_type == "logmel23": - n_fft = 2 * (Y.shape[1] - 1) - sr = 8000 - n_mels = 23 - mel_basis = librosa.filters.mel(sr, n_fft, n_mels) - Y = np.dot(Y**2, mel_basis.T) - Y = np.log10(np.maximum(Y, 1e-10)) - elif transform_type == "logmel23_mn": - n_fft = 2 * (Y.shape[1] - 1) - sr = 8000 - n_mels = 23 - mel_basis = librosa.filters.mel(sr, n_fft, n_mels) - Y = np.dot(Y**2, mel_basis.T) - Y = np.log10(np.maximum(Y, 1e-10)) - mean = np.mean(Y, axis=0) - Y = Y - mean - elif transform_type == "logmel23_swn": - n_fft = 2 * (Y.shape[1] - 1) - sr = 8000 - n_mels = 23 - mel_basis = librosa.filters.mel(sr, n_fft, n_mels) - Y = np.dot(Y**2, mel_basis.T) - Y = np.log10(np.maximum(Y, 1e-10)) - # b = np.ones(300)/300 - # mean = scipy.signal.convolve2d(Y, b[:, None], mode='same') - - # simple 2-means based threshoding for mean calculation - powers = np.sum(Y, axis=1) - th = (np.max(powers) + np.min(powers)) / 2.0 - for i in range(10): - th = (np.mean(powers[powers >= th]) + np.mean(powers[powers < th])) / 2 - mean = np.mean(Y[powers > th, :], axis=0) - Y = Y - mean - elif transform_type == "logmel23_mvn": - n_fft = 2 * (Y.shape[1] - 1) - sr = 8000 - n_mels = 23 - mel_basis = librosa.filters.mel(sr, n_fft, n_mels) - Y = np.dot(Y**2, mel_basis.T) - Y = np.log10(np.maximum(Y, 1e-10)) - mean = np.mean(Y, axis=0) - Y = Y - mean - std = np.maximum(np.std(Y, axis=0), 1e-10) - Y = Y / std - else: - raise ValueError("Unknown transform_type: %s" % transform_type) - return Y.astype(dtype) - - -def subsample(Y, T, subsampling=1): - """Frame subsampling""" - Y_ss = Y[::subsampling] - T_ss = T[::subsampling] - return Y_ss, T_ss - - -def splice(Y, context_size=0): - """Frame splicing - - Args: - Y: feature - (n_frames, n_featdim)-shaped numpy array - context_size: - number of frames concatenated on left-side - if context_size = 5, 11 frames are concatenated. - - Returns: - Y_spliced: spliced feature - (n_frames, n_featdim * (2 * context_size + 1))-shaped - """ - Y_pad = np.pad(Y, [(context_size, context_size), (0, 0)], "constant") - Y_spliced = np.lib.stride_tricks.as_strided( - np.ascontiguousarray(Y_pad), - (Y.shape[0], Y.shape[1] * (2 * context_size + 1)), - (Y.itemsize * Y.shape[1], Y.itemsize), - writeable=False, - ) - return Y_spliced - - -def stft(data, frame_size=1024, frame_shift=256): - """Compute STFT features - - Args: - data: audio signal - (n_samples,)-shaped np.float32 array - frame_size: number of samples in a frame (must be a power of two) - frame_shift: number of samples between frames - - Returns: - stft: STFT frames - (n_frames, n_bins)-shaped np.complex64 array - """ - # round up to nearest power of 2 - fft_size = 1 << (frame_size - 1).bit_length() - # HACK: The last frame is ommited - # as librosa.stft produces such an excessive frame - if len(data) % frame_shift == 0: - return librosa.stft( - data, n_fft=fft_size, win_length=frame_size, hop_length=frame_shift - ).T[:-1] - else: - return librosa.stft( - data, n_fft=fft_size, win_length=frame_size, hop_length=frame_shift - ).T - - -def _count_frames(data_len, size, shift): - # HACK: Assuming librosa.stft(..., center=True) - n_frames = 1 + int(data_len / shift) - if data_len % shift == 0: - n_frames = n_frames - 1 - return n_frames - - -def get_frame_labels( - kaldi_obj, rec, start=0, end=None, frame_size=1024, frame_shift=256, n_speakers=None -): - """Get frame-aligned labels of given recording - Args: - kaldi_obj (KaldiData) - rec (str): recording id - start (int): start frame index - end (int): end frame index - None means the last frame of recording - frame_size (int): number of frames in a frame - frame_shift (int): number of shift samples - n_speakers (int): number of speakers - if None, the value is given from data - Returns: - T: label - (n_frames, n_speakers)-shaped np.int32 array - """ - filtered_segments = kaldi_obj.segments[kaldi_obj.segments["rec"] == rec] - speakers = np.unique( - [kaldi_obj.utt2spk[seg["utt"]] for seg in filtered_segments] - ).tolist() - if n_speakers is None: - n_speakers = len(speakers) - es = end * frame_shift if end is not None else None - data, rate = kaldi_obj.load_wav(rec, start * frame_shift, es) - n_frames = _count_frames(len(data), frame_size, frame_shift) - T = np.zeros((n_frames, n_speakers), dtype=np.int32) - if end is None: - end = n_frames - - for seg in filtered_segments: - speaker_index = speakers.index(kaldi_obj.utt2spk[seg["utt"]]) - start_frame = np.rint(seg["st"] * rate / frame_shift).astype(int) - end_frame = np.rint(seg["et"] * rate / frame_shift).astype(int) - rel_start = rel_end = None - if start <= start_frame and start_frame < end: - rel_start = start_frame - start - if start < end_frame and end_frame <= end: - rel_end = end_frame - start - if rel_start is not None or rel_end is not None: - T[rel_start:rel_end, speaker_index] = 1 - return T - - -def get_labeledSTFT( - kaldi_obj, - rec, - start, - end, - frame_size, - frame_shift, - n_speakers=None, - use_speaker_id=False, -): - """Extracts STFT and corresponding labels - - Extracts STFT and corresponding diarization labels for - given recording id and start/end times - - Args: - kaldi_obj (KaldiData) - rec (str): recording id - start (int): start frame index - end (int): end frame index - frame_size (int): number of samples in a frame - frame_shift (int): number of shift samples - n_speakers (int): number of speakers - if None, the value is given from data - Returns: - Y: STFT - (n_frames, n_bins)-shaped np.complex64 array, - T: label - (n_frmaes, n_speakers)-shaped np.int32 array. - """ - data, rate = kaldi_obj.load_wav(rec, start * frame_shift, end * frame_shift) - Y = stft(data, frame_size, frame_shift) - filtered_segments = kaldi_obj.segments[rec] - # filtered_segments = kaldi_obj.segments[kaldi_obj.segments['rec'] == rec] - speakers = np.unique( - [kaldi_obj.utt2spk[seg["utt"]] for seg in filtered_segments] - ).tolist() - if n_speakers is None: - n_speakers = len(speakers) - T = np.zeros((Y.shape[0], n_speakers), dtype=np.int32) - - if use_speaker_id: - all_speakers = sorted(kaldi_obj.spk2utt.keys()) - S = np.zeros((Y.shape[0], len(all_speakers)), dtype=np.int32) - - for seg in filtered_segments: - speaker_index = speakers.index(kaldi_obj.utt2spk[seg["utt"]]) - if use_speaker_id: - all_speaker_index = all_speakers.index(kaldi_obj.utt2spk[seg["utt"]]) - start_frame = np.rint(seg["st"] * rate / frame_shift).astype(int) - end_frame = np.rint(seg["et"] * rate / frame_shift).astype(int) - rel_start = rel_end = None - if start <= start_frame and start_frame < end: - rel_start = start_frame - start - if start < end_frame and end_frame <= end: - rel_end = end_frame - start - if rel_start is not None or rel_end is not None: - T[rel_start:rel_end, speaker_index] = 1 - if use_speaker_id: - S[rel_start:rel_end, all_speaker_index] = 1 - - if use_speaker_id: - return Y, T, S - else: - return Y, T diff --git a/funasr_detach/models/eend/utils/kaldi_data.py b/funasr_detach/models/eend/utils/kaldi_data.py deleted file mode 100644 index 59e7a16f8b55d7c27ad7bd0075b6a66338d8bbb2..0000000000000000000000000000000000000000 --- a/funasr_detach/models/eend/utils/kaldi_data.py +++ /dev/null @@ -1,152 +0,0 @@ -# Copyright 2019 Hitachi, Ltd. (author: Yusuke Fujita) -# Licensed under the MIT license. -# -# This library provides utilities for kaldi-style data directory. - - -from __future__ import print_function -import os -import sys -import numpy as np -import subprocess -import librosa as sf -import io -from functools import lru_cache - - -def load_segments(segments_file): - """load segments file as array""" - if not os.path.exists(segments_file): - return None - return np.loadtxt( - segments_file, - dtype=[("utt", "object"), ("rec", "object"), ("st", "f"), ("et", "f")], - ndmin=1, - ) - - -def load_segments_hash(segments_file): - ret = {} - if not os.path.exists(segments_file): - return None - for line in open(segments_file): - utt, rec, st, et = line.strip().split() - ret[utt] = (rec, float(st), float(et)) - return ret - - -def load_segments_rechash(segments_file): - ret = {} - if not os.path.exists(segments_file): - return None - for line in open(segments_file): - utt, rec, st, et = line.strip().split() - if rec not in ret: - ret[rec] = [] - ret[rec].append({"utt": utt, "st": float(st), "et": float(et)}) - return ret - - -def load_wav_scp(wav_scp_file): - """return dictionary { rec: wav_rxfilename }""" - lines = [line.strip().split(None, 1) for line in open(wav_scp_file)] - return {x[0]: x[1] for x in lines} - - -@lru_cache(maxsize=1) -def load_wav(wav_rxfilename, start=0, end=None): - """This function reads audio file and return data in numpy.float32 array. - "lru_cache" holds recently loaded audio so that can be called - many times on the same audio file. - OPTIMIZE: controls lru_cache size for random access, - considering memory size - """ - if wav_rxfilename.endswith("|"): - # input piped command - p = subprocess.Popen(wav_rxfilename[:-1], shell=True, stdout=subprocess.PIPE) - data, samplerate = sf.load(io.BytesIO(p.stdout.read()), dtype="float32") - # cannot seek - data = data[start:end] - elif wav_rxfilename == "-": - # stdin - data, samplerate = sf.load(sys.stdin, dtype="float32") - # cannot seek - data = data[start:end] - else: - # normal wav file - data, samplerate = sf.load(wav_rxfilename, start=start, stop=end) - return data, samplerate - - -def load_utt2spk(utt2spk_file): - """returns dictionary { uttid: spkid }""" - lines = [line.strip().split(None, 1) for line in open(utt2spk_file)] - return {x[0]: x[1] for x in lines} - - -def load_spk2utt(spk2utt_file): - """returns dictionary { spkid: list of uttids }""" - if not os.path.exists(spk2utt_file): - return None - lines = [line.strip().split() for line in open(spk2utt_file)] - return {x[0]: x[1:] for x in lines} - - -def load_reco2dur(reco2dur_file): - """returns dictionary { recid: duration }""" - if not os.path.exists(reco2dur_file): - return None - lines = [line.strip().split(None, 1) for line in open(reco2dur_file)] - return {x[0]: float(x[1]) for x in lines} - - -def process_wav(wav_rxfilename, process): - """This function returns preprocessed wav_rxfilename - Args: - wav_rxfilename: input - process: command which can be connected via pipe, - use stdin and stdout - Returns: - wav_rxfilename: output piped command - """ - if wav_rxfilename.endswith("|"): - # input piped command - return wav_rxfilename + process + "|" - else: - # stdin "-" or normal file - return "cat {} | {} |".format(wav_rxfilename, process) - - -def extract_segments(wavs, segments=None): - """This function returns generator of segmented audio as - (utterance id, numpy.float32 array) - TODO?: sampling rate is not converted. - """ - if segments is not None: - # segments should be sorted by rec-id - for seg in segments: - wav = wavs[seg["rec"]] - data, samplerate = load_wav(wav) - st_sample = np.rint(seg["st"] * samplerate).astype(int) - et_sample = np.rint(seg["et"] * samplerate).astype(int) - yield seg["utt"], data[st_sample:et_sample] - else: - # segments file not found, - # wav.scp is used as segmented audio list - for rec in wavs: - data, samplerate = load_wav(wavs[rec]) - yield rec, data - - -class KaldiData: - def __init__(self, data_dir): - self.data_dir = data_dir - self.segments = load_segments_rechash(os.path.join(self.data_dir, "segments")) - self.utt2spk = load_utt2spk(os.path.join(self.data_dir, "utt2spk")) - self.wavs = load_wav_scp(os.path.join(self.data_dir, "wav.scp")) - self.reco2dur = load_reco2dur(os.path.join(self.data_dir, "reco2dur")) - self.spk2utt = load_spk2utt(os.path.join(self.data_dir, "spk2utt")) - - def load_wav(self, recid, start=0, end=None): - data, rate = load_wav(self.wavs[recid], start, end) - return data, rate diff --git a/funasr_detach/models/eend/utils/losses.py b/funasr_detach/models/eend/utils/losses.py deleted file mode 100644 index 957e8b291784edbc3b9e2a5076b8156206c0560d..0000000000000000000000000000000000000000 --- a/funasr_detach/models/eend/utils/losses.py +++ /dev/null @@ -1,53 +0,0 @@ -import numpy as np -import torch -import torch.nn.functional as F -from scipy.optimize import linear_sum_assignment - - -def standard_loss(ys, ts): - losses = [ - F.binary_cross_entropy(torch.sigmoid(y), t) * len(y) for y, t in zip(ys, ts) - ] - loss = torch.sum(torch.stack(losses)) - n_frames = ( - torch.from_numpy(np.array(np.sum([t.shape[0] for t in ts]))) - .to(torch.float32) - .to(ys[0].device) - ) - loss = loss / n_frames - return loss - - -def fast_batch_pit_n_speaker_loss(ys, ts): - with torch.no_grad(): - bs = len(ys) - indices = [] - for b in range(bs): - y = ys[b].transpose(0, 1) - t = ts[b].transpose(0, 1) - C, _ = t.shape - y = y[:, None, :].repeat(1, C, 1) - t = t[None, :, :].repeat(C, 1, 1) - bce_loss = F.binary_cross_entropy( - torch.sigmoid(y), t, reduction="none" - ).mean(-1) - C = bce_loss.cpu() - indices.append(linear_sum_assignment(C)) - labels_perm = [t[:, idx[1]] for t, idx in zip(ts, indices)] - - return labels_perm - - -def cal_power_loss(logits, power_ts): - losses = [ - F.cross_entropy(input=logit, target=power_t.to(torch.long)) * len(logit) - for logit, power_t in zip(logits, power_ts) - ] - loss = torch.sum(torch.stack(losses)) - n_frames = ( - torch.from_numpy(np.array(np.sum([power_t.shape[0] for power_t in power_ts]))) - .to(torch.float32) - .to(power_ts[0].device) - ) - loss = loss / n_frames - return loss diff --git a/funasr_detach/models/eend/utils/power.py b/funasr_detach/models/eend/utils/power.py deleted file mode 100644 index b0b3203bb90302d85904acc590ce3bae3330208b..0000000000000000000000000000000000000000 --- a/funasr_detach/models/eend/utils/power.py +++ /dev/null @@ -1,114 +0,0 @@ -import numpy as np -import torch -import torch.multiprocessing -import torch.nn.functional as F -from itertools import combinations -from itertools import permutations - - -def generate_mapping_dict(max_speaker_num=6, max_olp_speaker_num=3): - all_kinds = [] - all_kinds.append(0) - for i in range(max_olp_speaker_num): - selected_num = i + 1 - coms = np.array(list(combinations(np.arange(max_speaker_num), selected_num))) - for com in coms: - tmp = np.zeros(max_speaker_num) - tmp[com] = 1 - item = int(raw_dec_trans(tmp.reshape(1, -1), max_speaker_num)[0]) - all_kinds.append(item) - all_kinds_order = sorted(all_kinds) - - mapping_dict = {} - mapping_dict["dec2label"] = {} - mapping_dict["label2dec"] = {} - for i in range(len(all_kinds_order)): - dec = all_kinds_order[i] - mapping_dict["dec2label"][dec] = i - mapping_dict["label2dec"][i] = dec - oov_id = len(all_kinds_order) - mapping_dict["oov"] = oov_id - return mapping_dict - - -def raw_dec_trans(x, max_speaker_num): - num_list = [] - for i in range(max_speaker_num): - num_list.append(x[:, i]) - base = 1 - T = x.shape[0] - res = np.zeros((T)) - for num in num_list: - res += num * base - base = base * 2 - return res - - -def mapping_func(num, mapping_dict): - if num in mapping_dict["dec2label"].keys(): - label = mapping_dict["dec2label"][num] - else: - label = mapping_dict["oov"] - return label - - -def dec_trans(x, max_speaker_num, mapping_dict): - num_list = [] - for i in range(max_speaker_num): - num_list.append(x[:, i]) - base = 1 - T = x.shape[0] - res = np.zeros((T)) - for num in num_list: - res += num * base - base = base * 2 - res = np.array([mapping_func(i, mapping_dict) for i in res]) - return res - - -def create_powerlabel(label, mapping_dict, max_speaker_num=6, max_olp_speaker_num=3): - T, C = label.shape - padding_label = np.zeros((T, max_speaker_num)) - padding_label[:, :C] = label - out_label = dec_trans(padding_label, max_speaker_num, mapping_dict) - out_label = torch.from_numpy(out_label) - return out_label - - -def generate_perm_pse( - label, n_speaker, mapping_dict, max_speaker_num, max_olp_speaker_num=3 -): - perms = np.array(list(permutations(range(n_speaker)))).astype(np.float32) - perms = torch.from_numpy(perms).to(label.device).to(torch.int64) - perm_labels = [label[:, perm] for perm in perms] - perm_pse_labels = [ - create_powerlabel(perm_label.cpu().numpy(), mapping_dict, max_speaker_num).to( - perm_label.device, non_blocking=True - ) - for perm_label in perm_labels - ] - return perm_labels, perm_pse_labels - - -def generate_min_pse( - label, n_speaker, mapping_dict, max_speaker_num, pse_logit, max_olp_speaker_num=3 -): - perm_labels, perm_pse_labels = generate_perm_pse( - label, - n_speaker, - mapping_dict, - max_speaker_num, - max_olp_speaker_num=max_olp_speaker_num, - ) - losses = [ - F.cross_entropy(input=pse_logit, target=perm_pse_label.to(torch.long)) - * len(pse_logit) - for perm_pse_label in perm_pse_labels - ] - loss = torch.stack(losses) - min_index = torch.argmin(loss) - selected_perm_label, selected_pse_label = ( - perm_labels[min_index], - perm_pse_labels[min_index], - ) - return selected_perm_label, selected_pse_label diff --git a/funasr_detach/models/eend/utils/report.py b/funasr_detach/models/eend/utils/report.py deleted file mode 100644 index 23382072427580b7b1cf168653ae59bc7fd983a8..0000000000000000000000000000000000000000 --- a/funasr_detach/models/eend/utils/report.py +++ /dev/null @@ -1,201 +0,0 @@ -import copy -import numpy as np -import time -import torch -from funasr_detach.models.eend.utils.power import create_powerlabel -from itertools import combinations - -metrics = [ - ("diarization_error", "speaker_scored", "DER"), - ("speech_miss", "speech_scored", "SAD_MR"), - ("speech_falarm", "speech_scored", "SAD_FR"), - ("speaker_miss", "speaker_scored", "MI"), - ("speaker_falarm", "speaker_scored", "FA"), - ("speaker_error", "speaker_scored", "CF"), - ("correct", "frames", "accuracy"), -] - - -def recover_prediction(y, n_speaker): - if n_speaker <= 1: - return y - elif n_speaker == 2: - com_index = torch.from_numpy( - np.array(list(combinations(np.arange(n_speaker), 2))) - ).to(y.dtype) - num_coms = com_index.shape[0] - y_single = y[:, :-num_coms] - y_olp = y[:, -num_coms:] - olp_map_index = torch.where(y_olp > 0.5) - olp_map_index = torch.stack(olp_map_index, dim=1) - com_map_index = com_index[olp_map_index[:, -1]] - speaker_map_index = ( - torch.from_numpy(np.array(com_map_index)).view(-1).to(torch.int64) - ) - frame_map_index = ( - olp_map_index[:, 0][:, None].repeat([1, 2]).view(-1).to(torch.int64) - ) - y_single[frame_map_index] = 0 - y_single[frame_map_index, speaker_map_index] = 1 - return y_single - else: - olp2_com_index = torch.from_numpy( - np.array(list(combinations(np.arange(n_speaker), 2))) - ).to(y.dtype) - olp2_num_coms = olp2_com_index.shape[0] - olp3_com_index = torch.from_numpy( - np.array(list(combinations(np.arange(n_speaker), 3))) - ).to(y.dtype) - olp3_num_coms = olp3_com_index.shape[0] - y_single = y[:, :n_speaker] - y_olp2 = y[:, n_speaker : n_speaker + olp2_num_coms] - y_olp3 = y[:, -olp3_num_coms:] - - olp3_map_index = torch.where(y_olp3 > 0.5) - olp3_map_index = torch.stack(olp3_map_index, dim=1) - olp3_com_map_index = olp3_com_index[olp3_map_index[:, -1]] - olp3_speaker_map_index = ( - torch.from_numpy(np.array(olp3_com_map_index)).view(-1).to(torch.int64) - ) - olp3_frame_map_index = ( - olp3_map_index[:, 0][:, None].repeat([1, 3]).view(-1).to(torch.int64) - ) - y_single[olp3_frame_map_index] = 0 - y_single[olp3_frame_map_index, olp3_speaker_map_index] = 1 - y_olp2[olp3_frame_map_index] = 0 - - olp2_map_index = torch.where(y_olp2 > 0.5) - olp2_map_index = torch.stack(olp2_map_index, dim=1) - olp2_com_map_index = olp2_com_index[olp2_map_index[:, -1]] - olp2_speaker_map_index = ( - torch.from_numpy(np.array(olp2_com_map_index)).view(-1).to(torch.int64) - ) - olp2_frame_map_index = ( - olp2_map_index[:, 0][:, None].repeat([1, 2]).view(-1).to(torch.int64) - ) - y_single[olp2_frame_map_index] = 0 - y_single[olp2_frame_map_index, olp2_speaker_map_index] = 1 - return y_single - - -class PowerReporter: - def __init__(self, valid_data_loader, mapping_dict, max_n_speaker): - valid_data_loader_cp = copy.deepcopy(valid_data_loader) - self.valid_data_loader = valid_data_loader_cp - del valid_data_loader - self.mapping_dict = mapping_dict - self.max_n_speaker = max_n_speaker - - def report(self, model, eidx, device): - self.report_val(model, eidx, device) - - def report_val(self, model, eidx, device): - model.eval() - ud_valid_start = time.time() - valid_res, valid_loss, stats_keys, vad_valid_accuracy = self.report_core( - model, self.valid_data_loader, device - ) - - # Epoch Display - valid_der = valid_res["diarization_error"] / valid_res["speaker_scored"] - valid_accuracy = ( - valid_res["correct"].to(torch.float32) / valid_res["frames"] * 100 - ) - vad_valid_accuracy = vad_valid_accuracy * 100 - print( - "Epoch ", - eidx + 1, - "Valid Loss ", - valid_loss, - "Valid_DER %.5f" % valid_der, - "Valid_Accuracy %.5f%% " % valid_accuracy, - "VAD_Valid_Accuracy %.5f%% " % vad_valid_accuracy, - ) - ud_valid = (time.time() - ud_valid_start) / 60.0 - print("Valid cost time ... ", ud_valid) - - def inv_mapping_func(self, label, mapping_dict): - if not isinstance(label, int): - label = int(label) - if label in mapping_dict["label2dec"].keys(): - num = mapping_dict["label2dec"][label] - else: - num = -1 - return num - - def report_core(self, model, data_loader, device): - res = {} - for item in metrics: - res[item[0]] = 0.0 - res[item[1]] = 0.0 - with torch.no_grad(): - loss_s = 0.0 - uidx = 0 - for xs, ts, orders in data_loader: - xs = [x.to(device) for x in xs] - ts = [t.to(device) for t in ts] - orders = [o.to(device) for o in orders] - loss, pit_loss, mpit_loss, att_loss, ys, logits, labels, attractors = ( - model(xs, ts, orders) - ) - loss_s += loss.item() - uidx += 1 - - for logit, t, att in zip(logits, labels, attractors): - pred = torch.argmax(torch.softmax(logit, dim=-1), dim=-1) # (T, ) - oov_index = torch.where(pred == self.mapping_dict["oov"])[0] - for i in oov_index: - if i > 0: - pred[i] = pred[i - 1] - else: - pred[i] = 0 - pred = [self.inv_mapping_func(i, self.mapping_dict) for i in pred] - decisions = [ - bin(num)[2:].zfill(self.max_n_speaker)[::-1] for num in pred - ] - decisions = ( - torch.from_numpy( - np.stack( - [np.array([int(i) for i in dec]) for dec in decisions], - axis=0, - ) - ) - .to(att.device) - .to(torch.float32) - ) - decisions = decisions[:, : att.shape[0]] - - stats = self.calc_diarization_error(decisions, t) - res["speaker_scored"] += stats["speaker_scored"] - res["speech_scored"] += stats["speech_scored"] - res["frames"] += stats["frames"] - for item in metrics: - res[item[0]] += stats[item[0]] - loss_s /= uidx - vad_acc = 0 - - return res, loss_s, stats.keys(), vad_acc - - def calc_diarization_error(self, decisions, label, label_delay=0): - label = label[: len(label) - label_delay, ...] - n_ref = torch.sum(label, dim=-1) - n_sys = torch.sum(decisions, dim=-1) - res = {} - res["speech_scored"] = torch.sum(n_ref > 0) - res["speech_miss"] = torch.sum((n_ref > 0) & (n_sys == 0)) - res["speech_falarm"] = torch.sum((n_ref == 0) & (n_sys > 0)) - res["speaker_scored"] = torch.sum(n_ref) - res["speaker_miss"] = torch.sum( - torch.max(n_ref - n_sys, torch.zeros_like(n_ref)) - ) - res["speaker_falarm"] = torch.sum( - torch.max(n_sys - n_ref, torch.zeros_like(n_ref)) - ) - n_map = torch.sum(((label == 1) & (decisions == 1)), dim=-1).to(torch.float32) - res["speaker_error"] = torch.sum(torch.min(n_ref, n_sys) - n_map) - res["correct"] = torch.sum(label == decisions) / label.shape[1] - res["diarization_error"] = ( - res["speaker_miss"] + res["speaker_falarm"] + res["speaker_error"] - ) - res["frames"] = len(label) - return res diff --git a/funasr_detach/models/emotion2vec/__init__.py b/funasr_detach/models/emotion2vec/__init__.py deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/funasr_detach/models/emotion2vec/audio.py b/funasr_detach/models/emotion2vec/audio.py deleted file mode 100644 index 0f68ab497360713f4ff01cbca4b3cee4e231ac2d..0000000000000000000000000000000000000000 --- a/funasr_detach/models/emotion2vec/audio.py +++ /dev/null @@ -1,165 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import torch -import numpy as np -import torch.nn as nn -from functools import partial -import torch.nn.functional as F -from typing import Callable, Dict - -from funasr_detach.models.emotion2vec.fairseq_modules import ( - LayerNorm, - SamePad, - TransposeLast, - ConvFeatureExtractionModel, -) -from funasr_detach.models.emotion2vec.modules import Modality, BlockEncoder, Decoder1d -from funasr_detach.models.emotion2vec.base import ( - ModalitySpecificEncoder, - get_alibi_bias, -) - - -class AudioEncoder(ModalitySpecificEncoder): - - def __init__( - self, - modality_cfg, - embed_dim: int, - make_block: Callable[[float], nn.ModuleList], - norm_layer: Callable[[int], nn.LayerNorm], - layer_norm_first: bool, - alibi_biases: Dict, - ): - - self.feature_enc_layers = eval(modality_cfg.feature_encoder_spec) - feature_embed_dim = self.feature_enc_layers[-1][0] - - local_encoder = ConvFeatureExtractionModel( - conv_layers=self.feature_enc_layers, - dropout=0.0, - mode=modality_cfg.extractor_mode, - conv_bias=False, - ) - - project_features = nn.Sequential( - TransposeLast(), - nn.LayerNorm(feature_embed_dim), - nn.Linear(feature_embed_dim, embed_dim), - ) - - num_pos_layers = modality_cfg.conv_pos_depth - k = max(3, modality_cfg.conv_pos_width // num_pos_layers) - - positional_encoder = nn.Sequential( - TransposeLast(), - *[ - nn.Sequential( - nn.Conv1d( - embed_dim, - embed_dim, - kernel_size=k, - padding=k // 2, - groups=modality_cfg.conv_pos_groups, - ), - SamePad(k), - TransposeLast(), - LayerNorm(embed_dim, elementwise_affine=False), - TransposeLast(), - nn.GELU(), - ) - for _ in range(num_pos_layers) - ], - TransposeLast(), - ) - - if modality_cfg.conv_pos_pre_ln: - positional_encoder = nn.Sequential(LayerNorm(embed_dim), positional_encoder) - - dpr = np.linspace( - modality_cfg.start_drop_path_rate, - modality_cfg.end_drop_path_rate, - modality_cfg.prenet_depth, - ) - context_encoder = BlockEncoder( - nn.ModuleList(make_block(dpr[i]) for i in range(modality_cfg.prenet_depth)), - norm_layer(embed_dim) if not layer_norm_first else None, - layer_norm_first, - modality_cfg.prenet_layerdrop, - modality_cfg.prenet_dropout, - ) - - decoder = ( - Decoder1d(modality_cfg.decoder, embed_dim) - if modality_cfg.decoder is not None - else None - ) - - alibi_bias_fn = partial(get_alibi_bias, alibi_biases=alibi_biases) - - super().__init__( - modality_cfg=modality_cfg, - embed_dim=embed_dim, - local_encoder=local_encoder, - project_features=project_features, - fixed_positional_encoder=None, - relative_positional_encoder=positional_encoder, - context_encoder=context_encoder, - decoder=decoder, - get_alibi_bias=alibi_bias_fn, - ) - - def convert_padding_mask(self, x, padding_mask): - def get_feat_extract_output_lengths(input_lengths: torch.LongTensor): - """ - Computes the output length of the convolutional layers - """ - - def _conv_out_length(input_length, kernel_size, stride): - return torch.floor((input_length - kernel_size) / stride + 1) - - for i in range(len(self.feature_enc_layers)): - input_lengths = _conv_out_length( - input_lengths, - self.feature_enc_layers[i][1], - self.feature_enc_layers[i][2], - ) - - return input_lengths.to(torch.long) - - if padding_mask is not None: - input_lengths = (1 - padding_mask.long()).sum(-1) - # apply conv formula to get real output_lengths - output_lengths = get_feat_extract_output_lengths(input_lengths) - - if padding_mask.any(): - padding_mask = torch.zeros(x.shape[:2], dtype=x.dtype, device=x.device) - - # these two operations makes sure that all values - # before the output lengths indices are attended to - padding_mask[ - ( - torch.arange(padding_mask.shape[0], device=padding_mask.device), - output_lengths - 1, - ) - ] = 1 - padding_mask = ( - 1 - padding_mask.flip([-1]).cumsum(-1).flip([-1]) - ).bool() - else: - padding_mask = torch.zeros( - x.shape[:2], dtype=torch.bool, device=x.device - ) - - return padding_mask - - def reset_parameters(self): - super().reset_parameters() - for mod in self.project_features.children(): - if isinstance(mod, nn.Linear): - mod.reset_parameters() - if self.decoder is not None: - self.decoder.reset_parameters() diff --git a/funasr_detach/models/emotion2vec/base.py b/funasr_detach/models/emotion2vec/base.py deleted file mode 100644 index 0c006a1f73301ceec7c063d57b609c2e476197b2..0000000000000000000000000000000000000000 --- a/funasr_detach/models/emotion2vec/base.py +++ /dev/null @@ -1,646 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import logging -import math -import numpy as np -import torch -import torch.nn as nn -import torch.nn.functional as F -from collections import namedtuple -from dataclasses import dataclass -from functools import partial -from omegaconf import MISSING, II -from typing import Optional, Callable -from funasr_detach.models.emotion2vec.fairseq_modules import compute_mask_indices -from funasr_detach.models.emotion2vec.fairseq_modules import GradMultiply -from funasr_detach.models.emotion2vec.fairseq_modules import index_put - - -logger = logging.getLogger(__name__) - - -MaskSeed = namedtuple("MaskSeed", ["seed", "update", "ids"]) -MaskInfo = namedtuple("MaskInfo", ["x_unmasked", "mask", "ids_restore", "ids_keep"]) - - -class ModalitySpecificEncoder(nn.Module): - def __init__( - self, - modality_cfg, - embed_dim: int, - local_encoder: nn.Module, - project_features: nn.Module, - fixed_positional_encoder: Optional[nn.Module], - relative_positional_encoder: Optional[nn.Module], - context_encoder: nn.Module, - decoder: nn.Module, - get_alibi_bias: Optional[Callable[[int, int, str, str], torch.Tensor]], - ): - super().__init__() - - self.modality_cfg = modality_cfg - self.local_encoder = local_encoder - self.project_features = project_features - self.fixed_positional_encoder = fixed_positional_encoder - self.relative_positional_encoder = relative_positional_encoder - self.context_encoder = context_encoder - - self.decoder = decoder - self.get_alibi_bias = get_alibi_bias if modality_cfg.use_alibi_encoder else None - - self.local_grad_mult = self.modality_cfg.local_grad_mult - - self.extra_tokens = None - if modality_cfg.num_extra_tokens > 0: - self.extra_tokens = nn.Parameter( - torch.zeros(1, modality_cfg.num_extra_tokens, embed_dim) - ) - if not modality_cfg.init_extra_token_zero: - nn.init.normal_(self.extra_tokens) - elif self.extra_tokens.size(1) > 1: - nn.init.normal_(self.extra_tokens[:, 1:]) - - self.alibi_scale = None - if self.get_alibi_bias is not None: - self.alibi_scale = nn.Parameter( - torch.full( - ( - ( - (modality_cfg.prenet_depth + modality_cfg.model_depth) - if modality_cfg.learned_alibi_scale_per_layer - else 1 - ), - 1, - ( - self.modality_cfg.num_alibi_heads - if modality_cfg.learned_alibi_scale_per_head - else 1 - ), - 1, - 1, - ), - modality_cfg.alibi_scale, - dtype=torch.float, - ), - requires_grad=modality_cfg.learned_alibi_scale, - ) - - if modality_cfg.learned_alibi and self.get_alibi_bias is not None: - assert modality_cfg.alibi_max_pos is not None - alibi_bias = self.get_alibi_bias( - batch_size=1, - time_steps=modality_cfg.alibi_max_pos, - heads=modality_cfg.num_alibi_heads, - scale=1.0, - dtype=torch.float, - device="cpu", - ) - self.alibi_bias = nn.Parameter(alibi_bias) - self.get_alibi_bias = partial( - _learned_alibi_bias, alibi_bias=self.alibi_bias - ) - - def upgrade_state_dict_named(self, state_dict, name): - k = f"{name}.alibi_scale" - if k in state_dict and state_dict[k].dim() == 4: - state_dict[k] = state_dict[k].unsqueeze(0) - - return state_dict - - def convert_padding_mask(self, x, padding_mask): - return padding_mask - - def decoder_input(self, x, mask_info: MaskInfo): - inp_drop = self.modality_cfg.decoder.input_dropout - if inp_drop > 0: - x = F.dropout(x, inp_drop, training=self.training, inplace=True) - - num_extra = self.modality_cfg.num_extra_tokens - - if mask_info is not None: - num_masked = mask_info.ids_restore.shape[1] - x.shape[1] + num_extra - - mask_tokens = x.new_empty( - x.size(0), - num_masked, - x.size(-1), - ).normal_(0, self.modality_cfg.mask_noise_std) - - x_ = torch.cat([x[:, num_extra:], mask_tokens], dim=1) - x = torch.gather(x_, dim=1, index=mask_info.ids_restore) - - if self.modality_cfg.decoder.add_positions_masked: - assert self.fixed_positional_encoder is not None - pos = self.fixed_positional_encoder(x, None) - x = x + (pos * mask_info.mask.unsqueeze(-1)) - else: - x = x[:, num_extra:] - - if self.modality_cfg.decoder.add_positions_all: - assert self.fixed_positional_encoder is not None - x = x + self.fixed_positional_encoder(x, None) - - return x, mask_info - - def local_features(self, features): - if self.local_grad_mult > 0: - if self.local_grad_mult == 1.0: - x = self.local_encoder(features) - else: - x = GradMultiply.apply( - self.local_encoder(features), self.local_grad_mult - ) - else: - with torch.no_grad(): - x = self.local_encoder(features) - - x = self.project_features(x) - return x - - def contextualized_features( - self, - x, - padding_mask, - mask, - remove_masked, - clone_batch: int = 1, - mask_seeds: Optional[torch.Tensor] = None, - precomputed_mask=None, - ): - - if padding_mask is not None: - padding_mask = self.convert_padding_mask(x, padding_mask) - - local_features = x - if mask and clone_batch == 1: - local_features = local_features.clone() - - orig_B, orig_T, _ = x.shape - pre_mask_B = orig_B - mask_info = None - - x_pos = None - if self.fixed_positional_encoder is not None: - x = x + self.fixed_positional_encoder(x, padding_mask) - - if mask: - if clone_batch > 1: - x = x.repeat_interleave(clone_batch, 0) - if mask_seeds is not None: - clone_hash = [ - int(hash((mask_seeds.seed, ind)) % 1e10) - for ind in range(clone_batch - 1) - ] - clone_hash = torch.tensor([0] + clone_hash).long().view(1, -1) - - id = mask_seeds.ids - id = id.repeat_interleave(clone_batch, 0) - id = id.view(-1, clone_batch) + clone_hash.to(id) - id = id.view(-1) - mask_seeds = MaskSeed( - seed=mask_seeds.seed, update=mask_seeds.update, ids=id - ) - if padding_mask is not None: - padding_mask = padding_mask.repeat_interleave(clone_batch, 0) - - x, mask_info = self.compute_mask( - x, - padding_mask, - mask_seed=mask_seeds, - apply=self.relative_positional_encoder is not None or not remove_masked, - precomputed_mask=precomputed_mask, - ) - - if self.relative_positional_encoder is not None: - x_pos = self.relative_positional_encoder(x) - - masked_padding_mask = padding_mask - if mask and remove_masked: - x = mask_info.x_unmasked - if x_pos is not None: - x = x + gather_unmasked(x_pos, mask_info) - - if padding_mask is not None and padding_mask.any(): - masked_padding_mask = gather_unmasked_mask(padding_mask, mask_info) - if not masked_padding_mask.any(): - masked_padding_mask = None - else: - masked_padding_mask = None - - elif x_pos is not None: - x = x + x_pos - - alibi_bias = None - alibi_scale = self.alibi_scale - - if self.get_alibi_bias is not None: - alibi_bias = self.get_alibi_bias( - batch_size=pre_mask_B, - time_steps=orig_T, - heads=self.modality_cfg.num_alibi_heads, - dtype=torch.float32, - device=x.device, - ) - - if alibi_scale is not None: - alibi_scale = alibi_scale.clamp_min(0) - if alibi_scale.size(0) == 1: - alibi_bias = alibi_bias * alibi_scale.squeeze(0).type_as(alibi_bias) - alibi_scale = None - - if clone_batch > 1: - alibi_bias = alibi_bias.repeat_interleave(clone_batch, 0) - - if mask_info is not None and remove_masked: - alibi_bias = masked_alibi(alibi_bias, mask_info) - - if self.extra_tokens is not None: - num = self.extra_tokens.size(1) - x = torch.cat([self.extra_tokens.expand(x.size(0), -1, -1), x], dim=1) - if masked_padding_mask is not None: - # B x T - masked_padding_mask = F.pad(masked_padding_mask, (num, 0)) - if alibi_bias is not None: - # B x H x T x T - alibi_bias = F.pad(alibi_bias, (num, 0, num, 0)) - - x = self.context_encoder( - x, - masked_padding_mask, - alibi_bias, - ( - alibi_scale[: self.modality_cfg.prenet_depth] - if alibi_scale is not None - else None - ), - ) - - return { - "x": x, - "local_features": local_features, - "padding_mask": masked_padding_mask, - "alibi_bias": alibi_bias, - "alibi_scale": ( - alibi_scale[self.modality_cfg.prenet_depth :] - if alibi_scale is not None and alibi_scale.size(0) > 1 - else alibi_scale - ), - "encoder_mask": mask_info, - } - - def forward( - self, - features, - padding_mask, - mask: bool, - remove_masked: bool, - clone_batch: int = 1, - mask_seeds: Optional[torch.Tensor] = None, - precomputed_mask=None, - ): - x = self.local_features(features) - return self.contextualized_features( - x, - padding_mask, - mask, - remove_masked, - clone_batch, - mask_seeds, - precomputed_mask, - ) - - def reset_parameters(self): - pass - - def compute_mask( - self, - x, - padding_mask, - mask_seed: Optional[MaskSeed], - apply, - precomputed_mask, - ): - if precomputed_mask is not None: - mask = precomputed_mask - mask_info = self.make_maskinfo(x, mask) - else: - B, T, C = x.shape - cfg = self.modality_cfg - - mask_prob = cfg.mask_prob - - if ( - cfg.mask_prob_min is not None - and cfg.mask_prob_min >= 0 - and cfg.mask_prob_min < mask_prob - ): - mask_prob = np.random.uniform(cfg.mask_prob_min, mask_prob) - - if mask_prob > 0: - if cfg.mask_length == 1: - mask_info = random_masking(x, mask_prob, mask_seed) - else: - if self.modality_cfg.inverse_mask: - mask_prob = 1 - mask_prob - - mask = compute_mask_indices( - (B, T), - padding_mask, - mask_prob, - cfg.mask_length, - min_masks=1, - require_same_masks=True, - mask_dropout=cfg.mask_dropout, - add_masks=cfg.add_masks, - seed=mask_seed.seed if mask_seed is not None else None, - epoch=mask_seed.update if mask_seed is not None else None, - indices=mask_seed.ids if mask_seed is not None else None, - ) - - mask = torch.from_numpy(mask).to(device=x.device) - if self.modality_cfg.inverse_mask: - mask = 1 - mask - mask_info = self.make_maskinfo(x, mask) - else: - mask_info = None - - if apply: - x = self.apply_mask(x, mask_info) - - return x, mask_info - - def make_maskinfo(self, x, mask, shape=None): - if shape is None: - B, T, D = x.shape - else: - B, T, D = shape - - mask = mask.to(torch.uint8) - ids_shuffle = mask.argsort(dim=1) - ids_restore = ids_shuffle.argsort(dim=1).unsqueeze(-1).expand(-1, -1, D) - - len_keep = T - mask[0].sum() - if self.modality_cfg.keep_masked_pct > 0: - len_keep += round((T - int(len_keep)) * self.modality_cfg.keep_masked_pct) - - ids_keep = ids_shuffle[:, :len_keep] - - if shape is not None: - x_unmasked = None - else: - ids_keep = ids_keep.unsqueeze(-1).expand(-1, -1, D) - x_unmasked = torch.gather(x, dim=1, index=ids_keep) - - mask_info = MaskInfo( - x_unmasked=x_unmasked, - mask=mask, - ids_restore=ids_restore, - ids_keep=ids_keep, - ) - return mask_info - - def apply_mask(self, x, mask_info): - cfg = self.modality_cfg - B, T, C = x.shape - - if mask_info is not None: - mask = mask_info.mask - if cfg.encoder_zero_mask: - x = x * (1 - mask.type_as(x).unsqueeze(-1)) - else: - num_masks = mask.sum().item() - masks = x.new_empty(num_masks, x.size(-1)).normal_( - 0, cfg.mask_noise_std - ) - x = index_put(x, mask, masks) - if cfg.mask_channel_prob > 0: - mask_channel = compute_mask_indices( - (B, C), - None, - cfg.mask_channel_prob, - cfg.mask_channel_length, - ) - mask_channel = ( - torch.from_numpy(mask_channel) - .to(x.device) - .unsqueeze(1) - .expand(-1, T, -1) - ) - x = index_put(x, mask_channel, 0) - return x - - def remove_pretraining_modules(self, keep_decoder=False): - if not keep_decoder: - self.decoder = None - - -def get_annealed_rate(start, end, curr_step, total_steps): - if curr_step >= total_steps: - return end - r = end - start - pct_remaining = 1 - curr_step / total_steps - return end - r * pct_remaining - - -# adapted from MAE -def random_masking(x, mask_ratio, mask_seed: Optional[MaskSeed]): - N, L, D = x.shape # batch, length, dim - len_keep = int(L * (1 - mask_ratio)) - - generator = None - if mask_seed is not None: - seed = int( - hash((mask_seed.seed, mask_seed.update, mask_seed.ids.sum().item())) % 1e6 - ) - generator = torch.Generator(device=x.device) - generator.manual_seed(seed) - - noise = torch.rand(N, L, generator=generator, device=x.device) # noise in [0, 1] - - # sort noise for each sample - ids_shuffle = noise.argsort(dim=1) # ascend: small is keep, large is remove - ids_restore = ids_shuffle.argsort(dim=1) - - # keep the first subset - ids_keep = ids_shuffle[:, :len_keep] - ids_keep = ids_keep.unsqueeze(-1).expand(-1, -1, D) - x_unmasked = torch.gather(x, dim=1, index=ids_keep) - - # generate the binary mask: 0 is keep, 1 is remove - mask = torch.ones([N, L], dtype=x.dtype, device=x.device) - mask[:, :len_keep] = 0 - # unshuffle to get the binary mask - mask = torch.gather(mask, dim=1, index=ids_restore) - - ids_restore = ids_restore.unsqueeze(-1).expand(-1, -1, D) - - return MaskInfo( - x_unmasked=x_unmasked, mask=mask, ids_restore=ids_restore, ids_keep=ids_keep - ) - - -def gather_unmasked(x: torch.Tensor, mask_info: MaskInfo) -> torch.Tensor: - return torch.gather( - x, - dim=1, - index=mask_info.ids_keep, - ) - - -def gather_unmasked_mask(x: torch.Tensor, mask_info: MaskInfo) -> torch.Tensor: - return torch.gather( - x, - dim=1, - index=mask_info.ids_keep[..., 0], # ignore the feature dimension - ) - - -def get_alibi( - max_positions: int, - attention_heads: int, - dims: int = 1, - distance: str = "manhattan", -): - def get_slopes(n): - def get_slopes_power_of_2(n): - start = 2 ** (-(2 ** -(math.log2(n) - 3))) - ratio = start - return [start * ratio**i for i in range(n)] - - # In the paper, we only train models that have 2^a heads for some - # a. This function has some good properties that only occur when - # the input is a power of 2. To maintain that even when the number - # of heads is not a power of 2, we use this workaround. - if math.log2(n).is_integer(): - return get_slopes_power_of_2(n) - else: - closest_power_of_2 = 2 ** math.floor(math.log2(n)) - return ( - get_slopes_power_of_2(closest_power_of_2) - + get_slopes(2 * closest_power_of_2)[0::2][: n - closest_power_of_2] - ) - - maxpos = max_positions - attn_heads = attention_heads - slopes = torch.Tensor(get_slopes(attn_heads)) - - if dims == 1: - # prepare alibi position linear bias. Note that wav2vec2 is non - # autoregressive model so we want a symmetric mask with 0 on the - # diagonal and other wise linear decreasing valuees - pos_bias = ( - torch.abs( - torch.arange(maxpos).unsqueeze(0) - torch.arange(maxpos).unsqueeze(1) - ) - * -1 - ) - elif dims == 2: - if distance == "manhattan": - df = lambda x1, y1, x2, y2: abs(x1 - x2) + abs(y1 - y2) - elif distance == "euclidean": - df = lambda x1, y1, x2, y2: math.sqrt((x1 - x2) ** 2 + (y1 - y2) ** 2) - - n = math.sqrt(max_positions) - assert n.is_integer(), n - n = int(n) - - pos_bias = torch.zeros((max_positions, max_positions)) - - for i in range(n): - for j in range(n): - for k in range(n): - for l in range(n): - new_x = i * n + j - new_y = k * n + l - pos_bias[new_x, new_y] = -df(i, j, k, l) - - else: - raise Exception(f"unsupported number of alibi dims: {dims}") - - alibi_bias = slopes.unsqueeze(1).unsqueeze(1) * pos_bias.unsqueeze(0).expand( - attn_heads, -1, -1 - ) - - return alibi_bias - - -def get_alibi_bias( - alibi_biases, - batch_size, - time_steps, - heads, - dtype, - device, - dims=1, - distance="manhattan", -): - cache_key = f"{dims}_{heads}_{distance}" - - buffered = alibi_biases.get(cache_key, None) - - target_size = heads * batch_size - if ( - buffered is None - or buffered.size(0) < target_size - or buffered.size(1) < time_steps - or buffered.dtype != dtype - or buffered.device != device - ): - bt = max(time_steps, buffered.size(1) if buffered is not None else 0) - bn = max(target_size, buffered.size(0) if buffered is not None else 0) // heads - - buffered = ( - get_alibi(bt, heads, dims=dims, distance=distance) - .to(dtype=dtype, device=device) - .repeat(bn, 1, 1) - ) - - alibi_biases[cache_key] = buffered - - b = buffered[:target_size, :time_steps, :time_steps] - b = b.view(batch_size, heads, time_steps, time_steps) - return b - - -def _learned_alibi_bias( - alibi_bias, - batch_size, - time_steps, - heads, - scale, - dtype, - device, -): - assert alibi_bias.size(1) == heads, alibi_bias.shape - assert alibi_bias.dtype == dtype, alibi_bias.dtype - assert alibi_bias.device == device, alibi_bias.device - - if alibi_bias.size(-1) < time_steps: - psz = math.ceil((time_steps - alibi_bias.size(-1)) / 2) - alibi_bias = F.pad(alibi_bias, (psz, psz, psz, psz), mode="replicate") - - alibi_bias = alibi_bias.expand(batch_size, -1, -1, -1) * scale - return alibi_bias[..., :time_steps, :time_steps] - - -def masked_alibi(alibi_bias, mask_info): - H = alibi_bias.size(1) - - orig_bias = alibi_bias - - index = mask_info.ids_keep.unsqueeze(1)[..., 0].unsqueeze(-1) - alibi_bias = torch.gather( - orig_bias, - dim=-2, - index=index.expand(-1, H, -1, mask_info.ids_restore.size(1)), - ) - alibi_bias = torch.gather( - alibi_bias, - dim=-1, - index=index.transpose(-1, -2).expand(-1, H, alibi_bias.size(-2), -1), - ) - - return alibi_bias diff --git a/funasr_detach/models/emotion2vec/fairseq_modules.py b/funasr_detach/models/emotion2vec/fairseq_modules.py deleted file mode 100644 index 46dd225dc82898ee163f5f50228978591970b670..0000000000000000000000000000000000000000 --- a/funasr_detach/models/emotion2vec/fairseq_modules.py +++ /dev/null @@ -1,310 +0,0 @@ -import torch -import torch.nn as nn -import torch.nn.functional as F -from typing import Optional, Tuple, List -import numpy as np - - -def LayerNorm(normalized_shape, eps=1e-5, elementwise_affine=True, export=False): - return torch.nn.LayerNorm(normalized_shape, eps, elementwise_affine) - - -class SamePad(nn.Module): - def __init__(self, kernel_size, causal=False): - super().__init__() - if causal: - self.remove = kernel_size - 1 - else: - self.remove = 1 if kernel_size % 2 == 0 else 0 - - def forward(self, x): - if self.remove > 0: - x = x[:, :, : -self.remove] - return x - - -class TransposeLast(nn.Module): - def __init__(self, deconstruct_idx=None): - super().__init__() - self.deconstruct_idx = deconstruct_idx - - def forward(self, x): - if self.deconstruct_idx is not None: - x = x[self.deconstruct_idx] - return x.transpose(-2, -1) - - -class Fp32LayerNorm(nn.LayerNorm): - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - - def forward(self, input): - output = F.layer_norm( - input.float(), - self.normalized_shape, - self.weight.float() if self.weight is not None else None, - self.bias.float() if self.bias is not None else None, - self.eps, - ) - return output.type_as(input) - - -class Fp32GroupNorm(nn.GroupNorm): - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - - def forward(self, input): - output = F.group_norm( - input.float(), - self.num_groups, - self.weight.float() if self.weight is not None else None, - self.bias.float() if self.bias is not None else None, - self.eps, - ) - return output.type_as(input) - - -class ConvFeatureExtractionModel(nn.Module): - def __init__( - self, - conv_layers: List[Tuple[int, int, int]], - dropout: float = 0.0, - mode: str = "default", - conv_bias: bool = False, - ): - super().__init__() - - assert mode in {"default", "layer_norm"} - - def block( - n_in, - n_out, - k, - stride, - is_layer_norm=False, - is_group_norm=False, - conv_bias=False, - ): - def make_conv(): - conv = nn.Conv1d(n_in, n_out, k, stride=stride, bias=conv_bias) - nn.init.kaiming_normal_(conv.weight) - return conv - - assert ( - is_layer_norm and is_group_norm - ) == False, "layer norm and group norm are exclusive" - - if is_layer_norm: - return nn.Sequential( - make_conv(), - nn.Dropout(p=dropout), - nn.Sequential( - TransposeLast(), - Fp32LayerNorm(dim, elementwise_affine=True), - TransposeLast(), - ), - nn.GELU(), - ) - elif is_group_norm: - return nn.Sequential( - make_conv(), - nn.Dropout(p=dropout), - Fp32GroupNorm(dim, dim, affine=True), - nn.GELU(), - ) - else: - return nn.Sequential(make_conv(), nn.Dropout(p=dropout), nn.GELU()) - - in_d = 1 - self.conv_layers = nn.ModuleList() - for i, cl in enumerate(conv_layers): - assert len(cl) == 3, "invalid conv definition: " + str(cl) - (dim, k, stride) = cl - - self.conv_layers.append( - block( - in_d, - dim, - k, - stride, - is_layer_norm=mode == "layer_norm", - is_group_norm=mode == "default" and i == 0, - conv_bias=conv_bias, - ) - ) - in_d = dim - - def forward(self, x): - - # BxT -> BxCxT - x = x.unsqueeze(1) - - for conv in self.conv_layers: - x = conv(x) - - return x - - -def compute_mask_indices( - shape: Tuple[int, int], - padding_mask: Optional[torch.Tensor], - mask_prob: float, - mask_length: int, - mask_type: str = "static", - mask_other: float = 0.0, - min_masks: int = 0, - no_overlap: bool = False, - min_space: int = 0, - require_same_masks: bool = True, - mask_dropout: float = 0.0, -) -> np.ndarray: - """ - Computes random mask spans for a given shape - - Args: - shape: the the shape for which to compute masks. - should be of size 2 where first element is batch size and 2nd is timesteps - padding_mask: optional padding mask of the same size as shape, which will prevent masking padded elements - mask_prob: probability for each token to be chosen as start of the span to be masked. this will be multiplied by - number of timesteps divided by length of mask span to mask approximately this percentage of all elements. - however due to overlaps, the actual number will be smaller (unless no_overlap is True) - mask_type: how to compute mask lengths - static = fixed size - uniform = sample from uniform distribution [mask_other, mask_length*2] - normal = sample from normal distribution with mean mask_length and stdev mask_other. mask is min 1 element - poisson = sample from possion distribution with lambda = mask length - min_masks: minimum number of masked spans - no_overlap: if false, will switch to an alternative recursive algorithm that prevents spans from overlapping - min_space: only used if no_overlap is True, this is how many elements to keep unmasked between spans - require_same_masks: if true, will randomly drop out masks until same amount of masks remains in each sample - mask_dropout: randomly dropout this percentage of masks in each example - """ - - bsz, all_sz = shape - mask = np.full((bsz, all_sz), False) - - all_num_mask = int( - # add a random number for probabilistic rounding - mask_prob * all_sz / float(mask_length) - + np.random.rand() - ) - - all_num_mask = max(min_masks, all_num_mask) - - mask_idcs = [] - for i in range(bsz): - if padding_mask is not None: - sz = all_sz - padding_mask[i].long().sum().item() - num_mask = int( - # add a random number for probabilistic rounding - mask_prob * sz / float(mask_length) - + np.random.rand() - ) - num_mask = max(min_masks, num_mask) - else: - sz = all_sz - num_mask = all_num_mask - - if mask_type == "static": - lengths = np.full(num_mask, mask_length) - elif mask_type == "uniform": - lengths = np.random.randint(mask_other, mask_length * 2 + 1, size=num_mask) - elif mask_type == "normal": - lengths = np.random.normal(mask_length, mask_other, size=num_mask) - lengths = [max(1, int(round(x))) for x in lengths] - elif mask_type == "poisson": - lengths = np.random.poisson(mask_length, size=num_mask) - lengths = [int(round(x)) for x in lengths] - else: - raise Exception("unknown mask selection " + mask_type) - - if sum(lengths) == 0: - lengths[0] = min(mask_length, sz - 1) - - if no_overlap: - mask_idc = [] - - def arrange(s, e, length, keep_length): - span_start = np.random.randint(s, e - length) - mask_idc.extend(span_start + i for i in range(length)) - - new_parts = [] - if span_start - s - min_space >= keep_length: - new_parts.append((s, span_start - min_space + 1)) - if e - span_start - length - min_space > keep_length: - new_parts.append((span_start + length + min_space, e)) - return new_parts - - parts = [(0, sz)] - min_length = min(lengths) - for length in sorted(lengths, reverse=True): - lens = np.fromiter( - (e - s if e - s >= length + min_space else 0 for s, e in parts), - np.int, - ) - l_sum = np.sum(lens) - if l_sum == 0: - break - probs = lens / np.sum(lens) - c = np.random.choice(len(parts), p=probs) - s, e = parts.pop(c) - parts.extend(arrange(s, e, length, min_length)) - mask_idc = np.asarray(mask_idc) - else: - min_len = min(lengths) - if sz - min_len <= num_mask: - min_len = sz - num_mask - 1 - - mask_idc = np.random.choice(sz - min_len, num_mask, replace=False) - - mask_idc = np.asarray( - [ - mask_idc[j] + offset - for j in range(len(mask_idc)) - for offset in range(lengths[j]) - ] - ) - - mask_idcs.append(np.unique(mask_idc[mask_idc < sz])) - - min_len = min([len(m) for m in mask_idcs]) - for i, mask_idc in enumerate(mask_idcs): - if len(mask_idc) > min_len and require_same_masks: - mask_idc = np.random.choice(mask_idc, min_len, replace=False) - if mask_dropout > 0: - num_holes = np.rint(len(mask_idc) * mask_dropout).astype(int) - mask_idc = np.random.choice( - mask_idc, len(mask_idc) - num_holes, replace=False - ) - - mask[i, mask_idc] = True - - return mask - - -class GradMultiply(torch.autograd.Function): - @staticmethod - def forward(ctx, x, scale): - ctx.scale = scale - res = x.new(x) - return res - - @staticmethod - def backward(ctx, grad): - return grad * ctx.scale, None - - -def is_xla_tensor(tensor): - return torch.is_tensor(tensor) and tensor.device.type == "xla" - - -def index_put(tensor, indices, value): - if is_xla_tensor(tensor): - for _ in range(indices.dim(), tensor.dim()): - indices = indices.unsqueeze(-1) - if indices.size(-1) < tensor.size(-1): - indices = indices.expand_as(tensor) - tensor = torch.mul(tensor, ~indices) + torch.mul(value, indices) - else: - tensor[indices] = value - return tensor diff --git a/funasr_detach/models/emotion2vec/model.py b/funasr_detach/models/emotion2vec/model.py deleted file mode 100644 index f0d971bcd01a0663375c472258762882f14ff4d8..0000000000000000000000000000000000000000 --- a/funasr_detach/models/emotion2vec/model.py +++ /dev/null @@ -1,272 +0,0 @@ -#!/usr/bin/env python3 -# -*- encoding: utf-8 -*- -# Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved. -# MIT License (https://opensource.org/licenses/MIT) -# Modified from https://github.com/ddlBoJack/emotion2vec/tree/main - -import os -import time -import torch -import logging -import numpy as np -from functools import partial -from omegaconf import OmegaConf -import torch.nn.functional as F -from contextlib import contextmanager -from distutils.version import LooseVersion - -from funasr_detach.register import tables -from funasr_detach.models.emotion2vec.modules import AltBlock -from funasr_detach.models.emotion2vec.audio import AudioEncoder -from funasr_detach.utils.load_utils import load_audio_text_image_video - - -logger = logging.getLogger(__name__) -if LooseVersion(torch.__version__) >= LooseVersion("1.6.0"): - from torch.cuda.amp import autocast -else: - # Nothing to do if torch<1.6.0 - @contextmanager - def autocast(enabled=True): - yield - - -@tables.register("model_classes", "Emotion2vec") -class Emotion2vec(torch.nn.Module): - """ - Author: Ziyang Ma, Zhisheng Zheng, Jiaxin Ye, Jinchao Li, Zhifu Gao, Shiliang Zhang, Xie Chen - emotion2vec: Self-Supervised Pre-Training for Speech Emotion Representation - https://arxiv.org/abs/2312.15185 - """ - - def __init__(self, **kwargs): - super().__init__() - # import pdb; pdb.set_trace() - cfg = OmegaConf.create(kwargs["model_conf"]) - self.cfg = cfg - - make_layer_norm = partial( - torch.nn.LayerNorm, - eps=cfg.get("norm_eps"), - elementwise_affine=cfg.get("norm_affine"), - ) - - def make_block(drop_path, dim=None, heads=None): - return AltBlock( - cfg.get("embed_dim") if dim is None else dim, - cfg.get("num_heads") if heads is None else heads, - cfg.get("mlp_ratio"), - qkv_bias=True, - drop=cfg.get("encoder_dropout"), - attn_drop=cfg.get("attention_dropout"), - mlp_drop=cfg.get("activation_dropout"), - post_mlp_drop=cfg.get("post_mlp_drop"), - drop_path=drop_path, - norm_layer=make_layer_norm, - layer_norm_first=cfg.get("layer_norm_first"), - ffn_targets=not cfg.get("end_of_block_targets"), - ) - - self.alibi_biases = {} - self.modality_encoders = torch.nn.ModuleDict() - - enc = AudioEncoder( - cfg.modalities.audio, - cfg.get("embed_dim"), - make_block, - make_layer_norm, - cfg.get("layer_norm_first"), - self.alibi_biases, - ) - self.modality_encoders["AUDIO"] = enc - - self.ema = None - - self.average_top_k_layers = cfg.get("average_top_k_layers") - self.loss_beta = cfg.get("loss_beta") - self.loss_scale = cfg.get("loss_scale") - - self.dropout_input = torch.nn.Dropout(cfg.get("dropout_input")) - - dpr = np.linspace( - cfg.get("start_drop_path_rate"), - cfg.get("end_drop_path_rate"), - cfg.get("depth"), - ) - - self.blocks = torch.nn.ModuleList( - [make_block(dpr[i]) for i in range(cfg.get("depth"))] - ) - - self.norm = None - if cfg.get("layer_norm_first"): - self.norm = make_layer_norm(cfg.get("embed_dim")) - - vocab_size = kwargs.get("vocab_size", -1) - self.proj = None - if vocab_size > 0: - self.proj = torch.nn.Linear(cfg.get("embed_dim"), vocab_size) - - def forward( - self, - source, - target=None, - id=None, - mode=None, - padding_mask=None, - mask=True, - features_only=False, - force_remove_masked=False, - remove_extra_tokens=True, - precomputed_mask=None, - **kwargs, - ): - - feature_extractor = self.modality_encoders["AUDIO"] - - mask_seeds = None - - extractor_out = feature_extractor( - source, - padding_mask, - mask, - remove_masked=not features_only or force_remove_masked, - clone_batch=self.cfg.get("clone_batch") if not features_only else 1, - mask_seeds=mask_seeds, - precomputed_mask=precomputed_mask, - ) - - x = extractor_out["x"] - encoder_mask = extractor_out["encoder_mask"] - masked_padding_mask = extractor_out["padding_mask"] - masked_alibi_bias = extractor_out.get("alibi_bias", None) - alibi_scale = extractor_out.get("alibi_scale", None) - - if self.dropout_input is not None: - x = self.dropout_input(x) - - layer_results = [] - for i, blk in enumerate(self.blocks): - if ( - not self.training - or self.cfg.get("layerdrop", 0) == 0 - or (np.random.random() > self.cfg.get("layerdrop", 0)) - ): - ab = masked_alibi_bias - if ab is not None and alibi_scale is not None: - scale = ( - alibi_scale[i] - if alibi_scale.size(0) > 1 - else alibi_scale.squeeze(0) - ) - ab = ab * scale.type_as(ab) - - x, lr = blk( - x, - padding_mask=masked_padding_mask, - alibi_bias=ab, - ) - if features_only: - layer_results.append(lr) - - if self.norm is not None: - x = self.norm(x) - - if features_only: - if remove_extra_tokens: - x = x[:, feature_extractor.modality_cfg.num_extra_tokens :] - if masked_padding_mask is not None: - masked_padding_mask = masked_padding_mask[ - :, feature_extractor.modality_cfg.num_extra_tokens : - ] - - return { - "x": x, - "padding_mask": masked_padding_mask, - "layer_results": layer_results, - "mask": encoder_mask, - } - - def extract_features( - self, source, mode=None, padding_mask=None, mask=False, remove_extra_tokens=True - ): - res = self.forward( - source, - mode=mode, - padding_mask=padding_mask, - mask=mask, - features_only=True, - remove_extra_tokens=remove_extra_tokens, - ) - return res - - def inference( - self, - data_in, - data_lengths=None, - key: list = None, - tokenizer=None, - frontend=None, - **kwargs, - ): - - # if source_file.endswith('.wav'): - # wav, sr = sf.read(source_file) - # channel = sf.info(source_file).channels - # assert sr == 16e3, "Sample rate should be 16kHz, but got {}in file {}".format(sr, source_file) - # assert channel == 1, "Channel should be 1, but got {} in file {}".format(channel, source_file) - granularity = kwargs.get("granularity", "utterance") - extract_embedding = kwargs.get("extract_embedding", True) - if self.proj is None: - extract_embedding = True - meta_data = {} - # extract fbank feats - time1 = time.perf_counter() - audio_sample_list = load_audio_text_image_video( - data_in, - fs=16000, - audio_fs=kwargs.get("fs", 16000), - data_type=kwargs.get("data_type", "sound"), - tokenizer=tokenizer, - ) - time2 = time.perf_counter() - meta_data["load_data"] = f"{time2 - time1:0.3f}" - meta_data["batch_data_time"] = len(audio_sample_list[0]) / kwargs.get( - "fs", 16000 - ) - - results = [] - output_dir = kwargs.get("output_dir") - if output_dir: - os.makedirs(output_dir, exist_ok=True) - for i, wav in enumerate(audio_sample_list): - source = wav.to(device=kwargs["device"]) - if self.cfg.normalize: - source = F.layer_norm(source, source.shape) - source = source.view(1, -1) - - feats = self.extract_features(source, padding_mask=None) - x = feats["x"] - feats = feats["x"].squeeze(0).cpu().numpy() - if granularity == "frame": - feats = feats - elif granularity == "utterance": - feats = np.mean(feats, axis=0) - - if output_dir and extract_embedding: - np.save(os.path.join(output_dir, "{}.npy".format(key[i])), feats) - - labels = tokenizer.token_list if tokenizer is not None else [] - scores = [] - if self.proj: - x = x.mean(dim=1) - x = self.proj(x) - x = torch.softmax(x, dim=-1) - scores = x[0].tolist() - - result_i = {"key": key[i], "labels": labels, "scores": scores} - if extract_embedding: - result_i["feats"] = feats - results.append(result_i) - - return results, meta_data diff --git a/funasr_detach/models/emotion2vec/modules.py b/funasr_detach/models/emotion2vec/modules.py deleted file mode 100644 index 9d2bb6454e5bd1f2e3178a3d66c18a2eac23e592..0000000000000000000000000000000000000000 --- a/funasr_detach/models/emotion2vec/modules.py +++ /dev/null @@ -1,323 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import torch -import numpy as np -import torch.nn as nn -from enum import Enum, auto -import torch.nn.functional as F -from dataclasses import dataclass -from funasr_detach.models.emotion2vec.fairseq_modules import ( - LayerNorm, - SamePad, - TransposeLast, -) - - -class Modality(Enum): - AUDIO = auto() - - -@dataclass -class D2vDecoderConfig: - decoder_dim: int = 384 - decoder_groups: int = 16 - decoder_kernel: int = 5 - decoder_layers: int = 5 - input_dropout: float = 0.1 - - add_positions_masked: bool = False - add_positions_all: bool = False - - decoder_residual: bool = True - projection_layers: int = 1 - projection_ratio: float = 2.0 - - -class FixedPositionalEncoder(nn.Module): - def __init__(self, pos_embed): - super().__init__() - self.positions = pos_embed - - def forward(self, x, padding_mask): - return self.positions - - -class TextFeatPositionalEncoder(nn.Module): - """ - Original encoder expects (B, T) long input. This module wraps it to take - local_encoder output which are (B, T, D) float tensors - """ - - def __init__(self, pos_encoder): - super().__init__() - self.pos_encoder = pos_encoder - - def forward(self, x, padding_mask): - # assume padded token embeddings are 0s - # TODO: consider using padding_mask as input - return self.pos_encoder(x[..., 0]) - - -class BlockEncoder(nn.Module): - def __init__(self, blocks, norm_layer, layer_norm_first, layerdrop, dropout): - super().__init__() - self.blocks = blocks - self.norm = norm_layer - self.layer_norm_first = layer_norm_first - self.layerdrop = layerdrop - self.dropout = nn.Dropout(dropout, inplace=True) - - def forward(self, x, padding_mask, alibi_bias, alibi_scale): - if self.norm is not None and not self.layer_norm_first: - x = self.norm(x) - - x = self.dropout(x) - - for i, blk in enumerate(self.blocks): - if ( - not self.training - or self.layerdrop == 0 - or (np.random.random() > self.layerdrop) - ): - ab = alibi_bias - if ab is not None and alibi_scale is not None: - scale = ( - alibi_scale[i] - if alibi_scale.size(0) > 1 - else alibi_scale.squeeze(0) - ) - ab = ab * scale.type_as(ab) - x, _ = blk(x, padding_mask, ab) - - if self.norm is not None and self.layer_norm_first: - x = self.norm(x) - - return x - - -class DecoderBase(nn.Module): - decoder_cfg: D2vDecoderConfig - - def __init__(self, cfg: D2vDecoderConfig): - super().__init__() - - self.decoder_cfg = cfg - - def reset_parameters(self): - for mod in self.proj.modules(): - if isinstance(mod, nn.Linear): - mod.reset_parameters() - - def add_residual(self, x, residual, i, mask_info): - if ( - residual is None - or not self.decoder_cfg.decoder_residual - or residual.size(1) != x.size(1) - ): - return x - - ret = x + residual - - return ret - - -class Decoder1d(DecoderBase): - def __init__(self, cfg: D2vDecoderConfig, input_dim): - super().__init__(cfg) - - def make_block(in_dim): - block = [ - nn.Conv1d( - in_dim, - cfg.decoder_dim, - kernel_size=cfg.decoder_kernel, - padding=cfg.decoder_kernel // 2, - groups=cfg.decoder_groups, - ), - SamePad(cfg.decoder_kernel), - TransposeLast(), - LayerNorm(cfg.decoder_dim, elementwise_affine=False), - TransposeLast(), - nn.GELU(), - ] - - return nn.Sequential(*block) - - self.blocks = nn.Sequential( - *[ - make_block(input_dim if i == 0 else cfg.decoder_dim) - for i in range(cfg.decoder_layers) - ] - ) - - projs = [] - curr_dim = cfg.decoder_dim - for i in range(cfg.projection_layers - 1): - next_dim = int(curr_dim * cfg.projection_ratio) if i == 0 else curr_dim - projs.append(nn.Linear(curr_dim, next_dim)) - projs.append(nn.GELU()) - curr_dim = next_dim - projs.append(nn.Linear(curr_dim, input_dim)) - if len(projs) == 1: - self.proj = projs[0] - else: - self.proj = nn.Sequential(*projs) - - def forward(self, x, mask_info): - - x = x.transpose(1, 2) - - residual = x - - for i, layer in enumerate(self.blocks): - x = layer(x) - x = self.add_residual(x, residual, i, mask_info) - residual = x - - x = x.transpose(1, 2) - x = self.proj(x) - return x - - -class AltBlock(nn.Module): - def __init__( - self, - dim, - num_heads, - mlp_ratio=4.0, - qkv_bias=False, - qk_scale=None, - drop=0.0, - attn_drop=0.0, - mlp_drop=0.0, - post_mlp_drop=0.0, - drop_path=0.0, - act_layer=nn.GELU, - norm_layer=nn.LayerNorm, - layer_norm_first=True, - ffn_targets=False, - cosine_attention=False, - ): - super().__init__() - - self.layer_norm_first = layer_norm_first - self.ffn_targets = ffn_targets - - from funasr_detach.models.emotion2vec.timm_modules import DropPath, Mlp - - self.norm1 = norm_layer(dim) - self.attn = AltAttention( - dim, - num_heads=num_heads, - qkv_bias=qkv_bias, - qk_scale=qk_scale, - attn_drop=attn_drop, - proj_drop=drop, - cosine_attention=cosine_attention, - ) - - self.drop_path = DropPath(drop_path) if drop_path > 0.0 else nn.Identity() - self.norm2 = norm_layer(dim) - mlp_hidden_dim = int(dim * mlp_ratio) - self.mlp = Mlp( - in_features=dim, - hidden_features=mlp_hidden_dim, - act_layer=act_layer, - drop=mlp_drop, - ) - self.post_mlp_dropout = nn.Dropout(post_mlp_drop, inplace=False) - - def forward(self, x, padding_mask=None, alibi_bias=None): - if self.layer_norm_first: - x = x + self.drop_path(self.attn(self.norm1(x), padding_mask, alibi_bias)) - r = x = self.mlp(self.norm2(x)) - t = x - x = r + self.drop_path(self.post_mlp_dropout(x)) - if not self.ffn_targets: - t = x - else: - x = x + self.drop_path(self.attn(x, padding_mask, alibi_bias)) - r = x = self.norm1(x) - x = self.mlp(x) - t = x - x = self.norm2(r + self.drop_path(self.post_mlp_dropout(x))) - if not self.ffn_targets: - t = x - - return x, t - - -class AltAttention(nn.Module): - def __init__( - self, - dim, - num_heads=8, - qkv_bias=False, - qk_scale=None, - attn_drop=0.0, - proj_drop=0.0, - cosine_attention=False, - ): - super().__init__() - self.num_heads = num_heads - head_dim = dim // num_heads - self.scale = qk_scale or head_dim**-0.5 - - self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias) - self.attn_drop = nn.Dropout(attn_drop) - self.proj = nn.Linear(dim, dim) - self.proj_drop = nn.Dropout(proj_drop) - - self.cosine_attention = cosine_attention - - if cosine_attention: - self.logit_scale = nn.Parameter( - torch.log(10 * torch.ones((num_heads, 1, 1))), requires_grad=True - ) - - def forward(self, x, padding_mask=None, alibi_bias=None): - B, N, C = x.shape - qkv = ( - self.qkv(x) - .reshape(B, N, 3, self.num_heads, C // self.num_heads) - .permute(2, 0, 3, 1, 4) # qkv x B x H x L x D - ) - q, k, v = ( - qkv[0], - qkv[1], - qkv[2], - ) # make torchscript happy (cannot use tensor as tuple) - - dtype = q.dtype - - if self.cosine_attention: - # cosine attention - attn = F.normalize(q, dim=-1) @ F.normalize(k, dim=-1).transpose(-2, -1) - logit_scale = torch.clamp( - self.logit_scale, max=torch.log(torch.tensor(1.0 / 0.01)) - ).exp() - attn = attn * logit_scale - else: - q = q * self.scale - attn = q @ k.transpose(-2, -1) - - if alibi_bias is not None: - attn = attn.type_as(alibi_bias) - attn[:, : alibi_bias.size(1)] += alibi_bias - - if padding_mask is not None and padding_mask.any(): - attn = attn.masked_fill( - padding_mask.unsqueeze(1).unsqueeze(2).to(torch.bool), - float("-inf"), - ) - - attn = attn.softmax(dim=-1, dtype=torch.float32).to(dtype=dtype) - attn = self.attn_drop(attn) - x = (attn @ v).transpose(1, 2) # - x = x.reshape(B, N, C) - x = self.proj(x) - x = self.proj_drop(x) - return x diff --git a/funasr_detach/models/emotion2vec/template.yaml b/funasr_detach/models/emotion2vec/template.yaml deleted file mode 100644 index 53bca63742d2954229afd58ff406e5d3d7ae97a6..0000000000000000000000000000000000000000 --- a/funasr_detach/models/emotion2vec/template.yaml +++ /dev/null @@ -1,113 +0,0 @@ -# This is an example that demonstrates how to configure a model file. -# You can modify the configuration according to your own requirements. - -# to print the register_table: -# from funasr.register import tables -# tables.print() - -# network architecture -model: Emotion2vec -model_conf: - loss_beta: 0.0 - loss_scale: null - depth: 8 - start_drop_path_rate: 0.0 - end_drop_path_rate: 0.0 - num_heads: 12 - norm_eps: 1e-05 - norm_affine: true - encoder_dropout: 0.1 - post_mlp_drop: 0.1 - attention_dropout: 0.1 - activation_dropout: 0.0 - dropout_input: 0.0 - layerdrop: 0.05 - embed_dim: 768 - mlp_ratio: 4.0 - layer_norm_first: false - average_top_k_layers: 8 - end_of_block_targets: false - clone_batch: 8 - layer_norm_target_layer: false - batch_norm_target_layer: false - instance_norm_target_layer: true - instance_norm_targets: false - layer_norm_targets: false - ema_decay: 0.999 - ema_same_dtype: true - log_norms: true - ema_end_decay: 0.99999 - ema_anneal_end_step: 20000 - ema_encoder_only: false - max_update: 100000 - extractor_mode: layer_norm - shared_decoder: null - min_target_var: 0.1 - min_pred_var: 0.01 - supported_modality: AUDIO - mae_init: false - seed: 1 - skip_ema: false - cls_loss: 1.0 - recon_loss: 0.0 - d2v_loss: 1.0 - decoder_group: false - adversarial_training: false - adversarial_hidden_dim: 128 - adversarial_weight: 0.1 - cls_type: chunk - normalize: true - - modalities: - audio: - type: AUDIO - prenet_depth: 4 - prenet_layerdrop: 0.05 - prenet_dropout: 0.1 - start_drop_path_rate: 0.0 - end_drop_path_rate: 0.0 - num_extra_tokens: 10 - init_extra_token_zero: true - mask_noise_std: 0.01 - mask_prob_min: null - mask_prob: 0.5 - inverse_mask: false - mask_prob_adjust: 0.05 - keep_masked_pct: 0.0 - mask_length: 5 - add_masks: false - remove_masks: false - mask_dropout: 0.0 - encoder_zero_mask: true - mask_channel_prob: 0.0 - mask_channel_length: 64 - ema_local_encoder: false - local_grad_mult: 1.0 - use_alibi_encoder: true - alibi_scale: 1.0 - learned_alibi: false - alibi_max_pos: null - learned_alibi_scale: true - learned_alibi_scale_per_head: true - learned_alibi_scale_per_layer: false - num_alibi_heads: 12 - model_depth: 8 - decoder: - decoder_dim: 384 - decoder_groups: 16 - decoder_kernel: 7 - decoder_layers: 4 - input_dropout: 0.1 - add_positions_masked: false - add_positions_all: false - decoder_residual: true - projection_layers: 1 - projection_ratio: 2.0 - extractor_mode: layer_norm - feature_encoder_spec: '[(512, 10, 5)] + [(512, 3, 2)] * 4 + [(512,2,2)] + [(512,2,2)]' - conv_pos_width: 95 - conv_pos_groups: 16 - conv_pos_depth: 5 - conv_pos_pre_ln: false - - diff --git a/funasr_detach/models/emotion2vec/timm_modules.py b/funasr_detach/models/emotion2vec/timm_modules.py deleted file mode 100644 index 60d7076a7a5beeab86f8f0f98e7d92d5eea4225d..0000000000000000000000000000000000000000 --- a/funasr_detach/models/emotion2vec/timm_modules.py +++ /dev/null @@ -1,100 +0,0 @@ -import torch.nn as nn -import collections.abc -from itertools import repeat -from functools import partial - - -def drop_path( - x, drop_prob: float = 0.0, training: bool = False, scale_by_keep: bool = True -): - """Drop paths (Stochastic Depth) per sample (when applied in main path of residual blocks). - - This is the same as the DropConnect impl I created for EfficientNet, etc networks, however, - the original name is misleading as 'Drop Connect' is a different form of dropout in a separate paper... - See discussion: https://github.com/tensorflow/tpu/issues/494#issuecomment-532968956 ... I've opted for - changing the layer and argument names to 'drop path' rather than mix DropConnect as a layer name and use - 'survival rate' as the argument. - - """ - if drop_prob == 0.0 or not training: - return x - keep_prob = 1 - drop_prob - shape = (x.shape[0],) + (1,) * ( - x.ndim - 1 - ) # work with diff dim tensors, not just 2D ConvNets - random_tensor = x.new_empty(shape).bernoulli_(keep_prob) - if keep_prob > 0.0 and scale_by_keep: - random_tensor.div_(keep_prob) - return x * random_tensor - - -class DropPath(nn.Module): - """Drop paths (Stochastic Depth) per sample (when applied in main path of residual blocks).""" - - def __init__(self, drop_prob: float = 0.0, scale_by_keep: bool = True): - super(DropPath, self).__init__() - self.drop_prob = drop_prob - self.scale_by_keep = scale_by_keep - - def forward(self, x): - return drop_path(x, self.drop_prob, self.training, self.scale_by_keep) - - def extra_repr(self): - return f"drop_prob={round(self.drop_prob,3):0.3f}" - - -# From PyTorch internals -def _ntuple(n): - def parse(x): - if isinstance(x, collections.abc.Iterable) and not isinstance(x, str): - return tuple(x) - return tuple(repeat(x, n)) - - return parse - - -to_1tuple = _ntuple(1) -to_2tuple = _ntuple(2) -to_3tuple = _ntuple(3) -to_4tuple = _ntuple(4) -to_ntuple = _ntuple - - -class Mlp(nn.Module): - """MLP as used in Vision Transformer, MLP-Mixer and related networks""" - - def __init__( - self, - in_features, - hidden_features=None, - out_features=None, - act_layer=nn.GELU, - norm_layer=None, - bias=True, - drop=0.0, - use_conv=False, - ): - super().__init__() - out_features = out_features or in_features - hidden_features = hidden_features or in_features - bias = to_2tuple(bias) - drop_probs = to_2tuple(drop) - linear_layer = partial(nn.Conv2d, kernel_size=1) if use_conv else nn.Linear - - self.fc1 = linear_layer(in_features, hidden_features, bias=bias[0]) - self.act = act_layer() - self.drop1 = nn.Dropout(drop_probs[0]) - self.norm = ( - norm_layer(hidden_features) if norm_layer is not None else nn.Identity() - ) - self.fc2 = linear_layer(hidden_features, out_features, bias=bias[1]) - self.drop2 = nn.Dropout(drop_probs[1]) - - def forward(self, x): - x = self.fc1(x) - x = self.act(x) - x = self.drop1(x) - x = self.norm(x) - x = self.fc2(x) - x = self.drop2(x) - return x diff --git a/funasr_detach/models/eres2net/__init__.py b/funasr_detach/models/eres2net/__init__.py deleted file mode 100644 index 570bbb1f155ab288ceec70c73a377d2e46e31fac..0000000000000000000000000000000000000000 --- a/funasr_detach/models/eres2net/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -from .eres2net import ERes2Net -from .eres2net_aug import ERes2NetAug diff --git a/funasr_detach/models/eres2net/eres2net.py b/funasr_detach/models/eres2net/eres2net.py deleted file mode 100644 index afe981510c3e644caf5cf7e0dded941c1ec679fd..0000000000000000000000000000000000000000 --- a/funasr_detach/models/eres2net/eres2net.py +++ /dev/null @@ -1,431 +0,0 @@ -# Copyright 3D-Speaker (https://github.com/alibaba-damo-academy/3D-Speaker). All Rights Reserved. -# Licensed under the Apache License, Version 2.0 (http://www.apache.org/licenses/LICENSE-2.0) - -"""Res2Net implementation is adapted from https://github.com/wenet-e2e/wespeaker. -ERes2Net incorporates both local and global feature fusion techniques to improve the performance. -The local feature fusion (LFF) fuses the features within one single residual block to extract the local signal. -The global feature fusion (GFF) takes acoustic features of different scales as input to aggregate global signal. -ERes2Net-Large is an upgraded version of ERes2Net that uses a larger number of parameters to achieve better -recognition performance. Parameters expansion, baseWidth, and scale can be modified to obtain optimal performance. -""" - -import math - -import torch -import torch.nn as nn -import torch.nn.functional as F - -import funasr_detach.models.sond.pooling.pooling_layers as pooling_layers - -from funasr_detach.models.eres2net.fusion import AFF - - -class ReLU(nn.Hardtanh): - - def __init__(self, inplace=False): - super(ReLU, self).__init__(0, 20, inplace) - - def __repr__(self): - inplace_str = "inplace" if self.inplace else "" - return self.__class__.__name__ + " (" + inplace_str + ")" - - -def conv1x1(in_planes, out_planes, stride=1): - "1x1 convolution without padding" - return nn.Conv2d( - in_planes, out_planes, kernel_size=1, stride=stride, padding=0, bias=False - ) - - -def conv3x3(in_planes, out_planes, stride=1): - "3x3 convolution with padding" - return nn.Conv2d( - in_planes, out_planes, kernel_size=3, stride=stride, padding=1, bias=False - ) - - -class BasicBlockERes2Net(nn.Module): - expansion = 2 - - def __init__(self, in_planes, planes, stride=1, baseWidth=32, scale=2): - super(BasicBlockERes2Net, self).__init__() - width = int(math.floor(planes * (baseWidth / 64.0))) - self.conv1 = conv1x1(in_planes, width * scale, stride) - self.bn1 = nn.BatchNorm2d(width * scale) - self.nums = scale - - convs = [] - bns = [] - for i in range(self.nums): - convs.append(conv3x3(width, width)) - bns.append(nn.BatchNorm2d(width)) - self.convs = nn.ModuleList(convs) - self.bns = nn.ModuleList(bns) - self.relu = ReLU(inplace=True) - - self.conv3 = conv1x1(width * scale, planes * self.expansion) - self.bn3 = nn.BatchNorm2d(planes * self.expansion) - self.shortcut = nn.Sequential() - if stride != 1 or in_planes != self.expansion * planes: - self.shortcut = nn.Sequential( - nn.Conv2d( - in_planes, - self.expansion * planes, - kernel_size=1, - stride=stride, - bias=False, - ), - nn.BatchNorm2d(self.expansion * planes), - ) - self.stride = stride - self.width = width - self.scale = scale - - def forward(self, x): - residual = x - - out = self.conv1(x) - out = self.bn1(out) - out = self.relu(out) - spx = torch.split(out, self.width, 1) - for i in range(self.nums): - if i == 0: - sp = spx[i] - else: - sp = sp + spx[i] - sp = self.convs[i](sp) - sp = self.relu(self.bns[i](sp)) - if i == 0: - out = sp - else: - out = torch.cat((out, sp), 1) - - out = self.conv3(out) - out = self.bn3(out) - - residual = self.shortcut(x) - out += residual - out = self.relu(out) - - return out - - -class BasicBlockERes2Net_diff_AFF(nn.Module): - expansion = 2 - - def __init__(self, in_planes, planes, stride=1, baseWidth=32, scale=2): - super(BasicBlockERes2Net_diff_AFF, self).__init__() - width = int(math.floor(planes * (baseWidth / 64.0))) - self.conv1 = conv1x1(in_planes, width * scale, stride) - self.bn1 = nn.BatchNorm2d(width * scale) - self.nums = scale - - convs = [] - fuse_models = [] - bns = [] - for i in range(self.nums): - convs.append(conv3x3(width, width)) - bns.append(nn.BatchNorm2d(width)) - for j in range(self.nums - 1): - fuse_models.append(AFF(channels=width)) - - self.convs = nn.ModuleList(convs) - self.bns = nn.ModuleList(bns) - self.fuse_models = nn.ModuleList(fuse_models) - self.relu = ReLU(inplace=True) - - self.conv3 = conv1x1(width * scale, planes * self.expansion) - self.bn3 = nn.BatchNorm2d(planes * self.expansion) - self.shortcut = nn.Sequential() - if stride != 1 or in_planes != self.expansion * planes: - self.shortcut = nn.Sequential( - nn.Conv2d( - in_planes, - self.expansion * planes, - kernel_size=1, - stride=stride, - bias=False, - ), - nn.BatchNorm2d(self.expansion * planes), - ) - self.stride = stride - self.width = width - self.scale = scale - - def forward(self, x): - residual = x - - out = self.conv1(x) - out = self.bn1(out) - out = self.relu(out) - spx = torch.split(out, self.width, 1) - for i in range(self.nums): - if i == 0: - sp = spx[i] - else: - sp = self.fuse_models[i - 1](sp, spx[i]) - - sp = self.convs[i](sp) - sp = self.relu(self.bns[i](sp)) - if i == 0: - out = sp - else: - out = torch.cat((out, sp), 1) - - out = self.conv3(out) - out = self.bn3(out) - - residual = self.shortcut(x) - out += residual - out = self.relu(out) - - return out - - -class ERes2Net(nn.Module): - def __init__( - self, - block=BasicBlockERes2Net, - block_fuse=BasicBlockERes2Net_diff_AFF, - num_blocks=[3, 4, 6, 3], - m_channels=32, - feat_dim=80, - embedding_size=192, - pooling_func="TSTP", - two_emb_layer=False, - ): - super(ERes2Net, self).__init__() - self.in_planes = m_channels - self.feat_dim = feat_dim - self.embedding_size = embedding_size - self.stats_dim = int(feat_dim / 8) * m_channels * 8 - self.two_emb_layer = two_emb_layer - - self.conv1 = nn.Conv2d( - 1, m_channels, kernel_size=3, stride=1, padding=1, bias=False - ) - self.bn1 = nn.BatchNorm2d(m_channels) - self.layer1 = self._make_layer(block, m_channels, num_blocks[0], stride=1) - self.layer2 = self._make_layer(block, m_channels * 2, num_blocks[1], stride=2) - self.layer3 = self._make_layer( - block_fuse, m_channels * 4, num_blocks[2], stride=2 - ) - self.layer4 = self._make_layer( - block_fuse, m_channels * 8, num_blocks[3], stride=2 - ) - - # Downsampling module for each layer - self.layer1_downsample = nn.Conv2d( - m_channels * 2, - m_channels * 4, - kernel_size=3, - stride=2, - padding=1, - bias=False, - ) - self.layer2_downsample = nn.Conv2d( - m_channels * 4, - m_channels * 8, - kernel_size=3, - padding=1, - stride=2, - bias=False, - ) - self.layer3_downsample = nn.Conv2d( - m_channels * 8, - m_channels * 16, - kernel_size=3, - padding=1, - stride=2, - bias=False, - ) - - # Bottom-up fusion module - self.fuse_mode12 = AFF(channels=m_channels * 4) - self.fuse_mode123 = AFF(channels=m_channels * 8) - self.fuse_mode1234 = AFF(channels=m_channels * 16) - - self.n_stats = 1 if pooling_func == "TAP" or pooling_func == "TSDP" else 2 - self.pool = getattr(pooling_layers, pooling_func)( - in_dim=self.stats_dim * block.expansion - ) - self.seg_1 = nn.Linear( - self.stats_dim * block.expansion * self.n_stats, embedding_size - ) - if self.two_emb_layer: - self.seg_bn_1 = nn.BatchNorm1d(embedding_size, affine=False) - self.seg_2 = nn.Linear(embedding_size, embedding_size) - else: - self.seg_bn_1 = nn.Identity() - self.seg_2 = nn.Identity() - - def _make_layer(self, block, planes, num_blocks, stride): - strides = [stride] + [1] * (num_blocks - 1) - layers = [] - for stride in strides: - layers.append(block(self.in_planes, planes, stride)) - self.in_planes = planes * block.expansion - return nn.Sequential(*layers) - - def forward(self, x): - x = x.permute(0, 2, 1) # (B,T,F) => (B,F,T) - x = x.unsqueeze_(1) - out = F.relu(self.bn1(self.conv1(x))) - out1 = self.layer1(out) - out2 = self.layer2(out1) - out1_downsample = self.layer1_downsample(out1) - fuse_out12 = self.fuse_mode12(out2, out1_downsample) - out3 = self.layer3(out2) - fuse_out12_downsample = self.layer2_downsample(fuse_out12) - fuse_out123 = self.fuse_mode123(out3, fuse_out12_downsample) - out4 = self.layer4(out3) - fuse_out123_downsample = self.layer3_downsample(fuse_out123) - fuse_out1234 = self.fuse_mode1234(out4, fuse_out123_downsample) - stats = self.pool(fuse_out1234) - - embed_a = self.seg_1(stats) - if self.two_emb_layer: - out = F.relu(embed_a) - out = self.seg_bn_1(out) - embed_b = self.seg_2(out) - return embed_b - else: - return embed_a - - -class BasicBlockRes2Net(nn.Module): - expansion = 2 - - def __init__(self, in_planes, planes, stride=1, baseWidth=32, scale=2): - super(BasicBlockRes2Net, self).__init__() - width = int(math.floor(planes * (baseWidth / 64.0))) - self.conv1 = conv1x1(in_planes, width * scale, stride) - self.bn1 = nn.BatchNorm2d(width * scale) - self.nums = scale - 1 - convs = [] - bns = [] - for i in range(self.nums): - convs.append(conv3x3(width, width)) - bns.append(nn.BatchNorm2d(width)) - self.convs = nn.ModuleList(convs) - self.bns = nn.ModuleList(bns) - self.relu = ReLU(inplace=True) - - self.conv3 = conv1x1(width * scale, planes * self.expansion) - self.bn3 = nn.BatchNorm2d(planes * self.expansion) - self.shortcut = nn.Sequential() - if stride != 1 or in_planes != self.expansion * planes: - self.shortcut = nn.Sequential( - nn.Conv2d( - in_planes, - self.expansion * planes, - kernel_size=1, - stride=stride, - bias=False, - ), - nn.BatchNorm2d(self.expansion * planes), - ) - self.stride = stride - self.width = width - self.scale = scale - - def forward(self, x): - residual = x - - out = self.conv1(x) - out = self.bn1(out) - out = self.relu(out) - spx = torch.split(out, self.width, 1) - for i in range(self.nums): - if i == 0: - sp = spx[i] - else: - sp = sp + spx[i] - sp = self.convs[i](sp) - sp = self.relu(self.bns[i](sp)) - if i == 0: - out = sp - else: - out = torch.cat((out, sp), 1) - - out = torch.cat((out, spx[self.nums]), 1) - - out = self.conv3(out) - out = self.bn3(out) - - residual = self.shortcut(x) - out += residual - out = self.relu(out) - - return out - - -class Res2Net(nn.Module): - def __init__( - self, - block=BasicBlockRes2Net, - num_blocks=[3, 4, 6, 3], - m_channels=32, - feat_dim=80, - embedding_size=192, - pooling_func="TSTP", - two_emb_layer=False, - ): - super(Res2Net, self).__init__() - self.in_planes = m_channels - self.feat_dim = feat_dim - self.embedding_size = embedding_size - self.stats_dim = int(feat_dim / 8) * m_channels * 8 - self.two_emb_layer = two_emb_layer - - self.conv1 = nn.Conv2d( - 1, m_channels, kernel_size=3, stride=1, padding=1, bias=False - ) - self.bn1 = nn.BatchNorm2d(m_channels) - self.layer1 = self._make_layer(block, m_channels, num_blocks[0], stride=1) - self.layer2 = self._make_layer(block, m_channels * 2, num_blocks[1], stride=2) - self.layer3 = self._make_layer(block, m_channels * 4, num_blocks[2], stride=2) - self.layer4 = self._make_layer(block, m_channels * 8, num_blocks[3], stride=2) - - self.n_stats = 1 if pooling_func == "TAP" or pooling_func == "TSDP" else 2 - self.pool = getattr(pooling_layers, pooling_func)( - in_dim=self.stats_dim * block.expansion - ) - self.seg_1 = nn.Linear( - self.stats_dim * block.expansion * self.n_stats, embedding_size - ) - if self.two_emb_layer: - self.seg_bn_1 = nn.BatchNorm1d(embedding_size, affine=False) - self.seg_2 = nn.Linear(embedding_size, embedding_size) - else: - self.seg_bn_1 = nn.Identity() - self.seg_2 = nn.Identity() - - def _make_layer(self, block, planes, num_blocks, stride): - strides = [stride] + [1] * (num_blocks - 1) - layers = [] - for stride in strides: - layers.append(block(self.in_planes, planes, stride)) - self.in_planes = planes * block.expansion - return nn.Sequential(*layers) - - def forward(self, x): - x = x.permute(0, 2, 1) # (B,T,F) => (B,F,T) - - x = x.unsqueeze_(1) - out = F.relu(self.bn1(self.conv1(x))) - out = self.layer1(out) - out = self.layer2(out) - out = self.layer3(out) - out = self.layer4(out) - - stats = self.pool(out) - - embed_a = self.seg_1(stats) - if self.two_emb_layer: - out = F.relu(embed_a) - out = self.seg_bn_1(out) - embed_b = self.seg_2(out) - return embed_b - else: - return embed_a diff --git a/funasr_detach/models/eres2net/eres2net_aug.py b/funasr_detach/models/eres2net/eres2net_aug.py deleted file mode 100644 index 1ad19e7e2ff1c29ce78f81abc9b1ef2531683f9e..0000000000000000000000000000000000000000 --- a/funasr_detach/models/eres2net/eres2net_aug.py +++ /dev/null @@ -1,292 +0,0 @@ -# Copyright 3D-Speaker (https://github.com/alibaba-damo-academy/3D-Speaker). All Rights Reserved. -# Licensed under the Apache License, Version 2.0 (http://www.apache.org/licenses/LICENSE-2.0) - -"""Res2Net implementation is adapted from https://github.com/wenet-e2e/wespeaker. -ERes2Net incorporates both local and global feature fusion techniques to improve the performance. -The local feature fusion (LFF) fuses the features within one single residual block to extract the local signal. -The global feature fusion (GFF) takes acoustic features of different scales as input to aggregate global signal. -ERes2Net-Large is an upgraded version of ERes2Net that uses a larger number of parameters to achieve better -recognition performance. Parameters expansion, baseWidth, and scale can be modified to obtain optimal performance. -""" - -import math - -import torch -import torch.nn as nn -import torch.nn.functional as F - -import funasr_detach.models.sond.pooling.pooling_layers as pooling_layers - -from funasr_detach.models.eres2net.fusion import AFF - - -class ReLU(nn.Hardtanh): - - def __init__(self, inplace=False): - super(ReLU, self).__init__(0, 20, inplace) - - def __repr__(self): - inplace_str = "inplace" if self.inplace else "" - return self.__class__.__name__ + " (" + inplace_str + ")" - - -def conv1x1(in_planes, out_planes, stride=1): - "1x1 convolution without padding" - return nn.Conv2d( - in_planes, out_planes, kernel_size=1, stride=stride, padding=0, bias=False - ) - - -def conv3x3(in_planes, out_planes, stride=1): - "3x3 convolution with padding" - return nn.Conv2d( - in_planes, out_planes, kernel_size=3, stride=stride, padding=1, bias=False - ) - - -class BasicBlockERes2Net(nn.Module): - expansion = 4 - - def __init__(self, in_planes, planes, stride=1, baseWidth=24, scale=3): - super(BasicBlockERes2Net, self).__init__() - width = int(math.floor(planes * (baseWidth / 64.0))) - self.conv1 = conv1x1(in_planes, width * scale, stride) - self.bn1 = nn.BatchNorm2d(width * scale) - self.nums = scale - - convs = [] - bns = [] - for i in range(self.nums): - convs.append(conv3x3(width, width)) - bns.append(nn.BatchNorm2d(width)) - self.convs = nn.ModuleList(convs) - self.bns = nn.ModuleList(bns) - self.relu = ReLU(inplace=True) - - self.conv3 = conv1x1(width * scale, planes * self.expansion) - self.bn3 = nn.BatchNorm2d(planes * self.expansion) - self.shortcut = nn.Sequential() - if stride != 1 or in_planes != self.expansion * planes: - self.shortcut = nn.Sequential( - nn.Conv2d( - in_planes, - self.expansion * planes, - kernel_size=1, - stride=stride, - bias=False, - ), - nn.BatchNorm2d(self.expansion * planes), - ) - self.stride = stride - self.width = width - self.scale = scale - - def forward(self, x): - residual = x - - out = self.conv1(x) - out = self.bn1(out) - out = self.relu(out) - spx = torch.split(out, self.width, 1) - for i in range(self.nums): - if i == 0: - sp = spx[i] - else: - sp = sp + spx[i] - sp = self.convs[i](sp) - sp = self.relu(self.bns[i](sp)) - if i == 0: - out = sp - else: - out = torch.cat((out, sp), 1) - - out = self.conv3(out) - out = self.bn3(out) - - residual = self.shortcut(x) - out += residual - out = self.relu(out) - - return out - - -class BasicBlockERes2Net_diff_AFF(nn.Module): - expansion = 4 - - def __init__(self, in_planes, planes, stride=1, baseWidth=24, scale=3): - super(BasicBlockERes2Net_diff_AFF, self).__init__() - width = int(math.floor(planes * (baseWidth / 64.0))) - self.conv1 = conv1x1(in_planes, width * scale, stride) - self.bn1 = nn.BatchNorm2d(width * scale) - - self.nums = scale - - convs = [] - fuse_models = [] - bns = [] - for i in range(self.nums): - convs.append(conv3x3(width, width)) - bns.append(nn.BatchNorm2d(width)) - for j in range(self.nums - 1): - fuse_models.append(AFF(channels=width)) - - self.convs = nn.ModuleList(convs) - self.bns = nn.ModuleList(bns) - self.fuse_models = nn.ModuleList(fuse_models) - self.relu = ReLU(inplace=True) - - self.conv3 = conv1x1(width * scale, planes * self.expansion) - self.bn3 = nn.BatchNorm2d(planes * self.expansion) - self.shortcut = nn.Sequential() - if stride != 1 or in_planes != self.expansion * planes: - self.shortcut = nn.Sequential( - nn.Conv2d( - in_planes, - self.expansion * planes, - kernel_size=1, - stride=stride, - bias=False, - ), - nn.BatchNorm2d(self.expansion * planes), - ) - self.stride = stride - self.width = width - self.scale = scale - - def forward(self, x): - residual = x - - out = self.conv1(x) - out = self.bn1(out) - out = self.relu(out) - spx = torch.split(out, self.width, 1) - for i in range(self.nums): - if i == 0: - sp = spx[i] - else: - sp = self.fuse_models[i - 1](sp, spx[i]) - - sp = self.convs[i](sp) - sp = self.relu(self.bns[i](sp)) - if i == 0: - out = sp - else: - out = torch.cat((out, sp), 1) - - out = self.conv3(out) - out = self.bn3(out) - - residual = self.shortcut(x) - out += residual - out = self.relu(out) - - return out - - -class ERes2NetAug(nn.Module): - def __init__( - self, - block=BasicBlockERes2Net, - block_fuse=BasicBlockERes2Net_diff_AFF, - num_blocks=[3, 4, 6, 3], - m_channels=64, - feat_dim=80, - embedding_size=192, - pooling_func="TSTP", - two_emb_layer=False, - ): - super(ERes2NetAug, self).__init__() - self.in_planes = m_channels - self.feat_dim = feat_dim - self.embedding_size = embedding_size - self.stats_dim = int(feat_dim / 8) * m_channels * 8 - self.two_emb_layer = two_emb_layer - - self.conv1 = nn.Conv2d( - 1, m_channels, kernel_size=3, stride=1, padding=1, bias=False - ) - self.bn1 = nn.BatchNorm2d(m_channels) - self.layer1 = self._make_layer(block, m_channels, num_blocks[0], stride=1) - self.layer2 = self._make_layer(block, m_channels * 2, num_blocks[1], stride=2) - self.layer3 = self._make_layer( - block_fuse, m_channels * 4, num_blocks[2], stride=2 - ) - self.layer4 = self._make_layer( - block_fuse, m_channels * 8, num_blocks[3], stride=2 - ) - - self.layer1_downsample = nn.Conv2d( - m_channels * 4, - m_channels * 8, - kernel_size=3, - padding=1, - stride=2, - bias=False, - ) - self.layer2_downsample = nn.Conv2d( - m_channels * 8, - m_channels * 16, - kernel_size=3, - padding=1, - stride=2, - bias=False, - ) - self.layer3_downsample = nn.Conv2d( - m_channels * 16, - m_channels * 32, - kernel_size=3, - padding=1, - stride=2, - bias=False, - ) - self.fuse_mode12 = AFF(channels=m_channels * 8) - self.fuse_mode123 = AFF(channels=m_channels * 16) - self.fuse_mode1234 = AFF(channels=m_channels * 32) - - self.n_stats = 1 if pooling_func == "TAP" or pooling_func == "TSDP" else 2 - self.pool = getattr(pooling_layers, pooling_func)( - in_dim=self.stats_dim * block.expansion - ) - self.seg_1 = nn.Linear( - self.stats_dim * block.expansion * self.n_stats, embedding_size - ) - if self.two_emb_layer: - self.seg_bn_1 = nn.BatchNorm1d(embedding_size, affine=False) - self.seg_2 = nn.Linear(embedding_size, embedding_size) - else: - self.seg_bn_1 = nn.Identity() - self.seg_2 = nn.Identity() - - def _make_layer(self, block, planes, num_blocks, stride): - strides = [stride] + [1] * (num_blocks - 1) - layers = [] - for stride in strides: - layers.append(block(self.in_planes, planes, stride)) - self.in_planes = planes * block.expansion - return nn.Sequential(*layers) - - def forward(self, x): - x = x.permute(0, 2, 1) # (B,T,F) => (B,F,T) - - x = x.unsqueeze_(1) - out = F.relu(self.bn1(self.conv1(x))) - out1 = self.layer1(out) - out2 = self.layer2(out1) - out1_downsample = self.layer1_downsample(out1) - fuse_out12 = self.fuse_mode12(out2, out1_downsample) - out3 = self.layer3(out2) - fuse_out12_downsample = self.layer2_downsample(fuse_out12) - fuse_out123 = self.fuse_mode123(out3, fuse_out12_downsample) - out4 = self.layer4(out3) - fuse_out123_downsample = self.layer3_downsample(fuse_out123) - fuse_out1234 = self.fuse_mode1234(out4, fuse_out123_downsample) - stats = self.pool(fuse_out1234) - - embed_a = self.seg_1(stats) - if self.two_emb_layer: - out = F.relu(embed_a) - out = self.seg_bn_1(out) - embed_b = self.seg_2(out) - return embed_b - else: - return embed_a diff --git a/funasr_detach/models/eres2net/fusion.py b/funasr_detach/models/eres2net/fusion.py deleted file mode 100644 index fbe699efbaa33380594df2c5068b78e0ecbe7303..0000000000000000000000000000000000000000 --- a/funasr_detach/models/eres2net/fusion.py +++ /dev/null @@ -1,28 +0,0 @@ -# Copyright 3D-Speaker (https://github.com/alibaba-damo-academy/3D-Speaker). All Rights Reserved. -# Licensed under the Apache License, Version 2.0 (http://www.apache.org/licenses/LICENSE-2.0) - -import torch -import torch.nn as nn - - -class AFF(nn.Module): - - def __init__(self, channels=64, r=4): - super(AFF, self).__init__() - inter_channels = int(channels // r) - - self.local_att = nn.Sequential( - nn.Conv2d(channels * 2, inter_channels, kernel_size=1, stride=1, padding=0), - nn.BatchNorm2d(inter_channels), - nn.SiLU(inplace=True), - nn.Conv2d(inter_channels, channels, kernel_size=1, stride=1, padding=0), - nn.BatchNorm2d(channels), - ) - - def forward(self, x, ds_y): - xa = torch.cat((x, ds_y), dim=1) - x_att = self.local_att(xa) - x_att = 1.0 + torch.tanh(x_att) - xo = torch.mul(x, x_att) + torch.mul(ds_y, 2.0 - x_att) - - return xo diff --git a/funasr_detach/models/fsmn_vad_streaming/__init__.py b/funasr_detach/models/fsmn_vad_streaming/__init__.py deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/funasr_detach/models/fsmn_vad_streaming/encoder.py b/funasr_detach/models/fsmn_vad_streaming/encoder.py deleted file mode 100755 index 60169c08430ba45526caeef7f3b868b2bc5ef4c0..0000000000000000000000000000000000000000 --- a/funasr_detach/models/fsmn_vad_streaming/encoder.py +++ /dev/null @@ -1,338 +0,0 @@ -from typing import Tuple, Dict -import copy - -import numpy as np -import torch -import torch.nn as nn -import torch.nn.functional as F - -from funasr_detach.register import tables - - -class LinearTransform(nn.Module): - - def __init__(self, input_dim, output_dim): - super(LinearTransform, self).__init__() - self.input_dim = input_dim - self.output_dim = output_dim - self.linear = nn.Linear(input_dim, output_dim, bias=False) - - def forward(self, input): - output = self.linear(input) - - return output - - -class AffineTransform(nn.Module): - - def __init__(self, input_dim, output_dim): - super(AffineTransform, self).__init__() - self.input_dim = input_dim - self.output_dim = output_dim - self.linear = nn.Linear(input_dim, output_dim) - - def forward(self, input): - output = self.linear(input) - - return output - - -class RectifiedLinear(nn.Module): - - def __init__(self, input_dim, output_dim): - super(RectifiedLinear, self).__init__() - self.dim = input_dim - self.relu = nn.ReLU() - self.dropout = nn.Dropout(0.1) - - def forward(self, input): - out = self.relu(input) - return out - - -class FSMNBlock(nn.Module): - - def __init__( - self, - input_dim: int, - output_dim: int, - lorder=None, - rorder=None, - lstride=1, - rstride=1, - ): - super(FSMNBlock, self).__init__() - - self.dim = input_dim - - if lorder is None: - return - - self.lorder = lorder - self.rorder = rorder - self.lstride = lstride - self.rstride = rstride - - self.conv_left = nn.Conv2d( - self.dim, - self.dim, - [lorder, 1], - dilation=[lstride, 1], - groups=self.dim, - bias=False, - ) - - if self.rorder > 0: - self.conv_right = nn.Conv2d( - self.dim, - self.dim, - [rorder, 1], - dilation=[rstride, 1], - groups=self.dim, - bias=False, - ) - else: - self.conv_right = None - - def forward(self, input: torch.Tensor, cache: torch.Tensor): - x = torch.unsqueeze(input, 1) - x_per = x.permute(0, 3, 2, 1) # B D T C - - cache = cache.to(x_per.device) - y_left = torch.cat((cache, x_per), dim=2) - cache = y_left[:, :, -(self.lorder - 1) * self.lstride :, :] - y_left = self.conv_left(y_left) - out = x_per + y_left - - if self.conv_right is not None: - # maybe need to check - y_right = F.pad(x_per, [0, 0, 0, self.rorder * self.rstride]) - y_right = y_right[:, :, self.rstride :, :] - y_right = self.conv_right(y_right) - out += y_right - - out_per = out.permute(0, 3, 2, 1) - output = out_per.squeeze(1) - - return output, cache - - -class BasicBlock(nn.Module): - def __init__( - self, - linear_dim: int, - proj_dim: int, - lorder: int, - rorder: int, - lstride: int, - rstride: int, - stack_layer: int, - ): - super(BasicBlock, self).__init__() - self.lorder = lorder - self.rorder = rorder - self.lstride = lstride - self.rstride = rstride - self.stack_layer = stack_layer - self.linear = LinearTransform(linear_dim, proj_dim) - self.fsmn_block = FSMNBlock( - proj_dim, proj_dim, lorder, rorder, lstride, rstride - ) - self.affine = AffineTransform(proj_dim, linear_dim) - self.relu = RectifiedLinear(linear_dim, linear_dim) - - def forward(self, input: torch.Tensor, cache: Dict[str, torch.Tensor]): - x1 = self.linear(input) # B T D - cache_layer_name = "cache_layer_{}".format(self.stack_layer) - if cache_layer_name not in cache: - cache[cache_layer_name] = torch.zeros( - x1.shape[0], x1.shape[-1], (self.lorder - 1) * self.lstride, 1 - ) - x2, cache[cache_layer_name] = self.fsmn_block(x1, cache[cache_layer_name]) - x3 = self.affine(x2) - x4 = self.relu(x3) - return x4 - - -class FsmnStack(nn.Sequential): - def __init__(self, *args): - super(FsmnStack, self).__init__(*args) - - def forward(self, input: torch.Tensor, cache: Dict[str, torch.Tensor]): - x = input - for module in self._modules.values(): - x = module(x, cache) - return x - - -""" -FSMN net for keyword spotting -input_dim: input dimension -linear_dim: fsmn input dimensionll -proj_dim: fsmn projection dimension -lorder: fsmn left order -rorder: fsmn right order -num_syn: output dimension -fsmn_layers: no. of sequential fsmn layers -""" - - -@tables.register("encoder_classes", "FSMN") -class FSMN(nn.Module): - def __init__( - self, - input_dim: int, - input_affine_dim: int, - fsmn_layers: int, - linear_dim: int, - proj_dim: int, - lorder: int, - rorder: int, - lstride: int, - rstride: int, - output_affine_dim: int, - output_dim: int, - ): - super(FSMN, self).__init__() - - self.input_dim = input_dim - self.input_affine_dim = input_affine_dim - self.fsmn_layers = fsmn_layers - self.linear_dim = linear_dim - self.proj_dim = proj_dim - self.output_affine_dim = output_affine_dim - self.output_dim = output_dim - - self.in_linear1 = AffineTransform(input_dim, input_affine_dim) - self.in_linear2 = AffineTransform(input_affine_dim, linear_dim) - self.relu = RectifiedLinear(linear_dim, linear_dim) - self.fsmn = FsmnStack( - *[ - BasicBlock(linear_dim, proj_dim, lorder, rorder, lstride, rstride, i) - for i in range(fsmn_layers) - ] - ) - self.out_linear1 = AffineTransform(linear_dim, output_affine_dim) - self.out_linear2 = AffineTransform(output_affine_dim, output_dim) - self.softmax = nn.Softmax(dim=-1) - - def fuse_modules(self): - pass - - def forward( - self, input: torch.Tensor, cache: Dict[str, torch.Tensor] - ) -> Tuple[torch.Tensor, Dict[str, torch.Tensor]]: - """ - Args: - input (torch.Tensor): Input tensor (B, T, D) - cache: when cache is not None, the forward is in streaming. The type of cache is a dict, egs, - {'cache_layer_1': torch.Tensor(B, T1, D)}, T1 is equal to self.lorder. It is {} for the 1st frame - """ - - x1 = self.in_linear1(input) - x2 = self.in_linear2(x1) - x3 = self.relu(x2) - x4 = self.fsmn(x3, cache) # self.cache will update automatically in self.fsmn - x5 = self.out_linear1(x4) - x6 = self.out_linear2(x5) - x7 = self.softmax(x6) - - return x7 - - -""" -one deep fsmn layer -dimproj: projection dimension, input and output dimension of memory blocks -dimlinear: dimension of mapping layer -lorder: left order -rorder: right order -lstride: left stride -rstride: right stride -""" - - -@tables.register("encoder_classes", "DFSMN") -class DFSMN(nn.Module): - - def __init__( - self, dimproj=64, dimlinear=128, lorder=20, rorder=1, lstride=1, rstride=1 - ): - super(DFSMN, self).__init__() - - self.lorder = lorder - self.rorder = rorder - self.lstride = lstride - self.rstride = rstride - - self.expand = AffineTransform(dimproj, dimlinear) - self.shrink = LinearTransform(dimlinear, dimproj) - - self.conv_left = nn.Conv2d( - dimproj, - dimproj, - [lorder, 1], - dilation=[lstride, 1], - groups=dimproj, - bias=False, - ) - - if rorder > 0: - self.conv_right = nn.Conv2d( - dimproj, - dimproj, - [rorder, 1], - dilation=[rstride, 1], - groups=dimproj, - bias=False, - ) - else: - self.conv_right = None - - def forward(self, input): - f1 = F.relu(self.expand(input)) - p1 = self.shrink(f1) - - x = torch.unsqueeze(p1, 1) - x_per = x.permute(0, 3, 2, 1) - - y_left = F.pad(x_per, [0, 0, (self.lorder - 1) * self.lstride, 0]) - - if self.conv_right is not None: - y_right = F.pad(x_per, [0, 0, 0, (self.rorder) * self.rstride]) - y_right = y_right[:, :, self.rstride :, :] - out = x_per + self.conv_left(y_left) + self.conv_right(y_right) - else: - out = x_per + self.conv_left(y_left) - - out1 = out.permute(0, 3, 2, 1) - output = input + out1.squeeze(1) - - return output - - -""" -build stacked dfsmn layers -""" - - -def buildDFSMNRepeats(linear_dim=128, proj_dim=64, lorder=20, rorder=1, fsmn_layers=6): - repeats = [ - nn.Sequential(DFSMN(proj_dim, linear_dim, lorder, rorder, 1, 1)) - for i in range(fsmn_layers) - ] - - return nn.Sequential(*repeats) - - -if __name__ == "__main__": - fsmn = FSMN(400, 140, 4, 250, 128, 10, 2, 1, 1, 140, 2599) - print(fsmn) - - num_params = sum(p.numel() for p in fsmn.parameters()) - print("the number of model params: {}".format(num_params)) - x = torch.zeros(128, 200, 400) # batch-size * time * dim - y, _ = fsmn(x) # batch-size * time * dim - print("input shape: {}".format(x.shape)) - print("output shape: {}".format(y.shape)) - - print(fsmn.to_kaldi_net()) diff --git a/funasr_detach/models/fsmn_vad_streaming/model.py b/funasr_detach/models/fsmn_vad_streaming/model.py deleted file mode 100644 index 77ca870e2720205d97d596f86e4a44bdf42a904c..0000000000000000000000000000000000000000 --- a/funasr_detach/models/fsmn_vad_streaming/model.py +++ /dev/null @@ -1,1048 +0,0 @@ -#!/usr/bin/env python3 -# -*- encoding: utf-8 -*- -# Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved. -# MIT License (https://opensource.org/licenses/MIT) - -import os -import json -import time -import math -import torch -from torch import nn -from enum import Enum -from dataclasses import dataclass -from funasr_detach.register import tables -from typing import List, Tuple, Dict, Any, Optional - -from funasr_detach.utils.datadir_writer import DatadirWriter -from funasr_detach.utils.load_utils import load_audio_text_image_video, extract_fbank - - -class VadStateMachine(Enum): - kVadInStateStartPointNotDetected = 1 - kVadInStateInSpeechSegment = 2 - kVadInStateEndPointDetected = 3 - - -class FrameState(Enum): - kFrameStateInvalid = -1 - kFrameStateSpeech = 1 - kFrameStateSil = 0 - - -# final voice/unvoice state per frame -class AudioChangeState(Enum): - kChangeStateSpeech2Speech = 0 - kChangeStateSpeech2Sil = 1 - kChangeStateSil2Sil = 2 - kChangeStateSil2Speech = 3 - kChangeStateNoBegin = 4 - kChangeStateInvalid = 5 - - -class VadDetectMode(Enum): - kVadSingleUtteranceDetectMode = 0 - kVadMutipleUtteranceDetectMode = 1 - - -class VADXOptions: - """ - Author: Speech Lab of DAMO Academy, Alibaba Group - Deep-FSMN for Large Vocabulary Continuous Speech Recognition - https://arxiv.org/abs/1803.05030 - """ - - def __init__( - self, - sample_rate: int = 16000, - detect_mode: int = VadDetectMode.kVadMutipleUtteranceDetectMode.value, - snr_mode: int = 0, - max_end_silence_time: int = 800, - max_start_silence_time: int = 3000, - do_start_point_detection: bool = True, - do_end_point_detection: bool = True, - window_size_ms: int = 200, - sil_to_speech_time_thres: int = 150, - speech_to_sil_time_thres: int = 150, - speech_2_noise_ratio: float = 1.0, - do_extend: int = 1, - lookback_time_start_point: int = 200, - lookahead_time_end_point: int = 100, - max_single_segment_time: int = 60000, - nn_eval_block_size: int = 8, - dcd_block_size: int = 4, - snr_thres: int = -100.0, - noise_frame_num_used_for_snr: int = 100, - decibel_thres: int = -100.0, - speech_noise_thres: float = 0.6, - fe_prior_thres: float = 1e-4, - silence_pdf_num: int = 1, - sil_pdf_ids: List[int] = [0], - speech_noise_thresh_low: float = -0.1, - speech_noise_thresh_high: float = 0.3, - output_frame_probs: bool = False, - frame_in_ms: int = 10, - frame_length_ms: int = 25, - **kwargs, - ): - self.sample_rate = sample_rate - self.detect_mode = detect_mode - self.snr_mode = snr_mode - self.max_end_silence_time = max_end_silence_time - self.max_start_silence_time = max_start_silence_time - self.do_start_point_detection = do_start_point_detection - self.do_end_point_detection = do_end_point_detection - self.window_size_ms = window_size_ms - self.sil_to_speech_time_thres = sil_to_speech_time_thres - self.speech_to_sil_time_thres = speech_to_sil_time_thres - self.speech_2_noise_ratio = speech_2_noise_ratio - self.do_extend = do_extend - self.lookback_time_start_point = lookback_time_start_point - self.lookahead_time_end_point = lookahead_time_end_point - self.max_single_segment_time = max_single_segment_time - self.nn_eval_block_size = nn_eval_block_size - self.dcd_block_size = dcd_block_size - self.snr_thres = snr_thres - self.noise_frame_num_used_for_snr = noise_frame_num_used_for_snr - self.decibel_thres = decibel_thres - self.speech_noise_thres = speech_noise_thres - self.fe_prior_thres = fe_prior_thres - self.silence_pdf_num = silence_pdf_num - self.sil_pdf_ids = sil_pdf_ids - self.speech_noise_thresh_low = speech_noise_thresh_low - self.speech_noise_thresh_high = speech_noise_thresh_high - self.output_frame_probs = output_frame_probs - self.frame_in_ms = frame_in_ms - self.frame_length_ms = frame_length_ms - - -class E2EVadSpeechBufWithDoa(object): - """ - Author: Speech Lab of DAMO Academy, Alibaba Group - Deep-FSMN for Large Vocabulary Continuous Speech Recognition - https://arxiv.org/abs/1803.05030 - """ - - def __init__(self): - self.start_ms = 0 - self.end_ms = 0 - self.buffer = [] - self.contain_seg_start_point = False - self.contain_seg_end_point = False - self.doa = 0 - - def Reset(self): - self.start_ms = 0 - self.end_ms = 0 - self.buffer = [] - self.contain_seg_start_point = False - self.contain_seg_end_point = False - self.doa = 0 - - -class E2EVadFrameProb(object): - """ - Author: Speech Lab of DAMO Academy, Alibaba Group - Deep-FSMN for Large Vocabulary Continuous Speech Recognition - https://arxiv.org/abs/1803.05030 - """ - - def __init__(self): - self.noise_prob = 0.0 - self.speech_prob = 0.0 - self.score = 0.0 - self.frame_id = 0 - self.frm_state = 0 - - -class WindowDetector(object): - """ - Author: Speech Lab of DAMO Academy, Alibaba Group - Deep-FSMN for Large Vocabulary Continuous Speech Recognition - https://arxiv.org/abs/1803.05030 - """ - - def __init__( - self, - window_size_ms: int, - sil_to_speech_time: int, - speech_to_sil_time: int, - frame_size_ms: int, - ): - self.window_size_ms = window_size_ms - self.sil_to_speech_time = sil_to_speech_time - self.speech_to_sil_time = speech_to_sil_time - self.frame_size_ms = frame_size_ms - - self.win_size_frame = int(window_size_ms / frame_size_ms) - self.win_sum = 0 - self.win_state = [0] * self.win_size_frame # 初始化窗 - - self.cur_win_pos = 0 - self.pre_frame_state = FrameState.kFrameStateSil - self.cur_frame_state = FrameState.kFrameStateSil - self.sil_to_speech_frmcnt_thres = int(sil_to_speech_time / frame_size_ms) - self.speech_to_sil_frmcnt_thres = int(speech_to_sil_time / frame_size_ms) - - self.voice_last_frame_count = 0 - self.noise_last_frame_count = 0 - self.hydre_frame_count = 0 - - def Reset(self) -> None: - self.cur_win_pos = 0 - self.win_sum = 0 - self.win_state = [0] * self.win_size_frame - self.pre_frame_state = FrameState.kFrameStateSil - self.cur_frame_state = FrameState.kFrameStateSil - self.voice_last_frame_count = 0 - self.noise_last_frame_count = 0 - self.hydre_frame_count = 0 - - def GetWinSize(self) -> int: - return int(self.win_size_frame) - - def DetectOneFrame( - self, frameState: FrameState, frame_count: int, cache: dict = {} - ) -> AudioChangeState: - cur_frame_state = FrameState.kFrameStateSil - if frameState == FrameState.kFrameStateSpeech: - cur_frame_state = 1 - elif frameState == FrameState.kFrameStateSil: - cur_frame_state = 0 - else: - return AudioChangeState.kChangeStateInvalid - self.win_sum -= self.win_state[self.cur_win_pos] - self.win_sum += cur_frame_state - self.win_state[self.cur_win_pos] = cur_frame_state - self.cur_win_pos = (self.cur_win_pos + 1) % self.win_size_frame - - if ( - self.pre_frame_state == FrameState.kFrameStateSil - and self.win_sum >= self.sil_to_speech_frmcnt_thres - ): - self.pre_frame_state = FrameState.kFrameStateSpeech - return AudioChangeState.kChangeStateSil2Speech - - if ( - self.pre_frame_state == FrameState.kFrameStateSpeech - and self.win_sum <= self.speech_to_sil_frmcnt_thres - ): - self.pre_frame_state = FrameState.kFrameStateSil - return AudioChangeState.kChangeStateSpeech2Sil - - if self.pre_frame_state == FrameState.kFrameStateSil: - return AudioChangeState.kChangeStateSil2Sil - if self.pre_frame_state == FrameState.kFrameStateSpeech: - return AudioChangeState.kChangeStateSpeech2Speech - return AudioChangeState.kChangeStateInvalid - - def FrameSizeMs(self) -> int: - return int(self.frame_size_ms) - - -class Stats(object): - def __init__( - self, - sil_pdf_ids, - max_end_sil_frame_cnt_thresh, - speech_noise_thres, - ): - self.data_buf_start_frame = 0 - self.frm_cnt = 0 - self.latest_confirmed_speech_frame = 0 - self.lastest_confirmed_silence_frame = -1 - self.continous_silence_frame_count = 0 - self.vad_state_machine = VadStateMachine.kVadInStateStartPointNotDetected - self.confirmed_start_frame = -1 - self.confirmed_end_frame = -1 - self.number_end_time_detected = 0 - self.sil_frame = 0 - self.sil_pdf_ids = sil_pdf_ids - self.noise_average_decibel = -100.0 - self.pre_end_silence_detected = False - self.next_seg = True - - self.output_data_buf = [] - self.output_data_buf_offset = 0 - self.frame_probs = [] - self.max_end_sil_frame_cnt_thresh = max_end_sil_frame_cnt_thresh - self.speech_noise_thres = speech_noise_thres - self.scores = None - self.max_time_out = False - self.decibel = [] - self.data_buf = None - self.data_buf_all = None - self.waveform = None - self.last_drop_frames = 0 - - -@tables.register("model_classes", "FsmnVADStreaming") -class FsmnVADStreaming(nn.Module): - """ - Author: Speech Lab of DAMO Academy, Alibaba Group - Deep-FSMN for Large Vocabulary Continuous Speech Recognition - https://arxiv.org/abs/1803.05030 - """ - - def __init__( - self, - encoder: str = None, - encoder_conf: Optional[Dict] = None, - vad_post_args: Dict[str, Any] = None, - **kwargs, - ): - super().__init__() - self.vad_opts = VADXOptions(**kwargs) - - encoder_class = tables.encoder_classes.get(encoder) - encoder = encoder_class(**encoder_conf) - self.encoder = encoder - - def ResetDetection(self, cache: dict = {}): - cache["stats"].continous_silence_frame_count = 0 - cache["stats"].latest_confirmed_speech_frame = 0 - cache["stats"].lastest_confirmed_silence_frame = -1 - cache["stats"].confirmed_start_frame = -1 - cache["stats"].confirmed_end_frame = -1 - cache["stats"].vad_state_machine = ( - VadStateMachine.kVadInStateStartPointNotDetected - ) - cache["windows_detector"].Reset() - cache["stats"].sil_frame = 0 - cache["stats"].frame_probs = [] - - if cache["stats"].output_data_buf: - assert cache["stats"].output_data_buf[-1].contain_seg_end_point == True - drop_frames = int( - cache["stats"].output_data_buf[-1].end_ms / self.vad_opts.frame_in_ms - ) - real_drop_frames = drop_frames - cache["stats"].last_drop_frames - cache["stats"].last_drop_frames = drop_frames - cache["stats"].data_buf_all = cache["stats"].data_buf_all[ - real_drop_frames - * int(self.vad_opts.frame_in_ms * self.vad_opts.sample_rate / 1000) : - ] - cache["stats"].decibel = cache["stats"].decibel[real_drop_frames:] - cache["stats"].scores = cache["stats"].scores[:, real_drop_frames:, :] - - def ComputeDecibel(self, cache: dict = {}) -> None: - frame_sample_length = int( - self.vad_opts.frame_length_ms * self.vad_opts.sample_rate / 1000 - ) - frame_shift_length = int( - self.vad_opts.frame_in_ms * self.vad_opts.sample_rate / 1000 - ) - if cache["stats"].data_buf_all is None: - cache["stats"].data_buf_all = cache["stats"].waveform[ - 0 - ] # cache["stats"].data_buf is pointed to cache["stats"].waveform[0] - cache["stats"].data_buf = cache["stats"].data_buf_all - else: - cache["stats"].data_buf_all = torch.cat( - (cache["stats"].data_buf_all, cache["stats"].waveform[0]) - ) - for offset in range( - 0, - cache["stats"].waveform.shape[1] - frame_sample_length + 1, - frame_shift_length, - ): - cache["stats"].decibel.append( - 10 - * math.log10( - (cache["stats"].waveform[0][offset : offset + frame_sample_length]) - .square() - .sum() - + 0.000001 - ) - ) - - def ComputeScores(self, feats: torch.Tensor, cache: dict = {}) -> None: - scores = self.encoder(feats, cache=cache["encoder"]).to( - "cpu" - ) # return B * T * D - assert ( - scores.shape[1] == feats.shape[1] - ), "The shape between feats and scores does not match" - self.vad_opts.nn_eval_block_size = scores.shape[1] - cache["stats"].frm_cnt += scores.shape[1] # count total frames - if cache["stats"].scores is None: - cache["stats"].scores = scores # the first calculation - else: - cache["stats"].scores = torch.cat((cache["stats"].scores, scores), dim=1) - - def PopDataBufTillFrame( - self, frame_idx: int, cache: dict = {} - ) -> None: # need check again - while cache["stats"].data_buf_start_frame < frame_idx: - if len(cache["stats"].data_buf) >= int( - self.vad_opts.frame_in_ms * self.vad_opts.sample_rate / 1000 - ): - cache["stats"].data_buf_start_frame += 1 - cache["stats"].data_buf = cache["stats"].data_buf_all[ - ( - cache["stats"].data_buf_start_frame - - cache["stats"].last_drop_frames - ) - * int( - self.vad_opts.frame_in_ms * self.vad_opts.sample_rate / 1000 - ) : - ] - - def PopDataToOutputBuf( - self, - start_frm: int, - frm_cnt: int, - first_frm_is_start_point: bool, - last_frm_is_end_point: bool, - end_point_is_sent_end: bool, - cache: dict = {}, - ) -> None: - self.PopDataBufTillFrame(start_frm, cache=cache) - expected_sample_number = int( - frm_cnt * self.vad_opts.sample_rate * self.vad_opts.frame_in_ms / 1000 - ) - if last_frm_is_end_point: - extra_sample = max( - 0, - int( - self.vad_opts.frame_length_ms * self.vad_opts.sample_rate / 1000 - - self.vad_opts.sample_rate * self.vad_opts.frame_in_ms / 1000 - ), - ) - expected_sample_number += int(extra_sample) - if end_point_is_sent_end: - expected_sample_number = max( - expected_sample_number, len(cache["stats"].data_buf) - ) - if len(cache["stats"].data_buf) < expected_sample_number: - print("error in calling pop data_buf\n") - - if len(cache["stats"].output_data_buf) == 0 or first_frm_is_start_point: - cache["stats"].output_data_buf.append(E2EVadSpeechBufWithDoa()) - cache["stats"].output_data_buf[-1].Reset() - cache["stats"].output_data_buf[-1].start_ms = ( - start_frm * self.vad_opts.frame_in_ms - ) - cache["stats"].output_data_buf[-1].end_ms = ( - cache["stats"].output_data_buf[-1].start_ms - ) - cache["stats"].output_data_buf[-1].doa = 0 - cur_seg = cache["stats"].output_data_buf[-1] - if cur_seg.end_ms != start_frm * self.vad_opts.frame_in_ms: - print("warning\n") - out_pos = len(cur_seg.buffer) # cur_seg.buff现在没做任何操作 - data_to_pop = 0 - if end_point_is_sent_end: - data_to_pop = expected_sample_number - else: - data_to_pop = int( - frm_cnt * self.vad_opts.frame_in_ms * self.vad_opts.sample_rate / 1000 - ) - if data_to_pop > len(cache["stats"].data_buf): - print('VAD data_to_pop is bigger than cache["stats"].data_buf.size()!!!\n') - data_to_pop = len(cache["stats"].data_buf) - expected_sample_number = len(cache["stats"].data_buf) - - cur_seg.doa = 0 - for sample_cpy_out in range(0, data_to_pop): - # cur_seg.buffer[out_pos ++] = data_buf_.back(); - out_pos += 1 - for sample_cpy_out in range(data_to_pop, expected_sample_number): - # cur_seg.buffer[out_pos++] = data_buf_.back() - out_pos += 1 - if cur_seg.end_ms != start_frm * self.vad_opts.frame_in_ms: - print("Something wrong with the VAD algorithm\n") - cache["stats"].data_buf_start_frame += frm_cnt - cur_seg.end_ms = (start_frm + frm_cnt) * self.vad_opts.frame_in_ms - if first_frm_is_start_point: - cur_seg.contain_seg_start_point = True - if last_frm_is_end_point: - cur_seg.contain_seg_end_point = True - - def OnSilenceDetected(self, valid_frame: int, cache: dict = {}): - cache["stats"].lastest_confirmed_silence_frame = valid_frame - if ( - cache["stats"].vad_state_machine - == VadStateMachine.kVadInStateStartPointNotDetected - ): - self.PopDataBufTillFrame(valid_frame, cache=cache) - - # silence_detected_callback_ - # pass - - def OnVoiceDetected(self, valid_frame: int, cache: dict = {}) -> None: - cache["stats"].latest_confirmed_speech_frame = valid_frame - self.PopDataToOutputBuf(valid_frame, 1, False, False, False, cache=cache) - - def OnVoiceStart( - self, start_frame: int, fake_result: bool = False, cache: dict = {} - ) -> None: - if self.vad_opts.do_start_point_detection: - pass - if cache["stats"].confirmed_start_frame != -1: - print("not reset vad properly\n") - else: - cache["stats"].confirmed_start_frame = start_frame - - if ( - not fake_result - and cache["stats"].vad_state_machine - == VadStateMachine.kVadInStateStartPointNotDetected - ): - self.PopDataToOutputBuf( - cache["stats"].confirmed_start_frame, 1, True, False, False, cache=cache - ) - - def OnVoiceEnd( - self, end_frame: int, fake_result: bool, is_last_frame: bool, cache: dict = {} - ) -> None: - for t in range(cache["stats"].latest_confirmed_speech_frame + 1, end_frame): - self.OnVoiceDetected(t, cache=cache) - if self.vad_opts.do_end_point_detection: - pass - if cache["stats"].confirmed_end_frame != -1: - print("not reset vad properly\n") - else: - cache["stats"].confirmed_end_frame = end_frame - if not fake_result: - cache["stats"].sil_frame = 0 - self.PopDataToOutputBuf( - cache["stats"].confirmed_end_frame, - 1, - False, - True, - is_last_frame, - cache=cache, - ) - cache["stats"].number_end_time_detected += 1 - - def MaybeOnVoiceEndIfLastFrame( - self, is_final_frame: bool, cur_frm_idx: int, cache: dict = {} - ) -> None: - if is_final_frame: - self.OnVoiceEnd(cur_frm_idx, False, True, cache=cache) - cache["stats"].vad_state_machine = ( - VadStateMachine.kVadInStateEndPointDetected - ) - - def GetLatency(self, cache: dict = {}) -> int: - return int( - self.LatencyFrmNumAtStartPoint(cache=cache) * self.vad_opts.frame_in_ms - ) - - def LatencyFrmNumAtStartPoint(self, cache: dict = {}) -> int: - vad_latency = cache["windows_detector"].GetWinSize() - if self.vad_opts.do_extend: - vad_latency += int( - self.vad_opts.lookback_time_start_point / self.vad_opts.frame_in_ms - ) - return vad_latency - - def GetFrameState(self, t: int, cache: dict = {}): - frame_state = FrameState.kFrameStateInvalid - cur_decibel = cache["stats"].decibel[t] - cur_snr = cur_decibel - cache["stats"].noise_average_decibel - # for each frame, calc log posterior probability of each state - if cur_decibel < self.vad_opts.decibel_thres: - frame_state = FrameState.kFrameStateSil - self.DetectOneFrame(frame_state, t, False, cache=cache) - return frame_state - - sum_score = 0.0 - noise_prob = 0.0 - assert len(cache["stats"].sil_pdf_ids) == self.vad_opts.silence_pdf_num - if len(cache["stats"].sil_pdf_ids) > 0: - assert len(cache["stats"].scores) == 1 # 只支持batch_size = 1的测试 - sil_pdf_scores = [ - cache["stats"].scores[0][t][sil_pdf_id] - for sil_pdf_id in cache["stats"].sil_pdf_ids - ] - sum_score = sum(sil_pdf_scores) - noise_prob = math.log(sum_score) * self.vad_opts.speech_2_noise_ratio - total_score = 1.0 - sum_score = total_score - sum_score - speech_prob = math.log(sum_score) - if self.vad_opts.output_frame_probs: - frame_prob = E2EVadFrameProb() - frame_prob.noise_prob = noise_prob - frame_prob.speech_prob = speech_prob - frame_prob.score = sum_score - frame_prob.frame_id = t - cache["stats"].frame_probs.append(frame_prob) - if ( - math.exp(speech_prob) - >= math.exp(noise_prob) + cache["stats"].speech_noise_thres - ): - if ( - cur_snr >= self.vad_opts.snr_thres - and cur_decibel >= self.vad_opts.decibel_thres - ): - frame_state = FrameState.kFrameStateSpeech - else: - frame_state = FrameState.kFrameStateSil - else: - frame_state = FrameState.kFrameStateSil - if cache["stats"].noise_average_decibel < -99.9: - cache["stats"].noise_average_decibel = cur_decibel - else: - cache["stats"].noise_average_decibel = ( - cur_decibel - + cache["stats"].noise_average_decibel - * (self.vad_opts.noise_frame_num_used_for_snr - 1) - ) / self.vad_opts.noise_frame_num_used_for_snr - - return frame_state - - def forward( - self, - feats: torch.Tensor, - waveform: torch.tensor, - cache: dict = {}, - is_final: bool = False, - **kwargs, - ): - # if len(cache) == 0: - # self.AllResetDetection() - # self.waveform = waveform # compute decibel for each frame - cache["stats"].waveform = waveform - is_streaming_input = kwargs.get("is_streaming_input", True) - self.ComputeDecibel(cache=cache) - self.ComputeScores(feats, cache=cache) - if not is_final: - self.DetectCommonFrames(cache=cache) - else: - self.DetectLastFrames(cache=cache) - segments = [] - for batch_num in range(0, feats.shape[0]): # only support batch_size = 1 now - segment_batch = [] - if len(cache["stats"].output_data_buf) > 0: - for i in range( - cache["stats"].output_data_buf_offset, - len(cache["stats"].output_data_buf), - ): - if ( - is_streaming_input - ): # in this case, return [beg, -1], [], [-1, end], [beg, end] - if ( - not cache["stats"] - .output_data_buf[i] - .contain_seg_start_point - ): - continue - if ( - not cache["stats"].next_seg - and not cache["stats"] - .output_data_buf[i] - .contain_seg_end_point - ): - continue - start_ms = ( - cache["stats"].output_data_buf[i].start_ms - if cache["stats"].next_seg - else -1 - ) - if cache["stats"].output_data_buf[i].contain_seg_end_point: - end_ms = cache["stats"].output_data_buf[i].end_ms - cache["stats"].next_seg = True - cache["stats"].output_data_buf_offset += 1 - else: - end_ms = -1 - cache["stats"].next_seg = False - segment = [start_ms, end_ms] - - else: # in this case, return [beg, end] - - if not is_final and ( - not cache["stats"] - .output_data_buf[i] - .contain_seg_start_point - or not cache["stats"] - .output_data_buf[i] - .contain_seg_end_point - ): - continue - segment = [ - cache["stats"].output_data_buf[i].start_ms, - cache["stats"].output_data_buf[i].end_ms, - ] - cache[ - "stats" - ].output_data_buf_offset += 1 # need update this parameter - - segment_batch.append(segment) - - if segment_batch: - segments.append(segment_batch) - # if is_final: - # # reset class variables and clear the dict for the next query - # self.AllResetDetection() - return segments - - def init_cache(self, cache: dict = {}, **kwargs): - - cache["frontend"] = {} - cache["prev_samples"] = torch.empty(0) - cache["encoder"] = {} - windows_detector = WindowDetector( - self.vad_opts.window_size_ms, - self.vad_opts.sil_to_speech_time_thres, - self.vad_opts.speech_to_sil_time_thres, - self.vad_opts.frame_in_ms, - ) - windows_detector.Reset() - - stats = Stats( - sil_pdf_ids=self.vad_opts.sil_pdf_ids, - max_end_sil_frame_cnt_thresh=self.vad_opts.max_end_silence_time - - self.vad_opts.speech_to_sil_time_thres, - speech_noise_thres=self.vad_opts.speech_noise_thres, - ) - cache["windows_detector"] = windows_detector - cache["stats"] = stats - return cache - - def inference( - self, - data_in, - data_lengths=None, - key: list = None, - tokenizer=None, - frontend=None, - cache: dict = {}, - **kwargs, - ): - - if len(cache) == 0: - self.init_cache(cache, **kwargs) - - meta_data = {} - chunk_size = kwargs.get("chunk_size", 60000) # 50ms - chunk_stride_samples = int(chunk_size * frontend.fs / 1000) - - time1 = time.perf_counter() - is_streaming_input = ( - kwargs.get("is_streaming_input", False) - if chunk_size >= 15000 - else kwargs.get("is_streaming_input", True) - ) - is_final = ( - kwargs.get("is_final", False) - if is_streaming_input - else kwargs.get("is_final", True) - ) - cfg = {"is_final": is_final, "is_streaming_input": is_streaming_input} - audio_sample_list = load_audio_text_image_video( - data_in, - fs=frontend.fs, - audio_fs=kwargs.get("fs", 16000), - data_type=kwargs.get("data_type", "sound"), - tokenizer=tokenizer, - cache=cfg, - ) - _is_final = cfg["is_final"] # if data_in is a file or url, set is_final=True - is_streaming_input = cfg["is_streaming_input"] - time2 = time.perf_counter() - meta_data["load_data"] = f"{time2 - time1:0.3f}" - assert len(audio_sample_list) == 1, "batch_size must be set 1" - - audio_sample = torch.cat((cache["prev_samples"], audio_sample_list[0])) - - n = int(len(audio_sample) // chunk_stride_samples + int(_is_final)) - m = int(len(audio_sample) % chunk_stride_samples * (1 - int(_is_final))) - segments = [] - for i in range(n): - kwargs["is_final"] = _is_final and i == n - 1 - audio_sample_i = audio_sample[ - i * chunk_stride_samples : (i + 1) * chunk_stride_samples - ] - - # extract fbank feats - speech, speech_lengths = extract_fbank( - [audio_sample_i], - data_type=kwargs.get("data_type", "sound"), - frontend=frontend, - cache=cache["frontend"], - is_final=kwargs["is_final"], - ) - time3 = time.perf_counter() - meta_data["extract_feat"] = f"{time3 - time2:0.3f}" - meta_data["batch_data_time"] = ( - speech_lengths.sum().item() - * frontend.frame_shift - * frontend.lfr_n - / 1000 - ) - speech = speech.to(device=kwargs["device"]) - speech_lengths = speech_lengths.to(device=kwargs["device"]) - - batch = { - "feats": speech, - "waveform": cache["frontend"]["waveforms"], - "is_final": kwargs["is_final"], - "cache": cache, - "is_streaming_input": is_streaming_input, - } - segments_i = self.forward(**batch) - if len(segments_i) > 0: - segments.extend(*segments_i) - - cache["prev_samples"] = audio_sample[:-m] - if _is_final: - self.init_cache(cache) - - ibest_writer = None - if kwargs.get("output_dir") is not None: - if not hasattr(self, "writer"): - self.writer = DatadirWriter(kwargs.get("output_dir")) - ibest_writer = self.writer[f"{1}best_recog"] - - results = [] - result_i = {"key": key[0], "value": segments} - if ( - "MODELSCOPE_ENVIRONMENT" in os.environ - and os.environ["MODELSCOPE_ENVIRONMENT"] == "eas" - ): - result_i = json.dumps(result_i) - - results.append(result_i) - - if ibest_writer is not None: - ibest_writer["text"][key[0]] = segments - - return results, meta_data - - def DetectCommonFrames(self, cache: dict = {}) -> int: - if ( - cache["stats"].vad_state_machine - == VadStateMachine.kVadInStateEndPointDetected - ): - return 0 - for i in range(self.vad_opts.nn_eval_block_size - 1, -1, -1): - frame_state = FrameState.kFrameStateInvalid - frame_state = self.GetFrameState( - cache["stats"].frm_cnt - 1 - i - cache["stats"].last_drop_frames, - cache=cache, - ) - self.DetectOneFrame( - frame_state, cache["stats"].frm_cnt - 1 - i, False, cache=cache - ) - - return 0 - - def DetectLastFrames(self, cache: dict = {}) -> int: - if ( - cache["stats"].vad_state_machine - == VadStateMachine.kVadInStateEndPointDetected - ): - return 0 - for i in range(self.vad_opts.nn_eval_block_size - 1, -1, -1): - frame_state = FrameState.kFrameStateInvalid - frame_state = self.GetFrameState( - cache["stats"].frm_cnt - 1 - i - cache["stats"].last_drop_frames, - cache=cache, - ) - if i != 0: - self.DetectOneFrame( - frame_state, cache["stats"].frm_cnt - 1 - i, False, cache=cache - ) - else: - self.DetectOneFrame( - frame_state, cache["stats"].frm_cnt - 1, True, cache=cache - ) - - return 0 - - def DetectOneFrame( - self, - cur_frm_state: FrameState, - cur_frm_idx: int, - is_final_frame: bool, - cache: dict = {}, - ) -> None: - tmp_cur_frm_state = FrameState.kFrameStateInvalid - if cur_frm_state == FrameState.kFrameStateSpeech: - if math.fabs(1.0) > self.vad_opts.fe_prior_thres: - tmp_cur_frm_state = FrameState.kFrameStateSpeech - else: - tmp_cur_frm_state = FrameState.kFrameStateSil - elif cur_frm_state == FrameState.kFrameStateSil: - tmp_cur_frm_state = FrameState.kFrameStateSil - state_change = cache["windows_detector"].DetectOneFrame( - tmp_cur_frm_state, cur_frm_idx, cache=cache - ) - frm_shift_in_ms = self.vad_opts.frame_in_ms - if AudioChangeState.kChangeStateSil2Speech == state_change: - silence_frame_count = cache["stats"].continous_silence_frame_count - cache["stats"].continous_silence_frame_count = 0 - cache["stats"].pre_end_silence_detected = False - start_frame = 0 - if ( - cache["stats"].vad_state_machine - == VadStateMachine.kVadInStateStartPointNotDetected - ): - start_frame = max( - cache["stats"].data_buf_start_frame, - cur_frm_idx - self.LatencyFrmNumAtStartPoint(cache=cache), - ) - self.OnVoiceStart(start_frame, cache=cache) - cache["stats"].vad_state_machine = ( - VadStateMachine.kVadInStateInSpeechSegment - ) - for t in range(start_frame + 1, cur_frm_idx + 1): - self.OnVoiceDetected(t, cache=cache) - elif ( - cache["stats"].vad_state_machine - == VadStateMachine.kVadInStateInSpeechSegment - ): - for t in range( - cache["stats"].latest_confirmed_speech_frame + 1, cur_frm_idx - ): - self.OnVoiceDetected(t, cache=cache) - if ( - cur_frm_idx - cache["stats"].confirmed_start_frame + 1 - > self.vad_opts.max_single_segment_time / frm_shift_in_ms - ): - self.OnVoiceEnd(cur_frm_idx, False, False, cache=cache) - cache["stats"].vad_state_machine = ( - VadStateMachine.kVadInStateEndPointDetected - ) - elif not is_final_frame: - self.OnVoiceDetected(cur_frm_idx, cache=cache) - else: - self.MaybeOnVoiceEndIfLastFrame( - is_final_frame, cur_frm_idx, cache=cache - ) - else: - pass - elif AudioChangeState.kChangeStateSpeech2Sil == state_change: - cache["stats"].continous_silence_frame_count = 0 - if ( - cache["stats"].vad_state_machine - == VadStateMachine.kVadInStateStartPointNotDetected - ): - pass - elif ( - cache["stats"].vad_state_machine - == VadStateMachine.kVadInStateInSpeechSegment - ): - if ( - cur_frm_idx - cache["stats"].confirmed_start_frame + 1 - > self.vad_opts.max_single_segment_time / frm_shift_in_ms - ): - self.OnVoiceEnd(cur_frm_idx, False, False, cache=cache) - cache["stats"].vad_state_machine = ( - VadStateMachine.kVadInStateEndPointDetected - ) - elif not is_final_frame: - self.OnVoiceDetected(cur_frm_idx, cache=cache) - else: - self.MaybeOnVoiceEndIfLastFrame( - is_final_frame, cur_frm_idx, cache=cache - ) - else: - pass - elif AudioChangeState.kChangeStateSpeech2Speech == state_change: - cache["stats"].continous_silence_frame_count = 0 - if ( - cache["stats"].vad_state_machine - == VadStateMachine.kVadInStateInSpeechSegment - ): - if ( - cur_frm_idx - cache["stats"].confirmed_start_frame + 1 - > self.vad_opts.max_single_segment_time / frm_shift_in_ms - ): - cache["stats"].max_time_out = True - self.OnVoiceEnd(cur_frm_idx, False, False, cache=cache) - cache["stats"].vad_state_machine = ( - VadStateMachine.kVadInStateEndPointDetected - ) - elif not is_final_frame: - self.OnVoiceDetected(cur_frm_idx, cache=cache) - else: - self.MaybeOnVoiceEndIfLastFrame( - is_final_frame, cur_frm_idx, cache=cache - ) - else: - pass - elif AudioChangeState.kChangeStateSil2Sil == state_change: - cache["stats"].continous_silence_frame_count += 1 - if ( - cache["stats"].vad_state_machine - == VadStateMachine.kVadInStateStartPointNotDetected - ): - # silence timeout, return zero length decision - if ( - ( - self.vad_opts.detect_mode - == VadDetectMode.kVadSingleUtteranceDetectMode.value - ) - and ( - cache["stats"].continous_silence_frame_count * frm_shift_in_ms - > self.vad_opts.max_start_silence_time - ) - ) or (is_final_frame and cache["stats"].number_end_time_detected == 0): - for t in range( - cache["stats"].lastest_confirmed_silence_frame + 1, cur_frm_idx - ): - self.OnSilenceDetected(t, cache=cache) - self.OnVoiceStart(0, True, cache=cache) - self.OnVoiceEnd(0, True, False, cache=cache) - cache["stats"].vad_state_machine = ( - VadStateMachine.kVadInStateEndPointDetected - ) - else: - if cur_frm_idx >= self.LatencyFrmNumAtStartPoint(cache=cache): - self.OnSilenceDetected( - cur_frm_idx - self.LatencyFrmNumAtStartPoint(cache=cache), - cache=cache, - ) - elif ( - cache["stats"].vad_state_machine - == VadStateMachine.kVadInStateInSpeechSegment - ): - if ( - cache["stats"].continous_silence_frame_count * frm_shift_in_ms - >= cache["stats"].max_end_sil_frame_cnt_thresh - ): - lookback_frame = int( - cache["stats"].max_end_sil_frame_cnt_thresh / frm_shift_in_ms - ) - if self.vad_opts.do_extend: - lookback_frame -= int( - self.vad_opts.lookahead_time_end_point / frm_shift_in_ms - ) - lookback_frame -= 1 - lookback_frame = max(0, lookback_frame) - self.OnVoiceEnd( - cur_frm_idx - lookback_frame, False, False, cache=cache - ) - cache["stats"].vad_state_machine = ( - VadStateMachine.kVadInStateEndPointDetected - ) - elif ( - cur_frm_idx - cache["stats"].confirmed_start_frame + 1 - > self.vad_opts.max_single_segment_time / frm_shift_in_ms - ): - self.OnVoiceEnd(cur_frm_idx, False, False, cache=cache) - cache["stats"].vad_state_machine = ( - VadStateMachine.kVadInStateEndPointDetected - ) - elif self.vad_opts.do_extend and not is_final_frame: - if cache["stats"].continous_silence_frame_count <= int( - self.vad_opts.lookahead_time_end_point / frm_shift_in_ms - ): - self.OnVoiceDetected(cur_frm_idx, cache=cache) - else: - self.MaybeOnVoiceEndIfLastFrame( - is_final_frame, cur_frm_idx, cache=cache - ) - else: - pass - - if ( - cache["stats"].vad_state_machine - == VadStateMachine.kVadInStateEndPointDetected - and self.vad_opts.detect_mode - == VadDetectMode.kVadMutipleUtteranceDetectMode.value - ): - self.ResetDetection(cache=cache) diff --git a/funasr_detach/models/fsmn_vad_streaming/template.yaml b/funasr_detach/models/fsmn_vad_streaming/template.yaml deleted file mode 100644 index e8a3a4f30d823305e9b698a44b695cbcb33feea3..0000000000000000000000000000000000000000 --- a/funasr_detach/models/fsmn_vad_streaming/template.yaml +++ /dev/null @@ -1,62 +0,0 @@ -# This is an example that demonstrates how to configure a model file. -# You can modify the configuration according to your own requirements. - -# to print the register_table: -# from funasr.register import tables -# tables.print() - -# network architecture -model: FsmnVADStreaming -model_conf: - sample_rate: 16000 - detect_mode: 1 - snr_mode: 0 - max_end_silence_time: 800 - max_start_silence_time: 3000 - do_start_point_detection: True - do_end_point_detection: True - window_size_ms: 200 - sil_to_speech_time_thres: 150 - speech_to_sil_time_thres: 150 - speech_2_noise_ratio: 1.0 - do_extend: 1 - lookback_time_start_point: 200 - lookahead_time_end_point: 100 - max_single_segment_time: 60000 - snr_thres: -100.0 - noise_frame_num_used_for_snr: 100 - decibel_thres: -100.0 - speech_noise_thres: 0.6 - fe_prior_thres: 0.0001 - silence_pdf_num: 1 - sil_pdf_ids: [0] - speech_noise_thresh_low: -0.1 - speech_noise_thresh_high: 0.3 - output_frame_probs: False - frame_in_ms: 10 - frame_length_ms: 25 - -encoder: FSMN -encoder_conf: - input_dim: 400 - input_affine_dim: 140 - fsmn_layers: 4 - linear_dim: 250 - proj_dim: 128 - lorder: 20 - rorder: 0 - lstride: 1 - rstride: 0 - output_affine_dim: 140 - output_dim: 248 - -frontend: WavFrontend -frontend_conf: - fs: 16000 - window: hamming - n_mels: 80 - frame_length: 25 - frame_shift: 10 - dither: 0.0 - lfr_m: 5 - lfr_n: 1 diff --git a/funasr_detach/models/language_model/__init__.py b/funasr_detach/models/language_model/__init__.py deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/funasr_detach/models/language_model/rnn/__init__.py b/funasr_detach/models/language_model/rnn/__init__.py deleted file mode 100644 index b7f177368e62a5578b8706300e101f831a3972ac..0000000000000000000000000000000000000000 --- a/funasr_detach/models/language_model/rnn/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Initialize sub package.""" diff --git a/funasr_detach/models/language_model/rnn/argument.py b/funasr_detach/models/language_model/rnn/argument.py deleted file mode 100644 index b4c89d25f52882f0c99ec3e8c8a182e3b6dc5ee7..0000000000000000000000000000000000000000 --- a/funasr_detach/models/language_model/rnn/argument.py +++ /dev/null @@ -1,156 +0,0 @@ -# Copyright 2020 Hirofumi Inaguma -# Apache 2.0 (http://www.apache.org/licenses/LICENSE-2.0) - -"""Conformer common arguments.""" - - -def add_arguments_rnn_encoder_common(group): - """Define common arguments for RNN encoder.""" - group.add_argument( - "--etype", - default="blstmp", - type=str, - choices=[ - "lstm", - "blstm", - "lstmp", - "blstmp", - "vgglstmp", - "vggblstmp", - "vgglstm", - "vggblstm", - "gru", - "bgru", - "grup", - "bgrup", - "vgggrup", - "vggbgrup", - "vgggru", - "vggbgru", - ], - help="Type of encoder network architecture", - ) - group.add_argument( - "--elayers", - default=4, - type=int, - help="Number of encoder layers", - ) - group.add_argument( - "--eunits", - "-u", - default=300, - type=int, - help="Number of encoder hidden units", - ) - group.add_argument( - "--eprojs", default=320, type=int, help="Number of encoder projection units" - ) - group.add_argument( - "--subsample", - default="1", - type=str, - help="Subsample input frames x_y_z means " - "subsample every x frame at 1st layer, " - "every y frame at 2nd layer etc.", - ) - return group - - -def add_arguments_rnn_decoder_common(group): - """Define common arguments for RNN decoder.""" - group.add_argument( - "--dtype", - default="lstm", - type=str, - choices=["lstm", "gru"], - help="Type of decoder network architecture", - ) - group.add_argument( - "--dlayers", default=1, type=int, help="Number of decoder layers" - ) - group.add_argument( - "--dunits", default=320, type=int, help="Number of decoder hidden units" - ) - group.add_argument( - "--dropout-rate-decoder", - default=0.0, - type=float, - help="Dropout rate for the decoder", - ) - group.add_argument( - "--sampling-probability", - default=0.0, - type=float, - help="Ratio of predicted labels fed back to decoder", - ) - group.add_argument( - "--lsm-type", - const="", - default="", - type=str, - nargs="?", - choices=["", "unigram"], - help="Apply label smoothing with a specified distribution type", - ) - return group - - -def add_arguments_rnn_attention_common(group): - """Define common arguments for RNN attention.""" - group.add_argument( - "--atype", - default="dot", - type=str, - choices=[ - "noatt", - "dot", - "add", - "location", - "coverage", - "coverage_location", - "location2d", - "location_recurrent", - "multi_head_dot", - "multi_head_add", - "multi_head_loc", - "multi_head_multi_res_loc", - ], - help="Type of attention architecture", - ) - group.add_argument( - "--adim", - default=320, - type=int, - help="Number of attention transformation dimensions", - ) - group.add_argument( - "--awin", default=5, type=int, help="Window size for location2d attention" - ) - group.add_argument( - "--aheads", - default=4, - type=int, - help="Number of heads for multi head attention", - ) - group.add_argument( - "--aconv-chans", - default=-1, - type=int, - help="Number of attention convolution channels \ - (negative value indicates no location-aware attention)", - ) - group.add_argument( - "--aconv-filts", - default=100, - type=int, - help="Number of attention convolution filters \ - (negative value indicates no location-aware attention)", - ) - group.add_argument( - "--dropout-rate", - default=0.0, - type=float, - help="Dropout rate for the encoder", - ) - return group diff --git a/funasr_detach/models/language_model/rnn/attentions.py b/funasr_detach/models/language_model/rnn/attentions.py deleted file mode 100644 index 0d0a29310cd340db066fb23eaed69e5cd461568e..0000000000000000000000000000000000000000 --- a/funasr_detach/models/language_model/rnn/attentions.py +++ /dev/null @@ -1,1808 +0,0 @@ -"""Attention modules for RNN.""" - -import math -import six - -import torch -import torch.nn.functional as F - -from funasr_detach.models.transformer.utils.nets_utils import make_pad_mask -from funasr_detach.models.transformer.utils.nets_utils import to_device - - -def _apply_attention_constraint( - e, last_attended_idx, backward_window=1, forward_window=3 -): - """Apply monotonic attention constraint. - - This function apply the monotonic attention constraint - introduced in `Deep Voice 3: Scaling - Text-to-Speech with Convolutional Sequence Learning`_. - - Args: - e (Tensor): Attention energy before applying softmax (1, T). - last_attended_idx (int): The index of the inputs of the last attended [0, T]. - backward_window (int, optional): Backward window size in attention constraint. - forward_window (int, optional): Forward window size in attetion constraint. - - Returns: - Tensor: Monotonic constrained attention energy (1, T). - - .. _`Deep Voice 3: Scaling Text-to-Speech with Convolutional Sequence Learning`: - https://arxiv.org/abs/1710.07654 - - """ - if e.size(0) != 1: - raise NotImplementedError("Batch attention constraining is not yet supported.") - backward_idx = last_attended_idx - backward_window - forward_idx = last_attended_idx + forward_window - if backward_idx > 0: - e[:, :backward_idx] = -float("inf") - if forward_idx < e.size(1): - e[:, forward_idx:] = -float("inf") - return e - - -class NoAtt(torch.nn.Module): - """No attention""" - - def __init__(self): - super(NoAtt, self).__init__() - self.h_length = None - self.enc_h = None - self.pre_compute_enc_h = None - self.c = None - - def reset(self): - """reset states""" - self.h_length = None - self.enc_h = None - self.pre_compute_enc_h = None - self.c = None - - def forward(self, enc_hs_pad, enc_hs_len, dec_z, att_prev): - """NoAtt forward - - :param torch.Tensor enc_hs_pad: padded encoder hidden state (B, T_max, D_enc) - :param list enc_hs_len: padded encoder hidden state length (B) - :param torch.Tensor dec_z: dummy (does not use) - :param torch.Tensor att_prev: dummy (does not use) - :return: attention weighted encoder state (B, D_enc) - :rtype: torch.Tensor - :return: previous attention weights - :rtype: torch.Tensor - """ - batch = len(enc_hs_pad) - # pre-compute all h outside the decoder loop - if self.pre_compute_enc_h is None: - self.enc_h = enc_hs_pad # utt x frame x hdim - self.h_length = self.enc_h.size(1) - - # initialize attention weight with uniform dist. - if att_prev is None: - # if no bias, 0 0-pad goes 0 - mask = 1.0 - make_pad_mask(enc_hs_len).float() - att_prev = mask / mask.new(enc_hs_len).unsqueeze(-1) - att_prev = att_prev.to(self.enc_h) - self.c = torch.sum( - self.enc_h * att_prev.view(batch, self.h_length, 1), dim=1 - ) - - return self.c, att_prev - - -class AttDot(torch.nn.Module): - """Dot product attention - - :param int eprojs: # projection-units of encoder - :param int dunits: # units of decoder - :param int att_dim: attention dimension - :param bool han_mode: flag to swith on mode of hierarchical attention - and not store pre_compute_enc_h - """ - - def __init__(self, eprojs, dunits, att_dim, han_mode=False): - super(AttDot, self).__init__() - self.mlp_enc = torch.nn.Linear(eprojs, att_dim) - self.mlp_dec = torch.nn.Linear(dunits, att_dim) - - self.dunits = dunits - self.eprojs = eprojs - self.att_dim = att_dim - self.h_length = None - self.enc_h = None - self.pre_compute_enc_h = None - self.mask = None - self.han_mode = han_mode - - def reset(self): - """reset states""" - self.h_length = None - self.enc_h = None - self.pre_compute_enc_h = None - self.mask = None - - def forward(self, enc_hs_pad, enc_hs_len, dec_z, att_prev, scaling=2.0): - """AttDot forward - - :param torch.Tensor enc_hs_pad: padded encoder hidden state (B x T_max x D_enc) - :param list enc_hs_len: padded encoder hidden state length (B) - :param torch.Tensor dec_z: dummy (does not use) - :param torch.Tensor att_prev: dummy (does not use) - :param float scaling: scaling parameter before applying softmax - :return: attention weighted encoder state (B, D_enc) - :rtype: torch.Tensor - :return: previous attention weight (B x T_max) - :rtype: torch.Tensor - """ - - batch = enc_hs_pad.size(0) - # pre-compute all h outside the decoder loop - if self.pre_compute_enc_h is None or self.han_mode: - self.enc_h = enc_hs_pad # utt x frame x hdim - self.h_length = self.enc_h.size(1) - # utt x frame x att_dim - self.pre_compute_enc_h = torch.tanh(self.mlp_enc(self.enc_h)) - - if dec_z is None: - dec_z = enc_hs_pad.new_zeros(batch, self.dunits) - else: - dec_z = dec_z.view(batch, self.dunits) - - e = torch.sum( - self.pre_compute_enc_h - * torch.tanh(self.mlp_dec(dec_z)).view(batch, 1, self.att_dim), - dim=2, - ) # utt x frame - - # NOTE consider zero padding when compute w. - if self.mask is None: - self.mask = to_device(enc_hs_pad, make_pad_mask(enc_hs_len)) - e.masked_fill_(self.mask, -float("inf")) - w = F.softmax(scaling * e, dim=1) - - # weighted sum over flames - # utt x hdim - # NOTE use bmm instead of sum(*) - c = torch.sum(self.enc_h * w.view(batch, self.h_length, 1), dim=1) - return c, w - - -class AttAdd(torch.nn.Module): - """Additive attention - - :param int eprojs: # projection-units of encoder - :param int dunits: # units of decoder - :param int att_dim: attention dimension - :param bool han_mode: flag to swith on mode of hierarchical attention - and not store pre_compute_enc_h - """ - - def __init__(self, eprojs, dunits, att_dim, han_mode=False): - super(AttAdd, self).__init__() - self.mlp_enc = torch.nn.Linear(eprojs, att_dim) - self.mlp_dec = torch.nn.Linear(dunits, att_dim, bias=False) - self.gvec = torch.nn.Linear(att_dim, 1) - self.dunits = dunits - self.eprojs = eprojs - self.att_dim = att_dim - self.h_length = None - self.enc_h = None - self.pre_compute_enc_h = None - self.mask = None - self.han_mode = han_mode - - def reset(self): - """reset states""" - self.h_length = None - self.enc_h = None - self.pre_compute_enc_h = None - self.mask = None - - def forward(self, enc_hs_pad, enc_hs_len, dec_z, att_prev, scaling=2.0): - """AttAdd forward - - :param torch.Tensor enc_hs_pad: padded encoder hidden state (B x T_max x D_enc) - :param list enc_hs_len: padded encoder hidden state length (B) - :param torch.Tensor dec_z: decoder hidden state (B x D_dec) - :param torch.Tensor att_prev: dummy (does not use) - :param float scaling: scaling parameter before applying softmax - :return: attention weighted encoder state (B, D_enc) - :rtype: torch.Tensor - :return: previous attention weights (B x T_max) - :rtype: torch.Tensor - """ - - batch = len(enc_hs_pad) - # pre-compute all h outside the decoder loop - if self.pre_compute_enc_h is None or self.han_mode: - self.enc_h = enc_hs_pad # utt x frame x hdim - self.h_length = self.enc_h.size(1) - # utt x frame x att_dim - self.pre_compute_enc_h = self.mlp_enc(self.enc_h) - - if dec_z is None: - dec_z = enc_hs_pad.new_zeros(batch, self.dunits) - else: - dec_z = dec_z.view(batch, self.dunits) - - # dec_z_tiled: utt x frame x att_dim - dec_z_tiled = self.mlp_dec(dec_z).view(batch, 1, self.att_dim) - - # dot with gvec - # utt x frame x att_dim -> utt x frame - e = self.gvec(torch.tanh(self.pre_compute_enc_h + dec_z_tiled)).squeeze(2) - - # NOTE consider zero padding when compute w. - if self.mask is None: - self.mask = to_device(enc_hs_pad, make_pad_mask(enc_hs_len)) - e.masked_fill_(self.mask, -float("inf")) - w = F.softmax(scaling * e, dim=1) - - # weighted sum over flames - # utt x hdim - # NOTE use bmm instead of sum(*) - c = torch.sum(self.enc_h * w.view(batch, self.h_length, 1), dim=1) - - return c, w - - -class AttLoc(torch.nn.Module): - """location-aware attention module. - - Reference: Attention-Based Models for Speech Recognition - (https://arxiv.org/pdf/1506.07503.pdf) - - :param int eprojs: # projection-units of encoder - :param int dunits: # units of decoder - :param int att_dim: attention dimension - :param int aconv_chans: # channels of attention convolution - :param int aconv_filts: filter size of attention convolution - :param bool han_mode: flag to swith on mode of hierarchical attention - and not store pre_compute_enc_h - """ - - def __init__( - self, eprojs, dunits, att_dim, aconv_chans, aconv_filts, han_mode=False - ): - super(AttLoc, self).__init__() - self.mlp_enc = torch.nn.Linear(eprojs, att_dim) - self.mlp_dec = torch.nn.Linear(dunits, att_dim, bias=False) - self.mlp_att = torch.nn.Linear(aconv_chans, att_dim, bias=False) - self.loc_conv = torch.nn.Conv2d( - 1, - aconv_chans, - (1, 2 * aconv_filts + 1), - padding=(0, aconv_filts), - bias=False, - ) - self.gvec = torch.nn.Linear(att_dim, 1) - - self.dunits = dunits - self.eprojs = eprojs - self.att_dim = att_dim - self.h_length = None - self.enc_h = None - self.pre_compute_enc_h = None - self.mask = None - self.han_mode = han_mode - - def reset(self): - """reset states""" - self.h_length = None - self.enc_h = None - self.pre_compute_enc_h = None - self.mask = None - - def forward( - self, - enc_hs_pad, - enc_hs_len, - dec_z, - att_prev, - scaling=2.0, - last_attended_idx=None, - backward_window=1, - forward_window=3, - ): - """Calculate AttLoc forward propagation. - - :param torch.Tensor enc_hs_pad: padded encoder hidden state (B x T_max x D_enc) - :param list enc_hs_len: padded encoder hidden state length (B) - :param torch.Tensor dec_z: decoder hidden state (B x D_dec) - :param torch.Tensor att_prev: previous attention weight (B x T_max) - :param float scaling: scaling parameter before applying softmax - :param torch.Tensor forward_window: - forward window size when constraining attention - :param int last_attended_idx: index of the inputs of the last attended - :param int backward_window: backward window size in attention constraint - :param int forward_window: forward window size in attetion constraint - :return: attention weighted encoder state (B, D_enc) - :rtype: torch.Tensor - :return: previous attention weights (B x T_max) - :rtype: torch.Tensor - """ - batch = len(enc_hs_pad) - # pre-compute all h outside the decoder loop - if self.pre_compute_enc_h is None or self.han_mode: - self.enc_h = enc_hs_pad # utt x frame x hdim - self.h_length = self.enc_h.size(1) - # utt x frame x att_dim - self.pre_compute_enc_h = self.mlp_enc(self.enc_h) - - if dec_z is None: - dec_z = enc_hs_pad.new_zeros(batch, self.dunits) - else: - dec_z = dec_z.view(batch, self.dunits) - - # initialize attention weight with uniform dist. - if att_prev is None: - # if no bias, 0 0-pad goes 0 - att_prev = 1.0 - make_pad_mask(enc_hs_len).to( - device=dec_z.device, dtype=dec_z.dtype - ) - att_prev = att_prev / att_prev.new(enc_hs_len).unsqueeze(-1) - - # att_prev: utt x frame -> utt x 1 x 1 x frame - # -> utt x att_conv_chans x 1 x frame - att_conv = self.loc_conv(att_prev.view(batch, 1, 1, self.h_length)) - # att_conv: utt x att_conv_chans x 1 x frame -> utt x frame x att_conv_chans - att_conv = att_conv.squeeze(2).transpose(1, 2) - # att_conv: utt x frame x att_conv_chans -> utt x frame x att_dim - att_conv = self.mlp_att(att_conv) - - # dec_z_tiled: utt x frame x att_dim - dec_z_tiled = self.mlp_dec(dec_z).view(batch, 1, self.att_dim) - - # dot with gvec - # utt x frame x att_dim -> utt x frame - e = self.gvec( - torch.tanh(att_conv + self.pre_compute_enc_h + dec_z_tiled) - ).squeeze(2) - - # NOTE: consider zero padding when compute w. - if self.mask is None: - self.mask = to_device(enc_hs_pad, make_pad_mask(enc_hs_len)) - e.masked_fill_(self.mask, -float("inf")) - - # apply monotonic attention constraint (mainly for TTS) - if last_attended_idx is not None: - e = _apply_attention_constraint( - e, last_attended_idx, backward_window, forward_window - ) - - w = F.softmax(scaling * e, dim=1) - - # weighted sum over flames - # utt x hdim - c = torch.sum(self.enc_h * w.view(batch, self.h_length, 1), dim=1) - - return c, w - - -class AttCov(torch.nn.Module): - """Coverage mechanism attention - - Reference: Get To The Point: Summarization with Pointer-Generator Network - (https://arxiv.org/abs/1704.04368) - - :param int eprojs: # projection-units of encoder - :param int dunits: # units of decoder - :param int att_dim: attention dimension - :param bool han_mode: flag to swith on mode of hierarchical attention - and not store pre_compute_enc_h - """ - - def __init__(self, eprojs, dunits, att_dim, han_mode=False): - super(AttCov, self).__init__() - self.mlp_enc = torch.nn.Linear(eprojs, att_dim) - self.mlp_dec = torch.nn.Linear(dunits, att_dim, bias=False) - self.wvec = torch.nn.Linear(1, att_dim) - self.gvec = torch.nn.Linear(att_dim, 1) - - self.dunits = dunits - self.eprojs = eprojs - self.att_dim = att_dim - self.h_length = None - self.enc_h = None - self.pre_compute_enc_h = None - self.mask = None - self.han_mode = han_mode - - def reset(self): - """reset states""" - self.h_length = None - self.enc_h = None - self.pre_compute_enc_h = None - self.mask = None - - def forward(self, enc_hs_pad, enc_hs_len, dec_z, att_prev_list, scaling=2.0): - """AttCov forward - - :param torch.Tensor enc_hs_pad: padded encoder hidden state (B x T_max x D_enc) - :param list enc_hs_len: padded encoder hidden state length (B) - :param torch.Tensor dec_z: decoder hidden state (B x D_dec) - :param list att_prev_list: list of previous attention weight - :param float scaling: scaling parameter before applying softmax - :return: attention weighted encoder state (B, D_enc) - :rtype: torch.Tensor - :return: list of previous attention weights - :rtype: list - """ - - batch = len(enc_hs_pad) - # pre-compute all h outside the decoder loop - if self.pre_compute_enc_h is None or self.han_mode: - self.enc_h = enc_hs_pad # utt x frame x hdim - self.h_length = self.enc_h.size(1) - # utt x frame x att_dim - self.pre_compute_enc_h = self.mlp_enc(self.enc_h) - - if dec_z is None: - dec_z = enc_hs_pad.new_zeros(batch, self.dunits) - else: - dec_z = dec_z.view(batch, self.dunits) - - # initialize attention weight with uniform dist. - if att_prev_list is None: - # if no bias, 0 0-pad goes 0 - att_prev_list = to_device( - enc_hs_pad, (1.0 - make_pad_mask(enc_hs_len).float()) - ) - att_prev_list = [ - att_prev_list / att_prev_list.new(enc_hs_len).unsqueeze(-1) - ] - - # att_prev_list: L' * [B x T] => cov_vec B x T - cov_vec = sum(att_prev_list) - # cov_vec: B x T => B x T x 1 => B x T x att_dim - cov_vec = self.wvec(cov_vec.unsqueeze(-1)) - - # dec_z_tiled: utt x frame x att_dim - dec_z_tiled = self.mlp_dec(dec_z).view(batch, 1, self.att_dim) - - # dot with gvec - # utt x frame x att_dim -> utt x frame - e = self.gvec( - torch.tanh(cov_vec + self.pre_compute_enc_h + dec_z_tiled) - ).squeeze(2) - - # NOTE consider zero padding when compute w. - if self.mask is None: - self.mask = to_device(enc_hs_pad, make_pad_mask(enc_hs_len)) - e.masked_fill_(self.mask, -float("inf")) - w = F.softmax(scaling * e, dim=1) - att_prev_list += [w] - - # weighted sum over flames - # utt x hdim - # NOTE use bmm instead of sum(*) - c = torch.sum(self.enc_h * w.view(batch, self.h_length, 1), dim=1) - - return c, att_prev_list - - -class AttLoc2D(torch.nn.Module): - """2D location-aware attention - - This attention is an extended version of location aware attention. - It take not only one frame before attention weights, - but also earlier frames into account. - - :param int eprojs: # projection-units of encoder - :param int dunits: # units of decoder - :param int att_dim: attention dimension - :param int aconv_chans: # channels of attention convolution - :param int aconv_filts: filter size of attention convolution - :param int att_win: attention window size (default=5) - :param bool han_mode: - flag to swith on mode of hierarchical attention and not store pre_compute_enc_h - """ - - def __init__( - self, eprojs, dunits, att_dim, att_win, aconv_chans, aconv_filts, han_mode=False - ): - super(AttLoc2D, self).__init__() - self.mlp_enc = torch.nn.Linear(eprojs, att_dim) - self.mlp_dec = torch.nn.Linear(dunits, att_dim, bias=False) - self.mlp_att = torch.nn.Linear(aconv_chans, att_dim, bias=False) - self.loc_conv = torch.nn.Conv2d( - 1, - aconv_chans, - (att_win, 2 * aconv_filts + 1), - padding=(0, aconv_filts), - bias=False, - ) - self.gvec = torch.nn.Linear(att_dim, 1) - - self.dunits = dunits - self.eprojs = eprojs - self.att_dim = att_dim - self.h_length = None - self.enc_h = None - self.pre_compute_enc_h = None - self.aconv_chans = aconv_chans - self.att_win = att_win - self.mask = None - self.han_mode = han_mode - - def reset(self): - """reset states""" - self.h_length = None - self.enc_h = None - self.pre_compute_enc_h = None - self.mask = None - - def forward(self, enc_hs_pad, enc_hs_len, dec_z, att_prev, scaling=2.0): - """AttLoc2D forward - - :param torch.Tensor enc_hs_pad: padded encoder hidden state (B x T_max x D_enc) - :param list enc_hs_len: padded encoder hidden state length (B) - :param torch.Tensor dec_z: decoder hidden state (B x D_dec) - :param torch.Tensor att_prev: previous attention weight (B x att_win x T_max) - :param float scaling: scaling parameter before applying softmax - :return: attention weighted encoder state (B, D_enc) - :rtype: torch.Tensor - :return: previous attention weights (B x att_win x T_max) - :rtype: torch.Tensor - """ - - batch = len(enc_hs_pad) - # pre-compute all h outside the decoder loop - if self.pre_compute_enc_h is None or self.han_mode: - self.enc_h = enc_hs_pad # utt x frame x hdim - self.h_length = self.enc_h.size(1) - # utt x frame x att_dim - self.pre_compute_enc_h = self.mlp_enc(self.enc_h) - - if dec_z is None: - dec_z = enc_hs_pad.new_zeros(batch, self.dunits) - else: - dec_z = dec_z.view(batch, self.dunits) - - # initialize attention weight with uniform dist. - if att_prev is None: - # B * [Li x att_win] - # if no bias, 0 0-pad goes 0 - att_prev = to_device(enc_hs_pad, (1.0 - make_pad_mask(enc_hs_len).float())) - att_prev = att_prev / att_prev.new(enc_hs_len).unsqueeze(-1) - att_prev = att_prev.unsqueeze(1).expand(-1, self.att_win, -1) - - # att_prev: B x att_win x Tmax -> B x 1 x att_win x Tmax -> B x C x 1 x Tmax - att_conv = self.loc_conv(att_prev.unsqueeze(1)) - # att_conv: B x C x 1 x Tmax -> B x Tmax x C - att_conv = att_conv.squeeze(2).transpose(1, 2) - # att_conv: utt x frame x att_conv_chans -> utt x frame x att_dim - att_conv = self.mlp_att(att_conv) - - # dec_z_tiled: utt x frame x att_dim - dec_z_tiled = self.mlp_dec(dec_z).view(batch, 1, self.att_dim) - - # dot with gvec - # utt x frame x att_dim -> utt x frame - e = self.gvec( - torch.tanh(att_conv + self.pre_compute_enc_h + dec_z_tiled) - ).squeeze(2) - - # NOTE consider zero padding when compute w. - if self.mask is None: - self.mask = to_device(enc_hs_pad, make_pad_mask(enc_hs_len)) - e.masked_fill_(self.mask, -float("inf")) - w = F.softmax(scaling * e, dim=1) - - # weighted sum over flames - # utt x hdim - # NOTE use bmm instead of sum(*) - c = torch.sum(self.enc_h * w.view(batch, self.h_length, 1), dim=1) - - # update att_prev: B x att_win x Tmax -> B x att_win+1 x Tmax - # -> B x att_win x Tmax - att_prev = torch.cat([att_prev, w.unsqueeze(1)], dim=1) - att_prev = att_prev[:, 1:] - - return c, att_prev - - -class AttLocRec(torch.nn.Module): - """location-aware recurrent attention - - This attention is an extended version of location aware attention. - With the use of RNN, - it take the effect of the history of attention weights into account. - - :param int eprojs: # projection-units of encoder - :param int dunits: # units of decoder - :param int att_dim: attention dimension - :param int aconv_chans: # channels of attention convolution - :param int aconv_filts: filter size of attention convolution - :param bool han_mode: - flag to swith on mode of hierarchical attention and not store pre_compute_enc_h - """ - - def __init__( - self, eprojs, dunits, att_dim, aconv_chans, aconv_filts, han_mode=False - ): - super(AttLocRec, self).__init__() - self.mlp_enc = torch.nn.Linear(eprojs, att_dim) - self.mlp_dec = torch.nn.Linear(dunits, att_dim, bias=False) - self.loc_conv = torch.nn.Conv2d( - 1, - aconv_chans, - (1, 2 * aconv_filts + 1), - padding=(0, aconv_filts), - bias=False, - ) - self.att_lstm = torch.nn.LSTMCell(aconv_chans, att_dim, bias=False) - self.gvec = torch.nn.Linear(att_dim, 1) - - self.dunits = dunits - self.eprojs = eprojs - self.att_dim = att_dim - self.h_length = None - self.enc_h = None - self.pre_compute_enc_h = None - self.mask = None - self.han_mode = han_mode - - def reset(self): - """reset states""" - self.h_length = None - self.enc_h = None - self.pre_compute_enc_h = None - self.mask = None - - def forward(self, enc_hs_pad, enc_hs_len, dec_z, att_prev_states, scaling=2.0): - """AttLocRec forward - - :param torch.Tensor enc_hs_pad: padded encoder hidden state (B x T_max x D_enc) - :param list enc_hs_len: padded encoder hidden state length (B) - :param torch.Tensor dec_z: decoder hidden state (B x D_dec) - :param tuple att_prev_states: previous attention weight and lstm states - ((B, T_max), ((B, att_dim), (B, att_dim))) - :param float scaling: scaling parameter before applying softmax - :return: attention weighted encoder state (B, D_enc) - :rtype: torch.Tensor - :return: previous attention weights and lstm states (w, (hx, cx)) - ((B, T_max), ((B, att_dim), (B, att_dim))) - :rtype: tuple - """ - - batch = len(enc_hs_pad) - # pre-compute all h outside the decoder loop - if self.pre_compute_enc_h is None or self.han_mode: - self.enc_h = enc_hs_pad # utt x frame x hdim - self.h_length = self.enc_h.size(1) - # utt x frame x att_dim - self.pre_compute_enc_h = self.mlp_enc(self.enc_h) - - if dec_z is None: - dec_z = enc_hs_pad.new_zeros(batch, self.dunits) - else: - dec_z = dec_z.view(batch, self.dunits) - - if att_prev_states is None: - # initialize attention weight with uniform dist. - # if no bias, 0 0-pad goes 0 - att_prev = to_device(enc_hs_pad, (1.0 - make_pad_mask(enc_hs_len).float())) - att_prev = att_prev / att_prev.new(enc_hs_len).unsqueeze(-1) - - # initialize lstm states - att_h = enc_hs_pad.new_zeros(batch, self.att_dim) - att_c = enc_hs_pad.new_zeros(batch, self.att_dim) - att_states = (att_h, att_c) - else: - att_prev = att_prev_states[0] - att_states = att_prev_states[1] - - # B x 1 x 1 x T -> B x C x 1 x T - att_conv = self.loc_conv(att_prev.view(batch, 1, 1, self.h_length)) - # apply non-linear - att_conv = F.relu(att_conv) - # B x C x 1 x T -> B x C x 1 x 1 -> B x C - att_conv = F.max_pool2d(att_conv, (1, att_conv.size(3))).view(batch, -1) - - att_h, att_c = self.att_lstm(att_conv, att_states) - - # dec_z_tiled: utt x frame x att_dim - dec_z_tiled = self.mlp_dec(dec_z).view(batch, 1, self.att_dim) - - # dot with gvec - # utt x frame x att_dim -> utt x frame - e = self.gvec( - torch.tanh(att_h.unsqueeze(1) + self.pre_compute_enc_h + dec_z_tiled) - ).squeeze(2) - - # NOTE consider zero padding when compute w. - if self.mask is None: - self.mask = to_device(enc_hs_pad, make_pad_mask(enc_hs_len)) - e.masked_fill_(self.mask, -float("inf")) - w = F.softmax(scaling * e, dim=1) - - # weighted sum over flames - # utt x hdim - # NOTE use bmm instead of sum(*) - c = torch.sum(self.enc_h * w.view(batch, self.h_length, 1), dim=1) - - return c, (w, (att_h, att_c)) - - -class AttCovLoc(torch.nn.Module): - """Coverage mechanism location aware attention - - This attention is a combination of coverage and location-aware attentions. - - :param int eprojs: # projection-units of encoder - :param int dunits: # units of decoder - :param int att_dim: attention dimension - :param int aconv_chans: # channels of attention convolution - :param int aconv_filts: filter size of attention convolution - :param bool han_mode: - flag to swith on mode of hierarchical attention and not store pre_compute_enc_h - """ - - def __init__( - self, eprojs, dunits, att_dim, aconv_chans, aconv_filts, han_mode=False - ): - super(AttCovLoc, self).__init__() - self.mlp_enc = torch.nn.Linear(eprojs, att_dim) - self.mlp_dec = torch.nn.Linear(dunits, att_dim, bias=False) - self.mlp_att = torch.nn.Linear(aconv_chans, att_dim, bias=False) - self.loc_conv = torch.nn.Conv2d( - 1, - aconv_chans, - (1, 2 * aconv_filts + 1), - padding=(0, aconv_filts), - bias=False, - ) - self.gvec = torch.nn.Linear(att_dim, 1) - - self.dunits = dunits - self.eprojs = eprojs - self.att_dim = att_dim - self.h_length = None - self.enc_h = None - self.pre_compute_enc_h = None - self.aconv_chans = aconv_chans - self.mask = None - self.han_mode = han_mode - - def reset(self): - """reset states""" - self.h_length = None - self.enc_h = None - self.pre_compute_enc_h = None - self.mask = None - - def forward(self, enc_hs_pad, enc_hs_len, dec_z, att_prev_list, scaling=2.0): - """AttCovLoc forward - - :param torch.Tensor enc_hs_pad: padded encoder hidden state (B x T_max x D_enc) - :param list enc_hs_len: padded encoder hidden state length (B) - :param torch.Tensor dec_z: decoder hidden state (B x D_dec) - :param list att_prev_list: list of previous attention weight - :param float scaling: scaling parameter before applying softmax - :return: attention weighted encoder state (B, D_enc) - :rtype: torch.Tensor - :return: list of previous attention weights - :rtype: list - """ - - batch = len(enc_hs_pad) - # pre-compute all h outside the decoder loop - if self.pre_compute_enc_h is None or self.han_mode: - self.enc_h = enc_hs_pad # utt x frame x hdim - self.h_length = self.enc_h.size(1) - # utt x frame x att_dim - self.pre_compute_enc_h = self.mlp_enc(self.enc_h) - - if dec_z is None: - dec_z = enc_hs_pad.new_zeros(batch, self.dunits) - else: - dec_z = dec_z.view(batch, self.dunits) - - # initialize attention weight with uniform dist. - if att_prev_list is None: - # if no bias, 0 0-pad goes 0 - mask = 1.0 - make_pad_mask(enc_hs_len).float() - att_prev_list = [ - to_device(enc_hs_pad, mask / mask.new(enc_hs_len).unsqueeze(-1)) - ] - - # att_prev_list: L' * [B x T] => cov_vec B x T - cov_vec = sum(att_prev_list) - - # cov_vec: B x T -> B x 1 x 1 x T -> B x C x 1 x T - att_conv = self.loc_conv(cov_vec.view(batch, 1, 1, self.h_length)) - # att_conv: utt x att_conv_chans x 1 x frame -> utt x frame x att_conv_chans - att_conv = att_conv.squeeze(2).transpose(1, 2) - # att_conv: utt x frame x att_conv_chans -> utt x frame x att_dim - att_conv = self.mlp_att(att_conv) - - # dec_z_tiled: utt x frame x att_dim - dec_z_tiled = self.mlp_dec(dec_z).view(batch, 1, self.att_dim) - - # dot with gvec - # utt x frame x att_dim -> utt x frame - e = self.gvec( - torch.tanh(att_conv + self.pre_compute_enc_h + dec_z_tiled) - ).squeeze(2) - - # NOTE consider zero padding when compute w. - if self.mask is None: - self.mask = to_device(enc_hs_pad, make_pad_mask(enc_hs_len)) - e.masked_fill_(self.mask, -float("inf")) - w = F.softmax(scaling * e, dim=1) - att_prev_list += [w] - - # weighted sum over flames - # utt x hdim - # NOTE use bmm instead of sum(*) - c = torch.sum(self.enc_h * w.view(batch, self.h_length, 1), dim=1) - - return c, att_prev_list - - -class AttMultiHeadDot(torch.nn.Module): - """Multi head dot product attention - - Reference: Attention is all you need - (https://arxiv.org/abs/1706.03762) - - :param int eprojs: # projection-units of encoder - :param int dunits: # units of decoder - :param int aheads: # heads of multi head attention - :param int att_dim_k: dimension k in multi head attention - :param int att_dim_v: dimension v in multi head attention - :param bool han_mode: flag to swith on mode of hierarchical attention - and not store pre_compute_k and pre_compute_v - """ - - def __init__(self, eprojs, dunits, aheads, att_dim_k, att_dim_v, han_mode=False): - super(AttMultiHeadDot, self).__init__() - self.mlp_q = torch.nn.ModuleList() - self.mlp_k = torch.nn.ModuleList() - self.mlp_v = torch.nn.ModuleList() - for _ in six.moves.range(aheads): - self.mlp_q += [torch.nn.Linear(dunits, att_dim_k)] - self.mlp_k += [torch.nn.Linear(eprojs, att_dim_k, bias=False)] - self.mlp_v += [torch.nn.Linear(eprojs, att_dim_v, bias=False)] - self.mlp_o = torch.nn.Linear(aheads * att_dim_v, eprojs, bias=False) - self.dunits = dunits - self.eprojs = eprojs - self.aheads = aheads - self.att_dim_k = att_dim_k - self.att_dim_v = att_dim_v - self.scaling = 1.0 / math.sqrt(att_dim_k) - self.h_length = None - self.enc_h = None - self.pre_compute_k = None - self.pre_compute_v = None - self.mask = None - self.han_mode = han_mode - - def reset(self): - """reset states""" - self.h_length = None - self.enc_h = None - self.pre_compute_k = None - self.pre_compute_v = None - self.mask = None - - def forward(self, enc_hs_pad, enc_hs_len, dec_z, att_prev): - """AttMultiHeadDot forward - - :param torch.Tensor enc_hs_pad: padded encoder hidden state (B x T_max x D_enc) - :param list enc_hs_len: padded encoder hidden state length (B) - :param torch.Tensor dec_z: decoder hidden state (B x D_dec) - :param torch.Tensor att_prev: dummy (does not use) - :return: attention weighted encoder state (B x D_enc) - :rtype: torch.Tensor - :return: list of previous attention weight (B x T_max) * aheads - :rtype: list - """ - - batch = enc_hs_pad.size(0) - # pre-compute all k and v outside the decoder loop - if self.pre_compute_k is None or self.han_mode: - self.enc_h = enc_hs_pad # utt x frame x hdim - self.h_length = self.enc_h.size(1) - # utt x frame x att_dim - self.pre_compute_k = [ - torch.tanh(self.mlp_k[h](self.enc_h)) - for h in six.moves.range(self.aheads) - ] - - if self.pre_compute_v is None or self.han_mode: - self.enc_h = enc_hs_pad # utt x frame x hdim - self.h_length = self.enc_h.size(1) - # utt x frame x att_dim - self.pre_compute_v = [ - self.mlp_v[h](self.enc_h) for h in six.moves.range(self.aheads) - ] - - if dec_z is None: - dec_z = enc_hs_pad.new_zeros(batch, self.dunits) - else: - dec_z = dec_z.view(batch, self.dunits) - - c = [] - w = [] - for h in six.moves.range(self.aheads): - e = torch.sum( - self.pre_compute_k[h] - * torch.tanh(self.mlp_q[h](dec_z)).view(batch, 1, self.att_dim_k), - dim=2, - ) # utt x frame - - # NOTE consider zero padding when compute w. - if self.mask is None: - self.mask = to_device(enc_hs_pad, make_pad_mask(enc_hs_len)) - e.masked_fill_(self.mask, -float("inf")) - w += [F.softmax(self.scaling * e, dim=1)] - - # weighted sum over flames - # utt x hdim - # NOTE use bmm instead of sum(*) - c += [ - torch.sum( - self.pre_compute_v[h] * w[h].view(batch, self.h_length, 1), dim=1 - ) - ] - - # concat all of c - c = self.mlp_o(torch.cat(c, dim=1)) - - return c, w - - -class AttMultiHeadAdd(torch.nn.Module): - """Multi head additive attention - - Reference: Attention is all you need - (https://arxiv.org/abs/1706.03762) - - This attention is multi head attention using additive attention for each head. - - :param int eprojs: # projection-units of encoder - :param int dunits: # units of decoder - :param int aheads: # heads of multi head attention - :param int att_dim_k: dimension k in multi head attention - :param int att_dim_v: dimension v in multi head attention - :param bool han_mode: flag to swith on mode of hierarchical attention - and not store pre_compute_k and pre_compute_v - """ - - def __init__(self, eprojs, dunits, aheads, att_dim_k, att_dim_v, han_mode=False): - super(AttMultiHeadAdd, self).__init__() - self.mlp_q = torch.nn.ModuleList() - self.mlp_k = torch.nn.ModuleList() - self.mlp_v = torch.nn.ModuleList() - self.gvec = torch.nn.ModuleList() - for _ in six.moves.range(aheads): - self.mlp_q += [torch.nn.Linear(dunits, att_dim_k)] - self.mlp_k += [torch.nn.Linear(eprojs, att_dim_k, bias=False)] - self.mlp_v += [torch.nn.Linear(eprojs, att_dim_v, bias=False)] - self.gvec += [torch.nn.Linear(att_dim_k, 1)] - self.mlp_o = torch.nn.Linear(aheads * att_dim_v, eprojs, bias=False) - self.dunits = dunits - self.eprojs = eprojs - self.aheads = aheads - self.att_dim_k = att_dim_k - self.att_dim_v = att_dim_v - self.scaling = 1.0 / math.sqrt(att_dim_k) - self.h_length = None - self.enc_h = None - self.pre_compute_k = None - self.pre_compute_v = None - self.mask = None - self.han_mode = han_mode - - def reset(self): - """reset states""" - self.h_length = None - self.enc_h = None - self.pre_compute_k = None - self.pre_compute_v = None - self.mask = None - - def forward(self, enc_hs_pad, enc_hs_len, dec_z, att_prev): - """AttMultiHeadAdd forward - - :param torch.Tensor enc_hs_pad: padded encoder hidden state (B x T_max x D_enc) - :param list enc_hs_len: padded encoder hidden state length (B) - :param torch.Tensor dec_z: decoder hidden state (B x D_dec) - :param torch.Tensor att_prev: dummy (does not use) - :return: attention weighted encoder state (B, D_enc) - :rtype: torch.Tensor - :return: list of previous attention weight (B x T_max) * aheads - :rtype: list - """ - - batch = enc_hs_pad.size(0) - # pre-compute all k and v outside the decoder loop - if self.pre_compute_k is None or self.han_mode: - self.enc_h = enc_hs_pad # utt x frame x hdim - self.h_length = self.enc_h.size(1) - # utt x frame x att_dim - self.pre_compute_k = [ - self.mlp_k[h](self.enc_h) for h in six.moves.range(self.aheads) - ] - - if self.pre_compute_v is None or self.han_mode: - self.enc_h = enc_hs_pad # utt x frame x hdim - self.h_length = self.enc_h.size(1) - # utt x frame x att_dim - self.pre_compute_v = [ - self.mlp_v[h](self.enc_h) for h in six.moves.range(self.aheads) - ] - - if dec_z is None: - dec_z = enc_hs_pad.new_zeros(batch, self.dunits) - else: - dec_z = dec_z.view(batch, self.dunits) - - c = [] - w = [] - for h in six.moves.range(self.aheads): - e = self.gvec[h]( - torch.tanh( - self.pre_compute_k[h] - + self.mlp_q[h](dec_z).view(batch, 1, self.att_dim_k) - ) - ).squeeze(2) - - # NOTE consider zero padding when compute w. - if self.mask is None: - self.mask = to_device(enc_hs_pad, make_pad_mask(enc_hs_len)) - e.masked_fill_(self.mask, -float("inf")) - w += [F.softmax(self.scaling * e, dim=1)] - - # weighted sum over flames - # utt x hdim - # NOTE use bmm instead of sum(*) - c += [ - torch.sum( - self.pre_compute_v[h] * w[h].view(batch, self.h_length, 1), dim=1 - ) - ] - - # concat all of c - c = self.mlp_o(torch.cat(c, dim=1)) - - return c, w - - -class AttMultiHeadLoc(torch.nn.Module): - """Multi head location based attention - - Reference: Attention is all you need - (https://arxiv.org/abs/1706.03762) - - This attention is multi head attention using location-aware attention for each head. - - :param int eprojs: # projection-units of encoder - :param int dunits: # units of decoder - :param int aheads: # heads of multi head attention - :param int att_dim_k: dimension k in multi head attention - :param int att_dim_v: dimension v in multi head attention - :param int aconv_chans: # channels of attention convolution - :param int aconv_filts: filter size of attention convolution - :param bool han_mode: flag to swith on mode of hierarchical attention - and not store pre_compute_k and pre_compute_v - """ - - def __init__( - self, - eprojs, - dunits, - aheads, - att_dim_k, - att_dim_v, - aconv_chans, - aconv_filts, - han_mode=False, - ): - super(AttMultiHeadLoc, self).__init__() - self.mlp_q = torch.nn.ModuleList() - self.mlp_k = torch.nn.ModuleList() - self.mlp_v = torch.nn.ModuleList() - self.gvec = torch.nn.ModuleList() - self.loc_conv = torch.nn.ModuleList() - self.mlp_att = torch.nn.ModuleList() - for _ in six.moves.range(aheads): - self.mlp_q += [torch.nn.Linear(dunits, att_dim_k)] - self.mlp_k += [torch.nn.Linear(eprojs, att_dim_k, bias=False)] - self.mlp_v += [torch.nn.Linear(eprojs, att_dim_v, bias=False)] - self.gvec += [torch.nn.Linear(att_dim_k, 1)] - self.loc_conv += [ - torch.nn.Conv2d( - 1, - aconv_chans, - (1, 2 * aconv_filts + 1), - padding=(0, aconv_filts), - bias=False, - ) - ] - self.mlp_att += [torch.nn.Linear(aconv_chans, att_dim_k, bias=False)] - self.mlp_o = torch.nn.Linear(aheads * att_dim_v, eprojs, bias=False) - self.dunits = dunits - self.eprojs = eprojs - self.aheads = aheads - self.att_dim_k = att_dim_k - self.att_dim_v = att_dim_v - self.scaling = 1.0 / math.sqrt(att_dim_k) - self.h_length = None - self.enc_h = None - self.pre_compute_k = None - self.pre_compute_v = None - self.mask = None - self.han_mode = han_mode - - def reset(self): - """reset states""" - self.h_length = None - self.enc_h = None - self.pre_compute_k = None - self.pre_compute_v = None - self.mask = None - - def forward(self, enc_hs_pad, enc_hs_len, dec_z, att_prev, scaling=2.0): - """AttMultiHeadLoc forward - - :param torch.Tensor enc_hs_pad: padded encoder hidden state (B x T_max x D_enc) - :param list enc_hs_len: padded encoder hidden state length (B) - :param torch.Tensor dec_z: decoder hidden state (B x D_dec) - :param torch.Tensor att_prev: - list of previous attention weight (B x T_max) * aheads - :param float scaling: scaling parameter before applying softmax - :return: attention weighted encoder state (B x D_enc) - :rtype: torch.Tensor - :return: list of previous attention weight (B x T_max) * aheads - :rtype: list - """ - - batch = enc_hs_pad.size(0) - # pre-compute all k and v outside the decoder loop - if self.pre_compute_k is None or self.han_mode: - self.enc_h = enc_hs_pad # utt x frame x hdim - self.h_length = self.enc_h.size(1) - # utt x frame x att_dim - self.pre_compute_k = [ - self.mlp_k[h](self.enc_h) for h in six.moves.range(self.aheads) - ] - - if self.pre_compute_v is None or self.han_mode: - self.enc_h = enc_hs_pad # utt x frame x hdim - self.h_length = self.enc_h.size(1) - # utt x frame x att_dim - self.pre_compute_v = [ - self.mlp_v[h](self.enc_h) for h in six.moves.range(self.aheads) - ] - - if dec_z is None: - dec_z = enc_hs_pad.new_zeros(batch, self.dunits) - else: - dec_z = dec_z.view(batch, self.dunits) - - if att_prev is None: - att_prev = [] - for _ in six.moves.range(self.aheads): - # if no bias, 0 0-pad goes 0 - mask = 1.0 - make_pad_mask(enc_hs_len).float() - att_prev += [ - to_device(enc_hs_pad, mask / mask.new(enc_hs_len).unsqueeze(-1)) - ] - - c = [] - w = [] - for h in six.moves.range(self.aheads): - att_conv = self.loc_conv[h](att_prev[h].view(batch, 1, 1, self.h_length)) - att_conv = att_conv.squeeze(2).transpose(1, 2) - att_conv = self.mlp_att[h](att_conv) - - e = self.gvec[h]( - torch.tanh( - self.pre_compute_k[h] - + att_conv - + self.mlp_q[h](dec_z).view(batch, 1, self.att_dim_k) - ) - ).squeeze(2) - - # NOTE consider zero padding when compute w. - if self.mask is None: - self.mask = to_device(enc_hs_pad, make_pad_mask(enc_hs_len)) - e.masked_fill_(self.mask, -float("inf")) - w += [F.softmax(scaling * e, dim=1)] - - # weighted sum over flames - # utt x hdim - # NOTE use bmm instead of sum(*) - c += [ - torch.sum( - self.pre_compute_v[h] * w[h].view(batch, self.h_length, 1), dim=1 - ) - ] - - # concat all of c - c = self.mlp_o(torch.cat(c, dim=1)) - - return c, w - - -class AttMultiHeadMultiResLoc(torch.nn.Module): - """Multi head multi resolution location based attention - - Reference: Attention is all you need - (https://arxiv.org/abs/1706.03762) - - This attention is multi head attention using location-aware attention for each head. - Furthermore, it uses different filter size for each head. - - :param int eprojs: # projection-units of encoder - :param int dunits: # units of decoder - :param int aheads: # heads of multi head attention - :param int att_dim_k: dimension k in multi head attention - :param int att_dim_v: dimension v in multi head attention - :param int aconv_chans: maximum # channels of attention convolution - each head use #ch = aconv_chans * (head + 1) / aheads - e.g. aheads=4, aconv_chans=100 => filter size = 25, 50, 75, 100 - :param int aconv_filts: filter size of attention convolution - :param bool han_mode: flag to swith on mode of hierarchical attention - and not store pre_compute_k and pre_compute_v - """ - - def __init__( - self, - eprojs, - dunits, - aheads, - att_dim_k, - att_dim_v, - aconv_chans, - aconv_filts, - han_mode=False, - ): - super(AttMultiHeadMultiResLoc, self).__init__() - self.mlp_q = torch.nn.ModuleList() - self.mlp_k = torch.nn.ModuleList() - self.mlp_v = torch.nn.ModuleList() - self.gvec = torch.nn.ModuleList() - self.loc_conv = torch.nn.ModuleList() - self.mlp_att = torch.nn.ModuleList() - for h in six.moves.range(aheads): - self.mlp_q += [torch.nn.Linear(dunits, att_dim_k)] - self.mlp_k += [torch.nn.Linear(eprojs, att_dim_k, bias=False)] - self.mlp_v += [torch.nn.Linear(eprojs, att_dim_v, bias=False)] - self.gvec += [torch.nn.Linear(att_dim_k, 1)] - afilts = aconv_filts * (h + 1) // aheads - self.loc_conv += [ - torch.nn.Conv2d( - 1, aconv_chans, (1, 2 * afilts + 1), padding=(0, afilts), bias=False - ) - ] - self.mlp_att += [torch.nn.Linear(aconv_chans, att_dim_k, bias=False)] - self.mlp_o = torch.nn.Linear(aheads * att_dim_v, eprojs, bias=False) - self.dunits = dunits - self.eprojs = eprojs - self.aheads = aheads - self.att_dim_k = att_dim_k - self.att_dim_v = att_dim_v - self.scaling = 1.0 / math.sqrt(att_dim_k) - self.h_length = None - self.enc_h = None - self.pre_compute_k = None - self.pre_compute_v = None - self.mask = None - self.han_mode = han_mode - - def reset(self): - """reset states""" - self.h_length = None - self.enc_h = None - self.pre_compute_k = None - self.pre_compute_v = None - self.mask = None - - def forward(self, enc_hs_pad, enc_hs_len, dec_z, att_prev): - """AttMultiHeadMultiResLoc forward - - :param torch.Tensor enc_hs_pad: padded encoder hidden state (B x T_max x D_enc) - :param list enc_hs_len: padded encoder hidden state length (B) - :param torch.Tensor dec_z: decoder hidden state (B x D_dec) - :param torch.Tensor att_prev: list of previous attention weight - (B x T_max) * aheads - :return: attention weighted encoder state (B x D_enc) - :rtype: torch.Tensor - :return: list of previous attention weight (B x T_max) * aheads - :rtype: list - """ - - batch = enc_hs_pad.size(0) - # pre-compute all k and v outside the decoder loop - if self.pre_compute_k is None or self.han_mode: - self.enc_h = enc_hs_pad # utt x frame x hdim - self.h_length = self.enc_h.size(1) - # utt x frame x att_dim - self.pre_compute_k = [ - self.mlp_k[h](self.enc_h) for h in six.moves.range(self.aheads) - ] - - if self.pre_compute_v is None or self.han_mode: - self.enc_h = enc_hs_pad # utt x frame x hdim - self.h_length = self.enc_h.size(1) - # utt x frame x att_dim - self.pre_compute_v = [ - self.mlp_v[h](self.enc_h) for h in six.moves.range(self.aheads) - ] - - if dec_z is None: - dec_z = enc_hs_pad.new_zeros(batch, self.dunits) - else: - dec_z = dec_z.view(batch, self.dunits) - - if att_prev is None: - att_prev = [] - for _ in six.moves.range(self.aheads): - # if no bias, 0 0-pad goes 0 - mask = 1.0 - make_pad_mask(enc_hs_len).float() - att_prev += [ - to_device(enc_hs_pad, mask / mask.new(enc_hs_len).unsqueeze(-1)) - ] - - c = [] - w = [] - for h in six.moves.range(self.aheads): - att_conv = self.loc_conv[h](att_prev[h].view(batch, 1, 1, self.h_length)) - att_conv = att_conv.squeeze(2).transpose(1, 2) - att_conv = self.mlp_att[h](att_conv) - - e = self.gvec[h]( - torch.tanh( - self.pre_compute_k[h] - + att_conv - + self.mlp_q[h](dec_z).view(batch, 1, self.att_dim_k) - ) - ).squeeze(2) - - # NOTE consider zero padding when compute w. - if self.mask is None: - self.mask = to_device(enc_hs_pad, make_pad_mask(enc_hs_len)) - e.masked_fill_(self.mask, -float("inf")) - w += [F.softmax(self.scaling * e, dim=1)] - - # weighted sum over flames - # utt x hdim - # NOTE use bmm instead of sum(*) - c += [ - torch.sum( - self.pre_compute_v[h] * w[h].view(batch, self.h_length, 1), dim=1 - ) - ] - - # concat all of c - c = self.mlp_o(torch.cat(c, dim=1)) - - return c, w - - -class AttForward(torch.nn.Module): - """Forward attention module. - - Reference: - Forward attention in sequence-to-sequence acoustic modeling for speech synthesis - (https://arxiv.org/pdf/1807.06736.pdf) - - :param int eprojs: # projection-units of encoder - :param int dunits: # units of decoder - :param int att_dim: attention dimension - :param int aconv_chans: # channels of attention convolution - :param int aconv_filts: filter size of attention convolution - """ - - def __init__(self, eprojs, dunits, att_dim, aconv_chans, aconv_filts): - super(AttForward, self).__init__() - self.mlp_enc = torch.nn.Linear(eprojs, att_dim) - self.mlp_dec = torch.nn.Linear(dunits, att_dim, bias=False) - self.mlp_att = torch.nn.Linear(aconv_chans, att_dim, bias=False) - self.loc_conv = torch.nn.Conv2d( - 1, - aconv_chans, - (1, 2 * aconv_filts + 1), - padding=(0, aconv_filts), - bias=False, - ) - self.gvec = torch.nn.Linear(att_dim, 1) - self.dunits = dunits - self.eprojs = eprojs - self.att_dim = att_dim - self.h_length = None - self.enc_h = None - self.pre_compute_enc_h = None - self.mask = None - - def reset(self): - """reset states""" - self.h_length = None - self.enc_h = None - self.pre_compute_enc_h = None - self.mask = None - - def forward( - self, - enc_hs_pad, - enc_hs_len, - dec_z, - att_prev, - scaling=1.0, - last_attended_idx=None, - backward_window=1, - forward_window=3, - ): - """Calculate AttForward forward propagation. - - :param torch.Tensor enc_hs_pad: padded encoder hidden state (B x T_max x D_enc) - :param list enc_hs_len: padded encoder hidden state length (B) - :param torch.Tensor dec_z: decoder hidden state (B x D_dec) - :param torch.Tensor att_prev: attention weights of previous step - :param float scaling: scaling parameter before applying softmax - :param int last_attended_idx: index of the inputs of the last attended - :param int backward_window: backward window size in attention constraint - :param int forward_window: forward window size in attetion constraint - :return: attention weighted encoder state (B, D_enc) - :rtype: torch.Tensor - :return: previous attention weights (B x T_max) - :rtype: torch.Tensor - """ - batch = len(enc_hs_pad) - # pre-compute all h outside the decoder loop - if self.pre_compute_enc_h is None: - self.enc_h = enc_hs_pad # utt x frame x hdim - self.h_length = self.enc_h.size(1) - # utt x frame x att_dim - self.pre_compute_enc_h = self.mlp_enc(self.enc_h) - - if dec_z is None: - dec_z = enc_hs_pad.new_zeros(batch, self.dunits) - else: - dec_z = dec_z.view(batch, self.dunits) - - if att_prev is None: - # initial attention will be [1, 0, 0, ...] - att_prev = enc_hs_pad.new_zeros(*enc_hs_pad.size()[:2]) - att_prev[:, 0] = 1.0 - - # att_prev: utt x frame -> utt x 1 x 1 x frame - # -> utt x att_conv_chans x 1 x frame - att_conv = self.loc_conv(att_prev.view(batch, 1, 1, self.h_length)) - # att_conv: utt x att_conv_chans x 1 x frame -> utt x frame x att_conv_chans - att_conv = att_conv.squeeze(2).transpose(1, 2) - # att_conv: utt x frame x att_conv_chans -> utt x frame x att_dim - att_conv = self.mlp_att(att_conv) - - # dec_z_tiled: utt x frame x att_dim - dec_z_tiled = self.mlp_dec(dec_z).unsqueeze(1) - - # dot with gvec - # utt x frame x att_dim -> utt x frame - e = self.gvec( - torch.tanh(self.pre_compute_enc_h + dec_z_tiled + att_conv) - ).squeeze(2) - - # NOTE: consider zero padding when compute w. - if self.mask is None: - self.mask = to_device(enc_hs_pad, make_pad_mask(enc_hs_len)) - e.masked_fill_(self.mask, -float("inf")) - - # apply monotonic attention constraint (mainly for TTS) - if last_attended_idx is not None: - e = _apply_attention_constraint( - e, last_attended_idx, backward_window, forward_window - ) - - w = F.softmax(scaling * e, dim=1) - - # forward attention - att_prev_shift = F.pad(att_prev, (1, 0))[:, :-1] - w = (att_prev + att_prev_shift) * w - # NOTE: clamp is needed to avoid nan gradient - w = F.normalize(torch.clamp(w, 1e-6), p=1, dim=1) - - # weighted sum over flames - # utt x hdim - # NOTE use bmm instead of sum(*) - c = torch.sum(self.enc_h * w.unsqueeze(-1), dim=1) - - return c, w - - -class AttForwardTA(torch.nn.Module): - """Forward attention with transition agent module. - - Reference: - Forward attention in sequence-to-sequence acoustic modeling for speech synthesis - (https://arxiv.org/pdf/1807.06736.pdf) - - :param int eunits: # units of encoder - :param int dunits: # units of decoder - :param int att_dim: attention dimension - :param int aconv_chans: # channels of attention convolution - :param int aconv_filts: filter size of attention convolution - :param int odim: output dimension - """ - - def __init__(self, eunits, dunits, att_dim, aconv_chans, aconv_filts, odim): - super(AttForwardTA, self).__init__() - self.mlp_enc = torch.nn.Linear(eunits, att_dim) - self.mlp_dec = torch.nn.Linear(dunits, att_dim, bias=False) - self.mlp_ta = torch.nn.Linear(eunits + dunits + odim, 1) - self.mlp_att = torch.nn.Linear(aconv_chans, att_dim, bias=False) - self.loc_conv = torch.nn.Conv2d( - 1, - aconv_chans, - (1, 2 * aconv_filts + 1), - padding=(0, aconv_filts), - bias=False, - ) - self.gvec = torch.nn.Linear(att_dim, 1) - self.dunits = dunits - self.eunits = eunits - self.att_dim = att_dim - self.h_length = None - self.enc_h = None - self.pre_compute_enc_h = None - self.mask = None - self.trans_agent_prob = 0.5 - - def reset(self): - self.h_length = None - self.enc_h = None - self.pre_compute_enc_h = None - self.mask = None - self.trans_agent_prob = 0.5 - - def forward( - self, - enc_hs_pad, - enc_hs_len, - dec_z, - att_prev, - out_prev, - scaling=1.0, - last_attended_idx=None, - backward_window=1, - forward_window=3, - ): - """Calculate AttForwardTA forward propagation. - - :param torch.Tensor enc_hs_pad: padded encoder hidden state (B, Tmax, eunits) - :param list enc_hs_len: padded encoder hidden state length (B) - :param torch.Tensor dec_z: decoder hidden state (B, dunits) - :param torch.Tensor att_prev: attention weights of previous step - :param torch.Tensor out_prev: decoder outputs of previous step (B, odim) - :param float scaling: scaling parameter before applying softmax - :param int last_attended_idx: index of the inputs of the last attended - :param int backward_window: backward window size in attention constraint - :param int forward_window: forward window size in attetion constraint - :return: attention weighted encoder state (B, dunits) - :rtype: torch.Tensor - :return: previous attention weights (B, Tmax) - :rtype: torch.Tensor - """ - batch = len(enc_hs_pad) - # pre-compute all h outside the decoder loop - if self.pre_compute_enc_h is None: - self.enc_h = enc_hs_pad # utt x frame x hdim - self.h_length = self.enc_h.size(1) - # utt x frame x att_dim - self.pre_compute_enc_h = self.mlp_enc(self.enc_h) - - if dec_z is None: - dec_z = enc_hs_pad.new_zeros(batch, self.dunits) - else: - dec_z = dec_z.view(batch, self.dunits) - - if att_prev is None: - # initial attention will be [1, 0, 0, ...] - att_prev = enc_hs_pad.new_zeros(*enc_hs_pad.size()[:2]) - att_prev[:, 0] = 1.0 - - # att_prev: utt x frame -> utt x 1 x 1 x frame - # -> utt x att_conv_chans x 1 x frame - att_conv = self.loc_conv(att_prev.view(batch, 1, 1, self.h_length)) - # att_conv: utt x att_conv_chans x 1 x frame -> utt x frame x att_conv_chans - att_conv = att_conv.squeeze(2).transpose(1, 2) - # att_conv: utt x frame x att_conv_chans -> utt x frame x att_dim - att_conv = self.mlp_att(att_conv) - - # dec_z_tiled: utt x frame x att_dim - dec_z_tiled = self.mlp_dec(dec_z).view(batch, 1, self.att_dim) - - # dot with gvec - # utt x frame x att_dim -> utt x frame - e = self.gvec( - torch.tanh(att_conv + self.pre_compute_enc_h + dec_z_tiled) - ).squeeze(2) - - # NOTE consider zero padding when compute w. - if self.mask is None: - self.mask = to_device(enc_hs_pad, make_pad_mask(enc_hs_len)) - e.masked_fill_(self.mask, -float("inf")) - - # apply monotonic attention constraint (mainly for TTS) - if last_attended_idx is not None: - e = _apply_attention_constraint( - e, last_attended_idx, backward_window, forward_window - ) - - w = F.softmax(scaling * e, dim=1) - - # forward attention - att_prev_shift = F.pad(att_prev, (1, 0))[:, :-1] - w = ( - self.trans_agent_prob * att_prev - + (1 - self.trans_agent_prob) * att_prev_shift - ) * w - # NOTE: clamp is needed to avoid nan gradient - w = F.normalize(torch.clamp(w, 1e-6), p=1, dim=1) - - # weighted sum over flames - # utt x hdim - # NOTE use bmm instead of sum(*) - c = torch.sum(self.enc_h * w.view(batch, self.h_length, 1), dim=1) - - # update transition agent prob - self.trans_agent_prob = torch.sigmoid( - self.mlp_ta(torch.cat([c, out_prev, dec_z], dim=1)) - ) - - return c, w - - -def att_for(args, num_att=1, han_mode=False): - """Instantiates an attention module given the program arguments - - :param Namespace args: The arguments - :param int num_att: number of attention modules - (in multi-speaker case, it can be 2 or more) - :param bool han_mode: switch on/off mode of hierarchical attention network (HAN) - :rtype torch.nn.Module - :return: The attention module - """ - att_list = torch.nn.ModuleList() - num_encs = getattr(args, "num_encs", 1) # use getattr to keep compatibility - aheads = getattr(args, "aheads", None) - awin = getattr(args, "awin", None) - aconv_chans = getattr(args, "aconv_chans", None) - aconv_filts = getattr(args, "aconv_filts", None) - - if num_encs == 1: - for i in range(num_att): - att = initial_att( - args.atype, - args.eprojs, - args.dunits, - aheads, - args.adim, - awin, - aconv_chans, - aconv_filts, - ) - att_list.append(att) - elif num_encs > 1: # no multi-speaker mode - if han_mode: - att = initial_att( - args.han_type, - args.eprojs, - args.dunits, - args.han_heads, - args.han_dim, - args.han_win, - args.han_conv_chans, - args.han_conv_filts, - han_mode=True, - ) - return att - else: - att_list = torch.nn.ModuleList() - for idx in range(num_encs): - att = initial_att( - args.atype[idx], - args.eprojs, - args.dunits, - aheads[idx], - args.adim[idx], - awin[idx], - aconv_chans[idx], - aconv_filts[idx], - ) - att_list.append(att) - else: - raise ValueError( - "Number of encoders needs to be more than one. {}".format(num_encs) - ) - return att_list - - -def initial_att( - atype, eprojs, dunits, aheads, adim, awin, aconv_chans, aconv_filts, han_mode=False -): - """Instantiates a single attention module - - :param str atype: attention type - :param int eprojs: # projection-units of encoder - :param int dunits: # units of decoder - :param int aheads: # heads of multi head attention - :param int adim: attention dimension - :param int awin: attention window size - :param int aconv_chans: # channels of attention convolution - :param int aconv_filts: filter size of attention convolution - :param bool han_mode: flag to swith on mode of hierarchical attention - :return: The attention module - """ - - if atype == "noatt": - att = NoAtt() - elif atype == "dot": - att = AttDot(eprojs, dunits, adim, han_mode) - elif atype == "add": - att = AttAdd(eprojs, dunits, adim, han_mode) - elif atype == "location": - att = AttLoc(eprojs, dunits, adim, aconv_chans, aconv_filts, han_mode) - elif atype == "location2d": - att = AttLoc2D(eprojs, dunits, adim, awin, aconv_chans, aconv_filts, han_mode) - elif atype == "location_recurrent": - att = AttLocRec(eprojs, dunits, adim, aconv_chans, aconv_filts, han_mode) - elif atype == "coverage": - att = AttCov(eprojs, dunits, adim, han_mode) - elif atype == "coverage_location": - att = AttCovLoc(eprojs, dunits, adim, aconv_chans, aconv_filts, han_mode) - elif atype == "multi_head_dot": - att = AttMultiHeadDot(eprojs, dunits, aheads, adim, adim, han_mode) - elif atype == "multi_head_add": - att = AttMultiHeadAdd(eprojs, dunits, aheads, adim, adim, han_mode) - elif atype == "multi_head_loc": - att = AttMultiHeadLoc( - eprojs, dunits, aheads, adim, adim, aconv_chans, aconv_filts, han_mode - ) - elif atype == "multi_head_multi_res_loc": - att = AttMultiHeadMultiResLoc( - eprojs, dunits, aheads, adim, adim, aconv_chans, aconv_filts, han_mode - ) - return att - - -def att_to_numpy(att_ws, att): - """Converts attention weights to a numpy array given the attention - - :param list att_ws: The attention weights - :param torch.nn.Module att: The attention - :rtype: np.ndarray - :return: The numpy array of the attention weights - """ - # convert to numpy array with the shape (B, Lmax, Tmax) - if isinstance(att, AttLoc2D): - # att_ws => list of previous concate attentions - att_ws = torch.stack([aw[:, -1] for aw in att_ws], dim=1).cpu().numpy() - elif isinstance(att, (AttCov, AttCovLoc)): - # att_ws => list of list of previous attentions - att_ws = ( - torch.stack([aw[idx] for idx, aw in enumerate(att_ws)], dim=1).cpu().numpy() - ) - elif isinstance(att, AttLocRec): - # att_ws => list of tuple of attention and hidden states - att_ws = torch.stack([aw[0] for aw in att_ws], dim=1).cpu().numpy() - elif isinstance( - att, - (AttMultiHeadDot, AttMultiHeadAdd, AttMultiHeadLoc, AttMultiHeadMultiResLoc), - ): - # att_ws => list of list of each head attention - n_heads = len(att_ws[0]) - att_ws_sorted_by_head = [] - for h in six.moves.range(n_heads): - att_ws_head = torch.stack([aw[h] for aw in att_ws], dim=1) - att_ws_sorted_by_head += [att_ws_head] - att_ws = torch.stack(att_ws_sorted_by_head, dim=1).cpu().numpy() - else: - # att_ws => list of attentions - att_ws = torch.stack(att_ws, dim=1).cpu().numpy() - return att_ws diff --git a/funasr_detach/models/language_model/rnn/decoders.py b/funasr_detach/models/language_model/rnn/decoders.py deleted file mode 100644 index 3ccbe14e52bce42a783231607e57f00b7ba497e9..0000000000000000000000000000000000000000 --- a/funasr_detach/models/language_model/rnn/decoders.py +++ /dev/null @@ -1,1224 +0,0 @@ -"""RNN decoder module.""" - -import logging -import math -import random -from argparse import Namespace - -import numpy as np -import six -import torch -import torch.nn.functional as F - -from funasr_detach.models.transformer.utils.scorers.ctc_prefix_score import ( - CTCPrefixScore, -) -from funasr_detach.models.transformer.utils.scorers.ctc_prefix_score import ( - CTCPrefixScoreTH, -) -from funasr_detach.models.transformer.utils.scorers.scorer_interface import ( - ScorerInterface, -) -from funasr_detach.metrics import end_detect -from funasr_detach.models.transformer.utils.nets_utils import mask_by_length -from funasr_detach.models.transformer.utils.nets_utils import pad_list -from funasr_detach.metrics.compute_acc import th_accuracy -from funasr_detach.models.transformer.utils.nets_utils import to_device -from funasr_detach.models.language_model.rnn.attentions import att_to_numpy - -MAX_DECODER_OUTPUT = 5 -CTC_SCORING_RATIO = 1.5 - - -class Decoder(torch.nn.Module, ScorerInterface): - """Decoder module - - :param int eprojs: encoder projection units - :param int odim: dimension of outputs - :param str dtype: gru or lstm - :param int dlayers: decoder layers - :param int dunits: decoder units - :param int sos: start of sequence symbol id - :param int eos: end of sequence symbol id - :param torch.nn.Module att: attention module - :param int verbose: verbose level - :param list char_list: list of character strings - :param ndarray labeldist: distribution of label smoothing - :param float lsm_weight: label smoothing weight - :param float sampling_probability: scheduled sampling probability - :param float dropout: dropout rate - :param float context_residual: if True, use context vector for token generation - :param float replace_sos: use for multilingual (speech/text) translation - """ - - def __init__( - self, - eprojs, - odim, - dtype, - dlayers, - dunits, - sos, - eos, - att, - verbose=0, - char_list=None, - labeldist=None, - lsm_weight=0.0, - sampling_probability=0.0, - dropout=0.0, - context_residual=False, - replace_sos=False, - num_encs=1, - ): - - torch.nn.Module.__init__(self) - self.dtype = dtype - self.dunits = dunits - self.dlayers = dlayers - self.context_residual = context_residual - self.embed = torch.nn.Embedding(odim, dunits) - self.dropout_emb = torch.nn.Dropout(p=dropout) - - self.decoder = torch.nn.ModuleList() - self.dropout_dec = torch.nn.ModuleList() - self.decoder += [ - ( - torch.nn.LSTMCell(dunits + eprojs, dunits) - if self.dtype == "lstm" - else torch.nn.GRUCell(dunits + eprojs, dunits) - ) - ] - self.dropout_dec += [torch.nn.Dropout(p=dropout)] - for _ in six.moves.range(1, self.dlayers): - self.decoder += [ - ( - torch.nn.LSTMCell(dunits, dunits) - if self.dtype == "lstm" - else torch.nn.GRUCell(dunits, dunits) - ) - ] - self.dropout_dec += [torch.nn.Dropout(p=dropout)] - # NOTE: dropout is applied only for the vertical connections - # see https://arxiv.org/pdf/1409.2329.pdf - self.ignore_id = -1 - - if context_residual: - self.output = torch.nn.Linear(dunits + eprojs, odim) - else: - self.output = torch.nn.Linear(dunits, odim) - - self.loss = None - self.att = att - self.dunits = dunits - self.sos = sos - self.eos = eos - self.odim = odim - self.verbose = verbose - self.char_list = char_list - # for label smoothing - self.labeldist = labeldist - self.vlabeldist = None - self.lsm_weight = lsm_weight - self.sampling_probability = sampling_probability - self.dropout = dropout - self.num_encs = num_encs - - # for multilingual E2E-ST - self.replace_sos = replace_sos - - self.logzero = -10000000000.0 - - def zero_state(self, hs_pad): - return hs_pad.new_zeros(hs_pad.size(0), self.dunits) - - def rnn_forward(self, ey, z_list, c_list, z_prev, c_prev): - if self.dtype == "lstm": - z_list[0], c_list[0] = self.decoder[0](ey, (z_prev[0], c_prev[0])) - for i in six.moves.range(1, self.dlayers): - z_list[i], c_list[i] = self.decoder[i]( - self.dropout_dec[i - 1](z_list[i - 1]), (z_prev[i], c_prev[i]) - ) - else: - z_list[0] = self.decoder[0](ey, z_prev[0]) - for i in six.moves.range(1, self.dlayers): - z_list[i] = self.decoder[i]( - self.dropout_dec[i - 1](z_list[i - 1]), z_prev[i] - ) - return z_list, c_list - - def forward(self, hs_pad, hlens, ys_pad, strm_idx=0, lang_ids=None): - """Decoder forward - - :param torch.Tensor hs_pad: batch of padded hidden state sequences (B, Tmax, D) - [in multi-encoder case, - list of torch.Tensor, - [(B, Tmax_1, D), (B, Tmax_2, D), ..., ] ] - :param torch.Tensor hlens: batch of lengths of hidden state sequences (B) - [in multi-encoder case, list of torch.Tensor, - [(B), (B), ..., ] - :param torch.Tensor ys_pad: batch of padded character id sequence tensor - (B, Lmax) - :param int strm_idx: stream index indicates the index of decoding stream. - :param torch.Tensor lang_ids: batch of target language id tensor (B, 1) - :return: attention loss value - :rtype: torch.Tensor - :return: accuracy - :rtype: float - """ - # to support mutiple encoder asr mode, in single encoder mode, - # convert torch.Tensor to List of torch.Tensor - if self.num_encs == 1: - hs_pad = [hs_pad] - hlens = [hlens] - - # TODO(kan-bayashi): need to make more smart way - ys = [y[y != self.ignore_id] for y in ys_pad] # parse padded ys - # attention index for the attention module - # in SPA (speaker parallel attention), - # att_idx is used to select attention module. In other cases, it is 0. - att_idx = min(strm_idx, len(self.att) - 1) - - # hlens should be list of list of integer - hlens = [list(map(int, hlens[idx])) for idx in range(self.num_encs)] - - self.loss = None - # prepare input and output word sequences with sos/eos IDs - eos = ys[0].new([self.eos]) - sos = ys[0].new([self.sos]) - if self.replace_sos: - ys_in = [torch.cat([idx, y], dim=0) for idx, y in zip(lang_ids, ys)] - else: - ys_in = [torch.cat([sos, y], dim=0) for y in ys] - ys_out = [torch.cat([y, eos], dim=0) for y in ys] - - # padding for ys with -1 - # pys: utt x olen - ys_in_pad = pad_list(ys_in, self.eos) - ys_out_pad = pad_list(ys_out, self.ignore_id) - - # get dim, length info - batch = ys_out_pad.size(0) - olength = ys_out_pad.size(1) - for idx in range(self.num_encs): - logging.info( - self.__class__.__name__ - + "Number of Encoder:{}; enc{}: input lengths: {}.".format( - self.num_encs, idx + 1, hlens[idx] - ) - ) - logging.info( - self.__class__.__name__ - + " output lengths: " - + str([y.size(0) for y in ys_out]) - ) - - # initialization - c_list = [self.zero_state(hs_pad[0])] - z_list = [self.zero_state(hs_pad[0])] - for _ in six.moves.range(1, self.dlayers): - c_list.append(self.zero_state(hs_pad[0])) - z_list.append(self.zero_state(hs_pad[0])) - z_all = [] - if self.num_encs == 1: - att_w = None - self.att[att_idx].reset() # reset pre-computation of h - else: - att_w_list = [None] * (self.num_encs + 1) # atts + han - att_c_list = [None] * (self.num_encs) # atts - for idx in range(self.num_encs + 1): - self.att[idx].reset() # reset pre-computation of h in atts and han - - # pre-computation of embedding - eys = self.dropout_emb(self.embed(ys_in_pad)) # utt x olen x zdim - - # loop for an output sequence - for i in six.moves.range(olength): - if self.num_encs == 1: - att_c, att_w = self.att[att_idx]( - hs_pad[0], hlens[0], self.dropout_dec[0](z_list[0]), att_w - ) - else: - for idx in range(self.num_encs): - att_c_list[idx], att_w_list[idx] = self.att[idx]( - hs_pad[idx], - hlens[idx], - self.dropout_dec[0](z_list[0]), - att_w_list[idx], - ) - hs_pad_han = torch.stack(att_c_list, dim=1) - hlens_han = [self.num_encs] * len(ys_in) - att_c, att_w_list[self.num_encs] = self.att[self.num_encs]( - hs_pad_han, - hlens_han, - self.dropout_dec[0](z_list[0]), - att_w_list[self.num_encs], - ) - if i > 0 and random.random() < self.sampling_probability: - logging.info(" scheduled sampling ") - z_out = self.output(z_all[-1]) - z_out = np.argmax(z_out.detach().cpu(), axis=1) - z_out = self.dropout_emb(self.embed(to_device(hs_pad[0], z_out))) - ey = torch.cat((z_out, att_c), dim=1) # utt x (zdim + hdim) - else: - ey = torch.cat((eys[:, i, :], att_c), dim=1) # utt x (zdim + hdim) - z_list, c_list = self.rnn_forward(ey, z_list, c_list, z_list, c_list) - if self.context_residual: - z_all.append( - torch.cat((self.dropout_dec[-1](z_list[-1]), att_c), dim=-1) - ) # utt x (zdim + hdim) - else: - z_all.append(self.dropout_dec[-1](z_list[-1])) # utt x (zdim) - - z_all = torch.stack(z_all, dim=1).view(batch * olength, -1) - # compute loss - y_all = self.output(z_all) - self.loss = F.cross_entropy( - y_all, - ys_out_pad.view(-1), - ignore_index=self.ignore_id, - reduction="mean", - ) - # compute perplexity - ppl = math.exp(self.loss.item()) - # -1: eos, which is removed in the loss computation - self.loss *= np.mean([len(x) for x in ys_in]) - 1 - acc = th_accuracy(y_all, ys_out_pad, ignore_label=self.ignore_id) - logging.info("att loss:" + "".join(str(self.loss.item()).split("\n"))) - - # show predicted character sequence for debug - if self.verbose > 0 and self.char_list is not None: - ys_hat = y_all.view(batch, olength, -1) - ys_true = ys_out_pad - for (i, y_hat), y_true in zip( - enumerate(ys_hat.detach().cpu().numpy()), ys_true.detach().cpu().numpy() - ): - if i == MAX_DECODER_OUTPUT: - break - idx_hat = np.argmax(y_hat[y_true != self.ignore_id], axis=1) - idx_true = y_true[y_true != self.ignore_id] - seq_hat = [self.char_list[int(idx)] for idx in idx_hat] - seq_true = [self.char_list[int(idx)] for idx in idx_true] - seq_hat = "".join(seq_hat) - seq_true = "".join(seq_true) - logging.info("groundtruth[%d]: " % i + seq_true) - logging.info("prediction [%d]: " % i + seq_hat) - - if self.labeldist is not None: - if self.vlabeldist is None: - self.vlabeldist = to_device(hs_pad[0], torch.from_numpy(self.labeldist)) - loss_reg = -torch.sum( - (F.log_softmax(y_all, dim=1) * self.vlabeldist).view(-1), dim=0 - ) / len(ys_in) - self.loss = (1.0 - self.lsm_weight) * self.loss + self.lsm_weight * loss_reg - - return self.loss, acc, ppl - - def recognize_beam(self, h, lpz, recog_args, char_list, rnnlm=None, strm_idx=0): - """beam search implementation - - :param torch.Tensor h: encoder hidden state (T, eprojs) - [in multi-encoder case, list of torch.Tensor, - [(T1, eprojs), (T2, eprojs), ...] ] - :param torch.Tensor lpz: ctc log softmax output (T, odim) - [in multi-encoder case, list of torch.Tensor, - [(T1, odim), (T2, odim), ...] ] - :param Namespace recog_args: argument Namespace containing options - :param char_list: list of character strings - :param torch.nn.Module rnnlm: language module - :param int strm_idx: - stream index for speaker parallel attention in multi-speaker case - :return: N-best decoding results - :rtype: list of dicts - """ - # to support mutiple encoder asr mode, in single encoder mode, - # convert torch.Tensor to List of torch.Tensor - if self.num_encs == 1: - h = [h] - lpz = [lpz] - if self.num_encs > 1 and lpz is None: - lpz = [lpz] * self.num_encs - - for idx in range(self.num_encs): - logging.info( - "Number of Encoder:{}; enc{}: input lengths: {}.".format( - self.num_encs, idx + 1, h[0].size(0) - ) - ) - att_idx = min(strm_idx, len(self.att) - 1) - # initialization - c_list = [self.zero_state(h[0].unsqueeze(0))] - z_list = [self.zero_state(h[0].unsqueeze(0))] - for _ in six.moves.range(1, self.dlayers): - c_list.append(self.zero_state(h[0].unsqueeze(0))) - z_list.append(self.zero_state(h[0].unsqueeze(0))) - if self.num_encs == 1: - a = None - self.att[att_idx].reset() # reset pre-computation of h - else: - a = [None] * (self.num_encs + 1) # atts + han - att_w_list = [None] * (self.num_encs + 1) # atts + han - att_c_list = [None] * (self.num_encs) # atts - for idx in range(self.num_encs + 1): - self.att[idx].reset() # reset pre-computation of h in atts and han - - # search parms - beam = recog_args.beam_size - penalty = recog_args.penalty - ctc_weight = getattr(recog_args, "ctc_weight", False) # for NMT - - if lpz[0] is not None and self.num_encs > 1: - # weights-ctc, - # e.g. ctc_loss = w_1*ctc_1_loss + w_2 * ctc_2_loss + w_N * ctc_N_loss - weights_ctc_dec = recog_args.weights_ctc_dec / np.sum( - recog_args.weights_ctc_dec - ) # normalize - logging.info( - "ctc weights (decoding): " + " ".join([str(x) for x in weights_ctc_dec]) - ) - else: - weights_ctc_dec = [1.0] - - # preprate sos - if self.replace_sos and recog_args.tgt_lang: - y = char_list.index(recog_args.tgt_lang) - else: - y = self.sos - logging.info(" index: " + str(y)) - logging.info(" mark: " + char_list[y]) - vy = h[0].new_zeros(1).long() - - maxlen = np.amin([h[idx].size(0) for idx in range(self.num_encs)]) - if recog_args.maxlenratio != 0: - # maxlen >= 1 - maxlen = max(1, int(recog_args.maxlenratio * maxlen)) - minlen = int(recog_args.minlenratio * maxlen) - logging.info("max output length: " + str(maxlen)) - logging.info("min output length: " + str(minlen)) - - # initialize hypothesis - if rnnlm: - hyp = { - "score": 0.0, - "yseq": [y], - "c_prev": c_list, - "z_prev": z_list, - "a_prev": a, - "rnnlm_prev": None, - } - else: - hyp = { - "score": 0.0, - "yseq": [y], - "c_prev": c_list, - "z_prev": z_list, - "a_prev": a, - } - if lpz[0] is not None: - ctc_prefix_score = [ - CTCPrefixScore(lpz[idx].detach().numpy(), 0, self.eos, np) - for idx in range(self.num_encs) - ] - hyp["ctc_state_prev"] = [ - ctc_prefix_score[idx].initial_state() for idx in range(self.num_encs) - ] - hyp["ctc_score_prev"] = [0.0] * self.num_encs - if ctc_weight != 1.0: - # pre-pruning based on attention scores - ctc_beam = min(lpz[0].shape[-1], int(beam * CTC_SCORING_RATIO)) - else: - ctc_beam = lpz[0].shape[-1] - hyps = [hyp] - ended_hyps = [] - - for i in six.moves.range(maxlen): - logging.debug("position " + str(i)) - - hyps_best_kept = [] - for hyp in hyps: - vy[0] = hyp["yseq"][i] - ey = self.dropout_emb(self.embed(vy)) # utt list (1) x zdim - if self.num_encs == 1: - att_c, att_w = self.att[att_idx]( - h[0].unsqueeze(0), - [h[0].size(0)], - self.dropout_dec[0](hyp["z_prev"][0]), - hyp["a_prev"], - ) - else: - for idx in range(self.num_encs): - att_c_list[idx], att_w_list[idx] = self.att[idx]( - h[idx].unsqueeze(0), - [h[idx].size(0)], - self.dropout_dec[0](hyp["z_prev"][0]), - hyp["a_prev"][idx], - ) - h_han = torch.stack(att_c_list, dim=1) - att_c, att_w_list[self.num_encs] = self.att[self.num_encs]( - h_han, - [self.num_encs], - self.dropout_dec[0](hyp["z_prev"][0]), - hyp["a_prev"][self.num_encs], - ) - ey = torch.cat((ey, att_c), dim=1) # utt(1) x (zdim + hdim) - z_list, c_list = self.rnn_forward( - ey, z_list, c_list, hyp["z_prev"], hyp["c_prev"] - ) - - # get nbest local scores and their ids - if self.context_residual: - logits = self.output( - torch.cat((self.dropout_dec[-1](z_list[-1]), att_c), dim=-1) - ) - else: - logits = self.output(self.dropout_dec[-1](z_list[-1])) - local_att_scores = F.log_softmax(logits, dim=1) - if rnnlm: - rnnlm_state, local_lm_scores = rnnlm.predict(hyp["rnnlm_prev"], vy) - local_scores = ( - local_att_scores + recog_args.lm_weight * local_lm_scores - ) - else: - local_scores = local_att_scores - - if lpz[0] is not None: - local_best_scores, local_best_ids = torch.topk( - local_att_scores, ctc_beam, dim=1 - ) - ctc_scores, ctc_states = ( - [None] * self.num_encs, - [None] * self.num_encs, - ) - for idx in range(self.num_encs): - ctc_scores[idx], ctc_states[idx] = ctc_prefix_score[idx]( - hyp["yseq"], local_best_ids[0], hyp["ctc_state_prev"][idx] - ) - local_scores = (1.0 - ctc_weight) * local_att_scores[ - :, local_best_ids[0] - ] - if self.num_encs == 1: - local_scores += ctc_weight * torch.from_numpy( - ctc_scores[0] - hyp["ctc_score_prev"][0] - ) - else: - for idx in range(self.num_encs): - local_scores += ( - ctc_weight - * weights_ctc_dec[idx] - * torch.from_numpy( - ctc_scores[idx] - hyp["ctc_score_prev"][idx] - ) - ) - if rnnlm: - local_scores += ( - recog_args.lm_weight * local_lm_scores[:, local_best_ids[0]] - ) - local_best_scores, joint_best_ids = torch.topk( - local_scores, beam, dim=1 - ) - local_best_ids = local_best_ids[:, joint_best_ids[0]] - else: - local_best_scores, local_best_ids = torch.topk( - local_scores, beam, dim=1 - ) - - for j in six.moves.range(beam): - new_hyp = {} - # [:] is needed! - new_hyp["z_prev"] = z_list[:] - new_hyp["c_prev"] = c_list[:] - if self.num_encs == 1: - new_hyp["a_prev"] = att_w[:] - else: - new_hyp["a_prev"] = [ - att_w_list[idx][:] for idx in range(self.num_encs + 1) - ] - new_hyp["score"] = hyp["score"] + local_best_scores[0, j] - new_hyp["yseq"] = [0] * (1 + len(hyp["yseq"])) - new_hyp["yseq"][: len(hyp["yseq"])] = hyp["yseq"] - new_hyp["yseq"][len(hyp["yseq"])] = int(local_best_ids[0, j]) - if rnnlm: - new_hyp["rnnlm_prev"] = rnnlm_state - if lpz[0] is not None: - new_hyp["ctc_state_prev"] = [ - ctc_states[idx][joint_best_ids[0, j]] - for idx in range(self.num_encs) - ] - new_hyp["ctc_score_prev"] = [ - ctc_scores[idx][joint_best_ids[0, j]] - for idx in range(self.num_encs) - ] - # will be (2 x beam) hyps at most - hyps_best_kept.append(new_hyp) - - hyps_best_kept = sorted( - hyps_best_kept, key=lambda x: x["score"], reverse=True - )[:beam] - - # sort and get nbest - hyps = hyps_best_kept - logging.debug("number of pruned hypotheses: " + str(len(hyps))) - logging.debug( - "best hypo: " - + "".join([char_list[int(x)] for x in hyps[0]["yseq"][1:]]) - ) - - # add eos in the final loop to avoid that there are no ended hyps - if i == maxlen - 1: - logging.info("adding in the last position in the loop") - for hyp in hyps: - hyp["yseq"].append(self.eos) - - # add ended hypotheses to a final list, - # and removed them from current hypotheses - # (this will be a problem, number of hyps < beam) - remained_hyps = [] - for hyp in hyps: - if hyp["yseq"][-1] == self.eos: - # only store the sequence that has more than minlen outputs - # also add penalty - if len(hyp["yseq"]) > minlen: - hyp["score"] += (i + 1) * penalty - if rnnlm: # Word LM needs to add final score - hyp["score"] += recog_args.lm_weight * rnnlm.final( - hyp["rnnlm_prev"] - ) - ended_hyps.append(hyp) - else: - remained_hyps.append(hyp) - - # end detection - if end_detect(ended_hyps, i) and recog_args.maxlenratio == 0.0: - logging.info("end detected at %d", i) - break - - hyps = remained_hyps - if len(hyps) > 0: - logging.debug("remaining hypotheses: " + str(len(hyps))) - else: - logging.info("no hypothesis. Finish decoding.") - break - - for hyp in hyps: - logging.debug( - "hypo: " + "".join([char_list[int(x)] for x in hyp["yseq"][1:]]) - ) - - logging.debug("number of ended hypotheses: " + str(len(ended_hyps))) - - nbest_hyps = sorted(ended_hyps, key=lambda x: x["score"], reverse=True)[ - : min(len(ended_hyps), recog_args.nbest) - ] - - # check number of hypotheses - if len(nbest_hyps) == 0: - logging.warning( - "there is no N-best results, " - "perform recognition again with smaller minlenratio." - ) - # should copy because Namespace will be overwritten globally - recog_args = Namespace(**vars(recog_args)) - recog_args.minlenratio = max(0.0, recog_args.minlenratio - 0.1) - if self.num_encs == 1: - return self.recognize_beam(h[0], lpz[0], recog_args, char_list, rnnlm) - else: - return self.recognize_beam(h, lpz, recog_args, char_list, rnnlm) - - logging.info("total log probability: " + str(nbest_hyps[0]["score"])) - logging.info( - "normalized log probability: " - + str(nbest_hyps[0]["score"] / len(nbest_hyps[0]["yseq"])) - ) - - # remove sos - return nbest_hyps - - def recognize_beam_batch( - self, - h, - hlens, - lpz, - recog_args, - char_list, - rnnlm=None, - normalize_score=True, - strm_idx=0, - lang_ids=None, - ): - # to support mutiple encoder asr mode, in single encoder mode, - # convert torch.Tensor to List of torch.Tensor - if self.num_encs == 1: - h = [h] - hlens = [hlens] - lpz = [lpz] - if self.num_encs > 1 and lpz is None: - lpz = [lpz] * self.num_encs - - att_idx = min(strm_idx, len(self.att) - 1) - for idx in range(self.num_encs): - logging.info( - "Number of Encoder:{}; enc{}: input lengths: {}.".format( - self.num_encs, idx + 1, h[idx].size(1) - ) - ) - h[idx] = mask_by_length(h[idx], hlens[idx], 0.0) - - # search params - batch = len(hlens[0]) - beam = recog_args.beam_size - penalty = recog_args.penalty - ctc_weight = getattr(recog_args, "ctc_weight", 0) # for NMT - att_weight = 1.0 - ctc_weight - ctc_margin = getattr( - recog_args, "ctc_window_margin", 0 - ) # use getattr to keep compatibility - # weights-ctc, - # e.g. ctc_loss = w_1*ctc_1_loss + w_2 * ctc_2_loss + w_N * ctc_N_loss - if lpz[0] is not None and self.num_encs > 1: - weights_ctc_dec = recog_args.weights_ctc_dec / np.sum( - recog_args.weights_ctc_dec - ) # normalize - logging.info( - "ctc weights (decoding): " + " ".join([str(x) for x in weights_ctc_dec]) - ) - else: - weights_ctc_dec = [1.0] - - n_bb = batch * beam - pad_b = to_device(h[0], torch.arange(batch) * beam).view(-1, 1) - - max_hlen = np.amin([max(hlens[idx]) for idx in range(self.num_encs)]) - if recog_args.maxlenratio == 0: - maxlen = max_hlen - else: - maxlen = max(1, int(recog_args.maxlenratio * max_hlen)) - minlen = int(recog_args.minlenratio * max_hlen) - logging.info("max output length: " + str(maxlen)) - logging.info("min output length: " + str(minlen)) - - # initialization - c_prev = [ - to_device(h[0], torch.zeros(n_bb, self.dunits)) for _ in range(self.dlayers) - ] - z_prev = [ - to_device(h[0], torch.zeros(n_bb, self.dunits)) for _ in range(self.dlayers) - ] - c_list = [ - to_device(h[0], torch.zeros(n_bb, self.dunits)) for _ in range(self.dlayers) - ] - z_list = [ - to_device(h[0], torch.zeros(n_bb, self.dunits)) for _ in range(self.dlayers) - ] - vscores = to_device(h[0], torch.zeros(batch, beam)) - - rnnlm_state = None - if self.num_encs == 1: - a_prev = [None] - att_w_list, ctc_scorer, ctc_state = [None], [None], [None] - self.att[att_idx].reset() # reset pre-computation of h - else: - a_prev = [None] * (self.num_encs + 1) # atts + han - att_w_list = [None] * (self.num_encs + 1) # atts + han - att_c_list = [None] * (self.num_encs) # atts - ctc_scorer, ctc_state = [None] * (self.num_encs), [None] * (self.num_encs) - for idx in range(self.num_encs + 1): - self.att[idx].reset() # reset pre-computation of h in atts and han - - if self.replace_sos and recog_args.tgt_lang: - logging.info(" index: " + str(char_list.index(recog_args.tgt_lang))) - logging.info(" mark: " + recog_args.tgt_lang) - yseq = [ - [char_list.index(recog_args.tgt_lang)] for _ in six.moves.range(n_bb) - ] - elif lang_ids is not None: - # NOTE: used for evaluation during training - yseq = [ - [lang_ids[b // recog_args.beam_size]] for b in six.moves.range(n_bb) - ] - else: - logging.info(" index: " + str(self.sos)) - logging.info(" mark: " + char_list[self.sos]) - yseq = [[self.sos] for _ in six.moves.range(n_bb)] - - accum_odim_ids = [self.sos for _ in six.moves.range(n_bb)] - stop_search = [False for _ in six.moves.range(batch)] - nbest_hyps = [[] for _ in six.moves.range(batch)] - ended_hyps = [[] for _ in range(batch)] - - exp_hlens = [ - hlens[idx].repeat(beam).view(beam, batch).transpose(0, 1).contiguous() - for idx in range(self.num_encs) - ] - exp_hlens = [exp_hlens[idx].view(-1).tolist() for idx in range(self.num_encs)] - exp_h = [ - h[idx].unsqueeze(1).repeat(1, beam, 1, 1).contiguous() - for idx in range(self.num_encs) - ] - exp_h = [ - exp_h[idx].view(n_bb, h[idx].size()[1], h[idx].size()[2]) - for idx in range(self.num_encs) - ] - - if lpz[0] is not None: - scoring_num = min( - ( - int(beam * CTC_SCORING_RATIO) - if att_weight > 0.0 and not lpz[0].is_cuda - else 0 - ), - lpz[0].size(-1), - ) - ctc_scorer = [ - CTCPrefixScoreTH( - lpz[idx], - hlens[idx], - 0, - self.eos, - margin=ctc_margin, - ) - for idx in range(self.num_encs) - ] - - for i in six.moves.range(maxlen): - logging.debug("position " + str(i)) - - vy = to_device(h[0], torch.LongTensor(self._get_last_yseq(yseq))) - ey = self.dropout_emb(self.embed(vy)) - if self.num_encs == 1: - att_c, att_w = self.att[att_idx]( - exp_h[0], exp_hlens[0], self.dropout_dec[0](z_prev[0]), a_prev[0] - ) - att_w_list = [att_w] - else: - for idx in range(self.num_encs): - att_c_list[idx], att_w_list[idx] = self.att[idx]( - exp_h[idx], - exp_hlens[idx], - self.dropout_dec[0](z_prev[0]), - a_prev[idx], - ) - exp_h_han = torch.stack(att_c_list, dim=1) - att_c, att_w_list[self.num_encs] = self.att[self.num_encs]( - exp_h_han, - [self.num_encs] * n_bb, - self.dropout_dec[0](z_prev[0]), - a_prev[self.num_encs], - ) - ey = torch.cat((ey, att_c), dim=1) - - # attention decoder - z_list, c_list = self.rnn_forward(ey, z_list, c_list, z_prev, c_prev) - if self.context_residual: - logits = self.output( - torch.cat((self.dropout_dec[-1](z_list[-1]), att_c), dim=-1) - ) - else: - logits = self.output(self.dropout_dec[-1](z_list[-1])) - local_scores = att_weight * F.log_softmax(logits, dim=1) - - # rnnlm - if rnnlm: - rnnlm_state, local_lm_scores = rnnlm.buff_predict(rnnlm_state, vy, n_bb) - local_scores = local_scores + recog_args.lm_weight * local_lm_scores - - # ctc - if ctc_scorer[0]: - local_scores[:, 0] = self.logzero # avoid choosing blank - part_ids = ( - torch.topk(local_scores, scoring_num, dim=-1)[1] - if scoring_num > 0 - else None - ) - for idx in range(self.num_encs): - att_w = att_w_list[idx] - att_w_ = att_w if isinstance(att_w, torch.Tensor) else att_w[0] - local_ctc_scores, ctc_state[idx] = ctc_scorer[idx]( - yseq, ctc_state[idx], part_ids, att_w_ - ) - local_scores = ( - local_scores - + ctc_weight * weights_ctc_dec[idx] * local_ctc_scores - ) - - local_scores = local_scores.view(batch, beam, self.odim) - if i == 0: - local_scores[:, 1:, :] = self.logzero - - # accumulate scores - eos_vscores = local_scores[:, :, self.eos] + vscores - vscores = vscores.view(batch, beam, 1).repeat(1, 1, self.odim) - vscores[:, :, self.eos] = self.logzero - vscores = (vscores + local_scores).view(batch, -1) - - # global pruning - accum_best_scores, accum_best_ids = torch.topk(vscores, beam, 1) - accum_odim_ids = ( - torch.fmod(accum_best_ids, self.odim).view(-1).data.cpu().tolist() - ) - accum_padded_beam_ids = ( - (accum_best_ids // self.odim + pad_b).view(-1).data.cpu().tolist() - ) - - y_prev = yseq[:][:] - yseq = self._index_select_list(yseq, accum_padded_beam_ids) - yseq = self._append_ids(yseq, accum_odim_ids) - vscores = accum_best_scores - vidx = to_device(h[0], torch.LongTensor(accum_padded_beam_ids)) - - a_prev = [] - num_atts = self.num_encs if self.num_encs == 1 else self.num_encs + 1 - for idx in range(num_atts): - if isinstance(att_w_list[idx], torch.Tensor): - _a_prev = torch.index_select( - att_w_list[idx].view(n_bb, *att_w_list[idx].shape[1:]), 0, vidx - ) - elif isinstance(att_w_list[idx], list): - # handle the case of multi-head attention - _a_prev = [ - torch.index_select(att_w_one.view(n_bb, -1), 0, vidx) - for att_w_one in att_w_list[idx] - ] - else: - # handle the case of location_recurrent when return is a tuple - _a_prev_ = torch.index_select( - att_w_list[idx][0].view(n_bb, -1), 0, vidx - ) - _h_prev_ = torch.index_select( - att_w_list[idx][1][0].view(n_bb, -1), 0, vidx - ) - _c_prev_ = torch.index_select( - att_w_list[idx][1][1].view(n_bb, -1), 0, vidx - ) - _a_prev = (_a_prev_, (_h_prev_, _c_prev_)) - a_prev.append(_a_prev) - z_prev = [ - torch.index_select(z_list[li].view(n_bb, -1), 0, vidx) - for li in range(self.dlayers) - ] - c_prev = [ - torch.index_select(c_list[li].view(n_bb, -1), 0, vidx) - for li in range(self.dlayers) - ] - - # pick ended hyps - if i >= minlen: - k = 0 - penalty_i = (i + 1) * penalty - thr = accum_best_scores[:, -1] - for samp_i in six.moves.range(batch): - if stop_search[samp_i]: - k = k + beam - continue - for beam_j in six.moves.range(beam): - _vscore = None - if eos_vscores[samp_i, beam_j] > thr[samp_i]: - yk = y_prev[k][:] - if len(yk) <= min( - hlens[idx][samp_i] for idx in range(self.num_encs) - ): - _vscore = eos_vscores[samp_i][beam_j] + penalty_i - elif i == maxlen - 1: - yk = yseq[k][:] - _vscore = vscores[samp_i][beam_j] + penalty_i - if _vscore: - yk.append(self.eos) - if rnnlm: - _vscore += recog_args.lm_weight * rnnlm.final( - rnnlm_state, index=k - ) - _score = _vscore.data.cpu().numpy() - ended_hyps[samp_i].append( - {"yseq": yk, "vscore": _vscore, "score": _score} - ) - k = k + 1 - - # end detection - stop_search = [ - stop_search[samp_i] or end_detect(ended_hyps[samp_i], i) - for samp_i in six.moves.range(batch) - ] - stop_search_summary = list(set(stop_search)) - if len(stop_search_summary) == 1 and stop_search_summary[0]: - break - - if rnnlm: - rnnlm_state = self._index_select_lm_state(rnnlm_state, 0, vidx) - if ctc_scorer[0]: - for idx in range(self.num_encs): - ctc_state[idx] = ctc_scorer[idx].index_select_state( - ctc_state[idx], accum_best_ids - ) - - torch.cuda.empty_cache() - - dummy_hyps = [ - {"yseq": [self.sos, self.eos], "score": np.array([-float("inf")])} - ] - ended_hyps = [ - ended_hyps[samp_i] if len(ended_hyps[samp_i]) != 0 else dummy_hyps - for samp_i in six.moves.range(batch) - ] - if normalize_score: - for samp_i in six.moves.range(batch): - for x in ended_hyps[samp_i]: - x["score"] /= len(x["yseq"]) - - nbest_hyps = [ - sorted(ended_hyps[samp_i], key=lambda x: x["score"], reverse=True)[ - : min(len(ended_hyps[samp_i]), recog_args.nbest) - ] - for samp_i in six.moves.range(batch) - ] - - return nbest_hyps - - def calculate_all_attentions(self, hs_pad, hlen, ys_pad, strm_idx=0, lang_ids=None): - """Calculate all of attentions - - :param torch.Tensor hs_pad: batch of padded hidden state sequences - (B, Tmax, D) - in multi-encoder case, list of torch.Tensor, - [(B, Tmax_1, D), (B, Tmax_2, D), ..., ] ] - :param torch.Tensor hlen: batch of lengths of hidden state sequences (B) - [in multi-encoder case, list of torch.Tensor, - [(B), (B), ..., ] - :param torch.Tensor ys_pad: - batch of padded character id sequence tensor (B, Lmax) - :param int strm_idx: - stream index for parallel speaker attention in multi-speaker case - :param torch.Tensor lang_ids: batch of target language id tensor (B, 1) - :return: attention weights with the following shape, - 1) multi-head case => attention weights (B, H, Lmax, Tmax), - 2) multi-encoder case => - [(B, Lmax, Tmax1), (B, Lmax, Tmax2), ..., (B, Lmax, NumEncs)] - 3) other case => attention weights (B, Lmax, Tmax). - :rtype: float ndarray - """ - # to support mutiple encoder asr mode, in single encoder mode, - # convert torch.Tensor to List of torch.Tensor - if self.num_encs == 1: - hs_pad = [hs_pad] - hlen = [hlen] - - # TODO(kan-bayashi): need to make more smart way - ys = [y[y != self.ignore_id] for y in ys_pad] # parse padded ys - att_idx = min(strm_idx, len(self.att) - 1) - - # hlen should be list of list of integer - hlen = [list(map(int, hlen[idx])) for idx in range(self.num_encs)] - - self.loss = None - # prepare input and output word sequences with sos/eos IDs - eos = ys[0].new([self.eos]) - sos = ys[0].new([self.sos]) - if self.replace_sos: - ys_in = [torch.cat([idx, y], dim=0) for idx, y in zip(lang_ids, ys)] - else: - ys_in = [torch.cat([sos, y], dim=0) for y in ys] - ys_out = [torch.cat([y, eos], dim=0) for y in ys] - - # padding for ys with -1 - # pys: utt x olen - ys_in_pad = pad_list(ys_in, self.eos) - ys_out_pad = pad_list(ys_out, self.ignore_id) - - # get length info - olength = ys_out_pad.size(1) - - # initialization - c_list = [self.zero_state(hs_pad[0])] - z_list = [self.zero_state(hs_pad[0])] - for _ in six.moves.range(1, self.dlayers): - c_list.append(self.zero_state(hs_pad[0])) - z_list.append(self.zero_state(hs_pad[0])) - att_ws = [] - if self.num_encs == 1: - att_w = None - self.att[att_idx].reset() # reset pre-computation of h - else: - att_w_list = [None] * (self.num_encs + 1) # atts + han - att_c_list = [None] * (self.num_encs) # atts - for idx in range(self.num_encs + 1): - self.att[idx].reset() # reset pre-computation of h in atts and han - - # pre-computation of embedding - eys = self.dropout_emb(self.embed(ys_in_pad)) # utt x olen x zdim - - # loop for an output sequence - for i in six.moves.range(olength): - if self.num_encs == 1: - att_c, att_w = self.att[att_idx]( - hs_pad[0], hlen[0], self.dropout_dec[0](z_list[0]), att_w - ) - att_ws.append(att_w) - else: - for idx in range(self.num_encs): - att_c_list[idx], att_w_list[idx] = self.att[idx]( - hs_pad[idx], - hlen[idx], - self.dropout_dec[0](z_list[0]), - att_w_list[idx], - ) - hs_pad_han = torch.stack(att_c_list, dim=1) - hlen_han = [self.num_encs] * len(ys_in) - att_c, att_w_list[self.num_encs] = self.att[self.num_encs]( - hs_pad_han, - hlen_han, - self.dropout_dec[0](z_list[0]), - att_w_list[self.num_encs], - ) - att_ws.append(att_w_list.copy()) - ey = torch.cat((eys[:, i, :], att_c), dim=1) # utt x (zdim + hdim) - z_list, c_list = self.rnn_forward(ey, z_list, c_list, z_list, c_list) - - if self.num_encs == 1: - # convert to numpy array with the shape (B, Lmax, Tmax) - att_ws = att_to_numpy(att_ws, self.att[att_idx]) - else: - _att_ws = [] - for idx, ws in enumerate(zip(*att_ws)): - ws = att_to_numpy(ws, self.att[idx]) - _att_ws.append(ws) - att_ws = _att_ws - return att_ws - - @staticmethod - def _get_last_yseq(exp_yseq): - last = [] - for y_seq in exp_yseq: - last.append(y_seq[-1]) - return last - - @staticmethod - def _append_ids(yseq, ids): - if isinstance(ids, list): - for i, j in enumerate(ids): - yseq[i].append(j) - else: - for i in range(len(yseq)): - yseq[i].append(ids) - return yseq - - @staticmethod - def _index_select_list(yseq, lst): - new_yseq = [] - for i in lst: - new_yseq.append(yseq[i][:]) - return new_yseq - - @staticmethod - def _index_select_lm_state(rnnlm_state, dim, vidx): - if isinstance(rnnlm_state, dict): - new_state = {} - for k, v in rnnlm_state.items(): - new_state[k] = [torch.index_select(vi, dim, vidx) for vi in v] - elif isinstance(rnnlm_state, list): - new_state = [] - for i in vidx: - new_state.append(rnnlm_state[int(i)][:]) - return new_state - - # scorer interface methods - def init_state(self, x): - # to support mutiple encoder asr mode, in single encoder mode, - # convert torch.Tensor to List of torch.Tensor - if self.num_encs == 1: - x = [x] - - c_list = [self.zero_state(x[0].unsqueeze(0))] - z_list = [self.zero_state(x[0].unsqueeze(0))] - for _ in six.moves.range(1, self.dlayers): - c_list.append(self.zero_state(x[0].unsqueeze(0))) - z_list.append(self.zero_state(x[0].unsqueeze(0))) - # TODO(karita): support strm_index for `asr_mix` - strm_index = 0 - att_idx = min(strm_index, len(self.att) - 1) - if self.num_encs == 1: - a = None - self.att[att_idx].reset() # reset pre-computation of h - else: - a = [None] * (self.num_encs + 1) # atts + han - for idx in range(self.num_encs + 1): - self.att[idx].reset() # reset pre-computation of h in atts and han - return dict( - c_prev=c_list[:], - z_prev=z_list[:], - a_prev=a, - workspace=(att_idx, z_list, c_list), - ) - - def score(self, yseq, state, x): - # to support mutiple encoder asr mode, in single encoder mode, - # convert torch.Tensor to List of torch.Tensor - if self.num_encs == 1: - x = [x] - - att_idx, z_list, c_list = state["workspace"] - vy = yseq[-1].unsqueeze(0) - ey = self.dropout_emb(self.embed(vy)) # utt list (1) x zdim - if self.num_encs == 1: - att_c, att_w = self.att[att_idx]( - x[0].unsqueeze(0), - [x[0].size(0)], - self.dropout_dec[0](state["z_prev"][0]), - state["a_prev"], - ) - else: - att_w = [None] * (self.num_encs + 1) # atts + han - att_c_list = [None] * (self.num_encs) # atts - for idx in range(self.num_encs): - att_c_list[idx], att_w[idx] = self.att[idx]( - x[idx].unsqueeze(0), - [x[idx].size(0)], - self.dropout_dec[0](state["z_prev"][0]), - state["a_prev"][idx], - ) - h_han = torch.stack(att_c_list, dim=1) - att_c, att_w[self.num_encs] = self.att[self.num_encs]( - h_han, - [self.num_encs], - self.dropout_dec[0](state["z_prev"][0]), - state["a_prev"][self.num_encs], - ) - ey = torch.cat((ey, att_c), dim=1) # utt(1) x (zdim + hdim) - z_list, c_list = self.rnn_forward( - ey, z_list, c_list, state["z_prev"], state["c_prev"] - ) - if self.context_residual: - logits = self.output( - torch.cat((self.dropout_dec[-1](z_list[-1]), att_c), dim=-1) - ) - else: - logits = self.output(self.dropout_dec[-1](z_list[-1])) - logp = F.log_softmax(logits, dim=1).squeeze(0) - return ( - logp, - dict( - c_prev=c_list[:], - z_prev=z_list[:], - a_prev=att_w, - workspace=(att_idx, z_list, c_list), - ), - ) - - -def decoder_for(args, odim, sos, eos, att, labeldist): - return Decoder( - args.eprojs, - odim, - args.dtype, - args.dlayers, - args.dunits, - sos, - eos, - att, - args.verbose, - args.char_list, - labeldist, - args.lsm_weight, - args.sampling_probability, - args.dropout_rate_decoder, - getattr(args, "context_residual", False), # use getattr to keep compatibility - getattr(args, "replace_sos", False), # use getattr to keep compatibility - getattr(args, "num_encs", 1), - ) # use getattr to keep compatibility diff --git a/funasr_detach/models/language_model/rnn/encoders.py b/funasr_detach/models/language_model/rnn/encoders.py deleted file mode 100644 index e75687e480733aa3aa9859feff198430c75133b6..0000000000000000000000000000000000000000 --- a/funasr_detach/models/language_model/rnn/encoders.py +++ /dev/null @@ -1,372 +0,0 @@ -import logging - -import numpy as np -import six -import torch -import torch.nn.functional as F -from torch.nn.utils.rnn import pack_padded_sequence -from torch.nn.utils.rnn import pad_packed_sequence - -from funasr_detach.metrics.common import get_vgg2l_odim -from funasr_detach.models.transformer.utils.nets_utils import make_pad_mask -from funasr_detach.models.transformer.utils.nets_utils import to_device - - -class RNNP(torch.nn.Module): - """RNN with projection layer module - - :param int idim: dimension of inputs - :param int elayers: number of encoder layers - :param int cdim: number of rnn units (resulted in cdim * 2 if bidirectional) - :param int hdim: number of projection units - :param np.ndarray subsample: list of subsampling numbers - :param float dropout: dropout rate - :param str typ: The RNN type - """ - - def __init__(self, idim, elayers, cdim, hdim, subsample, dropout, typ="blstm"): - super(RNNP, self).__init__() - bidir = typ[0] == "b" - for i in six.moves.range(elayers): - if i == 0: - inputdim = idim - else: - inputdim = hdim - - RNN = torch.nn.LSTM if "lstm" in typ else torch.nn.GRU - rnn = RNN( - inputdim, cdim, num_layers=1, bidirectional=bidir, batch_first=True - ) - - setattr(self, "%s%d" % ("birnn" if bidir else "rnn", i), rnn) - - # bottleneck layer to merge - if bidir: - setattr(self, "bt%d" % i, torch.nn.Linear(2 * cdim, hdim)) - else: - setattr(self, "bt%d" % i, torch.nn.Linear(cdim, hdim)) - - self.elayers = elayers - self.cdim = cdim - self.subsample = subsample - self.typ = typ - self.bidir = bidir - self.dropout = dropout - - def forward(self, xs_pad, ilens, prev_state=None): - """RNNP forward - - :param torch.Tensor xs_pad: batch of padded input sequences (B, Tmax, idim) - :param torch.Tensor ilens: batch of lengths of input sequences (B) - :param torch.Tensor prev_state: batch of previous RNN states - :return: batch of hidden state sequences (B, Tmax, hdim) - :rtype: torch.Tensor - """ - logging.debug(self.__class__.__name__ + " input lengths: " + str(ilens)) - elayer_states = [] - for layer in six.moves.range(self.elayers): - if not isinstance(ilens, torch.Tensor): - ilens = torch.tensor(ilens) - xs_pack = pack_padded_sequence(xs_pad, ilens.cpu(), batch_first=True) - rnn = getattr(self, ("birnn" if self.bidir else "rnn") + str(layer)) - rnn.flatten_parameters() - if prev_state is not None and rnn.bidirectional: - prev_state = reset_backward_rnn_state(prev_state) - ys, states = rnn( - xs_pack, hx=None if prev_state is None else prev_state[layer] - ) - elayer_states.append(states) - # ys: utt list of frame x cdim x 2 (2: means bidirectional) - ys_pad, ilens = pad_packed_sequence(ys, batch_first=True) - sub = self.subsample[layer + 1] - if sub > 1: - ys_pad = ys_pad[:, ::sub] - ilens = torch.tensor([int(i + 1) // sub for i in ilens]) - # (sum _utt frame_utt) x dim - projection_layer = getattr(self, "bt%d" % layer) - projected = projection_layer(ys_pad.contiguous().view(-1, ys_pad.size(2))) - xs_pad = projected.view(ys_pad.size(0), ys_pad.size(1), -1) - if layer < self.elayers - 1: - xs_pad = torch.tanh(F.dropout(xs_pad, p=self.dropout)) - - return xs_pad, ilens, elayer_states # x: utt list of frame x dim - - -class RNN(torch.nn.Module): - """RNN module - - :param int idim: dimension of inputs - :param int elayers: number of encoder layers - :param int cdim: number of rnn units (resulted in cdim * 2 if bidirectional) - :param int hdim: number of final projection units - :param float dropout: dropout rate - :param str typ: The RNN type - """ - - def __init__(self, idim, elayers, cdim, hdim, dropout, typ="blstm"): - super(RNN, self).__init__() - bidir = typ[0] == "b" - self.nbrnn = ( - torch.nn.LSTM( - idim, - cdim, - elayers, - batch_first=True, - dropout=dropout, - bidirectional=bidir, - ) - if "lstm" in typ - else torch.nn.GRU( - idim, - cdim, - elayers, - batch_first=True, - dropout=dropout, - bidirectional=bidir, - ) - ) - if bidir: - self.l_last = torch.nn.Linear(cdim * 2, hdim) - else: - self.l_last = torch.nn.Linear(cdim, hdim) - self.typ = typ - - def forward(self, xs_pad, ilens, prev_state=None): - """RNN forward - - :param torch.Tensor xs_pad: batch of padded input sequences (B, Tmax, D) - :param torch.Tensor ilens: batch of lengths of input sequences (B) - :param torch.Tensor prev_state: batch of previous RNN states - :return: batch of hidden state sequences (B, Tmax, eprojs) - :rtype: torch.Tensor - """ - logging.debug(self.__class__.__name__ + " input lengths: " + str(ilens)) - if not isinstance(ilens, torch.Tensor): - ilens = torch.tensor(ilens) - xs_pack = pack_padded_sequence(xs_pad, ilens.cpu(), batch_first=True) - self.nbrnn.flatten_parameters() - if prev_state is not None and self.nbrnn.bidirectional: - # We assume that when previous state is passed, - # it means that we're streaming the input - # and therefore cannot propagate backward BRNN state - # (otherwise it goes in the wrong direction) - prev_state = reset_backward_rnn_state(prev_state) - ys, states = self.nbrnn(xs_pack, hx=prev_state) - # ys: utt list of frame x cdim x 2 (2: means bidirectional) - ys_pad, ilens = pad_packed_sequence(ys, batch_first=True) - # (sum _utt frame_utt) x dim - projected = torch.tanh( - self.l_last(ys_pad.contiguous().view(-1, ys_pad.size(2))) - ) - xs_pad = projected.view(ys_pad.size(0), ys_pad.size(1), -1) - return xs_pad, ilens, states # x: utt list of frame x dim - - -def reset_backward_rnn_state(states): - """Sets backward BRNN states to zeroes - - Useful in processing of sliding windows over the inputs - """ - if isinstance(states, (list, tuple)): - for state in states: - state[1::2] = 0.0 - else: - states[1::2] = 0.0 - return states - - -class VGG2L(torch.nn.Module): - """VGG-like module - - :param int in_channel: number of input channels - """ - - def __init__(self, in_channel=1): - super(VGG2L, self).__init__() - # CNN layer (VGG motivated) - self.conv1_1 = torch.nn.Conv2d(in_channel, 64, 3, stride=1, padding=1) - self.conv1_2 = torch.nn.Conv2d(64, 64, 3, stride=1, padding=1) - self.conv2_1 = torch.nn.Conv2d(64, 128, 3, stride=1, padding=1) - self.conv2_2 = torch.nn.Conv2d(128, 128, 3, stride=1, padding=1) - - self.in_channel = in_channel - - def forward(self, xs_pad, ilens, **kwargs): - """VGG2L forward - - :param torch.Tensor xs_pad: batch of padded input sequences (B, Tmax, D) - :param torch.Tensor ilens: batch of lengths of input sequences (B) - :return: batch of padded hidden state sequences (B, Tmax // 4, 128 * D // 4) - :rtype: torch.Tensor - """ - logging.debug(self.__class__.__name__ + " input lengths: " + str(ilens)) - - # x: utt x frame x dim - # xs_pad = F.pad_sequence(xs_pad) - - # x: utt x 1 (input channel num) x frame x dim - xs_pad = xs_pad.view( - xs_pad.size(0), - xs_pad.size(1), - self.in_channel, - xs_pad.size(2) // self.in_channel, - ).transpose(1, 2) - - # NOTE: max_pool1d ? - xs_pad = F.relu(self.conv1_1(xs_pad)) - xs_pad = F.relu(self.conv1_2(xs_pad)) - xs_pad = F.max_pool2d(xs_pad, 2, stride=2, ceil_mode=True) - - xs_pad = F.relu(self.conv2_1(xs_pad)) - xs_pad = F.relu(self.conv2_2(xs_pad)) - xs_pad = F.max_pool2d(xs_pad, 2, stride=2, ceil_mode=True) - if torch.is_tensor(ilens): - ilens = ilens.cpu().numpy() - else: - ilens = np.array(ilens, dtype=np.float32) - ilens = np.array(np.ceil(ilens / 2), dtype=np.int64) - ilens = np.array( - np.ceil(np.array(ilens, dtype=np.float32) / 2), dtype=np.int64 - ).tolist() - - # x: utt_list of frame (remove zeropaded frames) x (input channel num x dim) - xs_pad = xs_pad.transpose(1, 2) - xs_pad = xs_pad.contiguous().view( - xs_pad.size(0), xs_pad.size(1), xs_pad.size(2) * xs_pad.size(3) - ) - return xs_pad, ilens, None # no state in this layer - - -class Encoder(torch.nn.Module): - """Encoder module - - :param str etype: type of encoder network - :param int idim: number of dimensions of encoder network - :param int elayers: number of layers of encoder network - :param int eunits: number of lstm units of encoder network - :param int eprojs: number of projection units of encoder network - :param np.ndarray subsample: list of subsampling numbers - :param float dropout: dropout rate - :param int in_channel: number of input channels - """ - - def __init__( - self, etype, idim, elayers, eunits, eprojs, subsample, dropout, in_channel=1 - ): - super(Encoder, self).__init__() - typ = etype.lstrip("vgg").rstrip("p") - if typ not in ["lstm", "gru", "blstm", "bgru"]: - logging.error("Error: need to specify an appropriate encoder architecture") - - if etype.startswith("vgg"): - if etype[-1] == "p": - self.enc = torch.nn.ModuleList( - [ - VGG2L(in_channel), - RNNP( - get_vgg2l_odim(idim, in_channel=in_channel), - elayers, - eunits, - eprojs, - subsample, - dropout, - typ=typ, - ), - ] - ) - logging.info("Use CNN-VGG + " + typ.upper() + "P for encoder") - else: - self.enc = torch.nn.ModuleList( - [ - VGG2L(in_channel), - RNN( - get_vgg2l_odim(idim, in_channel=in_channel), - elayers, - eunits, - eprojs, - dropout, - typ=typ, - ), - ] - ) - logging.info("Use CNN-VGG + " + typ.upper() + " for encoder") - self.conv_subsampling_factor = 4 - else: - if etype[-1] == "p": - self.enc = torch.nn.ModuleList( - [RNNP(idim, elayers, eunits, eprojs, subsample, dropout, typ=typ)] - ) - logging.info(typ.upper() + " with every-layer projection for encoder") - else: - self.enc = torch.nn.ModuleList( - [RNN(idim, elayers, eunits, eprojs, dropout, typ=typ)] - ) - logging.info(typ.upper() + " without projection for encoder") - self.conv_subsampling_factor = 1 - - def forward(self, xs_pad, ilens, prev_states=None): - """Encoder forward - - :param torch.Tensor xs_pad: batch of padded input sequences (B, Tmax, D) - :param torch.Tensor ilens: batch of lengths of input sequences (B) - :param torch.Tensor prev_state: batch of previous encoder hidden states (?, ...) - :return: batch of hidden state sequences (B, Tmax, eprojs) - :rtype: torch.Tensor - """ - if prev_states is None: - prev_states = [None] * len(self.enc) - assert len(prev_states) == len(self.enc) - - current_states = [] - for module, prev_state in zip(self.enc, prev_states): - xs_pad, ilens, states = module(xs_pad, ilens, prev_state=prev_state) - current_states.append(states) - - # make mask to remove bias value in padded part - mask = to_device(xs_pad, make_pad_mask(ilens).unsqueeze(-1)) - - return xs_pad.masked_fill(mask, 0.0), ilens, current_states - - -def encoder_for(args, idim, subsample): - """Instantiates an encoder module given the program arguments - - :param Namespace args: The arguments - :param int or List of integer idim: dimension of input, e.g. 83, or - List of dimensions of inputs, e.g. [83,83] - :param List or List of List subsample: subsample factors, e.g. [1,2,2,1,1], or - List of subsample factors of each encoder. - e.g. [[1,2,2,1,1], [1,2,2,1,1]] - :rtype torch.nn.Module - :return: The encoder module - """ - num_encs = getattr(args, "num_encs", 1) # use getattr to keep compatibility - if num_encs == 1: - # compatible with single encoder asr mode - return Encoder( - args.etype, - idim, - args.elayers, - args.eunits, - args.eprojs, - subsample, - args.dropout_rate, - ) - elif num_encs >= 1: - enc_list = torch.nn.ModuleList() - for idx in range(num_encs): - enc = Encoder( - args.etype[idx], - idim[idx], - args.elayers[idx], - args.eunits[idx], - args.eprojs, - subsample[idx], - args.dropout_rate[idx], - ) - enc_list.append(enc) - return enc_list - else: - raise ValueError( - "Number of encoders needs to be more than one. {}".format(num_encs) - ) diff --git a/funasr_detach/models/language_model/seq_rnn_lm.py b/funasr_detach/models/language_model/seq_rnn_lm.py deleted file mode 100644 index c143eac8ef58d630e576c2a8e6c3bfc0ee5e9b0e..0000000000000000000000000000000000000000 --- a/funasr_detach/models/language_model/seq_rnn_lm.py +++ /dev/null @@ -1,172 +0,0 @@ -"""Sequential implementation of Recurrent Neural Network Language Model.""" - -from typing import Tuple -from typing import Union - -import torch -import torch.nn as nn -from funasr_detach.train.abs_model import AbsLM - - -class SequentialRNNLM(AbsLM): - """Sequential RNNLM. - - See also: - https://github.com/pytorch/examples/blob/4581968193699de14b56527296262dd76ab43557/word_language_model/model.py - - """ - - def __init__( - self, - vocab_size: int, - unit: int = 650, - nhid: int = None, - nlayers: int = 2, - dropout_rate: float = 0.0, - tie_weights: bool = False, - rnn_type: str = "lstm", - ignore_id: int = 0, - ): - super().__init__() - - ninp = unit - if nhid is None: - nhid = unit - rnn_type = rnn_type.upper() - - self.drop = nn.Dropout(dropout_rate) - self.encoder = nn.Embedding(vocab_size, ninp, padding_idx=ignore_id) - if rnn_type in ["LSTM", "GRU"]: - rnn_class = getattr(nn, rnn_type) - self.rnn = rnn_class( - ninp, nhid, nlayers, dropout=dropout_rate, batch_first=True - ) - else: - try: - nonlinearity = {"RNN_TANH": "tanh", "RNN_RELU": "relu"}[rnn_type] - except KeyError: - raise ValueError( - """An invalid option for `--model` was supplied, - options are ['LSTM', 'GRU', 'RNN_TANH' or 'RNN_RELU']""" - ) - self.rnn = nn.RNN( - ninp, - nhid, - nlayers, - nonlinearity=nonlinearity, - dropout=dropout_rate, - batch_first=True, - ) - self.decoder = nn.Linear(nhid, vocab_size) - - # Optionally tie weights as in: - # "Using the Output Embedding to Improve Language Models" - # (Press & Wolf 2016) https://arxiv.org/abs/1608.05859 - # and - # "Tying Word Vectors and Word Classifiers: - # A Loss Framework for Language Modeling" (Inan et al. 2016) - # https://arxiv.org/abs/1611.01462 - if tie_weights: - if nhid != ninp: - raise ValueError( - "When using the tied flag, nhid must be equal to emsize" - ) - self.decoder.weight = self.encoder.weight - - self.rnn_type = rnn_type - self.nhid = nhid - self.nlayers = nlayers - - def zero_state(self): - """Initialize LM state filled with zero values.""" - if isinstance(self.rnn, torch.nn.LSTM): - h = torch.zeros((self.nlayers, self.nhid), dtype=torch.float) - c = torch.zeros((self.nlayers, self.nhid), dtype=torch.float) - state = h, c - else: - state = torch.zeros((self.nlayers, self.nhid), dtype=torch.float) - - return state - - def forward( - self, input: torch.Tensor, hidden: torch.Tensor - ) -> Tuple[torch.Tensor, torch.Tensor]: - emb = self.drop(self.encoder(input)) - output, hidden = self.rnn(emb, hidden) - output = self.drop(output) - decoded = self.decoder( - output.contiguous().view(output.size(0) * output.size(1), output.size(2)) - ) - return ( - decoded.view(output.size(0), output.size(1), decoded.size(1)), - hidden, - ) - - def score( - self, - y: torch.Tensor, - state: Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]], - x: torch.Tensor, - ) -> Tuple[torch.Tensor, Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]]: - """Score new token. - - Args: - y: 1D torch.int64 prefix tokens. - state: Scorer state for prefix tokens - x: 2D encoder feature that generates ys. - - Returns: - Tuple of - torch.float32 scores for next token (n_vocab) - and next state for ys - - """ - y, new_state = self(y[-1].view(1, 1), state) - logp = y.log_softmax(dim=-1).view(-1) - return logp, new_state - - def batch_score( - self, ys: torch.Tensor, states: torch.Tensor, xs: torch.Tensor - ) -> Tuple[torch.Tensor, torch.Tensor]: - """Score new token batch. - - Args: - ys (torch.Tensor): torch.int64 prefix tokens (n_batch, ylen). - states (List[Any]): Scorer states for prefix tokens. - xs (torch.Tensor): - The encoder feature that generates ys (n_batch, xlen, n_feat). - - Returns: - tuple[torch.Tensor, List[Any]]: Tuple of - batchfied scores for next token with shape of `(n_batch, n_vocab)` - and next state list for ys. - - """ - if states[0] is None: - states = None - elif isinstance(self.rnn, torch.nn.LSTM): - # states: Batch x 2 x (Nlayers, Dim) -> 2 x (Nlayers, Batch, Dim) - h = torch.stack([h for h, c in states], dim=1) - c = torch.stack([c for h, c in states], dim=1) - states = h, c - else: - # states: Batch x (Nlayers, Dim) -> (Nlayers, Batch, Dim) - states = torch.stack(states, dim=1) - - ys, states = self(ys[:, -1:], states) - # ys: (Batch, 1, Nvocab) -> (Batch, NVocab) - assert ys.size(1) == 1, ys.shape - ys = ys.squeeze(1) - logp = ys.log_softmax(dim=-1) - - # state: Change to batch first - if isinstance(self.rnn, torch.nn.LSTM): - # h, c: (Nlayers, Batch, Dim) - h, c = states - # states: Batch x 2 x (Nlayers, Dim) - states = [(h[:, i], c[:, i]) for i in range(h.size(1))] - else: - # states: (Nlayers, Batch, Dim) -> Batch x (Nlayers, Dim) - states = [states[:, i] for i in range(states.size(1))] - - return logp, states diff --git a/funasr_detach/models/language_model/transformer_encoder.py b/funasr_detach/models/language_model/transformer_encoder.py deleted file mode 100644 index 967e810e79dcfe65165c9eb8e85799cd66503bdb..0000000000000000000000000000000000000000 --- a/funasr_detach/models/language_model/transformer_encoder.py +++ /dev/null @@ -1,468 +0,0 @@ -# Copyright 2019 Shigeki Karita -# Apache 2.0 (http://www.apache.org/licenses/LICENSE-2.0) - -"""Transformer encoder definition.""" - -from typing import List -from typing import Optional -from typing import Tuple - -import torch -from torch import nn -import logging - -from funasr_detach.models.transformer.attention import MultiHeadedAttention -from funasr_detach.models.transformer.embedding import PositionalEncoding -from funasr_detach.models.transformer.layer_norm import LayerNorm -from funasr_detach.models.transformer.utils.multi_layer_conv import Conv1dLinear -from funasr_detach.models.transformer.utils.multi_layer_conv import MultiLayeredConv1d -from funasr_detach.models.transformer.utils.nets_utils import make_pad_mask -from funasr_detach.models.transformer.positionwise_feed_forward import ( - PositionwiseFeedForward, # noqa: H301 -) -from funasr_detach.models.transformer.utils.repeat import repeat -from funasr_detach.models.transformer.utils.dynamic_conv import DynamicConvolution -from funasr_detach.models.transformer.utils.dynamic_conv2d import DynamicConvolution2D -from funasr_detach.models.transformer.utils.lightconv import LightweightConvolution -from funasr_detach.models.transformer.utils.lightconv2d import LightweightConvolution2D -from funasr_detach.models.transformer.utils.subsampling import Conv2dSubsampling -from funasr_detach.models.transformer.utils.subsampling import Conv2dSubsampling2 -from funasr_detach.models.transformer.utils.subsampling import Conv2dSubsampling6 -from funasr_detach.models.transformer.utils.subsampling import Conv2dSubsampling8 -from funasr_detach.models.transformer.utils.subsampling import TooShortUttError -from funasr_detach.models.transformer.utils.subsampling import check_short_utt - - -class EncoderLayer(nn.Module): - """Encoder layer module. - - Args: - size (int): Input dimension. - self_attn (torch.nn.Module): Self-attention module instance. - `MultiHeadedAttention` or `RelPositionMultiHeadedAttention` instance - can be used as the argument. - feed_forward (torch.nn.Module): Feed-forward module instance. - `PositionwiseFeedForward`, `MultiLayeredConv1d`, or `Conv1dLinear` instance - can be used as the argument. - dropout_rate (float): Dropout rate. - normalize_before (bool): Whether to use layer_norm before the first block. - concat_after (bool): Whether to concat attention layer's input and output. - if True, additional linear will be applied. - i.e. x -> x + linear(concat(x, att(x))) - if False, no additional linear will be applied. i.e. x -> x + att(x) - stochastic_depth_rate (float): Proability to skip this layer. - During training, the layer may skip residual computation and return input - as-is with given probability. - """ - - def __init__( - self, - size, - self_attn, - feed_forward, - dropout_rate, - normalize_before=True, - concat_after=False, - stochastic_depth_rate=0.0, - ): - """Construct an EncoderLayer object.""" - super(EncoderLayer, self).__init__() - self.self_attn = self_attn - self.feed_forward = feed_forward - self.norm1 = LayerNorm(size) - self.norm2 = LayerNorm(size) - self.dropout = nn.Dropout(dropout_rate) - self.size = size - self.normalize_before = normalize_before - self.concat_after = concat_after - if self.concat_after: - self.concat_linear = nn.Linear(size + size, size) - self.stochastic_depth_rate = stochastic_depth_rate - - def forward(self, x, mask, cache=None): - """Compute encoded features. - - Args: - x_input (torch.Tensor): Input tensor (#batch, time, size). - mask (torch.Tensor): Mask tensor for the input (#batch, time). - cache (torch.Tensor): Cache tensor of the input (#batch, time - 1, size). - - Returns: - torch.Tensor: Output tensor (#batch, time, size). - torch.Tensor: Mask tensor (#batch, time). - - """ - skip_layer = False - # with stochastic depth, residual connection `x + f(x)` becomes - # `x <- x + 1 / (1 - p) * f(x)` at training time. - stoch_layer_coeff = 1.0 - if self.training and self.stochastic_depth_rate > 0: - skip_layer = torch.rand(1).item() < self.stochastic_depth_rate - stoch_layer_coeff = 1.0 / (1 - self.stochastic_depth_rate) - - if skip_layer: - if cache is not None: - x = torch.cat([cache, x], dim=1) - return x, mask - - residual = x - if self.normalize_before: - x = self.norm1(x) - - if cache is None: - x_q = x - else: - assert cache.shape == (x.shape[0], x.shape[1] - 1, self.size) - x_q = x[:, -1:, :] - residual = residual[:, -1:, :] - mask = None if mask is None else mask[:, -1:, :] - - if self.concat_after: - x_concat = torch.cat((x, self.self_attn(x_q, x, x, mask)), dim=-1) - x = residual + stoch_layer_coeff * self.concat_linear(x_concat) - else: - x = residual + stoch_layer_coeff * self.dropout( - self.self_attn(x_q, x, x, mask) - ) - if not self.normalize_before: - x = self.norm1(x) - - residual = x - if self.normalize_before: - x = self.norm2(x) - x = residual + stoch_layer_coeff * self.dropout(self.feed_forward(x)) - if not self.normalize_before: - x = self.norm2(x) - - if cache is not None: - x = torch.cat([cache, x], dim=1) - - return x, mask - - -class TransformerEncoder_lm(nn.Module): - """Transformer encoder module. - - Args: - idim (int): Input dimension. - attention_dim (int): Dimension of attention. - attention_heads (int): The number of heads of multi head attention. - conv_wshare (int): The number of kernel of convolution. Only used in - selfattention_layer_type == "lightconv*" or "dynamiconv*". - conv_kernel_length (Union[int, str]): Kernel size str of convolution - (e.g. 71_71_71_71_71_71). Only used in selfattention_layer_type - == "lightconv*" or "dynamiconv*". - conv_usebias (bool): Whether to use bias in convolution. Only used in - selfattention_layer_type == "lightconv*" or "dynamiconv*". - linear_units (int): The number of units of position-wise feed forward. - num_blocks (int): The number of decoder blocks. - dropout_rate (float): Dropout rate. - positional_dropout_rate (float): Dropout rate after adding positional encoding. - attention_dropout_rate (float): Dropout rate in attention. - input_layer (Union[str, torch.nn.Module]): Input layer type. - pos_enc_class (torch.nn.Module): Positional encoding module class. - `PositionalEncoding `or `ScaledPositionalEncoding` - normalize_before (bool): Whether to use layer_norm before the first block. - concat_after (bool): Whether to concat attention layer's input and output. - if True, additional linear will be applied. - i.e. x -> x + linear(concat(x, att(x))) - if False, no additional linear will be applied. i.e. x -> x + att(x) - positionwise_layer_type (str): "linear", "conv1d", or "conv1d-linear". - positionwise_conv_kernel_size (int): Kernel size of positionwise conv1d layer. - selfattention_layer_type (str): Encoder attention layer type. - padding_idx (int): Padding idx for input_layer=embed. - stochastic_depth_rate (float): Maximum probability to skip the encoder layer. - intermediate_layers (Union[List[int], None]): indices of intermediate CTC layer. - indices start from 1. - if not None, intermediate outputs are returned (which changes return type - signature.) - - """ - - def __init__( - self, - idim, - attention_dim=256, - attention_heads=4, - conv_wshare=4, - conv_kernel_length="11", - conv_usebias=False, - linear_units=2048, - num_blocks=6, - dropout_rate=0.1, - positional_dropout_rate=0.1, - attention_dropout_rate=0.0, - input_layer="conv2d", - pos_enc_class=PositionalEncoding, - normalize_before=True, - concat_after=False, - positionwise_layer_type="linear", - positionwise_conv_kernel_size=1, - selfattention_layer_type="selfattn", - padding_idx=-1, - stochastic_depth_rate=0.0, - intermediate_layers=None, - ctc_softmax=None, - conditioning_layer_dim=None, - ): - """Construct an Encoder object.""" - super().__init__() - - self.conv_subsampling_factor = 1 - if input_layer == "linear": - self.embed = torch.nn.Sequential( - torch.nn.Linear(idim, attention_dim), - torch.nn.LayerNorm(attention_dim), - torch.nn.Dropout(dropout_rate), - torch.nn.ReLU(), - pos_enc_class(attention_dim, positional_dropout_rate), - ) - elif input_layer == "conv2d": - self.embed = Conv2dSubsampling(idim, attention_dim, dropout_rate) - self.conv_subsampling_factor = 4 - elif input_layer == "conv2d-scaled-pos-enc": - self.embed = Conv2dSubsampling( - idim, - attention_dim, - dropout_rate, - pos_enc_class(attention_dim, positional_dropout_rate), - ) - self.conv_subsampling_factor = 4 - elif input_layer == "conv2d6": - self.embed = Conv2dSubsampling6(idim, attention_dim, dropout_rate) - self.conv_subsampling_factor = 6 - elif input_layer == "conv2d8": - self.embed = Conv2dSubsampling8(idim, attention_dim, dropout_rate) - self.conv_subsampling_factor = 8 - elif input_layer == "embed": - self.embed = torch.nn.Sequential( - torch.nn.Embedding(idim, attention_dim, padding_idx=padding_idx), - pos_enc_class(attention_dim, positional_dropout_rate), - ) - elif isinstance(input_layer, torch.nn.Module): - self.embed = torch.nn.Sequential( - input_layer, - pos_enc_class(attention_dim, positional_dropout_rate), - ) - elif input_layer is None: - self.embed = torch.nn.Sequential( - pos_enc_class(attention_dim, positional_dropout_rate) - ) - else: - raise ValueError("unknown input_layer: " + input_layer) - self.normalize_before = normalize_before - positionwise_layer, positionwise_layer_args = self.get_positionwise_layer( - positionwise_layer_type, - attention_dim, - linear_units, - dropout_rate, - positionwise_conv_kernel_size, - ) - if selfattention_layer_type in [ - "selfattn", - "rel_selfattn", - "legacy_rel_selfattn", - ]: - logging.info("encoder self-attention layer type = self-attention") - encoder_selfattn_layer = MultiHeadedAttention - encoder_selfattn_layer_args = [ - ( - attention_heads, - attention_dim, - attention_dropout_rate, - ) - ] * num_blocks - elif selfattention_layer_type == "lightconv": - logging.info("encoder self-attention layer type = lightweight convolution") - encoder_selfattn_layer = LightweightConvolution - encoder_selfattn_layer_args = [ - ( - conv_wshare, - attention_dim, - attention_dropout_rate, - int(conv_kernel_length.split("_")[lnum]), - False, - conv_usebias, - ) - for lnum in range(num_blocks) - ] - elif selfattention_layer_type == "lightconv2d": - logging.info( - "encoder self-attention layer " - "type = lightweight convolution 2-dimensional" - ) - encoder_selfattn_layer = LightweightConvolution2D - encoder_selfattn_layer_args = [ - ( - conv_wshare, - attention_dim, - attention_dropout_rate, - int(conv_kernel_length.split("_")[lnum]), - False, - conv_usebias, - ) - for lnum in range(num_blocks) - ] - elif selfattention_layer_type == "dynamicconv": - logging.info("encoder self-attention layer type = dynamic convolution") - encoder_selfattn_layer = DynamicConvolution - encoder_selfattn_layer_args = [ - ( - conv_wshare, - attention_dim, - attention_dropout_rate, - int(conv_kernel_length.split("_")[lnum]), - False, - conv_usebias, - ) - for lnum in range(num_blocks) - ] - elif selfattention_layer_type == "dynamicconv2d": - logging.info( - "encoder self-attention layer type = dynamic convolution 2-dimensional" - ) - encoder_selfattn_layer = DynamicConvolution2D - encoder_selfattn_layer_args = [ - ( - conv_wshare, - attention_dim, - attention_dropout_rate, - int(conv_kernel_length.split("_")[lnum]), - False, - conv_usebias, - ) - for lnum in range(num_blocks) - ] - else: - raise NotImplementedError(selfattention_layer_type) - - self.encoders = repeat( - num_blocks, - lambda lnum: EncoderLayer( - attention_dim, - encoder_selfattn_layer(*encoder_selfattn_layer_args[lnum]), - positionwise_layer(*positionwise_layer_args), - dropout_rate, - normalize_before, - concat_after, - stochastic_depth_rate * float(1 + lnum) / num_blocks, - ), - ) - if self.normalize_before: - self.after_norm = LayerNorm(attention_dim) - - self.intermediate_layers = intermediate_layers - self.use_conditioning = True if ctc_softmax is not None else False - if self.use_conditioning: - self.ctc_softmax = ctc_softmax - self.conditioning_layer = torch.nn.Linear( - conditioning_layer_dim, attention_dim - ) - - def get_positionwise_layer( - self, - positionwise_layer_type="linear", - attention_dim=256, - linear_units=2048, - dropout_rate=0.1, - positionwise_conv_kernel_size=1, - ): - """Define positionwise layer.""" - if positionwise_layer_type == "linear": - positionwise_layer = PositionwiseFeedForward - positionwise_layer_args = (attention_dim, linear_units, dropout_rate) - elif positionwise_layer_type == "conv1d": - positionwise_layer = MultiLayeredConv1d - positionwise_layer_args = ( - attention_dim, - linear_units, - positionwise_conv_kernel_size, - dropout_rate, - ) - elif positionwise_layer_type == "conv1d-linear": - positionwise_layer = Conv1dLinear - positionwise_layer_args = ( - attention_dim, - linear_units, - positionwise_conv_kernel_size, - dropout_rate, - ) - else: - raise NotImplementedError("Support only linear or conv1d.") - return positionwise_layer, positionwise_layer_args - - def forward(self, xs, masks): - """Encode input sequence. - - Args: - xs (torch.Tensor): Input tensor (#batch, time, idim). - masks (torch.Tensor): Mask tensor (#batch, time). - - Returns: - torch.Tensor: Output tensor (#batch, time, attention_dim). - torch.Tensor: Mask tensor (#batch, time). - - """ - if isinstance( - self.embed, - (Conv2dSubsampling, Conv2dSubsampling6, Conv2dSubsampling8), - ): - xs, masks = self.embed(xs, masks) - else: - xs = self.embed(xs) - - if self.intermediate_layers is None: - xs, masks = self.encoders(xs, masks) - else: - intermediate_outputs = [] - for layer_idx, encoder_layer in enumerate(self.encoders): - xs, masks = encoder_layer(xs, masks) - - if ( - self.intermediate_layers is not None - and layer_idx + 1 in self.intermediate_layers - ): - encoder_output = xs - # intermediate branches also require normalization. - if self.normalize_before: - encoder_output = self.after_norm(encoder_output) - intermediate_outputs.append(encoder_output) - - if self.use_conditioning: - intermediate_result = self.ctc_softmax(encoder_output) - xs = xs + self.conditioning_layer(intermediate_result) - - if self.normalize_before: - xs = self.after_norm(xs) - - if self.intermediate_layers is not None: - return xs, masks, intermediate_outputs - return xs, masks - - def forward_one_step(self, xs, masks, cache=None): - """Encode input frame. - - Args: - xs (torch.Tensor): Input tensor. - masks (torch.Tensor): Mask tensor. - cache (List[torch.Tensor]): List of cache tensors. - - Returns: - torch.Tensor: Output tensor. - torch.Tensor: Mask tensor. - List[torch.Tensor]: List of new cache tensors. - - """ - if isinstance(self.embed, Conv2dSubsampling): - xs, masks = self.embed(xs, masks) - else: - xs = self.embed(xs) - if cache is None: - cache = [None for _ in range(len(self.encoders))] - new_cache = [] - for c, e in zip(cache, self.encoders): - xs, masks = e(xs, masks, cache=c) - new_cache.append(xs) - if self.normalize_before: - xs = self.after_norm(xs) - return xs, masks, new_cache diff --git a/funasr_detach/models/language_model/transformer_lm.py b/funasr_detach/models/language_model/transformer_lm.py deleted file mode 100644 index dbd0f4f0e189c10f41991c552c8e08bd2c6a44bf..0000000000000000000000000000000000000000 --- a/funasr_detach/models/language_model/transformer_lm.py +++ /dev/null @@ -1,133 +0,0 @@ -from typing import Any -from typing import List -from typing import Tuple - -import torch -import torch.nn as nn - -from funasr_detach.models.transformer.embedding import PositionalEncoding -from funasr_detach.models.encoder.transformer_encoder import ( - TransformerEncoder_s0 as Encoder, -) -from funasr_detach.models.transformer.utils.mask import subsequent_mask -from funasr_detach.train.abs_model import AbsLM - - -class TransformerLM(AbsLM): - def __init__( - self, - vocab_size: int, - pos_enc: str = None, - embed_unit: int = 128, - att_unit: int = 256, - head: int = 2, - unit: int = 1024, - layer: int = 4, - dropout_rate: float = 0.5, - ): - super().__init__() - if pos_enc == "sinusoidal": - pos_enc_class = PositionalEncoding - elif pos_enc is None: - - def pos_enc_class(*args, **kwargs): - return nn.Sequential() # indentity - - else: - raise ValueError(f"unknown pos-enc option: {pos_enc}") - - self.embed = nn.Embedding(vocab_size, embed_unit) - self.encoder = Encoder( - idim=embed_unit, - attention_dim=att_unit, - attention_heads=head, - linear_units=unit, - num_blocks=layer, - dropout_rate=dropout_rate, - input_layer="linear", - pos_enc_class=pos_enc_class, - ) - self.decoder = nn.Linear(att_unit, vocab_size) - - def _target_mask(self, ys_in_pad): - ys_mask = ys_in_pad != 0 - m = subsequent_mask(ys_mask.size(-1), device=ys_mask.device).unsqueeze(0) - return ys_mask.unsqueeze(-2) & m - - def forward(self, input: torch.Tensor, hidden: None) -> Tuple[torch.Tensor, None]: - """Compute LM loss value from buffer sequences. - - Args: - input (torch.Tensor): Input ids. (batch, len) - hidden (torch.Tensor): Target ids. (batch, len) - - """ - x = self.embed(input) - mask = self._target_mask(input) - h, _ = self.encoder(x, mask) - y = self.decoder(h) - return y, None - - def score( - self, y: torch.Tensor, state: Any, x: torch.Tensor - ) -> Tuple[torch.Tensor, Any]: - """Score new token. - - Args: - y (torch.Tensor): 1D torch.int64 prefix tokens. - state: Scorer state for prefix tokens - x (torch.Tensor): encoder feature that generates ys. - - Returns: - tuple[torch.Tensor, Any]: Tuple of - torch.float32 scores for next token (vocab_size) - and next state for ys - - """ - y = y.unsqueeze(0) - h, _, cache = self.encoder.forward_one_step( - self.embed(y), self._target_mask(y), cache=state - ) - h = self.decoder(h[:, -1]) - logp = h.log_softmax(dim=-1).squeeze(0) - return logp, cache - - def batch_score( - self, ys: torch.Tensor, states: List[Any], xs: torch.Tensor - ) -> Tuple[torch.Tensor, List[Any]]: - """Score new token batch. - - Args: - ys (torch.Tensor): torch.int64 prefix tokens (n_batch, ylen). - states (List[Any]): Scorer states for prefix tokens. - xs (torch.Tensor): - The encoder feature that generates ys (n_batch, xlen, n_feat). - - Returns: - tuple[torch.Tensor, List[Any]]: Tuple of - batchfied scores for next token with shape of `(n_batch, vocab_size)` - and next state list for ys. - - """ - # merge states - n_batch = len(ys) - n_layers = len(self.encoder.encoders) - if states[0] is None: - batch_state = None - else: - # transpose state of [batch, layer] into [layer, batch] - batch_state = [ - torch.stack([states[b][i] for b in range(n_batch)]) - for i in range(n_layers) - ] - - # batch decoding - h, _, states = self.encoder.forward_one_step( - self.embed(ys), self._target_mask(ys), cache=batch_state - ) - h = self.decoder(h[:, -1]) - logp = h.log_softmax(dim=-1) - - # transpose state of [layer, batch] into [batch, layer] - state_list = [[states[i][b] for i in range(n_layers)] for b in range(n_batch)] - return logp, state_list diff --git a/funasr_detach/models/lora/__init__.py b/funasr_detach/models/lora/__init__.py deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/funasr_detach/models/lora/layers.py b/funasr_detach/models/lora/layers.py deleted file mode 100644 index 680e22b4ef5feda079323df5fbc2240fd1eee409..0000000000000000000000000000000000000000 --- a/funasr_detach/models/lora/layers.py +++ /dev/null @@ -1,369 +0,0 @@ -# ------------------------------------------------------------------------------------------ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. -# ------------------------------------------------------------------------------------------ -import torch -import torch.nn as nn -import torch.nn.functional as F - -import math -from typing import Optional, List - - -class LoRALayer: - def __init__( - self, - r: int, - lora_alpha: int, - lora_dropout: float, - merge_weights: bool, - ): - self.r = r - self.lora_alpha = lora_alpha - # Optional dropout - if lora_dropout > 0.0: - self.lora_dropout = nn.Dropout(p=lora_dropout) - else: - self.lora_dropout = lambda x: x - # Mark the weight as unmerged - self.merged = False - self.merge_weights = merge_weights - - -class Embedding(nn.Embedding, LoRALayer): - # LoRA implemented in a dense layer - def __init__( - self, - num_embeddings: int, - embedding_dim: int, - r: int = 0, - lora_alpha: int = 1, - merge_weights: bool = True, - **kwargs - ): - nn.Embedding.__init__(self, num_embeddings, embedding_dim, **kwargs) - LoRALayer.__init__( - self, - r=r, - lora_alpha=lora_alpha, - lora_dropout=0, - merge_weights=merge_weights, - ) - # Actual trainable parameters - if r > 0: - self.lora_A = nn.Parameter(self.weight.new_zeros((r, num_embeddings))) - self.lora_B = nn.Parameter(self.weight.new_zeros((embedding_dim, r))) - self.scaling = self.lora_alpha / self.r - # Freezing the pre-trained weight matrix - self.weight.requires_grad = False - self.reset_parameters() - - def reset_parameters(self): - nn.Embedding.reset_parameters(self) - if hasattr(self, "lora_A"): - # initialize A the same way as the default for nn.Linear and B to zero - nn.init.zeros_(self.lora_A) - nn.init.normal_(self.lora_B) - - def train(self, mode: bool = True): - nn.Embedding.train(self, mode) - if self.merge_weights and self.merged: - # Make sure that the weights are not merged - if self.r > 0: - self.weight.data -= (self.lora_B @ self.lora_A).T * self.scaling - self.merged = False - - def eval(self): - nn.Linear.eval(self) - if self.merge_weights and not self.merged: - # Merge the weights and mark it - if self.r > 0: - self.weight.data += (self.lora_B @ self.lora_A) * self.scaling - self.merged = True - - def forward(self, x: torch.Tensor): - if self.r > 0 and not self.merged: - result = nn.Embedding.forward(self, x) - if self.r > 0: - after_A = F.embedding( - x, - self.lora_A.T, - self.padding_idx, - self.max_norm, - self.norm_type, - self.scale_grad_by_freq, - self.sparse, - ) - result += (after_A @ self.lora_B.T) * self.scaling - return result - else: - return nn.Embedding.forward(self, x) - - -class Linear(nn.Linear, LoRALayer): - # LoRA implemented in a dense layer - def __init__( - self, - in_features: int, - out_features: int, - r: int = 0, - lora_alpha: int = 1, - lora_dropout: float = 0.0, - fan_in_fan_out: bool = False, # Set this to True if the layer to replace stores weight like (fan_in, fan_out) - merge_weights: bool = True, - **kwargs - ): - nn.Linear.__init__(self, in_features, out_features, **kwargs) - LoRALayer.__init__( - self, - r=r, - lora_alpha=lora_alpha, - lora_dropout=lora_dropout, - merge_weights=merge_weights, - ) - - self.fan_in_fan_out = fan_in_fan_out - # Actual trainable parameters - if r > 0: - self.lora_A = nn.Parameter(self.weight.new_zeros((r, in_features))) - self.lora_B = nn.Parameter(self.weight.new_zeros((out_features, r))) - self.scaling = self.lora_alpha / self.r - # Freezing the pre-trained weight matrix - self.weight.requires_grad = False - self.reset_parameters() - if fan_in_fan_out: - self.weight.data = self.weight.data.T - - def reset_parameters(self): - nn.Linear.reset_parameters(self) - if hasattr(self, "lora_A"): - # initialize A the same way as the default for nn.Linear and B to zero - nn.init.kaiming_uniform_(self.lora_A, a=math.sqrt(5)) - nn.init.zeros_(self.lora_B) - - def train(self, mode: bool = True): - def T(w): - return w.T if self.fan_in_fan_out else w - - nn.Linear.train(self, mode) - if self.merge_weights and self.merged: - # Make sure that the weights are not merged - if self.r > 0: - self.weight.data -= T(self.lora_B @ self.lora_A) * self.scaling - self.merged = False - - def eval(self): - def T(w): - return w.T if self.fan_in_fan_out else w - - nn.Linear.eval(self) - if self.merge_weights and not self.merged: - # Merge the weights and mark it - if self.r > 0: - self.weight.data += T(self.lora_B @ self.lora_A) * self.scaling - self.merged = True - - def forward(self, x: torch.Tensor): - def T(w): - return w.T if self.fan_in_fan_out else w - - if self.r > 0 and not self.merged: - result = F.linear(x, T(self.weight), bias=self.bias) - if self.r > 0: - result += ( - self.lora_dropout(x) @ self.lora_A.T @ self.lora_B.T - ) * self.scaling - return result - else: - return F.linear(x, T(self.weight), bias=self.bias) - - -class MergedLinear(nn.Linear, LoRALayer): - # LoRA implemented in a dense layer - def __init__( - self, - in_features: int, - out_features: int, - r: int = 0, - lora_alpha: int = 1, - lora_dropout: float = 0.0, - enable_lora: List[bool] = [False], - fan_in_fan_out: bool = False, - merge_weights: bool = True, - **kwargs - ): - nn.Linear.__init__(self, in_features, out_features, **kwargs) - LoRALayer.__init__( - self, - r=r, - lora_alpha=lora_alpha, - lora_dropout=lora_dropout, - merge_weights=merge_weights, - ) - assert ( - out_features % len(enable_lora) == 0 - ), "The length of enable_lora must divide out_features" - self.enable_lora = enable_lora - self.fan_in_fan_out = fan_in_fan_out - # Actual trainable parameters - if r > 0 and any(enable_lora): - self.lora_A = nn.Parameter( - self.weight.new_zeros((r * sum(enable_lora), in_features)) - ) - self.lora_B = nn.Parameter( - self.weight.new_zeros( - (out_features // len(enable_lora) * sum(enable_lora), r) - ) - ) # weights for Conv1D with groups=sum(enable_lora) - self.scaling = self.lora_alpha / self.r - # Freezing the pre-trained weight matrix - self.weight.requires_grad = False - # Compute the indices - self.lora_ind = self.weight.new_zeros( - (out_features,), dtype=torch.bool - ).view(len(enable_lora), -1) - self.lora_ind[enable_lora, :] = True - self.lora_ind = self.lora_ind.view(-1) - self.reset_parameters() - if fan_in_fan_out: - self.weight.data = self.weight.data.T - - def reset_parameters(self): - nn.Linear.reset_parameters(self) - if hasattr(self, "lora_A"): - # initialize A the same way as the default for nn.Linear and B to zero - nn.init.kaiming_uniform_(self.lora_A, a=math.sqrt(5)) - nn.init.zeros_(self.lora_B) - - def zero_pad(self, x): - result = x.new_zeros((*x.shape[:-1], self.out_features)) - result = result.view(-1, self.out_features) - result[:, self.lora_ind] = x.reshape( - -1, self.out_features // len(self.enable_lora) * sum(self.enable_lora) - ) - return result.view((*x.shape[:-1], self.out_features)) - - def train(self, mode: bool = True): - def T(w): - return w.T if self.fan_in_fan_out else w - - nn.Linear.train(self, mode) - if self.merge_weights and self.merged: - # Make sure that the weights are not merged - if self.r > 0 and any(self.enable_lora): - delta_w = F.conv1d( - self.lora_A.data.unsqueeze(0), - self.lora_B.data.unsqueeze(-1), - groups=sum(self.enable_lora), - ).squeeze(0) - self.weight.data -= self.zero_pad(T(delta_w * self.scaling)) - self.merged = False - - def eval(self): - def T(w): - return w.T if self.fan_in_fan_out else w - - nn.Linear.eval(self) - if self.merge_weights and not self.merged: - # Merge the weights and mark it - if self.r > 0 and any(self.enable_lora): - delta_w = F.conv1d( - self.lora_A.data.unsqueeze(0), - self.lora_B.data.unsqueeze(-1), - groups=sum(self.enable_lora), - ).squeeze(0) - self.weight.data += self.zero_pad(T(delta_w * self.scaling)) - self.merged = True - - def forward(self, x: torch.Tensor): - def T(w): - return w.T if self.fan_in_fan_out else w - - if self.merged: - return F.linear(x, T(self.weight), bias=self.bias) - else: - result = F.linear(x, T(self.weight), bias=self.bias) - if self.r > 0: - after_A = F.linear(self.lora_dropout(x), self.lora_A) - after_B = F.conv1d( - after_A.transpose(-2, -1), - self.lora_B.unsqueeze(-1), - groups=sum(self.enable_lora), - ).transpose(-2, -1) - result += self.zero_pad(after_B) * self.scaling - return result - - -class Conv2d(nn.Conv2d, LoRALayer): - # LoRA implemented in a dense layer - def __init__( - self, - in_channels: int, - out_channels: int, - kernel_size: int, - r: int = 0, - lora_alpha: int = 1, - lora_dropout: float = 0.0, - merge_weights: bool = True, - **kwargs - ): - nn.Conv2d.__init__(self, in_channels, out_channels, kernel_size, **kwargs) - LoRALayer.__init__( - self, - r=r, - lora_alpha=lora_alpha, - lora_dropout=lora_dropout, - merge_weights=merge_weights, - ) - assert type(kernel_size) is int - # Actual trainable parameters - if r > 0: - self.lora_A = nn.Parameter( - self.weight.new_zeros((r * kernel_size, in_channels * kernel_size)) - ) - self.lora_B = nn.Parameter( - self.weight.new_zeros((out_channels * kernel_size, r * kernel_size)) - ) - self.scaling = self.lora_alpha / self.r - # Freezing the pre-trained weight matrix - self.weight.requires_grad = False - self.reset_parameters() - - def reset_parameters(self): - nn.Conv2d.reset_parameters(self) - if hasattr(self, "lora_A"): - # initialize A the same way as the default for nn.Linear and B to zero - nn.init.kaiming_uniform_(self.lora_A, a=math.sqrt(5)) - nn.init.zeros_(self.lora_B) - - def train(self, mode: bool = True): - nn.Conv2d.train(self, mode) - if self.merge_weights and self.merged: - # Make sure that the weights are not merged - self.weight.data -= (self.lora_B @ self.lora_A).view( - self.weight.shape - ) * self.scaling - self.merged = False - - def eval(self): - nn.Conv2d.eval(self) - if self.merge_weights and not self.merged: - # Merge the weights and mark it - self.weight.data += (self.lora_B @ self.lora_A).view( - self.weight.shape - ) * self.scaling - self.merged = True - - def forward(self, x: torch.Tensor): - if self.r > 0 and not self.merged: - return F.conv2d( - x, - self.weight - + (self.lora_B @ self.lora_A).view(self.weight.shape) * self.scaling, - self.bias, - self.stride, - self.padding, - self.dilation, - self.groups, - ) - return nn.Conv2d.forward(self, x) diff --git a/funasr_detach/models/lora/utils.py b/funasr_detach/models/lora/utils.py deleted file mode 100644 index 204b251f6a0446d6cc9b3531cac41c5e74283abf..0000000000000000000000000000000000000000 --- a/funasr_detach/models/lora/utils.py +++ /dev/null @@ -1,49 +0,0 @@ -# ------------------------------------------------------------------------------------------ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. -# ------------------------------------------------------------------------------------------ -import torch -import torch.nn as nn - -from typing import Dict - -from .layers import LoRALayer - - -def mark_only_lora_as_trainable(model: nn.Module, bias: str = "none") -> None: - for n, p in model.named_parameters(): - if "lora_" not in n and "cif" not in n: - p.requires_grad = False - if bias == "none": - return - elif bias == "all": - for n, p in model.named_parameters(): - if "bias" in n: - p.requires_grad = True - elif bias == "lora_only": - for m in model.modules(): - if isinstance(m, LoRALayer) and hasattr(m, "bias") and m.bias is not None: - m.bias.requires_grad = True - else: - raise NotImplementedError - - -def lora_state_dict(model: nn.Module, bias: str = "none") -> Dict[str, torch.Tensor]: - my_state_dict = model.state_dict() - if bias == "none": - return {k: my_state_dict[k] for k in my_state_dict if "lora_" in k} - elif bias == "all": - return { - k: my_state_dict[k] for k in my_state_dict if "lora_" in k or "bias" in k - } - elif bias == "lora_only": - to_return = {} - for k in my_state_dict: - if "lora_" in k: - to_return[k] = my_state_dict[k] - bias_name = k.split("lora_")[0] + "bias" - if bias_name in my_state_dict: - to_return[bias_name] = my_state_dict[bias_name] - return to_return - else: - raise NotImplementedError diff --git a/funasr_detach/models/mfcca/__init__.py b/funasr_detach/models/mfcca/__init__.py deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/funasr_detach/models/mfcca/e2e_asr_mfcca.py b/funasr_detach/models/mfcca/e2e_asr_mfcca.py deleted file mode 100644 index fba061fb56d2283c1740c63cd7049f167f5c9d63..0000000000000000000000000000000000000000 --- a/funasr_detach/models/mfcca/e2e_asr_mfcca.py +++ /dev/null @@ -1,331 +0,0 @@ -from contextlib import contextmanager -from distutils.version import LooseVersion -from typing import Dict -from typing import List -from typing import Optional -from typing import Tuple -from typing import Union -import logging -import torch - -from funasr_detach.metrics import ErrorCalculator -from funasr_detach.metrics.compute_acc import th_accuracy -from funasr_detach.models.transformer.utils.add_sos_eos import add_sos_eos -from funasr_detach.losses.label_smoothing_loss import ( - LabelSmoothingLoss, # noqa: H301 -) -from funasr_detach.models.ctc import CTC -from funasr_detach.models.decoder.abs_decoder import AbsDecoder -from funasr_detach.models.encoder.abs_encoder import AbsEncoder -from funasr_detach.frontends.abs_frontend import AbsFrontend -from funasr_detach.models.preencoder.abs_preencoder import AbsPreEncoder -from funasr_detach.models.specaug.abs_specaug import AbsSpecAug -from funasr_detach.layers.abs_normalize import AbsNormalize -from funasr_detach.train_utils.device_funcs import force_gatherable -from funasr_detach.models.base_model import FunASRModel - -if LooseVersion(torch.__version__) >= LooseVersion("1.6.0"): - from torch.cuda.amp import autocast -else: - # Nothing to do if torch<1.6.0 - @contextmanager - def autocast(enabled=True): - yield - - -import pdb -import random -import math - - -class MFCCA(FunASRModel): - """ - Author: Audio, Speech and Language Processing Group (ASLP@NPU), Northwestern Polytechnical University - MFCCA:Multi-Frame Cross-Channel attention for multi-speaker ASR in Multi-party meeting scenario - https://arxiv.org/abs/2210.05265 - """ - - def __init__( - self, - vocab_size: int, - token_list: Union[Tuple[str, ...], List[str]], - frontend: Optional[AbsFrontend], - specaug: Optional[AbsSpecAug], - normalize: Optional[AbsNormalize], - encoder: AbsEncoder, - decoder: AbsDecoder, - ctc: CTC, - rnnt_decoder: None = None, - ctc_weight: float = 0.5, - ignore_id: int = -1, - lsm_weight: float = 0.0, - mask_ratio: float = 0.0, - length_normalized_loss: bool = False, - report_cer: bool = True, - report_wer: bool = True, - sym_space: str = "", - sym_blank: str = "", - preencoder: Optional[AbsPreEncoder] = None, - ): - assert 0.0 <= ctc_weight <= 1.0, ctc_weight - assert rnnt_decoder is None, "Not implemented" - - super().__init__() - # note that eos is the same as sos (equivalent ID) - self.sos = vocab_size - 1 - self.eos = vocab_size - 1 - self.vocab_size = vocab_size - self.ignore_id = ignore_id - self.ctc_weight = ctc_weight - self.token_list = token_list.copy() - - self.mask_ratio = mask_ratio - - self.frontend = frontend - self.specaug = specaug - self.normalize = normalize - self.preencoder = preencoder - self.encoder = encoder - # we set self.decoder = None in the CTC mode since - # self.decoder parameters were never used and PyTorch complained - # and threw an Exception in the multi-GPU experiment. - # thanks Jeff Farris for pointing out the issue. - if ctc_weight == 1.0: - self.decoder = None - else: - self.decoder = decoder - if ctc_weight == 0.0: - self.ctc = None - else: - self.ctc = ctc - self.rnnt_decoder = rnnt_decoder - self.criterion_att = LabelSmoothingLoss( - size=vocab_size, - padding_idx=ignore_id, - smoothing=lsm_weight, - normalize_length=length_normalized_loss, - ) - - if report_cer or report_wer: - self.error_calculator = ErrorCalculator( - token_list, sym_space, sym_blank, report_cer, report_wer - ) - else: - self.error_calculator = None - - def forward( - self, - speech: torch.Tensor, - speech_lengths: torch.Tensor, - text: torch.Tensor, - text_lengths: torch.Tensor, - ) -> Tuple[torch.Tensor, Dict[str, torch.Tensor], torch.Tensor]: - """Frontend + Encoder + Decoder + Calc loss - Args: - speech: (Batch, Length, ...) - speech_lengths: (Batch, ) - text: (Batch, Length) - text_lengths: (Batch,) - """ - assert text_lengths.dim() == 1, text_lengths.shape - # Check that batch_size is unified - assert ( - speech.shape[0] - == speech_lengths.shape[0] - == text.shape[0] - == text_lengths.shape[0] - ), (speech.shape, speech_lengths.shape, text.shape, text_lengths.shape) - # pdb.set_trace() - if speech.dim() == 3 and speech.size(2) == 8 and self.mask_ratio != 0: - rate_num = random.random() - # rate_num = 0.1 - if rate_num <= self.mask_ratio: - retain_channel = math.ceil(random.random() * 8) - if retain_channel > 1: - speech = speech[ - :, :, torch.randperm(8)[0:retain_channel].sort().values - ] - else: - speech = speech[:, :, torch.randperm(8)[0]] - # pdb.set_trace() - batch_size = speech.shape[0] - # for data-parallel - text = text[:, : text_lengths.max()] - - # 1. Encoder - encoder_out, encoder_out_lens = self.encode(speech, speech_lengths) - - # 2a. Attention-decoder branch - if self.ctc_weight == 1.0: - loss_att, acc_att, cer_att, wer_att = None, None, None, None - else: - loss_att, acc_att, cer_att, wer_att = self._calc_att_loss( - encoder_out, encoder_out_lens, text, text_lengths - ) - - # 2b. CTC branch - if self.ctc_weight == 0.0: - loss_ctc, cer_ctc = None, None - else: - loss_ctc, cer_ctc = self._calc_ctc_loss( - encoder_out, encoder_out_lens, text, text_lengths - ) - - # 2c. RNN-T branch - if self.rnnt_decoder is not None: - _ = self._calc_rnnt_loss(encoder_out, encoder_out_lens, text, text_lengths) - - if self.ctc_weight == 0.0: - loss = loss_att - elif self.ctc_weight == 1.0: - loss = loss_ctc - else: - loss = self.ctc_weight * loss_ctc + (1 - self.ctc_weight) * loss_att - - stats = dict( - loss=loss.detach(), - loss_att=loss_att.detach() if loss_att is not None else None, - loss_ctc=loss_ctc.detach() if loss_ctc is not None else None, - acc=acc_att, - cer=cer_att, - wer=wer_att, - cer_ctc=cer_ctc, - ) - - # force_gatherable: to-device and to-tensor if scalar for DataParallel - loss, stats, weight = force_gatherable((loss, stats, batch_size), loss.device) - return loss, stats, weight - - def collect_feats( - self, - speech: torch.Tensor, - speech_lengths: torch.Tensor, - text: torch.Tensor, - text_lengths: torch.Tensor, - ) -> Dict[str, torch.Tensor]: - feats, feats_lengths, channel_size = self._extract_feats(speech, speech_lengths) - return {"feats": feats, "feats_lengths": feats_lengths} - - def encode( - self, speech: torch.Tensor, speech_lengths: torch.Tensor - ) -> Tuple[torch.Tensor, torch.Tensor]: - """Frontend + Encoder. Note that this method is used by asr_inference.py - Args: - speech: (Batch, Length, ...) - speech_lengths: (Batch, ) - """ - with autocast(False): - # 1. Extract feats - feats, feats_lengths, channel_size = self._extract_feats( - speech, speech_lengths - ) - # 2. Data augmentation - if self.specaug is not None and self.training: - feats, feats_lengths = self.specaug(feats, feats_lengths) - - # 3. Normalization for feature: e.g. Global-CMVN, Utterance-CMVN - if self.normalize is not None: - feats, feats_lengths = self.normalize(feats, feats_lengths) - - # Pre-encoder, e.g. used for raw input data - if self.preencoder is not None: - feats, feats_lengths = self.preencoder(feats, feats_lengths) - # pdb.set_trace() - encoder_out, encoder_out_lens, _ = self.encoder( - feats, feats_lengths, channel_size - ) - - assert encoder_out.size(0) == speech.size(0), ( - encoder_out.size(), - speech.size(0), - ) - if encoder_out.dim() == 4: - assert encoder_out.size(2) <= encoder_out_lens.max(), ( - encoder_out.size(), - encoder_out_lens.max(), - ) - else: - assert encoder_out.size(1) <= encoder_out_lens.max(), ( - encoder_out.size(), - encoder_out_lens.max(), - ) - - return encoder_out, encoder_out_lens - - def _extract_feats( - self, speech: torch.Tensor, speech_lengths: torch.Tensor - ) -> Tuple[torch.Tensor, torch.Tensor]: - assert speech_lengths.dim() == 1, speech_lengths.shape - # for data-parallel - speech = speech[:, : speech_lengths.max()] - if self.frontend is not None: - # Frontend - # e.g. STFT and Feature extract - # data_loader may send time-domain signal in this case - # speech (Batch, NSamples) -> feats: (Batch, NFrames, Dim) - feats, feats_lengths, channel_size = self.frontend(speech, speech_lengths) - else: - # No frontend and no feature extract - feats, feats_lengths = speech, speech_lengths - channel_size = 1 - return feats, feats_lengths, channel_size - - def _calc_att_loss( - self, - encoder_out: torch.Tensor, - encoder_out_lens: torch.Tensor, - ys_pad: torch.Tensor, - ys_pad_lens: torch.Tensor, - ): - ys_in_pad, ys_out_pad = add_sos_eos(ys_pad, self.sos, self.eos, self.ignore_id) - ys_in_lens = ys_pad_lens + 1 - - # 1. Forward decoder - decoder_out, _ = self.decoder( - encoder_out, encoder_out_lens, ys_in_pad, ys_in_lens - ) - - # 2. Compute attention loss - loss_att = self.criterion_att(decoder_out, ys_out_pad) - acc_att = th_accuracy( - decoder_out.view(-1, self.vocab_size), - ys_out_pad, - ignore_label=self.ignore_id, - ) - - # Compute cer/wer using attention-decoder - if self.training or self.error_calculator is None: - cer_att, wer_att = None, None - else: - ys_hat = decoder_out.argmax(dim=-1) - cer_att, wer_att = self.error_calculator(ys_hat.cpu(), ys_pad.cpu()) - - return loss_att, acc_att, cer_att, wer_att - - def _calc_ctc_loss( - self, - encoder_out: torch.Tensor, - encoder_out_lens: torch.Tensor, - ys_pad: torch.Tensor, - ys_pad_lens: torch.Tensor, - ): - # Calc CTC loss - if encoder_out.dim() == 4: - encoder_out = encoder_out.mean(1) - loss_ctc = self.ctc(encoder_out, encoder_out_lens, ys_pad, ys_pad_lens) - - # Calc CER using CTC - cer_ctc = None - if not self.training and self.error_calculator is not None: - ys_hat = self.ctc.argmax(encoder_out).data - cer_ctc = self.error_calculator(ys_hat.cpu(), ys_pad.cpu(), is_ctc=True) - return loss_ctc, cer_ctc - - def _calc_rnnt_loss( - self, - encoder_out: torch.Tensor, - encoder_out_lens: torch.Tensor, - ys_pad: torch.Tensor, - ys_pad_lens: torch.Tensor, - ): - raise NotImplementedError diff --git a/funasr_detach/models/mfcca/encoder_layer_mfcca.py b/funasr_detach/models/mfcca/encoder_layer_mfcca.py deleted file mode 100644 index 09567f07a7218a3004cbaa3c07084a9c6d8cd2f8..0000000000000000000000000000000000000000 --- a/funasr_detach/models/mfcca/encoder_layer_mfcca.py +++ /dev/null @@ -1,275 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- - -# Copyright 2020 Johns Hopkins University (Shinji Watanabe) -# Northwestern Polytechnical University (Pengcheng Guo) -# Apache 2.0 (http://www.apache.org/licenses/LICENSE-2.0) - -"""Encoder self-attention layer definition.""" - -import torch - -from torch import nn - -from funasr_detach.models.transformer.layer_norm import LayerNorm -from torch.autograd import Variable - - -class Encoder_Conformer_Layer(nn.Module): - """Encoder layer module. - - Args: - size (int): Input dimension. - self_attn (torch.nn.Module): Self-attention module instance. - `MultiHeadedAttention` or `RelPositionMultiHeadedAttention` instance - can be used as the argument. - feed_forward (torch.nn.Module): Feed-forward module instance. - `PositionwiseFeedForward`, `MultiLayeredConv1d`, or `Conv1dLinear` instance - can be used as the argument. - feed_forward_macaron (torch.nn.Module): Additional feed-forward module instance. - `PositionwiseFeedForward`, `MultiLayeredConv1d`, or `Conv1dLinear` instance - can be used as the argument. - conv_module (torch.nn.Module): Convolution module instance. - `ConvlutionModule` instance can be used as the argument. - dropout_rate (float): Dropout rate. - normalize_before (bool): Whether to use layer_norm before the first block. - concat_after (bool): Whether to concat attention layer's input and output. - if True, additional linear will be applied. - i.e. x -> x + linear(concat(x, att(x))) - if False, no additional linear will be applied. i.e. x -> x + att(x) - - """ - - def __init__( - self, - size, - self_attn, - feed_forward, - feed_forward_macaron, - conv_module, - dropout_rate, - normalize_before=True, - concat_after=False, - cca_pos=0, - ): - """Construct an Encoder_Conformer_Layer object.""" - super(Encoder_Conformer_Layer, self).__init__() - self.self_attn = self_attn - self.feed_forward = feed_forward - self.feed_forward_macaron = feed_forward_macaron - self.conv_module = conv_module - self.norm_ff = LayerNorm(size) # for the FNN module - self.norm_mha = LayerNorm(size) # for the MHA module - if feed_forward_macaron is not None: - self.norm_ff_macaron = LayerNorm(size) - self.ff_scale = 0.5 - else: - self.ff_scale = 1.0 - if self.conv_module is not None: - self.norm_conv = LayerNorm(size) # for the CNN module - self.norm_final = LayerNorm(size) # for the final output of the block - self.dropout = nn.Dropout(dropout_rate) - self.size = size - self.normalize_before = normalize_before - self.concat_after = concat_after - self.cca_pos = cca_pos - - if self.concat_after: - self.concat_linear = nn.Linear(size + size, size) - - def forward(self, x_input, mask, cache=None): - """Compute encoded features. - - Args: - x_input (Union[Tuple, torch.Tensor]): Input tensor w/ or w/o pos emb. - - w/ pos emb: Tuple of tensors [(#batch, time, size), (1, time, size)]. - - w/o pos emb: Tensor (#batch, time, size). - mask (torch.Tensor): Mask tensor for the input (#batch, time). - cache (torch.Tensor): Cache tensor of the input (#batch, time - 1, size). - - Returns: - torch.Tensor: Output tensor (#batch, time, size). - torch.Tensor: Mask tensor (#batch, time). - - """ - if isinstance(x_input, tuple): - x, pos_emb = x_input[0], x_input[1] - else: - x, pos_emb = x_input, None - # whether to use macaron style - if self.feed_forward_macaron is not None: - residual = x - if self.normalize_before: - x = self.norm_ff_macaron(x) - x = residual + self.ff_scale * self.dropout(self.feed_forward_macaron(x)) - if not self.normalize_before: - x = self.norm_ff_macaron(x) - - # multi-headed self-attention module - residual = x - if self.normalize_before: - x = self.norm_mha(x) - - if cache is None: - x_q = x - else: - assert cache.shape == (x.shape[0], x.shape[1] - 1, self.size) - x_q = x[:, -1:, :] - residual = residual[:, -1:, :] - mask = None if mask is None else mask[:, -1:, :] - - if self.cca_pos < 2: - if pos_emb is not None: - x_att = self.self_attn(x_q, x, x, pos_emb, mask) - else: - x_att = self.self_attn(x_q, x, x, mask) - else: - x_att = self.self_attn(x_q, x, x, mask) - - if self.concat_after: - x_concat = torch.cat((x, x_att), dim=-1) - x = residual + self.concat_linear(x_concat) - else: - x = residual + self.dropout(x_att) - if not self.normalize_before: - x = self.norm_mha(x) - - # convolution module - if self.conv_module is not None: - residual = x - if self.normalize_before: - x = self.norm_conv(x) - x = residual + self.dropout(self.conv_module(x)) - if not self.normalize_before: - x = self.norm_conv(x) - - # feed forward module - residual = x - if self.normalize_before: - x = self.norm_ff(x) - x = residual + self.ff_scale * self.dropout(self.feed_forward(x)) - if not self.normalize_before: - x = self.norm_ff(x) - - if self.conv_module is not None: - x = self.norm_final(x) - - if cache is not None: - x = torch.cat([cache, x], dim=1) - - if pos_emb is not None: - return (x, pos_emb), mask - - return x, mask - - -class EncoderLayer(nn.Module): - """Encoder layer module. - - Args: - size (int): Input dimension. - self_attn (torch.nn.Module): Self-attention module instance. - `MultiHeadedAttention` or `RelPositionMultiHeadedAttention` instance - can be used as the argument. - feed_forward (torch.nn.Module): Feed-forward module instance. - `PositionwiseFeedForward`, `MultiLayeredConv1d`, or `Conv1dLinear` instance - can be used as the argument. - feed_forward_macaron (torch.nn.Module): Additional feed-forward module instance. - `PositionwiseFeedForward`, `MultiLayeredConv1d`, or `Conv1dLinear` instance - can be used as the argument. - conv_module (torch.nn.Module): Convolution module instance. - `ConvlutionModule` instance can be used as the argument. - dropout_rate (float): Dropout rate. - normalize_before (bool): Whether to use layer_norm before the first block. - concat_after (bool): Whether to concat attention layer's input and output. - if True, additional linear will be applied. - i.e. x -> x + linear(concat(x, att(x))) - if False, no additional linear will be applied. i.e. x -> x + att(x) - - """ - - def __init__( - self, - size, - self_attn_cros_channel, - self_attn_conformer, - feed_forward_csa, - feed_forward_macaron_csa, - conv_module_csa, - dropout_rate, - normalize_before=True, - concat_after=False, - ): - """Construct an EncoderLayer object.""" - super(EncoderLayer, self).__init__() - - self.encoder_cros_channel_atten = self_attn_cros_channel - self.encoder_csa = Encoder_Conformer_Layer( - size, - self_attn_conformer, - feed_forward_csa, - feed_forward_macaron_csa, - conv_module_csa, - dropout_rate, - normalize_before, - concat_after, - cca_pos=0, - ) - self.norm_mha = LayerNorm(size) # for the MHA module - self.dropout = nn.Dropout(dropout_rate) - - def forward(self, x_input, mask, channel_size, cache=None): - """Compute encoded features. - - Args: - x_input (Union[Tuple, torch.Tensor]): Input tensor w/ or w/o pos emb. - - w/ pos emb: Tuple of tensors [(#batch, time, size), (1, time, size)]. - - w/o pos emb: Tensor (#batch, time, size). - mask (torch.Tensor): Mask tensor for the input (#batch, time). - cache (torch.Tensor): Cache tensor of the input (#batch, time - 1, size). - - Returns: - torch.Tensor: Output tensor (#batch, time, size). - torch.Tensor: Mask tensor (#batch, time). - - """ - if isinstance(x_input, tuple): - x, pos_emb = x_input[0], x_input[1] - else: - x, pos_emb = x_input, None - residual = x - x = self.norm_mha(x) - t_leng = x.size(1) - d_dim = x.size(2) - x_new = x.reshape(-1, channel_size, t_leng, d_dim).transpose( - 1, 2 - ) # x_new B*T * C * D - x_k_v = x_new.new(x_new.size(0), x_new.size(1), 5, x_new.size(2), x_new.size(3)) - pad_before = Variable( - torch.zeros(x_new.size(0), 2, x_new.size(2), x_new.size(3)) - ).type(x_new.type()) - pad_after = Variable( - torch.zeros(x_new.size(0), 2, x_new.size(2), x_new.size(3)) - ).type(x_new.type()) - x_pad = torch.cat([pad_before, x_new, pad_after], 1) - x_k_v[:, :, 0, :, :] = x_pad[:, 0:-4, :, :] - x_k_v[:, :, 1, :, :] = x_pad[:, 1:-3, :, :] - x_k_v[:, :, 2, :, :] = x_pad[:, 2:-2, :, :] - x_k_v[:, :, 3, :, :] = x_pad[:, 3:-1, :, :] - x_k_v[:, :, 4, :, :] = x_pad[:, 4:, :, :] - x_new = x_new.reshape(-1, channel_size, d_dim) - x_k_v = x_k_v.reshape(-1, 5 * channel_size, d_dim) - x_att = self.encoder_cros_channel_atten(x_new, x_k_v, x_k_v, None) - x_att = ( - x_att.reshape(-1, t_leng, channel_size, d_dim) - .transpose(1, 2) - .reshape(-1, t_leng, d_dim) - ) - x = residual + self.dropout(x_att) - if pos_emb is not None: - x_input = (x, pos_emb) - else: - x_input = x - x_input, mask = self.encoder_csa(x_input, mask) - - return x_input, mask, channel_size diff --git a/funasr_detach/models/mfcca/mfcca_encoder.py b/funasr_detach/models/mfcca/mfcca_encoder.py deleted file mode 100644 index 8358cd3927d33ee95a0eb822c1fd37e9089c1abd..0000000000000000000000000000000000000000 --- a/funasr_detach/models/mfcca/mfcca_encoder.py +++ /dev/null @@ -1,436 +0,0 @@ -from typing import Optional -from typing import Tuple - -import logging -import torch -from torch import nn - - -from funasr_detach.models.encoder.encoder_layer_mfcca import EncoderLayer -from funasr_detach.models.transformer.utils.nets_utils import get_activation -from funasr_detach.models.transformer.utils.nets_utils import make_pad_mask -from funasr_detach.models.transformer.attention import ( - MultiHeadedAttention, # noqa: H301 - RelPositionMultiHeadedAttention, # noqa: H301 - LegacyRelPositionMultiHeadedAttention, # noqa: H301 -) -from funasr_detach.models.transformer.embedding import ( - PositionalEncoding, # noqa: H301 - ScaledPositionalEncoding, # noqa: H301 - RelPositionalEncoding, # noqa: H301 - LegacyRelPositionalEncoding, # noqa: H301 -) -from funasr_detach.models.transformer.layer_norm import LayerNorm -from funasr_detach.models.transformer.utils.multi_layer_conv import Conv1dLinear -from funasr_detach.models.transformer.utils.multi_layer_conv import MultiLayeredConv1d -from funasr_detach.models.transformer.positionwise_feed_forward import ( - PositionwiseFeedForward, # noqa: H301 -) -from funasr_detach.models.transformer.utils.repeat import repeat -from funasr_detach.models.transformer.utils.subsampling import Conv2dSubsampling -from funasr_detach.models.transformer.utils.subsampling import Conv2dSubsampling2 -from funasr_detach.models.transformer.utils.subsampling import Conv2dSubsampling6 -from funasr_detach.models.transformer.utils.subsampling import Conv2dSubsampling8 -from funasr_detach.models.transformer.utils.subsampling import TooShortUttError -from funasr_detach.models.transformer.utils.subsampling import check_short_utt -from funasr_detach.models.encoder.abs_encoder import AbsEncoder -import pdb -import math - - -class ConvolutionModule(nn.Module): - """ConvolutionModule in Conformer model. - Args: - channels (int): The number of channels of conv layers. - kernel_size (int): Kernerl size of conv layers. - """ - - def __init__(self, channels, kernel_size, activation=nn.ReLU(), bias=True): - """Construct an ConvolutionModule object.""" - super(ConvolutionModule, self).__init__() - # kernerl_size should be a odd number for 'SAME' padding - assert (kernel_size - 1) % 2 == 0 - - self.pointwise_conv1 = nn.Conv1d( - channels, - 2 * channels, - kernel_size=1, - stride=1, - padding=0, - bias=bias, - ) - self.depthwise_conv = nn.Conv1d( - channels, - channels, - kernel_size, - stride=1, - padding=(kernel_size - 1) // 2, - groups=channels, - bias=bias, - ) - self.norm = nn.BatchNorm1d(channels) - self.pointwise_conv2 = nn.Conv1d( - channels, - channels, - kernel_size=1, - stride=1, - padding=0, - bias=bias, - ) - self.activation = activation - - def forward(self, x): - """Compute convolution module. - Args: - x (torch.Tensor): Input tensor (#batch, time, channels). - Returns: - torch.Tensor: Output tensor (#batch, time, channels). - """ - # exchange the temporal dimension and the feature dimension - x = x.transpose(1, 2) - - # GLU mechanism - x = self.pointwise_conv1(x) # (batch, 2*channel, dim) - x = nn.functional.glu(x, dim=1) # (batch, channel, dim) - - # 1D Depthwise Conv - x = self.depthwise_conv(x) - x = self.activation(self.norm(x)) - - x = self.pointwise_conv2(x) - - return x.transpose(1, 2) - - -class MFCCAEncoder(AbsEncoder): - """Conformer encoder module. - Args: - input_size (int): Input dimension. - output_size (int): Dimention of attention. - attention_heads (int): The number of heads of multi head attention. - linear_units (int): The number of units of position-wise feed forward. - num_blocks (int): The number of decoder blocks. - dropout_rate (float): Dropout rate. - attention_dropout_rate (float): Dropout rate in attention. - positional_dropout_rate (float): Dropout rate after adding positional encoding. - input_layer (Union[str, torch.nn.Module]): Input layer type. - normalize_before (bool): Whether to use layer_norm before the first block. - concat_after (bool): Whether to concat attention layer's input and output. - If True, additional linear will be applied. - i.e. x -> x + linear(concat(x, att(x))) - If False, no additional linear will be applied. i.e. x -> x + att(x) - positionwise_layer_type (str): "linear", "conv1d", or "conv1d-linear". - positionwise_conv_kernel_size (int): Kernel size of positionwise conv1d layer. - rel_pos_type (str): Whether to use the latest relative positional encoding or - the legacy one. The legacy relative positional encoding will be deprecated - in the future. More Details can be found in - https://github.com/espnet/espnet/pull/2816. - encoder_pos_enc_layer_type (str): Encoder positional encoding layer type. - encoder_attn_layer_type (str): Encoder attention layer type. - activation_type (str): Encoder activation function type. - macaron_style (bool): Whether to use macaron style for positionwise layer. - use_cnn_module (bool): Whether to use convolution module. - zero_triu (bool): Whether to zero the upper triangular part of attention matrix. - cnn_module_kernel (int): Kernerl size of convolution module. - padding_idx (int): Padding idx for input_layer=embed. - """ - - def __init__( - self, - input_size: int, - output_size: int = 256, - attention_heads: int = 4, - linear_units: int = 2048, - num_blocks: int = 6, - dropout_rate: float = 0.1, - positional_dropout_rate: float = 0.1, - attention_dropout_rate: float = 0.0, - input_layer: str = "conv2d", - normalize_before: bool = True, - concat_after: bool = False, - positionwise_layer_type: str = "linear", - positionwise_conv_kernel_size: int = 3, - macaron_style: bool = False, - rel_pos_type: str = "legacy", - pos_enc_layer_type: str = "rel_pos", - selfattention_layer_type: str = "rel_selfattn", - activation_type: str = "swish", - use_cnn_module: bool = True, - zero_triu: bool = False, - cnn_module_kernel: int = 31, - padding_idx: int = -1, - ): - super().__init__() - self._output_size = output_size - - if rel_pos_type == "legacy": - if pos_enc_layer_type == "rel_pos": - pos_enc_layer_type = "legacy_rel_pos" - if selfattention_layer_type == "rel_selfattn": - selfattention_layer_type = "legacy_rel_selfattn" - elif rel_pos_type == "latest": - assert selfattention_layer_type != "legacy_rel_selfattn" - assert pos_enc_layer_type != "legacy_rel_pos" - else: - raise ValueError("unknown rel_pos_type: " + rel_pos_type) - - activation = get_activation(activation_type) - if pos_enc_layer_type == "abs_pos": - pos_enc_class = PositionalEncoding - elif pos_enc_layer_type == "scaled_abs_pos": - pos_enc_class = ScaledPositionalEncoding - elif pos_enc_layer_type == "rel_pos": - assert selfattention_layer_type == "rel_selfattn" - pos_enc_class = RelPositionalEncoding - elif pos_enc_layer_type == "legacy_rel_pos": - assert selfattention_layer_type == "legacy_rel_selfattn" - pos_enc_class = LegacyRelPositionalEncoding - logging.warning( - "Using legacy_rel_pos and it will be deprecated in the future." - ) - else: - raise ValueError("unknown pos_enc_layer: " + pos_enc_layer_type) - - if input_layer == "linear": - self.embed = torch.nn.Sequential( - torch.nn.Linear(input_size, output_size), - torch.nn.LayerNorm(output_size), - torch.nn.Dropout(dropout_rate), - pos_enc_class(output_size, positional_dropout_rate), - ) - elif input_layer == "conv2d": - self.embed = Conv2dSubsampling( - input_size, - output_size, - dropout_rate, - pos_enc_class(output_size, positional_dropout_rate), - ) - elif input_layer == "conv2d6": - self.embed = Conv2dSubsampling6( - input_size, - output_size, - dropout_rate, - pos_enc_class(output_size, positional_dropout_rate), - ) - elif input_layer == "conv2d8": - self.embed = Conv2dSubsampling8( - input_size, - output_size, - dropout_rate, - pos_enc_class(output_size, positional_dropout_rate), - ) - elif input_layer == "embed": - self.embed = torch.nn.Sequential( - torch.nn.Embedding(input_size, output_size, padding_idx=padding_idx), - pos_enc_class(output_size, positional_dropout_rate), - ) - elif isinstance(input_layer, torch.nn.Module): - self.embed = torch.nn.Sequential( - input_layer, - pos_enc_class(output_size, positional_dropout_rate), - ) - elif input_layer is None: - self.embed = torch.nn.Sequential( - pos_enc_class(output_size, positional_dropout_rate) - ) - else: - raise ValueError("unknown input_layer: " + input_layer) - self.normalize_before = normalize_before - if positionwise_layer_type == "linear": - positionwise_layer = PositionwiseFeedForward - positionwise_layer_args = ( - output_size, - linear_units, - dropout_rate, - activation, - ) - elif positionwise_layer_type == "conv1d": - positionwise_layer = MultiLayeredConv1d - positionwise_layer_args = ( - output_size, - linear_units, - positionwise_conv_kernel_size, - dropout_rate, - ) - elif positionwise_layer_type == "conv1d-linear": - positionwise_layer = Conv1dLinear - positionwise_layer_args = ( - output_size, - linear_units, - positionwise_conv_kernel_size, - dropout_rate, - ) - else: - raise NotImplementedError("Support only linear or conv1d.") - - if selfattention_layer_type == "selfattn": - encoder_selfattn_layer = MultiHeadedAttention - encoder_selfattn_layer_args = ( - attention_heads, - output_size, - attention_dropout_rate, - ) - elif selfattention_layer_type == "legacy_rel_selfattn": - assert pos_enc_layer_type == "legacy_rel_pos" - encoder_selfattn_layer = LegacyRelPositionMultiHeadedAttention - encoder_selfattn_layer_args = ( - attention_heads, - output_size, - attention_dropout_rate, - ) - logging.warning( - "Using legacy_rel_selfattn and it will be deprecated in the future." - ) - elif selfattention_layer_type == "rel_selfattn": - assert pos_enc_layer_type == "rel_pos" - encoder_selfattn_layer = RelPositionMultiHeadedAttention - encoder_selfattn_layer_args = ( - attention_heads, - output_size, - attention_dropout_rate, - zero_triu, - ) - else: - raise ValueError("unknown encoder_attn_layer: " + selfattention_layer_type) - - convolution_layer = ConvolutionModule - convolution_layer_args = (output_size, cnn_module_kernel, activation) - encoder_selfattn_layer_raw = MultiHeadedAttention - encoder_selfattn_layer_args_raw = ( - attention_heads, - output_size, - attention_dropout_rate, - ) - self.encoders = repeat( - num_blocks, - lambda lnum: EncoderLayer( - output_size, - encoder_selfattn_layer_raw(*encoder_selfattn_layer_args_raw), - encoder_selfattn_layer(*encoder_selfattn_layer_args), - positionwise_layer(*positionwise_layer_args), - positionwise_layer(*positionwise_layer_args) if macaron_style else None, - convolution_layer(*convolution_layer_args) if use_cnn_module else None, - dropout_rate, - normalize_before, - concat_after, - ), - ) - if self.normalize_before: - self.after_norm = LayerNorm(output_size) - self.conv1 = torch.nn.Conv2d(8, 16, [5, 7], stride=[1, 1], padding=(2, 3)) - - self.conv2 = torch.nn.Conv2d(16, 32, [5, 7], stride=[1, 1], padding=(2, 3)) - - self.conv3 = torch.nn.Conv2d(32, 16, [5, 7], stride=[1, 1], padding=(2, 3)) - - self.conv4 = torch.nn.Conv2d(16, 1, [5, 7], stride=[1, 1], padding=(2, 3)) - - def output_size(self) -> int: - return self._output_size - - def forward( - self, - xs_pad: torch.Tensor, - ilens: torch.Tensor, - channel_size: torch.Tensor, - prev_states: torch.Tensor = None, - ) -> Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]: - """Calculate forward propagation. - Args: - xs_pad (torch.Tensor): Input tensor (#batch, L, input_size). - ilens (torch.Tensor): Input length (#batch). - prev_states (torch.Tensor): Not to be used now. - Returns: - torch.Tensor: Output tensor (#batch, L, output_size). - torch.Tensor: Output length (#batch). - torch.Tensor: Not to be used now. - """ - masks = (~make_pad_mask(ilens)[:, None, :]).to(xs_pad.device) - if ( - isinstance(self.embed, Conv2dSubsampling) - or isinstance(self.embed, Conv2dSubsampling6) - or isinstance(self.embed, Conv2dSubsampling8) - ): - short_status, limit_size = check_short_utt(self.embed, xs_pad.size(1)) - if short_status: - raise TooShortUttError( - f"has {xs_pad.size(1)} frames and is too short for subsampling " - + f"(it needs more than {limit_size} frames), return empty results", - xs_pad.size(1), - limit_size, - ) - xs_pad, masks = self.embed(xs_pad, masks) - else: - xs_pad = self.embed(xs_pad) - xs_pad, masks, channel_size = self.encoders(xs_pad, masks, channel_size) - if isinstance(xs_pad, tuple): - xs_pad = xs_pad[0] - - t_leng = xs_pad.size(1) - d_dim = xs_pad.size(2) - xs_pad = xs_pad.reshape(-1, channel_size, t_leng, d_dim) - # pdb.set_trace() - if channel_size < 8: - repeat_num = math.ceil(8 / channel_size) - xs_pad = xs_pad.repeat(1, repeat_num, 1, 1)[:, 0:8, :, :] - xs_pad = self.conv1(xs_pad) - xs_pad = self.conv2(xs_pad) - xs_pad = self.conv3(xs_pad) - xs_pad = self.conv4(xs_pad) - xs_pad = xs_pad.squeeze().reshape(-1, t_leng, d_dim) - mask_tmp = masks.size(1) - masks = masks.reshape(-1, channel_size, mask_tmp, t_leng)[:, 0, :, :] - - if self.normalize_before: - xs_pad = self.after_norm(xs_pad) - - olens = masks.squeeze(1).sum(1) - return xs_pad, olens, None - - def forward_hidden( - self, - xs_pad: torch.Tensor, - ilens: torch.Tensor, - prev_states: torch.Tensor = None, - ) -> Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]: - """Calculate forward propagation. - Args: - xs_pad (torch.Tensor): Input tensor (#batch, L, input_size). - ilens (torch.Tensor): Input length (#batch). - prev_states (torch.Tensor): Not to be used now. - Returns: - torch.Tensor: Output tensor (#batch, L, output_size). - torch.Tensor: Output length (#batch). - torch.Tensor: Not to be used now. - """ - masks = (~make_pad_mask(ilens)[:, None, :]).to(xs_pad.device) - if ( - isinstance(self.embed, Conv2dSubsampling) - or isinstance(self.embed, Conv2dSubsampling6) - or isinstance(self.embed, Conv2dSubsampling8) - ): - short_status, limit_size = check_short_utt(self.embed, xs_pad.size(1)) - if short_status: - raise TooShortUttError( - f"has {xs_pad.size(1)} frames and is too short for subsampling " - + f"(it needs more than {limit_size} frames), return empty results", - xs_pad.size(1), - limit_size, - ) - xs_pad, masks = self.embed(xs_pad, masks) - else: - xs_pad = self.embed(xs_pad) - num_layer = len(self.encoders) - for idx, encoder in enumerate(self.encoders): - xs_pad, masks = encoder(xs_pad, masks) - if idx == num_layer // 2 - 1: - hidden_feature = xs_pad - if isinstance(xs_pad, tuple): - xs_pad = xs_pad[0] - hidden_feature = hidden_feature[0] - if self.normalize_before: - xs_pad = self.after_norm(xs_pad) - self.hidden_feature = self.after_norm(hidden_feature) - - olens = masks.squeeze(1).sum(1) - return xs_pad, olens, None diff --git a/funasr_detach/models/model_hf/__init__.py b/funasr_detach/models/model_hf/__init__.py deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/funasr_detach/models/monotonic_aligner/__init__.py b/funasr_detach/models/monotonic_aligner/__init__.py deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/funasr_detach/models/monotonic_aligner/model.py b/funasr_detach/models/monotonic_aligner/model.py deleted file mode 100644 index 6c4ed9f024db40f97b1dda9e07456db3f9a490f3..0000000000000000000000000000000000000000 --- a/funasr_detach/models/monotonic_aligner/model.py +++ /dev/null @@ -1,238 +0,0 @@ -#!/usr/bin/env python3 -# -*- encoding: utf-8 -*- -# Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved. -# MIT License (https://opensource.org/licenses/MIT) - -import time -import copy -import torch -from torch.cuda.amp import autocast -from typing import Union, Dict, List, Tuple, Optional - -from funasr_detach.register import tables -from funasr_detach.models.ctc.ctc import CTC -from funasr_detach.utils import postprocess_utils -from funasr_detach.utils.datadir_writer import DatadirWriter -from funasr_detach.models.paraformer.cif_predictor import mae_loss -from funasr_detach.train_utils.device_funcs import force_gatherable -from funasr_detach.models.transformer.utils.add_sos_eos import add_sos_eos -from funasr_detach.models.transformer.utils.nets_utils import make_pad_mask -from funasr_detach.utils.timestamp_tools import ts_prediction_lfr6_standard -from funasr_detach.utils.load_utils import load_audio_text_image_video, extract_fbank - - -@tables.register("model_classes", "MonotonicAligner") -class MonotonicAligner(torch.nn.Module): - """ - Author: Speech Lab of DAMO Academy, Alibaba Group - Achieving timestamp prediction while recognizing with non-autoregressive end-to-end ASR model - https://arxiv.org/abs/2301.12343 - """ - - def __init__( - self, - input_size: int = 80, - specaug: Optional[str] = None, - specaug_conf: Optional[Dict] = None, - normalize: str = None, - normalize_conf: Optional[Dict] = None, - encoder: str = None, - encoder_conf: Optional[Dict] = None, - predictor: str = None, - predictor_conf: Optional[Dict] = None, - predictor_bias: int = 0, - length_normalized_loss: bool = False, - **kwargs, - ): - super().__init__() - - if specaug is not None: - specaug_class = tables.specaug_classes.get(specaug) - specaug = specaug_class(**specaug_conf) - if normalize is not None: - normalize_class = tables.normalize_classes.get(normalize) - normalize = normalize_class(**normalize_conf) - encoder_class = tables.encoder_classes.get(encoder) - encoder = encoder_class(input_size=input_size, **encoder_conf) - encoder_output_size = encoder.output_size() - predictor_class = tables.predictor_classes.get(predictor) - predictor = predictor_class(**predictor_conf) - self.specaug = specaug - self.normalize = normalize - self.encoder = encoder - self.predictor = predictor - self.criterion_pre = mae_loss(normalize_length=length_normalized_loss) - self.predictor_bias = predictor_bias - - def forward( - self, - speech: torch.Tensor, - speech_lengths: torch.Tensor, - text: torch.Tensor, - text_lengths: torch.Tensor, - ) -> Tuple[torch.Tensor, Dict[str, torch.Tensor], torch.Tensor]: - """Frontend + Encoder + Decoder + Calc loss - Args: - speech: (Batch, Length, ...) - speech_lengths: (Batch, ) - text: (Batch, Length) - text_lengths: (Batch,) - """ - assert text_lengths.dim() == 1, text_lengths.shape - # Check that batch_size is unified - assert ( - speech.shape[0] - == speech_lengths.shape[0] - == text.shape[0] - == text_lengths.shape[0] - ), (speech.shape, speech_lengths.shape, text.shape, text_lengths.shape) - batch_size = speech.shape[0] - # for data-parallel - text = text[:, : text_lengths.max()] - speech = speech[:, : speech_lengths.max()] - - # 1. Encoder - encoder_out, encoder_out_lens = self.encode(speech, speech_lengths) - - encoder_out_mask = ( - ~make_pad_mask(encoder_out_lens, maxlen=encoder_out.size(1))[:, None, :] - ).to(encoder_out.device) - if self.predictor_bias == 1: - _, text = add_sos_eos(text, 1, 2, -1) - text_lengths = text_lengths + self.predictor_bias - _, _, _, _, pre_token_length2 = self.predictor( - encoder_out, text, encoder_out_mask, ignore_id=-1 - ) - - # loss_pre = self.criterion_pre(ys_pad_lens.type_as(pre_token_length), pre_token_length) - loss_pre = self.criterion_pre( - text_lengths.type_as(pre_token_length2), pre_token_length2 - ) - - loss = loss_pre - stats = dict() - - # Collect Attn branch stats - stats["loss_pre"] = loss_pre.detach().cpu() if loss_pre is not None else None - stats["loss"] = torch.clone(loss.detach()) - - # force_gatherable: to-device and to-tensor if scalar for DataParallel - loss, stats, weight = force_gatherable((loss, stats, batch_size), loss.device) - return loss, stats, weight - - def calc_predictor_timestamp(self, encoder_out, encoder_out_lens, token_num): - encoder_out_mask = ( - ~make_pad_mask(encoder_out_lens, maxlen=encoder_out.size(1))[:, None, :] - ).to(encoder_out.device) - ds_alphas, ds_cif_peak, us_alphas, us_peaks = ( - self.predictor.get_upsample_timestamp( - encoder_out, encoder_out_mask, token_num - ) - ) - return ds_alphas, ds_cif_peak, us_alphas, us_peaks - - def encode( - self, - speech: torch.Tensor, - speech_lengths: torch.Tensor, - **kwargs, - ) -> Tuple[torch.Tensor, torch.Tensor]: - """Encoder. Note that this method is used by asr_inference.py - Args: - speech: (Batch, Length, ...) - speech_lengths: (Batch, ) - ind: int - """ - with autocast(False): - - # Data augmentation - if self.specaug is not None and self.training: - speech, speech_lengths = self.specaug(speech, speech_lengths) - - # Normalization for feature: e.g. Global-CMVN, Utterance-CMVN - if self.normalize is not None: - speech, speech_lengths = self.normalize(speech, speech_lengths) - - # Forward encoder - encoder_out, encoder_out_lens, _ = self.encoder(speech, speech_lengths) - if isinstance(encoder_out, tuple): - encoder_out = encoder_out[0] - - return encoder_out, encoder_out_lens - - def inference( - self, - data_in, - data_lengths=None, - key: list = None, - tokenizer=None, - frontend=None, - **kwargs, - ): - meta_data = {} - # extract fbank feats - time1 = time.perf_counter() - audio_list, text_token_int_list = load_audio_text_image_video( - data_in, - fs=frontend.fs, - audio_fs=kwargs.get("fs", 16000), - data_type=kwargs.get("data_type", "sound"), - tokenizer=tokenizer, - ) - time2 = time.perf_counter() - meta_data["load_data"] = f"{time2 - time1:0.3f}" - speech, speech_lengths = extract_fbank( - audio_list, data_type=kwargs.get("data_type", "sound"), frontend=frontend - ) - time3 = time.perf_counter() - meta_data["extract_feat"] = f"{time3 - time2:0.3f}" - meta_data["batch_data_time"] = ( - speech_lengths.sum().item() * frontend.frame_shift * frontend.lfr_n / 1000 - ) - - speech = speech.to(device=kwargs["device"]) - speech_lengths = speech_lengths.to(device=kwargs["device"]) - - # Encoder - encoder_out, encoder_out_lens = self.encode(speech, speech_lengths) - - # predictor - text_lengths = torch.tensor([len(i) + 1 for i in text_token_int_list]).to( - encoder_out.device - ) - _, _, us_alphas, us_peaks = self.calc_predictor_timestamp( - encoder_out, encoder_out_lens, token_num=text_lengths - ) - - results = [] - ibest_writer = None - if kwargs.get("output_dir") is not None: - if not hasattr(self, "writer"): - self.writer = DatadirWriter(kwargs.get("output_dir")) - ibest_writer = self.writer["tp_res"] - - for i, (us_alpha, us_peak, token_int) in enumerate( - zip(us_alphas, us_peaks, text_token_int_list) - ): - token = tokenizer.ids2tokens(token_int) - timestamp_str, timestamp = ts_prediction_lfr6_standard( - us_alpha[: encoder_out_lens[i] * 3], - us_peak[: encoder_out_lens[i] * 3], - copy.copy(token), - ) - text_postprocessed, time_stamp_postprocessed, _ = ( - postprocess_utils.sentence_postprocess(token, timestamp) - ) - result_i = { - "key": key[i], - "text": text_postprocessed, - "timestamp": time_stamp_postprocessed, - } - results.append(result_i) - - if ibest_writer: - # ibest_writer["token"][key[i]] = " ".join(token) - ibest_writer["timestamp_list"][key[i]] = time_stamp_postprocessed - ibest_writer["timestamp_str"][key[i]] = timestamp_str - - return results, meta_data diff --git a/funasr_detach/models/monotonic_aligner/template.yaml b/funasr_detach/models/monotonic_aligner/template.yaml deleted file mode 100644 index b1379de787be8ccefb848244b7186bf55cb6856b..0000000000000000000000000000000000000000 --- a/funasr_detach/models/monotonic_aligner/template.yaml +++ /dev/null @@ -1,115 +0,0 @@ -# This is an example that demonstrates how to configure a model file. -# You can modify the configuration according to your own requirements. - -# to print the register_table: -# from funasr.register import tables -# tables.print() - -# network architecture -model: MonotonicAligner -model_conf: - length_normalized_loss: False - predictor_bias: 1 - -# encoder -encoder: SANMEncoder -encoder_conf: - output_size: 320 - attention_heads: 4 - linear_units: 1280 - num_blocks: 30 - dropout_rate: 0.1 - positional_dropout_rate: 0.1 - attention_dropout_rate: 0.1 - input_layer: pe - pos_enc_class: SinusoidalPositionEncoder - normalize_before: true - kernel_size: 11 - sanm_shfit: 0 - selfattention_layer_type: sanm - -predictor: CifPredictorV3 -predictor_conf: - idim: 320 - threshold: 1.0 - l_order: 1 - r_order: 1 - tail_threshold: 0.45 - smooth_factor2: 0.25 - noise_threshold2: 0.01 - upsample_times: 3 - use_cif1_cnn: false - upsample_type: cnn_blstm - -# frontend related -frontend: WavFrontend -frontend_conf: - fs: 16000 - window: hamming - n_mels: 80 - frame_length: 25 - frame_shift: 10 - lfr_m: 7 - lfr_n: 6 - -specaug: SpecAugLFR -specaug_conf: - apply_time_warp: false - time_warp_window: 5 - time_warp_mode: bicubic - apply_freq_mask: true - freq_mask_width_range: - - 0 - - 30 - lfr_rate: 6 - num_freq_mask: 1 - apply_time_mask: true - time_mask_width_range: - - 0 - - 12 - num_time_mask: 1 - -train_conf: - accum_grad: 1 - grad_clip: 5 - max_epoch: 150 - val_scheduler_criterion: - - valid - - acc - best_model_criterion: - - - valid - - acc - - max - keep_nbest_models: 10 - log_interval: 50 - -optim: adam -optim_conf: - lr: 0.0005 -scheduler: warmuplr -scheduler_conf: - warmup_steps: 30000 - -dataset: AudioDataset -dataset_conf: - index_ds: IndexDSJsonl - batch_sampler: DynamicBatchLocalShuffleSampler - batch_type: example # example or length - batch_size: 1 # if batch_type is example, batch_size is the numbers of samples; if length, batch_size is source_token_len+target_token_len; - max_token_length: 2048 # filter samples if source_token_len+target_token_len > max_token_length, - buffer_size: 500 - shuffle: True - num_workers: 0 - -tokenizer: CharTokenizer -tokenizer_conf: - unk_symbol: - split_with_space: true - -ctc_conf: - dropout_rate: 0.0 - ctc_type: builtin - reduce: true - ignore_nan_grad: true - -normalize: null diff --git a/funasr_detach/models/mossformer/__init__.py b/funasr_detach/models/mossformer/__init__.py deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/funasr_detach/models/mossformer/e2e_ss.py b/funasr_detach/models/mossformer/e2e_ss.py deleted file mode 100644 index 7d2463ec68f04ed62fd79efe1ffdfec759e60458..0000000000000000000000000000000000000000 --- a/funasr_detach/models/mossformer/e2e_ss.py +++ /dev/null @@ -1,98 +0,0 @@ -import math -import torch -import torch.nn as nn -import torch.nn.functional as F -import copy -from funasr_detach.models.base_model import FunASRModel -from funasr_detach.models.encoder.mossformer_encoder import ( - MossFormerEncoder, - MossFormer_MaskNet, -) -from funasr_detach.models.decoder.mossformer_decoder import MossFormerDecoder - - -class MossFormer(FunASRModel): - """The MossFormer model for separating input mixed speech into different speaker's speech. - - Arguments - --------- - in_channels : int - Number of channels at the output of the encoder. - out_channels : int - Number of channels that would be inputted to the intra and inter blocks. - num_blocks : int - Number of layers of Dual Computation Block. - norm : str - Normalization type. - num_spks : int - Number of sources (speakers). - skip_around_intra : bool - Skip connection around intra. - use_global_pos_enc : bool - Global positional encodings. - max_length : int - Maximum sequence length. - kernel_size: int - Encoder and decoder kernel size - """ - - def __init__( - self, - in_channels=512, - out_channels=512, - num_blocks=24, - kernel_size=16, - norm="ln", - num_spks=2, - skip_around_intra=True, - use_global_pos_enc=True, - max_length=20000, - ): - super(MossFormer, self).__init__() - self.num_spks = num_spks - # Encoding - self.enc = MossFormerEncoder( - kernel_size=kernel_size, out_channels=in_channels, in_channels=1 - ) - - ##Compute Mask - self.mask_net = MossFormer_MaskNet( - in_channels=in_channels, - out_channels=out_channels, - num_blocks=num_blocks, - norm=norm, - num_spks=num_spks, - skip_around_intra=skip_around_intra, - use_global_pos_enc=use_global_pos_enc, - max_length=max_length, - ) - self.dec = MossFormerDecoder( - in_channels=out_channels, - out_channels=1, - kernel_size=kernel_size, - stride=kernel_size // 2, - bias=False, - ) - - def forward(self, input): - x = self.enc(input) - mask = self.mask_net(x) - x = torch.stack([x] * self.num_spks) - sep_x = x * mask - - # Decoding - est_source = torch.cat( - [self.dec(sep_x[i]).unsqueeze(-1) for i in range(self.num_spks)], - dim=-1, - ) - T_origin = input.size(1) - T_est = est_source.size(1) - if T_origin > T_est: - est_source = F.pad(est_source, (0, 0, 0, T_origin - T_est)) - else: - est_source = est_source[:, :T_origin, :] - - out = [] - for spk in range(self.num_spks): - out.append(est_source[:, :, spk]) - return out diff --git a/funasr_detach/models/mossformer/mossformer.py b/funasr_detach/models/mossformer/mossformer.py deleted file mode 100644 index be352895c478406224b4706e509752cfa04b3e52..0000000000000000000000000000000000000000 --- a/funasr_detach/models/mossformer/mossformer.py +++ /dev/null @@ -1,324 +0,0 @@ -import torch -import torch.nn.functional as F -from torch import nn, einsum -from einops import rearrange - - -def identity(t, *args, **kwargs): - return t - - -def append_dims(x, num_dims): - if num_dims <= 0: - return x - return x.view(*x.shape, *((1,) * num_dims)) - - -def exists(val): - return val is not None - - -def default(val, d): - return val if exists(val) else d - - -def padding_to_multiple_of(n, mult): - remainder = n % mult - if remainder == 0: - return 0 - return mult - remainder - - -class Transpose(nn.Module): - """Wrapper class of torch.transpose() for Sequential module.""" - - def __init__(self, shape: tuple): - super(Transpose, self).__init__() - self.shape = shape - - def forward(self, x): - return x.transpose(*self.shape) - - -class DepthwiseConv1d(nn.Module): - """ - When groups == in_channels and out_channels == K * in_channels, where K is a positive integer, - this operation is termed in literature as depthwise convolution. - Args: - in_channels (int): Number of channels in the input - out_channels (int): Number of channels produced by the convolution - kernel_size (int or tuple): Size of the convolving kernel - stride (int, optional): Stride of the convolution. Default: 1 - padding (int or tuple, optional): Zero-padding added to both sides of the input. Default: 0 - bias (bool, optional): If True, adds a learnable bias to the output. Default: True - Inputs: inputs - - **inputs** (batch, in_channels, time): Tensor containing input vector - Returns: outputs - - **outputs** (batch, out_channels, time): Tensor produces by depthwise 1-D convolution. - """ - - def __init__( - self, - in_channels: int, - out_channels: int, - kernel_size: int, - stride: int = 1, - padding: int = 0, - bias: bool = False, - ) -> None: - super(DepthwiseConv1d, self).__init__() - assert ( - out_channels % in_channels == 0 - ), "out_channels should be constant multiple of in_channels" - self.conv = nn.Conv1d( - in_channels=in_channels, - out_channels=out_channels, - kernel_size=kernel_size, - groups=in_channels, - stride=stride, - padding=padding, - bias=bias, - ) - - def forward(self, inputs): - return self.conv(inputs) - - -class ConvModule(nn.Module): - """ - Conformer convolution module starts with a pointwise convolution and a gated linear unit (GLU). - This is followed by a single 1-D depthwise convolution layer. Batchnorm is deployed just after the convolution - to aid training deep models. - Args: - in_channels (int): Number of channels in the input - kernel_size (int or tuple, optional): Size of the convolving kernel Default: 31 - dropout_p (float, optional): probability of dropout - Inputs: inputs - inputs (batch, time, dim): Tensor contains input sequences - Outputs: outputs - outputs (batch, time, dim): Tensor produces by conformer convolution module. - """ - - def __init__( - self, - in_channels: int, - kernel_size: int = 17, - expansion_factor: int = 2, - dropout_p: float = 0.1, - ) -> None: - super(ConvModule, self).__init__() - assert ( - kernel_size - 1 - ) % 2 == 0, "kernel_size should be a odd number for 'SAME' padding" - assert expansion_factor == 2, "Currently, Only Supports expansion_factor 2" - - self.sequential = nn.Sequential( - Transpose(shape=(1, 2)), - DepthwiseConv1d( - in_channels, - in_channels, - kernel_size, - stride=1, - padding=(kernel_size - 1) // 2, - ), - ) - - def forward(self, inputs): - return inputs + self.sequential(inputs).transpose(1, 2) - - -class OffsetScale(nn.Module): - def __init__(self, dim, heads=1): - super().__init__() - self.gamma = nn.Parameter(torch.ones(heads, dim)) - self.beta = nn.Parameter(torch.zeros(heads, dim)) - nn.init.normal_(self.gamma, std=0.02) - - def forward(self, x): - out = einsum("... d, h d -> ... h d", x, self.gamma) + self.beta - return out.unbind(dim=-2) - - -class FFConvM(nn.Module): - def __init__(self, dim_in, dim_out, norm_klass=nn.LayerNorm, dropout=0.1): - super().__init__() - self.mdl = nn.Sequential( - norm_klass(dim_in), - nn.Linear(dim_in, dim_out), - nn.SiLU(), - ConvModule(dim_out), - nn.Dropout(dropout), - ) - - def forward( - self, - x, - ): - output = self.mdl(x) - return output - - -class FLASH_ShareA_FFConvM(nn.Module): - def __init__( - self, - *, - dim, - group_size=256, - query_key_dim=128, - expansion_factor=1.0, - causal=False, - dropout=0.1, - rotary_pos_emb=None, - norm_klass=nn.LayerNorm, - shift_tokens=True - ): - super().__init__() - hidden_dim = int(dim * expansion_factor) - self.group_size = group_size - self.causal = causal - self.shift_tokens = shift_tokens - - # positional embeddings - self.rotary_pos_emb = rotary_pos_emb - # norm - self.dropout = nn.Dropout(dropout) - # projections - - self.to_hidden = FFConvM( - dim_in=dim, - dim_out=hidden_dim, - norm_klass=norm_klass, - dropout=dropout, - ) - self.to_qk = FFConvM( - dim_in=dim, - dim_out=query_key_dim, - norm_klass=norm_klass, - dropout=dropout, - ) - - self.qk_offset_scale = OffsetScale(query_key_dim, heads=4) - - self.to_out = FFConvM( - dim_in=dim * 2, - dim_out=dim, - norm_klass=norm_klass, - dropout=dropout, - ) - - self.gateActivate = nn.Sigmoid() - - def forward(self, x, *, mask=None): - """ - b - batch - n - sequence length (within groups) - g - group dimension - d - feature dimension (keys) - e - feature dimension (values) - i - sequence dimension (source) - j - sequence dimension (target) - """ - - normed_x = x - - # do token shift - a great, costless trick from an independent AI researcher in Shenzhen - residual = x - - if self.shift_tokens: - x_shift, x_pass = normed_x.chunk(2, dim=-1) - x_shift = F.pad(x_shift, (0, 0, 1, -1), value=0.0) - normed_x = torch.cat((x_shift, x_pass), dim=-1) - - # initial projections - - v, u = self.to_hidden(normed_x).chunk(2, dim=-1) - qk = self.to_qk(normed_x) - - # offset and scale - quad_q, lin_q, quad_k, lin_k = self.qk_offset_scale(qk) - att_v, att_u = self.cal_attention(x, quad_q, lin_q, quad_k, lin_k, v, u) - out = (att_u * v) * self.gateActivate(att_v * u) - x = x + self.to_out(out) - return x - - def cal_attention(self, x, quad_q, lin_q, quad_k, lin_k, v, u, mask=None): - b, n, device, g = x.shape[0], x.shape[-2], x.device, self.group_size - - if exists(mask): - lin_mask = rearrange(mask, "... -> ... 1") - lin_k = lin_k.masked_fill(~lin_mask, 0.0) - - # rotate queries and keys - - if exists(self.rotary_pos_emb): - quad_q, lin_q, quad_k, lin_k = map( - self.rotary_pos_emb.rotate_queries_or_keys, - (quad_q, lin_q, quad_k, lin_k), - ) - - # padding for groups - - padding = padding_to_multiple_of(n, g) - - if padding > 0: - quad_q, quad_k, lin_q, lin_k, v, u = map( - lambda t: F.pad(t, (0, 0, 0, padding), value=0.0), - (quad_q, quad_k, lin_q, lin_k, v, u), - ) - - mask = default(mask, torch.ones((b, n), device=device, dtype=torch.bool)) - mask = F.pad(mask, (0, padding), value=False) - - # group along sequence - - quad_q, quad_k, lin_q, lin_k, v, u = map( - lambda t: rearrange(t, "b (g n) d -> b g n d", n=self.group_size), - (quad_q, quad_k, lin_q, lin_k, v, u), - ) - - if exists(mask): - mask = rearrange(mask, "b (g j) -> b g 1 j", j=g) - - # calculate quadratic attention output - - sim = einsum("... i d, ... j d -> ... i j", quad_q, quad_k) / g - - attn = F.relu(sim) ** 2 - attn = self.dropout(attn) - - if exists(mask): - attn = attn.masked_fill(~mask, 0.0) - - if self.causal: - causal_mask = torch.ones((g, g), dtype=torch.bool, device=device).triu(1) - attn = attn.masked_fill(causal_mask, 0.0) - - quad_out_v = einsum("... i j, ... j d -> ... i d", attn, v) - quad_out_u = einsum("... i j, ... j d -> ... i d", attn, u) - - # calculate linear attention output - - if self.causal: - lin_kv = einsum("b g n d, b g n e -> b g d e", lin_k, v) / g - # exclusive cumulative sum along group dimension - lin_kv = lin_kv.cumsum(dim=1) - lin_kv = F.pad(lin_kv, (0, 0, 0, 0, 1, -1), value=0.0) - lin_out_v = einsum("b g d e, b g n d -> b g n e", lin_kv, lin_q) - - lin_ku = einsum("b g n d, b g n e -> b g d e", lin_k, u) / g - # exclusive cumulative sum along group dimension - lin_ku = lin_ku.cumsum(dim=1) - lin_ku = F.pad(lin_ku, (0, 0, 0, 0, 1, -1), value=0.0) - lin_out_u = einsum("b g d e, b g n d -> b g n e", lin_ku, lin_q) - else: - lin_kv = einsum("b g n d, b g n e -> b d e", lin_k, v) / n - lin_out_v = einsum("b g n d, b d e -> b g n e", lin_q, lin_kv) - - lin_ku = einsum("b g n d, b g n e -> b d e", lin_k, u) / n - lin_out_u = einsum("b g n d, b d e -> b g n e", lin_q, lin_ku) - - # fold back groups into full sequence, and excise out padding - return map( - lambda t: rearrange(t, "b g n d -> b (g n) d")[:, :n], - (quad_out_v + lin_out_v, quad_out_u + lin_out_u), - ) diff --git a/funasr_detach/models/mossformer/mossformer_decoder.py b/funasr_detach/models/mossformer/mossformer_decoder.py deleted file mode 100644 index 56ad2ecb325df5499a6d9559391b8565927ea199..0000000000000000000000000000000000000000 --- a/funasr_detach/models/mossformer/mossformer_decoder.py +++ /dev/null @@ -1,50 +0,0 @@ -import torch -import torch.nn as nn - - -class MossFormerDecoder(nn.ConvTranspose1d): - """A decoder layer that consists of ConvTranspose1d. - - Arguments - --------- - kernel_size : int - Length of filters. - in_channels : int - Number of input channels. - out_channels : int - Number of output channels. - - - Example - --------- - >>> x = torch.randn(2, 100, 1000) - >>> decoder = Decoder(kernel_size=4, in_channels=100, out_channels=1) - >>> h = decoder(x) - >>> h.shape - torch.Size([2, 1003]) - """ - - def __init__(self, *args, **kwargs): - super(MossFormerDecoder, self).__init__(*args, **kwargs) - - def forward(self, x): - """Return the decoded output. - - Arguments - --------- - x : torch.Tensor - Input tensor with dimensionality [B, N, L]. - where, B = Batchsize, - N = number of filters - L = time points - """ - - if x.dim() not in [2, 3]: - raise RuntimeError("{} accept 3/4D tensor as input".format(self.__name__)) - x = super().forward(x if x.dim() == 3 else torch.unsqueeze(x, 1)) - - if torch.squeeze(x).dim() == 1: - x = torch.squeeze(x, dim=1) - else: - x = torch.squeeze(x) - return x diff --git a/funasr_detach/models/mossformer/mossformer_encoder.py b/funasr_detach/models/mossformer/mossformer_encoder.py deleted file mode 100644 index 7458b2ebb7add76d90089eb33b0b18a43ed73415..0000000000000000000000000000000000000000 --- a/funasr_detach/models/mossformer/mossformer_encoder.py +++ /dev/null @@ -1,436 +0,0 @@ -import torch -import torch.nn as nn -import torch.nn.functional as F - -try: - from rotary_embedding_torch import RotaryEmbedding -except: - print( - "If you want use mossformer, lease install rotary_embedding_torch by: \n pip install -U rotary_embedding_torch" - ) -from funasr_detach.models.transformer.layer_norm import ( - GlobalLayerNorm, - CumulativeLayerNorm, - ScaleNorm, -) -from funasr_detach.models.transformer.embedding import ScaledSinuEmbedding -from funasr_detach.models.transformer.mossformer import FLASH_ShareA_FFConvM - - -def select_norm(norm, dim, shape): - """Just a wrapper to select the normalization type.""" - - if norm == "gln": - return GlobalLayerNorm(dim, shape, elementwise_affine=True) - if norm == "cln": - return CumulativeLayerNorm(dim, elementwise_affine=True) - if norm == "ln": - return nn.GroupNorm(1, dim, eps=1e-8) - else: - return nn.BatchNorm1d(dim) - - -class MossformerBlock(nn.Module): - def __init__( - self, - *, - dim, - depth, - group_size=256, - query_key_dim=128, - expansion_factor=4.0, - causal=False, - attn_dropout=0.1, - norm_type="scalenorm", - shift_tokens=True - ): - super().__init__() - assert norm_type in ( - "scalenorm", - "layernorm", - ), "norm_type must be one of scalenorm or layernorm" - - if norm_type == "scalenorm": - norm_klass = ScaleNorm - elif norm_type == "layernorm": - norm_klass = nn.LayerNorm - - self.group_size = group_size - - rotary_pos_emb = RotaryEmbedding(dim=min(32, query_key_dim)) - # max rotary embedding dimensions of 32, partial Rotary embeddings, from Wang et al - GPT-J - self.layers = nn.ModuleList( - [ - FLASH_ShareA_FFConvM( - dim=dim, - group_size=group_size, - query_key_dim=query_key_dim, - expansion_factor=expansion_factor, - causal=causal, - dropout=attn_dropout, - rotary_pos_emb=rotary_pos_emb, - norm_klass=norm_klass, - shift_tokens=shift_tokens, - ) - for _ in range(depth) - ] - ) - - def forward(self, x, *, mask=None): - ii = 0 - for flash in self.layers: - x = flash(x, mask=mask) - ii = ii + 1 - return x - - -class MossFormer_MaskNet(nn.Module): - """The MossFormer module for computing output masks. - - Arguments - --------- - in_channels : int - Number of channels at the output of the encoder. - out_channels : int - Number of channels that would be inputted to the intra and inter blocks. - num_blocks : int - Number of layers of Dual Computation Block. - norm : str - Normalization type. - num_spks : int - Number of sources (speakers). - skip_around_intra : bool - Skip connection around intra. - use_global_pos_enc : bool - Global positional encodings. - max_length : int - Maximum sequence length. - - Example - --------- - >>> mossformer_block = MossFormerM(1, 64, 8) - >>> mossformer_masknet = MossFormer_MaskNet(64, 64, intra_block, num_spks=2) - >>> x = torch.randn(10, 64, 2000) - >>> x = mossformer_masknet(x) - >>> x.shape - torch.Size([2, 10, 64, 2000]) - """ - - def __init__( - self, - in_channels, - out_channels, - num_blocks=24, - norm="ln", - num_spks=2, - skip_around_intra=True, - use_global_pos_enc=True, - max_length=20000, - ): - super(MossFormer_MaskNet, self).__init__() - self.num_spks = num_spks - self.num_blocks = num_blocks - self.norm = select_norm(norm, in_channels, 3) - self.conv1d_encoder = nn.Conv1d(in_channels, out_channels, 1, bias=False) - self.use_global_pos_enc = use_global_pos_enc - - if self.use_global_pos_enc: - self.pos_enc = ScaledSinuEmbedding(out_channels) - - self.mdl = Computation_Block( - num_blocks, - out_channels, - norm, - skip_around_intra=skip_around_intra, - ) - - self.conv1d_out = nn.Conv1d( - out_channels, out_channels * num_spks, kernel_size=1 - ) - self.conv1_decoder = nn.Conv1d(out_channels, in_channels, 1, bias=False) - self.prelu = nn.PReLU() - self.activation = nn.ReLU() - # gated output layer - self.output = nn.Sequential(nn.Conv1d(out_channels, out_channels, 1), nn.Tanh()) - self.output_gate = nn.Sequential( - nn.Conv1d(out_channels, out_channels, 1), nn.Sigmoid() - ) - - def forward(self, x): - """Returns the output tensor. - - Arguments - --------- - x : torch.Tensor - Input tensor of dimension [B, N, S]. - - Returns - ------- - out : torch.Tensor - Output tensor of dimension [spks, B, N, S] - where, spks = Number of speakers - B = Batchsize, - N = number of filters - S = the number of time frames - """ - - # before each line we indicate the shape after executing the line - - # [B, N, L] - x = self.norm(x) - - # [B, N, L] - x = self.conv1d_encoder(x) - if self.use_global_pos_enc: - # x = self.pos_enc(x.transpose(1, -1)).transpose(1, -1) + x * ( - # x.size(1) ** 0.5) - base = x - x = x.transpose(1, -1) - emb = self.pos_enc(x) - emb = emb.transpose(0, -1) - # print('base: {}, emb: {}'.format(base.shape, emb.shape)) - x = base + emb - - # [B, N, S] - # for i in range(self.num_modules): - # x = self.dual_mdl[i](x) - x = self.mdl(x) - x = self.prelu(x) - - # [B, N*spks, S] - x = self.conv1d_out(x) - B, _, S = x.shape - - # [B*spks, N, S] - x = x.view(B * self.num_spks, -1, S) - - # [B*spks, N, S] - x = self.output(x) * self.output_gate(x) - - # [B*spks, N, S] - x = self.conv1_decoder(x) - - # [B, spks, N, S] - _, N, L = x.shape - x = x.view(B, self.num_spks, N, L) - x = self.activation(x) - - # [spks, B, N, S] - x = x.transpose(0, 1) - - return x - - -class MossFormerEncoder(nn.Module): - """Convolutional Encoder Layer. - - Arguments - --------- - kernel_size : int - Length of filters. - in_channels : int - Number of input channels. - out_channels : int - Number of output channels. - - Example - ------- - >>> x = torch.randn(2, 1000) - >>> encoder = Encoder(kernel_size=4, out_channels=64) - >>> h = encoder(x) - >>> h.shape - torch.Size([2, 64, 499]) - """ - - def __init__(self, kernel_size=2, out_channels=64, in_channels=1): - super(MossFormerEncoder, self).__init__() - self.conv1d = nn.Conv1d( - in_channels=in_channels, - out_channels=out_channels, - kernel_size=kernel_size, - stride=kernel_size // 2, - groups=1, - bias=False, - ) - self.in_channels = in_channels - - def forward(self, x): - """Return the encoded output. - - Arguments - --------- - x : torch.Tensor - Input tensor with dimensionality [B, L]. - Return - ------ - x : torch.Tensor - Encoded tensor with dimensionality [B, N, T_out]. - - where B = Batchsize - L = Number of timepoints - N = Number of filters - T_out = Number of timepoints at the output of the encoder - """ - # B x L -> B x 1 x L - if self.in_channels == 1: - x = torch.unsqueeze(x, dim=1) - # B x 1 x L -> B x N x T_out - x = self.conv1d(x) - x = F.relu(x) - - return x - - -class MossFormerM(nn.Module): - """This class implements the transformer encoder. - - Arguments - --------- - num_blocks : int - Number of mossformer blocks to include. - d_model : int - The dimension of the input embedding. - attn_dropout : float - Dropout for the self-attention (Optional). - group_size: int - the chunk size - query_key_dim: int - the attention vector dimension - expansion_factor: int - the expansion factor for the linear projection in conv module - causal: bool - true for causal / false for non causal - - Example - ------- - >>> import torch - >>> x = torch.rand((8, 60, 512)) - >>> net = TransformerEncoder_MossFormerM(num_blocks=8, d_model=512) - >>> output, _ = net(x) - >>> output.shape - torch.Size([8, 60, 512]) - """ - - def __init__( - self, - num_blocks, - d_model=None, - causal=False, - group_size=256, - query_key_dim=128, - expansion_factor=4.0, - attn_dropout=0.1, - ): - super().__init__() - - self.mossformerM = MossformerBlock( - dim=d_model, - depth=num_blocks, - group_size=group_size, - query_key_dim=query_key_dim, - expansion_factor=expansion_factor, - causal=causal, - attn_dropout=attn_dropout, - ) - self.norm = nn.LayerNorm(d_model, eps=1e-6) - - def forward( - self, - src, - ): - """ - Arguments - ---------- - src : torch.Tensor - Tensor shape [B, L, N], - where, B = Batchsize, - L = time points - N = number of filters - The sequence to the encoder layer (required). - src_mask : tensor - The mask for the src sequence (optional). - src_key_padding_mask : tensor - The mask for the src keys per batch (optional). - """ - output = self.mossformerM(src) - output = self.norm(output) - - return output - - -class Computation_Block(nn.Module): - """Computation block for dual-path processing. - - Arguments - --------- - out_channels : int - Dimensionality of inter/intra model. - norm : str - Normalization type. - skip_around_intra : bool - Skip connection around the intra layer. - - Example - --------- - >>> comp_block = Computation_Block(64) - >>> x = torch.randn(10, 64, 100) - >>> x = comp_block(x) - >>> x.shape - torch.Size([10, 64, 100]) - """ - - def __init__( - self, - num_blocks, - out_channels, - norm="ln", - skip_around_intra=True, - ): - super(Computation_Block, self).__init__() - - ##MossFormer2M: MossFormer with recurrence - # self.intra_mdl = MossFormer2M(num_blocks=num_blocks, d_model=out_channels) - ##MossFormerM: the orignal MossFormer - self.intra_mdl = MossFormerM(num_blocks=num_blocks, d_model=out_channels) - self.skip_around_intra = skip_around_intra - - # Norm - self.norm = norm - if norm is not None: - self.intra_norm = select_norm(norm, out_channels, 3) - - def forward(self, x): - """Returns the output tensor. - - Arguments - --------- - x : torch.Tensor - Input tensor of dimension [B, N, S]. - - - Return - --------- - out: torch.Tensor - Output tensor of dimension [B, N, S]. - where, B = Batchsize, - N = number of filters - S = sequence time index - """ - B, N, S = x.shape - # intra RNN - # [B, S, N] - intra = x.permute(0, 2, 1).contiguous() # .view(B, S, N) - - intra = self.intra_mdl(intra) - - # [B, N, S] - intra = intra.permute(0, 2, 1).contiguous() - if self.norm is not None: - intra = self.intra_norm(intra) - - # [B, N, S] - if self.skip_around_intra: - intra = intra + x - - out = intra - return out diff --git a/funasr_detach/models/normalize/__init__.py b/funasr_detach/models/normalize/__init__.py deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/funasr_detach/models/normalize/global_mvn.py b/funasr_detach/models/normalize/global_mvn.py deleted file mode 100644 index 9809abc52b296252bce35a28694a062d49a035c9..0000000000000000000000000000000000000000 --- a/funasr_detach/models/normalize/global_mvn.py +++ /dev/null @@ -1,116 +0,0 @@ -from pathlib import Path -from typing import Tuple -from typing import Union - -import numpy as np -import torch - -from funasr_detach.models.transformer.utils.nets_utils import make_pad_mask -from funasr_detach.register import tables - - -@tables.register("normalize_classes", "GlobalMVN") -class GlobalMVN(torch.nn.Module): - """Apply global mean and variance normalization - TODO(kamo): Make this class portable somehow - Args: - stats_file: npy file - norm_means: Apply mean normalization - norm_vars: Apply var normalization - eps: - """ - - def __init__( - self, - stats_file: Union[Path, str], - norm_means: bool = True, - norm_vars: bool = True, - eps: float = 1.0e-20, - ): - super().__init__() - self.norm_means = norm_means - self.norm_vars = norm_vars - self.eps = eps - stats_file = Path(stats_file) - - self.stats_file = stats_file - stats = np.load(stats_file) - if isinstance(stats, np.ndarray): - # Kaldi like stats - count = stats[0].flatten()[-1] - mean = stats[0, :-1] / count - var = stats[1, :-1] / count - mean * mean - else: - # New style: Npz file - count = stats["count"] - sum_v = stats["sum"] - sum_square_v = stats["sum_square"] - mean = sum_v / count - var = sum_square_v / count - mean * mean - std = np.sqrt(np.maximum(var, eps)) - - self.register_buffer("mean", torch.from_numpy(mean)) - self.register_buffer("std", torch.from_numpy(std)) - - def extra_repr(self): - return ( - f"stats_file={self.stats_file}, " - f"norm_means={self.norm_means}, norm_vars={self.norm_vars}" - ) - - def forward( - self, x: torch.Tensor, ilens: torch.Tensor = None - ) -> Tuple[torch.Tensor, torch.Tensor]: - """Forward function - Args: - x: (B, L, ...) - ilens: (B,) - """ - if ilens is None: - ilens = x.new_full([x.size(0)], x.size(1)) - norm_means = self.norm_means - norm_vars = self.norm_vars - self.mean = self.mean.to(x.device, x.dtype) - self.std = self.std.to(x.device, x.dtype) - mask = make_pad_mask(ilens, x, 1) - - # feat: (B, T, D) - if norm_means: - if x.requires_grad: - x = x - self.mean - else: - x -= self.mean - if x.requires_grad: - x = x.masked_fill(mask, 0.0) - else: - x.masked_fill_(mask, 0.0) - - if norm_vars: - x /= self.std - - return x, ilens - - def inverse( - self, x: torch.Tensor, ilens: torch.Tensor = None - ) -> Tuple[torch.Tensor, torch.Tensor]: - if ilens is None: - ilens = x.new_full([x.size(0)], x.size(1)) - norm_means = self.norm_means - norm_vars = self.norm_vars - self.mean = self.mean.to(x.device, x.dtype) - self.std = self.std.to(x.device, x.dtype) - mask = make_pad_mask(ilens, x, 1) - - if x.requires_grad: - x = x.masked_fill(mask, 0.0) - else: - x.masked_fill_(mask, 0.0) - - if norm_vars: - x *= self.std - - # feat: (B, T, D) - if norm_means: - x += self.mean - x.masked_fill_(make_pad_mask(ilens, x, 1), 0.0) - return x, ilens diff --git a/funasr_detach/models/normalize/utterance_mvn.py b/funasr_detach/models/normalize/utterance_mvn.py deleted file mode 100644 index 0016c352163938cc64be870d08c4b9e74533e475..0000000000000000000000000000000000000000 --- a/funasr_detach/models/normalize/utterance_mvn.py +++ /dev/null @@ -1,87 +0,0 @@ -from typing import Tuple - -import torch - -from funasr_detach.models.transformer.utils.nets_utils import make_pad_mask -from funasr_detach.register import tables - - -@tables.register("normalize_classes", "UtteranceMVN") -class UtteranceMVN(torch.nn.Module): - def __init__( - self, - norm_means: bool = True, - norm_vars: bool = False, - eps: float = 1.0e-20, - ): - super().__init__() - self.norm_means = norm_means - self.norm_vars = norm_vars - self.eps = eps - - def extra_repr(self): - return f"norm_means={self.norm_means}, norm_vars={self.norm_vars}" - - def forward( - self, x: torch.Tensor, ilens: torch.Tensor = None - ) -> Tuple[torch.Tensor, torch.Tensor]: - """Forward function - - Args: - x: (B, L, ...) - ilens: (B,) - - """ - return utterance_mvn( - x, - ilens, - norm_means=self.norm_means, - norm_vars=self.norm_vars, - eps=self.eps, - ) - - -def utterance_mvn( - x: torch.Tensor, - ilens: torch.Tensor = None, - norm_means: bool = True, - norm_vars: bool = False, - eps: float = 1.0e-20, -) -> Tuple[torch.Tensor, torch.Tensor]: - """Apply utterance mean and variance normalization - - Args: - x: (B, T, D), assumed zero padded - ilens: (B,) - norm_means: - norm_vars: - eps: - - """ - if ilens is None: - ilens = x.new_full([x.size(0)], x.size(1)) - ilens_ = ilens.to(x.device, x.dtype).view(-1, *[1 for _ in range(x.dim() - 1)]) - # Zero padding - if x.requires_grad: - x = x.masked_fill(make_pad_mask(ilens, x, 1), 0.0) - else: - x.masked_fill_(make_pad_mask(ilens, x, 1), 0.0) - # mean: (B, 1, D) - mean = x.sum(dim=1, keepdim=True) / ilens_ - - if norm_means: - x -= mean - - if norm_vars: - var = x.pow(2).sum(dim=1, keepdim=True) / ilens_ - std = torch.clamp(var.sqrt(), min=eps) - x = x / std.sqrt() - return x, ilens - else: - if norm_vars: - y = x - mean - y.masked_fill_(make_pad_mask(ilens, y, 1), 0.0) - var = y.pow(2).sum(dim=1, keepdim=True) / ilens_ - std = torch.clamp(var.sqrt(), min=eps) - x /= std - return x, ilens diff --git a/funasr_detach/models/paraformer/__init__.py b/funasr_detach/models/paraformer/__init__.py deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/funasr_detach/models/paraformer/cif_predictor.py b/funasr_detach/models/paraformer/cif_predictor.py deleted file mode 100644 index d9d556d278b2e8cd895d631721c7b35d088835c7..0000000000000000000000000000000000000000 --- a/funasr_detach/models/paraformer/cif_predictor.py +++ /dev/null @@ -1,622 +0,0 @@ -#!/usr/bin/env python3 -# -*- encoding: utf-8 -*- -# Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved. -# MIT License (https://opensource.org/licenses/MIT) - -import torch -import logging -import numpy as np - -from funasr_detach.register import tables -from funasr_detach.train_utils.device_funcs import to_device -from funasr_detach.models.transformer.utils.nets_utils import make_pad_mask - - -@tables.register("predictor_classes", "CifPredictor") -class CifPredictor(torch.nn.Module): - def __init__( - self, - idim, - l_order, - r_order, - threshold=1.0, - dropout=0.1, - smooth_factor=1.0, - noise_threshold=0, - tail_threshold=0.45, - ): - super().__init__() - - self.pad = torch.nn.ConstantPad1d((l_order, r_order), 0) - self.cif_conv1d = torch.nn.Conv1d( - idim, idim, l_order + r_order + 1, groups=idim - ) - self.cif_output = torch.nn.Linear(idim, 1) - self.dropout = torch.nn.Dropout(p=dropout) - self.threshold = threshold - self.smooth_factor = smooth_factor - self.noise_threshold = noise_threshold - self.tail_threshold = tail_threshold - - def forward( - self, - hidden, - target_label=None, - mask=None, - ignore_id=-1, - mask_chunk_predictor=None, - target_label_length=None, - ): - h = hidden - context = h.transpose(1, 2) - queries = self.pad(context) - memory = self.cif_conv1d(queries) - output = memory + context - output = self.dropout(output) - output = output.transpose(1, 2) - output = torch.relu(output) - output = self.cif_output(output) - alphas = torch.sigmoid(output) - alphas = torch.nn.functional.relu( - alphas * self.smooth_factor - self.noise_threshold - ) - if mask is not None: - mask = mask.transpose(-1, -2).float() - alphas = alphas * mask - if mask_chunk_predictor is not None: - alphas = alphas * mask_chunk_predictor - alphas = alphas.squeeze(-1) - mask = mask.squeeze(-1) - if target_label_length is not None: - target_length = target_label_length - elif target_label is not None: - target_length = (target_label != ignore_id).float().sum(-1) - else: - target_length = None - token_num = alphas.sum(-1) - if target_length is not None: - alphas *= (target_length / token_num)[:, None].repeat(1, alphas.size(1)) - elif self.tail_threshold > 0.0: - hidden, alphas, token_num = self.tail_process_fn( - hidden, alphas, token_num, mask=mask - ) - - acoustic_embeds, cif_peak = cif(hidden, alphas, self.threshold) - - if target_length is None and self.tail_threshold > 0.0: - token_num_int = torch.max(token_num).type(torch.int32).item() - acoustic_embeds = acoustic_embeds[:, :token_num_int, :] - - return acoustic_embeds, token_num, alphas, cif_peak - - def tail_process_fn(self, hidden, alphas, token_num=None, mask=None): - b, t, d = hidden.size() - tail_threshold = self.tail_threshold - if mask is not None: - zeros_t = torch.zeros((b, 1), dtype=torch.float32, device=alphas.device) - ones_t = torch.ones_like(zeros_t) - mask_1 = torch.cat([mask, zeros_t], dim=1) - mask_2 = torch.cat([ones_t, mask], dim=1) - mask = mask_2 - mask_1 - tail_threshold = mask * tail_threshold - alphas = torch.cat([alphas, zeros_t], dim=1) - alphas = torch.add(alphas, tail_threshold) - else: - tail_threshold = torch.tensor([tail_threshold], dtype=alphas.dtype).to( - alphas.device - ) - tail_threshold = torch.reshape(tail_threshold, (1, 1)) - alphas = torch.cat([alphas, tail_threshold], dim=1) - zeros = torch.zeros((b, 1, d), dtype=hidden.dtype).to(hidden.device) - hidden = torch.cat([hidden, zeros], dim=1) - token_num = alphas.sum(dim=-1) - token_num_floor = torch.floor(token_num) - - return hidden, alphas, token_num_floor - - def gen_frame_alignments( - self, alphas: torch.Tensor = None, encoder_sequence_length: torch.Tensor = None - ): - batch_size, maximum_length = alphas.size() - int_type = torch.int32 - - is_training = self.training - if is_training: - token_num = torch.round(torch.sum(alphas, dim=1)).type(int_type) - else: - token_num = torch.floor(torch.sum(alphas, dim=1)).type(int_type) - - max_token_num = torch.max(token_num).item() - - alphas_cumsum = torch.cumsum(alphas, dim=1) - alphas_cumsum = torch.floor(alphas_cumsum).type(int_type) - alphas_cumsum = alphas_cumsum[:, None, :].repeat(1, max_token_num, 1) - - index = torch.ones([batch_size, max_token_num], dtype=int_type) - index = torch.cumsum(index, dim=1) - index = index[:, :, None].repeat(1, 1, maximum_length).to(alphas_cumsum.device) - - index_div = torch.floor(torch.true_divide(alphas_cumsum, index)).type(int_type) - index_div_bool_zeros = index_div.eq(0) - index_div_bool_zeros_count = torch.sum(index_div_bool_zeros, dim=-1) + 1 - index_div_bool_zeros_count = torch.clamp( - index_div_bool_zeros_count, 0, encoder_sequence_length.max() - ) - token_num_mask = (~make_pad_mask(token_num, maxlen=max_token_num)).to( - token_num.device - ) - index_div_bool_zeros_count *= token_num_mask - - index_div_bool_zeros_count_tile = index_div_bool_zeros_count[:, :, None].repeat( - 1, 1, maximum_length - ) - ones = torch.ones_like(index_div_bool_zeros_count_tile) - zeros = torch.zeros_like(index_div_bool_zeros_count_tile) - ones = torch.cumsum(ones, dim=2) - cond = index_div_bool_zeros_count_tile == ones - index_div_bool_zeros_count_tile = torch.where(cond, zeros, ones) - - index_div_bool_zeros_count_tile_bool = index_div_bool_zeros_count_tile.type( - torch.bool - ) - index_div_bool_zeros_count_tile = 1 - index_div_bool_zeros_count_tile_bool.type( - int_type - ) - index_div_bool_zeros_count_tile_out = torch.sum( - index_div_bool_zeros_count_tile, dim=1 - ) - index_div_bool_zeros_count_tile_out = index_div_bool_zeros_count_tile_out.type( - int_type - ) - predictor_mask = ( - ( - ~make_pad_mask( - encoder_sequence_length, maxlen=encoder_sequence_length.max() - ) - ) - .type(int_type) - .to(encoder_sequence_length.device) - ) - index_div_bool_zeros_count_tile_out = ( - index_div_bool_zeros_count_tile_out * predictor_mask - ) - - predictor_alignments = index_div_bool_zeros_count_tile_out - predictor_alignments_length = predictor_alignments.sum(-1).type( - encoder_sequence_length.dtype - ) - return predictor_alignments.detach(), predictor_alignments_length.detach() - - -@tables.register("predictor_classes", "CifPredictorV2") -class CifPredictorV2(torch.nn.Module): - def __init__( - self, - idim, - l_order, - r_order, - threshold=1.0, - dropout=0.1, - smooth_factor=1.0, - noise_threshold=0, - tail_threshold=0.0, - tf2torch_tensor_name_prefix_torch="predictor", - tf2torch_tensor_name_prefix_tf="seq2seq/cif", - tail_mask=True, - ): - super(CifPredictorV2, self).__init__() - - self.pad = torch.nn.ConstantPad1d((l_order, r_order), 0) - self.cif_conv1d = torch.nn.Conv1d(idim, idim, l_order + r_order + 1) - self.cif_output = torch.nn.Linear(idim, 1) - self.dropout = torch.nn.Dropout(p=dropout) - self.threshold = threshold - self.smooth_factor = smooth_factor - self.noise_threshold = noise_threshold - self.tail_threshold = tail_threshold - self.tf2torch_tensor_name_prefix_torch = tf2torch_tensor_name_prefix_torch - self.tf2torch_tensor_name_prefix_tf = tf2torch_tensor_name_prefix_tf - self.tail_mask = tail_mask - - def forward( - self, - hidden, - target_label=None, - mask=None, - ignore_id=-1, - mask_chunk_predictor=None, - target_label_length=None, - ): - h = hidden - context = h.transpose(1, 2) - queries = self.pad(context) - output = torch.relu(self.cif_conv1d(queries)) - output = output.transpose(1, 2) - - output = self.cif_output(output) - alphas = torch.sigmoid(output) - alphas = torch.nn.functional.relu( - alphas * self.smooth_factor - self.noise_threshold - ) - if mask is not None: - mask = mask.transpose(-1, -2).float() - alphas = alphas * mask - if mask_chunk_predictor is not None: - alphas = alphas * mask_chunk_predictor - alphas = alphas.squeeze(-1) - mask = mask.squeeze(-1) - if target_label_length is not None: - target_length = target_label_length.squeeze(-1) - elif target_label is not None: - target_length = (target_label != ignore_id).float().sum(-1) - else: - target_length = None - token_num = alphas.sum(-1) - if target_length is not None: - alphas *= (target_length / token_num)[:, None].repeat(1, alphas.size(1)) - elif self.tail_threshold > 0.0: - if self.tail_mask: - hidden, alphas, token_num = self.tail_process_fn( - hidden, alphas, token_num, mask=mask - ) - else: - hidden, alphas, token_num = self.tail_process_fn( - hidden, alphas, token_num, mask=None - ) - - acoustic_embeds, cif_peak = cif(hidden, alphas, self.threshold) - if target_length is None and self.tail_threshold > 0.0: - token_num_int = torch.max(token_num).type(torch.int32).item() - acoustic_embeds = acoustic_embeds[:, :token_num_int, :] - - return acoustic_embeds, token_num, alphas, cif_peak - - def forward_chunk(self, hidden, cache=None, **kwargs): - is_final = kwargs.get("is_final", False) - batch_size, len_time, hidden_size = hidden.shape - h = hidden - context = h.transpose(1, 2) - queries = self.pad(context) - output = torch.relu(self.cif_conv1d(queries)) - output = output.transpose(1, 2) - output = self.cif_output(output) - alphas = torch.sigmoid(output) - alphas = torch.nn.functional.relu( - alphas * self.smooth_factor - self.noise_threshold - ) - - alphas = alphas.squeeze(-1) - - token_length = [] - list_fires = [] - list_frames = [] - cache_alphas = [] - cache_hiddens = [] - - if cache is not None and "chunk_size" in cache: - alphas[:, : cache["chunk_size"][0]] = 0.0 - if not is_final: - alphas[:, sum(cache["chunk_size"][:2]) :] = 0.0 - if cache is not None and "cif_alphas" in cache and "cif_hidden" in cache: - cache["cif_hidden"] = to_device(cache["cif_hidden"], device=hidden.device) - cache["cif_alphas"] = to_device(cache["cif_alphas"], device=alphas.device) - hidden = torch.cat((cache["cif_hidden"], hidden), dim=1) - alphas = torch.cat((cache["cif_alphas"], alphas), dim=1) - if cache is not None and is_final: - tail_hidden = torch.zeros( - (batch_size, 1, hidden_size), device=hidden.device - ) - tail_alphas = torch.tensor([[self.tail_threshold]], device=alphas.device) - tail_alphas = torch.tile(tail_alphas, (batch_size, 1)) - hidden = torch.cat((hidden, tail_hidden), dim=1) - alphas = torch.cat((alphas, tail_alphas), dim=1) - - len_time = alphas.shape[1] - for b in range(batch_size): - integrate = 0.0 - frames = torch.zeros((hidden_size), device=hidden.device) - list_frame = [] - list_fire = [] - for t in range(len_time): - alpha = alphas[b][t] - if alpha + integrate < self.threshold: - integrate += alpha - list_fire.append(integrate) - frames += alpha * hidden[b][t] - else: - frames += (self.threshold - integrate) * hidden[b][t] - list_frame.append(frames) - integrate += alpha - list_fire.append(integrate) - integrate -= self.threshold - frames = integrate * hidden[b][t] - - cache_alphas.append(integrate) - if integrate > 0.0: - cache_hiddens.append(frames / integrate) - else: - cache_hiddens.append(frames) - - token_length.append(torch.tensor(len(list_frame), device=alphas.device)) - list_fires.append(list_fire) - list_frames.append(list_frame) - - cache["cif_alphas"] = torch.stack(cache_alphas, axis=0) - cache["cif_alphas"] = torch.unsqueeze(cache["cif_alphas"], axis=0) - cache["cif_hidden"] = torch.stack(cache_hiddens, axis=0) - cache["cif_hidden"] = torch.unsqueeze(cache["cif_hidden"], axis=0) - - max_token_len = max(token_length) - if max_token_len == 0: - return hidden, torch.stack(token_length, 0), None, None - list_ls = [] - for b in range(batch_size): - pad_frames = torch.zeros( - (max_token_len - token_length[b], hidden_size), device=alphas.device - ) - if token_length[b] == 0: - list_ls.append(pad_frames) - else: - list_frames[b] = torch.stack(list_frames[b]) - list_ls.append(torch.cat((list_frames[b], pad_frames), dim=0)) - - cache["cif_alphas"] = torch.stack(cache_alphas, axis=0) - cache["cif_alphas"] = torch.unsqueeze(cache["cif_alphas"], axis=0) - cache["cif_hidden"] = torch.stack(cache_hiddens, axis=0) - cache["cif_hidden"] = torch.unsqueeze(cache["cif_hidden"], axis=0) - return torch.stack(list_ls, 0), torch.stack(token_length, 0), None, None - - def tail_process_fn(self, hidden, alphas, token_num=None, mask=None): - b, t, d = hidden.size() - tail_threshold = self.tail_threshold - if mask is not None: - zeros_t = torch.zeros((b, 1), dtype=torch.float32, device=alphas.device) - ones_t = torch.ones_like(zeros_t) - mask_1 = torch.cat([mask, zeros_t], dim=1) - mask_2 = torch.cat([ones_t, mask], dim=1) - mask = mask_2 - mask_1 - tail_threshold = mask * tail_threshold - alphas = torch.cat([alphas, zeros_t], dim=1) - alphas = torch.add(alphas, tail_threshold) - else: - tail_threshold = torch.tensor([tail_threshold], dtype=alphas.dtype).to( - alphas.device - ) - tail_threshold = torch.reshape(tail_threshold, (1, 1)) - if b > 1: - alphas = torch.cat([alphas, tail_threshold.repeat(b, 1)], dim=1) - else: - alphas = torch.cat([alphas, tail_threshold], dim=1) - zeros = torch.zeros((b, 1, d), dtype=hidden.dtype).to(hidden.device) - hidden = torch.cat([hidden, zeros], dim=1) - token_num = alphas.sum(dim=-1) - token_num_floor = torch.floor(token_num) - - return hidden, alphas, token_num_floor - - def gen_frame_alignments( - self, alphas: torch.Tensor = None, encoder_sequence_length: torch.Tensor = None - ): - batch_size, maximum_length = alphas.size() - int_type = torch.int32 - - is_training = self.training - if is_training: - token_num = torch.round(torch.sum(alphas, dim=1)).type(int_type) - else: - token_num = torch.floor(torch.sum(alphas, dim=1)).type(int_type) - - max_token_num = torch.max(token_num).item() - - alphas_cumsum = torch.cumsum(alphas, dim=1) - alphas_cumsum = torch.floor(alphas_cumsum).type(int_type) - alphas_cumsum = alphas_cumsum[:, None, :].repeat(1, max_token_num, 1) - - index = torch.ones([batch_size, max_token_num], dtype=int_type) - index = torch.cumsum(index, dim=1) - index = index[:, :, None].repeat(1, 1, maximum_length).to(alphas_cumsum.device) - - index_div = torch.floor(torch.true_divide(alphas_cumsum, index)).type(int_type) - index_div_bool_zeros = index_div.eq(0) - index_div_bool_zeros_count = torch.sum(index_div_bool_zeros, dim=-1) + 1 - index_div_bool_zeros_count = torch.clamp( - index_div_bool_zeros_count, 0, encoder_sequence_length.max() - ) - token_num_mask = (~make_pad_mask(token_num, maxlen=max_token_num)).to( - token_num.device - ) - index_div_bool_zeros_count *= token_num_mask - - index_div_bool_zeros_count_tile = index_div_bool_zeros_count[:, :, None].repeat( - 1, 1, maximum_length - ) - ones = torch.ones_like(index_div_bool_zeros_count_tile) - zeros = torch.zeros_like(index_div_bool_zeros_count_tile) - ones = torch.cumsum(ones, dim=2) - cond = index_div_bool_zeros_count_tile == ones - index_div_bool_zeros_count_tile = torch.where(cond, zeros, ones) - - index_div_bool_zeros_count_tile_bool = index_div_bool_zeros_count_tile.type( - torch.bool - ) - index_div_bool_zeros_count_tile = 1 - index_div_bool_zeros_count_tile_bool.type( - int_type - ) - index_div_bool_zeros_count_tile_out = torch.sum( - index_div_bool_zeros_count_tile, dim=1 - ) - index_div_bool_zeros_count_tile_out = index_div_bool_zeros_count_tile_out.type( - int_type - ) - predictor_mask = ( - ( - ~make_pad_mask( - encoder_sequence_length, maxlen=encoder_sequence_length.max() - ) - ) - .type(int_type) - .to(encoder_sequence_length.device) - ) - index_div_bool_zeros_count_tile_out = ( - index_div_bool_zeros_count_tile_out * predictor_mask - ) - - predictor_alignments = index_div_bool_zeros_count_tile_out - predictor_alignments_length = predictor_alignments.sum(-1).type( - encoder_sequence_length.dtype - ) - return predictor_alignments.detach(), predictor_alignments_length.detach() - - def gen_tf2torch_map_dict(self): - - tensor_name_prefix_torch = self.tf2torch_tensor_name_prefix_torch - tensor_name_prefix_tf = self.tf2torch_tensor_name_prefix_tf - map_dict_local = { - ## predictor - "{}.cif_conv1d.weight".format(tensor_name_prefix_torch): { - "name": "{}/conv1d/kernel".format(tensor_name_prefix_tf), - "squeeze": None, - "transpose": (2, 1, 0), - }, # (256,256,3),(3,256,256) - "{}.cif_conv1d.bias".format(tensor_name_prefix_torch): { - "name": "{}/conv1d/bias".format(tensor_name_prefix_tf), - "squeeze": None, - "transpose": None, - }, # (256,),(256,) - "{}.cif_output.weight".format(tensor_name_prefix_torch): { - "name": "{}/conv1d_1/kernel".format(tensor_name_prefix_tf), - "squeeze": 0, - "transpose": (1, 0), - }, # (1,256),(1,256,1) - "{}.cif_output.bias".format(tensor_name_prefix_torch): { - "name": "{}/conv1d_1/bias".format(tensor_name_prefix_tf), - "squeeze": None, - "transpose": None, - }, # (1,),(1,) - } - return map_dict_local - - def convert_tf2torch( - self, - var_dict_tf, - var_dict_torch, - ): - map_dict = self.gen_tf2torch_map_dict() - var_dict_torch_update = dict() - for name in sorted(var_dict_torch.keys(), reverse=False): - names = name.split(".") - if names[0] == self.tf2torch_tensor_name_prefix_torch: - name_tf = map_dict[name]["name"] - data_tf = var_dict_tf[name_tf] - if map_dict[name]["squeeze"] is not None: - data_tf = np.squeeze(data_tf, axis=map_dict[name]["squeeze"]) - if map_dict[name]["transpose"] is not None: - data_tf = np.transpose(data_tf, map_dict[name]["transpose"]) - data_tf = torch.from_numpy(data_tf).type(torch.float32).to("cpu") - assert ( - var_dict_torch[name].size() == data_tf.size() - ), "{}, {}, {} != {}".format( - name, name_tf, var_dict_torch[name].size(), data_tf.size() - ) - var_dict_torch_update[name] = data_tf - logging.info( - "torch tensor: {}, {}, loading from tf tensor: {}, {}".format( - name, data_tf.size(), name_tf, var_dict_tf[name_tf].shape - ) - ) - - return var_dict_torch_update - - -class mae_loss(torch.nn.Module): - - def __init__(self, normalize_length=False): - super(mae_loss, self).__init__() - self.normalize_length = normalize_length - self.criterion = torch.nn.L1Loss(reduction="sum") - - def forward(self, token_length, pre_token_length): - loss_token_normalizer = token_length.size(0) - if self.normalize_length: - loss_token_normalizer = token_length.sum().type(torch.float32) - loss = self.criterion(token_length, pre_token_length) - loss = loss / loss_token_normalizer - return loss - - -def cif(hidden, alphas, threshold): - batch_size, len_time, hidden_size = hidden.size() - - # loop varss - integrate = torch.zeros([batch_size], device=hidden.device) - frame = torch.zeros([batch_size, hidden_size], device=hidden.device) - # intermediate vars along time - list_fires = [] - list_frames = [] - - for t in range(len_time): - alpha = alphas[:, t] - distribution_completion = ( - torch.ones([batch_size], device=hidden.device) - integrate - ) - - integrate += alpha - list_fires.append(integrate) - - fire_place = integrate >= threshold - integrate = torch.where( - fire_place, - integrate - torch.ones([batch_size], device=hidden.device), - integrate, - ) - cur = torch.where(fire_place, distribution_completion, alpha) - remainds = alpha - cur - - frame += cur[:, None] * hidden[:, t, :] - list_frames.append(frame) - frame = torch.where( - fire_place[:, None].repeat(1, hidden_size), - remainds[:, None] * hidden[:, t, :], - frame, - ) - - fires = torch.stack(list_fires, 1) - frames = torch.stack(list_frames, 1) - list_ls = [] - len_labels = torch.round(alphas.sum(-1)).int() - max_label_len = len_labels.max() - for b in range(batch_size): - fire = fires[b, :] - l = torch.index_select( - frames[b, :, :], 0, torch.nonzero(fire >= threshold).squeeze() - ) - pad_l = torch.zeros( - [max_label_len - l.size(0), hidden_size], device=hidden.device - ) - list_ls.append(torch.cat([l, pad_l], 0)) - return torch.stack(list_ls, 0), fires - - -def cif_wo_hidden(alphas, threshold): - batch_size, len_time = alphas.size() - - # loop varss - integrate = torch.zeros([batch_size], device=alphas.device) - # intermediate vars along time - list_fires = [] - - for t in range(len_time): - alpha = alphas[:, t] - - integrate += alpha - list_fires.append(integrate) - - fire_place = integrate >= threshold - integrate = torch.where( - fire_place, - integrate - torch.ones([batch_size], device=alphas.device) * threshold, - integrate, - ) - - fires = torch.stack(list_fires, 1) - return fires diff --git a/funasr_detach/models/paraformer/decoder.py b/funasr_detach/models/paraformer/decoder.py deleted file mode 100644 index a3d6dd784cf05b9da9ae029a60e96fa631feb867..0000000000000000000000000000000000000000 --- a/funasr_detach/models/paraformer/decoder.py +++ /dev/null @@ -1,680 +0,0 @@ -#!/usr/bin/env python3 -# -*- encoding: utf-8 -*- -# Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved. -# MIT License (https://opensource.org/licenses/MIT) - -import torch -from typing import List, Tuple - -from funasr_detach.register import tables -from funasr_detach.models.scama import utils as myutils -from funasr_detach.models.transformer.utils.repeat import repeat -from funasr_detach.models.transformer.decoder import DecoderLayer -from funasr_detach.models.transformer.layer_norm import LayerNorm -from funasr_detach.models.transformer.embedding import PositionalEncoding -from funasr_detach.models.transformer.attention import MultiHeadedAttention -from funasr_detach.models.transformer.utils.nets_utils import make_pad_mask -from funasr_detach.models.transformer.decoder import BaseTransformerDecoder -from funasr_detach.models.transformer.positionwise_feed_forward import ( - PositionwiseFeedForward, -) -from funasr_detach.models.sanm.positionwise_feed_forward import ( - PositionwiseFeedForwardDecoderSANM, -) -from funasr_detach.models.sanm.attention import ( - MultiHeadedAttentionSANMDecoder, - MultiHeadedAttentionCrossAtt, -) - - -class DecoderLayerSANM(torch.nn.Module): - """Single decoder layer module. - - Args: - size (int): Input dimension. - self_attn (torch.nn.Module): Self-attention module instance. - `MultiHeadedAttention` instance can be used as the argument. - src_attn (torch.nn.Module): Self-attention module instance. - `MultiHeadedAttention` instance can be used as the argument. - feed_forward (torch.nn.Module): Feed-forward module instance. - `PositionwiseFeedForward`, `MultiLayeredConv1d`, or `Conv1dLinear` instance - can be used as the argument. - dropout_rate (float): Dropout rate. - normalize_before (bool): Whether to use layer_norm before the first block. - concat_after (bool): Whether to concat attention layer's input and output. - if True, additional linear will be applied. - i.e. x -> x + linear(concat(x, att(x))) - if False, no additional linear will be applied. i.e. x -> x + att(x) - - - """ - - def __init__( - self, - size, - self_attn, - src_attn, - feed_forward, - dropout_rate, - normalize_before=True, - concat_after=False, - ): - """Construct an DecoderLayer object.""" - super(DecoderLayerSANM, self).__init__() - self.size = size - self.self_attn = self_attn - self.src_attn = src_attn - self.feed_forward = feed_forward - self.norm1 = LayerNorm(size) - if self_attn is not None: - self.norm2 = LayerNorm(size) - if src_attn is not None: - self.norm3 = LayerNorm(size) - self.dropout = torch.nn.Dropout(dropout_rate) - self.normalize_before = normalize_before - self.concat_after = concat_after - if self.concat_after: - self.concat_linear1 = torch.nn.Linear(size + size, size) - self.concat_linear2 = torch.nn.Linear(size + size, size) - self.reserve_attn = False - self.attn_mat = [] - - def forward(self, tgt, tgt_mask, memory, memory_mask=None, cache=None): - """Compute decoded features. - - Args: - tgt (torch.Tensor): Input tensor (#batch, maxlen_out, size). - tgt_mask (torch.Tensor): Mask for input tensor (#batch, maxlen_out). - memory (torch.Tensor): Encoded memory, float32 (#batch, maxlen_in, size). - memory_mask (torch.Tensor): Encoded memory mask (#batch, maxlen_in). - cache (List[torch.Tensor]): List of cached tensors. - Each tensor shape should be (#batch, maxlen_out - 1, size). - - Returns: - torch.Tensor: Output tensor(#batch, maxlen_out, size). - torch.Tensor: Mask for output tensor (#batch, maxlen_out). - torch.Tensor: Encoded memory (#batch, maxlen_in, size). - torch.Tensor: Encoded memory mask (#batch, maxlen_in). - - """ - # tgt = self.dropout(tgt) - residual = tgt - if self.normalize_before: - tgt = self.norm1(tgt) - tgt = self.feed_forward(tgt) - - x = tgt - if self.self_attn: - if self.normalize_before: - tgt = self.norm2(tgt) - x, _ = self.self_attn(tgt, tgt_mask) - x = residual + self.dropout(x) - - if self.src_attn is not None: - residual = x - if self.normalize_before: - x = self.norm3(x) - if self.reserve_attn: - x_src_attn, attn_mat = self.src_attn( - x, memory, memory_mask, ret_attn=True - ) - self.attn_mat.append(attn_mat) - else: - x_src_attn = self.src_attn(x, memory, memory_mask, ret_attn=False) - x = residual + self.dropout(x_src_attn) - # x = residual + self.dropout(self.src_attn(x, memory, memory_mask)) - - return x, tgt_mask, memory, memory_mask, cache - - def forward_one_step(self, tgt, tgt_mask, memory, memory_mask=None, cache=None): - """Compute decoded features. - - Args: - tgt (torch.Tensor): Input tensor (#batch, maxlen_out, size). - tgt_mask (torch.Tensor): Mask for input tensor (#batch, maxlen_out). - memory (torch.Tensor): Encoded memory, float32 (#batch, maxlen_in, size). - memory_mask (torch.Tensor): Encoded memory mask (#batch, maxlen_in). - cache (List[torch.Tensor]): List of cached tensors. - Each tensor shape should be (#batch, maxlen_out - 1, size). - - Returns: - torch.Tensor: Output tensor(#batch, maxlen_out, size). - torch.Tensor: Mask for output tensor (#batch, maxlen_out). - torch.Tensor: Encoded memory (#batch, maxlen_in, size). - torch.Tensor: Encoded memory mask (#batch, maxlen_in). - - """ - # tgt = self.dropout(tgt) - residual = tgt - if self.normalize_before: - tgt = self.norm1(tgt) - tgt = self.feed_forward(tgt) - - x = tgt - if self.self_attn: - if self.normalize_before: - tgt = self.norm2(tgt) - if self.training: - cache = None - x, cache = self.self_attn(tgt, tgt_mask, cache=cache) - x = residual + self.dropout(x) - - if self.src_attn is not None: - residual = x - if self.normalize_before: - x = self.norm3(x) - - x = residual + self.dropout(self.src_attn(x, memory, memory_mask)) - - return x, tgt_mask, memory, memory_mask, cache - - def forward_chunk( - self, tgt, memory, fsmn_cache=None, opt_cache=None, chunk_size=None, look_back=0 - ): - """Compute decoded features. - - Args: - tgt (torch.Tensor): Input tensor (#batch, maxlen_out, size). - tgt_mask (torch.Tensor): Mask for input tensor (#batch, maxlen_out). - memory (torch.Tensor): Encoded memory, float32 (#batch, maxlen_in, size). - memory_mask (torch.Tensor): Encoded memory mask (#batch, maxlen_in). - cache (List[torch.Tensor]): List of cached tensors. - Each tensor shape should be (#batch, maxlen_out - 1, size). - - Returns: - torch.Tensor: Output tensor(#batch, maxlen_out, size). - torch.Tensor: Mask for output tensor (#batch, maxlen_out). - torch.Tensor: Encoded memory (#batch, maxlen_in, size). - torch.Tensor: Encoded memory mask (#batch, maxlen_in). - - """ - residual = tgt - if self.normalize_before: - tgt = self.norm1(tgt) - tgt = self.feed_forward(tgt) - - x = tgt - if self.self_attn: - if self.normalize_before: - tgt = self.norm2(tgt) - x, fsmn_cache = self.self_attn(tgt, None, fsmn_cache) - x = residual + self.dropout(x) - - if self.src_attn is not None: - residual = x - if self.normalize_before: - x = self.norm3(x) - - x, opt_cache = self.src_attn.forward_chunk( - x, memory, opt_cache, chunk_size, look_back - ) - x = residual + x - - return x, memory, fsmn_cache, opt_cache - - -@tables.register("decoder_classes", "ParaformerSANMDecoder") -class ParaformerSANMDecoder(BaseTransformerDecoder): - """ - Author: Speech Lab of DAMO Academy, Alibaba Group - Paraformer: Fast and Accurate Parallel Transformer for Non-autoregressive End-to-End Speech Recognition - https://arxiv.org/abs/2006.01713 - """ - - def __init__( - self, - vocab_size: int, - encoder_output_size: int, - attention_heads: int = 4, - linear_units: int = 2048, - num_blocks: int = 6, - dropout_rate: float = 0.1, - positional_dropout_rate: float = 0.1, - self_attention_dropout_rate: float = 0.0, - src_attention_dropout_rate: float = 0.0, - input_layer: str = "embed", - use_output_layer: bool = True, - wo_input_layer: bool = False, - pos_enc_class=PositionalEncoding, - normalize_before: bool = True, - concat_after: bool = False, - att_layer_num: int = 6, - kernel_size: int = 21, - sanm_shfit: int = 0, - lora_list: List[str] = None, - lora_rank: int = 8, - lora_alpha: int = 16, - lora_dropout: float = 0.1, - chunk_multiply_factor: tuple = (1,), - tf2torch_tensor_name_prefix_torch: str = "decoder", - tf2torch_tensor_name_prefix_tf: str = "seq2seq/decoder", - ): - super().__init__( - vocab_size=vocab_size, - encoder_output_size=encoder_output_size, - dropout_rate=dropout_rate, - positional_dropout_rate=positional_dropout_rate, - input_layer=input_layer, - use_output_layer=use_output_layer, - pos_enc_class=pos_enc_class, - normalize_before=normalize_before, - ) - - attention_dim = encoder_output_size - if wo_input_layer: - self.embed = None - else: - if input_layer == "embed": - self.embed = torch.nn.Sequential( - torch.nn.Embedding(vocab_size, attention_dim), - # pos_enc_class(attention_dim, positional_dropout_rate), - ) - elif input_layer == "linear": - self.embed = torch.nn.Sequential( - torch.nn.Linear(vocab_size, attention_dim), - torch.nn.LayerNorm(attention_dim), - torch.nn.Dropout(dropout_rate), - torch.nn.ReLU(), - pos_enc_class(attention_dim, positional_dropout_rate), - ) - else: - raise ValueError( - f"only 'embed' or 'linear' is supported: {input_layer}" - ) - - self.normalize_before = normalize_before - if self.normalize_before: - self.after_norm = LayerNorm(attention_dim) - if use_output_layer: - self.output_layer = torch.nn.Linear(attention_dim, vocab_size) - else: - self.output_layer = None - - self.att_layer_num = att_layer_num - self.num_blocks = num_blocks - if sanm_shfit is None: - sanm_shfit = (kernel_size - 1) // 2 - self.decoders = repeat( - att_layer_num, - lambda lnum: DecoderLayerSANM( - attention_dim, - MultiHeadedAttentionSANMDecoder( - attention_dim, - self_attention_dropout_rate, - kernel_size, - sanm_shfit=sanm_shfit, - ), - MultiHeadedAttentionCrossAtt( - attention_heads, - attention_dim, - src_attention_dropout_rate, - lora_list, - lora_rank, - lora_alpha, - lora_dropout, - ), - PositionwiseFeedForwardDecoderSANM( - attention_dim, linear_units, dropout_rate - ), - dropout_rate, - normalize_before, - concat_after, - ), - ) - if num_blocks - att_layer_num <= 0: - self.decoders2 = None - else: - self.decoders2 = repeat( - num_blocks - att_layer_num, - lambda lnum: DecoderLayerSANM( - attention_dim, - MultiHeadedAttentionSANMDecoder( - attention_dim, - self_attention_dropout_rate, - kernel_size, - sanm_shfit=0, - ), - None, - PositionwiseFeedForwardDecoderSANM( - attention_dim, linear_units, dropout_rate - ), - dropout_rate, - normalize_before, - concat_after, - ), - ) - - self.decoders3 = repeat( - 1, - lambda lnum: DecoderLayerSANM( - attention_dim, - None, - None, - PositionwiseFeedForwardDecoderSANM( - attention_dim, linear_units, dropout_rate - ), - dropout_rate, - normalize_before, - concat_after, - ), - ) - self.tf2torch_tensor_name_prefix_torch = tf2torch_tensor_name_prefix_torch - self.tf2torch_tensor_name_prefix_tf = tf2torch_tensor_name_prefix_tf - self.chunk_multiply_factor = chunk_multiply_factor - - def forward( - self, - hs_pad: torch.Tensor, - hlens: torch.Tensor, - ys_in_pad: torch.Tensor, - ys_in_lens: torch.Tensor, - return_hidden: bool = False, - return_both: bool = False, - chunk_mask: torch.Tensor = None, - ) -> Tuple[torch.Tensor, torch.Tensor]: - """Forward decoder. - - Args: - hs_pad: encoded memory, float32 (batch, maxlen_in, feat) - hlens: (batch) - ys_in_pad: - input token ids, int64 (batch, maxlen_out) - if input_layer == "embed" - input tensor (batch, maxlen_out, #mels) in the other cases - ys_in_lens: (batch) - Returns: - (tuple): tuple containing: - - x: decoded token score before softmax (batch, maxlen_out, token) - if use_output_layer is True, - olens: (batch, ) - """ - tgt = ys_in_pad - tgt_mask = myutils.sequence_mask(ys_in_lens, device=tgt.device)[:, :, None] - - memory = hs_pad - memory_mask = myutils.sequence_mask(hlens, device=memory.device)[:, None, :] - if chunk_mask is not None: - memory_mask = memory_mask * chunk_mask - if tgt_mask.size(1) != memory_mask.size(1): - memory_mask = torch.cat((memory_mask, memory_mask[:, -2:-1, :]), dim=1) - - x = tgt - x, tgt_mask, memory, memory_mask, _ = self.decoders( - x, tgt_mask, memory, memory_mask - ) - if self.decoders2 is not None: - x, tgt_mask, memory, memory_mask, _ = self.decoders2( - x, tgt_mask, memory, memory_mask - ) - x, tgt_mask, memory, memory_mask, _ = self.decoders3( - x, tgt_mask, memory, memory_mask - ) - if self.normalize_before: - hidden = self.after_norm(x) - - olens = tgt_mask.sum(1) - if self.output_layer is not None and return_hidden is False: - x = self.output_layer(hidden) - return x, olens - if return_both: - x = self.output_layer(hidden) - return x, hidden, olens - return hidden, olens - - def score(self, ys, state, x): - """Score.""" - ys_mask = myutils.sequence_mask( - torch.tensor([len(ys)], dtype=torch.int32), device=x.device - )[:, :, None] - logp, state = self.forward_one_step( - ys.unsqueeze(0), ys_mask, x.unsqueeze(0), cache=state - ) - return logp.squeeze(0), state - - def forward_chunk( - self, - memory: torch.Tensor, - tgt: torch.Tensor, - cache: dict = None, - ) -> Tuple[torch.Tensor, torch.Tensor]: - """Forward decoder. - - Args: - hs_pad: encoded memory, float32 (batch, maxlen_in, feat) - hlens: (batch) - ys_in_pad: - input token ids, int64 (batch, maxlen_out) - if input_layer == "embed" - input tensor (batch, maxlen_out, #mels) in the other cases - ys_in_lens: (batch) - Returns: - (tuple): tuple containing: - - x: decoded token score before softmax (batch, maxlen_out, token) - if use_output_layer is True, - olens: (batch, ) - """ - x = tgt - if cache["decode_fsmn"] is None: - cache_layer_num = len(self.decoders) - if self.decoders2 is not None: - cache_layer_num += len(self.decoders2) - fsmn_cache = [None] * cache_layer_num - else: - fsmn_cache = cache["decode_fsmn"] - - if cache["opt"] is None: - cache_layer_num = len(self.decoders) - opt_cache = [None] * cache_layer_num - else: - opt_cache = cache["opt"] - - for i in range(self.att_layer_num): - decoder = self.decoders[i] - x, memory, fsmn_cache[i], opt_cache[i] = decoder.forward_chunk( - x, - memory, - fsmn_cache=fsmn_cache[i], - opt_cache=opt_cache[i], - chunk_size=cache["chunk_size"], - look_back=cache["decoder_chunk_look_back"], - ) - - if self.num_blocks - self.att_layer_num > 1: - for i in range(self.num_blocks - self.att_layer_num): - j = i + self.att_layer_num - decoder = self.decoders2[i] - x, memory, fsmn_cache[j], _ = decoder.forward_chunk( - x, memory, fsmn_cache=fsmn_cache[j] - ) - - for decoder in self.decoders3: - x, memory, _, _ = decoder.forward_chunk(x, memory) - if self.normalize_before: - x = self.after_norm(x) - if self.output_layer is not None: - x = self.output_layer(x) - - cache["decode_fsmn"] = fsmn_cache - if ( - cache["decoder_chunk_look_back"] > 0 - or cache["decoder_chunk_look_back"] == -1 - ): - cache["opt"] = opt_cache - return x - - def forward_one_step( - self, - tgt: torch.Tensor, - tgt_mask: torch.Tensor, - memory: torch.Tensor, - cache: List[torch.Tensor] = None, - ) -> Tuple[torch.Tensor, List[torch.Tensor]]: - """Forward one step. - - Args: - tgt: input token ids, int64 (batch, maxlen_out) - tgt_mask: input token mask, (batch, maxlen_out) - dtype=torch.uint8 in PyTorch 1.2- - dtype=torch.bool in PyTorch 1.2+ (include 1.2) - memory: encoded memory, float32 (batch, maxlen_in, feat) - cache: cached output list of (batch, max_time_out-1, size) - Returns: - y, cache: NN output value and cache per `self.decoders`. - y.shape` is (batch, maxlen_out, token) - """ - x = self.embed(tgt) - if cache is None: - cache_layer_num = len(self.decoders) - if self.decoders2 is not None: - cache_layer_num += len(self.decoders2) - cache = [None] * cache_layer_num - new_cache = [] - # for c, decoder in zip(cache, self.decoders): - for i in range(self.att_layer_num): - decoder = self.decoders[i] - c = cache[i] - x, tgt_mask, memory, memory_mask, c_ret = decoder.forward_one_step( - x, tgt_mask, memory, None, cache=c - ) - new_cache.append(c_ret) - - if self.num_blocks - self.att_layer_num > 1: - for i in range(self.num_blocks - self.att_layer_num): - j = i + self.att_layer_num - decoder = self.decoders2[i] - c = cache[j] - x, tgt_mask, memory, memory_mask, c_ret = decoder.forward_one_step( - x, tgt_mask, memory, None, cache=c - ) - new_cache.append(c_ret) - - for decoder in self.decoders3: - - x, tgt_mask, memory, memory_mask, _ = decoder.forward_one_step( - x, tgt_mask, memory, None, cache=None - ) - - if self.normalize_before: - y = self.after_norm(x[:, -1]) - else: - y = x[:, -1] - if self.output_layer is not None: - y = torch.log_softmax(self.output_layer(y), dim=-1) - - return y, new_cache - - -@tables.register("decoder_classes", "ParaformerSANDecoder") -class ParaformerSANDecoder(BaseTransformerDecoder): - """ - Author: Speech Lab of DAMO Academy, Alibaba Group - Paraformer: Fast and Accurate Parallel Transformer for Non-autoregressive End-to-End Speech Recognition - https://arxiv.org/abs/2006.01713 - """ - - def __init__( - self, - vocab_size: int, - encoder_output_size: int, - attention_heads: int = 4, - linear_units: int = 2048, - num_blocks: int = 6, - dropout_rate: float = 0.1, - positional_dropout_rate: float = 0.1, - self_attention_dropout_rate: float = 0.0, - src_attention_dropout_rate: float = 0.0, - input_layer: str = "embed", - use_output_layer: bool = True, - pos_enc_class=PositionalEncoding, - normalize_before: bool = True, - concat_after: bool = False, - embeds_id: int = -1, - ): - super().__init__( - vocab_size=vocab_size, - encoder_output_size=encoder_output_size, - dropout_rate=dropout_rate, - positional_dropout_rate=positional_dropout_rate, - input_layer=input_layer, - use_output_layer=use_output_layer, - pos_enc_class=pos_enc_class, - normalize_before=normalize_before, - ) - - attention_dim = encoder_output_size - self.decoders = repeat( - num_blocks, - lambda lnum: DecoderLayer( - attention_dim, - MultiHeadedAttention( - attention_heads, attention_dim, self_attention_dropout_rate - ), - MultiHeadedAttention( - attention_heads, attention_dim, src_attention_dropout_rate - ), - PositionwiseFeedForward(attention_dim, linear_units, dropout_rate), - dropout_rate, - normalize_before, - concat_after, - ), - ) - self.embeds_id = embeds_id - self.attention_dim = attention_dim - - def forward( - self, - hs_pad: torch.Tensor, - hlens: torch.Tensor, - ys_in_pad: torch.Tensor, - ys_in_lens: torch.Tensor, - ) -> Tuple[torch.Tensor, torch.Tensor]: - """Forward decoder. - - Args: - hs_pad: encoded memory, float32 (batch, maxlen_in, feat) - hlens: (batch) - ys_in_pad: - input token ids, int64 (batch, maxlen_out) - if input_layer == "embed" - input tensor (batch, maxlen_out, #mels) in the other cases - ys_in_lens: (batch) - Returns: - (tuple): tuple containing: - - x: decoded token score before softmax (batch, maxlen_out, token) - if use_output_layer is True, - olens: (batch, ) - """ - tgt = ys_in_pad - tgt_mask = (~make_pad_mask(ys_in_lens)[:, None, :]).to(tgt.device) - - memory = hs_pad - memory_mask = (~make_pad_mask(hlens, maxlen=memory.size(1)))[:, None, :].to( - memory.device - ) - # Padding for Longformer - if memory_mask.shape[-1] != memory.shape[1]: - padlen = memory.shape[1] - memory_mask.shape[-1] - memory_mask = torch.nn.functional.pad( - memory_mask, (0, padlen), "constant", False - ) - - # x = self.embed(tgt) - x = tgt - embeds_outputs = None - for layer_id, decoder in enumerate(self.decoders): - x, tgt_mask, memory, memory_mask = decoder(x, tgt_mask, memory, memory_mask) - if layer_id == self.embeds_id: - embeds_outputs = x - if self.normalize_before: - x = self.after_norm(x) - if self.output_layer is not None: - x = self.output_layer(x) - - olens = tgt_mask.sum(1) - if embeds_outputs is not None: - return x, olens, embeds_outputs - else: - return x, olens diff --git a/funasr_detach/models/paraformer/model.py b/funasr_detach/models/paraformer/model.py deleted file mode 100644 index fa7da5f6495f5285aec839cb395b1c70bdd13e3f..0000000000000000000000000000000000000000 --- a/funasr_detach/models/paraformer/model.py +++ /dev/null @@ -1,611 +0,0 @@ -#!/usr/bin/env python3 -# -*- encoding: utf-8 -*- -# Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved. -# MIT License (https://opensource.org/licenses/MIT) - -import time -import torch -import logging -from torch.cuda.amp import autocast -from typing import Union, Dict, List, Tuple, Optional - -from funasr_detach.register import tables -from funasr_detach.models.ctc.ctc import CTC -from funasr_detach.utils import postprocess_utils -from funasr_detach.metrics.compute_acc import th_accuracy -from funasr_detach.utils.datadir_writer import DatadirWriter -from funasr_detach.models.paraformer.search import Hypothesis -from funasr_detach.models.paraformer.cif_predictor import mae_loss -from funasr_detach.train_utils.device_funcs import force_gatherable -from funasr_detach.losses.label_smoothing_loss import LabelSmoothingLoss -from funasr_detach.models.transformer.utils.add_sos_eos import add_sos_eos -from funasr_detach.models.transformer.utils.nets_utils import make_pad_mask, pad_list -from funasr_detach.utils.load_utils import load_audio_text_image_video, extract_fbank - - -@tables.register("model_classes", "Paraformer") -class Paraformer(torch.nn.Module): - """ - Author: Speech Lab of DAMO Academy, Alibaba Group - Paraformer: Fast and Accurate Parallel Transformer for Non-autoregressive End-to-End Speech Recognition - https://arxiv.org/abs/2206.08317 - """ - - def __init__( - self, - specaug: Optional[str] = None, - specaug_conf: Optional[Dict] = None, - normalize: str = None, - normalize_conf: Optional[Dict] = None, - encoder: str = None, - encoder_conf: Optional[Dict] = None, - decoder: str = None, - decoder_conf: Optional[Dict] = None, - ctc: str = None, - ctc_conf: Optional[Dict] = None, - predictor: str = None, - predictor_conf: Optional[Dict] = None, - ctc_weight: float = 0.5, - input_size: int = 80, - vocab_size: int = -1, - ignore_id: int = -1, - blank_id: int = 0, - sos: int = 1, - eos: int = 2, - lsm_weight: float = 0.0, - length_normalized_loss: bool = False, - # report_cer: bool = True, - # report_wer: bool = True, - # sym_space: str = "", - # sym_blank: str = "", - # extract_feats_in_collect_stats: bool = True, - # predictor=None, - predictor_weight: float = 0.0, - predictor_bias: int = 0, - sampling_ratio: float = 0.2, - share_embedding: bool = False, - # preencoder: Optional[AbsPreEncoder] = None, - # postencoder: Optional[AbsPostEncoder] = None, - use_1st_decoder_loss: bool = False, - **kwargs, - ): - - super().__init__() - - if specaug is not None: - specaug_class = tables.specaug_classes.get(specaug) - specaug = specaug_class(**specaug_conf) - if normalize is not None: - normalize_class = tables.normalize_classes.get(normalize) - normalize = normalize_class(**normalize_conf) - encoder_class = tables.encoder_classes.get(encoder) - encoder = encoder_class(input_size=input_size, **encoder_conf) - encoder_output_size = encoder.output_size() - - if decoder is not None: - decoder_class = tables.decoder_classes.get(decoder) - decoder = decoder_class( - vocab_size=vocab_size, - encoder_output_size=encoder_output_size, - **decoder_conf, - ) - if ctc_weight > 0.0: - - if ctc_conf is None: - ctc_conf = {} - - ctc = CTC( - odim=vocab_size, encoder_output_size=encoder_output_size, **ctc_conf - ) - if predictor is not None: - predictor_class = tables.predictor_classes.get(predictor) - predictor = predictor_class(**predictor_conf) - - # note that eos is the same as sos (equivalent ID) - self.blank_id = blank_id - self.sos = sos if sos is not None else vocab_size - 1 - self.eos = eos if eos is not None else vocab_size - 1 - self.vocab_size = vocab_size - self.ignore_id = ignore_id - self.ctc_weight = ctc_weight - # self.token_list = token_list.copy() - # - # self.frontend = frontend - self.specaug = specaug - self.normalize = normalize - # self.preencoder = preencoder - # self.postencoder = postencoder - self.encoder = encoder - # - # if not hasattr(self.encoder, "interctc_use_conditioning"): - # self.encoder.interctc_use_conditioning = False - # if self.encoder.interctc_use_conditioning: - # self.encoder.conditioning_layer = torch.nn.Linear( - # vocab_size, self.encoder.output_size() - # ) - # - # self.error_calculator = None - # - if ctc_weight == 1.0: - self.decoder = None - else: - self.decoder = decoder - - self.criterion_att = LabelSmoothingLoss( - size=vocab_size, - padding_idx=ignore_id, - smoothing=lsm_weight, - normalize_length=length_normalized_loss, - ) - # - # if report_cer or report_wer: - # self.error_calculator = ErrorCalculator( - # token_list, sym_space, sym_blank, report_cer, report_wer - # ) - # - if ctc_weight == 0.0: - self.ctc = None - else: - self.ctc = ctc - # - # self.extract_feats_in_collect_stats = extract_feats_in_collect_stats - self.predictor = predictor - self.predictor_weight = predictor_weight - self.predictor_bias = predictor_bias - self.sampling_ratio = sampling_ratio - self.criterion_pre = mae_loss(normalize_length=length_normalized_loss) - # self.step_cur = 0 - # - self.share_embedding = share_embedding - if self.share_embedding: - self.decoder.embed = None - - self.use_1st_decoder_loss = use_1st_decoder_loss - self.length_normalized_loss = length_normalized_loss - self.beam_search = None - self.error_calculator = None - - def forward( - self, - speech: torch.Tensor, - speech_lengths: torch.Tensor, - text: torch.Tensor, - text_lengths: torch.Tensor, - **kwargs, - ) -> Tuple[torch.Tensor, Dict[str, torch.Tensor], torch.Tensor]: - """Encoder + Decoder + Calc loss - Args: - speech: (Batch, Length, ...) - speech_lengths: (Batch, ) - text: (Batch, Length) - text_lengths: (Batch,) - """ - # import pdb; - # pdb.set_trace() - if len(text_lengths.size()) > 1: - text_lengths = text_lengths[:, 0] - if len(speech_lengths.size()) > 1: - speech_lengths = speech_lengths[:, 0] - - batch_size = speech.shape[0] - - # Encoder - encoder_out, encoder_out_lens = self.encode(speech, speech_lengths) - - loss_ctc, cer_ctc = None, None - loss_pre = None - stats = dict() - - # decoder: CTC branch - if self.ctc_weight != 0.0: - loss_ctc, cer_ctc = self._calc_ctc_loss( - encoder_out, encoder_out_lens, text, text_lengths - ) - - # Collect CTC branch stats - stats["loss_ctc"] = loss_ctc.detach() if loss_ctc is not None else None - stats["cer_ctc"] = cer_ctc - - # decoder: Attention decoder branch - loss_att, acc_att, cer_att, wer_att, loss_pre, pre_loss_att = ( - self._calc_att_loss(encoder_out, encoder_out_lens, text, text_lengths) - ) - - # 3. CTC-Att loss definition - if self.ctc_weight == 0.0: - loss = loss_att + loss_pre * self.predictor_weight - else: - loss = ( - self.ctc_weight * loss_ctc - + (1 - self.ctc_weight) * loss_att - + loss_pre * self.predictor_weight - ) - - # Collect Attn branch stats - stats["loss_att"] = loss_att.detach() if loss_att is not None else None - stats["pre_loss_att"] = ( - pre_loss_att.detach() if pre_loss_att is not None else None - ) - stats["acc"] = acc_att - stats["cer"] = cer_att - stats["wer"] = wer_att - stats["loss_pre"] = loss_pre.detach().cpu() if loss_pre is not None else None - - stats["loss"] = torch.clone(loss.detach()) - - # force_gatherable: to-device and to-tensor if scalar for DataParallel - if self.length_normalized_loss: - batch_size = (text_lengths + self.predictor_bias).sum() - loss, stats, weight = force_gatherable((loss, stats, batch_size), loss.device) - return loss, stats, weight - - def encode( - self, - speech: torch.Tensor, - speech_lengths: torch.Tensor, - **kwargs, - ) -> Tuple[torch.Tensor, torch.Tensor]: - """Encoder. Note that this method is used by asr_inference.py - Args: - speech: (Batch, Length, ...) - speech_lengths: (Batch, ) - ind: int - """ - with autocast(False): - - # Data augmentation - if self.specaug is not None and self.training: - speech, speech_lengths = self.specaug(speech, speech_lengths) - - # Normalization for feature: e.g. Global-CMVN, Utterance-CMVN - if self.normalize is not None: - speech, speech_lengths = self.normalize(speech, speech_lengths) - - # Forward encoder - encoder_out, encoder_out_lens, _ = self.encoder(speech, speech_lengths) - if isinstance(encoder_out, tuple): - encoder_out = encoder_out[0] - - return encoder_out, encoder_out_lens - - def calc_predictor(self, encoder_out, encoder_out_lens): - - encoder_out_mask = ( - ~make_pad_mask(encoder_out_lens, maxlen=encoder_out.size(1))[:, None, :] - ).to(encoder_out.device) - pre_acoustic_embeds, pre_token_length, alphas, pre_peak_index = self.predictor( - encoder_out, None, encoder_out_mask, ignore_id=self.ignore_id - ) - return pre_acoustic_embeds, pre_token_length, alphas, pre_peak_index - - def cal_decoder_with_predictor( - self, encoder_out, encoder_out_lens, sematic_embeds, ys_pad_lens - ): - - decoder_outs = self.decoder( - encoder_out, encoder_out_lens, sematic_embeds, ys_pad_lens - ) - decoder_out = decoder_outs[0] - decoder_out = torch.log_softmax(decoder_out, dim=-1) - return decoder_out, ys_pad_lens - - def _calc_att_loss( - self, - encoder_out: torch.Tensor, - encoder_out_lens: torch.Tensor, - ys_pad: torch.Tensor, - ys_pad_lens: torch.Tensor, - ): - encoder_out_mask = ( - ~make_pad_mask(encoder_out_lens, maxlen=encoder_out.size(1))[:, None, :] - ).to(encoder_out.device) - if self.predictor_bias == 1: - _, ys_pad = add_sos_eos(ys_pad, self.sos, self.eos, self.ignore_id) - ys_pad_lens = ys_pad_lens + self.predictor_bias - pre_acoustic_embeds, pre_token_length, _, pre_peak_index = self.predictor( - encoder_out, ys_pad, encoder_out_mask, ignore_id=self.ignore_id - ) - - # 0. sampler - decoder_out_1st = None - pre_loss_att = None - if self.sampling_ratio > 0.0: - - sematic_embeds, decoder_out_1st = self.sampler( - encoder_out, encoder_out_lens, ys_pad, ys_pad_lens, pre_acoustic_embeds - ) - else: - sematic_embeds = pre_acoustic_embeds - - # 1. Forward decoder - decoder_outs = self.decoder( - encoder_out, encoder_out_lens, sematic_embeds, ys_pad_lens - ) - decoder_out, _ = decoder_outs[0], decoder_outs[1] - - if decoder_out_1st is None: - decoder_out_1st = decoder_out - # 2. Compute attention loss - loss_att = self.criterion_att(decoder_out, ys_pad) - acc_att = th_accuracy( - decoder_out_1st.view(-1, self.vocab_size), - ys_pad, - ignore_label=self.ignore_id, - ) - loss_pre = self.criterion_pre( - ys_pad_lens.type_as(pre_token_length), pre_token_length - ) - - # Compute cer/wer using attention-decoder - if self.training or self.error_calculator is None: - cer_att, wer_att = None, None - else: - ys_hat = decoder_out_1st.argmax(dim=-1) - cer_att, wer_att = self.error_calculator(ys_hat.cpu(), ys_pad.cpu()) - - return loss_att, acc_att, cer_att, wer_att, loss_pre, pre_loss_att - - def sampler( - self, encoder_out, encoder_out_lens, ys_pad, ys_pad_lens, pre_acoustic_embeds - ): - - tgt_mask = ( - ~make_pad_mask(ys_pad_lens, maxlen=ys_pad_lens.max())[:, :, None] - ).to(ys_pad.device) - ys_pad_masked = ys_pad * tgt_mask[:, :, 0] - if self.share_embedding: - ys_pad_embed = self.decoder.output_layer.weight[ys_pad_masked] - else: - ys_pad_embed = self.decoder.embed(ys_pad_masked) - with torch.no_grad(): - decoder_outs = self.decoder( - encoder_out, encoder_out_lens, pre_acoustic_embeds, ys_pad_lens - ) - decoder_out, _ = decoder_outs[0], decoder_outs[1] - pred_tokens = decoder_out.argmax(-1) - nonpad_positions = ys_pad.ne(self.ignore_id) - seq_lens = (nonpad_positions).sum(1) - same_num = ((pred_tokens == ys_pad) & nonpad_positions).sum(1) - input_mask = torch.ones_like(nonpad_positions) - bsz, seq_len = ys_pad.size() - for li in range(bsz): - target_num = ( - ((seq_lens[li] - same_num[li].sum()).float()) * self.sampling_ratio - ).long() - if target_num > 0: - input_mask[li].scatter_( - dim=0, - index=torch.randperm(seq_lens[li])[:target_num].to( - input_mask.device - ), - value=0, - ) - input_mask = input_mask.eq(1) - input_mask = input_mask.masked_fill(~nonpad_positions, False) - input_mask_expand_dim = input_mask.unsqueeze(2).to( - pre_acoustic_embeds.device - ) - - sematic_embeds = pre_acoustic_embeds.masked_fill( - ~input_mask_expand_dim, 0 - ) + ys_pad_embed.masked_fill(input_mask_expand_dim, 0) - return sematic_embeds * tgt_mask, decoder_out * tgt_mask - - def _calc_ctc_loss( - self, - encoder_out: torch.Tensor, - encoder_out_lens: torch.Tensor, - ys_pad: torch.Tensor, - ys_pad_lens: torch.Tensor, - ): - # Calc CTC loss - loss_ctc = self.ctc(encoder_out, encoder_out_lens, ys_pad, ys_pad_lens) - - # Calc CER using CTC - cer_ctc = None - if not self.training and self.error_calculator is not None: - ys_hat = self.ctc.argmax(encoder_out).data - cer_ctc = self.error_calculator(ys_hat.cpu(), ys_pad.cpu(), is_ctc=True) - return loss_ctc, cer_ctc - - def init_beam_search( - self, - **kwargs, - ): - from funasr_detach.models.paraformer.search import BeamSearchPara - from funasr_detach.models.transformer.scorers.ctc import CTCPrefixScorer - from funasr_detach.models.transformer.scorers.length_bonus import LengthBonus - - # 1. Build ASR model - scorers = {} - - if self.ctc != None: - ctc = CTCPrefixScorer(ctc=self.ctc, eos=self.eos) - scorers.update(ctc=ctc) - token_list = kwargs.get("token_list") - scorers.update( - length_bonus=LengthBonus(len(token_list)), - ) - - # 3. Build ngram model - # ngram is not supported now - ngram = None - scorers["ngram"] = ngram - - weights = dict( - decoder=1.0 - kwargs.get("decoding_ctc_weight"), - ctc=kwargs.get("decoding_ctc_weight", 0.0), - lm=kwargs.get("lm_weight", 0.0), - ngram=kwargs.get("ngram_weight", 0.0), - length_bonus=kwargs.get("penalty", 0.0), - ) - beam_search = BeamSearchPara( - beam_size=kwargs.get("beam_size", 2), - weights=weights, - scorers=scorers, - sos=self.sos, - eos=self.eos, - vocab_size=len(token_list), - token_list=token_list, - pre_beam_score_key=None if self.ctc_weight == 1.0 else "full", - ) - # beam_search.to(device=kwargs.get("device", "cpu"), dtype=getattr(torch, kwargs.get("dtype", "float32"))).eval() - # for scorer in scorers.values(): - # if isinstance(scorer, torch.nn.Module): - # scorer.to(device=kwargs.get("device", "cpu"), dtype=getattr(torch, kwargs.get("dtype", "float32"))).eval() - self.beam_search = beam_search - - def inference( - self, - data_in, - data_lengths=None, - key: list = None, - tokenizer=None, - frontend=None, - **kwargs, - ): - # init beamsearch - is_use_ctc = ( - kwargs.get("decoding_ctc_weight", 0.0) > 0.00001 and self.ctc != None - ) - is_use_lm = ( - kwargs.get("lm_weight", 0.0) > 0.00001 - and kwargs.get("lm_file", None) is not None - ) - if self.beam_search is None and (is_use_lm or is_use_ctc): - logging.info("enable beam_search") - self.init_beam_search(**kwargs) - self.nbest = kwargs.get("nbest", 1) - - meta_data = {} - if ( - isinstance(data_in, torch.Tensor) - and kwargs.get("data_type", "sound") == "fbank" - ): # fbank - speech, speech_lengths = data_in, data_lengths - if len(speech.shape) < 3: - speech = speech[None, :, :] - if speech_lengths is not None: - speech_lengths = speech_lengths.squeeze(-1) - else: - speech_lengths = speech.shape[1] - else: - # extract fbank feats - time1 = time.perf_counter() - audio_sample_list = load_audio_text_image_video( - data_in, - fs=frontend.fs, - audio_fs=kwargs.get("fs", 16000), - data_type=kwargs.get("data_type", "sound"), - tokenizer=tokenizer, - ) - time2 = time.perf_counter() - meta_data["load_data"] = f"{time2 - time1:0.3f}" - speech, speech_lengths = extract_fbank( - audio_sample_list, - data_type=kwargs.get("data_type", "sound"), - frontend=frontend, - ) - time3 = time.perf_counter() - meta_data["extract_feat"] = f"{time3 - time2:0.3f}" - meta_data["batch_data_time"] = ( - speech_lengths.sum().item() - * frontend.frame_shift - * frontend.lfr_n - / 1000 - ) - - speech = speech.to(device=kwargs["device"]) - speech_lengths = speech_lengths.to(device=kwargs["device"]) - # Encoder - encoder_out, encoder_out_lens = self.encode(speech, speech_lengths) - if isinstance(encoder_out, tuple): - encoder_out = encoder_out[0] - - # predictor - predictor_outs = self.calc_predictor(encoder_out, encoder_out_lens) - pre_acoustic_embeds, pre_token_length, alphas, pre_peak_index = ( - predictor_outs[0], - predictor_outs[1], - predictor_outs[2], - predictor_outs[3], - ) - pre_token_length = pre_token_length.round().long() - if torch.max(pre_token_length) < 1: - return [] - decoder_outs = self.cal_decoder_with_predictor( - encoder_out, encoder_out_lens, pre_acoustic_embeds, pre_token_length - ) - decoder_out, ys_pad_lens = decoder_outs[0], decoder_outs[1] - - results = [] - b, n, d = decoder_out.size() - if isinstance(key[0], (list, tuple)): - key = key[0] - if len(key) < b: - key = key * b - for i in range(b): - x = encoder_out[i, : encoder_out_lens[i], :] - am_scores = decoder_out[i, : pre_token_length[i], :] - if self.beam_search is not None: - nbest_hyps = self.beam_search( - x=x, - am_scores=am_scores, - maxlenratio=kwargs.get("maxlenratio", 0.0), - minlenratio=kwargs.get("minlenratio", 0.0), - ) - - nbest_hyps = nbest_hyps[: self.nbest] - else: - - yseq = am_scores.argmax(dim=-1) - score = am_scores.max(dim=-1)[0] - score = torch.sum(score, dim=-1) - # pad with mask tokens to ensure compatibility with sos/eos tokens - yseq = torch.tensor( - [self.sos] + yseq.tolist() + [self.eos], device=yseq.device - ) - nbest_hyps = [Hypothesis(yseq=yseq, score=score)] - for nbest_idx, hyp in enumerate(nbest_hyps): - ibest_writer = None - if kwargs.get("output_dir") is not None: - if not hasattr(self, "writer"): - self.writer = DatadirWriter(kwargs.get("output_dir")) - ibest_writer = self.writer[f"{nbest_idx+1}best_recog"] - # remove sos/eos and get results - last_pos = -1 - if isinstance(hyp.yseq, list): - token_int = hyp.yseq[1:last_pos] - else: - token_int = hyp.yseq[1:last_pos].tolist() - - # remove blank symbol id, which is assumed to be 0 - token_int = list( - filter( - lambda x: x != self.eos - and x != self.sos - and x != self.blank_id, - token_int, - ) - ) - - if tokenizer is not None: - # Change integer-ids to tokens - token = tokenizer.ids2tokens(token_int) - text_postprocessed = tokenizer.tokens2text(token) - if not hasattr(tokenizer, "bpemodel"): - text_postprocessed, _ = postprocess_utils.sentence_postprocess( - token - ) - - result_i = {"key": key[i], "text": text_postprocessed} - - if ibest_writer is not None: - ibest_writer["token"][key[i]] = " ".join(token) - # ibest_writer["text"][key[i]] = text - ibest_writer["text"][key[i]] = text_postprocessed - else: - result_i = {"key": key[i], "token_int": token_int} - results.append(result_i) - - return results, meta_data diff --git a/funasr_detach/models/paraformer/search.py b/funasr_detach/models/paraformer/search.py deleted file mode 100644 index cc3ed45de8a3dedf57245bdbe877f29b8005cefc..0000000000000000000000000000000000000000 --- a/funasr_detach/models/paraformer/search.py +++ /dev/null @@ -1,459 +0,0 @@ -#!/usr/bin/env python3 -# -*- encoding: utf-8 -*- -# Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved. -# MIT License (https://opensource.org/licenses/MIT) - -import torch -import logging -from itertools import chain -from typing import Any, Dict, List, NamedTuple, Tuple, Union - -from funasr_detach.metrics.common import end_detect -from funasr_detach.models.transformer.scorers.scorer_interface import ( - PartialScorerInterface, - ScorerInterface, -) - - -class Hypothesis(NamedTuple): - """Hypothesis data type.""" - - yseq: torch.Tensor - score: Union[float, torch.Tensor] = 0 - scores: Dict[str, Union[float, torch.Tensor]] = dict() - states: Dict[str, Any] = dict() - - def asdict(self) -> dict: - """Convert data to JSON-friendly dict.""" - return self._replace( - yseq=self.yseq.tolist(), - score=float(self.score), - scores={k: float(v) for k, v in self.scores.items()}, - )._asdict() - - -class BeamSearchPara(torch.nn.Module): - """Beam search implementation.""" - - def __init__( - self, - scorers: Dict[str, ScorerInterface], - weights: Dict[str, float], - beam_size: int, - vocab_size: int, - sos: int, - eos: int, - token_list: List[str] = None, - pre_beam_ratio: float = 1.5, - pre_beam_score_key: str = None, - ): - """Initialize beam search. - - Args: - scorers (dict[str, ScorerInterface]): Dict of decoder modules - e.g., Decoder, CTCPrefixScorer, LM - The scorer will be ignored if it is `None` - weights (dict[str, float]): Dict of weights for each scorers - The scorer will be ignored if its weight is 0 - beam_size (int): The number of hypotheses kept during search - vocab_size (int): The number of vocabulary - sos (int): Start of sequence id - eos (int): End of sequence id - token_list (list[str]): List of tokens for debug log - pre_beam_score_key (str): key of scores to perform pre-beam search - pre_beam_ratio (float): beam size in the pre-beam search - will be `int(pre_beam_ratio * beam_size)` - - """ - super().__init__() - # set scorers - self.weights = weights - self.scorers = dict() - self.full_scorers = dict() - self.part_scorers = dict() - # this module dict is required for recursive cast - # `self.to(device, dtype)` in `recog.py` - self.nn_dict = torch.nn.ModuleDict() - for k, v in scorers.items(): - w = weights.get(k, 0) - if w == 0 or v is None: - continue - assert isinstance( - v, ScorerInterface - ), f"{k} ({type(v)}) does not implement ScorerInterface" - self.scorers[k] = v - if isinstance(v, PartialScorerInterface): - self.part_scorers[k] = v - else: - self.full_scorers[k] = v - if isinstance(v, torch.nn.Module): - self.nn_dict[k] = v - - # set configurations - self.sos = sos - self.eos = eos - self.token_list = token_list - self.pre_beam_size = int(pre_beam_ratio * beam_size) - self.beam_size = beam_size - self.n_vocab = vocab_size - if ( - pre_beam_score_key is not None - and pre_beam_score_key != "full" - and pre_beam_score_key not in self.full_scorers - ): - raise KeyError(f"{pre_beam_score_key} is not found in {self.full_scorers}") - self.pre_beam_score_key = pre_beam_score_key - self.do_pre_beam = ( - self.pre_beam_score_key is not None - and self.pre_beam_size < self.n_vocab - and len(self.part_scorers) > 0 - ) - - def init_hyp(self, x: torch.Tensor) -> List[Hypothesis]: - """Get an initial hypothesis data. - - Args: - x (torch.Tensor): The encoder output feature - - Returns: - Hypothesis: The initial hypothesis. - - """ - init_states = dict() - init_scores = dict() - for k, d in self.scorers.items(): - init_states[k] = d.init_state(x) - init_scores[k] = 0.0 - return [ - Hypothesis( - score=0.0, - scores=init_scores, - states=init_states, - yseq=torch.tensor([self.sos], device=x.device), - ) - ] - - @staticmethod - def append_token(xs: torch.Tensor, x: int) -> torch.Tensor: - """Append new token to prefix tokens. - - Args: - xs (torch.Tensor): The prefix token - x (int): The new token to append - - Returns: - torch.Tensor: New tensor contains: xs + [x] with xs.dtype and xs.device - - """ - x = torch.tensor([x], dtype=xs.dtype, device=xs.device) - return torch.cat((xs, x)) - - def score_full( - self, hyp: Hypothesis, x: torch.Tensor - ) -> Tuple[Dict[str, torch.Tensor], Dict[str, Any]]: - """Score new hypothesis by `self.full_scorers`. - - Args: - hyp (Hypothesis): Hypothesis with prefix tokens to score - x (torch.Tensor): Corresponding input feature - - Returns: - Tuple[Dict[str, torch.Tensor], Dict[str, Any]]: Tuple of - score dict of `hyp` that has string keys of `self.full_scorers` - and tensor score values of shape: `(self.n_vocab,)`, - and state dict that has string keys - and state values of `self.full_scorers` - - """ - scores = dict() - states = dict() - for k, d in self.full_scorers.items(): - scores[k], states[k] = d.score(hyp.yseq, hyp.states[k], x) - return scores, states - - def score_partial( - self, hyp: Hypothesis, ids: torch.Tensor, x: torch.Tensor - ) -> Tuple[Dict[str, torch.Tensor], Dict[str, Any]]: - """Score new hypothesis by `self.part_scorers`. - - Args: - hyp (Hypothesis): Hypothesis with prefix tokens to score - ids (torch.Tensor): 1D tensor of new partial tokens to score - x (torch.Tensor): Corresponding input feature - - Returns: - Tuple[Dict[str, torch.Tensor], Dict[str, Any]]: Tuple of - score dict of `hyp` that has string keys of `self.part_scorers` - and tensor score values of shape: `(len(ids),)`, - and state dict that has string keys - and state values of `self.part_scorers` - - """ - scores = dict() - states = dict() - for k, d in self.part_scorers.items(): - scores[k], states[k] = d.score_partial(hyp.yseq, ids, hyp.states[k], x) - return scores, states - - def beam( - self, weighted_scores: torch.Tensor, ids: torch.Tensor - ) -> Tuple[torch.Tensor, torch.Tensor]: - """Compute topk full token ids and partial token ids. - - Args: - weighted_scores (torch.Tensor): The weighted sum scores for each tokens. - Its shape is `(self.n_vocab,)`. - ids (torch.Tensor): The partial token ids to compute topk - - Returns: - Tuple[torch.Tensor, torch.Tensor]: - The topk full token ids and partial token ids. - Their shapes are `(self.beam_size,)` - - """ - # no pre beam performed - if weighted_scores.size(0) == ids.size(0): - top_ids = weighted_scores.topk(self.beam_size)[1] - return top_ids, top_ids - - # mask pruned in pre-beam not to select in topk - tmp = weighted_scores[ids] - weighted_scores[:] = -float("inf") - weighted_scores[ids] = tmp - top_ids = weighted_scores.topk(self.beam_size)[1] - local_ids = weighted_scores[ids].topk(self.beam_size)[1] - return top_ids, local_ids - - @staticmethod - def merge_scores( - prev_scores: Dict[str, float], - next_full_scores: Dict[str, torch.Tensor], - full_idx: int, - next_part_scores: Dict[str, torch.Tensor], - part_idx: int, - ) -> Dict[str, torch.Tensor]: - """Merge scores for new hypothesis. - - Args: - prev_scores (Dict[str, float]): - The previous hypothesis scores by `self.scorers` - next_full_scores (Dict[str, torch.Tensor]): scores by `self.full_scorers` - full_idx (int): The next token id for `next_full_scores` - next_part_scores (Dict[str, torch.Tensor]): - scores of partial tokens by `self.part_scorers` - part_idx (int): The new token id for `next_part_scores` - - Returns: - Dict[str, torch.Tensor]: The new score dict. - Its keys are names of `self.full_scorers` and `self.part_scorers`. - Its values are scalar tensors by the scorers. - - """ - new_scores = dict() - for k, v in next_full_scores.items(): - new_scores[k] = prev_scores[k] + v[full_idx] - for k, v in next_part_scores.items(): - new_scores[k] = prev_scores[k] + v[part_idx] - return new_scores - - def merge_states(self, states: Any, part_states: Any, part_idx: int) -> Any: - """Merge states for new hypothesis. - - Args: - states: states of `self.full_scorers` - part_states: states of `self.part_scorers` - part_idx (int): The new token id for `part_scores` - - Returns: - Dict[str, torch.Tensor]: The new score dict. - Its keys are names of `self.full_scorers` and `self.part_scorers`. - Its values are states of the scorers. - - """ - new_states = dict() - for k, v in states.items(): - new_states[k] = v - for k, d in self.part_scorers.items(): - new_states[k] = d.select_state(part_states[k], part_idx) - return new_states - - def search( - self, running_hyps: List[Hypothesis], x: torch.Tensor, am_score: torch.Tensor - ) -> List[Hypothesis]: - """Search new tokens for running hypotheses and encoded speech x. - - Args: - running_hyps (List[Hypothesis]): Running hypotheses on beam - x (torch.Tensor): Encoded speech feature (T, D) - - Returns: - List[Hypotheses]: Best sorted hypotheses - - """ - best_hyps = [] - part_ids = torch.arange(self.n_vocab, device=x.device) # no pre-beam - for hyp in running_hyps: - # scoring - weighted_scores = torch.zeros(self.n_vocab, dtype=x.dtype, device=x.device) - weighted_scores += am_score - scores, states = self.score_full(hyp, x) - for k in self.full_scorers: - weighted_scores += self.weights[k] * scores[k] - # partial scoring - if self.do_pre_beam: - pre_beam_scores = ( - weighted_scores - if self.pre_beam_score_key == "full" - else scores[self.pre_beam_score_key] - ) - part_ids = torch.topk(pre_beam_scores, self.pre_beam_size)[1] - part_scores, part_states = self.score_partial(hyp, part_ids, x) - for k in self.part_scorers: - weighted_scores[part_ids] += self.weights[k] * part_scores[k] - # add previous hyp score - weighted_scores += hyp.score - - # update hyps - for j, part_j in zip(*self.beam(weighted_scores, part_ids)): - # will be (2 x beam at most) - best_hyps.append( - Hypothesis( - score=weighted_scores[j], - yseq=self.append_token(hyp.yseq, j), - scores=self.merge_scores( - hyp.scores, scores, j, part_scores, part_j - ), - states=self.merge_states(states, part_states, part_j), - ) - ) - - # sort and prune 2 x beam -> beam - best_hyps = sorted(best_hyps, key=lambda x: x.score, reverse=True)[ - : min(len(best_hyps), self.beam_size) - ] - return best_hyps - - def forward( - self, - x: torch.Tensor, - am_scores: torch.Tensor, - maxlenratio: float = 0.0, - minlenratio: float = 0.0, - ) -> List[Hypothesis]: - """Perform beam search. - - Args: - x (torch.Tensor): Encoded speech feature (T, D) - maxlenratio (float): Input length ratio to obtain max output length. - If maxlenratio=0.0 (default), it uses a end-detect function - to automatically find maximum hypothesis lengths - If maxlenratio<0.0, its absolute value is interpreted - as a constant max output length. - minlenratio (float): Input length ratio to obtain min output length. - - Returns: - list[Hypothesis]: N-best decoding results - - """ - # set length bounds - maxlen = am_scores.shape[0] - logging.info("decoder input length: " + str(x.shape[0])) - logging.info("max output length: " + str(maxlen)) - - # main loop of prefix search - running_hyps = self.init_hyp(x) - ended_hyps = [] - for i in range(maxlen): - logging.debug("position " + str(i)) - best = self.search(running_hyps, x, am_scores[i]) - # post process of one iteration - running_hyps = self.post_process(i, maxlen, maxlenratio, best, ended_hyps) - # end detection - if maxlenratio == 0.0 and end_detect([h.asdict() for h in ended_hyps], i): - logging.info(f"end detected at {i}") - break - if len(running_hyps) == 0: - logging.info("no hypothesis. Finish decoding.") - break - else: - logging.debug(f"remained hypotheses: {len(running_hyps)}") - - nbest_hyps = sorted(ended_hyps, key=lambda x: x.score, reverse=True) - # check the number of hypotheses reaching to eos - if len(nbest_hyps) == 0: - logging.warning( - "there is no N-best results, perform recognition " - "again with smaller minlenratio." - ) - return ( - [] - if minlenratio < 0.1 - else self.forward(x, maxlenratio, max(0.0, minlenratio - 0.1)) - ) - - # report the best result - best = nbest_hyps[0] - for k, v in best.scores.items(): - logging.info( - f"{v:6.2f} * {self.weights[k]:3} = {v * self.weights[k]:6.2f} for {k}" - ) - logging.info(f"total log probability: {best.score:.2f}") - logging.info(f"normalized log probability: {best.score / len(best.yseq):.2f}") - logging.info(f"total number of ended hypotheses: {len(nbest_hyps)}") - if self.token_list is not None: - logging.info( - "best hypo: " - + "".join([self.token_list[x.item()] for x in best.yseq[1:-1]]) - + "\n" - ) - return nbest_hyps - - def post_process( - self, - i: int, - maxlen: int, - maxlenratio: float, - running_hyps: List[Hypothesis], - ended_hyps: List[Hypothesis], - ) -> List[Hypothesis]: - """Perform post-processing of beam search iterations. - - Args: - i (int): The length of hypothesis tokens. - maxlen (int): The maximum length of tokens in beam search. - maxlenratio (int): The maximum length ratio in beam search. - running_hyps (List[Hypothesis]): The running hypotheses in beam search. - ended_hyps (List[Hypothesis]): The ended hypotheses in beam search. - - Returns: - List[Hypothesis]: The new running hypotheses. - - """ - logging.debug(f"the number of running hypotheses: {len(running_hyps)}") - if self.token_list is not None: - logging.debug( - "best hypo: " - + "".join([self.token_list[x.item()] for x in running_hyps[0].yseq[1:]]) - ) - # add eos in the final loop to avoid that there are no ended hyps - if i == maxlen - 1: - logging.info("adding in the last position in the loop") - running_hyps = [ - h._replace(yseq=self.append_token(h.yseq, self.eos)) - for h in running_hyps - ] - - # add ended hypotheses to a final list, and removed them from current hypotheses - # (this will be a problem, number of hyps < beam) - remained_hyps = [] - for hyp in running_hyps: - if hyp.yseq[-1] == self.eos: - # e.g., Word LM needs to add final score - for k, d in chain(self.full_scorers.items(), self.part_scorers.items()): - s = d.final_score(hyp.states[k]) - hyp.scores[k] += s - hyp = hyp._replace(score=hyp.score + self.weights[k] * s) - ended_hyps.append(hyp) - else: - remained_hyps.append(hyp) - return remained_hyps diff --git a/funasr_detach/models/paraformer/template.yaml b/funasr_detach/models/paraformer/template.yaml deleted file mode 100644 index bccf6387199fbf82c7fc6aca5646c60fb603fb6e..0000000000000000000000000000000000000000 --- a/funasr_detach/models/paraformer/template.yaml +++ /dev/null @@ -1,122 +0,0 @@ -# This is an example that demonstrates how to configure a model file. -# You can modify the configuration according to your own requirements. - -# to print the register_table: -# from funasr.register import tables -# tables.print() - -# network architecture -model: Paraformer -model_conf: - ctc_weight: 0.0 - lsm_weight: 0.1 - length_normalized_loss: true - predictor_weight: 1.0 - predictor_bias: 1 - sampling_ratio: 0.75 - -# encoder -encoder: SANMEncoder -encoder_conf: - output_size: 512 - attention_heads: 4 - linear_units: 2048 - num_blocks: 50 - dropout_rate: 0.1 - positional_dropout_rate: 0.1 - attention_dropout_rate: 0.1 - input_layer: pe - pos_enc_class: SinusoidalPositionEncoder - normalize_before: true - kernel_size: 11 - sanm_shfit: 0 - selfattention_layer_type: sanm - -# decoder -decoder: ParaformerSANMDecoder -decoder_conf: - attention_heads: 4 - linear_units: 2048 - num_blocks: 16 - dropout_rate: 0.1 - positional_dropout_rate: 0.1 - self_attention_dropout_rate: 0.1 - src_attention_dropout_rate: 0.1 - att_layer_num: 16 - kernel_size: 11 - sanm_shfit: 0 - -predictor: CifPredictorV2 -predictor_conf: - idim: 512 - threshold: 1.0 - l_order: 1 - r_order: 1 - tail_threshold: 0.45 - -# frontend related -frontend: WavFrontend -frontend_conf: - fs: 16000 - window: hamming - n_mels: 80 - frame_length: 25 - frame_shift: 10 - lfr_m: 7 - lfr_n: 6 - -specaug: SpecAugLFR -specaug_conf: - apply_time_warp: false - time_warp_window: 5 - time_warp_mode: bicubic - apply_freq_mask: true - freq_mask_width_range: - - 0 - - 30 - lfr_rate: 6 - num_freq_mask: 1 - apply_time_mask: true - time_mask_width_range: - - 0 - - 12 - num_time_mask: 1 - -train_conf: - accum_grad: 1 - grad_clip: 5 - max_epoch: 150 - keep_nbest_models: 10 - avg_nbest_model: 5 - log_interval: 50 - -optim: adam -optim_conf: - lr: 0.0005 -scheduler: warmuplr -scheduler_conf: - warmup_steps: 30000 - -dataset: AudioDataset -dataset_conf: - index_ds: IndexDSJsonl - batch_sampler: DynamicBatchLocalShuffleSampler - batch_type: example # example or length - batch_size: 1 # if batch_type is example, batch_size is the numbers of samples; if length, batch_size is source_token_len+target_token_len; - max_token_length: 2048 # filter samples if source_token_len+target_token_len > max_token_length, - buffer_size: 500 - shuffle: True - num_workers: 0 - -tokenizer: CharTokenizer -tokenizer_conf: - unk_symbol: - split_with_space: true - - -ctc_conf: - dropout_rate: 0.0 - ctc_type: builtin - reduce: true - ignore_nan_grad: true -normalize: null diff --git a/funasr_detach/models/paraformer_streaming/__init__.py b/funasr_detach/models/paraformer_streaming/__init__.py deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/funasr_detach/models/paraformer_streaming/model.py b/funasr_detach/models/paraformer_streaming/model.py deleted file mode 100644 index cdcaa723000ed83fefe860c96918d0075043043c..0000000000000000000000000000000000000000 --- a/funasr_detach/models/paraformer_streaming/model.py +++ /dev/null @@ -1,834 +0,0 @@ -#!/usr/bin/env python3 -# -*- encoding: utf-8 -*- -# Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved. -# MIT License (https://opensource.org/licenses/MIT) - -import time -import torch -import logging -from typing import Dict, Tuple -from contextlib import contextmanager -from distutils.version import LooseVersion - -from funasr_detach.register import tables -from funasr_detach.models.ctc.ctc import CTC -from funasr_detach.utils import postprocess_utils -from funasr_detach.metrics.compute_acc import th_accuracy -from funasr_detach.utils.datadir_writer import DatadirWriter -from funasr_detach.models.paraformer.model import Paraformer -from funasr_detach.models.paraformer.search import Hypothesis -from funasr_detach.models.paraformer.cif_predictor import mae_loss -from funasr_detach.train_utils.device_funcs import force_gatherable -from funasr_detach.losses.label_smoothing_loss import LabelSmoothingLoss -from funasr_detach.models.transformer.utils.add_sos_eos import add_sos_eos -from funasr_detach.models.transformer.utils.nets_utils import make_pad_mask, pad_list -from funasr_detach.utils.load_utils import load_audio_text_image_video, extract_fbank - - -if LooseVersion(torch.__version__) >= LooseVersion("1.6.0"): - from torch.cuda.amp import autocast -else: - # Nothing to do if torch<1.6.0 - @contextmanager - def autocast(enabled=True): - yield - - -@tables.register("model_classes", "ParaformerStreaming") -class ParaformerStreaming(Paraformer): - """ - Author: Speech Lab of DAMO Academy, Alibaba Group - Paraformer: Fast and Accurate Parallel Transformer for Non-autoregressive End-to-End Speech Recognition - https://arxiv.org/abs/2206.08317 - """ - - def __init__( - self, - *args, - **kwargs, - ): - - super().__init__(*args, **kwargs) - - # import pdb; - # pdb.set_trace() - self.sampling_ratio = kwargs.get("sampling_ratio", 0.2) - - self.scama_mask = None - if ( - hasattr(self.encoder, "overlap_chunk_cls") - and self.encoder.overlap_chunk_cls is not None - ): - from funasr_detach.models.scama.chunk_utilis import ( - build_scama_mask_for_cross_attention_decoder, - ) - - self.build_scama_mask_for_cross_attention_decoder_fn = ( - build_scama_mask_for_cross_attention_decoder - ) - self.decoder_attention_chunk_type = kwargs.get( - "decoder_attention_chunk_type", "chunk" - ) - - def forward( - self, - speech: torch.Tensor, - speech_lengths: torch.Tensor, - text: torch.Tensor, - text_lengths: torch.Tensor, - **kwargs, - ) -> Tuple[torch.Tensor, Dict[str, torch.Tensor], torch.Tensor]: - """Encoder + Decoder + Calc loss - Args: - speech: (Batch, Length, ...) - speech_lengths: (Batch, ) - text: (Batch, Length) - text_lengths: (Batch,) - """ - # import pdb; - # pdb.set_trace() - decoding_ind = kwargs.get("decoding_ind") - if len(text_lengths.size()) > 1: - text_lengths = text_lengths[:, 0] - if len(speech_lengths.size()) > 1: - speech_lengths = speech_lengths[:, 0] - - batch_size = speech.shape[0] - - # Encoder - if hasattr(self.encoder, "overlap_chunk_cls"): - ind = self.encoder.overlap_chunk_cls.random_choice( - self.training, decoding_ind - ) - encoder_out, encoder_out_lens = self.encode(speech, speech_lengths, ind=ind) - else: - encoder_out, encoder_out_lens = self.encode(speech, speech_lengths) - - loss_ctc, cer_ctc = None, None - loss_pre = None - stats = dict() - - # decoder: CTC branch - - if self.ctc_weight > 0.0: - if hasattr(self.encoder, "overlap_chunk_cls"): - encoder_out_ctc, encoder_out_lens_ctc = ( - self.encoder.overlap_chunk_cls.remove_chunk( - encoder_out, encoder_out_lens, chunk_outs=None - ) - ) - else: - encoder_out_ctc, encoder_out_lens_ctc = encoder_out, encoder_out_lens - - loss_ctc, cer_ctc = self._calc_ctc_loss( - encoder_out_ctc, encoder_out_lens_ctc, text, text_lengths - ) - # Collect CTC branch stats - stats["loss_ctc"] = loss_ctc.detach() if loss_ctc is not None else None - stats["cer_ctc"] = cer_ctc - - # decoder: Attention decoder branch - loss_att, acc_att, cer_att, wer_att, loss_pre, pre_loss_att = ( - self._calc_att_predictor_loss( - encoder_out, encoder_out_lens, text, text_lengths - ) - ) - - # 3. CTC-Att loss definition - if self.ctc_weight == 0.0: - loss = loss_att + loss_pre * self.predictor_weight - else: - loss = ( - self.ctc_weight * loss_ctc - + (1 - self.ctc_weight) * loss_att - + loss_pre * self.predictor_weight - ) - - # Collect Attn branch stats - stats["loss_att"] = loss_att.detach() if loss_att is not None else None - stats["pre_loss_att"] = ( - pre_loss_att.detach() if pre_loss_att is not None else None - ) - stats["acc"] = acc_att - stats["cer"] = cer_att - stats["wer"] = wer_att - stats["loss_pre"] = loss_pre.detach().cpu() if loss_pre is not None else None - - stats["loss"] = torch.clone(loss.detach()) - - # force_gatherable: to-device and to-tensor if scalar for DataParallel - if self.length_normalized_loss: - batch_size = (text_lengths + self.predictor_bias).sum() - loss, stats, weight = force_gatherable((loss, stats, batch_size), loss.device) - return loss, stats, weight - - def encode_chunk( - self, - speech: torch.Tensor, - speech_lengths: torch.Tensor, - cache: dict = None, - **kwargs, - ) -> Tuple[torch.Tensor, torch.Tensor]: - """Frontend + Encoder. Note that this method is used by asr_inference.py - Args: - speech: (Batch, Length, ...) - speech_lengths: (Batch, ) - ind: int - """ - with autocast(False): - - # Data augmentation - if self.specaug is not None and self.training: - speech, speech_lengths = self.specaug(speech, speech_lengths) - - # Normalization for feature: e.g. Global-CMVN, Utterance-CMVN - if self.normalize is not None: - speech, speech_lengths = self.normalize(speech, speech_lengths) - - # Forward encoder - encoder_out, encoder_out_lens, _ = self.encoder.forward_chunk( - speech, speech_lengths, cache=cache["encoder"] - ) - if isinstance(encoder_out, tuple): - encoder_out = encoder_out[0] - - return encoder_out, torch.tensor([encoder_out.size(1)]) - - def _calc_att_predictor_loss( - self, - encoder_out: torch.Tensor, - encoder_out_lens: torch.Tensor, - ys_pad: torch.Tensor, - ys_pad_lens: torch.Tensor, - ): - encoder_out_mask = ( - ~make_pad_mask(encoder_out_lens, maxlen=encoder_out.size(1))[:, None, :] - ).to(encoder_out.device) - if self.predictor_bias == 1: - _, ys_pad = add_sos_eos(ys_pad, self.sos, self.eos, self.ignore_id) - ys_pad_lens = ys_pad_lens + self.predictor_bias - mask_chunk_predictor = None - if self.encoder.overlap_chunk_cls is not None: - mask_chunk_predictor = ( - self.encoder.overlap_chunk_cls.get_mask_chunk_predictor( - None, device=encoder_out.device, batch_size=encoder_out.size(0) - ) - ) - mask_shfit_chunk = self.encoder.overlap_chunk_cls.get_mask_shfit_chunk( - None, device=encoder_out.device, batch_size=encoder_out.size(0) - ) - encoder_out = encoder_out * mask_shfit_chunk - pre_acoustic_embeds, pre_token_length, pre_alphas, _ = self.predictor( - encoder_out, - ys_pad, - encoder_out_mask, - ignore_id=self.ignore_id, - mask_chunk_predictor=mask_chunk_predictor, - target_label_length=ys_pad_lens, - ) - predictor_alignments, predictor_alignments_len = ( - self.predictor.gen_frame_alignments(pre_alphas, encoder_out_lens) - ) - - scama_mask = None - if ( - self.encoder.overlap_chunk_cls is not None - and self.decoder_attention_chunk_type == "chunk" - ): - encoder_chunk_size = self.encoder.overlap_chunk_cls.chunk_size_pad_shift_cur - attention_chunk_center_bias = 0 - attention_chunk_size = encoder_chunk_size - decoder_att_look_back_factor = ( - self.encoder.overlap_chunk_cls.decoder_att_look_back_factor_cur - ) - mask_shift_att_chunk_decoder = ( - self.encoder.overlap_chunk_cls.get_mask_shift_att_chunk_decoder( - None, device=encoder_out.device, batch_size=encoder_out.size(0) - ) - ) - scama_mask = self.build_scama_mask_for_cross_attention_decoder_fn( - predictor_alignments=predictor_alignments, - encoder_sequence_length=encoder_out_lens, - chunk_size=1, - encoder_chunk_size=encoder_chunk_size, - attention_chunk_center_bias=attention_chunk_center_bias, - attention_chunk_size=attention_chunk_size, - attention_chunk_type=self.decoder_attention_chunk_type, - step=None, - predictor_mask_chunk_hopping=mask_chunk_predictor, - decoder_att_look_back_factor=decoder_att_look_back_factor, - mask_shift_att_chunk_decoder=mask_shift_att_chunk_decoder, - target_length=ys_pad_lens, - is_training=self.training, - ) - elif self.encoder.overlap_chunk_cls is not None: - encoder_out, encoder_out_lens = self.encoder.overlap_chunk_cls.remove_chunk( - encoder_out, encoder_out_lens, chunk_outs=None - ) - # 0. sampler - decoder_out_1st = None - pre_loss_att = None - if self.sampling_ratio > 0.0: - if self.step_cur < 2: - logging.info( - "enable sampler in paraformer, sampling_ratio: {}".format( - self.sampling_ratio - ) - ) - if self.use_1st_decoder_loss: - sematic_embeds, decoder_out_1st, pre_loss_att = self.sampler_with_grad( - encoder_out, - encoder_out_lens, - ys_pad, - ys_pad_lens, - pre_acoustic_embeds, - scama_mask, - ) - else: - sematic_embeds, decoder_out_1st = self.sampler( - encoder_out, - encoder_out_lens, - ys_pad, - ys_pad_lens, - pre_acoustic_embeds, - scama_mask, - ) - else: - if self.step_cur < 2: - logging.info( - "disable sampler in paraformer, sampling_ratio: {}".format( - self.sampling_ratio - ) - ) - sematic_embeds = pre_acoustic_embeds - - # 1. Forward decoder - decoder_outs = self.decoder( - encoder_out, encoder_out_lens, sematic_embeds, ys_pad_lens, scama_mask - ) - decoder_out, _ = decoder_outs[0], decoder_outs[1] - - if decoder_out_1st is None: - decoder_out_1st = decoder_out - # 2. Compute attention loss - loss_att = self.criterion_att(decoder_out, ys_pad) - acc_att = th_accuracy( - decoder_out_1st.view(-1, self.vocab_size), - ys_pad, - ignore_label=self.ignore_id, - ) - loss_pre = self.criterion_pre( - ys_pad_lens.type_as(pre_token_length), pre_token_length - ) - - # Compute cer/wer using attention-decoder - if self.training or self.error_calculator is None: - cer_att, wer_att = None, None - else: - ys_hat = decoder_out_1st.argmax(dim=-1) - cer_att, wer_att = self.error_calculator(ys_hat.cpu(), ys_pad.cpu()) - - return loss_att, acc_att, cer_att, wer_att, loss_pre, pre_loss_att - - def sampler( - self, - encoder_out, - encoder_out_lens, - ys_pad, - ys_pad_lens, - pre_acoustic_embeds, - chunk_mask=None, - ): - - tgt_mask = ( - ~make_pad_mask(ys_pad_lens, maxlen=ys_pad_lens.max())[:, :, None] - ).to(ys_pad.device) - ys_pad_masked = ys_pad * tgt_mask[:, :, 0] - if self.share_embedding: - ys_pad_embed = self.decoder.output_layer.weight[ys_pad_masked] - else: - ys_pad_embed = self.decoder.embed(ys_pad_masked) - with torch.no_grad(): - decoder_outs = self.decoder( - encoder_out, - encoder_out_lens, - pre_acoustic_embeds, - ys_pad_lens, - chunk_mask, - ) - decoder_out, _ = decoder_outs[0], decoder_outs[1] - pred_tokens = decoder_out.argmax(-1) - nonpad_positions = ys_pad.ne(self.ignore_id) - seq_lens = (nonpad_positions).sum(1) - same_num = ((pred_tokens == ys_pad) & nonpad_positions).sum(1) - input_mask = torch.ones_like(nonpad_positions) - bsz, seq_len = ys_pad.size() - for li in range(bsz): - target_num = ( - ((seq_lens[li] - same_num[li].sum()).float()) * self.sampling_ratio - ).long() - if target_num > 0: - input_mask[li].scatter_( - dim=0, - index=torch.randperm(seq_lens[li])[:target_num].cuda(), - value=0, - ) - input_mask = input_mask.eq(1) - input_mask = input_mask.masked_fill(~nonpad_positions, False) - input_mask_expand_dim = input_mask.unsqueeze(2).to( - pre_acoustic_embeds.device - ) - - sematic_embeds = pre_acoustic_embeds.masked_fill( - ~input_mask_expand_dim, 0 - ) + ys_pad_embed.masked_fill(input_mask_expand_dim, 0) - return sematic_embeds * tgt_mask, decoder_out * tgt_mask - - def calc_predictor(self, encoder_out, encoder_out_lens): - - encoder_out_mask = ( - ~make_pad_mask(encoder_out_lens, maxlen=encoder_out.size(1))[:, None, :] - ).to(encoder_out.device) - mask_chunk_predictor = None - if self.encoder.overlap_chunk_cls is not None: - mask_chunk_predictor = ( - self.encoder.overlap_chunk_cls.get_mask_chunk_predictor( - None, device=encoder_out.device, batch_size=encoder_out.size(0) - ) - ) - mask_shfit_chunk = self.encoder.overlap_chunk_cls.get_mask_shfit_chunk( - None, device=encoder_out.device, batch_size=encoder_out.size(0) - ) - encoder_out = encoder_out * mask_shfit_chunk - pre_acoustic_embeds, pre_token_length, pre_alphas, pre_peak_index = ( - self.predictor( - encoder_out, - None, - encoder_out_mask, - ignore_id=self.ignore_id, - mask_chunk_predictor=mask_chunk_predictor, - target_label_length=None, - ) - ) - predictor_alignments, predictor_alignments_len = ( - self.predictor.gen_frame_alignments( - pre_alphas, - ( - encoder_out_lens + 1 - if self.predictor.tail_threshold > 0.0 - else encoder_out_lens - ), - ) - ) - - scama_mask = None - if ( - self.encoder.overlap_chunk_cls is not None - and self.decoder_attention_chunk_type == "chunk" - ): - encoder_chunk_size = self.encoder.overlap_chunk_cls.chunk_size_pad_shift_cur - attention_chunk_center_bias = 0 - attention_chunk_size = encoder_chunk_size - decoder_att_look_back_factor = ( - self.encoder.overlap_chunk_cls.decoder_att_look_back_factor_cur - ) - mask_shift_att_chunk_decoder = ( - self.encoder.overlap_chunk_cls.get_mask_shift_att_chunk_decoder( - None, device=encoder_out.device, batch_size=encoder_out.size(0) - ) - ) - scama_mask = self.build_scama_mask_for_cross_attention_decoder_fn( - predictor_alignments=predictor_alignments, - encoder_sequence_length=encoder_out_lens, - chunk_size=1, - encoder_chunk_size=encoder_chunk_size, - attention_chunk_center_bias=attention_chunk_center_bias, - attention_chunk_size=attention_chunk_size, - attention_chunk_type=self.decoder_attention_chunk_type, - step=None, - predictor_mask_chunk_hopping=mask_chunk_predictor, - decoder_att_look_back_factor=decoder_att_look_back_factor, - mask_shift_att_chunk_decoder=mask_shift_att_chunk_decoder, - target_length=None, - is_training=self.training, - ) - self.scama_mask = scama_mask - - return pre_acoustic_embeds, pre_token_length, pre_alphas, pre_peak_index - - def calc_predictor_chunk(self, encoder_out, encoder_out_lens, cache=None, **kwargs): - is_final = kwargs.get("is_final", False) - - return self.predictor.forward_chunk( - encoder_out, cache["encoder"], is_final=is_final - ) - - def cal_decoder_with_predictor( - self, encoder_out, encoder_out_lens, sematic_embeds, ys_pad_lens - ): - decoder_outs = self.decoder( - encoder_out, encoder_out_lens, sematic_embeds, ys_pad_lens, self.scama_mask - ) - decoder_out = decoder_outs[0] - decoder_out = torch.log_softmax(decoder_out, dim=-1) - return decoder_out, ys_pad_lens - - def cal_decoder_with_predictor_chunk( - self, encoder_out, encoder_out_lens, sematic_embeds, ys_pad_lens, cache=None - ): - decoder_outs = self.decoder.forward_chunk( - encoder_out, sematic_embeds, cache["decoder"] - ) - decoder_out = decoder_outs - decoder_out = torch.log_softmax(decoder_out, dim=-1) - return decoder_out, ys_pad_lens - - def init_cache(self, cache: dict = {}, **kwargs): - chunk_size = kwargs.get("chunk_size", [0, 10, 5]) - encoder_chunk_look_back = kwargs.get("encoder_chunk_look_back", 0) - decoder_chunk_look_back = kwargs.get("decoder_chunk_look_back", 0) - batch_size = 1 - - enc_output_size = kwargs["encoder_conf"]["output_size"] - feats_dims = ( - kwargs["frontend_conf"]["n_mels"] * kwargs["frontend_conf"]["lfr_m"] - ) - cache_encoder = { - "start_idx": 0, - "cif_hidden": torch.zeros((batch_size, 1, enc_output_size)), - "cif_alphas": torch.zeros((batch_size, 1)), - "chunk_size": chunk_size, - "encoder_chunk_look_back": encoder_chunk_look_back, - "last_chunk": False, - "opt": None, - "feats": torch.zeros( - (batch_size, chunk_size[0] + chunk_size[2], feats_dims) - ), - "tail_chunk": False, - } - cache["encoder"] = cache_encoder - - cache_decoder = { - "decode_fsmn": None, - "decoder_chunk_look_back": decoder_chunk_look_back, - "opt": None, - "chunk_size": chunk_size, - } - cache["decoder"] = cache_decoder - cache["frontend"] = {} - cache["prev_samples"] = torch.empty(0) - - return cache - - def generate_chunk( - self, - speech, - speech_lengths=None, - key: list = None, - tokenizer=None, - frontend=None, - **kwargs, - ): - cache = kwargs.get("cache", {}) - speech = speech.to(device=kwargs["device"]) - speech_lengths = speech_lengths.to(device=kwargs["device"]) - - # Encoder - # - encoder_out, encoder_out_lens = self.encode_chunk( - speech, speech_lengths, cache=cache, is_final=kwargs.get("is_final", False) - ) - print(speech.shape, encoder_out.shape, encoder_out_lens) - if isinstance(encoder_out, tuple): - encoder_out = encoder_out[0] - - # predictor - predictor_outs = self.calc_predictor_chunk( - encoder_out, - encoder_out_lens, - cache=cache, - is_final=kwargs.get("is_final", False), - ) - pre_acoustic_embeds, pre_token_length, alphas, pre_peak_index = ( - predictor_outs[0], - predictor_outs[1], - predictor_outs[2], - predictor_outs[3], - ) - pre_token_length = pre_token_length.round().long() - if torch.max(pre_token_length) < 1: - return [] - decoder_outs = self.cal_decoder_with_predictor_chunk( - encoder_out, - encoder_out_lens, - pre_acoustic_embeds, - pre_token_length, - cache=cache, - ) - decoder_out, ys_pad_lens = decoder_outs[0], decoder_outs[1] - - results = [] - b, n, d = decoder_out.size() - if isinstance(key[0], (list, tuple)): - key = key[0] - for i in range(b): - x = encoder_out[i, : encoder_out_lens[i], :] - am_scores = decoder_out[i, : pre_token_length[i], :] - if self.beam_search is not None: - nbest_hyps = self.beam_search( - x=x, - am_scores=am_scores, - maxlenratio=kwargs.get("maxlenratio", 0.0), - minlenratio=kwargs.get("minlenratio", 0.0), - ) - - nbest_hyps = nbest_hyps[: self.nbest] - else: - - yseq = am_scores.argmax(dim=-1) - score = am_scores.max(dim=-1)[0] - score = torch.sum(score, dim=-1) - # pad with mask tokens to ensure compatibility with sos/eos tokens - yseq = torch.tensor( - [self.sos] + yseq.tolist() + [self.eos], device=yseq.device - ) - nbest_hyps = [Hypothesis(yseq=yseq, score=score)] - for nbest_idx, hyp in enumerate(nbest_hyps): - - # remove sos/eos and get results - last_pos = -1 - if isinstance(hyp.yseq, list): - token_int = hyp.yseq[1:last_pos] - else: - token_int = hyp.yseq[1:last_pos].tolist() - - # remove blank symbol id, which is assumed to be 0 - token_int = list( - filter( - lambda x: x != self.eos - and x != self.sos - and x != self.blank_id, - token_int, - ) - ) - - # Change integer-ids to tokens - token = tokenizer.ids2tokens(token_int) - # text = tokenizer.tokens2text(token) - - result_i = token - - results.extend(result_i) - - return results - - def inference( - self, - data_in, - data_lengths=None, - key: list = None, - tokenizer=None, - frontend=None, - cache: dict = {}, - **kwargs, - ): - - # init beamsearch - is_use_ctc = ( - kwargs.get("decoding_ctc_weight", 0.0) > 0.00001 and self.ctc != None - ) - is_use_lm = ( - kwargs.get("lm_weight", 0.0) > 0.00001 - and kwargs.get("lm_file", None) is not None - ) - if self.beam_search is None and (is_use_lm or is_use_ctc): - logging.info("enable beam_search") - self.init_beam_search(**kwargs) - self.nbest = kwargs.get("nbest", 1) - - if len(cache) == 0: - self.init_cache(cache, **kwargs) - - meta_data = {} - chunk_size = kwargs.get("chunk_size", [0, 10, 5]) - chunk_stride_samples = int(chunk_size[1] * 960) # 600ms - - time1 = time.perf_counter() - cfg = {"is_final": kwargs.get("is_final", False)} - audio_sample_list = load_audio_text_image_video( - data_in, - fs=frontend.fs, - audio_fs=kwargs.get("fs", 16000), - data_type=kwargs.get("data_type", "sound"), - tokenizer=tokenizer, - cache=cfg, - ) - # import pdb; pdb.set_trace() - _is_final = cfg["is_final"] # if data_in is a file or url, set is_final=True - - time2 = time.perf_counter() - meta_data["load_data"] = f"{time2 - time1:0.3f}" - assert len(audio_sample_list) == 1, "batch_size must be set 1" - - audio_sample = torch.cat((cache["prev_samples"], audio_sample_list[0])) - - n = int(len(audio_sample) // chunk_stride_samples + int(_is_final)) - m = int(len(audio_sample) % chunk_stride_samples * (1 - int(_is_final))) - tokens = [] - for i in range(n): - kwargs["is_final"] = _is_final and i == n - 1 - audio_sample_i = audio_sample[ - i * chunk_stride_samples : (i + 1) * chunk_stride_samples - ] - - # extract fbank feats - speech, speech_lengths = extract_fbank( - [audio_sample_i], - data_type=kwargs.get("data_type", "sound"), - frontend=frontend, - cache=cache["frontend"], - is_final=kwargs["is_final"], - ) - time3 = time.perf_counter() - meta_data["extract_feat"] = f"{time3 - time2:0.3f}" - meta_data["batch_data_time"] = ( - speech_lengths.sum().item() - * frontend.frame_shift - * frontend.lfr_n - / 1000 - ) - if len(speech) == 0: - break - tokens_i = self.generate_chunk( - speech, - speech_lengths, - key=key, - tokenizer=tokenizer, - cache=cache, - frontend=frontend, - **kwargs, - ) - tokens.extend(tokens_i) - - text_postprocessed, _ = postprocess_utils.sentence_postprocess(tokens) - - result_i = {"key": key[0], "text": text_postprocessed} - result = [result_i] - - cache["prev_samples"] = audio_sample[:-m] - if _is_final: - self.init_cache(cache, **kwargs) - - if kwargs.get("output_dir"): - if not hasattr(self, "writer"): - self.writer = DatadirWriter(kwargs.get("output_dir")) - ibest_writer = self.writer[f"{1}best_recog"] - ibest_writer["token"][key[0]] = " ".join(tokens) - ibest_writer["text"][key[0]] = text_postprocessed - - return result, meta_data - - def infer_encoder( - self, - data_in, - data_lengths=None, - key: list = None, - tokenizer=None, - frontend=None, - cache: dict = {}, - **kwargs, - ): - if len(cache) == 0: - self.init_cache(cache, **kwargs) - - meta_data = {} - chunk_size = kwargs.get("chunk_size", [0, 10, 5]) - chunk_stride_samples = int(chunk_size[1] * 960) # 600ms - - time1 = time.perf_counter() - cfg = {"is_final": kwargs.get("is_final", False)} - if isinstance(data_in[0], torch.Tensor): - audio_sample_list = data_in - else: - audio_sample_list = load_audio_text_image_video( - data_in, - fs=frontend.fs, - audio_fs=kwargs.get("fs", 16000), - data_type=kwargs.get("data_type", "sound"), - tokenizer=tokenizer, - cache=cfg, - ) - - _is_final = cfg["is_final"] # if data_in is a file or url, set is_final=True - time2 = time.perf_counter() - meta_data["load_data"] = f"{time2 - time1:0.3f}" - assert len(audio_sample_list) == 1, "batch_size must be set 1" - - audio_sample = torch.cat((cache["prev_samples"], audio_sample_list[0])) - - n = int(len(audio_sample) // chunk_stride_samples + int(_is_final)) - m = int(len(audio_sample) % chunk_stride_samples * (1 - int(_is_final))) - encoder_outs = [] - meta_data["batch_data_time"] = 0.0 - meta_data["extract_feat"] = 0.0 - for i in range(n): - kwargs["is_final"] = _is_final and i == n - 1 - audio_sample_i = audio_sample[ - i * chunk_stride_samples : (i + 1) * chunk_stride_samples - ] - time2 = time.perf_counter() - # extract fbank feats - if kwargs["is_final"] and len(audio_sample_i) == 0: - break - try: - speech, speech_lengths = extract_fbank( - [audio_sample_i], - data_type=kwargs.get("data_type", "sound"), - frontend=frontend, - cache=cache["frontend"], - is_final=kwargs["is_final"], - ) - except: - if i == n - 1 and audio_sample_i.shape[0] < 480: - print(f"Warning!!!, skip {audio_sample_i.shape[0]} samples") - break - else: - raise RuntimeError("infer failed") - time3 = time.perf_counter() - if len(speech) == 0 and kwargs["is_final"]: - break - meta_data["extract_feat"] = meta_data["extract_feat"] + time3 - time2 - meta_data["batch_data_time"] = ( - meta_data["batch_data_time"] - + speech_lengths.sum().item() - * frontend.frame_shift - * frontend.lfr_n - / 1000 - ) - speech = speech.to(device=kwargs["device"]) - speech_lengths = speech_lengths.to(device=kwargs["device"]) - encoder_out, encoder_out_lens = self.encode_chunk( - speech, - speech_lengths, - cache=cache, - is_final=kwargs.get("is_final", False), - ) - encoder_outs.append(encoder_out[:, (-speech_lengths[0]) :]) - - if i == n - 1: - break - speech_out = [] - if len(encoder_outs) > 0: - speech_out = torch.cat(encoder_outs, dim=1) - result_i = {"key": key[0], "enc_out": speech_out} - result = [result_i] - - if m > 0: # tail exists - cache["prev_samples"] = audio_sample[-m:] - else: - cache["prev_samples"] = torch.empty(0) - - if _is_final: - self.init_cache(cache, **kwargs) - - return result, meta_data, cache diff --git a/funasr_detach/models/paraformer_streaming/template.yaml b/funasr_detach/models/paraformer_streaming/template.yaml deleted file mode 100644 index d1300ac79f65d2c5d127bae70ea3e3aaa53cb83c..0000000000000000000000000000000000000000 --- a/funasr_detach/models/paraformer_streaming/template.yaml +++ /dev/null @@ -1,143 +0,0 @@ -# This is an example that demonstrates how to configure a model file. -# You can modify the configuration according to your own requirements. - -# to print the register_table: -# from funasr.register import tables -# tables.print() - -# network architecture -model: ParaformerStreaming -model_conf: - ctc_weight: 0.0 - lsm_weight: 0.1 - length_normalized_loss: true - predictor_weight: 1.0 - predictor_bias: 1 - sampling_ratio: 0.75 - -# encoder -encoder: SANMEncoderChunkOpt -encoder_conf: - output_size: 512 - attention_heads: 4 - linear_units: 2048 - num_blocks: 50 - dropout_rate: 0.1 - positional_dropout_rate: 0.1 - attention_dropout_rate: 0.1 - input_layer: pe_online - pos_enc_class: SinusoidalPositionEncoder - normalize_before: true - kernel_size: 11 - sanm_shfit: 0 - selfattention_layer_type: sanm - chunk_size: - - 12 - - 15 - stride: - - 8 - - 10 - pad_left: - - 0 - - 0 - encoder_att_look_back_factor: - - 4 - - 4 - decoder_att_look_back_factor: - - 1 - - 1 - -# decoder -decoder: ParaformerSANMDecoder -decoder_conf: - attention_heads: 4 - linear_units: 2048 - num_blocks: 16 - dropout_rate: 0.1 - positional_dropout_rate: 0.1 - self_attention_dropout_rate: 0.1 - src_attention_dropout_rate: 0.1 - att_layer_num: 16 - kernel_size: 11 - sanm_shfit: 5 - -predictor: CifPredictorV2 -predictor_conf: - idim: 512 - threshold: 1.0 - l_order: 1 - r_order: 1 - tail_threshold: 0.45 - -# frontend related -frontend: WavFrontendOnline -frontend_conf: - fs: 16000 - window: hamming - n_mels: 80 - frame_length: 25 - frame_shift: 10 - lfr_m: 7 - lfr_n: 6 - -specaug: SpecAugLFR -specaug_conf: - apply_time_warp: false - time_warp_window: 5 - time_warp_mode: bicubic - apply_freq_mask: true - freq_mask_width_range: - - 0 - - 30 - lfr_rate: 6 - num_freq_mask: 1 - apply_time_mask: true - time_mask_width_range: - - 0 - - 12 - num_time_mask: 1 - -train_conf: - accum_grad: 1 - grad_clip: 5 - max_epoch: 150 - val_scheduler_criterion: - - valid - - acc - best_model_criterion: - - - valid - - acc - - max - keep_nbest_models: 10 - log_interval: 50 - -optim: adam -optim_conf: - lr: 0.0005 -scheduler: warmuplr -scheduler_conf: - warmup_steps: 30000 - -dataset: AudioDataset -dataset_conf: - index_ds: IndexDSJsonl - batch_sampler: DynamicBatchLocalShuffleSampler - batch_type: example # example or length - batch_size: 1 # if batch_type is example, batch_size is the numbers of samples; if length, batch_size is source_token_len+target_token_len; - max_token_length: 2048 # filter samples if source_token_len+target_token_len > max_token_length, - buffer_size: 500 - shuffle: True - num_workers: 0 - -tokenizer: CharTokenizer -tokenizer_conf: - unk_symbol: - split_with_space: true - - -ctc_conf: - dropout_rate: 0.0 - ctc_type: builtin - reduce: true - ignore_nan_grad: true -normalize: null diff --git a/funasr_detach/models/rwkv_bat/__init__.py b/funasr_detach/models/rwkv_bat/__init__.py deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/funasr_detach/models/rwkv_bat/cuda_decoder/wkv_cuda.cu b/funasr_detach/models/rwkv_bat/cuda_decoder/wkv_cuda.cu deleted file mode 100644 index 1dcbc7141dc69655534214f6df642731c8c1260f..0000000000000000000000000000000000000000 --- a/funasr_detach/models/rwkv_bat/cuda_decoder/wkv_cuda.cu +++ /dev/null @@ -1,135 +0,0 @@ -// Copied from https://github.com/BlinkDL/RWKV-LM/blob/main/RWKV-v4/cuda/wkv_cuda.cu - -#include -#include - -#define MIN_VALUE (-1e38) - -template -__global__ void kernel_forward(const int B, const int T, const int C, - const F *__restrict__ const _w, const F *__restrict__ const _u, const F *__restrict__ const _k, const F *__restrict__ const _v, - F *__restrict__ const _y) { - const int idx = blockIdx.x * blockDim.x + threadIdx.x; - const int _b = idx / C; - const int _c = idx % C; - const int _offset = _b * T * C + _c; - - F u = _u[_c]; - F w = _w[_c]; - const F *__restrict__ const k = _k + _offset; - const F *__restrict__ const v = _v + _offset; - F *__restrict__ const y = _y + _offset; - - // aa and bb are running sums divided by exp(pp) (to avoid overflow) - F aa = 0, bb = 0, pp = MIN_VALUE; - for (int i = 0; i < T; i++) { - const int ii = i * C; - const F kk = k[ii]; - const F vv = v[ii]; - - F ww = u + kk; - F p = max(pp, ww); - F e1 = exp(pp - p); - F e2 = exp(ww - p); - y[ii] = (e1 * aa + e2 * vv) / (e1 * bb + e2); - - ww = w + pp; - p = max(ww, kk); - e1 = exp(ww - p); - e2 = exp(kk - p); - aa = e1 * aa + e2 * vv; - bb = e1 * bb + e2; - pp = p; - } -} - -template -__global__ void kernel_backward(const int B, const int T, const int C, - const F *__restrict__ const _w, const F *__restrict__ const _u, const F *__restrict__ const _k, const F *__restrict__ const _v, - const F *__restrict__ const _y, const F *__restrict__ const _gy, - F *__restrict__ const _gw, F *__restrict__ const _gu, F *__restrict__ const _gk, F *__restrict__ const _gv) { - const int idx = blockIdx.x * blockDim.x + threadIdx.x; - const int _b = idx / C; - const int _c = idx % C; - const int _offset = _b * T * C + _c; - - F u = _u[_c]; - F w = _w[_c]; - const F *__restrict__ const k = _k + _offset; - const F *__restrict__ const v = _v + _offset; - const F *__restrict__ const y = _y + _offset; - const F *__restrict__ const gy = _gy + _offset; - F *__restrict__ const gk = _gk + _offset; - F *__restrict__ const gv = _gv + _offset; - - F q[Tmax], r[Tmax]; - - F gw = 0, gu = 0, aa = 0, bb = 0, ga = 0, gb = 0, pp = MIN_VALUE; - for (int i = 0; i < T; i++) { - const int ii = i * C; - const F kk = k[ii]; - const F vv = v[ii]; - const F yy = y[ii]; - - F ww = u + kk; - F p = max(pp, ww); - F e1 = exp(pp - p); - F e2 = exp(ww - p); - const F qq = gy[ii] / (e1 * bb + e2); - gw += (ga - gb * yy) * e1 * qq; - gu += (vv - yy) * e2 * qq; - q[i] = qq; - r[i] = ww - p; - - ww = w + pp; - p = max(ww, kk); - e1 = exp(ww - p); - e2 = exp(kk - p); - ga = e1 * (aa + ga); - gb = e1 * (bb + gb); - aa = e1 * aa + e2 * vv; - bb = e1 * bb + e2; - pp = p; - } - const int _offsetBC = _b * C + _c; - _gw[_offsetBC] = gw * _w[_c]; // multiply by w because of w -> -exp(w) in python forward() - _gu[_offsetBC] = gu; - - aa = 0, bb = 0, pp = MIN_VALUE; - for (int i = T - 1; i >= 0; i--) { - const int ii = i * C; - const F kk = k[ii]; - const F vv = v[ii]; - const F yy = y[ii]; - const F qq = q[i]; - const F rr = r[i]; - - F e1 = qq * exp(rr); - F e2 = exp(kk + pp); - gk[ii] = e1 * (vv - yy) + e2 * (aa * vv + bb); - gv[ii] = e1 + e2 * aa; - - const F ww = w + pp; - const F www = rr - u - kk; - const F p = max(ww, www); - e1 = exp(ww - p); - e2 = qq * exp(www - p); - aa = e1 * aa + e2; - bb = e1 * bb - e2 * yy; - pp = p; - } -} - -void cuda_forward(int B, int T, int C, float *w, float *u, float *k, float *v, float *y) { - dim3 threadsPerBlock( min(C, 32) ); // requires --maxrregcount 60 for optimal performance - assert(B * C % threadsPerBlock.x == 0); - dim3 numBlocks(B * C / threadsPerBlock.x); - kernel_forward<<>>(B, T, C, w, u, k, v, y); -} - -void cuda_backward(int B, int T, int C, float *w, float *u, float *k, float *v, float *y, float *gy, float *gw, float *gu, float *gk, float *gv) { - dim3 threadsPerBlock( min(C, 32) ); // requires --maxrregcount 60 for optimal performance - assert(B * C % threadsPerBlock.x == 0); - dim3 numBlocks(B * C / threadsPerBlock.x); - kernel_backward<<>>(B, T, C, w, u, k, v, y, gy, gw, gu, gk, gv); -} diff --git a/funasr_detach/models/rwkv_bat/cuda_decoder/wkv_op.cpp b/funasr_detach/models/rwkv_bat/cuda_decoder/wkv_op.cpp deleted file mode 100644 index 1024219999ffd7ced493a63802911e6c52c0507a..0000000000000000000000000000000000000000 --- a/funasr_detach/models/rwkv_bat/cuda_decoder/wkv_op.cpp +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Bsed on https://github.com/BlinkDL/RWKV-LM/blob/main/RWKV-v4/cuda/wkv_op.cpp - Function signatures were modified based on https://github.com/huggingface/transformers/blob/main/src/transformers/kernels/rwkv/wkv_op.cpp - - */ - -#include - -void cuda_forward(int B, int T, int C, float *w, float *u, float *k, float *v, float *y); - -void cuda_backward(int B, int T, int C, float *w, float *u, float *k, float *v, float *y, float *gy, float *gw, float *gu, float *gk, float *gv); - -void forward(torch::Tensor &w, torch::Tensor &u, torch::Tensor &k, torch::Tensor &v, torch::Tensor &y) { - const int B = k.size(0); - const int T = k.size(1); - const int C = k.size(2); - - cuda_forward(B, T, C, w.data_ptr(), u.data_ptr(), k.data_ptr(), v.data_ptr(), y.data_ptr()); -} - -void backward(torch::Tensor &w, torch::Tensor &u, torch::Tensor &k, torch::Tensor &v, torch::Tensor &y, torch::Tensor &gy, torch::Tensor &gw, torch::Tensor &gu, torch::Tensor &gk, torch::Tensor &gv) { - const int B = k.size(0); - const int T = k.size(1); - const int C = k.size(2); - - cuda_backward(B, T, C, w.data_ptr(), u.data_ptr(), k.data_ptr(), v.data_ptr(), y.data_ptr(), gy.data_ptr(), gw.data_ptr(), gu.data_ptr(), gk.data_ptr(), gv.data_ptr()); -} - -PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { - m.def("forward", &forward, "wkv forward"); - m.def("backward", &backward, "wkv backward"); -} - -TORCH_LIBRARY(wkv_decoder, m) { - m.def("forward", forward); - m.def("backward", backward); -} diff --git a/funasr_detach/models/rwkv_bat/cuda_encoder/wkv_cuda.cu b/funasr_detach/models/rwkv_bat/cuda_encoder/wkv_cuda.cu deleted file mode 100644 index 1dcbc7141dc69655534214f6df642731c8c1260f..0000000000000000000000000000000000000000 --- a/funasr_detach/models/rwkv_bat/cuda_encoder/wkv_cuda.cu +++ /dev/null @@ -1,135 +0,0 @@ -// Copied from https://github.com/BlinkDL/RWKV-LM/blob/main/RWKV-v4/cuda/wkv_cuda.cu - -#include -#include - -#define MIN_VALUE (-1e38) - -template -__global__ void kernel_forward(const int B, const int T, const int C, - const F *__restrict__ const _w, const F *__restrict__ const _u, const F *__restrict__ const _k, const F *__restrict__ const _v, - F *__restrict__ const _y) { - const int idx = blockIdx.x * blockDim.x + threadIdx.x; - const int _b = idx / C; - const int _c = idx % C; - const int _offset = _b * T * C + _c; - - F u = _u[_c]; - F w = _w[_c]; - const F *__restrict__ const k = _k + _offset; - const F *__restrict__ const v = _v + _offset; - F *__restrict__ const y = _y + _offset; - - // aa and bb are running sums divided by exp(pp) (to avoid overflow) - F aa = 0, bb = 0, pp = MIN_VALUE; - for (int i = 0; i < T; i++) { - const int ii = i * C; - const F kk = k[ii]; - const F vv = v[ii]; - - F ww = u + kk; - F p = max(pp, ww); - F e1 = exp(pp - p); - F e2 = exp(ww - p); - y[ii] = (e1 * aa + e2 * vv) / (e1 * bb + e2); - - ww = w + pp; - p = max(ww, kk); - e1 = exp(ww - p); - e2 = exp(kk - p); - aa = e1 * aa + e2 * vv; - bb = e1 * bb + e2; - pp = p; - } -} - -template -__global__ void kernel_backward(const int B, const int T, const int C, - const F *__restrict__ const _w, const F *__restrict__ const _u, const F *__restrict__ const _k, const F *__restrict__ const _v, - const F *__restrict__ const _y, const F *__restrict__ const _gy, - F *__restrict__ const _gw, F *__restrict__ const _gu, F *__restrict__ const _gk, F *__restrict__ const _gv) { - const int idx = blockIdx.x * blockDim.x + threadIdx.x; - const int _b = idx / C; - const int _c = idx % C; - const int _offset = _b * T * C + _c; - - F u = _u[_c]; - F w = _w[_c]; - const F *__restrict__ const k = _k + _offset; - const F *__restrict__ const v = _v + _offset; - const F *__restrict__ const y = _y + _offset; - const F *__restrict__ const gy = _gy + _offset; - F *__restrict__ const gk = _gk + _offset; - F *__restrict__ const gv = _gv + _offset; - - F q[Tmax], r[Tmax]; - - F gw = 0, gu = 0, aa = 0, bb = 0, ga = 0, gb = 0, pp = MIN_VALUE; - for (int i = 0; i < T; i++) { - const int ii = i * C; - const F kk = k[ii]; - const F vv = v[ii]; - const F yy = y[ii]; - - F ww = u + kk; - F p = max(pp, ww); - F e1 = exp(pp - p); - F e2 = exp(ww - p); - const F qq = gy[ii] / (e1 * bb + e2); - gw += (ga - gb * yy) * e1 * qq; - gu += (vv - yy) * e2 * qq; - q[i] = qq; - r[i] = ww - p; - - ww = w + pp; - p = max(ww, kk); - e1 = exp(ww - p); - e2 = exp(kk - p); - ga = e1 * (aa + ga); - gb = e1 * (bb + gb); - aa = e1 * aa + e2 * vv; - bb = e1 * bb + e2; - pp = p; - } - const int _offsetBC = _b * C + _c; - _gw[_offsetBC] = gw * _w[_c]; // multiply by w because of w -> -exp(w) in python forward() - _gu[_offsetBC] = gu; - - aa = 0, bb = 0, pp = MIN_VALUE; - for (int i = T - 1; i >= 0; i--) { - const int ii = i * C; - const F kk = k[ii]; - const F vv = v[ii]; - const F yy = y[ii]; - const F qq = q[i]; - const F rr = r[i]; - - F e1 = qq * exp(rr); - F e2 = exp(kk + pp); - gk[ii] = e1 * (vv - yy) + e2 * (aa * vv + bb); - gv[ii] = e1 + e2 * aa; - - const F ww = w + pp; - const F www = rr - u - kk; - const F p = max(ww, www); - e1 = exp(ww - p); - e2 = qq * exp(www - p); - aa = e1 * aa + e2; - bb = e1 * bb - e2 * yy; - pp = p; - } -} - -void cuda_forward(int B, int T, int C, float *w, float *u, float *k, float *v, float *y) { - dim3 threadsPerBlock( min(C, 32) ); // requires --maxrregcount 60 for optimal performance - assert(B * C % threadsPerBlock.x == 0); - dim3 numBlocks(B * C / threadsPerBlock.x); - kernel_forward<<>>(B, T, C, w, u, k, v, y); -} - -void cuda_backward(int B, int T, int C, float *w, float *u, float *k, float *v, float *y, float *gy, float *gw, float *gu, float *gk, float *gv) { - dim3 threadsPerBlock( min(C, 32) ); // requires --maxrregcount 60 for optimal performance - assert(B * C % threadsPerBlock.x == 0); - dim3 numBlocks(B * C / threadsPerBlock.x); - kernel_backward<<>>(B, T, C, w, u, k, v, y, gy, gw, gu, gk, gv); -} diff --git a/funasr_detach/models/rwkv_bat/cuda_encoder/wkv_op.cpp b/funasr_detach/models/rwkv_bat/cuda_encoder/wkv_op.cpp deleted file mode 100644 index 16f364637eb12623c3ba9b982493794fd7f01a89..0000000000000000000000000000000000000000 --- a/funasr_detach/models/rwkv_bat/cuda_encoder/wkv_op.cpp +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Bsed on https://github.com/BlinkDL/RWKV-LM/blob/main/RWKV-v4/cuda/wkv_op.cpp - Function signatures were modified based on https://github.com/huggingface/transformers/blob/main/src/transformers/kernels/rwkv/wkv_op.cpp - - */ - -#include - -void cuda_forward(int B, int T, int C, float *w, float *u, float *k, float *v, float *y); - -void cuda_backward(int B, int T, int C, float *w, float *u, float *k, float *v, float *y, float *gy, float *gw, float *gu, float *gk, float *gv); - -void forward(torch::Tensor &w, torch::Tensor &u, torch::Tensor &k, torch::Tensor &v, torch::Tensor &y) { - const int B = k.size(0); - const int T = k.size(1); - const int C = k.size(2); - - cuda_forward(B, T, C, w.data_ptr(), u.data_ptr(), k.data_ptr(), v.data_ptr(), y.data_ptr()); -} - -void backward(torch::Tensor &w, torch::Tensor &u, torch::Tensor &k, torch::Tensor &v, torch::Tensor &y, torch::Tensor &gy, torch::Tensor &gw, torch::Tensor &gu, torch::Tensor &gk, torch::Tensor &gv) { - const int B = k.size(0); - const int T = k.size(1); - const int C = k.size(2); - - cuda_backward(B, T, C, w.data_ptr(), u.data_ptr(), k.data_ptr(), v.data_ptr(), y.data_ptr(), gy.data_ptr(), gw.data_ptr(), gu.data_ptr(), gk.data_ptr(), gv.data_ptr()); -} - -PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { - m.def("forward", &forward, "wkv forward"); - m.def("backward", &backward, "wkv backward"); -} - -TORCH_LIBRARY(wkv_encoder, m) { - m.def("forward", forward); - m.def("backward", backward); -} diff --git a/funasr_detach/models/rwkv_bat/rwkv.py b/funasr_detach/models/rwkv_bat/rwkv.py deleted file mode 100644 index c8341f9a04d6abaffc80dc1abf49bc998e75f11b..0000000000000000000000000000000000000000 --- a/funasr_detach/models/rwkv_bat/rwkv.py +++ /dev/null @@ -1,152 +0,0 @@ -#!/usr/bin/env python3 -# -*- encoding: utf-8 -*- -# Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved. -# MIT License (https://opensource.org/licenses/MIT) - -import torch -from typing import Dict, Optional, Tuple - -from funasr_detach.models.transformer.layer_norm import LayerNorm -from funasr_detach.models.rwkv_bat.rwkv_feed_forward import FeedForward -from funasr_detach.models.rwkv_bat.rwkv_attention import ( - EncoderSelfAttention, - DecoderSelfAttention, -) - - -class RWKV(torch.nn.Module): - """RWKV module. - - Args: - size: Input/Output size. - linear_size: Feed-forward hidden size. - attention_size: SelfAttention hidden size. - context_size: Context size for WKV computation. - block_id: Block index. - num_blocks: Number of blocks in the architecture. - normalization_class: Normalization layer class. - normalization_args: Normalization layer arguments. - att_dropout_rate: Dropout rate for the attention module. - ffn_dropout_rate: Dropout rate for the feed-forward module. - - """ - - def __init__( - self, - size: int, - linear_size: int, - attention_size: int, - context_size: int, - block_id: int, - num_blocks: int, - att_dropout_rate: float = 0.0, - ffn_dropout_rate: float = 0.0, - dropout_rate: float = 0.0, - ) -> None: - """Construct a RWKV object.""" - super().__init__() - - self.layer_norm_att = LayerNorm(size) - self.layer_norm_ffn = LayerNorm(size) - - self.att = EncoderSelfAttention( - size, attention_size, context_size, block_id, att_dropout_rate, num_blocks - ) - self.dropout_att = torch.nn.Dropout(p=dropout_rate) - - self.ffn = FeedForward( - size, linear_size, block_id, ffn_dropout_rate, num_blocks - ) - self.dropout_ffn = torch.nn.Dropout(p=dropout_rate) - - def forward( - self, - x: torch.Tensor, - state: Optional[torch.Tensor] = None, - ) -> Tuple[torch.Tensor, Optional[torch.Tensor]]: - """Compute receptance weighted key value. - - Args: - x: RWKV input sequences. (B, L, size) - state: Decoder hidden states. [5 x (B, D_att/size, N)] - - Returns: - x: RWKV output sequences. (B, L, size) - x: Decoder hidden states. [5 x (B, D_att/size, N)] - - """ - att, state = self.att(self.layer_norm_att(x), state=state) - x = x + self.dropout_att(att) - ffn, state = self.ffn(self.layer_norm_ffn(x), state=state) - x = x + self.dropout_ffn(ffn) - return x, state - - -class RWKVDecoderLayer(torch.nn.Module): - """RWKV module. - - Args: - size: Input/Output size. - linear_size: Feed-forward hidden size. - attention_size: SelfAttention hidden size. - context_size: Context size for WKV computation. - block_id: Block index. - num_blocks: Number of blocks in the architecture. - normalization_class: Normalization layer class. - normalization_args: Normalization layer arguments. - att_dropout_rate: Dropout rate for the attention module. - ffn_dropout_rate: Dropout rate for the feed-forward module. - - """ - - def __init__( - self, - size: int, - linear_size: int, - attention_size: int, - context_size: int, - block_id: int, - num_blocks: int, - att_dropout_rate: float = 0.0, - ffn_dropout_rate: float = 0.0, - dropout_rate: float = 0.0, - ) -> None: - """Construct a RWKV object.""" - super().__init__() - - self.layer_norm_att = LayerNorm(size) - self.layer_norm_ffn = LayerNorm(size) - - self.att = DecoderSelfAttention( - size, attention_size, context_size, block_id, att_dropout_rate, num_blocks - ) - self.dropout_att = torch.nn.Dropout(p=dropout_rate) - - self.ffn = FeedForward( - size, linear_size, block_id, ffn_dropout_rate, num_blocks - ) - self.dropout_ffn = torch.nn.Dropout(p=dropout_rate) - - def forward( - self, - x: torch.Tensor, - state: Optional[torch.Tensor] = None, - ) -> Tuple[torch.Tensor, Optional[torch.Tensor]]: - """Compute receptance weighted key value. - - Args: - x: RWKV input sequences. (B, L, size) - state: Decoder hidden states. [5 x (B, D_att/size, N)] - - Returns: - x: RWKV output sequences. (B, L, size) - x: Decoder hidden states. [5 x (B, D_att/size, N)] - - """ - att, state = self.att(self.layer_norm_att(x), state=state) - x = x + self.dropout_att(att) - - ffn, state = self.ffn(self.layer_norm_ffn(x), state=state) - x = x + self.dropout_ffn(ffn) - - return x, state diff --git a/funasr_detach/models/rwkv_bat/rwkv_attention.py b/funasr_detach/models/rwkv_bat/rwkv_attention.py deleted file mode 100644 index c4b0343c7eb820b2fb56c94322f9f953b1d8d278..0000000000000000000000000000000000000000 --- a/funasr_detach/models/rwkv_bat/rwkv_attention.py +++ /dev/null @@ -1,632 +0,0 @@ -#!/usr/bin/env python3 -# -*- encoding: utf-8 -*- -# Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved. -# MIT License (https://opensource.org/licenses/MIT) - -import math -import torch -from pathlib import Path -from importlib.util import find_spec -from typing import List, Optional, Tuple, Union - - -wkv_kernel_encoder = None -wkv_kernel_decoder = None - - -class WKVLinearAttentionEncoder(torch.autograd.Function): - """WKVLinearAttention function definition.""" - - @staticmethod - def forward( - ctx, - time_decay: torch.Tensor, - time_first: torch.Tensor, - key: torch.Tensor, - value: torch.tensor, - ) -> torch.Tensor: - """WKVLinearAttention function forward pass. - - Args: - time_decay: Channel-wise time decay vector. (D_att) - time_first: Channel-wise time first vector. (D_att) - key: Key tensor. (B, U, D_att) - value: Value tensor. (B, U, D_att) - - Returns: - out: Weighted Key-Value tensor. (B, U, D_att) - - """ - batch, length, dim = key.size() - - assert length <= wkv_kernel_encoder.context_size, ( - f"Cannot process key of length {length} while context_size " - f"is ({wkv_kernel_encoder.context_size}). Limit should be increased." - ) - - assert batch * dim % min(dim, 32) == 0, ( - f"batch size ({batch}) by dimension ({dim}) should be a multiple of " - f"{min(dim, 32)}" - ) - - ctx.input_dtype = key.dtype - - time_decay = -torch.exp(time_decay.float().contiguous()) - time_first = time_first.float().contiguous() - - key = key.float().contiguous() - value = value.float().contiguous() - - out = torch.empty_like(key, memory_format=torch.contiguous_format) - - wkv_kernel_encoder.forward(time_decay, time_first, key, value, out) - ctx.save_for_backward(time_decay, time_first, key, value, out) - - return out - - @staticmethod - def backward( - ctx, grad_output: torch.Tensor - ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: - """WKVLinearAttention function backward pass. - - Args: - grad_output: Output gradient. (B, U, D_att) - - Returns: - grad_time_decay: Gradient for channel-wise time decay vector. (D_att) - grad_time_first: Gradient for channel-wise time first vector. (D_att) - grad_key: Gradient for key tensor. (B, U, D_att) - grad_value: Gradient for value tensor. (B, U, D_att) - - """ - time_decay, time_first, key, value, output = ctx.saved_tensors - grad_dtype = ctx.input_dtype - - batch, _, dim = key.size() - - grad_time_decay = torch.empty( - (batch, dim), - memory_format=torch.contiguous_format, - dtype=time_decay.dtype, - device=time_decay.device, - ) - - grad_time_first = torch.empty( - (batch, dim), - memory_format=torch.contiguous_format, - dtype=time_decay.dtype, - device=time_decay.device, - ) - - grad_key = torch.empty_like(key, memory_format=torch.contiguous_format) - grad_value = torch.empty_like(value, memory_format=torch.contiguous_format) - - wkv_kernel_encoder.backward( - time_decay, - time_first, - key, - value, - output, - grad_output.contiguous(), - grad_time_decay, - grad_time_first, - grad_key, - grad_value, - ) - - grad_time_decay = torch.sum(grad_time_decay, dim=0) - grad_time_first = torch.sum(grad_time_first, dim=0) - - return ( - grad_time_decay, - grad_time_first, - grad_key, - grad_value, - ) - - -class WKVLinearAttentionDecoder(torch.autograd.Function): - """WKVLinearAttention function definition.""" - - @staticmethod - def forward( - ctx, - time_decay: torch.Tensor, - time_first: torch.Tensor, - key: torch.Tensor, - value: torch.tensor, - ) -> torch.Tensor: - """WKVLinearAttention function forward pass. - - Args: - time_decay: Channel-wise time decay vector. (D_att) - time_first: Channel-wise time first vector. (D_att) - key: Key tensor. (B, U, D_att) - value: Value tensor. (B, U, D_att) - - Returns: - out: Weighted Key-Value tensor. (B, U, D_att) - - """ - batch, length, dim = key.size() - - assert length <= wkv_kernel_decoder.context_size, ( - f"Cannot process key of length {length} while context_size " - f"is ({wkv_kernel.context_size}). Limit should be increased." - ) - - assert batch * dim % min(dim, 32) == 0, ( - f"batch size ({batch}) by dimension ({dim}) should be a multiple of " - f"{min(dim, 32)}" - ) - - ctx.input_dtype = key.dtype - - time_decay = -torch.exp(time_decay.float().contiguous()) - time_first = time_first.float().contiguous() - - key = key.float().contiguous() - value = value.float().contiguous() - - out = torch.empty_like(key, memory_format=torch.contiguous_format) - - wkv_kernel_decoder.forward(time_decay, time_first, key, value, out) - ctx.save_for_backward(time_decay, time_first, key, value, out) - - return out - - @staticmethod - def backward( - ctx, grad_output: torch.Tensor - ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: - """WKVLinearAttention function backward pass. - - Args: - grad_output: Output gradient. (B, U, D_att) - - Returns: - grad_time_decay: Gradient for channel-wise time decay vector. (D_att) - grad_time_first: Gradient for channel-wise time first vector. (D_att) - grad_key: Gradient for key tensor. (B, U, D_att) - grad_value: Gradient for value tensor. (B, U, D_att) - - """ - time_decay, time_first, key, value, output = ctx.saved_tensors - grad_dtype = ctx.input_dtype - - batch, _, dim = key.size() - - grad_time_decay = torch.empty( - (batch, dim), - memory_format=torch.contiguous_format, - dtype=time_decay.dtype, - device=time_decay.device, - ) - - grad_time_first = torch.empty( - (batch, dim), - memory_format=torch.contiguous_format, - dtype=time_decay.dtype, - device=time_decay.device, - ) - - grad_key = torch.empty_like(key, memory_format=torch.contiguous_format) - grad_value = torch.empty_like(value, memory_format=torch.contiguous_format) - - wkv_kernel_decoder.backward( - time_decay, - time_first, - key, - value, - output, - grad_output.contiguous(), - grad_time_decay, - grad_time_first, - grad_key, - grad_value, - ) - - grad_time_decay = torch.sum(grad_time_decay, dim=0) - grad_time_first = torch.sum(grad_time_first, dim=0) - - return ( - grad_time_decay, - grad_time_first, - grad_key, - grad_value, - ) - - -def load_encoder_wkv_kernel(context_size: int) -> None: - """Load WKV CUDA kernel. - - Args: - context_size: Context size. - - """ - from torch.utils.cpp_extension import load - - global wkv_kernel_encoder - - if ( - wkv_kernel_encoder is not None - and wkv_kernel_encoder.context_size == context_size - ): - return - - if find_spec("ninja") is None: - raise ImportError( - "Ninja package was not found. WKV kernel module can't be loaded " - "for training. Please, 'pip install ninja' in your environment." - ) - - if not torch.cuda.is_available(): - raise ImportError( - "CUDA is currently a requirement for WKV kernel loading. " - "Please set your devices properly and launch again." - ) - - kernel_folder = Path(__file__).resolve().parent / "cuda_encoder" - kernel_files = [kernel_folder / f for f in ["wkv_op.cpp", "wkv_cuda.cu"]] - - kernel_cflags = [ - "-res-usage", - "--maxrregcount 60", - "--use_fast_math", - "-O3", - "-Xptxas -O3", - f"-DTmax={context_size}", - ] - wkv_kernel_encoder = load( - name=f"encoder_wkv_{context_size}", - sources=kernel_files, - verbose=True, - extra_cuda_cflags=kernel_cflags, - ) - wkv_kernel_encoder.context_size = context_size - - -def load_decoder_wkv_kernel(context_size: int) -> None: - """Load WKV CUDA kernel. - - Args: - context_size: Context size. - - """ - from torch.utils.cpp_extension import load - - global wkv_kernel_decoder - - if ( - wkv_kernel_decoder is not None - and wkv_kernel_decoder.context_size == context_size - ): - return - - if find_spec("ninja") is None: - raise ImportError( - "Ninja package was not found. WKV kernel module can't be loaded " - "for training. Please, 'pip install ninja' in your environment." - ) - - if not torch.cuda.is_available(): - raise ImportError( - "CUDA is currently a requirement for WKV kernel loading. " - "Please set your devices properly and launch again." - ) - - kernel_folder = Path(__file__).resolve().parent / "cuda_decoder" - kernel_files = [kernel_folder / f for f in ["wkv_op.cpp", "wkv_cuda.cu"]] - - kernel_cflags = [ - "-res-usage", - "--maxrregcount 60", - "--use_fast_math", - "-O3", - "-Xptxas -O3", - f"-DTmax={context_size}", - ] - wkv_kernel_decoder = load( - name=f"decoder_wkv_{context_size}", - sources=kernel_files, - verbose=True, - extra_cuda_cflags=kernel_cflags, - ) - wkv_kernel_decoder.context_size = context_size - - -class SelfAttention(torch.nn.Module): - """SelfAttention module definition. - - Args: - size: Input/Output size. - attention_size: Attention hidden size. - context_size: Context size for WKV kernel. - block_id: Block index. - num_blocks: Number of blocks in the architecture. - - """ - - def __init__( - self, - size: int, - attention_size: int, - block_id: int, - dropout_rate: float, - num_blocks: int, - ) -> None: - """Construct a SelfAttention object.""" - super().__init__() - self.time_shift = torch.nn.ZeroPad2d((0, 0, 1, -1)) - - self.time_decay = torch.nn.Parameter(torch.empty(attention_size)) - self.time_first = torch.nn.Parameter(torch.empty(attention_size)) - - self.time_mix_key = torch.nn.Parameter(torch.empty(1, 1, size)) - self.time_mix_value = torch.nn.Parameter(torch.empty(1, 1, size)) - self.time_mix_receptance = torch.nn.Parameter(torch.empty(1, 1, size)) - - self.proj_key = torch.nn.Linear(size, attention_size, bias=True) - self.proj_value = torch.nn.Linear(size, attention_size, bias=True) - self.proj_receptance = torch.nn.Linear(size, attention_size, bias=True) - - self.proj_output = torch.nn.Linear(attention_size, size, bias=True) - - self.block_id = block_id - - self.reset_parameters(size, attention_size, block_id, num_blocks) - self.dropout = torch.nn.Dropout(p=dropout_rate) - - def reset_parameters( - self, size: int, attention_size: int, block_id: int, num_blocks: int - ) -> None: - """Reset module parameters. - - Args: - size: Block size. - attention_size: Attention hidden size. - block_id: Block index. - num_blocks: Number of blocks in the architecture. - - """ - ratio_0_to_1 = block_id / (num_blocks - 1) - ratio_1_to_almost0 = 1.0 - (block_id / num_blocks) - - time_weight = torch.ones(1, 1, size) - - for i in range(size): - time_weight[0, 0, i] = i / size - - decay_speed = [ - -5 + 8 * (h / (attention_size - 1)) ** (0.7 + 1.3 * ratio_0_to_1) - for h in range(attention_size) - ] - decay_speed = torch.tensor( - decay_speed, dtype=self.time_decay.dtype, device=self.time_decay.device - ) - - zigzag = ( - torch.tensor( - [(i + 1) % 3 - 1 for i in range(attention_size)], - dtype=self.time_first.dtype, - device=self.time_first.device, - ) - * 0.5 - ) - - with torch.no_grad(): - self.time_decay.data = decay_speed - self.time_first.data = torch.ones_like( - self.time_first * math.log(0.3) + zigzag - ) - - self.time_mix_key.data = torch.pow(time_weight, ratio_1_to_almost0) - self.time_mix_value.data = ( - torch.pow(time_weight, ratio_1_to_almost0) + 0.3 * ratio_0_to_1 - ) - self.time_mix_receptance.data = torch.pow( - time_weight, 0.5 * ratio_1_to_almost0 - ) - - @torch.no_grad() - def wkv_linear_attention( - self, - time_decay: torch.Tensor, - time_first: torch.Tensor, - key: torch.Tensor, - value: torch.Tensor, - state: Tuple[torch.Tensor, torch.Tensor, torch.Tensor], - ) -> Tuple[torch.Tensor, Tuple[torch.Tensor, torch.Tensor, torch.Tensor]]: - """Compute WKV with state (i.e.: for inference). - - Args: - time_decay: Channel-wise time decay vector. (D_att) - time_first: Channel-wise time first vector. (D_att) - key: Key tensor. (B, 1, D_att) - value: Value tensor. (B, 1, D_att) - state: Decoder hidden states. [3 x (B, D_att)] - - Returns: - output: Weighted Key-Value. (B, 1, D_att) - state: Decoder hidden states. [3 x (B, 1, D_att)] - - """ - num_state, den_state, max_state = state - time_decay = -torch.exp(time_decay) - max_for_output = torch.maximum(max_state, (time_first + key)) - - e1 = torch.exp(max_state - max_for_output) - e2 = torch.exp((time_first + key) - max_for_output) - - numerator = e1 * num_state + e2 * value - denominator = e1 * den_state + e2 - - max_for_state = torch.maximum(key, (max_state + time_decay)) - - e1 = torch.exp((max_state + time_decay) - max_for_state) - e2 = torch.exp(key - max_for_state) - - wkv = numerator / denominator - - state = [e1 * num_state + e2 * value, e1 * den_state + e2, max_for_state] - - return wkv, state - - -class DecoderSelfAttention(SelfAttention): - """SelfAttention module definition. - - Args: - size: Input/Output size. - attention_size: Attention hidden size. - context_size: Context size for WKV kernel. - block_id: Block index. - num_blocks: Number of blocks in the architecture. - - """ - - def __init__( - self, - size: int, - attention_size: int, - context_size: int, - block_id: int, - dropout_rate: float, - num_blocks: int, - ) -> None: - """Construct a SelfAttention object.""" - super().__init__(size, attention_size, block_id, dropout_rate, num_blocks) - # load_decoder_wkv_kernel(context_size) - - def forward( - self, - x: torch.Tensor, - state: Optional[List[torch.Tensor]] = None, - ) -> Tuple[torch.Tensor, Optional[List[torch.Tensor]]]: - """Compute time mixing. - - Args: - x: SelfAttention input sequences. (B, U, size) - state: Decoder hidden states. [5 x (B, 1, D_att, N)] - - Returns: - x: SelfAttention output sequences. (B, U, size) - - """ - shifted_x = ( - self.time_shift(x) if state is None else state[1][..., self.block_id] - ) - - key = x * self.time_mix_key + shifted_x * (1 - self.time_mix_key) - value = x * self.time_mix_value + shifted_x * (1 - self.time_mix_value) - receptance = x * self.time_mix_receptance + shifted_x * ( - 1 - self.time_mix_receptance - ) - - key = self.proj_key(key) - value = self.proj_value(value) - receptance = torch.sigmoid(self.proj_receptance(receptance)) - - if state is not None: - state[1][..., self.block_id] = x - - wkv, att_state = self.wkv_linear_attention( - self.time_decay, - self.time_first, - key, - value, - tuple(s[..., self.block_id] for s in state[2:]), - ) - - state[2][..., self.block_id] = att_state[0] - state[3][..., self.block_id] = att_state[1] - state[4][..., self.block_id] = att_state[2] - else: - wkv = WKVLinearAttentionDecoder.apply( - self.time_decay, self.time_first, key, value - ) - - wkv = self.dropout(wkv) - x = self.proj_output(receptance * wkv) - - return x, state - - -class EncoderSelfAttention(SelfAttention): - """SelfAttention module definition. - - Args: - size: Input/Output size. - attention_size: Attention hidden size. - context_size: Context size for WKV kernel. - block_id: Block index. - num_blocks: Number of blocks in the architecture. - - """ - - def __init__( - self, - size: int, - attention_size: int, - context_size: int, - block_id: int, - dropout_rate: float, - num_blocks: int, - ) -> None: - """Construct a SelfAttention object.""" - super().__init__(size, attention_size, block_id, dropout_rate, num_blocks) - # load_encoder_wkv_kernel(context_size) - - def forward( - self, - x: torch.Tensor, - state: Optional[List[torch.Tensor]] = None, - ) -> Tuple[torch.Tensor, Optional[List[torch.Tensor]]]: - """Compute time mixing. - - Args: - x: SelfAttention input sequences. (B, U, size) - state: Decoder hidden states. [5 x (B, 1, D_att, N)] - - Returns: - x: SelfAttention output sequences. (B, U, size) - - """ - shifted_x = ( - self.time_shift(x) if state is None else state[1][..., self.block_id] - ) - - key = x * self.time_mix_key + shifted_x * (1 - self.time_mix_key) - value = x * self.time_mix_value + shifted_x * (1 - self.time_mix_value) - receptance = x * self.time_mix_receptance + shifted_x * ( - 1 - self.time_mix_receptance - ) - - key = self.proj_key(key) - value = self.proj_value(value) - receptance = torch.sigmoid(self.proj_receptance(receptance)) - - if state is not None: - state[1][..., self.block_id] = x - - wkv, att_state = self.wkv_linear_attention( - self.time_decay, - self.time_first, - key, - value, - tuple(s[..., self.block_id] for s in state[2:]), - ) - - state[2][..., self.block_id] = att_state[0] - state[3][..., self.block_id] = att_state[1] - state[4][..., self.block_id] = att_state[2] - else: - wkv = WKVLinearAttentionEncoder.apply( - self.time_decay, self.time_first, key, value - ) - - wkv = self.dropout(wkv) - x = self.proj_output(receptance * wkv) - - return x, state diff --git a/funasr_detach/models/rwkv_bat/rwkv_encoder.py b/funasr_detach/models/rwkv_bat/rwkv_encoder.py deleted file mode 100644 index 41df895d9a22009362b1cfd90e0c1e31c6eda862..0000000000000000000000000000000000000000 --- a/funasr_detach/models/rwkv_bat/rwkv_encoder.py +++ /dev/null @@ -1,158 +0,0 @@ -#!/usr/bin/env python3 -# -*- encoding: utf-8 -*- -# Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved. -# MIT License (https://opensource.org/licenses/MIT) - -import torch -from typing import Dict, List, Optional, Tuple - -from funasr_detach.register import tables -from funasr_detach.models.rwkv_bat.rwkv import RWKV -from funasr_detach.models.transformer.layer_norm import LayerNorm -from funasr_detach.models.transformer.utils.nets_utils import make_source_mask -from funasr_detach.models.rwkv_bat.rwkv_subsampling import RWKVConvInput - - -@tables.register("encoder_classes", "RWKVEncoder") -class RWKVEncoder(torch.nn.Module): - """RWKV encoder module. - - Based on https://arxiv.org/pdf/2305.13048.pdf. - - Args: - vocab_size: Vocabulary size. - output_size: Input/Output size. - context_size: Context size for WKV computation. - linear_size: FeedForward hidden size. - attention_size: SelfAttention hidden size. - normalization_type: Normalization layer type. - normalization_args: Normalization layer arguments. - num_blocks: Number of RWKV blocks. - embed_dropout_rate: Dropout rate for embedding layer. - att_dropout_rate: Dropout rate for the attention module. - ffn_dropout_rate: Dropout rate for the feed-forward module. - """ - - def __init__( - self, - input_size: int, - output_size: int = 512, - context_size: int = 1024, - linear_size: Optional[int] = None, - attention_size: Optional[int] = None, - num_blocks: int = 4, - att_dropout_rate: float = 0.0, - ffn_dropout_rate: float = 0.0, - dropout_rate: float = 0.0, - subsampling_factor: int = 4, - time_reduction_factor: int = 1, - kernel: int = 3, - **kwargs, - ) -> None: - """Construct a RWKVEncoder object.""" - super().__init__() - - self.embed = RWKVConvInput( - input_size, - [output_size // 4, output_size // 2, output_size], - subsampling_factor, - conv_kernel_size=kernel, - output_size=output_size, - ) - - self.subsampling_factor = subsampling_factor - - linear_size = output_size * 4 if linear_size is None else linear_size - attention_size = output_size if attention_size is None else attention_size - - self.rwkv_blocks = torch.nn.ModuleList( - [ - RWKV( - output_size, - linear_size, - attention_size, - context_size, - block_id, - num_blocks, - att_dropout_rate=att_dropout_rate, - ffn_dropout_rate=ffn_dropout_rate, - dropout_rate=dropout_rate, - ) - for block_id in range(num_blocks) - ] - ) - - self.embed_norm = LayerNorm(output_size) - self.final_norm = LayerNorm(output_size) - - self._output_size = output_size - self.context_size = context_size - - self.num_blocks = num_blocks - self.time_reduction_factor = time_reduction_factor - - def output_size(self) -> int: - return self._output_size - - def forward(self, x: torch.Tensor, x_len) -> torch.Tensor: - """Encode source label sequences. - - Args: - x: Encoder input sequences. (B, L) - - Returns: - out: Encoder output sequences. (B, U, D) - - """ - _, length, _ = x.size() - - assert ( - length <= self.context_size * self.subsampling_factor - ), "Context size is too short for current length: %d versus %d" % ( - length, - self.context_size * self.subsampling_factor, - ) - mask = make_source_mask(x_len).to(x.device) - x, mask = self.embed(x, mask, None) - x = self.embed_norm(x) - olens = mask.eq(0).sum(1) - - if self.training: - for block in self.rwkv_blocks: - x, _ = block(x) - else: - x = self.rwkv_infer(x) - - x = self.final_norm(x) - - if self.time_reduction_factor > 1: - x = x[:, :: self.time_reduction_factor, :] - olens = torch.floor_divide(olens - 1, self.time_reduction_factor) + 1 - - return x, olens, None - - def rwkv_infer(self, xs_pad): - - batch_size = xs_pad.shape[0] - - hidden_sizes = [self._output_size for i in range(5)] - - state = [ - torch.zeros( - (batch_size, 1, hidden_sizes[i], self.num_blocks), - dtype=torch.float32, - device=xs_pad.device, - ) - for i in range(5) - ] - - state[4] -= 1e-30 - - xs_out = [] - for t in range(xs_pad.shape[1]): - x_t = xs_pad[:, t, :] - for idx, block in enumerate(self.rwkv_blocks): - x_t, state = block(x_t, state=state) - xs_out.append(x_t) - xs_out = torch.cat(xs_out, dim=1) - return xs_out diff --git a/funasr_detach/models/rwkv_bat/rwkv_feed_forward.py b/funasr_detach/models/rwkv_bat/rwkv_feed_forward.py deleted file mode 100644 index 927a3cb4f9cf6c2a9c8762949fbb98f5df8aa67b..0000000000000000000000000000000000000000 --- a/funasr_detach/models/rwkv_bat/rwkv_feed_forward.py +++ /dev/null @@ -1,98 +0,0 @@ -#!/usr/bin/env python3 -# -*- encoding: utf-8 -*- -# Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved. -# MIT License (https://opensource.org/licenses/MIT) - -import torch -from typing import List, Optional, Tuple - - -class FeedForward(torch.nn.Module): - """FeedForward module definition. - - Args: - size: Input/Output size. - hidden_size: Hidden size. - block_id: Block index. - num_blocks: Number of blocks in the architecture. - - """ - - def __init__( - self, - size: int, - hidden_size: int, - block_id: int, - dropout_rate: float, - num_blocks: int, - ) -> None: - """Construct a FeedForward object.""" - super().__init__() - - self.time_shift = torch.nn.ZeroPad2d((0, 0, 1, -1)) - - self.time_mix_key = torch.nn.Parameter(torch.empty(1, 1, size)) - self.time_mix_receptance = torch.nn.Parameter(torch.empty(1, 1, size)) - - self.proj_key = torch.nn.Linear(size, hidden_size, bias=True) - self.proj_value = torch.nn.Linear(hidden_size, size, bias=True) - self.proj_receptance = torch.nn.Linear(size, size, bias=True) - - self.block_id = block_id - - self.reset_parameters(size, block_id, num_blocks) - self.dropout = torch.nn.Dropout(p=dropout_rate) - - def reset_parameters(self, size: int, block_id: int, num_blocks: int) -> None: - """Reset module parameters. - - Args: - size: Block size. - block_id: Block index. - num_blocks: Number of blocks in the architecture. - - """ - ratio_1_to_almost0 = 1.0 - (block_id / num_blocks) - - time_weight = torch.ones(1, 1, size) - - for i in range(size): - time_weight[0, 0, i] = i / size - - with torch.no_grad(): - self.time_mix_key.data = torch.pow(time_weight, ratio_1_to_almost0) - self.time_mix_receptance.data = torch.pow(time_weight, ratio_1_to_almost0) - - def forward( - self, x: torch.Tensor, state: Optional[List[torch.Tensor]] = None - ) -> Tuple[torch.Tensor, Optional[List[torch.Tensor]]]: - """Compute channel mixing. - - Args: - x: FeedForward input sequences. (B, U, size) - state: Decoder hidden state. [5 x (B, 1, size, N)] - - Returns: - x: FeedForward output sequences. (B, U, size) - state: Decoder hidden state. [5 x (B, 1, size, N)] - - """ - shifted_x = ( - self.time_shift(x) if state is None else state[0][..., self.block_id] - ) - - key = x * self.time_mix_key + shifted_x * (1 - self.time_mix_key) - receptance = x * self.time_mix_receptance + shifted_x * ( - 1 - self.time_mix_receptance - ) - - key = torch.square(torch.relu(self.proj_key(key))) - value = self.proj_value(self.dropout(key)) - receptance = torch.sigmoid(self.proj_receptance(receptance)) - - if state is not None: - state[0][..., self.block_id] = x - - x = receptance * value - - return x, state diff --git a/funasr_detach/models/rwkv_bat/rwkv_subsampling.py b/funasr_detach/models/rwkv_bat/rwkv_subsampling.py deleted file mode 100644 index 205f8ad8998c7aaff2228a07ff250598ac0718c1..0000000000000000000000000000000000000000 --- a/funasr_detach/models/rwkv_bat/rwkv_subsampling.py +++ /dev/null @@ -1,261 +0,0 @@ -#!/usr/bin/env python3 -# -*- encoding: utf-8 -*- -# Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved. -# MIT License (https://opensource.org/licenses/MIT) - -import math -import torch -from typing import Optional, Tuple, Union -from funasr_detach.models.transformer.utils.nets_utils import pad_to_len - - -class TooShortUttError(Exception): - """Raised when the utt is too short for subsampling. - - Args: - message (str): Message for error catch - actual_size (int): the short size that cannot pass the subsampling - limit (int): the limit size for subsampling - - """ - - def __init__(self, message, actual_size, limit): - """Construct a TooShortUttError for error handler.""" - super().__init__(message) - self.actual_size = actual_size - self.limit = limit - - -def check_short_utt(ins, size): - """Check if the utterance is too short for subsampling.""" - if isinstance(ins, Conv2dSubsampling2) and size < 3: - return True, 3 - if isinstance(ins, Conv2dSubsampling) and size < 7: - return True, 7 - if isinstance(ins, Conv2dSubsampling6) and size < 11: - return True, 11 - if isinstance(ins, Conv2dSubsampling8) and size < 15: - return True, 15 - return False, -1 - - -class RWKVConvInput(torch.nn.Module): - """Streaming ConvInput module definition. - Args: - input_size: Input size. - conv_size: Convolution size. - subsampling_factor: Subsampling factor. - output_size: Block output dimension. - """ - - def __init__( - self, - input_size: int, - conv_size: Union[int, Tuple], - subsampling_factor: int = 4, - conv_kernel_size: int = 3, - output_size: Optional[int] = None, - ) -> None: - """Construct a ConvInput object.""" - super().__init__() - if subsampling_factor == 1: - conv_size1, conv_size2, conv_size3 = conv_size - - self.conv = torch.nn.Sequential( - torch.nn.Conv2d( - 1, - conv_size1, - conv_kernel_size, - stride=1, - padding=(conv_kernel_size - 1) // 2, - ), - torch.nn.ReLU(), - torch.nn.Conv2d( - conv_size1, - conv_size1, - conv_kernel_size, - stride=[1, 2], - padding=(conv_kernel_size - 1) // 2, - ), - torch.nn.ReLU(), - torch.nn.Conv2d( - conv_size1, - conv_size2, - conv_kernel_size, - stride=1, - padding=(conv_kernel_size - 1) // 2, - ), - torch.nn.ReLU(), - torch.nn.Conv2d( - conv_size2, - conv_size2, - conv_kernel_size, - stride=[1, 2], - padding=(conv_kernel_size - 1) // 2, - ), - torch.nn.ReLU(), - torch.nn.Conv2d( - conv_size2, - conv_size3, - conv_kernel_size, - stride=1, - padding=(conv_kernel_size - 1) // 2, - ), - torch.nn.ReLU(), - torch.nn.Conv2d( - conv_size3, - conv_size3, - conv_kernel_size, - stride=[1, 2], - padding=(conv_kernel_size - 1) // 2, - ), - torch.nn.ReLU(), - ) - - output_proj = conv_size3 * ((input_size // 2) // 2) - - self.subsampling_factor = 1 - - self.stride_1 = 1 - - self.create_new_mask = self.create_new_vgg_mask - - else: - conv_size1, conv_size2, conv_size3 = conv_size - - kernel_1 = int(subsampling_factor / 2) - - self.conv = torch.nn.Sequential( - torch.nn.Conv2d( - 1, - conv_size1, - conv_kernel_size, - stride=1, - padding=(conv_kernel_size - 1) // 2, - ), - torch.nn.ReLU(), - torch.nn.Conv2d( - conv_size1, - conv_size1, - conv_kernel_size, - stride=[kernel_1, 2], - padding=(conv_kernel_size - 1) // 2, - ), - torch.nn.ReLU(), - torch.nn.Conv2d( - conv_size1, - conv_size2, - conv_kernel_size, - stride=1, - padding=(conv_kernel_size - 1) // 2, - ), - torch.nn.ReLU(), - torch.nn.Conv2d( - conv_size2, - conv_size2, - conv_kernel_size, - stride=[2, 2], - padding=(conv_kernel_size - 1) // 2, - ), - torch.nn.ReLU(), - torch.nn.Conv2d( - conv_size2, - conv_size3, - conv_kernel_size, - stride=1, - padding=(conv_kernel_size - 1) // 2, - ), - torch.nn.ReLU(), - torch.nn.Conv2d( - conv_size3, - conv_size3, - conv_kernel_size, - stride=1, - padding=(conv_kernel_size - 1) // 2, - ), - torch.nn.ReLU(), - ) - - output_proj = conv_size3 * ((input_size // 2) // 2) - - self.subsampling_factor = subsampling_factor - - self.create_new_mask = self.create_new_vgg_mask - - self.stride_1 = kernel_1 - - self.min_frame_length = 7 - - if output_size is not None: - self.output = torch.nn.Linear(output_proj, output_size) - self.output_size = output_size - else: - self.output = None - self.output_size = output_proj - - def forward( - self, - x: torch.Tensor, - mask: Optional[torch.Tensor], - chunk_size: Optional[torch.Tensor], - ) -> Tuple[torch.Tensor, torch.Tensor]: - """Encode input sequences. - Args: - x: ConvInput input sequences. (B, T, D_feats) - mask: Mask of input sequences. (B, 1, T) - Returns: - x: ConvInput output sequences. (B, sub(T), D_out) - mask: Mask of output sequences. (B, 1, sub(T)) - """ - if mask is not None: - mask = self.create_new_mask(mask) - olens = max(mask.eq(0).sum(1)) - - b, t, f = x.size() - x = x.unsqueeze(1) # (b. 1. t. f) - - if chunk_size is not None: - max_input_length = int( - chunk_size - * self.subsampling_factor - * (math.ceil(float(t) / (chunk_size * self.subsampling_factor))) - ) - x = map(lambda inputs: pad_to_len(inputs, max_input_length, 1), x) - x = list(x) - x = torch.stack(x, dim=0) - N_chunks = max_input_length // (chunk_size * self.subsampling_factor) - x = x.view(b * N_chunks, 1, chunk_size * self.subsampling_factor, f) - - x = self.conv(x) - - _, c, _, f = x.size() - if chunk_size is not None: - x = x.transpose(1, 2).contiguous().view(b, -1, c * f)[:, :olens, :] - else: - x = x.transpose(1, 2).contiguous().view(b, -1, c * f) - - if self.output is not None: - x = self.output(x) - - return x, mask[:, :olens][:, : x.size(1)] - - def create_new_vgg_mask(self, mask: torch.Tensor) -> torch.Tensor: - """Create a new mask for VGG output sequences. - Args: - mask: Mask of input sequences. (B, T) - Returns: - mask: Mask of output sequences. (B, sub(T)) - """ - if self.subsampling_factor > 1: - return mask[:, ::2][:, :: self.stride_1] - else: - return mask - - def get_size_before_subsampling(self, size: int) -> int: - """Return the original size before subsampling for a given size. - Args: - size: Number of frames after subsampling. - Returns: - : Number of frames before subsampling. - """ - return size * self.subsampling_factor diff --git a/funasr_detach/models/sa_asr/__init__.py b/funasr_detach/models/sa_asr/__init__.py deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/funasr_detach/models/sa_asr/attention.py b/funasr_detach/models/sa_asr/attention.py deleted file mode 100644 index 2bb26f5231f77ba2e2e0102c05b3810ab6350f4c..0000000000000000000000000000000000000000 --- a/funasr_detach/models/sa_asr/attention.py +++ /dev/null @@ -1,56 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- - - -"""Multi-Head Attention layer definition.""" - -import math - -import numpy -import torch -from torch import nn -from typing import Optional, Tuple - -import torch.nn.functional as F -from funasr_detach.models.transformer.utils.nets_utils import make_pad_mask -import funasr_detach.models.lora.layers as lora - - -class CosineDistanceAttention(nn.Module): - """Compute Cosine Distance between spk decoder output and speaker profile - Args: - profile_path: speaker profile file path (.npy file) - """ - - def __init__(self): - super().__init__() - self.softmax = nn.Softmax(dim=-1) - - def forward(self, spk_decoder_out, profile, profile_lens=None): - """ - Args: - spk_decoder_out(torch.Tensor):(B, L, D) - spk_profiles(torch.Tensor):(B, N, D) - """ - x = spk_decoder_out.unsqueeze(2) # (B, L, 1, D) - if profile_lens is not None: - - mask = (make_pad_mask(profile_lens)[:, None, :]).to(profile.device) - min_value = float( - numpy.finfo(torch.tensor(0, dtype=x.dtype).numpy().dtype).min - ) - weights_not_softmax = F.cosine_similarity( - x, profile.unsqueeze(1), dim=-1 - ).masked_fill(mask, min_value) - weights = self.softmax(weights_not_softmax).masked_fill( - mask, 0.0 - ) # (B, L, N) - else: - x = x[:, -1:, :, :] - weights_not_softmax = F.cosine_similarity( - x, profile.unsqueeze(1).to(x.device), dim=-1 - ) - weights = self.softmax(weights_not_softmax) # (B, 1, N) - spk_embedding = torch.matmul(weights, profile.to(weights.device)) # (B, L, D) - - return spk_embedding, weights diff --git a/funasr_detach/models/sa_asr/beam_search_sa_asr.py b/funasr_detach/models/sa_asr/beam_search_sa_asr.py deleted file mode 100755 index 85f7c35c4a7e625682e74c693be728116459f4ba..0000000000000000000000000000000000000000 --- a/funasr_detach/models/sa_asr/beam_search_sa_asr.py +++ /dev/null @@ -1,559 +0,0 @@ -"""Beam search module.""" - -from itertools import chain -import logging -from typing import Any -from typing import Dict -from typing import List -from typing import NamedTuple -from typing import Tuple -from typing import Union - -import torch - -from funasr_detach.metrics import end_detect -from funasr_detach.models.transformer.scorers.scorer_interface import ( - PartialScorerInterface, -) -from funasr_detach.models.transformer.scorers.scorer_interface import ScorerInterface -from funasr_detach.models.decoder.abs_decoder import AbsDecoder - - -class Hypothesis(NamedTuple): - """Hypothesis data type.""" - - yseq: torch.Tensor - spk_weigths: List - score: Union[float, torch.Tensor] = 0 - scores: Dict[str, Union[float, torch.Tensor]] = dict() - states: Dict[str, Any] = dict() - - def asdict(self) -> dict: - """Convert data to JSON-friendly dict.""" - return self._replace( - yseq=self.yseq.tolist(), - score=float(self.score), - scores={k: float(v) for k, v in self.scores.items()}, - )._asdict() - - -class BeamSearch(torch.nn.Module): - """Beam search implementation.""" - - def __init__( - self, - scorers: Dict[str, ScorerInterface], - weights: Dict[str, float], - beam_size: int, - vocab_size: int, - sos: int, - eos: int, - token_list: List[str] = None, - pre_beam_ratio: float = 1.5, - pre_beam_score_key: str = None, - ): - """Initialize beam search. - - Args: - scorers (dict[str, ScorerInterface]): Dict of decoder modules - e.g., Decoder, CTCPrefixScorer, LM - The scorer will be ignored if it is `None` - weights (dict[str, float]): Dict of weights for each scorers - The scorer will be ignored if its weight is 0 - beam_size (int): The number of hypotheses kept during search - vocab_size (int): The number of vocabulary - sos (int): Start of sequence id - eos (int): End of sequence id - token_list (list[str]): List of tokens for debug log - pre_beam_score_key (str): key of scores to perform pre-beam search - pre_beam_ratio (float): beam size in the pre-beam search - will be `int(pre_beam_ratio * beam_size)` - - """ - super().__init__() - # set scorers - self.weights = weights - self.scorers = dict() - self.full_scorers = dict() - self.part_scorers = dict() - # this module dict is required for recursive cast - # `self.to(device, dtype)` in `recog.py` - self.nn_dict = torch.nn.ModuleDict() - for k, v in scorers.items(): - w = weights.get(k, 0) - if w == 0 or v is None: - continue - assert isinstance( - v, ScorerInterface - ), f"{k} ({type(v)}) does not implement ScorerInterface" - self.scorers[k] = v - if isinstance(v, PartialScorerInterface): - self.part_scorers[k] = v - else: - self.full_scorers[k] = v - if isinstance(v, torch.nn.Module): - self.nn_dict[k] = v - - # set configurations - self.sos = sos - self.eos = eos - self.token_list = token_list - self.pre_beam_size = int(pre_beam_ratio * beam_size) - self.beam_size = beam_size - self.n_vocab = vocab_size - if ( - pre_beam_score_key is not None - and pre_beam_score_key != "full" - and pre_beam_score_key not in self.full_scorers - ): - raise KeyError(f"{pre_beam_score_key} is not found in {self.full_scorers}") - self.pre_beam_score_key = pre_beam_score_key - self.do_pre_beam = ( - self.pre_beam_score_key is not None - and self.pre_beam_size < self.n_vocab - and len(self.part_scorers) > 0 - ) - - def init_hyp(self, x: torch.Tensor) -> List[Hypothesis]: - """Get an initial hypothesis data. - - Args: - x (torch.Tensor): The encoder output feature - - Returns: - Hypothesis: The initial hypothesis. - - """ - init_states = dict() - init_scores = dict() - for k, d in self.scorers.items(): - init_states[k] = d.init_state(x) - init_scores[k] = 0.0 - return [ - Hypothesis( - score=0.0, - scores=init_scores, - states=init_states, - yseq=torch.tensor([self.sos], device=x.device), - spk_weigths=[], - ) - ] - - @staticmethod - def append_token(xs: torch.Tensor, x: int) -> torch.Tensor: - """Append new token to prefix tokens. - - Args: - xs (torch.Tensor): The prefix token - x (int): The new token to append - - Returns: - torch.Tensor: New tensor contains: xs + [x] with xs.dtype and xs.device - - """ - x = torch.tensor([x], dtype=xs.dtype, device=xs.device) - return torch.cat((xs, x)) - - def score_full( - self, - hyp: Hypothesis, - asr_enc: torch.Tensor, - spk_enc: torch.Tensor, - profile: torch.Tensor, - ) -> Tuple[Dict[str, torch.Tensor], Dict[str, Any]]: - """Score new hypothesis by `self.full_scorers`. - - Args: - hyp (Hypothesis): Hypothesis with prefix tokens to score - x (torch.Tensor): Corresponding input feature - - Returns: - Tuple[Dict[str, torch.Tensor], Dict[str, Any]]: Tuple of - score dict of `hyp` that has string keys of `self.full_scorers` - and tensor score values of shape: `(self.n_vocab,)`, - and state dict that has string keys - and state values of `self.full_scorers` - - """ - scores = dict() - states = dict() - for k, d in self.full_scorers.items(): - if isinstance(d, AbsDecoder): - scores[k], spk_weigths, states[k] = d.score( - hyp.yseq, hyp.states[k], asr_enc, spk_enc, profile - ) - else: - scores[k], states[k] = d.score(hyp.yseq, hyp.states[k], asr_enc) - return scores, spk_weigths, states - - def score_partial( - self, - hyp: Hypothesis, - ids: torch.Tensor, - asr_enc: torch.Tensor, - spk_enc: torch.Tensor, - profile: torch.Tensor, - ) -> Tuple[Dict[str, torch.Tensor], Dict[str, Any]]: - """Score new hypothesis by `self.part_scorers`. - - Args: - hyp (Hypothesis): Hypothesis with prefix tokens to score - ids (torch.Tensor): 1D tensor of new partial tokens to score - x (torch.Tensor): Corresponding input feature - - Returns: - Tuple[Dict[str, torch.Tensor], Dict[str, Any]]: Tuple of - score dict of `hyp` that has string keys of `self.part_scorers` - and tensor score values of shape: `(len(ids),)`, - and state dict that has string keys - and state values of `self.part_scorers` - - """ - scores = dict() - states = dict() - for k, d in self.part_scorers.items(): - if isinstance(d, AbsDecoder): - scores[k], states[k] = d.score_partial( - hyp.yseq, ids, hyp.states[k], asr_enc, spk_enc, profile - ) - else: - scores[k], states[k] = d.score_partial( - hyp.yseq, ids, hyp.states[k], asr_enc - ) - return scores, states - - def beam( - self, weighted_scores: torch.Tensor, ids: torch.Tensor - ) -> Tuple[torch.Tensor, torch.Tensor]: - """Compute topk full token ids and partial token ids. - - Args: - weighted_scores (torch.Tensor): The weighted sum scores for each tokens. - Its shape is `(self.n_vocab,)`. - ids (torch.Tensor): The partial token ids to compute topk - - Returns: - Tuple[torch.Tensor, torch.Tensor]: - The topk full token ids and partial token ids. - Their shapes are `(self.beam_size,)` - - """ - # no pre beam performed - if weighted_scores.size(0) == ids.size(0): - top_ids = weighted_scores.topk(self.beam_size)[1] - return top_ids, top_ids - - # mask pruned in pre-beam not to select in topk - tmp = weighted_scores[ids] - weighted_scores[:] = -float("inf") - weighted_scores[ids] = tmp - top_ids = weighted_scores.topk(self.beam_size)[1] - local_ids = weighted_scores[ids].topk(self.beam_size)[1] - return top_ids, local_ids - - @staticmethod - def merge_scores( - prev_scores: Dict[str, float], - next_full_scores: Dict[str, torch.Tensor], - full_idx: int, - next_part_scores: Dict[str, torch.Tensor], - part_idx: int, - ) -> Dict[str, torch.Tensor]: - """Merge scores for new hypothesis. - - Args: - prev_scores (Dict[str, float]): - The previous hypothesis scores by `self.scorers` - next_full_scores (Dict[str, torch.Tensor]): scores by `self.full_scorers` - full_idx (int): The next token id for `next_full_scores` - next_part_scores (Dict[str, torch.Tensor]): - scores of partial tokens by `self.part_scorers` - part_idx (int): The new token id for `next_part_scores` - - Returns: - Dict[str, torch.Tensor]: The new score dict. - Its keys are names of `self.full_scorers` and `self.part_scorers`. - Its values are scalar tensors by the scorers. - - """ - new_scores = dict() - for k, v in next_full_scores.items(): - new_scores[k] = prev_scores[k] + v[full_idx] - for k, v in next_part_scores.items(): - new_scores[k] = prev_scores[k] + v[part_idx] - return new_scores - - def merge_states(self, states: Any, part_states: Any, part_idx: int) -> Any: - """Merge states for new hypothesis. - - Args: - states: states of `self.full_scorers` - part_states: states of `self.part_scorers` - part_idx (int): The new token id for `part_scores` - - Returns: - Dict[str, torch.Tensor]: The new score dict. - Its keys are names of `self.full_scorers` and `self.part_scorers`. - Its values are states of the scorers. - - """ - new_states = dict() - for k, v in states.items(): - new_states[k] = v - for k, d in self.part_scorers.items(): - new_states[k] = d.select_state(part_states[k], part_idx) - return new_states - - def search( - self, - running_hyps: List[Hypothesis], - asr_enc: torch.Tensor, - spk_enc: torch.Tensor, - profile: torch.Tensor, - ) -> List[Hypothesis]: - """Search new tokens for running hypotheses and encoded speech x. - - Args: - running_hyps (List[Hypothesis]): Running hypotheses on beam - x (torch.Tensor): Encoded speech feature (T, D) - - Returns: - List[Hypotheses]: Best sorted hypotheses - - """ - # import ipdb;ipdb.set_trace() - best_hyps = [] - part_ids = torch.arange(self.n_vocab, device=asr_enc.device) # no pre-beam - for hyp in running_hyps: - # scoring - weighted_scores = torch.zeros( - self.n_vocab, dtype=asr_enc.dtype, device=asr_enc.device - ) - scores, spk_weigths, states = self.score_full( - hyp, asr_enc, spk_enc, profile - ) - for k in self.full_scorers: - weighted_scores += self.weights[k] * scores[k] - # partial scoring - if self.do_pre_beam: - pre_beam_scores = ( - weighted_scores - if self.pre_beam_score_key == "full" - else scores[self.pre_beam_score_key] - ) - part_ids = torch.topk(pre_beam_scores, self.pre_beam_size)[1] - part_scores, part_states = self.score_partial( - hyp, part_ids, asr_enc, spk_enc, profile - ) - for k in self.part_scorers: - weighted_scores[part_ids] += self.weights[k] * part_scores[k] - # add previous hyp score - weighted_scores += hyp.score - - # update hyps - for j, part_j in zip(*self.beam(weighted_scores, part_ids)): - # will be (2 x beam at most) - best_hyps.append( - Hypothesis( - score=weighted_scores[j], - yseq=self.append_token(hyp.yseq, j), - scores=self.merge_scores( - hyp.scores, scores, j, part_scores, part_j - ), - states=self.merge_states(states, part_states, part_j), - spk_weigths=hyp.spk_weigths + [spk_weigths], - ) - ) - - # sort and prune 2 x beam -> beam - best_hyps = sorted(best_hyps, key=lambda x: x.score, reverse=True)[ - : min(len(best_hyps), self.beam_size) - ] - return best_hyps - - def forward( - self, - asr_enc: torch.Tensor, - spk_enc: torch.Tensor, - profile: torch.Tensor, - maxlenratio: float = 0.0, - minlenratio: float = 0.0, - ) -> List[Hypothesis]: - """Perform beam search. - - Args: - x (torch.Tensor): Encoded speech feature (T, D) - maxlenratio (float): Input length ratio to obtain max output length. - If maxlenratio=0.0 (default), it uses a end-detect function - to automatically find maximum hypothesis lengths - minlenratio (float): Input length ratio to obtain min output length. - - Returns: - list[Hypothesis]: N-best decoding results - - """ - # import ipdb;ipdb.set_trace() - # set length bounds - if maxlenratio == 0: - maxlen = asr_enc.shape[0] - else: - maxlen = max(1, int(maxlenratio * asr_enc.size(0))) - minlen = int(minlenratio * asr_enc.size(0)) - logging.info("decoder input length: " + str(asr_enc.shape[0])) - logging.info("max output length: " + str(maxlen)) - logging.info("min output length: " + str(minlen)) - - # main loop of prefix search - running_hyps = self.init_hyp(asr_enc) - ended_hyps = [] - for i in range(maxlen): - logging.debug("position " + str(i)) - best = self.search(running_hyps, asr_enc, spk_enc, profile) - # import pdb;pdb.set_trace() - # post process of one iteration - running_hyps = self.post_process(i, maxlen, maxlenratio, best, ended_hyps) - # end detection - if maxlenratio == 0.0 and end_detect([h.asdict() for h in ended_hyps], i): - logging.info(f"end detected at {i}") - break - if len(running_hyps) == 0: - logging.info("no hypothesis. Finish decoding.") - break - else: - logging.debug(f"remained hypotheses: {len(running_hyps)}") - - nbest_hyps = sorted(ended_hyps, key=lambda x: x.score, reverse=True) - # check the number of hypotheses reaching to eos - if len(nbest_hyps) == 0: - logging.warning( - "there is no N-best results, perform recognition " - "again with smaller minlenratio." - ) - return ( - [] - if minlenratio < 0.1 - else self.forward( - asr_enc, spk_enc, profile, maxlenratio, max(0.0, minlenratio - 0.1) - ) - ) - - # report the best result - best = nbest_hyps[0] - for k, v in best.scores.items(): - logging.info( - f"{v:6.2f} * {self.weights[k]:3} = {v * self.weights[k]:6.2f} for {k}" - ) - logging.info(f"total log probability: {best.score:.2f}") - logging.info(f"normalized log probability: {best.score / len(best.yseq):.2f}") - logging.info(f"total number of ended hypotheses: {len(nbest_hyps)}") - if self.token_list is not None: - logging.info( - "best hypo: " - + "".join([self.token_list[x] for x in best.yseq[1:-1]]) - + "\n" - ) - return nbest_hyps - - def post_process( - self, - i: int, - maxlen: int, - maxlenratio: float, - running_hyps: List[Hypothesis], - ended_hyps: List[Hypothesis], - ) -> List[Hypothesis]: - """Perform post-processing of beam search iterations. - - Args: - i (int): The length of hypothesis tokens. - maxlen (int): The maximum length of tokens in beam search. - maxlenratio (int): The maximum length ratio in beam search. - running_hyps (List[Hypothesis]): The running hypotheses in beam search. - ended_hyps (List[Hypothesis]): The ended hypotheses in beam search. - - Returns: - List[Hypothesis]: The new running hypotheses. - - """ - logging.debug(f"the number of running hypotheses: {len(running_hyps)}") - if self.token_list is not None: - logging.debug( - "best hypo: " - + "".join([self.token_list[x] for x in running_hyps[0].yseq[1:]]) - ) - # add eos in the final loop to avoid that there are no ended hyps - if i == maxlen - 1: - logging.info("adding in the last position in the loop") - running_hyps = [ - h._replace(yseq=self.append_token(h.yseq, self.eos)) - for h in running_hyps - ] - - # add ended hypotheses to a final list, and removed them from current hypotheses - # (this will be a problem, number of hyps < beam) - remained_hyps = [] - for hyp in running_hyps: - if hyp.yseq[-1] == self.eos: - # e.g., Word LM needs to add final score - for k, d in chain(self.full_scorers.items(), self.part_scorers.items()): - s = d.final_score(hyp.states[k]) - hyp.scores[k] += s - hyp = hyp._replace(score=hyp.score + self.weights[k] * s) - ended_hyps.append(hyp) - else: - remained_hyps.append(hyp) - return remained_hyps - - -def beam_search( - x: torch.Tensor, - sos: int, - eos: int, - beam_size: int, - vocab_size: int, - scorers: Dict[str, ScorerInterface], - weights: Dict[str, float], - token_list: List[str] = None, - maxlenratio: float = 0.0, - minlenratio: float = 0.0, - pre_beam_ratio: float = 1.5, - pre_beam_score_key: str = "full", -) -> list: - """Perform beam search with scorers. - - Args: - x (torch.Tensor): Encoded speech feature (T, D) - sos (int): Start of sequence id - eos (int): End of sequence id - beam_size (int): The number of hypotheses kept during search - vocab_size (int): The number of vocabulary - scorers (dict[str, ScorerInterface]): Dict of decoder modules - e.g., Decoder, CTCPrefixScorer, LM - The scorer will be ignored if it is `None` - weights (dict[str, float]): Dict of weights for each scorers - The scorer will be ignored if its weight is 0 - token_list (list[str]): List of tokens for debug log - maxlenratio (float): Input length ratio to obtain max output length. - If maxlenratio=0.0 (default), it uses a end-detect function - to automatically find maximum hypothesis lengths - minlenratio (float): Input length ratio to obtain min output length. - pre_beam_score_key (str): key of scores to perform pre-beam search - pre_beam_ratio (float): beam size in the pre-beam search - will be `int(pre_beam_ratio * beam_size)` - - Returns: - list: N-best decoding results - - """ - ret = BeamSearch( - scorers, - weights, - beam_size=beam_size, - vocab_size=vocab_size, - pre_beam_ratio=pre_beam_ratio, - pre_beam_score_key=pre_beam_score_key, - sos=sos, - eos=eos, - token_list=token_list, - ).forward(x=x, maxlenratio=maxlenratio, minlenratio=minlenratio) - return [h.asdict() for h in ret] diff --git a/funasr_detach/models/sa_asr/e2e_sa_asr.py b/funasr_detach/models/sa_asr/e2e_sa_asr.py deleted file mode 100644 index cebe5d3fce5894d790c53ba7191d16a561630544..0000000000000000000000000000000000000000 --- a/funasr_detach/models/sa_asr/e2e_sa_asr.py +++ /dev/null @@ -1,531 +0,0 @@ -# Copyright ESPnet (https://github.com/espnet/espnet). All Rights Reserved. -# Apache 2.0 (http://www.apache.org/licenses/LICENSE-2.0) - -import logging -from contextlib import contextmanager -from distutils.version import LooseVersion -from typing import Dict -from typing import List -from typing import Optional -from typing import Tuple -from typing import Union - -import torch -import torch.nn.functional as F - -from funasr_detach.layers.abs_normalize import AbsNormalize -from funasr_detach.losses.label_smoothing_loss import ( - LabelSmoothingLoss, - NllLoss, -) # noqa: H301 -from funasr_detach.models.ctc import CTC -from funasr_detach.models.decoder.abs_decoder import AbsDecoder -from funasr_detach.models.encoder.abs_encoder import AbsEncoder -from funasr_detach.frontends.abs_frontend import AbsFrontend -from funasr_detach.models.postencoder.abs_postencoder import AbsPostEncoder -from funasr_detach.models.preencoder.abs_preencoder import AbsPreEncoder -from funasr_detach.models.specaug.abs_specaug import AbsSpecAug -from funasr_detach.models.transformer.utils.add_sos_eos import add_sos_eos -from funasr_detach.metrics import ErrorCalculator -from funasr_detach.metrics.compute_acc import th_accuracy -from funasr_detach.train_utils.device_funcs import force_gatherable -from funasr_detach.models.base_model import FunASRModel - -if LooseVersion(torch.__version__) >= LooseVersion("1.6.0"): - from torch.cuda.amp import autocast -else: - # Nothing to do if torch<1.6.0 - @contextmanager - def autocast(enabled=True): - yield - - -class SAASRModel(FunASRModel): - """CTC-attention hybrid Encoder-Decoder model""" - - def __init__( - self, - vocab_size: int, - max_spk_num: int, - token_list: Union[Tuple[str, ...], List[str]], - frontend: Optional[AbsFrontend], - specaug: Optional[AbsSpecAug], - normalize: Optional[AbsNormalize], - asr_encoder: AbsEncoder, - spk_encoder: torch.nn.Module, - decoder: AbsDecoder, - ctc: CTC, - spk_weight: float = 0.5, - ctc_weight: float = 0.5, - interctc_weight: float = 0.0, - ignore_id: int = -1, - lsm_weight: float = 0.0, - length_normalized_loss: bool = False, - report_cer: bool = True, - report_wer: bool = True, - sym_space: str = "", - sym_blank: str = "", - extract_feats_in_collect_stats: bool = True, - ): - assert 0.0 <= ctc_weight <= 1.0, ctc_weight - assert 0.0 <= interctc_weight < 1.0, interctc_weight - - super().__init__() - # note that eos is the same as sos (equivalent ID) - self.blank_id = 0 - self.sos = 1 - self.eos = 2 - self.vocab_size = vocab_size - self.max_spk_num = max_spk_num - self.ignore_id = ignore_id - self.spk_weight = spk_weight - self.ctc_weight = ctc_weight - self.interctc_weight = interctc_weight - self.token_list = token_list.copy() - - self.frontend = frontend - self.specaug = specaug - self.normalize = normalize - self.asr_encoder = asr_encoder - self.spk_encoder = spk_encoder - - if not hasattr(self.asr_encoder, "interctc_use_conditioning"): - self.asr_encoder.interctc_use_conditioning = False - if self.asr_encoder.interctc_use_conditioning: - self.asr_encoder.conditioning_layer = torch.nn.Linear( - vocab_size, self.asr_encoder.output_size() - ) - - self.error_calculator = None - - # we set self.decoder = None in the CTC mode since - # self.decoder parameters were never used and PyTorch complained - # and threw an Exception in the multi-GPU experiment. - # thanks Jeff Farris for pointing out the issue. - if ctc_weight == 1.0: - self.decoder = None - else: - self.decoder = decoder - - self.criterion_att = LabelSmoothingLoss( - size=vocab_size, - padding_idx=ignore_id, - smoothing=lsm_weight, - normalize_length=length_normalized_loss, - ) - - self.criterion_spk = NllLoss( - size=max_spk_num, - padding_idx=ignore_id, - normalize_length=length_normalized_loss, - ) - - if report_cer or report_wer: - self.error_calculator = ErrorCalculator( - token_list, sym_space, sym_blank, report_cer, report_wer - ) - - if ctc_weight == 0.0: - self.ctc = None - else: - self.ctc = ctc - - self.extract_feats_in_collect_stats = extract_feats_in_collect_stats - - def forward( - self, - speech: torch.Tensor, - speech_lengths: torch.Tensor, - text: torch.Tensor, - text_lengths: torch.Tensor, - profile: torch.Tensor, - profile_lengths: torch.Tensor, - text_id: torch.Tensor, - text_id_lengths: torch.Tensor, - ) -> Tuple[torch.Tensor, Dict[str, torch.Tensor], torch.Tensor]: - """Frontend + Encoder + Decoder + Calc loss - - Args: - speech: (Batch, Length, ...) - speech_lengths: (Batch, ) - text: (Batch, Length) - text_lengths: (Batch,) - profile: (Batch, Length, Dim) - profile_lengths: (Batch,) - """ - assert text_lengths.dim() == 1, text_lengths.shape - # Check that batch_size is unified - assert ( - speech.shape[0] - == speech_lengths.shape[0] - == text.shape[0] - == text_lengths.shape[0] - ), (speech.shape, speech_lengths.shape, text.shape, text_lengths.shape) - batch_size = speech.shape[0] - - # for data-parallel - text = text[:, : text_lengths.max()] - - # 1. Encoder - asr_encoder_out, encoder_out_lens, spk_encoder_out = self.encode( - speech, speech_lengths - ) - intermediate_outs = None - if isinstance(asr_encoder_out, tuple): - intermediate_outs = asr_encoder_out[1] - asr_encoder_out = asr_encoder_out[0] - - loss_att, loss_spk, acc_att, acc_spk, cer_att, wer_att = ( - None, - None, - None, - None, - None, - None, - ) - loss_ctc, cer_ctc = None, None - stats = dict() - - # 1. CTC branch - if self.ctc_weight != 0.0: - loss_ctc, cer_ctc = self._calc_ctc_loss( - asr_encoder_out, encoder_out_lens, text, text_lengths - ) - - # Intermediate CTC (optional) - loss_interctc = 0.0 - if self.interctc_weight != 0.0 and intermediate_outs is not None: - for layer_idx, intermediate_out in intermediate_outs: - # we assume intermediate_out has the same length & padding - # as those of encoder_out - loss_ic, cer_ic = self._calc_ctc_loss( - intermediate_out, encoder_out_lens, text, text_lengths - ) - loss_interctc = loss_interctc + loss_ic - - # Collect Intermedaite CTC stats - stats["loss_interctc_layer{}".format(layer_idx)] = ( - loss_ic.detach() if loss_ic is not None else None - ) - stats["cer_interctc_layer{}".format(layer_idx)] = cer_ic - - loss_interctc = loss_interctc / len(intermediate_outs) - - # calculate whole encoder loss - loss_ctc = ( - 1 - self.interctc_weight - ) * loss_ctc + self.interctc_weight * loss_interctc - - # 2b. Attention decoder branch - if self.ctc_weight != 1.0: - loss_att, loss_spk, acc_att, acc_spk, cer_att, wer_att = ( - self._calc_att_loss( - asr_encoder_out, - spk_encoder_out, - encoder_out_lens, - text, - text_lengths, - profile, - profile_lengths, - text_id, - text_id_lengths, - ) - ) - - # 3. CTC-Att loss definition - if self.ctc_weight == 0.0: - loss_asr = loss_att - elif self.ctc_weight == 1.0: - loss_asr = loss_ctc - else: - loss_asr = self.ctc_weight * loss_ctc + (1 - self.ctc_weight) * loss_att - - if self.spk_weight == 0.0: - loss = loss_asr - else: - loss = self.spk_weight * loss_spk + (1 - self.spk_weight) * loss_asr - - stats = dict( - loss=loss.detach(), - loss_asr=loss_asr.detach(), - loss_att=loss_att.detach() if loss_att is not None else None, - loss_ctc=loss_ctc.detach() if loss_ctc is not None else None, - loss_spk=loss_spk.detach() if loss_spk is not None else None, - acc=acc_att, - acc_spk=acc_spk, - cer=cer_att, - wer=wer_att, - cer_ctc=cer_ctc, - ) - - # force_gatherable: to-device and to-tensor if scalar for DataParallel - loss, stats, weight = force_gatherable((loss, stats, batch_size), loss.device) - return loss, stats, weight - - def collect_feats( - self, - speech: torch.Tensor, - speech_lengths: torch.Tensor, - text: torch.Tensor, - text_lengths: torch.Tensor, - ) -> Dict[str, torch.Tensor]: - if self.extract_feats_in_collect_stats: - feats, feats_lengths = self._extract_feats(speech, speech_lengths) - else: - # Generate dummy stats if extract_feats_in_collect_stats is False - logging.warning( - "Generating dummy stats for feats and feats_lengths, " - "because encoder_conf.extract_feats_in_collect_stats is " - f"{self.extract_feats_in_collect_stats}" - ) - feats, feats_lengths = speech, speech_lengths - return {"feats": feats, "feats_lengths": feats_lengths} - - def encode( - self, speech: torch.Tensor, speech_lengths: torch.Tensor - ) -> Tuple[torch.Tensor, torch.Tensor]: - """Frontend + Encoder. Note that this method is used by asr_inference.py - - Args: - speech: (Batch, Length, ...) - speech_lengths: (Batch, ) - """ - with autocast(False): - # 1. Extract feats - feats, feats_lengths = self._extract_feats(speech, speech_lengths) - - # 2. Data augmentation - feats_raw = feats.clone() - if self.specaug is not None and self.training: - feats, feats_lengths = self.specaug(feats, feats_lengths) - - # 3. Normalization for feature: e.g. Global-CMVN, Utterance-CMVN - if self.normalize is not None: - feats, feats_lengths = self.normalize(feats, feats_lengths) - - # 4. Forward encoder - # feats: (Batch, Length, Dim) - # -> encoder_out: (Batch, Length2, Dim2) - if self.asr_encoder.interctc_use_conditioning: - encoder_out, encoder_out_lens, _ = self.asr_encoder( - feats, feats_lengths, ctc=self.ctc - ) - else: - encoder_out, encoder_out_lens, _ = self.asr_encoder(feats, feats_lengths) - intermediate_outs = None - if isinstance(encoder_out, tuple): - intermediate_outs = encoder_out[1] - encoder_out = encoder_out[0] - - encoder_out_spk_ori = self.spk_encoder(feats_raw, feats_lengths)[0] - # import ipdb;ipdb.set_trace() - if encoder_out_spk_ori.size(1) != encoder_out.size(1): - encoder_out_spk = F.interpolate( - encoder_out_spk_ori.transpose(-2, -1), - size=(encoder_out.size(1)), - mode="nearest", - ).transpose(-2, -1) - else: - encoder_out_spk = encoder_out_spk_ori - - assert encoder_out.size(0) == speech.size(0), ( - encoder_out.size(), - speech.size(0), - ) - assert encoder_out.size(1) <= encoder_out_lens.max(), ( - encoder_out.size(), - encoder_out_lens.max(), - ) - assert encoder_out_spk.size(0) == speech.size(0), ( - encoder_out_spk.size(), - speech.size(0), - ) - - if intermediate_outs is not None: - return (encoder_out, intermediate_outs), encoder_out_lens, encoder_out_spk - - return encoder_out, encoder_out_lens, encoder_out_spk - - def _extract_feats( - self, speech: torch.Tensor, speech_lengths: torch.Tensor - ) -> Tuple[torch.Tensor, torch.Tensor]: - assert speech_lengths.dim() == 1, speech_lengths.shape - - # for data-parallel - speech = speech[:, : speech_lengths.max()] - - if self.frontend is not None: - # Frontend - # e.g. STFT and Feature extract - # data_loader may send time-domain signal in this case - # speech (Batch, NSamples) -> feats: (Batch, NFrames, Dim) - feats, feats_lengths = self.frontend(speech, speech_lengths) - else: - # No frontend and no feature extract - feats, feats_lengths = speech, speech_lengths - return feats, feats_lengths - - def nll( - self, - encoder_out: torch.Tensor, - encoder_out_lens: torch.Tensor, - ys_pad: torch.Tensor, - ys_pad_lens: torch.Tensor, - ) -> torch.Tensor: - """Compute negative log likelihood(nll) from transformer-decoder - - Normally, this function is called in batchify_nll. - - Args: - encoder_out: (Batch, Length, Dim) - encoder_out_lens: (Batch,) - ys_pad: (Batch, Length) - ys_pad_lens: (Batch,) - """ - ys_in_pad, ys_out_pad = add_sos_eos(ys_pad, self.sos, self.eos, self.ignore_id) - ys_in_lens = ys_pad_lens + 1 - - # 1. Forward decoder - decoder_out, _ = self.decoder( - encoder_out, encoder_out_lens, ys_in_pad, ys_in_lens - ) # [batch, seqlen, dim] - batch_size = decoder_out.size(0) - decoder_num_class = decoder_out.size(2) - # nll: negative log-likelihood - nll = torch.nn.functional.cross_entropy( - decoder_out.view(-1, decoder_num_class), - ys_out_pad.view(-1), - ignore_index=self.ignore_id, - reduction="none", - ) - nll = nll.view(batch_size, -1) - nll = nll.sum(dim=1) - assert nll.size(0) == batch_size - return nll - - def batchify_nll( - self, - encoder_out: torch.Tensor, - encoder_out_lens: torch.Tensor, - ys_pad: torch.Tensor, - ys_pad_lens: torch.Tensor, - batch_size: int = 100, - ): - """Compute negative log likelihood(nll) from transformer-decoder - - To avoid OOM, this fuction seperate the input into batches. - Then call nll for each batch and combine and return results. - Args: - encoder_out: (Batch, Length, Dim) - encoder_out_lens: (Batch,) - ys_pad: (Batch, Length) - ys_pad_lens: (Batch,) - batch_size: int, samples each batch contain when computing nll, - you may change this to avoid OOM or increase - GPU memory usage - """ - total_num = encoder_out.size(0) - if total_num <= batch_size: - nll = self.nll(encoder_out, encoder_out_lens, ys_pad, ys_pad_lens) - else: - nll = [] - start_idx = 0 - while True: - end_idx = min(start_idx + batch_size, total_num) - batch_encoder_out = encoder_out[start_idx:end_idx, :, :] - batch_encoder_out_lens = encoder_out_lens[start_idx:end_idx] - batch_ys_pad = ys_pad[start_idx:end_idx, :] - batch_ys_pad_lens = ys_pad_lens[start_idx:end_idx] - batch_nll = self.nll( - batch_encoder_out, - batch_encoder_out_lens, - batch_ys_pad, - batch_ys_pad_lens, - ) - nll.append(batch_nll) - start_idx = end_idx - if start_idx == total_num: - break - nll = torch.cat(nll) - assert nll.size(0) == total_num - return nll - - def _calc_att_loss( - self, - asr_encoder_out: torch.Tensor, - spk_encoder_out: torch.Tensor, - encoder_out_lens: torch.Tensor, - ys_pad: torch.Tensor, - ys_pad_lens: torch.Tensor, - profile: torch.Tensor, - profile_lens: torch.Tensor, - text_id: torch.Tensor, - text_id_lengths: torch.Tensor, - ): - ys_in_pad, ys_out_pad = add_sos_eos(ys_pad, self.sos, self.eos, self.ignore_id) - ys_in_lens = ys_pad_lens + 1 - - # 1. Forward decoder - decoder_out, weights_no_pad, _ = self.decoder( - asr_encoder_out, - spk_encoder_out, - encoder_out_lens, - ys_in_pad, - ys_in_lens, - profile, - profile_lens, - ) - - spk_num_no_pad = weights_no_pad.size(-1) - pad = (0, self.max_spk_num - spk_num_no_pad) - weights = F.pad(weights_no_pad, pad, mode="constant", value=0) - - # pre_id=weights.argmax(-1) - # pre_text=decoder_out.argmax(-1) - # id_mask=(pre_id==text_id).to(dtype=text_id.dtype) - # pre_text_mask=pre_text*id_mask+1-id_mask #相同的地方不变，不同的地方设为1() - # padding_mask= ys_out_pad != self.ignore_id - # numerator = torch.sum(pre_text_mask.masked_select(padding_mask) == ys_out_pad.masked_select(padding_mask)) - # denominator = torch.sum(padding_mask) - # sd_acc = float(numerator) / float(denominator) - - # 2. Compute attention loss - loss_att = self.criterion_att(decoder_out, ys_out_pad) - loss_spk = self.criterion_spk(torch.log(weights), text_id) - - acc_spk = th_accuracy( - weights.view(-1, self.max_spk_num), - text_id, - ignore_label=self.ignore_id, - ) - acc_att = th_accuracy( - decoder_out.view(-1, self.vocab_size), - ys_out_pad, - ignore_label=self.ignore_id, - ) - - # Compute cer/wer using attention-decoder - if self.training or self.error_calculator is None: - cer_att, wer_att = None, None - else: - ys_hat = decoder_out.argmax(dim=-1) - cer_att, wer_att = self.error_calculator(ys_hat.cpu(), ys_pad.cpu()) - - return loss_att, loss_spk, acc_att, acc_spk, cer_att, wer_att - - def _calc_ctc_loss( - self, - encoder_out: torch.Tensor, - encoder_out_lens: torch.Tensor, - ys_pad: torch.Tensor, - ys_pad_lens: torch.Tensor, - ): - # Calc CTC loss - loss_ctc = self.ctc(encoder_out, encoder_out_lens, ys_pad, ys_pad_lens) - - # Calc CER using CTC - cer_ctc = None - if not self.training and self.error_calculator is not None: - ys_hat = self.ctc.argmax(encoder_out).data - cer_ctc = self.error_calculator(ys_hat.cpu(), ys_pad.cpu(), is_ctc=True) - return loss_ctc, cer_ctc diff --git a/funasr_detach/models/sa_asr/transformer_decoder.py b/funasr_detach/models/sa_asr/transformer_decoder.py deleted file mode 100644 index ba5e594fce04826da0d6bd153fcfc107440b931c..0000000000000000000000000000000000000000 --- a/funasr_detach/models/sa_asr/transformer_decoder.py +++ /dev/null @@ -1,768 +0,0 @@ -# Copyright 2019 Shigeki Karita -# Apache 2.0 (http://www.apache.org/licenses/LICENSE-2.0) - -"""Decoder definition.""" -from typing import Any -from typing import List -from typing import Sequence -from typing import Tuple - -import torch -from torch import nn - - -from funasr_detach.models.transformer.attention import MultiHeadedAttention -from funasr_detach.models.sa_asr.attention import CosineDistanceAttention -from funasr_detach.models.transformer.utils.dynamic_conv import DynamicConvolution -from funasr_detach.models.transformer.utils.dynamic_conv2d import DynamicConvolution2D -from funasr_detach.models.transformer.embedding import PositionalEncoding -from funasr_detach.models.transformer.layer_norm import LayerNorm -from funasr_detach.models.transformer.utils.lightconv import LightweightConvolution -from funasr_detach.models.transformer.utils.lightconv2d import LightweightConvolution2D -from funasr_detach.models.transformer.utils.mask import subsequent_mask -from funasr_detach.models.transformer.utils.nets_utils import make_pad_mask -from funasr_detach.models.transformer.positionwise_feed_forward import ( - PositionwiseFeedForward, # noqa: H301 -) -from funasr_detach.models.transformer.utils.repeat import repeat -from funasr_detach.models.transformer.scorers.scorer_interface import ( - BatchScorerInterface, -) - -from funasr_detach.register import tables - - -class DecoderLayer(nn.Module): - """Single decoder layer module. - - Args: - size (int): Input dimension. - self_attn (torch.nn.Module): Self-attention module instance. - `MultiHeadedAttention` instance can be used as the argument. - src_attn (torch.nn.Module): Self-attention module instance. - `MultiHeadedAttention` instance can be used as the argument. - feed_forward (torch.nn.Module): Feed-forward module instance. - `PositionwiseFeedForward`, `MultiLayeredConv1d`, or `Conv1dLinear` instance - can be used as the argument. - dropout_rate (float): Dropout rate. - normalize_before (bool): Whether to use layer_norm before the first block. - concat_after (bool): Whether to concat attention layer's input and output. - if True, additional linear will be applied. - i.e. x -> x + linear(concat(x, att(x))) - if False, no additional linear will be applied. i.e. x -> x + att(x) - - - """ - - def __init__( - self, - size, - self_attn, - src_attn, - feed_forward, - dropout_rate, - normalize_before=True, - concat_after=False, - ): - """Construct an DecoderLayer object.""" - super(DecoderLayer, self).__init__() - self.size = size - self.self_attn = self_attn - self.src_attn = src_attn - self.feed_forward = feed_forward - self.norm1 = LayerNorm(size) - self.norm2 = LayerNorm(size) - self.norm3 = LayerNorm(size) - self.dropout = nn.Dropout(dropout_rate) - self.normalize_before = normalize_before - self.concat_after = concat_after - if self.concat_after: - self.concat_linear1 = nn.Linear(size + size, size) - self.concat_linear2 = nn.Linear(size + size, size) - - def forward(self, tgt, tgt_mask, memory, memory_mask, cache=None): - """Compute decoded features. - - Args: - tgt (torch.Tensor): Input tensor (#batch, maxlen_out, size). - tgt_mask (torch.Tensor): Mask for input tensor (#batch, maxlen_out). - memory (torch.Tensor): Encoded memory, float32 (#batch, maxlen_in, size). - memory_mask (torch.Tensor): Encoded memory mask (#batch, maxlen_in). - cache (List[torch.Tensor]): List of cached tensors. - Each tensor shape should be (#batch, maxlen_out - 1, size). - - Returns: - torch.Tensor: Output tensor(#batch, maxlen_out, size). - torch.Tensor: Mask for output tensor (#batch, maxlen_out). - torch.Tensor: Encoded memory (#batch, maxlen_in, size). - torch.Tensor: Encoded memory mask (#batch, maxlen_in). - - """ - residual = tgt - if self.normalize_before: - tgt = self.norm1(tgt) - - if cache is None: - tgt_q = tgt - tgt_q_mask = tgt_mask - else: - # compute only the last frame query keeping dim: max_time_out -> 1 - assert cache.shape == ( - tgt.shape[0], - tgt.shape[1] - 1, - self.size, - ), f"{cache.shape} == {(tgt.shape[0], tgt.shape[1] - 1, self.size)}" - tgt_q = tgt[:, -1:, :] - residual = residual[:, -1:, :] - tgt_q_mask = None - if tgt_mask is not None: - tgt_q_mask = tgt_mask[:, -1:, :] - - if self.concat_after: - tgt_concat = torch.cat( - (tgt_q, self.self_attn(tgt_q, tgt, tgt, tgt_q_mask)), dim=-1 - ) - x = residual + self.concat_linear1(tgt_concat) - else: - x = residual + self.dropout(self.self_attn(tgt_q, tgt, tgt, tgt_q_mask)) - if not self.normalize_before: - x = self.norm1(x) - - residual = x - if self.normalize_before: - x = self.norm2(x) - if self.concat_after: - x_concat = torch.cat( - (x, self.src_attn(x, memory, memory, memory_mask)), dim=-1 - ) - x = residual + self.concat_linear2(x_concat) - else: - x = residual + self.dropout(self.src_attn(x, memory, memory, memory_mask)) - if not self.normalize_before: - x = self.norm2(x) - - residual = x - if self.normalize_before: - x = self.norm3(x) - x = residual + self.dropout(self.feed_forward(x)) - if not self.normalize_before: - x = self.norm3(x) - - if cache is not None: - x = torch.cat([cache, x], dim=1) - - return x, tgt_mask, memory, memory_mask - - -class BaseTransformerDecoder(nn.Module, BatchScorerInterface): - """Base class of Transfomer decoder module. - - Args: - vocab_size: output dim - encoder_output_size: dimension of attention - attention_heads: the number of heads of multi head attention - linear_units: the number of units of position-wise feed forward - num_blocks: the number of decoder blocks - dropout_rate: dropout rate - self_attention_dropout_rate: dropout rate for attention - input_layer: input layer type - use_output_layer: whether to use output layer - pos_enc_class: PositionalEncoding or ScaledPositionalEncoding - normalize_before: whether to use layer_norm before the first block - concat_after: whether to concat attention layer's input and output - if True, additional linear will be applied. - i.e. x -> x + linear(concat(x, att(x))) - if False, no additional linear will be applied. - i.e. x -> x + att(x) - """ - - def __init__( - self, - vocab_size: int, - encoder_output_size: int, - dropout_rate: float = 0.1, - positional_dropout_rate: float = 0.1, - input_layer: str = "embed", - use_output_layer: bool = True, - pos_enc_class=PositionalEncoding, - normalize_before: bool = True, - ): - super().__init__() - attention_dim = encoder_output_size - - if input_layer == "embed": - self.embed = torch.nn.Sequential( - torch.nn.Embedding(vocab_size, attention_dim), - pos_enc_class(attention_dim, positional_dropout_rate), - ) - elif input_layer == "linear": - self.embed = torch.nn.Sequential( - torch.nn.Linear(vocab_size, attention_dim), - torch.nn.LayerNorm(attention_dim), - torch.nn.Dropout(dropout_rate), - torch.nn.ReLU(), - pos_enc_class(attention_dim, positional_dropout_rate), - ) - else: - raise ValueError(f"only 'embed' or 'linear' is supported: {input_layer}") - - self.normalize_before = normalize_before - if self.normalize_before: - self.after_norm = LayerNorm(attention_dim) - if use_output_layer: - self.output_layer = torch.nn.Linear(attention_dim, vocab_size) - else: - self.output_layer = None - - # Must set by the inheritance - self.decoders = None - - def forward( - self, - hs_pad: torch.Tensor, - hlens: torch.Tensor, - ys_in_pad: torch.Tensor, - ys_in_lens: torch.Tensor, - ) -> Tuple[torch.Tensor, torch.Tensor]: - """Forward decoder. - - Args: - hs_pad: encoded memory, float32 (batch, maxlen_in, feat) - hlens: (batch) - ys_in_pad: - input token ids, int64 (batch, maxlen_out) - if input_layer == "embed" - input tensor (batch, maxlen_out, #mels) in the other cases - ys_in_lens: (batch) - Returns: - (tuple): tuple containing: - - x: decoded token score before softmax (batch, maxlen_out, token) - if use_output_layer is True, - olens: (batch, ) - """ - tgt = ys_in_pad - # tgt_mask: (B, 1, L) - tgt_mask = (~make_pad_mask(ys_in_lens)[:, None, :]).to(tgt.device) - # m: (1, L, L) - m = subsequent_mask(tgt_mask.size(-1), device=tgt_mask.device).unsqueeze(0) - # tgt_mask: (B, L, L) - tgt_mask = tgt_mask & m - - memory = hs_pad - memory_mask = (~make_pad_mask(hlens, maxlen=memory.size(1)))[:, None, :].to( - memory.device - ) - # Padding for Longformer - if memory_mask.shape[-1] != memory.shape[1]: - padlen = memory.shape[1] - memory_mask.shape[-1] - memory_mask = torch.nn.functional.pad( - memory_mask, (0, padlen), "constant", False - ) - - x = self.embed(tgt) - x, tgt_mask, memory, memory_mask = self.decoders( - x, tgt_mask, memory, memory_mask - ) - if self.normalize_before: - x = self.after_norm(x) - if self.output_layer is not None: - x = self.output_layer(x) - - olens = tgt_mask.sum(1) - return x, olens - - def forward_one_step( - self, - tgt: torch.Tensor, - tgt_mask: torch.Tensor, - memory: torch.Tensor, - cache: List[torch.Tensor] = None, - ) -> Tuple[torch.Tensor, List[torch.Tensor]]: - """Forward one step. - - Args: - tgt: input token ids, int64 (batch, maxlen_out) - tgt_mask: input token mask, (batch, maxlen_out) - dtype=torch.uint8 in PyTorch 1.2- - dtype=torch.bool in PyTorch 1.2+ (include 1.2) - memory: encoded memory, float32 (batch, maxlen_in, feat) - cache: cached output list of (batch, max_time_out-1, size) - Returns: - y, cache: NN output value and cache per `self.decoders`. - y.shape` is (batch, maxlen_out, token) - """ - x = self.embed(tgt) - if cache is None: - cache = [None] * len(self.decoders) - new_cache = [] - for c, decoder in zip(cache, self.decoders): - x, tgt_mask, memory, memory_mask = decoder( - x, tgt_mask, memory, None, cache=c - ) - new_cache.append(x) - - if self.normalize_before: - y = self.after_norm(x[:, -1]) - else: - y = x[:, -1] - if self.output_layer is not None: - y = torch.log_softmax(self.output_layer(y), dim=-1) - - return y, new_cache - - def score(self, ys, state, x): - """Score.""" - ys_mask = subsequent_mask(len(ys), device=x.device).unsqueeze(0) - logp, state = self.forward_one_step( - ys.unsqueeze(0), ys_mask, x.unsqueeze(0), cache=state - ) - return logp.squeeze(0), state - - def batch_score( - self, ys: torch.Tensor, states: List[Any], xs: torch.Tensor - ) -> Tuple[torch.Tensor, List[Any]]: - """Score new token batch. - - Args: - ys (torch.Tensor): torch.int64 prefix tokens (n_batch, ylen). - states (List[Any]): Scorer states for prefix tokens. - xs (torch.Tensor): - The encoder feature that generates ys (n_batch, xlen, n_feat). - - Returns: - tuple[torch.Tensor, List[Any]]: Tuple of - batchfied scores for next token with shape of `(n_batch, n_vocab)` - and next state list for ys. - - """ - # merge states - n_batch = len(ys) - n_layers = len(self.decoders) - if states[0] is None: - batch_state = None - else: - # transpose state of [batch, layer] into [layer, batch] - batch_state = [ - torch.stack([states[b][i] for b in range(n_batch)]) - for i in range(n_layers) - ] - - # batch decoding - ys_mask = subsequent_mask(ys.size(-1), device=xs.device).unsqueeze(0) - logp, states = self.forward_one_step(ys, ys_mask, xs, cache=batch_state) - - # transpose state of [layer, batch] into [batch, layer] - state_list = [[states[i][b] for i in range(n_layers)] for b in range(n_batch)] - return logp, state_list - - -@tables.register("decoder_classes", "TransformerDecoder") -class TransformerDecoder(BaseTransformerDecoder): - def __init__( - self, - vocab_size: int, - encoder_output_size: int, - attention_heads: int = 4, - linear_units: int = 2048, - num_blocks: int = 6, - dropout_rate: float = 0.1, - positional_dropout_rate: float = 0.1, - self_attention_dropout_rate: float = 0.0, - src_attention_dropout_rate: float = 0.0, - input_layer: str = "embed", - use_output_layer: bool = True, - pos_enc_class=PositionalEncoding, - normalize_before: bool = True, - concat_after: bool = False, - ): - super().__init__( - vocab_size=vocab_size, - encoder_output_size=encoder_output_size, - dropout_rate=dropout_rate, - positional_dropout_rate=positional_dropout_rate, - input_layer=input_layer, - use_output_layer=use_output_layer, - pos_enc_class=pos_enc_class, - normalize_before=normalize_before, - ) - - attention_dim = encoder_output_size - self.decoders = repeat( - num_blocks, - lambda lnum: DecoderLayer( - attention_dim, - MultiHeadedAttention( - attention_heads, attention_dim, self_attention_dropout_rate - ), - MultiHeadedAttention( - attention_heads, attention_dim, src_attention_dropout_rate - ), - PositionwiseFeedForward(attention_dim, linear_units, dropout_rate), - dropout_rate, - normalize_before, - concat_after, - ), - ) - - -@tables.register("decoder_classes", "ParaformerDecoderSAN") -class ParaformerDecoderSAN(BaseTransformerDecoder): - """ - Author: Speech Lab of DAMO Academy, Alibaba Group - Paraformer: Fast and Accurate Parallel Transformer for Non-autoregressive End-to-End Speech Recognition - https://arxiv.org/abs/2006.01713 - """ - - def __init__( - self, - vocab_size: int, - encoder_output_size: int, - attention_heads: int = 4, - linear_units: int = 2048, - num_blocks: int = 6, - dropout_rate: float = 0.1, - positional_dropout_rate: float = 0.1, - self_attention_dropout_rate: float = 0.0, - src_attention_dropout_rate: float = 0.0, - input_layer: str = "embed", - use_output_layer: bool = True, - pos_enc_class=PositionalEncoding, - normalize_before: bool = True, - concat_after: bool = False, - embeds_id: int = -1, - ): - super().__init__( - vocab_size=vocab_size, - encoder_output_size=encoder_output_size, - dropout_rate=dropout_rate, - positional_dropout_rate=positional_dropout_rate, - input_layer=input_layer, - use_output_layer=use_output_layer, - pos_enc_class=pos_enc_class, - normalize_before=normalize_before, - ) - - attention_dim = encoder_output_size - self.decoders = repeat( - num_blocks, - lambda lnum: DecoderLayer( - attention_dim, - MultiHeadedAttention( - attention_heads, attention_dim, self_attention_dropout_rate - ), - MultiHeadedAttention( - attention_heads, attention_dim, src_attention_dropout_rate - ), - PositionwiseFeedForward(attention_dim, linear_units, dropout_rate), - dropout_rate, - normalize_before, - concat_after, - ), - ) - self.embeds_id = embeds_id - self.attention_dim = attention_dim - - def forward( - self, - hs_pad: torch.Tensor, - hlens: torch.Tensor, - ys_in_pad: torch.Tensor, - ys_in_lens: torch.Tensor, - ) -> Tuple[torch.Tensor, torch.Tensor]: - """Forward decoder. - - Args: - hs_pad: encoded memory, float32 (batch, maxlen_in, feat) - hlens: (batch) - ys_in_pad: - input token ids, int64 (batch, maxlen_out) - if input_layer == "embed" - input tensor (batch, maxlen_out, #mels) in the other cases - ys_in_lens: (batch) - Returns: - (tuple): tuple containing: - - x: decoded token score before softmax (batch, maxlen_out, token) - if use_output_layer is True, - olens: (batch, ) - """ - tgt = ys_in_pad - tgt_mask = (~make_pad_mask(ys_in_lens)[:, None, :]).to(tgt.device) - - memory = hs_pad - memory_mask = (~make_pad_mask(hlens, maxlen=memory.size(1)))[:, None, :].to( - memory.device - ) - # Padding for Longformer - if memory_mask.shape[-1] != memory.shape[1]: - padlen = memory.shape[1] - memory_mask.shape[-1] - memory_mask = torch.nn.functional.pad( - memory_mask, (0, padlen), "constant", False - ) - - # x = self.embed(tgt) - x = tgt - embeds_outputs = None - for layer_id, decoder in enumerate(self.decoders): - x, tgt_mask, memory, memory_mask = decoder(x, tgt_mask, memory, memory_mask) - if layer_id == self.embeds_id: - embeds_outputs = x - if self.normalize_before: - x = self.after_norm(x) - if self.output_layer is not None: - x = self.output_layer(x) - - olens = tgt_mask.sum(1) - if embeds_outputs is not None: - return x, olens, embeds_outputs - else: - return x, olens - - -@tables.register("decoder_classes", "LightweightConvolutionTransformerDecoder") -class LightweightConvolutionTransformerDecoder(BaseTransformerDecoder): - def __init__( - self, - vocab_size: int, - encoder_output_size: int, - attention_heads: int = 4, - linear_units: int = 2048, - num_blocks: int = 6, - dropout_rate: float = 0.1, - positional_dropout_rate: float = 0.1, - self_attention_dropout_rate: float = 0.0, - src_attention_dropout_rate: float = 0.0, - input_layer: str = "embed", - use_output_layer: bool = True, - pos_enc_class=PositionalEncoding, - normalize_before: bool = True, - concat_after: bool = False, - conv_wshare: int = 4, - conv_kernel_length: Sequence[int] = (11, 11, 11, 11, 11, 11), - conv_usebias: int = False, - ): - if len(conv_kernel_length) != num_blocks: - raise ValueError( - "conv_kernel_length must have equal number of values to num_blocks: " - f"{len(conv_kernel_length)} != {num_blocks}" - ) - super().__init__( - vocab_size=vocab_size, - encoder_output_size=encoder_output_size, - dropout_rate=dropout_rate, - positional_dropout_rate=positional_dropout_rate, - input_layer=input_layer, - use_output_layer=use_output_layer, - pos_enc_class=pos_enc_class, - normalize_before=normalize_before, - ) - - attention_dim = encoder_output_size - self.decoders = repeat( - num_blocks, - lambda lnum: DecoderLayer( - attention_dim, - LightweightConvolution( - wshare=conv_wshare, - n_feat=attention_dim, - dropout_rate=self_attention_dropout_rate, - kernel_size=conv_kernel_length[lnum], - use_kernel_mask=True, - use_bias=conv_usebias, - ), - MultiHeadedAttention( - attention_heads, attention_dim, src_attention_dropout_rate - ), - PositionwiseFeedForward(attention_dim, linear_units, dropout_rate), - dropout_rate, - normalize_before, - concat_after, - ), - ) - - -@tables.register("decoder_classes", "LightweightConvolution2DTransformerDecoder") -class LightweightConvolution2DTransformerDecoder(BaseTransformerDecoder): - def __init__( - self, - vocab_size: int, - encoder_output_size: int, - attention_heads: int = 4, - linear_units: int = 2048, - num_blocks: int = 6, - dropout_rate: float = 0.1, - positional_dropout_rate: float = 0.1, - self_attention_dropout_rate: float = 0.0, - src_attention_dropout_rate: float = 0.0, - input_layer: str = "embed", - use_output_layer: bool = True, - pos_enc_class=PositionalEncoding, - normalize_before: bool = True, - concat_after: bool = False, - conv_wshare: int = 4, - conv_kernel_length: Sequence[int] = (11, 11, 11, 11, 11, 11), - conv_usebias: int = False, - ): - if len(conv_kernel_length) != num_blocks: - raise ValueError( - "conv_kernel_length must have equal number of values to num_blocks: " - f"{len(conv_kernel_length)} != {num_blocks}" - ) - super().__init__( - vocab_size=vocab_size, - encoder_output_size=encoder_output_size, - dropout_rate=dropout_rate, - positional_dropout_rate=positional_dropout_rate, - input_layer=input_layer, - use_output_layer=use_output_layer, - pos_enc_class=pos_enc_class, - normalize_before=normalize_before, - ) - - attention_dim = encoder_output_size - self.decoders = repeat( - num_blocks, - lambda lnum: DecoderLayer( - attention_dim, - LightweightConvolution2D( - wshare=conv_wshare, - n_feat=attention_dim, - dropout_rate=self_attention_dropout_rate, - kernel_size=conv_kernel_length[lnum], - use_kernel_mask=True, - use_bias=conv_usebias, - ), - MultiHeadedAttention( - attention_heads, attention_dim, src_attention_dropout_rate - ), - PositionwiseFeedForward(attention_dim, linear_units, dropout_rate), - dropout_rate, - normalize_before, - concat_after, - ), - ) - - -@tables.register("decoder_classes", "DynamicConvolutionTransformerDecoder") -class DynamicConvolutionTransformerDecoder(BaseTransformerDecoder): - def __init__( - self, - vocab_size: int, - encoder_output_size: int, - attention_heads: int = 4, - linear_units: int = 2048, - num_blocks: int = 6, - dropout_rate: float = 0.1, - positional_dropout_rate: float = 0.1, - self_attention_dropout_rate: float = 0.0, - src_attention_dropout_rate: float = 0.0, - input_layer: str = "embed", - use_output_layer: bool = True, - pos_enc_class=PositionalEncoding, - normalize_before: bool = True, - concat_after: bool = False, - conv_wshare: int = 4, - conv_kernel_length: Sequence[int] = (11, 11, 11, 11, 11, 11), - conv_usebias: int = False, - ): - if len(conv_kernel_length) != num_blocks: - raise ValueError( - "conv_kernel_length must have equal number of values to num_blocks: " - f"{len(conv_kernel_length)} != {num_blocks}" - ) - super().__init__( - vocab_size=vocab_size, - encoder_output_size=encoder_output_size, - dropout_rate=dropout_rate, - positional_dropout_rate=positional_dropout_rate, - input_layer=input_layer, - use_output_layer=use_output_layer, - pos_enc_class=pos_enc_class, - normalize_before=normalize_before, - ) - attention_dim = encoder_output_size - - self.decoders = repeat( - num_blocks, - lambda lnum: DecoderLayer( - attention_dim, - DynamicConvolution( - wshare=conv_wshare, - n_feat=attention_dim, - dropout_rate=self_attention_dropout_rate, - kernel_size=conv_kernel_length[lnum], - use_kernel_mask=True, - use_bias=conv_usebias, - ), - MultiHeadedAttention( - attention_heads, attention_dim, src_attention_dropout_rate - ), - PositionwiseFeedForward(attention_dim, linear_units, dropout_rate), - dropout_rate, - normalize_before, - concat_after, - ), - ) - - -@tables.register("decoder_classes", "DynamicConvolution2DTransformerDecoder") -class DynamicConvolution2DTransformerDecoder(BaseTransformerDecoder): - def __init__( - self, - vocab_size: int, - encoder_output_size: int, - attention_heads: int = 4, - linear_units: int = 2048, - num_blocks: int = 6, - dropout_rate: float = 0.1, - positional_dropout_rate: float = 0.1, - self_attention_dropout_rate: float = 0.0, - src_attention_dropout_rate: float = 0.0, - input_layer: str = "embed", - use_output_layer: bool = True, - pos_enc_class=PositionalEncoding, - normalize_before: bool = True, - concat_after: bool = False, - conv_wshare: int = 4, - conv_kernel_length: Sequence[int] = (11, 11, 11, 11, 11, 11), - conv_usebias: int = False, - ): - if len(conv_kernel_length) != num_blocks: - raise ValueError( - "conv_kernel_length must have equal number of values to num_blocks: " - f"{len(conv_kernel_length)} != {num_blocks}" - ) - super().__init__( - vocab_size=vocab_size, - encoder_output_size=encoder_output_size, - dropout_rate=dropout_rate, - positional_dropout_rate=positional_dropout_rate, - input_layer=input_layer, - use_output_layer=use_output_layer, - pos_enc_class=pos_enc_class, - normalize_before=normalize_before, - ) - attention_dim = encoder_output_size - - self.decoders = repeat( - num_blocks, - lambda lnum: DecoderLayer( - attention_dim, - DynamicConvolution2D( - wshare=conv_wshare, - n_feat=attention_dim, - dropout_rate=self_attention_dropout_rate, - kernel_size=conv_kernel_length[lnum], - use_kernel_mask=True, - use_bias=conv_usebias, - ), - MultiHeadedAttention( - attention_heads, attention_dim, src_attention_dropout_rate - ), - PositionwiseFeedForward(attention_dim, linear_units, dropout_rate), - dropout_rate, - normalize_before, - concat_after, - ), - ) diff --git a/funasr_detach/models/sanm/__init__.py b/funasr_detach/models/sanm/__init__.py deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/funasr_detach/models/sanm/attention.py b/funasr_detach/models/sanm/attention.py deleted file mode 100644 index d8c221ad7517265a709cf3f6fe0c447bdbe6b2b5..0000000000000000000000000000000000000000 --- a/funasr_detach/models/sanm/attention.py +++ /dev/null @@ -1,718 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- - -# Copyright 2019 Shigeki Karita -# Apache 2.0 (http://www.apache.org/licenses/LICENSE-2.0) - -"""Multi-Head Attention layer definition.""" - -import math - -import numpy -import torch -from torch import nn -from typing import Optional, Tuple - -import torch.nn.functional as F -from funasr_detach.models.transformer.utils.nets_utils import make_pad_mask -import funasr_detach.models.lora.layers as lora - - -class MultiHeadedAttention(nn.Module): - """Multi-Head Attention layer. - - Args: - n_head (int): The number of heads. - n_feat (int): The number of features. - dropout_rate (float): Dropout rate. - - """ - - def __init__(self, n_head, n_feat, dropout_rate): - """Construct an MultiHeadedAttention object.""" - super(MultiHeadedAttention, self).__init__() - assert n_feat % n_head == 0 - # We assume d_v always equals d_k - self.d_k = n_feat // n_head - self.h = n_head - self.linear_q = nn.Linear(n_feat, n_feat) - self.linear_k = nn.Linear(n_feat, n_feat) - self.linear_v = nn.Linear(n_feat, n_feat) - self.linear_out = nn.Linear(n_feat, n_feat) - self.attn = None - self.dropout = nn.Dropout(p=dropout_rate) - - def forward_qkv(self, query, key, value): - """Transform query, key and value. - - Args: - query (torch.Tensor): Query tensor (#batch, time1, size). - key (torch.Tensor): Key tensor (#batch, time2, size). - value (torch.Tensor): Value tensor (#batch, time2, size). - - Returns: - torch.Tensor: Transformed query tensor (#batch, n_head, time1, d_k). - torch.Tensor: Transformed key tensor (#batch, n_head, time2, d_k). - torch.Tensor: Transformed value tensor (#batch, n_head, time2, d_k). - - """ - n_batch = query.size(0) - q = self.linear_q(query).view(n_batch, -1, self.h, self.d_k) - k = self.linear_k(key).view(n_batch, -1, self.h, self.d_k) - v = self.linear_v(value).view(n_batch, -1, self.h, self.d_k) - q = q.transpose(1, 2) # (batch, head, time1, d_k) - k = k.transpose(1, 2) # (batch, head, time2, d_k) - v = v.transpose(1, 2) # (batch, head, time2, d_k) - - return q, k, v - - def forward_attention(self, value, scores, mask): - """Compute attention context vector. - - Args: - value (torch.Tensor): Transformed value (#batch, n_head, time2, d_k). - scores (torch.Tensor): Attention score (#batch, n_head, time1, time2). - mask (torch.Tensor): Mask (#batch, 1, time2) or (#batch, time1, time2). - - Returns: - torch.Tensor: Transformed value (#batch, time1, d_model) - weighted by the attention score (#batch, time1, time2). - - """ - n_batch = value.size(0) - if mask is not None: - mask = mask.unsqueeze(1).eq(0) # (batch, 1, *, time2) - min_value = float( - numpy.finfo(torch.tensor(0, dtype=scores.dtype).numpy().dtype).min - ) - scores = scores.masked_fill(mask, min_value) - self.attn = torch.softmax(scores, dim=-1).masked_fill( - mask, 0.0 - ) # (batch, head, time1, time2) - else: - self.attn = torch.softmax(scores, dim=-1) # (batch, head, time1, time2) - - p_attn = self.dropout(self.attn) - x = torch.matmul(p_attn, value) # (batch, head, time1, d_k) - x = ( - x.transpose(1, 2).contiguous().view(n_batch, -1, self.h * self.d_k) - ) # (batch, time1, d_model) - - return self.linear_out(x) # (batch, time1, d_model) - - def forward(self, query, key, value, mask): - """Compute scaled dot product attention. - - Args: - query (torch.Tensor): Query tensor (#batch, time1, size). - key (torch.Tensor): Key tensor (#batch, time2, size). - value (torch.Tensor): Value tensor (#batch, time2, size). - mask (torch.Tensor): Mask tensor (#batch, 1, time2) or - (#batch, time1, time2). - - Returns: - torch.Tensor: Output tensor (#batch, time1, d_model). - - """ - q, k, v = self.forward_qkv(query, key, value) - scores = torch.matmul(q, k.transpose(-2, -1)) / math.sqrt(self.d_k) - return self.forward_attention(v, scores, mask) - - -class MultiHeadedAttentionSANM(nn.Module): - """Multi-Head Attention layer. - - Args: - n_head (int): The number of heads. - n_feat (int): The number of features. - dropout_rate (float): Dropout rate. - - """ - - def __init__( - self, - n_head, - in_feat, - n_feat, - dropout_rate, - kernel_size, - sanm_shfit=0, - lora_list=None, - lora_rank=8, - lora_alpha=16, - lora_dropout=0.1, - ): - """Construct an MultiHeadedAttention object.""" - super().__init__() - assert n_feat % n_head == 0 - # We assume d_v always equals d_k - self.d_k = n_feat // n_head - self.h = n_head - # self.linear_q = nn.Linear(n_feat, n_feat) - # self.linear_k = nn.Linear(n_feat, n_feat) - # self.linear_v = nn.Linear(n_feat, n_feat) - if lora_list is not None: - if "o" in lora_list: - self.linear_out = lora.Linear( - n_feat, - n_feat, - r=lora_rank, - lora_alpha=lora_alpha, - lora_dropout=lora_dropout, - ) - else: - self.linear_out = nn.Linear(n_feat, n_feat) - lora_qkv_list = ["q" in lora_list, "k" in lora_list, "v" in lora_list] - if lora_qkv_list == [False, False, False]: - self.linear_q_k_v = nn.Linear(in_feat, n_feat * 3) - else: - self.linear_q_k_v = lora.MergedLinear( - in_feat, - n_feat * 3, - r=lora_rank, - lora_alpha=lora_alpha, - lora_dropout=lora_dropout, - enable_lora=lora_qkv_list, - ) - else: - self.linear_out = nn.Linear(n_feat, n_feat) - self.linear_q_k_v = nn.Linear(in_feat, n_feat * 3) - self.attn = None - self.dropout = nn.Dropout(p=dropout_rate) - - self.fsmn_block = nn.Conv1d( - n_feat, n_feat, kernel_size, stride=1, padding=0, groups=n_feat, bias=False - ) - # padding - left_padding = (kernel_size - 1) // 2 - if sanm_shfit > 0: - left_padding = left_padding + sanm_shfit - right_padding = kernel_size - 1 - left_padding - self.pad_fn = nn.ConstantPad1d((left_padding, right_padding), 0.0) - - def forward_fsmn(self, inputs, mask, mask_shfit_chunk=None): - b, t, d = inputs.size() - if mask is not None: - mask = torch.reshape(mask, (b, -1, 1)) - if mask_shfit_chunk is not None: - mask = mask * mask_shfit_chunk - inputs = inputs * mask - - x = inputs.transpose(1, 2) - x = self.pad_fn(x) - x = self.fsmn_block(x) - x = x.transpose(1, 2) - x += inputs - x = self.dropout(x) - if mask is not None: - x = x * mask - return x - - def forward_qkv(self, x): - """Transform query, key and value. - - Args: - query (torch.Tensor): Query tensor (#batch, time1, size). - key (torch.Tensor): Key tensor (#batch, time2, size). - value (torch.Tensor): Value tensor (#batch, time2, size). - - Returns: - torch.Tensor: Transformed query tensor (#batch, n_head, time1, d_k). - torch.Tensor: Transformed key tensor (#batch, n_head, time2, d_k). - torch.Tensor: Transformed value tensor (#batch, n_head, time2, d_k). - - """ - b, t, d = x.size() - q_k_v = self.linear_q_k_v(x) - q, k, v = torch.split(q_k_v, int(self.h * self.d_k), dim=-1) - q_h = torch.reshape(q, (b, t, self.h, self.d_k)).transpose( - 1, 2 - ) # (batch, head, time1, d_k) - k_h = torch.reshape(k, (b, t, self.h, self.d_k)).transpose( - 1, 2 - ) # (batch, head, time2, d_k) - v_h = torch.reshape(v, (b, t, self.h, self.d_k)).transpose( - 1, 2 - ) # (batch, head, time2, d_k) - - return q_h, k_h, v_h, v - - def forward_attention(self, value, scores, mask, mask_att_chunk_encoder=None): - """Compute attention context vector. - - Args: - value (torch.Tensor): Transformed value (#batch, n_head, time2, d_k). - scores (torch.Tensor): Attention score (#batch, n_head, time1, time2). - mask (torch.Tensor): Mask (#batch, 1, time2) or (#batch, time1, time2). - - Returns: - torch.Tensor: Transformed value (#batch, time1, d_model) - weighted by the attention score (#batch, time1, time2). - - """ - n_batch = value.size(0) - if mask is not None: - if mask_att_chunk_encoder is not None: - mask = mask * mask_att_chunk_encoder - - mask = mask.unsqueeze(1).eq(0) # (batch, 1, *, time2) - - min_value = float( - numpy.finfo(torch.tensor(0, dtype=scores.dtype).numpy().dtype).min - ) - scores = scores.masked_fill(mask, min_value) - self.attn = torch.softmax(scores, dim=-1).masked_fill( - mask, 0.0 - ) # (batch, head, time1, time2) - else: - self.attn = torch.softmax(scores, dim=-1) # (batch, head, time1, time2) - - p_attn = self.dropout(self.attn) - x = torch.matmul(p_attn, value) # (batch, head, time1, d_k) - x = ( - x.transpose(1, 2).contiguous().view(n_batch, -1, self.h * self.d_k) - ) # (batch, time1, d_model) - - return self.linear_out(x) # (batch, time1, d_model) - - def forward(self, x, mask, mask_shfit_chunk=None, mask_att_chunk_encoder=None): - """Compute scaled dot product attention. - - Args: - query (torch.Tensor): Query tensor (#batch, time1, size). - key (torch.Tensor): Key tensor (#batch, time2, size). - value (torch.Tensor): Value tensor (#batch, time2, size). - mask (torch.Tensor): Mask tensor (#batch, 1, time2) or - (#batch, time1, time2). - - Returns: - torch.Tensor: Output tensor (#batch, time1, d_model). - - """ - q_h, k_h, v_h, v = self.forward_qkv(x) - fsmn_memory = self.forward_fsmn(v, mask, mask_shfit_chunk) - q_h = q_h * self.d_k ** (-0.5) - scores = torch.matmul(q_h, k_h.transpose(-2, -1)) - att_outs = self.forward_attention(v_h, scores, mask, mask_att_chunk_encoder) - return att_outs + fsmn_memory - - def forward_chunk(self, x, cache=None, chunk_size=None, look_back=0): - """Compute scaled dot product attention. - - Args: - query (torch.Tensor): Query tensor (#batch, time1, size). - key (torch.Tensor): Key tensor (#batch, time2, size). - value (torch.Tensor): Value tensor (#batch, time2, size). - mask (torch.Tensor): Mask tensor (#batch, 1, time2) or - (#batch, time1, time2). - - Returns: - torch.Tensor: Output tensor (#batch, time1, d_model). - - """ - q_h, k_h, v_h, v = self.forward_qkv(x) - if chunk_size is not None and look_back > 0 or look_back == -1: - if cache is not None: - k_h_stride = k_h[:, :, : -(chunk_size[2]), :] - v_h_stride = v_h[:, :, : -(chunk_size[2]), :] - k_h = torch.cat((cache["k"], k_h), dim=2) - v_h = torch.cat((cache["v"], v_h), dim=2) - - cache["k"] = torch.cat((cache["k"], k_h_stride), dim=2) - cache["v"] = torch.cat((cache["v"], v_h_stride), dim=2) - if look_back != -1: - cache["k"] = cache["k"][:, :, -(look_back * chunk_size[1]) :, :] - cache["v"] = cache["v"][:, :, -(look_back * chunk_size[1]) :, :] - else: - cache_tmp = { - "k": k_h[:, :, : -(chunk_size[2]), :], - "v": v_h[:, :, : -(chunk_size[2]), :], - } - cache = cache_tmp - fsmn_memory = self.forward_fsmn(v, None) - q_h = q_h * self.d_k ** (-0.5) - scores = torch.matmul(q_h, k_h.transpose(-2, -1)) - att_outs = self.forward_attention(v_h, scores, None) - return att_outs + fsmn_memory, cache - - -class MultiHeadedAttentionSANMDecoder(nn.Module): - """Multi-Head Attention layer. - - Args: - n_head (int): The number of heads. - n_feat (int): The number of features. - dropout_rate (float): Dropout rate. - - """ - - def __init__(self, n_feat, dropout_rate, kernel_size, sanm_shfit=0): - """Construct an MultiHeadedAttention object.""" - super(MultiHeadedAttentionSANMDecoder, self).__init__() - - self.dropout = nn.Dropout(p=dropout_rate) - - self.fsmn_block = nn.Conv1d( - n_feat, n_feat, kernel_size, stride=1, padding=0, groups=n_feat, bias=False - ) - # padding - # padding - left_padding = (kernel_size - 1) // 2 - if sanm_shfit > 0: - left_padding = left_padding + sanm_shfit - right_padding = kernel_size - 1 - left_padding - self.pad_fn = nn.ConstantPad1d((left_padding, right_padding), 0.0) - self.kernel_size = kernel_size - - def forward(self, inputs, mask, cache=None, mask_shfit_chunk=None): - """ - :param x: (#batch, time1, size). - :param mask: Mask tensor (#batch, 1, time) - :return: - """ - # print("in fsmn, inputs", inputs.size()) - b, t, d = inputs.size() - # logging.info( - # "mask: {}".format(mask.size())) - if mask is not None: - mask = torch.reshape(mask, (b, -1, 1)) - # logging.info("in fsmn, mask: {}, {}".format(mask.size(), mask[0:100:50, :, :])) - if mask_shfit_chunk is not None: - # logging.info("in fsmn, mask_fsmn: {}, {}".format(mask_shfit_chunk.size(), mask_shfit_chunk[0:100:50, :, :])) - mask = mask * mask_shfit_chunk - # logging.info("in fsmn, mask_after_fsmn: {}, {}".format(mask.size(), mask[0:100:50, :, :])) - # print("in fsmn, mask", mask.size()) - # print("in fsmn, inputs", inputs.size()) - inputs = inputs * mask - - x = inputs.transpose(1, 2) - b, d, t = x.size() - if cache is None: - # print("in fsmn, cache is None, x", x.size()) - - x = self.pad_fn(x) - if not self.training: - cache = x - else: - # print("in fsmn, cache is not None, x", x.size()) - # x = torch.cat((x, cache), dim=2)[:, :, :-1] - # if t < self.kernel_size: - # x = self.pad_fn(x) - x = torch.cat((cache[:, :, 1:], x), dim=2) - x = x[:, :, -(self.kernel_size + t - 1) :] - # print("in fsmn, cache is not None, x_cat", x.size()) - cache = x - x = self.fsmn_block(x) - x = x.transpose(1, 2) - # print("in fsmn, fsmn_out", x.size()) - if x.size(1) != inputs.size(1): - inputs = inputs[:, -1, :] - - x = x + inputs - x = self.dropout(x) - if mask is not None: - x = x * mask - return x, cache - - -class MultiHeadedAttentionCrossAtt(nn.Module): - """Multi-Head Attention layer. - - Args: - n_head (int): The number of heads. - n_feat (int): The number of features. - dropout_rate (float): Dropout rate. - - """ - - def __init__( - self, - n_head, - n_feat, - dropout_rate, - lora_list=None, - lora_rank=8, - lora_alpha=16, - lora_dropout=0.1, - encoder_output_size=None, - ): - """Construct an MultiHeadedAttention object.""" - super(MultiHeadedAttentionCrossAtt, self).__init__() - assert n_feat % n_head == 0 - # We assume d_v always equals d_k - self.d_k = n_feat // n_head - self.h = n_head - if lora_list is not None: - if "q" in lora_list: - self.linear_q = lora.Linear( - n_feat, - n_feat, - r=lora_rank, - lora_alpha=lora_alpha, - lora_dropout=lora_dropout, - ) - else: - self.linear_q = nn.Linear(n_feat, n_feat) - lora_kv_list = ["k" in lora_list, "v" in lora_list] - if lora_kv_list == [False, False]: - self.linear_k_v = nn.Linear( - n_feat if encoder_output_size is None else encoder_output_size, - n_feat * 2, - ) - else: - self.linear_k_v = lora.MergedLinear( - n_feat if encoder_output_size is None else encoder_output_size, - n_feat * 2, - r=lora_rank, - lora_alpha=lora_alpha, - lora_dropout=lora_dropout, - enable_lora=lora_kv_list, - ) - if "o" in lora_list: - self.linear_out = lora.Linear( - n_feat, - n_feat, - r=lora_rank, - lora_alpha=lora_alpha, - lora_dropout=lora_dropout, - ) - else: - self.linear_out = nn.Linear(n_feat, n_feat) - else: - self.linear_q = nn.Linear(n_feat, n_feat) - self.linear_k_v = nn.Linear( - n_feat if encoder_output_size is None else encoder_output_size, - n_feat * 2, - ) - self.linear_out = nn.Linear(n_feat, n_feat) - self.attn = None - self.dropout = nn.Dropout(p=dropout_rate) - - def forward_qkv(self, x, memory): - """Transform query, key and value. - - Args: - query (torch.Tensor): Query tensor (#batch, time1, size). - key (torch.Tensor): Key tensor (#batch, time2, size). - value (torch.Tensor): Value tensor (#batch, time2, size). - - Returns: - torch.Tensor: Transformed query tensor (#batch, n_head, time1, d_k). - torch.Tensor: Transformed key tensor (#batch, n_head, time2, d_k). - torch.Tensor: Transformed value tensor (#batch, n_head, time2, d_k). - - """ - - # print("in forward_qkv, x", x.size()) - b = x.size(0) - q = self.linear_q(x) - q_h = torch.reshape(q, (b, -1, self.h, self.d_k)).transpose( - 1, 2 - ) # (batch, head, time1, d_k) - - k_v = self.linear_k_v(memory) - k, v = torch.split(k_v, int(self.h * self.d_k), dim=-1) - k_h = torch.reshape(k, (b, -1, self.h, self.d_k)).transpose( - 1, 2 - ) # (batch, head, time2, d_k) - v_h = torch.reshape(v, (b, -1, self.h, self.d_k)).transpose( - 1, 2 - ) # (batch, head, time2, d_k) - - return q_h, k_h, v_h - - def forward_attention(self, value, scores, mask, ret_attn=False): - """Compute attention context vector. - - Args: - value (torch.Tensor): Transformed value (#batch, n_head, time2, d_k). - scores (torch.Tensor): Attention score (#batch, n_head, time1, time2). - mask (torch.Tensor): Mask (#batch, 1, time2) or (#batch, time1, time2). - - Returns: - torch.Tensor: Transformed value (#batch, time1, d_model) - weighted by the attention score (#batch, time1, time2). - - """ - n_batch = value.size(0) - if mask is not None: - mask = mask.unsqueeze(1).eq(0) # (batch, 1, *, time2) - min_value = float( - numpy.finfo(torch.tensor(0, dtype=scores.dtype).numpy().dtype).min - ) - # logging.info( - # "scores: {}, mask_size: {}".format(scores.size(), mask.size())) - scores = scores.masked_fill(mask, min_value) - self.attn = torch.softmax(scores, dim=-1).masked_fill( - mask, 0.0 - ) # (batch, head, time1, time2) - else: - self.attn = torch.softmax(scores, dim=-1) # (batch, head, time1, time2) - p_attn = self.dropout(self.attn) - x = torch.matmul(p_attn, value) # (batch, head, time1, d_k) - x = ( - x.transpose(1, 2).contiguous().view(n_batch, -1, self.h * self.d_k) - ) # (batch, time1, d_model) - if ret_attn: - return self.linear_out(x), self.attn # (batch, time1, d_model) - return self.linear_out(x) # (batch, time1, d_model) - - def forward(self, x, memory, memory_mask, ret_attn=False): - """Compute scaled dot product attention. - - Args: - query (torch.Tensor): Query tensor (#batch, time1, size). - key (torch.Tensor): Key tensor (#batch, time2, size). - value (torch.Tensor): Value tensor (#batch, time2, size). - mask (torch.Tensor): Mask tensor (#batch, 1, time2) or - (#batch, time1, time2). - - Returns: - torch.Tensor: Output tensor (#batch, time1, d_model). - - """ - q_h, k_h, v_h = self.forward_qkv(x, memory) - q_h = q_h * self.d_k ** (-0.5) - scores = torch.matmul(q_h, k_h.transpose(-2, -1)) - return self.forward_attention(v_h, scores, memory_mask, ret_attn=ret_attn) - - def forward_chunk(self, x, memory, cache=None, chunk_size=None, look_back=0): - """Compute scaled dot product attention. - - Args: - query (torch.Tensor): Query tensor (#batch, time1, size). - key (torch.Tensor): Key tensor (#batch, time2, size). - value (torch.Tensor): Value tensor (#batch, time2, size). - mask (torch.Tensor): Mask tensor (#batch, 1, time2) or - (#batch, time1, time2). - - Returns: - torch.Tensor: Output tensor (#batch, time1, d_model). - - """ - q_h, k_h, v_h = self.forward_qkv(x, memory) - if chunk_size is not None and look_back > 0: - if cache is not None: - k_h = torch.cat((cache["k"], k_h), dim=2) - v_h = torch.cat((cache["v"], v_h), dim=2) - cache["k"] = k_h[:, :, -(look_back * chunk_size[1]) :, :] - cache["v"] = v_h[:, :, -(look_back * chunk_size[1]) :, :] - else: - cache_tmp = { - "k": k_h[:, :, -(look_back * chunk_size[1]) :, :], - "v": v_h[:, :, -(look_back * chunk_size[1]) :, :], - } - cache = cache_tmp - q_h = q_h * self.d_k ** (-0.5) - scores = torch.matmul(q_h, k_h.transpose(-2, -1)) - return self.forward_attention(v_h, scores, None), cache - - -class MultiHeadSelfAttention(nn.Module): - """Multi-Head Attention layer. - - Args: - n_head (int): The number of heads. - n_feat (int): The number of features. - dropout_rate (float): Dropout rate. - - """ - - def __init__(self, n_head, in_feat, n_feat, dropout_rate): - """Construct an MultiHeadedAttention object.""" - super(MultiHeadSelfAttention, self).__init__() - assert n_feat % n_head == 0 - # We assume d_v always equals d_k - self.d_k = n_feat // n_head - self.h = n_head - self.linear_out = nn.Linear(n_feat, n_feat) - self.linear_q_k_v = nn.Linear(in_feat, n_feat * 3) - self.attn = None - self.dropout = nn.Dropout(p=dropout_rate) - - def forward_qkv(self, x): - """Transform query, key and value. - - Args: - query (torch.Tensor): Query tensor (#batch, time1, size). - key (torch.Tensor): Key tensor (#batch, time2, size). - value (torch.Tensor): Value tensor (#batch, time2, size). - - Returns: - torch.Tensor: Transformed query tensor (#batch, n_head, time1, d_k). - torch.Tensor: Transformed key tensor (#batch, n_head, time2, d_k). - torch.Tensor: Transformed value tensor (#batch, n_head, time2, d_k). - - """ - b, t, d = x.size() - q_k_v = self.linear_q_k_v(x) - q, k, v = torch.split(q_k_v, int(self.h * self.d_k), dim=-1) - q_h = torch.reshape(q, (b, t, self.h, self.d_k)).transpose( - 1, 2 - ) # (batch, head, time1, d_k) - k_h = torch.reshape(k, (b, t, self.h, self.d_k)).transpose( - 1, 2 - ) # (batch, head, time2, d_k) - v_h = torch.reshape(v, (b, t, self.h, self.d_k)).transpose( - 1, 2 - ) # (batch, head, time2, d_k) - - return q_h, k_h, v_h, v - - def forward_attention(self, value, scores, mask, mask_att_chunk_encoder=None): - """Compute attention context vector. - - Args: - value (torch.Tensor): Transformed value (#batch, n_head, time2, d_k). - scores (torch.Tensor): Attention score (#batch, n_head, time1, time2). - mask (torch.Tensor): Mask (#batch, 1, time2) or (#batch, time1, time2). - - Returns: - torch.Tensor: Transformed value (#batch, time1, d_model) - weighted by the attention score (#batch, time1, time2). - - """ - n_batch = value.size(0) - if mask is not None: - if mask_att_chunk_encoder is not None: - mask = mask * mask_att_chunk_encoder - - mask = mask.unsqueeze(1).eq(0) # (batch, 1, *, time2) - - min_value = float( - numpy.finfo(torch.tensor(0, dtype=scores.dtype).numpy().dtype).min - ) - scores = scores.masked_fill(mask, min_value) - self.attn = torch.softmax(scores, dim=-1).masked_fill( - mask, 0.0 - ) # (batch, head, time1, time2) - else: - self.attn = torch.softmax(scores, dim=-1) # (batch, head, time1, time2) - - p_attn = self.dropout(self.attn) - x = torch.matmul(p_attn, value) # (batch, head, time1, d_k) - x = ( - x.transpose(1, 2).contiguous().view(n_batch, -1, self.h * self.d_k) - ) # (batch, time1, d_model) - - return self.linear_out(x) # (batch, time1, d_model) - - def forward(self, x, mask, mask_att_chunk_encoder=None): - """Compute scaled dot product attention. - - Args: - query (torch.Tensor): Query tensor (#batch, time1, size). - key (torch.Tensor): Key tensor (#batch, time2, size). - value (torch.Tensor): Value tensor (#batch, time2, size). - mask (torch.Tensor): Mask tensor (#batch, 1, time2) or - (#batch, time1, time2). - - Returns: - torch.Tensor: Output tensor (#batch, time1, d_model). - - """ - q_h, k_h, v_h, v = self.forward_qkv(x) - q_h = q_h * self.d_k ** (-0.5) - scores = torch.matmul(q_h, k_h.transpose(-2, -1)) - att_outs = self.forward_attention(v_h, scores, mask, mask_att_chunk_encoder) - return att_outs diff --git a/funasr_detach/models/sanm/decoder.py b/funasr_detach/models/sanm/decoder.py deleted file mode 100644 index 3700534f651f870aab04eb6053e99dde09eb6288..0000000000000000000000000000000000000000 --- a/funasr_detach/models/sanm/decoder.py +++ /dev/null @@ -1,517 +0,0 @@ -#!/usr/bin/env python3 -# -*- encoding: utf-8 -*- -# Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved. -# MIT License (https://opensource.org/licenses/MIT) - -from typing import List -from typing import Tuple -import logging -import torch -import torch.nn as nn -import numpy as np - -from funasr_detach.models.scama import utils as myutils -from funasr_detach.models.transformer.decoder import BaseTransformerDecoder - -from funasr_detach.models.sanm.attention import ( - MultiHeadedAttentionSANMDecoder, - MultiHeadedAttentionCrossAtt, -) -from funasr_detach.models.transformer.embedding import PositionalEncoding -from funasr_detach.models.transformer.layer_norm import LayerNorm -from funasr_detach.models.sanm.positionwise_feed_forward import ( - PositionwiseFeedForwardDecoderSANM, -) -from funasr_detach.models.transformer.utils.repeat import repeat - -from funasr_detach.register import tables - - -class DecoderLayerSANM(nn.Module): - """Single decoder layer module. - - Args: - size (int): Input dimension. - self_attn (torch.nn.Module): Self-attention module instance. - `MultiHeadedAttention` instance can be used as the argument. - src_attn (torch.nn.Module): Self-attention module instance. - `MultiHeadedAttention` instance can be used as the argument. - feed_forward (torch.nn.Module): Feed-forward module instance. - `PositionwiseFeedForward`, `MultiLayeredConv1d`, or `Conv1dLinear` instance - can be used as the argument. - dropout_rate (float): Dropout rate. - normalize_before (bool): Whether to use layer_norm before the first block. - concat_after (bool): Whether to concat attention layer's input and output. - if True, additional linear will be applied. - i.e. x -> x + linear(concat(x, att(x))) - if False, no additional linear will be applied. i.e. x -> x + att(x) - - - """ - - def __init__( - self, - size, - self_attn, - src_attn, - feed_forward, - dropout_rate, - normalize_before=True, - concat_after=False, - ): - """Construct an DecoderLayer object.""" - super(DecoderLayerSANM, self).__init__() - self.size = size - self.self_attn = self_attn - self.src_attn = src_attn - self.feed_forward = feed_forward - self.norm1 = LayerNorm(size) - if self_attn is not None: - self.norm2 = LayerNorm(size) - if src_attn is not None: - self.norm3 = LayerNorm(size) - self.dropout = nn.Dropout(dropout_rate) - self.normalize_before = normalize_before - self.concat_after = concat_after - if self.concat_after: - self.concat_linear1 = nn.Linear(size + size, size) - self.concat_linear2 = nn.Linear(size + size, size) - - def forward(self, tgt, tgt_mask, memory, memory_mask=None, cache=None): - """Compute decoded features. - - Args: - tgt (torch.Tensor): Input tensor (#batch, maxlen_out, size). - tgt_mask (torch.Tensor): Mask for input tensor (#batch, maxlen_out). - memory (torch.Tensor): Encoded memory, float32 (#batch, maxlen_in, size). - memory_mask (torch.Tensor): Encoded memory mask (#batch, maxlen_in). - cache (List[torch.Tensor]): List of cached tensors. - Each tensor shape should be (#batch, maxlen_out - 1, size). - - Returns: - torch.Tensor: Output tensor(#batch, maxlen_out, size). - torch.Tensor: Mask for output tensor (#batch, maxlen_out). - torch.Tensor: Encoded memory (#batch, maxlen_in, size). - torch.Tensor: Encoded memory mask (#batch, maxlen_in). - - """ - # tgt = self.dropout(tgt) - residual = tgt - if self.normalize_before: - tgt = self.norm1(tgt) - tgt = self.feed_forward(tgt) - - x = tgt - if self.self_attn: - if self.normalize_before: - tgt = self.norm2(tgt) - x, _ = self.self_attn(tgt, tgt_mask) - x = residual + self.dropout(x) - - if self.src_attn is not None: - residual = x - if self.normalize_before: - x = self.norm3(x) - - x = residual + self.dropout(self.src_attn(x, memory, memory_mask)) - - return x, tgt_mask, memory, memory_mask, cache - - def forward_one_step(self, tgt, tgt_mask, memory, memory_mask=None, cache=None): - """Compute decoded features. - - Args: - tgt (torch.Tensor): Input tensor (#batch, maxlen_out, size). - tgt_mask (torch.Tensor): Mask for input tensor (#batch, maxlen_out). - memory (torch.Tensor): Encoded memory, float32 (#batch, maxlen_in, size). - memory_mask (torch.Tensor): Encoded memory mask (#batch, maxlen_in). - cache (List[torch.Tensor]): List of cached tensors. - Each tensor shape should be (#batch, maxlen_out - 1, size). - - Returns: - torch.Tensor: Output tensor(#batch, maxlen_out, size). - torch.Tensor: Mask for output tensor (#batch, maxlen_out). - torch.Tensor: Encoded memory (#batch, maxlen_in, size). - torch.Tensor: Encoded memory mask (#batch, maxlen_in). - - """ - # tgt = self.dropout(tgt) - residual = tgt - if self.normalize_before: - tgt = self.norm1(tgt) - tgt = self.feed_forward(tgt) - - x = tgt - if self.self_attn: - if self.normalize_before: - tgt = self.norm2(tgt) - if self.training: - cache = None - x, cache = self.self_attn(tgt, tgt_mask, cache=cache) - x = residual + self.dropout(x) - - if self.src_attn is not None: - residual = x - if self.normalize_before: - x = self.norm3(x) - - x = residual + self.dropout(self.src_attn(x, memory, memory_mask)) - - return x, tgt_mask, memory, memory_mask, cache - - def forward_chunk( - self, tgt, memory, fsmn_cache=None, opt_cache=None, chunk_size=None, look_back=0 - ): - """Compute decoded features. - - Args: - tgt (torch.Tensor): Input tensor (#batch, maxlen_out, size). - tgt_mask (torch.Tensor): Mask for input tensor (#batch, maxlen_out). - memory (torch.Tensor): Encoded memory, float32 (#batch, maxlen_in, size). - memory_mask (torch.Tensor): Encoded memory mask (#batch, maxlen_in). - cache (List[torch.Tensor]): List of cached tensors. - Each tensor shape should be (#batch, maxlen_out - 1, size). - - Returns: - torch.Tensor: Output tensor(#batch, maxlen_out, size). - torch.Tensor: Mask for output tensor (#batch, maxlen_out). - torch.Tensor: Encoded memory (#batch, maxlen_in, size). - torch.Tensor: Encoded memory mask (#batch, maxlen_in). - - """ - residual = tgt - if self.normalize_before: - tgt = self.norm1(tgt) - tgt = self.feed_forward(tgt) - - x = tgt - if self.self_attn: - if self.normalize_before: - tgt = self.norm2(tgt) - x, fsmn_cache = self.self_attn(tgt, None, fsmn_cache) - x = residual + self.dropout(x) - - if self.src_attn is not None: - residual = x - if self.normalize_before: - x = self.norm3(x) - - x, opt_cache = self.src_attn.forward_chunk( - x, memory, opt_cache, chunk_size, look_back - ) - x = residual + x - - return x, memory, fsmn_cache, opt_cache - - -@tables.register("decoder_classes", "FsmnDecoder") -class FsmnDecoder(BaseTransformerDecoder): - """ - Author: Zhifu Gao, Shiliang Zhang, Ming Lei, Ian McLoughlin - San-m: Memory equipped self-attention for end-to-end speech recognition - https://arxiv.org/abs/2006.01713 - """ - - def __init__( - self, - vocab_size: int, - encoder_output_size: int, - attention_heads: int = 4, - linear_units: int = 2048, - num_blocks: int = 6, - dropout_rate: float = 0.1, - positional_dropout_rate: float = 0.1, - self_attention_dropout_rate: float = 0.0, - src_attention_dropout_rate: float = 0.0, - input_layer: str = "embed", - use_output_layer: bool = True, - pos_enc_class=PositionalEncoding, - normalize_before: bool = True, - concat_after: bool = False, - att_layer_num: int = 6, - kernel_size: int = 21, - sanm_shfit: int = None, - concat_embeds: bool = False, - attention_dim: int = None, - tf2torch_tensor_name_prefix_torch: str = "decoder", - tf2torch_tensor_name_prefix_tf: str = "seq2seq/decoder", - embed_tensor_name_prefix_tf: str = None, - ): - super().__init__( - vocab_size=vocab_size, - encoder_output_size=encoder_output_size, - dropout_rate=dropout_rate, - positional_dropout_rate=positional_dropout_rate, - input_layer=input_layer, - use_output_layer=use_output_layer, - pos_enc_class=pos_enc_class, - normalize_before=normalize_before, - ) - if attention_dim is None: - attention_dim = encoder_output_size - - if input_layer == "embed": - self.embed = torch.nn.Sequential( - torch.nn.Embedding(vocab_size, attention_dim), - ) - elif input_layer == "linear": - self.embed = torch.nn.Sequential( - torch.nn.Linear(vocab_size, attention_dim), - torch.nn.LayerNorm(attention_dim), - torch.nn.Dropout(dropout_rate), - torch.nn.ReLU(), - pos_enc_class(attention_dim, positional_dropout_rate), - ) - else: - raise ValueError(f"only 'embed' or 'linear' is supported: {input_layer}") - - self.normalize_before = normalize_before - if self.normalize_before: - self.after_norm = LayerNorm(attention_dim) - if use_output_layer: - self.output_layer = torch.nn.Linear(attention_dim, vocab_size) - else: - self.output_layer = None - - self.att_layer_num = att_layer_num - self.num_blocks = num_blocks - if sanm_shfit is None: - sanm_shfit = (kernel_size - 1) // 2 - self.decoders = repeat( - att_layer_num, - lambda lnum: DecoderLayerSANM( - attention_dim, - MultiHeadedAttentionSANMDecoder( - attention_dim, - self_attention_dropout_rate, - kernel_size, - sanm_shfit=sanm_shfit, - ), - MultiHeadedAttentionCrossAtt( - attention_heads, - attention_dim, - src_attention_dropout_rate, - encoder_output_size=encoder_output_size, - ), - PositionwiseFeedForwardDecoderSANM( - attention_dim, linear_units, dropout_rate - ), - dropout_rate, - normalize_before, - concat_after, - ), - ) - if num_blocks - att_layer_num <= 0: - self.decoders2 = None - else: - self.decoders2 = repeat( - num_blocks - att_layer_num, - lambda lnum: DecoderLayerSANM( - attention_dim, - MultiHeadedAttentionSANMDecoder( - attention_dim, - self_attention_dropout_rate, - kernel_size, - sanm_shfit=sanm_shfit, - ), - None, - PositionwiseFeedForwardDecoderSANM( - attention_dim, linear_units, dropout_rate - ), - dropout_rate, - normalize_before, - concat_after, - ), - ) - - self.decoders3 = repeat( - 1, - lambda lnum: DecoderLayerSANM( - attention_dim, - None, - None, - PositionwiseFeedForwardDecoderSANM( - attention_dim, linear_units, dropout_rate - ), - dropout_rate, - normalize_before, - concat_after, - ), - ) - if concat_embeds: - self.embed_concat_ffn = repeat( - 1, - lambda lnum: DecoderLayerSANM( - attention_dim + encoder_output_size, - None, - None, - PositionwiseFeedForwardDecoderSANM( - attention_dim + encoder_output_size, - linear_units, - dropout_rate, - adim=attention_dim, - ), - dropout_rate, - normalize_before, - concat_after, - ), - ) - else: - self.embed_concat_ffn = None - self.concat_embeds = concat_embeds - self.tf2torch_tensor_name_prefix_torch = tf2torch_tensor_name_prefix_torch - self.tf2torch_tensor_name_prefix_tf = tf2torch_tensor_name_prefix_tf - self.embed_tensor_name_prefix_tf = embed_tensor_name_prefix_tf - - def forward( - self, - hs_pad: torch.Tensor, - hlens: torch.Tensor, - ys_in_pad: torch.Tensor, - ys_in_lens: torch.Tensor, - chunk_mask: torch.Tensor = None, - pre_acoustic_embeds: torch.Tensor = None, - ) -> Tuple[torch.Tensor, torch.Tensor]: - """Forward decoder. - - Args: - hs_pad: encoded memory, float32 (batch, maxlen_in, feat) - hlens: (batch) - ys_in_pad: - input token ids, int64 (batch, maxlen_out) - if input_layer == "embed" - input tensor (batch, maxlen_out, #mels) in the other cases - ys_in_lens: (batch) - Returns: - (tuple): tuple containing: - - x: decoded token score before softmax (batch, maxlen_out, token) - if use_output_layer is True, - olens: (batch, ) - """ - tgt = ys_in_pad - tgt_mask = myutils.sequence_mask(ys_in_lens, device=tgt.device)[:, :, None] - - memory = hs_pad - memory_mask = myutils.sequence_mask(hlens, device=memory.device)[:, None, :] - if chunk_mask is not None: - memory_mask = memory_mask * chunk_mask - if tgt_mask.size(1) != memory_mask.size(1): - memory_mask = torch.cat((memory_mask, memory_mask[:, -2:-1, :]), dim=1) - - x = self.embed(tgt) - - if pre_acoustic_embeds is not None and self.concat_embeds: - x = torch.cat((x, pre_acoustic_embeds), dim=-1) - x, _, _, _, _ = self.embed_concat_ffn(x, None, None, None, None) - - x, tgt_mask, memory, memory_mask, _ = self.decoders( - x, tgt_mask, memory, memory_mask - ) - if self.decoders2 is not None: - x, tgt_mask, memory, memory_mask, _ = self.decoders2( - x, tgt_mask, memory, memory_mask - ) - x, tgt_mask, memory, memory_mask, _ = self.decoders3( - x, tgt_mask, memory, memory_mask - ) - if self.normalize_before: - x = self.after_norm(x) - if self.output_layer is not None: - x = self.output_layer(x) - - olens = tgt_mask.sum(1) - return x, olens - - def score( - self, - ys, - state, - x, - x_mask=None, - pre_acoustic_embeds: torch.Tensor = None, - ): - """Score.""" - ys_mask = myutils.sequence_mask( - torch.tensor([len(ys)], dtype=torch.int32), device=x.device - )[:, :, None] - logp, state = self.forward_one_step( - ys.unsqueeze(0), - ys_mask, - x.unsqueeze(0), - memory_mask=x_mask, - pre_acoustic_embeds=pre_acoustic_embeds, - cache=state, - ) - return logp.squeeze(0), state - - def forward_one_step( - self, - tgt: torch.Tensor, - tgt_mask: torch.Tensor, - memory: torch.Tensor, - memory_mask: torch.Tensor = None, - pre_acoustic_embeds: torch.Tensor = None, - cache: List[torch.Tensor] = None, - ) -> Tuple[torch.Tensor, List[torch.Tensor]]: - """Forward one step. - - Args: - tgt: input token ids, int64 (batch, maxlen_out) - tgt_mask: input token mask, (batch, maxlen_out) - dtype=torch.uint8 in PyTorch 1.2- - dtype=torch.bool in PyTorch 1.2+ (include 1.2) - memory: encoded memory, float32 (batch, maxlen_in, feat) - cache: cached output list of (batch, max_time_out-1, size) - Returns: - y, cache: NN output value and cache per `self.decoders`. - y.shape` is (batch, maxlen_out, token) - """ - - x = tgt[:, -1:] - tgt_mask = None - x = self.embed(x) - - if pre_acoustic_embeds is not None and self.concat_embeds: - x = torch.cat((x, pre_acoustic_embeds), dim=-1) - x, _, _, _, _ = self.embed_concat_ffn(x, None, None, None, None) - - if cache is None: - cache_layer_num = len(self.decoders) - if self.decoders2 is not None: - cache_layer_num += len(self.decoders2) - cache = [None] * cache_layer_num - new_cache = [] - # for c, decoder in zip(cache, self.decoders): - for i in range(self.att_layer_num): - decoder = self.decoders[i] - c = cache[i] - x, tgt_mask, memory, memory_mask, c_ret = decoder.forward_one_step( - x, tgt_mask, memory, memory_mask, cache=c - ) - new_cache.append(c_ret) - - if self.num_blocks - self.att_layer_num >= 1: - for i in range(self.num_blocks - self.att_layer_num): - j = i + self.att_layer_num - decoder = self.decoders2[i] - c = cache[j] - x, tgt_mask, memory, memory_mask, c_ret = decoder.forward_one_step( - x, tgt_mask, memory, memory_mask, cache=c - ) - new_cache.append(c_ret) - - for decoder in self.decoders3: - x, tgt_mask, memory, memory_mask, _ = decoder.forward_one_step( - x, tgt_mask, memory, None, cache=None - ) - - if self.normalize_before: - y = self.after_norm(x[:, -1]) - else: - y = x[:, -1] - if self.output_layer is not None: - y = self.output_layer(y) - y = torch.log_softmax(y, dim=-1) - - return y, new_cache diff --git a/funasr_detach/models/sanm/encoder.py b/funasr_detach/models/sanm/encoder.py deleted file mode 100644 index 8556d9bf5cab3c72ba4d20ab4512957fdef798e2..0000000000000000000000000000000000000000 --- a/funasr_detach/models/sanm/encoder.py +++ /dev/null @@ -1,491 +0,0 @@ -#!/usr/bin/env python3 -# -*- encoding: utf-8 -*- -# Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved. -# MIT License (https://opensource.org/licenses/MIT) - -from typing import List -from typing import Optional -from typing import Sequence -from typing import Tuple -from typing import Union -import logging -import torch -import torch.nn as nn -import torch.nn.functional as F - -import numpy as np -from funasr_detach.train_utils.device_funcs import to_device -from funasr_detach.models.transformer.utils.nets_utils import make_pad_mask -from funasr_detach.models.sanm.attention import ( - MultiHeadedAttention, - MultiHeadedAttentionSANM, -) -from funasr_detach.models.transformer.embedding import ( - SinusoidalPositionEncoder, - StreamSinusoidalPositionEncoder, -) -from funasr_detach.models.transformer.layer_norm import LayerNorm -from funasr_detach.models.transformer.utils.multi_layer_conv import Conv1dLinear -from funasr_detach.models.transformer.utils.multi_layer_conv import MultiLayeredConv1d -from funasr_detach.models.transformer.positionwise_feed_forward import ( - PositionwiseFeedForward, # noqa: H301 -) -from funasr_detach.models.transformer.utils.repeat import repeat -from funasr_detach.models.transformer.utils.subsampling import Conv2dSubsampling -from funasr_detach.models.transformer.utils.subsampling import Conv2dSubsampling2 -from funasr_detach.models.transformer.utils.subsampling import Conv2dSubsampling6 -from funasr_detach.models.transformer.utils.subsampling import Conv2dSubsampling8 -from funasr_detach.models.transformer.utils.subsampling import TooShortUttError -from funasr_detach.models.transformer.utils.subsampling import check_short_utt - - -from funasr_detach.models.ctc.ctc import CTC - -from funasr_detach.register import tables - - -class EncoderLayerSANM(nn.Module): - def __init__( - self, - in_size, - size, - self_attn, - feed_forward, - dropout_rate, - normalize_before=True, - concat_after=False, - stochastic_depth_rate=0.0, - ): - """Construct an EncoderLayer object.""" - super(EncoderLayerSANM, self).__init__() - self.self_attn = self_attn - self.feed_forward = feed_forward - self.norm1 = LayerNorm(in_size) - self.norm2 = LayerNorm(size) - self.dropout = nn.Dropout(dropout_rate) - self.in_size = in_size - self.size = size - self.normalize_before = normalize_before - self.concat_after = concat_after - if self.concat_after: - self.concat_linear = nn.Linear(size + size, size) - self.stochastic_depth_rate = stochastic_depth_rate - self.dropout_rate = dropout_rate - - def forward( - self, x, mask, cache=None, mask_shfit_chunk=None, mask_att_chunk_encoder=None - ): - """Compute encoded features. - - Args: - x_input (torch.Tensor): Input tensor (#batch, time, size). - mask (torch.Tensor): Mask tensor for the input (#batch, time). - cache (torch.Tensor): Cache tensor of the input (#batch, time - 1, size). - - Returns: - torch.Tensor: Output tensor (#batch, time, size). - torch.Tensor: Mask tensor (#batch, time). - - """ - skip_layer = False - # with stochastic depth, residual connection `x + f(x)` becomes - # `x <- x + 1 / (1 - p) * f(x)` at training time. - stoch_layer_coeff = 1.0 - if self.training and self.stochastic_depth_rate > 0: - skip_layer = torch.rand(1).item() < self.stochastic_depth_rate - stoch_layer_coeff = 1.0 / (1 - self.stochastic_depth_rate) - - if skip_layer: - if cache is not None: - x = torch.cat([cache, x], dim=1) - return x, mask - - residual = x - if self.normalize_before: - x = self.norm1(x) - - if self.concat_after: - x_concat = torch.cat( - ( - x, - self.self_attn( - x, - mask, - mask_shfit_chunk=mask_shfit_chunk, - mask_att_chunk_encoder=mask_att_chunk_encoder, - ), - ), - dim=-1, - ) - if self.in_size == self.size: - x = residual + stoch_layer_coeff * self.concat_linear(x_concat) - else: - x = stoch_layer_coeff * self.concat_linear(x_concat) - else: - if self.in_size == self.size: - x = residual + stoch_layer_coeff * self.dropout( - self.self_attn( - x, - mask, - mask_shfit_chunk=mask_shfit_chunk, - mask_att_chunk_encoder=mask_att_chunk_encoder, - ) - ) - else: - x = stoch_layer_coeff * self.dropout( - self.self_attn( - x, - mask, - mask_shfit_chunk=mask_shfit_chunk, - mask_att_chunk_encoder=mask_att_chunk_encoder, - ) - ) - if not self.normalize_before: - x = self.norm1(x) - - residual = x - if self.normalize_before: - x = self.norm2(x) - x = residual + stoch_layer_coeff * self.dropout(self.feed_forward(x)) - if not self.normalize_before: - x = self.norm2(x) - - return x, mask, cache, mask_shfit_chunk, mask_att_chunk_encoder - - def forward_chunk(self, x, cache=None, chunk_size=None, look_back=0): - """Compute encoded features. - - Args: - x_input (torch.Tensor): Input tensor (#batch, time, size). - mask (torch.Tensor): Mask tensor for the input (#batch, time). - cache (torch.Tensor): Cache tensor of the input (#batch, time - 1, size). - - Returns: - torch.Tensor: Output tensor (#batch, time, size). - torch.Tensor: Mask tensor (#batch, time). - - """ - - residual = x - if self.normalize_before: - x = self.norm1(x) - - if self.in_size == self.size: - attn, cache = self.self_attn.forward_chunk(x, cache, chunk_size, look_back) - x = residual + attn - else: - x, cache = self.self_attn.forward_chunk(x, cache, chunk_size, look_back) - - if not self.normalize_before: - x = self.norm1(x) - - residual = x - if self.normalize_before: - x = self.norm2(x) - x = residual + self.feed_forward(x) - if not self.normalize_before: - x = self.norm2(x) - - return x, cache - - -@tables.register("encoder_classes", "SANMEncoder") -class SANMEncoder(nn.Module): - """ - Author: Zhifu Gao, Shiliang Zhang, Ming Lei, Ian McLoughlin - San-m: Memory equipped self-attention for end-to-end speech recognition - https://arxiv.org/abs/2006.01713 - """ - - def __init__( - self, - input_size: int, - output_size: int = 256, - attention_heads: int = 4, - linear_units: int = 2048, - num_blocks: int = 6, - dropout_rate: float = 0.1, - positional_dropout_rate: float = 0.1, - attention_dropout_rate: float = 0.0, - input_layer: Optional[str] = "conv2d", - pos_enc_class=SinusoidalPositionEncoder, - normalize_before: bool = True, - concat_after: bool = False, - positionwise_layer_type: str = "linear", - positionwise_conv_kernel_size: int = 1, - padding_idx: int = -1, - interctc_layer_idx: List[int] = [], - interctc_use_conditioning: bool = False, - kernel_size: int = 11, - sanm_shfit: int = 0, - lora_list: List[str] = None, - lora_rank: int = 8, - lora_alpha: int = 16, - lora_dropout: float = 0.1, - selfattention_layer_type: str = "sanm", - tf2torch_tensor_name_prefix_torch: str = "encoder", - tf2torch_tensor_name_prefix_tf: str = "seq2seq/encoder", - ): - super().__init__() - self._output_size = output_size - - if input_layer == "linear": - self.embed = torch.nn.Sequential( - torch.nn.Linear(input_size, output_size), - torch.nn.LayerNorm(output_size), - torch.nn.Dropout(dropout_rate), - torch.nn.ReLU(), - pos_enc_class(output_size, positional_dropout_rate), - ) - elif input_layer == "conv2d": - self.embed = Conv2dSubsampling(input_size, output_size, dropout_rate) - elif input_layer == "conv2d2": - self.embed = Conv2dSubsampling2(input_size, output_size, dropout_rate) - elif input_layer == "conv2d6": - self.embed = Conv2dSubsampling6(input_size, output_size, dropout_rate) - elif input_layer == "conv2d8": - self.embed = Conv2dSubsampling8(input_size, output_size, dropout_rate) - elif input_layer == "embed": - self.embed = torch.nn.Sequential( - torch.nn.Embedding(input_size, output_size, padding_idx=padding_idx), - SinusoidalPositionEncoder(), - ) - elif input_layer is None: - if input_size == output_size: - self.embed = None - else: - self.embed = torch.nn.Linear(input_size, output_size) - elif input_layer == "pe": - self.embed = SinusoidalPositionEncoder() - elif input_layer == "pe_online": - self.embed = StreamSinusoidalPositionEncoder() - else: - raise ValueError("unknown input_layer: " + input_layer) - self.normalize_before = normalize_before - if positionwise_layer_type == "linear": - positionwise_layer = PositionwiseFeedForward - positionwise_layer_args = ( - output_size, - linear_units, - dropout_rate, - ) - elif positionwise_layer_type == "conv1d": - positionwise_layer = MultiLayeredConv1d - positionwise_layer_args = ( - output_size, - linear_units, - positionwise_conv_kernel_size, - dropout_rate, - ) - elif positionwise_layer_type == "conv1d-linear": - positionwise_layer = Conv1dLinear - positionwise_layer_args = ( - output_size, - linear_units, - positionwise_conv_kernel_size, - dropout_rate, - ) - else: - raise NotImplementedError("Support only linear or conv1d.") - - if selfattention_layer_type == "selfattn": - encoder_selfattn_layer = MultiHeadedAttention - encoder_selfattn_layer_args = ( - attention_heads, - output_size, - attention_dropout_rate, - ) - - elif selfattention_layer_type == "sanm": - encoder_selfattn_layer = MultiHeadedAttentionSANM - encoder_selfattn_layer_args0 = ( - attention_heads, - input_size, - output_size, - attention_dropout_rate, - kernel_size, - sanm_shfit, - lora_list, - lora_rank, - lora_alpha, - lora_dropout, - ) - - encoder_selfattn_layer_args = ( - attention_heads, - output_size, - output_size, - attention_dropout_rate, - kernel_size, - sanm_shfit, - lora_list, - lora_rank, - lora_alpha, - lora_dropout, - ) - self.encoders0 = repeat( - 1, - lambda lnum: EncoderLayerSANM( - input_size, - output_size, - encoder_selfattn_layer(*encoder_selfattn_layer_args0), - positionwise_layer(*positionwise_layer_args), - dropout_rate, - normalize_before, - concat_after, - ), - ) - - self.encoders = repeat( - num_blocks - 1, - lambda lnum: EncoderLayerSANM( - output_size, - output_size, - encoder_selfattn_layer(*encoder_selfattn_layer_args), - positionwise_layer(*positionwise_layer_args), - dropout_rate, - normalize_before, - concat_after, - ), - ) - if self.normalize_before: - self.after_norm = LayerNorm(output_size) - - self.interctc_layer_idx = interctc_layer_idx - if len(interctc_layer_idx) > 0: - assert 0 < min(interctc_layer_idx) and max(interctc_layer_idx) < num_blocks - self.interctc_use_conditioning = interctc_use_conditioning - self.conditioning_layer = None - self.dropout = nn.Dropout(dropout_rate) - self.tf2torch_tensor_name_prefix_torch = tf2torch_tensor_name_prefix_torch - self.tf2torch_tensor_name_prefix_tf = tf2torch_tensor_name_prefix_tf - - def output_size(self) -> int: - return self._output_size - - def forward( - self, - xs_pad: torch.Tensor, - ilens: torch.Tensor, - prev_states: torch.Tensor = None, - ctc: CTC = None, - ) -> Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]: - """Embed positions in tensor. - - Args: - xs_pad: input tensor (B, L, D) - ilens: input length (B) - prev_states: Not to be used now. - Returns: - position embedded tensor and mask - """ - masks = (~make_pad_mask(ilens)[:, None, :]).to(xs_pad.device) - xs_pad = xs_pad * self.output_size() ** 0.5 - if self.embed is None: - xs_pad = xs_pad - elif ( - isinstance(self.embed, Conv2dSubsampling) - or isinstance(self.embed, Conv2dSubsampling2) - or isinstance(self.embed, Conv2dSubsampling6) - or isinstance(self.embed, Conv2dSubsampling8) - ): - short_status, limit_size = check_short_utt(self.embed, xs_pad.size(1)) - if short_status: - raise TooShortUttError( - f"has {xs_pad.size(1)} frames and is too short for subsampling " - + f"(it needs more than {limit_size} frames), return empty results", - xs_pad.size(1), - limit_size, - ) - xs_pad, masks = self.embed(xs_pad, masks) - else: - xs_pad = self.embed(xs_pad) - - # xs_pad = self.dropout(xs_pad) - encoder_outs = self.encoders0(xs_pad, masks) - xs_pad, masks = encoder_outs[0], encoder_outs[1] - intermediate_outs = [] - if len(self.interctc_layer_idx) == 0: - encoder_outs = self.encoders(xs_pad, masks) - xs_pad, masks = encoder_outs[0], encoder_outs[1] - else: - for layer_idx, encoder_layer in enumerate(self.encoders): - encoder_outs = encoder_layer(xs_pad, masks) - xs_pad, masks = encoder_outs[0], encoder_outs[1] - - if layer_idx + 1 in self.interctc_layer_idx: - encoder_out = xs_pad - - # intermediate outputs are also normalized - if self.normalize_before: - encoder_out = self.after_norm(encoder_out) - - intermediate_outs.append((layer_idx + 1, encoder_out)) - - if self.interctc_use_conditioning: - ctc_out = ctc.softmax(encoder_out) - xs_pad = xs_pad + self.conditioning_layer(ctc_out) - - if self.normalize_before: - xs_pad = self.after_norm(xs_pad) - - olens = masks.squeeze(1).sum(1) - if len(intermediate_outs) > 0: - return (xs_pad, intermediate_outs), olens, None - return xs_pad, olens, None - - def _add_overlap_chunk(self, feats: np.ndarray, cache: dict = {}): - if len(cache) == 0: - return feats - cache["feats"] = to_device(cache["feats"], device=feats.device) - overlap_feats = torch.cat((cache["feats"], feats), dim=1) - cache["feats"] = overlap_feats[ - :, -(cache["chunk_size"][0] + cache["chunk_size"][2]) :, : - ] - return overlap_feats - - def forward_chunk( - self, - xs_pad: torch.Tensor, - ilens: torch.Tensor, - cache: dict = None, - ctc: CTC = None, - ): - xs_pad *= self.output_size() ** 0.5 - if self.embed is None: - xs_pad = xs_pad - else: - xs_pad = self.embed(xs_pad, cache) - if cache["tail_chunk"]: - xs_pad = to_device(cache["feats"], device=xs_pad.device) - else: - xs_pad = self._add_overlap_chunk(xs_pad, cache) - encoder_outs = self.encoders0(xs_pad, None, None, None, None) - xs_pad, masks = encoder_outs[0], encoder_outs[1] - intermediate_outs = [] - if len(self.interctc_layer_idx) == 0: - encoder_outs = self.encoders(xs_pad, None, None, None, None) - xs_pad, masks = encoder_outs[0], encoder_outs[1] - else: - for layer_idx, encoder_layer in enumerate(self.encoders): - encoder_outs = encoder_layer(xs_pad, None, None, None, None) - xs_pad, masks = encoder_outs[0], encoder_outs[1] - if layer_idx + 1 in self.interctc_layer_idx: - encoder_out = xs_pad - - # intermediate outputs are also normalized - if self.normalize_before: - encoder_out = self.after_norm(encoder_out) - - intermediate_outs.append((layer_idx + 1, encoder_out)) - - if self.interctc_use_conditioning: - ctc_out = ctc.softmax(encoder_out) - xs_pad = xs_pad + self.conditioning_layer(ctc_out) - - if self.normalize_before: - xs_pad = self.after_norm(xs_pad) - - if len(intermediate_outs) > 0: - return (xs_pad, intermediate_outs), None, None - return xs_pad, ilens, None diff --git a/funasr_detach/models/sanm/model.py b/funasr_detach/models/sanm/model.py deleted file mode 100644 index 497d2277555cfc70f71bd3fb06afabefab1e56e3..0000000000000000000000000000000000000000 --- a/funasr_detach/models/sanm/model.py +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/env python3 -# -*- encoding: utf-8 -*- -# Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved. -# MIT License (https://opensource.org/licenses/MIT) - -import logging - -import torch - -from funasr_detach.models.transformer.model import Transformer -from funasr_detach.register import tables - - -@tables.register("model_classes", "SANM") -class SANM(Transformer): - """ - Author: Zhifu Gao, Shiliang Zhang, Ming Lei, Ian McLoughlin - San-m: Memory equipped self-attention for end-to-end speech recognition - https://arxiv.org/abs/2006.01713 - """ - - def __init__( - self, - *args, - **kwargs, - ): - - super().__init__(*args, **kwargs) diff --git a/funasr_detach/models/sanm/positionwise_feed_forward.py b/funasr_detach/models/sanm/positionwise_feed_forward.py deleted file mode 100644 index bdea48c4b5dd4ae7403c4f0d10f7843b3bf054ce..0000000000000000000000000000000000000000 --- a/funasr_detach/models/sanm/positionwise_feed_forward.py +++ /dev/null @@ -1,37 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- - - -"""Positionwise feed forward layer definition.""" - -import torch - -from funasr_detach.models.transformer.layer_norm import LayerNorm - - -class PositionwiseFeedForwardDecoderSANM(torch.nn.Module): - """Positionwise feed forward layer. - - Args: - idim (int): Input dimenstion. - hidden_units (int): The number of hidden units. - dropout_rate (float): Dropout rate. - - """ - - def __init__( - self, idim, hidden_units, dropout_rate, adim=None, activation=torch.nn.ReLU() - ): - """Construct an PositionwiseFeedForward object.""" - super(PositionwiseFeedForwardDecoderSANM, self).__init__() - self.w_1 = torch.nn.Linear(idim, hidden_units) - self.w_2 = torch.nn.Linear( - hidden_units, idim if adim is None else adim, bias=False - ) - self.dropout = torch.nn.Dropout(dropout_rate) - self.activation = activation - self.norm = LayerNorm(hidden_units) - - def forward(self, x): - """Forward function.""" - return self.w_2(self.norm(self.dropout(self.activation(self.w_1(x))))) diff --git a/funasr_detach/models/sanm/template.yaml b/funasr_detach/models/sanm/template.yaml deleted file mode 100644 index 156926f2c44f26978985fe68d3f96112e640688b..0000000000000000000000000000000000000000 --- a/funasr_detach/models/sanm/template.yaml +++ /dev/null @@ -1,121 +0,0 @@ -# This is an example that demonstrates how to configure a model file. -# You can modify the configuration according to your own requirements. - -# to print the register_table: -# from funasr.register import tables -# tables.print() - -# network architecture -model: SANM -model_conf: - ctc_weight: 0.0 - lsm_weight: 0.1 - length_normalized_loss: true - -# encoder -encoder: SANMEncoder -encoder_conf: - output_size: 512 - attention_heads: 4 - linear_units: 2048 - num_blocks: 50 - dropout_rate: 0.1 - positional_dropout_rate: 0.1 - attention_dropout_rate: 0.1 - input_layer: pe - pos_enc_class: SinusoidalPositionEncoder - normalize_before: true - kernel_size: 11 - sanm_shfit: 0 - selfattention_layer_type: sanm - -# decoder -decoder: FsmnDecoder -decoder_conf: - attention_heads: 4 - linear_units: 2048 - num_blocks: 16 - dropout_rate: 0.1 - positional_dropout_rate: 0.1 - self_attention_dropout_rate: 0.1 - src_attention_dropout_rate: 0.1 - att_layer_num: 16 - kernel_size: 11 - sanm_shfit: 0 - - - -# frontend related -frontend: WavFrontend -frontend_conf: - fs: 16000 - window: hamming - n_mels: 80 - frame_length: 25 - frame_shift: 10 - lfr_m: 7 - lfr_n: 6 - -specaug: SpecAugLFR -specaug_conf: - apply_time_warp: false - time_warp_window: 5 - time_warp_mode: bicubic - apply_freq_mask: true - freq_mask_width_range: - - 0 - - 30 - lfr_rate: 6 - num_freq_mask: 1 - apply_time_mask: true - time_mask_width_range: - - 0 - - 12 - num_time_mask: 1 - -train_conf: - accum_grad: 1 - grad_clip: 5 - max_epoch: 150 - val_scheduler_criterion: - - valid - - acc - best_model_criterion: - - - valid - - acc - - max - keep_nbest_models: 10 - avg_nbest_model: 5 - log_interval: 50 - -optim: adam -optim_conf: - lr: 0.0005 -scheduler: warmuplr -scheduler_conf: - warmup_steps: 30000 - -dataset: AudioDataset -dataset_conf: - index_ds: IndexDSJsonl - batch_sampler: DynamicBatchLocalShuffleSampler - batch_type: example # example or length - batch_size: 1 # if batch_type is example, batch_size is the numbers of samples; if length, batch_size is source_token_len+target_token_len; - max_token_length: 2048 # filter samples if source_token_len+target_token_len > max_token_length, - buffer_size: 500 - shuffle: True - num_workers: 0 - -tokenizer: CharTokenizer -tokenizer_conf: - unk_symbol: - split_with_space: true - - -ctc_conf: - dropout_rate: 0.0 - ctc_type: builtin - reduce: true - ignore_nan_grad: true - -normalize: null diff --git a/funasr_detach/models/scama/__init__.py b/funasr_detach/models/scama/__init__.py deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/funasr_detach/models/scama/beam_search.py b/funasr_detach/models/scama/beam_search.py deleted file mode 100644 index 03f5783c740fe10856bc1f86211ef1408b470aad..0000000000000000000000000000000000000000 --- a/funasr_detach/models/scama/beam_search.py +++ /dev/null @@ -1,1013 +0,0 @@ -"""Beam search module.""" - -from itertools import chain -import logging -from typing import Any -from typing import Dict -from typing import List -from typing import NamedTuple -from typing import Tuple -from typing import Union - -import torch - -from funasr_detach.metrics.common import end_detect -from funasr_detach.models.transformer.scorers.scorer_interface import ( - PartialScorerInterface, -) -from funasr_detach.models.transformer.scorers.scorer_interface import ScorerInterface - - -class Hypothesis(NamedTuple): - """Hypothesis data type.""" - - yseq: torch.Tensor - score: Union[float, torch.Tensor] = 0 - scores: Dict[str, Union[float, torch.Tensor]] = dict() - states: Dict[str, Any] = dict() - - def asdict(self) -> dict: - """Convert data to JSON-friendly dict.""" - return self._replace( - yseq=self.yseq.tolist(), - score=float(self.score), - scores={k: float(v) for k, v in self.scores.items()}, - )._asdict() - - -class BeamSearchScama(torch.nn.Module): - """Beam search implementation.""" - - def __init__( - self, - scorers: Dict[str, ScorerInterface], - weights: Dict[str, float], - beam_size: int, - vocab_size: int, - sos: int, - eos: int, - token_list: List[str] = None, - pre_beam_ratio: float = 1.5, - pre_beam_score_key: str = None, - ): - """Initialize beam search. - - Args: - scorers (dict[str, ScorerInterface]): Dict of decoder modules - e.g., Decoder, CTCPrefixScorer, LM - The scorer will be ignored if it is `None` - weights (dict[str, float]): Dict of weights for each scorers - The scorer will be ignored if its weight is 0 - beam_size (int): The number of hypotheses kept during search - vocab_size (int): The number of vocabulary - sos (int): Start of sequence id - eos (int): End of sequence id - token_list (list[str]): List of tokens for debug log - pre_beam_score_key (str): key of scores to perform pre-beam search - pre_beam_ratio (float): beam size in the pre-beam search - will be `int(pre_beam_ratio * beam_size)` - - """ - super().__init__() - # set scorers - self.weights = weights - self.scorers = dict() - self.full_scorers = dict() - self.part_scorers = dict() - # this module dict is required for recursive cast - # `self.to(device, dtype)` in `recog.py` - self.nn_dict = torch.nn.ModuleDict() - for k, v in scorers.items(): - w = weights.get(k, 0) - if w == 0 or v is None: - continue - assert isinstance( - v, ScorerInterface - ), f"{k} ({type(v)}) does not implement ScorerInterface" - self.scorers[k] = v - if isinstance(v, PartialScorerInterface): - self.part_scorers[k] = v - else: - self.full_scorers[k] = v - if isinstance(v, torch.nn.Module): - self.nn_dict[k] = v - - # set configurations - self.sos = sos - self.eos = eos - self.token_list = token_list - self.pre_beam_size = int(pre_beam_ratio * beam_size) - self.beam_size = beam_size - self.n_vocab = vocab_size - if ( - pre_beam_score_key is not None - and pre_beam_score_key != "full" - and pre_beam_score_key not in self.full_scorers - ): - raise KeyError(f"{pre_beam_score_key} is not found in {self.full_scorers}") - self.pre_beam_score_key = pre_beam_score_key - self.do_pre_beam = ( - self.pre_beam_score_key is not None - and self.pre_beam_size < self.n_vocab - and len(self.part_scorers) > 0 - ) - - def init_hyp(self, x: torch.Tensor) -> List[Hypothesis]: - """Get an initial hypothesis data. - - Args: - x (torch.Tensor): The encoder output feature - - Returns: - Hypothesis: The initial hypothesis. - - """ - init_states = dict() - init_scores = dict() - for k, d in self.scorers.items(): - init_states[k] = d.init_state(x) - init_scores[k] = 0.0 - return [ - Hypothesis( - score=0.0, - scores=init_scores, - states=init_states, - yseq=torch.tensor([self.sos], device=x.device), - ) - ] - - @staticmethod - def append_token(xs: torch.Tensor, x: int) -> torch.Tensor: - """Append new token to prefix tokens. - - Args: - xs (torch.Tensor): The prefix token - x (int): The new token to append - - Returns: - torch.Tensor: New tensor contains: xs + [x] with xs.dtype and xs.device - - """ - x = torch.tensor([x], dtype=xs.dtype, device=xs.device) - return torch.cat((xs, x)) - - def score_full( - self, - hyp: Hypothesis, - x: torch.Tensor, - x_mask: torch.Tensor = None, - pre_acoustic_embeds: torch.Tensor = None, - ) -> Tuple[Dict[str, torch.Tensor], Dict[str, Any]]: - """Score new hypothesis by `self.full_scorers`. - - Args: - hyp (Hypothesis): Hypothesis with prefix tokens to score - x (torch.Tensor): Corresponding input feature - - Returns: - Tuple[Dict[str, torch.Tensor], Dict[str, Any]]: Tuple of - score dict of `hyp` that has string keys of `self.full_scorers` - and tensor score values of shape: `(self.n_vocab,)`, - and state dict that has string keys - and state values of `self.full_scorers` - - """ - scores = dict() - states = dict() - for k, d in self.full_scorers.items(): - scores[k], states[k] = d.score( - hyp.yseq, - hyp.states[k], - x, - x_mask=x_mask, - pre_acoustic_embeds=pre_acoustic_embeds, - ) - return scores, states - - def score_partial( - self, hyp: Hypothesis, ids: torch.Tensor, x: torch.Tensor - ) -> Tuple[Dict[str, torch.Tensor], Dict[str, Any]]: - """Score new hypothesis by `self.part_scorers`. - - Args: - hyp (Hypothesis): Hypothesis with prefix tokens to score - ids (torch.Tensor): 1D tensor of new partial tokens to score - x (torch.Tensor): Corresponding input feature - - Returns: - Tuple[Dict[str, torch.Tensor], Dict[str, Any]]: Tuple of - score dict of `hyp` that has string keys of `self.part_scorers` - and tensor score values of shape: `(len(ids),)`, - and state dict that has string keys - and state values of `self.part_scorers` - - """ - scores = dict() - states = dict() - for k, d in self.part_scorers.items(): - scores[k], states[k] = d.score_partial(hyp.yseq, ids, hyp.states[k], x) - return scores, states - - def beam( - self, weighted_scores: torch.Tensor, ids: torch.Tensor - ) -> Tuple[torch.Tensor, torch.Tensor]: - """Compute topk full token ids and partial token ids. - - Args: - weighted_scores (torch.Tensor): The weighted sum scores for each tokens. - Its shape is `(self.n_vocab,)`. - ids (torch.Tensor): The partial token ids to compute topk - - Returns: - Tuple[torch.Tensor, torch.Tensor]: - The topk full token ids and partial token ids. - Their shapes are `(self.beam_size,)` - - """ - # no pre beam performed - if weighted_scores.size(0) == ids.size(0): - top_ids = weighted_scores.topk(self.beam_size)[1] - return top_ids, top_ids - - # mask pruned in pre-beam not to select in topk - tmp = weighted_scores[ids] - weighted_scores[:] = -float("inf") - weighted_scores[ids] = tmp - top_ids = weighted_scores.topk(self.beam_size)[1] - local_ids = weighted_scores[ids].topk(self.beam_size)[1] - return top_ids, local_ids - - @staticmethod - def merge_scores( - prev_scores: Dict[str, float], - next_full_scores: Dict[str, torch.Tensor], - full_idx: int, - next_part_scores: Dict[str, torch.Tensor], - part_idx: int, - ) -> Dict[str, torch.Tensor]: - """Merge scores for new hypothesis. - - Args: - prev_scores (Dict[str, float]): - The previous hypothesis scores by `self.scorers` - next_full_scores (Dict[str, torch.Tensor]): scores by `self.full_scorers` - full_idx (int): The next token id for `next_full_scores` - next_part_scores (Dict[str, torch.Tensor]): - scores of partial tokens by `self.part_scorers` - part_idx (int): The new token id for `next_part_scores` - - Returns: - Dict[str, torch.Tensor]: The new score dict. - Its keys are names of `self.full_scorers` and `self.part_scorers`. - Its values are scalar tensors by the scorers. - - """ - new_scores = dict() - for k, v in next_full_scores.items(): - new_scores[k] = prev_scores[k] + v[full_idx] - for k, v in next_part_scores.items(): - new_scores[k] = prev_scores[k] + v[part_idx] - return new_scores - - def merge_states(self, states: Any, part_states: Any, part_idx: int) -> Any: - """Merge states for new hypothesis. - - Args: - states: states of `self.full_scorers` - part_states: states of `self.part_scorers` - part_idx (int): The new token id for `part_scores` - - Returns: - Dict[str, torch.Tensor]: The new score dict. - Its keys are names of `self.full_scorers` and `self.part_scorers`. - Its values are states of the scorers. - - """ - new_states = dict() - for k, v in states.items(): - new_states[k] = v - for k, d in self.part_scorers.items(): - new_states[k] = d.select_state(part_states[k], part_idx) - return new_states - - def search( - self, - running_hyps: List[Hypothesis], - x: torch.Tensor, - x_mask: torch.Tensor = None, - pre_acoustic_embeds: torch.Tensor = None, - ) -> List[Hypothesis]: - """Search new tokens for running hypotheses and encoded speech x. - - Args: - running_hyps (List[Hypothesis]): Running hypotheses on beam - x (torch.Tensor): Encoded speech feature (T, D) - - Returns: - List[Hypotheses]: Best sorted hypotheses - - """ - best_hyps = [] - part_ids = torch.arange(self.n_vocab, device=x.device) # no pre-beam - for hyp in running_hyps: - # scoring - weighted_scores = torch.zeros(self.n_vocab, dtype=x.dtype, device=x.device) - scores, states = self.score_full( - hyp, x, x_mask=x_mask, pre_acoustic_embeds=pre_acoustic_embeds - ) - for k in self.full_scorers: - weighted_scores += self.weights[k] * scores[k] - # partial scoring - if self.do_pre_beam: - pre_beam_scores = ( - weighted_scores - if self.pre_beam_score_key == "full" - else scores[self.pre_beam_score_key] - ) - part_ids = torch.topk(pre_beam_scores, self.pre_beam_size)[1] - part_scores, part_states = self.score_partial(hyp, part_ids, x) - for k in self.part_scorers: - weighted_scores[part_ids] += self.weights[k] * part_scores[k] - # add previous hyp score - weighted_scores += hyp.score - - # update hyps - for j, part_j in zip(*self.beam(weighted_scores, part_ids)): - # will be (2 x beam at most) - best_hyps.append( - Hypothesis( - score=weighted_scores[j], - yseq=self.append_token(hyp.yseq, j), - scores=self.merge_scores( - hyp.scores, scores, j, part_scores, part_j - ), - states=self.merge_states(states, part_states, part_j), - ) - ) - - # sort and prune 2 x beam -> beam - best_hyps = sorted(best_hyps, key=lambda x: x.score, reverse=True)[ - : min(len(best_hyps), self.beam_size) - ] - return best_hyps - - def forward( - self, - x: torch.Tensor, - scama_mask: torch.Tensor = None, - pre_acoustic_embeds: torch.Tensor = None, - maxlenratio: float = 0.0, - minlenratio: float = 0.0, - maxlen: int = None, - minlen: int = 0, - ) -> List[Hypothesis]: - """Perform beam search. - - Args: - x (torch.Tensor): Encoded speech feature (T, D) - maxlenratio (float): Input length ratio to obtain max output length. - If maxlenratio=0.0 (default), it uses a end-detect function - to automatically find maximum hypothesis lengths - If maxlenratio<0.0, its absolute value is interpreted - as a constant max output length. - minlenratio (float): Input length ratio to obtain min output length. - - Returns: - list[Hypothesis]: N-best decoding results - - """ - if maxlen is None: - # set length bounds - if maxlenratio == 0: - maxlen = x.shape[0] - elif maxlenratio < 0: - maxlen = -1 * int(maxlenratio) - else: - maxlen = max(1, int(maxlenratio * x.size(0))) - minlen = int(minlenratio * x.size(0)) - - logging.info("decoder input length: " + str(x.shape[0])) - logging.info("max output length: " + str(maxlen)) - logging.info("min output length: " + str(minlen)) - - # main loop of prefix search - running_hyps = self.init_hyp(x) - ended_hyps = [] - for i in range(maxlen): - logging.debug("position " + str(i)) - mask_enc = None - if scama_mask is not None: - token_num_predictor = scama_mask.size(1) - token_id_slice = min(i, token_num_predictor - 1) - mask_enc = scama_mask[:, token_id_slice : token_id_slice + 1, :] - # if mask_enc.size(1) == 0: - # mask_enc = scama_mask[:, -2:-1, :] - # # mask_enc = torch.zeros_like(mask_enc) - pre_acoustic_embeds_cur = None - if pre_acoustic_embeds is not None: - b, t, d = pre_acoustic_embeds.size() - pad = torch.zeros((b, 1, d), dtype=pre_acoustic_embeds.dtype).to( - device=pre_acoustic_embeds.device - ) - pre_acoustic_embeds = torch.cat((pre_acoustic_embeds, pad), dim=1) - token_id_slice = min(i, t) - pre_acoustic_embeds_cur = pre_acoustic_embeds[ - :, token_id_slice : token_id_slice + 1, : - ] - - best = self.search( - running_hyps, - x, - x_mask=mask_enc, - pre_acoustic_embeds=pre_acoustic_embeds_cur, - ) - # post process of one iteration - running_hyps = self.post_process(i, maxlen, maxlenratio, best, ended_hyps) - # end detection - if maxlenratio == 0.0 and end_detect([h.asdict() for h in ended_hyps], i): - logging.info(f"end detected at {i}") - break - if len(running_hyps) == 0: - logging.info("no hypothesis. Finish decoding.") - break - else: - logging.debug(f"remained hypotheses: {len(running_hyps)}") - - nbest_hyps = sorted(ended_hyps, key=lambda x: x.score, reverse=True) - # check the number of hypotheses reaching to eos - if len(nbest_hyps) == 0: - logging.warning( - "there is no N-best results, perform recognition " - "again with smaller minlenratio." - ) - return ( - [] - if minlenratio < 0.1 - else self.forward(x, maxlenratio, max(0.0, minlenratio - 0.1)) - ) - - # report the best result - for x in nbest_hyps: - yseq = "".join([self.token_list[x] for x in x.yseq]) - logging.debug( - "nbest: y: {}, yseq: {}, score: {}".format(x.yseq, yseq, x.score) - ) - best = nbest_hyps[0] - for k, v in best.scores.items(): - logging.info( - f"{v:6.2f} * {self.weights[k]:3} = {v * self.weights[k]:6.2f} for {k}" - ) - logging.info(f"total log probability: {best.score:.2f}") - logging.info(f"normalized log probability: {best.score / len(best.yseq):.2f}") - logging.info(f"total number of ended hypotheses: {len(nbest_hyps)}") - if self.token_list is not None: - logging.info( - "best hypo: " - + "".join([self.token_list[x] for x in best.yseq[1:-1]]) - + "\n" - ) - return nbest_hyps - - def post_process( - self, - i: int, - maxlen: int, - maxlenratio: float, - running_hyps: List[Hypothesis], - ended_hyps: List[Hypothesis], - ) -> List[Hypothesis]: - """Perform post-processing of beam search iterations. - - Args: - i (int): The length of hypothesis tokens. - maxlen (int): The maximum length of tokens in beam search. - maxlenratio (int): The maximum length ratio in beam search. - running_hyps (List[Hypothesis]): The running hypotheses in beam search. - ended_hyps (List[Hypothesis]): The ended hypotheses in beam search. - - Returns: - List[Hypothesis]: The new running hypotheses. - - """ - logging.debug(f"the number of running hypotheses: {len(running_hyps)}") - if self.token_list is not None: - logging.debug( - "best hypo: " - + "".join([self.token_list[x] for x in running_hyps[0].yseq[1:]]) - ) - # add eos in the final loop to avoid that there are no ended hyps - if i == maxlen - 1: - logging.info("adding in the last position in the loop") - running_hyps = [ - h._replace(yseq=self.append_token(h.yseq, self.eos)) - for h in running_hyps - ] - - # add ended hypotheses to a final list, and removed them from current hypotheses - # (this will be a problem, number of hyps < beam) - remained_hyps = [] - for hyp in running_hyps: - if hyp.yseq[-1] == self.eos: - # e.g., Word LM needs to add final score - for k, d in chain(self.full_scorers.items(), self.part_scorers.items()): - s = d.final_score(hyp.states[k]) - hyp.scores[k] += s - hyp = hyp._replace(score=hyp.score + self.weights[k] * s) - ended_hyps.append(hyp) - else: - remained_hyps.append(hyp) - return remained_hyps - - -class BeamSearchScamaStreaming(torch.nn.Module): - """Beam search implementation.""" - - def __init__( - self, - scorers: Dict[str, ScorerInterface], - weights: Dict[str, float], - beam_size: int, - vocab_size: int, - sos: int, - eos: int, - token_list: List[str] = None, - pre_beam_ratio: float = 1.5, - pre_beam_score_key: str = None, - ): - """Initialize beam search. - - Args: - scorers (dict[str, ScorerInterface]): Dict of decoder modules - e.g., Decoder, CTCPrefixScorer, LM - The scorer will be ignored if it is `None` - weights (dict[str, float]): Dict of weights for each scorers - The scorer will be ignored if its weight is 0 - beam_size (int): The number of hypotheses kept during search - vocab_size (int): The number of vocabulary - sos (int): Start of sequence id - eos (int): End of sequence id - token_list (list[str]): List of tokens for debug log - pre_beam_score_key (str): key of scores to perform pre-beam search - pre_beam_ratio (float): beam size in the pre-beam search - will be `int(pre_beam_ratio * beam_size)` - - """ - super().__init__() - # set scorers - self.weights = weights - self.scorers = dict() - self.full_scorers = dict() - self.part_scorers = dict() - # this module dict is required for recursive cast - # `self.to(device, dtype)` in `recog.py` - self.nn_dict = torch.nn.ModuleDict() - for k, v in scorers.items(): - w = weights.get(k, 0) - if w == 0 or v is None: - continue - assert isinstance( - v, ScorerInterface - ), f"{k} ({type(v)}) does not implement ScorerInterface" - self.scorers[k] = v - if isinstance(v, PartialScorerInterface): - self.part_scorers[k] = v - else: - self.full_scorers[k] = v - if isinstance(v, torch.nn.Module): - self.nn_dict[k] = v - - # set configurations - self.sos = sos - self.eos = eos - self.token_list = token_list - self.pre_beam_size = int(pre_beam_ratio * beam_size) - self.beam_size = beam_size - self.n_vocab = vocab_size - if ( - pre_beam_score_key is not None - and pre_beam_score_key != "full" - and pre_beam_score_key not in self.full_scorers - ): - raise KeyError(f"{pre_beam_score_key} is not found in {self.full_scorers}") - self.pre_beam_score_key = pre_beam_score_key - self.do_pre_beam = ( - self.pre_beam_score_key is not None - and self.pre_beam_size < self.n_vocab - and len(self.part_scorers) > 0 - ) - - def init_hyp(self, x) -> List[Hypothesis]: - """Get an initial hypothesis data. - - Args: - x (torch.Tensor): The encoder output feature - - Returns: - Hypothesis: The initial hypothesis. - - """ - init_states = dict() - init_scores = dict() - for k, d in self.scorers.items(): - init_states[k] = d.init_state(x) - init_scores[k] = 0.0 - return [ - Hypothesis( - score=0.0, - scores=init_scores, - states=init_states, - yseq=torch.tensor([self.sos], device=x.device), - ) - ] - - @staticmethod - def append_token(xs: torch.Tensor, x: int) -> torch.Tensor: - """Append new token to prefix tokens. - - Args: - xs (torch.Tensor): The prefix token - x (int): The new token to append - - Returns: - torch.Tensor: New tensor contains: xs + [x] with xs.dtype and xs.device - - """ - x = torch.tensor([x], dtype=xs.dtype, device=xs.device) - return torch.cat((xs, x)) - - def score_full( - self, - hyp: Hypothesis, - x: torch.Tensor, - x_mask: torch.Tensor = None, - pre_acoustic_embeds: torch.Tensor = None, - cache: dict = {}, - ) -> Tuple[Dict[str, torch.Tensor], Dict[str, Any]]: - """Score new hypothesis by `self.full_scorers`. - - Args: - hyp (Hypothesis): Hypothesis with prefix tokens to score - x (torch.Tensor): Corresponding input feature - - Returns: - Tuple[Dict[str, torch.Tensor], Dict[str, Any]]: Tuple of - score dict of `hyp` that has string keys of `self.full_scorers` - and tensor score values of shape: `(self.n_vocab,)`, - and state dict that has string keys - and state values of `self.full_scorers` - - """ - scores = dict() - states = dict() - for k, d in self.full_scorers.items(): - scores[k], states[k] = d.score( - hyp.yseq, - hyp.states[k], - x, - x_mask=x_mask, - pre_acoustic_embeds=pre_acoustic_embeds, - cache=cache, - ) - return scores, states - - def score_partial( - self, hyp: Hypothesis, ids: torch.Tensor, x: torch.Tensor - ) -> Tuple[Dict[str, torch.Tensor], Dict[str, Any]]: - """Score new hypothesis by `self.part_scorers`. - - Args: - hyp (Hypothesis): Hypothesis with prefix tokens to score - ids (torch.Tensor): 1D tensor of new partial tokens to score - x (torch.Tensor): Corresponding input feature - - Returns: - Tuple[Dict[str, torch.Tensor], Dict[str, Any]]: Tuple of - score dict of `hyp` that has string keys of `self.part_scorers` - and tensor score values of shape: `(len(ids),)`, - and state dict that has string keys - and state values of `self.part_scorers` - - """ - scores = dict() - states = dict() - for k, d in self.part_scorers.items(): - scores[k], states[k] = d.score_partial(hyp.yseq, ids, hyp.states[k], x) - return scores, states - - def beam( - self, weighted_scores: torch.Tensor, ids: torch.Tensor - ) -> Tuple[torch.Tensor, torch.Tensor]: - """Compute topk full token ids and partial token ids. - - Args: - weighted_scores (torch.Tensor): The weighted sum scores for each tokens. - Its shape is `(self.n_vocab,)`. - ids (torch.Tensor): The partial token ids to compute topk - - Returns: - Tuple[torch.Tensor, torch.Tensor]: - The topk full token ids and partial token ids. - Their shapes are `(self.beam_size,)` - - """ - # no pre beam performed - if weighted_scores.size(0) == ids.size(0): - top_ids = weighted_scores.topk(self.beam_size)[1] - return top_ids, top_ids - - # mask pruned in pre-beam not to select in topk - tmp = weighted_scores[ids] - weighted_scores[:] = -float("inf") - weighted_scores[ids] = tmp - top_ids = weighted_scores.topk(self.beam_size)[1] - local_ids = weighted_scores[ids].topk(self.beam_size)[1] - return top_ids, local_ids - - @staticmethod - def merge_scores( - prev_scores: Dict[str, float], - next_full_scores: Dict[str, torch.Tensor], - full_idx: int, - next_part_scores: Dict[str, torch.Tensor], - part_idx: int, - ) -> Dict[str, torch.Tensor]: - """Merge scores for new hypothesis. - - Args: - prev_scores (Dict[str, float]): - The previous hypothesis scores by `self.scorers` - next_full_scores (Dict[str, torch.Tensor]): scores by `self.full_scorers` - full_idx (int): The next token id for `next_full_scores` - next_part_scores (Dict[str, torch.Tensor]): - scores of partial tokens by `self.part_scorers` - part_idx (int): The new token id for `next_part_scores` - - Returns: - Dict[str, torch.Tensor]: The new score dict. - Its keys are names of `self.full_scorers` and `self.part_scorers`. - Its values are scalar tensors by the scorers. - - """ - new_scores = dict() - for k, v in next_full_scores.items(): - new_scores[k] = prev_scores[k] + v[full_idx] - for k, v in next_part_scores.items(): - new_scores[k] = prev_scores[k] + v[part_idx] - return new_scores - - def merge_states(self, states: Any, part_states: Any, part_idx: int) -> Any: - """Merge states for new hypothesis. - - Args: - states: states of `self.full_scorers` - part_states: states of `self.part_scorers` - part_idx (int): The new token id for `part_scores` - - Returns: - Dict[str, torch.Tensor]: The new score dict. - Its keys are names of `self.full_scorers` and `self.part_scorers`. - Its values are states of the scorers. - - """ - new_states = dict() - for k, v in states.items(): - new_states[k] = v - for k, d in self.part_scorers.items(): - new_states[k] = d.select_state(part_states[k], part_idx) - return new_states - - def search( - self, - running_hyps: List[Hypothesis], - x: torch.Tensor, - x_mask: torch.Tensor = None, - pre_acoustic_embeds: torch.Tensor = None, - cache: dict = {}, - ) -> List[Hypothesis]: - """Search new tokens for running hypotheses and encoded speech x. - - Args: - running_hyps (List[Hypothesis]): Running hypotheses on beam - x (torch.Tensor): Encoded speech feature (T, D) - - Returns: - List[Hypotheses]: Best sorted hypotheses - - """ - best_hyps = [] - part_ids = torch.arange(self.n_vocab, device=x.device) # no pre-beam - for hyp in running_hyps: - # scoring - weighted_scores = torch.zeros(self.n_vocab, dtype=x.dtype, device=x.device) - scores, states = self.score_full( - hyp, - x, - x_mask=x_mask, - pre_acoustic_embeds=pre_acoustic_embeds, - cache=cache, - ) - for k in self.full_scorers: - weighted_scores += self.weights[k] * scores[k] - # partial scoring - if self.do_pre_beam: - pre_beam_scores = ( - weighted_scores - if self.pre_beam_score_key == "full" - else scores[self.pre_beam_score_key] - ) - part_ids = torch.topk(pre_beam_scores, self.pre_beam_size)[1] - part_scores, part_states = self.score_partial(hyp, part_ids, x) - for k in self.part_scorers: - weighted_scores[part_ids] += self.weights[k] * part_scores[k] - # add previous hyp score - weighted_scores += hyp.score - - # update hyps - for j, part_j in zip(*self.beam(weighted_scores, part_ids)): - # will be (2 x beam at most) - best_hyps.append( - Hypothesis( - score=weighted_scores[j], - yseq=self.append_token(hyp.yseq, j), - scores=self.merge_scores( - hyp.scores, scores, j, part_scores, part_j - ), - states=self.merge_states(states, part_states, part_j), - ) - ) - - # sort and prune 2 x beam -> beam - best_hyps = sorted(best_hyps, key=lambda x: x.score, reverse=True)[ - : min(len(best_hyps), self.beam_size) - ] - return best_hyps - - def forward( - self, - x: torch.Tensor, - scama_mask: torch.Tensor = None, - pre_acoustic_embeds: torch.Tensor = None, - maxlenratio: float = 0.0, - minlenratio: float = 0.0, - maxlen: int = None, - minlen: int = 0, - cache: dict = {}, - ) -> List[Hypothesis]: - """Perform beam search. - - Args: - x (torch.Tensor): Encoded speech feature (T, D) - maxlenratio (float): Input length ratio to obtain max output length. - If maxlenratio=0.0 (default), it uses a end-detect function - to automatically find maximum hypothesis lengths - If maxlenratio<0.0, its absolute value is interpreted - as a constant max output length. - minlenratio (float): Input length ratio to obtain min output length. - - Returns: - list[Hypothesis]: N-best decoding results - - """ - if maxlen is None: - # set length bounds - if maxlenratio == 0: - maxlen = x.shape[0] - elif maxlenratio < 0: - maxlen = -1 * int(maxlenratio) - else: - maxlen = max(1, int(maxlenratio * x.size(0))) - minlen = int(minlenratio * x.size(0)) - - logging.info("decoder input length: " + str(x.shape[0])) - logging.info("max output length: " + str(maxlen)) - logging.info("min output length: " + str(minlen)) - - # main loop of prefix search - # running_hyps = self.init_hyp(x) - running_hyps = cache["running_hyps"] - ended_hyps = [] - for i in range(maxlen): - logging.debug("position " + str(i)) - mask_enc = None - # if scama_mask is not None: - # token_num_predictor = scama_mask.size(1) - # token_id_slice = min(i, token_num_predictor-1) - # mask_enc = scama_mask[:, token_id_slice:token_id_slice+1, :] - # # if mask_enc.size(1) == 0: - # # mask_enc = scama_mask[:, -2:-1, :] - # # # mask_enc = torch.zeros_like(mask_enc) - pre_acoustic_embeds_cur = None - if pre_acoustic_embeds is not None: - b, t, d = pre_acoustic_embeds.size() - pad = torch.zeros((b, 1, d), dtype=pre_acoustic_embeds.dtype).to( - device=pre_acoustic_embeds.device - ) - pre_acoustic_embeds = torch.cat((pre_acoustic_embeds, pad), dim=1) - token_id_slice = min(i, t) - pre_acoustic_embeds_cur = pre_acoustic_embeds[ - :, token_id_slice : token_id_slice + 1, : - ] - - best = self.search( - running_hyps, - x, - x_mask=mask_enc, - pre_acoustic_embeds=pre_acoustic_embeds_cur, - cache=cache["decoder"], - ) - # post process of one iteration - running_hyps = self.post_process(i, maxlen, maxlenratio, best, ended_hyps) - # end detection - if maxlenratio == 0.0 and end_detect([h.asdict() for h in ended_hyps], i): - logging.info(f"end detected at {i}") - break - if len(running_hyps) == 0: - logging.info("no hypothesis. Finish decoding.") - break - else: - logging.debug(f"remained hypotheses: {len(running_hyps)}") - - nbest_hyps = sorted(ended_hyps, key=lambda x: x.score, reverse=True) - # check the number of hypotheses reaching to eos - if len(nbest_hyps) == 0: - logging.warning( - "there is no N-best results, perform recognition " - "again with smaller minlenratio." - ) - return ( - [] - if minlenratio < 0.1 - else self.forward(x, maxlenratio, max(0.0, minlenratio - 0.1)) - ) - - # report the best result - for x in nbest_hyps: - yseq = "".join([self.token_list[x] for x in x.yseq]) - logging.debug( - "nbest: y: {}, yseq: {}, score: {}".format(x.yseq, yseq, x.score) - ) - best = nbest_hyps[0] - for k, v in best.scores.items(): - logging.info( - f"{v:6.2f} * {self.weights[k]:3} = {v * self.weights[k]:6.2f} for {k}" - ) - logging.info(f"total log probability: {best.score:.2f}") - logging.info(f"normalized log probability: {best.score / len(best.yseq):.2f}") - logging.info(f"total number of ended hypotheses: {len(nbest_hyps)}") - if self.token_list is not None: - logging.info( - "best hypo: " - + "".join([self.token_list[x] for x in best.yseq[1:-1]]) - + "\n" - ) - return nbest_hyps - - def post_process( - self, - i: int, - maxlen: int, - maxlenratio: float, - running_hyps: List[Hypothesis], - ended_hyps: List[Hypothesis], - ) -> List[Hypothesis]: - """Perform post-processing of beam search iterations. - - Args: - i (int): The length of hypothesis tokens. - maxlen (int): The maximum length of tokens in beam search. - maxlenratio (int): The maximum length ratio in beam search. - running_hyps (List[Hypothesis]): The running hypotheses in beam search. - ended_hyps (List[Hypothesis]): The ended hypotheses in beam search. - - Returns: - List[Hypothesis]: The new running hypotheses. - - """ - logging.debug(f"the number of running hypotheses: {len(running_hyps)}") - if self.token_list is not None: - logging.debug( - "best hypo: " - + "".join([self.token_list[x] for x in running_hyps[0].yseq[1:]]) - ) - # add eos in the final loop to avoid that there are no ended hyps - if i == maxlen - 1: - logging.info("adding in the last position in the loop") - running_hyps = [ - h._replace(yseq=self.append_token(h.yseq, self.eos)) - for h in running_hyps - ] - - # add ended hypotheses to a final list, and removed them from current hypotheses - # (this will be a problem, number of hyps < beam) - remained_hyps = [] - for hyp in running_hyps: - if hyp.yseq[-1] == self.eos: - # e.g., Word LM needs to add final score - for k, d in chain(self.full_scorers.items(), self.part_scorers.items()): - s = d.final_score(hyp.states[k]) - hyp.scores[k] += s - hyp = hyp._replace(score=hyp.score + self.weights[k] * s) - ended_hyps.append(hyp) - else: - remained_hyps.append(hyp) - return remained_hyps diff --git a/funasr_detach/models/scama/chunk_utilis.py b/funasr_detach/models/scama/chunk_utilis.py deleted file mode 100644 index d7b005690690a081487dafbde3b778c67d8d9e3c..0000000000000000000000000000000000000000 --- a/funasr_detach/models/scama/chunk_utilis.py +++ /dev/null @@ -1,591 +0,0 @@ -import math -import torch -import numpy as np -import torch.nn.functional as F - -from funasr_detach.models.scama.utils import sequence_mask -from funasr_detach.models.transformer.utils.nets_utils import make_pad_mask - - -class overlap_chunk: - """ - Author: Speech Lab of DAMO Academy, Alibaba Group - San-m: Memory equipped self-attention for end-to-end speech recognition - https://arxiv.org/abs/2006.01713 - - """ - - def __init__( - self, - chunk_size: tuple = (16,), - stride: tuple = (10,), - pad_left: tuple = (0,), - encoder_att_look_back_factor: tuple = (1,), - shfit_fsmn: int = 0, - decoder_att_look_back_factor: tuple = (1,), - ): - - pad_left = self.check_chunk_size_args(chunk_size, pad_left) - encoder_att_look_back_factor = self.check_chunk_size_args( - chunk_size, encoder_att_look_back_factor - ) - decoder_att_look_back_factor = self.check_chunk_size_args( - chunk_size, decoder_att_look_back_factor - ) - ( - self.chunk_size, - self.stride, - self.pad_left, - self.encoder_att_look_back_factor, - self.decoder_att_look_back_factor, - ) = ( - chunk_size, - stride, - pad_left, - encoder_att_look_back_factor, - decoder_att_look_back_factor, - ) - self.shfit_fsmn = shfit_fsmn - self.x_add_mask = None - self.x_rm_mask = None - self.x_len = None - self.mask_shfit_chunk = None - self.mask_chunk_predictor = None - self.mask_att_chunk_encoder = None - self.mask_shift_att_chunk_decoder = None - self.chunk_outs = None - ( - self.chunk_size_cur, - self.stride_cur, - self.pad_left_cur, - self.encoder_att_look_back_factor_cur, - self.chunk_size_pad_shift_cur, - ) = (None, None, None, None, None) - - def check_chunk_size_args(self, chunk_size, x): - if len(x) < len(chunk_size): - x = [x[0] for i in chunk_size] - return x - - def get_chunk_size(self, ind: int = 0): - # with torch.no_grad: - ( - chunk_size, - stride, - pad_left, - encoder_att_look_back_factor, - decoder_att_look_back_factor, - ) = ( - self.chunk_size[ind], - self.stride[ind], - self.pad_left[ind], - self.encoder_att_look_back_factor[ind], - self.decoder_att_look_back_factor[ind], - ) - ( - self.chunk_size_cur, - self.stride_cur, - self.pad_left_cur, - self.encoder_att_look_back_factor_cur, - self.chunk_size_pad_shift_cur, - self.decoder_att_look_back_factor_cur, - ) = ( - chunk_size, - stride, - pad_left, - encoder_att_look_back_factor, - chunk_size + self.shfit_fsmn, - decoder_att_look_back_factor, - ) - return ( - self.chunk_size_cur, - self.stride_cur, - self.pad_left_cur, - self.encoder_att_look_back_factor_cur, - self.chunk_size_pad_shift_cur, - ) - - def random_choice(self, training=True, decoding_ind=None): - chunk_num = len(self.chunk_size) - ind = 0 - if training and chunk_num > 1: - ind = torch.randint(0, chunk_num, ()).cpu().item() - if not training and decoding_ind is not None: - ind = int(decoding_ind) - - return ind - - def gen_chunk_mask(self, x_len, ind=0, num_units=1, num_units_predictor=1): - - with torch.no_grad(): - x_len = x_len.cpu().numpy() - x_len_max = x_len.max() - - ( - chunk_size, - stride, - pad_left, - encoder_att_look_back_factor, - chunk_size_pad_shift, - ) = self.get_chunk_size(ind) - shfit_fsmn = self.shfit_fsmn - pad_right = chunk_size - stride - pad_left - - chunk_num_batch = np.ceil(x_len / stride).astype(np.int32) - x_len_chunk = ( - (chunk_num_batch - 1) * chunk_size_pad_shift - + shfit_fsmn - + pad_left - + 0 - + x_len - - (chunk_num_batch - 1) * stride - ) - x_len_chunk = x_len_chunk.astype(x_len.dtype) - x_len_chunk_max = x_len_chunk.max() - - chunk_num = int(math.ceil(x_len_max / stride)) - dtype = np.int32 - max_len_for_x_mask_tmp = max(chunk_size, x_len_max + pad_left) - x_add_mask = np.zeros([0, max_len_for_x_mask_tmp], dtype=dtype) - x_rm_mask = np.zeros([max_len_for_x_mask_tmp, 0], dtype=dtype) - mask_shfit_chunk = np.zeros([0, num_units], dtype=dtype) - mask_chunk_predictor = np.zeros([0, num_units_predictor], dtype=dtype) - mask_shift_att_chunk_decoder = np.zeros([0, 1], dtype=dtype) - mask_att_chunk_encoder = np.zeros( - [0, chunk_num * chunk_size_pad_shift], dtype=dtype - ) - for chunk_ids in range(chunk_num): - # x_mask add - fsmn_padding = np.zeros( - (shfit_fsmn, max_len_for_x_mask_tmp), dtype=dtype - ) - x_mask_cur = np.diag(np.ones(chunk_size, dtype=np.float32)) - x_mask_pad_left = np.zeros( - (chunk_size, chunk_ids * stride), dtype=dtype - ) - x_mask_pad_right = np.zeros( - (chunk_size, max_len_for_x_mask_tmp), dtype=dtype - ) - x_cur_pad = np.concatenate( - [x_mask_pad_left, x_mask_cur, x_mask_pad_right], axis=1 - ) - x_cur_pad = x_cur_pad[:chunk_size, :max_len_for_x_mask_tmp] - x_add_mask_fsmn = np.concatenate([fsmn_padding, x_cur_pad], axis=0) - x_add_mask = np.concatenate([x_add_mask, x_add_mask_fsmn], axis=0) - - # x_mask rm - fsmn_padding = np.zeros( - (max_len_for_x_mask_tmp, shfit_fsmn), dtype=dtype - ) - padding_mask_left = np.zeros( - (max_len_for_x_mask_tmp, pad_left), dtype=dtype - ) - padding_mask_right = np.zeros( - (max_len_for_x_mask_tmp, pad_right), dtype=dtype - ) - x_mask_cur = np.diag(np.ones(stride, dtype=dtype)) - x_mask_cur_pad_top = np.zeros((chunk_ids * stride, stride), dtype=dtype) - x_mask_cur_pad_bottom = np.zeros( - (max_len_for_x_mask_tmp, stride), dtype=dtype - ) - x_rm_mask_cur = np.concatenate( - [x_mask_cur_pad_top, x_mask_cur, x_mask_cur_pad_bottom], axis=0 - ) - x_rm_mask_cur = x_rm_mask_cur[:max_len_for_x_mask_tmp, :stride] - x_rm_mask_cur_fsmn = np.concatenate( - [ - fsmn_padding, - padding_mask_left, - x_rm_mask_cur, - padding_mask_right, - ], - axis=1, - ) - x_rm_mask = np.concatenate([x_rm_mask, x_rm_mask_cur_fsmn], axis=1) - - # fsmn_padding_mask - pad_shfit_mask = np.zeros([shfit_fsmn, num_units], dtype=dtype) - ones_1 = np.ones([chunk_size, num_units], dtype=dtype) - mask_shfit_chunk_cur = np.concatenate([pad_shfit_mask, ones_1], axis=0) - mask_shfit_chunk = np.concatenate( - [mask_shfit_chunk, mask_shfit_chunk_cur], axis=0 - ) - - # predictor mask - zeros_1 = np.zeros( - [shfit_fsmn + pad_left, num_units_predictor], dtype=dtype - ) - ones_2 = np.ones([stride, num_units_predictor], dtype=dtype) - zeros_3 = np.zeros( - [chunk_size - stride - pad_left, num_units_predictor], dtype=dtype - ) - ones_zeros = np.concatenate([ones_2, zeros_3], axis=0) - mask_chunk_predictor_cur = np.concatenate([zeros_1, ones_zeros], axis=0) - mask_chunk_predictor = np.concatenate( - [mask_chunk_predictor, mask_chunk_predictor_cur], axis=0 - ) - - # encoder att mask - zeros_1_top = np.zeros( - [shfit_fsmn, chunk_num * chunk_size_pad_shift], dtype=dtype - ) - - zeros_2_num = max(chunk_ids - encoder_att_look_back_factor, 0) - zeros_2 = np.zeros( - [chunk_size, zeros_2_num * chunk_size_pad_shift], dtype=dtype - ) - - encoder_att_look_back_num = max(chunk_ids - zeros_2_num, 0) - zeros_2_left = np.zeros([chunk_size, shfit_fsmn], dtype=dtype) - ones_2_mid = np.ones([stride, stride], dtype=dtype) - zeros_2_bottom = np.zeros([chunk_size - stride, stride], dtype=dtype) - zeros_2_right = np.zeros([chunk_size, chunk_size - stride], dtype=dtype) - ones_2 = np.concatenate([ones_2_mid, zeros_2_bottom], axis=0) - ones_2 = np.concatenate([zeros_2_left, ones_2, zeros_2_right], axis=1) - ones_2 = np.tile(ones_2, [1, encoder_att_look_back_num]) - - zeros_3_left = np.zeros([chunk_size, shfit_fsmn], dtype=dtype) - ones_3_right = np.ones([chunk_size, chunk_size], dtype=dtype) - ones_3 = np.concatenate([zeros_3_left, ones_3_right], axis=1) - - zeros_remain_num = max(chunk_num - 1 - chunk_ids, 0) - zeros_remain = np.zeros( - [chunk_size, zeros_remain_num * chunk_size_pad_shift], dtype=dtype - ) - - ones2_bottom = np.concatenate( - [zeros_2, ones_2, ones_3, zeros_remain], axis=1 - ) - mask_att_chunk_encoder_cur = np.concatenate( - [zeros_1_top, ones2_bottom], axis=0 - ) - mask_att_chunk_encoder = np.concatenate( - [mask_att_chunk_encoder, mask_att_chunk_encoder_cur], axis=0 - ) - - # decoder fsmn_shift_att_mask - zeros_1 = np.zeros([shfit_fsmn, 1]) - ones_1 = np.ones([chunk_size, 1]) - mask_shift_att_chunk_decoder_cur = np.concatenate( - [zeros_1, ones_1], axis=0 - ) - mask_shift_att_chunk_decoder = np.concatenate( - [mask_shift_att_chunk_decoder, mask_shift_att_chunk_decoder_cur], - axis=0, - ) - - self.x_add_mask = x_add_mask[:x_len_chunk_max, : x_len_max + pad_left] - self.x_len_chunk = x_len_chunk - self.x_rm_mask = x_rm_mask[:x_len_max, :x_len_chunk_max] - self.x_len = x_len - self.mask_shfit_chunk = mask_shfit_chunk[:x_len_chunk_max, :] - self.mask_chunk_predictor = mask_chunk_predictor[:x_len_chunk_max, :] - self.mask_att_chunk_encoder = mask_att_chunk_encoder[ - :x_len_chunk_max, :x_len_chunk_max - ] - self.mask_shift_att_chunk_decoder = mask_shift_att_chunk_decoder[ - :x_len_chunk_max, : - ] - self.chunk_outs = ( - self.x_add_mask, - self.x_len_chunk, - self.x_rm_mask, - self.x_len, - self.mask_shfit_chunk, - self.mask_chunk_predictor, - self.mask_att_chunk_encoder, - self.mask_shift_att_chunk_decoder, - ) - - return self.chunk_outs - - def split_chunk(self, x, x_len, chunk_outs): - """ - :param x: (b, t, d) - :param x_length: (b) - :param ind: int - :return: - """ - x = x[:, : x_len.max(), :] - b, t, d = x.size() - x_len_mask = (~make_pad_mask(x_len, maxlen=t)).to(x.device) - x *= x_len_mask[:, :, None] - - x_add_mask = self.get_x_add_mask(chunk_outs, x.device, dtype=x.dtype) - x_len_chunk = self.get_x_len_chunk(chunk_outs, x_len.device, dtype=x_len.dtype) - pad = (0, 0, self.pad_left_cur, 0) - x = F.pad(x, pad, "constant", 0.0) - b, t, d = x.size() - x = torch.transpose(x, 1, 0) - x = torch.reshape(x, [t, -1]) - x_chunk = torch.mm(x_add_mask, x) - x_chunk = torch.reshape(x_chunk, [-1, b, d]).transpose(1, 0) - - return x_chunk, x_len_chunk - - def remove_chunk(self, x_chunk, x_len_chunk, chunk_outs): - x_chunk = x_chunk[:, : x_len_chunk.max(), :] - b, t, d = x_chunk.size() - x_len_chunk_mask = (~make_pad_mask(x_len_chunk, maxlen=t)).to(x_chunk.device) - x_chunk *= x_len_chunk_mask[:, :, None] - - x_rm_mask = self.get_x_rm_mask(chunk_outs, x_chunk.device, dtype=x_chunk.dtype) - x_len = self.get_x_len(chunk_outs, x_len_chunk.device, dtype=x_len_chunk.dtype) - x_chunk = torch.transpose(x_chunk, 1, 0) - x_chunk = torch.reshape(x_chunk, [t, -1]) - x = torch.mm(x_rm_mask, x_chunk) - x = torch.reshape(x, [-1, b, d]).transpose(1, 0) - - return x, x_len - - def get_x_add_mask(self, chunk_outs=None, device="cpu", idx=0, dtype=torch.float32): - with torch.no_grad(): - x = chunk_outs[idx] if chunk_outs is not None else self.chunk_outs[idx] - x = torch.from_numpy(x).type(dtype).to(device) - return x - - def get_x_len_chunk( - self, chunk_outs=None, device="cpu", idx=1, dtype=torch.float32 - ): - with torch.no_grad(): - x = chunk_outs[idx] if chunk_outs is not None else self.chunk_outs[idx] - x = torch.from_numpy(x).type(dtype).to(device) - return x - - def get_x_rm_mask(self, chunk_outs=None, device="cpu", idx=2, dtype=torch.float32): - with torch.no_grad(): - x = chunk_outs[idx] if chunk_outs is not None else self.chunk_outs[idx] - x = torch.from_numpy(x).type(dtype).to(device) - return x - - def get_x_len(self, chunk_outs=None, device="cpu", idx=3, dtype=torch.float32): - with torch.no_grad(): - x = chunk_outs[idx] if chunk_outs is not None else self.chunk_outs[idx] - x = torch.from_numpy(x).type(dtype).to(device) - return x - - def get_mask_shfit_chunk( - self, - chunk_outs=None, - device="cpu", - batch_size=1, - num_units=1, - idx=4, - dtype=torch.float32, - ): - with torch.no_grad(): - x = chunk_outs[idx] if chunk_outs is not None else self.chunk_outs[idx] - x = np.tile( - x[ - None, - :, - :, - ], - [batch_size, 1, num_units], - ) - x = torch.from_numpy(x).type(dtype).to(device) - return x - - def get_mask_chunk_predictor( - self, - chunk_outs=None, - device="cpu", - batch_size=1, - num_units=1, - idx=5, - dtype=torch.float32, - ): - with torch.no_grad(): - x = chunk_outs[idx] if chunk_outs is not None else self.chunk_outs[idx] - x = np.tile( - x[ - None, - :, - :, - ], - [batch_size, 1, num_units], - ) - x = torch.from_numpy(x).type(dtype).to(device) - return x - - def get_mask_att_chunk_encoder( - self, chunk_outs=None, device="cpu", batch_size=1, idx=6, dtype=torch.float32 - ): - with torch.no_grad(): - x = chunk_outs[idx] if chunk_outs is not None else self.chunk_outs[idx] - x = np.tile( - x[ - None, - :, - :, - ], - [batch_size, 1, 1], - ) - x = torch.from_numpy(x).type(dtype).to(device) - return x - - def get_mask_shift_att_chunk_decoder( - self, chunk_outs=None, device="cpu", batch_size=1, idx=7, dtype=torch.float32 - ): - with torch.no_grad(): - x = chunk_outs[idx] if chunk_outs is not None else self.chunk_outs[idx] - x = np.tile(x[None, None, :, 0], [batch_size, 1, 1]) - x = torch.from_numpy(x).type(dtype).to(device) - return x - - -def build_scama_mask_for_cross_attention_decoder( - predictor_alignments: torch.Tensor, - encoder_sequence_length: torch.Tensor, - chunk_size: int = 5, - encoder_chunk_size: int = 5, - attention_chunk_center_bias: int = 0, - attention_chunk_size: int = 1, - attention_chunk_type: str = "chunk", - step=None, - predictor_mask_chunk_hopping: torch.Tensor = None, - decoder_att_look_back_factor: int = 1, - mask_shift_att_chunk_decoder: torch.Tensor = None, - target_length: torch.Tensor = None, - is_training=True, - dtype: torch.dtype = torch.float32, -): - with torch.no_grad(): - device = predictor_alignments.device - batch_size, chunk_num = predictor_alignments.size() - maximum_encoder_length = encoder_sequence_length.max().item() - int_type = predictor_alignments.dtype - if not is_training: - target_length = predictor_alignments.sum(dim=-1).type( - encoder_sequence_length.dtype - ) - maximum_target_length = target_length.max() - predictor_alignments_cumsum = torch.cumsum(predictor_alignments, dim=1) - predictor_alignments_cumsum = predictor_alignments_cumsum[:, None, :].repeat( - 1, maximum_target_length, 1 - ) - - index = torch.ones([batch_size, maximum_target_length], dtype=int_type).to( - device - ) - index = torch.cumsum(index, dim=1) - index = index[:, :, None].repeat(1, 1, chunk_num) - - index_div = torch.floor(torch.divide(predictor_alignments_cumsum, index)).type( - int_type - ) - index_div_bool_zeros = index_div == 0 - index_div_bool_zeros_count = ( - torch.sum(index_div_bool_zeros.type(int_type), dim=-1) + 1 - ) - - index_div_bool_zeros_count = torch.clip( - index_div_bool_zeros_count, min=1, max=chunk_num - ) - - index_div_bool_zeros_count *= chunk_size - index_div_bool_zeros_count += attention_chunk_center_bias - index_div_bool_zeros_count = torch.clip( - index_div_bool_zeros_count - 1, min=0, max=maximum_encoder_length - ) - index_div_bool_zeros_count_ori = index_div_bool_zeros_count - - index_div_bool_zeros_count = ( - torch.floor(index_div_bool_zeros_count / encoder_chunk_size) + 1 - ) * encoder_chunk_size - max_len_chunk = ( - math.ceil(maximum_encoder_length / encoder_chunk_size) * encoder_chunk_size - ) - - mask_flip, mask_flip2 = None, None - if attention_chunk_size is not None: - index_div_bool_zeros_count_beg = ( - index_div_bool_zeros_count - attention_chunk_size - ) - index_div_bool_zeros_count_beg = torch.clip( - index_div_bool_zeros_count_beg, 0, max_len_chunk - ) - index_div_bool_zeros_count_beg_mask = sequence_mask( - index_div_bool_zeros_count_beg, - maxlen=max_len_chunk, - dtype=int_type, - device=device, - ) - mask_flip = 1 - index_div_bool_zeros_count_beg_mask - attention_chunk_size2 = attention_chunk_size * ( - decoder_att_look_back_factor + 1 - ) - index_div_bool_zeros_count_beg = ( - index_div_bool_zeros_count - attention_chunk_size2 - ) - - index_div_bool_zeros_count_beg = torch.clip( - index_div_bool_zeros_count_beg, 0, max_len_chunk - ) - index_div_bool_zeros_count_beg_mask = sequence_mask( - index_div_bool_zeros_count_beg, - maxlen=max_len_chunk, - dtype=int_type, - device=device, - ) - mask_flip2 = 1 - index_div_bool_zeros_count_beg_mask - - mask = sequence_mask( - index_div_bool_zeros_count, maxlen=max_len_chunk, dtype=dtype, device=device - ) - - if predictor_mask_chunk_hopping is not None: - b, k, t = mask.size() - predictor_mask_chunk_hopping = predictor_mask_chunk_hopping[ - :, None, :, 0 - ].repeat(1, k, 1) - - mask_mask_flip = mask - if mask_flip is not None: - mask_mask_flip = mask_flip * mask - - def _fn(): - mask_sliced = mask[:b, :k, encoder_chunk_size:t] - zero_pad_right = torch.zeros( - [b, k, encoder_chunk_size], dtype=mask_sliced.dtype - ).to(device) - mask_sliced = torch.cat([mask_sliced, zero_pad_right], dim=2) - _, _, tt = predictor_mask_chunk_hopping.size() - pad_right_p = max_len_chunk - tt - predictor_mask_chunk_hopping_pad = torch.nn.functional.pad( - predictor_mask_chunk_hopping, [0, pad_right_p], "constant", 0 - ) - masked = mask_sliced * predictor_mask_chunk_hopping_pad - - mask_true = mask_mask_flip + masked - return mask_true - - mask = _fn() if t > chunk_size else mask_mask_flip - - if mask_flip2 is not None: - mask *= mask_flip2 - - mask_target = sequence_mask( - target_length, maxlen=maximum_target_length, dtype=mask.dtype, device=device - ) - mask = mask[:, :maximum_target_length, :] * mask_target[:, :, None] - - mask_len = sequence_mask( - encoder_sequence_length, - maxlen=maximum_encoder_length, - dtype=mask.dtype, - device=device, - ) - mask = mask[:, :, :maximum_encoder_length] * mask_len[:, None, :] - - if attention_chunk_type == "full": - mask = torch.ones_like(mask).to(device) - if mask_shift_att_chunk_decoder is not None: - mask = mask * mask_shift_att_chunk_decoder - mask = ( - mask[:, :maximum_target_length, :maximum_encoder_length] - .type(dtype) - .to(device) - ) - - return mask diff --git a/funasr_detach/models/scama/decoder.py b/funasr_detach/models/scama/decoder.py deleted file mode 100644 index d4abbfbb64472feea82e9a556b5411e9dac73108..0000000000000000000000000000000000000000 --- a/funasr_detach/models/scama/decoder.py +++ /dev/null @@ -1,1057 +0,0 @@ -#!/usr/bin/env python3 -# -*- encoding: utf-8 -*- -# Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved. -# MIT License (https://opensource.org/licenses/MIT) - -from typing import List -from typing import Tuple -import logging -import torch -import torch.nn as nn -import numpy as np - -from funasr_detach.models.scama import utils as myutils -from funasr_detach.models.transformer.decoder import BaseTransformerDecoder - -from funasr_detach.models.sanm.attention import ( - MultiHeadedAttentionSANMDecoder, - MultiHeadedAttentionCrossAtt, -) -from funasr_detach.models.transformer.embedding import PositionalEncoding -from funasr_detach.models.transformer.layer_norm import LayerNorm -from funasr_detach.models.sanm.positionwise_feed_forward import ( - PositionwiseFeedForwardDecoderSANM, -) -from funasr_detach.models.transformer.utils.repeat import repeat - -from funasr_detach.register import tables - - -class DecoderLayerSANM(nn.Module): - """Single decoder layer module. - - Args: - size (int): Input dimension. - self_attn (torch.nn.Module): Self-attention module instance. - `MultiHeadedAttention` instance can be used as the argument. - src_attn (torch.nn.Module): Self-attention module instance. - `MultiHeadedAttention` instance can be used as the argument. - feed_forward (torch.nn.Module): Feed-forward module instance. - `PositionwiseFeedForward`, `MultiLayeredConv1d`, or `Conv1dLinear` instance - can be used as the argument. - dropout_rate (float): Dropout rate. - normalize_before (bool): Whether to use layer_norm before the first block. - concat_after (bool): Whether to concat attention layer's input and output. - if True, additional linear will be applied. - i.e. x -> x + linear(concat(x, att(x))) - if False, no additional linear will be applied. i.e. x -> x + att(x) - - - """ - - def __init__( - self, - size, - self_attn, - src_attn, - feed_forward, - dropout_rate, - normalize_before=True, - concat_after=False, - ): - """Construct an DecoderLayer object.""" - super(DecoderLayerSANM, self).__init__() - self.size = size - self.self_attn = self_attn - self.src_attn = src_attn - self.feed_forward = feed_forward - self.norm1 = LayerNorm(size) - if self_attn is not None: - self.norm2 = LayerNorm(size) - if src_attn is not None: - self.norm3 = LayerNorm(size) - self.dropout = nn.Dropout(dropout_rate) - self.normalize_before = normalize_before - self.concat_after = concat_after - if self.concat_after: - self.concat_linear1 = nn.Linear(size + size, size) - self.concat_linear2 = nn.Linear(size + size, size) - - def forward(self, tgt, tgt_mask, memory, memory_mask=None, cache=None): - """Compute decoded features. - - Args: - tgt (torch.Tensor): Input tensor (#batch, maxlen_out, size). - tgt_mask (torch.Tensor): Mask for input tensor (#batch, maxlen_out). - memory (torch.Tensor): Encoded memory, float32 (#batch, maxlen_in, size). - memory_mask (torch.Tensor): Encoded memory mask (#batch, maxlen_in). - cache (List[torch.Tensor]): List of cached tensors. - Each tensor shape should be (#batch, maxlen_out - 1, size). - - Returns: - torch.Tensor: Output tensor(#batch, maxlen_out, size). - torch.Tensor: Mask for output tensor (#batch, maxlen_out). - torch.Tensor: Encoded memory (#batch, maxlen_in, size). - torch.Tensor: Encoded memory mask (#batch, maxlen_in). - - """ - # tgt = self.dropout(tgt) - residual = tgt - if self.normalize_before: - tgt = self.norm1(tgt) - tgt = self.feed_forward(tgt) - - x = tgt - if self.self_attn: - if self.normalize_before: - tgt = self.norm2(tgt) - x, _ = self.self_attn(tgt, tgt_mask) - x = residual + self.dropout(x) - - if self.src_attn is not None: - residual = x - if self.normalize_before: - x = self.norm3(x) - - x = residual + self.dropout(self.src_attn(x, memory, memory_mask)) - - return x, tgt_mask, memory, memory_mask, cache - - def forward_one_step(self, tgt, tgt_mask, memory, memory_mask=None, cache=None): - """Compute decoded features. - - Args: - tgt (torch.Tensor): Input tensor (#batch, maxlen_out, size). - tgt_mask (torch.Tensor): Mask for input tensor (#batch, maxlen_out). - memory (torch.Tensor): Encoded memory, float32 (#batch, maxlen_in, size). - memory_mask (torch.Tensor): Encoded memory mask (#batch, maxlen_in). - cache (List[torch.Tensor]): List of cached tensors. - Each tensor shape should be (#batch, maxlen_out - 1, size). - - Returns: - torch.Tensor: Output tensor(#batch, maxlen_out, size). - torch.Tensor: Mask for output tensor (#batch, maxlen_out). - torch.Tensor: Encoded memory (#batch, maxlen_in, size). - torch.Tensor: Encoded memory mask (#batch, maxlen_in). - - """ - # tgt = self.dropout(tgt) - residual = tgt - if self.normalize_before: - tgt = self.norm1(tgt) - tgt = self.feed_forward(tgt) - - x = tgt - if self.self_attn: - if self.normalize_before: - tgt = self.norm2(tgt) - if self.training: - cache = None - x, cache = self.self_attn(tgt, tgt_mask, cache=cache) - x = residual + self.dropout(x) - - if self.src_attn is not None: - residual = x - if self.normalize_before: - x = self.norm3(x) - - x = residual + self.dropout(self.src_attn(x, memory, memory_mask)) - - return x, tgt_mask, memory, memory_mask, cache - - def forward_chunk( - self, tgt, memory, fsmn_cache=None, opt_cache=None, chunk_size=None, look_back=0 - ): - """Compute decoded features. - - Args: - tgt (torch.Tensor): Input tensor (#batch, maxlen_out, size). - tgt_mask (torch.Tensor): Mask for input tensor (#batch, maxlen_out). - memory (torch.Tensor): Encoded memory, float32 (#batch, maxlen_in, size). - memory_mask (torch.Tensor): Encoded memory mask (#batch, maxlen_in). - cache (List[torch.Tensor]): List of cached tensors. - Each tensor shape should be (#batch, maxlen_out - 1, size). - - Returns: - torch.Tensor: Output tensor(#batch, maxlen_out, size). - torch.Tensor: Mask for output tensor (#batch, maxlen_out). - torch.Tensor: Encoded memory (#batch, maxlen_in, size). - torch.Tensor: Encoded memory mask (#batch, maxlen_in). - - """ - residual = tgt - if self.normalize_before: - tgt = self.norm1(tgt) - tgt = self.feed_forward(tgt) - - x = tgt - if self.self_attn: - if self.normalize_before: - tgt = self.norm2(tgt) - x, fsmn_cache = self.self_attn(tgt, None, fsmn_cache) - x = residual + self.dropout(x) - - if self.src_attn is not None: - residual = x - if self.normalize_before: - x = self.norm3(x) - - x, opt_cache = self.src_attn.forward_chunk( - x, memory, opt_cache, chunk_size, look_back - ) - x = residual + x - - return x, memory, fsmn_cache, opt_cache - - -@tables.register("decoder_classes", "FsmnDecoderSCAMAOpt") -class FsmnDecoderSCAMAOpt(BaseTransformerDecoder): - """ - Author: Shiliang Zhang, Zhifu Gao, Haoneng Luo, Ming Lei, Jie Gao, Zhijie Yan, Lei Xie - SCAMA: Streaming chunk-aware multihead attention for online end-to-end speech recognition - https://arxiv.org/abs/2006.01712 - """ - - def __init__( - self, - vocab_size: int, - encoder_output_size: int, - attention_heads: int = 4, - linear_units: int = 2048, - num_blocks: int = 6, - dropout_rate: float = 0.1, - positional_dropout_rate: float = 0.1, - self_attention_dropout_rate: float = 0.0, - src_attention_dropout_rate: float = 0.0, - input_layer: str = "embed", - use_output_layer: bool = True, - pos_enc_class=PositionalEncoding, - normalize_before: bool = True, - concat_after: bool = False, - att_layer_num: int = 6, - kernel_size: int = 21, - sanm_shfit: int = None, - concat_embeds: bool = False, - attention_dim: int = None, - tf2torch_tensor_name_prefix_torch: str = "decoder", - tf2torch_tensor_name_prefix_tf: str = "seq2seq/decoder", - embed_tensor_name_prefix_tf: str = None, - ): - super().__init__( - vocab_size=vocab_size, - encoder_output_size=encoder_output_size, - dropout_rate=dropout_rate, - positional_dropout_rate=positional_dropout_rate, - input_layer=input_layer, - use_output_layer=use_output_layer, - pos_enc_class=pos_enc_class, - normalize_before=normalize_before, - ) - if attention_dim is None: - attention_dim = encoder_output_size - - if input_layer == "embed": - self.embed = torch.nn.Sequential( - torch.nn.Embedding(vocab_size, attention_dim), - ) - elif input_layer == "linear": - self.embed = torch.nn.Sequential( - torch.nn.Linear(vocab_size, attention_dim), - torch.nn.LayerNorm(attention_dim), - torch.nn.Dropout(dropout_rate), - torch.nn.ReLU(), - pos_enc_class(attention_dim, positional_dropout_rate), - ) - else: - raise ValueError(f"only 'embed' or 'linear' is supported: {input_layer}") - - self.normalize_before = normalize_before - if self.normalize_before: - self.after_norm = LayerNorm(attention_dim) - if use_output_layer: - self.output_layer = torch.nn.Linear(attention_dim, vocab_size) - else: - self.output_layer = None - - self.att_layer_num = att_layer_num - self.num_blocks = num_blocks - if sanm_shfit is None: - sanm_shfit = (kernel_size - 1) // 2 - self.decoders = repeat( - att_layer_num, - lambda lnum: DecoderLayerSANM( - attention_dim, - MultiHeadedAttentionSANMDecoder( - attention_dim, - self_attention_dropout_rate, - kernel_size, - sanm_shfit=sanm_shfit, - ), - MultiHeadedAttentionCrossAtt( - attention_heads, - attention_dim, - src_attention_dropout_rate, - encoder_output_size=encoder_output_size, - ), - PositionwiseFeedForwardDecoderSANM( - attention_dim, linear_units, dropout_rate - ), - dropout_rate, - normalize_before, - concat_after, - ), - ) - if num_blocks - att_layer_num <= 0: - self.decoders2 = None - else: - self.decoders2 = repeat( - num_blocks - att_layer_num, - lambda lnum: DecoderLayerSANM( - attention_dim, - MultiHeadedAttentionSANMDecoder( - attention_dim, - self_attention_dropout_rate, - kernel_size, - sanm_shfit=sanm_shfit, - ), - None, - PositionwiseFeedForwardDecoderSANM( - attention_dim, linear_units, dropout_rate - ), - dropout_rate, - normalize_before, - concat_after, - ), - ) - - self.decoders3 = repeat( - 1, - lambda lnum: DecoderLayerSANM( - attention_dim, - None, - None, - PositionwiseFeedForwardDecoderSANM( - attention_dim, linear_units, dropout_rate - ), - dropout_rate, - normalize_before, - concat_after, - ), - ) - if concat_embeds: - self.embed_concat_ffn = repeat( - 1, - lambda lnum: DecoderLayerSANM( - attention_dim + encoder_output_size, - None, - None, - PositionwiseFeedForwardDecoderSANM( - attention_dim + encoder_output_size, - linear_units, - dropout_rate, - adim=attention_dim, - ), - dropout_rate, - normalize_before, - concat_after, - ), - ) - else: - self.embed_concat_ffn = None - self.concat_embeds = concat_embeds - self.tf2torch_tensor_name_prefix_torch = tf2torch_tensor_name_prefix_torch - self.tf2torch_tensor_name_prefix_tf = tf2torch_tensor_name_prefix_tf - self.embed_tensor_name_prefix_tf = embed_tensor_name_prefix_tf - - def forward( - self, - hs_pad: torch.Tensor, - hlens: torch.Tensor, - ys_in_pad: torch.Tensor, - ys_in_lens: torch.Tensor, - chunk_mask: torch.Tensor = None, - pre_acoustic_embeds: torch.Tensor = None, - ) -> Tuple[torch.Tensor, torch.Tensor]: - """Forward decoder. - - Args: - hs_pad: encoded memory, float32 (batch, maxlen_in, feat) - hlens: (batch) - ys_in_pad: - input token ids, int64 (batch, maxlen_out) - if input_layer == "embed" - input tensor (batch, maxlen_out, #mels) in the other cases - ys_in_lens: (batch) - Returns: - (tuple): tuple containing: - - x: decoded token score before softmax (batch, maxlen_out, token) - if use_output_layer is True, - olens: (batch, ) - """ - tgt = ys_in_pad - tgt_mask = myutils.sequence_mask(ys_in_lens, device=tgt.device)[:, :, None] - - memory = hs_pad - memory_mask = myutils.sequence_mask(hlens, device=memory.device)[:, None, :] - if chunk_mask is not None: - memory_mask = memory_mask * chunk_mask - if tgt_mask.size(1) != memory_mask.size(1): - memory_mask = torch.cat((memory_mask, memory_mask[:, -2:-1, :]), dim=1) - - x = self.embed(tgt) - - if pre_acoustic_embeds is not None and self.concat_embeds: - x = torch.cat((x, pre_acoustic_embeds), dim=-1) - x, _, _, _, _ = self.embed_concat_ffn(x, None, None, None, None) - - x, tgt_mask, memory, memory_mask, _ = self.decoders( - x, tgt_mask, memory, memory_mask - ) - if self.decoders2 is not None: - x, tgt_mask, memory, memory_mask, _ = self.decoders2( - x, tgt_mask, memory, memory_mask - ) - x, tgt_mask, memory, memory_mask, _ = self.decoders3( - x, tgt_mask, memory, memory_mask - ) - if self.normalize_before: - x = self.after_norm(x) - if self.output_layer is not None: - x = self.output_layer(x) - - olens = tgt_mask.sum(1) - return x, olens - - def score( - self, - ys, - state, - x, - x_mask=None, - pre_acoustic_embeds: torch.Tensor = None, - ): - """Score.""" - ys_mask = myutils.sequence_mask( - torch.tensor([len(ys)], dtype=torch.int32), device=x.device - )[:, :, None] - logp, state = self.forward_one_step( - ys.unsqueeze(0), - ys_mask, - x.unsqueeze(0), - memory_mask=x_mask, - pre_acoustic_embeds=pre_acoustic_embeds, - cache=state, - ) - return logp.squeeze(0), state - - def forward_one_step( - self, - tgt: torch.Tensor, - tgt_mask: torch.Tensor, - memory: torch.Tensor, - memory_mask: torch.Tensor = None, - pre_acoustic_embeds: torch.Tensor = None, - cache: List[torch.Tensor] = None, - ) -> Tuple[torch.Tensor, List[torch.Tensor]]: - """Forward one step. - - Args: - tgt: input token ids, int64 (batch, maxlen_out) - tgt_mask: input token mask, (batch, maxlen_out) - dtype=torch.uint8 in PyTorch 1.2- - dtype=torch.bool in PyTorch 1.2+ (include 1.2) - memory: encoded memory, float32 (batch, maxlen_in, feat) - cache: cached output list of (batch, max_time_out-1, size) - Returns: - y, cache: NN output value and cache per `self.decoders`. - y.shape` is (batch, maxlen_out, token) - """ - - x = tgt[:, -1:] - tgt_mask = None - x = self.embed(x) - - if pre_acoustic_embeds is not None and self.concat_embeds: - x = torch.cat((x, pre_acoustic_embeds), dim=-1) - x, _, _, _, _ = self.embed_concat_ffn(x, None, None, None, None) - - if cache is None: - cache_layer_num = len(self.decoders) - if self.decoders2 is not None: - cache_layer_num += len(self.decoders2) - cache = [None] * cache_layer_num - new_cache = [] - # for c, decoder in zip(cache, self.decoders): - for i in range(self.att_layer_num): - decoder = self.decoders[i] - c = cache[i] - x, tgt_mask, memory, memory_mask, c_ret = decoder.forward_one_step( - x, tgt_mask, memory, memory_mask, cache=c - ) - new_cache.append(c_ret) - - if self.num_blocks - self.att_layer_num >= 1: - for i in range(self.num_blocks - self.att_layer_num): - j = i + self.att_layer_num - decoder = self.decoders2[i] - c = cache[j] - x, tgt_mask, memory, memory_mask, c_ret = decoder.forward_one_step( - x, tgt_mask, memory, memory_mask, cache=c - ) - new_cache.append(c_ret) - - for decoder in self.decoders3: - x, tgt_mask, memory, memory_mask, _ = decoder.forward_one_step( - x, tgt_mask, memory, None, cache=None - ) - - if self.normalize_before: - y = self.after_norm(x[:, -1]) - else: - y = x[:, -1] - if self.output_layer is not None: - y = self.output_layer(y) - y = torch.log_softmax(y, dim=-1) - - return y, new_cache - - def gen_tf2torch_map_dict(self): - - tensor_name_prefix_torch = self.tf2torch_tensor_name_prefix_torch - tensor_name_prefix_tf = self.tf2torch_tensor_name_prefix_tf - embed_tensor_name_prefix_tf = ( - self.embed_tensor_name_prefix_tf - if self.embed_tensor_name_prefix_tf is not None - else tensor_name_prefix_tf - ) - map_dict_local = { - ## decoder - # ffn - "{}.decoders.layeridx.norm1.weight".format(tensor_name_prefix_torch): { - "name": "{}/decoder_fsmn_layer_layeridx/decoder_ffn/LayerNorm/gamma".format( - tensor_name_prefix_tf - ), - "squeeze": None, - "transpose": None, - }, # (256,),(256,) - "{}.decoders.layeridx.norm1.bias".format(tensor_name_prefix_torch): { - "name": "{}/decoder_fsmn_layer_layeridx/decoder_ffn/LayerNorm/beta".format( - tensor_name_prefix_tf - ), - "squeeze": None, - "transpose": None, - }, # (256,),(256,) - "{}.decoders.layeridx.feed_forward.w_1.weight".format( - tensor_name_prefix_torch - ): { - "name": "{}/decoder_fsmn_layer_layeridx/decoder_ffn/conv1d/kernel".format( - tensor_name_prefix_tf - ), - "squeeze": 0, - "transpose": (1, 0), - }, # (1024,256),(1,256,1024) - "{}.decoders.layeridx.feed_forward.w_1.bias".format( - tensor_name_prefix_torch - ): { - "name": "{}/decoder_fsmn_layer_layeridx/decoder_ffn/conv1d/bias".format( - tensor_name_prefix_tf - ), - "squeeze": None, - "transpose": None, - }, # (1024,),(1024,) - "{}.decoders.layeridx.feed_forward.norm.weight".format( - tensor_name_prefix_torch - ): { - "name": "{}/decoder_fsmn_layer_layeridx/decoder_ffn/LayerNorm_1/gamma".format( - tensor_name_prefix_tf - ), - "squeeze": None, - "transpose": None, - }, # (1024,),(1024,) - "{}.decoders.layeridx.feed_forward.norm.bias".format( - tensor_name_prefix_torch - ): { - "name": "{}/decoder_fsmn_layer_layeridx/decoder_ffn/LayerNorm_1/beta".format( - tensor_name_prefix_tf - ), - "squeeze": None, - "transpose": None, - }, # (1024,),(1024,) - "{}.decoders.layeridx.feed_forward.w_2.weight".format( - tensor_name_prefix_torch - ): { - "name": "{}/decoder_fsmn_layer_layeridx/decoder_ffn/conv1d_1/kernel".format( - tensor_name_prefix_tf - ), - "squeeze": 0, - "transpose": (1, 0), - }, # (256,1024),(1,1024,256) - # fsmn - "{}.decoders.layeridx.norm2.weight".format(tensor_name_prefix_torch): { - "name": "{}/decoder_fsmn_layer_layeridx/decoder_memory_block/LayerNorm/gamma".format( - tensor_name_prefix_tf - ), - "squeeze": None, - "transpose": None, - }, # (256,),(256,) - "{}.decoders.layeridx.norm2.bias".format(tensor_name_prefix_torch): { - "name": "{}/decoder_fsmn_layer_layeridx/decoder_memory_block/LayerNorm/beta".format( - tensor_name_prefix_tf - ), - "squeeze": None, - "transpose": None, - }, # (256,),(256,) - "{}.decoders.layeridx.self_attn.fsmn_block.weight".format( - tensor_name_prefix_torch - ): { - "name": "{}/decoder_fsmn_layer_layeridx/decoder_memory_block/depth_conv_w".format( - tensor_name_prefix_tf - ), - "squeeze": 0, - "transpose": (1, 2, 0), - }, # (256,1,31),(1,31,256,1) - # src att - "{}.decoders.layeridx.norm3.weight".format(tensor_name_prefix_torch): { - "name": "{}/decoder_fsmn_layer_layeridx/multi_head/LayerNorm/gamma".format( - tensor_name_prefix_tf - ), - "squeeze": None, - "transpose": None, - }, # (256,),(256,) - "{}.decoders.layeridx.norm3.bias".format(tensor_name_prefix_torch): { - "name": "{}/decoder_fsmn_layer_layeridx/multi_head/LayerNorm/beta".format( - tensor_name_prefix_tf - ), - "squeeze": None, - "transpose": None, - }, # (256,),(256,) - "{}.decoders.layeridx.src_attn.linear_q.weight".format( - tensor_name_prefix_torch - ): { - "name": "{}/decoder_fsmn_layer_layeridx/multi_head/conv1d/kernel".format( - tensor_name_prefix_tf - ), - "squeeze": 0, - "transpose": (1, 0), - }, # (256,256),(1,256,256) - "{}.decoders.layeridx.src_attn.linear_q.bias".format( - tensor_name_prefix_torch - ): { - "name": "{}/decoder_fsmn_layer_layeridx/multi_head/conv1d/bias".format( - tensor_name_prefix_tf - ), - "squeeze": None, - "transpose": None, - }, # (256,),(256,) - "{}.decoders.layeridx.src_attn.linear_k_v.weight".format( - tensor_name_prefix_torch - ): { - "name": "{}/decoder_fsmn_layer_layeridx/multi_head/conv1d_1/kernel".format( - tensor_name_prefix_tf - ), - "squeeze": 0, - "transpose": (1, 0), - }, # (1024,256),(1,256,1024) - "{}.decoders.layeridx.src_attn.linear_k_v.bias".format( - tensor_name_prefix_torch - ): { - "name": "{}/decoder_fsmn_layer_layeridx/multi_head/conv1d_1/bias".format( - tensor_name_prefix_tf - ), - "squeeze": None, - "transpose": None, - }, # (1024,),(1024,) - "{}.decoders.layeridx.src_attn.linear_out.weight".format( - tensor_name_prefix_torch - ): { - "name": "{}/decoder_fsmn_layer_layeridx/multi_head/conv1d_2/kernel".format( - tensor_name_prefix_tf - ), - "squeeze": 0, - "transpose": (1, 0), - }, # (256,256),(1,256,256) - "{}.decoders.layeridx.src_attn.linear_out.bias".format( - tensor_name_prefix_torch - ): { - "name": "{}/decoder_fsmn_layer_layeridx/multi_head/conv1d_2/bias".format( - tensor_name_prefix_tf - ), - "squeeze": None, - "transpose": None, - }, # (256,),(256,) - # dnn - "{}.decoders3.layeridx.norm1.weight".format(tensor_name_prefix_torch): { - "name": "{}/decoder_dnn_layer_layeridx/LayerNorm/gamma".format( - tensor_name_prefix_tf - ), - "squeeze": None, - "transpose": None, - }, # (256,),(256,) - "{}.decoders3.layeridx.norm1.bias".format(tensor_name_prefix_torch): { - "name": "{}/decoder_dnn_layer_layeridx/LayerNorm/beta".format( - tensor_name_prefix_tf - ), - "squeeze": None, - "transpose": None, - }, # (256,),(256,) - "{}.decoders3.layeridx.feed_forward.w_1.weight".format( - tensor_name_prefix_torch - ): { - "name": "{}/decoder_dnn_layer_layeridx/conv1d/kernel".format( - tensor_name_prefix_tf - ), - "squeeze": 0, - "transpose": (1, 0), - }, # (1024,256),(1,256,1024) - "{}.decoders3.layeridx.feed_forward.w_1.bias".format( - tensor_name_prefix_torch - ): { - "name": "{}/decoder_dnn_layer_layeridx/conv1d/bias".format( - tensor_name_prefix_tf - ), - "squeeze": None, - "transpose": None, - }, # (1024,),(1024,) - "{}.decoders3.layeridx.feed_forward.norm.weight".format( - tensor_name_prefix_torch - ): { - "name": "{}/decoder_dnn_layer_layeridx/LayerNorm_1/gamma".format( - tensor_name_prefix_tf - ), - "squeeze": None, - "transpose": None, - }, # (1024,),(1024,) - "{}.decoders3.layeridx.feed_forward.norm.bias".format( - tensor_name_prefix_torch - ): { - "name": "{}/decoder_dnn_layer_layeridx/LayerNorm_1/beta".format( - tensor_name_prefix_tf - ), - "squeeze": None, - "transpose": None, - }, # (1024,),(1024,) - "{}.decoders3.layeridx.feed_forward.w_2.weight".format( - tensor_name_prefix_torch - ): { - "name": "{}/decoder_dnn_layer_layeridx/conv1d_1/kernel".format( - tensor_name_prefix_tf - ), - "squeeze": 0, - "transpose": (1, 0), - }, # (256,1024),(1,1024,256) - # embed_concat_ffn - "{}.embed_concat_ffn.layeridx.norm1.weight".format( - tensor_name_prefix_torch - ): { - "name": "{}/cif_concat/LayerNorm/gamma".format(tensor_name_prefix_tf), - "squeeze": None, - "transpose": None, - }, # (256,),(256,) - "{}.embed_concat_ffn.layeridx.norm1.bias".format( - tensor_name_prefix_torch - ): { - "name": "{}/cif_concat/LayerNorm/beta".format(tensor_name_prefix_tf), - "squeeze": None, - "transpose": None, - }, # (256,),(256,) - "{}.embed_concat_ffn.layeridx.feed_forward.w_1.weight".format( - tensor_name_prefix_torch - ): { - "name": "{}/cif_concat/conv1d/kernel".format(tensor_name_prefix_tf), - "squeeze": 0, - "transpose": (1, 0), - }, # (1024,256),(1,256,1024) - "{}.embed_concat_ffn.layeridx.feed_forward.w_1.bias".format( - tensor_name_prefix_torch - ): { - "name": "{}/cif_concat/conv1d/bias".format(tensor_name_prefix_tf), - "squeeze": None, - "transpose": None, - }, # (1024,),(1024,) - "{}.embed_concat_ffn.layeridx.feed_forward.norm.weight".format( - tensor_name_prefix_torch - ): { - "name": "{}/cif_concat/LayerNorm_1/gamma".format(tensor_name_prefix_tf), - "squeeze": None, - "transpose": None, - }, # (1024,),(1024,) - "{}.embed_concat_ffn.layeridx.feed_forward.norm.bias".format( - tensor_name_prefix_torch - ): { - "name": "{}/cif_concat/LayerNorm_1/beta".format(tensor_name_prefix_tf), - "squeeze": None, - "transpose": None, - }, # (1024,),(1024,) - "{}.embed_concat_ffn.layeridx.feed_forward.w_2.weight".format( - tensor_name_prefix_torch - ): { - "name": "{}/cif_concat/conv1d_1/kernel".format(tensor_name_prefix_tf), - "squeeze": 0, - "transpose": (1, 0), - }, # (256,1024),(1,1024,256) - # out norm - "{}.after_norm.weight".format(tensor_name_prefix_torch): { - "name": "{}/LayerNorm/gamma".format(tensor_name_prefix_tf), - "squeeze": None, - "transpose": None, - }, # (256,),(256,) - "{}.after_norm.bias".format(tensor_name_prefix_torch): { - "name": "{}/LayerNorm/beta".format(tensor_name_prefix_tf), - "squeeze": None, - "transpose": None, - }, # (256,),(256,) - # in embed - "{}.embed.0.weight".format(tensor_name_prefix_torch): { - "name": "{}/w_embs".format(embed_tensor_name_prefix_tf), - "squeeze": None, - "transpose": None, - }, # (4235,256),(4235,256) - # out layer - "{}.output_layer.weight".format(tensor_name_prefix_torch): { - "name": [ - "{}/dense/kernel".format(tensor_name_prefix_tf), - "{}/w_embs".format(embed_tensor_name_prefix_tf), - ], - "squeeze": [None, None], - "transpose": [(1, 0), None], - }, # (4235,256),(256,4235) - "{}.output_layer.bias".format(tensor_name_prefix_torch): { - "name": [ - "{}/dense/bias".format(tensor_name_prefix_tf), - ( - "seq2seq/2bias" - if tensor_name_prefix_tf == "seq2seq/decoder/inputter_1" - else "seq2seq/bias" - ), - ], - "squeeze": [None, None], - "transpose": [None, None], - }, # (4235,),(4235,) - } - return map_dict_local - - def convert_tf2torch( - self, - var_dict_tf, - var_dict_torch, - ): - - map_dict = self.gen_tf2torch_map_dict() - var_dict_torch_update = dict() - decoder_layeridx_sets = set() - for name in sorted(var_dict_torch.keys(), reverse=False): - names = name.split(".") - if names[0] == self.tf2torch_tensor_name_prefix_torch: - if names[1] == "decoders": - layeridx = int(names[2]) - name_q = name.replace(".{}.".format(layeridx), ".layeridx.") - layeridx_bias = 0 - layeridx += layeridx_bias - decoder_layeridx_sets.add(layeridx) - if name_q in map_dict.keys(): - name_v = map_dict[name_q]["name"] - name_tf = name_v.replace("layeridx", "{}".format(layeridx)) - data_tf = var_dict_tf[name_tf] - if map_dict[name_q]["squeeze"] is not None: - data_tf = np.squeeze( - data_tf, axis=map_dict[name_q]["squeeze"] - ) - if map_dict[name_q]["transpose"] is not None: - data_tf = np.transpose( - data_tf, map_dict[name_q]["transpose"] - ) - data_tf = ( - torch.from_numpy(data_tf).type(torch.float32).to("cpu") - ) - assert ( - var_dict_torch[name].size() == data_tf.size() - ), "{}, {}, {} != {}".format( - name, name_tf, var_dict_torch[name].size(), data_tf.size() - ) - var_dict_torch_update[name] = data_tf - logging.info( - "torch tensor: {}, {}, loading from tf tensor: {}, {}".format( - name, data_tf.size(), name_v, var_dict_tf[name_tf].shape - ) - ) - - elif names[1] == "decoders2": - layeridx = int(names[2]) - name_q = name.replace(".{}.".format(layeridx), ".layeridx.") - name_q = name_q.replace("decoders2", "decoders") - layeridx_bias = len(decoder_layeridx_sets) - - layeridx += layeridx_bias - if "decoders." in name: - decoder_layeridx_sets.add(layeridx) - if name_q in map_dict.keys(): - name_v = map_dict[name_q]["name"] - name_tf = name_v.replace("layeridx", "{}".format(layeridx)) - data_tf = var_dict_tf[name_tf] - if map_dict[name_q]["squeeze"] is not None: - data_tf = np.squeeze( - data_tf, axis=map_dict[name_q]["squeeze"] - ) - if map_dict[name_q]["transpose"] is not None: - data_tf = np.transpose( - data_tf, map_dict[name_q]["transpose"] - ) - data_tf = ( - torch.from_numpy(data_tf).type(torch.float32).to("cpu") - ) - assert ( - var_dict_torch[name].size() == data_tf.size() - ), "{}, {}, {} != {}".format( - name, name_tf, var_dict_torch[name].size(), data_tf.size() - ) - var_dict_torch_update[name] = data_tf - logging.info( - "torch tensor: {}, {}, loading from tf tensor: {}, {}".format( - name, data_tf.size(), name_v, var_dict_tf[name_tf].shape - ) - ) - - elif names[1] == "decoders3": - layeridx = int(names[2]) - name_q = name.replace(".{}.".format(layeridx), ".layeridx.") - - layeridx_bias = 0 - layeridx += layeridx_bias - if "decoders." in name: - decoder_layeridx_sets.add(layeridx) - if name_q in map_dict.keys(): - name_v = map_dict[name_q]["name"] - name_tf = name_v.replace("layeridx", "{}".format(layeridx)) - data_tf = var_dict_tf[name_tf] - if map_dict[name_q]["squeeze"] is not None: - data_tf = np.squeeze( - data_tf, axis=map_dict[name_q]["squeeze"] - ) - if map_dict[name_q]["transpose"] is not None: - data_tf = np.transpose( - data_tf, map_dict[name_q]["transpose"] - ) - data_tf = ( - torch.from_numpy(data_tf).type(torch.float32).to("cpu") - ) - assert ( - var_dict_torch[name].size() == data_tf.size() - ), "{}, {}, {} != {}".format( - name, name_tf, var_dict_torch[name].size(), data_tf.size() - ) - var_dict_torch_update[name] = data_tf - logging.info( - "torch tensor: {}, {}, loading from tf tensor: {}, {}".format( - name, data_tf.size(), name_v, var_dict_tf[name_tf].shape - ) - ) - - elif names[1] == "embed" or names[1] == "output_layer": - name_tf = map_dict[name]["name"] - if isinstance(name_tf, list): - idx_list = 0 - if name_tf[idx_list] in var_dict_tf.keys(): - pass - else: - idx_list = 1 - data_tf = var_dict_tf[name_tf[idx_list]] - if map_dict[name]["squeeze"][idx_list] is not None: - data_tf = np.squeeze( - data_tf, axis=map_dict[name]["squeeze"][idx_list] - ) - if map_dict[name]["transpose"][idx_list] is not None: - data_tf = np.transpose( - data_tf, map_dict[name]["transpose"][idx_list] - ) - data_tf = ( - torch.from_numpy(data_tf).type(torch.float32).to("cpu") - ) - assert ( - var_dict_torch[name].size() == data_tf.size() - ), "{}, {}, {} != {}".format( - name, name_tf, var_dict_torch[name].size(), data_tf.size() - ) - var_dict_torch_update[name] = data_tf - logging.info( - "torch tensor: {}, {}, loading from tf tensor: {}, {}".format( - name, - data_tf.size(), - name_tf[idx_list], - var_dict_tf[name_tf[idx_list]].shape, - ) - ) - - else: - data_tf = var_dict_tf[name_tf] - if map_dict[name]["squeeze"] is not None: - data_tf = np.squeeze( - data_tf, axis=map_dict[name]["squeeze"] - ) - if map_dict[name]["transpose"] is not None: - data_tf = np.transpose(data_tf, map_dict[name]["transpose"]) - data_tf = ( - torch.from_numpy(data_tf).type(torch.float32).to("cpu") - ) - assert ( - var_dict_torch[name].size() == data_tf.size() - ), "{}, {}, {} != {}".format( - name, name_tf, var_dict_torch[name].size(), data_tf.size() - ) - var_dict_torch_update[name] = data_tf - logging.info( - "torch tensor: {}, {}, loading from tf tensor: {}, {}".format( - name, - data_tf.size(), - name_tf, - var_dict_tf[name_tf].shape, - ) - ) - - elif names[1] == "after_norm": - name_tf = map_dict[name]["name"] - data_tf = var_dict_tf[name_tf] - data_tf = torch.from_numpy(data_tf).type(torch.float32).to("cpu") - var_dict_torch_update[name] = data_tf - logging.info( - "torch tensor: {}, {}, loading from tf tensor: {}, {}".format( - name, data_tf.size(), name_tf, var_dict_tf[name_tf].shape - ) - ) - - elif names[1] == "embed_concat_ffn": - layeridx = int(names[2]) - name_q = name.replace(".{}.".format(layeridx), ".layeridx.") - - layeridx_bias = 0 - layeridx += layeridx_bias - if "decoders." in name: - decoder_layeridx_sets.add(layeridx) - if name_q in map_dict.keys(): - name_v = map_dict[name_q]["name"] - name_tf = name_v.replace("layeridx", "{}".format(layeridx)) - data_tf = var_dict_tf[name_tf] - if map_dict[name_q]["squeeze"] is not None: - data_tf = np.squeeze( - data_tf, axis=map_dict[name_q]["squeeze"] - ) - if map_dict[name_q]["transpose"] is not None: - data_tf = np.transpose( - data_tf, map_dict[name_q]["transpose"] - ) - data_tf = ( - torch.from_numpy(data_tf).type(torch.float32).to("cpu") - ) - assert ( - var_dict_torch[name].size() == data_tf.size() - ), "{}, {}, {} != {}".format( - name, name_tf, var_dict_torch[name].size(), data_tf.size() - ) - var_dict_torch_update[name] = data_tf - logging.info( - "torch tensor: {}, {}, loading from tf tensor: {}, {}".format( - name, data_tf.size(), name_v, var_dict_tf[name_tf].shape - ) - ) - - return var_dict_torch_update diff --git a/funasr_detach/models/scama/encoder.py b/funasr_detach/models/scama/encoder.py deleted file mode 100644 index eade514212add6e80fe46dfc335fe2636d668a3f..0000000000000000000000000000000000000000 --- a/funasr_detach/models/scama/encoder.py +++ /dev/null @@ -1,740 +0,0 @@ -#!/usr/bin/env python3 -# -*- encoding: utf-8 -*- -# Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved. -# MIT License (https://opensource.org/licenses/MIT) - -from typing import List -from typing import Optional -from typing import Sequence -from typing import Tuple -from typing import Union -import logging -import torch -import torch.nn as nn -import torch.nn.functional as F -from funasr_detach.models.scama.chunk_utilis import overlap_chunk -import numpy as np -from funasr_detach.train_utils.device_funcs import to_device -from funasr_detach.models.transformer.utils.nets_utils import make_pad_mask -from funasr_detach.models.sanm.attention import ( - MultiHeadedAttention, - MultiHeadedAttentionSANM, -) -from funasr_detach.models.transformer.embedding import ( - SinusoidalPositionEncoder, - StreamSinusoidalPositionEncoder, -) -from funasr_detach.models.transformer.layer_norm import LayerNorm -from funasr_detach.models.transformer.utils.multi_layer_conv import Conv1dLinear -from funasr_detach.models.transformer.utils.multi_layer_conv import MultiLayeredConv1d -from funasr_detach.models.transformer.positionwise_feed_forward import ( - PositionwiseFeedForward, # noqa: H301 -) -from funasr_detach.models.transformer.utils.repeat import repeat -from funasr_detach.models.transformer.utils.subsampling import Conv2dSubsampling -from funasr_detach.models.transformer.utils.subsampling import Conv2dSubsampling2 -from funasr_detach.models.transformer.utils.subsampling import Conv2dSubsampling6 -from funasr_detach.models.transformer.utils.subsampling import Conv2dSubsampling8 -from funasr_detach.models.transformer.utils.subsampling import TooShortUttError -from funasr_detach.models.transformer.utils.subsampling import check_short_utt -from funasr_detach.models.transformer.utils.mask import subsequent_mask, vad_mask - -from funasr_detach.models.ctc.ctc import CTC - -from funasr_detach.register import tables - - -class EncoderLayerSANM(nn.Module): - def __init__( - self, - in_size, - size, - self_attn, - feed_forward, - dropout_rate, - normalize_before=True, - concat_after=False, - stochastic_depth_rate=0.0, - ): - """Construct an EncoderLayer object.""" - super(EncoderLayerSANM, self).__init__() - self.self_attn = self_attn - self.feed_forward = feed_forward - self.norm1 = LayerNorm(in_size) - self.norm2 = LayerNorm(size) - self.dropout = nn.Dropout(dropout_rate) - self.in_size = in_size - self.size = size - self.normalize_before = normalize_before - self.concat_after = concat_after - if self.concat_after: - self.concat_linear = nn.Linear(size + size, size) - self.stochastic_depth_rate = stochastic_depth_rate - self.dropout_rate = dropout_rate - - def forward( - self, x, mask, cache=None, mask_shfit_chunk=None, mask_att_chunk_encoder=None - ): - """Compute encoded features. - - Args: - x_input (torch.Tensor): Input tensor (#batch, time, size). - mask (torch.Tensor): Mask tensor for the input (#batch, time). - cache (torch.Tensor): Cache tensor of the input (#batch, time - 1, size). - - Returns: - torch.Tensor: Output tensor (#batch, time, size). - torch.Tensor: Mask tensor (#batch, time). - - """ - skip_layer = False - # with stochastic depth, residual connection `x + f(x)` becomes - # `x <- x + 1 / (1 - p) * f(x)` at training time. - stoch_layer_coeff = 1.0 - if self.training and self.stochastic_depth_rate > 0: - skip_layer = torch.rand(1).item() < self.stochastic_depth_rate - stoch_layer_coeff = 1.0 / (1 - self.stochastic_depth_rate) - - if skip_layer: - if cache is not None: - x = torch.cat([cache, x], dim=1) - return x, mask - - residual = x - if self.normalize_before: - x = self.norm1(x) - - if self.concat_after: - x_concat = torch.cat( - ( - x, - self.self_attn( - x, - mask, - mask_shfit_chunk=mask_shfit_chunk, - mask_att_chunk_encoder=mask_att_chunk_encoder, - ), - ), - dim=-1, - ) - if self.in_size == self.size: - x = residual + stoch_layer_coeff * self.concat_linear(x_concat) - else: - x = stoch_layer_coeff * self.concat_linear(x_concat) - else: - if self.in_size == self.size: - x = residual + stoch_layer_coeff * self.dropout( - self.self_attn( - x, - mask, - mask_shfit_chunk=mask_shfit_chunk, - mask_att_chunk_encoder=mask_att_chunk_encoder, - ) - ) - else: - x = stoch_layer_coeff * self.dropout( - self.self_attn( - x, - mask, - mask_shfit_chunk=mask_shfit_chunk, - mask_att_chunk_encoder=mask_att_chunk_encoder, - ) - ) - if not self.normalize_before: - x = self.norm1(x) - - residual = x - if self.normalize_before: - x = self.norm2(x) - x = residual + stoch_layer_coeff * self.dropout(self.feed_forward(x)) - if not self.normalize_before: - x = self.norm2(x) - - return x, mask, cache, mask_shfit_chunk, mask_att_chunk_encoder - - def forward_chunk(self, x, cache=None, chunk_size=None, look_back=0): - """Compute encoded features. - - Args: - x_input (torch.Tensor): Input tensor (#batch, time, size). - mask (torch.Tensor): Mask tensor for the input (#batch, time). - cache (torch.Tensor): Cache tensor of the input (#batch, time - 1, size). - - Returns: - torch.Tensor: Output tensor (#batch, time, size). - torch.Tensor: Mask tensor (#batch, time). - - """ - - residual = x - if self.normalize_before: - x = self.norm1(x) - - if self.in_size == self.size: - attn, cache = self.self_attn.forward_chunk(x, cache, chunk_size, look_back) - x = residual + attn - else: - x, cache = self.self_attn.forward_chunk(x, cache, chunk_size, look_back) - - if not self.normalize_before: - x = self.norm1(x) - - residual = x - if self.normalize_before: - x = self.norm2(x) - x = residual + self.feed_forward(x) - if not self.normalize_before: - x = self.norm2(x) - - return x, cache - - -@tables.register("encoder_classes", "SANMEncoderChunkOpt") -class SANMEncoderChunkOpt(nn.Module): - """ - Author: Shiliang Zhang, Zhifu Gao, Haoneng Luo, Ming Lei, Jie Gao, Zhijie Yan, Lei Xie - SCAMA: Streaming chunk-aware multihead attention for online end-to-end speech recognition - https://arxiv.org/abs/2006.01712 - """ - - def __init__( - self, - input_size: int, - output_size: int = 256, - attention_heads: int = 4, - linear_units: int = 2048, - num_blocks: int = 6, - dropout_rate: float = 0.1, - positional_dropout_rate: float = 0.1, - attention_dropout_rate: float = 0.0, - input_layer: Optional[str] = "conv2d", - pos_enc_class=SinusoidalPositionEncoder, - normalize_before: bool = True, - concat_after: bool = False, - positionwise_layer_type: str = "linear", - positionwise_conv_kernel_size: int = 1, - padding_idx: int = -1, - interctc_layer_idx: List[int] = [], - interctc_use_conditioning: bool = False, - kernel_size: int = 11, - sanm_shfit: int = 0, - selfattention_layer_type: str = "sanm", - chunk_size: Union[int, Sequence[int]] = (16,), - stride: Union[int, Sequence[int]] = (10,), - pad_left: Union[int, Sequence[int]] = (0,), - encoder_att_look_back_factor: Union[int, Sequence[int]] = (1,), - decoder_att_look_back_factor: Union[int, Sequence[int]] = (1,), - tf2torch_tensor_name_prefix_torch: str = "encoder", - tf2torch_tensor_name_prefix_tf: str = "seq2seq/encoder", - ): - super().__init__() - self._output_size = output_size - - if input_layer == "linear": - self.embed = torch.nn.Sequential( - torch.nn.Linear(input_size, output_size), - torch.nn.LayerNorm(output_size), - torch.nn.Dropout(dropout_rate), - torch.nn.ReLU(), - pos_enc_class(output_size, positional_dropout_rate), - ) - elif input_layer == "conv2d": - self.embed = Conv2dSubsampling(input_size, output_size, dropout_rate) - elif input_layer == "conv2d2": - self.embed = Conv2dSubsampling2(input_size, output_size, dropout_rate) - elif input_layer == "conv2d6": - self.embed = Conv2dSubsampling6(input_size, output_size, dropout_rate) - elif input_layer == "conv2d8": - self.embed = Conv2dSubsampling8(input_size, output_size, dropout_rate) - elif input_layer == "embed": - self.embed = torch.nn.Sequential( - torch.nn.Embedding(input_size, output_size, padding_idx=padding_idx), - pos_enc_class(output_size, positional_dropout_rate), - ) - elif input_layer is None: - if input_size == output_size: - self.embed = None - else: - self.embed = torch.nn.Linear(input_size, output_size) - elif input_layer == "pe": - self.embed = SinusoidalPositionEncoder() - elif input_layer == "pe_online": - self.embed = StreamSinusoidalPositionEncoder() - else: - raise ValueError("unknown input_layer: " + input_layer) - self.normalize_before = normalize_before - if positionwise_layer_type == "linear": - positionwise_layer = PositionwiseFeedForward - positionwise_layer_args = ( - output_size, - linear_units, - dropout_rate, - ) - elif positionwise_layer_type == "conv1d": - positionwise_layer = MultiLayeredConv1d - positionwise_layer_args = ( - output_size, - linear_units, - positionwise_conv_kernel_size, - dropout_rate, - ) - elif positionwise_layer_type == "conv1d-linear": - positionwise_layer = Conv1dLinear - positionwise_layer_args = ( - output_size, - linear_units, - positionwise_conv_kernel_size, - dropout_rate, - ) - else: - raise NotImplementedError("Support only linear or conv1d.") - - if selfattention_layer_type == "selfattn": - encoder_selfattn_layer = MultiHeadedAttention - encoder_selfattn_layer_args = ( - attention_heads, - output_size, - attention_dropout_rate, - ) - elif selfattention_layer_type == "sanm": - encoder_selfattn_layer = MultiHeadedAttentionSANM - encoder_selfattn_layer_args0 = ( - attention_heads, - input_size, - output_size, - attention_dropout_rate, - kernel_size, - sanm_shfit, - ) - - encoder_selfattn_layer_args = ( - attention_heads, - output_size, - output_size, - attention_dropout_rate, - kernel_size, - sanm_shfit, - ) - self.encoders0 = repeat( - 1, - lambda lnum: EncoderLayerSANM( - input_size, - output_size, - encoder_selfattn_layer(*encoder_selfattn_layer_args0), - positionwise_layer(*positionwise_layer_args), - dropout_rate, - normalize_before, - concat_after, - ), - ) - - self.encoders = repeat( - num_blocks - 1, - lambda lnum: EncoderLayerSANM( - output_size, - output_size, - encoder_selfattn_layer(*encoder_selfattn_layer_args), - positionwise_layer(*positionwise_layer_args), - dropout_rate, - normalize_before, - concat_after, - ), - ) - if self.normalize_before: - self.after_norm = LayerNorm(output_size) - - self.interctc_layer_idx = interctc_layer_idx - if len(interctc_layer_idx) > 0: - assert 0 < min(interctc_layer_idx) and max(interctc_layer_idx) < num_blocks - self.interctc_use_conditioning = interctc_use_conditioning - self.conditioning_layer = None - shfit_fsmn = (kernel_size - 1) // 2 - self.overlap_chunk_cls = overlap_chunk( - chunk_size=chunk_size, - stride=stride, - pad_left=pad_left, - shfit_fsmn=shfit_fsmn, - encoder_att_look_back_factor=encoder_att_look_back_factor, - decoder_att_look_back_factor=decoder_att_look_back_factor, - ) - self.tf2torch_tensor_name_prefix_torch = tf2torch_tensor_name_prefix_torch - self.tf2torch_tensor_name_prefix_tf = tf2torch_tensor_name_prefix_tf - - def output_size(self) -> int: - return self._output_size - - def forward( - self, - xs_pad: torch.Tensor, - ilens: torch.Tensor, - prev_states: torch.Tensor = None, - ctc: CTC = None, - ind: int = 0, - ) -> Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]: - """Embed positions in tensor. - - Args: - xs_pad: input tensor (B, L, D) - ilens: input length (B) - prev_states: Not to be used now. - Returns: - position embedded tensor and mask - """ - masks = (~make_pad_mask(ilens)[:, None, :]).to(xs_pad.device) - xs_pad *= self.output_size() ** 0.5 - if self.embed is None: - xs_pad = xs_pad - elif ( - isinstance(self.embed, Conv2dSubsampling) - or isinstance(self.embed, Conv2dSubsampling2) - or isinstance(self.embed, Conv2dSubsampling6) - or isinstance(self.embed, Conv2dSubsampling8) - ): - short_status, limit_size = check_short_utt(self.embed, xs_pad.size(1)) - if short_status: - raise TooShortUttError( - f"has {xs_pad.size(1)} frames and is too short for subsampling " - + f"(it needs more than {limit_size} frames), return empty results", - xs_pad.size(1), - limit_size, - ) - xs_pad, masks = self.embed(xs_pad, masks) - else: - xs_pad = self.embed(xs_pad) - - mask_shfit_chunk, mask_att_chunk_encoder = None, None - if self.overlap_chunk_cls is not None: - ilens = masks.squeeze(1).sum(1) - chunk_outs = self.overlap_chunk_cls.gen_chunk_mask(ilens, ind) - xs_pad, ilens = self.overlap_chunk_cls.split_chunk( - xs_pad, ilens, chunk_outs=chunk_outs - ) - masks = (~make_pad_mask(ilens)[:, None, :]).to(xs_pad.device) - mask_shfit_chunk = self.overlap_chunk_cls.get_mask_shfit_chunk( - chunk_outs, xs_pad.device, xs_pad.size(0), dtype=xs_pad.dtype - ) - mask_att_chunk_encoder = self.overlap_chunk_cls.get_mask_att_chunk_encoder( - chunk_outs, xs_pad.device, xs_pad.size(0), dtype=xs_pad.dtype - ) - - encoder_outs = self.encoders0( - xs_pad, masks, None, mask_shfit_chunk, mask_att_chunk_encoder - ) - xs_pad, masks = encoder_outs[0], encoder_outs[1] - intermediate_outs = [] - if len(self.interctc_layer_idx) == 0: - encoder_outs = self.encoders( - xs_pad, masks, None, mask_shfit_chunk, mask_att_chunk_encoder - ) - xs_pad, masks = encoder_outs[0], encoder_outs[1] - else: - for layer_idx, encoder_layer in enumerate(self.encoders): - encoder_outs = encoder_layer( - xs_pad, masks, None, mask_shfit_chunk, mask_att_chunk_encoder - ) - xs_pad, masks = encoder_outs[0], encoder_outs[1] - if layer_idx + 1 in self.interctc_layer_idx: - encoder_out = xs_pad - - # intermediate outputs are also normalized - if self.normalize_before: - encoder_out = self.after_norm(encoder_out) - - intermediate_outs.append((layer_idx + 1, encoder_out)) - - if self.interctc_use_conditioning: - ctc_out = ctc.softmax(encoder_out) - xs_pad = xs_pad + self.conditioning_layer(ctc_out) - - if self.normalize_before: - xs_pad = self.after_norm(xs_pad) - - olens = masks.squeeze(1).sum(1) - if len(intermediate_outs) > 0: - return (xs_pad, intermediate_outs), olens, None - return xs_pad, olens, None - - def _add_overlap_chunk(self, feats: np.ndarray, cache: dict = {}): - if len(cache) == 0: - return feats - cache["feats"] = to_device(cache["feats"], device=feats.device) - overlap_feats = torch.cat((cache["feats"], feats), dim=1) - cache["feats"] = overlap_feats[ - :, -(cache["chunk_size"][0] + cache["chunk_size"][2]) :, : - ] - return overlap_feats - - def forward_chunk( - self, - xs_pad: torch.Tensor, - ilens: torch.Tensor, - cache: dict = None, - **kwargs, - ): - is_final = kwargs.get("is_final", False) - xs_pad *= self.output_size() ** 0.5 - if self.embed is None: - xs_pad = xs_pad - else: - xs_pad = self.embed(xs_pad, cache) - if cache["tail_chunk"]: - xs_pad = to_device(cache["feats"], device=xs_pad.device) - else: - xs_pad = self._add_overlap_chunk(xs_pad, cache) - if cache["opt"] is None: - cache_layer_num = len(self.encoders0) + len(self.encoders) - new_cache = [None] * cache_layer_num - else: - new_cache = cache["opt"] - - for layer_idx, encoder_layer in enumerate(self.encoders0): - encoder_outs = encoder_layer.forward_chunk( - xs_pad, - new_cache[layer_idx], - cache["chunk_size"], - cache["encoder_chunk_look_back"], - ) - xs_pad, new_cache[0] = encoder_outs[0], encoder_outs[1] - - for layer_idx, encoder_layer in enumerate(self.encoders): - encoder_outs = encoder_layer.forward_chunk( - xs_pad, - new_cache[layer_idx + len(self.encoders0)], - cache["chunk_size"], - cache["encoder_chunk_look_back"], - ) - xs_pad, new_cache[layer_idx + len(self.encoders0)] = ( - encoder_outs[0], - encoder_outs[1], - ) - - if self.normalize_before: - xs_pad = self.after_norm(xs_pad) - if ( - cache["encoder_chunk_look_back"] > 0 - or cache["encoder_chunk_look_back"] == -1 - ): - cache["opt"] = new_cache - - return xs_pad, ilens, None - - def gen_tf2torch_map_dict(self): - tensor_name_prefix_torch = self.tf2torch_tensor_name_prefix_torch - tensor_name_prefix_tf = self.tf2torch_tensor_name_prefix_tf - map_dict_local = { - ## encoder - # cicd - "{}.encoders.layeridx.norm1.weight".format(tensor_name_prefix_torch): { - "name": "{}/layer_layeridx/multi_head/LayerNorm/gamma".format( - tensor_name_prefix_tf - ), - "squeeze": None, - "transpose": None, - }, # (256,),(256,) - "{}.encoders.layeridx.norm1.bias".format(tensor_name_prefix_torch): { - "name": "{}/layer_layeridx/multi_head/LayerNorm/beta".format( - tensor_name_prefix_tf - ), - "squeeze": None, - "transpose": None, - }, # (256,),(256,) - "{}.encoders.layeridx.self_attn.linear_q_k_v.weight".format( - tensor_name_prefix_torch - ): { - "name": "{}/layer_layeridx/multi_head/conv1d/kernel".format( - tensor_name_prefix_tf - ), - "squeeze": 0, - "transpose": (1, 0), - }, # (768,256),(1,256,768) - "{}.encoders.layeridx.self_attn.linear_q_k_v.bias".format( - tensor_name_prefix_torch - ): { - "name": "{}/layer_layeridx/multi_head/conv1d/bias".format( - tensor_name_prefix_tf - ), - "squeeze": None, - "transpose": None, - }, # (768,),(768,) - "{}.encoders.layeridx.self_attn.fsmn_block.weight".format( - tensor_name_prefix_torch - ): { - "name": "{}/layer_layeridx/multi_head/depth_conv_w".format( - tensor_name_prefix_tf - ), - "squeeze": 0, - "transpose": (1, 2, 0), - }, # (256,1,31),(1,31,256,1) - "{}.encoders.layeridx.self_attn.linear_out.weight".format( - tensor_name_prefix_torch - ): { - "name": "{}/layer_layeridx/multi_head/conv1d_1/kernel".format( - tensor_name_prefix_tf - ), - "squeeze": 0, - "transpose": (1, 0), - }, # (256,256),(1,256,256) - "{}.encoders.layeridx.self_attn.linear_out.bias".format( - tensor_name_prefix_torch - ): { - "name": "{}/layer_layeridx/multi_head/conv1d_1/bias".format( - tensor_name_prefix_tf - ), - "squeeze": None, - "transpose": None, - }, # (256,),(256,) - # ffn - "{}.encoders.layeridx.norm2.weight".format(tensor_name_prefix_torch): { - "name": "{}/layer_layeridx/ffn/LayerNorm/gamma".format( - tensor_name_prefix_tf - ), - "squeeze": None, - "transpose": None, - }, # (256,),(256,) - "{}.encoders.layeridx.norm2.bias".format(tensor_name_prefix_torch): { - "name": "{}/layer_layeridx/ffn/LayerNorm/beta".format( - tensor_name_prefix_tf - ), - "squeeze": None, - "transpose": None, - }, # (256,),(256,) - "{}.encoders.layeridx.feed_forward.w_1.weight".format( - tensor_name_prefix_torch - ): { - "name": "{}/layer_layeridx/ffn/conv1d/kernel".format( - tensor_name_prefix_tf - ), - "squeeze": 0, - "transpose": (1, 0), - }, # (1024,256),(1,256,1024) - "{}.encoders.layeridx.feed_forward.w_1.bias".format( - tensor_name_prefix_torch - ): { - "name": "{}/layer_layeridx/ffn/conv1d/bias".format( - tensor_name_prefix_tf - ), - "squeeze": None, - "transpose": None, - }, # (1024,),(1024,) - "{}.encoders.layeridx.feed_forward.w_2.weight".format( - tensor_name_prefix_torch - ): { - "name": "{}/layer_layeridx/ffn/conv1d_1/kernel".format( - tensor_name_prefix_tf - ), - "squeeze": 0, - "transpose": (1, 0), - }, # (256,1024),(1,1024,256) - "{}.encoders.layeridx.feed_forward.w_2.bias".format( - tensor_name_prefix_torch - ): { - "name": "{}/layer_layeridx/ffn/conv1d_1/bias".format( - tensor_name_prefix_tf - ), - "squeeze": None, - "transpose": None, - }, # (256,),(256,) - # out norm - "{}.after_norm.weight".format(tensor_name_prefix_torch): { - "name": "{}/LayerNorm/gamma".format(tensor_name_prefix_tf), - "squeeze": None, - "transpose": None, - }, # (256,),(256,) - "{}.after_norm.bias".format(tensor_name_prefix_torch): { - "name": "{}/LayerNorm/beta".format(tensor_name_prefix_tf), - "squeeze": None, - "transpose": None, - }, # (256,),(256,) - } - - return map_dict_local - - def convert_tf2torch( - self, - var_dict_tf, - var_dict_torch, - ): - - map_dict = self.gen_tf2torch_map_dict() - - var_dict_torch_update = dict() - for name in sorted(var_dict_torch.keys(), reverse=False): - names = name.split(".") - if names[0] == self.tf2torch_tensor_name_prefix_torch: - if names[1] == "encoders0": - layeridx = int(names[2]) - name_q = name.replace(".{}.".format(layeridx), ".layeridx.") - - name_q = name_q.replace("encoders0", "encoders") - layeridx_bias = 0 - layeridx += layeridx_bias - if name_q in map_dict.keys(): - name_v = map_dict[name_q]["name"] - name_tf = name_v.replace("layeridx", "{}".format(layeridx)) - data_tf = var_dict_tf[name_tf] - if map_dict[name_q]["squeeze"] is not None: - data_tf = np.squeeze( - data_tf, axis=map_dict[name_q]["squeeze"] - ) - if map_dict[name_q]["transpose"] is not None: - data_tf = np.transpose( - data_tf, map_dict[name_q]["transpose"] - ) - data_tf = ( - torch.from_numpy(data_tf).type(torch.float32).to("cpu") - ) - assert ( - var_dict_torch[name].size() == data_tf.size() - ), "{}, {}, {} != {}".format( - name, name_tf, var_dict_torch[name].size(), data_tf.size() - ) - var_dict_torch_update[name] = data_tf - logging.info( - "torch tensor: {}, {}, loading from tf tensor: {}, {}".format( - name, data_tf.size(), name_v, var_dict_tf[name_tf].shape - ) - ) - elif names[1] == "encoders": - layeridx = int(names[2]) - name_q = name.replace(".{}.".format(layeridx), ".layeridx.") - layeridx_bias = 1 - layeridx += layeridx_bias - if name_q in map_dict.keys(): - name_v = map_dict[name_q]["name"] - name_tf = name_v.replace("layeridx", "{}".format(layeridx)) - data_tf = var_dict_tf[name_tf] - if map_dict[name_q]["squeeze"] is not None: - data_tf = np.squeeze( - data_tf, axis=map_dict[name_q]["squeeze"] - ) - if map_dict[name_q]["transpose"] is not None: - data_tf = np.transpose( - data_tf, map_dict[name_q]["transpose"] - ) - data_tf = ( - torch.from_numpy(data_tf).type(torch.float32).to("cpu") - ) - assert ( - var_dict_torch[name].size() == data_tf.size() - ), "{}, {}, {} != {}".format( - name, name_tf, var_dict_torch[name].size(), data_tf.size() - ) - var_dict_torch_update[name] = data_tf - logging.info( - "torch tensor: {}, {}, loading from tf tensor: {}, {}".format( - name, data_tf.size(), name_v, var_dict_tf[name_tf].shape - ) - ) - - elif names[1] == "after_norm": - name_tf = map_dict[name]["name"] - data_tf = var_dict_tf[name_tf] - data_tf = torch.from_numpy(data_tf).type(torch.float32).to("cpu") - var_dict_torch_update[name] = data_tf - logging.info( - "torch tensor: {}, {}, loading from tf tensor: {}, {}".format( - name, data_tf.size(), name_tf, var_dict_tf[name_tf].shape - ) - ) - - return var_dict_torch_update diff --git a/funasr_detach/models/scama/model.py b/funasr_detach/models/scama/model.py deleted file mode 100644 index 3123b997f917512f0055764dc825e1c09e44f56f..0000000000000000000000000000000000000000 --- a/funasr_detach/models/scama/model.py +++ /dev/null @@ -1,766 +0,0 @@ -#!/usr/bin/env python3 -# -*- encoding: utf-8 -*- -# Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved. -# MIT License (https://opensource.org/licenses/MIT) - -import time -import torch -import torch.nn as nn -import torch.functional as F -import logging -from typing import Dict, Tuple -from contextlib import contextmanager -from distutils.version import LooseVersion - -from funasr_detach.register import tables -from funasr_detach.models.ctc.ctc import CTC -from funasr_detach.utils import postprocess_utils -from funasr_detach.metrics.compute_acc import th_accuracy -from funasr_detach.utils.datadir_writer import DatadirWriter -from funasr_detach.models.paraformer.model import Paraformer -from funasr_detach.models.paraformer.search import Hypothesis -from funasr_detach.models.paraformer.cif_predictor import mae_loss -from funasr_detach.train_utils.device_funcs import force_gatherable -from funasr_detach.losses.label_smoothing_loss import LabelSmoothingLoss -from funasr_detach.models.transformer.utils.add_sos_eos import add_sos_eos -from funasr_detach.models.transformer.utils.nets_utils import make_pad_mask, pad_list -from funasr_detach.utils.load_utils import load_audio_text_image_video, extract_fbank -from funasr_detach.models.scama.utils import sequence_mask - -if LooseVersion(torch.__version__) >= LooseVersion("1.6.0"): - from torch.cuda.amp import autocast -else: - # Nothing to do if torch<1.6.0 - @contextmanager - def autocast(enabled=True): - yield - - -@tables.register("model_classes", "SCAMA") -class SCAMA(nn.Module): - """ - Author: Shiliang Zhang, Zhifu Gao, Haoneng Luo, Ming Lei, Jie Gao, Zhijie Yan, Lei Xie - SCAMA: Streaming chunk-aware multihead attention for online end-to-end speech recognition - https://arxiv.org/abs/2006.01712 - """ - - def __init__( - self, - specaug: str = None, - specaug_conf: dict = None, - normalize: str = None, - normalize_conf: dict = None, - encoder: str = None, - encoder_conf: dict = None, - decoder: str = None, - decoder_conf: dict = None, - ctc: str = None, - ctc_conf: dict = None, - ctc_weight: float = 0.5, - predictor: str = None, - predictor_conf: dict = None, - predictor_bias: int = 0, - predictor_weight: float = 0.0, - input_size: int = 80, - vocab_size: int = -1, - ignore_id: int = -1, - blank_id: int = 0, - sos: int = 1, - eos: int = 2, - lsm_weight: float = 0.0, - length_normalized_loss: bool = False, - share_embedding: bool = False, - **kwargs, - ): - - super().__init__() - - if specaug is not None: - specaug_class = tables.specaug_classes.get(specaug) - specaug = specaug_class(**specaug_conf) - - if normalize is not None: - normalize_class = tables.normalize_classes.get(normalize) - normalize = normalize_class(**normalize_conf) - - encoder_class = tables.encoder_classes.get(encoder) - encoder = encoder_class(input_size=input_size, **encoder_conf) - encoder_output_size = encoder.output_size() - - decoder_class = tables.decoder_classes.get(decoder) - decoder = decoder_class( - vocab_size=vocab_size, - encoder_output_size=encoder_output_size, - **decoder_conf, - ) - if ctc_weight > 0.0: - - if ctc_conf is None: - ctc_conf = {} - - ctc = CTC( - odim=vocab_size, encoder_output_size=encoder_output_size, **ctc_conf - ) - - predictor_class = tables.predictor_classes.get(predictor) - predictor = predictor_class(**predictor_conf) - - # note that eos is the same as sos (equivalent ID) - self.blank_id = blank_id - self.sos = sos if sos is not None else vocab_size - 1 - self.eos = eos if eos is not None else vocab_size - 1 - self.vocab_size = vocab_size - self.ignore_id = ignore_id - self.ctc_weight = ctc_weight - - self.specaug = specaug - self.normalize = normalize - - self.encoder = encoder - - if ctc_weight == 1.0: - self.decoder = None - else: - self.decoder = decoder - - self.criterion_att = LabelSmoothingLoss( - size=vocab_size, - padding_idx=ignore_id, - smoothing=lsm_weight, - normalize_length=length_normalized_loss, - ) - - if ctc_weight == 0.0: - self.ctc = None - else: - self.ctc = ctc - - self.predictor = predictor - self.predictor_weight = predictor_weight - self.predictor_bias = predictor_bias - - self.criterion_pre = mae_loss(normalize_length=length_normalized_loss) - - self.share_embedding = share_embedding - if self.share_embedding: - self.decoder.embed = None - - self.length_normalized_loss = length_normalized_loss - self.beam_search = None - self.error_calculator = None - - if self.encoder.overlap_chunk_cls is not None: - from funasr_detach.models.scama.chunk_utilis import ( - build_scama_mask_for_cross_attention_decoder, - ) - - self.build_scama_mask_for_cross_attention_decoder_fn = ( - build_scama_mask_for_cross_attention_decoder - ) - self.decoder_attention_chunk_type = kwargs.get( - "decoder_attention_chunk_type", "chunk" - ) - - def forward( - self, - speech: torch.Tensor, - speech_lengths: torch.Tensor, - text: torch.Tensor, - text_lengths: torch.Tensor, - **kwargs, - ) -> Tuple[torch.Tensor, Dict[str, torch.Tensor], torch.Tensor]: - """Encoder + Decoder + Calc loss - Args: - speech: (Batch, Length, ...) - speech_lengths: (Batch, ) - text: (Batch, Length) - text_lengths: (Batch,) - """ - - decoding_ind = kwargs.get("decoding_ind") - if len(text_lengths.size()) > 1: - text_lengths = text_lengths[:, 0] - if len(speech_lengths.size()) > 1: - speech_lengths = speech_lengths[:, 0] - - batch_size = speech.shape[0] - - # Encoder - ind = self.encoder.overlap_chunk_cls.random_choice(self.training, decoding_ind) - encoder_out, encoder_out_lens = self.encode(speech, speech_lengths, ind=ind) - - loss_ctc, cer_ctc = None, None - loss_pre = None - stats = dict() - - # decoder: CTC branch - - if self.ctc_weight > 0.0: - - encoder_out_ctc, encoder_out_lens_ctc = ( - self.encoder.overlap_chunk_cls.remove_chunk( - encoder_out, encoder_out_lens, chunk_outs=None - ) - ) - - loss_ctc, cer_ctc = self._calc_ctc_loss( - encoder_out_ctc, encoder_out_lens_ctc, text, text_lengths - ) - # Collect CTC branch stats - stats["loss_ctc"] = loss_ctc.detach() if loss_ctc is not None else None - stats["cer_ctc"] = cer_ctc - - # decoder: Attention decoder branch - loss_att, acc_att, cer_att, wer_att, loss_pre = self._calc_att_predictor_loss( - encoder_out, encoder_out_lens, text, text_lengths - ) - - # 3. CTC-Att loss definition - if self.ctc_weight == 0.0: - loss = loss_att + loss_pre * self.predictor_weight - else: - loss = ( - self.ctc_weight * loss_ctc - + (1 - self.ctc_weight) * loss_att - + loss_pre * self.predictor_weight - ) - - # Collect Attn branch stats - stats["loss_att"] = loss_att.detach() if loss_att is not None else None - stats["acc"] = acc_att - stats["cer"] = cer_att - stats["wer"] = wer_att - stats["loss_pre"] = loss_pre.detach().cpu() if loss_pre is not None else None - - stats["loss"] = torch.clone(loss.detach()) - - # force_gatherable: to-device and to-tensor if scalar for DataParallel - if self.length_normalized_loss: - batch_size = (text_lengths + self.predictor_bias).sum() - loss, stats, weight = force_gatherable((loss, stats, batch_size), loss.device) - return loss, stats, weight - - def encode( - self, - speech: torch.Tensor, - speech_lengths: torch.Tensor, - **kwargs, - ) -> Tuple[torch.Tensor, torch.Tensor]: - """Encoder. Note that this method is used by asr_inference.py - Args: - speech: (Batch, Length, ...) - speech_lengths: (Batch, ) - ind: int - """ - with autocast(False): - - # Data augmentation - if self.specaug is not None and self.training: - speech, speech_lengths = self.specaug(speech, speech_lengths) - - # Normalization for feature: e.g. Global-CMVN, Utterance-CMVN - if self.normalize is not None: - speech, speech_lengths = self.normalize(speech, speech_lengths) - - # Forward encoder - encoder_out, encoder_out_lens, _ = self.encoder(speech, speech_lengths) - if isinstance(encoder_out, tuple): - encoder_out = encoder_out[0] - - return encoder_out, encoder_out_lens - - def encode_chunk( - self, - speech: torch.Tensor, - speech_lengths: torch.Tensor, - cache: dict = None, - **kwargs, - ) -> Tuple[torch.Tensor, torch.Tensor]: - """Frontend + Encoder. Note that this method is used by asr_inference.py - Args: - speech: (Batch, Length, ...) - speech_lengths: (Batch, ) - ind: int - """ - with autocast(False): - - # Data augmentation - if self.specaug is not None and self.training: - speech, speech_lengths = self.specaug(speech, speech_lengths) - - # Normalization for feature: e.g. Global-CMVN, Utterance-CMVN - if self.normalize is not None: - speech, speech_lengths = self.normalize(speech, speech_lengths) - - # Forward encoder - encoder_out, encoder_out_lens, _ = self.encoder.forward_chunk( - speech, speech_lengths, cache=cache["encoder"] - ) - if isinstance(encoder_out, tuple): - encoder_out = encoder_out[0] - - return encoder_out, torch.tensor([encoder_out.size(1)]) - - def calc_predictor_chunk(self, encoder_out, encoder_out_lens, cache=None, **kwargs): - is_final = kwargs.get("is_final", False) - - return self.predictor.forward_chunk( - encoder_out, cache["encoder"], is_final=is_final - ) - - def _calc_att_predictor_loss( - self, - encoder_out: torch.Tensor, - encoder_out_lens: torch.Tensor, - ys_pad: torch.Tensor, - ys_pad_lens: torch.Tensor, - ): - ys_in_pad, ys_out_pad = add_sos_eos(ys_pad, self.sos, self.eos, self.ignore_id) - ys_in_lens = ys_pad_lens + 1 - - encoder_out_mask = sequence_mask( - encoder_out_lens, - maxlen=encoder_out.size(1), - dtype=encoder_out.dtype, - device=encoder_out.device, - )[:, None, :] - mask_chunk_predictor = None - if self.encoder.overlap_chunk_cls is not None: - mask_chunk_predictor = ( - self.encoder.overlap_chunk_cls.get_mask_chunk_predictor( - None, device=encoder_out.device, batch_size=encoder_out.size(0) - ) - ) - mask_shfit_chunk = self.encoder.overlap_chunk_cls.get_mask_shfit_chunk( - None, device=encoder_out.device, batch_size=encoder_out.size(0) - ) - encoder_out = encoder_out * mask_shfit_chunk - pre_acoustic_embeds, pre_token_length, pre_alphas, _ = self.predictor( - encoder_out, - ys_out_pad, - encoder_out_mask, - ignore_id=self.ignore_id, - mask_chunk_predictor=mask_chunk_predictor, - target_label_length=ys_in_lens, - ) - predictor_alignments, predictor_alignments_len = ( - self.predictor.gen_frame_alignments(pre_alphas, encoder_out_lens) - ) - - encoder_chunk_size = self.encoder.overlap_chunk_cls.chunk_size_pad_shift_cur - attention_chunk_center_bias = 0 - attention_chunk_size = encoder_chunk_size - decoder_att_look_back_factor = ( - self.encoder.overlap_chunk_cls.decoder_att_look_back_factor_cur - ) - mask_shift_att_chunk_decoder = ( - self.encoder.overlap_chunk_cls.get_mask_shift_att_chunk_decoder( - None, device=encoder_out.device, batch_size=encoder_out.size(0) - ) - ) - scama_mask = self.build_scama_mask_for_cross_attention_decoder_fn( - predictor_alignments=predictor_alignments, - encoder_sequence_length=encoder_out_lens, - chunk_size=1, - encoder_chunk_size=encoder_chunk_size, - attention_chunk_center_bias=attention_chunk_center_bias, - attention_chunk_size=attention_chunk_size, - attention_chunk_type=self.decoder_attention_chunk_type, - step=None, - predictor_mask_chunk_hopping=mask_chunk_predictor, - decoder_att_look_back_factor=decoder_att_look_back_factor, - mask_shift_att_chunk_decoder=mask_shift_att_chunk_decoder, - target_length=ys_in_lens, - is_training=self.training, - ) - - # try: - # 1. Forward decoder - decoder_out, _ = self.decoder( - encoder_out, - encoder_out_lens, - ys_in_pad, - ys_in_lens, - chunk_mask=scama_mask, - pre_acoustic_embeds=pre_acoustic_embeds, - ) - - # 2. Compute attention loss - loss_att = self.criterion_att(decoder_out, ys_out_pad) - acc_att = th_accuracy( - decoder_out.view(-1, self.vocab_size), - ys_out_pad, - ignore_label=self.ignore_id, - ) - # predictor loss - loss_pre = self.criterion_pre( - ys_in_lens.type_as(pre_token_length), pre_token_length - ) - # Compute cer/wer using attention-decoder - if self.training or self.error_calculator is None: - cer_att, wer_att = None, None - else: - ys_hat = decoder_out.argmax(dim=-1) - cer_att, wer_att = self.error_calculator(ys_hat.cpu(), ys_pad.cpu()) - - return loss_att, acc_att, cer_att, wer_att, loss_pre - - def calc_predictor_mask( - self, - encoder_out: torch.Tensor, - encoder_out_lens: torch.Tensor, - ys_pad: torch.Tensor = None, - ys_pad_lens: torch.Tensor = None, - ): - # ys_in_pad, ys_out_pad = add_sos_eos(ys_pad, self.sos, self.eos, self.ignore_id) - # ys_in_lens = ys_pad_lens + 1 - ys_out_pad, ys_in_lens = None, None - - encoder_out_mask = sequence_mask( - encoder_out_lens, - maxlen=encoder_out.size(1), - dtype=encoder_out.dtype, - device=encoder_out.device, - )[:, None, :] - mask_chunk_predictor = None - - mask_chunk_predictor = self.encoder.overlap_chunk_cls.get_mask_chunk_predictor( - None, device=encoder_out.device, batch_size=encoder_out.size(0) - ) - mask_shfit_chunk = self.encoder.overlap_chunk_cls.get_mask_shfit_chunk( - None, device=encoder_out.device, batch_size=encoder_out.size(0) - ) - encoder_out = encoder_out * mask_shfit_chunk - pre_acoustic_embeds, pre_token_length, pre_alphas, _ = self.predictor( - encoder_out, - ys_out_pad, - encoder_out_mask, - ignore_id=self.ignore_id, - mask_chunk_predictor=mask_chunk_predictor, - target_label_length=ys_in_lens, - ) - predictor_alignments, predictor_alignments_len = ( - self.predictor.gen_frame_alignments(pre_alphas, encoder_out_lens) - ) - - encoder_chunk_size = self.encoder.overlap_chunk_cls.chunk_size_pad_shift_cur - attention_chunk_center_bias = 0 - attention_chunk_size = encoder_chunk_size - decoder_att_look_back_factor = ( - self.encoder.overlap_chunk_cls.decoder_att_look_back_factor_cur - ) - mask_shift_att_chunk_decoder = ( - self.encoder.overlap_chunk_cls.get_mask_shift_att_chunk_decoder( - None, device=encoder_out.device, batch_size=encoder_out.size(0) - ) - ) - scama_mask = self.build_scama_mask_for_cross_attention_decoder_fn( - predictor_alignments=predictor_alignments, - encoder_sequence_length=encoder_out_lens, - chunk_size=1, - encoder_chunk_size=encoder_chunk_size, - attention_chunk_center_bias=attention_chunk_center_bias, - attention_chunk_size=attention_chunk_size, - attention_chunk_type=self.decoder_attention_chunk_type, - step=None, - predictor_mask_chunk_hopping=mask_chunk_predictor, - decoder_att_look_back_factor=decoder_att_look_back_factor, - mask_shift_att_chunk_decoder=mask_shift_att_chunk_decoder, - target_length=ys_in_lens, - is_training=self.training, - ) - - return ( - pre_acoustic_embeds, - pre_token_length, - predictor_alignments, - predictor_alignments_len, - scama_mask, - ) - - def init_beam_search( - self, - **kwargs, - ): - - from funasr_detach.models.scama.beam_search import BeamSearchScamaStreaming - - from funasr_detach.models.transformer.scorers.ctc import CTCPrefixScorer - from funasr_detach.models.transformer.scorers.length_bonus import LengthBonus - - # 1. Build ASR model - scorers = {} - - if self.ctc != None: - ctc = CTCPrefixScorer(ctc=self.ctc, eos=self.eos) - scorers.update(ctc=ctc) - token_list = kwargs.get("token_list") - scorers.update( - decoder=self.decoder, - length_bonus=LengthBonus(len(token_list)), - ) - - # 3. Build ngram model - # ngram is not supported now - ngram = None - scorers["ngram"] = ngram - - weights = dict( - decoder=1.0 - kwargs.get("decoding_ctc_weight", 0.0), - ctc=kwargs.get("decoding_ctc_weight", 0.0), - lm=kwargs.get("lm_weight", 0.0), - ngram=kwargs.get("ngram_weight", 0.0), - length_bonus=kwargs.get("penalty", 0.0), - ) - - beam_search = BeamSearchScamaStreaming( - beam_size=kwargs.get("beam_size", 2), - weights=weights, - scorers=scorers, - sos=self.sos, - eos=self.eos, - vocab_size=len(token_list), - token_list=token_list, - pre_beam_score_key=None if self.ctc_weight == 1.0 else "full", - ) - # beam_search.to(device=kwargs.get("device", "cpu"), dtype=getattr(torch, kwargs.get("dtype", "float32"))).eval() - # for scorer in scorers.values(): - # if isinstance(scorer, torch.nn.Module): - # scorer.to(device=kwargs.get("device", "cpu"), dtype=getattr(torch, kwargs.get("dtype", "float32"))).eval() - self.beam_search = beam_search - - def generate_chunk( - self, - speech, - speech_lengths=None, - key: list = None, - tokenizer=None, - frontend=None, - **kwargs, - ): - cache = kwargs.get("cache", {}) - speech = speech.to(device=kwargs["device"]) - speech_lengths = speech_lengths.to(device=kwargs["device"]) - - # Encoder - encoder_out, encoder_out_lens = self.encode_chunk( - speech, speech_lengths, cache=cache, is_final=kwargs.get("is_final", False) - ) - if isinstance(encoder_out, tuple): - encoder_out = encoder_out[0] - if "running_hyps" not in cache: - running_hyps = self.beam_search.init_hyp(encoder_out) - cache["running_hyps"] = running_hyps - - # predictor - predictor_outs = self.calc_predictor_chunk( - encoder_out, - encoder_out_lens, - cache=cache, - is_final=kwargs.get("is_final", False), - ) - pre_acoustic_embeds, pre_token_length, alphas, pre_peak_index = ( - predictor_outs[0], - predictor_outs[1], - predictor_outs[2], - predictor_outs[3], - ) - pre_token_length = pre_token_length.round().long() - - if torch.max(pre_token_length) < 1: - return [] - maxlen = minlen = pre_token_length - if kwargs.get("is_final", False): - maxlen += kwargs.get("token_num_relax", 5) - minlen = max(0, minlen - kwargs.get("token_num_relax", 5)) - # c. Passed the encoder result and the beam search - nbest_hyps = self.beam_search( - x=encoder_out[0], - scama_mask=None, - pre_acoustic_embeds=pre_acoustic_embeds, - maxlen=int(maxlen), - minlen=int(minlen), - cache=cache, - ) - - cache["running_hyps"] = nbest_hyps - nbest_hyps = nbest_hyps[: self.nbest] - - results = [] - for hyp in nbest_hyps: - # assert isinstance(hyp, (Hypothesis)), type(hyp) - - # remove sos/eos and get results - last_pos = -1 - if isinstance(hyp.yseq, list): - token_int = hyp.yseq[1:last_pos] - else: - token_int = hyp.yseq[1:last_pos].tolist() - - # remove blank symbol id, which is assumed to be 0 - token_int = list( - filter( - lambda x: x != self.eos - and x != self.sos - and x != self.blank_id, - token_int, - ) - ) - - # Change integer-ids to tokens - token = tokenizer.ids2tokens(token_int) - # text = tokenizer.tokens2text(token) - - result_i = token - - results.extend(result_i) - - return results - - def init_cache(self, cache: dict = {}, **kwargs): - device = kwargs.get("device", "cuda") - - chunk_size = kwargs.get("chunk_size", [0, 10, 5]) - encoder_chunk_look_back = kwargs.get("encoder_chunk_look_back", 0) - decoder_chunk_look_back = kwargs.get("decoder_chunk_look_back", 0) - batch_size = 1 - - enc_output_size = kwargs["encoder_conf"]["output_size"] - feats_dims = ( - kwargs["frontend_conf"]["n_mels"] * kwargs["frontend_conf"]["lfr_m"] - ) - - cache_encoder = { - "start_idx": 0, - "cif_hidden": torch.zeros((batch_size, 1, enc_output_size)).to( - device=device - ), - "cif_alphas": torch.zeros((batch_size, 1)).to(device=device), - "chunk_size": chunk_size, - "encoder_chunk_look_back": encoder_chunk_look_back, - "last_chunk": False, - "opt": None, - "feats": torch.zeros( - (batch_size, chunk_size[0] + chunk_size[2], feats_dims) - ).to(device=device), - "tail_chunk": False, - } - cache["encoder"] = cache_encoder - - cache_decoder = { - "decode_fsmn": None, - "decoder_chunk_look_back": decoder_chunk_look_back, - "opt": None, - "chunk_size": chunk_size, - } - cache["decoder"] = cache_decoder - cache["frontend"] = {} - - cache["prev_samples"] = torch.empty(0).to(device=device) - - return cache - - def inference( - self, - data_in, - data_lengths=None, - key: list = None, - tokenizer=None, - frontend=None, - cache: dict = {}, - **kwargs, - ): - - # init beamsearch - is_use_ctc = ( - kwargs.get("decoding_ctc_weight", 0.0) > 0.00001 and self.ctc != None - ) - is_use_lm = ( - kwargs.get("lm_weight", 0.0) > 0.00001 - and kwargs.get("lm_file", None) is not None - ) - - if self.beam_search is None: - - logging.info("enable beam_search") - self.init_beam_search(**kwargs) - self.nbest = kwargs.get("nbest", 1) - - if len(cache) == 0: - self.init_cache(cache, **kwargs) - - meta_data = {} - chunk_size = kwargs.get("chunk_size", [0, 10, 5]) - chunk_stride_samples = int(chunk_size[1] * 960) # 600ms - - time1 = time.perf_counter() - cfg = {"is_final": kwargs.get("is_final", False)} - audio_sample_list = load_audio_text_image_video( - data_in, - fs=frontend.fs, - audio_fs=kwargs.get("fs", 16000), - data_type=kwargs.get("data_type", "sound"), - tokenizer=tokenizer, - cache=cfg, - ) - _is_final = cfg["is_final"] # if data_in is a file or url, set is_final=True - - time2 = time.perf_counter() - meta_data["load_data"] = f"{time2 - time1:0.3f}" - assert len(audio_sample_list) == 1, "batch_size must be set 1" - - audio_sample = torch.cat((cache["prev_samples"], audio_sample_list[0])) - - n = int(len(audio_sample) // chunk_stride_samples + int(_is_final)) - m = int(len(audio_sample) % chunk_stride_samples * (1 - int(_is_final))) - tokens = [] - for i in range(n): - kwargs["is_final"] = _is_final and i == n - 1 - audio_sample_i = audio_sample[ - i * chunk_stride_samples : (i + 1) * chunk_stride_samples - ] - - # extract fbank feats - speech, speech_lengths = extract_fbank( - [audio_sample_i], - data_type=kwargs.get("data_type", "sound"), - frontend=frontend, - cache=cache["frontend"], - is_final=kwargs["is_final"], - ) - time3 = time.perf_counter() - meta_data["extract_feat"] = f"{time3 - time2:0.3f}" - meta_data["batch_data_time"] = ( - speech_lengths.sum().item() - * frontend.frame_shift - * frontend.lfr_n - / 1000 - ) - - tokens_i = self.generate_chunk( - speech, - speech_lengths, - key=key, - tokenizer=tokenizer, - cache=cache, - frontend=frontend, - **kwargs, - ) - tokens.extend(tokens_i) - - text_postprocessed, _ = postprocess_utils.sentence_postprocess(tokens) - - result_i = {"key": key[0], "text": text_postprocessed} - result = [result_i] - - cache["prev_samples"] = audio_sample[:-m] - if _is_final: - self.init_cache(cache, **kwargs) - - if kwargs.get("output_dir"): - writer = DatadirWriter(kwargs.get("output_dir")) - ibest_writer = writer[f"{1}best_recog"] - ibest_writer["token"][key[0]] = " ".join(tokens) - ibest_writer["text"][key[0]] = text_postprocessed - - return result, meta_data diff --git a/funasr_detach/models/scama/template.yaml b/funasr_detach/models/scama/template.yaml deleted file mode 100644 index f647a922277b63ae2273f301aff8579fe8da9be4..0000000000000000000000000000000000000000 --- a/funasr_detach/models/scama/template.yaml +++ /dev/null @@ -1,127 +0,0 @@ -# This is an example that demonstrates how to configure a model file. -# You can modify the configuration according to your own requirements. - -# to print the register_table: -# from funasr.register import tables -# tables.print() - -# network architecture -model: SCAMA -model_conf: - ctc_weight: 0.0 - lsm_weight: 0.1 - length_normalized_loss: true - -# encoder -encoder: SANMEncoderChunkOpt -encoder_conf: - output_size: 512 - attention_heads: 4 - linear_units: 2048 - num_blocks: 50 - dropout_rate: 0.1 - positional_dropout_rate: 0.1 - attention_dropout_rate: 0.1 - input_layer: pe - pos_enc_class: SinusoidalPositionEncoder - normalize_before: true - kernel_size: 11 - sanm_shfit: 0 - selfattention_layer_type: sanm - -# decoder -decoder: FsmnDecoderSCAMAOpt -decoder_conf: - attention_heads: 4 - linear_units: 2048 - num_blocks: 16 - dropout_rate: 0.1 - positional_dropout_rate: 0.1 - self_attention_dropout_rate: 0.1 - src_attention_dropout_rate: 0.1 - att_layer_num: 16 - kernel_size: 11 - sanm_shfit: 0 - -predictor: CifPredictorV2 -predictor_conf: - idim: 512 - threshold: 1.0 - l_order: 1 - r_order: 1 - tail_threshold: 0.45 - -# frontend related -frontend: WavFrontend -frontend_conf: - fs: 16000 - window: hamming - n_mels: 80 - frame_length: 25 - frame_shift: 10 - lfr_m: 7 - lfr_n: 6 - -specaug: SpecAugLFR -specaug_conf: - apply_time_warp: false - time_warp_window: 5 - time_warp_mode: bicubic - apply_freq_mask: true - freq_mask_width_range: - - 0 - - 30 - lfr_rate: 6 - num_freq_mask: 1 - apply_time_mask: true - time_mask_width_range: - - 0 - - 12 - num_time_mask: 1 - -train_conf: - accum_grad: 1 - grad_clip: 5 - max_epoch: 150 - val_scheduler_criterion: - - valid - - acc - best_model_criterion: - - - valid - - acc - - max - keep_nbest_models: 10 - avg_nbest_model: 5 - log_interval: 50 - -optim: adam -optim_conf: - lr: 0.0005 -scheduler: warmuplr -scheduler_conf: - warmup_steps: 30000 - -dataset: AudioDataset -dataset_conf: - index_ds: IndexDSJsonl - batch_sampler: DynamicBatchLocalShuffleSampler - batch_type: example # example or length - batch_size: 1 # if batch_type is example, batch_size is the numbers of samples; if length, batch_size is source_token_len+target_token_len; - max_token_length: 2048 # filter samples if source_token_len+target_token_len > max_token_length, - buffer_size: 500 - shuffle: True - num_workers: 0 - -tokenizer: CharTokenizer -tokenizer_conf: - unk_symbol: - split_with_space: true - - -ctc_conf: - dropout_rate: 0.0 - ctc_type: builtin - reduce: true - ignore_nan_grad: true - -normalize: null diff --git a/funasr_detach/models/scama/utils.py b/funasr_detach/models/scama/utils.py deleted file mode 100644 index edca6f4d3367ec392d3b4597bd3d0f2c27ee2dae..0000000000000000000000000000000000000000 --- a/funasr_detach/models/scama/utils.py +++ /dev/null @@ -1,91 +0,0 @@ -import os -import yaml -import torch -import numpy as np -from torch.nn import functional as F - - -def sequence_mask(lengths, maxlen=None, dtype=torch.float32, device=None): - if maxlen is None: - maxlen = lengths.max() - row_vector = torch.arange(0, maxlen, 1).to(lengths.device) - matrix = torch.unsqueeze(lengths, dim=-1) - mask = row_vector < matrix - mask = mask.detach() - - return mask.type(dtype).to(device) if device is not None else mask.type(dtype) - - -def apply_cmvn(inputs, mvn): - device = inputs.device - dtype = inputs.dtype - frame, dim = inputs.shape - meams = np.tile(mvn[0:1, :dim], (frame, 1)) - vars = np.tile(mvn[1:2, :dim], (frame, 1)) - inputs -= torch.from_numpy(meams).type(dtype).to(device) - inputs *= torch.from_numpy(vars).type(dtype).to(device) - - return inputs.type(torch.float32) - - -def drop_and_add( - inputs: torch.Tensor, - outputs: torch.Tensor, - training: bool, - dropout_rate: float = 0.1, - stoch_layer_coeff: float = 1.0, -): - - outputs = F.dropout(outputs, p=dropout_rate, training=training, inplace=True) - outputs *= stoch_layer_coeff - - input_dim = inputs.size(-1) - output_dim = outputs.size(-1) - - if input_dim == output_dim: - outputs += inputs - return outputs - - -def proc_tf_vocab(vocab_path): - with open(vocab_path, encoding="utf-8") as f: - token_list = [line.rstrip() for line in f] - if "" not in token_list: - token_list.append("") - return token_list - - -def gen_config_for_tfmodel(config_path, vocab_path, output_dir): - token_list = proc_tf_vocab(vocab_path) - with open(config_path, encoding="utf-8") as f: - config = yaml.safe_load(f) - - config["token_list"] = token_list - - if not os.path.exists(output_dir): - os.makedirs(output_dir) - - with open(os.path.join(output_dir, "config.yaml"), "w", encoding="utf-8") as f: - yaml_no_alias_safe_dump(config, f, indent=4, sort_keys=False) - - -class NoAliasSafeDumper(yaml.SafeDumper): - # Disable anchor/alias in yaml because looks ugly - def ignore_aliases(self, data): - return True - - -def yaml_no_alias_safe_dump(data, stream=None, **kwargs): - """Safe-dump in yaml with no anchor/alias""" - return yaml.dump( - data, stream, allow_unicode=True, Dumper=NoAliasSafeDumper, **kwargs - ) - - -if __name__ == "__main__": - import sys - - config_path = sys.argv[1] - vocab_path = sys.argv[2] - output_dir = sys.argv[3] - gen_config_for_tfmodel(config_path, vocab_path, output_dir) diff --git a/funasr_detach/models/seaco_paraformer/__init__.py b/funasr_detach/models/seaco_paraformer/__init__.py deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/funasr_detach/models/seaco_paraformer/model.py b/funasr_detach/models/seaco_paraformer/model.py deleted file mode 100644 index 2c4bfe0e5e9646ea18190f7cf819dcf00ec085b3..0000000000000000000000000000000000000000 --- a/funasr_detach/models/seaco_paraformer/model.py +++ /dev/null @@ -1,646 +0,0 @@ -#!/usr/bin/env python3 -# -*- encoding: utf-8 -*- -# Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved. -# MIT License (https://opensource.org/licenses/MIT) - -import os -import re -import time -import copy -import torch -import codecs -import logging -import tempfile -import requests -import numpy as np -from typing import Dict, Tuple -from contextlib import contextmanager -from distutils.version import LooseVersion - -from funasr_detach.register import tables -from funasr_detach.utils import postprocess_utils -from funasr_detach.metrics.compute_acc import th_accuracy -from funasr_detach.models.paraformer.model import Paraformer -from funasr_detach.utils.datadir_writer import DatadirWriter -from funasr_detach.models.paraformer.search import Hypothesis -from funasr_detach.models.paraformer.cif_predictor import mae_loss -from funasr_detach.train_utils.device_funcs import force_gatherable -from funasr_detach.models.bicif_paraformer.model import BiCifParaformer -from funasr_detach.losses.label_smoothing_loss import LabelSmoothingLoss -from funasr_detach.utils.timestamp_tools import ts_prediction_lfr6_standard -from funasr_detach.models.transformer.utils.add_sos_eos import add_sos_eos -from funasr_detach.models.transformer.utils.nets_utils import make_pad_mask, pad_list -from funasr_detach.utils.load_utils import load_audio_text_image_video, extract_fbank - - -if LooseVersion(torch.__version__) >= LooseVersion("1.6.0"): - from torch.cuda.amp import autocast -else: - # Nothing to do if torch<1.6.0 - @contextmanager - def autocast(enabled=True): - yield - - -@tables.register("model_classes", "SeacoParaformer") -class SeacoParaformer(BiCifParaformer, Paraformer): - """ - Author: Speech Lab of DAMO Academy, Alibaba Group - SeACo-Paraformer: A Non-Autoregressive ASR System with Flexible and Effective Hotword Customization Ability - https://arxiv.org/abs/2308.03266 - """ - - def __init__( - self, - *args, - **kwargs, - ): - super().__init__(*args, **kwargs) - - self.inner_dim = kwargs.get("inner_dim", 256) - self.bias_encoder_type = kwargs.get("bias_encoder_type", "lstm") - bias_encoder_dropout_rate = kwargs.get("bias_encoder_dropout_rate", 0.0) - bias_encoder_bid = kwargs.get("bias_encoder_bid", False) - seaco_lsm_weight = kwargs.get("seaco_lsm_weight", 0.0) - seaco_length_normalized_loss = kwargs.get("seaco_length_normalized_loss", True) - - # bias encoder - if self.bias_encoder_type == "lstm": - self.bias_encoder = torch.nn.LSTM( - self.inner_dim, - self.inner_dim, - 2, - batch_first=True, - dropout=bias_encoder_dropout_rate, - bidirectional=bias_encoder_bid, - ) - if bias_encoder_bid: - self.lstm_proj = torch.nn.Linear(self.inner_dim * 2, self.inner_dim) - else: - self.lstm_proj = None - self.bias_embed = torch.nn.Embedding(self.vocab_size, self.inner_dim) - elif self.bias_encoder_type == "mean": - self.bias_embed = torch.nn.Embedding(self.vocab_size, self.inner_dim) - else: - logging.error( - "Unsupport bias encoder type: {}".format(self.bias_encoder_type) - ) - - # seaco decoder - seaco_decoder = kwargs.get("seaco_decoder", None) - if seaco_decoder is not None: - seaco_decoder_conf = kwargs.get("seaco_decoder_conf") - seaco_decoder_class = tables.decoder_classes.get(seaco_decoder) - self.seaco_decoder = seaco_decoder_class( - vocab_size=self.vocab_size, - encoder_output_size=self.inner_dim, - **seaco_decoder_conf, - ) - self.hotword_output_layer = torch.nn.Linear(self.inner_dim, self.vocab_size) - self.criterion_seaco = LabelSmoothingLoss( - size=self.vocab_size, - padding_idx=self.ignore_id, - smoothing=seaco_lsm_weight, - normalize_length=seaco_length_normalized_loss, - ) - self.train_decoder = kwargs.get("train_decoder", False) - self.NO_BIAS = kwargs.get("NO_BIAS", 8377) - - def forward( - self, - speech: torch.Tensor, - speech_lengths: torch.Tensor, - text: torch.Tensor, - text_lengths: torch.Tensor, - **kwargs, - ) -> Tuple[torch.Tensor, Dict[str, torch.Tensor], torch.Tensor]: - """Frontend + Encoder + Decoder + Calc loss - - Args: - speech: (Batch, Length, ...) - speech_lengths: (Batch, ) - text: (Batch, Length) - text_lengths: (Batch,) - """ - assert text_lengths.dim() == 1, text_lengths.shape - # Check that batch_size is unified - assert ( - speech.shape[0] - == speech_lengths.shape[0] - == text.shape[0] - == text_lengths.shape[0] - ), (speech.shape, speech_lengths.shape, text.shape, text_lengths.shape) - - hotword_pad = kwargs.get("hotword_pad") - hotword_lengths = kwargs.get("hotword_lengths") - dha_pad = kwargs.get("dha_pad") - - batch_size = speech.shape[0] - self.step_cur += 1 - # for data-parallel - text = text[:, : text_lengths.max()] - speech = speech[:, : speech_lengths.max()] - - # 1. Encoder - encoder_out, encoder_out_lens = self.encode(speech, speech_lengths) - if self.predictor_bias == 1: - _, ys_pad = add_sos_eos(text, self.sos, self.eos, self.ignore_id) - ys_lengths = text_lengths + self.predictor_bias - - stats = dict() - loss_seaco = self._calc_seaco_loss( - encoder_out, - encoder_out_lens, - ys_pad, - ys_lengths, - hotword_pad, - hotword_lengths, - dha_pad, - ) - if self.train_decoder: - loss_att, acc_att = self._calc_att_loss( - encoder_out, encoder_out_lens, text, text_lengths - ) - loss = loss_seaco + loss_att - stats["loss_att"] = torch.clone(loss_att.detach()) - stats["acc_att"] = acc_att - else: - loss = loss_seaco - stats["loss_seaco"] = torch.clone(loss_seaco.detach()) - stats["loss"] = torch.clone(loss.detach()) - - # force_gatherable: to-device and to-tensor if scalar for DataParallel - if self.length_normalized_loss: - batch_size = (text_lengths + self.predictor_bias).sum().type_as(batch_size) - loss, stats, weight = force_gatherable((loss, stats, batch_size), loss.device) - return loss, stats, weight - - def _merge(self, cif_attended, dec_attended): - return cif_attended + dec_attended - - def _calc_seaco_loss( - self, - encoder_out: torch.Tensor, - encoder_out_lens: torch.Tensor, - ys_pad: torch.Tensor, - ys_lengths: torch.Tensor, - hotword_pad: torch.Tensor, - hotword_lengths: torch.Tensor, - dha_pad: torch.Tensor, - ): - # predictor forward - encoder_out_mask = ( - ~make_pad_mask(encoder_out_lens, maxlen=encoder_out.size(1))[:, None, :] - ).to(encoder_out.device) - pre_acoustic_embeds, _, _, _ = self.predictor( - encoder_out, ys_pad, encoder_out_mask, ignore_id=self.ignore_id - ) - # decoder forward - decoder_out, _ = self.decoder( - encoder_out, - encoder_out_lens, - pre_acoustic_embeds, - ys_lengths, - return_hidden=True, - ) - selected = self._hotword_representation(hotword_pad, hotword_lengths) - contextual_info = ( - selected.squeeze(0) - .repeat(encoder_out.shape[0], 1, 1) - .to(encoder_out.device) - ) - num_hot_word = contextual_info.shape[1] - _contextual_length = ( - torch.Tensor([num_hot_word]) - .int() - .repeat(encoder_out.shape[0]) - .to(encoder_out.device) - ) - # dha core - cif_attended, _ = self.seaco_decoder( - contextual_info, _contextual_length, pre_acoustic_embeds, ys_lengths - ) - dec_attended, _ = self.seaco_decoder( - contextual_info, _contextual_length, decoder_out, ys_lengths - ) - merged = self._merge(cif_attended, dec_attended) - dha_output = self.hotword_output_layer( - merged[:, :-1] - ) # remove the last token in loss calculation - loss_att = self.criterion_seaco(dha_output, dha_pad) - return loss_att - - def _seaco_decode_with_ASF( - self, - encoder_out, - encoder_out_lens, - sematic_embeds, - ys_pad_lens, - hw_list, - nfilter=50, - seaco_weight=1.0, - ): - # decoder forward - decoder_out, decoder_hidden, _ = self.decoder( - encoder_out, - encoder_out_lens, - sematic_embeds, - ys_pad_lens, - return_hidden=True, - return_both=True, - ) - decoder_pred = torch.log_softmax(decoder_out, dim=-1) - if hw_list is not None: - hw_lengths = [len(i) for i in hw_list] - hw_list_ = [torch.Tensor(i).long() for i in hw_list] - hw_list_pad = pad_list(hw_list_, 0).to(encoder_out.device) - selected = self._hotword_representation( - hw_list_pad, torch.Tensor(hw_lengths).int().to(encoder_out.device) - ) - contextual_info = ( - selected.squeeze(0) - .repeat(encoder_out.shape[0], 1, 1) - .to(encoder_out.device) - ) - num_hot_word = contextual_info.shape[1] - _contextual_length = ( - torch.Tensor([num_hot_word]) - .int() - .repeat(encoder_out.shape[0]) - .to(encoder_out.device) - ) - - # ASF Core - if nfilter > 0 and nfilter < num_hot_word: - for dec in self.seaco_decoder.decoders: - dec.reserve_attn = True - # cif_attended, _ = self.decoder2(contextual_info, _contextual_length, sematic_embeds, ys_pad_lens) - dec_attended, _ = self.seaco_decoder( - contextual_info, _contextual_length, decoder_hidden, ys_pad_lens - ) - # cif_filter = torch.topk(self.decoder2.decoders[-1].attn_mat[0][0].sum(0).sum(0)[:-1], min(nfilter, num_hot_word-1))[1].tolist() - hotword_scores = ( - self.seaco_decoder.decoders[-1].attn_mat[0][0].sum(0).sum(0)[:-1] - ) - # hotword_scores /= torch.sqrt(torch.tensor(hw_lengths)[:-1].float()).to(hotword_scores.device) - dec_filter = torch.topk(hotword_scores, min(nfilter, num_hot_word - 1))[ - 1 - ].tolist() - add_filter = dec_filter - add_filter.append(len(hw_list_pad) - 1) - # filter hotword embedding - selected = selected[add_filter] - # again - contextual_info = ( - selected.squeeze(0) - .repeat(encoder_out.shape[0], 1, 1) - .to(encoder_out.device) - ) - num_hot_word = contextual_info.shape[1] - _contextual_length = ( - torch.Tensor([num_hot_word]) - .int() - .repeat(encoder_out.shape[0]) - .to(encoder_out.device) - ) - for dec in self.seaco_decoder.decoders: - dec.attn_mat = [] - dec.reserve_attn = False - - # SeACo Core - cif_attended, _ = self.seaco_decoder( - contextual_info, _contextual_length, sematic_embeds, ys_pad_lens - ) - dec_attended, _ = self.seaco_decoder( - contextual_info, _contextual_length, decoder_hidden, ys_pad_lens - ) - merged = self._merge(cif_attended, dec_attended) - - dha_output = self.hotword_output_layer( - merged - ) # remove the last token in loss calculation - dha_pred = torch.log_softmax(dha_output, dim=-1) - - def _merge_res(dec_output, dha_output): - lmbd = torch.Tensor([seaco_weight] * dha_output.shape[0]) - dha_ids = dha_output.max(-1)[-1] # [0] - dha_mask = (dha_ids == 8377).int().unsqueeze(-1) - a = (1 - lmbd) / lmbd - b = 1 / lmbd - a, b = a.to(dec_output.device), b.to(dec_output.device) - dha_mask = (dha_mask + a.reshape(-1, 1, 1)) / b.reshape(-1, 1, 1) - # logits = dec_output * dha_mask + dha_output[:,:,:-1] * (1-dha_mask) - logits = dec_output * dha_mask + dha_output[:, :, :] * (1 - dha_mask) - return logits - - merged_pred = _merge_res(decoder_pred, dha_pred) - # import pdb; pdb.set_trace() - return merged_pred - else: - return decoder_pred - - def _hotword_representation(self, hotword_pad, hotword_lengths): - if self.bias_encoder_type != "lstm": - logging.error("Unsupported bias encoder type") - hw_embed = self.decoder.embed(hotword_pad) - hw_embed, (_, _) = self.bias_encoder(hw_embed) - if self.lstm_proj is not None: - hw_embed = self.lstm_proj(hw_embed) - _ind = np.arange(0, hw_embed.shape[0]).tolist() - selected = hw_embed[ - _ind, [i - 1 for i in hotword_lengths.detach().cpu().tolist()] - ] - return selected - - """ - def calc_predictor(self, encoder_out, encoder_out_lens): - encoder_out_mask = (~make_pad_mask(encoder_out_lens, maxlen=encoder_out.size(1))[:, None, :]).to( - encoder_out.device) - pre_acoustic_embeds, pre_token_length, alphas, pre_peak_index, pre_token_length2 = self.predictor(encoder_out, - None, - encoder_out_mask, - ignore_id=self.ignore_id) - return pre_acoustic_embeds, pre_token_length, alphas, pre_peak_index - - - def calc_predictor_timestamp(self, encoder_out, encoder_out_lens, token_num): - encoder_out_mask = (~make_pad_mask(encoder_out_lens, maxlen=encoder_out.size(1))[:, None, :]).to( - encoder_out.device) - ds_alphas, ds_cif_peak, us_alphas, us_peaks = self.predictor.get_upsample_timestamp(encoder_out, - encoder_out_mask, - token_num) - return ds_alphas, ds_cif_peak, us_alphas, us_peaks - """ - - def inference( - self, - data_in, - data_lengths=None, - key: list = None, - tokenizer=None, - frontend=None, - **kwargs, - ): - - # init beamsearch - is_use_ctc = ( - kwargs.get("decoding_ctc_weight", 0.0) > 0.00001 and self.ctc != None - ) - is_use_lm = ( - kwargs.get("lm_weight", 0.0) > 0.00001 - and kwargs.get("lm_file", None) is not None - ) - if self.beam_search is None and (is_use_lm or is_use_ctc): - logging.info("enable beam_search") - self.init_beam_search(**kwargs) - self.nbest = kwargs.get("nbest", 1) - - meta_data = {} - - # extract fbank feats - time1 = time.perf_counter() - audio_sample_list = load_audio_text_image_video( - data_in, fs=frontend.fs, audio_fs=kwargs.get("fs", 16000) - ) - time2 = time.perf_counter() - meta_data["load_data"] = f"{time2 - time1:0.3f}" - speech, speech_lengths = extract_fbank( - audio_sample_list, - data_type=kwargs.get("data_type", "sound"), - frontend=frontend, - ) - time3 = time.perf_counter() - meta_data["extract_feat"] = f"{time3 - time2:0.3f}" - meta_data["batch_data_time"] = ( - speech_lengths.sum().item() * frontend.frame_shift * frontend.lfr_n / 1000 - ) - - speech = speech.to(device=kwargs["device"]) - speech_lengths = speech_lengths.to(device=kwargs["device"]) - - # hotword - self.hotword_list = self.generate_hotwords_list( - kwargs.get("hotword", None), tokenizer=tokenizer, frontend=frontend - ) - - # Encoder - encoder_out, encoder_out_lens = self.encode(speech, speech_lengths) - if isinstance(encoder_out, tuple): - encoder_out = encoder_out[0] - - # predictor - predictor_outs = self.calc_predictor(encoder_out, encoder_out_lens) - pre_acoustic_embeds, pre_token_length, _, _ = ( - predictor_outs[0], - predictor_outs[1], - predictor_outs[2], - predictor_outs[3], - ) - pre_token_length = pre_token_length.round().long() - if torch.max(pre_token_length) < 1: - return [] - - decoder_out = self._seaco_decode_with_ASF( - encoder_out, - encoder_out_lens, - pre_acoustic_embeds, - pre_token_length, - hw_list=self.hotword_list, - ) - # decoder_out, _ = decoder_outs[0], decoder_outs[1] - _, _, us_alphas, us_peaks = self.calc_predictor_timestamp( - encoder_out, encoder_out_lens, pre_token_length - ) - - results = [] - b, n, d = decoder_out.size() - for i in range(b): - x = encoder_out[i, : encoder_out_lens[i], :] - am_scores = decoder_out[i, : pre_token_length[i], :] - if self.beam_search is not None: - nbest_hyps = self.beam_search( - x=x, - am_scores=am_scores, - maxlenratio=kwargs.get("maxlenratio", 0.0), - minlenratio=kwargs.get("minlenratio", 0.0), - ) - - nbest_hyps = nbest_hyps[: self.nbest] - else: - - yseq = am_scores.argmax(dim=-1) - score = am_scores.max(dim=-1)[0] - score = torch.sum(score, dim=-1) - # pad with mask tokens to ensure compatibility with sos/eos tokens - yseq = torch.tensor( - [self.sos] + yseq.tolist() + [self.eos], device=yseq.device - ) - nbest_hyps = [Hypothesis(yseq=yseq, score=score)] - for nbest_idx, hyp in enumerate(nbest_hyps): - ibest_writer = None - if kwargs.get("output_dir") is not None: - if not hasattr(self, "writer"): - self.writer = DatadirWriter(kwargs.get("output_dir")) - ibest_writer = self.writer[f"{nbest_idx + 1}best_recog"] - - # remove sos/eos and get results - last_pos = -1 - if isinstance(hyp.yseq, list): - token_int = hyp.yseq[1:last_pos] - else: - token_int = hyp.yseq[1:last_pos].tolist() - - # remove blank symbol id, which is assumed to be 0 - token_int = list( - filter( - lambda x: x != self.eos - and x != self.sos - and x != self.blank_id, - token_int, - ) - ) - - if tokenizer is not None: - # Change integer-ids to tokens - token = tokenizer.ids2tokens(token_int) - text = tokenizer.tokens2text(token) - - _, timestamp = ts_prediction_lfr6_standard( - us_alphas[i][: encoder_out_lens[i] * 3], - us_peaks[i][: encoder_out_lens[i] * 3], - copy.copy(token), - vad_offset=kwargs.get("begin_time", 0), - ) - - text_postprocessed, time_stamp_postprocessed, word_lists = ( - postprocess_utils.sentence_postprocess(token, timestamp) - ) - - result_i = { - "key": key[i], - "text": text_postprocessed, - "timestamp": time_stamp_postprocessed, - } - - if ibest_writer is not None: - ibest_writer["token"][key[i]] = " ".join(token) - ibest_writer["timestamp"][key[i]] = time_stamp_postprocessed - ibest_writer["text"][key[i]] = text_postprocessed - else: - result_i = {"key": key[i], "token_int": token_int} - results.append(result_i) - - return results, meta_data - - def generate_hotwords_list( - self, hotword_list_or_file, tokenizer=None, frontend=None - ): - def load_seg_dict(seg_dict_file): - seg_dict = {} - assert isinstance(seg_dict_file, str) - with open(seg_dict_file, "r", encoding="utf8") as f: - lines = f.readlines() - for line in lines: - s = line.strip().split() - key = s[0] - value = s[1:] - seg_dict[key] = " ".join(value) - return seg_dict - - def seg_tokenize(txt, seg_dict): - pattern = re.compile(r"^[\u4E00-\u9FA50-9]+$") - out_txt = "" - for word in txt: - word = word.lower() - if word in seg_dict: - out_txt += seg_dict[word] + " " - else: - if pattern.match(word): - for char in word: - if char in seg_dict: - out_txt += seg_dict[char] + " " - else: - out_txt += "" + " " - else: - out_txt += "" + " " - return out_txt.strip().split() - - seg_dict = None - if frontend.cmvn_file is not None: - model_dir = os.path.dirname(frontend.cmvn_file) - seg_dict_file = os.path.join(model_dir, "seg_dict") - if os.path.exists(seg_dict_file): - seg_dict = load_seg_dict(seg_dict_file) - else: - seg_dict = None - # for None - if hotword_list_or_file is None: - hotword_list = None - # for local txt inputs - elif os.path.exists(hotword_list_or_file) and hotword_list_or_file.endswith( - ".txt" - ): - logging.info("Attempting to parse hotwords from local txt...") - hotword_list = [] - hotword_str_list = [] - with codecs.open(hotword_list_or_file, "r") as fin: - for line in fin.readlines(): - hw = line.strip() - hw_list = hw.split() - if seg_dict is not None: - hw_list = seg_tokenize(hw_list, seg_dict) - hotword_str_list.append(hw) - hotword_list.append(tokenizer.tokens2ids(hw_list)) - hotword_list.append([self.sos]) - hotword_str_list.append("") - logging.info( - "Initialized hotword list from file: {}, hotword list: {}.".format( - hotword_list_or_file, hotword_str_list - ) - ) - # for url, download and generate txt - elif hotword_list_or_file.startswith("http"): - logging.info("Attempting to parse hotwords from url...") - work_dir = tempfile.TemporaryDirectory().name - if not os.path.exists(work_dir): - os.makedirs(work_dir) - text_file_path = os.path.join( - work_dir, os.path.basename(hotword_list_or_file) - ) - local_file = requests.get(hotword_list_or_file) - open(text_file_path, "wb").write(local_file.content) - hotword_list_or_file = text_file_path - hotword_list = [] - hotword_str_list = [] - with codecs.open(hotword_list_or_file, "r") as fin: - for line in fin.readlines(): - hw = line.strip() - hw_list = hw.split() - if seg_dict is not None: - hw_list = seg_tokenize(hw_list, seg_dict) - hotword_str_list.append(hw) - hotword_list.append(tokenizer.tokens2ids(hw_list)) - hotword_list.append([self.sos]) - hotword_str_list.append("") - logging.info( - "Initialized hotword list from file: {}, hotword list: {}.".format( - hotword_list_or_file, hotword_str_list - ) - ) - # for text str input - elif not hotword_list_or_file.endswith(".txt"): - logging.info("Attempting to parse hotwords as str...") - hotword_list = [] - hotword_str_list = [] - for hw in hotword_list_or_file.strip().split(): - hotword_str_list.append(hw) - hw_list = hw.strip().split() - if seg_dict is not None: - hw_list = seg_tokenize(hw_list, seg_dict) - hotword_list.append(tokenizer.tokens2ids(hw_list)) - hotword_list.append([self.sos]) - hotword_str_list.append("") - logging.info("Hotword list: {}.".format(hotword_str_list)) - else: - hotword_list = None - return hotword_list diff --git a/funasr_detach/models/seaco_paraformer/template.yaml b/funasr_detach/models/seaco_paraformer/template.yaml deleted file mode 100644 index ab2301a32bb864010790f51ebcfcbdf47e6df47b..0000000000000000000000000000000000000000 --- a/funasr_detach/models/seaco_paraformer/template.yaml +++ /dev/null @@ -1,156 +0,0 @@ -# This is an example that demonstrates how to configure a model file. -# You can modify the configuration according to your own requirements. - -# to print the register_table: -# from funasr.register import tables -# tables.print() - -# network architecture -model: SeacoParaformer -model_conf: - ctc_weight: 0.0 - lsm_weight: 0.1 - length_normalized_loss: true - predictor_weight: 1.0 - predictor_bias: 1 - sampling_ratio: 0.75 - inner_dim: 512 - bias_encoder_type: lstm - bias_encoder_bid: false - seaco_lsm_weight: 0.1 - seaco_length_normal: true - train_decoder: false - NO_BIAS: 8377 - -# encoder -encoder: SANMEncoder -encoder_conf: - output_size: 512 - attention_heads: 4 - linear_units: 2048 - num_blocks: 50 - dropout_rate: 0.1 - positional_dropout_rate: 0.1 - attention_dropout_rate: 0.1 - input_layer: pe - pos_enc_class: SinusoidalPositionEncoder - normalize_before: true - kernel_size: 11 - sanm_shfit: 0 - selfattention_layer_type: sanm - -# decoder -decoder: ParaformerSANMDecoder -decoder_conf: - attention_heads: 4 - linear_units: 2048 - num_blocks: 16 - dropout_rate: 0.1 - positional_dropout_rate: 0.1 - self_attention_dropout_rate: 0.1 - src_attention_dropout_rate: 0.1 - att_layer_num: 16 - kernel_size: 11 - sanm_shfit: 0 - -# seaco decoder -seaco_decoder: ParaformerSANMDecoder -seaco_decoder_conf: - attention_heads: 4 - linear_units: 1024 - num_blocks: 4 - dropout_rate: 0.1 - positional_dropout_rate: 0.1 - self_attention_dropout_rate: 0.1 - src_attention_dropout_rate: 0.1 - kernel_size: 21 - sanm_shfit: 0 - use_output_layer: false - wo_input_layer: true - -predictor: CifPredictorV3 -predictor_conf: - idim: 512 - threshold: 1.0 - l_order: 1 - r_order: 1 - tail_threshold: 0.45 - smooth_factor2: 0.25 - noise_threshold2: 0.01 - upsample_times: 3 - use_cif1_cnn: false - upsample_type: cnn_blstm - -# frontend related -frontend: WavFrontend -frontend_conf: - fs: 16000 - window: hamming - n_mels: 80 - frame_length: 25 - frame_shift: 10 - lfr_m: 7 - lfr_n: 6 - dither: 0.0 - -specaug: SpecAugLFR -specaug_conf: - apply_time_warp: false - time_warp_window: 5 - time_warp_mode: bicubic - apply_freq_mask: true - freq_mask_width_range: - - 0 - - 30 - lfr_rate: 6 - num_freq_mask: 1 - apply_time_mask: true - time_mask_width_range: - - 0 - - 12 - num_time_mask: 1 - -train_conf: - accum_grad: 1 - grad_clip: 5 - max_epoch: 150 - val_scheduler_criterion: - - valid - - acc - best_model_criterion: - - - valid - - acc - - max - keep_nbest_models: 10 - log_interval: 50 - -optim: adam -optim_conf: - lr: 0.0005 -scheduler: warmuplr -scheduler_conf: - warmup_steps: 30000 - -dataset: AudioDataset -dataset_conf: - index_ds: IndexDSJsonl - batch_sampler: DynamicBatchLocalShuffleSampler - batch_type: example # example or length - batch_size: 1 # if batch_type is example, batch_size is the numbers of samples; if length, batch_size is source_token_len+target_token_len; - max_token_length: 2048 # filter samples if source_token_len+target_token_len > max_token_length, - buffer_size: 500 - shuffle: True - num_workers: 0 - -tokenizer: CharTokenizer -tokenizer_conf: - unk_symbol: - split_with_space: true - - -ctc_conf: - dropout_rate: 0.0 - ctc_type: builtin - reduce: true - ignore_nan_grad: true -normalize: null diff --git a/funasr_detach/models/sond/__init__.py b/funasr_detach/models/sond/__init__.py deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/funasr_detach/models/sond/attention.py b/funasr_detach/models/sond/attention.py deleted file mode 100644 index eb6cb7ed865086baf893ff5ae76b14c430bd0e38..0000000000000000000000000000000000000000 --- a/funasr_detach/models/sond/attention.py +++ /dev/null @@ -1,328 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- - -# Copyright 2019 Shigeki Karita -# Apache 2.0 (http://www.apache.org/licenses/LICENSE-2.0) - -"""Multi-Head Attention layer definition.""" - -import math - -import numpy -import torch -from torch import nn -from typing import Optional, Tuple - -import torch.nn.functional as F -from funasr_detach.models.transformer.utils.nets_utils import make_pad_mask -import funasr_detach.models.lora.layers as lora - - -class MultiHeadedAttention(nn.Module): - """Multi-Head Attention layer. - - Args: - n_head (int): The number of heads. - n_feat (int): The number of features. - dropout_rate (float): Dropout rate. - - """ - - def __init__(self, n_head, n_feat, dropout_rate): - """Construct an MultiHeadedAttention object.""" - super(MultiHeadedAttention, self).__init__() - assert n_feat % n_head == 0 - # We assume d_v always equals d_k - self.d_k = n_feat // n_head - self.h = n_head - self.linear_q = nn.Linear(n_feat, n_feat) - self.linear_k = nn.Linear(n_feat, n_feat) - self.linear_v = nn.Linear(n_feat, n_feat) - self.linear_out = nn.Linear(n_feat, n_feat) - self.attn = None - self.dropout = nn.Dropout(p=dropout_rate) - - def forward_qkv(self, query, key, value): - """Transform query, key and value. - - Args: - query (torch.Tensor): Query tensor (#batch, time1, size). - key (torch.Tensor): Key tensor (#batch, time2, size). - value (torch.Tensor): Value tensor (#batch, time2, size). - - Returns: - torch.Tensor: Transformed query tensor (#batch, n_head, time1, d_k). - torch.Tensor: Transformed key tensor (#batch, n_head, time2, d_k). - torch.Tensor: Transformed value tensor (#batch, n_head, time2, d_k). - - """ - n_batch = query.size(0) - q = self.linear_q(query).view(n_batch, -1, self.h, self.d_k) - k = self.linear_k(key).view(n_batch, -1, self.h, self.d_k) - v = self.linear_v(value).view(n_batch, -1, self.h, self.d_k) - q = q.transpose(1, 2) # (batch, head, time1, d_k) - k = k.transpose(1, 2) # (batch, head, time2, d_k) - v = v.transpose(1, 2) # (batch, head, time2, d_k) - - return q, k, v - - def forward_attention(self, value, scores, mask): - """Compute attention context vector. - - Args: - value (torch.Tensor): Transformed value (#batch, n_head, time2, d_k). - scores (torch.Tensor): Attention score (#batch, n_head, time1, time2). - mask (torch.Tensor): Mask (#batch, 1, time2) or (#batch, time1, time2). - - Returns: - torch.Tensor: Transformed value (#batch, time1, d_model) - weighted by the attention score (#batch, time1, time2). - - """ - n_batch = value.size(0) - if mask is not None: - mask = mask.unsqueeze(1).eq(0) # (batch, 1, *, time2) - min_value = float( - numpy.finfo(torch.tensor(0, dtype=scores.dtype).numpy().dtype).min - ) - scores = scores.masked_fill(mask, min_value) - self.attn = torch.softmax(scores, dim=-1).masked_fill( - mask, 0.0 - ) # (batch, head, time1, time2) - else: - self.attn = torch.softmax(scores, dim=-1) # (batch, head, time1, time2) - - p_attn = self.dropout(self.attn) - x = torch.matmul(p_attn, value) # (batch, head, time1, d_k) - x = ( - x.transpose(1, 2).contiguous().view(n_batch, -1, self.h * self.d_k) - ) # (batch, time1, d_model) - - return self.linear_out(x) # (batch, time1, d_model) - - def forward(self, query, key, value, mask): - """Compute scaled dot product attention. - - Args: - query (torch.Tensor): Query tensor (#batch, time1, size). - key (torch.Tensor): Key tensor (#batch, time2, size). - value (torch.Tensor): Value tensor (#batch, time2, size). - mask (torch.Tensor): Mask tensor (#batch, 1, time2) or - (#batch, time1, time2). - - Returns: - torch.Tensor: Output tensor (#batch, time1, d_model). - - """ - q, k, v = self.forward_qkv(query, key, value) - scores = torch.matmul(q, k.transpose(-2, -1)) / math.sqrt(self.d_k) - return self.forward_attention(v, scores, mask) - - -class RelPositionMultiHeadedAttention(MultiHeadedAttention): - """Multi-Head Attention layer with relative position encoding (new implementation). - - Details can be found in https://github.com/espnet/espnet/pull/2816. - - Paper: https://arxiv.org/abs/1901.02860 - - Args: - n_head (int): The number of heads. - n_feat (int): The number of features. - dropout_rate (float): Dropout rate. - zero_triu (bool): Whether to zero the upper triangular part of attention matrix. - - """ - - def __init__(self, n_head, n_feat, dropout_rate, zero_triu=False): - """Construct an RelPositionMultiHeadedAttention object.""" - super().__init__(n_head, n_feat, dropout_rate) - self.zero_triu = zero_triu - # linear transformation for positional encoding - self.linear_pos = nn.Linear(n_feat, n_feat, bias=False) - # these two learnable bias are used in matrix c and matrix d - # as described in https://arxiv.org/abs/1901.02860 Section 3.3 - self.pos_bias_u = nn.Parameter(torch.Tensor(self.h, self.d_k)) - self.pos_bias_v = nn.Parameter(torch.Tensor(self.h, self.d_k)) - torch.nn.init.xavier_uniform_(self.pos_bias_u) - torch.nn.init.xavier_uniform_(self.pos_bias_v) - - def rel_shift(self, x): - """Compute relative positional encoding. - - Args: - x (torch.Tensor): Input tensor (batch, head, time1, 2*time1-1). - time1 means the length of query vector. - - Returns: - torch.Tensor: Output tensor. - - """ - zero_pad = torch.zeros((*x.size()[:3], 1), device=x.device, dtype=x.dtype) - x_padded = torch.cat([zero_pad, x], dim=-1) - - x_padded = x_padded.view(*x.size()[:2], x.size(3) + 1, x.size(2)) - x = x_padded[:, :, 1:].view_as(x)[ - :, :, :, : x.size(-1) // 2 + 1 - ] # only keep the positions from 0 to time2 - - if self.zero_triu: - ones = torch.ones((x.size(2), x.size(3)), device=x.device) - x = x * torch.tril(ones, x.size(3) - x.size(2))[None, None, :, :] - - return x - - def forward(self, query, key, value, pos_emb, mask): - """Compute 'Scaled Dot Product Attention' with rel. positional encoding. - - Args: - query (torch.Tensor): Query tensor (#batch, time1, size). - key (torch.Tensor): Key tensor (#batch, time2, size). - value (torch.Tensor): Value tensor (#batch, time2, size). - pos_emb (torch.Tensor): Positional embedding tensor - (#batch, 2*time1-1, size). - mask (torch.Tensor): Mask tensor (#batch, 1, time2) or - (#batch, time1, time2). - - Returns: - torch.Tensor: Output tensor (#batch, time1, d_model). - - """ - q, k, v = self.forward_qkv(query, key, value) - q = q.transpose(1, 2) # (batch, time1, head, d_k) - - n_batch_pos = pos_emb.size(0) - p = self.linear_pos(pos_emb).view(n_batch_pos, -1, self.h, self.d_k) - p = p.transpose(1, 2) # (batch, head, 2*time1-1, d_k) - - # (batch, head, time1, d_k) - q_with_bias_u = (q + self.pos_bias_u).transpose(1, 2) - # (batch, head, time1, d_k) - q_with_bias_v = (q + self.pos_bias_v).transpose(1, 2) - - # compute attention score - # first compute matrix a and matrix c - # as described in https://arxiv.org/abs/1901.02860 Section 3.3 - # (batch, head, time1, time2) - matrix_ac = torch.matmul(q_with_bias_u, k.transpose(-2, -1)) - - # compute matrix b and matrix d - # (batch, head, time1, 2*time1-1) - matrix_bd = torch.matmul(q_with_bias_v, p.transpose(-2, -1)) - matrix_bd = self.rel_shift(matrix_bd) - - scores = (matrix_ac + matrix_bd) / math.sqrt( - self.d_k - ) # (batch, head, time1, time2) - - return self.forward_attention(v, scores, mask) - - -class MultiHeadSelfAttention(nn.Module): - """Multi-Head Attention layer. - - Args: - n_head (int): The number of heads. - n_feat (int): The number of features. - dropout_rate (float): Dropout rate. - - """ - - def __init__(self, n_head, in_feat, n_feat, dropout_rate): - """Construct an MultiHeadedAttention object.""" - super(MultiHeadSelfAttention, self).__init__() - assert n_feat % n_head == 0 - # We assume d_v always equals d_k - self.d_k = n_feat // n_head - self.h = n_head - self.linear_out = nn.Linear(n_feat, n_feat) - self.linear_q_k_v = nn.Linear(in_feat, n_feat * 3) - self.attn = None - self.dropout = nn.Dropout(p=dropout_rate) - - def forward_qkv(self, x): - """Transform query, key and value. - - Args: - query (torch.Tensor): Query tensor (#batch, time1, size). - key (torch.Tensor): Key tensor (#batch, time2, size). - value (torch.Tensor): Value tensor (#batch, time2, size). - - Returns: - torch.Tensor: Transformed query tensor (#batch, n_head, time1, d_k). - torch.Tensor: Transformed key tensor (#batch, n_head, time2, d_k). - torch.Tensor: Transformed value tensor (#batch, n_head, time2, d_k). - - """ - b, t, d = x.size() - q_k_v = self.linear_q_k_v(x) - q, k, v = torch.split(q_k_v, int(self.h * self.d_k), dim=-1) - q_h = torch.reshape(q, (b, t, self.h, self.d_k)).transpose( - 1, 2 - ) # (batch, head, time1, d_k) - k_h = torch.reshape(k, (b, t, self.h, self.d_k)).transpose( - 1, 2 - ) # (batch, head, time2, d_k) - v_h = torch.reshape(v, (b, t, self.h, self.d_k)).transpose( - 1, 2 - ) # (batch, head, time2, d_k) - - return q_h, k_h, v_h, v - - def forward_attention(self, value, scores, mask, mask_att_chunk_encoder=None): - """Compute attention context vector. - - Args: - value (torch.Tensor): Transformed value (#batch, n_head, time2, d_k). - scores (torch.Tensor): Attention score (#batch, n_head, time1, time2). - mask (torch.Tensor): Mask (#batch, 1, time2) or (#batch, time1, time2). - - Returns: - torch.Tensor: Transformed value (#batch, time1, d_model) - weighted by the attention score (#batch, time1, time2). - - """ - n_batch = value.size(0) - if mask is not None: - if mask_att_chunk_encoder is not None: - mask = mask * mask_att_chunk_encoder - - mask = mask.unsqueeze(1).eq(0) # (batch, 1, *, time2) - - min_value = float( - numpy.finfo(torch.tensor(0, dtype=scores.dtype).numpy().dtype).min - ) - scores = scores.masked_fill(mask, min_value) - self.attn = torch.softmax(scores, dim=-1).masked_fill( - mask, 0.0 - ) # (batch, head, time1, time2) - else: - self.attn = torch.softmax(scores, dim=-1) # (batch, head, time1, time2) - - p_attn = self.dropout(self.attn) - x = torch.matmul(p_attn, value) # (batch, head, time1, d_k) - x = ( - x.transpose(1, 2).contiguous().view(n_batch, -1, self.h * self.d_k) - ) # (batch, time1, d_model) - - return self.linear_out(x) # (batch, time1, d_model) - - def forward(self, x, mask, mask_att_chunk_encoder=None): - """Compute scaled dot product attention. - - Args: - query (torch.Tensor): Query tensor (#batch, time1, size). - key (torch.Tensor): Key tensor (#batch, time2, size). - value (torch.Tensor): Value tensor (#batch, time2, size). - mask (torch.Tensor): Mask tensor (#batch, 1, time2) or - (#batch, time1, time2). - - Returns: - torch.Tensor: Output tensor (#batch, time1, d_model). - - """ - q_h, k_h, v_h, v = self.forward_qkv(x) - q_h = q_h * self.d_k ** (-0.5) - scores = torch.matmul(q_h, k_h.transpose(-2, -1)) - att_outs = self.forward_attention(v_h, scores, mask, mask_att_chunk_encoder) - return att_outs diff --git a/funasr_detach/models/sond/e2e_diar_sond.py b/funasr_detach/models/sond/e2e_diar_sond.py deleted file mode 100644 index e3c828b26c662dd4edb00c8f225bea8329e4e790..0000000000000000000000000000000000000000 --- a/funasr_detach/models/sond/e2e_diar_sond.py +++ /dev/null @@ -1,626 +0,0 @@ -#!/usr/bin/env python3 -# Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved. -# MIT License (https://opensource.org/licenses/MIT) -import logging -import random -from contextlib import contextmanager -from distutils.version import LooseVersion -from itertools import permutations -from typing import Dict -from typing import Optional -from typing import Tuple, List - -import numpy as np -import torch -from torch.nn import functional as F - -from funasr_detach.models.transformer.utils.nets_utils import to_device -from funasr_detach.models.transformer.utils.nets_utils import make_pad_mask -from funasr_detach.models.decoder.abs_decoder import AbsDecoder -from funasr_detach.models.encoder.abs_encoder import AbsEncoder -from funasr_detach.frontends.abs_frontend import AbsFrontend -from funasr_detach.models.specaug.abs_specaug import AbsSpecAug -from funasr_detach.models.specaug.abs_profileaug import AbsProfileAug -from funasr_detach.layers.abs_normalize import AbsNormalize -from funasr_detach.train_utils.device_funcs import force_gatherable -from funasr_detach.models.base_model import FunASRModel -from funasr_detach.losses.label_smoothing_loss import ( - LabelSmoothingLoss, - SequenceBinaryCrossEntropy, -) -from funasr_detach.utils.misc import int2vec -from funasr_detach.utils.hinter import hint_once - -if LooseVersion(torch.__version__) >= LooseVersion("1.6.0"): - from torch.cuda.amp import autocast -else: - # Nothing to do if torch<1.6.0 - @contextmanager - def autocast(enabled=True): - yield - - -class DiarSondModel(FunASRModel): - """Speaker overlap-aware neural diarization model - reference: https://arxiv.org/abs/2211.10243 - """ - - def __init__( - self, - vocab_size: int, - frontend: Optional[AbsFrontend], - specaug: Optional[AbsSpecAug], - profileaug: Optional[AbsProfileAug], - normalize: Optional[AbsNormalize], - encoder: torch.nn.Module, - speaker_encoder: Optional[torch.nn.Module], - ci_scorer: torch.nn.Module, - cd_scorer: Optional[torch.nn.Module], - decoder: torch.nn.Module, - token_list: list, - lsm_weight: float = 0.1, - length_normalized_loss: bool = False, - max_spk_num: int = 16, - label_aggregator: Optional[torch.nn.Module] = None, - normalize_speech_speaker: bool = False, - ignore_id: int = -1, - speaker_discrimination_loss_weight: float = 1.0, - inter_score_loss_weight: float = 0.0, - inputs_type: str = "raw", - model_regularizer_weight: float = 0.0, - freeze_encoder: bool = False, - onfly_shuffle_speaker: bool = True, - ): - - super().__init__() - - self.encoder = encoder - self.speaker_encoder = speaker_encoder - self.ci_scorer = ci_scorer - self.cd_scorer = cd_scorer - self.normalize = normalize - self.frontend = frontend - self.specaug = specaug - self.profileaug = profileaug - self.label_aggregator = label_aggregator - self.decoder = decoder - self.token_list = token_list - self.max_spk_num = max_spk_num - self.normalize_speech_speaker = normalize_speech_speaker - self.ignore_id = ignore_id - self.model_regularizer_weight = model_regularizer_weight - self.freeze_encoder = freeze_encoder - self.onfly_shuffle_speaker = onfly_shuffle_speaker - self.criterion_diar = LabelSmoothingLoss( - size=vocab_size, - padding_idx=ignore_id, - smoothing=lsm_weight, - normalize_length=length_normalized_loss, - ) - self.criterion_bce = SequenceBinaryCrossEntropy( - normalize_length=length_normalized_loss - ) - self.pse_embedding = self.generate_pse_embedding() - self.power_weight = torch.from_numpy( - 2 ** np.arange(max_spk_num)[np.newaxis, np.newaxis, :] - ).float() - self.int_token_arr = torch.from_numpy( - np.array(self.token_list).astype(int)[np.newaxis, np.newaxis, :] - ).int() - self.speaker_discrimination_loss_weight = speaker_discrimination_loss_weight - self.inter_score_loss_weight = inter_score_loss_weight - self.forward_steps = 0 - self.inputs_type = inputs_type - self.to_regularize_parameters = None - - def get_regularize_parameters(self): - to_regularize_parameters, normal_parameters = [], [] - for name, param in self.named_parameters(): - if ( - "encoder" in name - and "weight" in name - and "bn" not in name - and ( - "conv2" in name - or "conv1" in name - or "conv_sc" in name - or "dense" in name - ) - ): - to_regularize_parameters.append((name, param)) - else: - normal_parameters.append((name, param)) - self.to_regularize_parameters = to_regularize_parameters - return to_regularize_parameters, normal_parameters - - def generate_pse_embedding(self): - embedding = np.zeros((len(self.token_list), self.max_spk_num), dtype=np.float32) - for idx, pse_label in enumerate(self.token_list): - emb = int2vec(int(pse_label), vec_dim=self.max_spk_num, dtype=np.float32) - embedding[idx] = emb - return torch.from_numpy(embedding) - - def rand_permute_speaker(self, raw_profile, raw_binary_labels): - """ - raw_profile: B, N, D - raw_binary_labels: B, T, N - """ - assert ( - raw_profile.shape[1] == raw_binary_labels.shape[2] - ), "Num profile: {}, Num label: {}".format( - raw_profile.shape[1], raw_binary_labels.shape[-1] - ) - profile = torch.clone(raw_profile) - binary_labels = torch.clone(raw_binary_labels) - bsz, num_spk = profile.shape[0], profile.shape[1] - for i in range(bsz): - idx = list(range(num_spk)) - random.shuffle(idx) - profile[i] = profile[i][idx, :] - binary_labels[i] = binary_labels[i][:, idx] - - return profile, binary_labels - - def forward( - self, - speech: torch.Tensor, - speech_lengths: torch.Tensor = None, - profile: torch.Tensor = None, - profile_lengths: torch.Tensor = None, - binary_labels: torch.Tensor = None, - binary_labels_lengths: torch.Tensor = None, - ) -> Tuple[torch.Tensor, Dict[str, torch.Tensor], torch.Tensor]: - """Frontend + Encoder + Speaker Encoder + CI Scorer + CD Scorer + Decoder + Calc loss - - Args: - speech: (Batch, samples) or (Batch, frames, input_size) - speech_lengths: (Batch,) default None for chunk interator, - because the chunk-iterator does not - have the speech_lengths returned. - see in - espnet2/iterators/chunk_iter_factory.py - profile: (Batch, N_spk, dim) - profile_lengths: (Batch,) - binary_labels: (Batch, frames, max_spk_num) - binary_labels_lengths: (Batch,) - """ - assert speech.shape[0] <= binary_labels.shape[0], ( - speech.shape, - binary_labels.shape, - ) - batch_size = speech.shape[0] - if self.freeze_encoder: - hint_once("Freeze encoder", "freeze_encoder", rank=0) - self.encoder.eval() - self.forward_steps = self.forward_steps + 1 - if self.pse_embedding.device != speech.device: - self.pse_embedding = self.pse_embedding.to(speech.device) - self.power_weight = self.power_weight.to(speech.device) - self.int_token_arr = self.int_token_arr.to(speech.device) - - if self.onfly_shuffle_speaker: - hint_once( - "On-the-fly shuffle speaker permutation.", - "onfly_shuffle_speaker", - rank=0, - ) - profile, binary_labels = self.rand_permute_speaker(profile, binary_labels) - - # 0a. Aggregate time-domain labels to match forward outputs - if self.label_aggregator is not None: - binary_labels, binary_labels_lengths = self.label_aggregator( - binary_labels, binary_labels_lengths - ) - # 0b. augment profiles - if self.profileaug is not None and self.training: - speech, profile, binary_labels = self.profileaug( - speech, - speech_lengths, - profile, - profile_lengths, - binary_labels, - binary_labels_lengths, - ) - - # 1. Calculate power-set encoding (PSE) labels - pad_bin_labels = F.pad( - binary_labels, - (0, self.max_spk_num - binary_labels.shape[2]), - "constant", - 0.0, - ) - raw_pse_labels = torch.sum( - pad_bin_labels * self.power_weight, dim=2, keepdim=True - ) - pse_labels = torch.argmax( - (raw_pse_labels.int() == self.int_token_arr).float(), dim=2 - ) - - # 2. Network forward - pred, inter_outputs = self.prediction_forward( - speech, speech_lengths, profile, profile_lengths, return_inter_outputs=True - ) - (speech, speech_lengths), (profile, profile_lengths), (ci_score, cd_score) = ( - inter_outputs - ) - - # If encoder uses conv* as input_layer (i.e., subsampling), - # the sequence length of 'pred' might be slightly less than the - # length of 'spk_labels'. Here we force them to be equal. - length_diff_tolerance = 2 - length_diff = abs(pse_labels.shape[1] - pred.shape[1]) - if length_diff <= length_diff_tolerance: - min_len = min(pred.shape[1], pse_labels.shape[1]) - pse_labels = pse_labels[:, :min_len] - pred = pred[:, :min_len] - cd_score = cd_score[:, :min_len] - ci_score = ci_score[:, :min_len] - - loss_diar = self.classification_loss(pred, pse_labels, binary_labels_lengths) - loss_spk_dis = self.speaker_discrimination_loss(profile, profile_lengths) - loss_inter_ci, loss_inter_cd = self.internal_score_loss( - cd_score, ci_score, pse_labels, binary_labels_lengths - ) - regularizer_loss = None - if ( - self.model_regularizer_weight > 0 - and self.to_regularize_parameters is not None - ): - regularizer_loss = self.calculate_regularizer_loss() - label_mask = make_pad_mask( - binary_labels_lengths, maxlen=pse_labels.shape[1] - ).to(pse_labels.device) - loss = ( - loss_diar - + self.speaker_discrimination_loss_weight * loss_spk_dis - + self.inter_score_loss_weight * (loss_inter_ci + loss_inter_cd) - ) - # if regularizer_loss is not None: - # loss = loss + regularizer_loss * self.model_regularizer_weight - - ( - correct, - num_frames, - speech_scored, - speech_miss, - speech_falarm, - speaker_scored, - speaker_miss, - speaker_falarm, - speaker_error, - ) = self.calc_diarization_error( - pred=F.embedding(pred.argmax(dim=2) * (~label_mask), self.pse_embedding), - label=F.embedding(pse_labels * (~label_mask), self.pse_embedding), - length=binary_labels_lengths, - ) - - if speech_scored > 0 and num_frames > 0: - sad_mr, sad_fr, mi, fa, cf, acc, der = ( - speech_miss / speech_scored, - speech_falarm / speech_scored, - speaker_miss / speaker_scored, - speaker_falarm / speaker_scored, - speaker_error / speaker_scored, - correct / num_frames, - (speaker_miss + speaker_falarm + speaker_error) / speaker_scored, - ) - else: - sad_mr, sad_fr, mi, fa, cf, acc, der = 0, 0, 0, 0, 0, 0, 0 - - stats = dict( - loss=loss.detach(), - loss_diar=loss_diar.detach() if loss_diar is not None else None, - loss_spk_dis=loss_spk_dis.detach() if loss_spk_dis is not None else None, - loss_inter_ci=loss_inter_ci.detach() if loss_inter_ci is not None else None, - loss_inter_cd=loss_inter_cd.detach() if loss_inter_cd is not None else None, - regularizer_loss=( - regularizer_loss.detach() if regularizer_loss is not None else None - ), - sad_mr=sad_mr, - sad_fr=sad_fr, - mi=mi, - fa=fa, - cf=cf, - acc=acc, - der=der, - forward_steps=self.forward_steps, - ) - - loss, stats, weight = force_gatherable((loss, stats, batch_size), loss.device) - return loss, stats, weight - - def calculate_regularizer_loss(self): - regularizer_loss = 0.0 - for name, param in self.to_regularize_parameters: - regularizer_loss = regularizer_loss + torch.norm(param, p=2) - return regularizer_loss - - def classification_loss( - self, - predictions: torch.Tensor, - labels: torch.Tensor, - prediction_lengths: torch.Tensor, - ) -> torch.Tensor: - mask = make_pad_mask(prediction_lengths, maxlen=labels.shape[1]) - pad_labels = labels.masked_fill( - mask.to(predictions.device), value=self.ignore_id - ) - loss = self.criterion_diar(predictions.contiguous(), pad_labels) - - return loss - - def speaker_discrimination_loss( - self, profile: torch.Tensor, profile_lengths: torch.Tensor - ) -> torch.Tensor: - profile_mask = ( - torch.linalg.norm(profile, ord=2, dim=2, keepdim=True) > 0 - ).float() # (B, N, 1) - mask = torch.matmul(profile_mask, profile_mask.transpose(1, 2)) # (B, N, N) - mask = mask * (1.0 - torch.eye(self.max_spk_num).unsqueeze(0).to(mask)) - - eps = 1e-12 - coding_norm = ( - torch.linalg.norm( - profile * profile_mask + (1 - profile_mask) * eps, dim=2, keepdim=True - ) - * profile_mask - ) - # profile: Batch, N, dim - cos_theta = ( - F.cosine_similarity( - profile.unsqueeze(2), profile.unsqueeze(1), dim=-1, eps=eps - ) - * mask - ) - cos_theta = torch.clip(cos_theta, -1 + eps, 1 - eps) - loss = (F.relu(mask * coding_norm * (cos_theta - 0.0))).sum() / mask.sum() - - return loss - - def calculate_multi_labels(self, pse_labels, pse_labels_lengths): - mask = make_pad_mask(pse_labels_lengths, maxlen=pse_labels.shape[1]) - padding_labels = pse_labels.masked_fill(mask.to(pse_labels.device), value=0).to( - pse_labels - ) - multi_labels = F.embedding(padding_labels, self.pse_embedding) - - return multi_labels - - def internal_score_loss( - self, - cd_score: torch.Tensor, - ci_score: torch.Tensor, - pse_labels: torch.Tensor, - pse_labels_lengths: torch.Tensor, - ) -> Tuple[torch.Tensor, torch.Tensor]: - multi_labels = self.calculate_multi_labels(pse_labels, pse_labels_lengths) - ci_loss = self.criterion_bce(ci_score, multi_labels, pse_labels_lengths) - cd_loss = self.criterion_bce(cd_score, multi_labels, pse_labels_lengths) - return ci_loss, cd_loss - - def collect_feats( - self, - speech: torch.Tensor, - speech_lengths: torch.Tensor, - profile: torch.Tensor = None, - profile_lengths: torch.Tensor = None, - binary_labels: torch.Tensor = None, - binary_labels_lengths: torch.Tensor = None, - ) -> Dict[str, torch.Tensor]: - feats, feats_lengths = self._extract_feats(speech, speech_lengths) - return {"feats": feats, "feats_lengths": feats_lengths} - - def encode_speaker( - self, - profile: torch.Tensor, - profile_lengths: torch.Tensor, - ) -> Tuple[torch.Tensor, torch.Tensor]: - with autocast(False): - if profile.shape[1] < self.max_spk_num: - profile = F.pad( - profile, - [0, 0, 0, self.max_spk_num - profile.shape[1], 0, 0], - "constant", - 0.0, - ) - profile_mask = ( - torch.linalg.norm(profile, ord=2, dim=2, keepdim=True) > 0 - ).float() - profile = F.normalize(profile, dim=2) - if self.speaker_encoder is not None: - profile = self.speaker_encoder(profile, profile_lengths)[0] - return profile * profile_mask, profile_lengths - else: - return profile, profile_lengths - - def encode_speech( - self, - speech: torch.Tensor, - speech_lengths: torch.Tensor, - ) -> Tuple[torch.Tensor, torch.Tensor]: - if self.encoder is not None and self.inputs_type == "raw": - speech, speech_lengths = self.encode(speech, speech_lengths) - speech_mask = ~make_pad_mask(speech_lengths, maxlen=speech.shape[1]) - speech_mask = speech_mask.to(speech.device).unsqueeze(-1).float() - return speech * speech_mask, speech_lengths - else: - return speech, speech_lengths - - @staticmethod - def concate_speech_ivc(speech: torch.Tensor, ivc: torch.Tensor) -> torch.Tensor: - nn, tt = ivc.shape[1], speech.shape[1] - speech = speech.unsqueeze(dim=1) # B x 1 x T x D - speech = speech.expand(-1, nn, -1, -1) # B x N x T x D - ivc = ivc.unsqueeze(dim=2) # B x N x 1 x D - ivc = ivc.expand(-1, -1, tt, -1) # B x N x T x D - sd_in = torch.cat([speech, ivc], dim=3) # B x N x T x 2D - return sd_in - - def calc_similarity( - self, - speech_encoder_outputs: torch.Tensor, - speaker_encoder_outputs: torch.Tensor, - seq_len: torch.Tensor = None, - spk_len: torch.Tensor = None, - ) -> Tuple[torch.Tensor, torch.Tensor]: - bb, tt = speech_encoder_outputs.shape[0], speech_encoder_outputs.shape[1] - d_sph, d_spk = speech_encoder_outputs.shape[2], speaker_encoder_outputs.shape[2] - if self.normalize_speech_speaker: - speech_encoder_outputs = F.normalize(speech_encoder_outputs, dim=2) - speaker_encoder_outputs = F.normalize(speaker_encoder_outputs, dim=2) - ge_in = self.concate_speech_ivc(speech_encoder_outputs, speaker_encoder_outputs) - ge_in = torch.reshape(ge_in, [bb * self.max_spk_num, tt, d_sph + d_spk]) - ge_len = seq_len.unsqueeze(1).expand(-1, self.max_spk_num) - ge_len = torch.reshape(ge_len, [bb * self.max_spk_num]) - cd_simi = self.cd_scorer(ge_in, ge_len)[0] - cd_simi = torch.reshape(cd_simi, [bb, self.max_spk_num, tt, 1]) - cd_simi = cd_simi.squeeze(dim=3).permute([0, 2, 1]) - - if isinstance(self.ci_scorer, AbsEncoder): - ci_simi = self.ci_scorer(ge_in, ge_len)[0] - ci_simi = torch.reshape(ci_simi, [bb, self.max_spk_num, tt]).permute( - [0, 2, 1] - ) - else: - ci_simi = self.ci_scorer(speech_encoder_outputs, speaker_encoder_outputs) - - return ci_simi, cd_simi - - def post_net_forward(self, simi, seq_len): - logits = self.decoder(simi, seq_len)[0] - - return logits - - def prediction_forward( - self, - speech: torch.Tensor, - speech_lengths: torch.Tensor, - profile: torch.Tensor, - profile_lengths: torch.Tensor, - return_inter_outputs: bool = False, - ) -> [torch.Tensor, Optional[list]]: - # speech encoding - speech, speech_lengths = self.encode_speech(speech, speech_lengths) - # speaker encoding - profile, profile_lengths = self.encode_speaker(profile, profile_lengths) - # calculating similarity - ci_simi, cd_simi = self.calc_similarity( - speech, profile, speech_lengths, profile_lengths - ) - similarity = torch.cat([cd_simi, ci_simi], dim=2) - # post net forward - logits = self.post_net_forward(similarity, speech_lengths) - - if return_inter_outputs: - return logits, [ - (speech, speech_lengths), - (profile, profile_lengths), - (ci_simi, cd_simi), - ] - return logits - - def encode( - self, speech: torch.Tensor, speech_lengths: torch.Tensor - ) -> Tuple[torch.Tensor, torch.Tensor]: - """Frontend + Encoder - - Args: - speech: (Batch, Length, ...) - speech_lengths: (Batch,) - """ - with autocast(False): - # 1. Extract feats - feats, feats_lengths = self._extract_feats(speech, speech_lengths) - - # 2. Data augmentation - if self.specaug is not None and self.training: - feats, feats_lengths = self.specaug(feats, feats_lengths) - - # 3. Normalization for feature: e.g. Global-CMVN, Utterance-CMVN - if self.normalize is not None: - feats, feats_lengths = self.normalize(feats, feats_lengths) - - # 4. Forward encoder - # feats: (Batch, Length, Dim) - # -> encoder_out: (Batch, Length2, Dim) - encoder_outputs = self.encoder(feats, feats_lengths) - encoder_out, encoder_out_lens = encoder_outputs[:2] - - assert encoder_out.size(0) == speech.size(0), ( - encoder_out.size(), - speech.size(0), - ) - assert encoder_out.size(1) <= encoder_out_lens.max(), ( - encoder_out.size(), - encoder_out_lens.max(), - ) - - return encoder_out, encoder_out_lens - - def _extract_feats( - self, speech: torch.Tensor, speech_lengths: torch.Tensor - ) -> Tuple[torch.Tensor, torch.Tensor]: - batch_size = speech.shape[0] - speech_lengths = ( - speech_lengths - if speech_lengths is not None - else torch.ones(batch_size).int() * speech.shape[1] - ) - - assert speech_lengths.dim() == 1, speech_lengths.shape - - # for data-parallel - speech = speech[:, : speech_lengths.max()] - - if self.frontend is not None: - # Frontend - # e.g. STFT and Feature extract - # data_loader may send time-domain signal in this case - # speech (Batch, NSamples) -> feats: (Batch, NFrames, Dim) - feats, feats_lengths = self.frontend(speech, speech_lengths) - else: - # No frontend and no feature extract - feats, feats_lengths = speech, speech_lengths - return feats, feats_lengths - - @staticmethod - def calc_diarization_error(pred, label, length): - # Note (jiatong): Credit to https://github.com/hitachi-speech/EEND - - (batch_size, max_len, num_output) = label.size() - # mask the padding part - mask = ~make_pad_mask(length, maxlen=label.shape[1]).unsqueeze(-1).numpy() - - # pred and label have the shape (batch_size, max_len, num_output) - label_np = label.data.cpu().numpy().astype(int) - pred_np = (pred.data.cpu().numpy() > 0).astype(int) - label_np = label_np * mask - pred_np = pred_np * mask - length = length.data.cpu().numpy() - - # compute speech activity detection error - n_ref = np.sum(label_np, axis=2) - n_sys = np.sum(pred_np, axis=2) - speech_scored = float(np.sum(n_ref > 0)) - speech_miss = float(np.sum(np.logical_and(n_ref > 0, n_sys == 0))) - speech_falarm = float(np.sum(np.logical_and(n_ref == 0, n_sys > 0))) - - # compute speaker diarization error - speaker_scored = float(np.sum(n_ref)) - speaker_miss = float(np.sum(np.maximum(n_ref - n_sys, 0))) - speaker_falarm = float(np.sum(np.maximum(n_sys - n_ref, 0))) - n_map = np.sum(np.logical_and(label_np == 1, pred_np == 1), axis=2) - speaker_error = float(np.sum(np.minimum(n_ref, n_sys) - n_map)) - correct = float(1.0 * np.sum((label_np == pred_np) * mask) / num_output) - num_frames = np.sum(length) - return ( - correct, - num_frames, - speech_scored, - speech_miss, - speech_falarm, - speaker_scored, - speaker_miss, - speaker_falarm, - speaker_error, - ) diff --git a/funasr_detach/models/sond/encoder/__init__.py b/funasr_detach/models/sond/encoder/__init__.py deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/funasr_detach/models/sond/encoder/ci_scorers.py b/funasr_detach/models/sond/encoder/ci_scorers.py deleted file mode 100644 index fd3b3be9c70448bcc673e4e7d824b7d85c8ada16..0000000000000000000000000000000000000000 --- a/funasr_detach/models/sond/encoder/ci_scorers.py +++ /dev/null @@ -1,38 +0,0 @@ -import torch -from torch.nn import functional as F - - -class DotScorer(torch.nn.Module): - def __init__(self): - super().__init__() - - def forward( - self, - xs_pad: torch.Tensor, - spk_emb: torch.Tensor, - ): - # xs_pad: B, T, D - # spk_emb: B, N, D - scores = torch.matmul(xs_pad, spk_emb.transpose(1, 2)) - return scores - - def convert_tf2torch(self, var_dict_tf, var_dict_torch): - return {} - - -class CosScorer(torch.nn.Module): - def __init__(self): - super().__init__() - - def forward( - self, - xs_pad: torch.Tensor, - spk_emb: torch.Tensor, - ): - # xs_pad: B, T, D - # spk_emb: B, N, D - scores = F.cosine_similarity(xs_pad.unsqueeze(2), spk_emb.unsqueeze(1), dim=-1) - return scores - - def convert_tf2torch(self, var_dict_tf, var_dict_torch): - return {} diff --git a/funasr_detach/models/sond/encoder/conv_encoder.py b/funasr_detach/models/sond/encoder/conv_encoder.py deleted file mode 100644 index 318adc73162ab5892a7645ed45645f3e47c61e22..0000000000000000000000000000000000000000 --- a/funasr_detach/models/sond/encoder/conv_encoder.py +++ /dev/null @@ -1,299 +0,0 @@ -from typing import List -from typing import Optional -from typing import Sequence -from typing import Tuple -from typing import Union -import logging -import torch -import torch.nn as nn -from torch.nn import functional as F -import numpy as np -from funasr_detach.models.transformer.utils.nets_utils import make_pad_mask -from funasr_detach.models.transformer.layer_norm import LayerNorm -from funasr_detach.models.encoder.abs_encoder import AbsEncoder -import math -from funasr_detach.models.transformer.utils.repeat import repeat - - -class EncoderLayer(nn.Module): - def __init__( - self, - input_units, - num_units, - kernel_size=3, - activation="tanh", - stride=1, - include_batch_norm=False, - residual=False, - ): - super().__init__() - left_padding = math.ceil((kernel_size - stride) / 2) - right_padding = kernel_size - stride - left_padding - self.conv_padding = nn.ConstantPad1d((left_padding, right_padding), 0.0) - self.conv1d = nn.Conv1d( - input_units, - num_units, - kernel_size, - stride, - ) - self.activation = self.get_activation(activation) - if include_batch_norm: - self.bn = nn.BatchNorm1d(num_units, momentum=0.99, eps=1e-3) - self.residual = residual - self.include_batch_norm = include_batch_norm - self.input_units = input_units - self.num_units = num_units - self.stride = stride - - @staticmethod - def get_activation(activation): - if activation == "tanh": - return nn.Tanh() - else: - return nn.ReLU() - - def forward(self, xs_pad, ilens=None): - outputs = self.conv1d(self.conv_padding(xs_pad)) - if self.residual and self.stride == 1 and self.input_units == self.num_units: - outputs = outputs + xs_pad - - if self.include_batch_norm: - outputs = self.bn(outputs) - - # add parenthesis for repeat module - return self.activation(outputs), ilens - - -class ConvEncoder(AbsEncoder): - """ - Author: Speech Lab of DAMO Academy, Alibaba Group - Convolution encoder in OpenNMT framework - """ - - def __init__( - self, - num_layers, - input_units, - num_units, - kernel_size=3, - dropout_rate=0.3, - position_encoder=None, - activation="tanh", - auxiliary_states=True, - out_units=None, - out_norm=False, - out_residual=False, - include_batchnorm=False, - regularization_weight=0.0, - stride=1, - tf2torch_tensor_name_prefix_torch: str = "speaker_encoder", - tf2torch_tensor_name_prefix_tf: str = "EAND/speaker_encoder", - ): - super().__init__() - self._output_size = num_units - - self.num_layers = num_layers - self.input_units = input_units - self.num_units = num_units - self.kernel_size = kernel_size - self.dropout_rate = dropout_rate - self.position_encoder = position_encoder - self.out_units = out_units - self.auxiliary_states = auxiliary_states - self.out_norm = out_norm - self.activation = activation - self.out_residual = out_residual - self.include_batch_norm = include_batchnorm - self.regularization_weight = regularization_weight - self.tf2torch_tensor_name_prefix_torch = tf2torch_tensor_name_prefix_torch - self.tf2torch_tensor_name_prefix_tf = tf2torch_tensor_name_prefix_tf - if isinstance(stride, int): - self.stride = [stride] * self.num_layers - else: - self.stride = stride - self.downsample_rate = 1 - for s in self.stride: - self.downsample_rate *= s - - self.dropout = nn.Dropout(dropout_rate) - self.cnn_a = repeat( - self.num_layers, - lambda lnum: EncoderLayer( - input_units if lnum == 0 else num_units, - num_units, - kernel_size, - activation, - self.stride[lnum], - include_batchnorm, - residual=True if lnum > 0 else False, - ), - ) - - if self.out_units is not None: - left_padding = math.ceil((kernel_size - stride) / 2) - right_padding = kernel_size - stride - left_padding - self.out_padding = nn.ConstantPad1d((left_padding, right_padding), 0.0) - self.conv_out = nn.Conv1d( - num_units, - out_units, - kernel_size, - ) - - if self.out_norm: - self.after_norm = LayerNorm(out_units) - - def output_size(self) -> int: - return self.num_units - - def forward( - self, - xs_pad: torch.Tensor, - ilens: torch.Tensor, - prev_states: torch.Tensor = None, - ) -> Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]: - - inputs = xs_pad - if self.position_encoder is not None: - inputs = self.position_encoder(inputs) - - if self.dropout_rate > 0: - inputs = self.dropout(inputs) - - outputs, _ = self.cnn_a(inputs.transpose(1, 2), ilens) - - if self.out_units is not None: - outputs = self.conv_out(self.out_padding(outputs)) - - outputs = outputs.transpose(1, 2) - if self.out_norm: - outputs = self.after_norm(outputs) - - if self.out_residual: - outputs = outputs + inputs - - return outputs, ilens, None - - def gen_tf2torch_map_dict(self): - tensor_name_prefix_torch = self.tf2torch_tensor_name_prefix_torch - tensor_name_prefix_tf = self.tf2torch_tensor_name_prefix_tf - map_dict_local = { - # torch: conv1d.weight in "out_channel in_channel kernel_size" - # tf : conv1d.weight in "kernel_size in_channel out_channel" - # torch: linear.weight in "out_channel in_channel" - # tf : dense.weight in "in_channel out_channel" - "{}.cnn_a.0.conv1d.weight".format(tensor_name_prefix_torch): { - "name": "{}/cnn_a/conv1d/kernel".format(tensor_name_prefix_tf), - "squeeze": None, - "transpose": (2, 1, 0), - }, - "{}.cnn_a.0.conv1d.bias".format(tensor_name_prefix_torch): { - "name": "{}/cnn_a/conv1d/bias".format(tensor_name_prefix_tf), - "squeeze": None, - "transpose": None, - }, - "{}.cnn_a.layeridx.conv1d.weight".format(tensor_name_prefix_torch): { - "name": "{}/cnn_a/conv1d_layeridx/kernel".format(tensor_name_prefix_tf), - "squeeze": None, - "transpose": (2, 1, 0), - }, - "{}.cnn_a.layeridx.conv1d.bias".format(tensor_name_prefix_torch): { - "name": "{}/cnn_a/conv1d_layeridx/bias".format(tensor_name_prefix_tf), - "squeeze": None, - "transpose": None, - }, - } - if self.out_units is not None: - # add output layer - map_dict_local.update( - { - "{}.conv_out.weight".format(tensor_name_prefix_torch): { - "name": "{}/cnn_a/conv1d_{}/kernel".format( - tensor_name_prefix_tf, self.num_layers - ), - "squeeze": None, - "transpose": (2, 1, 0), - }, # tf: (1, 256, 256) -> torch: (256, 256, 1) - "{}.conv_out.bias".format(tensor_name_prefix_torch): { - "name": "{}/cnn_a/conv1d_{}/bias".format( - tensor_name_prefix_tf, self.num_layers - ), - "squeeze": None, - "transpose": None, - }, # tf: (256,) -> torch: (256,) - } - ) - - return map_dict_local - - def convert_tf2torch( - self, - var_dict_tf, - var_dict_torch, - ): - - map_dict = self.gen_tf2torch_map_dict() - - var_dict_torch_update = dict() - for name in sorted(var_dict_torch.keys(), reverse=False): - if name.startswith(self.tf2torch_tensor_name_prefix_torch): - # process special (first and last) layers - if name in map_dict: - name_tf = map_dict[name]["name"] - data_tf = var_dict_tf[name_tf] - if map_dict[name]["squeeze"] is not None: - data_tf = np.squeeze(data_tf, axis=map_dict[name]["squeeze"]) - if map_dict[name]["transpose"] is not None: - data_tf = np.transpose(data_tf, map_dict[name]["transpose"]) - data_tf = torch.from_numpy(data_tf).type(torch.float32).to("cpu") - assert ( - var_dict_torch[name].size() == data_tf.size() - ), "{}, {}, {} != {}".format( - name, name_tf, var_dict_torch[name].size(), data_tf.size() - ) - var_dict_torch_update[name] = data_tf - logging.info( - "torch tensor: {}, {}, loading from tf tensor: {}, {}".format( - name, data_tf.size(), name_tf, var_dict_tf[name_tf].shape - ) - ) - # process general layers - else: - # self.tf2torch_tensor_name_prefix_torch may include ".", solve this case - names = name.replace( - self.tf2torch_tensor_name_prefix_torch, "todo" - ).split(".") - layeridx = int(names[2]) - name_q = name.replace(".{}.".format(layeridx), ".layeridx.") - if name_q in map_dict.keys(): - name_v = map_dict[name_q]["name"] - name_tf = name_v.replace("layeridx", "{}".format(layeridx)) - data_tf = var_dict_tf[name_tf] - if map_dict[name_q]["squeeze"] is not None: - data_tf = np.squeeze( - data_tf, axis=map_dict[name_q]["squeeze"] - ) - if map_dict[name_q]["transpose"] is not None: - data_tf = np.transpose( - data_tf, map_dict[name_q]["transpose"] - ) - data_tf = ( - torch.from_numpy(data_tf).type(torch.float32).to("cpu") - ) - assert ( - var_dict_torch[name].size() == data_tf.size() - ), "{}, {}, {} != {}".format( - name, name_tf, var_dict_torch[name].size(), data_tf.size() - ) - var_dict_torch_update[name] = data_tf - logging.info( - "torch tensor: {}, {}, loading from tf tensor: {}, {}".format( - name, - data_tf.size(), - name_tf, - var_dict_tf[name_tf].shape, - ) - ) - else: - logging.warning("{} is missed from tf checkpoint".format(name)) - - return var_dict_torch_update diff --git a/funasr_detach/models/sond/encoder/ecapa_tdnn_encoder.py b/funasr_detach/models/sond/encoder/ecapa_tdnn_encoder.py deleted file mode 100644 index 0c2b4f1eeda0297bf634f7f88e30356f7904f918..0000000000000000000000000000000000000000 --- a/funasr_detach/models/sond/encoder/ecapa_tdnn_encoder.py +++ /dev/null @@ -1,683 +0,0 @@ -import math -import torch -import torch.nn as nn -import torch.nn.functional as F - - -class _BatchNorm1d(nn.Module): - def __init__( - self, - input_shape=None, - input_size=None, - eps=1e-05, - momentum=0.1, - affine=True, - track_running_stats=True, - combine_batch_time=False, - skip_transpose=False, - ): - super().__init__() - self.combine_batch_time = combine_batch_time - self.skip_transpose = skip_transpose - - if input_size is None and skip_transpose: - input_size = input_shape[1] - elif input_size is None: - input_size = input_shape[-1] - - self.norm = nn.BatchNorm1d( - input_size, - eps=eps, - momentum=momentum, - affine=affine, - track_running_stats=track_running_stats, - ) - - def forward(self, x): - shape_or = x.shape - if self.combine_batch_time: - if x.ndim == 3: - x = x.reshape(shape_or[0] * shape_or[1], shape_or[2]) - else: - x = x.reshape(shape_or[0] * shape_or[1], shape_or[3], shape_or[2]) - - elif not self.skip_transpose: - x = x.transpose(-1, 1) - - x_n = self.norm(x) - - if self.combine_batch_time: - x_n = x_n.reshape(shape_or) - elif not self.skip_transpose: - x_n = x_n.transpose(1, -1) - - return x_n - - -class _Conv1d(nn.Module): - def __init__( - self, - out_channels, - kernel_size, - input_shape=None, - in_channels=None, - stride=1, - dilation=1, - padding="same", - groups=1, - bias=True, - padding_mode="reflect", - skip_transpose=False, - ): - super().__init__() - self.kernel_size = kernel_size - self.stride = stride - self.dilation = dilation - self.padding = padding - self.padding_mode = padding_mode - self.unsqueeze = False - self.skip_transpose = skip_transpose - - if input_shape is None and in_channels is None: - raise ValueError("Must provide one of input_shape or in_channels") - - if in_channels is None: - in_channels = self._check_input_shape(input_shape) - - self.conv = nn.Conv1d( - in_channels, - out_channels, - self.kernel_size, - stride=self.stride, - dilation=self.dilation, - padding=0, - groups=groups, - bias=bias, - ) - - def forward(self, x): - if not self.skip_transpose: - x = x.transpose(1, -1) - - if self.unsqueeze: - x = x.unsqueeze(1) - - if self.padding == "same": - x = self._manage_padding(x, self.kernel_size, self.dilation, self.stride) - - elif self.padding == "causal": - num_pad = (self.kernel_size - 1) * self.dilation - x = F.pad(x, (num_pad, 0)) - - elif self.padding == "valid": - pass - - else: - raise ValueError( - "Padding must be 'same', 'valid' or 'causal'. Got " + self.padding - ) - - wx = self.conv(x) - - if self.unsqueeze: - wx = wx.squeeze(1) - - if not self.skip_transpose: - wx = wx.transpose(1, -1) - - return wx - - def _manage_padding( - self, - x, - kernel_size: int, - dilation: int, - stride: int, - ): - # Detecting input shape - L_in = x.shape[-1] - - # Time padding - padding = get_padding_elem(L_in, stride, kernel_size, dilation) - - # Applying padding - x = F.pad(x, padding, mode=self.padding_mode) - - return x - - def _check_input_shape(self, shape): - """Checks the input shape and returns the number of input channels.""" - - if len(shape) == 2: - self.unsqueeze = True - in_channels = 1 - elif self.skip_transpose: - in_channels = shape[1] - elif len(shape) == 3: - in_channels = shape[2] - else: - raise ValueError("conv1d expects 2d, 3d inputs. Got " + str(len(shape))) - - # Kernel size must be odd - if self.kernel_size % 2 == 0: - raise ValueError( - "The field kernel size must be an odd number. Got %s." - % (self.kernel_size) - ) - return in_channels - - -def get_padding_elem(L_in: int, stride: int, kernel_size: int, dilation: int): - if stride > 1: - n_steps = math.ceil(((L_in - kernel_size * dilation) / stride) + 1) - L_out = stride * (n_steps - 1) + kernel_size * dilation - padding = [kernel_size // 2, kernel_size // 2] - - else: - L_out = (L_in - dilation * (kernel_size - 1) - 1) // stride + 1 - - padding = [(L_in - L_out) // 2, (L_in - L_out) // 2] - return padding - - -# Skip transpose as much as possible for efficiency -class Conv1d(_Conv1d): - def __init__(self, *args, **kwargs): - super().__init__(skip_transpose=True, *args, **kwargs) - - -class BatchNorm1d(_BatchNorm1d): - def __init__(self, *args, **kwargs): - super().__init__(skip_transpose=True, *args, **kwargs) - - -def length_to_mask(length, max_len=None, dtype=None, device=None): - assert len(length.shape) == 1 - - if max_len is None: - max_len = length.max().long().item() # using arange to generate mask - mask = torch.arange(max_len, device=length.device, dtype=length.dtype).expand( - len(length), max_len - ) < length.unsqueeze(1) - - if dtype is None: - dtype = length.dtype - - if device is None: - device = length.device - - mask = torch.as_tensor(mask, dtype=dtype, device=device) - return mask - - -class TDNNBlock(nn.Module): - def __init__( - self, - in_channels, - out_channels, - kernel_size, - dilation, - activation=nn.ReLU, - groups=1, - ): - super(TDNNBlock, self).__init__() - self.conv = Conv1d( - in_channels=in_channels, - out_channels=out_channels, - kernel_size=kernel_size, - dilation=dilation, - groups=groups, - ) - self.activation = activation() - self.norm = BatchNorm1d(input_size=out_channels) - - def forward(self, x): - return self.norm(self.activation(self.conv(x))) - - -class Res2NetBlock(torch.nn.Module): - """An implementation of Res2NetBlock w/ dilation. - - Arguments - --------- - in_channels : int - The number of channels expected in the input. - out_channels : int - The number of output channels. - scale : int - The scale of the Res2Net block. - kernel_size: int - The kernel size of the Res2Net block. - dilation : int - The dilation of the Res2Net block. - - Example - ------- - >>> inp_tensor = torch.rand([8, 120, 64]).transpose(1, 2) - >>> layer = Res2NetBlock(64, 64, scale=4, dilation=3) - >>> out_tensor = layer(inp_tensor).transpose(1, 2) - >>> out_tensor.shape - torch.Size([8, 120, 64]) - """ - - def __init__(self, in_channels, out_channels, scale=8, kernel_size=3, dilation=1): - super(Res2NetBlock, self).__init__() - assert in_channels % scale == 0 - assert out_channels % scale == 0 - - in_channel = in_channels // scale - hidden_channel = out_channels // scale - - self.blocks = nn.ModuleList( - [ - TDNNBlock( - in_channel, - hidden_channel, - kernel_size=kernel_size, - dilation=dilation, - ) - for i in range(scale - 1) - ] - ) - self.scale = scale - - def forward(self, x): - y = [] - for i, x_i in enumerate(torch.chunk(x, self.scale, dim=1)): - if i == 0: - y_i = x_i - elif i == 1: - y_i = self.blocks[i - 1](x_i) - else: - y_i = self.blocks[i - 1](x_i + y_i) - y.append(y_i) - y = torch.cat(y, dim=1) - return y - - -class SEBlock(nn.Module): - """An implementation of squeeze-and-excitation block. - - Arguments - --------- - in_channels : int - The number of input channels. - se_channels : int - The number of output channels after squeeze. - out_channels : int - The number of output channels. - - Example - ------- - >>> inp_tensor = torch.rand([8, 120, 64]).transpose(1, 2) - >>> se_layer = SEBlock(64, 16, 64) - >>> lengths = torch.rand((8,)) - >>> out_tensor = se_layer(inp_tensor, lengths).transpose(1, 2) - >>> out_tensor.shape - torch.Size([8, 120, 64]) - """ - - def __init__(self, in_channels, se_channels, out_channels): - super(SEBlock, self).__init__() - - self.conv1 = Conv1d( - in_channels=in_channels, out_channels=se_channels, kernel_size=1 - ) - self.relu = torch.nn.ReLU(inplace=True) - self.conv2 = Conv1d( - in_channels=se_channels, out_channels=out_channels, kernel_size=1 - ) - self.sigmoid = torch.nn.Sigmoid() - - def forward(self, x, lengths=None): - L = x.shape[-1] - if lengths is not None: - mask = length_to_mask(lengths * L, max_len=L, device=x.device) - mask = mask.unsqueeze(1) - total = mask.sum(dim=2, keepdim=True) - s = (x * mask).sum(dim=2, keepdim=True) / total - else: - s = x.mean(dim=2, keepdim=True) - - s = self.relu(self.conv1(s)) - s = self.sigmoid(self.conv2(s)) - - return s * x - - -class AttentiveStatisticsPooling(nn.Module): - """This class implements an attentive statistic pooling layer for each channel. - It returns the concatenated mean and std of the input tensor. - - Arguments - --------- - channels: int - The number of input channels. - attention_channels: int - The number of attention channels. - - Example - ------- - >>> inp_tensor = torch.rand([8, 120, 64]).transpose(1, 2) - >>> asp_layer = AttentiveStatisticsPooling(64) - >>> lengths = torch.rand((8,)) - >>> out_tensor = asp_layer(inp_tensor, lengths).transpose(1, 2) - >>> out_tensor.shape - torch.Size([8, 1, 128]) - """ - - def __init__(self, channels, attention_channels=128, global_context=True): - super().__init__() - - self.eps = 1e-12 - self.global_context = global_context - if global_context: - self.tdnn = TDNNBlock(channels * 3, attention_channels, 1, 1) - else: - self.tdnn = TDNNBlock(channels, attention_channels, 1, 1) - self.tanh = nn.Tanh() - self.conv = Conv1d( - in_channels=attention_channels, out_channels=channels, kernel_size=1 - ) - - def forward(self, x, lengths=None): - """Calculates mean and std for a batch (input tensor). - - Arguments - --------- - x : torch.Tensor - Tensor of shape [N, C, L]. - """ - L = x.shape[-1] - - def _compute_statistics(x, m, dim=2, eps=self.eps): - mean = (m * x).sum(dim) - std = torch.sqrt((m * (x - mean.unsqueeze(dim)).pow(2)).sum(dim).clamp(eps)) - return mean, std - - if lengths is None: - lengths = torch.ones(x.shape[0], device=x.device) - - # Make binary mask of shape [N, 1, L] - mask = length_to_mask(lengths * L, max_len=L, device=x.device) - mask = mask.unsqueeze(1) - - # Expand the temporal context of the pooling layer by allowing the - # self-attention to look at global properties of the utterance. - if self.global_context: - # torch.std is unstable for backward computation - # https://github.com/pytorch/pytorch/issues/4320 - total = mask.sum(dim=2, keepdim=True).float() - mean, std = _compute_statistics(x, mask / total) - mean = mean.unsqueeze(2).repeat(1, 1, L) - std = std.unsqueeze(2).repeat(1, 1, L) - attn = torch.cat([x, mean, std], dim=1) - else: - attn = x - - # Apply layers - attn = self.conv(self.tanh(self.tdnn(attn))) - - # Filter out zero-paddings - attn = attn.masked_fill(mask == 0, float("-inf")) - - attn = F.softmax(attn, dim=2) - mean, std = _compute_statistics(x, attn) - # Append mean and std of the batch - pooled_stats = torch.cat((mean, std), dim=1) - pooled_stats = pooled_stats.unsqueeze(2) - - return pooled_stats - - -class SERes2NetBlock(nn.Module): - """An implementation of building block in ECAPA-TDNN, i.e., - TDNN-Res2Net-TDNN-SEBlock. - - Arguments - ---------- - out_channels: int - The number of output channels. - res2net_scale: int - The scale of the Res2Net block. - kernel_size: int - The kernel size of the TDNN blocks. - dilation: int - The dilation of the Res2Net block. - activation : torch class - A class for constructing the activation layers. - groups: int - Number of blocked connections from input channels to output channels. - - Example - ------- - >>> x = torch.rand(8, 120, 64).transpose(1, 2) - >>> conv = SERes2NetBlock(64, 64, res2net_scale=4) - >>> out = conv(x).transpose(1, 2) - >>> out.shape - torch.Size([8, 120, 64]) - """ - - def __init__( - self, - in_channels, - out_channels, - res2net_scale=8, - se_channels=128, - kernel_size=1, - dilation=1, - activation=torch.nn.ReLU, - groups=1, - ): - super().__init__() - self.out_channels = out_channels - self.tdnn1 = TDNNBlock( - in_channels, - out_channels, - kernel_size=1, - dilation=1, - activation=activation, - groups=groups, - ) - self.res2net_block = Res2NetBlock( - out_channels, out_channels, res2net_scale, kernel_size, dilation - ) - self.tdnn2 = TDNNBlock( - out_channels, - out_channels, - kernel_size=1, - dilation=1, - activation=activation, - groups=groups, - ) - self.se_block = SEBlock(out_channels, se_channels, out_channels) - - self.shortcut = None - if in_channels != out_channels: - self.shortcut = Conv1d( - in_channels=in_channels, - out_channels=out_channels, - kernel_size=1, - ) - - def forward(self, x, lengths=None): - residual = x - if self.shortcut: - residual = self.shortcut(x) - - x = self.tdnn1(x) - x = self.res2net_block(x) - x = self.tdnn2(x) - x = self.se_block(x, lengths) - - return x + residual - - -class ECAPA_TDNN(torch.nn.Module): - """An implementation of the speaker embedding model in a paper. - "ECAPA-TDNN: Emphasized Channel Attention, Propagation and Aggregation in - TDNN Based Speaker Verification" (https://arxiv.org/abs/2005.07143). - - Arguments - --------- - activation : torch class - A class for constructing the activation layers. - channels : list of ints - Output channels for TDNN/SERes2Net layer. - kernel_sizes : list of ints - List of kernel sizes for each layer. - dilations : list of ints - List of dilations for kernels in each layer. - lin_neurons : int - Number of neurons in linear layers. - groups : list of ints - List of groups for kernels in each layer. - - Example - ------- - >>> input_feats = torch.rand([5, 120, 80]) - >>> compute_embedding = ECAPA_TDNN(80, lin_neurons=192) - >>> outputs = compute_embedding(input_feats) - >>> outputs.shape - torch.Size([5, 1, 192]) - """ - - def __init__( - self, - input_size, - lin_neurons=192, - activation=torch.nn.ReLU, - channels=[512, 512, 512, 512, 1536], - kernel_sizes=[5, 3, 3, 3, 1], - dilations=[1, 2, 3, 4, 1], - attention_channels=128, - res2net_scale=8, - se_channels=128, - global_context=True, - groups=[1, 1, 1, 1, 1], - window_size=20, - window_shift=1, - ): - - super().__init__() - assert len(channels) == len(kernel_sizes) - assert len(channels) == len(dilations) - self.channels = channels - self.blocks = nn.ModuleList() - self.window_size = window_size - self.window_shift = window_shift - - # The initial TDNN layer - self.blocks.append( - TDNNBlock( - input_size, - channels[0], - kernel_sizes[0], - dilations[0], - activation, - groups[0], - ) - ) - - # SE-Res2Net layers - for i in range(1, len(channels) - 1): - self.blocks.append( - SERes2NetBlock( - channels[i - 1], - channels[i], - res2net_scale=res2net_scale, - se_channels=se_channels, - kernel_size=kernel_sizes[i], - dilation=dilations[i], - activation=activation, - groups=groups[i], - ) - ) - - # Multi-layer feature aggregation - self.mfa = TDNNBlock( - channels[-1], - channels[-1], - kernel_sizes[-1], - dilations[-1], - activation, - groups=groups[-1], - ) - - # Attentive Statistical Pooling - self.asp = AttentiveStatisticsPooling( - channels[-1], - attention_channels=attention_channels, - global_context=global_context, - ) - self.asp_bn = BatchNorm1d(input_size=channels[-1] * 2) - - # Final linear transformation - self.fc = Conv1d( - in_channels=channels[-1] * 2, - out_channels=lin_neurons, - kernel_size=1, - ) - - def windowed_pooling(self, x, lengths=None): - # x: Batch, Channel, Time - tt = x.shape[2] - num_chunk = int(math.ceil(tt / self.window_shift)) - pad = self.window_size // 2 - x = F.pad(x, (pad, pad, 0, 0), "reflect") - stat_list = [] - - for i in range(num_chunk): - # B x C - st, ed = i * self.window_shift, i * self.window_shift + self.window_size - x = self.asp( - x[:, :, st:ed], - lengths=( - torch.clamp(lengths - i, 0, self.window_size) - if lengths is not None - else None - ), - ) - x = self.asp_bn(x) - x = self.fc(x) - stat_list.append(x) - - return torch.cat(stat_list, dim=2) - - def forward(self, x, lengths=None): - """Returns the embedding vector. - - Arguments - --------- - x : torch.Tensor - Tensor of shape (batch, time, channel). - lengths: torch.Tensor - Tensor of shape (batch, ) - """ - # Minimize transpose for efficiency - x = x.transpose(1, 2) - - xl = [] - for layer in self.blocks: - try: - x = layer(x, lengths=lengths) - except TypeError: - x = layer(x) - xl.append(x) - - # Multi-layer feature aggregation - x = torch.cat(xl[1:], dim=1) - x = self.mfa(x) - - if self.window_size is None: - # Attentive Statistical Pooling - x = self.asp(x, lengths=lengths) - x = self.asp_bn(x) - # Final linear transformation - x = self.fc(x) - # x = x.transpose(1, 2) - x = x.squeeze(2) # -> B, C - else: - x = self.windowed_pooling(x, lengths) - x = x.transpose(1, 2) # -> B, T, C - return x diff --git a/funasr_detach/models/sond/encoder/fsmn_encoder.py b/funasr_detach/models/sond/encoder/fsmn_encoder.py deleted file mode 100644 index ecc17d0d39ba935b502d36b5b6e4ccda59eed1ca..0000000000000000000000000000000000000000 --- a/funasr_detach/models/sond/encoder/fsmn_encoder.py +++ /dev/null @@ -1,369 +0,0 @@ -from typing import List -from typing import Optional -from typing import Sequence -from typing import Tuple -from typing import Union -import logging -import torch -import torch.nn as nn -from torch.nn import functional as F -import numpy as np -from funasr_detach.models.transformer.utils.nets_utils import make_pad_mask -from funasr_detach.models.transformer.layer_norm import LayerNorm -from funasr_detach.models.encoder.abs_encoder import AbsEncoder -import math -from funasr_detach.models.transformer.utils.repeat import repeat -from funasr_detach.models.transformer.utils.multi_layer_conv import FsmnFeedForward - - -class FsmnBlock(torch.nn.Module): - def __init__( - self, - n_feat, - dropout_rate, - kernel_size, - fsmn_shift=0, - ): - super().__init__() - self.dropout = nn.Dropout(p=dropout_rate) - self.fsmn_block = nn.Conv1d( - n_feat, n_feat, kernel_size, stride=1, padding=0, groups=n_feat, bias=False - ) - # padding - left_padding = (kernel_size - 1) // 2 - if fsmn_shift > 0: - left_padding = left_padding + fsmn_shift - right_padding = kernel_size - 1 - left_padding - self.pad_fn = nn.ConstantPad1d((left_padding, right_padding), 0.0) - - def forward(self, inputs, mask, mask_shfit_chunk=None): - b, t, d = inputs.size() - if mask is not None: - mask = torch.reshape(mask, (b, -1, 1)) - if mask_shfit_chunk is not None: - mask = mask * mask_shfit_chunk - - inputs = inputs * mask - x = inputs.transpose(1, 2) - x = self.pad_fn(x) - x = self.fsmn_block(x) - x = x.transpose(1, 2) - x = x + inputs - x = self.dropout(x) - return x * mask - - -class EncoderLayer(torch.nn.Module): - def __init__(self, in_size, size, feed_forward, fsmn_block, dropout_rate=0.0): - super().__init__() - self.in_size = in_size - self.size = size - self.ffn = feed_forward - self.memory = fsmn_block - self.dropout = nn.Dropout(dropout_rate) - - def forward( - self, xs_pad: torch.Tensor, mask: torch.Tensor - ) -> Tuple[torch.Tensor, torch.Tensor]: - # xs_pad in Batch, Time, Dim - - context = self.ffn(xs_pad)[0] - memory = self.memory(context, mask) - - memory = self.dropout(memory) - if self.in_size == self.size: - return memory + xs_pad, mask - - return memory, mask - - -class FsmnEncoder(AbsEncoder): - """Encoder using Fsmn""" - - def __init__( - self, - in_units, - filter_size, - fsmn_num_layers, - dnn_num_layers, - num_memory_units=512, - ffn_inner_dim=2048, - dropout_rate=0.0, - shift=0, - position_encoder=None, - sample_rate=1, - out_units=None, - tf2torch_tensor_name_prefix_torch="post_net", - tf2torch_tensor_name_prefix_tf="EAND/post_net", - ): - """Initializes the parameters of the encoder. - - Args: - filter_size: the total order of memory block - fsmn_num_layers: The number of fsmn layers. - dnn_num_layers: The number of dnn layers - num_units: The number of memory units. - ffn_inner_dim: The number of units of the inner linear transformation - in the feed forward layer. - dropout_rate: The probability to drop units from the outputs. - shift: left padding, to control delay - position_encoder: The :class:`opennmt.layers.position.PositionEncoder` to - apply on inputs or ``None``. - """ - super(FsmnEncoder, self).__init__() - self.in_units = in_units - self.filter_size = filter_size - self.fsmn_num_layers = fsmn_num_layers - self.dnn_num_layers = dnn_num_layers - self.num_memory_units = num_memory_units - self.ffn_inner_dim = ffn_inner_dim - self.dropout_rate = dropout_rate - self.shift = shift - if not isinstance(shift, list): - self.shift = [shift for _ in range(self.fsmn_num_layers)] - self.sample_rate = sample_rate - if not isinstance(sample_rate, list): - self.sample_rate = [sample_rate for _ in range(self.fsmn_num_layers)] - self.position_encoder = position_encoder - self.dropout = nn.Dropout(dropout_rate) - self.out_units = out_units - self.tf2torch_tensor_name_prefix_torch = tf2torch_tensor_name_prefix_torch - self.tf2torch_tensor_name_prefix_tf = tf2torch_tensor_name_prefix_tf - - self.fsmn_layers = repeat( - self.fsmn_num_layers, - lambda lnum: EncoderLayer( - in_units if lnum == 0 else num_memory_units, - num_memory_units, - FsmnFeedForward( - in_units if lnum == 0 else num_memory_units, - ffn_inner_dim, - num_memory_units, - 1, - dropout_rate, - ), - FsmnBlock( - num_memory_units, dropout_rate, filter_size, self.shift[lnum] - ), - ), - ) - - self.dnn_layers = repeat( - dnn_num_layers, - lambda lnum: FsmnFeedForward( - num_memory_units, - ffn_inner_dim, - num_memory_units, - 1, - dropout_rate, - ), - ) - if out_units is not None: - self.conv1d = nn.Conv1d(num_memory_units, out_units, 1, 1) - - def output_size(self) -> int: - return self.num_memory_units - - def forward( - self, - xs_pad: torch.Tensor, - ilens: torch.Tensor, - prev_states: torch.Tensor = None, - ) -> Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]: - inputs = xs_pad - if self.position_encoder is not None: - inputs = self.position_encoder(inputs) - - inputs = self.dropout(inputs) - masks = (~make_pad_mask(ilens)[:, None, :]).to(xs_pad.device) - inputs = self.fsmn_layers(inputs, masks)[0] - inputs = self.dnn_layers(inputs)[0] - - if self.out_units is not None: - inputs = self.conv1d(inputs.transpose(1, 2)).transpose(1, 2) - - return inputs, ilens, None - - def gen_tf2torch_map_dict(self): - tensor_name_prefix_torch = self.tf2torch_tensor_name_prefix_torch - tensor_name_prefix_tf = self.tf2torch_tensor_name_prefix_tf - map_dict_local = { - # torch: conv1d.weight in "out_channel in_channel kernel_size" - # tf : conv1d.weight in "kernel_size in_channel out_channel" - # torch: linear.weight in "out_channel in_channel" - # tf : dense.weight in "in_channel out_channel" - # for fsmn_layers - "{}.fsmn_layers.layeridx.ffn.norm.bias".format(tensor_name_prefix_torch): { - "name": "{}/fsmn_layer_layeridx/ffn/LayerNorm/beta".format( - tensor_name_prefix_tf - ), - "squeeze": None, - "transpose": None, - }, - "{}.fsmn_layers.layeridx.ffn.norm.weight".format( - tensor_name_prefix_torch - ): { - "name": "{}/fsmn_layer_layeridx/ffn/LayerNorm/gamma".format( - tensor_name_prefix_tf - ), - "squeeze": None, - "transpose": None, - }, - "{}.fsmn_layers.layeridx.ffn.w_1.bias".format(tensor_name_prefix_torch): { - "name": "{}/fsmn_layer_layeridx/ffn/conv1d/bias".format( - tensor_name_prefix_tf - ), - "squeeze": None, - "transpose": None, - }, - "{}.fsmn_layers.layeridx.ffn.w_1.weight".format(tensor_name_prefix_torch): { - "name": "{}/fsmn_layer_layeridx/ffn/conv1d/kernel".format( - tensor_name_prefix_tf - ), - "squeeze": None, - "transpose": (2, 1, 0), - }, - "{}.fsmn_layers.layeridx.ffn.w_2.weight".format(tensor_name_prefix_torch): { - "name": "{}/fsmn_layer_layeridx/ffn/conv1d_1/kernel".format( - tensor_name_prefix_tf - ), - "squeeze": None, - "transpose": (2, 1, 0), - }, - "{}.fsmn_layers.layeridx.memory.fsmn_block.weight".format( - tensor_name_prefix_torch - ): { - "name": "{}/fsmn_layer_layeridx/memory/depth_conv_w".format( - tensor_name_prefix_tf - ), - "squeeze": 0, - "transpose": (1, 2, 0), - }, # (1, 31, 512, 1) -> (31, 512, 1) -> (512, 1, 31) - # for dnn_layers - "{}.dnn_layers.layeridx.norm.bias".format(tensor_name_prefix_torch): { - "name": "{}/dnn_layer_layeridx/LayerNorm/beta".format( - tensor_name_prefix_tf - ), - "squeeze": None, - "transpose": None, - }, - "{}.dnn_layers.layeridx.norm.weight".format(tensor_name_prefix_torch): { - "name": "{}/dnn_layer_layeridx/LayerNorm/gamma".format( - tensor_name_prefix_tf - ), - "squeeze": None, - "transpose": None, - }, - "{}.dnn_layers.layeridx.w_1.bias".format(tensor_name_prefix_torch): { - "name": "{}/dnn_layer_layeridx/conv1d/bias".format( - tensor_name_prefix_tf - ), - "squeeze": None, - "transpose": None, - }, - "{}.dnn_layers.layeridx.w_1.weight".format(tensor_name_prefix_torch): { - "name": "{}/dnn_layer_layeridx/conv1d/kernel".format( - tensor_name_prefix_tf - ), - "squeeze": None, - "transpose": (2, 1, 0), - }, - "{}.dnn_layers.layeridx.w_2.weight".format(tensor_name_prefix_torch): { - "name": "{}/dnn_layer_layeridx/conv1d_1/kernel".format( - tensor_name_prefix_tf - ), - "squeeze": None, - "transpose": (2, 1, 0), - }, - } - if self.out_units is not None: - # add output layer - map_dict_local.update( - { - "{}.conv1d.weight".format(tensor_name_prefix_torch): { - "name": "{}/conv1d/kernel".format(tensor_name_prefix_tf), - "squeeze": None, - "transpose": (2, 1, 0), - }, - "{}.conv1d.bias".format(tensor_name_prefix_torch): { - "name": "{}/conv1d/bias".format(tensor_name_prefix_tf), - "squeeze": None, - "transpose": None, - }, - } - ) - - return map_dict_local - - def convert_tf2torch( - self, - var_dict_tf, - var_dict_torch, - ): - - map_dict = self.gen_tf2torch_map_dict() - - var_dict_torch_update = dict() - for name in sorted(var_dict_torch.keys(), reverse=False): - if name.startswith(self.tf2torch_tensor_name_prefix_torch): - # process special (first and last) layers - if name in map_dict: - name_tf = map_dict[name]["name"] - data_tf = var_dict_tf[name_tf] - if map_dict[name]["squeeze"] is not None: - data_tf = np.squeeze(data_tf, axis=map_dict[name]["squeeze"]) - if map_dict[name]["transpose"] is not None: - data_tf = np.transpose(data_tf, map_dict[name]["transpose"]) - data_tf = torch.from_numpy(data_tf).type(torch.float32).to("cpu") - assert ( - var_dict_torch[name].size() == data_tf.size() - ), "{}, {}, {} != {}".format( - name, name_tf, var_dict_torch[name].size(), data_tf.size() - ) - var_dict_torch_update[name] = data_tf - logging.info( - "torch tensor: {}, {}, loading from tf tensor: {}, {}".format( - name, data_tf.size(), name_tf, var_dict_tf[name_tf].shape - ) - ) - # process general layers - else: - # self.tf2torch_tensor_name_prefix_torch may include ".", solve this case - names = name.replace( - self.tf2torch_tensor_name_prefix_torch, "todo" - ).split(".") - layeridx = int(names[2]) - name_q = name.replace(".{}.".format(layeridx), ".layeridx.") - if name_q in map_dict.keys(): - name_v = map_dict[name_q]["name"] - name_tf = name_v.replace("layeridx", "{}".format(layeridx)) - data_tf = var_dict_tf[name_tf] - if map_dict[name_q]["squeeze"] is not None: - data_tf = np.squeeze( - data_tf, axis=map_dict[name_q]["squeeze"] - ) - if map_dict[name_q]["transpose"] is not None: - data_tf = np.transpose( - data_tf, map_dict[name_q]["transpose"] - ) - data_tf = ( - torch.from_numpy(data_tf).type(torch.float32).to("cpu") - ) - assert ( - var_dict_torch[name].size() == data_tf.size() - ), "{}, {}, {} != {}".format( - name, name_tf, var_dict_torch[name].size(), data_tf.size() - ) - var_dict_torch_update[name] = data_tf - logging.info( - "torch tensor: {}, {}, loading from tf tensor: {}, {}".format( - name, - data_tf.size(), - name_tf, - var_dict_tf[name_tf].shape, - ) - ) - else: - logging.warning("{} is missed from tf checkpoint".format(name)) - - return var_dict_torch_update diff --git a/funasr_detach/models/sond/encoder/resnet34_encoder.py b/funasr_detach/models/sond/encoder/resnet34_encoder.py deleted file mode 100644 index 851068bbac290e5af80d2d88b61faae8fcfc5b8a..0000000000000000000000000000000000000000 --- a/funasr_detach/models/sond/encoder/resnet34_encoder.py +++ /dev/null @@ -1,1117 +0,0 @@ -import torch -from torch.nn import functional as F -from funasr_detach.models.encoder.abs_encoder import AbsEncoder -from typing import Tuple, Optional -from funasr_detach.models.pooling.statistic_pooling import ( - statistic_pooling, - windowed_statistic_pooling, -) -from collections import OrderedDict -import logging -import numpy as np - - -class BasicLayer(torch.nn.Module): - - def __init__( - self, in_filters: int, filters: int, stride: int, bn_momentum: float = 0.5 - ): - - super().__init__() - self.stride = stride - self.in_filters = in_filters - self.filters = filters - - self.bn1 = torch.nn.BatchNorm2d( - in_filters, eps=1e-3, momentum=bn_momentum, affine=True - ) - self.relu1 = torch.nn.ReLU() - self.conv1 = torch.nn.Conv2d(in_filters, filters, 3, stride, bias=False) - - self.bn2 = torch.nn.BatchNorm2d( - filters, eps=1e-3, momentum=bn_momentum, affine=True - ) - self.relu2 = torch.nn.ReLU() - self.conv2 = torch.nn.Conv2d(filters, filters, 3, 1, bias=False) - - if in_filters != filters or stride > 1: - self.conv_sc = torch.nn.Conv2d(in_filters, filters, 1, stride, bias=False) - self.bn_sc = torch.nn.BatchNorm2d( - filters, eps=1e-3, momentum=bn_momentum, affine=True - ) - - def proper_padding(self, x, stride): - # align padding mode to tf.layers.conv2d with padding_mod="same" - if stride == 1: - return F.pad(x, (1, 1, 1, 1), "constant", 0) - elif stride == 2: - h, w = x.size(2), x.size(3) - # (left, right, top, bottom) - return F.pad(x, (w % 2, 1, h % 2, 1), "constant", 0) - - def forward(self, xs_pad, ilens): - identity = xs_pad - if self.in_filters != self.filters or self.stride > 1: - identity = self.conv_sc(identity) - identity = self.bn_sc(identity) - - xs_pad = self.relu1(self.bn1(xs_pad)) - xs_pad = self.proper_padding(xs_pad, self.stride) - xs_pad = self.conv1(xs_pad) - - xs_pad = self.relu2(self.bn2(xs_pad)) - xs_pad = self.proper_padding(xs_pad, 1) - xs_pad = self.conv2(xs_pad) - - if self.stride == 2: - ilens = (ilens + 1) // self.stride - - return xs_pad + identity, ilens - - -class BasicBlock(torch.nn.Module): - def __init__(self, in_filters, filters, num_layer, stride, bn_momentum=0.5): - super().__init__() - self.num_layer = num_layer - - for i in range(num_layer): - layer = BasicLayer( - in_filters if i == 0 else filters, - filters, - stride if i == 0 else 1, - bn_momentum, - ) - self.add_module("layer_{}".format(i), layer) - - def forward(self, xs_pad, ilens): - - for i in range(self.num_layer): - xs_pad, ilens = self._modules["layer_{}".format(i)](xs_pad, ilens) - - return xs_pad, ilens - - -class ResNet34(AbsEncoder): - def __init__( - self, - input_size, - use_head_conv=True, - batchnorm_momentum=0.5, - use_head_maxpool=False, - num_nodes_pooling_layer=256, - layers_in_block=(3, 4, 6, 3), - filters_in_block=(32, 64, 128, 256), - ): - super(ResNet34, self).__init__() - - self.use_head_conv = use_head_conv - self.use_head_maxpool = use_head_maxpool - self.num_nodes_pooling_layer = num_nodes_pooling_layer - self.layers_in_block = layers_in_block - self.filters_in_block = filters_in_block - self.input_size = input_size - - pre_filters = filters_in_block[0] - if use_head_conv: - self.pre_conv = torch.nn.Conv2d( - 1, pre_filters, 3, 1, 1, bias=False, padding_mode="zeros" - ) - self.pre_conv_bn = torch.nn.BatchNorm2d( - pre_filters, eps=1e-3, momentum=batchnorm_momentum - ) - - if use_head_maxpool: - self.head_maxpool = torch.nn.MaxPool2d(3, 1, padding=1) - - for i in range(len(layers_in_block)): - if i == 0: - in_filters = pre_filters if self.use_head_conv else 1 - else: - in_filters = filters_in_block[i - 1] - - block = BasicBlock( - in_filters, - filters=filters_in_block[i], - num_layer=layers_in_block[i], - stride=1 if i == 0 else 2, - bn_momentum=batchnorm_momentum, - ) - self.add_module("block_{}".format(i), block) - - self.resnet0_dense = torch.nn.Conv2d( - filters_in_block[-1], num_nodes_pooling_layer, 1 - ) - self.resnet0_bn = torch.nn.BatchNorm2d( - num_nodes_pooling_layer, eps=1e-3, momentum=batchnorm_momentum - ) - - self.time_ds_ratio = 8 - - def output_size(self) -> int: - return self.num_nodes_pooling_layer - - def forward( - self, - xs_pad: torch.Tensor, - ilens: torch.Tensor, - prev_states: torch.Tensor = None, - ) -> Tuple[torch.Tensor, torch.Tensor]: - - features = xs_pad - assert ( - features.size(-1) == self.input_size - ), "Dimension of features {} doesn't match the input_size {}.".format( - features.size(-1), self.input_size - ) - features = torch.unsqueeze(features, dim=1) - if self.use_head_conv: - features = self.pre_conv(features) - features = self.pre_conv_bn(features) - features = F.relu(features) - - if self.use_head_maxpool: - features = self.head_maxpool(features) - - resnet_outs, resnet_out_lens = features, ilens - for i in range(len(self.layers_in_block)): - block = self._modules["block_{}".format(i)] - resnet_outs, resnet_out_lens = block(resnet_outs, resnet_out_lens) - - features = self.resnet0_dense(resnet_outs) - features = F.relu(features) - features = self.resnet0_bn(features) - - return features, resnet_out_lens - - -# Note: For training, this implement is not equivalent to tf because of the kernel_regularizer in tf.layers. -# TODO: implement kernel_regularizer in torch with munal loss addition or weigth_decay in the optimizer -class ResNet34_SP_L2Reg(AbsEncoder): - def __init__( - self, - input_size, - use_head_conv=True, - batchnorm_momentum=0.5, - use_head_maxpool=False, - num_nodes_pooling_layer=256, - layers_in_block=(3, 4, 6, 3), - filters_in_block=(32, 64, 128, 256), - tf2torch_tensor_name_prefix_torch="encoder", - tf2torch_tensor_name_prefix_tf="EAND/speech_encoder", - tf_train_steps=720000, - ): - super(ResNet34_SP_L2Reg, self).__init__() - - self.use_head_conv = use_head_conv - self.use_head_maxpool = use_head_maxpool - self.num_nodes_pooling_layer = num_nodes_pooling_layer - self.layers_in_block = layers_in_block - self.filters_in_block = filters_in_block - self.input_size = input_size - self.tf2torch_tensor_name_prefix_torch = tf2torch_tensor_name_prefix_torch - self.tf2torch_tensor_name_prefix_tf = tf2torch_tensor_name_prefix_tf - self.tf_train_steps = tf_train_steps - - pre_filters = filters_in_block[0] - if use_head_conv: - self.pre_conv = torch.nn.Conv2d( - 1, pre_filters, 3, 1, 1, bias=False, padding_mode="zeros" - ) - self.pre_conv_bn = torch.nn.BatchNorm2d( - pre_filters, eps=1e-3, momentum=batchnorm_momentum - ) - - if use_head_maxpool: - self.head_maxpool = torch.nn.MaxPool2d(3, 1, padding=1) - - for i in range(len(layers_in_block)): - if i == 0: - in_filters = pre_filters if self.use_head_conv else 1 - else: - in_filters = filters_in_block[i - 1] - - block = BasicBlock( - in_filters, - filters=filters_in_block[i], - num_layer=layers_in_block[i], - stride=1 if i == 0 else 2, - bn_momentum=batchnorm_momentum, - ) - self.add_module("block_{}".format(i), block) - - self.resnet0_dense = torch.nn.Conv1d( - filters_in_block[-1] * input_size // 8, num_nodes_pooling_layer, 1 - ) - self.resnet0_bn = torch.nn.BatchNorm1d( - num_nodes_pooling_layer, eps=1e-3, momentum=batchnorm_momentum - ) - - self.time_ds_ratio = 8 - - def output_size(self) -> int: - return self.num_nodes_pooling_layer - - def forward( - self, - xs_pad: torch.Tensor, - ilens: torch.Tensor, - prev_states: torch.Tensor = None, - ) -> Tuple[torch.Tensor, torch.Tensor]: - - features = xs_pad - assert ( - features.size(-1) == self.input_size - ), "Dimension of features {} doesn't match the input_size {}.".format( - features.size(-1), self.input_size - ) - features = torch.unsqueeze(features, dim=1) - if self.use_head_conv: - features = self.pre_conv(features) - features = self.pre_conv_bn(features) - features = F.relu(features) - - if self.use_head_maxpool: - features = self.head_maxpool(features) - - resnet_outs, resnet_out_lens = features, ilens - for i in range(len(self.layers_in_block)): - block = self._modules["block_{}".format(i)] - resnet_outs, resnet_out_lens = block(resnet_outs, resnet_out_lens) - - # B, C, T, F - bb, cc, tt, ff = resnet_outs.shape - resnet_outs = torch.reshape(resnet_outs.permute(0, 3, 1, 2), [bb, ff * cc, tt]) - features = self.resnet0_dense(resnet_outs) - features = F.relu(features) - features = self.resnet0_bn(features) - - return features, resnet_out_lens - - def gen_tf2torch_map_dict(self): - tensor_name_prefix_torch = self.tf2torch_tensor_name_prefix_torch - tensor_name_prefix_tf = self.tf2torch_tensor_name_prefix_tf - train_steps = self.tf_train_steps - map_dict_local = { - # torch: conv1d.weight in "out_channel in_channel kernel_size" - # tf : conv1d.weight in "kernel_size in_channel out_channel" - # torch: linear.weight in "out_channel in_channel" - # tf : dense.weight in "in_channel out_channel" - "{}.pre_conv.weight".format(tensor_name_prefix_torch): { - "name": "{}/pre_conv/kernel".format(tensor_name_prefix_tf), - "squeeze": None, - "transpose": (3, 2, 0, 1), - }, - "{}.pre_conv_bn.bias".format(tensor_name_prefix_torch): { - "name": "{}/pre_conv_bn/beta".format(tensor_name_prefix_tf), - "squeeze": None, - "transpose": None, - }, - "{}.pre_conv_bn.weight".format(tensor_name_prefix_torch): { - "name": "{}/pre_conv_bn/gamma".format(tensor_name_prefix_tf), - "squeeze": None, - "transpose": None, - }, - "{}.pre_conv_bn.running_mean".format(tensor_name_prefix_torch): { - "name": "{}/pre_conv_bn/moving_mean".format(tensor_name_prefix_tf), - "squeeze": None, - "transpose": None, - }, - "{}.pre_conv_bn.running_var".format(tensor_name_prefix_torch): { - "name": "{}/pre_conv_bn/moving_variance".format(tensor_name_prefix_tf), - "squeeze": None, - "transpose": None, - }, - "{}.pre_conv_bn.num_batches_tracked".format( - tensor_name_prefix_torch - ): train_steps, - } - for layer_idx in range(3): - map_dict_local.update( - { - "{}.resnet{}_dense.weight".format( - tensor_name_prefix_torch, layer_idx - ): { - "name": "{}/resnet{}_dense/kernel".format( - tensor_name_prefix_tf, layer_idx - ), - "squeeze": None, - "transpose": (2, 1, 0) if layer_idx == 0 else (1, 0), - }, - "{}.resnet{}_dense.bias".format( - tensor_name_prefix_torch, layer_idx - ): { - "name": "{}/resnet{}_dense/bias".format( - tensor_name_prefix_tf, layer_idx - ), - "squeeze": None, - "transpose": None, - }, - "{}.resnet{}_bn.weight".format( - tensor_name_prefix_torch, layer_idx - ): { - "name": "{}/resnet{}_bn/gamma".format( - tensor_name_prefix_tf, layer_idx - ), - "squeeze": None, - "transpose": None, - }, - "{}.resnet{}_bn.bias".format(tensor_name_prefix_torch, layer_idx): { - "name": "{}/resnet{}_bn/beta".format( - tensor_name_prefix_tf, layer_idx - ), - "squeeze": None, - "transpose": None, - }, - "{}.resnet{}_bn.running_mean".format( - tensor_name_prefix_torch, layer_idx - ): { - "name": "{}/resnet{}_bn/moving_mean".format( - tensor_name_prefix_tf, layer_idx - ), - "squeeze": None, - "transpose": None, - }, - "{}.resnet{}_bn.running_var".format( - tensor_name_prefix_torch, layer_idx - ): { - "name": "{}/resnet{}_bn/moving_variance".format( - tensor_name_prefix_tf, layer_idx - ), - "squeeze": None, - "transpose": None, - }, - "{}.resnet{}_bn.num_batches_tracked".format( - tensor_name_prefix_torch, layer_idx - ): train_steps, - } - ) - - for block_idx in range(len(self.layers_in_block)): - for layer_idx in range(self.layers_in_block[block_idx]): - for i in ["1", "2", "_sc"]: - map_dict_local.update( - { - "{}.block_{}.layer_{}.conv{}.weight".format( - tensor_name_prefix_torch, block_idx, layer_idx, i - ): { - "name": "{}/block_{}/layer_{}/conv{}/kernel".format( - tensor_name_prefix_tf, block_idx, layer_idx, i - ), - "squeeze": None, - "transpose": (3, 2, 0, 1), - }, - "{}.block_{}.layer_{}.bn{}.weight".format( - tensor_name_prefix_torch, block_idx, layer_idx, i - ): { - "name": "{}/block_{}/layer_{}/bn{}/gamma".format( - tensor_name_prefix_tf, block_idx, layer_idx, i - ), - "squeeze": None, - "transpose": None, - }, - "{}.block_{}.layer_{}.bn{}.bias".format( - tensor_name_prefix_torch, block_idx, layer_idx, i - ): { - "name": "{}/block_{}/layer_{}/bn{}/beta".format( - tensor_name_prefix_tf, block_idx, layer_idx, i - ), - "squeeze": None, - "transpose": None, - }, - "{}.block_{}.layer_{}.bn{}.running_mean".format( - tensor_name_prefix_torch, block_idx, layer_idx, i - ): { - "name": "{}/block_{}/layer_{}/bn{}/moving_mean".format( - tensor_name_prefix_tf, block_idx, layer_idx, i - ), - "squeeze": None, - "transpose": None, - }, - "{}.block_{}.layer_{}.bn{}.running_var".format( - tensor_name_prefix_torch, block_idx, layer_idx, i - ): { - "name": "{}/block_{}/layer_{}/bn{}/moving_variance".format( - tensor_name_prefix_tf, block_idx, layer_idx, i - ), - "squeeze": None, - "transpose": None, - }, - "{}.block_{}.layer_{}.bn{}.num_batches_tracked".format( - tensor_name_prefix_torch, block_idx, layer_idx, i - ): train_steps, - } - ) - - return map_dict_local - - def convert_tf2torch( - self, - var_dict_tf, - var_dict_torch, - ): - - map_dict = self.gen_tf2torch_map_dict() - - var_dict_torch_update = dict() - for name in sorted(var_dict_torch.keys(), reverse=False): - if name.startswith(self.tf2torch_tensor_name_prefix_torch): - if name in map_dict: - if "num_batches_tracked" not in name: - name_tf = map_dict[name]["name"] - data_tf = var_dict_tf[name_tf] - if map_dict[name]["squeeze"] is not None: - data_tf = np.squeeze( - data_tf, axis=map_dict[name]["squeeze"] - ) - if map_dict[name]["transpose"] is not None: - data_tf = np.transpose(data_tf, map_dict[name]["transpose"]) - data_tf = ( - torch.from_numpy(data_tf).type(torch.float32).to("cpu") - ) - assert ( - var_dict_torch[name].size() == data_tf.size() - ), "{}, {}, {} != {}".format( - name, name_tf, var_dict_torch[name].size(), data_tf.size() - ) - var_dict_torch_update[name] = data_tf - logging.info( - "torch tensor: {}, {}, loading from tf tensor: {}, {}".format( - name, - data_tf.size(), - name_tf, - var_dict_tf[name_tf].shape, - ) - ) - else: - var_dict_torch_update[name] = ( - torch.Tensor(map_dict[name]).type(torch.int64).to("cpu") - ) - logging.info( - "torch tensor: {}, manually assigning to: {}".format( - name, map_dict[name] - ) - ) - else: - logging.warning("{} is missed from tf checkpoint".format(name)) - - return var_dict_torch_update - - -class ResNet34Diar(ResNet34): - def __init__( - self, - input_size, - embedding_node="resnet1_dense", - use_head_conv=True, - batchnorm_momentum=0.5, - use_head_maxpool=False, - num_nodes_pooling_layer=256, - layers_in_block=(3, 4, 6, 3), - filters_in_block=(32, 64, 128, 256), - num_nodes_resnet1=256, - num_nodes_last_layer=256, - pooling_type="window_shift", - pool_size=20, - stride=1, - tf2torch_tensor_name_prefix_torch="encoder", - tf2torch_tensor_name_prefix_tf="seq2seq/speech_encoder", - ): - """ - Author: Speech Lab, Alibaba Group, China - SOND: Speaker Overlap-aware Neural Diarization for Multi-party Meeting Analysis - https://arxiv.org/abs/2211.10243 - """ - - super(ResNet34Diar, self).__init__( - input_size, - use_head_conv=use_head_conv, - batchnorm_momentum=batchnorm_momentum, - use_head_maxpool=use_head_maxpool, - num_nodes_pooling_layer=num_nodes_pooling_layer, - layers_in_block=layers_in_block, - filters_in_block=filters_in_block, - ) - - self.embedding_node = embedding_node - self.num_nodes_resnet1 = num_nodes_resnet1 - self.num_nodes_last_layer = num_nodes_last_layer - self.pooling_type = pooling_type - self.pool_size = pool_size - self.stride = stride - self.tf2torch_tensor_name_prefix_torch = tf2torch_tensor_name_prefix_torch - self.tf2torch_tensor_name_prefix_tf = tf2torch_tensor_name_prefix_tf - - self.resnet1_dense = torch.nn.Linear( - num_nodes_pooling_layer * 2, num_nodes_resnet1 - ) - self.resnet1_bn = torch.nn.BatchNorm1d( - num_nodes_resnet1, eps=1e-3, momentum=batchnorm_momentum - ) - - self.resnet2_dense = torch.nn.Linear(num_nodes_resnet1, num_nodes_last_layer) - self.resnet2_bn = torch.nn.BatchNorm1d( - num_nodes_last_layer, eps=1e-3, momentum=batchnorm_momentum - ) - - def output_size(self) -> int: - if self.embedding_node.startswith("resnet1"): - return self.num_nodes_resnet1 - elif self.embedding_node.startswith("resnet2"): - return self.num_nodes_last_layer - - return self.num_nodes_pooling_layer - - def forward( - self, - xs_pad: torch.Tensor, - ilens: torch.Tensor, - prev_states: torch.Tensor = None, - ) -> Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]: - - endpoints = OrderedDict() - res_out, ilens = super().forward(xs_pad, ilens) - endpoints["resnet0_bn"] = res_out - if self.pooling_type == "frame_gsp": - features = statistic_pooling(res_out, ilens, (3,)) - else: - features, ilens = windowed_statistic_pooling( - res_out, ilens, (2, 3), self.pool_size, self.stride - ) - features = features.transpose(1, 2) - endpoints["pooling"] = features - - features = self.resnet1_dense(features) - endpoints["resnet1_dense"] = features - features = F.relu(features) - endpoints["resnet1_relu"] = features - features = self.resnet1_bn(features.transpose(1, 2)).transpose(1, 2) - endpoints["resnet1_bn"] = features - - features = self.resnet2_dense(features) - endpoints["resnet2_dense"] = features - features = F.relu(features) - endpoints["resnet2_relu"] = features - features = self.resnet2_bn(features.transpose(1, 2)).transpose(1, 2) - endpoints["resnet2_bn"] = features - - return endpoints[self.embedding_node], ilens, None - - def gen_tf2torch_map_dict(self): - tensor_name_prefix_torch = self.tf2torch_tensor_name_prefix_torch - tensor_name_prefix_tf = self.tf2torch_tensor_name_prefix_tf - train_steps = 300000 - map_dict_local = { - # torch: conv1d.weight in "out_channel in_channel kernel_size" - # tf : conv1d.weight in "kernel_size in_channel out_channel" - # torch: linear.weight in "out_channel in_channel" - # tf : dense.weight in "in_channel out_channel" - "{}.pre_conv.weight".format(tensor_name_prefix_torch): { - "name": "{}/pre_conv/kernel".format(tensor_name_prefix_tf), - "squeeze": None, - "transpose": (3, 2, 0, 1), - }, - "{}.pre_conv_bn.bias".format(tensor_name_prefix_torch): { - "name": "{}/pre_conv_bn/beta".format(tensor_name_prefix_tf), - "squeeze": None, - "transpose": None, - }, - "{}.pre_conv_bn.weight".format(tensor_name_prefix_torch): { - "name": "{}/pre_conv_bn/gamma".format(tensor_name_prefix_tf), - "squeeze": None, - "transpose": None, - }, - "{}.pre_conv_bn.running_mean".format(tensor_name_prefix_torch): { - "name": "{}/pre_conv_bn/moving_mean".format(tensor_name_prefix_tf), - "squeeze": None, - "transpose": None, - }, - "{}.pre_conv_bn.running_var".format(tensor_name_prefix_torch): { - "name": "{}/pre_conv_bn/moving_variance".format(tensor_name_prefix_tf), - "squeeze": None, - "transpose": None, - }, - "{}.pre_conv_bn.num_batches_tracked".format( - tensor_name_prefix_torch - ): train_steps, - } - for layer_idx in range(3): - map_dict_local.update( - { - "{}.resnet{}_dense.weight".format( - tensor_name_prefix_torch, layer_idx - ): { - "name": "{}/resnet{}_dense/kernel".format( - tensor_name_prefix_tf, layer_idx - ), - "squeeze": None, - "transpose": (3, 2, 0, 1) if layer_idx == 0 else (1, 0), - }, - "{}.resnet{}_dense.bias".format( - tensor_name_prefix_torch, layer_idx - ): { - "name": "{}/resnet{}_dense/bias".format( - tensor_name_prefix_tf, layer_idx - ), - "squeeze": None, - "transpose": None, - }, - "{}.resnet{}_bn.weight".format( - tensor_name_prefix_torch, layer_idx - ): { - "name": "{}/resnet{}_bn/gamma".format( - tensor_name_prefix_tf, layer_idx - ), - "squeeze": None, - "transpose": None, - }, - "{}.resnet{}_bn.bias".format(tensor_name_prefix_torch, layer_idx): { - "name": "{}/resnet{}_bn/beta".format( - tensor_name_prefix_tf, layer_idx - ), - "squeeze": None, - "transpose": None, - }, - "{}.resnet{}_bn.running_mean".format( - tensor_name_prefix_torch, layer_idx - ): { - "name": "{}/resnet{}_bn/moving_mean".format( - tensor_name_prefix_tf, layer_idx - ), - "squeeze": None, - "transpose": None, - }, - "{}.resnet{}_bn.running_var".format( - tensor_name_prefix_torch, layer_idx - ): { - "name": "{}/resnet{}_bn/moving_variance".format( - tensor_name_prefix_tf, layer_idx - ), - "squeeze": None, - "transpose": None, - }, - "{}.resnet{}_bn.num_batches_tracked".format( - tensor_name_prefix_torch, layer_idx - ): train_steps, - } - ) - - for block_idx in range(len(self.layers_in_block)): - for layer_idx in range(self.layers_in_block[block_idx]): - for i in ["1", "2", "_sc"]: - map_dict_local.update( - { - "{}.block_{}.layer_{}.conv{}.weight".format( - tensor_name_prefix_torch, block_idx, layer_idx, i - ): { - "name": "{}/block_{}/layer_{}/conv{}/kernel".format( - tensor_name_prefix_tf, block_idx, layer_idx, i - ), - "squeeze": None, - "transpose": (3, 2, 0, 1), - }, - "{}.block_{}.layer_{}.bn{}.weight".format( - tensor_name_prefix_torch, block_idx, layer_idx, i - ): { - "name": "{}/block_{}/layer_{}/bn{}/gamma".format( - tensor_name_prefix_tf, block_idx, layer_idx, i - ), - "squeeze": None, - "transpose": None, - }, - "{}.block_{}.layer_{}.bn{}.bias".format( - tensor_name_prefix_torch, block_idx, layer_idx, i - ): { - "name": "{}/block_{}/layer_{}/bn{}/beta".format( - tensor_name_prefix_tf, block_idx, layer_idx, i - ), - "squeeze": None, - "transpose": None, - }, - "{}.block_{}.layer_{}.bn{}.running_mean".format( - tensor_name_prefix_torch, block_idx, layer_idx, i - ): { - "name": "{}/block_{}/layer_{}/bn{}/moving_mean".format( - tensor_name_prefix_tf, block_idx, layer_idx, i - ), - "squeeze": None, - "transpose": None, - }, - "{}.block_{}.layer_{}.bn{}.running_var".format( - tensor_name_prefix_torch, block_idx, layer_idx, i - ): { - "name": "{}/block_{}/layer_{}/bn{}/moving_variance".format( - tensor_name_prefix_tf, block_idx, layer_idx, i - ), - "squeeze": None, - "transpose": None, - }, - "{}.block_{}.layer_{}.bn{}.num_batches_tracked".format( - tensor_name_prefix_torch, block_idx, layer_idx, i - ): train_steps, - } - ) - - return map_dict_local - - def convert_tf2torch( - self, - var_dict_tf, - var_dict_torch, - ): - - map_dict = self.gen_tf2torch_map_dict() - - var_dict_torch_update = dict() - for name in sorted(var_dict_torch.keys(), reverse=False): - if name.startswith(self.tf2torch_tensor_name_prefix_torch): - if name in map_dict: - if "num_batches_tracked" not in name: - name_tf = map_dict[name]["name"] - data_tf = var_dict_tf[name_tf] - if map_dict[name]["squeeze"] is not None: - data_tf = np.squeeze( - data_tf, axis=map_dict[name]["squeeze"] - ) - if map_dict[name]["transpose"] is not None: - data_tf = np.transpose(data_tf, map_dict[name]["transpose"]) - data_tf = ( - torch.from_numpy(data_tf).type(torch.float32).to("cpu") - ) - assert ( - var_dict_torch[name].size() == data_tf.size() - ), "{}, {}, {} != {}".format( - name, name_tf, var_dict_torch[name].size(), data_tf.size() - ) - var_dict_torch_update[name] = data_tf - logging.info( - "torch tensor: {}, {}, loading from tf tensor: {}, {}".format( - name, - data_tf.size(), - name_tf, - var_dict_tf[name_tf].shape, - ) - ) - else: - var_dict_torch_update[name] = ( - torch.Tensor(map_dict[name]).type(torch.int64).to("cpu") - ) - logging.info( - "torch tensor: {}, manually assigning to: {}".format( - name, map_dict[name] - ) - ) - else: - logging.warning("{} is missed from tf checkpoint".format(name)) - - return var_dict_torch_update - - -class ResNet34SpL2RegDiar(ResNet34_SP_L2Reg): - def __init__( - self, - input_size, - embedding_node="resnet1_dense", - use_head_conv=True, - batchnorm_momentum=0.5, - use_head_maxpool=False, - num_nodes_pooling_layer=256, - layers_in_block=(3, 4, 6, 3), - filters_in_block=(32, 64, 128, 256), - num_nodes_resnet1=256, - num_nodes_last_layer=256, - pooling_type="window_shift", - pool_size=20, - stride=1, - tf2torch_tensor_name_prefix_torch="encoder", - tf2torch_tensor_name_prefix_tf="seq2seq/speech_encoder", - ): - """ - Author: Speech Lab, Alibaba Group, China - TOLD: A Novel Two-Stage Overlap-Aware Framework for Speaker Diarization - https://arxiv.org/abs/2303.05397 - """ - - super(ResNet34SpL2RegDiar, self).__init__( - input_size, - use_head_conv=use_head_conv, - batchnorm_momentum=batchnorm_momentum, - use_head_maxpool=use_head_maxpool, - num_nodes_pooling_layer=num_nodes_pooling_layer, - layers_in_block=layers_in_block, - filters_in_block=filters_in_block, - ) - - self.embedding_node = embedding_node - self.num_nodes_resnet1 = num_nodes_resnet1 - self.num_nodes_last_layer = num_nodes_last_layer - self.pooling_type = pooling_type - self.pool_size = pool_size - self.stride = stride - self.tf2torch_tensor_name_prefix_torch = tf2torch_tensor_name_prefix_torch - self.tf2torch_tensor_name_prefix_tf = tf2torch_tensor_name_prefix_tf - - self.resnet1_dense = torch.nn.Linear( - num_nodes_pooling_layer * 2, num_nodes_resnet1 - ) - self.resnet1_bn = torch.nn.BatchNorm1d( - num_nodes_resnet1, eps=1e-3, momentum=batchnorm_momentum - ) - - self.resnet2_dense = torch.nn.Linear(num_nodes_resnet1, num_nodes_last_layer) - self.resnet2_bn = torch.nn.BatchNorm1d( - num_nodes_last_layer, eps=1e-3, momentum=batchnorm_momentum - ) - - def output_size(self) -> int: - if self.embedding_node.startswith("resnet1"): - return self.num_nodes_resnet1 - elif self.embedding_node.startswith("resnet2"): - return self.num_nodes_last_layer - - return self.num_nodes_pooling_layer - - def forward( - self, - xs_pad: torch.Tensor, - ilens: torch.Tensor, - prev_states: torch.Tensor = None, - ) -> Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]: - - endpoints = OrderedDict() - res_out, ilens = super().forward(xs_pad, ilens) - endpoints["resnet0_bn"] = res_out - if self.pooling_type == "frame_gsp": - features = statistic_pooling(res_out, ilens, (2,)) - else: - features, ilens = windowed_statistic_pooling( - res_out, ilens, (2,), self.pool_size, self.stride - ) - features = features.transpose(1, 2) - endpoints["pooling"] = features - - features = self.resnet1_dense(features) - endpoints["resnet1_dense"] = features - features = F.relu(features) - endpoints["resnet1_relu"] = features - features = self.resnet1_bn(features.transpose(1, 2)).transpose(1, 2) - endpoints["resnet1_bn"] = features - - features = self.resnet2_dense(features) - endpoints["resnet2_dense"] = features - features = F.relu(features) - endpoints["resnet2_relu"] = features - features = self.resnet2_bn(features.transpose(1, 2)).transpose(1, 2) - endpoints["resnet2_bn"] = features - - return endpoints[self.embedding_node], ilens, None - - def gen_tf2torch_map_dict(self): - tensor_name_prefix_torch = self.tf2torch_tensor_name_prefix_torch - tensor_name_prefix_tf = self.tf2torch_tensor_name_prefix_tf - train_steps = 720000 - map_dict_local = { - # torch: conv1d.weight in "out_channel in_channel kernel_size" - # tf : conv1d.weight in "kernel_size in_channel out_channel" - # torch: linear.weight in "out_channel in_channel" - # tf : dense.weight in "in_channel out_channel" - "{}.pre_conv.weight".format(tensor_name_prefix_torch): { - "name": "{}/pre_conv/kernel".format(tensor_name_prefix_tf), - "squeeze": None, - "transpose": (3, 2, 0, 1), - }, - "{}.pre_conv_bn.bias".format(tensor_name_prefix_torch): { - "name": "{}/pre_conv_bn/beta".format(tensor_name_prefix_tf), - "squeeze": None, - "transpose": None, - }, - "{}.pre_conv_bn.weight".format(tensor_name_prefix_torch): { - "name": "{}/pre_conv_bn/gamma".format(tensor_name_prefix_tf), - "squeeze": None, - "transpose": None, - }, - "{}.pre_conv_bn.running_mean".format(tensor_name_prefix_torch): { - "name": "{}/pre_conv_bn/moving_mean".format(tensor_name_prefix_tf), - "squeeze": None, - "transpose": None, - }, - "{}.pre_conv_bn.running_var".format(tensor_name_prefix_torch): { - "name": "{}/pre_conv_bn/moving_variance".format(tensor_name_prefix_tf), - "squeeze": None, - "transpose": None, - }, - "{}.pre_conv_bn.num_batches_tracked".format( - tensor_name_prefix_torch - ): train_steps, - } - for layer_idx in range(3): - map_dict_local.update( - { - "{}.resnet{}_dense.weight".format( - tensor_name_prefix_torch, layer_idx - ): { - "name": "{}/resnet{}_dense/kernel".format( - tensor_name_prefix_tf, layer_idx - ), - "squeeze": None, - "transpose": (2, 1, 0) if layer_idx == 0 else (1, 0), - }, - "{}.resnet{}_dense.bias".format( - tensor_name_prefix_torch, layer_idx - ): { - "name": "{}/resnet{}_dense/bias".format( - tensor_name_prefix_tf, layer_idx - ), - "squeeze": None, - "transpose": None, - }, - "{}.resnet{}_bn.weight".format( - tensor_name_prefix_torch, layer_idx - ): { - "name": "{}/resnet{}_bn/gamma".format( - tensor_name_prefix_tf, layer_idx - ), - "squeeze": None, - "transpose": None, - }, - "{}.resnet{}_bn.bias".format(tensor_name_prefix_torch, layer_idx): { - "name": "{}/resnet{}_bn/beta".format( - tensor_name_prefix_tf, layer_idx - ), - "squeeze": None, - "transpose": None, - }, - "{}.resnet{}_bn.running_mean".format( - tensor_name_prefix_torch, layer_idx - ): { - "name": "{}/resnet{}_bn/moving_mean".format( - tensor_name_prefix_tf, layer_idx - ), - "squeeze": None, - "transpose": None, - }, - "{}.resnet{}_bn.running_var".format( - tensor_name_prefix_torch, layer_idx - ): { - "name": "{}/resnet{}_bn/moving_variance".format( - tensor_name_prefix_tf, layer_idx - ), - "squeeze": None, - "transpose": None, - }, - "{}.resnet{}_bn.num_batches_tracked".format( - tensor_name_prefix_torch, layer_idx - ): train_steps, - } - ) - - for block_idx in range(len(self.layers_in_block)): - for layer_idx in range(self.layers_in_block[block_idx]): - for i in ["1", "2", "_sc"]: - map_dict_local.update( - { - "{}.block_{}.layer_{}.conv{}.weight".format( - tensor_name_prefix_torch, block_idx, layer_idx, i - ): { - "name": "{}/block_{}/layer_{}/conv{}/kernel".format( - tensor_name_prefix_tf, block_idx, layer_idx, i - ), - "squeeze": None, - "transpose": (3, 2, 0, 1), - }, - "{}.block_{}.layer_{}.bn{}.weight".format( - tensor_name_prefix_torch, block_idx, layer_idx, i - ): { - "name": "{}/block_{}/layer_{}/bn{}/gamma".format( - tensor_name_prefix_tf, block_idx, layer_idx, i - ), - "squeeze": None, - "transpose": None, - }, - "{}.block_{}.layer_{}.bn{}.bias".format( - tensor_name_prefix_torch, block_idx, layer_idx, i - ): { - "name": "{}/block_{}/layer_{}/bn{}/beta".format( - tensor_name_prefix_tf, block_idx, layer_idx, i - ), - "squeeze": None, - "transpose": None, - }, - "{}.block_{}.layer_{}.bn{}.running_mean".format( - tensor_name_prefix_torch, block_idx, layer_idx, i - ): { - "name": "{}/block_{}/layer_{}/bn{}/moving_mean".format( - tensor_name_prefix_tf, block_idx, layer_idx, i - ), - "squeeze": None, - "transpose": None, - }, - "{}.block_{}.layer_{}.bn{}.running_var".format( - tensor_name_prefix_torch, block_idx, layer_idx, i - ): { - "name": "{}/block_{}/layer_{}/bn{}/moving_variance".format( - tensor_name_prefix_tf, block_idx, layer_idx, i - ), - "squeeze": None, - "transpose": None, - }, - "{}.block_{}.layer_{}.bn{}.num_batches_tracked".format( - tensor_name_prefix_torch, block_idx, layer_idx, i - ): train_steps, - } - ) - - return map_dict_local - - def convert_tf2torch( - self, - var_dict_tf, - var_dict_torch, - ): - - map_dict = self.gen_tf2torch_map_dict() - - var_dict_torch_update = dict() - for name in sorted(var_dict_torch.keys(), reverse=False): - if name.startswith(self.tf2torch_tensor_name_prefix_torch): - if name in map_dict: - if "num_batches_tracked" not in name: - name_tf = map_dict[name]["name"] - data_tf = var_dict_tf[name_tf] - if map_dict[name]["squeeze"] is not None: - data_tf = np.squeeze( - data_tf, axis=map_dict[name]["squeeze"] - ) - if map_dict[name]["transpose"] is not None: - data_tf = np.transpose(data_tf, map_dict[name]["transpose"]) - data_tf = ( - torch.from_numpy(data_tf).type(torch.float32).to("cpu") - ) - assert ( - var_dict_torch[name].size() == data_tf.size() - ), "{}, {}, {} != {}".format( - name, name_tf, var_dict_torch[name].size(), data_tf.size() - ) - var_dict_torch_update[name] = data_tf - logging.info( - "torch tensor: {}, {}, loading from tf tensor: {}, {}".format( - name, - data_tf.size(), - name_tf, - var_dict_tf[name_tf].shape, - ) - ) - else: - var_dict_torch_update[name] = ( - torch.from_numpy(np.array(map_dict[name])) - .type(torch.int64) - .to("cpu") - ) - logging.info( - "torch tensor: {}, manually assigning to: {}".format( - name, map_dict[name] - ) - ) - else: - logging.warning("{} is missed from tf checkpoint".format(name)) - - return var_dict_torch_update diff --git a/funasr_detach/models/sond/encoder/self_attention_encoder.py b/funasr_detach/models/sond/encoder/self_attention_encoder.py deleted file mode 100644 index cdbbdfb1388038bb83da2e43b3a1bcbd29f662fc..0000000000000000000000000000000000000000 --- a/funasr_detach/models/sond/encoder/self_attention_encoder.py +++ /dev/null @@ -1,560 +0,0 @@ -from typing import List -from typing import Optional -from typing import Sequence -from typing import Tuple -from typing import Union -import logging -import torch -import torch.nn as nn -from funasr_detach.models.scama.chunk_utilis import overlap_chunk -import numpy as np -from funasr_detach.models.transformer.utils.nets_utils import make_pad_mask -from funasr_detach.models.sond.attention import MultiHeadSelfAttention -from funasr_detach.models.transformer.embedding import SinusoidalPositionEncoder -from funasr_detach.models.transformer.layer_norm import LayerNorm -from funasr_detach.models.transformer.utils.multi_layer_conv import Conv1dLinear -from funasr_detach.models.transformer.utils.multi_layer_conv import MultiLayeredConv1d -from funasr_detach.models.transformer.positionwise_feed_forward import ( - PositionwiseFeedForward, # noqa: H301 -) -from funasr_detach.models.transformer.utils.repeat import repeat -from funasr_detach.models.transformer.utils.subsampling import Conv2dSubsampling -from funasr_detach.models.transformer.utils.subsampling import Conv2dSubsampling2 -from funasr_detach.models.transformer.utils.subsampling import Conv2dSubsampling6 -from funasr_detach.models.transformer.utils.subsampling import Conv2dSubsampling8 -from funasr_detach.models.transformer.utils.subsampling import TooShortUttError -from funasr_detach.models.transformer.utils.subsampling import check_short_utt -from funasr_detach.models.ctc import CTC -from funasr_detach.models.encoder.abs_encoder import AbsEncoder - - -class EncoderLayer(nn.Module): - def __init__( - self, - in_size, - size, - self_attn, - feed_forward, - dropout_rate, - normalize_before=True, - concat_after=False, - stochastic_depth_rate=0.0, - ): - """Construct an EncoderLayer object.""" - super(EncoderLayer, self).__init__() - self.self_attn = self_attn - self.feed_forward = feed_forward - self.norm1 = LayerNorm(in_size) - self.norm2 = LayerNorm(size) - self.dropout = nn.Dropout(dropout_rate) - self.in_size = in_size - self.size = size - self.normalize_before = normalize_before - self.concat_after = concat_after - if self.concat_after: - self.concat_linear = nn.Linear(size + size, size) - self.stochastic_depth_rate = stochastic_depth_rate - self.dropout_rate = dropout_rate - - def forward(self, x, mask, cache=None, mask_att_chunk_encoder=None): - """Compute encoded features. - - Args: - x_input (torch.Tensor): Input tensor (#batch, time, size). - mask (torch.Tensor): Mask tensor for the input (#batch, time). - cache (torch.Tensor): Cache tensor of the input (#batch, time - 1, size). - - Returns: - torch.Tensor: Output tensor (#batch, time, size). - torch.Tensor: Mask tensor (#batch, time). - - """ - skip_layer = False - # with stochastic depth, residual connection `x + f(x)` becomes - # `x <- x + 1 / (1 - p) * f(x)` at training time. - stoch_layer_coeff = 1.0 - if self.training and self.stochastic_depth_rate > 0: - skip_layer = torch.rand(1).item() < self.stochastic_depth_rate - stoch_layer_coeff = 1.0 / (1 - self.stochastic_depth_rate) - - if skip_layer: - if cache is not None: - x = torch.cat([cache, x], dim=1) - return x, mask - - residual = x - if self.normalize_before: - x = self.norm1(x) - - if self.concat_after: - x_concat = torch.cat( - ( - x, - self.self_attn( - x, mask, mask_att_chunk_encoder=mask_att_chunk_encoder - ), - ), - dim=-1, - ) - if self.in_size == self.size: - x = residual + stoch_layer_coeff * self.concat_linear(x_concat) - else: - x = stoch_layer_coeff * self.concat_linear(x_concat) - else: - if self.in_size == self.size: - x = residual + stoch_layer_coeff * self.dropout( - self.self_attn( - x, mask, mask_att_chunk_encoder=mask_att_chunk_encoder - ) - ) - else: - x = stoch_layer_coeff * self.dropout( - self.self_attn( - x, mask, mask_att_chunk_encoder=mask_att_chunk_encoder - ) - ) - if not self.normalize_before: - x = self.norm1(x) - - residual = x - if self.normalize_before: - x = self.norm2(x) - x = residual + stoch_layer_coeff * self.dropout(self.feed_forward(x)) - if not self.normalize_before: - x = self.norm2(x) - - return x, mask, cache, mask_att_chunk_encoder - - -class SelfAttentionEncoder(AbsEncoder): - """ - Author: Speech Lab of DAMO Academy, Alibaba Group - Self attention encoder in OpenNMT framework - """ - - def __init__( - self, - input_size: int, - output_size: int = 256, - attention_heads: int = 4, - linear_units: int = 2048, - num_blocks: int = 6, - dropout_rate: float = 0.1, - positional_dropout_rate: float = 0.1, - attention_dropout_rate: float = 0.0, - input_layer: Optional[str] = "conv2d", - pos_enc_class=SinusoidalPositionEncoder, - normalize_before: bool = True, - concat_after: bool = False, - positionwise_layer_type: str = "linear", - positionwise_conv_kernel_size: int = 1, - padding_idx: int = -1, - interctc_layer_idx: List[int] = [], - interctc_use_conditioning: bool = False, - tf2torch_tensor_name_prefix_torch: str = "encoder", - tf2torch_tensor_name_prefix_tf: str = "seq2seq/encoder", - out_units=None, - ): - super().__init__() - self._output_size = output_size - - if input_layer == "linear": - self.embed = torch.nn.Sequential( - torch.nn.Linear(input_size, output_size), - torch.nn.LayerNorm(output_size), - torch.nn.Dropout(dropout_rate), - torch.nn.ReLU(), - pos_enc_class(output_size, positional_dropout_rate), - ) - elif input_layer == "conv2d": - self.embed = Conv2dSubsampling(input_size, output_size, dropout_rate) - elif input_layer == "conv2d2": - self.embed = Conv2dSubsampling2(input_size, output_size, dropout_rate) - elif input_layer == "conv2d6": - self.embed = Conv2dSubsampling6(input_size, output_size, dropout_rate) - elif input_layer == "conv2d8": - self.embed = Conv2dSubsampling8(input_size, output_size, dropout_rate) - elif input_layer == "embed": - self.embed = torch.nn.Sequential( - torch.nn.Embedding(input_size, output_size, padding_idx=padding_idx), - SinusoidalPositionEncoder(), - ) - elif input_layer is None: - if input_size == output_size: - self.embed = None - else: - self.embed = torch.nn.Linear(input_size, output_size) - elif input_layer == "pe": - self.embed = SinusoidalPositionEncoder() - elif input_layer == "null": - self.embed = None - else: - raise ValueError("unknown input_layer: " + input_layer) - self.normalize_before = normalize_before - if positionwise_layer_type == "linear": - positionwise_layer = PositionwiseFeedForward - positionwise_layer_args = ( - output_size, - linear_units, - dropout_rate, - ) - elif positionwise_layer_type == "conv1d": - positionwise_layer = MultiLayeredConv1d - positionwise_layer_args = ( - output_size, - linear_units, - positionwise_conv_kernel_size, - dropout_rate, - ) - elif positionwise_layer_type == "conv1d-linear": - positionwise_layer = Conv1dLinear - positionwise_layer_args = ( - output_size, - linear_units, - positionwise_conv_kernel_size, - dropout_rate, - ) - else: - raise NotImplementedError("Support only linear or conv1d.") - - self.encoders = repeat( - num_blocks, - lambda lnum: ( - EncoderLayer( - output_size, - output_size, - MultiHeadSelfAttention( - attention_heads, - output_size, - output_size, - attention_dropout_rate, - ), - positionwise_layer(*positionwise_layer_args), - dropout_rate, - normalize_before, - concat_after, - ) - if lnum > 0 - else EncoderLayer( - input_size, - output_size, - MultiHeadSelfAttention( - attention_heads, - ( - input_size - if input_layer == "pe" or input_layer == "null" - else output_size - ), - output_size, - attention_dropout_rate, - ), - positionwise_layer(*positionwise_layer_args), - dropout_rate, - normalize_before, - concat_after, - ) - ), - ) - if self.normalize_before: - self.after_norm = LayerNorm(output_size) - - self.interctc_layer_idx = interctc_layer_idx - if len(interctc_layer_idx) > 0: - assert 0 < min(interctc_layer_idx) and max(interctc_layer_idx) < num_blocks - self.interctc_use_conditioning = interctc_use_conditioning - self.conditioning_layer = None - self.dropout = nn.Dropout(dropout_rate) - self.tf2torch_tensor_name_prefix_torch = tf2torch_tensor_name_prefix_torch - self.tf2torch_tensor_name_prefix_tf = tf2torch_tensor_name_prefix_tf - self.out_units = out_units - if out_units is not None: - self.output_linear = nn.Linear(output_size, out_units) - - def output_size(self) -> int: - return self._output_size - - def forward( - self, - xs_pad: torch.Tensor, - ilens: torch.Tensor, - prev_states: torch.Tensor = None, - ctc: CTC = None, - ) -> Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]: - """Embed positions in tensor. - - Args: - xs_pad: input tensor (B, L, D) - ilens: input length (B) - prev_states: Not to be used now. - Returns: - position embedded tensor and mask - """ - masks = (~make_pad_mask(ilens)[:, None, :]).to(xs_pad.device) - xs_pad = xs_pad * self.output_size() ** 0.5 - if self.embed is None: - xs_pad = xs_pad - elif ( - isinstance(self.embed, Conv2dSubsampling) - or isinstance(self.embed, Conv2dSubsampling2) - or isinstance(self.embed, Conv2dSubsampling6) - or isinstance(self.embed, Conv2dSubsampling8) - ): - short_status, limit_size = check_short_utt(self.embed, xs_pad.size(1)) - if short_status: - raise TooShortUttError( - f"has {xs_pad.size(1)} frames and is too short for subsampling " - + f"(it needs more than {limit_size} frames), return empty results", - xs_pad.size(1), - limit_size, - ) - xs_pad, masks = self.embed(xs_pad, masks) - else: - xs_pad = self.embed(xs_pad) - - xs_pad = self.dropout(xs_pad) - # encoder_outs = self.encoders0(xs_pad, masks) - # xs_pad, masks = encoder_outs[0], encoder_outs[1] - intermediate_outs = [] - if len(self.interctc_layer_idx) == 0: - encoder_outs = self.encoders(xs_pad, masks) - xs_pad, masks = encoder_outs[0], encoder_outs[1] - else: - for layer_idx, encoder_layer in enumerate(self.encoders): - encoder_outs = encoder_layer(xs_pad, masks) - xs_pad, masks = encoder_outs[0], encoder_outs[1] - - if layer_idx + 1 in self.interctc_layer_idx: - encoder_out = xs_pad - - # intermediate outputs are also normalized - if self.normalize_before: - encoder_out = self.after_norm(encoder_out) - - intermediate_outs.append((layer_idx + 1, encoder_out)) - - if self.interctc_use_conditioning: - ctc_out = ctc.softmax(encoder_out) - xs_pad = xs_pad + self.conditioning_layer(ctc_out) - - if self.normalize_before: - xs_pad = self.after_norm(xs_pad) - - if self.out_units is not None: - xs_pad = self.output_linear(xs_pad) - olens = masks.squeeze(1).sum(1) - if len(intermediate_outs) > 0: - return (xs_pad, intermediate_outs), olens, None - return xs_pad, olens, None - - def gen_tf2torch_map_dict(self): - tensor_name_prefix_torch = self.tf2torch_tensor_name_prefix_torch - tensor_name_prefix_tf = self.tf2torch_tensor_name_prefix_tf - map_dict_local = { - # cicd - # torch: conv1d.weight in "out_channel in_channel kernel_size" - # tf : conv1d.weight in "kernel_size in_channel out_channel" - # torch: linear.weight in "out_channel in_channel" - # tf : dense.weight in "in_channel out_channel" - "{}.encoders.layeridx.norm1.weight".format(tensor_name_prefix_torch): { - "name": "{}/layer_layeridx/multi_head/LayerNorm/gamma".format( - tensor_name_prefix_tf - ), - "squeeze": None, - "transpose": None, - }, # (256,),(256,) - "{}.encoders.layeridx.norm1.bias".format(tensor_name_prefix_torch): { - "name": "{}/layer_layeridx/multi_head/LayerNorm/beta".format( - tensor_name_prefix_tf - ), - "squeeze": None, - "transpose": None, - }, # (256,),(256,) - "{}.encoders.layeridx.self_attn.linear_q_k_v.weight".format( - tensor_name_prefix_torch - ): { - "name": "{}/layer_layeridx/multi_head/conv1d/kernel".format( - tensor_name_prefix_tf - ), - "squeeze": 0, - "transpose": (1, 0), - }, # (768,256),(1,256,768) - "{}.encoders.layeridx.self_attn.linear_q_k_v.bias".format( - tensor_name_prefix_torch - ): { - "name": "{}/layer_layeridx/multi_head/conv1d/bias".format( - tensor_name_prefix_tf - ), - "squeeze": None, - "transpose": None, - }, # (768,),(768,) - "{}.encoders.layeridx.self_attn.linear_out.weight".format( - tensor_name_prefix_torch - ): { - "name": "{}/layer_layeridx/multi_head/conv1d_1/kernel".format( - tensor_name_prefix_tf - ), - "squeeze": 0, - "transpose": (1, 0), - }, # (256,256),(1,256,256) - "{}.encoders.layeridx.self_attn.linear_out.bias".format( - tensor_name_prefix_torch - ): { - "name": "{}/layer_layeridx/multi_head/conv1d_1/bias".format( - tensor_name_prefix_tf - ), - "squeeze": None, - "transpose": None, - }, # (256,),(256,) - # ffn - "{}.encoders.layeridx.norm2.weight".format(tensor_name_prefix_torch): { - "name": "{}/layer_layeridx/ffn/LayerNorm/gamma".format( - tensor_name_prefix_tf - ), - "squeeze": None, - "transpose": None, - }, # (256,),(256,) - "{}.encoders.layeridx.norm2.bias".format(tensor_name_prefix_torch): { - "name": "{}/layer_layeridx/ffn/LayerNorm/beta".format( - tensor_name_prefix_tf - ), - "squeeze": None, - "transpose": None, - }, # (256,),(256,) - "{}.encoders.layeridx.feed_forward.w_1.weight".format( - tensor_name_prefix_torch - ): { - "name": "{}/layer_layeridx/ffn/conv1d/kernel".format( - tensor_name_prefix_tf - ), - "squeeze": 0, - "transpose": (1, 0), - }, # (1024,256),(1,256,1024) - "{}.encoders.layeridx.feed_forward.w_1.bias".format( - tensor_name_prefix_torch - ): { - "name": "{}/layer_layeridx/ffn/conv1d/bias".format( - tensor_name_prefix_tf - ), - "squeeze": None, - "transpose": None, - }, # (1024,),(1024,) - "{}.encoders.layeridx.feed_forward.w_2.weight".format( - tensor_name_prefix_torch - ): { - "name": "{}/layer_layeridx/ffn/conv1d_1/kernel".format( - tensor_name_prefix_tf - ), - "squeeze": 0, - "transpose": (1, 0), - }, # (256,1024),(1,1024,256) - "{}.encoders.layeridx.feed_forward.w_2.bias".format( - tensor_name_prefix_torch - ): { - "name": "{}/layer_layeridx/ffn/conv1d_1/bias".format( - tensor_name_prefix_tf - ), - "squeeze": None, - "transpose": None, - }, # (256,),(256,) - # out norm - "{}.after_norm.weight".format(tensor_name_prefix_torch): { - "name": "{}/LayerNorm/gamma".format(tensor_name_prefix_tf), - "squeeze": None, - "transpose": None, - }, # (256,),(256,) - "{}.after_norm.bias".format(tensor_name_prefix_torch): { - "name": "{}/LayerNorm/beta".format(tensor_name_prefix_tf), - "squeeze": None, - "transpose": None, - }, # (256,),(256,) - } - if self.out_units is not None: - map_dict_local.update( - { - "{}.output_linear.weight".format(tensor_name_prefix_torch): { - "name": "{}/conv1d/kernel".format(tensor_name_prefix_tf), - "squeeze": 0, - "transpose": (1, 0), - }, - "{}.output_linear.bias".format(tensor_name_prefix_torch): { - "name": "{}/conv1d/bias".format(tensor_name_prefix_tf), - "squeeze": None, - "transpose": None, - }, # (256,),(256,) - } - ) - - return map_dict_local - - def convert_tf2torch( - self, - var_dict_tf, - var_dict_torch, - ): - - map_dict = self.gen_tf2torch_map_dict() - - var_dict_torch_update = dict() - for name in sorted(var_dict_torch.keys(), reverse=False): - if name.startswith(self.tf2torch_tensor_name_prefix_torch): - # process special (first and last) layers - if name in map_dict: - name_tf = map_dict[name]["name"] - data_tf = var_dict_tf[name_tf] - data_tf = torch.from_numpy(data_tf).type(torch.float32).to("cpu") - if map_dict[name]["squeeze"] is not None: - data_tf = np.squeeze(data_tf, axis=map_dict[name]["squeeze"]) - if map_dict[name]["transpose"] is not None: - data_tf = np.transpose(data_tf, map_dict[name]["transpose"]) - assert ( - var_dict_torch[name].size() == data_tf.size() - ), "{}, {}, {} != {}".format( - name, name_tf, var_dict_torch[name].size(), data_tf.size() - ) - var_dict_torch_update[name] = data_tf - logging.info( - "torch tensor: {}, {}, loading from tf tensor: {}, {}".format( - name, data_tf.size(), name_tf, var_dict_tf[name_tf].shape - ) - ) - # process general layers - else: - # self.tf2torch_tensor_name_prefix_torch may include ".", solve this case - names = name.replace( - self.tf2torch_tensor_name_prefix_torch, "todo" - ).split(".") - layeridx = int(names[2]) - name_q = name.replace(".{}.".format(layeridx), ".layeridx.") - if name_q in map_dict.keys(): - name_v = map_dict[name_q]["name"] - name_tf = name_v.replace("layeridx", "{}".format(layeridx)) - data_tf = var_dict_tf[name_tf] - if map_dict[name_q]["squeeze"] is not None: - data_tf = np.squeeze( - data_tf, axis=map_dict[name_q]["squeeze"] - ) - if map_dict[name_q]["transpose"] is not None: - data_tf = np.transpose( - data_tf, map_dict[name_q]["transpose"] - ) - data_tf = ( - torch.from_numpy(data_tf).type(torch.float32).to("cpu") - ) - assert ( - var_dict_torch[name].size() == data_tf.size() - ), "{}, {}, {} != {}".format( - name, name_tf, var_dict_torch[name].size(), data_tf.size() - ) - var_dict_torch_update[name] = data_tf - logging.info( - "torch tensor: {}, {}, loading from tf tensor: {}, {}".format( - name, - data_tf.size(), - name_tf, - var_dict_tf[name_tf].shape, - ) - ) - else: - logging.warning("{} is missed from tf checkpoint".format(name)) - - return var_dict_torch_update diff --git a/funasr_detach/models/sond/label_aggregation.py b/funasr_detach/models/sond/label_aggregation.py deleted file mode 100644 index 30322a0c761a3bc74d4e4fbd8f7c7c6d136ddc97..0000000000000000000000000000000000000000 --- a/funasr_detach/models/sond/label_aggregation.py +++ /dev/null @@ -1,113 +0,0 @@ -import torch -from typing import Optional -from typing import Tuple -from torch.nn import functional as F -from funasr_detach.models.transformer.utils.nets_utils import make_pad_mask - - -class LabelAggregate(torch.nn.Module): - def __init__( - self, - win_length: int = 512, - hop_length: int = 128, - center: bool = True, - ): - super().__init__() - - self.win_length = win_length - self.hop_length = hop_length - self.center = center - - def extra_repr(self): - return ( - f"win_length={self.win_length}, " - f"hop_length={self.hop_length}, " - f"center={self.center}, " - ) - - def forward( - self, input: torch.Tensor, ilens: torch.Tensor = None - ) -> Tuple[torch.Tensor, Optional[torch.Tensor]]: - """LabelAggregate forward function. - - Args: - input: (Batch, Nsamples, Label_dim) - ilens: (Batch) - Returns: - output: (Batch, Frames, Label_dim) - - """ - bs = input.size(0) - max_length = input.size(1) - label_dim = input.size(2) - - # NOTE(jiatong): - # The default behaviour of label aggregation is compatible with - # torch.stft about framing and padding. - - # Step1: center padding - if self.center: - pad = self.win_length // 2 - max_length = max_length + 2 * pad - input = torch.nn.functional.pad(input, (0, 0, pad, pad), "constant", 0) - input[:, :pad, :] = input[:, pad : (2 * pad), :] - input[:, (max_length - pad) : max_length, :] = input[ - :, (max_length - 2 * pad) : (max_length - pad), : - ] - nframe = (max_length - self.win_length) // self.hop_length + 1 - - # Step2: framing - output = input.as_strided( - (bs, nframe, self.win_length, label_dim), - (max_length * label_dim, self.hop_length * label_dim, label_dim, 1), - ) - - # Step3: aggregate label - output = torch.gt(output.sum(dim=2, keepdim=False), self.win_length // 2) - output = output.float() - - # Step4: process lengths - if ilens is not None: - if self.center: - pad = self.win_length // 2 - ilens = ilens + 2 * pad - - olens = (ilens - self.win_length) // self.hop_length + 1 - output.masked_fill_(make_pad_mask(olens, output, 1), 0.0) - else: - olens = None - - return output.to(input.dtype), olens - - -class LabelAggregateMaxPooling(torch.nn.Module): - def __init__( - self, - hop_length: int = 8, - ): - super().__init__() - - self.hop_length = hop_length - - def extra_repr(self): - return f"hop_length={self.hop_length}, " - - def forward( - self, input: torch.Tensor, ilens: torch.Tensor = None - ) -> Tuple[torch.Tensor, Optional[torch.Tensor]]: - """LabelAggregate forward function. - - Args: - input: (Batch, Nsamples, Label_dim) - ilens: (Batch) - Returns: - output: (Batch, Frames, Label_dim) - - """ - - output = F.max_pool1d( - input.transpose(1, 2), self.hop_length, self.hop_length - ).transpose(1, 2) - olens = ilens // self.hop_length - - return output.to(input.dtype), olens diff --git a/funasr_detach/models/sond/pooling/__init__.py b/funasr_detach/models/sond/pooling/__init__.py deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/funasr_detach/models/sond/pooling/pooling_layers.py b/funasr_detach/models/sond/pooling/pooling_layers.py deleted file mode 100644 index 7470cfba569e97e1f5c809475019b7e229b04074..0000000000000000000000000000000000000000 --- a/funasr_detach/models/sond/pooling/pooling_layers.py +++ /dev/null @@ -1,109 +0,0 @@ -# Copyright 3D-Speaker (https://github.com/alibaba-damo-academy/3D-Speaker). All Rights Reserved. -# Licensed under the Apache License, Version 2.0 (http://www.apache.org/licenses/LICENSE-2.0) - -"""This implementation is adapted from https://github.com/wenet-e2e/wespeaker.""" - -import torch -import torch.nn as nn - - -class TAP(nn.Module): - """ - Temporal average pooling, only first-order mean is considered - """ - - def __init__(self, **kwargs): - super(TAP, self).__init__() - - def forward(self, x): - pooling_mean = x.mean(dim=-1) - # To be compatable with 2D input - pooling_mean = pooling_mean.flatten(start_dim=1) - return pooling_mean - - -class TSDP(nn.Module): - """ - Temporal standard deviation pooling, only second-order std is considered - """ - - def __init__(self, **kwargs): - super(TSDP, self).__init__() - - def forward(self, x): - # The last dimension is the temporal axis - pooling_std = torch.sqrt(torch.var(x, dim=-1) + 1e-8) - pooling_std = pooling_std.flatten(start_dim=1) - return pooling_std - - -class TSTP(nn.Module): - """ - Temporal statistics pooling, concatenate mean and std, which is used in - x-vector - Comment: simple concatenation can not make full use of both statistics - """ - - def __init__(self, **kwargs): - super(TSTP, self).__init__() - - def forward(self, x): - # The last dimension is the temporal axis - pooling_mean = x.mean(dim=-1) - pooling_std = torch.sqrt(torch.var(x, dim=-1) + 1e-8) - pooling_mean = pooling_mean.flatten(start_dim=1) - pooling_std = pooling_std.flatten(start_dim=1) - - stats = torch.cat((pooling_mean, pooling_std), 1) - return stats - - -class ASTP(nn.Module): - """Attentive statistics pooling: Channel- and context-dependent - statistics pooling, first used in ECAPA_TDNN. - """ - - def __init__(self, in_dim, bottleneck_dim=128, global_context_att=False): - super(ASTP, self).__init__() - self.global_context_att = global_context_att - - # Use Conv1d with stride == 1 rather than Linear, then we don't - # need to transpose inputs. - if global_context_att: - self.linear1 = nn.Conv1d( - in_dim * 3, bottleneck_dim, kernel_size=1 - ) # equals W and b in the paper - else: - self.linear1 = nn.Conv1d( - in_dim, bottleneck_dim, kernel_size=1 - ) # equals W and b in the paper - self.linear2 = nn.Conv1d( - bottleneck_dim, in_dim, kernel_size=1 - ) # equals V and k in the paper - - def forward(self, x): - """ - x: a 3-dimensional tensor in tdnn-based architecture (B,F,T) - or a 4-dimensional tensor in resnet architecture (B,C,F,T) - 0-dim: batch-dimension, last-dim: time-dimension (frame-dimension) - """ - if len(x.shape) == 4: - x = x.reshape(x.shape[0], x.shape[1] * x.shape[2], x.shape[3]) - assert len(x.shape) == 3 - - if self.global_context_att: - context_mean = torch.mean(x, dim=-1, keepdim=True).expand_as(x) - context_std = torch.sqrt( - torch.var(x, dim=-1, keepdim=True) + 1e-10 - ).expand_as(x) - x_in = torch.cat((x, context_mean, context_std), dim=1) - else: - x_in = x - - # DON'T use ReLU here! ReLU may be hard to converge. - alpha = torch.tanh(self.linear1(x_in)) # alpha = F.relu(self.linear1(x_in)) - alpha = torch.softmax(self.linear2(alpha), dim=2) - mean = torch.sum(alpha * x, dim=2) - var = torch.sum(alpha * (x**2), dim=2) - mean**2 - std = torch.sqrt(var.clamp(min=1e-10)) - return torch.cat([mean, std], dim=1) diff --git a/funasr_detach/models/sond/pooling/statistic_pooling.py b/funasr_detach/models/sond/pooling/statistic_pooling.py deleted file mode 100644 index ff3c10ce084b2e1942cb80dbc84ca83812d19cd8..0000000000000000000000000000000000000000 --- a/funasr_detach/models/sond/pooling/statistic_pooling.py +++ /dev/null @@ -1,101 +0,0 @@ -import torch -from typing import Tuple -from typing import Union -from funasr_detach.models.transformer.utils.nets_utils import make_non_pad_mask -from torch.nn import functional as F -import math - -VAR2STD_EPSILON = 1e-12 - - -class StatisticPooling(torch.nn.Module): - def __init__(self, pooling_dim: Union[int, Tuple] = 2, eps=1e-12): - super(StatisticPooling, self).__init__() - if isinstance(pooling_dim, int): - pooling_dim = (pooling_dim,) - self.pooling_dim = pooling_dim - self.eps = eps - - def forward(self, xs_pad, ilens=None): - # xs_pad in (Batch, Channel, Time, Frequency) - - if ilens is None: - masks = torch.ones_like(xs_pad).to(xs_pad) - else: - masks = make_non_pad_mask(ilens, xs_pad, length_dim=2).to(xs_pad) - mean = torch.sum(xs_pad, dim=self.pooling_dim, keepdim=True) / torch.sum( - masks, dim=self.pooling_dim, keepdim=True - ) - squared_difference = torch.pow(xs_pad - mean, 2.0) - variance = torch.sum( - squared_difference, dim=self.pooling_dim, keepdim=True - ) / torch.sum(masks, dim=self.pooling_dim, keepdim=True) - for i in reversed(self.pooling_dim): - mean, variance = torch.squeeze(mean, dim=i), torch.squeeze(variance, dim=i) - - mask = torch.less_equal(variance, self.eps).float() - variance = (1.0 - mask) * variance + mask * self.eps - stddev = torch.sqrt(variance) - - stat_pooling = torch.cat([mean, stddev], dim=1) - - return stat_pooling - - def convert_tf2torch(self, var_dict_tf, var_dict_torch): - return {} - - -def statistic_pooling( - xs_pad: torch.Tensor, ilens: torch.Tensor = None, pooling_dim: Tuple = (2, 3) -) -> torch.Tensor: - # xs_pad in (Batch, Channel, Time, Frequency) - - if ilens is None: - seq_mask = torch.ones_like(xs_pad).to(xs_pad) - else: - seq_mask = make_non_pad_mask(ilens, xs_pad, length_dim=2).to(xs_pad) - mean = torch.sum(xs_pad, dim=pooling_dim, keepdim=True) / torch.sum( - seq_mask, dim=pooling_dim, keepdim=True - ) - squared_difference = torch.pow(xs_pad - mean, 2.0) - variance = torch.sum(squared_difference, dim=pooling_dim, keepdim=True) / torch.sum( - seq_mask, dim=pooling_dim, keepdim=True - ) - for i in reversed(pooling_dim): - mean, variance = torch.squeeze(mean, dim=i), torch.squeeze(variance, dim=i) - - value_mask = torch.less_equal(variance, VAR2STD_EPSILON).float() - variance = (1.0 - value_mask) * variance + value_mask * VAR2STD_EPSILON - stddev = torch.sqrt(variance) - - stat_pooling = torch.cat([mean, stddev], dim=1) - - return stat_pooling - - -def windowed_statistic_pooling( - xs_pad: torch.Tensor, - ilens: torch.Tensor = None, - pooling_dim: Tuple = (2, 3), - pooling_size: int = 20, - pooling_stride: int = 1, -) -> Tuple[torch.Tensor, int]: - # xs_pad in (Batch, Channel, Time, Frequency) - - tt = xs_pad.shape[2] - num_chunk = int(math.ceil(tt / pooling_stride)) - pad = pooling_size // 2 - if len(xs_pad.shape) == 4: - features = F.pad(xs_pad, (0, 0, pad, pad), "replicate") - else: - features = F.pad(xs_pad, (pad, pad), "replicate") - stat_list = [] - - for i in range(num_chunk): - # B x C - st, ed = i * pooling_stride, i * pooling_stride + pooling_size - stat = statistic_pooling(features[:, :, st:ed], pooling_dim=pooling_dim) - stat_list.append(stat.unsqueeze(2)) - - # B x C x T - return torch.cat(stat_list, dim=2), ilens / pooling_stride diff --git a/funasr_detach/models/sond/sv_decoder.py b/funasr_detach/models/sond/sv_decoder.py deleted file mode 100644 index 18f935781ebc1855bbb8dbc5abd9e63d2f5bc9eb..0000000000000000000000000000000000000000 --- a/funasr_detach/models/sond/sv_decoder.py +++ /dev/null @@ -1,43 +0,0 @@ -import torch -from torch.nn import functional as F -from funasr_detach.models.decoder.abs_decoder import AbsDecoder - - -class DenseDecoder(AbsDecoder): - def __init__( - self, - vocab_size, - encoder_output_size, - num_nodes_resnet1: int = 256, - num_nodes_last_layer: int = 256, - batchnorm_momentum: float = 0.5, - ): - super(DenseDecoder, self).__init__() - self.resnet1_dense = torch.nn.Linear(encoder_output_size, num_nodes_resnet1) - self.resnet1_bn = torch.nn.BatchNorm1d( - num_nodes_resnet1, eps=1e-3, momentum=batchnorm_momentum - ) - - self.resnet2_dense = torch.nn.Linear(num_nodes_resnet1, num_nodes_last_layer) - self.resnet2_bn = torch.nn.BatchNorm1d( - num_nodes_last_layer, eps=1e-3, momentum=batchnorm_momentum - ) - - self.output_dense = torch.nn.Linear( - num_nodes_last_layer, vocab_size, bias=False - ) - - def forward(self, features): - embeddings = {} - features = self.resnet1_dense(features) - embeddings["resnet1_dense"] = features - features = F.relu(features) - features = self.resnet1_bn(features) - - features = self.resnet2_dense(features) - embeddings["resnet2_dense"] = features - features = F.relu(features) - features = self.resnet2_bn(features) - - features = self.output_dense(features) - return features, embeddings diff --git a/funasr_detach/models/specaug/__init__.py b/funasr_detach/models/specaug/__init__.py deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/funasr_detach/models/specaug/mask_along_axis.py b/funasr_detach/models/specaug/mask_along_axis.py deleted file mode 100644 index 41c6a00438bafcf8ad5620dd95c1e5f5b69a8eab..0000000000000000000000000000000000000000 --- a/funasr_detach/models/specaug/mask_along_axis.py +++ /dev/null @@ -1,338 +0,0 @@ -import math -import torch -from typing import Sequence -from typing import Union - - -def mask_along_axis( - spec: torch.Tensor, - spec_lengths: torch.Tensor, - mask_width_range: Sequence[int] = (0, 30), - dim: int = 1, - num_mask: int = 2, - replace_with_zero: bool = True, -): - """Apply mask along the specified direction. - - Args: - spec: (Batch, Length, Freq) - spec_lengths: (Length): Not using lengths in this implementation - mask_width_range: Select the width randomly between this range - """ - - org_size = spec.size() - if spec.dim() == 4: - # spec: (Batch, Channel, Length, Freq) -> (Batch * Channel, Length, Freq) - spec = spec.view(-1, spec.size(2), spec.size(3)) - - B = spec.shape[0] - # D = Length or Freq - D = spec.shape[dim] - # mask_length: (B, num_mask, 1) - mask_length = torch.randint( - mask_width_range[0], - mask_width_range[1], - (B, num_mask), - device=spec.device, - ).unsqueeze(2) - - # mask_pos: (B, num_mask, 1) - mask_pos = torch.randint( - 0, max(1, D - mask_length.max()), (B, num_mask), device=spec.device - ).unsqueeze(2) - - # aran: (1, 1, D) - aran = torch.arange(D, device=spec.device)[None, None, :] - # mask: (Batch, num_mask, D) - mask = (mask_pos <= aran) * (aran < (mask_pos + mask_length)) - # Multiply masks: (Batch, num_mask, D) -> (Batch, D) - mask = mask.any(dim=1) - if dim == 1: - # mask: (Batch, Length, 1) - mask = mask.unsqueeze(2) - elif dim == 2: - # mask: (Batch, 1, Freq) - mask = mask.unsqueeze(1) - - if replace_with_zero: - value = 0.0 - else: - value = spec.mean() - - if spec.requires_grad: - spec = spec.masked_fill(mask, value) - else: - spec = spec.masked_fill_(mask, value) - spec = spec.view(*org_size) - return spec, spec_lengths - - -def mask_along_axis_lfr( - spec: torch.Tensor, - spec_lengths: torch.Tensor, - mask_width_range: Sequence[int] = (0, 30), - dim: int = 1, - num_mask: int = 2, - replace_with_zero: bool = True, - lfr_rate: int = 1, -): - """Apply mask along the specified direction. - - Args: - spec: (Batch, Length, Freq) - spec_lengths: (Length): Not using lengths in this implementation - mask_width_range: Select the width randomly between this range - lfr_rate：low frame rate - """ - - org_size = spec.size() - if spec.dim() == 4: - # spec: (Batch, Channel, Length, Freq) -> (Batch * Channel, Length, Freq) - spec = spec.view(-1, spec.size(2), spec.size(3)) - - B = spec.shape[0] - # D = Length or Freq - D = spec.shape[dim] // lfr_rate - # mask_length: (B, num_mask, 1) - mask_length = torch.randint( - mask_width_range[0], - mask_width_range[1], - (B, num_mask), - device=spec.device, - ).unsqueeze(2) - if lfr_rate > 1: - mask_length = mask_length.repeat(1, lfr_rate, 1) - # mask_pos: (B, num_mask, 1) - mask_pos = torch.randint( - 0, max(1, D - mask_length.max()), (B, num_mask), device=spec.device - ).unsqueeze(2) - if lfr_rate > 1: - mask_pos_raw = mask_pos.clone() - mask_pos = torch.zeros((B, 0, 1), device=spec.device, dtype=torch.int32) - for i in range(lfr_rate): - mask_pos_i = mask_pos_raw + D * i - mask_pos = torch.cat((mask_pos, mask_pos_i), dim=1) - # aran: (1, 1, D) - D = spec.shape[dim] - aran = torch.arange(D, device=spec.device)[None, None, :] - # mask: (Batch, num_mask, D) - mask = (mask_pos <= aran) * (aran < (mask_pos + mask_length)) - # Multiply masks: (Batch, num_mask, D) -> (Batch, D) - mask = mask.any(dim=1) - if dim == 1: - # mask: (Batch, Length, 1) - mask = mask.unsqueeze(2) - elif dim == 2: - # mask: (Batch, 1, Freq) - mask = mask.unsqueeze(1) - - if replace_with_zero: - value = 0.0 - else: - value = spec.mean() - - if spec.requires_grad: - spec = spec.masked_fill(mask, value) - else: - spec = spec.masked_fill_(mask, value) - spec = spec.view(*org_size) - return spec, spec_lengths - - -class MaskAlongAxis(torch.nn.Module): - def __init__( - self, - mask_width_range: Union[int, Sequence[int]] = (0, 30), - num_mask: int = 2, - dim: Union[int, str] = "time", - replace_with_zero: bool = True, - ): - if isinstance(mask_width_range, int): - mask_width_range = (0, mask_width_range) - if len(mask_width_range) != 2: - raise TypeError( - f"mask_width_range must be a tuple of int and int values: " - f"{mask_width_range}", - ) - - assert mask_width_range[1] > mask_width_range[0] - if isinstance(dim, str): - if dim == "time": - dim = 1 - elif dim == "freq": - dim = 2 - else: - raise ValueError("dim must be int, 'time' or 'freq'") - if dim == 1: - self.mask_axis = "time" - elif dim == 2: - self.mask_axis = "freq" - else: - self.mask_axis = "unknown" - - super().__init__() - self.mask_width_range = mask_width_range - self.num_mask = num_mask - self.dim = dim - self.replace_with_zero = replace_with_zero - - def extra_repr(self): - return ( - f"mask_width_range={self.mask_width_range}, " - f"num_mask={self.num_mask}, axis={self.mask_axis}" - ) - - def forward(self, spec: torch.Tensor, spec_lengths: torch.Tensor = None): - """Forward function. - - Args: - spec: (Batch, Length, Freq) - """ - - return mask_along_axis( - spec, - spec_lengths, - mask_width_range=self.mask_width_range, - dim=self.dim, - num_mask=self.num_mask, - replace_with_zero=self.replace_with_zero, - ) - - -class MaskAlongAxisVariableMaxWidth(torch.nn.Module): - """Mask input spec along a specified axis with variable maximum width. - - Formula: - max_width = max_width_ratio * seq_len - """ - - def __init__( - self, - mask_width_ratio_range: Union[float, Sequence[float]] = (0.0, 0.05), - num_mask: int = 2, - dim: Union[int, str] = "time", - replace_with_zero: bool = True, - ): - if isinstance(mask_width_ratio_range, float): - mask_width_ratio_range = (0.0, mask_width_ratio_range) - if len(mask_width_ratio_range) != 2: - raise TypeError( - f"mask_width_ratio_range must be a tuple of float and float values: " - f"{mask_width_ratio_range}", - ) - - assert mask_width_ratio_range[1] > mask_width_ratio_range[0] - if isinstance(dim, str): - if dim == "time": - dim = 1 - elif dim == "freq": - dim = 2 - else: - raise ValueError("dim must be int, 'time' or 'freq'") - if dim == 1: - self.mask_axis = "time" - elif dim == 2: - self.mask_axis = "freq" - else: - self.mask_axis = "unknown" - - super().__init__() - self.mask_width_ratio_range = mask_width_ratio_range - self.num_mask = num_mask - self.dim = dim - self.replace_with_zero = replace_with_zero - - def extra_repr(self): - return ( - f"mask_width_ratio_range={self.mask_width_ratio_range}, " - f"num_mask={self.num_mask}, axis={self.mask_axis}" - ) - - def forward(self, spec: torch.Tensor, spec_lengths: torch.Tensor = None): - """Forward function. - - Args: - spec: (Batch, Length, Freq) - """ - - max_seq_len = spec.shape[self.dim] - min_mask_width = math.floor(max_seq_len * self.mask_width_ratio_range[0]) - min_mask_width = max([0, min_mask_width]) - max_mask_width = math.floor(max_seq_len * self.mask_width_ratio_range[1]) - max_mask_width = min([max_seq_len, max_mask_width]) - - if max_mask_width > min_mask_width: - return mask_along_axis( - spec, - spec_lengths, - mask_width_range=(min_mask_width, max_mask_width), - dim=self.dim, - num_mask=self.num_mask, - replace_with_zero=self.replace_with_zero, - ) - return spec, spec_lengths - - -class MaskAlongAxisLFR(torch.nn.Module): - def __init__( - self, - mask_width_range: Union[int, Sequence[int]] = (0, 30), - num_mask: int = 2, - dim: Union[int, str] = "time", - replace_with_zero: bool = True, - lfr_rate: int = 1, - ): - if isinstance(mask_width_range, int): - mask_width_range = (0, mask_width_range) - if len(mask_width_range) != 2: - raise TypeError( - f"mask_width_range must be a tuple of int and int values: " - f"{mask_width_range}", - ) - - assert mask_width_range[1] > mask_width_range[0] - if isinstance(dim, str): - if dim == "time": - dim = 1 - lfr_rate = 1 - elif dim == "freq": - dim = 2 - else: - raise ValueError("dim must be int, 'time' or 'freq'") - if dim == 1: - self.mask_axis = "time" - lfr_rate = 1 - elif dim == 2: - self.mask_axis = "freq" - else: - self.mask_axis = "unknown" - - super().__init__() - self.mask_width_range = mask_width_range - self.num_mask = num_mask - self.dim = dim - self.replace_with_zero = replace_with_zero - self.lfr_rate = lfr_rate - - def extra_repr(self): - return ( - f"mask_width_range={self.mask_width_range}, " - f"num_mask={self.num_mask}, axis={self.mask_axis}" - ) - - def forward(self, spec: torch.Tensor, spec_lengths: torch.Tensor = None): - """Forward function. - - Args: - spec: (Batch, Length, Freq) - """ - - return mask_along_axis_lfr( - spec, - spec_lengths, - mask_width_range=self.mask_width_range, - dim=self.dim, - num_mask=self.num_mask, - replace_with_zero=self.replace_with_zero, - lfr_rate=self.lfr_rate, - ) diff --git a/funasr_detach/models/specaug/profileaug.py b/funasr_detach/models/specaug/profileaug.py deleted file mode 100644 index a5a003268ee9453ff36bc98cb73e79d4bdca183a..0000000000000000000000000000000000000000 --- a/funasr_detach/models/specaug/profileaug.py +++ /dev/null @@ -1,153 +0,0 @@ -from typing import Tuple, Optional -import numpy as np -import torch -from torch.nn import functional as F -import torch.nn as nn - - -class ProfileAug(nn.Module): - """ - Implement the augmentation for profiles including: - - Split aug: split one profile into two profiles, i.e., main and inaccurate, labels assigned to main - - Merge aug: merge two profiles into one, labels are also merged into one, the other set to zero - - Disturb aug: disturb some profile with others to simulate the inaccurate clustering centroids. - """ - - def __init__( - self, - apply_split_aug: bool = True, - split_aug_prob: float = 0.05, - apply_merge_aug: bool = True, - merge_aug_prob: float = 0.2, - apply_disturb_aug: bool = True, - disturb_aug_prob: float = 0.4, - disturb_alpha: float = 0.2, - ) -> None: - super().__init__() - self.apply_split_aug = apply_split_aug - self.split_aug_prob = split_aug_prob - self.apply_merge_aug = apply_merge_aug - self.merge_aug_prob = merge_aug_prob - self.apply_disturb_aug = apply_disturb_aug - self.disturb_aug_prob = disturb_aug_prob - self.disturb_alpha = disturb_alpha - - def split_aug( - self, profile: torch.Tensor, binary_labels: torch.Tensor, mask: torch.Tensor - ): - # B, N - bsz, dim = profile.shape[0], profile.shape[-1] - profile_norm = torch.linalg.norm(profile, dim=-1, keepdim=False) - spk_count = binary_labels.sum(dim=1) - prob = np.random.rand(bsz) - batch_indices = np.nonzero(prob < self.split_aug_prob)[0] - for idx in batch_indices: - valid_spk_idx = torch.nonzero(spk_count[idx] * mask[idx]) - pad_spk_idx = torch.nonzero((spk_count[idx] == 0) * mask[idx]) - if len(valid_spk_idx) == 0 or len(pad_spk_idx) == 0: - continue - split_spk_idx = valid_spk_idx[torch.randint(len(valid_spk_idx), ())] - to_cover_idx = pad_spk_idx[torch.randint(len(pad_spk_idx), ())] - disturb_vec = torch.randn((dim,)).to(profile) - disturb_vec = F.normalize(disturb_vec, dim=-1) - profile[idx, to_cover_idx] = F.normalize( - profile[idx, split_spk_idx] + self.disturb_alpha * disturb_vec - ) - mask[idx, split_spk_idx] = 0 - mask[idx, to_cover_idx] = 0 - return profile, binary_labels, mask - - def merge_aug( - self, profile: torch.Tensor, binary_labels: torch.Tensor, mask: torch.Tensor - ): - bsz, dim = profile.shape[0], profile.shape[-1] - profile_norm = torch.linalg.norm(profile, dim=-1, keepdim=False) - spk_count = binary_labels.sum(dim=1) - prob = np.random.rand(bsz) - batch_indices = np.nonzero(prob < self.merge_aug_prob)[0] - for idx in batch_indices: - valid_spk_idx = torch.nonzero(profile_norm[idx] * mask[idx]) - if len(valid_spk_idx) == 0: - continue - to_merge = torch.randint(len(valid_spk_idx), (2,)) - spk_idx_1, spk_idx_2 = ( - valid_spk_idx[to_merge[0]], - valid_spk_idx[to_merge[1]], - ) - # merge profile - profile[idx, spk_idx_1] = profile[idx, spk_idx_1] + profile[idx, spk_idx_2] - profile[idx, spk_idx_1] = F.normalize(profile[idx, spk_idx_1], dim=-1) - profile[idx, spk_idx_2] = 0 - # merge binary labels - binary_labels[idx, :, spk_idx_1] = ( - binary_labels[idx, :, spk_idx_1] + binary_labels[idx, :, spk_idx_2] - ) - binary_labels[idx, :, spk_idx_1] = ( - binary_labels[idx, :, spk_idx_1] > 0 - ).to(binary_labels) - binary_labels[idx, :, spk_idx_2] = 0 - - mask[idx, spk_idx_1] = 0 - mask[idx, spk_idx_2] = 0 - - return profile, binary_labels, mask - - def disturb_aug( - self, profile: torch.Tensor, binary_labels: torch.Tensor, mask: torch.Tensor - ): - bsz, dim = profile.shape[0], profile.shape[-1] - profile_norm = torch.linalg.norm(profile, dim=-1, keepdim=False) - spk_count = binary_labels.sum(dim=1) - prob = np.random.rand(bsz) - batch_indices = np.nonzero(prob < self.disturb_aug_prob)[0] - for idx in batch_indices: - pos_spk_idx = torch.nonzero(spk_count[idx] * mask[idx]) - valid_spk_idx = torch.nonzero(profile_norm[idx] * mask[idx]) - if len(pos_spk_idx) == 0 or len(valid_spk_idx) == 0: - continue - to_disturb_idx = pos_spk_idx[torch.randint(len(pos_spk_idx), ())] - disturb_idx = valid_spk_idx[torch.randint(len(valid_spk_idx), ())] - alpha = self.disturb_alpha * torch.rand(()).item() - profile[idx, to_disturb_idx] = (1 - alpha) * profile[ - idx, to_disturb_idx - ] + alpha * profile[idx, disturb_idx] - profile[idx, to_disturb_idx] = F.normalize( - profile[idx, to_disturb_idx], dim=-1 - ) - mask[idx, to_disturb_idx] = 0 - - return profile, binary_labels, mask - - def forward( - self, - speech: torch.Tensor, - speech_lengths: torch.Tensor = None, - profile: torch.Tensor = None, - profile_lengths: torch.Tensor = None, - binary_labels: torch.Tensor = None, - labels_length: torch.Tensor = None, - ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[torch.Tensor]]: - - # copy inputs to avoid inplace-operation - speech, profile, binary_labels = ( - torch.clone(speech), - torch.clone(profile), - torch.clone(binary_labels), - ) - profile = F.normalize(profile, dim=-1) - - profile_mask = torch.ones(profile.shape[:2]).to(profile) - if self.apply_disturb_aug: - profile, binary_labels, profile_mask = self.disturb_aug( - profile, binary_labels, profile_mask - ) - if self.apply_split_aug: - profile, binary_labels, profile_mask = self.split_aug( - profile, binary_labels, profile_mask - ) - if self.apply_merge_aug: - profile, binary_labels, profile_mask = self.merge_aug( - profile, binary_labels, profile_mask - ) - - return speech, profile, binary_labels diff --git a/funasr_detach/models/specaug/specaug.py b/funasr_detach/models/specaug/specaug.py deleted file mode 100644 index e34e448e59c452453a4cc52e140f835da60020ad..0000000000000000000000000000000000000000 --- a/funasr_detach/models/specaug/specaug.py +++ /dev/null @@ -1,190 +0,0 @@ -"""SpecAugment module.""" - -from typing import Optional -from typing import Sequence -from typing import Union - -from funasr_detach.models.specaug.mask_along_axis import MaskAlongAxis -from funasr_detach.models.specaug.mask_along_axis import MaskAlongAxisVariableMaxWidth -from funasr_detach.models.specaug.mask_along_axis import MaskAlongAxisLFR -from funasr_detach.models.specaug.time_warp import TimeWarp -from funasr_detach.register import tables - -import torch.nn as nn - - -@tables.register("specaug_classes", "SpecAug") -class SpecAug(nn.Module): - """Implementation of SpecAug. - - Reference: - Daniel S. Park et al. - "SpecAugment: A Simple Data - Augmentation Method for Automatic Speech Recognition" - - .. warning:: - When using cuda mode, time_warp doesn't have reproducibility - due to `torch.nn.functional.interpolate`. - - """ - - def __init__( - self, - apply_time_warp: bool = True, - time_warp_window: int = 5, - time_warp_mode: str = "bicubic", - apply_freq_mask: bool = True, - freq_mask_width_range: Union[int, Sequence[int]] = (0, 20), - num_freq_mask: int = 2, - apply_time_mask: bool = True, - time_mask_width_range: Optional[Union[int, Sequence[int]]] = None, - time_mask_width_ratio_range: Optional[Union[float, Sequence[float]]] = None, - num_time_mask: int = 2, - ): - if not apply_time_warp and not apply_time_mask and not apply_freq_mask: - raise ValueError( - "Either one of time_warp, time_mask, or freq_mask should be applied" - ) - if ( - apply_time_mask - and (time_mask_width_range is not None) - and (time_mask_width_ratio_range is not None) - ): - raise ValueError( - 'Either one of "time_mask_width_range" or ' - '"time_mask_width_ratio_range" can be used' - ) - super().__init__() - self.apply_time_warp = apply_time_warp - self.apply_freq_mask = apply_freq_mask - self.apply_time_mask = apply_time_mask - - if apply_time_warp: - self.time_warp = TimeWarp(window=time_warp_window, mode=time_warp_mode) - else: - self.time_warp = None - - if apply_freq_mask: - self.freq_mask = MaskAlongAxis( - dim="freq", - mask_width_range=freq_mask_width_range, - num_mask=num_freq_mask, - ) - else: - self.freq_mask = None - - if apply_time_mask: - if time_mask_width_range is not None: - self.time_mask = MaskAlongAxis( - dim="time", - mask_width_range=time_mask_width_range, - num_mask=num_time_mask, - ) - elif time_mask_width_ratio_range is not None: - self.time_mask = MaskAlongAxisVariableMaxWidth( - dim="time", - mask_width_ratio_range=time_mask_width_ratio_range, - num_mask=num_time_mask, - ) - else: - raise ValueError( - 'Either one of "time_mask_width_range" or ' - '"time_mask_width_ratio_range" should be used.' - ) - else: - self.time_mask = None - - def forward(self, x, x_lengths=None): - if self.time_warp is not None: - x, x_lengths = self.time_warp(x, x_lengths) - if self.freq_mask is not None: - x, x_lengths = self.freq_mask(x, x_lengths) - if self.time_mask is not None: - x, x_lengths = self.time_mask(x, x_lengths) - return x, x_lengths - - -@tables.register("specaug_classes", "SpecAugLFR") -class SpecAugLFR(nn.Module): - """Implementation of SpecAug. - lfr_rate：low frame rate - """ - - def __init__( - self, - apply_time_warp: bool = True, - time_warp_window: int = 5, - time_warp_mode: str = "bicubic", - apply_freq_mask: bool = True, - freq_mask_width_range: Union[int, Sequence[int]] = (0, 20), - num_freq_mask: int = 2, - lfr_rate: int = 0, - apply_time_mask: bool = True, - time_mask_width_range: Optional[Union[int, Sequence[int]]] = None, - time_mask_width_ratio_range: Optional[Union[float, Sequence[float]]] = None, - num_time_mask: int = 2, - ): - if not apply_time_warp and not apply_time_mask and not apply_freq_mask: - raise ValueError( - "Either one of time_warp, time_mask, or freq_mask should be applied" - ) - if ( - apply_time_mask - and (time_mask_width_range is not None) - and (time_mask_width_ratio_range is not None) - ): - raise ValueError( - 'Either one of "time_mask_width_range" or ' - '"time_mask_width_ratio_range" can be used' - ) - super().__init__() - self.apply_time_warp = apply_time_warp - self.apply_freq_mask = apply_freq_mask - self.apply_time_mask = apply_time_mask - - if apply_time_warp: - self.time_warp = TimeWarp(window=time_warp_window, mode=time_warp_mode) - else: - self.time_warp = None - - if apply_freq_mask: - self.freq_mask = MaskAlongAxisLFR( - dim="freq", - mask_width_range=freq_mask_width_range, - num_mask=num_freq_mask, - lfr_rate=lfr_rate + 1, - ) - - else: - self.freq_mask = None - - if apply_time_mask: - if time_mask_width_range is not None: - self.time_mask = MaskAlongAxisLFR( - dim="time", - mask_width_range=time_mask_width_range, - num_mask=num_time_mask, - lfr_rate=lfr_rate + 1, - ) - elif time_mask_width_ratio_range is not None: - self.time_mask = MaskAlongAxisVariableMaxWidth( - dim="time", - mask_width_ratio_range=time_mask_width_ratio_range, - num_mask=num_time_mask, - ) - else: - raise ValueError( - 'Either one of "time_mask_width_range" or ' - '"time_mask_width_ratio_range" should be used.' - ) - else: - self.time_mask = None - - def forward(self, x, x_lengths=None): - if self.time_warp is not None: - x, x_lengths = self.time_warp(x, x_lengths) - if self.freq_mask is not None: - x, x_lengths = self.freq_mask(x, x_lengths) - if self.time_mask is not None: - x, x_lengths = self.time_mask(x, x_lengths) - return x, x_lengths diff --git a/funasr_detach/models/specaug/time_warp.py b/funasr_detach/models/specaug/time_warp.py deleted file mode 100644 index 972a2c0f5d6f8d4507e5ee355194f03696a09fcb..0000000000000000000000000000000000000000 --- a/funasr_detach/models/specaug/time_warp.py +++ /dev/null @@ -1,89 +0,0 @@ -"""Time warp module.""" - -import torch - -from funasr_detach.models.transformer.utils.nets_utils import pad_list - -DEFAULT_TIME_WARP_MODE = "bicubic" - - -def time_warp(x: torch.Tensor, window: int = 80, mode: str = DEFAULT_TIME_WARP_MODE): - """Time warping using torch.interpolate. - - Args: - x: (Batch, Time, Freq) - window: time warp parameter - mode: Interpolate mode - """ - - # bicubic supports 4D or more dimension tensor - org_size = x.size() - if x.dim() == 3: - # x: (Batch, Time, Freq) -> (Batch, 1, Time, Freq) - x = x[:, None] - - t = x.shape[2] - if t - window <= window: - return x.view(*org_size) - - center = torch.randint(window, t - window, (1,))[0] - warped = torch.randint(center - window, center + window, (1,))[0] + 1 - - # left: (Batch, Channel, warped, Freq) - # right: (Batch, Channel, time - warped, Freq) - left = torch.nn.functional.interpolate( - x[:, :, :center], (warped, x.shape[3]), mode=mode, align_corners=False - ) - right = torch.nn.functional.interpolate( - x[:, :, center:], (t - warped, x.shape[3]), mode=mode, align_corners=False - ) - - if x.requires_grad: - x = torch.cat([left, right], dim=-2) - else: - x[:, :, :warped] = left - x[:, :, warped:] = right - - return x.view(*org_size) - - -class TimeWarp(torch.nn.Module): - """Time warping using torch.interpolate. - - Args: - window: time warp parameter - mode: Interpolate mode - """ - - def __init__(self, window: int = 80, mode: str = DEFAULT_TIME_WARP_MODE): - super().__init__() - self.window = window - self.mode = mode - - def extra_repr(self): - return f"window={self.window}, mode={self.mode}" - - def forward(self, x: torch.Tensor, x_lengths: torch.Tensor = None): - """Forward function. - - Args: - x: (Batch, Time, Freq) - x_lengths: (Batch,) - """ - - if x_lengths is None or all(le == x_lengths[0] for le in x_lengths): - # Note that applying same warping for each sample - y = time_warp(x, window=self.window, mode=self.mode) - else: - # FIXME(kamo): I have no idea to batchify Timewarp - ys = [] - for i in range(x.size(0)): - _y = time_warp( - x[i][None, : x_lengths[i]], - window=self.window, - mode=self.mode, - )[0] - ys.append(_y) - y = pad_list(ys, 0.0) - - return y, x_lengths diff --git a/funasr_detach/models/transducer/__init__.py b/funasr_detach/models/transducer/__init__.py deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/funasr_detach/models/transducer/beam_search_transducer.py b/funasr_detach/models/transducer/beam_search_transducer.py deleted file mode 100644 index 2fba447fe3f661d14c6c7bb5153b2f180ff2a931..0000000000000000000000000000000000000000 --- a/funasr_detach/models/transducer/beam_search_transducer.py +++ /dev/null @@ -1,706 +0,0 @@ -#!/usr/bin/env python3 -# -*- encoding: utf-8 -*- -# Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved. -# MIT License (https://opensource.org/licenses/MIT) - -import torch -import numpy as np -from dataclasses import dataclass -from typing import Any, Dict, List, Optional, Tuple, Union - -from funasr_detach.models.transducer.joint_network import JointNetwork - - -@dataclass -class Hypothesis: - """Default hypothesis definition for Transducer search algorithms. - - Args: - score: Total log-probability. - yseq: Label sequence as integer ID sequence. - dec_state: RNNDecoder or StatelessDecoder state. - ((N, 1, D_dec), (N, 1, D_dec) or None) or None - lm_state: RNNLM state. ((N, D_lm), (N, D_lm)) or None - - """ - - score: float - yseq: List[int] - dec_state: Optional[Tuple[torch.Tensor, Optional[torch.Tensor]]] = None - lm_state: Optional[Union[Dict[str, Any], List[Any]]] = None - - -@dataclass -class ExtendedHypothesis(Hypothesis): - """Extended hypothesis definition for NSC beam search and mAES. - - Args: - : Hypothesis dataclass arguments. - dec_out: Decoder output sequence. (B, D_dec) - lm_score: Log-probabilities of the LM for given label. (vocab_size) - - """ - - dec_out: torch.Tensor = None - lm_score: torch.Tensor = None - - -class BeamSearchTransducer: - """Beam search implementation for Transducer. - - Args: - decoder: Decoder module. - joint_network: Joint network module. - beam_size: Size of the beam. - lm: LM class. - lm_weight: LM weight for soft fusion. - search_type: Search algorithm to use during inference. - max_sym_exp: Number of maximum symbol expansions at each time step. (TSD) - u_max: Maximum expected target sequence length. (ALSD) - nstep: Number of maximum expansion steps at each time step. (mAES) - expansion_gamma: Allowed logp difference for prune-by-value method. (mAES) - expansion_beta: - Number of additional candidates for expanded hypotheses selection. (mAES) - score_norm: Normalize final scores by length. - nbest: Number of final hypothesis. - streaming: Whether to perform chunk-by-chunk beam search. - - """ - - def __init__( - self, - decoder, - joint_network: JointNetwork, - beam_size: int, - lm: Optional[torch.nn.Module] = None, - lm_weight: float = 0.1, - search_type: str = "default", - max_sym_exp: int = 3, - u_max: int = 50, - nstep: int = 2, - expansion_gamma: float = 2.3, - expansion_beta: int = 2, - score_norm: bool = False, - nbest: int = 1, - streaming: bool = False, - ) -> None: - """Construct a BeamSearchTransducer object.""" - super().__init__() - - self.decoder = decoder - self.joint_network = joint_network - - self.vocab_size = decoder.vocab_size - - assert beam_size <= self.vocab_size, ( - "beam_size (%d) should be smaller than or equal to vocabulary size (%d)." - % ( - beam_size, - self.vocab_size, - ) - ) - self.beam_size = beam_size - - if search_type == "default": - self.search_algorithm = self.default_beam_search - elif search_type == "tsd": - assert max_sym_exp > 1, "max_sym_exp (%d) should be greater than one." % ( - max_sym_exp - ) - self.max_sym_exp = max_sym_exp - - self.search_algorithm = self.time_sync_decoding - elif search_type == "alsd": - assert not streaming, "ALSD is not available in streaming mode." - - assert u_max >= 0, "u_max should be a positive integer, a portion of max_T." - self.u_max = u_max - - self.search_algorithm = self.align_length_sync_decoding - elif search_type == "maes": - assert self.vocab_size >= beam_size + expansion_beta, ( - "beam_size (%d) + expansion_beta (%d) " - " should be smaller than or equal to vocab size (%d)." - % (beam_size, expansion_beta, self.vocab_size) - ) - self.max_candidates = beam_size + expansion_beta - - self.nstep = nstep - self.expansion_gamma = expansion_gamma - - self.search_algorithm = self.modified_adaptive_expansion_search - else: - raise NotImplementedError( - "Specified search type (%s) is not supported." % search_type - ) - - self.use_lm = lm is not None - - if self.use_lm: - assert hasattr(lm, "rnn_type"), "Transformer LM is currently not supported." - - self.sos = self.vocab_size - 1 - - self.lm = lm - self.lm_weight = lm_weight - - self.score_norm = score_norm - self.nbest = nbest - - self.reset_inference_cache() - - def __call__( - self, - enc_out: torch.Tensor, - is_final: bool = True, - ) -> List[Hypothesis]: - """Perform beam search. - - Args: - enc_out: Encoder output sequence. (T, D_enc) - is_final: Whether enc_out is the final chunk of data. - - Returns: - nbest_hyps: N-best decoding results - - """ - self.decoder.set_device(enc_out.device) - - hyps = self.search_algorithm(enc_out) - - if is_final: - self.reset_inference_cache() - - return self.sort_nbest(hyps) - - self.search_cache = hyps - - return hyps - - def reset_inference_cache(self) -> None: - """Reset cache for decoder scoring and streaming.""" - self.decoder.score_cache = {} - self.search_cache = None - - def sort_nbest(self, hyps: List[Hypothesis]) -> List[Hypothesis]: - """Sort in-place hypotheses by score or score given sequence length. - - Args: - hyps: Hypothesis. - - Return: - hyps: Sorted hypothesis. - - """ - if self.score_norm: - hyps.sort(key=lambda x: x.score / len(x.yseq), reverse=True) - else: - hyps.sort(key=lambda x: x.score, reverse=True) - - return hyps[: self.nbest] - - def recombine_hyps(self, hyps: List[Hypothesis]) -> List[Hypothesis]: - """Recombine hypotheses with same label ID sequence. - - Args: - hyps: Hypotheses. - - Returns: - final: Recombined hypotheses. - - """ - final = {} - - for hyp in hyps: - str_yseq = "_".join(map(str, hyp.yseq)) - - if str_yseq in final: - final[str_yseq].score = np.logaddexp(final[str_yseq].score, hyp.score) - else: - final[str_yseq] = hyp - - return [*final.values()] - - def select_k_expansions( - self, - hyps: List[ExtendedHypothesis], - topk_idx: torch.Tensor, - topk_logp: torch.Tensor, - ) -> List[ExtendedHypothesis]: - """Return K hypotheses candidates for expansion from a list of hypothesis. - - K candidates are selected according to the extended hypotheses probabilities - and a prune-by-value method. Where K is equal to beam_size + beta. - - Args: - hyps: Hypotheses. - topk_idx: Indices of candidates hypothesis. - topk_logp: Log-probabilities of candidates hypothesis. - - Returns: - k_expansions: Best K expansion hypotheses candidates. - - """ - k_expansions = [] - - for i, hyp in enumerate(hyps): - hyp_i = [ - (int(k), hyp.score + float(v)) - for k, v in zip(topk_idx[i], topk_logp[i]) - ] - k_best_exp = max(hyp_i, key=lambda x: x[1])[1] - - k_expansions.append( - sorted( - filter( - lambda x: (k_best_exp - self.expansion_gamma) <= x[1], hyp_i - ), - key=lambda x: x[1], - reverse=True, - ) - ) - - return k_expansions - - def create_lm_batch_inputs(self, hyps_seq: List[List[int]]) -> torch.Tensor: - """Make batch of inputs with left padding for LM scoring. - - Args: - hyps_seq: Hypothesis sequences. - - Returns: - : Padded batch of sequences. - - """ - max_len = max([len(h) for h in hyps_seq]) - - return torch.LongTensor( - [[self.sos] + ([0] * (max_len - len(h))) + h[1:] for h in hyps_seq], - device=self.decoder.device, - ) - - def default_beam_search(self, enc_out: torch.Tensor) -> List[Hypothesis]: - """Beam search implementation without prefix search. - - Modified from https://arxiv.org/pdf/1211.3711.pdf - - Args: - enc_out: Encoder output sequence. (T, D) - - Returns: - nbest_hyps: N-best hypothesis. - - """ - beam_k = min(self.beam_size, (self.vocab_size - 1)) - max_t = len(enc_out) - - if self.search_cache is not None: - kept_hyps = self.search_cache - else: - kept_hyps = [ - Hypothesis( - score=0.0, - yseq=[0], - dec_state=self.decoder.init_state(1), - ) - ] - - for t in range(max_t): - hyps = kept_hyps - kept_hyps = [] - - while True: - max_hyp = max(hyps, key=lambda x: x.score) - hyps.remove(max_hyp) - - label = torch.full( - (1, 1), - max_hyp.yseq[-1], - dtype=torch.long, - device=self.decoder.device, - ) - dec_out, state = self.decoder.score( - label, - max_hyp.yseq, - max_hyp.dec_state, - ) - - logp = torch.log_softmax( - self.joint_network(enc_out[t : t + 1, :], dec_out), - dim=-1, - ).squeeze(0) - top_k = logp[1:].topk(beam_k, dim=-1) - - kept_hyps.append( - Hypothesis( - score=(max_hyp.score + float(logp[0:1])), - yseq=max_hyp.yseq, - dec_state=max_hyp.dec_state, - lm_state=max_hyp.lm_state, - ) - ) - - if self.use_lm: - lm_scores, lm_state = self.lm.score( - torch.LongTensor( - [self.sos] + max_hyp.yseq[1:], device=self.decoder.device - ), - max_hyp.lm_state, - None, - ) - else: - lm_state = max_hyp.lm_state - - for logp, k in zip(*top_k): - score = max_hyp.score + float(logp) - - if self.use_lm: - score += self.lm_weight * lm_scores[k + 1] - - hyps.append( - Hypothesis( - score=score, - yseq=max_hyp.yseq + [int(k + 1)], - dec_state=state, - lm_state=lm_state, - ) - ) - - hyps_max = float(max(hyps, key=lambda x: x.score).score) - kept_most_prob = sorted( - [hyp for hyp in kept_hyps if hyp.score > hyps_max], - key=lambda x: x.score, - ) - if len(kept_most_prob) >= self.beam_size: - kept_hyps = kept_most_prob - break - - return kept_hyps - - def align_length_sync_decoding( - self, - enc_out: torch.Tensor, - ) -> List[Hypothesis]: - """Alignment-length synchronous beam search implementation. - - Based on https://ieeexplore.ieee.org/document/9053040 - - Args: - h: Encoder output sequences. (T, D) - - Returns: - nbest_hyps: N-best hypothesis. - - """ - t_max = int(enc_out.size(0)) - u_max = min(self.u_max, (t_max - 1)) - - B = [Hypothesis(yseq=[0], score=0.0, dec_state=self.decoder.init_state(1))] - final = [] - - if self.use_lm: - B[0].lm_state = self.lm.zero_state() - - for i in range(t_max + u_max): - A = [] - - B_ = [] - B_enc_out = [] - for hyp in B: - u = len(hyp.yseq) - 1 - t = i - u - - if t > (t_max - 1): - continue - - B_.append(hyp) - B_enc_out.append((t, enc_out[t])) - - if B_: - beam_enc_out = torch.stack([b[1] for b in B_enc_out]) - beam_dec_out, beam_state = self.decoder.batch_score(B_) - - beam_logp = torch.log_softmax( - self.joint_network(beam_enc_out, beam_dec_out), - dim=-1, - ) - beam_topk = beam_logp[:, 1:].topk(self.beam_size, dim=-1) - - if self.use_lm: - beam_lm_scores, beam_lm_states = self.lm.batch_score( - self.create_lm_batch_inputs([b.yseq for b in B_]), - [b.lm_state for b in B_], - None, - ) - - for i, hyp in enumerate(B_): - new_hyp = Hypothesis( - score=(hyp.score + float(beam_logp[i, 0])), - yseq=hyp.yseq[:], - dec_state=hyp.dec_state, - lm_state=hyp.lm_state, - ) - - A.append(new_hyp) - - if B_enc_out[i][0] == (t_max - 1): - final.append(new_hyp) - - for logp, k in zip(beam_topk[0][i], beam_topk[1][i] + 1): - new_hyp = Hypothesis( - score=(hyp.score + float(logp)), - yseq=(hyp.yseq[:] + [int(k)]), - dec_state=self.decoder.select_state(beam_state, i), - lm_state=hyp.lm_state, - ) - - if self.use_lm: - new_hyp.score += self.lm_weight * beam_lm_scores[i, k] - new_hyp.lm_state = beam_lm_states[i] - - A.append(new_hyp) - - B = sorted(A, key=lambda x: x.score, reverse=True)[: self.beam_size] - B = self.recombine_hyps(B) - - if final: - return final - - return B - - def time_sync_decoding(self, enc_out: torch.Tensor) -> List[Hypothesis]: - """Time synchronous beam search implementation. - - Based on https://ieeexplore.ieee.org/document/9053040 - - Args: - enc_out: Encoder output sequence. (T, D) - - Returns: - nbest_hyps: N-best hypothesis. - - """ - if self.search_cache is not None: - B = self.search_cache - else: - B = [ - Hypothesis( - yseq=[0], - score=0.0, - dec_state=self.decoder.init_state(1), - ) - ] - - if self.use_lm: - B[0].lm_state = self.lm.zero_state() - - for enc_out_t in enc_out: - A = [] - C = B - - enc_out_t = enc_out_t.unsqueeze(0) - - for v in range(self.max_sym_exp): - D = [] - - beam_dec_out, beam_state = self.decoder.batch_score(C) - - beam_logp = torch.log_softmax( - self.joint_network(enc_out_t, beam_dec_out), - dim=-1, - ) - beam_topk = beam_logp[:, 1:].topk(self.beam_size, dim=-1) - - seq_A = [h.yseq for h in A] - - for i, hyp in enumerate(C): - if hyp.yseq not in seq_A: - A.append( - Hypothesis( - score=(hyp.score + float(beam_logp[i, 0])), - yseq=hyp.yseq[:], - dec_state=hyp.dec_state, - lm_state=hyp.lm_state, - ) - ) - else: - dict_pos = seq_A.index(hyp.yseq) - - A[dict_pos].score = np.logaddexp( - A[dict_pos].score, (hyp.score + float(beam_logp[i, 0])) - ) - - if v < (self.max_sym_exp - 1): - if self.use_lm: - beam_lm_scores, beam_lm_states = self.lm.batch_score( - self.create_lm_batch_inputs([c.yseq for c in C]), - [c.lm_state for c in C], - None, - ) - - for i, hyp in enumerate(C): - for logp, k in zip(beam_topk[0][i], beam_topk[1][i] + 1): - new_hyp = Hypothesis( - score=(hyp.score + float(logp)), - yseq=(hyp.yseq + [int(k)]), - dec_state=self.decoder.select_state(beam_state, i), - lm_state=hyp.lm_state, - ) - - if self.use_lm: - new_hyp.score += self.lm_weight * beam_lm_scores[i, k] - new_hyp.lm_state = beam_lm_states[i] - - D.append(new_hyp) - - C = sorted(D, key=lambda x: x.score, reverse=True)[: self.beam_size] - - B = sorted(A, key=lambda x: x.score, reverse=True)[: self.beam_size] - - return B - - def modified_adaptive_expansion_search( - self, - enc_out: torch.Tensor, - ) -> List[ExtendedHypothesis]: - """Modified version of Adaptive Expansion Search (mAES). - - Based on AES (https://ieeexplore.ieee.org/document/9250505) and - NSC (https://arxiv.org/abs/2201.05420). - - Args: - enc_out: Encoder output sequence. (T, D_enc) - - Returns: - nbest_hyps: N-best hypothesis. - - """ - if self.search_cache is not None: - kept_hyps = self.search_cache - else: - init_tokens = [ - ExtendedHypothesis( - yseq=[0], - score=0.0, - dec_state=self.decoder.init_state(1), - ) - ] - - beam_dec_out, beam_state = self.decoder.batch_score( - init_tokens, - ) - - if self.use_lm: - beam_lm_scores, beam_lm_states = self.lm.batch_score( - self.create_lm_batch_inputs([h.yseq for h in init_tokens]), - [h.lm_state for h in init_tokens], - None, - ) - - lm_state = beam_lm_states[0] - lm_score = beam_lm_scores[0] - else: - lm_state = None - lm_score = None - - kept_hyps = [ - ExtendedHypothesis( - yseq=[0], - score=0.0, - dec_state=self.decoder.select_state(beam_state, 0), - dec_out=beam_dec_out[0], - lm_state=lm_state, - lm_score=lm_score, - ) - ] - - for enc_out_t in enc_out: - hyps = kept_hyps - kept_hyps = [] - - beam_enc_out = enc_out_t.unsqueeze(0) - - list_b = [] - for n in range(self.nstep): - beam_dec_out = torch.stack([h.dec_out for h in hyps]) - - beam_logp, beam_idx = torch.log_softmax( - self.joint_network(beam_enc_out, beam_dec_out), - dim=-1, - ).topk(self.max_candidates, dim=-1) - - k_expansions = self.select_k_expansions(hyps, beam_idx, beam_logp) - - list_exp = [] - for i, hyp in enumerate(hyps): - for k, new_score in k_expansions[i]: - new_hyp = ExtendedHypothesis( - yseq=hyp.yseq[:], - score=new_score, - dec_out=hyp.dec_out, - dec_state=hyp.dec_state, - lm_state=hyp.lm_state, - lm_score=hyp.lm_score, - ) - - if k == 0: - list_b.append(new_hyp) - else: - new_hyp.yseq.append(int(k)) - - if self.use_lm: - new_hyp.score += self.lm_weight * float(hyp.lm_score[k]) - - list_exp.append(new_hyp) - - if not list_exp: - kept_hyps = sorted( - self.recombine_hyps(list_b), key=lambda x: x.score, reverse=True - )[: self.beam_size] - - break - else: - beam_dec_out, beam_state = self.decoder.batch_score( - list_exp, - ) - - if self.use_lm: - beam_lm_scores, beam_lm_states = self.lm.batch_score( - self.create_lm_batch_inputs([h.yseq for h in list_exp]), - [h.lm_state for h in list_exp], - None, - ) - - if n < (self.nstep - 1): - for i, hyp in enumerate(list_exp): - hyp.dec_out = beam_dec_out[i] - hyp.dec_state = self.decoder.select_state(beam_state, i) - - if self.use_lm: - hyp.lm_state = beam_lm_states[i] - hyp.lm_score = beam_lm_scores[i] - - hyps = list_exp[:] - else: - beam_logp = torch.log_softmax( - self.joint_network(beam_enc_out, beam_dec_out), - dim=-1, - ) - - for i, hyp in enumerate(list_exp): - hyp.score += float(beam_logp[i, 0]) - - hyp.dec_out = beam_dec_out[i] - hyp.dec_state = self.decoder.select_state(beam_state, i) - - if self.use_lm: - hyp.lm_state = beam_lm_states[i] - hyp.lm_score = beam_lm_scores[i] - - kept_hyps = sorted( - self.recombine_hyps(list_b + list_exp), - key=lambda x: x.score, - reverse=True, - )[: self.beam_size] - - return kept_hyps diff --git a/funasr_detach/models/transducer/joint_network.py b/funasr_detach/models/transducer/joint_network.py deleted file mode 100644 index b0916d9b810db281aff242408d2881e2ea4027f5..0000000000000000000000000000000000000000 --- a/funasr_detach/models/transducer/joint_network.py +++ /dev/null @@ -1,66 +0,0 @@ -#!/usr/bin/env python3 -# -*- encoding: utf-8 -*- -# Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved. -# MIT License (https://opensource.org/licenses/MIT) - -import torch - -from funasr_detach.register import tables -from funasr_detach.models.transformer.utils.nets_utils import get_activation - - -@tables.register("joint_network_classes", "joint_network") -class JointNetwork(torch.nn.Module): - """Transducer joint network module. - - Args: - output_size: Output size. - encoder_size: Encoder output size. - decoder_size: Decoder output size.. - joint_space_size: Joint space size. - joint_act_type: Type of activation for joint network. - **activation_parameters: Parameters for the activation function. - - """ - - def __init__( - self, - output_size: int, - encoder_size: int, - decoder_size: int, - joint_space_size: int = 256, - joint_activation_type: str = "tanh", - ) -> None: - """Construct a JointNetwork object.""" - super().__init__() - - self.lin_enc = torch.nn.Linear(encoder_size, joint_space_size) - self.lin_dec = torch.nn.Linear(decoder_size, joint_space_size, bias=False) - - self.lin_out = torch.nn.Linear(joint_space_size, output_size) - - self.joint_activation = get_activation(joint_activation_type) - - def forward( - self, - enc_out: torch.Tensor, - dec_out: torch.Tensor, - project_input: bool = True, - ) -> torch.Tensor: - """Joint computation of encoder and decoder hidden state sequences. - - Args: - enc_out: Expanded encoder output state sequences (B, T, 1, D_enc) - dec_out: Expanded decoder output state sequences (B, 1, U, D_dec) - - Returns: - joint_out: Joint output state sequences. (B, T, U, D_out) - - """ - if project_input: - joint_out = self.joint_activation( - self.lin_enc(enc_out) + self.lin_dec(dec_out) - ) - else: - joint_out = self.joint_activation(enc_out + dec_out) - return self.lin_out(joint_out) diff --git a/funasr_detach/models/transducer/model.py b/funasr_detach/models/transducer/model.py deleted file mode 100644 index 67615d9ceb02abbf6baa04436682f4d73314b336..0000000000000000000000000000000000000000 --- a/funasr_detach/models/transducer/model.py +++ /dev/null @@ -1,558 +0,0 @@ -#!/usr/bin/env python3 -# -*- encoding: utf-8 -*- -# Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved. -# MIT License (https://opensource.org/licenses/MIT) - -import time -import torch -import logging -from contextlib import contextmanager -from typing import Dict, Optional, Tuple -from distutils.version import LooseVersion - -from funasr_detach.register import tables -from funasr_detach.utils import postprocess_utils -from funasr_detach.utils.datadir_writer import DatadirWriter -from funasr_detach.train_utils.device_funcs import force_gatherable -from funasr_detach.models.transformer.scorers.ctc import CTCPrefixScorer -from funasr_detach.losses.label_smoothing_loss import LabelSmoothingLoss -from funasr_detach.models.transformer.scorers.length_bonus import LengthBonus -from funasr_detach.models.transformer.utils.nets_utils import get_transducer_task_io -from funasr_detach.utils.load_utils import load_audio_text_image_video, extract_fbank -from funasr_detach.models.transducer.beam_search_transducer import BeamSearchTransducer - - -if LooseVersion(torch.__version__) >= LooseVersion("1.6.0"): - from torch.cuda.amp import autocast -else: - # Nothing to do if torch<1.6.0 - @contextmanager - def autocast(enabled=True): - yield - - -@tables.register("model_classes", "Transducer") -class Transducer(torch.nn.Module): - def __init__( - self, - frontend: Optional[str] = None, - frontend_conf: Optional[Dict] = None, - specaug: Optional[str] = None, - specaug_conf: Optional[Dict] = None, - normalize: str = None, - normalize_conf: Optional[Dict] = None, - encoder: str = None, - encoder_conf: Optional[Dict] = None, - decoder: str = None, - decoder_conf: Optional[Dict] = None, - joint_network: str = None, - joint_network_conf: Optional[Dict] = None, - transducer_weight: float = 1.0, - fastemit_lambda: float = 0.0, - auxiliary_ctc_weight: float = 0.0, - auxiliary_ctc_dropout_rate: float = 0.0, - auxiliary_lm_loss_weight: float = 0.0, - auxiliary_lm_loss_smoothing: float = 0.0, - input_size: int = 80, - vocab_size: int = -1, - ignore_id: int = -1, - blank_id: int = 0, - sos: int = 1, - eos: int = 2, - lsm_weight: float = 0.0, - length_normalized_loss: bool = False, - # report_cer: bool = True, - # report_wer: bool = True, - # sym_space: str = "", - # sym_blank: str = "", - # extract_feats_in_collect_stats: bool = True, - share_embedding: bool = False, - # preencoder: Optional[AbsPreEncoder] = None, - # postencoder: Optional[AbsPostEncoder] = None, - **kwargs, - ): - - super().__init__() - - if specaug is not None: - specaug_class = tables.specaug_classes.get(specaug) - specaug = specaug_class(**specaug_conf) - if normalize is not None: - normalize_class = tables.normalize_classes.get(normalize) - normalize = normalize_class(**normalize_conf) - encoder_class = tables.encoder_classes.get(encoder) - encoder = encoder_class(input_size=input_size, **encoder_conf) - encoder_output_size = encoder.output_size() - - decoder_class = tables.decoder_classes.get(decoder) - decoder = decoder_class( - vocab_size=vocab_size, - **decoder_conf, - ) - decoder_output_size = decoder.output_size - - joint_network_class = tables.joint_network_classes.get(joint_network) - joint_network = joint_network_class( - vocab_size, - encoder_output_size, - decoder_output_size, - **joint_network_conf, - ) - - self.criterion_transducer = None - self.error_calculator = None - - self.use_auxiliary_ctc = auxiliary_ctc_weight > 0 - self.use_auxiliary_lm_loss = auxiliary_lm_loss_weight > 0 - - if self.use_auxiliary_ctc: - self.ctc_lin = torch.nn.Linear(encoder.output_size(), vocab_size) - self.ctc_dropout_rate = auxiliary_ctc_dropout_rate - - if self.use_auxiliary_lm_loss: - self.lm_lin = torch.nn.Linear(decoder.output_size, vocab_size) - self.lm_loss_smoothing = auxiliary_lm_loss_smoothing - - self.transducer_weight = transducer_weight - self.fastemit_lambda = fastemit_lambda - - self.auxiliary_ctc_weight = auxiliary_ctc_weight - self.auxiliary_lm_loss_weight = auxiliary_lm_loss_weight - self.blank_id = blank_id - self.sos = sos if sos is not None else vocab_size - 1 - self.eos = eos if eos is not None else vocab_size - 1 - self.vocab_size = vocab_size - self.ignore_id = ignore_id - self.frontend = frontend - self.specaug = specaug - self.normalize = normalize - self.encoder = encoder - self.decoder = decoder - self.joint_network = joint_network - - self.criterion_att = LabelSmoothingLoss( - size=vocab_size, - padding_idx=ignore_id, - smoothing=lsm_weight, - normalize_length=length_normalized_loss, - ) - - self.length_normalized_loss = length_normalized_loss - self.beam_search = None - self.ctc = None - self.ctc_weight = 0.0 - - def forward( - self, - speech: torch.Tensor, - speech_lengths: torch.Tensor, - text: torch.Tensor, - text_lengths: torch.Tensor, - **kwargs, - ) -> Tuple[torch.Tensor, Dict[str, torch.Tensor], torch.Tensor]: - """Encoder + Decoder + Calc loss - Args: - speech: (Batch, Length, ...) - speech_lengths: (Batch, ) - text: (Batch, Length) - text_lengths: (Batch,) - """ - if len(text_lengths.size()) > 1: - text_lengths = text_lengths[:, 0] - if len(speech_lengths.size()) > 1: - speech_lengths = speech_lengths[:, 0] - - batch_size = speech.shape[0] - # 1. Encoder - encoder_out, encoder_out_lens = self.encode(speech, speech_lengths) - if ( - hasattr(self.encoder, "overlap_chunk_cls") - and self.encoder.overlap_chunk_cls is not None - ): - encoder_out, encoder_out_lens = self.encoder.overlap_chunk_cls.remove_chunk( - encoder_out, encoder_out_lens, chunk_outs=None - ) - # 2. Transducer-related I/O preparation - decoder_in, target, t_len, u_len = get_transducer_task_io( - text, - encoder_out_lens, - ignore_id=self.ignore_id, - ) - - # 3. Decoder - self.decoder.set_device(encoder_out.device) - decoder_out = self.decoder(decoder_in, u_len) - - # 4. Joint Network - joint_out = self.joint_network( - encoder_out.unsqueeze(2), decoder_out.unsqueeze(1) - ) - - # 5. Losses - loss_trans, cer_trans, wer_trans = self._calc_transducer_loss( - encoder_out, - joint_out, - target, - t_len, - u_len, - ) - - loss_ctc, loss_lm = 0.0, 0.0 - - if self.use_auxiliary_ctc: - loss_ctc = self._calc_ctc_loss( - encoder_out, - target, - t_len, - u_len, - ) - - if self.use_auxiliary_lm_loss: - loss_lm = self._calc_lm_loss(decoder_out, target) - - loss = ( - self.transducer_weight * loss_trans - + self.auxiliary_ctc_weight * loss_ctc - + self.auxiliary_lm_loss_weight * loss_lm - ) - - stats = dict( - loss=loss.detach(), - loss_transducer=loss_trans.detach(), - aux_ctc_loss=loss_ctc.detach() if loss_ctc > 0.0 else None, - aux_lm_loss=loss_lm.detach() if loss_lm > 0.0 else None, - cer_transducer=cer_trans, - wer_transducer=wer_trans, - ) - - # force_gatherable: to-device and to-tensor if scalar for DataParallel - loss, stats, weight = force_gatherable((loss, stats, batch_size), loss.device) - - return loss, stats, weight - - def encode( - self, - speech: torch.Tensor, - speech_lengths: torch.Tensor, - **kwargs, - ) -> Tuple[torch.Tensor, torch.Tensor]: - """Frontend + Encoder. Note that this method is used by asr_inference.py - Args: - speech: (Batch, Length, ...) - speech_lengths: (Batch, ) - ind: int - """ - with autocast(False): - - # Data augmentation - if self.specaug is not None and self.training: - speech, speech_lengths = self.specaug(speech, speech_lengths) - - # Normalization for feature: e.g. Global-CMVN, Utterance-CMVN - if self.normalize is not None: - speech, speech_lengths = self.normalize(speech, speech_lengths) - - # Forward encoder - # feats: (Batch, Length, Dim) - # -> encoder_out: (Batch, Length2, Dim2) - encoder_out, encoder_out_lens, _ = self.encoder(speech, speech_lengths) - intermediate_outs = None - if isinstance(encoder_out, tuple): - intermediate_outs = encoder_out[1] - encoder_out = encoder_out[0] - - if intermediate_outs is not None: - return (encoder_out, intermediate_outs), encoder_out_lens - - return encoder_out, encoder_out_lens - - def _calc_transducer_loss( - self, - encoder_out: torch.Tensor, - joint_out: torch.Tensor, - target: torch.Tensor, - t_len: torch.Tensor, - u_len: torch.Tensor, - ) -> Tuple[torch.Tensor, Optional[float], Optional[float]]: - """Compute Transducer loss. - - Args: - encoder_out: Encoder output sequences. (B, T, D_enc) - joint_out: Joint Network output sequences (B, T, U, D_joint) - target: Target label ID sequences. (B, L) - t_len: Encoder output sequences lengths. (B,) - u_len: Target label ID sequences lengths. (B,) - - Return: - loss_transducer: Transducer loss value. - cer_transducer: Character error rate for Transducer. - wer_transducer: Word Error Rate for Transducer. - - """ - if self.criterion_transducer is None: - try: - from warp_rnnt import rnnt_loss as RNNTLoss - - self.criterion_transducer = RNNTLoss - - except ImportError: - logging.error( - "warp-rnnt was not installed." - "Please consult the installation documentation." - ) - exit(1) - - log_probs = torch.log_softmax(joint_out, dim=-1) - - loss_transducer = self.criterion_transducer( - log_probs, - target, - t_len, - u_len, - reduction="mean", - blank=self.blank_id, - fastemit_lambda=self.fastemit_lambda, - gather=True, - ) - - if not self.training and (self.report_cer or self.report_wer): - if self.error_calculator is None: - from funasr_detach.metrics import ( - ErrorCalculatorTransducer as ErrorCalculator, - ) - - self.error_calculator = ErrorCalculator( - self.decoder, - self.joint_network, - self.token_list, - self.sym_space, - self.sym_blank, - report_cer=self.report_cer, - report_wer=self.report_wer, - ) - - cer_transducer, wer_transducer = self.error_calculator( - encoder_out, target, t_len - ) - - return loss_transducer, cer_transducer, wer_transducer - - return loss_transducer, None, None - - def _calc_ctc_loss( - self, - encoder_out: torch.Tensor, - target: torch.Tensor, - t_len: torch.Tensor, - u_len: torch.Tensor, - ) -> torch.Tensor: - """Compute CTC loss. - - Args: - encoder_out: Encoder output sequences. (B, T, D_enc) - target: Target label ID sequences. (B, L) - t_len: Encoder output sequences lengths. (B,) - u_len: Target label ID sequences lengths. (B,) - - Return: - loss_ctc: CTC loss value. - - """ - ctc_in = self.ctc_lin( - torch.nn.functional.dropout(encoder_out, p=self.ctc_dropout_rate) - ) - ctc_in = torch.log_softmax(ctc_in.transpose(0, 1), dim=-1) - - target_mask = target != 0 - ctc_target = target[target_mask].cpu() - - with torch.backends.cudnn.flags(deterministic=True): - loss_ctc = torch.nn.functional.ctc_loss( - ctc_in, - ctc_target, - t_len, - u_len, - zero_infinity=True, - reduction="sum", - ) - loss_ctc /= target.size(0) - - return loss_ctc - - def _calc_lm_loss( - self, - decoder_out: torch.Tensor, - target: torch.Tensor, - ) -> torch.Tensor: - """Compute LM loss. - - Args: - decoder_out: Decoder output sequences. (B, U, D_dec) - target: Target label ID sequences. (B, L) - - Return: - loss_lm: LM loss value. - - """ - lm_loss_in = self.lm_lin(decoder_out[:, :-1, :]).view(-1, self.vocab_size) - lm_target = target.view(-1).type(torch.int64) - - with torch.no_grad(): - true_dist = lm_loss_in.clone() - true_dist.fill_(self.lm_loss_smoothing / (self.vocab_size - 1)) - - # Ignore blank ID (0) - ignore = lm_target == 0 - lm_target = lm_target.masked_fill(ignore, 0) - - true_dist.scatter_(1, lm_target.unsqueeze(1), (1 - self.lm_loss_smoothing)) - - loss_lm = torch.nn.functional.kl_div( - torch.log_softmax(lm_loss_in, dim=1), - true_dist, - reduction="none", - ) - loss_lm = loss_lm.masked_fill(ignore.unsqueeze(1), 0).sum() / decoder_out.size( - 0 - ) - - return loss_lm - - def init_beam_search( - self, - **kwargs, - ): - - # 1. Build ASR model - scorers = {} - - if self.ctc != None: - ctc = CTCPrefixScorer(ctc=self.ctc, eos=self.eos) - scorers.update(ctc=ctc) - token_list = kwargs.get("token_list") - scorers.update( - length_bonus=LengthBonus(len(token_list)), - ) - - # 3. Build ngram model - # ngram is not supported now - ngram = None - scorers["ngram"] = ngram - - beam_search = BeamSearchTransducer( - self.decoder, - self.joint_network, - kwargs.get("beam_size", 2), - nbest=1, - ) - # beam_search.to(device=kwargs.get("device", "cpu"), dtype=getattr(torch, kwargs.get("dtype", "float32"))).eval() - # for scorer in scorers.values(): - # if isinstance(scorer, torch.nn.Module): - # scorer.to(device=kwargs.get("device", "cpu"), dtype=getattr(torch, kwargs.get("dtype", "float32"))).eval() - self.beam_search = beam_search - - def inference( - self, - data_in: list, - data_lengths: list = None, - key: list = None, - tokenizer=None, - **kwargs, - ): - - if kwargs.get("batch_size", 1) > 1: - raise NotImplementedError("batch decoding is not implemented") - - # init beamsearch - is_use_ctc = ( - kwargs.get("decoding_ctc_weight", 0.0) > 0.00001 and self.ctc != None - ) - is_use_lm = ( - kwargs.get("lm_weight", 0.0) > 0.00001 - and kwargs.get("lm_file", None) is not None - ) - # if self.beam_search is None and (is_use_lm or is_use_ctc): - logging.info("enable beam_search") - self.init_beam_search(**kwargs) - self.nbest = kwargs.get("nbest", 1) - - meta_data = {} - # extract fbank feats - time1 = time.perf_counter() - audio_sample_list = load_audio_text_image_video( - data_in, fs=self.frontend.fs, audio_fs=kwargs.get("fs", 16000) - ) - time2 = time.perf_counter() - meta_data["load_data"] = f"{time2 - time1:0.3f}" - speech, speech_lengths = extract_fbank( - audio_sample_list, - data_type=kwargs.get("data_type", "sound"), - frontend=self.frontend, - ) - time3 = time.perf_counter() - meta_data["extract_feat"] = f"{time3 - time2:0.3f}" - meta_data["batch_data_time"] = ( - speech_lengths.sum().item() - * self.frontend.frame_shift - * self.frontend.lfr_n - / 1000 - ) - - speech = speech.to(device=kwargs["device"]) - speech_lengths = speech_lengths.to(device=kwargs["device"]) - - # Encoder - encoder_out, encoder_out_lens = self.encode(speech, speech_lengths) - if isinstance(encoder_out, tuple): - encoder_out = encoder_out[0] - - # c. Passed the encoder result and the beam search - nbest_hyps = self.beam_search(encoder_out[0], is_final=True) - nbest_hyps = nbest_hyps[: self.nbest] - - results = [] - b, n, d = encoder_out.size() - for i in range(b): - - for nbest_idx, hyp in enumerate(nbest_hyps): - ibest_writer = None - if kwargs.get("output_dir") is not None: - if not hasattr(self, "writer"): - self.writer = DatadirWriter(kwargs.get("output_dir")) - ibest_writer = self.writer[f"{nbest_idx + 1}best_recog"] - # remove sos/eos and get results - last_pos = -1 - if isinstance(hyp.yseq, list): - token_int = hyp.yseq # [1:last_pos] - else: - token_int = hyp.yseq # [1:last_pos].tolist() - - # remove blank symbol id, which is assumed to be 0 - token_int = list( - filter( - lambda x: x != self.eos - and x != self.sos - and x != self.blank_id, - token_int, - ) - ) - - # Change integer-ids to tokens - token = tokenizer.ids2tokens(token_int) - text = tokenizer.tokens2text(token) - - text_postprocessed, _ = postprocess_utils.sentence_postprocess(token) - result_i = { - "key": key[i], - "token": token, - "text": text, - "text_postprocessed": text_postprocessed, - } - results.append(result_i) - - if ibest_writer is not None: - ibest_writer["token"][key[i]] = " ".join(token) - ibest_writer["text"][key[i]] = text - ibest_writer["text_postprocessed"][key[i]] = text_postprocessed - - return results, meta_data diff --git a/funasr_detach/models/transducer/rnn_decoder.py b/funasr_detach/models/transducer/rnn_decoder.py deleted file mode 100644 index c4aa3299f00211922c637070a450bd3d7b1cb40c..0000000000000000000000000000000000000000 --- a/funasr_detach/models/transducer/rnn_decoder.py +++ /dev/null @@ -1,341 +0,0 @@ -#!/usr/bin/env python3 -# -*- encoding: utf-8 -*- -# Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved. -# MIT License (https://opensource.org/licenses/MIT) - -import torch -import random -import numpy as np -import torch.nn as nn -import torch.nn.functional as F - -from funasr_detach.register import tables -from funasr_detach.models.transformer.utils.nets_utils import make_pad_mask -from funasr_detach.models.transformer.utils.nets_utils import to_device -from funasr_detach.models.language_model.rnn.attentions import initial_att - - -def build_attention_list( - eprojs: int, - dunits: int, - atype: str = "location", - num_att: int = 1, - num_encs: int = 1, - aheads: int = 4, - adim: int = 320, - awin: int = 5, - aconv_chans: int = 10, - aconv_filts: int = 100, - han_mode: bool = False, - han_type=None, - han_heads: int = 4, - han_dim: int = 320, - han_conv_chans: int = -1, - han_conv_filts: int = 100, - han_win: int = 5, -): - - att_list = torch.nn.ModuleList() - if num_encs == 1: - for i in range(num_att): - att = initial_att( - atype, - eprojs, - dunits, - aheads, - adim, - awin, - aconv_chans, - aconv_filts, - ) - att_list.append(att) - elif num_encs > 1: # no multi-speaker mode - if han_mode: - att = initial_att( - han_type, - eprojs, - dunits, - han_heads, - han_dim, - han_win, - han_conv_chans, - han_conv_filts, - han_mode=True, - ) - return att - else: - att_list = torch.nn.ModuleList() - for idx in range(num_encs): - att = initial_att( - atype[idx], - eprojs, - dunits, - aheads[idx], - adim[idx], - awin[idx], - aconv_chans[idx], - aconv_filts[idx], - ) - att_list.append(att) - else: - raise ValueError( - "Number of encoders needs to be more than one. {}".format(num_encs) - ) - return att_list - - -@tables.register("decoder_classes", "rnn_decoder") -class RNNDecoder(nn.Module): - def __init__( - self, - vocab_size: int, - encoder_output_size: int, - rnn_type: str = "lstm", - num_layers: int = 1, - hidden_size: int = 320, - sampling_probability: float = 0.0, - dropout: float = 0.0, - context_residual: bool = False, - replace_sos: bool = False, - num_encs: int = 1, - att_conf: dict = None, - ): - # FIXME(kamo): The parts of num_spk should be refactored more more more - if rnn_type not in {"lstm", "gru"}: - raise ValueError(f"Not supported: rnn_type={rnn_type}") - - super().__init__() - eprojs = encoder_output_size - self.dtype = rnn_type - self.dunits = hidden_size - self.dlayers = num_layers - self.context_residual = context_residual - self.sos = vocab_size - 1 - self.eos = vocab_size - 1 - self.odim = vocab_size - self.sampling_probability = sampling_probability - self.dropout = dropout - self.num_encs = num_encs - - # for multilingual translation - self.replace_sos = replace_sos - - self.embed = torch.nn.Embedding(vocab_size, hidden_size) - self.dropout_emb = torch.nn.Dropout(p=dropout) - - self.decoder = torch.nn.ModuleList() - self.dropout_dec = torch.nn.ModuleList() - self.decoder += [ - ( - torch.nn.LSTMCell(hidden_size + eprojs, hidden_size) - if self.dtype == "lstm" - else torch.nn.GRUCell(hidden_size + eprojs, hidden_size) - ) - ] - self.dropout_dec += [torch.nn.Dropout(p=dropout)] - for _ in range(1, self.dlayers): - self.decoder += [ - ( - torch.nn.LSTMCell(hidden_size, hidden_size) - if self.dtype == "lstm" - else torch.nn.GRUCell(hidden_size, hidden_size) - ) - ] - self.dropout_dec += [torch.nn.Dropout(p=dropout)] - # NOTE: dropout is applied only for the vertical connections - # see https://arxiv.org/pdf/1409.2329.pdf - - if context_residual: - self.output = torch.nn.Linear(hidden_size + eprojs, vocab_size) - else: - self.output = torch.nn.Linear(hidden_size, vocab_size) - - self.att_list = build_attention_list( - eprojs=eprojs, dunits=hidden_size, **att_conf - ) - - def zero_state(self, hs_pad): - return hs_pad.new_zeros(hs_pad.size(0), self.dunits) - - def rnn_forward(self, ey, z_list, c_list, z_prev, c_prev): - if self.dtype == "lstm": - z_list[0], c_list[0] = self.decoder[0](ey, (z_prev[0], c_prev[0])) - for i in range(1, self.dlayers): - z_list[i], c_list[i] = self.decoder[i]( - self.dropout_dec[i - 1](z_list[i - 1]), - (z_prev[i], c_prev[i]), - ) - else: - z_list[0] = self.decoder[0](ey, z_prev[0]) - for i in range(1, self.dlayers): - z_list[i] = self.decoder[i]( - self.dropout_dec[i - 1](z_list[i - 1]), z_prev[i] - ) - return z_list, c_list - - def forward(self, hs_pad, hlens, ys_in_pad, ys_in_lens, strm_idx=0): - # to support mutiple encoder asr mode, in single encoder mode, - # convert torch.Tensor to List of torch.Tensor - if self.num_encs == 1: - hs_pad = [hs_pad] - hlens = [hlens] - - # attention index for the attention module - # in SPA (speaker parallel attention), - # att_idx is used to select attention module. In other cases, it is 0. - att_idx = min(strm_idx, len(self.att_list) - 1) - - # hlens should be list of list of integer - hlens = [list(map(int, hlens[idx])) for idx in range(self.num_encs)] - - # get dim, length info - olength = ys_in_pad.size(1) - - # initialization - c_list = [self.zero_state(hs_pad[0])] - z_list = [self.zero_state(hs_pad[0])] - for _ in range(1, self.dlayers): - c_list.append(self.zero_state(hs_pad[0])) - z_list.append(self.zero_state(hs_pad[0])) - z_all = [] - if self.num_encs == 1: - att_w = None - self.att_list[att_idx].reset() # reset pre-computation of h - else: - att_w_list = [None] * (self.num_encs + 1) # atts + han - att_c_list = [None] * self.num_encs # atts - for idx in range(self.num_encs + 1): - # reset pre-computation of h in atts and han - self.att_list[idx].reset() - - # pre-computation of embedding - eys = self.dropout_emb(self.embed(ys_in_pad)) # utt x olen x zdim - - # loop for an output sequence - for i in range(olength): - if self.num_encs == 1: - att_c, att_w = self.att_list[att_idx]( - hs_pad[0], hlens[0], self.dropout_dec[0](z_list[0]), att_w - ) - else: - for idx in range(self.num_encs): - att_c_list[idx], att_w_list[idx] = self.att_list[idx]( - hs_pad[idx], - hlens[idx], - self.dropout_dec[0](z_list[0]), - att_w_list[idx], - ) - hs_pad_han = torch.stack(att_c_list, dim=1) - hlens_han = [self.num_encs] * len(ys_in_pad) - att_c, att_w_list[self.num_encs] = self.att_list[self.num_encs]( - hs_pad_han, - hlens_han, - self.dropout_dec[0](z_list[0]), - att_w_list[self.num_encs], - ) - if i > 0 and random.random() < self.sampling_probability: - z_out = self.output(z_all[-1]) - z_out = np.argmax(z_out.detach().cpu(), axis=1) - z_out = self.dropout_emb(self.embed(to_device(self, z_out))) - ey = torch.cat((z_out, att_c), dim=1) # utt x (zdim + hdim) - else: - # utt x (zdim + hdim) - ey = torch.cat((eys[:, i, :], att_c), dim=1) - z_list, c_list = self.rnn_forward(ey, z_list, c_list, z_list, c_list) - if self.context_residual: - z_all.append( - torch.cat((self.dropout_dec[-1](z_list[-1]), att_c), dim=-1) - ) # utt x (zdim + hdim) - else: - z_all.append(self.dropout_dec[-1](z_list[-1])) # utt x (zdim) - - z_all = torch.stack(z_all, dim=1) - z_all = self.output(z_all) - z_all.masked_fill_( - make_pad_mask(ys_in_lens, z_all, 1), - 0, - ) - return z_all, ys_in_lens - - def init_state(self, x): - # to support mutiple encoder asr mode, in single encoder mode, - # convert torch.Tensor to List of torch.Tensor - if self.num_encs == 1: - x = [x] - - c_list = [self.zero_state(x[0].unsqueeze(0))] - z_list = [self.zero_state(x[0].unsqueeze(0))] - for _ in range(1, self.dlayers): - c_list.append(self.zero_state(x[0].unsqueeze(0))) - z_list.append(self.zero_state(x[0].unsqueeze(0))) - # TODO(karita): support strm_index for `asr_mix` - strm_index = 0 - att_idx = min(strm_index, len(self.att_list) - 1) - if self.num_encs == 1: - a = None - self.att_list[att_idx].reset() # reset pre-computation of h - else: - a = [None] * (self.num_encs + 1) # atts + han - for idx in range(self.num_encs + 1): - # reset pre-computation of h in atts and han - self.att_list[idx].reset() - return dict( - c_prev=c_list[:], - z_prev=z_list[:], - a_prev=a, - workspace=(att_idx, z_list, c_list), - ) - - def score(self, yseq, state, x): - # to support mutiple encoder asr mode, in single encoder mode, - # convert torch.Tensor to List of torch.Tensor - if self.num_encs == 1: - x = [x] - - att_idx, z_list, c_list = state["workspace"] - vy = yseq[-1].unsqueeze(0) - ey = self.dropout_emb(self.embed(vy)) # utt list (1) x zdim - if self.num_encs == 1: - att_c, att_w = self.att_list[att_idx]( - x[0].unsqueeze(0), - [x[0].size(0)], - self.dropout_dec[0](state["z_prev"][0]), - state["a_prev"], - ) - else: - att_w = [None] * (self.num_encs + 1) # atts + han - att_c_list = [None] * self.num_encs # atts - for idx in range(self.num_encs): - att_c_list[idx], att_w[idx] = self.att_list[idx]( - x[idx].unsqueeze(0), - [x[idx].size(0)], - self.dropout_dec[0](state["z_prev"][0]), - state["a_prev"][idx], - ) - h_han = torch.stack(att_c_list, dim=1) - att_c, att_w[self.num_encs] = self.att_list[self.num_encs]( - h_han, - [self.num_encs], - self.dropout_dec[0](state["z_prev"][0]), - state["a_prev"][self.num_encs], - ) - ey = torch.cat((ey, att_c), dim=1) # utt(1) x (zdim + hdim) - z_list, c_list = self.rnn_forward( - ey, z_list, c_list, state["z_prev"], state["c_prev"] - ) - if self.context_residual: - logits = self.output( - torch.cat((self.dropout_dec[-1](z_list[-1]), att_c), dim=-1) - ) - else: - logits = self.output(self.dropout_dec[-1](z_list[-1])) - logp = F.log_softmax(logits, dim=1).squeeze(0) - return ( - logp, - dict( - c_prev=c_list[:], - z_prev=z_list[:], - a_prev=att_w, - workspace=(att_idx, z_list, c_list), - ), - ) diff --git a/funasr_detach/models/transducer/rnnt_decoder.py b/funasr_detach/models/transducer/rnnt_decoder.py deleted file mode 100644 index 188f37e445a986a3f67c808866d34aa607b2570a..0000000000000000000000000000000000000000 --- a/funasr_detach/models/transducer/rnnt_decoder.py +++ /dev/null @@ -1,274 +0,0 @@ -#!/usr/bin/env python3 -# -*- encoding: utf-8 -*- -# Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved. -# MIT License (https://opensource.org/licenses/MIT) - -import torch -from typing import List, Optional, Tuple - -from funasr_detach.register import tables -from funasr_detach.models.specaug.specaug import SpecAug -from funasr_detach.models.transducer.beam_search_transducer import Hypothesis - - -@tables.register("decoder_classes", "rnnt_decoder") -class RNNTDecoder(torch.nn.Module): - """RNN decoder module. - - Args: - vocab_size: Vocabulary size. - embed_size: Embedding size. - hidden_size: Hidden size.. - rnn_type: Decoder layers type. - num_layers: Number of decoder layers. - dropout_rate: Dropout rate for decoder layers. - embed_dropout_rate: Dropout rate for embedding layer. - embed_pad: Embedding padding symbol ID. - - """ - - def __init__( - self, - vocab_size: int, - embed_size: int = 256, - hidden_size: int = 256, - rnn_type: str = "lstm", - num_layers: int = 1, - dropout_rate: float = 0.0, - embed_dropout_rate: float = 0.0, - embed_pad: int = 0, - use_embed_mask: bool = False, - ) -> None: - """Construct a RNNDecoder object.""" - super().__init__() - - if rnn_type not in ("lstm", "gru"): - raise ValueError(f"Not supported: rnn_type={rnn_type}") - - self.embed = torch.nn.Embedding(vocab_size, embed_size, padding_idx=embed_pad) - self.dropout_embed = torch.nn.Dropout(p=embed_dropout_rate) - - rnn_class = torch.nn.LSTM if rnn_type == "lstm" else torch.nn.GRU - - self.rnn = torch.nn.ModuleList( - [rnn_class(embed_size, hidden_size, 1, batch_first=True)] - ) - - for _ in range(1, num_layers): - self.rnn += [rnn_class(hidden_size, hidden_size, 1, batch_first=True)] - - self.dropout_rnn = torch.nn.ModuleList( - [torch.nn.Dropout(p=dropout_rate) for _ in range(num_layers)] - ) - - self.dlayers = num_layers - self.dtype = rnn_type - - self.output_size = hidden_size - self.vocab_size = vocab_size - - self.device = next(self.parameters()).device - self.score_cache = {} - - self.use_embed_mask = use_embed_mask - if self.use_embed_mask: - self._embed_mask = SpecAug( - time_mask_width_range=3, - num_time_mask=4, - apply_freq_mask=False, - apply_time_warp=False, - ) - - def forward( - self, - labels: torch.Tensor, - label_lens: torch.Tensor, - states: Optional[Tuple[torch.Tensor, Optional[torch.Tensor]]] = None, - ) -> torch.Tensor: - """Encode source label sequences. - - Args: - labels: Label ID sequences. (B, L) - states: Decoder hidden states. - ((N, B, D_dec), (N, B, D_dec) or None) or None - - Returns: - dec_out: Decoder output sequences. (B, U, D_dec) - - """ - if states is None: - states = self.init_state(labels.size(0)) - - dec_embed = self.dropout_embed(self.embed(labels)) - if self.use_embed_mask and self.training: - dec_embed = self._embed_mask(dec_embed, label_lens)[0] - dec_out, states = self.rnn_forward(dec_embed, states) - return dec_out - - def rnn_forward( - self, - x: torch.Tensor, - state: Tuple[torch.Tensor, Optional[torch.Tensor]], - ) -> Tuple[torch.Tensor, Tuple[torch.Tensor, Optional[torch.Tensor]]]: - """Encode source label sequences. - - Args: - x: RNN input sequences. (B, D_emb) - state: Decoder hidden states. ((N, B, D_dec), (N, B, D_dec) or None) - - Returns: - x: RNN output sequences. (B, D_dec) - (h_next, c_next): Decoder hidden states. - (N, B, D_dec), (N, B, D_dec) or None) - - """ - h_prev, c_prev = state - h_next, c_next = self.init_state(x.size(0)) - - for layer in range(self.dlayers): - if self.dtype == "lstm": - x, (h_next[layer : layer + 1], c_next[layer : layer + 1]) = self.rnn[ - layer - ](x, hx=(h_prev[layer : layer + 1], c_prev[layer : layer + 1])) - else: - x, h_next[layer : layer + 1] = self.rnn[layer]( - x, hx=h_prev[layer : layer + 1] - ) - - x = self.dropout_rnn[layer](x) - - return x, (h_next, c_next) - - def score( - self, - label: torch.Tensor, - label_sequence: List[int], - dec_state: Tuple[torch.Tensor, Optional[torch.Tensor]], - ) -> Tuple[torch.Tensor, Tuple[torch.Tensor, Optional[torch.Tensor]]]: - """One-step forward hypothesis. - - Args: - label: Previous label. (1, 1) - label_sequence: Current label sequence. - dec_state: Previous decoder hidden states. - ((N, 1, D_dec), (N, 1, D_dec) or None) - - Returns: - dec_out: Decoder output sequence. (1, D_dec) - dec_state: Decoder hidden states. - ((N, 1, D_dec), (N, 1, D_dec) or None) - - """ - str_labels = "_".join(map(str, label_sequence)) - - if str_labels in self.score_cache: - dec_out, dec_state = self.score_cache[str_labels] - else: - dec_embed = self.embed(label) - dec_out, dec_state = self.rnn_forward(dec_embed, dec_state) - - self.score_cache[str_labels] = (dec_out, dec_state) - - return dec_out[0], dec_state - - def batch_score( - self, - hyps: List[Hypothesis], - ) -> Tuple[torch.Tensor, Tuple[torch.Tensor, Optional[torch.Tensor]]]: - """One-step forward hypotheses. - - Args: - hyps: Hypotheses. - - Returns: - dec_out: Decoder output sequences. (B, D_dec) - states: Decoder hidden states. ((N, B, D_dec), (N, B, D_dec) or None) - - """ - labels = torch.LongTensor([[h.yseq[-1]] for h in hyps], device=self.device) - dec_embed = self.embed(labels) - - states = self.create_batch_states([h.dec_state for h in hyps]) - dec_out, states = self.rnn_forward(dec_embed, states) - - return dec_out.squeeze(1), states - - def set_device(self, device: torch.device) -> None: - """Set GPU device to use. - - Args: - device: Device ID. - - """ - self.device = device - - def init_state( - self, batch_size: int - ) -> Tuple[torch.Tensor, Optional[torch.tensor]]: - """Initialize decoder states. - - Args: - batch_size: Batch size. - - Returns: - : Initial decoder hidden states. ((N, B, D_dec), (N, B, D_dec) or None) - - """ - h_n = torch.zeros( - self.dlayers, - batch_size, - self.output_size, - device=self.device, - ) - - if self.dtype == "lstm": - c_n = torch.zeros( - self.dlayers, - batch_size, - self.output_size, - device=self.device, - ) - - return (h_n, c_n) - - return (h_n, None) - - def select_state( - self, states: Tuple[torch.Tensor, Optional[torch.Tensor]], idx: int - ) -> Tuple[torch.Tensor, Optional[torch.Tensor]]: - """Get specified ID state from decoder hidden states. - - Args: - states: Decoder hidden states. ((N, B, D_dec), (N, B, D_dec) or None) - idx: State ID to extract. - - Returns: - : Decoder hidden state for given ID. ((N, 1, D_dec), (N, 1, D_dec) or None) - - """ - return ( - states[0][:, idx : idx + 1, :], - states[1][:, idx : idx + 1, :] if self.dtype == "lstm" else None, - ) - - def create_batch_states( - self, - new_states: List[Tuple[torch.Tensor, Optional[torch.Tensor]]], - ) -> Tuple[torch.Tensor, Optional[torch.Tensor]]: - """Create decoder hidden states. - - Args: - new_states: Decoder hidden states. [N x ((1, D_dec), (1, D_dec) or None)] - - Returns: - states: Decoder hidden states. ((N, B, D_dec), (N, B, D_dec) or None) - - """ - return ( - torch.cat([s[0] for s in new_states], dim=1), - ( - torch.cat([s[1] for s in new_states], dim=1) - if self.dtype == "lstm" - else None - ), - ) diff --git a/funasr_detach/models/transformer/__init__.py b/funasr_detach/models/transformer/__init__.py deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/funasr_detach/models/transformer/attention.py b/funasr_detach/models/transformer/attention.py deleted file mode 100644 index bb9a79e825ed1fc86b8fb8a45341e6543f1e3521..0000000000000000000000000000000000000000 --- a/funasr_detach/models/transformer/attention.py +++ /dev/null @@ -1,532 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- - -# Copyright 2019 Shigeki Karita -# Apache 2.0 (http://www.apache.org/licenses/LICENSE-2.0) - -"""Multi-Head Attention layer definition.""" - -import math - -import numpy -import torch -from torch import nn -from typing import Optional, Tuple - -import torch.nn.functional as F -from funasr_detach.models.transformer.utils.nets_utils import make_pad_mask -import funasr_detach.models.lora.layers as lora - - -class MultiHeadedAttention(nn.Module): - """Multi-Head Attention layer. - - Args: - n_head (int): The number of heads. - n_feat (int): The number of features. - dropout_rate (float): Dropout rate. - - """ - - def __init__(self, n_head, n_feat, dropout_rate): - """Construct an MultiHeadedAttention object.""" - super(MultiHeadedAttention, self).__init__() - assert n_feat % n_head == 0 - # We assume d_v always equals d_k - self.d_k = n_feat // n_head - self.h = n_head - self.linear_q = nn.Linear(n_feat, n_feat) - self.linear_k = nn.Linear(n_feat, n_feat) - self.linear_v = nn.Linear(n_feat, n_feat) - self.linear_out = nn.Linear(n_feat, n_feat) - self.attn = None - self.dropout = nn.Dropout(p=dropout_rate) - - def forward_qkv(self, query, key, value): - """Transform query, key and value. - - Args: - query (torch.Tensor): Query tensor (#batch, time1, size). - key (torch.Tensor): Key tensor (#batch, time2, size). - value (torch.Tensor): Value tensor (#batch, time2, size). - - Returns: - torch.Tensor: Transformed query tensor (#batch, n_head, time1, d_k). - torch.Tensor: Transformed key tensor (#batch, n_head, time2, d_k). - torch.Tensor: Transformed value tensor (#batch, n_head, time2, d_k). - - """ - n_batch = query.size(0) - q = self.linear_q(query).view(n_batch, -1, self.h, self.d_k) - k = self.linear_k(key).view(n_batch, -1, self.h, self.d_k) - v = self.linear_v(value).view(n_batch, -1, self.h, self.d_k) - q = q.transpose(1, 2) # (batch, head, time1, d_k) - k = k.transpose(1, 2) # (batch, head, time2, d_k) - v = v.transpose(1, 2) # (batch, head, time2, d_k) - - return q, k, v - - def forward_attention(self, value, scores, mask): - """Compute attention context vector. - - Args: - value (torch.Tensor): Transformed value (#batch, n_head, time2, d_k). - scores (torch.Tensor): Attention score (#batch, n_head, time1, time2). - mask (torch.Tensor): Mask (#batch, 1, time2) or (#batch, time1, time2). - - Returns: - torch.Tensor: Transformed value (#batch, time1, d_model) - weighted by the attention score (#batch, time1, time2). - - """ - n_batch = value.size(0) - if mask is not None: - mask = mask.unsqueeze(1).eq(0) # (batch, 1, *, time2) - min_value = float( - numpy.finfo(torch.tensor(0, dtype=scores.dtype).numpy().dtype).min - ) - scores = scores.masked_fill(mask, min_value) - self.attn = torch.softmax(scores, dim=-1).masked_fill( - mask, 0.0 - ) # (batch, head, time1, time2) - else: - self.attn = torch.softmax(scores, dim=-1) # (batch, head, time1, time2) - - p_attn = self.dropout(self.attn) - x = torch.matmul(p_attn, value) # (batch, head, time1, d_k) - x = ( - x.transpose(1, 2).contiguous().view(n_batch, -1, self.h * self.d_k) - ) # (batch, time1, d_model) - - return self.linear_out(x) # (batch, time1, d_model) - - def forward(self, query, key, value, mask): - """Compute scaled dot product attention. - - Args: - query (torch.Tensor): Query tensor (#batch, time1, size). - key (torch.Tensor): Key tensor (#batch, time2, size). - value (torch.Tensor): Value tensor (#batch, time2, size). - mask (torch.Tensor): Mask tensor (#batch, 1, time2) or - (#batch, time1, time2). - - Returns: - torch.Tensor: Output tensor (#batch, time1, d_model). - - """ - q, k, v = self.forward_qkv(query, key, value) - scores = torch.matmul(q, k.transpose(-2, -1)) / math.sqrt(self.d_k) - return self.forward_attention(v, scores, mask) - - -class LegacyRelPositionMultiHeadedAttention(MultiHeadedAttention): - """Multi-Head Attention layer with relative position encoding (old version). - - Details can be found in https://github.com/espnet/espnet/pull/2816. - - Paper: https://arxiv.org/abs/1901.02860 - - Args: - n_head (int): The number of heads. - n_feat (int): The number of features. - dropout_rate (float): Dropout rate. - zero_triu (bool): Whether to zero the upper triangular part of attention matrix. - - """ - - def __init__(self, n_head, n_feat, dropout_rate, zero_triu=False): - """Construct an RelPositionMultiHeadedAttention object.""" - super().__init__(n_head, n_feat, dropout_rate) - self.zero_triu = zero_triu - # linear transformation for positional encoding - self.linear_pos = nn.Linear(n_feat, n_feat, bias=False) - # these two learnable bias are used in matrix c and matrix d - # as described in https://arxiv.org/abs/1901.02860 Section 3.3 - self.pos_bias_u = nn.Parameter(torch.Tensor(self.h, self.d_k)) - self.pos_bias_v = nn.Parameter(torch.Tensor(self.h, self.d_k)) - torch.nn.init.xavier_uniform_(self.pos_bias_u) - torch.nn.init.xavier_uniform_(self.pos_bias_v) - - def rel_shift(self, x): - """Compute relative positional encoding. - - Args: - x (torch.Tensor): Input tensor (batch, head, time1, time2). - - Returns: - torch.Tensor: Output tensor. - - """ - zero_pad = torch.zeros((*x.size()[:3], 1), device=x.device, dtype=x.dtype) - x_padded = torch.cat([zero_pad, x], dim=-1) - - x_padded = x_padded.view(*x.size()[:2], x.size(3) + 1, x.size(2)) - x = x_padded[:, :, 1:].view_as(x) - - if self.zero_triu: - ones = torch.ones((x.size(2), x.size(3))) - x = x * torch.tril(ones, x.size(3) - x.size(2))[None, None, :, :] - - return x - - def forward(self, query, key, value, pos_emb, mask): - """Compute 'Scaled Dot Product Attention' with rel. positional encoding. - - Args: - query (torch.Tensor): Query tensor (#batch, time1, size). - key (torch.Tensor): Key tensor (#batch, time2, size). - value (torch.Tensor): Value tensor (#batch, time2, size). - pos_emb (torch.Tensor): Positional embedding tensor (#batch, time1, size). - mask (torch.Tensor): Mask tensor (#batch, 1, time2) or - (#batch, time1, time2). - - Returns: - torch.Tensor: Output tensor (#batch, time1, d_model). - - """ - q, k, v = self.forward_qkv(query, key, value) - q = q.transpose(1, 2) # (batch, time1, head, d_k) - - n_batch_pos = pos_emb.size(0) - p = self.linear_pos(pos_emb).view(n_batch_pos, -1, self.h, self.d_k) - p = p.transpose(1, 2) # (batch, head, time1, d_k) - - # (batch, head, time1, d_k) - q_with_bias_u = (q + self.pos_bias_u).transpose(1, 2) - # (batch, head, time1, d_k) - q_with_bias_v = (q + self.pos_bias_v).transpose(1, 2) - - # compute attention score - # first compute matrix a and matrix c - # as described in https://arxiv.org/abs/1901.02860 Section 3.3 - # (batch, head, time1, time2) - matrix_ac = torch.matmul(q_with_bias_u, k.transpose(-2, -1)) - - # compute matrix b and matrix d - # (batch, head, time1, time1) - matrix_bd = torch.matmul(q_with_bias_v, p.transpose(-2, -1)) - matrix_bd = self.rel_shift(matrix_bd) - - scores = (matrix_ac + matrix_bd) / math.sqrt( - self.d_k - ) # (batch, head, time1, time2) - - return self.forward_attention(v, scores, mask) - - -class RelPositionMultiHeadedAttention(MultiHeadedAttention): - """Multi-Head Attention layer with relative position encoding (new implementation). - - Details can be found in https://github.com/espnet/espnet/pull/2816. - - Paper: https://arxiv.org/abs/1901.02860 - - Args: - n_head (int): The number of heads. - n_feat (int): The number of features. - dropout_rate (float): Dropout rate. - zero_triu (bool): Whether to zero the upper triangular part of attention matrix. - - """ - - def __init__(self, n_head, n_feat, dropout_rate, zero_triu=False): - """Construct an RelPositionMultiHeadedAttention object.""" - super().__init__(n_head, n_feat, dropout_rate) - self.zero_triu = zero_triu - # linear transformation for positional encoding - self.linear_pos = nn.Linear(n_feat, n_feat, bias=False) - # these two learnable bias are used in matrix c and matrix d - # as described in https://arxiv.org/abs/1901.02860 Section 3.3 - self.pos_bias_u = nn.Parameter(torch.Tensor(self.h, self.d_k)) - self.pos_bias_v = nn.Parameter(torch.Tensor(self.h, self.d_k)) - torch.nn.init.xavier_uniform_(self.pos_bias_u) - torch.nn.init.xavier_uniform_(self.pos_bias_v) - - def rel_shift(self, x): - """Compute relative positional encoding. - - Args: - x (torch.Tensor): Input tensor (batch, head, time1, 2*time1-1). - time1 means the length of query vector. - - Returns: - torch.Tensor: Output tensor. - - """ - zero_pad = torch.zeros((*x.size()[:3], 1), device=x.device, dtype=x.dtype) - x_padded = torch.cat([zero_pad, x], dim=-1) - - x_padded = x_padded.view(*x.size()[:2], x.size(3) + 1, x.size(2)) - x = x_padded[:, :, 1:].view_as(x)[ - :, :, :, : x.size(-1) // 2 + 1 - ] # only keep the positions from 0 to time2 - - if self.zero_triu: - ones = torch.ones((x.size(2), x.size(3)), device=x.device) - x = x * torch.tril(ones, x.size(3) - x.size(2))[None, None, :, :] - - return x - - def forward(self, query, key, value, pos_emb, mask): - """Compute 'Scaled Dot Product Attention' with rel. positional encoding. - - Args: - query (torch.Tensor): Query tensor (#batch, time1, size). - key (torch.Tensor): Key tensor (#batch, time2, size). - value (torch.Tensor): Value tensor (#batch, time2, size). - pos_emb (torch.Tensor): Positional embedding tensor - (#batch, 2*time1-1, size). - mask (torch.Tensor): Mask tensor (#batch, 1, time2) or - (#batch, time1, time2). - - Returns: - torch.Tensor: Output tensor (#batch, time1, d_model). - - """ - q, k, v = self.forward_qkv(query, key, value) - q = q.transpose(1, 2) # (batch, time1, head, d_k) - - n_batch_pos = pos_emb.size(0) - p = self.linear_pos(pos_emb).view(n_batch_pos, -1, self.h, self.d_k) - p = p.transpose(1, 2) # (batch, head, 2*time1-1, d_k) - - # (batch, head, time1, d_k) - q_with_bias_u = (q + self.pos_bias_u).transpose(1, 2) - # (batch, head, time1, d_k) - q_with_bias_v = (q + self.pos_bias_v).transpose(1, 2) - - # compute attention score - # first compute matrix a and matrix c - # as described in https://arxiv.org/abs/1901.02860 Section 3.3 - # (batch, head, time1, time2) - matrix_ac = torch.matmul(q_with_bias_u, k.transpose(-2, -1)) - - # compute matrix b and matrix d - # (batch, head, time1, 2*time1-1) - matrix_bd = torch.matmul(q_with_bias_v, p.transpose(-2, -1)) - matrix_bd = self.rel_shift(matrix_bd) - - scores = (matrix_ac + matrix_bd) / math.sqrt( - self.d_k - ) # (batch, head, time1, time2) - - return self.forward_attention(v, scores, mask) - - -class RelPositionMultiHeadedAttentionChunk(torch.nn.Module): - """RelPositionMultiHeadedAttention definition. - Args: - num_heads: Number of attention heads. - embed_size: Embedding size. - dropout_rate: Dropout rate. - """ - - def __init__( - self, - num_heads: int, - embed_size: int, - dropout_rate: float = 0.0, - simplified_attention_score: bool = False, - ) -> None: - """Construct an MultiHeadedAttention object.""" - super().__init__() - - self.d_k = embed_size // num_heads - self.num_heads = num_heads - - assert self.d_k * num_heads == embed_size, ( - "embed_size (%d) must be divisible by num_heads (%d)", - (embed_size, num_heads), - ) - - self.linear_q = torch.nn.Linear(embed_size, embed_size) - self.linear_k = torch.nn.Linear(embed_size, embed_size) - self.linear_v = torch.nn.Linear(embed_size, embed_size) - - self.linear_out = torch.nn.Linear(embed_size, embed_size) - - if simplified_attention_score: - self.linear_pos = torch.nn.Linear(embed_size, num_heads) - - self.compute_att_score = self.compute_simplified_attention_score - else: - self.linear_pos = torch.nn.Linear(embed_size, embed_size, bias=False) - - self.pos_bias_u = torch.nn.Parameter(torch.Tensor(num_heads, self.d_k)) - self.pos_bias_v = torch.nn.Parameter(torch.Tensor(num_heads, self.d_k)) - torch.nn.init.xavier_uniform_(self.pos_bias_u) - torch.nn.init.xavier_uniform_(self.pos_bias_v) - - self.compute_att_score = self.compute_attention_score - - self.dropout = torch.nn.Dropout(p=dropout_rate) - self.attn = None - - def rel_shift(self, x: torch.Tensor, left_context: int = 0) -> torch.Tensor: - """Compute relative positional encoding. - Args: - x: Input sequence. (B, H, T_1, 2 * T_1 - 1) - left_context: Number of frames in left context. - Returns: - x: Output sequence. (B, H, T_1, T_2) - """ - batch_size, n_heads, time1, n = x.shape - time2 = time1 + left_context - - batch_stride, n_heads_stride, time1_stride, n_stride = x.stride() - - return x.as_strided( - (batch_size, n_heads, time1, time2), - (batch_stride, n_heads_stride, time1_stride - n_stride, n_stride), - storage_offset=(n_stride * (time1 - 1)), - ) - - def compute_simplified_attention_score( - self, - query: torch.Tensor, - key: torch.Tensor, - pos_enc: torch.Tensor, - left_context: int = 0, - ) -> torch.Tensor: - """Simplified attention score computation. - Reference: https://github.com/k2-fsa/icefall/pull/458 - Args: - query: Transformed query tensor. (B, H, T_1, d_k) - key: Transformed key tensor. (B, H, T_2, d_k) - pos_enc: Positional embedding tensor. (B, 2 * T_1 - 1, size) - left_context: Number of frames in left context. - Returns: - : Attention score. (B, H, T_1, T_2) - """ - pos_enc = self.linear_pos(pos_enc) - - matrix_ac = torch.matmul(query, key.transpose(2, 3)) - - matrix_bd = self.rel_shift( - pos_enc.transpose(1, 2).unsqueeze(2).repeat(1, 1, query.size(2), 1), - left_context=left_context, - ) - - return (matrix_ac + matrix_bd) / math.sqrt(self.d_k) - - def compute_attention_score( - self, - query: torch.Tensor, - key: torch.Tensor, - pos_enc: torch.Tensor, - left_context: int = 0, - ) -> torch.Tensor: - """Attention score computation. - Args: - query: Transformed query tensor. (B, H, T_1, d_k) - key: Transformed key tensor. (B, H, T_2, d_k) - pos_enc: Positional embedding tensor. (B, 2 * T_1 - 1, size) - left_context: Number of frames in left context. - Returns: - : Attention score. (B, H, T_1, T_2) - """ - p = self.linear_pos(pos_enc).view(pos_enc.size(0), -1, self.num_heads, self.d_k) - - query = query.transpose(1, 2) - q_with_bias_u = (query + self.pos_bias_u).transpose(1, 2) - q_with_bias_v = (query + self.pos_bias_v).transpose(1, 2) - - matrix_ac = torch.matmul(q_with_bias_u, key.transpose(-2, -1)) - - matrix_bd = torch.matmul(q_with_bias_v, p.permute(0, 2, 3, 1)) - matrix_bd = self.rel_shift(matrix_bd, left_context=left_context) - - return (matrix_ac + matrix_bd) / math.sqrt(self.d_k) - - def forward_qkv( - self, query: torch.Tensor, key: torch.Tensor, value: torch.Tensor - ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: - """Transform query, key and value. - Args: - query: Query tensor. (B, T_1, size) - key: Key tensor. (B, T_2, size) - v: Value tensor. (B, T_2, size) - Returns: - q: Transformed query tensor. (B, H, T_1, d_k) - k: Transformed key tensor. (B, H, T_2, d_k) - v: Transformed value tensor. (B, H, T_2, d_k) - """ - n_batch = query.size(0) - - q = ( - self.linear_q(query) - .view(n_batch, -1, self.num_heads, self.d_k) - .transpose(1, 2) - ) - k = ( - self.linear_k(key) - .view(n_batch, -1, self.num_heads, self.d_k) - .transpose(1, 2) - ) - v = ( - self.linear_v(value) - .view(n_batch, -1, self.num_heads, self.d_k) - .transpose(1, 2) - ) - - return q, k, v - - def forward_attention( - self, - value: torch.Tensor, - scores: torch.Tensor, - mask: torch.Tensor, - chunk_mask: Optional[torch.Tensor] = None, - ) -> torch.Tensor: - """Compute attention context vector. - Args: - value: Transformed value. (B, H, T_2, d_k) - scores: Attention score. (B, H, T_1, T_2) - mask: Source mask. (B, T_2) - chunk_mask: Chunk mask. (T_1, T_1) - Returns: - attn_output: Transformed value weighted by attention score. (B, T_1, H * d_k) - """ - batch_size = scores.size(0) - mask = mask.unsqueeze(1).unsqueeze(2) - if chunk_mask is not None: - mask = chunk_mask.unsqueeze(0).unsqueeze(1) | mask - scores = scores.masked_fill(mask, float("-inf")) - self.attn = torch.softmax(scores, dim=-1).masked_fill(mask, 0.0) - - attn_output = self.dropout(self.attn) - attn_output = torch.matmul(attn_output, value) - - attn_output = self.linear_out( - attn_output.transpose(1, 2) - .contiguous() - .view(batch_size, -1, self.num_heads * self.d_k) - ) - - return attn_output - - def forward( - self, - query: torch.Tensor, - key: torch.Tensor, - value: torch.Tensor, - pos_enc: torch.Tensor, - mask: torch.Tensor, - chunk_mask: Optional[torch.Tensor] = None, - left_context: int = 0, - ) -> torch.Tensor: - """Compute scaled dot product attention with rel. positional encoding. - Args: - query: Query tensor. (B, T_1, size) - key: Key tensor. (B, T_2, size) - value: Value tensor. (B, T_2, size) - pos_enc: Positional embedding tensor. (B, 2 * T_1 - 1, size) - mask: Source mask. (B, T_2) - chunk_mask: Chunk mask. (T_1, T_1) - left_context: Number of frames in left context. - Returns: - : Output tensor. (B, T_1, H * d_k) - """ - q, k, v = self.forward_qkv(query, key, value) - scores = self.compute_att_score(q, k, pos_enc, left_context=left_context) - return self.forward_attention(v, scores, mask, chunk_mask=chunk_mask) diff --git a/funasr_detach/models/transformer/decoder.py b/funasr_detach/models/transformer/decoder.py deleted file mode 100644 index be38579742cd77957a853a4038d61a5a52ec5e04..0000000000000000000000000000000000000000 --- a/funasr_detach/models/transformer/decoder.py +++ /dev/null @@ -1,653 +0,0 @@ -# Copyright 2019 Shigeki Karita -# Apache 2.0 (http://www.apache.org/licenses/LICENSE-2.0) - -"""Decoder definition.""" -from typing import Any -from typing import List -from typing import Sequence -from typing import Tuple - -import torch -from torch import nn - - -from funasr_detach.models.transformer.attention import MultiHeadedAttention -from funasr_detach.models.transformer.utils.dynamic_conv import DynamicConvolution -from funasr_detach.models.transformer.utils.dynamic_conv2d import DynamicConvolution2D -from funasr_detach.models.transformer.embedding import PositionalEncoding -from funasr_detach.models.transformer.layer_norm import LayerNorm -from funasr_detach.models.transformer.utils.lightconv import LightweightConvolution -from funasr_detach.models.transformer.utils.lightconv2d import LightweightConvolution2D -from funasr_detach.models.transformer.utils.mask import subsequent_mask -from funasr_detach.models.transformer.utils.nets_utils import make_pad_mask -from funasr_detach.models.transformer.positionwise_feed_forward import ( - PositionwiseFeedForward, # noqa: H301 -) -from funasr_detach.models.transformer.utils.repeat import repeat -from funasr_detach.models.transformer.scorers.scorer_interface import ( - BatchScorerInterface, -) - -from funasr_detach.register import tables - - -class DecoderLayer(nn.Module): - """Single decoder layer module. - - Args: - size (int): Input dimension. - self_attn (torch.nn.Module): Self-attention module instance. - `MultiHeadedAttention` instance can be used as the argument. - src_attn (torch.nn.Module): Self-attention module instance. - `MultiHeadedAttention` instance can be used as the argument. - feed_forward (torch.nn.Module): Feed-forward module instance. - `PositionwiseFeedForward`, `MultiLayeredConv1d`, or `Conv1dLinear` instance - can be used as the argument. - dropout_rate (float): Dropout rate. - normalize_before (bool): Whether to use layer_norm before the first block. - concat_after (bool): Whether to concat attention layer's input and output. - if True, additional linear will be applied. - i.e. x -> x + linear(concat(x, att(x))) - if False, no additional linear will be applied. i.e. x -> x + att(x) - - - """ - - def __init__( - self, - size, - self_attn, - src_attn, - feed_forward, - dropout_rate, - normalize_before=True, - concat_after=False, - ): - """Construct an DecoderLayer object.""" - super(DecoderLayer, self).__init__() - self.size = size - self.self_attn = self_attn - self.src_attn = src_attn - self.feed_forward = feed_forward - self.norm1 = LayerNorm(size) - self.norm2 = LayerNorm(size) - self.norm3 = LayerNorm(size) - self.dropout = nn.Dropout(dropout_rate) - self.normalize_before = normalize_before - self.concat_after = concat_after - if self.concat_after: - self.concat_linear1 = nn.Linear(size + size, size) - self.concat_linear2 = nn.Linear(size + size, size) - - def forward(self, tgt, tgt_mask, memory, memory_mask, cache=None): - """Compute decoded features. - - Args: - tgt (torch.Tensor): Input tensor (#batch, maxlen_out, size). - tgt_mask (torch.Tensor): Mask for input tensor (#batch, maxlen_out). - memory (torch.Tensor): Encoded memory, float32 (#batch, maxlen_in, size). - memory_mask (torch.Tensor): Encoded memory mask (#batch, maxlen_in). - cache (List[torch.Tensor]): List of cached tensors. - Each tensor shape should be (#batch, maxlen_out - 1, size). - - Returns: - torch.Tensor: Output tensor(#batch, maxlen_out, size). - torch.Tensor: Mask for output tensor (#batch, maxlen_out). - torch.Tensor: Encoded memory (#batch, maxlen_in, size). - torch.Tensor: Encoded memory mask (#batch, maxlen_in). - - """ - residual = tgt - if self.normalize_before: - tgt = self.norm1(tgt) - - if cache is None: - tgt_q = tgt - tgt_q_mask = tgt_mask - else: - # compute only the last frame query keeping dim: max_time_out -> 1 - assert cache.shape == ( - tgt.shape[0], - tgt.shape[1] - 1, - self.size, - ), f"{cache.shape} == {(tgt.shape[0], tgt.shape[1] - 1, self.size)}" - tgt_q = tgt[:, -1:, :] - residual = residual[:, -1:, :] - tgt_q_mask = None - if tgt_mask is not None: - tgt_q_mask = tgt_mask[:, -1:, :] - - if self.concat_after: - tgt_concat = torch.cat( - (tgt_q, self.self_attn(tgt_q, tgt, tgt, tgt_q_mask)), dim=-1 - ) - x = residual + self.concat_linear1(tgt_concat) - else: - x = residual + self.dropout(self.self_attn(tgt_q, tgt, tgt, tgt_q_mask)) - if not self.normalize_before: - x = self.norm1(x) - - residual = x - if self.normalize_before: - x = self.norm2(x) - if self.concat_after: - x_concat = torch.cat( - (x, self.src_attn(x, memory, memory, memory_mask)), dim=-1 - ) - x = residual + self.concat_linear2(x_concat) - else: - x = residual + self.dropout(self.src_attn(x, memory, memory, memory_mask)) - if not self.normalize_before: - x = self.norm2(x) - - residual = x - if self.normalize_before: - x = self.norm3(x) - x = residual + self.dropout(self.feed_forward(x)) - if not self.normalize_before: - x = self.norm3(x) - - if cache is not None: - x = torch.cat([cache, x], dim=1) - - return x, tgt_mask, memory, memory_mask - - -class BaseTransformerDecoder(nn.Module, BatchScorerInterface): - """Base class of Transfomer decoder module. - - Args: - vocab_size: output dim - encoder_output_size: dimension of attention - attention_heads: the number of heads of multi head attention - linear_units: the number of units of position-wise feed forward - num_blocks: the number of decoder blocks - dropout_rate: dropout rate - self_attention_dropout_rate: dropout rate for attention - input_layer: input layer type - use_output_layer: whether to use output layer - pos_enc_class: PositionalEncoding or ScaledPositionalEncoding - normalize_before: whether to use layer_norm before the first block - concat_after: whether to concat attention layer's input and output - if True, additional linear will be applied. - i.e. x -> x + linear(concat(x, att(x))) - if False, no additional linear will be applied. - i.e. x -> x + att(x) - """ - - def __init__( - self, - vocab_size: int, - encoder_output_size: int, - dropout_rate: float = 0.1, - positional_dropout_rate: float = 0.1, - input_layer: str = "embed", - use_output_layer: bool = True, - pos_enc_class=PositionalEncoding, - normalize_before: bool = True, - ): - super().__init__() - attention_dim = encoder_output_size - - if input_layer == "embed": - self.embed = torch.nn.Sequential( - torch.nn.Embedding(vocab_size, attention_dim), - pos_enc_class(attention_dim, positional_dropout_rate), - ) - elif input_layer == "linear": - self.embed = torch.nn.Sequential( - torch.nn.Linear(vocab_size, attention_dim), - torch.nn.LayerNorm(attention_dim), - torch.nn.Dropout(dropout_rate), - torch.nn.ReLU(), - pos_enc_class(attention_dim, positional_dropout_rate), - ) - else: - raise ValueError(f"only 'embed' or 'linear' is supported: {input_layer}") - - self.normalize_before = normalize_before - if self.normalize_before: - self.after_norm = LayerNorm(attention_dim) - if use_output_layer: - self.output_layer = torch.nn.Linear(attention_dim, vocab_size) - else: - self.output_layer = None - - # Must set by the inheritance - self.decoders = None - - def forward( - self, - hs_pad: torch.Tensor, - hlens: torch.Tensor, - ys_in_pad: torch.Tensor, - ys_in_lens: torch.Tensor, - ) -> Tuple[torch.Tensor, torch.Tensor]: - """Forward decoder. - - Args: - hs_pad: encoded memory, float32 (batch, maxlen_in, feat) - hlens: (batch) - ys_in_pad: - input token ids, int64 (batch, maxlen_out) - if input_layer == "embed" - input tensor (batch, maxlen_out, #mels) in the other cases - ys_in_lens: (batch) - Returns: - (tuple): tuple containing: - - x: decoded token score before softmax (batch, maxlen_out, token) - if use_output_layer is True, - olens: (batch, ) - """ - tgt = ys_in_pad - # tgt_mask: (B, 1, L) - tgt_mask = (~make_pad_mask(ys_in_lens)[:, None, :]).to(tgt.device) - # m: (1, L, L) - m = subsequent_mask(tgt_mask.size(-1), device=tgt_mask.device).unsqueeze(0) - # tgt_mask: (B, L, L) - tgt_mask = tgt_mask & m - - memory = hs_pad - memory_mask = (~make_pad_mask(hlens, maxlen=memory.size(1)))[:, None, :].to( - memory.device - ) - # Padding for Longformer - if memory_mask.shape[-1] != memory.shape[1]: - padlen = memory.shape[1] - memory_mask.shape[-1] - memory_mask = torch.nn.functional.pad( - memory_mask, (0, padlen), "constant", False - ) - - x = self.embed(tgt) - x, tgt_mask, memory, memory_mask = self.decoders( - x, tgt_mask, memory, memory_mask - ) - if self.normalize_before: - x = self.after_norm(x) - if self.output_layer is not None: - x = self.output_layer(x) - - olens = tgt_mask.sum(1) - return x, olens - - def forward_one_step( - self, - tgt: torch.Tensor, - tgt_mask: torch.Tensor, - memory: torch.Tensor, - cache: List[torch.Tensor] = None, - ) -> Tuple[torch.Tensor, List[torch.Tensor]]: - """Forward one step. - - Args: - tgt: input token ids, int64 (batch, maxlen_out) - tgt_mask: input token mask, (batch, maxlen_out) - dtype=torch.uint8 in PyTorch 1.2- - dtype=torch.bool in PyTorch 1.2+ (include 1.2) - memory: encoded memory, float32 (batch, maxlen_in, feat) - cache: cached output list of (batch, max_time_out-1, size) - Returns: - y, cache: NN output value and cache per `self.decoders`. - y.shape` is (batch, maxlen_out, token) - """ - x = self.embed(tgt) - if cache is None: - cache = [None] * len(self.decoders) - new_cache = [] - for c, decoder in zip(cache, self.decoders): - x, tgt_mask, memory, memory_mask = decoder( - x, tgt_mask, memory, None, cache=c - ) - new_cache.append(x) - - if self.normalize_before: - y = self.after_norm(x[:, -1]) - else: - y = x[:, -1] - if self.output_layer is not None: - y = torch.log_softmax(self.output_layer(y), dim=-1) - - return y, new_cache - - def score(self, ys, state, x): - """Score.""" - ys_mask = subsequent_mask(len(ys), device=x.device).unsqueeze(0) - logp, state = self.forward_one_step( - ys.unsqueeze(0), ys_mask, x.unsqueeze(0), cache=state - ) - return logp.squeeze(0), state - - def batch_score( - self, ys: torch.Tensor, states: List[Any], xs: torch.Tensor - ) -> Tuple[torch.Tensor, List[Any]]: - """Score new token batch. - - Args: - ys (torch.Tensor): torch.int64 prefix tokens (n_batch, ylen). - states (List[Any]): Scorer states for prefix tokens. - xs (torch.Tensor): - The encoder feature that generates ys (n_batch, xlen, n_feat). - - Returns: - tuple[torch.Tensor, List[Any]]: Tuple of - batchfied scores for next token with shape of `(n_batch, n_vocab)` - and next state list for ys. - - """ - # merge states - n_batch = len(ys) - n_layers = len(self.decoders) - if states[0] is None: - batch_state = None - else: - # transpose state of [batch, layer] into [layer, batch] - batch_state = [ - torch.stack([states[b][i] for b in range(n_batch)]) - for i in range(n_layers) - ] - - # batch decoding - ys_mask = subsequent_mask(ys.size(-1), device=xs.device).unsqueeze(0) - logp, states = self.forward_one_step(ys, ys_mask, xs, cache=batch_state) - - # transpose state of [layer, batch] into [batch, layer] - state_list = [[states[i][b] for i in range(n_layers)] for b in range(n_batch)] - return logp, state_list - - -@tables.register("decoder_classes", "TransformerDecoder") -class TransformerDecoder(BaseTransformerDecoder): - def __init__( - self, - vocab_size: int, - encoder_output_size: int, - attention_heads: int = 4, - linear_units: int = 2048, - num_blocks: int = 6, - dropout_rate: float = 0.1, - positional_dropout_rate: float = 0.1, - self_attention_dropout_rate: float = 0.0, - src_attention_dropout_rate: float = 0.0, - input_layer: str = "embed", - use_output_layer: bool = True, - pos_enc_class=PositionalEncoding, - normalize_before: bool = True, - concat_after: bool = False, - ): - super().__init__( - vocab_size=vocab_size, - encoder_output_size=encoder_output_size, - dropout_rate=dropout_rate, - positional_dropout_rate=positional_dropout_rate, - input_layer=input_layer, - use_output_layer=use_output_layer, - pos_enc_class=pos_enc_class, - normalize_before=normalize_before, - ) - - attention_dim = encoder_output_size - self.decoders = repeat( - num_blocks, - lambda lnum: DecoderLayer( - attention_dim, - MultiHeadedAttention( - attention_heads, attention_dim, self_attention_dropout_rate - ), - MultiHeadedAttention( - attention_heads, attention_dim, src_attention_dropout_rate - ), - PositionwiseFeedForward(attention_dim, linear_units, dropout_rate), - dropout_rate, - normalize_before, - concat_after, - ), - ) - - -@tables.register("decoder_classes", "LightweightConvolutionTransformerDecoder") -class LightweightConvolutionTransformerDecoder(BaseTransformerDecoder): - def __init__( - self, - vocab_size: int, - encoder_output_size: int, - attention_heads: int = 4, - linear_units: int = 2048, - num_blocks: int = 6, - dropout_rate: float = 0.1, - positional_dropout_rate: float = 0.1, - self_attention_dropout_rate: float = 0.0, - src_attention_dropout_rate: float = 0.0, - input_layer: str = "embed", - use_output_layer: bool = True, - pos_enc_class=PositionalEncoding, - normalize_before: bool = True, - concat_after: bool = False, - conv_wshare: int = 4, - conv_kernel_length: Sequence[int] = (11, 11, 11, 11, 11, 11), - conv_usebias: int = False, - ): - if len(conv_kernel_length) != num_blocks: - raise ValueError( - "conv_kernel_length must have equal number of values to num_blocks: " - f"{len(conv_kernel_length)} != {num_blocks}" - ) - super().__init__( - vocab_size=vocab_size, - encoder_output_size=encoder_output_size, - dropout_rate=dropout_rate, - positional_dropout_rate=positional_dropout_rate, - input_layer=input_layer, - use_output_layer=use_output_layer, - pos_enc_class=pos_enc_class, - normalize_before=normalize_before, - ) - - attention_dim = encoder_output_size - self.decoders = repeat( - num_blocks, - lambda lnum: DecoderLayer( - attention_dim, - LightweightConvolution( - wshare=conv_wshare, - n_feat=attention_dim, - dropout_rate=self_attention_dropout_rate, - kernel_size=conv_kernel_length[lnum], - use_kernel_mask=True, - use_bias=conv_usebias, - ), - MultiHeadedAttention( - attention_heads, attention_dim, src_attention_dropout_rate - ), - PositionwiseFeedForward(attention_dim, linear_units, dropout_rate), - dropout_rate, - normalize_before, - concat_after, - ), - ) - - -@tables.register("decoder_classes", "LightweightConvolution2DTransformerDecoder") -class LightweightConvolution2DTransformerDecoder(BaseTransformerDecoder): - def __init__( - self, - vocab_size: int, - encoder_output_size: int, - attention_heads: int = 4, - linear_units: int = 2048, - num_blocks: int = 6, - dropout_rate: float = 0.1, - positional_dropout_rate: float = 0.1, - self_attention_dropout_rate: float = 0.0, - src_attention_dropout_rate: float = 0.0, - input_layer: str = "embed", - use_output_layer: bool = True, - pos_enc_class=PositionalEncoding, - normalize_before: bool = True, - concat_after: bool = False, - conv_wshare: int = 4, - conv_kernel_length: Sequence[int] = (11, 11, 11, 11, 11, 11), - conv_usebias: int = False, - ): - if len(conv_kernel_length) != num_blocks: - raise ValueError( - "conv_kernel_length must have equal number of values to num_blocks: " - f"{len(conv_kernel_length)} != {num_blocks}" - ) - super().__init__( - vocab_size=vocab_size, - encoder_output_size=encoder_output_size, - dropout_rate=dropout_rate, - positional_dropout_rate=positional_dropout_rate, - input_layer=input_layer, - use_output_layer=use_output_layer, - pos_enc_class=pos_enc_class, - normalize_before=normalize_before, - ) - - attention_dim = encoder_output_size - self.decoders = repeat( - num_blocks, - lambda lnum: DecoderLayer( - attention_dim, - LightweightConvolution2D( - wshare=conv_wshare, - n_feat=attention_dim, - dropout_rate=self_attention_dropout_rate, - kernel_size=conv_kernel_length[lnum], - use_kernel_mask=True, - use_bias=conv_usebias, - ), - MultiHeadedAttention( - attention_heads, attention_dim, src_attention_dropout_rate - ), - PositionwiseFeedForward(attention_dim, linear_units, dropout_rate), - dropout_rate, - normalize_before, - concat_after, - ), - ) - - -@tables.register("decoder_classes", "DynamicConvolutionTransformerDecoder") -class DynamicConvolutionTransformerDecoder(BaseTransformerDecoder): - def __init__( - self, - vocab_size: int, - encoder_output_size: int, - attention_heads: int = 4, - linear_units: int = 2048, - num_blocks: int = 6, - dropout_rate: float = 0.1, - positional_dropout_rate: float = 0.1, - self_attention_dropout_rate: float = 0.0, - src_attention_dropout_rate: float = 0.0, - input_layer: str = "embed", - use_output_layer: bool = True, - pos_enc_class=PositionalEncoding, - normalize_before: bool = True, - concat_after: bool = False, - conv_wshare: int = 4, - conv_kernel_length: Sequence[int] = (11, 11, 11, 11, 11, 11), - conv_usebias: int = False, - ): - if len(conv_kernel_length) != num_blocks: - raise ValueError( - "conv_kernel_length must have equal number of values to num_blocks: " - f"{len(conv_kernel_length)} != {num_blocks}" - ) - super().__init__( - vocab_size=vocab_size, - encoder_output_size=encoder_output_size, - dropout_rate=dropout_rate, - positional_dropout_rate=positional_dropout_rate, - input_layer=input_layer, - use_output_layer=use_output_layer, - pos_enc_class=pos_enc_class, - normalize_before=normalize_before, - ) - attention_dim = encoder_output_size - - self.decoders = repeat( - num_blocks, - lambda lnum: DecoderLayer( - attention_dim, - DynamicConvolution( - wshare=conv_wshare, - n_feat=attention_dim, - dropout_rate=self_attention_dropout_rate, - kernel_size=conv_kernel_length[lnum], - use_kernel_mask=True, - use_bias=conv_usebias, - ), - MultiHeadedAttention( - attention_heads, attention_dim, src_attention_dropout_rate - ), - PositionwiseFeedForward(attention_dim, linear_units, dropout_rate), - dropout_rate, - normalize_before, - concat_after, - ), - ) - - -@tables.register("decoder_classes", "DynamicConvolution2DTransformerDecoder") -class DynamicConvolution2DTransformerDecoder(BaseTransformerDecoder): - def __init__( - self, - vocab_size: int, - encoder_output_size: int, - attention_heads: int = 4, - linear_units: int = 2048, - num_blocks: int = 6, - dropout_rate: float = 0.1, - positional_dropout_rate: float = 0.1, - self_attention_dropout_rate: float = 0.0, - src_attention_dropout_rate: float = 0.0, - input_layer: str = "embed", - use_output_layer: bool = True, - pos_enc_class=PositionalEncoding, - normalize_before: bool = True, - concat_after: bool = False, - conv_wshare: int = 4, - conv_kernel_length: Sequence[int] = (11, 11, 11, 11, 11, 11), - conv_usebias: int = False, - ): - if len(conv_kernel_length) != num_blocks: - raise ValueError( - "conv_kernel_length must have equal number of values to num_blocks: " - f"{len(conv_kernel_length)} != {num_blocks}" - ) - super().__init__( - vocab_size=vocab_size, - encoder_output_size=encoder_output_size, - dropout_rate=dropout_rate, - positional_dropout_rate=positional_dropout_rate, - input_layer=input_layer, - use_output_layer=use_output_layer, - pos_enc_class=pos_enc_class, - normalize_before=normalize_before, - ) - attention_dim = encoder_output_size - - self.decoders = repeat( - num_blocks, - lambda lnum: DecoderLayer( - attention_dim, - DynamicConvolution2D( - wshare=conv_wshare, - n_feat=attention_dim, - dropout_rate=self_attention_dropout_rate, - kernel_size=conv_kernel_length[lnum], - use_kernel_mask=True, - use_bias=conv_usebias, - ), - MultiHeadedAttention( - attention_heads, attention_dim, src_attention_dropout_rate - ), - PositionwiseFeedForward(attention_dim, linear_units, dropout_rate), - dropout_rate, - normalize_before, - concat_after, - ), - ) diff --git a/funasr_detach/models/transformer/embedding.py b/funasr_detach/models/transformer/embedding.py deleted file mode 100644 index bbfe64235d63ae0ac58f7c27296f92bdaf6a912d..0000000000000000000000000000000000000000 --- a/funasr_detach/models/transformer/embedding.py +++ /dev/null @@ -1,557 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- - -# Copyright 2019 Shigeki Karita -# Apache 2.0 (http://www.apache.org/licenses/LICENSE-2.0) - -"""Positional Encoding Module.""" - -import math -import torch -import torch.nn.functional as F -from torch import einsum - - -def _pre_hook( - state_dict, - prefix, - local_metadata, - strict, - missing_keys, - unexpected_keys, - error_msgs, -): - """Perform pre-hook in load_state_dict for backward compatibility. - - Note: - We saved self.pe until v.0.5.2 but we have omitted it later. - Therefore, we remove the item "pe" from `state_dict` for backward compatibility. - - """ - k = prefix + "pe" - if k in state_dict: - state_dict.pop(k) - - -class PositionalEncoding(torch.nn.Module): - """Positional encoding. - - Args: - d_model (int): Embedding dimension. - dropout_rate (float): Dropout rate. - max_len (int): Maximum input length. - reverse (bool): Whether to reverse the input position. Only for - the class LegacyRelPositionalEncoding. We remove it in the current - class RelPositionalEncoding. - """ - - def __init__(self, d_model, dropout_rate, max_len=5000, reverse=False): - """Construct an PositionalEncoding object.""" - super(PositionalEncoding, self).__init__() - self.d_model = d_model - self.reverse = reverse - self.xscale = math.sqrt(self.d_model) - self.dropout = torch.nn.Dropout(p=dropout_rate) - self.pe = None - self.extend_pe(torch.tensor(0.0).expand(1, max_len)) - self._register_load_state_dict_pre_hook(_pre_hook) - - def extend_pe(self, x): - """Reset the positional encodings.""" - if self.pe is not None: - if self.pe.size(1) >= x.size(1): - if self.pe.dtype != x.dtype or self.pe.device != x.device: - self.pe = self.pe.to(dtype=x.dtype, device=x.device) - return - pe = torch.zeros(x.size(1), self.d_model) - if self.reverse: - position = torch.arange( - x.size(1) - 1, -1, -1.0, dtype=torch.float32 - ).unsqueeze(1) - else: - position = torch.arange(0, x.size(1), dtype=torch.float32).unsqueeze(1) - div_term = torch.exp( - torch.arange(0, self.d_model, 2, dtype=torch.float32) - * -(math.log(10000.0) / self.d_model) - ) - pe[:, 0::2] = torch.sin(position * div_term) - pe[:, 1::2] = torch.cos(position * div_term) - pe = pe.unsqueeze(0) - self.pe = pe.to(device=x.device, dtype=x.dtype) - - def forward(self, x: torch.Tensor): - """Add positional encoding. - - Args: - x (torch.Tensor): Input tensor (batch, time, `*`). - - Returns: - torch.Tensor: Encoded tensor (batch, time, `*`). - """ - self.extend_pe(x) - x = x * self.xscale + self.pe[:, : x.size(1)] - return self.dropout(x) - - -class ScaledPositionalEncoding(PositionalEncoding): - """Scaled positional encoding module. - - See Sec. 3.2 https://arxiv.org/abs/1809.08895 - - Args: - d_model (int): Embedding dimension. - dropout_rate (float): Dropout rate. - max_len (int): Maximum input length. - - """ - - def __init__(self, d_model, dropout_rate, max_len=5000): - """Initialize class.""" - super().__init__(d_model=d_model, dropout_rate=dropout_rate, max_len=max_len) - self.alpha = torch.nn.Parameter(torch.tensor(1.0)) - - def reset_parameters(self): - """Reset parameters.""" - self.alpha.data = torch.tensor(1.0) - - def forward(self, x): - """Add positional encoding. - - Args: - x (torch.Tensor): Input tensor (batch, time, `*`). - - Returns: - torch.Tensor: Encoded tensor (batch, time, `*`). - - """ - self.extend_pe(x) - x = x + self.alpha * self.pe[:, : x.size(1)] - return self.dropout(x) - - -class LearnableFourierPosEnc(torch.nn.Module): - """Learnable Fourier Features for Positional Encoding. - - See https://arxiv.org/pdf/2106.02795.pdf - - Args: - d_model (int): Embedding dimension. - dropout_rate (float): Dropout rate. - max_len (int): Maximum input length. - gamma (float): init parameter for the positional kernel variance - see https://arxiv.org/pdf/2106.02795.pdf. - apply_scaling (bool): Whether to scale the input before adding the pos encoding. - hidden_dim (int): if not None, we modulate the pos encodings with - an MLP whose hidden layer has hidden_dim neurons. - """ - - def __init__( - self, - d_model, - dropout_rate=0.0, - max_len=5000, - gamma=1.0, - apply_scaling=False, - hidden_dim=None, - ): - """Initialize class.""" - super(LearnableFourierPosEnc, self).__init__() - - self.d_model = d_model - - if apply_scaling: - self.xscale = math.sqrt(self.d_model) - else: - self.xscale = 1.0 - - self.dropout = torch.nn.Dropout(dropout_rate) - self.max_len = max_len - - self.gamma = gamma - if self.gamma is None: - self.gamma = self.d_model // 2 - - assert ( - d_model % 2 == 0 - ), "d_model should be divisible by two in order to use this layer." - self.w_r = torch.nn.Parameter(torch.empty(1, d_model // 2)) - self._reset() # init the weights - - self.hidden_dim = hidden_dim - if self.hidden_dim is not None: - self.mlp = torch.nn.Sequential( - torch.nn.Linear(d_model, hidden_dim), - torch.nn.GELU(), - torch.nn.Linear(hidden_dim, d_model), - ) - - def _reset(self): - self.w_r.data = torch.normal( - 0, (1 / math.sqrt(self.gamma)), (1, self.d_model // 2) - ) - - def extend_pe(self, x): - """Reset the positional encodings.""" - position_v = torch.arange(0, x.size(1), dtype=torch.float32).unsqueeze(1).to(x) - - cosine = torch.cos(torch.matmul(position_v, self.w_r)) - sine = torch.sin(torch.matmul(position_v, self.w_r)) - pos_enc = torch.cat((cosine, sine), -1) - pos_enc /= math.sqrt(self.d_model) - - if self.hidden_dim is None: - return pos_enc.unsqueeze(0) - else: - return self.mlp(pos_enc.unsqueeze(0)) - - def forward(self, x: torch.Tensor): - """Add positional encoding. - - Args: - x (torch.Tensor): Input tensor (batch, time, `*`). - - Returns: - torch.Tensor: Encoded tensor (batch, time, `*`). - """ - pe = self.extend_pe(x) - x = x * self.xscale + pe - return self.dropout(x) - - -class LegacyRelPositionalEncoding(PositionalEncoding): - """Relative positional encoding module (old version). - - Details can be found in https://github.com/espnet/espnet/pull/2816. - - See : Appendix B in https://arxiv.org/abs/1901.02860 - - Args: - d_model (int): Embedding dimension. - dropout_rate (float): Dropout rate. - max_len (int): Maximum input length. - - """ - - def __init__(self, d_model, dropout_rate, max_len=5000): - """Initialize class.""" - super().__init__( - d_model=d_model, - dropout_rate=dropout_rate, - max_len=max_len, - reverse=True, - ) - - def forward(self, x): - """Compute positional encoding. - - Args: - x (torch.Tensor): Input tensor (batch, time, `*`). - - Returns: - torch.Tensor: Encoded tensor (batch, time, `*`). - torch.Tensor: Positional embedding tensor (1, time, `*`). - - """ - self.extend_pe(x) - x = x * self.xscale - pos_emb = self.pe[:, : x.size(1)] - return self.dropout(x), self.dropout(pos_emb) - - -class RelPositionalEncoding(torch.nn.Module): - """Relative positional encoding module (new implementation). - - Details can be found in https://github.com/espnet/espnet/pull/2816. - - See : Appendix B in https://arxiv.org/abs/1901.02860 - - Args: - d_model (int): Embedding dimension. - dropout_rate (float): Dropout rate. - max_len (int): Maximum input length. - - """ - - def __init__(self, d_model, dropout_rate, max_len=5000): - """Construct an PositionalEncoding object.""" - super(RelPositionalEncoding, self).__init__() - self.d_model = d_model - self.xscale = math.sqrt(self.d_model) - self.dropout = torch.nn.Dropout(p=dropout_rate) - self.pe = None - self.extend_pe(torch.tensor(0.0).expand(1, max_len)) - - def extend_pe(self, x): - """Reset the positional encodings.""" - if self.pe is not None: - # self.pe contains both positive and negative parts - # the length of self.pe is 2 * input_len - 1 - if self.pe.size(1) >= x.size(1) * 2 - 1: - if self.pe.dtype != x.dtype or self.pe.device != x.device: - self.pe = self.pe.to(dtype=x.dtype, device=x.device) - return - # Suppose `i` means to the position of query vecotr and `j` means the - # position of key vector. We use position relative positions when keys - # are to the left (i>j) and negative relative positions otherwise (i= length: - if self.pe.dtype != dtype or self.pe.device != device: - self.pe = self.pe.to(dtype=dtype, device=device) - return - pe = torch.zeros(length, self.d_model) - position = torch.arange(0, length, dtype=torch.float32).unsqueeze(1) - div_term = torch.exp( - torch.arange(0, self.d_model, 2, dtype=torch.float32) - * -(math.log(10000.0) / self.d_model) - ) - pe[:, 0::2] = torch.sin(position * div_term) - pe[:, 1::2] = torch.cos(position * div_term) - pe = pe.unsqueeze(0) - self.pe = pe.to(device=device, dtype=dtype) - - def forward(self, x: torch.Tensor, start_idx: int = 0): - """Add positional encoding. - - Args: - x (torch.Tensor): Input tensor (batch, time, `*`). - - Returns: - torch.Tensor: Encoded tensor (batch, time, `*`). - - """ - self.extend_pe(x.size(1) + start_idx, x.device, x.dtype) - x = x * self.xscale + self.pe[:, start_idx : start_idx + x.size(1)] - return self.dropout(x) - - -class SinusoidalPositionEncoder(torch.nn.Module): - """ """ - - def __int__(self, d_model=80, dropout_rate=0.1): - pass - - def encode( - self, - positions: torch.Tensor = None, - depth: int = None, - dtype: torch.dtype = torch.float32, - ): - batch_size = positions.size(0) - positions = positions.type(dtype) - device = positions.device - log_timescale_increment = torch.log( - torch.tensor([10000], dtype=dtype, device=device) - ) / (depth / 2 - 1) - inv_timescales = torch.exp( - torch.arange(depth / 2, device=device).type(dtype) - * (-log_timescale_increment) - ) - inv_timescales = torch.reshape(inv_timescales, [batch_size, -1]) - scaled_time = torch.reshape(positions, [1, -1, 1]) * torch.reshape( - inv_timescales, [1, 1, -1] - ) - encoding = torch.cat([torch.sin(scaled_time), torch.cos(scaled_time)], dim=2) - return encoding.type(dtype) - - def forward(self, x): - batch_size, timesteps, input_dim = x.size() - positions = torch.arange(1, timesteps + 1, device=x.device)[None, :] - position_encoding = self.encode(positions, input_dim, x.dtype).to(x.device) - - return x + position_encoding - - -class StreamSinusoidalPositionEncoder(torch.nn.Module): - """ """ - - def __int__(self, d_model=80, dropout_rate=0.1): - pass - - def encode( - self, - positions: torch.Tensor = None, - depth: int = None, - dtype: torch.dtype = torch.float32, - ): - batch_size = positions.size(0) - positions = positions.type(dtype) - log_timescale_increment = torch.log(torch.tensor([10000], dtype=dtype)) / ( - depth / 2 - 1 - ) - inv_timescales = torch.exp( - torch.arange(depth / 2).type(dtype) * (-log_timescale_increment) - ) - inv_timescales = torch.reshape(inv_timescales, [batch_size, -1]) - scaled_time = torch.reshape(positions, [1, -1, 1]) * torch.reshape( - inv_timescales, [1, 1, -1] - ) - encoding = torch.cat([torch.sin(scaled_time), torch.cos(scaled_time)], dim=2) - return encoding.type(dtype) - - def forward(self, x, cache=None): - batch_size, timesteps, input_dim = x.size() - start_idx = 0 - if cache is not None: - start_idx = cache["start_idx"] - cache["start_idx"] += timesteps - positions = torch.arange(1, timesteps + start_idx + 1)[None, :] - position_encoding = self.encode(positions, input_dim, x.dtype).to(x.device) - return x + position_encoding[:, start_idx : start_idx + timesteps] - - -class StreamingRelPositionalEncoding(torch.nn.Module): - """Relative positional encoding. - Args: - size: Module size. - max_len: Maximum input length. - dropout_rate: Dropout rate. - """ - - def __init__( - self, size: int, dropout_rate: float = 0.0, max_len: int = 5000 - ) -> None: - """Construct a RelativePositionalEncoding object.""" - super().__init__() - - self.size = size - - self.pe = None - self.dropout = torch.nn.Dropout(p=dropout_rate) - - self.extend_pe(torch.tensor(0.0).expand(1, max_len)) - self._register_load_state_dict_pre_hook(_pre_hook) - - def extend_pe(self, x: torch.Tensor, left_context: int = 0) -> None: - """Reset positional encoding. - Args: - x: Input sequences. (B, T, ?) - left_context: Number of frames in left context. - """ - time1 = x.size(1) + left_context - - if self.pe is not None: - if self.pe.size(1) >= time1 * 2 - 1: - if self.pe.dtype != x.dtype or self.pe.device != x.device: - self.pe = self.pe.to(device=x.device, dtype=x.dtype) - return - - pe_positive = torch.zeros(time1, self.size) - pe_negative = torch.zeros(time1, self.size) - - position = torch.arange(0, time1, dtype=torch.float32).unsqueeze(1) - div_term = torch.exp( - torch.arange(0, self.size, 2, dtype=torch.float32) - * -(math.log(10000.0) / self.size) - ) - - pe_positive[:, 0::2] = torch.sin(position * div_term) - pe_positive[:, 1::2] = torch.cos(position * div_term) - pe_positive = torch.flip(pe_positive, [0]).unsqueeze(0) - - pe_negative[:, 0::2] = torch.sin(-1 * position * div_term) - pe_negative[:, 1::2] = torch.cos(-1 * position * div_term) - pe_negative = pe_negative[1:].unsqueeze(0) - - self.pe = torch.cat([pe_positive, pe_negative], dim=1).to( - dtype=x.dtype, device=x.device - ) - - def forward(self, x: torch.Tensor, left_context: int = 0) -> torch.Tensor: - """Compute positional encoding. - Args: - x: Input sequences. (B, T, ?) - left_context: Number of frames in left context. - Returns: - pos_enc: Positional embedding sequences. (B, 2 * (T - 1), ?) - """ - self.extend_pe(x, left_context=left_context) - - time1 = x.size(1) + left_context - - pos_enc = self.pe[ - :, self.pe.size(1) // 2 - time1 + 1 : self.pe.size(1) // 2 + x.size(1) - ] - pos_enc = self.dropout(pos_enc) - - return pos_enc - - -class ScaledSinuEmbedding(torch.nn.Module): - def __init__(self, dim): - super().__init__() - self.scale = torch.nn.Parameter( - torch.ones( - 1, - ) - ) - inv_freq = 1.0 / (10000 ** (torch.arange(0, dim, 2).float() / dim)) - self.register_buffer("inv_freq", inv_freq) - - def forward(self, x): - n, device = x.shape[1], x.device - t = torch.arange(n, device=device).type_as(self.inv_freq) - sinu = einsum("i , j -> i j", t, self.inv_freq) - emb = torch.cat((sinu.sin(), sinu.cos()), dim=-1) - return emb * self.scale diff --git a/funasr_detach/models/transformer/encoder.py b/funasr_detach/models/transformer/encoder.py deleted file mode 100644 index dbe5bfb60ec0dbe3821884f054aafb7701dc0d26..0000000000000000000000000000000000000000 --- a/funasr_detach/models/transformer/encoder.py +++ /dev/null @@ -1,335 +0,0 @@ -# Copyright 2019 Shigeki Karita -# Apache 2.0 (http://www.apache.org/licenses/LICENSE-2.0) - -"""Transformer encoder definition.""" - -from typing import List -from typing import Optional -from typing import Tuple - -import torch -from torch import nn -import logging - -from funasr_detach.models.transformer.attention import MultiHeadedAttention -from funasr_detach.models.transformer.embedding import PositionalEncoding -from funasr_detach.models.transformer.layer_norm import LayerNorm -from funasr_detach.models.transformer.utils.multi_layer_conv import Conv1dLinear -from funasr_detach.models.transformer.utils.multi_layer_conv import MultiLayeredConv1d -from funasr_detach.models.transformer.utils.nets_utils import make_pad_mask -from funasr_detach.models.transformer.positionwise_feed_forward import ( - PositionwiseFeedForward, -) -from funasr_detach.models.transformer.utils.repeat import repeat -from funasr_detach.models.ctc.ctc import CTC - -from funasr_detach.models.transformer.utils.subsampling import Conv2dSubsampling -from funasr_detach.models.transformer.utils.subsampling import Conv2dSubsampling2 -from funasr_detach.models.transformer.utils.subsampling import Conv2dSubsampling6 -from funasr_detach.models.transformer.utils.subsampling import Conv2dSubsampling8 -from funasr_detach.models.transformer.utils.subsampling import TooShortUttError -from funasr_detach.models.transformer.utils.subsampling import check_short_utt - -from funasr_detach.register import tables - - -class EncoderLayer(nn.Module): - """Encoder layer module. - - Args: - size (int): Input dimension. - self_attn (torch.nn.Module): Self-attention module instance. - `MultiHeadedAttention` or `RelPositionMultiHeadedAttention` instance - can be used as the argument. - feed_forward (torch.nn.Module): Feed-forward module instance. - `PositionwiseFeedForward`, `MultiLayeredConv1d`, or `Conv1dLinear` instance - can be used as the argument. - dropout_rate (float): Dropout rate. - normalize_before (bool): Whether to use layer_norm before the first block. - concat_after (bool): Whether to concat attention layer's input and output. - if True, additional linear will be applied. - i.e. x -> x + linear(concat(x, att(x))) - if False, no additional linear will be applied. i.e. x -> x + att(x) - stochastic_depth_rate (float): Proability to skip this layer. - During training, the layer may skip residual computation and return input - as-is with given probability. - """ - - def __init__( - self, - size, - self_attn, - feed_forward, - dropout_rate, - normalize_before=True, - concat_after=False, - stochastic_depth_rate=0.0, - ): - """Construct an EncoderLayer object.""" - super(EncoderLayer, self).__init__() - self.self_attn = self_attn - self.feed_forward = feed_forward - self.norm1 = LayerNorm(size) - self.norm2 = LayerNorm(size) - self.dropout = nn.Dropout(dropout_rate) - self.size = size - self.normalize_before = normalize_before - self.concat_after = concat_after - if self.concat_after: - self.concat_linear = nn.Linear(size + size, size) - self.stochastic_depth_rate = stochastic_depth_rate - - def forward(self, x, mask, cache=None): - """Compute encoded features. - - Args: - x_input (torch.Tensor): Input tensor (#batch, time, size). - mask (torch.Tensor): Mask tensor for the input (#batch, time). - cache (torch.Tensor): Cache tensor of the input (#batch, time - 1, size). - - Returns: - torch.Tensor: Output tensor (#batch, time, size). - torch.Tensor: Mask tensor (#batch, time). - - """ - skip_layer = False - # with stochastic depth, residual connection `x + f(x)` becomes - # `x <- x + 1 / (1 - p) * f(x)` at training time. - stoch_layer_coeff = 1.0 - if self.training and self.stochastic_depth_rate > 0: - skip_layer = torch.rand(1).item() < self.stochastic_depth_rate - stoch_layer_coeff = 1.0 / (1 - self.stochastic_depth_rate) - - if skip_layer: - if cache is not None: - x = torch.cat([cache, x], dim=1) - return x, mask - - residual = x - if self.normalize_before: - x = self.norm1(x) - - if cache is None: - x_q = x - else: - assert cache.shape == (x.shape[0], x.shape[1] - 1, self.size) - x_q = x[:, -1:, :] - residual = residual[:, -1:, :] - mask = None if mask is None else mask[:, -1:, :] - - if self.concat_after: - x_concat = torch.cat((x, self.self_attn(x_q, x, x, mask)), dim=-1) - x = residual + stoch_layer_coeff * self.concat_linear(x_concat) - else: - x = residual + stoch_layer_coeff * self.dropout( - self.self_attn(x_q, x, x, mask) - ) - if not self.normalize_before: - x = self.norm1(x) - - residual = x - if self.normalize_before: - x = self.norm2(x) - x = residual + stoch_layer_coeff * self.dropout(self.feed_forward(x)) - if not self.normalize_before: - x = self.norm2(x) - - if cache is not None: - x = torch.cat([cache, x], dim=1) - - return x, mask - - -@tables.register("encoder_classes", "TransformerEncoder") -class TransformerEncoder(nn.Module): - """Transformer encoder module. - - Args: - input_size: input dim - output_size: dimension of attention - attention_heads: the number of heads of multi head attention - linear_units: the number of units of position-wise feed forward - num_blocks: the number of decoder blocks - dropout_rate: dropout rate - attention_dropout_rate: dropout rate in attention - positional_dropout_rate: dropout rate after adding positional encoding - input_layer: input layer type - pos_enc_class: PositionalEncoding or ScaledPositionalEncoding - normalize_before: whether to use layer_norm before the first block - concat_after: whether to concat attention layer's input and output - if True, additional linear will be applied. - i.e. x -> x + linear(concat(x, att(x))) - if False, no additional linear will be applied. - i.e. x -> x + att(x) - positionwise_layer_type: linear of conv1d - positionwise_conv_kernel_size: kernel size of positionwise conv1d layer - padding_idx: padding_idx for input_layer=embed - """ - - def __init__( - self, - input_size: int, - output_size: int = 256, - attention_heads: int = 4, - linear_units: int = 2048, - num_blocks: int = 6, - dropout_rate: float = 0.1, - positional_dropout_rate: float = 0.1, - attention_dropout_rate: float = 0.0, - input_layer: Optional[str] = "conv2d", - pos_enc_class=PositionalEncoding, - normalize_before: bool = True, - concat_after: bool = False, - positionwise_layer_type: str = "linear", - positionwise_conv_kernel_size: int = 1, - padding_idx: int = -1, - interctc_layer_idx: List[int] = [], - interctc_use_conditioning: bool = False, - ): - super().__init__() - self._output_size = output_size - - if input_layer == "linear": - self.embed = torch.nn.Sequential( - torch.nn.Linear(input_size, output_size), - torch.nn.LayerNorm(output_size), - torch.nn.Dropout(dropout_rate), - torch.nn.ReLU(), - pos_enc_class(output_size, positional_dropout_rate), - ) - elif input_layer == "conv2d": - self.embed = Conv2dSubsampling(input_size, output_size, dropout_rate) - elif input_layer == "conv2d2": - self.embed = Conv2dSubsampling2(input_size, output_size, dropout_rate) - elif input_layer == "conv2d6": - self.embed = Conv2dSubsampling6(input_size, output_size, dropout_rate) - elif input_layer == "conv2d8": - self.embed = Conv2dSubsampling8(input_size, output_size, dropout_rate) - elif input_layer == "embed": - self.embed = torch.nn.Sequential( - torch.nn.Embedding(input_size, output_size, padding_idx=padding_idx), - pos_enc_class(output_size, positional_dropout_rate), - ) - elif input_layer is None: - if input_size == output_size: - self.embed = None - else: - self.embed = torch.nn.Linear(input_size, output_size) - else: - raise ValueError("unknown input_layer: " + input_layer) - self.normalize_before = normalize_before - if positionwise_layer_type == "linear": - positionwise_layer = PositionwiseFeedForward - positionwise_layer_args = ( - output_size, - linear_units, - dropout_rate, - ) - elif positionwise_layer_type == "conv1d": - positionwise_layer = MultiLayeredConv1d - positionwise_layer_args = ( - output_size, - linear_units, - positionwise_conv_kernel_size, - dropout_rate, - ) - elif positionwise_layer_type == "conv1d-linear": - positionwise_layer = Conv1dLinear - positionwise_layer_args = ( - output_size, - linear_units, - positionwise_conv_kernel_size, - dropout_rate, - ) - else: - raise NotImplementedError("Support only linear or conv1d.") - self.encoders = repeat( - num_blocks, - lambda lnum: EncoderLayer( - output_size, - MultiHeadedAttention( - attention_heads, output_size, attention_dropout_rate - ), - positionwise_layer(*positionwise_layer_args), - dropout_rate, - normalize_before, - concat_after, - ), - ) - if self.normalize_before: - self.after_norm = LayerNorm(output_size) - - self.interctc_layer_idx = interctc_layer_idx - if len(interctc_layer_idx) > 0: - assert 0 < min(interctc_layer_idx) and max(interctc_layer_idx) < num_blocks - self.interctc_use_conditioning = interctc_use_conditioning - self.conditioning_layer = None - - def output_size(self) -> int: - return self._output_size - - def forward( - self, - xs_pad: torch.Tensor, - ilens: torch.Tensor, - prev_states: torch.Tensor = None, - ctc: CTC = None, - ) -> Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]: - """Embed positions in tensor. - - Args: - xs_pad: input tensor (B, L, D) - ilens: input length (B) - prev_states: Not to be used now. - Returns: - position embedded tensor and mask - """ - masks = (~make_pad_mask(ilens)[:, None, :]).to(xs_pad.device) - - if self.embed is None: - xs_pad = xs_pad - elif ( - isinstance(self.embed, Conv2dSubsampling) - or isinstance(self.embed, Conv2dSubsampling2) - or isinstance(self.embed, Conv2dSubsampling6) - or isinstance(self.embed, Conv2dSubsampling8) - ): - short_status, limit_size = check_short_utt(self.embed, xs_pad.size(1)) - if short_status: - raise TooShortUttError( - f"has {xs_pad.size(1)} frames and is too short for subsampling " - + f"(it needs more than {limit_size} frames), return empty results", - xs_pad.size(1), - limit_size, - ) - xs_pad, masks = self.embed(xs_pad, masks) - else: - xs_pad = self.embed(xs_pad) - - intermediate_outs = [] - if len(self.interctc_layer_idx) == 0: - xs_pad, masks = self.encoders(xs_pad, masks) - else: - for layer_idx, encoder_layer in enumerate(self.encoders): - xs_pad, masks = encoder_layer(xs_pad, masks) - - if layer_idx + 1 in self.interctc_layer_idx: - encoder_out = xs_pad - - # intermediate outputs are also normalized - if self.normalize_before: - encoder_out = self.after_norm(encoder_out) - - intermediate_outs.append((layer_idx + 1, encoder_out)) - - if self.interctc_use_conditioning: - ctc_out = ctc.softmax(encoder_out) - xs_pad = xs_pad + self.conditioning_layer(ctc_out) - - if self.normalize_before: - xs_pad = self.after_norm(xs_pad) - - olens = masks.squeeze(1).sum(1) - if len(intermediate_outs) > 0: - return (xs_pad, intermediate_outs), olens, None - return xs_pad, olens, None diff --git a/funasr_detach/models/transformer/layer_norm.py b/funasr_detach/models/transformer/layer_norm.py deleted file mode 100644 index 9ee41d29e7f4f82b36b33d8ae0be7f0c647db55b..0000000000000000000000000000000000000000 --- a/funasr_detach/models/transformer/layer_norm.py +++ /dev/null @@ -1,170 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- - -# Copyright 2019 Shigeki Karita -# Apache 2.0 (http://www.apache.org/licenses/LICENSE-2.0) - -"""Layer normalization module.""" - -import torch -import torch.nn as nn - - -class LayerNorm(torch.nn.LayerNorm): - """Layer normalization module. - - Args: - nout (int): Output dim size. - dim (int): Dimension to be normalized. - - """ - - def __init__(self, nout, dim=-1): - """Construct an LayerNorm object.""" - super(LayerNorm, self).__init__(nout, eps=1e-12) - self.dim = dim - - def forward(self, x): - """Apply layer normalization. - - Args: - x (torch.Tensor): Input tensor. - - Returns: - torch.Tensor: Normalized tensor. - - """ - if self.dim == -1: - return super(LayerNorm, self).forward(x) - return ( - super(LayerNorm, self) - .forward(x.transpose(self.dim, -1)) - .transpose(self.dim, -1) - ) - - -class GlobalLayerNorm(nn.Module): - """Calculate Global Layer Normalization. - - Arguments - --------- - dim : (int or list or torch.Size) - Input shape from an expected input of size. - eps : float - A value added to the denominator for numerical stability. - elementwise_affine : bool - A boolean value that when set to True, - this module has learnable per-element affine parameters - initialized to ones (for weights) and zeros (for biases). - - Example - ------- - >>> x = torch.randn(5, 10, 20) - >>> GLN = GlobalLayerNorm(10, 3) - >>> x_norm = GLN(x) - """ - - def __init__(self, dim, shape, eps=1e-8, elementwise_affine=True): - super(GlobalLayerNorm, self).__init__() - self.dim = dim - self.eps = eps - self.elementwise_affine = elementwise_affine - - if self.elementwise_affine: - if shape == 3: - self.weight = nn.Parameter(torch.ones(self.dim, 1)) - self.bias = nn.Parameter(torch.zeros(self.dim, 1)) - if shape == 4: - self.weight = nn.Parameter(torch.ones(self.dim, 1, 1)) - self.bias = nn.Parameter(torch.zeros(self.dim, 1, 1)) - else: - self.register_parameter("weight", None) - self.register_parameter("bias", None) - - def forward(self, x): - """Returns the normalized tensor. - - Arguments - --------- - x : torch.Tensor - Tensor of size [N, C, K, S] or [N, C, L]. - """ - # x = N x C x K x S or N x C x L - # N x 1 x 1 - # cln: mean,var N x 1 x K x S - # gln: mean,var N x 1 x 1 - if x.dim() == 3: - mean = torch.mean(x, (1, 2), keepdim=True) - var = torch.mean((x - mean) ** 2, (1, 2), keepdim=True) - if self.elementwise_affine: - x = self.weight * (x - mean) / torch.sqrt(var + self.eps) + self.bias - else: - x = (x - mean) / torch.sqrt(var + self.eps) - - if x.dim() == 4: - mean = torch.mean(x, (1, 2, 3), keepdim=True) - var = torch.mean((x - mean) ** 2, (1, 2, 3), keepdim=True) - if self.elementwise_affine: - x = self.weight * (x - mean) / torch.sqrt(var + self.eps) + self.bias - else: - x = (x - mean) / torch.sqrt(var + self.eps) - return x - - -class CumulativeLayerNorm(nn.LayerNorm): - """Calculate Cumulative Layer Normalization. - - Arguments - --------- - dim : int - Dimension that you want to normalize. - elementwise_affine : True - Learnable per-element affine parameters. - - Example - ------- - >>> x = torch.randn(5, 10, 20) - >>> CLN = CumulativeLayerNorm(10) - >>> x_norm = CLN(x) - """ - - def __init__(self, dim, elementwise_affine=True): - super(CumulativeLayerNorm, self).__init__( - dim, elementwise_affine=elementwise_affine, eps=1e-8 - ) - - def forward(self, x): - """Returns the normalized tensor. - - Arguments - --------- - x : torch.Tensor - Tensor size [N, C, K, S] or [N, C, L] - """ - # x: N x C x K x S or N x C x L - # N x K x S x C - if x.dim() == 4: - x = x.permute(0, 2, 3, 1).contiguous() - # N x K x S x C == only channel norm - x = super().forward(x) - # N x C x K x S - x = x.permute(0, 3, 1, 2).contiguous() - if x.dim() == 3: - x = torch.transpose(x, 1, 2) - # N x L x C == only channel norm - x = super().forward(x) - # N x C x L - x = torch.transpose(x, 1, 2) - return x - - -class ScaleNorm(nn.Module): - def __init__(self, dim, eps=1e-5): - super().__init__() - self.scale = dim**-0.5 - self.eps = eps - self.g = nn.Parameter(torch.ones(1)) - - def forward(self, x): - norm = torch.norm(x, dim=-1, keepdim=True) * self.scale - return x / norm.clamp(min=self.eps) * self.g diff --git a/funasr_detach/models/transformer/model.py b/funasr_detach/models/transformer/model.py deleted file mode 100644 index eed8cebe5650012e149b626a8fe77089bf72f721..0000000000000000000000000000000000000000 --- a/funasr_detach/models/transformer/model.py +++ /dev/null @@ -1,478 +0,0 @@ -import logging -from typing import Union, Dict, List, Tuple, Optional - -import time -import torch -import torch.nn as nn -from torch.cuda.amp import autocast - -from funasr_detach.losses.label_smoothing_loss import LabelSmoothingLoss -from funasr_detach.models.ctc.ctc import CTC -from funasr_detach.models.transformer.utils.add_sos_eos import add_sos_eos -from funasr_detach.metrics.compute_acc import th_accuracy - -# from funasr_detach.models.e2e_asr_common import ErrorCalculator -from funasr_detach.train_utils.device_funcs import force_gatherable -from funasr_detach.utils.load_utils import load_audio_text_image_video, extract_fbank -from funasr_detach.utils import postprocess_utils -from funasr_detach.utils.datadir_writer import DatadirWriter -from funasr_detach.register import tables - - -@tables.register("model_classes", "Transformer") -class Transformer(nn.Module): - """CTC-attention hybrid Encoder-Decoder model""" - - def __init__( - self, - specaug: str = None, - specaug_conf: dict = None, - normalize: str = None, - normalize_conf: dict = None, - encoder: str = None, - encoder_conf: dict = None, - decoder: str = None, - decoder_conf: dict = None, - ctc: str = None, - ctc_conf: dict = None, - ctc_weight: float = 0.5, - interctc_weight: float = 0.0, - input_size: int = 80, - vocab_size: int = -1, - ignore_id: int = -1, - blank_id: int = 0, - sos: int = 1, - eos: int = 2, - lsm_weight: float = 0.0, - length_normalized_loss: bool = False, - report_cer: bool = True, - report_wer: bool = True, - sym_space: str = "", - sym_blank: str = "", - # extract_feats_in_collect_stats: bool = True, - share_embedding: bool = False, - # preencoder: Optional[AbsPreEncoder] = None, - # postencoder: Optional[AbsPostEncoder] = None, - **kwargs, - ): - - super().__init__() - - if specaug is not None: - specaug_class = tables.specaug_classes.get(specaug) - specaug = specaug_class(**specaug_conf) - if normalize is not None: - normalize_class = tables.normalize_classes.get(normalize) - normalize = normalize_class(**normalize_conf) - encoder_class = tables.encoder_classes.get(encoder) - encoder = encoder_class(input_size=input_size, **encoder_conf) - encoder_output_size = encoder.output_size() - if decoder is not None: - decoder_class = tables.decoder_classes.get(decoder) - decoder = decoder_class( - vocab_size=vocab_size, - encoder_output_size=encoder_output_size, - **decoder_conf, - ) - if ctc_weight > 0.0: - - if ctc_conf is None: - ctc_conf = {} - - ctc = CTC( - odim=vocab_size, encoder_output_size=encoder_output_size, **ctc_conf - ) - - self.blank_id = blank_id - self.sos = sos if sos is not None else vocab_size - 1 - self.eos = eos if eos is not None else vocab_size - 1 - self.vocab_size = vocab_size - self.ignore_id = ignore_id - self.ctc_weight = ctc_weight - self.specaug = specaug - self.normalize = normalize - self.encoder = encoder - - if not hasattr(self.encoder, "interctc_use_conditioning"): - self.encoder.interctc_use_conditioning = False - if self.encoder.interctc_use_conditioning: - self.encoder.conditioning_layer = torch.nn.Linear( - vocab_size, self.encoder.output_size() - ) - self.interctc_weight = interctc_weight - - # self.error_calculator = None - if ctc_weight == 1.0: - self.decoder = None - else: - self.decoder = decoder - - self.criterion_att = LabelSmoothingLoss( - size=vocab_size, - padding_idx=ignore_id, - smoothing=lsm_weight, - normalize_length=length_normalized_loss, - ) - # - # if report_cer or report_wer: - # self.error_calculator = ErrorCalculator( - # token_list, sym_space, sym_blank, report_cer, report_wer - # ) - # - self.error_calculator = None - if ctc_weight == 0.0: - self.ctc = None - else: - self.ctc = ctc - - self.share_embedding = share_embedding - if self.share_embedding: - self.decoder.embed = None - - self.length_normalized_loss = length_normalized_loss - self.beam_search = None - - def forward( - self, - speech: torch.Tensor, - speech_lengths: torch.Tensor, - text: torch.Tensor, - text_lengths: torch.Tensor, - **kwargs, - ) -> Tuple[torch.Tensor, Dict[str, torch.Tensor], torch.Tensor]: - """Encoder + Decoder + Calc loss - Args: - speech: (Batch, Length, ...) - speech_lengths: (Batch, ) - text: (Batch, Length) - text_lengths: (Batch,) - """ - # import pdb; - # pdb.set_trace() - if len(text_lengths.size()) > 1: - text_lengths = text_lengths[:, 0] - if len(speech_lengths.size()) > 1: - speech_lengths = speech_lengths[:, 0] - - batch_size = speech.shape[0] - - # 1. Encoder - encoder_out, encoder_out_lens = self.encode(speech, speech_lengths) - intermediate_outs = None - if isinstance(encoder_out, tuple): - intermediate_outs = encoder_out[1] - encoder_out = encoder_out[0] - - loss_att, acc_att, cer_att, wer_att = None, None, None, None - loss_ctc, cer_ctc = None, None - stats = dict() - - # decoder: CTC branch - if self.ctc_weight != 0.0: - loss_ctc, cer_ctc = self._calc_ctc_loss( - encoder_out, encoder_out_lens, text, text_lengths - ) - - # Collect CTC branch stats - stats["loss_ctc"] = loss_ctc.detach() if loss_ctc is not None else None - stats["cer_ctc"] = cer_ctc - - # Intermediate CTC (optional) - loss_interctc = 0.0 - if self.interctc_weight != 0.0 and intermediate_outs is not None: - for layer_idx, intermediate_out in intermediate_outs: - # we assume intermediate_out has the same length & padding - # as those of encoder_out - loss_ic, cer_ic = self._calc_ctc_loss( - intermediate_out, encoder_out_lens, text, text_lengths - ) - loss_interctc = loss_interctc + loss_ic - - # Collect Intermedaite CTC stats - stats["loss_interctc_layer{}".format(layer_idx)] = ( - loss_ic.detach() if loss_ic is not None else None - ) - stats["cer_interctc_layer{}".format(layer_idx)] = cer_ic - - loss_interctc = loss_interctc / len(intermediate_outs) - - # calculate whole encoder loss - loss_ctc = ( - 1 - self.interctc_weight - ) * loss_ctc + self.interctc_weight * loss_interctc - - # decoder: Attention decoder branch - loss_att, acc_att, cer_att, wer_att = self._calc_att_loss( - encoder_out, encoder_out_lens, text, text_lengths - ) - - # 3. CTC-Att loss definition - if self.ctc_weight == 0.0: - loss = loss_att - elif self.ctc_weight == 1.0: - loss = loss_ctc - else: - loss = self.ctc_weight * loss_ctc + (1 - self.ctc_weight) * loss_att - - # Collect Attn branch stats - stats["loss_att"] = loss_att.detach() if loss_att is not None else None - stats["acc"] = acc_att - stats["cer"] = cer_att - stats["wer"] = wer_att - - # Collect total loss stats - stats["loss"] = torch.clone(loss.detach()) - - # force_gatherable: to-device and to-tensor if scalar for DataParallel - if self.length_normalized_loss: - batch_size = int((text_lengths + 1).sum()) - loss, stats, weight = force_gatherable((loss, stats, batch_size), loss.device) - return loss, stats, weight - - def encode( - self, - speech: torch.Tensor, - speech_lengths: torch.Tensor, - **kwargs, - ) -> Tuple[torch.Tensor, torch.Tensor]: - """Frontend + Encoder. Note that this method is used by asr_inference.py - Args: - speech: (Batch, Length, ...) - speech_lengths: (Batch, ) - ind: int - """ - with autocast(False): - - # Data augmentation - if self.specaug is not None and self.training: - speech, speech_lengths = self.specaug(speech, speech_lengths) - - # Normalization for feature: e.g. Global-CMVN, Utterance-CMVN - if self.normalize is not None: - speech, speech_lengths = self.normalize(speech, speech_lengths) - - # Forward encoder - # feats: (Batch, Length, Dim) - # -> encoder_out: (Batch, Length2, Dim2) - if self.encoder.interctc_use_conditioning: - encoder_out, encoder_out_lens, _ = self.encoder( - speech, speech_lengths, ctc=self.ctc - ) - else: - encoder_out, encoder_out_lens, _ = self.encoder(speech, speech_lengths) - intermediate_outs = None - if isinstance(encoder_out, tuple): - intermediate_outs = encoder_out[1] - encoder_out = encoder_out[0] - - if intermediate_outs is not None: - return (encoder_out, intermediate_outs), encoder_out_lens - - return encoder_out, encoder_out_lens - - def _calc_att_loss( - self, - encoder_out: torch.Tensor, - encoder_out_lens: torch.Tensor, - ys_pad: torch.Tensor, - ys_pad_lens: torch.Tensor, - ): - ys_in_pad, ys_out_pad = add_sos_eos(ys_pad, self.sos, self.eos, self.ignore_id) - ys_in_lens = ys_pad_lens + 1 - - # 1. Forward decoder - decoder_out, _ = self.decoder( - encoder_out, encoder_out_lens, ys_in_pad, ys_in_lens - ) - - # 2. Compute attention loss - loss_att = self.criterion_att(decoder_out, ys_out_pad) - acc_att = th_accuracy( - decoder_out.view(-1, self.vocab_size), - ys_out_pad, - ignore_label=self.ignore_id, - ) - - # Compute cer/wer using attention-decoder - if self.training or self.error_calculator is None: - cer_att, wer_att = None, None - else: - ys_hat = decoder_out.argmax(dim=-1) - cer_att, wer_att = self.error_calculator(ys_hat.cpu(), ys_pad.cpu()) - - return loss_att, acc_att, cer_att, wer_att - - def _calc_ctc_loss( - self, - encoder_out: torch.Tensor, - encoder_out_lens: torch.Tensor, - ys_pad: torch.Tensor, - ys_pad_lens: torch.Tensor, - ): - # Calc CTC loss - loss_ctc = self.ctc(encoder_out, encoder_out_lens, ys_pad, ys_pad_lens) - - # Calc CER using CTC - cer_ctc = None - if not self.training and self.error_calculator is not None: - ys_hat = self.ctc.argmax(encoder_out).data - cer_ctc = self.error_calculator(ys_hat.cpu(), ys_pad.cpu(), is_ctc=True) - return loss_ctc, cer_ctc - - def init_beam_search( - self, - **kwargs, - ): - from funasr_detach.models.transformer.search import BeamSearch - from funasr_detach.models.transformer.scorers.ctc import CTCPrefixScorer - from funasr_detach.models.transformer.scorers.length_bonus import LengthBonus - - # 1. Build ASR model - scorers = {} - - if self.ctc != None: - ctc = CTCPrefixScorer(ctc=self.ctc, eos=self.eos) - scorers.update(ctc=ctc) - token_list = kwargs.get("token_list") - scorers.update( - decoder=self.decoder, - length_bonus=LengthBonus(len(token_list)), - ) - - # 3. Build ngram model - # ngram is not supported now - ngram = None - scorers["ngram"] = ngram - - weights = dict( - decoder=1.0 - kwargs.get("decoding_ctc_weight", 0.5), - ctc=kwargs.get("decoding_ctc_weight", 0.5), - lm=kwargs.get("lm_weight", 0.0), - ngram=kwargs.get("ngram_weight", 0.0), - length_bonus=kwargs.get("penalty", 0.0), - ) - beam_search = BeamSearch( - beam_size=kwargs.get("beam_size", 10), - weights=weights, - scorers=scorers, - sos=self.sos, - eos=self.eos, - vocab_size=len(token_list), - token_list=token_list, - pre_beam_score_key=None if self.ctc_weight == 1.0 else "full", - ) - - self.beam_search = beam_search - - def inference( - self, - data_in, - data_lengths=None, - key: list = None, - tokenizer=None, - frontend=None, - **kwargs, - ): - - if kwargs.get("batch_size", 1) > 1: - raise NotImplementedError("batch decoding is not implemented") - - # init beamsearch - if self.beam_search is None: - logging.info("enable beam_search") - self.init_beam_search(**kwargs) - self.nbest = kwargs.get("nbest", 1) - - meta_data = {} - if ( - isinstance(data_in, torch.Tensor) - and kwargs.get("data_type", "sound") == "fbank" - ): # fbank - speech, speech_lengths = data_in, data_lengths - if len(speech.shape) < 3: - speech = speech[None, :, :] - if speech_lengths is None: - speech_lengths = speech.shape[1] - else: - # extract fbank feats - time1 = time.perf_counter() - audio_sample_list = load_audio_text_image_video( - data_in, - fs=frontend.fs, - audio_fs=kwargs.get("fs", 16000), - data_type=kwargs.get("data_type", "sound"), - tokenizer=tokenizer, - ) - time2 = time.perf_counter() - meta_data["load_data"] = f"{time2 - time1:0.3f}" - speech, speech_lengths = extract_fbank( - audio_sample_list, - data_type=kwargs.get("data_type", "sound"), - frontend=frontend, - ) - time3 = time.perf_counter() - meta_data["extract_feat"] = f"{time3 - time2:0.3f}" - meta_data["batch_data_time"] = ( - speech_lengths.sum().item() - * frontend.frame_shift - * frontend.lfr_n - / 1000 - ) - - speech = speech.to(device=kwargs["device"]) - speech_lengths = speech_lengths.to(device=kwargs["device"]) - # Encoder - encoder_out, encoder_out_lens = self.encode(speech, speech_lengths) - if isinstance(encoder_out, tuple): - encoder_out = encoder_out[0] - - # c. Passed the encoder result and the beam search - nbest_hyps = self.beam_search( - x=encoder_out[0], - maxlenratio=kwargs.get("maxlenratio", 0.0), - minlenratio=kwargs.get("minlenratio", 0.0), - ) - - nbest_hyps = nbest_hyps[: self.nbest] - - results = [] - b, n, d = encoder_out.size() - for i in range(b): - - for nbest_idx, hyp in enumerate(nbest_hyps): - ibest_writer = None - if kwargs.get("output_dir") is not None: - if not hasattr(self, "writer"): - self.writer = DatadirWriter(kwargs.get("output_dir")) - ibest_writer = self.writer[f"{nbest_idx + 1}best_recog"] - - # remove sos/eos and get results - last_pos = -1 - if isinstance(hyp.yseq, list): - token_int = hyp.yseq[1:last_pos] - else: - token_int = hyp.yseq[1:last_pos].tolist() - - # remove blank symbol id, which is assumed to be 0 - token_int = list( - filter( - lambda x: x != self.eos - and x != self.sos - and x != self.blank_id, - token_int, - ) - ) - - # Change integer-ids to tokens - token = tokenizer.ids2tokens(token_int) - text = tokenizer.tokens2text(token) - - text_postprocessed, _ = postprocess_utils.sentence_postprocess(token) - result_i = {"key": key[i], "token": token, "text": text_postprocessed} - results.append(result_i) - - if ibest_writer is not None: - ibest_writer["token"][key[i]] = " ".join(token) - ibest_writer["text"][key[i]] = text_postprocessed - - return results, meta_data diff --git a/funasr_detach/models/transformer/positionwise_feed_forward.py b/funasr_detach/models/transformer/positionwise_feed_forward.py deleted file mode 100644 index 993b3245bfca5f4f6c2a08399c1e0814972a2379..0000000000000000000000000000000000000000 --- a/funasr_detach/models/transformer/positionwise_feed_forward.py +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- - -# Copyright 2019 Shigeki Karita -# Apache 2.0 (http://www.apache.org/licenses/LICENSE-2.0) - -"""Positionwise feed forward layer definition.""" - -import torch - -from funasr_detach.models.transformer.layer_norm import LayerNorm - - -class PositionwiseFeedForward(torch.nn.Module): - """Positionwise feed forward layer. - - Args: - idim (int): Input dimenstion. - hidden_units (int): The number of hidden units. - dropout_rate (float): Dropout rate. - - """ - - def __init__(self, idim, hidden_units, dropout_rate, activation=torch.nn.ReLU()): - """Construct an PositionwiseFeedForward object.""" - super(PositionwiseFeedForward, self).__init__() - self.w_1 = torch.nn.Linear(idim, hidden_units) - self.w_2 = torch.nn.Linear(hidden_units, idim) - self.dropout = torch.nn.Dropout(dropout_rate) - self.activation = activation - - def forward(self, x): - """Forward function.""" - return self.w_2(self.dropout(self.activation(self.w_1(x)))) diff --git a/funasr_detach/models/transformer/scorers/__init__.py b/funasr_detach/models/transformer/scorers/__init__.py deleted file mode 100644 index b7f177368e62a5578b8706300e101f831a3972ac..0000000000000000000000000000000000000000 --- a/funasr_detach/models/transformer/scorers/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Initialize sub package.""" diff --git a/funasr_detach/models/transformer/scorers/ctc.py b/funasr_detach/models/transformer/scorers/ctc.py deleted file mode 100644 index c4e1cf367fb78932e5db409d14259b3b12837bd2..0000000000000000000000000000000000000000 --- a/funasr_detach/models/transformer/scorers/ctc.py +++ /dev/null @@ -1,160 +0,0 @@ -"""ScorerInterface implementation for CTC.""" - -import numpy as np -import torch - -from funasr_detach.models.transformer.scorers.ctc_prefix_score import CTCPrefixScore -from funasr_detach.models.transformer.scorers.ctc_prefix_score import CTCPrefixScoreTH -from funasr_detach.models.transformer.scorers.scorer_interface import ( - BatchPartialScorerInterface, -) - - -class CTCPrefixScorer(BatchPartialScorerInterface): - """Decoder interface wrapper for CTCPrefixScore.""" - - def __init__(self, ctc: torch.nn.Module, eos: int): - """Initialize class. - - Args: - ctc (torch.nn.Module): The CTC implementation. - For example, :class:`espnet.nets.pytorch_backend.ctc.CTC` - eos (int): The end-of-sequence id. - - """ - self.ctc = ctc - self.eos = eos - self.impl = None - - def init_state(self, x: torch.Tensor): - """Get an initial state for decoding. - - Args: - x (torch.Tensor): The encoded feature tensor - - Returns: initial state - - """ - logp = self.ctc.log_softmax(x.unsqueeze(0)).detach().squeeze(0).cpu().numpy() - # TODO(karita): use CTCPrefixScoreTH - self.impl = CTCPrefixScore(logp, 0, self.eos, np) - return 0, self.impl.initial_state() - - def select_state(self, state, i, new_id=None): - """Select state with relative ids in the main beam search. - - Args: - state: Decoder state for prefix tokens - i (int): Index to select a state in the main beam search - new_id (int): New label id to select a state if necessary - - Returns: - state: pruned state - - """ - if type(state) == tuple: - if len(state) == 2: # for CTCPrefixScore - sc, st = state - return sc[i], st[i] - else: # for CTCPrefixScoreTH (need new_id > 0) - r, log_psi, f_min, f_max, scoring_idmap = state - s = log_psi[i, new_id].expand(log_psi.size(1)) - if scoring_idmap is not None: - return r[:, :, i, scoring_idmap[i, new_id]], s, f_min, f_max - else: - return r[:, :, i, new_id], s, f_min, f_max - return None if state is None else state[i] - - def score_partial(self, y, ids, state, x): - """Score new token. - - Args: - y (torch.Tensor): 1D prefix token - next_tokens (torch.Tensor): torch.int64 next token to score - state: decoder state for prefix tokens - x (torch.Tensor): 2D encoder feature that generates ys - - Returns: - tuple[torch.Tensor, Any]: - Tuple of a score tensor for y that has a shape `(len(next_tokens),)` - and next state for ys - - """ - prev_score, state = state - presub_score, new_st = self.impl(y.cpu(), ids.cpu(), state) - tscore = torch.as_tensor( - presub_score - prev_score, device=x.device, dtype=x.dtype - ) - return tscore, (presub_score, new_st) - - def batch_init_state(self, x: torch.Tensor): - """Get an initial state for decoding. - - Args: - x (torch.Tensor): The encoded feature tensor - - Returns: initial state - - """ - logp = self.ctc.log_softmax(x.unsqueeze(0)) # assuming batch_size = 1 - xlen = torch.tensor([logp.size(1)]) - self.impl = CTCPrefixScoreTH(logp, xlen, 0, self.eos) - return None - - def batch_score_partial(self, y, ids, state, x): - """Score new token. - - Args: - y (torch.Tensor): 1D prefix token - ids (torch.Tensor): torch.int64 next token to score - state: decoder state for prefix tokens - x (torch.Tensor): 2D encoder feature that generates ys - - Returns: - tuple[torch.Tensor, Any]: - Tuple of a score tensor for y that has a shape `(len(next_tokens),)` - and next state for ys - - """ - batch_state = ( - ( - torch.stack([s[0] for s in state], dim=2), - torch.stack([s[1] for s in state]), - state[0][2], - state[0][3], - ) - if state[0] is not None - else None - ) - return self.impl(y, batch_state, ids) - - def extend_prob(self, x: torch.Tensor): - """Extend probs for decoding. - - This extension is for streaming decoding - as in Eq (14) in https://arxiv.org/abs/2006.14941 - - Args: - x (torch.Tensor): The encoded feature tensor - - """ - logp = self.ctc.log_softmax(x.unsqueeze(0)) - self.impl.extend_prob(logp) - - def extend_state(self, state): - """Extend state for decoding. - - This extension is for streaming decoding - as in Eq (14) in https://arxiv.org/abs/2006.14941 - - Args: - state: The states of hyps - - Returns: exteded state - - """ - new_state = [] - for s in state: - new_state.append(self.impl.extend_state(s)) - - return new_state diff --git a/funasr_detach/models/transformer/scorers/ctc_prefix_score.py b/funasr_detach/models/transformer/scorers/ctc_prefix_score.py deleted file mode 100644 index 0c67ecd096de46ad00972cf3a8ba812852f38c97..0000000000000000000000000000000000000000 --- a/funasr_detach/models/transformer/scorers/ctc_prefix_score.py +++ /dev/null @@ -1,359 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright 2018 Mitsubishi Electric Research Labs (Takaaki Hori) -# Apache 2.0 (http://www.apache.org/licenses/LICENSE-2.0) - -import torch - -import numpy as np -import six - - -class CTCPrefixScoreTH(object): - """Batch processing of CTCPrefixScore - - which is based on Algorithm 2 in WATANABE et al. - "HYBRID CTC/ATTENTION ARCHITECTURE FOR END-TO-END SPEECH RECOGNITION," - but extended to efficiently compute the label probablities for multiple - hypotheses simultaneously - See also Seki et al. "Vectorized Beam Search for CTC-Attention-Based - Speech Recognition," In INTERSPEECH (pp. 3825-3829), 2019. - """ - - def __init__(self, x, xlens, blank, eos, margin=0): - """Construct CTC prefix scorer - - :param torch.Tensor x: input label posterior sequences (B, T, O) - :param torch.Tensor xlens: input lengths (B,) - :param int blank: blank label id - :param int eos: end-of-sequence id - :param int margin: margin parameter for windowing (0 means no windowing) - """ - # In the comment lines, - # we assume T: input_length, B: batch size, W: beam width, O: output dim. - self.logzero = -10000000000.0 - self.blank = blank - self.eos = eos - self.batch = x.size(0) - self.input_length = x.size(1) - self.odim = x.size(2) - self.dtype = x.dtype - self.device = ( - torch.device("cuda:%d" % x.get_device()) - if x.is_cuda - else torch.device("cpu") - ) - # Pad the rest of posteriors in the batch - # TODO(takaaki-hori): need a better way without for-loops - for i, l in enumerate(xlens): - if l < self.input_length: - x[i, l:, :] = self.logzero - x[i, l:, blank] = 0 - # Reshape input x - xn = x.transpose(0, 1) # (B, T, O) -> (T, B, O) - xb = xn[:, :, self.blank].unsqueeze(2).expand(-1, -1, self.odim) - self.x = torch.stack([xn, xb]) # (2, T, B, O) - self.end_frames = torch.as_tensor(xlens) - 1 - - # Setup CTC windowing - self.margin = margin - if margin > 0: - self.frame_ids = torch.arange( - self.input_length, dtype=self.dtype, device=self.device - ) - # Base indices for index conversion - self.idx_bh = None - self.idx_b = torch.arange(self.batch, device=self.device) - self.idx_bo = (self.idx_b * self.odim).unsqueeze(1) - - def __call__(self, y, state, scoring_ids=None, att_w=None): - """Compute CTC prefix scores for next labels - - :param list y: prefix label sequences - :param tuple state: previous CTC state - :param torch.Tensor pre_scores: scores for pre-selection of hypotheses (BW, O) - :param torch.Tensor att_w: attention weights to decide CTC window - :return new_state, ctc_local_scores (BW, O) - """ - output_length = len(y[0]) - 1 # ignore sos - last_ids = [yi[-1] for yi in y] # last output label ids - n_bh = len(last_ids) # batch * hyps - n_hyps = n_bh // self.batch # assuming each utterance has the same # of hyps - self.scoring_num = scoring_ids.size(-1) if scoring_ids is not None else 0 - # prepare state info - if state is None: - r_prev = torch.full( - (self.input_length, 2, self.batch, n_hyps), - self.logzero, - dtype=self.dtype, - device=self.device, - ) - r_prev[:, 1] = torch.cumsum(self.x[0, :, :, self.blank], 0).unsqueeze(2) - r_prev = r_prev.view(-1, 2, n_bh) - s_prev = 0.0 - f_min_prev = 0 - f_max_prev = 1 - else: - r_prev, s_prev, f_min_prev, f_max_prev = state - - # select input dimensions for scoring - if self.scoring_num > 0: - scoring_idmap = torch.full( - (n_bh, self.odim), -1, dtype=torch.long, device=self.device - ) - snum = self.scoring_num - if self.idx_bh is None or n_bh > len(self.idx_bh): - self.idx_bh = torch.arange(n_bh, device=self.device).view(-1, 1) - scoring_idmap[self.idx_bh[:n_bh], scoring_ids] = torch.arange( - snum, device=self.device - ) - scoring_idx = ( - scoring_ids + self.idx_bo.repeat(1, n_hyps).view(-1, 1) - ).view(-1) - x_ = torch.index_select( - self.x.view(2, -1, self.batch * self.odim), 2, scoring_idx - ).view(2, -1, n_bh, snum) - else: - scoring_ids = None - scoring_idmap = None - snum = self.odim - x_ = self.x.unsqueeze(3).repeat(1, 1, 1, n_hyps, 1).view(2, -1, n_bh, snum) - - # new CTC forward probs are prepared as a (T x 2 x BW x S) tensor - # that corresponds to r_t^n(h) and r_t^b(h) in a batch. - r = torch.full( - (self.input_length, 2, n_bh, snum), - self.logzero, - dtype=self.dtype, - device=self.device, - ) - if output_length == 0: - r[0, 0] = x_[0, 0] - - r_sum = torch.logsumexp(r_prev, 1) - log_phi = r_sum.unsqueeze(2).repeat(1, 1, snum) - if scoring_ids is not None: - for idx in range(n_bh): - pos = scoring_idmap[idx, last_ids[idx]] - if pos >= 0: - log_phi[:, idx, pos] = r_prev[:, 1, idx] - else: - for idx in range(n_bh): - log_phi[:, idx, last_ids[idx]] = r_prev[:, 1, idx] - - # decide start and end frames based on attention weights - if att_w is not None and self.margin > 0: - f_arg = torch.matmul(att_w, self.frame_ids) - f_min = max(int(f_arg.min().cpu()), f_min_prev) - f_max = max(int(f_arg.max().cpu()), f_max_prev) - start = min(f_max_prev, max(f_min - self.margin, output_length, 1)) - end = min(f_max + self.margin, self.input_length) - else: - f_min = f_max = 0 - start = max(output_length, 1) - end = self.input_length - - # compute forward probabilities log(r_t^n(h)) and log(r_t^b(h)) - for t in range(start, end): - rp = r[t - 1] - rr = torch.stack([rp[0], log_phi[t - 1], rp[0], rp[1]]).view( - 2, 2, n_bh, snum - ) - r[t] = torch.logsumexp(rr, 1) + x_[:, t] - - # compute log prefix probabilities log(psi) - log_phi_x = torch.cat((log_phi[0].unsqueeze(0), log_phi[:-1]), dim=0) + x_[0] - if scoring_ids is not None: - log_psi = torch.full( - (n_bh, self.odim), self.logzero, dtype=self.dtype, device=self.device - ) - log_psi_ = torch.logsumexp( - torch.cat((log_phi_x[start:end], r[start - 1, 0].unsqueeze(0)), dim=0), - dim=0, - ) - for si in range(n_bh): - log_psi[si, scoring_ids[si]] = log_psi_[si] - else: - log_psi = torch.logsumexp( - torch.cat((log_phi_x[start:end], r[start - 1, 0].unsqueeze(0)), dim=0), - dim=0, - ) - - for si in range(n_bh): - log_psi[si, self.eos] = r_sum[self.end_frames[si // n_hyps], si] - - # exclude blank probs - log_psi[:, self.blank] = self.logzero - - return (log_psi - s_prev), (r, log_psi, f_min, f_max, scoring_idmap) - - def index_select_state(self, state, best_ids): - """Select CTC states according to best ids - - :param state : CTC state - :param best_ids : index numbers selected by beam pruning (B, W) - :return selected_state - """ - r, s, f_min, f_max, scoring_idmap = state - # convert ids to BHO space - n_bh = len(s) - n_hyps = n_bh // self.batch - vidx = (best_ids + (self.idx_b * (n_hyps * self.odim)).view(-1, 1)).view(-1) - # select hypothesis scores - s_new = torch.index_select(s.view(-1), 0, vidx) - s_new = s_new.view(-1, 1).repeat(1, self.odim).view(n_bh, self.odim) - # convert ids to BHS space (S: scoring_num) - if scoring_idmap is not None: - snum = self.scoring_num - hyp_idx = (best_ids // self.odim + (self.idx_b * n_hyps).view(-1, 1)).view( - -1 - ) - label_ids = torch.fmod(best_ids, self.odim).view(-1) - score_idx = scoring_idmap[hyp_idx, label_ids] - score_idx[score_idx == -1] = 0 - vidx = score_idx + hyp_idx * snum - else: - snum = self.odim - # select forward probabilities - r_new = torch.index_select(r.view(-1, 2, n_bh * snum), 2, vidx).view( - -1, 2, n_bh - ) - return r_new, s_new, f_min, f_max - - def extend_prob(self, x): - """Extend CTC prob. - - :param torch.Tensor x: input label posterior sequences (B, T, O) - """ - - if self.x.shape[1] < x.shape[1]: # self.x (2,T,B,O); x (B,T,O) - # Pad the rest of posteriors in the batch - # TODO(takaaki-hori): need a better way without for-loops - xlens = [x.size(1)] - for i, l in enumerate(xlens): - if l < self.input_length: - x[i, l:, :] = self.logzero - x[i, l:, self.blank] = 0 - tmp_x = self.x - xn = x.transpose(0, 1) # (B, T, O) -> (T, B, O) - xb = xn[:, :, self.blank].unsqueeze(2).expand(-1, -1, self.odim) - self.x = torch.stack([xn, xb]) # (2, T, B, O) - self.x[:, : tmp_x.shape[1], :, :] = tmp_x - self.input_length = x.size(1) - self.end_frames = torch.as_tensor(xlens) - 1 - - def extend_state(self, state): - """Compute CTC prefix state. - - - :param state : CTC state - :return ctc_state - """ - - if state is None: - # nothing to do - return state - else: - r_prev, s_prev, f_min_prev, f_max_prev = state - - r_prev_new = torch.full( - (self.input_length, 2), - self.logzero, - dtype=self.dtype, - device=self.device, - ) - start = max(r_prev.shape[0], 1) - r_prev_new[0:start] = r_prev - for t in six.moves.range(start, self.input_length): - r_prev_new[t, 1] = r_prev_new[t - 1, 1] + self.x[0, t, :, self.blank] - - return (r_prev_new, s_prev, f_min_prev, f_max_prev) - - -class CTCPrefixScore(object): - """Compute CTC label sequence scores - - which is based on Algorithm 2 in WATANABE et al. - "HYBRID CTC/ATTENTION ARCHITECTURE FOR END-TO-END SPEECH RECOGNITION," - but extended to efficiently compute the probablities of multiple labels - simultaneously - """ - - def __init__(self, x, blank, eos, xp): - self.xp = xp - self.logzero = -10000000000.0 - self.blank = blank - self.eos = eos - self.input_length = len(x) - self.x = x - - def initial_state(self): - """Obtain an initial CTC state - - :return: CTC state - """ - # initial CTC state is made of a frame x 2 tensor that corresponds to - # r_t^n() and r_t^b(), where 0 and 1 of axis=1 represent - # superscripts n and b (non-blank and blank), respectively. - r = self.xp.full((self.input_length, 2), self.logzero, dtype=np.float32) - r[0, 1] = self.x[0, self.blank] - for i in six.moves.range(1, self.input_length): - r[i, 1] = r[i - 1, 1] + self.x[i, self.blank] - return r - - def __call__(self, y, cs, r_prev): - """Compute CTC prefix scores for next labels - - :param y : prefix label sequence - :param cs : array of next labels - :param r_prev: previous CTC state - :return ctc_scores, ctc_states - """ - # initialize CTC states - output_length = len(y) - 1 # ignore sos - # new CTC states are prepared as a frame x (n or b) x n_labels tensor - # that corresponds to r_t^n(h) and r_t^b(h). - r = self.xp.ndarray((self.input_length, 2, len(cs)), dtype=np.float32) - xs = self.x[:, cs] - if output_length == 0: - r[0, 0] = xs[0] - r[0, 1] = self.logzero - else: - r[output_length - 1] = self.logzero - - # prepare forward probabilities for the last label - r_sum = self.xp.logaddexp( - r_prev[:, 0], r_prev[:, 1] - ) # log(r_t^n(g) + r_t^b(g)) - last = y[-1] - if output_length > 0 and last in cs: - log_phi = self.xp.ndarray((self.input_length, len(cs)), dtype=np.float32) - for i in six.moves.range(len(cs)): - log_phi[:, i] = r_sum if cs[i] != last else r_prev[:, 1] - else: - log_phi = r_sum - - # compute forward probabilities log(r_t^n(h)), log(r_t^b(h)), - # and log prefix probabilities log(psi) - start = max(output_length, 1) - log_psi = r[start - 1, 0] - for t in six.moves.range(start, self.input_length): - r[t, 0] = self.xp.logaddexp(r[t - 1, 0], log_phi[t - 1]) + xs[t] - r[t, 1] = ( - self.xp.logaddexp(r[t - 1, 0], r[t - 1, 1]) + self.x[t, self.blank] - ) - log_psi = self.xp.logaddexp(log_psi, log_phi[t - 1] + xs[t]) - - # get P(...eos|X) that ends with the prefix itself - eos_pos = self.xp.where(cs == self.eos)[0] - if len(eos_pos) > 0: - log_psi[eos_pos] = r_sum[-1] # log(r_T^n(g) + r_T^b(g)) - - # exclude blank probs - blank_pos = self.xp.where(cs == self.blank)[0] - if len(blank_pos) > 0: - log_psi[blank_pos] = self.logzero - - # return the log prefix probability and CTC states, where the label axis - # of the CTC states is moved to the first axis to slice it easily - return log_psi, self.xp.rollaxis(r, 2) diff --git a/funasr_detach/models/transformer/scorers/length_bonus.py b/funasr_detach/models/transformer/scorers/length_bonus.py deleted file mode 100644 index c4ba0a47f0db1de45a616343c8919b979e4255ea..0000000000000000000000000000000000000000 --- a/funasr_detach/models/transformer/scorers/length_bonus.py +++ /dev/null @@ -1,64 +0,0 @@ -"""Length bonus module.""" - -from typing import Any -from typing import List -from typing import Tuple - -import torch - -from funasr_detach.models.transformer.scorers.scorer_interface import ( - BatchScorerInterface, -) - - -class LengthBonus(BatchScorerInterface): - """Length bonus in beam search.""" - - def __init__(self, n_vocab: int): - """Initialize class. - - Args: - n_vocab (int): The number of tokens in vocabulary for beam search - - """ - self.n = n_vocab - - def score(self, y, state, x): - """Score new token. - - Args: - y (torch.Tensor): 1D torch.int64 prefix tokens. - state: Scorer state for prefix tokens - x (torch.Tensor): 2D encoder feature that generates ys. - - Returns: - tuple[torch.Tensor, Any]: Tuple of - torch.float32 scores for next token (n_vocab) - and None - - """ - return torch.tensor([1.0], device=x.device, dtype=x.dtype).expand(self.n), None - - def batch_score( - self, ys: torch.Tensor, states: List[Any], xs: torch.Tensor - ) -> Tuple[torch.Tensor, List[Any]]: - """Score new token batch. - - Args: - ys (torch.Tensor): torch.int64 prefix tokens (n_batch, ylen). - states (List[Any]): Scorer states for prefix tokens. - xs (torch.Tensor): - The encoder feature that generates ys (n_batch, xlen, n_feat). - - Returns: - tuple[torch.Tensor, List[Any]]: Tuple of - batchfied scores for next token with shape of `(n_batch, n_vocab)` - and next state list for ys. - - """ - return ( - torch.tensor([1.0], device=xs.device, dtype=xs.dtype).expand( - ys.shape[0], self.n - ), - None, - ) diff --git a/funasr_detach/models/transformer/scorers/scorer_interface.py b/funasr_detach/models/transformer/scorers/scorer_interface.py deleted file mode 100644 index 946ec6be317603d87cfd938cc096d11b7bcbfbdf..0000000000000000000000000000000000000000 --- a/funasr_detach/models/transformer/scorers/scorer_interface.py +++ /dev/null @@ -1,188 +0,0 @@ -"""Scorer interface module.""" - -from typing import Any -from typing import List -from typing import Tuple - -import torch -import warnings - - -class ScorerInterface: - """Scorer interface for beam search. - - The scorer performs scoring of the all tokens in vocabulary. - - Examples: - * Search heuristics - * :class:`espnet.nets.scorers.length_bonus.LengthBonus` - * Decoder networks of the sequence-to-sequence models - * :class:`espnet.nets.pytorch_backend.nets.transformer.decoder.Decoder` - * :class:`espnet.nets.pytorch_backend.nets.rnn.decoders.Decoder` - * Neural language models - * :class:`espnet.nets.pytorch_backend.lm.transformer.TransformerLM` - * :class:`espnet.nets.pytorch_backend.lm.default.DefaultRNNLM` - * :class:`espnet.nets.pytorch_backend.lm.seq_rnn.SequentialRNNLM` - - """ - - def init_state(self, x: torch.Tensor) -> Any: - """Get an initial state for decoding (optional). - - Args: - x (torch.Tensor): The encoded feature tensor - - Returns: initial state - - """ - return None - - def select_state(self, state: Any, i: int, new_id: int = None) -> Any: - """Select state with relative ids in the main beam search. - - Args: - state: Decoder state for prefix tokens - i (int): Index to select a state in the main beam search - new_id (int): New label index to select a state if necessary - - Returns: - state: pruned state - - """ - return None if state is None else state[i] - - def score( - self, y: torch.Tensor, state: Any, x: torch.Tensor - ) -> Tuple[torch.Tensor, Any]: - """Score new token (required). - - Args: - y (torch.Tensor): 1D torch.int64 prefix tokens. - state: Scorer state for prefix tokens - x (torch.Tensor): The encoder feature that generates ys. - - Returns: - tuple[torch.Tensor, Any]: Tuple of - scores for next token that has a shape of `(n_vocab)` - and next state for ys - - """ - raise NotImplementedError - - def final_score(self, state: Any) -> float: - """Score eos (optional). - - Args: - state: Scorer state for prefix tokens - - Returns: - float: final score - - """ - return 0.0 - - -class BatchScorerInterface(ScorerInterface): - """Batch scorer interface.""" - - def batch_init_state(self, x: torch.Tensor) -> Any: - """Get an initial state for decoding (optional). - - Args: - x (torch.Tensor): The encoded feature tensor - - Returns: initial state - - """ - return self.init_state(x) - - def batch_score( - self, ys: torch.Tensor, states: List[Any], xs: torch.Tensor - ) -> Tuple[torch.Tensor, List[Any]]: - """Score new token batch (required). - - Args: - ys (torch.Tensor): torch.int64 prefix tokens (n_batch, ylen). - states (List[Any]): Scorer states for prefix tokens. - xs (torch.Tensor): - The encoder feature that generates ys (n_batch, xlen, n_feat). - - Returns: - tuple[torch.Tensor, List[Any]]: Tuple of - batchfied scores for next token with shape of `(n_batch, n_vocab)` - and next state list for ys. - - """ - warnings.warn( - "{} batch score is implemented through for loop not parallelized".format( - self.__class__.__name__ - ) - ) - scores = list() - outstates = list() - for i, (y, state, x) in enumerate(zip(ys, states, xs)): - score, outstate = self.score(y, state, x) - outstates.append(outstate) - scores.append(score) - scores = torch.cat(scores, 0).view(ys.shape[0], -1) - return scores, outstates - - -class PartialScorerInterface(ScorerInterface): - """Partial scorer interface for beam search. - - The partial scorer performs scoring when non-partial scorer finished scoring, - and receives pre-pruned next tokens to score because it is too heavy to score - all the tokens. - - Examples: - * Prefix search for connectionist-temporal-classification models - * :class:`espnet.nets.scorers.ctc.CTCPrefixScorer` - - """ - - def score_partial( - self, y: torch.Tensor, next_tokens: torch.Tensor, state: Any, x: torch.Tensor - ) -> Tuple[torch.Tensor, Any]: - """Score new token (required). - - Args: - y (torch.Tensor): 1D prefix token - next_tokens (torch.Tensor): torch.int64 next token to score - state: decoder state for prefix tokens - x (torch.Tensor): The encoder feature that generates ys - - Returns: - tuple[torch.Tensor, Any]: - Tuple of a score tensor for y that has a shape `(len(next_tokens),)` - and next state for ys - - """ - raise NotImplementedError - - -class BatchPartialScorerInterface(BatchScorerInterface, PartialScorerInterface): - """Batch partial scorer interface for beam search.""" - - def batch_score_partial( - self, - ys: torch.Tensor, - next_tokens: torch.Tensor, - states: List[Any], - xs: torch.Tensor, - ) -> Tuple[torch.Tensor, Any]: - """Score new token (required). - - Args: - ys (torch.Tensor): torch.int64 prefix tokens (n_batch, ylen). - next_tokens (torch.Tensor): torch.int64 tokens to score (n_batch, n_token). - states (List[Any]): Scorer states for prefix tokens. - xs (torch.Tensor): - The encoder feature that generates ys (n_batch, xlen, n_feat). - - Returns: - tuple[torch.Tensor, Any]: - Tuple of a score tensor for ys that has a shape `(n_batch, n_vocab)` - and next states for ys - """ - raise NotImplementedError diff --git a/funasr_detach/models/transformer/search.py b/funasr_detach/models/transformer/search.py deleted file mode 100644 index 6804cd8b32e13bc9d6ec482f03072113151d1a4d..0000000000000000000000000000000000000000 --- a/funasr_detach/models/transformer/search.py +++ /dev/null @@ -1,462 +0,0 @@ -from itertools import chain -import logging -from typing import Any -from typing import Dict -from typing import List -from typing import NamedTuple -from typing import Tuple -from typing import Union - -import torch - -from funasr_detach.metrics.common import end_detect -from funasr_detach.models.transformer.scorers.scorer_interface import ( - PartialScorerInterface, -) -from funasr_detach.models.transformer.scorers.scorer_interface import ScorerInterface - - -class Hypothesis(NamedTuple): - """Hypothesis data type.""" - - yseq: torch.Tensor - score: Union[float, torch.Tensor] = 0 - scores: Dict[str, Union[float, torch.Tensor]] = dict() - states: Dict[str, Any] = dict() - - def asdict(self) -> dict: - """Convert data to JSON-friendly dict.""" - return self._replace( - yseq=self.yseq.tolist(), - score=float(self.score), - scores={k: float(v) for k, v in self.scores.items()}, - )._asdict() - - -class BeamSearch(torch.nn.Module): - """Beam search implementation.""" - - def __init__( - self, - scorers: Dict[str, ScorerInterface], - weights: Dict[str, float], - beam_size: int, - vocab_size: int, - sos: int, - eos: int, - token_list: List[str] = None, - pre_beam_ratio: float = 1.5, - pre_beam_score_key: str = None, - ): - """Initialize beam search. - - Args: - scorers (dict[str, ScorerInterface]): Dict of decoder modules - e.g., Decoder, CTCPrefixScorer, LM - The scorer will be ignored if it is `None` - weights (dict[str, float]): Dict of weights for each scorers - The scorer will be ignored if its weight is 0 - beam_size (int): The number of hypotheses kept during search - vocab_size (int): The number of vocabulary - sos (int): Start of sequence id - eos (int): End of sequence id - token_list (list[str]): List of tokens for debug log - pre_beam_score_key (str): key of scores to perform pre-beam search - pre_beam_ratio (float): beam size in the pre-beam search - will be `int(pre_beam_ratio * beam_size)` - - """ - super().__init__() - # set scorers - self.weights = weights - self.scorers = dict() - self.full_scorers = dict() - self.part_scorers = dict() - # this module dict is required for recursive cast - # `self.to(device, dtype)` in `recog.py` - self.nn_dict = torch.nn.ModuleDict() - for k, v in scorers.items(): - w = weights.get(k, 0) - if w == 0 or v is None: - continue - assert isinstance( - v, ScorerInterface - ), f"{k} ({type(v)}) does not implement ScorerInterface" - self.scorers[k] = v - if isinstance(v, PartialScorerInterface): - self.part_scorers[k] = v - else: - self.full_scorers[k] = v - if isinstance(v, torch.nn.Module): - self.nn_dict[k] = v - - # set configurations - self.sos = sos - self.eos = eos - self.token_list = token_list - self.pre_beam_size = int(pre_beam_ratio * beam_size) - self.beam_size = beam_size - self.n_vocab = vocab_size - if ( - pre_beam_score_key is not None - and pre_beam_score_key != "full" - and pre_beam_score_key not in self.full_scorers - ): - raise KeyError(f"{pre_beam_score_key} is not found in {self.full_scorers}") - self.pre_beam_score_key = pre_beam_score_key - self.do_pre_beam = ( - self.pre_beam_score_key is not None - and self.pre_beam_size < self.n_vocab - and len(self.part_scorers) > 0 - ) - - def init_hyp(self, x: torch.Tensor) -> List[Hypothesis]: - """Get an initial hypothesis data. - - Args: - x (torch.Tensor): The encoder output feature - - Returns: - Hypothesis: The initial hypothesis. - - """ - init_states = dict() - init_scores = dict() - for k, d in self.scorers.items(): - init_states[k] = d.init_state(x) - init_scores[k] = 0.0 - return [ - Hypothesis( - score=0.0, - scores=init_scores, - states=init_states, - yseq=torch.tensor([self.sos], device=x.device), - ) - ] - - @staticmethod - def append_token(xs: torch.Tensor, x: int) -> torch.Tensor: - """Append new token to prefix tokens. - - Args: - xs (torch.Tensor): The prefix token - x (int): The new token to append - - Returns: - torch.Tensor: New tensor contains: xs + [x] with xs.dtype and xs.device - - """ - x = torch.tensor([x], dtype=xs.dtype, device=xs.device) - return torch.cat((xs, x)) - - def score_full( - self, hyp: Hypothesis, x: torch.Tensor - ) -> Tuple[Dict[str, torch.Tensor], Dict[str, Any]]: - """Score new hypothesis by `self.full_scorers`. - - Args: - hyp (Hypothesis): Hypothesis with prefix tokens to score - x (torch.Tensor): Corresponding input feature - - Returns: - Tuple[Dict[str, torch.Tensor], Dict[str, Any]]: Tuple of - score dict of `hyp` that has string keys of `self.full_scorers` - and tensor score values of shape: `(self.n_vocab,)`, - and state dict that has string keys - and state values of `self.full_scorers` - - """ - scores = dict() - states = dict() - for k, d in self.full_scorers.items(): - scores[k], states[k] = d.score(hyp.yseq, hyp.states[k], x) - return scores, states - - def score_partial( - self, hyp: Hypothesis, ids: torch.Tensor, x: torch.Tensor - ) -> Tuple[Dict[str, torch.Tensor], Dict[str, Any]]: - """Score new hypothesis by `self.part_scorers`. - - Args: - hyp (Hypothesis): Hypothesis with prefix tokens to score - ids (torch.Tensor): 1D tensor of new partial tokens to score - x (torch.Tensor): Corresponding input feature - - Returns: - Tuple[Dict[str, torch.Tensor], Dict[str, Any]]: Tuple of - score dict of `hyp` that has string keys of `self.part_scorers` - and tensor score values of shape: `(len(ids),)`, - and state dict that has string keys - and state values of `self.part_scorers` - - """ - scores = dict() - states = dict() - for k, d in self.part_scorers.items(): - scores[k], states[k] = d.score_partial(hyp.yseq, ids, hyp.states[k], x) - return scores, states - - def beam( - self, weighted_scores: torch.Tensor, ids: torch.Tensor - ) -> Tuple[torch.Tensor, torch.Tensor]: - """Compute topk full token ids and partial token ids. - - Args: - weighted_scores (torch.Tensor): The weighted sum scores for each tokens. - Its shape is `(self.n_vocab,)`. - ids (torch.Tensor): The partial token ids to compute topk - - Returns: - Tuple[torch.Tensor, torch.Tensor]: - The topk full token ids and partial token ids. - Their shapes are `(self.beam_size,)` - - """ - # no pre beam performed - if weighted_scores.size(0) == ids.size(0): - top_ids = weighted_scores.topk(self.beam_size)[1] - return top_ids, top_ids - - # mask pruned in pre-beam not to select in topk - tmp = weighted_scores[ids] - weighted_scores[:] = -float("inf") - weighted_scores[ids] = tmp - top_ids = weighted_scores.topk(self.beam_size)[1] - local_ids = weighted_scores[ids].topk(self.beam_size)[1] - return top_ids, local_ids - - @staticmethod - def merge_scores( - prev_scores: Dict[str, float], - next_full_scores: Dict[str, torch.Tensor], - full_idx: int, - next_part_scores: Dict[str, torch.Tensor], - part_idx: int, - ) -> Dict[str, torch.Tensor]: - """Merge scores for new hypothesis. - - Args: - prev_scores (Dict[str, float]): - The previous hypothesis scores by `self.scorers` - next_full_scores (Dict[str, torch.Tensor]): scores by `self.full_scorers` - full_idx (int): The next token id for `next_full_scores` - next_part_scores (Dict[str, torch.Tensor]): - scores of partial tokens by `self.part_scorers` - part_idx (int): The new token id for `next_part_scores` - - Returns: - Dict[str, torch.Tensor]: The new score dict. - Its keys are names of `self.full_scorers` and `self.part_scorers`. - Its values are scalar tensors by the scorers. - - """ - new_scores = dict() - for k, v in next_full_scores.items(): - new_scores[k] = prev_scores[k] + v[full_idx] - for k, v in next_part_scores.items(): - new_scores[k] = prev_scores[k] + v[part_idx] - return new_scores - - def merge_states(self, states: Any, part_states: Any, part_idx: int) -> Any: - """Merge states for new hypothesis. - - Args: - states: states of `self.full_scorers` - part_states: states of `self.part_scorers` - part_idx (int): The new token id for `part_scores` - - Returns: - Dict[str, torch.Tensor]: The new score dict. - Its keys are names of `self.full_scorers` and `self.part_scorers`. - Its values are states of the scorers. - - """ - new_states = dict() - for k, v in states.items(): - new_states[k] = v - for k, d in self.part_scorers.items(): - new_states[k] = d.select_state(part_states[k], part_idx) - return new_states - - def search( - self, running_hyps: List[Hypothesis], x: torch.Tensor - ) -> List[Hypothesis]: - """Search new tokens for running hypotheses and encoded speech x. - - Args: - running_hyps (List[Hypothesis]): Running hypotheses on beam - x (torch.Tensor): Encoded speech feature (T, D) - - Returns: - List[Hypotheses]: Best sorted hypotheses - - """ - best_hyps = [] - part_ids = torch.arange(self.n_vocab, device=x.device) # no pre-beam - for hyp in running_hyps: - # scoring - weighted_scores = torch.zeros(self.n_vocab, dtype=x.dtype, device=x.device) - scores, states = self.score_full(hyp, x) - for k in self.full_scorers: - weighted_scores += self.weights[k] * scores[k] - # partial scoring - if self.do_pre_beam: - pre_beam_scores = ( - weighted_scores - if self.pre_beam_score_key == "full" - else scores[self.pre_beam_score_key] - ) - part_ids = torch.topk(pre_beam_scores, self.pre_beam_size)[1] - part_scores, part_states = self.score_partial(hyp, part_ids, x) - for k in self.part_scorers: - weighted_scores[part_ids] += self.weights[k] * part_scores[k] - # add previous hyp score - weighted_scores += hyp.score - - # update hyps - for j, part_j in zip(*self.beam(weighted_scores, part_ids)): - # will be (2 x beam at most) - best_hyps.append( - Hypothesis( - score=weighted_scores[j], - yseq=self.append_token(hyp.yseq, j), - scores=self.merge_scores( - hyp.scores, scores, j, part_scores, part_j - ), - states=self.merge_states(states, part_states, part_j), - ) - ) - - # sort and prune 2 x beam -> beam - best_hyps = sorted(best_hyps, key=lambda x: x.score, reverse=True)[ - : min(len(best_hyps), self.beam_size) - ] - return best_hyps - - def forward( - self, x: torch.Tensor, maxlenratio: float = 0.0, minlenratio: float = 0.0 - ) -> List[Hypothesis]: - """Perform beam search. - - Args: - x (torch.Tensor): Encoded speech feature (T, D) - maxlenratio (float): Input length ratio to obtain max output length. - If maxlenratio=0.0 (default), it uses a end-detect function - to automatically find maximum hypothesis lengths - If maxlenratio<0.0, its absolute value is interpreted - as a constant max output length. - minlenratio (float): Input length ratio to obtain min output length. - - Returns: - list[Hypothesis]: N-best decoding results - - """ - # set length bounds - if maxlenratio == 0: - maxlen = x.shape[0] - elif maxlenratio < 0: - maxlen = -1 * int(maxlenratio) - else: - maxlen = max(1, int(maxlenratio * x.size(0))) - minlen = int(minlenratio * x.size(0)) - logging.info("decoder input length: " + str(x.shape[0])) - logging.info("max output length: " + str(maxlen)) - logging.info("min output length: " + str(minlen)) - - # main loop of prefix search - running_hyps = self.init_hyp(x) - ended_hyps = [] - for i in range(maxlen): - logging.debug("position " + str(i)) - best = self.search(running_hyps, x) - # post process of one iteration - running_hyps = self.post_process(i, maxlen, maxlenratio, best, ended_hyps) - # end detection - if maxlenratio == 0.0 and end_detect([h.asdict() for h in ended_hyps], i): - logging.info(f"end detected at {i}") - break - if len(running_hyps) == 0: - logging.info("no hypothesis. Finish decoding.") - break - else: - logging.debug(f"remained hypotheses: {len(running_hyps)}") - - nbest_hyps = sorted(ended_hyps, key=lambda x: x.score, reverse=True) - # check the number of hypotheses reaching to eos - if len(nbest_hyps) == 0: - logging.warning( - "there is no N-best results, perform recognition " - "again with smaller minlenratio." - ) - return ( - [] - if minlenratio < 0.1 - else self.forward(x, maxlenratio, max(0.0, minlenratio - 0.1)) - ) - - # report the best result - best = nbest_hyps[0] - for k, v in best.scores.items(): - logging.info( - f"{v:6.2f} * {self.weights[k]:3} = {v * self.weights[k]:6.2f} for {k}" - ) - logging.info(f"total log probability: {best.score:.2f}") - logging.info(f"normalized log probability: {best.score / len(best.yseq):.2f}") - logging.info(f"total number of ended hypotheses: {len(nbest_hyps)}") - if self.token_list is not None: - logging.info( - "best hypo: " - + "".join([self.token_list[x] for x in best.yseq[1:-1]]) - + "\n" - ) - return nbest_hyps - - def post_process( - self, - i: int, - maxlen: int, - maxlenratio: float, - running_hyps: List[Hypothesis], - ended_hyps: List[Hypothesis], - ) -> List[Hypothesis]: - """Perform post-processing of beam search iterations. - - Args: - i (int): The length of hypothesis tokens. - maxlen (int): The maximum length of tokens in beam search. - maxlenratio (int): The maximum length ratio in beam search. - running_hyps (List[Hypothesis]): The running hypotheses in beam search. - ended_hyps (List[Hypothesis]): The ended hypotheses in beam search. - - Returns: - List[Hypothesis]: The new running hypotheses. - - """ - logging.debug(f"the number of running hypotheses: {len(running_hyps)}") - if self.token_list is not None: - logging.debug( - "best hypo: " - + "".join([self.token_list[x] for x in running_hyps[0].yseq[1:]]) - ) - # add eos in the final loop to avoid that there are no ended hyps - if i == maxlen - 1: - logging.info("adding in the last position in the loop") - running_hyps = [ - h._replace(yseq=self.append_token(h.yseq, self.eos)) - for h in running_hyps - ] - - # add ended hypotheses to a final list, and removed them from current hypotheses - # (this will be a problem, number of hyps < beam) - remained_hyps = [] - for hyp in running_hyps: - if hyp.yseq[-1] == self.eos: - # e.g., Word LM needs to add final score - for k, d in chain(self.full_scorers.items(), self.part_scorers.items()): - s = d.final_score(hyp.states[k]) - hyp.scores[k] += s - hyp = hyp._replace(score=hyp.score + self.weights[k] * s) - ended_hyps.append(hyp) - else: - remained_hyps.append(hyp) - return remained_hyps diff --git a/funasr_detach/models/transformer/template.yaml b/funasr_detach/models/transformer/template.yaml deleted file mode 100644 index 87814dc3b3f3e562234b4b1b58ffad3edfae5f47..0000000000000000000000000000000000000000 --- a/funasr_detach/models/transformer/template.yaml +++ /dev/null @@ -1,110 +0,0 @@ -# This is an example that demonstrates how to configure a model file. -# You can modify the configuration according to your own requirements. - -# to print the register_table: -# from funasr.register import tables -# tables.print() - -# network architecture -model: Transformer -model_conf: - ctc_weight: 0.3 - lsm_weight: 0.1 # label smoothing option - length_normalized_loss: false - -# encoder -encoder: TransformerEncoder -encoder_conf: - output_size: 256 # dimension of attention - attention_heads: 4 - linear_units: 2048 # the number of units of position-wise feed forward - num_blocks: 12 # the number of encoder blocks - dropout_rate: 0.1 - positional_dropout_rate: 0.1 - attention_dropout_rate: 0.0 - input_layer: conv2d # encoder architecture type - normalize_before: true - -# decoder -decoder: TransformerDecoder -decoder_conf: - attention_heads: 4 - linear_units: 2048 - num_blocks: 6 - dropout_rate: 0.1 - positional_dropout_rate: 0.1 - self_attention_dropout_rate: 0.0 - src_attention_dropout_rate: 0.0 - - -# frontend related -frontend: WavFrontend -frontend_conf: - fs: 16000 - window: hamming - n_mels: 80 - frame_length: 25 - frame_shift: 10 - lfr_m: 1 - lfr_n: 1 - -specaug: SpecAug -specaug_conf: - apply_time_warp: true - time_warp_window: 5 - time_warp_mode: bicubic - apply_freq_mask: true - freq_mask_width_range: - - 0 - - 30 - num_freq_mask: 2 - apply_time_mask: true - time_mask_width_range: - - 0 - - 40 - num_time_mask: 2 - -train_conf: - accum_grad: 1 - grad_clip: 5 - max_epoch: 150 - val_scheduler_criterion: - - valid - - acc - best_model_criterion: - - - valid - - acc - - max - keep_nbest_models: 10 - log_interval: 50 - -optim: adam -optim_conf: - lr: 0.002 -scheduler: warmuplr -scheduler_conf: - warmup_steps: 30000 - -dataset: AudioDataset -dataset_conf: - index_ds: IndexDSJsonl - batch_sampler: DynamicBatchLocalShuffleSampler - batch_type: example # example or length - batch_size: 1 # if batch_type is example, batch_size is the numbers of samples; if length, batch_size is source_token_len+target_token_len; - max_token_length: 2048 # filter samples if source_token_len+target_token_len > max_token_length, - buffer_size: 500 - shuffle: True - num_workers: 0 - -tokenizer: CharTokenizer -tokenizer_conf: - unk_symbol: - split_with_space: true - - -ctc_conf: - dropout_rate: 0.0 - ctc_type: builtin - reduce: true - ignore_nan_grad: true -normalize: null diff --git a/funasr_detach/models/transformer/utils/__init__.py b/funasr_detach/models/transformer/utils/__init__.py deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/funasr_detach/models/transformer/utils/add_sos_eos.py b/funasr_detach/models/transformer/utils/add_sos_eos.py deleted file mode 100644 index 1edcf9559be18a561a8a60cb884b287819889a04..0000000000000000000000000000000000000000 --- a/funasr_detach/models/transformer/utils/add_sos_eos.py +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- - -# Copyright 2019 Shigeki Karita -# Apache 2.0 (http://www.apache.org/licenses/LICENSE-2.0) - -"""Unility functions for Transformer.""" - -import torch -from funasr_detach.models.transformer.utils.nets_utils import pad_list - - -def add_sos_eos(ys_pad, sos, eos, ignore_id): - """Add and labels. - - :param torch.Tensor ys_pad: batch of padded target sequences (B, Lmax) - :param int sos: index of - :param int eos: index of - :param int ignore_id: index of padding - :return: padded tensor (B, Lmax) - :rtype: torch.Tensor - :return: padded tensor (B, Lmax) - :rtype: torch.Tensor - """ - - _sos = ys_pad.new([sos]) - _eos = ys_pad.new([eos]) - ys = [y[y != ignore_id] for y in ys_pad] # parse padded ys - ys_in = [torch.cat([_sos, y], dim=0) for y in ys] - ys_out = [torch.cat([y, _eos], dim=0) for y in ys] - return pad_list(ys_in, eos), pad_list(ys_out, ignore_id) diff --git a/funasr_detach/models/transformer/utils/dynamic_conv.py b/funasr_detach/models/transformer/utils/dynamic_conv.py deleted file mode 100644 index 8a2a0c1eaf07b3e036f07e534639a52f91af56b4..0000000000000000000000000000000000000000 --- a/funasr_detach/models/transformer/utils/dynamic_conv.py +++ /dev/null @@ -1,125 +0,0 @@ -"""Dynamic Convolution module.""" - -import numpy -import torch -from torch import nn -import torch.nn.functional as F - - -MIN_VALUE = float(numpy.finfo(numpy.float32).min) - - -class DynamicConvolution(nn.Module): - """Dynamic Convolution layer. - - This implementation is based on - https://github.com/pytorch/fairseq/tree/master/fairseq - - Args: - wshare (int): the number of kernel of convolution - n_feat (int): the number of features - dropout_rate (float): dropout_rate - kernel_size (int): kernel size (length) - use_kernel_mask (bool): Use causal mask or not for convolution kernel - use_bias (bool): Use bias term or not. - - """ - - def __init__( - self, - wshare, - n_feat, - dropout_rate, - kernel_size, - use_kernel_mask=False, - use_bias=False, - ): - """Construct Dynamic Convolution layer.""" - super(DynamicConvolution, self).__init__() - - assert n_feat % wshare == 0 - self.wshare = wshare - self.use_kernel_mask = use_kernel_mask - self.dropout_rate = dropout_rate - self.kernel_size = kernel_size - self.attn = None - - # linear -> GLU -- -> lightconv -> linear - # \ / - # Linear - self.linear1 = nn.Linear(n_feat, n_feat * 2) - self.linear2 = nn.Linear(n_feat, n_feat) - self.linear_weight = nn.Linear(n_feat, self.wshare * 1 * kernel_size) - nn.init.xavier_uniform(self.linear_weight.weight) - self.act = nn.GLU() - - # dynamic conv related - self.use_bias = use_bias - if self.use_bias: - self.bias = nn.Parameter(torch.Tensor(n_feat)) - - def forward(self, query, key, value, mask): - """Forward of 'Dynamic Convolution'. - - This function takes query, key and value but uses only quert. - This is just for compatibility with self-attention layer (attention.py) - - Args: - query (torch.Tensor): (batch, time1, d_model) input tensor - key (torch.Tensor): (batch, time2, d_model) NOT USED - value (torch.Tensor): (batch, time2, d_model) NOT USED - mask (torch.Tensor): (batch, time1, time2) mask - - Return: - x (torch.Tensor): (batch, time1, d_model) output - - """ - # linear -> GLU -- -> lightconv -> linear - # \ / - # Linear - x = query - B, T, C = x.size() - H = self.wshare - k = self.kernel_size - - # first liner layer - x = self.linear1(x) - - # GLU activation - x = self.act(x) - - # get kernel of convolution - weight = self.linear_weight(x) # B x T x kH - weight = F.dropout(weight, self.dropout_rate, training=self.training) - weight = weight.view(B, T, H, k).transpose(1, 2).contiguous() # B x H x T x k - weight_new = torch.zeros(B * H * T * (T + k - 1), dtype=weight.dtype) - weight_new = weight_new.view(B, H, T, T + k - 1).fill_(float("-inf")) - weight_new = weight_new.to(x.device) # B x H x T x T+k-1 - weight_new.as_strided( - (B, H, T, k), ((T + k - 1) * T * H, (T + k - 1) * T, T + k, 1) - ).copy_(weight) - weight_new = weight_new.narrow(-1, int((k - 1) / 2), T) # B x H x T x T(k) - if self.use_kernel_mask: - kernel_mask = torch.tril(torch.ones(T, T, device=x.device)).unsqueeze(0) - weight_new = weight_new.masked_fill(kernel_mask == 0.0, float("-inf")) - weight_new = F.softmax(weight_new, dim=-1) - self.attn = weight_new - weight_new = weight_new.view(B * H, T, T) - - # convolution - x = x.transpose(1, 2).contiguous() # B x C x T - x = x.view(B * H, int(C / H), T).transpose(1, 2) - x = torch.bmm(weight_new, x) # BH x T x C/H - x = x.transpose(1, 2).contiguous().view(B, C, T) - - if self.use_bias: - x = x + self.bias.view(1, -1, 1) - x = x.transpose(1, 2) # B x T x C - - if mask is not None and not self.use_kernel_mask: - mask = mask.transpose(-1, -2) - x = x.masked_fill(mask == 0, 0.0) - - # second linear layer - x = self.linear2(x) - return x diff --git a/funasr_detach/models/transformer/utils/dynamic_conv2d.py b/funasr_detach/models/transformer/utils/dynamic_conv2d.py deleted file mode 100644 index f8a4dd6e9f62342dddd6f3d767e3e311f5195a0b..0000000000000000000000000000000000000000 --- a/funasr_detach/models/transformer/utils/dynamic_conv2d.py +++ /dev/null @@ -1,138 +0,0 @@ -"""Dynamic 2-Dimensional Convolution module.""" - -import numpy -import torch -from torch import nn -import torch.nn.functional as F - - -MIN_VALUE = float(numpy.finfo(numpy.float32).min) - - -class DynamicConvolution2D(nn.Module): - """Dynamic 2-Dimensional Convolution layer. - - This implementation is based on - https://github.com/pytorch/fairseq/tree/master/fairseq - - Args: - wshare (int): the number of kernel of convolution - n_feat (int): the number of features - dropout_rate (float): dropout_rate - kernel_size (int): kernel size (length) - use_kernel_mask (bool): Use causal mask or not for convolution kernel - use_bias (bool): Use bias term or not. - - """ - - def __init__( - self, - wshare, - n_feat, - dropout_rate, - kernel_size, - use_kernel_mask=False, - use_bias=False, - ): - """Construct Dynamic 2-Dimensional Convolution layer.""" - super(DynamicConvolution2D, self).__init__() - - assert n_feat % wshare == 0 - self.wshare = wshare - self.use_kernel_mask = use_kernel_mask - self.dropout_rate = dropout_rate - self.kernel_size = kernel_size - self.padding_size = int(kernel_size / 2) - self.attn_t = None - self.attn_f = None - - # linear -> GLU -- -> lightconv -> linear - # \ / - # Linear - self.linear1 = nn.Linear(n_feat, n_feat * 2) - self.linear2 = nn.Linear(n_feat * 2, n_feat) - self.linear_weight = nn.Linear(n_feat, self.wshare * 1 * kernel_size) - nn.init.xavier_uniform(self.linear_weight.weight) - self.linear_weight_f = nn.Linear(n_feat, kernel_size) - nn.init.xavier_uniform(self.linear_weight_f.weight) - self.act = nn.GLU() - - # dynamic conv related - self.use_bias = use_bias - if self.use_bias: - self.bias = nn.Parameter(torch.Tensor(n_feat)) - - def forward(self, query, key, value, mask): - """Forward of 'Dynamic 2-Dimensional Convolution'. - - This function takes query, key and value but uses only query. - This is just for compatibility with self-attention layer (attention.py) - - Args: - query (torch.Tensor): (batch, time1, d_model) input tensor - key (torch.Tensor): (batch, time2, d_model) NOT USED - value (torch.Tensor): (batch, time2, d_model) NOT USED - mask (torch.Tensor): (batch, time1, time2) mask - - Return: - x (torch.Tensor): (batch, time1, d_model) output - - """ - # linear -> GLU -- -> lightconv -> linear - # \ / - # Linear - x = query - B, T, C = x.size() - H = self.wshare - k = self.kernel_size - - # first liner layer - x = self.linear1(x) - - # GLU activation - x = self.act(x) - - # convolution of frequency axis - weight_f = self.linear_weight_f(x).view(B * T, 1, k) # B x T x k - self.attn_f = weight_f.view(B, T, k).unsqueeze(1) - xf = F.conv1d( - x.view(1, B * T, C), weight_f, padding=self.padding_size, groups=B * T - ) - xf = xf.view(B, T, C) - - # get kernel of convolution - weight = self.linear_weight(x) # B x T x kH - weight = F.dropout(weight, self.dropout_rate, training=self.training) - weight = weight.view(B, T, H, k).transpose(1, 2).contiguous() # B x H x T x k - weight_new = torch.zeros(B * H * T * (T + k - 1), dtype=weight.dtype) - weight_new = weight_new.view(B, H, T, T + k - 1).fill_(float("-inf")) - weight_new = weight_new.to(x.device) # B x H x T x T+k-1 - weight_new.as_strided( - (B, H, T, k), ((T + k - 1) * T * H, (T + k - 1) * T, T + k, 1) - ).copy_(weight) - weight_new = weight_new.narrow(-1, int((k - 1) / 2), T) # B x H x T x T(k) - if self.use_kernel_mask: - kernel_mask = torch.tril(torch.ones(T, T, device=x.device)).unsqueeze(0) - weight_new = weight_new.masked_fill(kernel_mask == 0.0, float("-inf")) - weight_new = F.softmax(weight_new, dim=-1) - self.attn_t = weight_new - weight_new = weight_new.view(B * H, T, T) - - # convolution - x = x.transpose(1, 2).contiguous() # B x C x T - x = x.view(B * H, int(C / H), T).transpose(1, 2) - x = torch.bmm(weight_new, x) - x = x.transpose(1, 2).contiguous().view(B, C, T) - - if self.use_bias: - x = x + self.bias.view(1, -1, 1) - x = x.transpose(1, 2) # B x T x C - x = torch.cat((x, xf), -1) # B x T x Cx2 - - if mask is not None and not self.use_kernel_mask: - mask = mask.transpose(-1, -2) - x = x.masked_fill(mask == 0, 0.0) - - # second linear layer - x = self.linear2(x) - return x diff --git a/funasr_detach/models/transformer/utils/lightconv.py b/funasr_detach/models/transformer/utils/lightconv.py deleted file mode 100644 index b249402591e7644194f7962283fe5d4b3fe0034a..0000000000000000000000000000000000000000 --- a/funasr_detach/models/transformer/utils/lightconv.py +++ /dev/null @@ -1,112 +0,0 @@ -"""Lightweight Convolution Module.""" - -import numpy -import torch -from torch import nn -import torch.nn.functional as F - - -MIN_VALUE = float(numpy.finfo(numpy.float32).min) - - -class LightweightConvolution(nn.Module): - """Lightweight Convolution layer. - - This implementation is based on - https://github.com/pytorch/fairseq/tree/master/fairseq - - Args: - wshare (int): the number of kernel of convolution - n_feat (int): the number of features - dropout_rate (float): dropout_rate - kernel_size (int): kernel size (length) - use_kernel_mask (bool): Use causal mask or not for convolution kernel - use_bias (bool): Use bias term or not. - - """ - - def __init__( - self, - wshare, - n_feat, - dropout_rate, - kernel_size, - use_kernel_mask=False, - use_bias=False, - ): - """Construct Lightweight Convolution layer.""" - super(LightweightConvolution, self).__init__() - - assert n_feat % wshare == 0 - self.wshare = wshare - self.use_kernel_mask = use_kernel_mask - self.dropout_rate = dropout_rate - self.kernel_size = kernel_size - self.padding_size = int(kernel_size / 2) - - # linear -> GLU -> lightconv -> linear - self.linear1 = nn.Linear(n_feat, n_feat * 2) - self.linear2 = nn.Linear(n_feat, n_feat) - self.act = nn.GLU() - - # lightconv related - self.weight = nn.Parameter( - torch.Tensor(self.wshare, 1, kernel_size).uniform_(0, 1) - ) - self.use_bias = use_bias - if self.use_bias: - self.bias = nn.Parameter(torch.Tensor(n_feat)) - - # mask of kernel - kernel_mask0 = torch.zeros(self.wshare, int(kernel_size / 2)) - kernel_mask1 = torch.ones(self.wshare, int(kernel_size / 2 + 1)) - self.kernel_mask = torch.cat((kernel_mask1, kernel_mask0), dim=-1).unsqueeze(1) - - def forward(self, query, key, value, mask): - """Forward of 'Lightweight Convolution'. - - This function takes query, key and value but uses only query. - This is just for compatibility with self-attention layer (attention.py) - - Args: - query (torch.Tensor): (batch, time1, d_model) input tensor - key (torch.Tensor): (batch, time2, d_model) NOT USED - value (torch.Tensor): (batch, time2, d_model) NOT USED - mask (torch.Tensor): (batch, time1, time2) mask - - Return: - x (torch.Tensor): (batch, time1, d_model) output - - """ - # linear -> GLU -> lightconv -> linear - x = query - B, T, C = x.size() - H = self.wshare - - # first liner layer - x = self.linear1(x) - - # GLU activation - x = self.act(x) - - # lightconv - x = x.transpose(1, 2).contiguous().view(-1, H, T) # B x C x T - weight = F.dropout(self.weight, self.dropout_rate, training=self.training) - if self.use_kernel_mask: - self.kernel_mask = self.kernel_mask.to(x.device) - weight = weight.masked_fill(self.kernel_mask == 0.0, float("-inf")) - weight = F.softmax(weight, dim=-1) - x = F.conv1d(x, weight, padding=self.padding_size, groups=self.wshare).view( - B, C, T - ) - if self.use_bias: - x = x + self.bias.view(1, -1, 1) - x = x.transpose(1, 2) # B x T x C - - if mask is not None and not self.use_kernel_mask: - mask = mask.transpose(-1, -2) - x = x.masked_fill(mask == 0, 0.0) - - # second linear layer - x = self.linear2(x) - return x diff --git a/funasr_detach/models/transformer/utils/lightconv2d.py b/funasr_detach/models/transformer/utils/lightconv2d.py deleted file mode 100644 index 294d23244e4c97639181357cfb16733dea9c4cee..0000000000000000000000000000000000000000 --- a/funasr_detach/models/transformer/utils/lightconv2d.py +++ /dev/null @@ -1,124 +0,0 @@ -"""Lightweight 2-Dimensional Convolution module.""" - -import numpy -import torch -from torch import nn -import torch.nn.functional as F - - -MIN_VALUE = float(numpy.finfo(numpy.float32).min) - - -class LightweightConvolution2D(nn.Module): - """Lightweight 2-Dimensional Convolution layer. - - This implementation is based on - https://github.com/pytorch/fairseq/tree/master/fairseq - - Args: - wshare (int): the number of kernel of convolution - n_feat (int): the number of features - dropout_rate (float): dropout_rate - kernel_size (int): kernel size (length) - use_kernel_mask (bool): Use causal mask or not for convolution kernel - use_bias (bool): Use bias term or not. - - """ - - def __init__( - self, - wshare, - n_feat, - dropout_rate, - kernel_size, - use_kernel_mask=False, - use_bias=False, - ): - """Construct Lightweight 2-Dimensional Convolution layer.""" - super(LightweightConvolution2D, self).__init__() - - assert n_feat % wshare == 0 - self.wshare = wshare - self.use_kernel_mask = use_kernel_mask - self.dropout_rate = dropout_rate - self.kernel_size = kernel_size - self.padding_size = int(kernel_size / 2) - - # linear -> GLU -> lightconv -> linear - self.linear1 = nn.Linear(n_feat, n_feat * 2) - self.linear2 = nn.Linear(n_feat * 2, n_feat) - self.act = nn.GLU() - - # lightconv related - self.weight = nn.Parameter( - torch.Tensor(self.wshare, 1, kernel_size).uniform_(0, 1) - ) - self.weight_f = nn.Parameter(torch.Tensor(1, 1, kernel_size).uniform_(0, 1)) - self.use_bias = use_bias - if self.use_bias: - self.bias = nn.Parameter(torch.Tensor(n_feat)) - - # mask of kernel - kernel_mask0 = torch.zeros(self.wshare, int(kernel_size / 2)) - kernel_mask1 = torch.ones(self.wshare, int(kernel_size / 2 + 1)) - self.kernel_mask = torch.cat((kernel_mask1, kernel_mask0), dim=-1).unsqueeze(1) - - def forward(self, query, key, value, mask): - """Forward of 'Lightweight 2-Dimensional Convolution'. - - This function takes query, key and value but uses only query. - This is just for compatibility with self-attention layer (attention.py) - - Args: - query (torch.Tensor): (batch, time1, d_model) input tensor - key (torch.Tensor): (batch, time2, d_model) NOT USED - value (torch.Tensor): (batch, time2, d_model) NOT USED - mask (torch.Tensor): (batch, time1, time2) mask - - Return: - x (torch.Tensor): (batch, time1, d_model) output - - """ - # linear -> GLU -> lightconv -> linear - x = query - B, T, C = x.size() - H = self.wshare - - # first liner layer - x = self.linear1(x) - - # GLU activation - x = self.act(x) - - # convolution along frequency axis - weight_f = F.softmax(self.weight_f, dim=-1) - weight_f = F.dropout(weight_f, self.dropout_rate, training=self.training) - weight_new = torch.zeros( - B * T, 1, self.kernel_size, device=x.device, dtype=x.dtype - ).copy_(weight_f) - xf = F.conv1d( - x.view(1, B * T, C), weight_new, padding=self.padding_size, groups=B * T - ).view(B, T, C) - - # lightconv - x = x.transpose(1, 2).contiguous().view(-1, H, T) # B x C x T - weight = F.dropout(self.weight, self.dropout_rate, training=self.training) - if self.use_kernel_mask: - self.kernel_mask = self.kernel_mask.to(x.device) - weight = weight.masked_fill(self.kernel_mask == 0.0, float("-inf")) - weight = F.softmax(weight, dim=-1) - x = F.conv1d(x, weight, padding=self.padding_size, groups=self.wshare).view( - B, C, T - ) - if self.use_bias: - x = x + self.bias.view(1, -1, 1) - x = x.transpose(1, 2) # B x T x C - x = torch.cat((x, xf), -1) # B x T x Cx2 - - if mask is not None and not self.use_kernel_mask: - mask = mask.transpose(-1, -2) - x = x.masked_fill(mask == 0, 0.0) - - # second linear layer - x = self.linear2(x) - return x diff --git a/funasr_detach/models/transformer/utils/mask.py b/funasr_detach/models/transformer/utils/mask.py deleted file mode 100644 index e859e0e06c20fba953de8a6a0973ddc282b21b33..0000000000000000000000000000000000000000 --- a/funasr_detach/models/transformer/utils/mask.py +++ /dev/null @@ -1,52 +0,0 @@ -# Copyright 2019 Shigeki Karita -# Apache 2.0 (http://www.apache.org/licenses/LICENSE-2.0) - -"""Mask module.""" - -import torch - - -def subsequent_mask(size, device="cpu", dtype=torch.bool): - """Create mask for subsequent steps (size, size). - - :param int size: size of mask - :param str device: "cpu" or "cuda" or torch.Tensor.device - :param torch.dtype dtype: result dtype - :rtype: torch.Tensor - >>> subsequent_mask(3) - [[1, 0, 0], - [1, 1, 0], - [1, 1, 1]] - """ - ret = torch.ones(size, size, device=device, dtype=dtype) - return torch.tril(ret, out=ret) - - -def target_mask(ys_in_pad, ignore_id): - """Create mask for decoder self-attention. - - :param torch.Tensor ys_pad: batch of padded target sequences (B, Lmax) - :param int ignore_id: index of padding - :param torch.dtype dtype: result dtype - :rtype: torch.Tensor (B, Lmax, Lmax) - """ - ys_mask = ys_in_pad != ignore_id - m = subsequent_mask(ys_mask.size(-1), device=ys_mask.device).unsqueeze(0) - return ys_mask.unsqueeze(-2) & m - - -def vad_mask(size, vad_pos, device="cpu", dtype=torch.bool): - """Create mask for decoder self-attention. - - :param int size: size of mask - :param int vad_pos: index of vad index - :param str device: "cpu" or "cuda" or torch.Tensor.device - :param torch.dtype dtype: result dtype - :rtype: torch.Tensor (B, Lmax, Lmax) - """ - ret = torch.ones(size, size, device=device, dtype=dtype) - if vad_pos <= 0 or vad_pos >= size: - return ret - sub_corner = torch.zeros(vad_pos - 1, size - vad_pos, device=device, dtype=dtype) - ret[0 : vad_pos - 1, vad_pos:] = sub_corner - return ret diff --git a/funasr_detach/models/transformer/utils/multi_layer_conv.py b/funasr_detach/models/transformer/utils/multi_layer_conv.py deleted file mode 100644 index 75583c38613288ca52b379168cbe74e25d9fcf3e..0000000000000000000000000000000000000000 --- a/funasr_detach/models/transformer/utils/multi_layer_conv.py +++ /dev/null @@ -1,160 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- - -# Copyright 2019 Tomoki Hayashi -# Apache 2.0 (http://www.apache.org/licenses/LICENSE-2.0) - -"""Layer modules for FFT block in FastSpeech (Feed-forward Transformer).""" - -import torch - - -class MultiLayeredConv1d(torch.nn.Module): - """Multi-layered conv1d for Transformer block. - - This is a module of multi-leyered conv1d designed - to replace positionwise feed-forward network - in Transforner block, which is introduced in - `FastSpeech: Fast, Robust and Controllable Text to Speech`_. - - .. _`FastSpeech: Fast, Robust and Controllable Text to Speech`: - https://arxiv.org/pdf/1905.09263.pdf - - """ - - def __init__(self, in_chans, hidden_chans, kernel_size, dropout_rate): - """Initialize MultiLayeredConv1d module. - - Args: - in_chans (int): Number of input channels. - hidden_chans (int): Number of hidden channels. - kernel_size (int): Kernel size of conv1d. - dropout_rate (float): Dropout rate. - - """ - super(MultiLayeredConv1d, self).__init__() - self.w_1 = torch.nn.Conv1d( - in_chans, - hidden_chans, - kernel_size, - stride=1, - padding=(kernel_size - 1) // 2, - ) - self.w_2 = torch.nn.Conv1d( - hidden_chans, - in_chans, - kernel_size, - stride=1, - padding=(kernel_size - 1) // 2, - ) - self.dropout = torch.nn.Dropout(dropout_rate) - - def forward(self, x): - """Calculate forward propagation. - - Args: - x (torch.Tensor): Batch of input tensors (B, T, in_chans). - - Returns: - torch.Tensor: Batch of output tensors (B, T, hidden_chans). - - """ - x = torch.relu(self.w_1(x.transpose(-1, 1))).transpose(-1, 1) - return self.w_2(self.dropout(x).transpose(-1, 1)).transpose(-1, 1) - - -class FsmnFeedForward(torch.nn.Module): - """Position-wise feed forward for FSMN blocks. - - This is a module of multi-leyered conv1d designed - to replace position-wise feed-forward network - in FSMN block. - """ - - def __init__(self, in_chans, hidden_chans, out_chans, kernel_size, dropout_rate): - """Initialize FsmnFeedForward module. - - Args: - in_chans (int): Number of input channels. - hidden_chans (int): Number of hidden channels. - out_chans (int): Number of output channels. - kernel_size (int): Kernel size of conv1d. - dropout_rate (float): Dropout rate. - - """ - super(FsmnFeedForward, self).__init__() - self.w_1 = torch.nn.Conv1d( - in_chans, - hidden_chans, - kernel_size, - stride=1, - padding=(kernel_size - 1) // 2, - ) - self.w_2 = torch.nn.Conv1d( - hidden_chans, - out_chans, - kernel_size, - stride=1, - padding=(kernel_size - 1) // 2, - bias=False, - ) - self.norm = torch.nn.LayerNorm(hidden_chans) - self.dropout = torch.nn.Dropout(dropout_rate) - - def forward(self, x, ilens=None): - """Calculate forward propagation. - - Args: - x (torch.Tensor): Batch of input tensors (B, T, in_chans). - - Returns: - torch.Tensor: Batch of output tensors (B, T, out_chans). - - """ - x = torch.relu(self.w_1(x.transpose(-1, 1))).transpose(-1, 1) - return ( - self.w_2(self.norm(self.dropout(x)).transpose(-1, 1)).transpose(-1, 1), - ilens, - ) - - -class Conv1dLinear(torch.nn.Module): - """Conv1D + Linear for Transformer block. - - A variant of MultiLayeredConv1d, which replaces second conv-layer to linear. - - """ - - def __init__(self, in_chans, hidden_chans, kernel_size, dropout_rate): - """Initialize Conv1dLinear module. - - Args: - in_chans (int): Number of input channels. - hidden_chans (int): Number of hidden channels. - kernel_size (int): Kernel size of conv1d. - dropout_rate (float): Dropout rate. - - """ - super(Conv1dLinear, self).__init__() - self.w_1 = torch.nn.Conv1d( - in_chans, - hidden_chans, - kernel_size, - stride=1, - padding=(kernel_size - 1) // 2, - ) - self.w_2 = torch.nn.Linear(hidden_chans, in_chans) - self.dropout = torch.nn.Dropout(dropout_rate) - - def forward(self, x): - """Calculate forward propagation. - - Args: - x (torch.Tensor): Batch of input tensors (B, T, in_chans). - - Returns: - torch.Tensor: Batch of output tensors (B, T, hidden_chans). - - """ - x = torch.relu(self.w_1(x.transpose(-1, 1))).transpose(-1, 1) - return self.w_2(self.dropout(x)) diff --git a/funasr_detach/models/transformer/utils/nets_utils.py b/funasr_detach/models/transformer/utils/nets_utils.py deleted file mode 100644 index 8b8332cc11ed181a292b9895f1aa5fba94cb138f..0000000000000000000000000000000000000000 --- a/funasr_detach/models/transformer/utils/nets_utils.py +++ /dev/null @@ -1,752 +0,0 @@ -# -*- coding: utf-8 -*- - -"""Network related utility tools.""" - -import logging -from typing import Dict, List, Tuple - -import numpy as np -import torch - - -def to_device(m, x): - """Send tensor into the device of the module. - - Args: - m (torch.nn.Module): Torch module. - x (Tensor): Torch tensor. - - Returns: - Tensor: Torch tensor located in the same place as torch module. - - """ - if isinstance(m, torch.nn.Module): - device = next(m.parameters()).device - elif isinstance(m, torch.Tensor): - device = m.device - else: - raise TypeError( - "Expected torch.nn.Module or torch.tensor, " f"bot got: {type(m)}" - ) - return x.to(device) - - -def pad_list(xs, pad_value): - """Perform padding for the list of tensors. - - Args: - xs (List): List of Tensors [(T_1, `*`), (T_2, `*`), ..., (T_B, `*`)]. - pad_value (float): Value for padding. - - Returns: - Tensor: Padded tensor (B, Tmax, `*`). - - Examples: - >>> x = [torch.ones(4), torch.ones(2), torch.ones(1)] - >>> x - [tensor([1., 1., 1., 1.]), tensor([1., 1.]), tensor([1.])] - >>> pad_list(x, 0) - tensor([[1., 1., 1., 1.], - [1., 1., 0., 0.], - [1., 0., 0., 0.]]) - - """ - n_batch = len(xs) - max_len = max(x.size(0) for x in xs) - pad = xs[0].new(n_batch, max_len, *xs[0].size()[1:]).fill_(pad_value) - - for i in range(n_batch): - pad[i, : xs[i].size(0)] = xs[i] - - return pad - - -def pad_list_all_dim(xs, pad_value): - """Perform padding for the list of tensors. - - Args: - xs (List): List of Tensors [(T_1, `*`), (T_2, `*`), ..., (T_B, `*`)]. - pad_value (float): Value for padding. - - Returns: - Tensor: Padded tensor (B, Tmax, `*`). - - Examples: - >>> x = [torch.ones(4), torch.ones(2), torch.ones(1)] - >>> x - [tensor([1., 1., 1., 1.]), tensor([1., 1.]), tensor([1.])] - >>> pad_list(x, 0) - tensor([[1., 1., 1., 1.], - [1., 1., 0., 0.], - [1., 0., 0., 0.]]) - - """ - n_batch = len(xs) - num_dim = len(xs[0].shape) - max_len_all_dim = [] - for i in range(num_dim): - max_len_all_dim.append(max(x.size(i) for x in xs)) - pad = xs[0].new(n_batch, *max_len_all_dim).fill_(pad_value) - - for i in range(n_batch): - if num_dim == 1: - pad[i, : xs[i].size(0)] = xs[i] - elif num_dim == 2: - pad[i, : xs[i].size(0), : xs[i].size(1)] = xs[i] - elif num_dim == 3: - pad[i, : xs[i].size(0), : xs[i].size(1), : xs[i].size(2)] = xs[i] - else: - raise ValueError( - "pad_list_all_dim only support 1-D, 2-D and 3-D tensors, not {}-D.".format( - num_dim - ) - ) - - return pad - - -def make_pad_mask(lengths, xs=None, length_dim=-1, maxlen=None): - """Make mask tensor containing indices of padded part. - - Args: - lengths (LongTensor or List): Batch of lengths (B,). - xs (Tensor, optional): The reference tensor. - If set, masks will be the same shape as this tensor. - length_dim (int, optional): Dimension indicator of the above tensor. - See the example. - - Returns: - Tensor: Mask tensor containing indices of padded part. - dtype=torch.uint8 in PyTorch 1.2- - dtype=torch.bool in PyTorch 1.2+ (including 1.2) - - Examples: - With only lengths. - - >>> lengths = [5, 3, 2] - >>> make_pad_mask(lengths) - masks = [[0, 0, 0, 0 ,0], - [0, 0, 0, 1, 1], - [0, 0, 1, 1, 1]] - - With the reference tensor. - - >>> xs = torch.zeros((3, 2, 4)) - >>> make_pad_mask(lengths, xs) - tensor([[[0, 0, 0, 0], - [0, 0, 0, 0]], - [[0, 0, 0, 1], - [0, 0, 0, 1]], - [[0, 0, 1, 1], - [0, 0, 1, 1]]], dtype=torch.uint8) - >>> xs = torch.zeros((3, 2, 6)) - >>> make_pad_mask(lengths, xs) - tensor([[[0, 0, 0, 0, 0, 1], - [0, 0, 0, 0, 0, 1]], - [[0, 0, 0, 1, 1, 1], - [0, 0, 0, 1, 1, 1]], - [[0, 0, 1, 1, 1, 1], - [0, 0, 1, 1, 1, 1]]], dtype=torch.uint8) - - With the reference tensor and dimension indicator. - - >>> xs = torch.zeros((3, 6, 6)) - >>> make_pad_mask(lengths, xs, 1) - tensor([[[0, 0, 0, 0, 0, 0], - [0, 0, 0, 0, 0, 0], - [0, 0, 0, 0, 0, 0], - [0, 0, 0, 0, 0, 0], - [0, 0, 0, 0, 0, 0], - [1, 1, 1, 1, 1, 1]], - [[0, 0, 0, 0, 0, 0], - [0, 0, 0, 0, 0, 0], - [0, 0, 0, 0, 0, 0], - [1, 1, 1, 1, 1, 1], - [1, 1, 1, 1, 1, 1], - [1, 1, 1, 1, 1, 1]], - [[0, 0, 0, 0, 0, 0], - [0, 0, 0, 0, 0, 0], - [1, 1, 1, 1, 1, 1], - [1, 1, 1, 1, 1, 1], - [1, 1, 1, 1, 1, 1], - [1, 1, 1, 1, 1, 1]]], dtype=torch.uint8) - >>> make_pad_mask(lengths, xs, 2) - tensor([[[0, 0, 0, 0, 0, 1], - [0, 0, 0, 0, 0, 1], - [0, 0, 0, 0, 0, 1], - [0, 0, 0, 0, 0, 1], - [0, 0, 0, 0, 0, 1], - [0, 0, 0, 0, 0, 1]], - [[0, 0, 0, 1, 1, 1], - [0, 0, 0, 1, 1, 1], - [0, 0, 0, 1, 1, 1], - [0, 0, 0, 1, 1, 1], - [0, 0, 0, 1, 1, 1], - [0, 0, 0, 1, 1, 1]], - [[0, 0, 1, 1, 1, 1], - [0, 0, 1, 1, 1, 1], - [0, 0, 1, 1, 1, 1], - [0, 0, 1, 1, 1, 1], - [0, 0, 1, 1, 1, 1], - [0, 0, 1, 1, 1, 1]]], dtype=torch.uint8) - - """ - if length_dim == 0: - raise ValueError("length_dim cannot be 0: {}".format(length_dim)) - - if not isinstance(lengths, list): - lengths = lengths.tolist() - bs = int(len(lengths)) - if maxlen is None: - if xs is None: - maxlen = int(max(lengths)) - else: - maxlen = xs.size(length_dim) - else: - assert xs is None - assert maxlen >= int(max(lengths)) - - seq_range = torch.arange(0, maxlen, dtype=torch.int64) - seq_range_expand = seq_range.unsqueeze(0).expand(bs, maxlen) - seq_length_expand = seq_range_expand.new(lengths).unsqueeze(-1) - mask = seq_range_expand >= seq_length_expand - - if xs is not None: - assert xs.size(0) == bs, (xs.size(0), bs) - - if length_dim < 0: - length_dim = xs.dim() + length_dim - # ind = (:, None, ..., None, :, , None, ..., None) - ind = tuple( - slice(None) if i in (0, length_dim) else None for i in range(xs.dim()) - ) - mask = mask[ind].expand_as(xs).to(xs.device) - return mask - - -def make_non_pad_mask(lengths, xs=None, length_dim=-1): - """Make mask tensor containing indices of non-padded part. - - Args: - lengths (LongTensor or List): Batch of lengths (B,). - xs (Tensor, optional): The reference tensor. - If set, masks will be the same shape as this tensor. - length_dim (int, optional): Dimension indicator of the above tensor. - See the example. - - Returns: - ByteTensor: mask tensor containing indices of padded part. - dtype=torch.uint8 in PyTorch 1.2- - dtype=torch.bool in PyTorch 1.2+ (including 1.2) - - Examples: - With only lengths. - - >>> lengths = [5, 3, 2] - >>> make_non_pad_mask(lengths) - masks = [[1, 1, 1, 1 ,1], - [1, 1, 1, 0, 0], - [1, 1, 0, 0, 0]] - - With the reference tensor. - - >>> xs = torch.zeros((3, 2, 4)) - >>> make_non_pad_mask(lengths, xs) - tensor([[[1, 1, 1, 1], - [1, 1, 1, 1]], - [[1, 1, 1, 0], - [1, 1, 1, 0]], - [[1, 1, 0, 0], - [1, 1, 0, 0]]], dtype=torch.uint8) - >>> xs = torch.zeros((3, 2, 6)) - >>> make_non_pad_mask(lengths, xs) - tensor([[[1, 1, 1, 1, 1, 0], - [1, 1, 1, 1, 1, 0]], - [[1, 1, 1, 0, 0, 0], - [1, 1, 1, 0, 0, 0]], - [[1, 1, 0, 0, 0, 0], - [1, 1, 0, 0, 0, 0]]], dtype=torch.uint8) - - With the reference tensor and dimension indicator. - - >>> xs = torch.zeros((3, 6, 6)) - >>> make_non_pad_mask(lengths, xs, 1) - tensor([[[1, 1, 1, 1, 1, 1], - [1, 1, 1, 1, 1, 1], - [1, 1, 1, 1, 1, 1], - [1, 1, 1, 1, 1, 1], - [1, 1, 1, 1, 1, 1], - [0, 0, 0, 0, 0, 0]], - [[1, 1, 1, 1, 1, 1], - [1, 1, 1, 1, 1, 1], - [1, 1, 1, 1, 1, 1], - [0, 0, 0, 0, 0, 0], - [0, 0, 0, 0, 0, 0], - [0, 0, 0, 0, 0, 0]], - [[1, 1, 1, 1, 1, 1], - [1, 1, 1, 1, 1, 1], - [0, 0, 0, 0, 0, 0], - [0, 0, 0, 0, 0, 0], - [0, 0, 0, 0, 0, 0], - [0, 0, 0, 0, 0, 0]]], dtype=torch.uint8) - >>> make_non_pad_mask(lengths, xs, 2) - tensor([[[1, 1, 1, 1, 1, 0], - [1, 1, 1, 1, 1, 0], - [1, 1, 1, 1, 1, 0], - [1, 1, 1, 1, 1, 0], - [1, 1, 1, 1, 1, 0], - [1, 1, 1, 1, 1, 0]], - [[1, 1, 1, 0, 0, 0], - [1, 1, 1, 0, 0, 0], - [1, 1, 1, 0, 0, 0], - [1, 1, 1, 0, 0, 0], - [1, 1, 1, 0, 0, 0], - [1, 1, 1, 0, 0, 0]], - [[1, 1, 0, 0, 0, 0], - [1, 1, 0, 0, 0, 0], - [1, 1, 0, 0, 0, 0], - [1, 1, 0, 0, 0, 0], - [1, 1, 0, 0, 0, 0], - [1, 1, 0, 0, 0, 0]]], dtype=torch.uint8) - - """ - return ~make_pad_mask(lengths, xs, length_dim) - - -def mask_by_length(xs, lengths, fill=0): - """Mask tensor according to length. - - Args: - xs (Tensor): Batch of input tensor (B, `*`). - lengths (LongTensor or List): Batch of lengths (B,). - fill (int or float): Value to fill masked part. - - Returns: - Tensor: Batch of masked input tensor (B, `*`). - - Examples: - >>> x = torch.arange(5).repeat(3, 1) + 1 - >>> x - tensor([[1, 2, 3, 4, 5], - [1, 2, 3, 4, 5], - [1, 2, 3, 4, 5]]) - >>> lengths = [5, 3, 2] - >>> mask_by_length(x, lengths) - tensor([[1, 2, 3, 4, 5], - [1, 2, 3, 0, 0], - [1, 2, 0, 0, 0]]) - - """ - assert xs.size(0) == len(lengths) - ret = xs.data.new(*xs.size()).fill_(fill) - for i, l in enumerate(lengths): - ret[i, :l] = xs[i, :l] - return ret - - -def to_torch_tensor(x): - """Change to torch.Tensor or ComplexTensor from numpy.ndarray. - - Args: - x: Inputs. It should be one of numpy.ndarray, Tensor, ComplexTensor, and dict. - - Returns: - Tensor or ComplexTensor: Type converted inputs. - - Examples: - >>> xs = np.ones(3, dtype=np.float32) - >>> xs = to_torch_tensor(xs) - tensor([1., 1., 1.]) - >>> xs = torch.ones(3, 4, 5) - >>> assert to_torch_tensor(xs) is xs - >>> xs = {'real': xs, 'imag': xs} - >>> to_torch_tensor(xs) - ComplexTensor( - Real: - tensor([1., 1., 1.]) - Imag; - tensor([1., 1., 1.]) - ) - - """ - # If numpy, change to torch tensor - if isinstance(x, np.ndarray): - if x.dtype.kind == "c": - # Dynamically importing because torch_complex requires python3 - from torch_complex.tensor import ComplexTensor - - return ComplexTensor(x) - else: - return torch.from_numpy(x) - - # If {'real': ..., 'imag': ...}, convert to ComplexTensor - elif isinstance(x, dict): - # Dynamically importing because torch_complex requires python3 - from torch_complex.tensor import ComplexTensor - - if "real" not in x or "imag" not in x: - raise ValueError("has 'real' and 'imag' keys: {}".format(list(x))) - # Relative importing because of using python3 syntax - return ComplexTensor(x["real"], x["imag"]) - - # If torch.Tensor, as it is - elif isinstance(x, torch.Tensor): - return x - - else: - error = ( - "x must be numpy.ndarray, torch.Tensor or a dict like " - "{{'real': torch.Tensor, 'imag': torch.Tensor}}, " - "but got {}".format(type(x)) - ) - try: - from torch_complex.tensor import ComplexTensor - except Exception: - # If PY2 - raise ValueError(error) - else: - # If PY3 - if isinstance(x, ComplexTensor): - return x - else: - raise ValueError(error) - - -def get_subsample(train_args, mode, arch): - """Parse the subsampling factors from the args for the specified `mode` and `arch`. - - Args: - train_args: argument Namespace containing options. - mode: one of ('asr', 'mt', 'st') - arch: one of ('rnn', 'rnn-t', 'rnn_mix', 'rnn_mulenc', 'transformer') - - Returns: - np.ndarray / List[np.ndarray]: subsampling factors. - """ - if arch == "transformer": - return np.array([1]) - - elif mode == "mt" and arch == "rnn": - # +1 means input (+1) and layers outputs (train_args.elayer) - subsample = np.ones(train_args.elayers + 1, dtype=np.int32) - logging.warning("Subsampling is not performed for machine translation.") - logging.info("subsample: " + " ".join([str(x) for x in subsample])) - return subsample - - elif ( - (mode == "asr" and arch in ("rnn", "rnn-t")) - or (mode == "mt" and arch == "rnn") - or (mode == "st" and arch == "rnn") - ): - subsample = np.ones(train_args.elayers + 1, dtype=np.int32) - if train_args.etype.endswith("p") and not train_args.etype.startswith("vgg"): - ss = train_args.subsample.split("_") - for j in range(min(train_args.elayers + 1, len(ss))): - subsample[j] = int(ss[j]) - else: - logging.warning( - "Subsampling is not performed for vgg*. " - "It is performed in max pooling layers at CNN." - ) - logging.info("subsample: " + " ".join([str(x) for x in subsample])) - return subsample - - elif mode == "asr" and arch == "rnn_mix": - subsample = np.ones( - train_args.elayers_sd + train_args.elayers + 1, dtype=np.int32 - ) - if train_args.etype.endswith("p") and not train_args.etype.startswith("vgg"): - ss = train_args.subsample.split("_") - for j in range( - min(train_args.elayers_sd + train_args.elayers + 1, len(ss)) - ): - subsample[j] = int(ss[j]) - else: - logging.warning( - "Subsampling is not performed for vgg*. " - "It is performed in max pooling layers at CNN." - ) - logging.info("subsample: " + " ".join([str(x) for x in subsample])) - return subsample - - elif mode == "asr" and arch == "rnn_mulenc": - subsample_list = [] - for idx in range(train_args.num_encs): - subsample = np.ones(train_args.elayers[idx] + 1, dtype=np.int32) - if train_args.etype[idx].endswith("p") and not train_args.etype[ - idx - ].startswith("vgg"): - ss = train_args.subsample[idx].split("_") - for j in range(min(train_args.elayers[idx] + 1, len(ss))): - subsample[j] = int(ss[j]) - else: - logging.warning( - "Encoder %d: Subsampling is not performed for vgg*. " - "It is performed in max pooling layers at CNN.", - idx + 1, - ) - logging.info("subsample: " + " ".join([str(x) for x in subsample])) - subsample_list.append(subsample) - return subsample_list - - else: - raise ValueError("Invalid options: mode={}, arch={}".format(mode, arch)) - - -def rename_state_dict( - old_prefix: str, new_prefix: str, state_dict: Dict[str, torch.Tensor] -): - """Replace keys of old prefix with new prefix in state dict.""" - # need this list not to break the dict iterator - old_keys = [k for k in state_dict if k.startswith(old_prefix)] - if len(old_keys) > 0: - logging.warning(f"Rename: {old_prefix} -> {new_prefix}") - for k in old_keys: - v = state_dict.pop(k) - new_k = k.replace(old_prefix, new_prefix) - state_dict[new_k] = v - - -class Swish(torch.nn.Module): - """Swish activation definition. - - Swish(x) = (beta * x) * sigmoid(x) - where beta = 1 defines standard Swish activation. - - References: - https://arxiv.org/abs/2108.12943 / https://arxiv.org/abs/1710.05941v1. - E-swish variant: https://arxiv.org/abs/1801.07145. - - Args: - beta: Beta parameter for E-Swish. - (beta >= 1. If beta < 1, use standard Swish). - use_builtin: Whether to use PyTorch function if available. - - """ - - def __init__(self, beta: float = 1.0, use_builtin: bool = False) -> None: - super().__init__() - - self.beta = beta - - if beta > 1: - self.swish = lambda x: (self.beta * x) * torch.sigmoid(x) - else: - if use_builtin: - self.swish = torch.nn.SiLU() - else: - self.swish = lambda x: x * torch.sigmoid(x) - - def forward(self, x: torch.Tensor) -> torch.Tensor: - """Forward computation.""" - return self.swish(x) - - -def get_activation(act): - """Return activation function.""" - - activation_funcs = { - "hardtanh": torch.nn.Hardtanh, - "tanh": torch.nn.Tanh, - "relu": torch.nn.ReLU, - "selu": torch.nn.SELU, - "swish": Swish, - } - - return activation_funcs[act]() - - -class TooShortUttError(Exception): - """Raised when the utt is too short for subsampling. - - Args: - message: Error message to display. - actual_size: The size that cannot pass the subsampling. - limit: The size limit for subsampling. - - """ - - def __init__(self, message: str, actual_size: int, limit: int) -> None: - """Construct a TooShortUttError module.""" - super().__init__(message) - - self.actual_size = actual_size - self.limit = limit - - -def check_short_utt(sub_factor: int, size: int) -> Tuple[bool, int]: - """Check if the input is too short for subsampling. - - Args: - sub_factor: Subsampling factor for Conv2DSubsampling. - size: Input size. - - Returns: - : Whether an error should be sent. - : Size limit for specified subsampling factor. - - """ - if sub_factor == 2 and size < 3: - return True, 7 - elif sub_factor == 4 and size < 7: - return True, 7 - elif sub_factor == 6 and size < 11: - return True, 11 - - return False, -1 - - -def sub_factor_to_params(sub_factor: int, input_size: int) -> Tuple[int, int, int]: - """Get conv2D second layer parameters for given subsampling factor. - - Args: - sub_factor: Subsampling factor (1/X). - input_size: Input size. - - Returns: - : Kernel size for second convolution. - : Stride for second convolution. - : Conv2DSubsampling output size. - - """ - if sub_factor == 2: - return 3, 1, (((input_size - 1) // 2 - 2)) - elif sub_factor == 4: - return 3, 2, (((input_size - 1) // 2 - 1) // 2) - elif sub_factor == 6: - return 5, 3, (((input_size - 1) // 2 - 2) // 3) - else: - raise ValueError( - "subsampling_factor parameter should be set to either 2, 4 or 6." - ) - - -def make_chunk_mask( - size: int, - chunk_size: int, - left_chunk_size: int = 0, - device: torch.device = None, -) -> torch.Tensor: - """Create chunk mask for the subsequent steps (size, size). - - Reference: https://github.com/k2-fsa/icefall/blob/master/icefall/utils.py - - Args: - size: Size of the source mask. - chunk_size: Number of frames in chunk. - left_chunk_size: Size of the left context in chunks (0 means full context). - device: Device for the mask tensor. - - Returns: - mask: Chunk mask. (size, size) - - """ - mask = torch.zeros(size, size, device=device, dtype=torch.bool) - - for i in range(size): - if left_chunk_size < 0: - start = 0 - else: - start = max((i // chunk_size - left_chunk_size) * chunk_size, 0) - - end = min((i // chunk_size + 1) * chunk_size, size) - mask[i, start:end] = True - - return ~mask - - -def make_source_mask(lengths: torch.Tensor) -> torch.Tensor: - """Create source mask for given lengths. - - Reference: https://github.com/k2-fsa/icefall/blob/master/icefall/utils.py - - Args: - lengths: Sequence lengths. (B,) - - Returns: - : Mask for the sequence lengths. (B, max_len) - - """ - max_len = lengths.max() - batch_size = lengths.size(0) - - expanded_lengths = torch.arange(max_len).expand(batch_size, max_len).to(lengths) - - return expanded_lengths >= lengths.unsqueeze(1) - - -def get_transducer_task_io( - labels: torch.Tensor, - encoder_out_lens: torch.Tensor, - ignore_id: int = -1, - blank_id: int = 0, -) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: - """Get Transducer loss I/O. - - Args: - labels: Label ID sequences. (B, L) - encoder_out_lens: Encoder output lengths. (B,) - ignore_id: Padding symbol ID. - blank_id: Blank symbol ID. - - Returns: - decoder_in: Decoder inputs. (B, U) - target: Target label ID sequences. (B, U) - t_len: Time lengths. (B,) - u_len: Label lengths. (B,) - - """ - - def pad_list(labels: List[torch.Tensor], padding_value: int = 0): - """Create padded batch of labels from a list of labels sequences. - - Args: - labels: Labels sequences. [B x (?)] - padding_value: Padding value. - - Returns: - labels: Batch of padded labels sequences. (B,) - - """ - batch_size = len(labels) - - padded = ( - labels[0] - .new(batch_size, max(x.size(0) for x in labels), *labels[0].size()[1:]) - .fill_(padding_value) - ) - - for i in range(batch_size): - padded[i, : labels[i].size(0)] = labels[i] - - return padded - - device = labels.device - - labels_unpad = [y[y != ignore_id] for y in labels] - blank = labels[0].new([blank_id]) - - decoder_in = pad_list( - [torch.cat([blank, label], dim=0) for label in labels_unpad], blank_id - ).to(device) - - target = pad_list(labels_unpad, blank_id).type(torch.int32).to(device) - - encoder_out_lens = list(map(int, encoder_out_lens)) - t_len = torch.IntTensor(encoder_out_lens).to(device) - - u_len = torch.IntTensor([y.size(0) for y in labels_unpad]).to(device) - - return decoder_in, target, t_len, u_len - - -def pad_to_len(t: torch.Tensor, pad_len: int, dim: int): - """Pad the tensor `t` at `dim` to the length `pad_len` with right padding zeros.""" - if t.size(dim) == pad_len: - return t - else: - pad_size = list(t.shape) - pad_size[dim] = pad_len - t.size(dim) - return torch.cat( - [t, torch.zeros(*pad_size, dtype=t.dtype, device=t.device)], dim=dim - ) diff --git a/funasr_detach/models/transformer/utils/repeat.py b/funasr_detach/models/transformer/utils/repeat.py deleted file mode 100644 index a16913bda858d3c647c213b6de39c6159a1ea009..0000000000000000000000000000000000000000 --- a/funasr_detach/models/transformer/utils/repeat.py +++ /dev/null @@ -1,137 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- - -# Copyright 2019 Shigeki Karita -# Apache 2.0 (http://www.apache.org/licenses/LICENSE-2.0) - -"""Repeat the same layer definition.""" - -from typing import Dict, List, Optional -from funasr_detach.models.transformer.layer_norm import LayerNorm -import torch - - -class MultiSequential(torch.nn.Sequential): - """Multi-input multi-output torch.nn.Sequential.""" - - def __init__(self, *args, layer_drop_rate=0.0): - """Initialize MultiSequential with layer_drop. - - Args: - layer_drop_rate (float): Probability of dropping out each fn (layer). - - """ - super(MultiSequential, self).__init__(*args) - self.layer_drop_rate = layer_drop_rate - - def forward(self, *args): - """Repeat.""" - _probs = torch.empty(len(self)).uniform_() - for idx, m in enumerate(self): - if not self.training or (_probs[idx] >= self.layer_drop_rate): - args = m(*args) - return args - - -def repeat(N, fn, layer_drop_rate=0.0): - """Repeat module N times. - - Args: - N (int): Number of repeat time. - fn (Callable): Function to generate module. - layer_drop_rate (float): Probability of dropping out each fn (layer). - - Returns: - MultiSequential: Repeated model instance. - - """ - return MultiSequential(*[fn(n) for n in range(N)], layer_drop_rate=layer_drop_rate) - - -class MultiBlocks(torch.nn.Module): - """MultiBlocks definition. - Args: - block_list: Individual blocks of the encoder architecture. - output_size: Architecture output size. - norm_class: Normalization module class. - norm_args: Normalization module arguments. - """ - - def __init__( - self, - block_list: List[torch.nn.Module], - output_size: int, - norm_class: torch.nn.Module = LayerNorm, - ) -> None: - """Construct a MultiBlocks object.""" - super().__init__() - - self.blocks = torch.nn.ModuleList(block_list) - self.norm_blocks = norm_class(output_size) - - self.num_blocks = len(block_list) - - def reset_streaming_cache(self, left_context: int, device: torch.device) -> None: - """Initialize/Reset encoder streaming cache. - Args: - left_context: Number of left frames during chunk-by-chunk inference. - device: Device to use for cache tensor. - """ - for idx in range(self.num_blocks): - self.blocks[idx].reset_streaming_cache(left_context, device) - - def forward( - self, - x: torch.Tensor, - pos_enc: torch.Tensor, - mask: torch.Tensor, - chunk_mask: Optional[torch.Tensor] = None, - ) -> torch.Tensor: - """Forward each block of the encoder architecture. - Args: - x: MultiBlocks input sequences. (B, T, D_block_1) - pos_enc: Positional embedding sequences. - mask: Source mask. (B, T) - chunk_mask: Chunk mask. (T_2, T_2) - Returns: - x: Output sequences. (B, T, D_block_N) - """ - for block_index, block in enumerate(self.blocks): - x, mask, pos_enc = block(x, pos_enc, mask, chunk_mask=chunk_mask) - - x = self.norm_blocks(x) - - return x - - def chunk_forward( - self, - x: torch.Tensor, - pos_enc: torch.Tensor, - mask: torch.Tensor, - chunk_size: int = 0, - left_context: int = 0, - right_context: int = 0, - ) -> torch.Tensor: - """Forward each block of the encoder architecture. - Args: - x: MultiBlocks input sequences. (B, T, D_block_1) - pos_enc: Positional embedding sequences. (B, 2 * (T - 1), D_att) - mask: Source mask. (B, T_2) - left_context: Number of frames in left context. - right_context: Number of frames in right context. - Returns: - x: MultiBlocks output sequences. (B, T, D_block_N) - """ - for block_idx, block in enumerate(self.blocks): - x, pos_enc = block.chunk_forward( - x, - pos_enc, - mask, - chunk_size=chunk_size, - left_context=left_context, - right_context=right_context, - ) - - x = self.norm_blocks(x) - - return x diff --git a/funasr_detach/models/transformer/utils/subsampling.py b/funasr_detach/models/transformer/utils/subsampling.py deleted file mode 100644 index 15605ba90325015ed8412ad05e96e4d33863b025..0000000000000000000000000000000000000000 --- a/funasr_detach/models/transformer/utils/subsampling.py +++ /dev/null @@ -1,688 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- - -# Copyright 2019 Shigeki Karita -# Apache 2.0 (http://www.apache.org/licenses/LICENSE-2.0) - -"""Subsampling layer definition.""" -import numpy as np -import torch -import torch.nn.functional as F -from funasr_detach.models.transformer.embedding import PositionalEncoding -import logging -from funasr_detach.models.scama.utils import sequence_mask -from funasr_detach.models.transformer.utils.nets_utils import ( - sub_factor_to_params, - pad_to_len, -) -from typing import Optional, Tuple, Union -import math - - -class TooShortUttError(Exception): - """Raised when the utt is too short for subsampling. - - Args: - message (str): Message for error catch - actual_size (int): the short size that cannot pass the subsampling - limit (int): the limit size for subsampling - - """ - - def __init__(self, message, actual_size, limit): - """Construct a TooShortUttError for error handler.""" - super().__init__(message) - self.actual_size = actual_size - self.limit = limit - - -def check_short_utt(ins, size): - """Check if the utterance is too short for subsampling.""" - if isinstance(ins, Conv2dSubsampling2) and size < 3: - return True, 3 - if isinstance(ins, Conv2dSubsampling) and size < 7: - return True, 7 - if isinstance(ins, Conv2dSubsampling6) and size < 11: - return True, 11 - if isinstance(ins, Conv2dSubsampling8) and size < 15: - return True, 15 - return False, -1 - - -class Conv2dSubsampling(torch.nn.Module): - """Convolutional 2D subsampling (to 1/4 length). - - Args: - idim (int): Input dimension. - odim (int): Output dimension. - dropout_rate (float): Dropout rate. - pos_enc (torch.nn.Module): Custom position encoding layer. - - """ - - def __init__(self, idim, odim, dropout_rate, pos_enc=None): - """Construct an Conv2dSubsampling object.""" - super(Conv2dSubsampling, self).__init__() - self.conv = torch.nn.Sequential( - torch.nn.Conv2d(1, odim, 3, 2), - torch.nn.ReLU(), - torch.nn.Conv2d(odim, odim, 3, 2), - torch.nn.ReLU(), - ) - self.out = torch.nn.Sequential( - torch.nn.Linear(odim * (((idim - 1) // 2 - 1) // 2), odim), - pos_enc if pos_enc is not None else PositionalEncoding(odim, dropout_rate), - ) - - def forward(self, x, x_mask): - """Subsample x. - - Args: - x (torch.Tensor): Input tensor (#batch, time, idim). - x_mask (torch.Tensor): Input mask (#batch, 1, time). - - Returns: - torch.Tensor: Subsampled tensor (#batch, time', odim), - where time' = time // 4. - torch.Tensor: Subsampled mask (#batch, 1, time'), - where time' = time // 4. - - """ - x = x.unsqueeze(1) # (b, c, t, f) - x = self.conv(x) - b, c, t, f = x.size() - x = self.out(x.transpose(1, 2).contiguous().view(b, t, c * f)) - if x_mask is None: - return x, None - return x, x_mask[:, :, :-2:2][:, :, :-2:2] - - def __getitem__(self, key): - """Get item. - - When reset_parameters() is called, if use_scaled_pos_enc is used, - return the positioning encoding. - - """ - if key != -1: - raise NotImplementedError("Support only `-1` (for `reset_parameters`).") - return self.out[key] - - -class Conv2dSubsamplingPad(torch.nn.Module): - """Convolutional 2D subsampling (to 1/4 length). - - Args: - idim (int): Input dimension. - odim (int): Output dimension. - dropout_rate (float): Dropout rate. - pos_enc (torch.nn.Module): Custom position encoding layer. - - """ - - def __init__(self, idim, odim, dropout_rate, pos_enc=None): - """Construct an Conv2dSubsampling object.""" - super(Conv2dSubsamplingPad, self).__init__() - self.conv = torch.nn.Sequential( - torch.nn.Conv2d(1, odim, 3, 2, padding=(0, 0)), - torch.nn.ReLU(), - torch.nn.Conv2d(odim, odim, 3, 2, padding=(0, 0)), - torch.nn.ReLU(), - ) - self.out = torch.nn.Sequential( - torch.nn.Linear(odim * (((idim - 1) // 2 - 1) // 2), odim), - pos_enc if pos_enc is not None else PositionalEncoding(odim, dropout_rate), - ) - self.pad_fn = torch.nn.ConstantPad1d((0, 4), 0.0) - - def forward(self, x, x_mask): - """Subsample x. - - Args: - x (torch.Tensor): Input tensor (#batch, time, idim). - x_mask (torch.Tensor): Input mask (#batch, 1, time). - - Returns: - torch.Tensor: Subsampled tensor (#batch, time', odim), - where time' = time // 4. - torch.Tensor: Subsampled mask (#batch, 1, time'), - where time' = time // 4. - - """ - x = x.transpose(1, 2) - x = self.pad_fn(x) - x = x.transpose(1, 2) - x = x.unsqueeze(1) # (b, c, t, f) - x = self.conv(x) - b, c, t, f = x.size() - x = self.out(x.transpose(1, 2).contiguous().view(b, t, c * f)) - if x_mask is None: - return x, None - x_len = torch.sum(x_mask[:, 0, :], dim=-1) - x_len = (x_len - 1) // 2 + 1 - x_len = (x_len - 1) // 2 + 1 - mask = sequence_mask(x_len, None, x_len.dtype, x[0].device) - return x, mask[:, None, :] - - def __getitem__(self, key): - """Get item. - - When reset_parameters() is called, if use_scaled_pos_enc is used, - return the positioning encoding. - - """ - if key != -1: - raise NotImplementedError("Support only `-1` (for `reset_parameters`).") - return self.out[key] - - -class Conv2dSubsampling2(torch.nn.Module): - """Convolutional 2D subsampling (to 1/2 length). - - Args: - idim (int): Input dimension. - odim (int): Output dimension. - dropout_rate (float): Dropout rate. - pos_enc (torch.nn.Module): Custom position encoding layer. - - """ - - def __init__(self, idim, odim, dropout_rate, pos_enc=None): - """Construct an Conv2dSubsampling2 object.""" - super(Conv2dSubsampling2, self).__init__() - self.conv = torch.nn.Sequential( - torch.nn.Conv2d(1, odim, 3, 2), - torch.nn.ReLU(), - torch.nn.Conv2d(odim, odim, 3, 1), - torch.nn.ReLU(), - ) - self.out = torch.nn.Sequential( - torch.nn.Linear(odim * (((idim - 1) // 2 - 2)), odim), - pos_enc if pos_enc is not None else PositionalEncoding(odim, dropout_rate), - ) - - def forward(self, x, x_mask): - """Subsample x. - - Args: - x (torch.Tensor): Input tensor (#batch, time, idim). - x_mask (torch.Tensor): Input mask (#batch, 1, time). - - Returns: - torch.Tensor: Subsampled tensor (#batch, time', odim), - where time' = time // 2. - torch.Tensor: Subsampled mask (#batch, 1, time'), - where time' = time // 2. - - """ - x = x.unsqueeze(1) # (b, c, t, f) - x = self.conv(x) - b, c, t, f = x.size() - x = self.out(x.transpose(1, 2).contiguous().view(b, t, c * f)) - if x_mask is None: - return x, None - return x, x_mask[:, :, :-2:2][:, :, :-2:1] - - def __getitem__(self, key): - """Get item. - - When reset_parameters() is called, if use_scaled_pos_enc is used, - return the positioning encoding. - - """ - if key != -1: - raise NotImplementedError("Support only `-1` (for `reset_parameters`).") - return self.out[key] - - -class Conv2dSubsampling6(torch.nn.Module): - """Convolutional 2D subsampling (to 1/6 length). - - Args: - idim (int): Input dimension. - odim (int): Output dimension. - dropout_rate (float): Dropout rate. - pos_enc (torch.nn.Module): Custom position encoding layer. - - """ - - def __init__(self, idim, odim, dropout_rate, pos_enc=None): - """Construct an Conv2dSubsampling6 object.""" - super(Conv2dSubsampling6, self).__init__() - self.conv = torch.nn.Sequential( - torch.nn.Conv2d(1, odim, 3, 2), - torch.nn.ReLU(), - torch.nn.Conv2d(odim, odim, 5, 3), - torch.nn.ReLU(), - ) - self.out = torch.nn.Sequential( - torch.nn.Linear(odim * (((idim - 1) // 2 - 2) // 3), odim), - pos_enc if pos_enc is not None else PositionalEncoding(odim, dropout_rate), - ) - - def forward(self, x, x_mask): - """Subsample x. - - Args: - x (torch.Tensor): Input tensor (#batch, time, idim). - x_mask (torch.Tensor): Input mask (#batch, 1, time). - - Returns: - torch.Tensor: Subsampled tensor (#batch, time', odim), - where time' = time // 6. - torch.Tensor: Subsampled mask (#batch, 1, time'), - where time' = time // 6. - - """ - x = x.unsqueeze(1) # (b, c, t, f) - x = self.conv(x) - b, c, t, f = x.size() - x = self.out(x.transpose(1, 2).contiguous().view(b, t, c * f)) - if x_mask is None: - return x, None - return x, x_mask[:, :, :-2:2][:, :, :-4:3] - - -class Conv2dSubsampling8(torch.nn.Module): - """Convolutional 2D subsampling (to 1/8 length). - - Args: - idim (int): Input dimension. - odim (int): Output dimension. - dropout_rate (float): Dropout rate. - pos_enc (torch.nn.Module): Custom position encoding layer. - - """ - - def __init__(self, idim, odim, dropout_rate, pos_enc=None): - """Construct an Conv2dSubsampling8 object.""" - super(Conv2dSubsampling8, self).__init__() - self.conv = torch.nn.Sequential( - torch.nn.Conv2d(1, odim, 3, 2), - torch.nn.ReLU(), - torch.nn.Conv2d(odim, odim, 3, 2), - torch.nn.ReLU(), - torch.nn.Conv2d(odim, odim, 3, 2), - torch.nn.ReLU(), - ) - self.out = torch.nn.Sequential( - torch.nn.Linear(odim * ((((idim - 1) // 2 - 1) // 2 - 1) // 2), odim), - pos_enc if pos_enc is not None else PositionalEncoding(odim, dropout_rate), - ) - - def forward(self, x, x_mask): - """Subsample x. - - Args: - x (torch.Tensor): Input tensor (#batch, time, idim). - x_mask (torch.Tensor): Input mask (#batch, 1, time). - - Returns: - torch.Tensor: Subsampled tensor (#batch, time', odim), - where time' = time // 8. - torch.Tensor: Subsampled mask (#batch, 1, time'), - where time' = time // 8. - - """ - x = x.unsqueeze(1) # (b, c, t, f) - x = self.conv(x) - b, c, t, f = x.size() - x = self.out(x.transpose(1, 2).contiguous().view(b, t, c * f)) - if x_mask is None: - return x, None - return x, x_mask[:, :, :-2:2][:, :, :-2:2][:, :, :-2:2] - - -class Conv1dSubsampling(torch.nn.Module): - """Convolutional 1D subsampling (to 1/2 length). - - Args: - idim (int): Input dimension. - odim (int): Output dimension. - dropout_rate (float): Dropout rate. - pos_enc (torch.nn.Module): Custom position encoding layer. - - """ - - def __init__( - self, - idim, - odim, - kernel_size, - stride, - pad, - tf2torch_tensor_name_prefix_torch: str = "stride_conv", - tf2torch_tensor_name_prefix_tf: str = "seq2seq/proj_encoder/downsampling", - ): - super(Conv1dSubsampling, self).__init__() - self.conv = torch.nn.Conv1d(idim, odim, kernel_size, stride) - self.pad_fn = torch.nn.ConstantPad1d(pad, 0.0) - self.stride = stride - self.odim = odim - self.tf2torch_tensor_name_prefix_torch = tf2torch_tensor_name_prefix_torch - self.tf2torch_tensor_name_prefix_tf = tf2torch_tensor_name_prefix_tf - - def output_size(self) -> int: - return self.odim - - def forward(self, x, x_len): - """Subsample x.""" - x = x.transpose(1, 2) # (b, d ,t) - x = self.pad_fn(x) - # x = F.relu(self.conv(x)) - x = F.leaky_relu(self.conv(x), negative_slope=0.0) - x = x.transpose(1, 2) # (b, t ,d) - - if x_len is None: - - return x, None - x_len = (x_len - 1) // self.stride + 1 - return x, x_len - - def gen_tf2torch_map_dict(self): - tensor_name_prefix_torch = self.tf2torch_tensor_name_prefix_torch - tensor_name_prefix_tf = self.tf2torch_tensor_name_prefix_tf - map_dict_local = { - ## predictor - "{}.conv.weight".format(tensor_name_prefix_torch): { - "name": "{}/conv1d/kernel".format(tensor_name_prefix_tf), - "squeeze": None, - "transpose": (2, 1, 0), - }, # (256,256,3),(3,256,256) - "{}.conv.bias".format(tensor_name_prefix_torch): { - "name": "{}/conv1d/bias".format(tensor_name_prefix_tf), - "squeeze": None, - "transpose": None, - }, # (256,),(256,) - } - return map_dict_local - - def convert_tf2torch( - self, - var_dict_tf, - var_dict_torch, - ): - - map_dict = self.gen_tf2torch_map_dict() - - var_dict_torch_update = dict() - for name in sorted(var_dict_torch.keys(), reverse=False): - names = name.split(".") - if names[0] == self.tf2torch_tensor_name_prefix_torch: - name_tf = map_dict[name]["name"] - data_tf = var_dict_tf[name_tf] - if map_dict[name]["squeeze"] is not None: - data_tf = np.squeeze(data_tf, axis=map_dict[name]["squeeze"]) - if map_dict[name]["transpose"] is not None: - data_tf = np.transpose(data_tf, map_dict[name]["transpose"]) - data_tf = torch.from_numpy(data_tf).type(torch.float32).to("cpu") - - var_dict_torch_update[name] = data_tf - - logging.info( - "torch tensor: {}, {}, loading from tf tensor: {}, {}".format( - name, data_tf.size(), name_tf, var_dict_tf[name_tf].shape - ) - ) - return var_dict_torch_update - - -class StreamingConvInput(torch.nn.Module): - """Streaming ConvInput module definition. - Args: - input_size: Input size. - conv_size: Convolution size. - subsampling_factor: Subsampling factor. - vgg_like: Whether to use a VGG-like network. - output_size: Block output dimension. - """ - - def __init__( - self, - input_size: int, - conv_size: Union[int, Tuple], - subsampling_factor: int = 4, - vgg_like: bool = True, - conv_kernel_size: int = 3, - output_size: Optional[int] = None, - ) -> None: - """Construct a ConvInput object.""" - super().__init__() - if vgg_like: - if subsampling_factor == 1: - conv_size1, conv_size2 = conv_size - - self.conv = torch.nn.Sequential( - torch.nn.Conv2d( - 1, - conv_size1, - conv_kernel_size, - stride=1, - padding=(conv_kernel_size - 1) // 2, - ), - torch.nn.ReLU(), - torch.nn.Conv2d( - conv_size1, - conv_size1, - conv_kernel_size, - stride=1, - padding=(conv_kernel_size - 1) // 2, - ), - torch.nn.ReLU(), - torch.nn.MaxPool2d((1, 2)), - torch.nn.Conv2d( - conv_size1, - conv_size2, - conv_kernel_size, - stride=1, - padding=(conv_kernel_size - 1) // 2, - ), - torch.nn.ReLU(), - torch.nn.Conv2d( - conv_size2, - conv_size2, - conv_kernel_size, - stride=1, - padding=(conv_kernel_size - 1) // 2, - ), - torch.nn.ReLU(), - torch.nn.MaxPool2d((1, 2)), - ) - - output_proj = conv_size2 * ((input_size // 2) // 2) - - self.subsampling_factor = 1 - - self.stride_1 = 1 - - self.create_new_mask = self.create_new_vgg_mask - - else: - conv_size1, conv_size2 = conv_size - - kernel_1 = int(subsampling_factor / 2) - - self.conv = torch.nn.Sequential( - torch.nn.Conv2d( - 1, - conv_size1, - conv_kernel_size, - stride=1, - padding=(conv_kernel_size - 1) // 2, - ), - torch.nn.ReLU(), - torch.nn.Conv2d( - conv_size1, - conv_size1, - conv_kernel_size, - stride=1, - padding=(conv_kernel_size - 1) // 2, - ), - torch.nn.ReLU(), - torch.nn.MaxPool2d((kernel_1, 2)), - torch.nn.Conv2d( - conv_size1, - conv_size2, - conv_kernel_size, - stride=1, - padding=(conv_kernel_size - 1) // 2, - ), - torch.nn.ReLU(), - torch.nn.Conv2d( - conv_size2, - conv_size2, - conv_kernel_size, - stride=1, - padding=(conv_kernel_size - 1) // 2, - ), - torch.nn.ReLU(), - torch.nn.MaxPool2d((2, 2)), - ) - - output_proj = conv_size2 * ((input_size // 2) // 2) - - self.subsampling_factor = subsampling_factor - - self.create_new_mask = self.create_new_vgg_mask - - self.stride_1 = kernel_1 - - else: - if subsampling_factor == 1: - self.conv = torch.nn.Sequential( - torch.nn.Conv2d(1, conv_size, 3, [1, 2], [1, 0]), - torch.nn.ReLU(), - torch.nn.Conv2d( - conv_size, conv_size, conv_kernel_size, [1, 2], [1, 0] - ), - torch.nn.ReLU(), - ) - - output_proj = conv_size * (((input_size - 1) // 2 - 1) // 2) - - self.subsampling_factor = subsampling_factor - self.kernel_2 = conv_kernel_size - self.stride_2 = 1 - - self.create_new_mask = self.create_new_conv2d_mask - - else: - kernel_2, stride_2, conv_2_output_size = sub_factor_to_params( - subsampling_factor, - input_size, - ) - - self.conv = torch.nn.Sequential( - torch.nn.Conv2d(1, conv_size, 3, 2, [1, 0]), - torch.nn.ReLU(), - torch.nn.Conv2d( - conv_size, - conv_size, - kernel_2, - stride_2, - [(kernel_2 - 1) // 2, 0], - ), - torch.nn.ReLU(), - ) - - output_proj = conv_size * conv_2_output_size - - self.subsampling_factor = subsampling_factor - self.kernel_2 = kernel_2 - self.stride_2 = stride_2 - - self.create_new_mask = self.create_new_conv2d_mask - - self.vgg_like = vgg_like - self.min_frame_length = 7 - - if output_size is not None: - self.output = torch.nn.Linear(output_proj, output_size) - self.output_size = output_size - else: - self.output = None - self.output_size = output_proj - - def forward( - self, - x: torch.Tensor, - mask: Optional[torch.Tensor], - chunk_size: Optional[torch.Tensor], - ) -> Tuple[torch.Tensor, torch.Tensor]: - """Encode input sequences. - Args: - x: ConvInput input sequences. (B, T, D_feats) - mask: Mask of input sequences. (B, 1, T) - Returns: - x: ConvInput output sequences. (B, sub(T), D_out) - mask: Mask of output sequences. (B, 1, sub(T)) - """ - if mask is not None: - mask = self.create_new_mask(mask) - olens = max(mask.eq(0).sum(1)) - - b, t, f = x.size() - x = x.unsqueeze(1) # (b. 1. t. f) - - if chunk_size is not None: - max_input_length = int( - chunk_size - * self.subsampling_factor - * (math.ceil(float(t) / (chunk_size * self.subsampling_factor))) - ) - x = map(lambda inputs: pad_to_len(inputs, max_input_length, 1), x) - x = list(x) - x = torch.stack(x, dim=0) - N_chunks = max_input_length // (chunk_size * self.subsampling_factor) - x = x.view(b * N_chunks, 1, chunk_size * self.subsampling_factor, f) - - x = self.conv(x) - - _, c, _, f = x.size() - if chunk_size is not None: - x = x.transpose(1, 2).contiguous().view(b, -1, c * f)[:, :olens, :] - else: - x = x.transpose(1, 2).contiguous().view(b, -1, c * f) - - if self.output is not None: - x = self.output(x) - - return x, mask[:, :olens][:, : x.size(1)] - - def create_new_vgg_mask(self, mask: torch.Tensor) -> torch.Tensor: - """Create a new mask for VGG output sequences. - Args: - mask: Mask of input sequences. (B, T) - Returns: - mask: Mask of output sequences. (B, sub(T)) - """ - if self.subsampling_factor > 1: - vgg1_t_len = mask.size(1) - (mask.size(1) % (self.subsampling_factor // 2)) - mask = mask[:, :vgg1_t_len][:, :: self.subsampling_factor // 2] - - vgg2_t_len = mask.size(1) - (mask.size(1) % 2) - mask = mask[:, :vgg2_t_len][:, ::2] - else: - mask = mask - - return mask - - def create_new_conv2d_mask(self, mask: torch.Tensor) -> torch.Tensor: - """Create new conformer mask for Conv2d output sequences. - Args: - mask: Mask of input sequences. (B, T) - Returns: - mask: Mask of output sequences. (B, sub(T)) - """ - if self.subsampling_factor > 1: - return mask[:, ::2][:, :: self.stride_2] - else: - return mask - - def get_size_before_subsampling(self, size: int) -> int: - """Return the original size before subsampling for a given size. - Args: - size: Number of frames after subsampling. - Returns: - : Number of frames before subsampling. - """ - return size * self.subsampling_factor diff --git a/funasr_detach/models/transformer/utils/subsampling_without_posenc.py b/funasr_detach/models/transformer/utils/subsampling_without_posenc.py deleted file mode 100644 index 239d3f1ade7f03435e44bf2414f7ab59cb055e6f..0000000000000000000000000000000000000000 --- a/funasr_detach/models/transformer/utils/subsampling_without_posenc.py +++ /dev/null @@ -1,61 +0,0 @@ -# Copyright 2020 Emiru Tsunoo -# Apache 2.0 (http://www.apache.org/licenses/LICENSE-2.0) - -"""Subsampling layer definition.""" - -import math -import torch - - -class Conv2dSubsamplingWOPosEnc(torch.nn.Module): - """Convolutional 2D subsampling. - - Args: - idim (int): Input dimension. - odim (int): Output dimension. - dropout_rate (float): Dropout rate. - kernels (list): kernel sizes - strides (list): stride sizes - - """ - - def __init__(self, idim, odim, dropout_rate, kernels, strides): - """Construct an Conv2dSubsamplingWOPosEnc object.""" - assert len(kernels) == len(strides) - super().__init__() - conv = [] - olen = idim - for i, (k, s) in enumerate(zip(kernels, strides)): - conv += [ - torch.nn.Conv2d(1 if i == 0 else odim, odim, k, s), - torch.nn.ReLU(), - ] - olen = math.floor((olen - k) / s + 1) - self.conv = torch.nn.Sequential(*conv) - self.out = torch.nn.Linear(odim * olen, odim) - self.strides = strides - self.kernels = kernels - - def forward(self, x, x_mask): - """Subsample x. - - Args: - x (torch.Tensor): Input tensor (#batch, time, idim). - x_mask (torch.Tensor): Input mask (#batch, 1, time). - - Returns: - torch.Tensor: Subsampled tensor (#batch, time', odim), - where time' = time // 4. - torch.Tensor: Subsampled mask (#batch, 1, time'), - where time' = time // 4. - - """ - x = x.unsqueeze(1) # (b, c, t, f) - x = self.conv(x) - b, c, t, f = x.size() - x = self.out(x.transpose(1, 2).contiguous().view(b, t, c * f)) - if x_mask is None: - return x, None - for k, s in zip(self.kernels, self.strides): - x_mask = x_mask[:, :, : -k + 1 : s] - return x, x_mask diff --git a/funasr_detach/models/transformer/utils/vgg2l.py b/funasr_detach/models/transformer/utils/vgg2l.py deleted file mode 100644 index ab448d6590ae8b502bfc004b2166343ed469e7a1..0000000000000000000000000000000000000000 --- a/funasr_detach/models/transformer/utils/vgg2l.py +++ /dev/null @@ -1,90 +0,0 @@ -"""VGG2L module definition for custom encoder.""" - -from typing import Tuple, Union - -import torch - - -class VGG2L(torch.nn.Module): - """VGG2L module for custom encoder. - - Args: - idim: Input dimension. - odim: Output dimension. - pos_enc: Positional encoding class. - - """ - - def __init__(self, idim: int, odim: int, pos_enc: torch.nn.Module = None): - """Construct a VGG2L object.""" - super().__init__() - - self.vgg2l = torch.nn.Sequential( - torch.nn.Conv2d(1, 64, 3, stride=1, padding=1), - torch.nn.ReLU(), - torch.nn.Conv2d(64, 64, 3, stride=1, padding=1), - torch.nn.ReLU(), - torch.nn.MaxPool2d((3, 2)), - torch.nn.Conv2d(64, 128, 3, stride=1, padding=1), - torch.nn.ReLU(), - torch.nn.Conv2d(128, 128, 3, stride=1, padding=1), - torch.nn.ReLU(), - torch.nn.MaxPool2d((2, 2)), - ) - - if pos_enc is not None: - self.output = torch.nn.Sequential( - torch.nn.Linear(128 * ((idim // 2) // 2), odim), pos_enc - ) - else: - self.output = torch.nn.Linear(128 * ((idim // 2) // 2), odim) - - def forward(self, feats: torch.Tensor, feats_mask: torch.Tensor) -> Union[ - Tuple[torch.Tensor, torch.Tensor], - Tuple[Tuple[torch.Tensor, torch.Tensor], torch.Tensor], - ]: - """Forward VGG2L bottleneck. - - Args: - feats: Feature sequences. (B, F, D_feats) - feats_mask: Mask of feature sequences. (B, 1, F) - - Returns: - vgg_output: VGG output sequences. - (B, sub(F), D_out) or ((B, sub(F), D_out), (B, sub(F), D_att)) - vgg_mask: Mask of VGG output sequences. (B, 1, sub(F)) - - """ - feats = feats.unsqueeze(1) - vgg_output = self.vgg2l(feats) - - b, c, t, f = vgg_output.size() - - vgg_output = self.output( - vgg_output.transpose(1, 2).contiguous().view(b, t, c * f) - ) - - if feats_mask is not None: - vgg_mask = self.create_new_mask(feats_mask) - else: - vgg_mask = feats_mask - - return vgg_output, vgg_mask - - def create_new_mask(self, feats_mask: torch.Tensor) -> torch.Tensor: - """Create a subsampled mask of feature sequences. - - Args: - feats_mask: Mask of feature sequences. (B, 1, F) - - Returns: - vgg_mask: Mask of VGG2L output sequences. (B, 1, sub(F)) - - """ - vgg1_t_len = feats_mask.size(2) - (feats_mask.size(2) % 3) - vgg_mask = feats_mask[:, :, :vgg1_t_len][:, :, ::3] - - vgg2_t_len = vgg_mask.size(2) - (vgg_mask.size(2) % 2) - vgg_mask = vgg_mask[:, :, :vgg2_t_len][:, :, ::2] - - return vgg_mask diff --git a/funasr_detach/models/uniasr/__init__.py b/funasr_detach/models/uniasr/__init__.py deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/funasr_detach/models/uniasr/beam_search.py b/funasr_detach/models/uniasr/beam_search.py deleted file mode 100644 index 8030b665f491d17224da22b42e4dc6940c4a8926..0000000000000000000000000000000000000000 --- a/funasr_detach/models/uniasr/beam_search.py +++ /dev/null @@ -1,519 +0,0 @@ -"""Beam search module.""" - -from itertools import chain -import logging -from typing import Any -from typing import Dict -from typing import List -from typing import NamedTuple -from typing import Tuple -from typing import Union - -import torch - -from funasr_detach.metrics.common import end_detect -from funasr_detach.models.transformer.scorers.scorer_interface import ( - PartialScorerInterface, -) -from funasr_detach.models.transformer.scorers.scorer_interface import ScorerInterface - - -class Hypothesis(NamedTuple): - """Hypothesis data type.""" - - yseq: torch.Tensor - score: Union[float, torch.Tensor] = 0 - scores: Dict[str, Union[float, torch.Tensor]] = dict() - states: Dict[str, Any] = dict() - - def asdict(self) -> dict: - """Convert data to JSON-friendly dict.""" - return self._replace( - yseq=self.yseq.tolist(), - score=float(self.score), - scores={k: float(v) for k, v in self.scores.items()}, - )._asdict() - - -class BeamSearchScama(torch.nn.Module): - """Beam search implementation.""" - - def __init__( - self, - scorers: Dict[str, ScorerInterface], - weights: Dict[str, float], - beam_size: int, - vocab_size: int, - sos: int, - eos: int, - token_list: List[str] = None, - pre_beam_ratio: float = 1.5, - pre_beam_score_key: str = None, - ): - """Initialize beam search. - - Args: - scorers (dict[str, ScorerInterface]): Dict of decoder modules - e.g., Decoder, CTCPrefixScorer, LM - The scorer will be ignored if it is `None` - weights (dict[str, float]): Dict of weights for each scorers - The scorer will be ignored if its weight is 0 - beam_size (int): The number of hypotheses kept during search - vocab_size (int): The number of vocabulary - sos (int): Start of sequence id - eos (int): End of sequence id - token_list (list[str]): List of tokens for debug log - pre_beam_score_key (str): key of scores to perform pre-beam search - pre_beam_ratio (float): beam size in the pre-beam search - will be `int(pre_beam_ratio * beam_size)` - - """ - super().__init__() - # set scorers - self.weights = weights - self.scorers = dict() - self.full_scorers = dict() - self.part_scorers = dict() - # this module dict is required for recursive cast - # `self.to(device, dtype)` in `recog.py` - self.nn_dict = torch.nn.ModuleDict() - for k, v in scorers.items(): - w = weights.get(k, 0) - if w == 0 or v is None: - continue - assert isinstance( - v, ScorerInterface - ), f"{k} ({type(v)}) does not implement ScorerInterface" - self.scorers[k] = v - if isinstance(v, PartialScorerInterface): - self.part_scorers[k] = v - else: - self.full_scorers[k] = v - if isinstance(v, torch.nn.Module): - self.nn_dict[k] = v - - # set configurations - self.sos = sos - self.eos = eos - self.token_list = token_list - self.pre_beam_size = int(pre_beam_ratio * beam_size) - self.beam_size = beam_size - self.n_vocab = vocab_size - if ( - pre_beam_score_key is not None - and pre_beam_score_key != "full" - and pre_beam_score_key not in self.full_scorers - ): - raise KeyError(f"{pre_beam_score_key} is not found in {self.full_scorers}") - self.pre_beam_score_key = pre_beam_score_key - self.do_pre_beam = ( - self.pre_beam_score_key is not None - and self.pre_beam_size < self.n_vocab - and len(self.part_scorers) > 0 - ) - - def init_hyp(self, x: torch.Tensor) -> List[Hypothesis]: - """Get an initial hypothesis data. - - Args: - x (torch.Tensor): The encoder output feature - - Returns: - Hypothesis: The initial hypothesis. - - """ - init_states = dict() - init_scores = dict() - for k, d in self.scorers.items(): - init_states[k] = d.init_state(x) - init_scores[k] = 0.0 - return [ - Hypothesis( - score=0.0, - scores=init_scores, - states=init_states, - yseq=torch.tensor([self.sos], device=x.device), - ) - ] - - @staticmethod - def append_token(xs: torch.Tensor, x: int) -> torch.Tensor: - """Append new token to prefix tokens. - - Args: - xs (torch.Tensor): The prefix token - x (int): The new token to append - - Returns: - torch.Tensor: New tensor contains: xs + [x] with xs.dtype and xs.device - - """ - x = torch.tensor([x], dtype=xs.dtype, device=xs.device) - return torch.cat((xs, x)) - - def score_full( - self, - hyp: Hypothesis, - x: torch.Tensor, - x_mask: torch.Tensor = None, - pre_acoustic_embeds: torch.Tensor = None, - ) -> Tuple[Dict[str, torch.Tensor], Dict[str, Any]]: - """Score new hypothesis by `self.full_scorers`. - - Args: - hyp (Hypothesis): Hypothesis with prefix tokens to score - x (torch.Tensor): Corresponding input feature - - Returns: - Tuple[Dict[str, torch.Tensor], Dict[str, Any]]: Tuple of - score dict of `hyp` that has string keys of `self.full_scorers` - and tensor score values of shape: `(self.n_vocab,)`, - and state dict that has string keys - and state values of `self.full_scorers` - - """ - scores = dict() - states = dict() - for k, d in self.full_scorers.items(): - scores[k], states[k] = d.score( - hyp.yseq, - hyp.states[k], - x, - x_mask=x_mask, - pre_acoustic_embeds=pre_acoustic_embeds, - ) - return scores, states - - def score_partial( - self, hyp: Hypothesis, ids: torch.Tensor, x: torch.Tensor - ) -> Tuple[Dict[str, torch.Tensor], Dict[str, Any]]: - """Score new hypothesis by `self.part_scorers`. - - Args: - hyp (Hypothesis): Hypothesis with prefix tokens to score - ids (torch.Tensor): 1D tensor of new partial tokens to score - x (torch.Tensor): Corresponding input feature - - Returns: - Tuple[Dict[str, torch.Tensor], Dict[str, Any]]: Tuple of - score dict of `hyp` that has string keys of `self.part_scorers` - and tensor score values of shape: `(len(ids),)`, - and state dict that has string keys - and state values of `self.part_scorers` - - """ - scores = dict() - states = dict() - for k, d in self.part_scorers.items(): - scores[k], states[k] = d.score_partial(hyp.yseq, ids, hyp.states[k], x) - return scores, states - - def beam( - self, weighted_scores: torch.Tensor, ids: torch.Tensor - ) -> Tuple[torch.Tensor, torch.Tensor]: - """Compute topk full token ids and partial token ids. - - Args: - weighted_scores (torch.Tensor): The weighted sum scores for each tokens. - Its shape is `(self.n_vocab,)`. - ids (torch.Tensor): The partial token ids to compute topk - - Returns: - Tuple[torch.Tensor, torch.Tensor]: - The topk full token ids and partial token ids. - Their shapes are `(self.beam_size,)` - - """ - # no pre beam performed - if weighted_scores.size(0) == ids.size(0): - top_ids = weighted_scores.topk(self.beam_size)[1] - return top_ids, top_ids - - # mask pruned in pre-beam not to select in topk - tmp = weighted_scores[ids] - weighted_scores[:] = -float("inf") - weighted_scores[ids] = tmp - top_ids = weighted_scores.topk(self.beam_size)[1] - local_ids = weighted_scores[ids].topk(self.beam_size)[1] - return top_ids, local_ids - - @staticmethod - def merge_scores( - prev_scores: Dict[str, float], - next_full_scores: Dict[str, torch.Tensor], - full_idx: int, - next_part_scores: Dict[str, torch.Tensor], - part_idx: int, - ) -> Dict[str, torch.Tensor]: - """Merge scores for new hypothesis. - - Args: - prev_scores (Dict[str, float]): - The previous hypothesis scores by `self.scorers` - next_full_scores (Dict[str, torch.Tensor]): scores by `self.full_scorers` - full_idx (int): The next token id for `next_full_scores` - next_part_scores (Dict[str, torch.Tensor]): - scores of partial tokens by `self.part_scorers` - part_idx (int): The new token id for `next_part_scores` - - Returns: - Dict[str, torch.Tensor]: The new score dict. - Its keys are names of `self.full_scorers` and `self.part_scorers`. - Its values are scalar tensors by the scorers. - - """ - new_scores = dict() - for k, v in next_full_scores.items(): - new_scores[k] = prev_scores[k] + v[full_idx] - for k, v in next_part_scores.items(): - new_scores[k] = prev_scores[k] + v[part_idx] - return new_scores - - def merge_states(self, states: Any, part_states: Any, part_idx: int) -> Any: - """Merge states for new hypothesis. - - Args: - states: states of `self.full_scorers` - part_states: states of `self.part_scorers` - part_idx (int): The new token id for `part_scores` - - Returns: - Dict[str, torch.Tensor]: The new score dict. - Its keys are names of `self.full_scorers` and `self.part_scorers`. - Its values are states of the scorers. - - """ - new_states = dict() - for k, v in states.items(): - new_states[k] = v - for k, d in self.part_scorers.items(): - new_states[k] = d.select_state(part_states[k], part_idx) - return new_states - - def search( - self, - running_hyps: List[Hypothesis], - x: torch.Tensor, - x_mask: torch.Tensor = None, - pre_acoustic_embeds: torch.Tensor = None, - ) -> List[Hypothesis]: - """Search new tokens for running hypotheses and encoded speech x. - - Args: - running_hyps (List[Hypothesis]): Running hypotheses on beam - x (torch.Tensor): Encoded speech feature (T, D) - - Returns: - List[Hypotheses]: Best sorted hypotheses - - """ - best_hyps = [] - part_ids = torch.arange(self.n_vocab, device=x.device) # no pre-beam - for hyp in running_hyps: - # scoring - weighted_scores = torch.zeros(self.n_vocab, dtype=x.dtype, device=x.device) - scores, states = self.score_full( - hyp, x, x_mask=x_mask, pre_acoustic_embeds=pre_acoustic_embeds - ) - for k in self.full_scorers: - weighted_scores += self.weights[k] * scores[k] - # partial scoring - if self.do_pre_beam: - pre_beam_scores = ( - weighted_scores - if self.pre_beam_score_key == "full" - else scores[self.pre_beam_score_key] - ) - part_ids = torch.topk(pre_beam_scores, self.pre_beam_size)[1] - part_scores, part_states = self.score_partial(hyp, part_ids, x) - for k in self.part_scorers: - weighted_scores[part_ids] += self.weights[k] * part_scores[k] - # add previous hyp score - weighted_scores += hyp.score - - # update hyps - for j, part_j in zip(*self.beam(weighted_scores, part_ids)): - # will be (2 x beam at most) - best_hyps.append( - Hypothesis( - score=weighted_scores[j], - yseq=self.append_token(hyp.yseq, j), - scores=self.merge_scores( - hyp.scores, scores, j, part_scores, part_j - ), - states=self.merge_states(states, part_states, part_j), - ) - ) - - # sort and prune 2 x beam -> beam - best_hyps = sorted(best_hyps, key=lambda x: x.score, reverse=True)[ - : min(len(best_hyps), self.beam_size) - ] - return best_hyps - - def forward( - self, - x: torch.Tensor, - scama_mask: torch.Tensor = None, - pre_acoustic_embeds: torch.Tensor = None, - maxlenratio: float = 0.0, - minlenratio: float = 0.0, - maxlen: int = None, - minlen: int = 0, - ) -> List[Hypothesis]: - """Perform beam search. - - Args: - x (torch.Tensor): Encoded speech feature (T, D) - maxlenratio (float): Input length ratio to obtain max output length. - If maxlenratio=0.0 (default), it uses a end-detect function - to automatically find maximum hypothesis lengths - If maxlenratio<0.0, its absolute value is interpreted - as a constant max output length. - minlenratio (float): Input length ratio to obtain min output length. - - Returns: - list[Hypothesis]: N-best decoding results - - """ - if maxlen is None: - # set length bounds - if maxlenratio == 0: - maxlen = x.shape[0] - elif maxlenratio < 0: - maxlen = -1 * int(maxlenratio) - else: - maxlen = max(1, int(maxlenratio * x.size(0))) - minlen = int(minlenratio * x.size(0)) - - logging.info("decoder input length: " + str(x.shape[0])) - logging.info("max output length: " + str(maxlen)) - logging.info("min output length: " + str(minlen)) - - # main loop of prefix search - running_hyps = self.init_hyp(x) - ended_hyps = [] - for i in range(maxlen): - logging.debug("position " + str(i)) - mask_enc = None - if scama_mask is not None: - token_num_predictor = scama_mask.size(1) - token_id_slice = min(i, token_num_predictor - 1) - mask_enc = scama_mask[:, token_id_slice : token_id_slice + 1, :] - # if mask_enc.size(1) == 0: - # mask_enc = scama_mask[:, -2:-1, :] - # # mask_enc = torch.zeros_like(mask_enc) - pre_acoustic_embeds_cur = None - if pre_acoustic_embeds is not None: - b, t, d = pre_acoustic_embeds.size() - pad = torch.zeros((b, 1, d), dtype=pre_acoustic_embeds.dtype).to( - device=pre_acoustic_embeds.device - ) - pre_acoustic_embeds = torch.cat((pre_acoustic_embeds, pad), dim=1) - token_id_slice = min(i, t) - pre_acoustic_embeds_cur = pre_acoustic_embeds[ - :, token_id_slice : token_id_slice + 1, : - ] - - best = self.search( - running_hyps, - x, - x_mask=mask_enc, - pre_acoustic_embeds=pre_acoustic_embeds_cur, - ) - # post process of one iteration - running_hyps = self.post_process(i, maxlen, maxlenratio, best, ended_hyps) - # end detection - if maxlenratio == 0.0 and end_detect([h.asdict() for h in ended_hyps], i): - logging.info(f"end detected at {i}") - break - if len(running_hyps) == 0: - logging.info("no hypothesis. Finish decoding.") - break - else: - logging.debug(f"remained hypotheses: {len(running_hyps)}") - - nbest_hyps = sorted(ended_hyps, key=lambda x: x.score, reverse=True) - # check the number of hypotheses reaching to eos - if len(nbest_hyps) == 0: - logging.warning( - "there is no N-best results, perform recognition " - "again with smaller minlenratio." - ) - return ( - [] - if minlenratio < 0.1 - else self.forward(x, maxlenratio, max(0.0, minlenratio - 0.1)) - ) - - # report the best result - for x in nbest_hyps: - yseq = "".join([self.token_list[x] for x in x.yseq]) - logging.debug( - "nbest: y: {}, yseq: {}, score: {}".format(x.yseq, yseq, x.score) - ) - best = nbest_hyps[0] - for k, v in best.scores.items(): - logging.info( - f"{v:6.2f} * {self.weights[k]:3} = {v * self.weights[k]:6.2f} for {k}" - ) - logging.info(f"total log probability: {best.score:.2f}") - logging.info(f"normalized log probability: {best.score / len(best.yseq):.2f}") - logging.info(f"total number of ended hypotheses: {len(nbest_hyps)}") - if self.token_list is not None: - logging.info( - "best hypo: " - + "".join([self.token_list[x] for x in best.yseq[1:-1]]) - + "\n" - ) - return nbest_hyps - - def post_process( - self, - i: int, - maxlen: int, - maxlenratio: float, - running_hyps: List[Hypothesis], - ended_hyps: List[Hypothesis], - ) -> List[Hypothesis]: - """Perform post-processing of beam search iterations. - - Args: - i (int): The length of hypothesis tokens. - maxlen (int): The maximum length of tokens in beam search. - maxlenratio (int): The maximum length ratio in beam search. - running_hyps (List[Hypothesis]): The running hypotheses in beam search. - ended_hyps (List[Hypothesis]): The ended hypotheses in beam search. - - Returns: - List[Hypothesis]: The new running hypotheses. - - """ - logging.debug(f"the number of running hypotheses: {len(running_hyps)}") - if self.token_list is not None: - logging.debug( - "best hypo: " - + "".join([self.token_list[x] for x in running_hyps[0].yseq[1:]]) - ) - # add eos in the final loop to avoid that there are no ended hyps - if i == maxlen - 1: - logging.info("adding in the last position in the loop") - running_hyps = [ - h._replace(yseq=self.append_token(h.yseq, self.eos)) - for h in running_hyps - ] - - # add ended hypotheses to a final list, and removed them from current hypotheses - # (this will be a problem, number of hyps < beam) - remained_hyps = [] - for hyp in running_hyps: - if hyp.yseq[-1] == self.eos: - # e.g., Word LM needs to add final score - for k, d in chain(self.full_scorers.items(), self.part_scorers.items()): - s = d.final_score(hyp.states[k]) - hyp.scores[k] += s - hyp = hyp._replace(score=hyp.score + self.weights[k] * s) - ended_hyps.append(hyp) - else: - remained_hyps.append(hyp) - return remained_hyps diff --git a/funasr_detach/models/uniasr/model.py b/funasr_detach/models/uniasr/model.py deleted file mode 100644 index 233c871596ba4dea3b85bb0e01d95960ed2ff593..0000000000000000000000000000000000000000 --- a/funasr_detach/models/uniasr/model.py +++ /dev/null @@ -1,1090 +0,0 @@ -#!/usr/bin/env python3 -# -*- encoding: utf-8 -*- -# Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved. -# MIT License (https://opensource.org/licenses/MIT) - -import time -import torch -import logging -from torch.cuda.amp import autocast -from typing import Union, Dict, List, Tuple, Optional - -from funasr_detach.register import tables -from funasr_detach.models.ctc.ctc import CTC -from funasr_detach.utils import postprocess_utils -from funasr_detach.metrics.compute_acc import th_accuracy -from funasr_detach.utils.datadir_writer import DatadirWriter -from funasr_detach.models.paraformer.cif_predictor import mae_loss -from funasr_detach.train_utils.device_funcs import force_gatherable -from funasr_detach.losses.label_smoothing_loss import LabelSmoothingLoss -from funasr_detach.models.transformer.utils.add_sos_eos import add_sos_eos -from funasr_detach.models.transformer.utils.nets_utils import make_pad_mask, pad_list -from funasr_detach.utils.load_utils import load_audio_text_image_video, extract_fbank -from funasr_detach.models.scama.utils import sequence_mask - - -@tables.register("model_classes", "UniASR") -class UniASR(torch.nn.Module): - """ - Author: Speech Lab of DAMO Academy, Alibaba Group - """ - - def __init__( - self, - specaug: str = None, - specaug_conf: dict = None, - normalize: str = None, - normalize_conf: dict = None, - encoder: str = None, - encoder_conf: dict = None, - encoder2: str = None, - encoder2_conf: dict = None, - decoder: str = None, - decoder_conf: dict = None, - decoder2: str = None, - decoder2_conf: dict = None, - predictor: str = None, - predictor_conf: dict = None, - predictor_bias: int = 0, - predictor_weight: float = 0.0, - predictor2: str = None, - predictor2_conf: dict = None, - predictor2_bias: int = 0, - predictor2_weight: float = 0.0, - ctc: str = None, - ctc_conf: dict = None, - ctc_weight: float = 0.5, - ctc2: str = None, - ctc2_conf: dict = None, - ctc2_weight: float = 0.5, - decoder_attention_chunk_type: str = "chunk", - decoder_attention_chunk_type2: str = "chunk", - stride_conv=None, - stride_conv_conf: dict = None, - loss_weight_model1: float = 0.5, - input_size: int = 80, - vocab_size: int = -1, - ignore_id: int = -1, - blank_id: int = 0, - sos: int = 1, - eos: int = 2, - lsm_weight: float = 0.0, - length_normalized_loss: bool = False, - share_embedding: bool = False, - **kwargs, - ): - super().__init__() - - if specaug is not None: - specaug_class = tables.specaug_classes.get(specaug) - specaug = specaug_class(**specaug_conf) - if normalize is not None: - normalize_class = tables.normalize_classes.get(normalize) - normalize = normalize_class(**normalize_conf) - - encoder_class = tables.encoder_classes.get(encoder) - encoder = encoder_class(input_size=input_size, **encoder_conf) - encoder_output_size = encoder.output_size() - - decoder_class = tables.decoder_classes.get(decoder) - decoder = decoder_class( - vocab_size=vocab_size, - encoder_output_size=encoder_output_size, - **decoder_conf, - ) - predictor_class = tables.predictor_classes.get(predictor) - predictor = predictor_class(**predictor_conf) - - from funasr_detach.models.transformer.utils.subsampling import Conv1dSubsampling - - stride_conv = Conv1dSubsampling( - **stride_conv_conf, - idim=input_size + encoder_output_size, - odim=input_size + encoder_output_size, - ) - stride_conv_output_size = stride_conv.output_size() - - encoder_class = tables.encoder_classes.get(encoder2) - encoder2 = encoder_class(input_size=stride_conv_output_size, **encoder2_conf) - encoder2_output_size = encoder2.output_size() - - decoder_class = tables.decoder_classes.get(decoder2) - decoder2 = decoder_class( - vocab_size=vocab_size, - encoder_output_size=encoder2_output_size, - **decoder2_conf, - ) - predictor_class = tables.predictor_classes.get(predictor2) - predictor2 = predictor_class(**predictor2_conf) - - self.blank_id = blank_id - self.sos = sos - self.eos = eos - self.vocab_size = vocab_size - self.ignore_id = ignore_id - self.ctc_weight = ctc_weight - self.ctc2_weight = ctc2_weight - - self.specaug = specaug - self.normalize = normalize - - self.encoder = encoder - - self.error_calculator = None - - self.decoder = decoder - self.ctc = None - self.ctc2 = None - - self.criterion_att = LabelSmoothingLoss( - size=vocab_size, - padding_idx=ignore_id, - smoothing=lsm_weight, - normalize_length=length_normalized_loss, - ) - - self.predictor = predictor - self.predictor_weight = predictor_weight - self.criterion_pre = mae_loss(normalize_length=length_normalized_loss) - self.encoder1_encoder2_joint_training = kwargs.get( - "encoder1_encoder2_joint_training", True - ) - - if self.encoder.overlap_chunk_cls is not None: - from funasr_detach.models.scama.chunk_utilis import ( - build_scama_mask_for_cross_attention_decoder, - ) - - self.build_scama_mask_for_cross_attention_decoder_fn = ( - build_scama_mask_for_cross_attention_decoder - ) - self.decoder_attention_chunk_type = decoder_attention_chunk_type - - self.encoder2 = encoder2 - self.decoder2 = decoder2 - self.ctc2_weight = ctc2_weight - - self.predictor2 = predictor2 - self.predictor2_weight = predictor2_weight - self.decoder_attention_chunk_type2 = decoder_attention_chunk_type2 - self.stride_conv = stride_conv - self.loss_weight_model1 = loss_weight_model1 - if self.encoder2.overlap_chunk_cls is not None: - from funasr_detach.models.scama.chunk_utilis import ( - build_scama_mask_for_cross_attention_decoder, - ) - - self.build_scama_mask_for_cross_attention_decoder_fn2 = ( - build_scama_mask_for_cross_attention_decoder - ) - self.decoder_attention_chunk_type2 = decoder_attention_chunk_type2 - - self.length_normalized_loss = length_normalized_loss - self.enable_maas_finetune = kwargs.get("enable_maas_finetune", False) - self.freeze_encoder2 = kwargs.get("freeze_encoder2", False) - self.beam_search = None - - def forward( - self, - speech: torch.Tensor, - speech_lengths: torch.Tensor, - text: torch.Tensor, - text_lengths: torch.Tensor, - **kwargs, - ) -> Tuple[torch.Tensor, Dict[str, torch.Tensor], torch.Tensor]: - """Frontend + Encoder + Decoder + Calc loss - Args: - speech: (Batch, Length, ...) - speech_lengths: (Batch, ) - text: (Batch, Length) - text_lengths: (Batch,) - """ - decoding_ind = kwargs.get("decoding_ind", None) - if len(text_lengths.size()) > 1: - text_lengths = text_lengths[:, 0] - if len(speech_lengths.size()) > 1: - speech_lengths = speech_lengths[:, 0] - - batch_size = speech.shape[0] - - ind = self.encoder.overlap_chunk_cls.random_choice(self.training, decoding_ind) - # 1. Encoder - if self.enable_maas_finetune: - with torch.no_grad(): - speech_raw, encoder_out, encoder_out_lens = self.encode( - speech, speech_lengths, ind=ind - ) - else: - speech_raw, encoder_out, encoder_out_lens = self.encode( - speech, speech_lengths, ind=ind - ) - - loss_att, acc_att, cer_att, wer_att = None, None, None, None - loss_ctc, cer_ctc = None, None - stats = dict() - loss_pre = None - loss, loss1, loss2 = 0.0, 0.0, 0.0 - - if self.loss_weight_model1 > 0.0: - ## model1 - # 1. CTC branch - if self.enable_maas_finetune: - with torch.no_grad(): - - loss_att, acc_att, cer_att, wer_att, loss_pre = ( - self._calc_att_predictor_loss( - encoder_out, encoder_out_lens, text, text_lengths - ) - ) - - loss = loss_att + loss_pre * self.predictor_weight - - # Collect Attn branch stats - stats["loss_att"] = ( - loss_att.detach() if loss_att is not None else None - ) - stats["acc"] = acc_att - stats["cer"] = cer_att - stats["wer"] = wer_att - stats["loss_pre"] = ( - loss_pre.detach().cpu() if loss_pre is not None else None - ) - else: - - loss_att, acc_att, cer_att, wer_att, loss_pre = ( - self._calc_att_predictor_loss( - encoder_out, encoder_out_lens, text, text_lengths - ) - ) - - loss = loss_att + loss_pre * self.predictor_weight - - # Collect Attn branch stats - stats["loss_att"] = loss_att.detach() if loss_att is not None else None - stats["acc"] = acc_att - stats["cer"] = cer_att - stats["wer"] = wer_att - stats["loss_pre"] = ( - loss_pre.detach().cpu() if loss_pre is not None else None - ) - - loss1 = loss - - if self.loss_weight_model1 < 1.0: - ## model2 - - # encoder2 - if self.freeze_encoder2: - with torch.no_grad(): - encoder_out, encoder_out_lens = self.encode2( - encoder_out, - encoder_out_lens, - speech_raw, - speech_lengths, - ind=ind, - ) - else: - encoder_out, encoder_out_lens = self.encode2( - encoder_out, encoder_out_lens, speech_raw, speech_lengths, ind=ind - ) - - intermediate_outs = None - if isinstance(encoder_out, tuple): - intermediate_outs = encoder_out[1] - encoder_out = encoder_out[0] - - loss_att, acc_att, cer_att, wer_att, loss_pre = ( - self._calc_att_predictor_loss2( - encoder_out, encoder_out_lens, text, text_lengths - ) - ) - - loss = loss_att + loss_pre * self.predictor2_weight - - # Collect Attn branch stats - stats["loss_att2"] = loss_att.detach() if loss_att is not None else None - stats["acc2"] = acc_att - stats["cer2"] = cer_att - stats["wer2"] = wer_att - stats["loss_pre2"] = ( - loss_pre.detach().cpu() if loss_pre is not None else None - ) - - loss2 = loss - - loss = loss1 * self.loss_weight_model1 + loss2 * (1 - self.loss_weight_model1) - stats["loss1"] = torch.clone(loss1.detach()) - stats["loss2"] = torch.clone(loss2.detach()) - stats["loss"] = torch.clone(loss.detach()) - # force_gatherable: to-device and to-tensor if scalar for DataParallel - if self.length_normalized_loss: - batch_size = int((text_lengths + 1).sum()) - - loss, stats, weight = force_gatherable((loss, stats, batch_size), loss.device) - return loss, stats, weight - - def collect_feats( - self, - speech: torch.Tensor, - speech_lengths: torch.Tensor, - text: torch.Tensor, - text_lengths: torch.Tensor, - ) -> Dict[str, torch.Tensor]: - if self.extract_feats_in_collect_stats: - feats, feats_lengths = self._extract_feats(speech, speech_lengths) - else: - # Generate dummy stats if extract_feats_in_collect_stats is False - logging.warning( - "Generating dummy stats for feats and feats_lengths, " - "because encoder_conf.extract_feats_in_collect_stats is " - f"{self.extract_feats_in_collect_stats}" - ) - feats, feats_lengths = speech, speech_lengths - return {"feats": feats, "feats_lengths": feats_lengths} - - def encode( - self, - speech: torch.Tensor, - speech_lengths: torch.Tensor, - **kwargs, - ): - """Frontend + Encoder. Note that this method is used by asr_inference.py - Args: - speech: (Batch, Length, ...) - speech_lengths: (Batch, ) - """ - ind = kwargs.get("ind", 0) - with autocast(False): - # Data augmentation - if self.specaug is not None and self.training: - speech, speech_lengths = self.specaug(speech, speech_lengths) - - # Normalization for feature: e.g. Global-CMVN, Utterance-CMVN - if self.normalize is not None: - speech, speech_lengths = self.normalize(speech, speech_lengths) - - speech_raw = speech.clone().to(speech.device) - - # 4. Forward encoder - encoder_out, encoder_out_lens, _ = self.encoder(speech, speech_lengths, ind=ind) - if isinstance(encoder_out, tuple): - encoder_out = encoder_out[0] - - return speech_raw, encoder_out, encoder_out_lens - - def encode2( - self, - encoder_out: torch.Tensor, - encoder_out_lens: torch.Tensor, - speech: torch.Tensor, - speech_lengths: torch.Tensor, - **kwargs, - ): - """Frontend + Encoder. Note that this method is used by asr_inference.py - Args: - speech: (Batch, Length, ...) - speech_lengths: (Batch, ) - """ - - ind = kwargs.get("ind", 0) - encoder_out_rm, encoder_out_lens_rm = ( - self.encoder.overlap_chunk_cls.remove_chunk( - encoder_out, - encoder_out_lens, - chunk_outs=None, - ) - ) - # residual_input - encoder_out = torch.cat((speech, encoder_out_rm), dim=-1) - encoder_out_lens = encoder_out_lens_rm - if self.stride_conv is not None: - speech, speech_lengths = self.stride_conv(encoder_out, encoder_out_lens) - if not self.encoder1_encoder2_joint_training: - speech = speech.detach() - speech_lengths = speech_lengths.detach() - # 4. Forward encoder - # feats: (Batch, Length, Dim) - # -> encoder_out: (Batch, Length2, Dim2) - - encoder_out, encoder_out_lens, _ = self.encoder2( - speech, speech_lengths, ind=ind - ) - if isinstance(encoder_out, tuple): - encoder_out = encoder_out[0] - - return encoder_out, encoder_out_lens - - def nll( - self, - encoder_out: torch.Tensor, - encoder_out_lens: torch.Tensor, - ys_pad: torch.Tensor, - ys_pad_lens: torch.Tensor, - ) -> torch.Tensor: - """Compute negative log likelihood(nll) from transformer-decoder - Normally, this function is called in batchify_nll. - Args: - encoder_out: (Batch, Length, Dim) - encoder_out_lens: (Batch,) - ys_pad: (Batch, Length) - ys_pad_lens: (Batch,) - """ - ys_in_pad, ys_out_pad = add_sos_eos(ys_pad, self.sos, self.eos, self.ignore_id) - ys_in_lens = ys_pad_lens + 1 - - # 1. Forward decoder - decoder_out, _ = self.decoder( - encoder_out, encoder_out_lens, ys_in_pad, ys_in_lens - ) # [batch, seqlen, dim] - batch_size = decoder_out.size(0) - decoder_num_class = decoder_out.size(2) - # nll: negative log-likelihood - nll = torch.nn.functional.cross_entropy( - decoder_out.view(-1, decoder_num_class), - ys_out_pad.view(-1), - ignore_index=self.ignore_id, - reduction="none", - ) - nll = nll.view(batch_size, -1) - nll = nll.sum(dim=1) - assert nll.size(0) == batch_size - return nll - - def batchify_nll( - self, - encoder_out: torch.Tensor, - encoder_out_lens: torch.Tensor, - ys_pad: torch.Tensor, - ys_pad_lens: torch.Tensor, - batch_size: int = 100, - ): - """Compute negative log likelihood(nll) from transformer-decoder - To avoid OOM, this fuction seperate the input into batches. - Then call nll for each batch and combine and return results. - Args: - encoder_out: (Batch, Length, Dim) - encoder_out_lens: (Batch,) - ys_pad: (Batch, Length) - ys_pad_lens: (Batch,) - batch_size: int, samples each batch contain when computing nll, - you may change this to avoid OOM or increase - GPU memory usage - """ - total_num = encoder_out.size(0) - if total_num <= batch_size: - nll = self.nll(encoder_out, encoder_out_lens, ys_pad, ys_pad_lens) - else: - nll = [] - start_idx = 0 - while True: - end_idx = min(start_idx + batch_size, total_num) - batch_encoder_out = encoder_out[start_idx:end_idx, :, :] - batch_encoder_out_lens = encoder_out_lens[start_idx:end_idx] - batch_ys_pad = ys_pad[start_idx:end_idx, :] - batch_ys_pad_lens = ys_pad_lens[start_idx:end_idx] - batch_nll = self.nll( - batch_encoder_out, - batch_encoder_out_lens, - batch_ys_pad, - batch_ys_pad_lens, - ) - nll.append(batch_nll) - start_idx = end_idx - if start_idx == total_num: - break - nll = torch.cat(nll) - assert nll.size(0) == total_num - return nll - - def _calc_att_loss( - self, - encoder_out: torch.Tensor, - encoder_out_lens: torch.Tensor, - ys_pad: torch.Tensor, - ys_pad_lens: torch.Tensor, - ): - ys_in_pad, ys_out_pad = add_sos_eos(ys_pad, self.sos, self.eos, self.ignore_id) - ys_in_lens = ys_pad_lens + 1 - - # 1. Forward decoder - decoder_out, _ = self.decoder( - encoder_out, encoder_out_lens, ys_in_pad, ys_in_lens - ) - - # 2. Compute attention loss - loss_att = self.criterion_att(decoder_out, ys_out_pad) - acc_att = th_accuracy( - decoder_out.view(-1, self.vocab_size), - ys_out_pad, - ignore_label=self.ignore_id, - ) - - # Compute cer/wer using attention-decoder - if self.training or self.error_calculator is None: - cer_att, wer_att = None, None - else: - ys_hat = decoder_out.argmax(dim=-1) - cer_att, wer_att = self.error_calculator(ys_hat.cpu(), ys_pad.cpu()) - - return loss_att, acc_att, cer_att, wer_att - - def _calc_att_predictor_loss( - self, - encoder_out: torch.Tensor, - encoder_out_lens: torch.Tensor, - ys_pad: torch.Tensor, - ys_pad_lens: torch.Tensor, - ): - ys_in_pad, ys_out_pad = add_sos_eos(ys_pad, self.sos, self.eos, self.ignore_id) - ys_in_lens = ys_pad_lens + 1 - - encoder_out_mask = sequence_mask( - encoder_out_lens, - maxlen=encoder_out.size(1), - dtype=encoder_out.dtype, - device=encoder_out.device, - )[:, None, :] - mask_chunk_predictor = None - if self.encoder.overlap_chunk_cls is not None: - mask_chunk_predictor = ( - self.encoder.overlap_chunk_cls.get_mask_chunk_predictor( - None, device=encoder_out.device, batch_size=encoder_out.size(0) - ) - ) - mask_shfit_chunk = self.encoder.overlap_chunk_cls.get_mask_shfit_chunk( - None, device=encoder_out.device, batch_size=encoder_out.size(0) - ) - encoder_out = encoder_out * mask_shfit_chunk - pre_acoustic_embeds, pre_token_length, pre_alphas, _ = self.predictor( - encoder_out, - ys_out_pad, - encoder_out_mask, - ignore_id=self.ignore_id, - mask_chunk_predictor=mask_chunk_predictor, - target_label_length=ys_in_lens, - ) - predictor_alignments, predictor_alignments_len = ( - self.predictor.gen_frame_alignments(pre_alphas, encoder_out_lens) - ) - - scama_mask = None - if ( - self.encoder.overlap_chunk_cls is not None - and self.decoder_attention_chunk_type == "chunk" - ): - encoder_chunk_size = self.encoder.overlap_chunk_cls.chunk_size_pad_shift_cur - attention_chunk_center_bias = 0 - attention_chunk_size = encoder_chunk_size - decoder_att_look_back_factor = ( - self.encoder.overlap_chunk_cls.decoder_att_look_back_factor_cur - ) - mask_shift_att_chunk_decoder = ( - self.encoder.overlap_chunk_cls.get_mask_shift_att_chunk_decoder( - None, device=encoder_out.device, batch_size=encoder_out.size(0) - ) - ) - scama_mask = self.build_scama_mask_for_cross_attention_decoder_fn( - predictor_alignments=predictor_alignments, - encoder_sequence_length=encoder_out_lens, - chunk_size=1, - encoder_chunk_size=encoder_chunk_size, - attention_chunk_center_bias=attention_chunk_center_bias, - attention_chunk_size=attention_chunk_size, - attention_chunk_type=self.decoder_attention_chunk_type, - step=None, - predictor_mask_chunk_hopping=mask_chunk_predictor, - decoder_att_look_back_factor=decoder_att_look_back_factor, - mask_shift_att_chunk_decoder=mask_shift_att_chunk_decoder, - target_length=ys_in_lens, - is_training=self.training, - ) - elif self.encoder.overlap_chunk_cls is not None: - encoder_out, encoder_out_lens = self.encoder.overlap_chunk_cls.remove_chunk( - encoder_out, encoder_out_lens, chunk_outs=None - ) - # try: - # 1. Forward decoder - decoder_out, _ = self.decoder( - encoder_out, - encoder_out_lens, - ys_in_pad, - ys_in_lens, - chunk_mask=scama_mask, - pre_acoustic_embeds=pre_acoustic_embeds, - ) - - # 2. Compute attention loss - loss_att = self.criterion_att(decoder_out, ys_out_pad) - acc_att = th_accuracy( - decoder_out.view(-1, self.vocab_size), - ys_out_pad, - ignore_label=self.ignore_id, - ) - # predictor loss - loss_pre = self.criterion_pre( - ys_in_lens.type_as(pre_token_length), pre_token_length - ) - # Compute cer/wer using attention-decoder - if self.training or self.error_calculator is None: - cer_att, wer_att = None, None - else: - ys_hat = decoder_out.argmax(dim=-1) - cer_att, wer_att = self.error_calculator(ys_hat.cpu(), ys_pad.cpu()) - - return loss_att, acc_att, cer_att, wer_att, loss_pre - - def _calc_att_predictor_loss2( - self, - encoder_out: torch.Tensor, - encoder_out_lens: torch.Tensor, - ys_pad: torch.Tensor, - ys_pad_lens: torch.Tensor, - ): - ys_in_pad, ys_out_pad = add_sos_eos(ys_pad, self.sos, self.eos, self.ignore_id) - ys_in_lens = ys_pad_lens + 1 - - encoder_out_mask = sequence_mask( - encoder_out_lens, - maxlen=encoder_out.size(1), - dtype=encoder_out.dtype, - device=encoder_out.device, - )[:, None, :] - mask_chunk_predictor = None - if self.encoder2.overlap_chunk_cls is not None: - mask_chunk_predictor = ( - self.encoder2.overlap_chunk_cls.get_mask_chunk_predictor( - None, device=encoder_out.device, batch_size=encoder_out.size(0) - ) - ) - mask_shfit_chunk = self.encoder2.overlap_chunk_cls.get_mask_shfit_chunk( - None, device=encoder_out.device, batch_size=encoder_out.size(0) - ) - encoder_out = encoder_out * mask_shfit_chunk - pre_acoustic_embeds, pre_token_length, pre_alphas, _ = self.predictor2( - encoder_out, - ys_out_pad, - encoder_out_mask, - ignore_id=self.ignore_id, - mask_chunk_predictor=mask_chunk_predictor, - target_label_length=ys_in_lens, - ) - predictor_alignments, predictor_alignments_len = ( - self.predictor2.gen_frame_alignments(pre_alphas, encoder_out_lens) - ) - - scama_mask = None - if ( - self.encoder2.overlap_chunk_cls is not None - and self.decoder_attention_chunk_type2 == "chunk" - ): - encoder_chunk_size = ( - self.encoder2.overlap_chunk_cls.chunk_size_pad_shift_cur - ) - attention_chunk_center_bias = 0 - attention_chunk_size = encoder_chunk_size - decoder_att_look_back_factor = ( - self.encoder2.overlap_chunk_cls.decoder_att_look_back_factor_cur - ) - mask_shift_att_chunk_decoder = ( - self.encoder2.overlap_chunk_cls.get_mask_shift_att_chunk_decoder( - None, device=encoder_out.device, batch_size=encoder_out.size(0) - ) - ) - scama_mask = self.build_scama_mask_for_cross_attention_decoder_fn2( - predictor_alignments=predictor_alignments, - encoder_sequence_length=encoder_out_lens, - chunk_size=1, - encoder_chunk_size=encoder_chunk_size, - attention_chunk_center_bias=attention_chunk_center_bias, - attention_chunk_size=attention_chunk_size, - attention_chunk_type=self.decoder_attention_chunk_type2, - step=None, - predictor_mask_chunk_hopping=mask_chunk_predictor, - decoder_att_look_back_factor=decoder_att_look_back_factor, - mask_shift_att_chunk_decoder=mask_shift_att_chunk_decoder, - target_length=ys_in_lens, - is_training=self.training, - ) - elif self.encoder2.overlap_chunk_cls is not None: - encoder_out, encoder_out_lens = ( - self.encoder2.overlap_chunk_cls.remove_chunk( - encoder_out, encoder_out_lens, chunk_outs=None - ) - ) - # try: - # 1. Forward decoder - decoder_out, _ = self.decoder2( - encoder_out, - encoder_out_lens, - ys_in_pad, - ys_in_lens, - chunk_mask=scama_mask, - pre_acoustic_embeds=pre_acoustic_embeds, - ) - - # 2. Compute attention loss - loss_att = self.criterion_att(decoder_out, ys_out_pad) - acc_att = th_accuracy( - decoder_out.view(-1, self.vocab_size), - ys_out_pad, - ignore_label=self.ignore_id, - ) - # predictor loss - loss_pre = self.criterion_pre( - ys_in_lens.type_as(pre_token_length), pre_token_length - ) - # Compute cer/wer using attention-decoder - if self.training or self.error_calculator is None: - cer_att, wer_att = None, None - else: - ys_hat = decoder_out.argmax(dim=-1) - cer_att, wer_att = self.error_calculator(ys_hat.cpu(), ys_pad.cpu()) - - return loss_att, acc_att, cer_att, wer_att, loss_pre - - def calc_predictor_mask( - self, - encoder_out: torch.Tensor, - encoder_out_lens: torch.Tensor, - ys_pad: torch.Tensor = None, - ys_pad_lens: torch.Tensor = None, - ): - # ys_in_pad, ys_out_pad = add_sos_eos(ys_pad, self.sos, self.eos, self.ignore_id) - # ys_in_lens = ys_pad_lens + 1 - ys_out_pad, ys_in_lens = None, None - - encoder_out_mask = sequence_mask( - encoder_out_lens, - maxlen=encoder_out.size(1), - dtype=encoder_out.dtype, - device=encoder_out.device, - )[:, None, :] - mask_chunk_predictor = None - if self.encoder.overlap_chunk_cls is not None: - mask_chunk_predictor = ( - self.encoder.overlap_chunk_cls.get_mask_chunk_predictor( - None, device=encoder_out.device, batch_size=encoder_out.size(0) - ) - ) - mask_shfit_chunk = self.encoder.overlap_chunk_cls.get_mask_shfit_chunk( - None, device=encoder_out.device, batch_size=encoder_out.size(0) - ) - encoder_out = encoder_out * mask_shfit_chunk - pre_acoustic_embeds, pre_token_length, pre_alphas, _ = self.predictor( - encoder_out, - ys_out_pad, - encoder_out_mask, - ignore_id=self.ignore_id, - mask_chunk_predictor=mask_chunk_predictor, - target_label_length=ys_in_lens, - ) - predictor_alignments, predictor_alignments_len = ( - self.predictor.gen_frame_alignments(pre_alphas, encoder_out_lens) - ) - - scama_mask = None - if ( - self.encoder.overlap_chunk_cls is not None - and self.decoder_attention_chunk_type == "chunk" - ): - encoder_chunk_size = self.encoder.overlap_chunk_cls.chunk_size_pad_shift_cur - attention_chunk_center_bias = 0 - attention_chunk_size = encoder_chunk_size - decoder_att_look_back_factor = ( - self.encoder.overlap_chunk_cls.decoder_att_look_back_factor_cur - ) - mask_shift_att_chunk_decoder = ( - self.encoder.overlap_chunk_cls.get_mask_shift_att_chunk_decoder( - None, device=encoder_out.device, batch_size=encoder_out.size(0) - ) - ) - scama_mask = self.build_scama_mask_for_cross_attention_decoder_fn( - predictor_alignments=predictor_alignments, - encoder_sequence_length=encoder_out_lens, - chunk_size=1, - encoder_chunk_size=encoder_chunk_size, - attention_chunk_center_bias=attention_chunk_center_bias, - attention_chunk_size=attention_chunk_size, - attention_chunk_type=self.decoder_attention_chunk_type, - step=None, - predictor_mask_chunk_hopping=mask_chunk_predictor, - decoder_att_look_back_factor=decoder_att_look_back_factor, - mask_shift_att_chunk_decoder=mask_shift_att_chunk_decoder, - target_length=ys_in_lens, - is_training=self.training, - ) - elif self.encoder.overlap_chunk_cls is not None: - encoder_out, encoder_out_lens = self.encoder.overlap_chunk_cls.remove_chunk( - encoder_out, encoder_out_lens, chunk_outs=None - ) - - return ( - pre_acoustic_embeds, - pre_token_length, - predictor_alignments, - predictor_alignments_len, - scama_mask, - ) - - def calc_predictor_mask2( - self, - encoder_out: torch.Tensor, - encoder_out_lens: torch.Tensor, - ys_pad: torch.Tensor = None, - ys_pad_lens: torch.Tensor = None, - ): - # ys_in_pad, ys_out_pad = add_sos_eos(ys_pad, self.sos, self.eos, self.ignore_id) - # ys_in_lens = ys_pad_lens + 1 - ys_out_pad, ys_in_lens = None, None - - encoder_out_mask = sequence_mask( - encoder_out_lens, - maxlen=encoder_out.size(1), - dtype=encoder_out.dtype, - device=encoder_out.device, - )[:, None, :] - mask_chunk_predictor = None - if self.encoder2.overlap_chunk_cls is not None: - mask_chunk_predictor = ( - self.encoder2.overlap_chunk_cls.get_mask_chunk_predictor( - None, device=encoder_out.device, batch_size=encoder_out.size(0) - ) - ) - mask_shfit_chunk = self.encoder2.overlap_chunk_cls.get_mask_shfit_chunk( - None, device=encoder_out.device, batch_size=encoder_out.size(0) - ) - encoder_out = encoder_out * mask_shfit_chunk - pre_acoustic_embeds, pre_token_length, pre_alphas, _ = self.predictor2( - encoder_out, - ys_out_pad, - encoder_out_mask, - ignore_id=self.ignore_id, - mask_chunk_predictor=mask_chunk_predictor, - target_label_length=ys_in_lens, - ) - predictor_alignments, predictor_alignments_len = ( - self.predictor2.gen_frame_alignments(pre_alphas, encoder_out_lens) - ) - - scama_mask = None - if ( - self.encoder2.overlap_chunk_cls is not None - and self.decoder_attention_chunk_type2 == "chunk" - ): - encoder_chunk_size = ( - self.encoder2.overlap_chunk_cls.chunk_size_pad_shift_cur - ) - attention_chunk_center_bias = 0 - attention_chunk_size = encoder_chunk_size - decoder_att_look_back_factor = ( - self.encoder2.overlap_chunk_cls.decoder_att_look_back_factor_cur - ) - mask_shift_att_chunk_decoder = ( - self.encoder2.overlap_chunk_cls.get_mask_shift_att_chunk_decoder( - None, device=encoder_out.device, batch_size=encoder_out.size(0) - ) - ) - scama_mask = self.build_scama_mask_for_cross_attention_decoder_fn2( - predictor_alignments=predictor_alignments, - encoder_sequence_length=encoder_out_lens, - chunk_size=1, - encoder_chunk_size=encoder_chunk_size, - attention_chunk_center_bias=attention_chunk_center_bias, - attention_chunk_size=attention_chunk_size, - attention_chunk_type=self.decoder_attention_chunk_type2, - step=None, - predictor_mask_chunk_hopping=mask_chunk_predictor, - decoder_att_look_back_factor=decoder_att_look_back_factor, - mask_shift_att_chunk_decoder=mask_shift_att_chunk_decoder, - target_length=ys_in_lens, - is_training=self.training, - ) - elif self.encoder2.overlap_chunk_cls is not None: - encoder_out, encoder_out_lens = ( - self.encoder2.overlap_chunk_cls.remove_chunk( - encoder_out, encoder_out_lens, chunk_outs=None - ) - ) - - return ( - pre_acoustic_embeds, - pre_token_length, - predictor_alignments, - predictor_alignments_len, - scama_mask, - ) - - def init_beam_search( - self, - **kwargs, - ): - from funasr_detach.models.uniasr.beam_search import BeamSearchScama - from funasr_detach.models.transformer.scorers.ctc import CTCPrefixScorer - from funasr_detach.models.transformer.scorers.length_bonus import LengthBonus - - decoding_mode = kwargs.get("decoding_mode", "model1") - if decoding_mode == "model1": - decoder = self.decoder - else: - decoder = self.decoder2 - # 1. Build ASR model - scorers = {} - - if self.ctc != None: - ctc = CTCPrefixScorer(ctc=self.ctc, eos=self.eos) - scorers.update(ctc=ctc) - token_list = kwargs.get("token_list") - scorers.update( - decoder=decoder, - length_bonus=LengthBonus(len(token_list)), - ) - - # 3. Build ngram model - # ngram is not supported now - ngram = None - scorers["ngram"] = ngram - - weights = dict( - decoder=1.0 - kwargs.get("decoding_ctc_weight", 0.0), - ctc=kwargs.get("decoding_ctc_weight", 0.0), - lm=kwargs.get("lm_weight", 0.0), - ngram=kwargs.get("ngram_weight", 0.0), - length_bonus=kwargs.get("penalty", 0.0), - ) - beam_search = BeamSearchScama( - beam_size=kwargs.get("beam_size", 5), - weights=weights, - scorers=scorers, - sos=self.sos, - eos=self.eos, - vocab_size=len(token_list), - token_list=token_list, - pre_beam_score_key=None if self.ctc_weight == 1.0 else "full", - ) - - self.beam_search = beam_search - - def inference( - self, - data_in, - data_lengths=None, - key: list = None, - tokenizer=None, - frontend=None, - **kwargs, - ): - - decoding_model = kwargs.get("decoding_model", "normal") - token_num_relax = kwargs.get("token_num_relax", 5) - if decoding_model == "fast": - decoding_ind = 0 - decoding_mode = "model1" - elif decoding_model == "offline": - decoding_ind = 1 - decoding_mode = "model2" - else: - decoding_ind = 0 - decoding_mode = "model2" - # init beamsearch - - if self.beam_search is None: - logging.info("enable beam_search") - self.init_beam_search(decoding_mode=decoding_mode, **kwargs) - self.nbest = kwargs.get("nbest", 1) - - meta_data = {} - if ( - isinstance(data_in, torch.Tensor) - and kwargs.get("data_type", "sound") == "fbank" - ): # fbank - speech, speech_lengths = data_in, data_lengths - if len(speech.shape) < 3: - speech = speech[None, :, :] - if speech_lengths is None: - speech_lengths = speech.shape[1] - else: - # extract fbank feats - time1 = time.perf_counter() - audio_sample_list = load_audio_text_image_video( - data_in, - fs=frontend.fs, - audio_fs=kwargs.get("fs", 16000), - data_type=kwargs.get("data_type", "sound"), - tokenizer=tokenizer, - ) - time2 = time.perf_counter() - meta_data["load_data"] = f"{time2 - time1:0.3f}" - speech, speech_lengths = extract_fbank( - audio_sample_list, - data_type=kwargs.get("data_type", "sound"), - frontend=frontend, - ) - time3 = time.perf_counter() - meta_data["extract_feat"] = f"{time3 - time2:0.3f}" - meta_data["batch_data_time"] = ( - speech_lengths.sum().item() - * frontend.frame_shift - * frontend.lfr_n - / 1000 - ) - - speech = speech.to(device=kwargs["device"]) - speech_lengths = speech_lengths.to(device=kwargs["device"]) - speech_raw = speech.clone().to(device=kwargs["device"]) - # Encoder - _, encoder_out, encoder_out_lens = self.encode( - speech, speech_lengths, ind=decoding_ind - ) - if decoding_mode == "model1": - predictor_outs = self.calc_predictor_mask(encoder_out, encoder_out_lens) - else: - encoder_out, encoder_out_lens = self.encode2( - encoder_out, - encoder_out_lens, - speech_raw, - speech_lengths, - ind=decoding_ind, - ) - predictor_outs = self.calc_predictor_mask2(encoder_out, encoder_out_lens) - - scama_mask = predictor_outs[4] - pre_token_length = predictor_outs[1] - pre_acoustic_embeds = predictor_outs[0] - maxlen = pre_token_length.sum().item() + token_num_relax - minlen = max(0, pre_token_length.sum().item() - token_num_relax) - # c. Passed the encoder result and the beam search - nbest_hyps = self.beam_search( - x=encoder_out[0], - scama_mask=scama_mask, - pre_acoustic_embeds=pre_acoustic_embeds, - maxlenratio=0.0, - minlenratio=0.0, - maxlen=int(maxlen), - minlen=int(minlen), - ) - - nbest_hyps = nbest_hyps[: self.nbest] - - results = [] - for hyp in nbest_hyps: - - # remove sos/eos and get results - last_pos = -1 - if isinstance(hyp.yseq, list): - token_int = hyp.yseq[1:last_pos] - else: - token_int = hyp.yseq[1:last_pos].tolist() - - # remove blank symbol id, which is assumed to be 0 - token_int = list(filter(lambda x: x != 0, token_int)) - - # Change integer-ids to tokens - token = tokenizer.ids2tokens(token_int) - text_postprocessed = tokenizer.tokens2text(token) - if not hasattr(tokenizer, "bpemodel"): - text_postprocessed, _ = postprocess_utils.sentence_postprocess(token) - - result_i = {"key": key[0], "text": text_postprocessed} - results.append(result_i) - - return results, meta_data diff --git a/funasr_detach/models/uniasr/template.yaml b/funasr_detach/models/uniasr/template.yaml deleted file mode 100644 index 35c6b2eb2812a7d64040dedddd720fcaf0bd69ba..0000000000000000000000000000000000000000 --- a/funasr_detach/models/uniasr/template.yaml +++ /dev/null @@ -1,206 +0,0 @@ -# This is an example that demonstrates how to configure a model file. -# You can modify the configuration according to your own requirements. - -# to print the register_table: -# from funasr.register import tables -# tables.print() - -# network architecture -model: UniASR -model_conf: - ctc_weight: 0.0 - lsm_weight: 0.1 - length_normalized_loss: true - predictor_weight: 1.0 - decoder_attention_chunk_type: chunk - ctc_weight2: 0.0 - predictor_weight2: 1.0 - decoder_attention_chunk_type2: chunk - loss_weight_model1: 0.5 - - -# encoder -encoder: SANMEncoderChunkOpt -encoder_conf: - output_size: 320 - attention_heads: 4 - linear_units: 1280 - num_blocks: 35 - dropout_rate: 0.1 - positional_dropout_rate: 0.1 - attention_dropout_rate: 0.1 - input_layer: pe - pos_enc_class: SinusoidalPositionEncoder - normalize_before: true - kernel_size: 11 - sanm_shfit: 0 - selfattention_layer_type: sanm - chunk_size: - - 20 - - 60 - stride: - - 10 - - 40 - pad_left: - - 5 - - 10 - encoder_att_look_back_factor: - - 0 - - 0 - decoder_att_look_back_factor: - - 0 - - 0 - -# decoder -decoder: FsmnDecoderSCAMAOpt -decoder_conf: - attention_dim: 256 - attention_heads: 4 - linear_units: 1024 - num_blocks: 12 - dropout_rate: 0.1 - positional_dropout_rate: 0.1 - self_attention_dropout_rate: 0.1 - src_attention_dropout_rate: 0.1 - att_layer_num: 6 - kernel_size: 11 - concat_embeds: true - -# predictor -predictor: CifPredictorV2 -predictor_conf: - idim: 320 - threshold: 1.0 - l_order: 1 - r_order: 1 - - -# encoder2 -encoder2: SANMEncoderChunkOpt -encoder2_conf: - output_size: 320 - attention_heads: 4 - linear_units: 1280 - num_blocks: 20 - dropout_rate: 0.1 - positional_dropout_rate: 0.1 - attention_dropout_rate: 0.1 - input_layer: pe - pos_enc_class: SinusoidalPositionEncoder - normalize_before: true - kernel_size: 21 - sanm_shfit: 0 - selfattention_layer_type: sanm - chunk_size: - - 45 - - 70 - stride: - - 35 - - 50 - pad_left: - - 5 - - 10 - encoder_att_look_back_factor: - - 0 - - 0 - decoder_att_look_back_factor: - - 0 - - 0 - -# decoder -decoder2: FsmnDecoderSCAMAOpt -decoder2_conf: - attention_dim: 320 - attention_heads: 4 - linear_units: 1280 - num_blocks: 12 - dropout_rate: 0.1 - positional_dropout_rate: 0.1 - self_attention_dropout_rate: 0.1 - src_attention_dropout_rate: 0.1 - att_layer_num: 6 - kernel_size: 11 - concat_embeds: true - -predictor2: CifPredictorV2 -predictor2_conf: - idim: 320 - threshold: 1.0 - l_order: 1 - r_order: 1 - -stride_conv: stride_conv1d -stride_conv_conf: - kernel_size: 2 - stride: 2 - pad: - - 0 - - 1 - -# frontend related -frontend: WavFrontend -frontend_conf: - fs: 16000 - window: hamming - n_mels: 80 - frame_length: 25 - frame_shift: 10 - lfr_m: 7 - lfr_n: 6 - dither: 0.0 - -specaug: SpecAugLFR -specaug_conf: - apply_time_warp: false - time_warp_window: 5 - time_warp_mode: bicubic - apply_freq_mask: true - freq_mask_width_range: - - 0 - - 30 - lfr_rate: 6 - num_freq_mask: 1 - apply_time_mask: true - time_mask_width_range: - - 0 - - 12 - num_time_mask: 1 - -train_conf: - accum_grad: 1 - grad_clip: 5 - max_epoch: 150 - keep_nbest_models: 10 - avg_nbest_model: 5 - log_interval: 50 - -optim: adam -optim_conf: - lr: 0.0001 -scheduler: warmuplr -scheduler_conf: - warmup_steps: 30000 - -dataset: AudioDataset -dataset_conf: - index_ds: IndexDSJsonl - batch_sampler: DynamicBatchLocalShuffleSampler - batch_type: example # example or length - batch_size: 1 # if batch_type is example, batch_size is the numbers of samples; if length, batch_size is source_token_len+target_token_len; - max_token_length: 2048 # filter samples if source_token_len+target_token_len > max_token_length, - buffer_size: 500 - shuffle: True - num_workers: 0 - -tokenizer: CharTokenizer -tokenizer_conf: - unk_symbol: - split_with_space: true - - -ctc_conf: - dropout_rate: 0.0 - ctc_type: builtin - reduce: true - ignore_nan_grad: true -normalize: null diff --git a/funasr_detach/models/whisper/__init__.py b/funasr_detach/models/whisper/__init__.py deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/funasr_detach/models/whisper/model.py b/funasr_detach/models/whisper/model.py deleted file mode 100644 index d921868df527f562a19d351923925e151cdb2f57..0000000000000000000000000000000000000000 --- a/funasr_detach/models/whisper/model.py +++ /dev/null @@ -1,306 +0,0 @@ -from dataclasses import dataclass -from typing import Dict -from typing import Iterable, Optional - -import numpy as np -import torch -import torch.nn.functional as F -from torch import Tensor -from torch import nn - - -from funasr_detach.models.whisper.utils.decoding import ( - detect_language as detect_language_function, - decode as decode_function, -) -from funasr_detach.register import tables - - -@dataclass -class ModelDimensions: - n_mels: int - n_audio_ctx: int - n_audio_state: int - n_audio_head: int - n_audio_layer: int - n_vocab: int - n_text_ctx: int - n_text_state: int - n_text_head: int - n_text_layer: int - - -class LayerNorm(nn.LayerNorm): - def forward(self, x: Tensor) -> Tensor: - return super().forward(x.float()).type(x.dtype) - - -class Linear(nn.Linear): - def forward(self, x: Tensor) -> Tensor: - return F.linear( - x, - self.weight.to(x.dtype), - None if self.bias is None else self.bias.to(x.dtype), - ) - - -class Conv1d(nn.Conv1d): - def _conv_forward( - self, x: Tensor, weight: Tensor, bias: Optional[Tensor] - ) -> Tensor: - return super()._conv_forward( - x, weight.to(x.dtype), None if bias is None else bias.to(x.dtype) - ) - - -def sinusoids(length, channels, max_timescale=10000): - """Returns sinusoids for positional embedding""" - assert channels % 2 == 0 - log_timescale_increment = np.log(max_timescale) / (channels // 2 - 1) - inv_timescales = torch.exp(-log_timescale_increment * torch.arange(channels // 2)) - scaled_time = torch.arange(length)[:, np.newaxis] * inv_timescales[np.newaxis, :] - return torch.cat([torch.sin(scaled_time), torch.cos(scaled_time)], dim=1) - - -class MultiHeadAttention(nn.Module): - def __init__(self, n_state: int, n_head: int): - super().__init__() - self.n_head = n_head - self.query = Linear(n_state, n_state) - self.key = Linear(n_state, n_state, bias=False) - self.value = Linear(n_state, n_state) - self.out = Linear(n_state, n_state) - - def forward( - self, - x: Tensor, - xa: Optional[Tensor] = None, - mask: Optional[Tensor] = None, - kv_cache: Optional[dict] = None, - ): - q = self.query(x) - - if kv_cache is None or xa is None or self.key not in kv_cache: - # hooks, if installed (i.e. kv_cache is not None), will prepend the cached kv tensors; - # otherwise, perform key/value projections for self- or cross-attention as usual. - k = self.key(x if xa is None else xa) - v = self.value(x if xa is None else xa) - else: - # for cross-attention, calculate keys and values once and reuse in subsequent calls. - k = kv_cache[self.key] - v = kv_cache[self.value] - - wv, qk = self.qkv_attention(q, k, v, mask) - return self.out(wv), qk - - def qkv_attention( - self, q: Tensor, k: Tensor, v: Tensor, mask: Optional[Tensor] = None - ): - n_batch, n_ctx, n_state = q.shape - scale = (n_state // self.n_head) ** -0.25 - q = q.view(*q.shape[:2], self.n_head, -1).permute(0, 2, 1, 3) * scale - k = k.view(*k.shape[:2], self.n_head, -1).permute(0, 2, 3, 1) * scale - v = v.view(*v.shape[:2], self.n_head, -1).permute(0, 2, 1, 3) - - qk = q @ k - if mask is not None: - qk = qk + mask[:n_ctx, :n_ctx] - qk = qk.float() - - w = F.softmax(qk, dim=-1).to(q.dtype) - return (w @ v).permute(0, 2, 1, 3).flatten(start_dim=2), qk.detach() - - -class ResidualAttentionBlock(nn.Module): - def __init__(self, n_state: int, n_head: int, cross_attention: bool = False): - super().__init__() - - self.attn = MultiHeadAttention(n_state, n_head) - self.attn_ln = LayerNorm(n_state) - - self.cross_attn = ( - MultiHeadAttention(n_state, n_head) if cross_attention else None - ) - self.cross_attn_ln = LayerNorm(n_state) if cross_attention else None - - n_mlp = n_state * 4 - self.mlp = nn.Sequential( - Linear(n_state, n_mlp), nn.GELU(), Linear(n_mlp, n_state) - ) - self.mlp_ln = LayerNorm(n_state) - - def forward( - self, - x: Tensor, - xa: Optional[Tensor] = None, - mask: Optional[Tensor] = None, - kv_cache: Optional[dict] = None, - ): - x = x + self.attn(self.attn_ln(x), mask=mask, kv_cache=kv_cache)[0] - if self.cross_attn: - x = x + self.cross_attn(self.cross_attn_ln(x), xa, kv_cache=kv_cache)[0] - x = x + self.mlp(self.mlp_ln(x)) - return x - - -@tables.register("encoder_classes", "WhisperEncoder") -class AudioEncoder(nn.Module): - def __init__( - self, n_mels: int, n_ctx: int, n_state: int, n_head: int, n_layer: int - ): - super().__init__() - self.conv1 = Conv1d(n_mels, n_state, kernel_size=3, padding=1) - self.conv2 = Conv1d(n_state, n_state, kernel_size=3, stride=2, padding=1) - self.register_buffer("positional_embedding", sinusoids(n_ctx, n_state)) - - self.blocks: Iterable[ResidualAttentionBlock] = nn.ModuleList( - [ResidualAttentionBlock(n_state, n_head) for _ in range(n_layer)] - ) - self.ln_post = LayerNorm(n_state) - - def forward(self, x: Tensor): - """ - x : torch.Tensor, shape = (batch_size, n_mels, n_ctx) - the mel spectrogram of the audio - """ - x = F.gelu(self.conv1(x)) - x = F.gelu(self.conv2(x)) - x = x.permute(0, 2, 1) - - assert x.shape[1:] == self.positional_embedding.shape, "incorrect audio shape" - x = (x + self.positional_embedding).to(x.dtype) - - for block in self.blocks: - x = block(x) - - x = self.ln_post(x) - return x - - -@tables.register("decoder_classes", "WhisperDecoder") -class TextDecoder(nn.Module): - def __init__( - self, n_vocab: int, n_ctx: int, n_state: int, n_head: int, n_layer: int - ): - super().__init__() - - self.token_embedding = nn.Embedding(n_vocab, n_state) - self.positional_embedding = nn.Parameter(torch.empty(n_ctx, n_state)) - - self.blocks: Iterable[ResidualAttentionBlock] = nn.ModuleList( - [ - ResidualAttentionBlock(n_state, n_head, cross_attention=True) - for _ in range(n_layer) - ] - ) - self.ln = LayerNorm(n_state) - - mask = torch.empty(n_ctx, n_ctx).fill_(-np.inf).triu_(1) - self.register_buffer("mask", mask, persistent=False) - - def forward(self, x: Tensor, xa: Tensor, kv_cache: Optional[dict] = None): - """ - x : torch.LongTensor, shape = (batch_size, <= n_ctx) - the text tokens - xa : torch.Tensor, shape = (batch_size, n_mels, n_audio_ctx) - the encoded audio features to be attended on - """ - offset = next(iter(kv_cache.values())).shape[1] if kv_cache else 0 - x = ( - self.token_embedding(x) - + self.positional_embedding[offset : offset + x.shape[-1]] - ) - x = x.to(xa.dtype) - - for block in self.blocks: - x = block(x, xa, mask=self.mask, kv_cache=kv_cache) - - x = self.ln(x) - logits = ( - x @ torch.transpose(self.token_embedding.weight.to(x.dtype), 0, 1) - ).float() - - return logits - - -@tables.register("model_classes", "Whisper") -class Whisper(nn.Module): - def __init__(self, dims: dict): - super().__init__() - dims = ModelDimensions(**dims) - self.dims = dims - self.sos = 1 - self.eos = 1 - self.encoder = AudioEncoder( - self.dims.n_mels, - self.dims.n_audio_ctx, - self.dims.n_audio_state, - self.dims.n_audio_head, - self.dims.n_audio_layer, - ) - self.decoder = TextDecoder( - self.dims.n_vocab, - self.dims.n_text_ctx, - self.dims.n_text_state, - self.dims.n_text_head, - self.dims.n_text_layer, - ) - - def embed_audio(self, mel: torch.Tensor): - return self.encoder(mel) - - def logits(self, tokens: torch.Tensor, audio_features: torch.Tensor): - return self.decoder(tokens, audio_features) - - def forward( - self, mel: torch.Tensor, tokens: torch.Tensor - ) -> Dict[str, torch.Tensor]: - return self.decoder(tokens, self.encoder(mel)) - - @property - def device(self): - return next(self.parameters()).device - - @property - def is_multilingual(self): - return self.dims.n_vocab == 51865 - - def install_kv_cache_hooks(self, cache: Optional[dict] = None): - """ - The `MultiHeadAttention` module optionally accepts `kv_cache` which stores the key and value - tensors calculated for the previous positions. This method returns a dictionary that stores - all caches, and the necessary hooks for the key and value projection modules that save the - intermediate tensors to be reused during later calculations. - - Returns - ------- - cache : Dict[nn.Module, torch.Tensor] - A dictionary object mapping the key/value projection modules to its cache - hooks : List[RemovableHandle] - List of PyTorch RemovableHandle objects to stop the hooks to be called - """ - cache = {**cache} if cache is not None else {} - hooks = [] - - def save_to_cache(module, _, output): - if ( - module not in cache - or output.shape[1] > self.decoder.positional_embedding.shape[0] - ): - cache[module] = ( - output # save as-is, for the first token or cross attention - ) - else: - cache[module] = torch.cat([cache[module], output], dim=1).detach() - return cache[module] - - def install_hooks(layer: nn.Module): - if isinstance(layer, MultiHeadAttention): - hooks.append(layer.key.register_forward_hook(save_to_cache)) - hooks.append(layer.value.register_forward_hook(save_to_cache)) - - self.decoder.apply(install_hooks) - return cache, hooks - - detect_language = detect_language_function - decode = decode_function diff --git a/funasr_detach/models/whisper/utils/__init__.py b/funasr_detach/models/whisper/utils/__init__.py deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/funasr_detach/models/whisper/utils/assets/gpt2/merges.txt b/funasr_detach/models/whisper/utils/assets/gpt2/merges.txt deleted file mode 100644 index 6636bda4a1fd7a63653dffb22683b8162c8de956..0000000000000000000000000000000000000000 --- a/funasr_detach/models/whisper/utils/assets/gpt2/merges.txt +++ /dev/null @@ -1,50001 +0,0 @@ -#version: 0.2 - Trained by `huggingface/tokenizers` -Ġ t -Ġ a -h e -i n -r e -o n -Ġt he -e r -Ġ s -a t -Ġ w -Ġ o -e n -Ġ c -i t -i s -a n -o r -e s -Ġ b -e d -Ġ f -in g -Ġ p -o u -Ġa n -a l -a r -Ġt o -Ġ m -Ġo f -Ġ in -Ġ d -Ġ h -Ġan d -i c -a s -l e -Ġt h -i on -o m -l l -en t -Ġ n -Ġ l -s t -Ġ re -v e -Ġ e -r o -l y -Ġb e -Ġ g -Ġ T -c t -Ġ S -i d -o t -Ġ I -u t -e t -Ġ A -Ġ is -Ġ on -i m -a m -o w -a y -a d -s e -Ġth at -Ġ C -i g -Ġf or -a c -Ġ y -v er -u r -Ġ u -l d -Ġs t -Ġ M -' s -Ġ he -Ġ it -at ion -it h -i r -c e -Ġy ou -i l -Ġ B -Ġw h -o l -Ġ P -Ġw ith -Ġ 1 -t er -c h -Ġa s -Ġw e -Ġ ( -n d -i ll -Ġ D -i f -Ġ 2 -a g -er s -k e -Ġ " -Ġ H -e m -Ġc on -Ġ W -Ġ R -he r -Ġw as -Ġ r -o d -Ġ F -u l -at e -Ġa t -r i -p p -o re -ĠT he -Ġs e -u s -Ġp ro -Ġh a -u m -Ġa re -Ġd e -a in -an d -Ġo r -ig h -es t -is t -a b -r om -Ġ N -t h -Ġc om -Ġ G -u n -o p -0 0 -Ġ L -Ġn ot -es s -Ġe x -Ġ v -re s -Ġ E -e w -it y -an t -Ġb y -e l -o s -or t -o c -q u -Ġf rom -Ġha ve -Ġs u -i ve -ou ld -Ġs h -Ġth is -n t -r a -p e -igh t -ar t -m ent -Ġa l -u st -en d -- - -al l -Ġ O -ac k -Ġc h -Ġ le -i es -re d -ar d -â Ģ -ou t -Ġ J -Ġa b -e ar -i v -al ly -ou r -o st -g h -p t -Ġp l -as t -Ġc an -a k -om e -u d -T he -Ġh is -Ġd o -Ġg o -Ġh as -g e -' t -Ġ U -r ou -Ġs a -Ġ j -Ġb ut -Ġw or -Ġa ll -e ct -Ġ k -am e -Ġw ill -o k -Ġw he -Ġthe y -id e -0 1 -f f -ic h -p l -t her -Ġt r -. . -Ġin t -i e -u re -ag e -Ġn e -i al -a p -in e -ic e -Ġm e -Ġo ut -an s -on e -on g -ion s -Ġwh o -Ġ K -Ġu p -Ġthe ir -Ġa d -Ġ 3 -Ġu s -at ed -ou s -Ġm ore -u e -o g -ĠS t -in d -i ke -Ġs o -im e -p er -. " -b er -i z -a ct -Ġon e -Ġsa id -Ġ - -a re -Ġyou r -c c -ĠT h -Ġc l -e p -a ke -ab le -i p -Ġcon t -Ġwh ich -i a -Ġ im -Ġab out -Ġwe re -ver y -u b -Ġh ad -Ġ en -Ġcom p -, " -ĠI n -Ġu n -Ġa g -i re -ac e -a u -ar y -Ġw ould -as s -r y -Ġ âĢ -c l -o ok -e re -s o -Ġ V -ig n -i b -Ġof f -Ġt e -v en -Ġ Y -i le -o se -it e -or m -Ġ2 01 -Ġre s -Ġm an -Ġp er -Ġo ther -or d -ul t -Ġbe en -Ġl ike -as e -an ce -k s -ay s -ow n -en ce -Ġd is -ct ion -Ġan y -Ġa pp -Ġs p -in t -res s -ation s -a il -Ġ 4 -ic al -Ġthe m -Ġhe r -ou nt -ĠC h -Ġa r -Ġ if -Ġthe re -Ġp e -Ġy ear -a v -Ġm y -Ġs ome -Ġwhe n -ou gh -ac h -Ġth an -r u -on d -ic k -Ġo ver -ve l -Ġ qu -Ċ Ċ -Ġs c -re at -re e -ĠI t -ou nd -p ort -Ġal so -Ġp art -f ter -Ġk n -Ġbe c -Ġt ime -en s -Ġ 5 -op le -Ġwh at -Ġn o -d u -m er -an g -Ġn ew --- -- -Ġg et -or y -it ion -ing s -Ġj ust -Ġint o -Ġ 0 -ent s -o ve -t e -Ġpe ople -Ġp re -Ġit s -Ġre c -Ġt w -i an -ir st -ar k -or s -Ġwor k -ad e -o b -Ġs he -Ġo ur -w n -in k -l ic -Ġ1 9 -ĠH e -is h -nd er -au se -Ġh im -on s -Ġ [ -Ġ ro -f orm -i ld -at es -ver s -Ġon ly -o ll -Ġs pe -c k -e ll -am p -Ġa cc -Ġb l -i ous -ur n -f t -o od -Ġh ow -he d -Ġ ' -Ġa fter -a w -Ġat t -o v -n e -Ġpl ay -er v -ic t -Ġc ould -it t -Ġa m -Ġf irst -Ġ 6 -Ġa ct -Ġ $ -e c -h ing -u al -u ll -Ġcom m -o y -o ld -c es -at er -Ġf e -Ġbe t -w e -if f -Ġtw o -oc k -Ġb ack -) . -id ent -Ġu nder -rou gh -se l -x t -Ġm ay -rou nd -Ġp o -p h -is s -Ġd es -Ġm ost -Ġd id -Ġad d -j ect -Ġin c -f ore -Ġp ol -on t -Ġag ain -cl ud -ter n -Ġkn ow -Ġne ed -Ġcon s -Ġc o -Ġ . -Ġw ant -Ġse e -Ġ 7 -n ing -i ew -ĠTh is -c ed -Ġe ven -Ġin d -t y -ĠW e -at h -Ġthe se -Ġp r -Ġu se -Ġbec ause -Ġf l -n g -Ġn ow -ĠâĢ ĵ -c om -is e -Ġm ake -Ġthe n -ow er -Ġe very -ĠU n -Ġse c -os s -u ch -Ġe m -Ġ = -ĠR e -i ed -r it -Ġin v -le ct -Ġsu pp -at ing -Ġl ook -m an -pe ct -Ġ 8 -ro w -Ġb u -Ġwhe re -if ic -Ġyear s -i ly -Ġd iff -Ġsh ould -Ġre m -T h -I n -Ġe v -d ay -' re -ri b -Ġre l -s s -Ġde f -Ġr ight -Ġs y -) , -l es -00 0 -he n -Ġth rough -ĠT r -_ _ -Ġw ay -Ġd on -Ġ , -Ġ1 0 -as ed -Ġas s -ub lic -Ġre g -ĠA nd -i x -Ġ very -Ġin clud -ot her -Ġim p -ot h -Ġsu b -ĠâĢ Ķ -Ġbe ing -ar g -ĠW h -= = -ib le -Ġdo es -an ge -r am -Ġ 9 -er t -p s -it ed -ation al -Ġb r -Ġd own -Ġman y -ak ing -Ġc all -ur ing -it ies -Ġp h -ic s -al s -Ġde c -at ive -en er -Ġbe fore -il ity -Ġwe ll -Ġm uch -ers on -Ġth ose -Ġsu ch -Ġ ke -Ġ end -ĠB ut -as on -t ing -Ġl ong -e f -Ġth ink -y s -Ġbe l -Ġs m -it s -a x -Ġo wn -Ġpro v -Ġs et -if e -ment s -b le -w ard -Ġsh ow -Ġp res -m s -om et -Ġo b -Ġs ay -ĠS h -t s -f ul -Ġe ff -Ġg u -Ġin st -u nd -re n -c ess -Ġ ent -ĠY ou -Ġgo od -Ġst art -in ce -Ġm ade -t t -st em -ol og -u p -Ġ | -um p -Ġhe l -ver n -ul ar -u ally -Ġa c -Ġm on -Ġl ast -Ġ2 00 -1 0 -Ġst ud -u res -ĠA r -sel f -ar s -mer ic -u es -c y -Ġm in -oll ow -Ġc ol -i o -Ġm od -Ġc ount -ĠC om -he s -Ġf in -a ir -i er -âĢ Ķ -re ad -an k -at ch -e ver -Ġst r -Ġpo int -or k -ĠN ew -Ġs ur -o ol -al k -em ent -Ġus ed -ra ct -we en -Ġs ame -ou n -ĠA l -c i -Ġdiff ere -Ġwh ile ----- ---- -Ġg ame -ce pt -Ġs im -.. . -Ġin ter -e k -Ġre port -Ġpro du -Ġst ill -l ed -a h -Ġhe re -Ġwor ld -Ġth ough -Ġn um -ar ch -im es -al e -ĠS e -ĠI f -/ / -ĠL e -Ġre t -Ġre f -Ġtr ans -n er -ut ion -ter s -Ġt ake -ĠC l -Ġcon f -w ay -a ve -Ġgo ing -Ġs l -u g -ĠA meric -Ġspe c -Ġh and -Ġbet ween -ist s -ĠD e -o ot -I t -Ġe ar -Ġagain st -Ġh igh -g an -a z -at her -Ġex p -Ġo p -Ġin s -Ġg r -Ġhel p -Ġre qu -et s -in s -ĠP ro -is m -Ġf ound -l and -at a -us s -am es -Ġp erson -Ġg reat -p r -Ġs ign -ĠA n -' ve -Ġs omet -Ġs er -h ip -Ġr un -Ġ : -Ġt er -ire ct -Ġf ollow -Ġd et -ic es -Ġf ind -1 2 -Ġm em -Ġc r -e red -e x -Ġex t -ut h -en se -c o -Ġte am -v ing -ou se -as h -at t -v ed -Ġsy stem -ĠA s -d er -iv es -m in -Ġle ad -ĠB l -c ent -Ġa round -Ġgo vern -Ġc ur -vel op -an y -Ġc our -al th -ag es -iz e -Ġc ar -od e -Ġl aw -Ġre ad -' m -c on -Ġre al -Ġsupp ort -Ġ1 2 -.. .. -Ġre ally -n ess -Ġf act -Ġd ay -Ġb oth -y ing -Ġs erv -ĠF or -Ġth ree -Ġw om -Ġm ed -od y -ĠThe y -5 0 -Ġex per -t on -Ġe ach -ak es -Ġc he -Ġc re -in es -Ġre p -1 9 -g g -ill ion -Ġg rou -ut e -i k -W e -g et -E R -Ġm et -Ġs ays -o x -Ġd uring -er n -iz ed -a red -Ġf am -ic ally -Ġha pp -ĠI s -Ġch ar -m ed -v ent -Ġg ener -i ent -p le -i et -re nt -1 1 -v es -pt ion -Ġ2 0 -form ation -Ġc or -Ġoff ic -ie ld -Ġto o -is ion -Ġin f -Ġ Z -t he -o ad -Ġp ublic -Ġpro g -r ic -* * -Ġw ar -Ġp ower -v iew -Ġf ew -Ġl oc -Ġdiffere nt -Ġst ate -Ġhe ad -' ll -Ġp oss -Ġst at -re t -ant s -Ġv al -Ġis s -Ġc le -i vers -an c -Ġex pl -Ġan other -Ġ Q -Ġa v -th ing -n ce -W h -Ġch ild -Ġs ince -i red -l ess -Ġl ife -Ġde velop -itt le -Ġde p -Ġp ass -ã ĥ -Ġt urn -or n -Th is -b ers -ro ss -ĠA d -Ġf r -Ġres p -Ġsec ond -o h -Ġ / -Ġdis c -Ġ & -Ġsomet hing -Ġcomp le -Ġ ed -Ġf il -Ġmon th -a j -u c -Ġgovern ment -Ġwith out -Ġle g -Ġd ist -Ġp ut -Ġqu est -an n -Ġpro t -2 0 -Ġne ver -i ence -Ġle vel -Ġar t -Ġth ings -Ġm ight -Ġeff ect -Ġcont ro -Ġc ent -Ġ1 8 -Ġall ow -Ġbel ie -ch ool -ot t -Ġinc re -Ġfe el -Ġres ult -Ġl ot -Ġf un -ot e -Ġt y -ere st -Ġcont in -Ġus ing -Ġb ig -2 01 -Ġas k -Ġb est -Ġ ) -I N -Ġo pp -3 0 -Ġnum ber -in ess -S t -le ase -Ġc a -Ġm ust -Ġd irect -Ġg l -Ġ < -Ġop en -Ġp ost -Ġcom e -Ġse em -ord ing -Ġwe ek -ate ly -it al -Ġe l -ri end -Ġf ar -Ġt ra -in al -Ġp ri -ĠU S -Ġpl ace -Ġfor m -Ġto ld -" : -ain s -at ure -ĠTr ump -Ġst and -Ġ # -id er -ĠF r -Ġne xt -Ġs oc -Ġp ur -Ġle t -Ġl ittle -Ġh um -Ġ i -r on -1 5 -Ġ1 5 -Ġcomm un -Ġm ark -ĠThe re -Ġw r -ĠTh at -Ġin formation -w ays -Ġb us -a pp -Ġinv est -m e -Ġh ard -ain ed -e ad -Ġim port -Ġapp ro -Ġt est -Ġt ri -Ġre st -os ed -Ġf ull -Ġc are -ĠS p -Ġc ase -O N -Ġs k -Ġl ess -Ġ + -Ġpart ic -ĠP l -ab ly -u ck -is hed -ch n -b e -Ġl ist -at or -Ġto p -Ġad v -ĠB e -ru ct -Ġd em -r ation -l ing -g y -re en -g er -Ġh ome -Ġle ft -Ġbet ter -Ġd ata -Ġ1 1 -Ġatt ack -Ġpro ble -l ine -ard s -Ġbe h -r al -ĠH ow -ĠS he -ar ge -Ġ -- -: // -Ġb ro -ĠP h -at s -Ġbu ild -w w -id ed -a im -as es -en cy -Ġm ain -in ed -Ġinclud ing -Ġ { -Ġg ot -Ġint erest -Ġke ep -Ġ X -Ġe as -ain ing -Ġcl ass -âĢ ¦ -ĠN o -Ġv ar -Ġsm all -amp le -A T -Ġ ide -ĠS o -Ġre ce -Ġpol it -Ġm ov -Ġpl an -Ġper cent -iv ing -Ġc amp -Ġp ay -1 4 -s c -is ed -Ġu nt -one y -pl oy -== == -Ġdid n -ĠI nd -el s -ert ain -Ġp os -__ __ -i ver -Ġpro cess -Ġprog ram -if ied -ĠR ep -1 6 -u ro -olog y -at ter -in a -Ġn ame -ĠA ll -Ġf our -Ġret urn -v ious -b s -Ġcall ed -Ġm ove -ĠS c -ir d -Ġgrou p -Ġb re -Ġm en -Ġc ap -t en -e e -Ġd ri -le g -he re -uth or -Ġp at -Ġcur rent -id es -Ġp op -t o -ent ion -Ġal ways -Ġm il -Ġwom en -Ġ1 6 -Ġo ld -iv en -ra ph -ĠO r -r or -ent ly -Ġn ear -ĠE x -re am -s h -Ġ1 4 -Ġf ree -iss ion -st and -ĠC on -al ity -us ed -1 3 -Ġdes ign -Ġch ange -Ġch ang -Ġb o -Ġv is -em ber -Ġb ook -read y -Ġk ill -2 5 -pp ed -Ġa way -Ġab le -Ġcount ry -Ġcon st -ar n -Ġor der -A R -i or -i um -or th -1 8 -ail able -Ġs w -Ġm illion -Ġ1 3 -at ic -t ed -ĠG o -Ġo per -en g -Ġth ing -aj or -con om -ĠCom m -Ġwh y -u red -ur al -Ġs chool -b y -ĠM ar -Ġa ff -Ġd ays -Ġan n -us h -an e -I f -e g -Ġpro f -Ġhe alth -ou th -B ut -ion al -. , -Ġs ol -Ġal ready -Ġ3 0 -Ġchar act -H e -Ġf riend -E S -i ans -ic le -' d -ĠO n -Ġle ast -Ġp rom -Ġd r -Ġh ist -it her -Ġ est -i qu -1 7 -s on -Ġte ll -Ġt alk -oh n -o int -le ction -A N -Ġunt il -au gh -Ġl ater -Ġ ve -Ġv iew -end ing -iv ed -Ġwor d -w are -Ġc ost -Ġen ough -Ġg ive -ĠUn ited -Ġte chn -are nt -O R -Ġp ar -ĠD r -Ġ201 6 -r ist -er ing -Ġ  -Ġl arge -s ide -ac y -cc ess -Ġw in -Ġimport ant -Ġ19 9 -Ġdoes n -Ġ1 7 -Ġbus iness -Ġcle ar -Ġre se -" , -ur y -Ġe qu -as ter -al f -ĠAmeric an -n ect -Ġex pect -ivers ity -Ġo cc -ĠF l -Ġk ind -Ġme an -Ġp ast -Ġde v -Ġb as -le t -ra ft -Ġor gan -Ġde l -Ġper form -Ġst ory -Ġse ason -ĠC ol -Ġcl aim -Ġc ame -Ġwith in -Ġl ine -Ġpro ject -ĠA t -Ġcontro l -end ed -ĠS y -Ġa ir -iz ation -Ġ * -le y -Ġm oney -id d -Y ou -f or -Ġfam ily -Ġm aking -Ġb it -Ġpol ice -Ġhapp en -Ġ vers -on y -u ff -ĠW hen -Ġs it -ide o -l f -is on -Ġsu re -g in -Ġapp ear -Ġl ight -Ġ es -o f -Ġw ater -Ġt imes -n ot -Ġg row -Ġcomp any -ĠT e -ow s -Ġm ar -our ce -i ol -ar m -b r -Ġex ample -Ġcon c -Ġf ore -ĠT o -p ro -E N -ri es -Ġ2 5 -ĠC an -ne y -Ġact ually -Ġe ver -ur ity -ak en -ap s -Ġt ax -Ġm ajor -am a -Ġof ten -er al -Ġhum an -Ġj ob -is ter -Ġav ailable -oc r -en n -a id -iv id -Ġrec ord -? " -Ġs ing -ĠA m -id ence -Ġnew s -st er -Ġe conom -Ġfollow ing -ĠB r -is ing -Ġh our -m ost -um ent -Ġse x -Ġdes c -Ġbec ome -ĠE d -Ġto ok -Ġha ving -Ġprodu ct -a ult -A s -ar ing -Ġme ans -Ġh op -un e -Ġch o -Ġc ertain -Ġn on -Ġde al -2 4 -le ment -oc i -en e -Ġs ide -ĠP r -ĠM ay -Ġre ason -u ed -c hed -ul ation -Ġe lect -Ġoffic ial -Ġposs ible -Ġh old -and s -ot s -Ġc ity -or ies -Ġse ver -Ġchild ren -Ġon ce -Ġact iv -l er -Ġn ight -it ions -ĠJ ohn -a pe -pl ay -Ġd one -Ġl im -Ġwork ing -ĠP res -or ld -e b -ĠC o -Ġb ody -ail s -ut es -ĠM r -Ġwhe ther -Ġa uthor -ro p -Ġpro per -Ġse en -) ; -Ġf ac -ĠS u -Ġcon d -it ing -Ġcour se -Ġ } --------- -------- -a ign -Ġev ent -Ġen g -Ġp ot -Ġin tern -i am -Ġsh ort -em pt -ã Ĥ -ĠG od -il ar -8 0 -Ġor ig -I S -our n -ab ility -it ive -Ġd am -Ġ1 00 -Ġp ress -Ġdo ing -Ġprot ect -r ing -Ġthough t -Ġquest ion -re w -ĠW ar -Ġsever al -ĠSt ate -Ġg iven -Ġf und -ĠT w -Ġw ent -an ces -w ork -p or -m y -4 0 -Ġar g -art ment -ust om -Ġpol ic -Ġme et -Ġc reat -2 2 -ĠSt ates -Ġg ames -ra w -ut ure -Ġunder stand -ur s -ĠO b -l ish -s y -Ġm akes -Ġw on -ag on -Ġh tt -Ġl ove -ent ial -Ġcomple te -p ar -ĠI m -A L -Ġacc ount - ł -ore d -ver t -Ġ ident -Ġ201 5 -Ġother s -ĠM in -i ber -ver age -The re -ition al -d d -Ġpro b -Ġyou ng -Ġal ong -Ġacc ording -Ġy et -Ġmem bers -ĠWh at -o id -ĠM an -A nd -Ġam ong -a i -Ġem ploy -ĠR es -Ġ > -Ġinv ol -Ġl ow -a f -ĠC ar -Ġh ig -ĠO ne -ĠS ec -in ation -Ġlike ly -Ġan t -ag ed -ĠR uss -Ġb en -Ġre le -F or -b ack -ĠN ot -Ġpres ident -b all -Ġacc ess -ivid ual -ĠD em -ĠE uro -6 0 -Ġkn own -ir l -ĠG r -Ġear ly -u se -iet y -âĢ ĵ -Ġf ight -Ġs ent -Ġto day -Ġmark et -" . -Ġb ased -Ġstr ong -ur ther -Ġde b -m ber -Ġproble m -Ġde ath -Ġsoc ial -im ate -A S -ort un -Ġcamp aign -er y -C h -Ġe y -i ally -Ġm us -w h -p os -Ġ er -Ġsa f -Ġmonth s -ir on -Ġv iol -Ġf ive -Ġst re -Ġplay ers -in c -al d -y ear -a un -Ġsu ccess -Ġpres ent -ere nce -Ġ201 4 -Ġsu gg -Ġpartic ular -Ġtr y -Ġsugg est -ĠCh rist -on es -Ġpri v -2 3 -Ġc rit -Ġl and -Ġloc al -if y -2 9 -Ġa ut -E D -ĠG u -Ġm ult -Ġpolit ical -Ġask ed -Ġfor mer -it ter -ri pt -Ġcl ose -Ġp ract -ĠY ork -Ġget ting -Ġac ross -Ġcom b -Ġbelie ve -Ġ z -Ġto get -Ġtoget her -ĠC ent -ir c -Ġind ividual -ĠM c -2 7 -is k -ĠE ng -Ġf ace -Ġ2 4 -Ġval ue -Ġare a -e v -Ġw rit -ĠPres ident -Ġv ot -Ġke y -Ġm om -p ut -Ġany thing -Ġexper ience -att le -Ġm ind -a ff -om m -Ġf uture -g ed -Ġc ut -Ġto t -it ch -Ġv ideo -Ġinvest ig -Ġn et -ĠM y -r ict -i en -. ) -Ġimp ro -th ough -ward s -Ġcon nect -ĠM ed -sel ves -ens ive -m b -o ber -at ors -A n -Ġ5 0 -Ġre du -res ent -Ġab ove -Ġf re -ĠEuro pe -s w -Ġam ount -ĠA pp -Ġe ither -Ġmil it -Ġan al -Ġf ail -ĠE n -al es -Ġspec ial -Ġbl ack -I T -c her -Ġlook ing -Ġf ire -y n -Ġal most -o on -Ġstud y -Ġm iss -c hes -ro wn -Ġt re -Ġcommun ity -Ġmed ia -Ġf ood -Ġcom es -ĠUn iversity -Ġsing le -Wh at -u ly -Ġh alf -ag ue -h od -ĠRep ublic -Ġstart ed -Ġqu ick -ot o -b ook -Ġiss ue -it or -Ġel se -Ġcons ider -2 6 -ro du -Ġt aken -2 8 -9 9 -ĠW ith -Ġtr ue -Ġw a -Ġtr ad -Ġag o -Ġm ess -ie f -Ġadd ed -o ke -Ġb ad -Ġf av -3 3 -Ġsim ilar -as k -ĠD on -Ġcharact er -ort s -ĠH ouse -Ġreport ed -Ġty pe -v al -i od -ĠHow ever -Ġt arg -Ġent ire -pp ing -Ġhist ory -Ġl ive -ff ic -.... .... -ed eral -Ġtr ying -Ġdisc uss -ĠH ar -ac es -l ished -Ġse lf -os p -re st -Ġro om -el t -Ġf all -ol ution -Ġe t -Ġ x -Ġis n -Ġide a -b o -Ġs ound -ĠD ep -Ġsome one -ci ally -ull y -Ġf oc -Ġob ject -if t -ap er -Ġplay er -Ġr ather -Ġserv ice -as hing -ĠD o -ĠP art -ru g -m on -p ly -Ġm or -Ġnot hing -Ġprov ide -I C -un g -Ġpart y -Ġex ist -Ġm ag -7 0 -Ġr ul -Ġh ouse -Ġbeh ind -Ġhow ever -ĠW orld -Ġs um -Ġapp lic -Ġ ; -Ġfun ction -g r -ĠP ol -Ġfr ont -2 00 -Ġser ies -Ġt em -Ġty p -ill s -Ġo pt -Ġpoint s -Ġbel ow -itt ed -Ġspec ific -Ġ201 7 -um b -Ġr a -Ġpre vious -Ġpre t -re me -Ġc ustom -Ġcour t -ĠM e -Ġre pl -Ġwho le -g o -c er -Ġt reat -ĠA ct -Ġprob ably -Ġle arn -end er -ĠA ss -Ġvers ion -n ow -Ġche ck -ĠC al -R E -min ist -O n -our ces -Ġben ef -Ġd oc -Ġdet er -Ġen c -Ġsu per -Ġadd ress -Ġv ict -Ġ201 3 -Ġme as -t r -Ġf ield -W hen -Ġsign ific -u ge -Ġfe at -Ġcomm on -l oad -Ġbe gin -Ġbr ing -Ġa ction -er man -Ġdesc rib -Ġind ust -Ġwant ed -ri ed -m ing -Ġatt empt -4 5 -f er -Ġd ue -ress ion -# # -Ġsh all -Ġs ix -o o -Ġst ep -Ġp ub -Ġhim self -Ġ2 3 -Ġc op -Ġd est -Ġst op -A C -ib ility -Ġl ab -ic ult -Ġhour s -Ġcre ate -Ġf urther -ĠAmeric a -ĠC ity -Ġd ou -he ad -S T -ĠN orth -c ing -Ġn ational -u le -ĠIn st -Ġt aking -ĠQ u -ir t -Ġre d -Ġrese arch -v iron -ĠG e -Ġbre ak -an a -Ġsp ace -ater ial -Ġrec ent -ĠA b -Ġgener al -Ġh it -Ġper iod -Ġevery thing -ive ly -Ġph ys -Ġsay ing -an ks -Ġc ou -Ġc ult -ac ed -e al -u ation -Ġc oun -l u -Ġinclud e -Ġpos ition -ĠA fter -ĠCan ad -ĠE m -Ġim m -ĠR ed -Ġp ick -Ġcom pl -Ġm atter -re g -e xt -ang u -is c -o le -a ut -Ġcomp et -e ed -f ect -Ġ2 1 -ĠS en -ĠThe se -as ing -Ġcan not -Ġin it -Ġrel ations -ac hed -Ġb ar -Ġ4 0 -ĠT H -Ġ201 2 -Ġv ol -Ġg round -Ġsec urity -Ġup d -il t -3 5 -Ġconc ern -ĠJ ust -Ġwh ite -Ġseem s -ĠH er -pe cially -i ents -Ġann oun -Ġf ig -ight s -Ġst ri -l ike -id s -Ġs us -Ġw atch -Ġ â -Ġw ind -ĠC ont -Ġit self -Ġm ass -A l -y le -iqu e -ĠN ational -Ġab s -Ġp ack -Ġout side -Ġan im -Ġp ain -et er -Ġman ag -du ct -og n -Ġ ] -ĠSe pt -se c -o ff -ĠJ an -Ġf oot -ad es -Ġth ird -Ġm ot -Ġev idence -int on -Ġth reat -a pt -pl es -c le -Ġl o -Ġde cl -Ġit em -med i -Ġrep resent -om b -am er -Ġsignific ant -og raph -s u -Ġc al -i res -00 00 -I D -A M -Ġsim ply -Ġlong er -Ġf ile -O T -c he -S o -ate g -or g -ĠH is -Ġen er -Ġd om -Ġup on -il i -": " -Ġthem selves -Ġcom ing -Ġqu ite -Ġdiff icult -ĠB ar -il ities -re l -end s -c ial -6 4 -Ġwom an -ra p -y r -Ġne cess -ip s -Ġte xt -Ġrequ ire -Ġmilit ary -Ġre view -Ġresp ons -7 5 -Ġsub ject -Ġinst ead -Ġiss ues -Ġg en -" ," -Ġmin utes -Ġwe ap -r ay -am ed -t ime -b l -H ow -Ġc ode -ĠS m -Ġhig her -ĠSt e -r is -Ġp age -Ġstud ents -ĠIn tern -Ġmet hod -ĠA ug -ĠP er -ĠA g -Ġpolic y -ĠS w -Ġex ec -Ġac cept -um e -rib ut -Ġword s -Ġfin al -Ġchang es -ĠDem ocr -Ġfriend s -Ġres pect -Ġe p -Ġcomp an -iv il -Ġdam age -** ** -og le -viron ment -Ġne g -ent al -Ġa p -Ġtot al -iv al -! " -l im -Ġneed s -Ġag re -Ġdevelop ment -Ġa ge -ip le -2 1 -Ġresult s -ĠA f -S h -Ġg un -ĠOb ama -ro ll -Ġ @ -Ġright s -ĠB rit -Ġrun ning -Ġwas n -Ġp ort -Ġr ate -Ġpret ty -Ġtarg et -Ġsa w -Ġc irc -Ġwor ks -ic ro -al t -o ver -ww w -Th at -l ier -Ġevery one -ud e -Ġp ie -idd le -ra el -Ġr ad -Ġbl ock -Ġw alk -T o -ã ģ -n es -ĠA ust -a ul -ro te -ĠS outh -ess ion -op h -Ġshow s -Ġs ite -Ġj o -Ġr isk -cl us -l t -Ġin j -id ing -ĠS pe -Ġch all -ir m -Ġ2 2 -itt ing -st r -Ġh y -L E -ke y -Ġbe gan -at ur -ashing ton -l am -ĠD av -b it -Ġs ize -ĠP ar -3 8 -ourn al -f ace -Ġdec ision -Ġl arg -Ġj ud -re ct -Ġcontin ue -ĠO ct -ove red -ĠI nt -==== ==== -Ġp arent -ĠW ill -Ġeas y -Ġd rug -ang er -Ġs ense -Ġd i -id ay -Ġener gy -ist ic -Ġass oci -ar ter -ob al -e ks -ĠE l -ur ch -Ġg irl -o e -it le -Ġ2 8 -ĠC he -Ġrequ est -Ġso on -Ġh ost -k y -Ġst ates -om es -Ġm aterial -le x -Ġmom ent -Ġan sw -on se -Ġes pecially -Ġn orm -Ġserv ices -p ite -r an -Ġro le -4 4 -) : -Ġc red -C l -____ ____ -Ġm at -Ġl og -ĠCl inton -O U -Ġoff ice -Ġ2 6 -Ġch arg -Ġtr ack -m a -Ġhe art -Ġb all -Ġperson al -Ġbuild ing -n a -s et -b ody -ĠBl ack -Ġincre ase -itt en -Ġneed ed -3 6 -3 2 -= " -Ġl ost -Ġbec ame -Ġgrou ps -ĠM us -Ġw rote -ĠP e -Ġpro p -j oy -à © -ĠWh ite -Ġde ad -. ' -Ġhtt p -Ġwe bs -O S -Ġins ide -Ġwr ong -Ġstat ement -Ġ ... -y l -Ġfil m -Ġmus ic -Ġsh are -ific ation -Ġre lease -Ġfor ward -Ġst ay -Ġcomp ut -it te -s er -Ġorig inal -Ġc ard -Ġc and -Ġd iv -at ural -Ġfav or -O M -Ġc ases -us es -Ġse ction -Ġle ave -g ing -ov ed -ĠW ashington -3 9 -ĠG l -Ġrequ ired -act ion -ap an -o or -it er -ĠK ing -Ġcount ries -ĠG erman -ll ing -Ġ2 7 -3 4 -Ġquest ions -Ġpr im -Ġc ell -Ġsh oot -Ġany one -ĠW est -Ġaff ect -ep end -Ġon line -ĠIs rael -ĠSept ember -Ġab ility -Ġcont ent -is es -Ġre ve -Ġl aun -Ġind ic -Ġfor ce -c ast -Ġso ld -av ing -f l -Ġso ft -Ġcompan ies -ce ed -Ġart icle -Ġa ud -Ġre v -Ġed uc -Ġplay ing -0 5 -Ġhe ld -ct or -Ġrele ased -Ġf ederal -3 7 -Ġad minist -Ġinter view -Ġinst all -Ġrece ived -Ġs ource -u k -P h -Ġser ious -Ġcre ated -Ġc ause -Ġim medi -Ġdef in -u el -ĠDep artment -ct ions -ĠC our -ĠN ow -z e -it es -it ution -Ġl ate -Ġspe ak -n ers -Ġleg al -ar i -ĠC or -Ġwe eks -Ġmod el -Ġp red -Ġex act -B C -ĠB y -IN G -os ing -Ġt akes -Ġreg ard -Ġopp ortun -Ġpr ice -Ġ19 8 -ĠA pr -f ully -Ġor d -Ġproble ms -ru ction -h am -ĠC ount -le ge -Ġlead ers -E T -le v -Ġde ep -olog ical -es e -h aps -ĠS ome -Ġp ers -Ġcont ract -Ġrelations hip -s p -ou d -Ġb ase -4 8 -m it -A d -anc ial -Ġcons um -Ġpot ential -Ġl angu -re m -et h -Ġrel ig -ress ed -6 6 -Ġl ink -Ġl ower -ay er -ĠJ une -Ġf em -un t -er c -ur d -Ġcont act -Ġ ill -Ġm other -Ġest ab -h tt -ĠM arch -ĠB ro -ĠCh ina -Ġ2 9 -Ġs qu -Ġprov ided -Ġa verage -as ons -Ġ201 1 -Ġex am -l in -5 5 -n ed -Ġper fect -Ġt ou -al se -u x -Ġbu y -Ġsh ot -Ġcol lect -Ġph ot -Ġplay ed -Ġsur pr -Ġofficial s -Ġsim ple -av y -Ġindust ry -Ġhand s -g round -Ġp ull -Ġr ound -Ġus er -Ġr ange -u ary -Ġpriv ate -op s -e es -Ġw ays -ĠM ich -Ġve h -Ġex cept -Ġter ms -im um -pp er -I ON -ore s -ĠDr agon -ou l -Ġd en -Ġperform ance -Ġb ill -c il -4 7 -Ġen vironment -Ġex c -ad d -Ġwor th -Ġp ict -Ġch ance -Ġ201 8 -b or -Ġspe ed -ict ion -Ġal leg -ĠJ apan -at ory -re et -Ġm atch -ĠI I -Ġst ru -ord er -Ġst e -Ġl iving -Ġst ruct -in o -Ġse par -her n -Ġresp onse -Ġen joy -Ġv ia -A D -um ents -ace book -Ġmem ber -ib r -iz ing -Ġto ol -ĠM on -ĠWh ile -h ood -ĠA ng -ĠD ef -Ġoff er -T r -a ur -Ġturn ed -ĠJ uly -d own -an ced -Ġrec ently -ĠE ar -Ġc e -ĠSt ar -ĠC ong -rough t -Ġbl ood -Ġhop e -Ġcom ment -ain t -Ġar ri -il es -Ġpartic ip -ough t -ri ption -0 8 -4 9 -Ġg ave -Ġse lect -Ġkill ed -sy ch -Ġgo es -i j -Ġc oll -Ġimp act -at ives -ĠS er -0 9 -ĠAug ust -Ġb oy -d e -ĠD es -Ġf elt -U S -Ġexpect ed -Ġim age -ĠM ark -cc ording -o ice -E C -ĠM ag -en ed -h old -ĠP ost -Ġpre vent -N o -Ġinvol ved -Ġey es -Ġquick ly -A t -un k -Ġbeh av -Ġ ur -Ġl ed -c ome -e y -Ġcand id -Ġear lier -Ġfoc us -et y -P ro -led ge -ix ed -ill ed -Ġpop ular -A P -Ġset t -l ight -Ġvar ious -in ks -Ġlevel s -Ġro ad -ell ig -ab les -he l -itte e -ĠG ener -y pe -Ġhe ard -ic les -Ġm is -Ġus ers -ĠS an -Ġimpro ve -Ġf ather -Ġse arch -The y -v il -Ġprof ess -Ġkn ew -Ġl oss -Ġev ents -6 5 -Ġb illion -0 7 -0 2 -ĠNew s -ĠA M -Ġco ver -w here -ens ion -Ġb ott -Ġare as -en ces -op e -ĠTw itter -a el -Ġget s -ĠGo ogle -Ġs n -i ant -Ġv ote -Ġnear ly -Ġinclud ed -Ġrec ogn -z z -m m -al ed -Ġhappen ed -0 4 -Ġh ot -Ġwho se -Ġc ivil -Ġsu ff -o es -it iz -ĠSy ri -Ġresp ond -Ġh on -Ġfeat ures -Ġeconom ic -ĠApr il -r im -Ġtechn ology -Ġo ption -ag ing -Ġpur ch -R e -Ġl at -ch ie -is l -Ġrec omm -u f -Ġtr aining -Ġeffect s -Ġf ast -Ġ201 0 -Ġocc ur -Ġwebs ite -Ġem ail -Ġs ens -e ch -Ġo il -Ġinf lu -Ġcurrent ly -ĠS ch -ĠAd d -Ġgo al -Ġsc ient -Ġcon v -1 00 -em y -Ġdec ided -Ġtra vel -Ġm ention -L L -0 3 -Ġe lection -Ġph one -Ġlook s -Ġsit uation -Ġc y -Ġh or -b ed -ĠCour t -a ily -av es -Ġqu ality -ĠCom p -w ise -Ġt able -Ġst aff -ĠW ind -et t -Ġtri ed -ide red -Ġadd ition -Ġb ox -Ġl ack -ar ily -Ġw ide -Ġm id -Ġbo ard -ys is -Ġant i -h a -Ġd ig -en ing -Ġd ro -C on -6 8 -Ġsl ow -b ased -se qu -Ġp ath -E x -ak er -Ġwork ed -Ġp en -Ġeng ine -Ġlook ed -ĠSu per -ĠS erv -Ġvict im -U n -Ġproper ty -Ġint rodu -Ġexec ut -ĠP M -L e -Ġcol or -ĠM ore -Ġ6 0 -Ġnet work -Ġd ate -c ul -id ge -Ġext ra -3 1 -Ġs le -6 7 -Ġw ond -Ġreport s -j ust -ĠAust ral -Ġcap ital -Ġen s -Ġcomm and -Ġallow ed -Ġpre p -Ġca pt -h ib -Ġnum bers -ch an -Ġf air -m p -om s -Ġre ach -W ith -t ain -Ġbro ad -Ġcou ple -ec ause -ly ing -ĠF eb -Ġsc reen -Ġl ives -Ġpri or -ĠCong ress -A r -Ġappro ach -Ġe mer -ar ies -ĠD is -s erv -ĠN e -Ġbu ilt -c ies -Ġre pe -Ġrul es -for ce -ĠP al -Ġfin ancial -Ġcons idered -ĠCh ar -n ces -ĠI S -Ġb rought -Ġb i -i ers -ĠS im -O P -Ġproduct s -Ġvis it -Ġdoc ument -Ġcon duct -Ġcomplete ly -in ing -ĠCal if -ib ly -Ġwr itten -ĠT V -em ents -Ġd raw -O ne -Ġpub lished -Ġsec ret -r ain -he t -ĠF acebook -ond ay -ĠU p -Ġsex ual -Ġth ous -ĠP at -Ġ ess -Ġstand ard -Ġar m -g es -ect ion -Ġf ell -Ġfore ign -an i -ĠFr iday -Ġreg ular -in ary -Ġincre ased -Ġus ually -Ġdem on -Ġd ark -Ġadd itional -ro l -ĠO f -Ġprodu ction -! ! -und red -Ġintern ational -id ents -ĠF ree -rou p -Ġr ace -Ġm ach -Ġh uge -A ll -le ar -ove mber -Ġto wn -Ġatt ention -ĠO ff -y ond -ĠThe n -f ield -Ġter ror -ra z -ĠB o -Ġmeet ing -ĠP ark -Ġar rest -Ġf ear -Ġa w -ĠV al -or ing -' , -Ġext reme -ar r -Ġwork ers -A fter -Ġ3 1 -n et -am ent -Ġdirect ly -Ġpop ulation -ub e -ĠOct ober -ĠI N -ĠJan uary -5 9 -ĠDav id -Ġc ross -ce mber -ĠF irst -Ġmess age -ir it -Ġn ation -Ġp oll -is ions -Ġansw er -n y -is ode -Ġcar ry -ĠRuss ia -Ġhe ar -eng th -ro y -Ġn atural -in ally -Ġdo g -m itted -Ġtr ade -Ġsub st -Ġmult iple -ĠAf ric -Ġf ans -Ġs ort -Ġgl obal -ic ation -ĠW ed -ar a -Ġa chie -Ġlangu age -ve y -Ġt al -Ġnecess ary -Ġdet ails -Ġs en -ĠS und -ĠRe g -ĠR ec -0 6 -Ġs il -ress ive -Ġmed ical -un ch -orn ia -Ġu nd -f ort -oc ks -ĠM onday -ues day -c raft -7 7 -ur t -Ġ ver -ĠH ill -Ġrece ive -Ġmor ning -es tern -Ġb ank -Ġs at -ir th -ĠH igh -Ġdev ice -ĠTH E -ĠCent er -Ġsaf e -Ġp le -ĠCanad a -Ġsystem s -Ġass ist -Ġsur v -Ġb attle -ĠS oc -vert is -S he -Ġp aper -Ġgrow th -Ġc ast -S c -Ġpl ans -ll ed -Ġpart s -Ġw all -Ġmove ment -Ġpract ice -im ately -Ġdis play -Ġsomet imes -om p -ĠP aul -ĠY es -k ing -5 8 -o ly -Ġs on -Ġav oid -ok es -ĠJ ew -Ġto wards -as c -Ġ // -ĠK ore -Ġtalk ing -Ġcor rect -Ġsp ent -ic ks -i able -e ared -Ġter m -Ġwant s -om ing -Ġ ut -Ġdou b -Ġfor ces -Ġp lease -6 9 -ĠN ovember -at form -ond on -Ġon es -Ġimmedi ately -ĠRuss ian -ĠM et -Ġde g -Ġparent s -C H -ĠAmeric ans -al y -ĠM od -Ġsh own -Ġcond itions -Ġst uff -Ġre b -ĠY our -Ġinclud es -n own -ĠS am -Ġexper ien -m ission -ĠE ven -augh t -Ġannoun ced -ĠRepublic an -Ġdeter min -Ġdescrib ed -ĠCount y -( ) -Ġdo or -Ġchang ed -Ġne igh -ĠH ere -Ġcle an -Ġp an -ĠDe cember -ĠEurope an -ir ing -ap ter -Ġcl ub -ĠT uesday -Ġp aid -ĠN et -Ġattack s -Ġcharact ers -Ġal one -Ġdirect or -d om -Ġ3 5 -Ġl oad -Ġr out -ĠCalif ornia -Ġfin ally -Ġr ac -Ġcont r -Ġexact ly -res h -p ri -ĠIs lam -Ġn ature -Ġcare er -Ġlat est -Ġcon vers -ĠS l -p ose -ci ent -ĠIn c -iv ity -8 8 -ĠA tt -ĠM or -nes day -Ġwe ight -k en -Ġnot e -Ġteam s -Ġ \ -air s -ĠG reen -Ġh undred -on ent -Ġstre ng -Ġcons ist -ic ated -Ġreg ul -Ġl ic -ast ic -Ġt en -urs day -ellig ence -ous ly -ĠU K -B I -Ġcost s -Ġind epend -ĠA P -Ġnorm al -Ġh om -Ġob vious -Ġs we -Ġst ar -Ġread y -ac her -Ġimp lement -g est -Ġs ong -ĠG et -ĠL ab -Ġinterest ing -us ing -Ġg iving -ĠSund ay -Ġet c -Ġm iddle -Ġrem ember -r ight -os ition -ut ions -Ġm ax -4 6 -Ġyour self -Ġdem and -Ġtreat ment -Ġd anger -ĠC ons -Ġgu y -ĠBrit ish -Ġphys ical -Ġrel ated -Ġrem ain -Ġcould n -Ġref er -Ġc itiz -b ox -EN T -bo ard -Ġin n -I G -er o -ĠSt reet -osp ital -ren ch -cher s -Ġst ra -O L -ag er -ĠA N -Ġeas ily -I A -en ge -in y -Ġcl os -ock ed -Ġus es -ĠC oun -I m -u ild -? ? -m ore -Ġan g -Ġwr ite -ol ute -5 7 -Ġlead er -Ġread ing -< / -Ġaut om -est s -4 3 -Ġleg isl -ĠG old -Ġdesign ed -ĠS T -ĠLe g -a res -Ġbe aut -ĠT ex -Ġappear s -Ġstru gg -ĠR om -Ġ 00 -Ġcho ice -Ġparticular ly -ĠF rom -op er -ĠL ondon -ann ed -Ġallow s -ob ile -Ġdiffere nce -âĢ ¢ -ĠV iew -ĠWed nesday -Ġal though -Ġrel ative -Ġapplic ation -ate ver -Ġare n -Ġmy self -Ġim ag -Ġdis e -Ġsoc iety -Ġfre qu -ĠEng lish -Ġpo or -ĠD ay -Ġwrit ing -Ġse ven -Ġstart ing -Ġb ud -Ġpr int -ĠTr ans -uf act -ĠSt ud -n ew -Ġcr im -Ġg ives -Ġco ol -a e -i ance -ĠGener al -Ġthink ing -Ġsa ve -Ġlim ited -ĠPart y -Ġmean ing -p en -ow ers -ĠJ ack -E M -Ġn ice -ru pt -Ġg as -Ġe ight -Ġfe et -Ġeff ort -Ġ ign -ic it -B l -co in -Ġop in -Ġbr ain -Wh ile -he st -ĠTh ursday -Ġwould n -augh ter -Ġtou ch -le ments -Ġstud ies -Ġcent er -c ont -or ge -Ġcomput er -Ġinvestig ation -P l -or ks -Ġ200 8 -Ġincre asing -Ġst ore -Ġcom ments -Ġb al -m en -Ġdo ll -Ġl iber -Ġw ife -Ġlaw s -atur day -it ness -Ġmod ern -ĠS k -Ġadminist ration -Ġopportun ity -Ġs al -Ġpower ful -M y -Ġclaim s -ĠEar th -ord s -Ġt itle -Ġes c -n ame -N ot -om en -Ġbe yond -Ġc amer -Ġse ll -it ute -ear ch -Ġapp l -im ent -4 2 -ĠAr t -Ġun f -Ġviol ence -ur g -ĠE ast -Ġcomp ared -Ġopt ions -Ġthrough out -Ġv s -ig r -. [ -ac hes -7 8 -Ġfil es -F L -E L -ar ian -ĠJ ames -ĠA ir -an ch -Ġdet ail -Ġpie ce -P S -Ġn amed -Ġeduc ation -Ġdri ve -Ġitem s -Ġstud ent -ic ed -: : -ic o -Ġth row -Ġsc ene -Ġcomple x -Ġ200 9 -Ġpre c -ĠB re -7 9 -Ġcon cept -Ġstat us -am ing -Ġd ied -Ġknow ledge -Ġbegin ning -O D -ru ary -Ġcertain ly -Ġgu ys -Ġsl ight -in n -ound s -Ġf ine -Ġf at -ic ations -Ġper haps -ĠA nt -Ġinc ome -Ġhtt ps -Ġmajor ity -port s -st on -Ġgreat er -Ġfe ed -ent ially -Ġsaf ety -Ġun ique -and om -Ġg one -Ġshow ed -Ġhist or -Ġcoun ter -i us -id a -Ġlead ing -i pe -Ġs end -ĠDon ald -er ve -Ġdef ense -ines e -Ġy es -ĠF ire -ĠMus lim -ra q -Ġcontin ued -os h -Ġprov ides -Ġpr ison -ĠP re -Ġhapp y -Ġeconom y -Ġtr ust -ag s -ĠG ame -Ġweap ons -um an -ĠC le -it ation -Ġanal ysis -ĠT imes -Ġsc ience -- > -Ġfig ure -Ġdis app -ent y -Ġsoft ware -Ġu lt -Ġoffic ers -N ew -I s -Ġrem ains -ĠInd ia -Ġp sych -ri ef -Ġc at -es c -Ġob serv -Ġst age -ĠD ark -Ġent er -ch ange -Ġpass ed -Ġdes pite -ĠO ut -Ġmov ie -r s -Ġv oice -m ine -ĠPl ay -Ġto ward -ĠT er -Ġreg ion -Ġval ues -or ters -Ġm ount -Ġoffic er -ĠO ther -b an -Ġh ous -w ood -ro om -I V -ĠS un -se e -ĠO ver -ro g -9 0 -Ġl ay -ĠT ur -a wn -Ġpress ure -ĠS ub -Ġbook s -ed om -ĠS and -A A -ag o -Ġre asons -f ord -Ġactiv ity -U T -N ow -ĠSen ate -ce ll -n ight -Ġcall s -in ter -Ġlet ter -ĠR ob -ĠJ e -Ġcho ose -ĠL aw -G et -B e -Ġro b -Ġtyp es -Ġpl atform -Ġqu arter -R A -ĠT ime -Ġmay be -ĠC r -9 5 -p re -Ġmov ing -Ġl if -Ġgo ld -Ġs om -Ġpat ients -Ġtr uth -ĠK e -ur ance -ant ly -m ar -Ġchar ge -ĠG reat -Ġce le ----------------- ---------------- -Ġro ck -ro id -an cy -Ġcred it -a ud -B y -ĠE very -Ġmov ed -ing er -rib ution -Ġn ames -Ġstra ight -ĠHe alth -ĠW ell -Ġfe ature -Ġr ule -Ġsc he -in ated -ĠMich ael -ber g -4 1 -il ed -b and -Ġcl ick -ĠAng el -on ents -Â Ń -ĠI raq -ĠS aturday -Ġa ware -p art -Ġpat tern -O W -ĠL et -Ġgr ad -ign ed -Ġassoci ated -Ġst yle -n o -i ation -a ith -il ies -Ġst ories -ur ation -Ġindividual s -ĠâĢ ¦ -m iss -ĠAss oci -ish ing -ab y -Ġsum mer -ĠB en -Ġ3 2 -Ġar ch -ut y -ĠTex as -h ol -Ġfull y -Ġm ill -Ġfollow ed -ĠB ill -ĠInd ian -ĠSec ret -ĠB el -ĠFeb ruary -Ġjob s -Ġseem ed -ĠGo vern -i pped -Ġreal ity -Ġl ines -Ġp ark -Ġmeas ure -ĠO ur -I M -Ġbro ther -Ġgrow ing -Ġb an -Ġest im -Ġc ry -ĠS chool -Ġme chan -ĠO F -ĠWind ows -Ġr ates -ĠO h -Ġpos itive -Ġcult ure -ist ics -ic a -Ġh ar -y a -ite ly -i pp -Ġm ap -en cies -ĠWill iam -I I -ak ers -5 6 -ĠM art -ĠR em -Ġal tern -it ude -Ġco ach -row d -D on -Ġk ids -Ġj ournal -Ġcor por -Ġf alse -Ġwe b -Ġsle ep -Ġcont ain -Ġst o -Ġb ed -iver se -ĠR ich -ĠCh inese -Ġp un -Ġme ant -k nown -Ġnot ice -Ġfavor ite -a ven -Ġcond ition -Ġpur pose -) ) -Ġorgan ization -Ġchall eng -Ġman ufact -Ġsus p -ĠA c -Ġcrit ic -un es -uc lear -Ġm er -vent ion -Ġ8 0 -Ġm ist -ĠU s -ĠT or -htt p -ol f -Ġlarg er -Ġadv ant -Ġrese ar -Ġact ions -m l -Ġke pt -Ġa im -, ' -c ol -Ġbenef its -if ying -Ġact ual -ĠIntern ational -Ġveh icle -Ġch ief -Ġeff orts -ĠLe ague -ĠM ost -Ġwa it -Ġad ult -Ġover all -Ġspe ech -Ġhigh ly -Ġfem ale -Ġer ror -Ġeffect ive -5 4 -Ġenc our -w ell -Ġfail ed -Ġcons erv -Ġprogram s -Ġt rou -Ġa head -5 00 -vertis ement -I P -ĠF ound -p ir -Ġ % -Ġcr ime -and er -Ġloc ation -ĠI ran -Ġbehav ior -az ing -Ġr are -Ġem b -Ġca used -Ġsh ip -Ġact ive -Ġcont ribut -Ġg reen -Ġac qu -Ġref lect -ven ue -Ġf irm -Ġb irth -] . -Ġclear ly -Ġem ot -Ġag ency -ri age -Ġmem ory -9 8 -S A -ĠSe e -ac ing -C C -Ġbig gest -Ġr ap -Ġbas ic -Ġb and -e at -Ġsus pect -ĠM ac -Ġ9 0 -m ark -ist an -Ġsp read -am s -k i -as y -ra v -ĠR ober -Ġdemon str -r ated -Ġabs olute -Ġpl aces -Ġim pl -ibr ary -Ġc ards -Ġdest roy -Ġv irt -ve re -Ġapp eared -y an -p oint -Ġbe g -Ġtem per -s pe -ant ed -ear s -ĠD irect -Ġl ength -Ġbl og -am b -Ġint eg -Ġres ources -ac c -if ul -Ġsp ot -Ġfor ced -Ġthous ands -ĠMin ister -Ġqu al -ĠF rench -at ically -Ġgener ally -Ġdr ink -Ġth us -I L -od es -Ġappro pri -ĠRe ad -Ġwh om -Ġey e -Ġcol lege -Ġ4 5 -ire ction -Ġens ure -Ġapp arent -id ers -Ġrelig ious -Ġmin or -ol ic -Ġt ro -ĠWh y -rib ute -m et -Ġprim ary -Ġdevelop ed -Ġpe ace -Ġsk in -st e -av a -Ġbl ue -Ġfam ilies -Ġ ir -Ġapp ly -Ġin form -ĠSm ith -C T -i i -Ġlim it -Ġres ist -........ ........ -um n -Ġconf lic -Ġtw e -ud d -ĠT om -Ġl iter -qu e -b on -Ġha ir -Ġevent ually -Ġp us -Ġhelp ed -Ġag g -or ney -ĠApp le -Ġf it -ĠS ur -Ġpre m -Ġs ales -Ġsecond s -Ġstreng th -Ġfeel ing -¿ ½ -Ġt our -Ġknow s -o om -Ġex erc -Ġsom ew -ï ¿½ -> > -Ġsp okes -Ġide as -Ġreg ist -so ft -ĠD el -ĠP C -Ġpro pos -Ġlaun ch -Ġbott om -T H -ĠP lease -v est -it z -ĠIn ter -Ġsc ript -Ġr at -ar ning -Ġ il -ĠJ er -ĠA re -Ġwh atever -ok en -ci ence -Ġmod e -Ġag ree -Ġs ources -Ġinit ial -Ġrest rict -Ġwond er -us ion -## ## -ĠS il -vil le -Ġb urn -t w -as ion -Ġ £ -Ġn or -u ing -Ġre ached -Ġs un -Ġc ateg -ig ration -Ġc ook -Ġprom ot -Ġm ale -Ġcl imate -Ġf ix -Ġalleg ed -U R -all ed -Ġim ages -C ont -ot a -Ġschool s -i os -Ġd rop -Ġst ream -ĠM o -Ġprevious ly -al ing -Ġp et -Ġdou ble -Ġ( @ -ann el -Ġdef ault -t ies -Ġr ank -ĠD ec -ĠCoun cil -Ġweap on -Ġst ock -Ġanal y -ĠSt r -Ġpict ure -ĠPol ice -f erence -Ġcent ury -Ġcitiz ens -Ġon to -Ġexp and -Ġhe ro -ĠS ol -Ġw ild -Ġupd ate -Ġcustom ers -r ont -d ef -Ġl ik -Ġcrim inal -ĠChrist ian -S P -7 6 -Ġle aving -Ġother wise -ĠD ist -Ġbas is -5 2 -5 3 -ic ip -ĠB er -Ġrecomm end -Ġfl oor -Ġc rowd -ol es -Ġ7 0 -Ġcent ral -ĠE v -Ġd ream -Ġdown load -Ġconf ir -ĠTh om -Ġwind ow -Ġhapp ens -Ġun it -Ġt end -Ġs pl -Ġbec omes -Ġfight ing -Ġpred ict -ĠP ress -ĠP ower -Ġhe avy -ak ed -Ġf an -or ter -ate gy -B A -iz es -Ġsp end -H ere -Ġ200 7 -Ġad op -ĠH am -Ġfoot ball -ĠP ort -od ay -5 1 -amp ions -Ġtrans fer -h t -Ġ3 8 -ter m -ac ity -Ġb ur -] , -tern al -r ig -b ut -Ġthere fore -ĠB ecause -res p -re y -Ġm ission -S ome -Ġnot ed -Ġass um -Ġdise ase -Ġed it -Ġprog ress -r d -ĠB rown -oc al -Ġadd ing -Ġra ised -ĠAn y -Ġt ick -Ġsee ing -ĠPe ople -Ġagre ement -Ġser ver -Ġw at -Ġdeb ate -Ġsupp osed -il ing -Ġlarg est -Ġsuccess ful -ĠP ri -ĠDemocr atic -Ġj ump -ĠSyri a -Ġown ers -Ġoff ers -Ġshoot ing -Ġeff ic -se y -Ġha ven -ver se -te red -ĠL ight -im al -ĠB ig -Ġdef end -Ġbe at -Ġrecord s -% ) -Ġsc en -Ġemploy ees -Ġdev ices -he m -Ġcom mer -ĠM ex -Ġbenef it -ĠPro f -Ġil leg -Ġsur face -ĠAl so -Ġh arm -ing ly -w ide -ĠA lex -Ġsh ut -ĠC ur -Ġl ose -p m -Ġchall enge -se mb -Ġst ation -Ġint elligence -Ġacc ur -ĠFl or -Ġrequ ires -ĠM al -b um -Ġh ospital -Ġsp irit -Ġoff ered -Ġprodu ce -ĠComm un -Ġcreat ing -Ġcr is -s pect -Ġend ed -Ġd aily -Ġvot ers -land s -i as -i h -on a -Ġsm art -ĠOff ice -ĠL ord -ri al -ĠIntern et -Ġcirc um -Ġextreme ly -' . -Ġopin ion -ĠM il -Ġg ain -B S -ĠF in -y p -Ġuse ful -Ġbud get -Ġcom fort -is f -Ġback ground -el ine -Ġep isode -Ġen emy -Ġtri al -Ġestab lish -d ate -ĠC ap -Ġcontin ues -Ġshow ing -ĠUn ion -w ith -Ġpost ed -ĠSy stem -Ġe at -ri an -Ġr ise -ĠGerman y -il s -Ġsign ed -Ġv ill -Ġgr and -m or -ĠEng land -Ġproject s -um ber -Ġconf erence -z a -Ġrespons ible -ĠAr ab -Ġlearn ed -âĢĶ âĢĶ -i pping -ĠGe orge -O C -Ġreturn ed -ĠAustral ia -Ġb rief -Q u -Ġbr and -ill ing -ab led -Ġhig hest -Ġtr ain -ĠComm ission -wh ile -Ġn om -cept ion -Ġm ut -ĠBl ue -Ġinc ident -v ant -8 6 -ĠI D -Ġn uclear -7 4 -ĠL ike -ĠR E -ĠM icro -l i -m ail -Ġcharg es -8 9 -Ġad just -ad o -Ġear th -N A -Ġpr ices -P A -Ġd raft -Ġrun s -Ġcandid ate -ens es -Ġmanag ement -ĠPh il -ĠM iss -Ġte ach -g ram -Ġunderstand ing -a it -ic ago -A dd -ĠE p -sec ut -Ġsepar ate -Ġinst ance -Ġe th -Ġun less -**** **** -ĠF ore -in ate -Ġoper ations -S p -Ġf aith -g ar -ĠCh urch -ron ic -Ġconf ig -os ure -Ġactiv ities -Ġtrad itional -Ġ3 6 -Ġd irection -Ġmach ine -Ġsur round -Ġp ush -un ction -ĠE U -Ġeas ier -Ġarg ument -G B -Ġm icro -Ġsp ending -iz ations -Ġthe ory -ad ow -Ġcall ing -ĠL ast -Ġd er -Ġinflu ence -Ġcomm it -Ġph oto -Ġun c -ist ry -g n -ast e -ack s -Ġdis p -ad y -d o -ĠG ood -Ġ ` -Ġw ish -Ġreve aled -Âł Âł -l ig -Ġen force -ĠComm ittee -Ġche m -Ġmil es -Ġinterest ed -Ġsol ution -ic y -in ct -Ġ- > -ĠD et -Ġrem oved -Ġcomp ar -e ah -Ġpl ant -ĠS ince -Ġachie ve -Ġadvant age -Ġslight ly -b ing -Ġpl aced -u nder -201 5 -ĠM ad -Ġt im -os es -Ġc ru -ĠR ock -Ġmost ly -Ġneg ative -Ġset ting -Ġprodu ced -Ġm ur -Ġconnect ion -ĠM er -Ġdri ver -Ġexecut ive -Ġass ault -Ġb orn -ĠV er -t ained -Ġstruct ure -Ġredu ce -Ġdec ades -Ġd ed -u ke -ĠM any -idd en -Ġle ague -S e -Ġjo in -Ġdis co -Ġd ie -c ks -act ions -Ġass ess -ag n -Ġgo als -our s -I R -Ġsen ior -ill er -m od -ip ment -oc ol -u y -ĠQ ue -Ġpart ies -ir gin -Ġle arning -it able -Ġstre et -Ġcamer a -A pp -Ġsk ills -b re -c ious -Ġcele br -ĠFr anc -Ġexist ing -Ġwill ing -l or -Ġ id -ĠSp ace -Ġcrit ical -ĠL a -ortun ately -Ġser ve -Ġc old -Ġspec ies -T S -Ġanim als -ĠB ay -Ġold er -ĠU nder -est ic -ĠT re -Ġte acher -Ġpre fer -v is -Ġth read -ĠM att -Ġmanag er -ãĥ » -Ġprofess ional -ĠV ol -Ġnot es -The se -ul a -Ġf resh -ent ed -u zz -ed y -clus ion -ĠR el -Ġdoub t -E O -Ġopen ed -ĠB it -Ad vertisement -Ġgu ess -ĠU N -Ġse qu -Ġexpl ain -ott en -Ġatt ract -ak s -Ġstr ing -Ġcont ext -oss ible -ĠRepublic ans -Ġsol id -Ġc ities -Ġask ing -Ġr andom -u ps -ur ies -ar ant -dd en -g l -ĠFlor ida -Ġdep end -ĠSc ott -Ġ3 3 -Ġi T -ic on -Ġmention ed -Ġ2 000 -Ġclaim ed -Ġdefin itely -ul f -Ġc ore -Ġopen ing -ĠCon st -wh ich -ĠT ra -A G -7 2 -Ġbelie ved -ad a -Ġ4 8 -ĠSec urity -yr ight -ĠP et -ĠL ou -Ġhold ing -======== ======== -Ġ ice -Ġb row -Ġauthor ities -h ost -w ord -Ġsc ore -ĠD iv -Ġcell s -Ġtrans l -Ġneigh bor -Ġrem ove -u ct -Ġdist rict -ĠA ccording -Ġwor se -Ġconcern s -Ġpresident ial -Ġpolic ies -ĠH all -7 3 -Ġh us -A Y -Ġ200 6 -ĠJ ud -Ġindepend ent -ĠJust ice -ili ar -pr int -igh ter -Ġprotect ion -z en -Ġsu dden -h ouse -ĠJ es -P R -ĠIn f -Ġb ul -Ġ _ -ĠServ ice -ĠP R -Ġstr ategy -ff ect -Ġgirl s -Ġmiss ing -oy al -ĠTe am -ul ated -Ġd at -Ġpolit ics -ab or -A ccording -Ġspe ll -Ġg raph -ort hern -T C -A b -Ġlab or -is her -Ġk ick -ĠiT unes -Ġstep s -pos es -Ġsmall er -E n -ber t -Ġro ll -Ġresear chers -Ġcl osed -Ġtrans port -Ġlaw y -________ ________ -ĠCh icago -Ġas pect -Ġn one -Ġmar riage -9 6 -Ġe lements -ĠF re -ĠS al -Ġd ram -F C -t op -e qu -Ġhe aring -Ġsupport ed -Ġtest ing -co hol -Ġmass ive -Ġst ick -Ġgu ard -is co -ph one -F rom -How ever -Ġb order -Ġcop y -ograph y -l ist -7 1 -Ġown er -cl ass -ru it -r ate -ĠO nce -Ġdig ital -Ġt ask -ER S -Ġinc red -t es -+ + -ĠFr ance -Ġb reat -ow l -Ġiss ued -ĠW estern -Ġdet ect -Ġpart ners -Ġsh ared -ĠC all -Ġcan cer -ac he -rib e -Ġexpl ained -Ġhe at -{ " -Ġinvest ment -ĠB ook -Ġw ood -Ġtool s -ĠAl though -Ġbelie f -Ġcris is -Ġg e -ĠM P -Ġoper ation -ty pe -~ ~ -g a -Ġcont ains -ant a -Ġexp ress -ĠG roup -ĠJ ournal -k a -Ġam b -ĠUS A -Ġfind ing -Ġfund ing -h ow -Ġestab lished -ide os -Ġdeg ree -Ġdanger ous -ang ing -Ġfre edom -pp ort -out hern -Ġch urch -Ġc atch -ĠTw o -Ġpres ence -ĠGu ard -U p -Ġauthor ity -ĠPro ject -Ġbut ton -Ġcon sequ -Ġval id -Ġwe ak -Ġstart s -Ġref erence -ĠM em -" ) -U N -or age -ĠO pen -Ġcol lection -y m -g ency -Ġbeaut iful -ro s -Ġtell s -Ġwa iting -n el -Ġprov iding -ĠDemocr ats -Ġd aughter -Ġm aster -Ġpur poses -ĠJapan ese -Ġequ al -Ġturn s -Ġdoc uments -Ġwatch ing -R es -Ġr an -201 4 -Ġre ject -ĠKore a -Ġvictim s -Le vel -ere nces -Ġw itness -Ġ3 4 -Ġre form -com ing -Ġocc up -Ġc aught -Ġtra ffic -ad ing -Ġmod els -ar io -Ġserv ed -Ġb atter -u ate -ĠSecret ary -Ġagre ed -Ġtr uly -yn am -ĠR et -Ġun its -ĠRes earch -h and -az ine -ĠM ike -Ġvar iety -ot al -Ġam azing -Ġconfir med -Ġentire ly -Ġpurch ase -Ġe lement -Ġc ash -Ġdeter mine -D e -Ġc ars -ĠW all -â ĸ -Ġview s -Ġdrug s -Ġdep artment -ĠSt ep -u it -Ġ3 9 -as ure -ĠCl ass -Ġc overed -ĠB ank -Ġme re -u ana -Ġmult i -Ġm ix -Ġun like -lev ision -Ġsto pped -Ġs em -ĠG al -ul es -Ġwe l -ĠJohn son -l a -Ġsk ill -Ġbec oming -ri e -Ġappropri ate -f e -ell ow -ĠPro t -ul ate -oc ation -Ġweek end -od ies -Ġsit es -Ġanim al -ĠT im -Ġsc ale -Ġcharg ed -Ġinst ruct -ill a -Ġmethod s -Ġc ert -Ġjud ge -ĠH el -Ġdoll ars -Ġstand ing -ĠS qu -Ġdeb t -l iam -Ġdri ving -ĠS um -ĠEd ition -Ġal bum -and on -I F -ĠU k -6 3 -ad er -Ġcommer cial -es h -ĠGovern ment -Ġdisc overed -Ġout put -ĠHill ary -ĠCar ol -Ġ200 5 -Ġab use -anc ing -Ġsw itch -Ġann ual -T w -Ġst ated -ag ement -in ner -Ġdem ocr -Ġres idents -Ġallow ing -Ġfact ors -od d -Ġf uck -em ies -Ġoccur red -ot i -Ġn orth -ĠP ublic -Ġinj ury -Ġins urance -C L -oll y -ã Ģ -Ġrepe ated -Ġar ms -ang ed -Ġconst ruction -Ġf le -P U -ic ians -Ġfor ms -ĠMc C -ant ic -Ġm ental -p ire -Ġequ ipment -Ġf ant -Ġdiscuss ion -Ġregard ing -k in -ar p -Ġch air -og ue -Ġpro ceed -ĠI d -O ur -Ġmur der -M an -Ġ4 9 -as p -Ġsupp ly -Ġin put -Ġwe alth -liam ent -Ġpro ced -or ial -ĠSt at -ĠN FL -hen s -ĠInst itute -Ġput ting -ourn ament -et ic -Ġloc ated -Ġk id -er ia -r un -Ġpr inc -Ġ ! -go ing -ĠB et -Ġcl ot -Ġtell ing -Ġprop osed -i ot -or ry -Ġfund s -g ment -ĠL ife -Ġb aby -ĠB ack -Ġsp oke -Im age -Ġear n -ĠA T -g u -Ġex change -ĠL in -ov ing -Ġp air -M ore -az on -Ġarrest ed -Ġkill ing -c an -ĠC ard -y d -Ġident ified -Ġm obile -Ġthan ks -ony m -ĠF orm -Ġhundred s -ĠCh ris -ĠC at -Ġtre nd -h at -ĠA v -om an -Ġelect ric -ĠW il -S E -O f -Ġrest aur -ot ed -Ġtr ig -Ġn ine -Ġb omb -Wh y - ¯ -Ġco verage -Ġapp eal -ĠRober t -ĠS up -Ġfin ished -Ġfl ow -Ġdel iver -Ġcal cul -Ġphot os -Ġph il -Ġpie ces -Ġapp re -k es -Ġr ough -D o -Ġpart ner -Ġconcern ed -Ġ3 7 -ĠG en -C ol -ct ors -Ġ= > -st ate -Ġsuggest ed -ĠFor ce -C E -Ġher self -ĠPl an -w orks -o oth -ren cy -Ġcor ner -Ġhus band -Ġintern et -ĠA ut -em s -os en -ĠAt l -g en -Ġbal ance -6 2 -Ġsound s -te xt -Ġar r -ov es -Ġmill ions -Ġrad io -Ġsat isf -ĠD am -M r -G o -S pe -Ġcomb at -r ant -ĠG ree -Ġf uel -Ġdist ance -Ġtest s -Ġdec re -ĠE r -Ġman aged -D S -Ġt it -Ġmeas ures -ĠL iber -Ġatt end -as hed -ĠJ ose -ĠN ight -d it -ĠN ov -ĠE nd -out s -Ġgener ation -Ġadv oc -y th -Ġconvers ation -ĠS ky -act ive -ce l -ri er -ĠFr ank -Ġg ender -Ġcon cent -Ġcar ried -and a -ĠV irgin -Ġarri ved -ic ide -ad ed -Ġfail ure -Ġmin imum -le ts -Ġwor st -Ġkeep ing -Ġint ended -Ġilleg al -Ġsub sc -Ġdetermin ed -Ġtri p -Y es -Ġra ise -Ġ ~ -Ġfeel s -Ġpack age -ĠJ o -h i -201 6 -re al -Ġf ra -Ġsy mb -M e -uck y -p ret -ĠK h -ĠEd it -ĠWe b -em ic -ĠCol or -Ġjust ice -I nt -Ġfar m -ck now -" > -el ess -Ġredu ced -Ġ5 00 -x x -ĠR ad -ĠW ood -Ġcl in -Ġhy p -il er -ur a -k ins -8 5 -6 1 -ĠThe ir -ĠM ary -Ġs an -Ġno vel -ĠWh o -Ġcap acity -Ġimp ossible -Ġpl ays -Ġmin ister -ij uana -ic ate -ĠS et -Ġf ram -Ġ ing -Ġcommun ities -ĠF BI -it a -Ġb on -Ġstr ateg -Ġinterest s -l ock -g ers -m as -ĠAN D -Ġconflic t -Ġrequire ments -Ġs ac -Ġoper ating -in i -rel ated -Ġcomm itted -Ġrelative ly -Ġs outh -¯ ¯ -Ġaff ord -Ġident ity -Ġdec isions -Ġacc used -pl ace -Ġvict ory -o ch -i at -N ame -C om -t ion -ed s -Ġsee k -Ġt ight -ĠIm ages -Ġinit i -Ġhum ans -Ġfam iliar -Ġaud ience -Ġintern al -vent ure -Ġs ides -ĠT O -Ġd im -Ġcon clud -Ġapp oint -Ġenforce ment -ĠJ im -ĠAssoci ation -Ġcircum st -ĠCanad ian -Ġjo ined -Ġdiffere nces -ĠL os -Ġprot est -Ġtw ice -w in -Ġgl ass -ars h -ĠAr my -Ġexp ression -Ġdec ide -Ġplan ning -an ia -Ġhand le -ĠMicro soft -ĠN or -Ġmax imum -ĠRe v -Ġse a -Ġev al -Ġhel ps -re f -Ġb ound -Ġm outh -Ġstand ards -Ġcl im -ĠC amp -ĠF ox -cl es -Ġar my -ĠTe chn -ack ing -x y -S S -Ġ4 2 -Ġbu g -ĠUk rain -ĠM ax -ĠJ ones -ĠSh ow -l o -Ġplan et -Ġ7 5 -Ġwin ning -Ġf aster -Ġspe ct -Ġbro ken -T R -Ġdef ined -Ġhealth y -Ġcompet ition -htt ps -ĠIs land -ĠF e -Ġannoun ce -ĠC up -ĠInst ead -Ġcl ient -Ġposs ibly -se ction -ock et -l ook -Ġfin ish -Ġcre w -Ġres erv -Ġed itor -Ġh ate -Ġs ale -Ġcontro vers -Ġp ages -w ing -Ġnum er -Ġopp osition -Ġ200 4 -Ġref uge -Ġfl ight -Ġap art -ĠL at -A meric -ĠAfric a -Ġapplic ations -ĠPal est -ĠB ur -Ġg ar -ĠSoc ial -Ġup gr -Ġsh ape -Ġspe aking -ans ion -a o -ĠS n -Ġwor ry -ĠBrit ain -P lease -rou d -Ġh un -Ġintrodu ced -Ġd iet -I nd -ĠSec ond -Ġfun ctions -ut s -ĠE ach -ĠJe ff -Ġst ress -Ġaccount s -Ġgu arant -ĠAn n -ed ia -Ġhon est -Ġt ree -ĠAfric an -ĠB ush -} , -Ġs ch -ĠOn ly -Ġf if -ig an -Ġexerc ise -ĠEx p -Ġscient ists -Ġlegisl ation -ĠW ork -ĠS pr -à Ĥ -ĠH uman -Ġ è -Ġsur vey -Ġr ich -ri p -Ġmain tain -Ġfl o -Ġleaders hip -st ream -ĠIslam ic -Ġ 01 -ĠCol lege -Ġmag ic -ĠPr ime -Ġfig ures -201 7 -ind er -x ual -ĠDe ad -Ġabsolute ly -Ġfour th -Ġpresent ed -resp ond -rib le -Ġal cohol -at o -ĠD E -por ary -Ġgr ab -Ġvar i -Ġqu ant -ĠPh oto -Ġpl us -r ick -ar ks -Ġaltern ative -Ġp il -Ġappro x -th at -Ġobject s -ĠR o -ĠAnd roid -Ġsignificant ly -ĠR oad -k ay -R ead -av or -Ġa cknow -ĠH D -ĠS ing -O r -ĠM ont -Ġun s -pro f -Ġneg oti -ĠAr ch -ik i -Ġte levision -ĠJew ish -Ġcomm ittee -Ġmot or -Ġappear ance -Ġs itting -Ġstri ke -ĠD own -com p -ĠH ist -Ġf old -ac ement -ĠLou is -Ġbel ong -ĠâĢ ¢ -Ġm ort -Ġprep ared -Ġ6 4 -ĠM aster -Ġind eed -ĠD en -Ġre nt -T A -our ney -ar c -S u -9 7 -Ġadv ice -Ġchang ing -Ġlist ed -Ġlaun ched -is ation -ĠP eter -is hes -Ġl ived -ĠM el -ĠSup reme -ĠF ederal -Ġ) ; -ruct ure -Ġset s -Ġphil os -u ous -Ġ ł -Ġappl ied -ĠN OT -Ġhous ing -ĠM ount -Ġo dd -Ġsu st -D A -ffic ient -Ġ ? -ol ved -Ġp owers -Ġth r -Ġrem aining -ĠW ater -L C -Ġca uses -ãģ ® -Ġman ner -ad s -Ġsuggest s -Ġend s -stand ing -f ig -ĠD un -id th -Ġg ay -Ġter min -ĠAngel es -M S -Ġscient ific -Ġco al -ap ers -b ar -ĠThom as -Ġsy m -ĠR un -th is -P C -igr ants -Ġmin ute -ĠDist rict -cell ent -Ġle aves -Ġcomple ted -am in -Ġfoc used -Ġmon itor -Ġveh icles -M A -ĠM ass -ĠGr and -Ġaffect ed -itution al -Ġconst ruct -Ġfollow s -Ġt on -re ens -Ġh omes -ĠE xt -ĠLe vel -r ast -ĠI r -Ġel im -Ġlarge ly -ĠJ oe -Ġvot es -all s -Ġbusiness es -ĠFound ation -ĠCent ral -Ġy ards -Ġmaterial s -ul ner -Ġgu ide -Ġclos er -um s -Ġsp orts -ed er -J ust -Ġtax es -8 4 -ĠO ld -Ġdec ade -ol a -Ġv ir -Ġdro pped -Ġdel ay -it ect -Ġsec ure -ste in -le vel -Ġtre ated -Ġfil ed -ain e -Ġv an -Ġm ir -Ġcol umn -ict ed -e per -Ġro t -Ġcons ult -Ġent ry -Ġmar ijuana -ĠD ou -Ġapparent ly -ok ing -clus ive -Ġincre ases -an o -Ġspecific ally -Ġte le -ens ions -Ġrelig ion -ab ilities -Ġfr ame -ĠN ote -ĠLe e -Ġhelp ing -Ġed ge -ost on -Ġorgan izations -à ĥ -ĠB oth -hip s -Ġbig ger -Ġbo ost -ĠSt and -Ġro w -ul s -ab ase -Ġr id -L et -are n -ra ve -Ġst ret -P D -Ġv ision -Ġwe aring -Ġappre ci -Ġa ward -ĠU se -Ġfact or -w ar -ul ations -) ( -Ġg od -Ġter rit -Ġpar am -ast s -8 7 -Ġen emies -ĠG ames -F F -Ġacc ident -W ell -ĠMart in -T ER -Ġat h -ĠHe ll -Ġfor g -Ġve ter -ĠMed ic -f ree -Ġst ars -Ġexp ensive -Ġac ad -ra wn -ĠW he -Ġl ock -Ġform at -Ġsold iers -s m -Ġag ent -Ġrespons ibility -or a -ĠS cience -Ġrap id -Ġt ough -ĠJes us -Ġbelie ves -M L -Ġwe ar -le te -Ãĥ ÃĤ -ĠD ri -Ġcomm ission -ĠB ob -O h -ap ed -Ġwar m -ÃĥÃĤ ÃĥÃĤ -Ġ200 3 -ort ion -Ġhas n -ust er -Ġun ivers -ĠI ll -Ġk ing -olog ies -9 4 -ĠT em -ĠM os -Ġpat ient -ĠMex ico -ce an -ĠDe ath -ĠSand ers -y ou -ĠC ast -ĠComp any -pt y -Ġhappen ing -F P -ĠB attle -Ġb ought -A m -M od -U s -ut ers -ĠC re -ĠTh ose -Ġ4 4 -is er -Ġs oul -ĠT op -ĠHar ry -ĠA w -Ġse at -ff ee -Ġrev olution -Ġ( " -ĠD uring -et te -Ġr ing -Ġoff ensive -Ġreturn s -Ġv ideos -Ġdis cl -Ġfam ous -en ced -ĠS ign -ĠR iver -Ġ3 00 -P M -ĠB us -ĠC H -Ġcandid ates -ard en -Ġpercent age -Ġvis ual -Ġthan k -Ġtrou ble -ner gy -Ġ200 1 -Ġpro ve -ash ion -Ġen h -ĠL ong -U M -Ġconnect ed -Ġposs ibility -O ver -Ġexper t -Ġl ibrary -art s -ĠDirect or -Ġfell ow -9 2 -ir ty -Ġd ry -Ġsign s -ĠL ove -Ġqu iet -f oot -Ġp ure -ĠH un -Ġf illed -ph as -ĠE lect -end ment -ĠEx pl -Ġun able -n s -m o -Ġv ast -ob e -Ġident ify -app ing -ĠCarol ina -g ress -Ġpro te -Ġf ish -Ġcircumst ances -raz y -ĠPh ot -Ġb odies -ĠM ur -Ġdevelop ing -ĠA R -Ġexperien ced -Ġsubst ant -ĠBo ard -es ome -Ġdom estic -Ġcomb ined -ĠP ut -Ġchem ical -ĠCh ild -Ġpo ol -ĠC y -Ġe gg -c ons -st ers -Ġh urt -Ġmark ets -Ġconserv ative -Ġsupp orters -Ġag encies -id el -O b -ur b -Ġ4 3 -ĠDef ense -y e -ĠA p -du le -Ġtemper ature -Ġconduct ed -ĠCh ief -Ġpull ed -Ġf ol -L ast -ont o -os is -V ER -D es -ĠP an -F irst -Ġadv ance -Ġlic ense -r ors -ĠJ on -Ġimag ine -Ġhe ll -Ġf ixed -Ġinc or -os ite -ĠL og -ick en -] : -Ġsurpr ise -h ab -Ġc raft -ol t -ĠJ ul -Ġd ial -Ġrele vant -Ġent ered -Ġlead s -ĠA D -ĠCle an -Ġpict ures -ess or -Ġal t -Ġpay ing -P er -ĠMark et -Ġupd ates -am ily -ĠT ype -ĠH ome -Ġ5 5 -semb ly -rom e -8 3 -Ġgreat est -Ġhe ight -Ġhe av -ain ts -Ġlist en -as er -ĠS H -Ġcap able -ac le -Ġpers pect -in ating -Ġoff ering -ry pt -ĠDe velop -ab in -r c -Ġbr ight -al ty -ar row -Ġsupp l -ind ing -ack ed -gy pt -ĠAn other -p g -ĠVirgin ia -ĠL u -Ġpl anned -Ġp it -Ġswe et -T ype -ĠD i -Ġtyp ically -ĠFranc isco -Ġpro spect -ĠD an -Ġte en -re es -Ġsc hed -Ġh ol -Ġsc r -Ġlot s -l ife -Ġnews p -Ġfor get -ĠN one -ĠM iddle -ĠR yan -ed d -Ġse vere -Ġsu it -ll er -9 3 -Ġcor respond -Ġexpl os -u ations -Ġfl ag -g ame -r id -Ġpr in -ĠD ata -Ġde ploy -ĠEn ter -su it -gh an -ĠM en -Ġthough ts -Ġmat ters -Ġad apt -ĠA ri -Ġf ill -Ġfor th -Ġs am -Ġ4 1 -Ġpay ment -ĠH or -Ġsp ring -du c -Ġl osing -Ġbring ing -F O -al a -Ġdist ribution -he red -b our -ĠIsrael i -om a -Ġcomb ination -Ġpl enty -V E -C an -ĠH aw -Ġper man -ĠSpe cial -Ġto w -Ġsee king -Ġexam ples -Ġclass es -c r -Ġbe er -Ġmov es -ĠI P -ĠK n -Ġpan el -E ven -Ġproper ly -Ġr is -Ġpl ug -Ġestim ated -E very -Ġdef ensive -ag raph -Ġpre gn -Ġinst it -ĠV ict -Ġvol ume -Ġpos itions -Ġl inks -ĠPro gram -ĠWe ek -ag ues -Ġtrans form -k er -ĠC EO -Ġc as -Ġopp onent -Ġtwe et -ĠC ode -Ġsh op -Ġf ly -Ġtal ks -Ġb ag -Ph one -Ġa id -Ġpl ants -Ġ6 5 -Ġatt orney -ar ters -qu est -ĠMag ic -Ġbeg ins -Ġmy ster -Ġenvironment al -Ġst orage -N N -Ġm arg -Ġs ke -Ġmet al -ell y -Ġord ered -Ġrem ained -Ġl oved -Ġprom pt -Ġupd ated -Ġexper ts -Ġwalk ing -Ġan cient -Ġperform ed -AT E -Ġne ither -i ency -Ġmanufact ure -ĠP ak -Ġselect ed -Ġm ine -Ġult imately -Ġexpl an -Ġlab el -ĠServ ices -ribut ed -Tr ump -Ġsy n -ĠU lt -S C -Ġme at -Ġg iant -ĠW ars -ĠO N -Ġad m -Ġinter pret -Ġeven ing -Ġev il -ĠB oston -ĠW ild -Ġ à -ĠBit coin -ĠAm azon -D r -ĠIn formation -Ġobvious ly -Ġadv anced -Ph oto -ol ar -Ġwe ather -Ġsymb ol -Ġso le -Ġpot entially -ost er -Ġorig inally -m un -3 00 -az e -ess ions -Ġde ck -Ġst ood -Ġyou th -ĠB ern -R ep -ĠT est -Ġbas ically -ot ic -Ġinvol ve -ol it -ly n -S ee -Ġair craft -Ġconf irm -E W -Ġmess ages -ĠRich ard -Ġk it -Ġpro hib -Ġv ulner -is ters -Ġexist ence -Ġturn ing -ĠS P -Ġdes ire -Ġfl at -Ġm ent -se ason -ang es -Ġneighbor hood -ĠL ake -AT ION -Ġpoint ed -b ur -Ġinn ov -uc ks -U L -Ġprofess or -Ġexp ressed -A B -ic ious -Ġ200 2 -ĠDe v -Ġs ession -Ġb are -s en -Ġdis s -ĠC ath -ĠP ass -ĠP oint -Ġdo ctor -or row -ail ed -ĠR ub -ĠD C -ĠChar l -p erson -Ġwrit er -igh ters -ure au -Ġob lig -Ġrecord ed -Ġbro ke -Ġord ers -il ty -Ġmot ion -in ity -l aw -ad ium -Ġimm igration -Ġcontr ast -Ġb att -Ġex cellent -Ġtechn ical -am i -Ġt un -Ġcl oud -ĠY ear -ge on -Ġcre ation -Ġstr ange -Ġa uth -Ġfor t -b orn -Ġext ent -ĠT oday -ĠCl ub -Ġr ain -Ġs ample -Ġaccept ed -Ġt act -Ġf ired -ĠS on -Ġstand s -Ġb oot -Ġ4 7 -Ġstat ements -Ġvers ions -Ġse lling -ound ed -Ġ199 0 -Ġwere n -ĠW atch -Ġexper iment -P ost -Ġret ail -ul ed -In st -un te -ãĥ ¼ -Ġdep art -Ġb ond -i very -om pl -Ġre action -ĠSyri an -ĠP ac -app ed -ani el -D P -Ġres olution -Ġre act -Ġappro ved -on om -m ond -ĠO ffic --- - -Ġrepl ace -Ġt ack -Ġsp ort -Ġch ain -Ġemer gency -r ad -ĠPalest in -Ġ4 6 -Ġautom atically -Ġrout e -Ġp al -Ġb anks -ĠPar is -ĠMed ia -ro ad -ic ing -i xt -ist ed -Ġg rew -Ġco ord -ĠW here -om in -Ġsub s -� � -Ġ ± -Ġcorpor ate -Ġse lection -n oon -ĠRep ort -c s -clud ing -ord ers -anc he -ĠIt s -Ġslow ly -ĠE gypt -ĠA cc -Ġcol le -iqu es -E X -Ġattempt s -ur l -ĠC ross -Ġfind ings -ĠS C -ĠO R -Ġind ex -ens ity -ĠW ay -ĠL and -Ġsh ock -d is -Ġd ynam -Ġc art -m osp -S ince -i est -ĠB oy -Ġst orm -ĠCont in -201 3 -he w -il it -Ġess ential -iqu id -O ther -ive red -Ġreason able -A ct -Ġsub sequ -ĠP ack -ĠF ort -Ġconsider ing -Ġun iversity -l og -Ġmar ried -Ġill ust -ĠTr ue -£ ı -Ġnumer ous -rast ructure -Ġserious ly -Ġrefer red -u a -Ġconsist ent -on na -ĠRe al -ru ption -ci ples -Ġfact s -9 1 -ot es -er g -The n -Ġacc ompl -N ote -Ġre venue -Ġpass ing -Ġm al -e en -ĠY et -Ġg ather -ter day -ew ork -ĠA uthor -P e -Ġopt im -Ġr ub -Ġè £ı -Ġun known -st one -Ġun ion -ol ve -Ġopportun ities -Ġbrow ser -ĠW al -ĠC ost -Ġreport ing -st s -p et -Ġs and -Ġsudden ly -Ġsurpr ising -ĠV R -Ġsomew hat -ĠB as -ult ure -iz z -ĠC D -Ġchalleng es -Ġsett ings -Ġexperien ces -ĠF ull -Ġcan n -Ġrece iving -ES T -Ġj oint -Ġcult ural -Ġa st -8 2 -as tern -ce ived -ĠC ru -Ġb ull -p ired -am m -Ġfac ing -p ower -Ġb oss -ĠH ol -Ġinst r -Ġincreasing ly -Ġsh ift -Ġstre ets -ĠWilliam s -ab b -Ġl ie -Ġl augh -ĠC a -P L -Ġadult s -Ġcustom er -Ġob tained -Ġsupport ing -ht ml -f ire -Ġdetail ed -Ġpick ed -ĠR ight -ld er -E E -st ood -ĠK im -Ġw ire -Ġs ight -Ġdevelop ers -Ġpers ons -Ġs ad -Ġc up -Ġwar ning -Ġboy s -l ong -Ġb ird -f o -Ġw al -Ġobserv ed -Ġz one -iven ess -Ġch annel -c ript -Ġref used -ĠAg ain -Ġsu c -Ġspokes man -ĠRe f -r ite -ou ston -ãĥ ³ -ĠS her -Ġact s -ĠN ame -Ġstrugg le -ar ry -omet imes -Ġdisc rim -H T -Ġcateg ory -Ġreal ize -Ġemploy ee -ĠAf ghan -en ger -Ġgun s -ĠSte ve -ĠM ot -ĠO l -ok ed -Ġth ick -Ġfair ly -ill y -Ġsur ve -ĠM at -we ight -â Ķ -Ġtro ops -Ġag ents -Ġbatter y -Ġmot iv -à ¡ -S ec -d en -o very -L S -Ġfl u -Ġconf ident -ĠO per -Ġem pty -Ġp hen -Ġse ctor -Ġexc ited -Ġrem ote -ap h -o en -Ġdestroy ed -Ġmor al -ĠH P -ĠR on -Ġd ress -ĠB at -Ġl it -ĠM S -Ġa f -H L -r um -is ms -Ġshould n -Ġsym pt -ĠTor onto -het ic -Ġcar bon -Ġinstall ed -Ġviol ent -Ġsol ar -j a -Ġpract ices -Ġr ide -ĠP enn -Ġimpro ved -Ġaud io -Ġbehav i -ĠP S -Ġe ating -D ata -ĠRe view -p ass -cl aim -u ated -ang ers -c hen -Ġproper ties -Ġany where -An other -Ġbl ow -ĠJack son -Ġp roud -Ġplan e -l ines -Ġsqu are -Ġpro of -ans as -Ġtalk ed -m akers -Ġs ister -Ġhold s -Ġres ident -Ġ= = -Ġresist ance -Ġspl it -Ġpro secut -Ġconf idence -res ents -Ġcut s -Ġexcept ion -Ġz ero -Get ty -Ġcop yright -Ġtot ally -orm al -ific ations -ĠAustral ian -Ġs ick -Ġ1 50 -Ġhouse hold -Ġfe es -Ġdri vers -og en -ĠN Y -Ġnecess arily -Ġregul ations -ear ing -s l -Ġperspect ive -c are -ic ial -H is -Ġesc ape -Ġsurpr ised -ĠV an -ur rent -Ġv ac -8 1 -ĠTh us -Ġem phas -ĠCh ampions -ĠI ce -Ġn arr -Ġhead s -Ġca using -b el -f ortunately -ĠM a -Ġtarg ets -ci pl -Ġafter noon -Ġadd s -ĠMay be -ĠF our -ess ed -ple te -Ġus ual -ch o -ing u -Ġwith d -ĠE nergy -ĠE conom -O O -Ġart icles -Ġinj ured -Ġman age -Ġexpl ains -Ġdi agn -R ec -at ures -Ġlink ed -Ġdiscuss ed -Ġexpl o -Ġocc asion -ath an -Ġopp osite -Ġfac es -Ġden ied -ĠK night -Ġn ut -Ġapprox imately -Ġdisapp oint -onym ous -ĠB est -ĠL o -ĠH y -ĠA ff -Ġvot ing -an while -ĠII I -Ġinstit utions -ag ram -ĠD aily -Ġdr ag -Ġnear by -Ġgu ilty -Ġcon ver -P re -s hip -Ġre ward -Ġphilos oph -ĠS S -u gh -Ġapp s -f riend -Ġu pper -Ġad vert -Ġs now -Ġfr ust -Ġour selves -F r -ĠD ie -amp ion -Ġdis miss -Ġc ere -Ġsign al -f rom -Ġ ). -Ġ5 2 -Ġcr imes -it ors -est ival -use um -Ġcoun cil -ĠS aud -M ay -ĠG un -ic ian -et her -Ġsu fficient -ĠH en -so le -Ġhistor ical -ĠF ar -ĠT urn -Ġp in -Ġsuc ceed -m at -ly mp -Ġtrad ition -ĠO k -Ġc ro -Ġdesc ription -al le -Ġsk y -T e -Ġwide ly -Ġw ave -Ġdefin ition -ĠJew s -Ġcy cle -Ġref ere -Ġbr ings -us al -Ġal ive -Ġfrequ ently -Ġint ention -ĠCont rol -l v -y stem -Ġpriv acy -g ent -ren ce -ĠQu est -ĠChrist mas -Ġr ail -Ġco oper -Ġtest ed -ĠC apt -as ks -Ġcomfort able -Ġdel ivered -sc ape -Ġdep th -ĠG OP -Ġwrit es -Ġass ets -Ġsa v -im ents -Ġtrans ition -Ġart ist -ĠL ook -Ġl ob -Ġcomp onents -ar ity -Ġwalk ed -Ġro ot -Ġparticip ants -Ġnot iced -Ġres c -Ġn av -ĠAd minist -d a -ut ral -pl ate -Ġimport ance -Ġass ert -ious ly -c ription -Ġinj uries -ĠChe ck -Ġregist ered -Ġint ent -Ġmiss ed -ograph ic -Ġsent ence -oun ter -Ġassist ance -ev in -Ġdat abase -Ġbuild ings -Ġclass ic -Ġth inks -ĠOh io -P r -ug g -Ġfe e -p an -Ġeffect ively -Ġfac ility -Ġbe ar -Ġch apter -Ġdog s -ĠCol umb -Ġl atter -it ial -Ġad mitted -T V -ĠGe org -Ġpost s -\ \ -Ġlawy er -Ġequ ival -Ġm and -Ġcontro lled -ĠW alk -ĠAnd rew -Ġmen u -am ental -Ġprotect ed -v a -Ġadminist r -or al -Ġre in -ĠS ar -Ġamount s -Ġn ative -ĠM oon -Ġrep resents -Ġab andon -Ġcarry ing -Ġt ank -m ary -Ġdecl ared -T ube -Ġh at -Ġpun ish -el lect -m es -Ġun iverse -ĠR od -ph y -Ġinf rastructure -Ġ5 1 -Ġopp osed -ow nt -c a -ĠM ake -Ġhard ware -Ġco ffee -R el -b al -w orld -ĠS af -ĠSe a -in als -Ġown ed -Ġh all -ers ion -Ġdescrib e -ĠP ot -Ġport ion -Ġat mosp -Ġgovern ments -Ġdep ending -Ġoff ense -Ġtr ick -aw a -ĠL ine -ĠV is -ĠH ard -ĠOr ig -ĠCl ick -Ġdes k -ĠVal ley -ĠS ov -Ġmov ies -Ġrem ark -Ġm ail -Ġcons cious -Ġrul ing -ĠR ights -Ġmed ic -he nt -ĠW omen -> < -Ġrepl aced -ĠP rem -ĠTh anks -Ġre new -ĠB all -if orm -Ġsh ots -C omm -Ġar med -Ġconst ant -Ġt aste -Ġreal ized -Ġbu ff -Ġm o -Ġeffic ient -M ost -or ation -if ies -Ġcommun ication -Ġfl ood -Ġconsequ ences -Ġany way -ig g -ĠG M -ĠTh ank -Ġ iron -Ġev olution -ĠC op -tw itter -Ġ9 5 -Ġrelationship s -ad el -ĠYou ng -Ġpropos al -ay ers -uild ing -ĠH ot -OR E -c os -Ġcoll abor -P G -ax y -Ġknow ing -Ġsupport s -ow ed -Ġcontrol s -Ġmere ly -um er -Ġath let -Ġf ashion -p ath -Ġg ift -Ġer a -AN D -Ġkind s -ĠKore an -Ġleg it -ul ous -Ġess entially -Ġthe rap -n ic -Ġsuff ered -Ġh ur -Ġprom ise -Ġex cess -Ġover w -Ġpr ime -ĠH ouston -er ry -ĠM s -R S -201 2 -Ġst ores -ĠO lymp -Ġj ourney -Al though -S ub -ĠE duc -ĠCh apter -Ġrequest s -Ġconsum ers -Ġt iny -Ġis ol -ĠF air -b a -ĠY OU -Ġcr ash -ce ler -Ġemot ional -Ġgood s -Ġelect ed -Ġmod er -ĠLin ux -Ġbl ocks -Ġis land -ĠSoc iety -Ġelect ions -Ġbroad cast -Ġche ap -Ġn ations -Ġse asons -4 00 -Ġwas te -ĠS at -Ġfield s -em ploy -Ġprof ile -Ġauth ors -AL L -ĠG ra -w est -ĠT y -Ġdeath s -Ġv acc -Ġfor med -Ġd u -Ġon going -ĠMuslim s -el f -ig ure -Ġass ume -ĠUkrain e -w ater -Ġco ast -Ġvot ed -g or -ĠA S -ĠMich igan -az a -ĠAr m -i ro -Ġf lex -as ters -' ' -Ġwel come -ar l -Ġloc ations -ig ation -ĠF il -Ġbu ying -Ġarch itect -Ġhard er -ĠC ub -Ġinter face -Ġrestaur ant -Ġdisco ver -Ġex ceed -Ġfav our -ger y -Ġd uty -Ġp itch -ad or -ĠM ach -b oy -Ġrespond ed -Ġext ended -her s -M any -ra id -if er -ĠIn s -S er -Ġmed ium -s he -ĠS ports -Ġmag azine -ut ation -Ġlim its -ĠG all -Ġex ternal -raz il -Ġyoung er -t le -Ġrem ind -ĠC ON -Ġimmedi ate -Ġh idden -Ġvol unte -Ġsim pl -od cast -Ġph ase -d r -Ġpl ot -Ġexp osure -R I -og rap -v in -an ish -ĠAc ad -ĠEng ine -Ġexp ansion -ĠP ay -Y our -Ġpus hed -ĠE ll -ĠHe ad -Ġmarket ing -ĠA C -k et -Ġh its -Ġg ro -ĠA ge -ĠSc ot -] [ -Ġst im -Ġi Phone -Ī Ĵ -Ġn arrow -ĠGet ty -ĠTur key -Ġperfect ly -Ġen able -ut ch -Ġprec ise -Ġreg ime -Ġsh if -Ġcomp ens -g un -d iv -Ġch osen -ĠK en -An y -Ġtre es -Ġrecomm ended -ĠR en -u able -ĠH T -F ollow -E G -ĠH and -ĠK enn -Ġarg uments -Ġex ists -Ġb ike -ĠCons erv -Ġbre aking -ĠG ar -Ġc razy -Ġvirt ual -ay lor -ix el -Ġ19 80 -Ġper mission -ĠSer ies -Ġconsum er -Ġclose ly -c alled -Ġ5 4 -Ġhop es -Ġar ray -ĠW in -ĠLab our -Ġsp ons -ĠI re -Ġp ow -Ġread ers -Ġemploy ment -Ġcreat ure -Ġresult ing -Ġaccur ate -Ġmom ents -Ġarg ued -Ġp ed -D uring -Ġ5 3 -ĠT al -Ġs ought -Ġsuff ering -Ġ icon -le e -Ġ( $ -al ian - ° -Ġp ra -Ġbon us -( " -k o -Ġact ing -D E -f all -Ġcompar ison -Ġsm ooth -ĠN AS -u pp -ĠJose ph -ep ing -ĠT ake -ĠM id -Ġs ending -f ast -ĠF all -Ġdeal ing -us er -ĠOr gan -C o -Ġatt ached -Ġse es -% . -Ġtyp ical -AR T -Ġfind s -ĠAs ia -um in -ĠC ore -ĠE nt -in ent -u ce -ĠBl ood -ĠN ever -Ġem ails -Ġhigh light -Ġconf ront -at us -ut ed -Ġun us -Ġtop ic -ĠAd am -Ġb le -at i -Ġunder stood -S et -st ruct -T P -Ġm ob -a a -ĠSt art -pect ed -se ll -Ġded icated -ĠC A -u an -Ġsong s -esc ription -Ġte ch -Ġr ape -Ġas ide -Ġgr ant -Ġ5 6 -s ub -Ġarg ue -Ġcont aining -Ġsche dule -Ġliber al -Ġpublic ly -Ġheav ily -ĠU t -in er -ĠS ection -ĠC are -we et -l s -D is -âĶ Ģ -ĠF ollow -B ack -ĠI T -Ġb es -j i -ĠH it -est ed -Ġevery body -ĠSw ed -Ġfem in -Ġfac ilities -Ġcon ven -C omp -ĠO S -c ore -Ġan x -Ġdiv ision -ĠC am -ĠSt an -m ates -Ġexpl ore -pl om -Ġsh ares -pl oad -an es -Ġide al -et ers -ĠB ase -Ġpl astic -Ġdist inct -ĠNet work -ĠSe attle -Ġtrad ing -ens us -int end -Ġex hib -Ġinit ially -ĠF ood -Ġthous and -ĠBus iness -act er -Ġpar agraph -Ġrough ly -Ġw ww -Ġcreat ive -ĠCon f -Ġconsum ption -Ġfil ms -ag an -Ġob tain -Ġt all -Ġt or -Ġacknow led -Ġg rown -al o -K E -Ġ4 00 -end ers -t aining -U G -Ġsu icide -Ġwat ched -ĠL ist -al i -re hens -Ġsurround ing -Ġp ip -Ġf lying -ĠJ ava -ord an -Ġserv ing -in ations -p ost -Ġsh o -A v -Ġj ail -z y -Ġ199 9 -Ġ< / -Ġliter ally -ĠS ir -Ġexp osed -Ġl ies -st ar -Ġb at -Ġear ned -ĠD ig -Ġspec ified -ĠSe ason -Ġdeg rees -Don ald -Ġcent re -Ġsh aring -Ġwin ter -ĠC O -C he -Ġ Î -M P -Ġun w -Ġfew er -ĠM ir -Ġsomew here -ĠK ey -Ġattack ed -ĠK ir -Ġdom ain -Ġstrong er -Ġ9 9 -Ġpen alty -I d -Sc ript -Ġdecl ined -Ġne ck -Ġfra ud -Ġcur rency -Ġr ising -R C -âĢ¦ âĢ¦ -H z -Ġt ab -Ġtal ent -n am -ĠN BA -Ġvill age -Ġleg s -ĠN ext -E d -Ġac id -Ġhy d -8 00 -Ġinvol ving -ĠIm age -ĠBe fore -F l -Ġyes terday -S ource -Ġterror ist -Ġsu p -Ġsy nt -ĠSaud i -Ġw est -Ġr u -b urg -Ġvis ible -Ġstru ck -r ison -Ġaw esome -Ġd rawn -Ġansw ers -ĠG irl -ĠR am -Ġthreat s -Ġdef eat -os it -Ġv ent -atur ally -Americ an -end a -ĠH oly -Ġr um -% , -c ase -ĠHist ory -ĠYou Tube -Ġsit uations -ĠD NA -S te -Ġsa ved -It em -Ġrec ip -olog ist -Ġfac ed -Ġel ig -O nce -ĠL i -u h -Ġmist ake -ĠDiv ision -ĠB ell -Ġsympt oms - ® -Ġdom in -Ġfall ing -Ġend ing -as hes -Ġmat ches -ĠOn line -Ġexplan ation -D ef -red it -Ġany more -ĠT otal -ĠF OR -us hed -Ġlet ters -Ġris ks -ĠO K -Ġreported ly -: \ -Ġpl ate -Ġsubject s -Ġattempt ed -if ier -ian a -Ġunlike ly -ĠTh ough -um a -ĠIn vest -ĠPr in -ic an -ĠD ar -ĠColor ado -au g -Ġve get -a os -ri a -Ġshe l -Ġmark ed -Ġ( ) -Ġsp r -p o -ĠL ink -Ġdef e -ĠJ r -Ġthem e -Ġpass ion -ĠP en -Ġinf o -iz er -Ġsh it -ĠC ivil -ap se -c re -Ġpo ly -Ġcomp onent -ĠChar les -ĠIre land -ĠPro v -Ġdo ctors -Ġgr anted -Ġpain t -Ġhon or -Ġsm oke -Ġpay ments -Ġprim arily -ĠKing dom -r ich -ate ll -Ġde als -Ġsched uled -Ġfund amental -Ġprote in -Ġnewsp aper -Ġcl ients -yth on -ĠD ate -h us -Ġfeed back -Ġstret ch -Ġc ock -Ġhot el -ĠQue en -Ġsu gar -Ġj u -Ġmil k -Ġappro val -ĠL ive -Ġequival ent -ef ully -Ġins ert -z ona -Ġext ension -d ri -J ohn -Ġacc omp -S m -ĠF und -Ġconst antly -Ġ` ` -Ġgener ated -ĠA ction -ĠP sych -ĠT ri -Ġrecogn ize -Ġv ary -ph a -ĠR a -d f -et ch -ĠSov iet -Tw o -Ġpattern s -Ġprof ession -an ing -T ime -ĠL im -Ġcol ors -ĠA z -ĠT R -Ġinf ect -Ġphen omen -Ġshe ll -Al so -Ġput s -Ġdel ivery -Ġbro wn -Ġprocess ing -Ġlight s -ess age -ĠBro ok -ĠA ud -l ation -Ġindust rial -L ike -ĠB razil -rou s -ES S -ĠL uc -Ġsome how -Ġ8 5 -Ġpro port -Ġpolit icians -Ġindic ate -Ġh ole -Ġtechn iques -Ġcompet itive -Ġph r -Ġv o -ist ent -ĠD ream -Ġcamp us -Ġaspect s -Ġhelp ful -Ġsh ield -or se -Ġtrig ger -m al -Ġ5 8 -Ġt ort -Ġperson ally -Ġt ag -Ġkeep s -ĠV ideo -Ġben ch -Ġg ap -a ire -Ġe ast -Ġrec overy -per ial -Ġprof it -ĠM ic -Ġ5 7 -Ġcol on -Ġstrong ly -st yle -Ġalleg ations -h an -Ġrep orters -j o -r ine -arg et -and al -Ġ0 3 -Ġfl ash -tr ans -Ġstr ict -Ġpark ing -ĠPak istan -Ġl i -Ġwe ird -ĠE ric -Ġreg ions -ĠJ un -Ġint ellect -ĠW H -od ing -rib utes -up id -ĠT it -Ġf inger -or ia -Ġe lev -ĠF ield -Ġcon clusion -; ; -Ġfeel ings -Ġext ensive -Ġm ixed -Ġne uro -v y -Ġhar ass -ĠC irc -ou ch -Ġterrit ory -Ġsuccess fully -M ar -Ġing red -Ġoverw hel -Ġl ayer -V iew -Ġall ies -ill ance -ĠTh ree -Ġb unch -Ġnorm ally -Ġnet works -Ġsac r -ĠC IA -b les -Ġch ose -Ġopp onents -Ġregard less -Ġfr anch -Ġpre f -ĠP o -Ġbr idge -ann a -ĠSil ver -Ġw age -p age -ri or -Ġrad ical -ĠL ittle -Ġman ip -Ġsecret ary -Ġg ang -D R -F A -Ġdec ent -ĠSp irit -Ġun cle -ĠDevelop ment -Ġinvest ors -Ġwall s -Ġpub lish -Ġgener ate -iss ions -c ar -Ġprom ote -Ġcut ting -Ġche st -Ġdrink ing -Ġcollect ed -Ġ7 2 -Ġhop ing -Ġem br -gor ith -Ġwar ned -Ġinstruct ions -O G -ĠD id -ĠAg ency -Ġg ear -Ġcritic ism -ĠF urther -Ġut il -ann y -R ed -Ġcoun sel -ĠAs ian -Ġredu ction -p ool -Ġteach ing -Ġdeep ly -i y -Ġestim ates -Ġcho ices -Ġperman ent -in em -ke l -Ġf asc -p se -f ile -ĠL ow -ĠP erson -Ġt ournament -st al -Ġm el -U ST -ĠR ay -az i -V al -Ġcont ained -ĠH olly -Ġw ake -Ġreve al -Ġprocess es -ĠIS IS -Ġ0 9 -Ġbl ind -Ġste el -ĠB ad -Ġcare fully -app y -ro it -Ġg aming -Ġhous es -ĠC oll -Ġtr uck -er m -Ġsc ored -Ġocc as -ret urn -b ound -v ar -Ġsh arp -Ġaf raid -ĠE X -am ber -c ific -Ġsche me -N C -ĠPol it -Ġdecl ine -Ġ199 8 -Ġpus hing -Ġposs ession -Ġpriv ile -Ġteacher s -Ġy ield -H A -ĠDav is -it led -#### #### -Ġr ig -ĠD aniel -ac on -Ġh ide -ut en -Ġcolle agues -Ġprin ciples -Ġl oud -Ġs in -ĠDem on -Ġst one -Ġ0 2 -Ġt aught -Ġter rible -Ġst uck -ĠPol icy -te en -Ġimplement ation -ĠB BC -ĠAP I -Ġwhe el -all as -Ġch ampions -ol ars -play er -Ġrepeated ly -ĠSt ill -Ġlik es -ast y -es ter -ĠCath olic -R L -Ġb ath -Ġno ise -t itle -Ġn orthern -P art -Ġmag n -Ġf ab -ĠAs h -Ġdis pl -Ġtick et -Ġm urd -Ġalong side -ĠMus ic -Ġr iver -ĠSte el -ĠC L -ĠPl ayer -ĠM ult -ow ing -re p -s ize -Ġt ur -ĠGeorg ia -isc al -ra ction -Ġc able -Ġ5 9 -Ġw ins -Ġup coming -Ġsurv ive -Ġins pired -ĠEduc ation -Ġstat istics -ĠF oot -iam i -Ġy ellow -ĠP age -. - -ĠH as -Ġur ban -Ġa x -es sel -\ " -Ġquarter back -Ġreg ister -ĠLab or -Ġab ilities -ĠF amily -Ġvar iable -ĠPr ice -Ġcont em -Ġth in -ĠE qu -d ata -Ġg otten -Ġconst it -Ġas ks -Ġt ail -Ġexc iting -ĠE ffect -ĠSp anish -Ġencour age -ins on -ĠA h -Ġcommit ment -C S -Ġr ally -Ġ: : -Ġsubs id -Ġsp in -Ġcapt ured -201 8 -Ġinn oc -Ġalleged ly -ĠC ome -Ġart ists -ĠN umber -Ġelect ronic -Ġreg ional -ap es -Ġw ra -Ġmy th -pr ise -ĠM iller -ĠC reat -ĠEp isode -b ell -Ġdirect ed -Ġext ract -Ġs orry -Ġv ice -ag ger -ĠSu pport -Ġ6 6 -ĠI ron -Ġwonder ful -Ġg ra -N et -ion e -E ng -Ġsh ips -ik es -ĠK evin -it ar -Ġactiv ists -tr ue -ĠAri zona -ent h -ĠDes pite -ĠS E -Ġha bit -ern el -Ġin qu -Ġab ortion -Ġv oid -Ġexpl icit -Ġeng aged -Ġang ry -Ġr ating -Ġfr ag -b ro -ick ing -d ev -Ġwor ried -Ġob ser -Ġap artment -ĠG T -Ġest ate -ĠConst itution -em on -ĠS now -Ġcount y -Ġdis ag -ĠStep hen -Ġimm igrants -w ind -ĠN ations -Ġfol ks -O ut -Ġg all -Ġtarget ed -Ġst ead -ĠB on -ĠL ib -Ġinform ed -Ġ12 0 -ch ain -idel ines -or ough -Ġdri ven -Ġregular ly -Ġbas ket -Ġprinc iple -oc ument -Ġst un -ib ilities -ĠRom an -ĠAb out -Ġal ert -Ġdemocr acy -Ġrepresent ed -H S -c ers -p arent -Ar t -p ack -Ġdi plom -re ts -ĠN O -Ġcapt ure -ĠAd v -Ħ ¢ -Ġannounce ment -ĠL ear -Ġh ook -Ġpur s -ĠS uch -ĠC amer -Ġrefuge es -ĠV e -P ol -Ġrecogn ized -l ib -Ġhad n -A ss -Ġpil ot -us hing -Ġreturn ing -Ġtra il -ĠSt one -Ġrout ine -Ġcour ts -Ġdes per -Ġfriend ly -ĠIt aly -Ġpl ed -Ġbreat h -Ġstud io -N S -Ġimp ressive -ĠAfghan istan -Ġf ing -Ġd ownt -ink ing -ĠR og -i ary -col or -se x -ar on -Ġf ault -ĠN ick -D own -ĠR ose -ĠS outhern -X X -is odes -L ist -6 00 -Ġout come -er r -Ġelse where -Ġret ire -Ġp ounds -ĠGl obal -Pe ople -Ġcommun ications -Ġlo an -Ġrat io -ĠEm pire -Ġg onna -Ġinv ent -D F -Ġ19 70 -ĠComm on -p at -Ġprom ised -Ġd inner -ĠH om -Ġcreat es -Ġoper ate -ver ty -ĠJ ordan -et ime -Ġsust ain -R eg -Ġincred ible -im a -Ġwar rant -Ġm m -A tt -Ġlaw suit -Ġreview s -it ure -ĠS ource -l ights -ĠF ord -Ġ6 3 -g roup -st ore -Ġfeat ured -Ġfore ver -Ġpo verty -ĠP op -ĠC NN -az z -ab is -ach ing -Ġl aid -ĠSu pp -Ġfil ter -en a -ĠCommun ity -Ġcreat ures -u ction -ĠR oyal -Ġassoci ation -ĠCon nect -ĠBr ad -âĸ Ī -l ers -the re -ĠG i -Ġval uable -AC K -ĠT aylor -Ġl iquid -ĠAtt orney -ĠCar l -ĠF inal -ag a -ĠWil son -B ecause -ĠProf essor -ak a -Ġincred ibly -r ance -! ) -R ef -s k -Ġsol utions -Ġatmosp here -Ġbl ame -um es -ĠN ob -C A -um ps -r ical -ĠPut in -ĠD est -or ic -ĠP A -Ġrespect ively -w an -Ġfif th -â Ħ¢ -ĠC ry -Ġgovern or -res ident -Ġpurch ased -Ġh ack -Ġint ense -ob s -Ġorig in -Ġdef ine -Ġcare ful -** * -Ġshould er -Cl ick -Ġt ied -Ġdest ruction -ou red -Ġno body -Ġh o -ĠEx per -Ġt ip -" ; -Ġtechn ique -Ġj ur -ĠP ok -b ow -Ġleg end -Ġacc ord -Ġbus y -ĠInt el -Ġh ang -ak i -. ] -âĢĶâĢĶ âĢĶâĢĶ -Ġsur gery -Ġrep rodu -Ġun iform -Ġscen es -c ode -Ġ6 2 -l isher -ĠH ave -ph ia -Ġcry pt -Ġrec on -Ġsc ream -Ġadop ted -Ġsc ores -N e -ĠIt alian -in cluding -B O -Ġindic ated -Ġent ertain -G u -T ext -i el -Ġtw enty -Ġeng age -off s -ĠPac ific -Ġsm ile -Ġperson nel -Ġto ler -Ġdo ors -Ġt one -Ġmach ines -Ġent ering -ten ance -C O -ĠJer sey -Ġfore st -Ġhor se -Ġcompl aint -ĠSpr ing -y o -ĠPl us -ed ing -ĠRet urn -qu arters -ial s -c ow -Ġacad emic -Ġf ruit -Ġ199 6 -og ether -Ġw ine -Ġpur su -ĠSte ven -Ġlic ens -Wh o -Ġclot hes -re ction -Ġsqu ad -Ġst able -Ġr aw -z ens -St ar -ut ies -anc er -Ġke ys -ĠM u -Ġcompl icated -ig er -ĠTe xt -Ġabs or -Ġ6 8 -Ġfun ny -Ġrel ief -ĠL ew -ĠC ook -Ġch art -Ġdraw ing -G E -Ġmod ule -ĠB ull -I LL -Ġs alt -0000 0000 -il le -Ġres ource -aw ay -adel phia -ĠB ru -Ġ6 7 -Ġsome body -Ġparticip ate -Ġro se -we red -Ġmus cle -Ġcons ent -Ġcontin uing -ĠGuard ian -ĠOr der -reg on -Ġre ar -Ġprov ision -Ġlik ed -ri ent -Ġb ra -Tr ans -Ġmeet ings -Ġto x -Ġcon vent -Ġaut o -Ġrec ording -ĠSo ft -00 1 -ĠR oll -Ġprogram ming -Ġp ic -Ġprov ed -Ġst ab -ĠA st -Ġca ption -ul ating -ĠAtt ack -Ġnew ly -Ġ199 7 -f r -Ġdis cipl -ĠGree k -Ġed ition -ĠDo es -ĠB ox -if le -ack et -Ġpass es -Ġgu est -Ġac celer -it als -U D -Ġaut hent -ĠR est -ov al -t a -u ine -Ġarm or -ĠT own -Ġcomp at -Ġinc hes -Des pite -Ġass ign -he rent -Ġprep are -ĠM eg -oc key -Ġdep ends -Ġtrack s -w atch -Ġl ists -ĠN orthern -Ġal ter -re c -ĠE astern -Ġcond em -Ġevery where -? ' -Ġaff ili -Ġf ought -": {" -Ġm ac -it arian -Ġsc ope -ĠA L -aw s -ar ms -Ġqu e -Ġenjoy ed -nes ota -Ġagg ressive -ĠSt ory -ĠI V -Ġrec ipe -Ġrare ly -ĠMed ical -val ue -ang el -ay ing -omet hing -Ġsub section -Ġs outhern -Ġfrequ ency -re te -roll ed -ult s -ĠN ic -Ġbeh alf -Ġsequ ence -ab et -Ġcontrovers ial -Ġcomp rom -Ġwork er -Ġmain ly -Ġal gorith -ĠM ajor -or ce -g ender -Ġorgan ized -Ġf ake -Ġconclud ed -ĠE D -ĠEx ec -r age -Ġch ances -ber ry -ĠTr ad -Ġconfig uration -Ġwithd raw -Ġf ro -ud es -ĠBro ther -ĠB rian -Ġtri es -Ġsam ples -Ġb id -ĠGold en -Ġphot ograph -if est -ĠD O -ĠPar liament -******** ******** -R em -Ġcont est -Ġsign ing -p x -ĠZ eal -âĶĢ âĶĢ -E ar -Ġex it -Be fore -ĠCor por -n ull -mon th -Ġrac ial -ott ed -ĠV eg -ĠRe uters -Ġsw ord -ps on -ĠRom ney -a ed -Ġt rib -Ġin ner -Ġprot ocol -ĠB i -ĠM iami -ever al -p ress -Ġsh ipping -ĠAm endment -ĠHow ard -con nect -ĠD isc -ĠJ ac -iam ond -ĠThere fore -s es -ĠPrin cess -ĠUS B -ĠAn th -Ġsurve illance -Ġap olog -Ġ6 1 -ow a -Ġf ulf -j s -Ġl uck -ust ed -Ġ § -n i -Ġant icip -em an -Ġwin ner -Ġsil ver -ll a -ic ity -Ġunus ual -Ġcr ack -Ġt ies -e z -Ġpract ical -Ġprov ince -ĠPl ace -Ġprior ity -IC E -Ġdescrib es -Ġbr anch -F orm -ask a -miss ions -b i -Ġp orn -ĠTur k -Ġent hus -Ġf ighters -Ġ0 8 -ĠDet roit -Ġfound ation -av id -A re -Ġjud gment -cl ing -Ġsol ve -ĠDes ign -W here -hes is -ĠT ro -a fter -Ġne utral -ĠPalestin ian -ĠHolly wood -Ġadv is -ĠN on -y es -ol is -Ġrep utation -Ġsm ell -Ġb read -ĠB ul -ĠBe ach -Ġclaim ing -Ġgen etic -Ġtechn ologies -Ġupgr ade -row s -Ġdevelop er -ĠJ osh -ĠDis ney -erv ed -ip al -Ġun ex -Ġbare ly -t hen -ĠP ub -Ġill ness -et ary -ĠB al -Ġp atch -Ġbut t -Ġst upid -ĠD og -ĠD allas -f ront -ie ce -Ġprot ests -Ġch at -oen ix -Ġw ing -Ġpar liament -Ġ7 7 -ose xual -Ġre nder -pt ions -ĠCo ast -os a -ĠG reg -h op -ĠMan agement -Ġbit coin -Ġrec over -Ġincor por -or ne -ĠUs ing -Ġpre ced -Ġthreat ened -Ġspirit ual -ĠE vent -ĠF red -Ġadvert ising -Ġimprove ments -ĠC ustom -Ġer rors -Ġsens itive -ĠN avy -Ġcre am -L ook -Ġex clusive -Ġcomp rehens -Ġde leg -Ġcon ce -Ġrem em -Ġstruct ures -Ġst ored -N D -Ġ1 000 -U P -ĠB udd -A F -w oman -ĠAcad emy -ð Ł -se a -Ġtem porary -Ab out -es ters -Ġtick ets -Ġposs ess -in ch -o z -Ġl a -Ġcontract s -Ġun p -Ġc ig -ĠK at -ult ural -as m -Ġmount ain -ĠCapt ain -St ep -m aking -ĠSp ain -Ġequ ally -Ġl ands -at ers -Ġreject ed -er a -im m -ri x -C D -Ġtrans action -g ener -less ly -Ġ| | -Ġc os -ĠHen ry -Ġprov isions -Ġg ained -Ġdirect ory -Ġra ising -ĠS ep -ol en -ond er -Ġcon sole -in st -Ġb om -Ġunc ertain -1 50 -ock ing -Ġmeas ured -Ġpl ain -Ġse ats -Ġd ict -S L -af e -Ġest imate -iz on -at hered -Ġcontribut ed -Ġep isodes -omm od -G r -AN T -Ġ6 9 -G ener -Ġ2 50 -vious ly -rog en -Ġterror ism -Ġmove ments -ent le -oun ce -ĠS oul -Ġpre v -ĠT able -act s -ri ors -t ab -Ġsuff er -Ġn erv -Ġmain stream -ĠW olf -Ġfranch ise -b at -Ġdem ands -Ġag enda -Ġdo zen -Ġclin ical -iz ard -ĠO p -t d -Ġvis ited -ĠPer haps -Ġact or -Ġde lic -Ġcont ribute -Ġin ject -ĠE s -ac co -Ġlist ening -Ġcon gress -epend ent -Ġprem ium -Ġ7 6 -ĠIr ish -Ġass igned -ĠPh ys -Ġworld wide -Ġnarr ative -ot ype -m ont -b ase -ĠB owl -ĠAdminist ration -Ġrel ation -ĠE V -C P -Ġco vers -Ġ7 8 -Ġcert ific -Ġgr ass -Ġ0 4 -pir acy -ir a -Ġengine ering -ĠM ars -Ġun employ -ĠFore ign -st ract -Ġv en -Ġst eal -Ġrepl ied -Ġult imate -Ġtit les -d ated -Ġj oy -a us -Ġhy per -ak u -Ġoffic ially -ĠPro duct -Ġdifficult y -per or -Ġresult ed -rib ed -l ink -wh o -~~ ~~ -ĠSpe ed -ĠV iet -W ind -ĠBar ack -Ġrestrict ions -ĠSh are -Ġ199 5 -ition ally -Ġbeaut y -op t -Ġm aps -ĠC R -ĠN ation -ĠCru z -W ill -Ġelectric ity -Ġor g -Ġb urd -Ġviol ation -Ġus age -Ġper mit -ĠCh ron -ĠF ant -Ġn aturally -Ġ0 7 -Ġth rown -ĠAw oken -Ġal ien -ĠHer o -ĠK ent -ĠR ick -ri ke -Ġp ace -}, {" -G L -Ġpo ison -ĠT ower -Ġform al -al ysis -Ġgen uine -Ġk il -a ver -Ġproced ure -ĠPro p -intend o -ĠM ain -as ant -Ġtr ained -G ame -ĠL oad -ĠM A -Ġcru cial -Ġle ts -ĠF R -Ġch ampion -1 01 -ĠCon ference -Ġwrit ers -Ġconnect ions -Ġo kay -ir ms -ĠR and -Ġenc ounter -ĠB uff -Ġachie ved -Ġche cks -isc ons -Ġassist ant -Ġwhen ever -ĠA ccess -ĠU r -b in -Ġcl ock -is p -op her -Ġb orrow -Ġm ad -Ġperson ality -on ly -IS T -ab ama -Ġg ains -Ġcommon ly -Ġter r -Ġhyp ot -Ġre ly -Ġt iss -iscons in -Ġrid ic -f unction -ĠO regon -Ġun com -r ating -el and -ĠN C -Ġm oon -ann on -Ġvulner able -ut ive -³³ ³³ -ĠRad io -Ġw estern -se ct -ĠT ony -Ġocc urs -ĠO s -ĠH on -Ã Ń -Ġv essel -ĠScot land -Ġdiscrim ination -Ġsubsequ ent -st ring -Ġfant asy -ĠSh adow -Ġtest im -W E -it i -r as -Ġbo at -Ġmar ks -Ġord inary -Ġre n -Ġrepresent ative -Ġpet ition -Ġ7 3 -Ġad venture -Ġign ore -ĠPhil adelphia -ĠS av -V P -Ġfact ory -Ġt asks -Ġdep ression -z ed -................ ................ -ĠSt orm -Ġc ogn -Ġelig ible -Ġredu cing -v ia -Ġ0 5 -Ġstri king -Ġdoll ar -h o -O V -Ġinstr ument -Ġphilosoph y -ĠMo ore -ĠA venue -Ġrul ed -ĠFr ont -IN E -ĠM ah -Ġscen ario -ĠNAS A -Ġen orm -Ġdeb ut -Ġte a -T oday -Ġabs ence -S im -Ġh am -le ep -Ġt ables -ĠHe art -M I -K e -re qu -V D -m ap -Ġchair man -Ġp ump -Ġrapid ly -v i -Ġsubstant ial -E P -d es -ch ant -ili pp -ĠS anta -ri ers -anche ster -L oad -ĠC ase -Ġsa ving -Ġ7 4 -ĠA FP -er ning -oun ced -ĠMin nesota -ĠW as -Ġrec ru -Ġassess ment -ĠB ron -U E -Ġdynam ic -Ġf urn -ul ator -Ġprop ag -h igh -Ġacc ommod -Ġst ack -ĠS us -w rit -Ġre ven -ĠGod d -ĠZeal and -ab s -Ġbr ut -Ġper pet -h ot -Ġhard ly -ĠB urn -ãĤ ¹ -Ġst y -Ġtrans actions -Ġg ate -Ġsc reens -Ġsub mitted -Ġ1 01 -Ġlangu ages -ugh t -em en -Ġfall s -Ġc oc -Ĥ ¬ -Ġstri kes -p a -Ġdel iber -ĠI M -Ġrel ax -ann els -ĠSen ator -Ġext rem -Ġ} , -ĠDe b -Ġbe ll -Ġdis order -c ut -Ġi OS -Ġl ocked -Ġem issions -Ġshort ly -" ] -ĠJud ge -ĠS ometimes -Ġr ival -Ġd ust -Ġreach ing -F ile -¯¯ ¯¯ -ino is -ĠJ ason -Ġs atell -are t -Ġst ations -Ġag ric -ĠTechn ology -com es -ĠUn fortunately -ĠChild ren -Ġappl ies -ast ed -Ġan ger -ail ability -ĠDam age -Ġcomp are -ĠStand ard -Ġaim ed -ĠB a -angu age -Ġreg ulation -Ġj ury -Ġair port -Ġse ctions -ĠPr ince -em ed -Ġmedic ine -Ġh itting -Ġsp ark -ol ves -Ġad s -St ate -Ġfood s -Ġrepl acement -Ġch icken -Ġlow est -Ġmind s -Ġinvol ves -u i -Ġarr ang -Ġproced ures -ĠWh ich -ivers ary -Ġb ills -Ġimprove ment -Ġin ev -Ġexpect ations -Ġintellect ual -Ġsp aces -Ġmechan ism -2 50 -bre ak -ĠZ e -ĠT enn -ĠB alt -Ġbar rel -Ġstat ic -man n -Pol ice -Ġt ips -Ġhand ling -c us -od ed -il ton -ir y -Ġjournal ists -our se -Ġcom ic -Ġnom ine -IT Y -Ġvers us -Ġlo op -Ġsur f -ĠInd ust -ĠHun ter -Ġbelief s -is an -Ġset up -Ġbre w -im age -Ġcomput ers -f ol -} ," -ĠMed al -Ġtax p -Ġdisplay ed -Ġg rav -Ġf iscal -M on -ĠMos cow -ĠK ong -ĠCent re -Ġcamer as -ĠMr s -ĠH ay -Ġa ver -ĠK elly -p y -Ġrequire ment -Ġent itled -omb ie -Ġsh adow -ag ic -ĠA k -Ġel ite -Ġdiv ided -Ġhead ing -Ġcop ies -Ġloss es -Ġv it -k ed -ĠB ry -Ġan s -ĠSte am -Ġrep orter -he im -ĠIt em -Ġsuper ior -d on -ere nt -à ¶ -Ġtherap y -Ġpe ak -ĠMod el -Ġl ying -Ġg am -z er -r itten -Ġrespons es -Ġconsider ation -ĠB ible -Ġl oyal -Ġinst ant -Ġp m -ĠFore st -à ¼ -Ġext end -Ġconv icted -Ġfound er -Ġconv in -ĠO ak -che ck -Ġsch olars -p ed -Ġover se -T op -c ount -ĠAr k - · -Ġ0 6 -ĠL A -m d -ĠLat in -im ental -ĠC PU -Ġsubst ance -Ġminor ity -Ġmanufact uring -E r -ocol ate -Ġatt ended -ĠMan ager -r ations -Ġappreci ate -om y -GB T -id ency -B L -Ġguarant ee -pos ition -Ġo cean -clud e -Ġhead ed -Ġt ape -Ġlo ose -Ġlog ic -Ġpro ven -Ġsp ir -Ġad mit -is a -Ġinvestig ate -Ġ199 4 -sy lv -ĠL ost -c est -Ġ7 1 -Ġrequest ed -Ġwind ows -ĠPok é -ĠWith out -M et -Ġbehavi our -Ġread er -Ġh ung -ĠKe ep -Ġro les -Ġimplement ed -Ġbl ank -Ġserv es -ĠJ ay -Ġc ited -ĠF riend -prof it -ap on -Ġrep air -it em -arr ass -Ġcrit ics -ad i -ĠF ather -Ġsh out -Ġf ool -Ġ8 8 -Ġprodu cing -Ġl ib -Ġround s -Ġcirc le -Ġpre par -Ġsub mit -Ġn ic -mor row -ãĥ « -U nder -Ġv ital -ater n -Ġpass word -Ġpublic ation -Ġprom inent -Ġspeak s -Ġb ars -Ġde eper -ĠM ill -port ed -Ġw id -Ġbut ter -Ġsm oking -Ġindic ates -K ey -rop ri -ĠF ile -all ing -ast ing -ĠR us -Ġad j -Ġ7 9 -av al -Ġpres um -bur gh -on ic -Ġf ur -Ġpoll s -ik a -Ġsecond ary -Ġmon ster -ig s -ĠCur rent -E vent -Ġowners hip -end ar -Ġarri ve -ĠT ax -Ġn ull -ĠPri v -Ġth ro -Ġk iss -c at -Ġup set -ang le -it ches -ect or -olog ists -ĠGal axy -Ġcor ruption -Ġh int -ent er -ĠH ospital -Ġgreat ly -Ġbeg un -es y -Ġso il -ĠAnt on -Ġmain tenance -ãĥ © -Ġdo zens -Ġhuman ity -ĠAl abama -Ġr om -w orth -ap ing -sylv ania -l ah -Ġg athered -G A -Ġattack ing -f ound -ĠSqu are -Ġar bit -ict ions -ĠW isconsin -Ġd ance -ĠS aint -arch y -Ġbase ball -Ġcontribut ions -Ġliter ature -Ġex ha -per ty -t est -Ġb ab -Ġcontain er -let ter -Ġfall en -Ġwebs ites -Ġbott le -ĠS ac -Ġbre ast -ĠP L -Ġveter an -Ġinterview s -ĠA le -Ġb anned -eng ers -ĠRev olution -in th -Ġconc erning -IV E -Ġexp enses -ĠMatt hew -ĠColumb ia -d s -ist ance -Ġent ity -.. ." -Ġrel iable -Ġpar alle -ĠChrist ians -Ġopin ions -Ġin du -l ow -Ġcompet e -Ġth orough -Ġemploy ed -Ġestablish ment -ig en -ĠC ro -Ġlawy ers -ĠSt ation -T E -ĠL ind -ĠP ur -it ary -Ġeffic iency -âĢ IJ -ĠL y -Ġm ask -Ġdis aster -Ġag es -ER E -es is -ĠH old -Ġcas ual -b led -Ġen abled -ĠEn vironment -ĠInt elligence -i per -ĠM ap -ĠB E -Ġemer ged -is dom -Ġc abin -Ġregist ration -Ġfing ers -Ġro ster -Ġfram ework -ĠDo ctor -et ts -Ġtransport ation -Ġaware ness -H er -Ġattempt ing -O ff -ĠSt ore -ÃĥÃĤÃĥÃĤ ÃĥÃĤÃĥÃĤ -ĠK now -Ġdef ence -Ġsc an -ĠT en -ĠCh air -ĠP H -ĠAtl anta -Ġfuck ing -Ġans wered -b n -ĠK ar -Ġcateg ories -Ġr ational -Ġc ust -Ġrob ot -Ġcorrect ly -Ġg if -Ġgraph ics -m ic -Ġground s -ĠO pp -i ate -Ġdist ributed -Ġsan ctions -Ġchalleng ing -ut o -Ġingred ients -Ġinv ited -Ġfound ed -ĠRe qu -d ed -Ġb owl -Ġbrother s -ĠH a -I O -Ġw ages -im ore -oc ial -Ġse ed -ative ly -Ġaddress es -ĠI owa -ab eth -Ġatt itude -is d -ch ild -Ġm ole -Ġdisco very -y ard -B r -Ġ8 2 -Ġsuppl ies -ell ing -Ġdist ingu -C R -Ġre cept -Ġ vert -Ġsw im -b ec -d oor -ĠY eah -Ġg al -Ġinter act -ĠE SP -ĠC S -amp s -Ġconvin ced -Ġobject ive -Ġdis h -ĠPhot os -l ad -Ġdownt own -o il -in ction -Ġto morrow -ĠC OM -Ġsurv ival -sh ot -Ġsett lement -C ons -ĠX box -int erest -ĠS M -arg o -en ess -Ġeth nic -b ered -M in -ĠT ok -Ġinc ent -ĠComm and -Ġmain tained -Ġbreak s -br idge -at ar -ag g -ĠF inally -un icip -ĠO nt -le ft -Ġrecogn ition -Ġ* / -ĠP ers -Ġwe lf -Ġaddress ed -ĠK ansas -Ġvir us -Ġwhere as -Ġp apers -ram s -ĠMin istry -Ġple asure -Ġacqu ired -Ġd uration -j pg -Ġcal m -ĠN HL -Ġburn ing -Ġfold er -ick ed -ĠP y -ĠIll inois -Cl ass -ĠGodd ess -Ġperform ing -Ġwelf are -j ar -In ter -Ġl in -Ġenh ance -Ġnot ion -f are -yp es -ĠAre a -Ġcann abis -ĠDie go -f s -ĠM anchester -com m -in ite -Ġcover ing -ĠS ound -Ġ19 60 -Ġ8 4 -e lect -z ing -Ġcitiz en -Ġph ones -Ġr aid -Ġign ored -ĠOb ject -Ġu pload -c ard -Ġmod ified -Ġroom s -ia h -r ange -he ast -ach us -Ġsuggest ing -âĢ ĭ -gr ade -E l -Ġclot hing -Ġr h -ĠH an -un ity -en cing -ĠAust in -sec ution -t ra -d em -ĠQ ual -Ġhe aven -Ġst ages -Ġw edd -pl us -ific ial -ĠIm m -ĠH o -iet ies -Ġphr ase -Ġbr ill -act ory -Ġprov iders -Ġsil ence -Ġa er -ĠA I -ĠAd venture -Ġplatform s -Ġdemonstr ated -Ġinter f -ing ton -Ġr aces -Ġgr ade -ult ane -ĠTh rough -f alse -Ġb ow -ĠA B -Ġfl avor -Ġhistor ic -g ov -Ġcol our -Ġview ed -ĠEm ail -el come -Ġinter vention -Ġd iversity -Ġperiod s -Ġre verse -ĠV ery -Ġqu ote -ĠLe ft -th rough -Ġsc rew -Ġland ing -Ġp ill -Ġw et -Ġprot esters -Ġrepe at -av ed -er k -Ġsal ary -ĠPenn sylvania -St ill -Ġmay or -Ġkit chen -Ġfeat uring -ĠM useum -ĠT ournament -ĠF al -Ġser vers -U C -Ġany body -im g -ĠTr ade -ixt ure -the less -Ġfin ance -Ġcl osing -ĠPat ri -i ac -ab el -Ġ> > -or ous -Ġf irms -sc reen -un a -Ġemb arrass -ul se -Ġlet ting -Ġth rew -ile y -Ġch annels -l an -ĠVeg as -Ġse ar -Ġfant astic -ar re -uzz le -ĠD er -Th ose -Ġsw ing -Ġshe et -ind ex -co ver -og an -Ġvari ables -ĠTe ch -Ġsp oken -ac hel -ĠD a -ĠMount ain -Ġload ed -Ġfoot age -vers ion -Ġun l -ĠPh oenix -Ġthrow ing -Ġf iring -Ġtrack ing -Ġw idth -Ġstrugg ling -ro oms -ot ion -Ġmonth ly -ĠSer ver -Ġegg s -op en -M C -Ġ199 3 -Ġh ired -Ġstay ed -ĠAll en -Ġst ro -Ġ9 8 -st ep -ĠTurk ish -Ġfab ric -ist ing -ĠD om -Ġd ates -Ġpr on -Ġbasket ball -Ġl ucky -ĠArab ia -Ġassum ed -est y -Ġaff airs -Ġgl ad -ĠInd eed -ĠF A -ĠW ord -Ġjo ining -if ice -p read -ir ts -ĠSe lect -Ġpop ulations -aw are -Ġn ose -Ġcompl aints -st art -Ġsc oring -Th anks -Ġmin ing -Ġvisit ors -S H -Ġdam aged -Ġcharacter istics -ĠP ent -D C -Ġ8 3 -ĠS ix -r ates -Ġfl ags -ĠB rew -d og -M ark -// // -Ġexec ution -Ġj oke -ph ones -Ġtestim ony -Ġob st -Q L -ĠC ut -Ġstud ied -ĠN intendo -ick et -ĠN BC -Ġl ad -ĠB ra -ĠM oh -Ġk ernel -Ġoverwhel ming -Ġag ed -Ġapplic able -ĠC ond -Ġroad s -ĠBl ock -m ade -od ge -Ġcomm ands -Ġoff ices -vel and -Ġt ut -Ġrece iver -ĠF ro -Ġsho pping -Ġi P -ĠSt re -ĠA BC -Ġentertain ment -ĠB ow -ort ed -M c -Ġread s -gr ad -ĠCol lect -Ġâ ĪĴ -ĠCap ital -eder ation -Ġemploy er -Ġinvolve ment -Ġanx iety -al ia -Ġro of -ĠAm ong -ĠDemocr at -Ġstat s -ĠV ill -Ġconst itutional -Ġrefer ring -itt y -Ġtack le -out ube -Ġback ed -ĠH ong -ĠBro ad -Ġe le -ĠO tt -Ġ199 2 -h our -achus etts -C al -Ġdefe ated -Ġ8 1 -es p -Ġseem ingly -w as -ĠJ enn -ĠK urd -Ġg ene -Ġdisc ount -R et -EC T -( ); -Ġclub s -Ġs id -ĠM arsh -Che ck -Ġp p -ĠE ag -ides pread -Ġbe ings -F T -Ġintrodu ction -ĠCh ange -AR D -Ġ1 10 -ad ows -ier ce -Ġme al -a uthor -ĠB ang -lah oma -Ġr anks -201 1 -?? ?? -m ax -Ġcoll apse -Ġop ens -Ġe cho -Ġs oph -Ġrac ist -Ġenorm ous -Ġw aves -Ġt ap -Ġcomprehens ive -. -- -ĠR oy -Ġfarm ers -Rel ated -a ired -ron es -ĠC rim -Ġproport ion -Ġdesign s -Ġnegoti ations -Ġvirt ually -ĠBat man -Ġwar n -Ġlegit imate -m ate -Ġcon vention -, , -net ic -ĠS D -Ġconsist ently -Ġcompens ation -Ġpunish ment -Ġy e -Ġt ie -ĠB ureau -ir lf -ĠB u -ĠA ren -ĠPh ilipp -Ġkn ife -Ġmem ories -ĠR oss -Ġang le -Ġ8 6 -ĠTh under -Ġre nd -ĠT our -Ġcount s -s ung -ĠIm p -Ġeduc ational -Ġaccess ible -C OM -Ġd rew -y er -G l -am ine -OR T -O B -I B -m aster -Ġtri als -og y -h ar -ĠTr ust -Ġprefer red -irlf riend -ĠN ev -Ġb in -Ġc ow -P age -Ġsign ature -ĠB L -7 00 -Ġret ired -Ġby tes -Ġneigh b -ĠLeg end -Ġdev ast -Ġsuspect ed -is ons -ĠPoké mon -sc ale -Ġcap abilities -Ġre vel -Ġche ese -d y -igr ant -Ġfail ing -b its -ĠHer oes -ĠG host -ĠS cient -Ġappoint ed -ur i -Ġinst itution -Ġexpand ed -g reg -Ġmonitor ing -Ġp odcast -Ġcoal ition -Ġ9 6 -J o -Ġst olen -ĠS ab -Ġstop s -Ġhol iday -Ġint r -C ar -Bl ack -ĠL GBT -Ġwar ming -ĠAnd erson -Ġ8 9 -Ġprodu cer -M ed -Ġaccur acy -ĠMar vel -iz abeth -ĠPat rick -m ony -Ġmin i -ac les -Ġover t -the y -Ġmembers hip -ĠV en -Ġex ch -Ġrem oval -ĠD ave -T Y -m ad -ĠF ind -Ġad equ -Ġe c -Ġte eth -Ġemot ion -Ġper m -Ġsole ly -d b -Ġextra ord -IG HT -c al -Ġgu idelines -Ġd ying -Ġsusp ended -ĠPrem ier -ĠAnth ony -el ve -Ġd ad -ĠE th -ĠFoot ball -Ġabandon ed -Ġ< < -Ġm arch -Ġhor ror -âĢ¦ " -Ġchild hood -Ġcampaign s -Ġl unch -ĠAl bert -bl ock -âĸĪ âĸĪ -ound ing -Ġb one -or gan -ad ers -ĠFl ash -ĠDri ve -Ġton ight -Ġw ars -ĠF L -Ġform ation -con st -New s -Ġcom pe -or ious -ĠSt aff -Ġdiscuss ions -ĠProt ection -ĠJ am -Ġcrit eria -Ġinstall ation -Ġaccompl ish -iz za -Ġpub lisher -Ġresc ue -ĠT ry -U LL -ĠS om -ĠH op -ore t -th s -ord on -Ġp ocket -ĠIn v -Down load -ĠCr ime -Ġb ene -ĠGu ide -ĠAs sembly -Ġparam eters -I E -ĠAlex ander -Ġconc ert -ĠSc he -Ġsh oes -Ġvis iting -Ġrec all -Ġb ub -Ġr ural -Ġconc rete -ĠR os -N ext -R uss -Ġlo ans -ĠSh ield -Ġtre m -hem at -k g -ĠHar ris -is ition -ĠM ove -ĠF C -Ġf ate -ĠCh o -Ġt ired -Ġprinc ipal -h ist -ien ces -ath y -Ġse vent -Ġm ood -Ġstrateg ic -Ġdise ases -Ġfor um -Ġtem por -Ġhead quarters -P ar -ig e -fl ix -Ġgu itar -Ġ9 4 -On ly -Ġrele ases -ro ph -================ ================ -Ġ6 00 -ĠContin ue -ig ate -ĠC rit -sy stem -Ġdis abled -Ġunex pected -ith ub -Ġuncle ar -ĠE st -Ġcontr ad -Ġstrateg ies -vent ures -Ġpass age -AM E -Ġimpro ving -Ġreve als -Ġdecre ase -ov a -Ġann oy -ĠSh ort -ĠL ibrary -Ġcy ber -n ell -ĠH ur -ĠC B -Ġphot ograp -U I -Ġs ed -G e -Ġ8 7 -Ġd iverse -Ġencour aged -Ġcons piracy -Ġbird s -Ġoper ator -Ġhand ful -Ġclass ified -? ) -Ġdram atic -Ġinvestig ators -it o -Ġw idespread -ĠR oom --------------------------------- -------------------------------- -Ġcollect ive -Ġjournal ist -St ring -Ġtemper atures -il a -Ġgu id -Ġins pect -Ġmiss ile -ĠMay or -Ġman ual -Ġsim ultane -Ġrat ings -Ġsu ck -Ġ9 7 -Ġunivers al -Ġph arm -Ġdis rupt -ian o -A V -Ġf t -Ġstat ist -old s -ĠWalk er -ph p -Ġunder t -ĠL as -ish op -nt il -res hold -ĠWhe ther -M s -Ġden y -ĠCl oud -Ġprov ider -Ġsurv iv -ĠUp date -h as -Ġmist akes -ch arge -pl ed -r ity -Ġn ode -ĠMass achusetts -ool s -lic ation -Ġf ails -em ale -or i -back s -Ġsh irt -Ġ' ' -ĠN AT -Ġwat ers -els on -Ġe ase -Ġsc ar -Ġcont ents -m ind -Ġcont ribution -Ġsh r -Ġhand ed -Ġst ability -Ġtra ve -E m -Ġmir ror -12 3 -Ġwe igh -Ġf iction -ou ver -ist ant -r ition -ĠF ed -Ġphys ically -Ġst ake -ĠArt icle -ĠAr c -ĠLew is -ĠM ind -Ġdemonstr ate -Ġprof its -v ision -om ic -ol id -Ġbatt les -Ġdri ves -Ġeas tern -ĠS ony -!! ! -ar ation -v ard -ĠG L -port ation -Ġ9 2 -Ġlaw makers -Ġprotect ing -ĠE PA -Ġy eah -Ġsh ame -ol ph -e ven -x it -Ġatt ach -Ġrepresent ing -Ġob s -ĠUt ah -iff s -ĠFre edom -à ³ -A K -Ġinc idents -it age -Ġview ers -c d -Ġm ouse -Ġcl ar -Ġaccord ance -Ġb ot -c or -ĠSum mer -he ld -Ġinnoc ent -Ġiniti ative -ol s -________________ ________________ -Ġsp ots -p ace -Ġconvent ional -Ġcorpor ations -Ġblock ed -H D -at tered -Ġref ers -Ġbu ck -ĠDig ital -12 0 -Ġtop ics -T F -Ä ģ -br id -re ement -Ġunder lying -ĠM ember -Ġinvestig ating -Ġpregn ancy -Ġtouch down -ĠB and -ĠCall er -Ġinst ances -P P -w a -G ood -Ġ199 1 -ĠC old -Ġfear s -Ġrem arks -Ĩ Ĵ -at al -Ġm it -Ġexper iments -i pt -Col or -ind u -Up date -Ġ9 3 -A g -Ġ å -anc ouver -B oth -Ġjud ges -Ob ject -Ġst ere -umb n -Ġparticip ation -ĠSt ars -ĠJ ere -Ġweek ly -ĠB an -Ġconvers ations -ĠP itt -u z -ĠIndian a -ĠK ick -Ġinf ection -Ġhero es -Ġsett led -Ġstri p -Ġh al -Ġd ump -ĠS ci -Ġl es -Ġref erences -ĠU RL -ĠBr idge -Ġwant ing -For ce -Ġex clus -Me anwhile -m n -Ġg entle -m aker -sen al -ĠG ro -ou ri -ĠR ain -ĠAll iance -Ġl ift -el a -S D -ĠCle veland -Ġrank ed -Ġst adium -Ġdead ly -ä ¸ -Ġr iding -ar ia -ĠAr mor -Ġdocument ation -ĠGree ce -ree k -Ġl ens -ĠS a -Ġg ross -ĠE mer -ag ers -ĠD ub -ĠR h -ĠAM D -Ġarri val -Ġdes ert -Ġsupp lement -ĠRes p -Ġkn ee -Ġmarg in -f ont -og g -201 0 -ĠP ir -ĠP rom -iv als -Ġint ake -Ġdifferent ly -ug s -Ġb its -clud ed -Ġsearch ing -ĠD u -um ble -Ġfunction al -ĠBalt imore -ĠC ould -Ġdes ired -Ġcirc uit -ĠL yn -ĠG O -ĠF alse -re pre -' : -alt ies -Ġmin im -Ġdro ve -ĠSh ould -Ġh ip -Ġpro s -Ġut ility -ĠN ature -ĠM ode -P resident -o pp -r at -form ance -Ġconcent ration -Ġf ont -ĠB ud -Ġam id -Ġre vers -ĠM L -B ar -Ġinter action -Ġjur isd -Ġspell s -d ep -f il -Ġcivil ians -ut ter -ĠCo oper -ĠBel ow -Ġent rance -Ġcon vert -Ġcontrovers y -ow ered -Ġcontr ary -Ġar c -ĠExec utive -ĠOffic er -Ġpack ages -Ġprog ressive -w idth -Ġreserv ed -v ol -ĠSam sung -Ġprint ed -Ġcent ers -Ġintrodu ce -ĠKenn edy -Ġodd s -Ġsure ly -Ġindepend ence -Ġpass engers -repre ne -ĠBe h -Ġl oves -ĠESP N -Ġfac ilit -Ġident ical -Ġdo ct -Ġpartners hip -con f -ĠH ide -Ġconf used -ĠC ow -M en -Ġw rest -ĠIraq i -Ġh oles -ĠStud ies -Ġpregn ant -h ard -Ġsign als -I X -Ġpull ing -Ġgrad uate -Ġnomine e -D ate -Ġper mitted -Ġâ Ĥ¬ -ĠOk lahoma -St art -Ġauthor ized -Ġal arm -ĠC os -v an -Ġgener ations -c ular -Ġdr agon -ĠSoft ware -ĠEd ward -Ġcontro ller -S en -ge red -ĠV ik -Ġappro ached -Th ank -Ġcan ce -Ġform ula -ĠSm all -Ġweak ness -Ġr amp -it udes -j ud -Ġbrill iant -Ġacc us -s ource -Ġ8 00 -ĠE vil -S w -Ġhom eless -we ek -i ens -r ics -ĠTh ird -T O -Ġorgan ic -Ġpresent ation -ag h -ĠDown load -v ation -Ġas sembly -or able -hold ers -ĠBern ie -ĠHel p -Ġt ong -ĠF ight -Ġbe ach -B ook -ĠL ic -Ġr ush -ĠR ound -ou p -ĠMar x -Ġcalcul ated -ĠDe vil -ĠSar ah -Ġoccasion ally -Ġbul let -Av ailable -g ate -Ġ9 1 -Ġh osp -Ġprom ises -ĠH IV -ĠSt adium -ĠSt ock -ĠCorpor ation -g age -N G -ĠC redit -Ġs ne -ib l -Ġacc um -s uch -Ġterror ists -Ġconscious ness -ĠZ h -Ġdram a -ool a -pir ation -Ġlab our -ĠN in -Ġut ter -Ġdemocr atic -Ġass ass -il ation -Ġg est -Ġab road -Ġmet ab -Ġs orts -Ġfl av -U B -Ġm g -ĠNot hing -ĠO d -Ġmus ical -200 9 -Ġdro ps -oc ated -ater al -0000 00 -Ġg re -Ġequ ality -Ġburd en -Ġv ig -ĠLe ader --------- ---- -Ġcere mony -Ġf ighter -Ġact ors -Ġ æ -am an -F i -Ġal ign -put er -Ġe lder -ĠN SA -Ġrepresent ation -ĠOnt ario -IT H -usal em -Ġharass ment -itz er -Ġsy mp -Ġbox es -ĠD R -Ġman ifest -at re -Ġ ^ -Ġd ies -le ton -Ġmiss ions -et he -Ġres olve -Ġfollow ers -Ġas c -Ġk m -l ord -am med -Ġsil ent -ĠAssoci ated -Ġtim ing -Ġprison ers -ĠK ings -ĠF ive -Ġtow er -Ġappro aches -Ġprecise ly -Ġb ureau -ĠM other -ĠI ss -Ġkey board -it ual -Ġfund ed -Ġstay ing -Ġpsych ological -Ġm ile -ĠLe on -ĠBar b -w ill -Ġw ider -ĠAtl antic -Ġt ill -ĠR ome -ro t -Ġaccomp an -Ġfl our -ac o -W orld -ĠExp ress -ĠY u -C or -Ġple ased -part y -Ġpoint ing -Ġinf lation -Ġro y -Ġ ), -ain er -Ġwedd ing -orm on -Ġrequ iring -Ġqual ified -Ġse gment -EN D -Ġs izes -e als -Ġcor rupt -ass ador -Ġcele b -Ġdream s -ĠM ess -Ġcheck ing -ĠV ersion -Ġprep aring -Ġact ively -ĠD iff -Ġl ux -ĠW inter -act eria -ĠN E -Ġdep uty -Ġtrans gender -Ġsum mary -Ġin her -er ies -ch ar -ĠY an -Ġkn ock -ĠP ath -Ġl ip -roll er -Ġimp ression -Ġcelebr ate -Ġsl ide -Ġgu ests -Ġcl ip -F S -Ġsav ings -Ġcapt ain -Ġleg acy -ĠDen ver -Ġw ounded -tab oola -AC T -Ġpurs ue -Ġo xy -Ġ q -Ġsem i -ĠN eed -ĠAff airs -Ġob sc -Ġcheck ed -Ġd ual -C ode -ĠM D -le m -ult y -Ġ © -ĠEl izabeth -Ġcent uries -ard ed -s rc -Ġev ident -enn is -at in -Ġunemploy ment -ĠMar io -Ġint im -Ch rist -Ġbi ological -Ġsold ier -ĠAdd ed -Ġm ath -ĠG il -Ġbi as -Ġd ating -ĠO cean -Ġm ice -M us -h ire -ĠT es -Ser ver -lim ited -S ize -Ġmet ers -Ġrock et -es see -Ġcertific ate -ĠIran ian -AS S -Ġgr id -D ec -Ġro lling -com mun -ĠSwed en -b ury -Ġtiss ue -Ġrac ism -ĠL ocal -Ġmyster y -Ġexam ine -Ġst em -Ġs its -Ġhop ed -ot ing -Ġdial ogue -Ġpers u -W atch -l ay -M AN -Ġch ronic -ĠPort land -mark et -ĠS EC -Ġparalle l -Ġsc andal -Ġcar ries -Ġphenomen on -h uman -ack er -ĠO x -Ġretire ment -tain ment -ov ie -ĠG ear -Ġd uties -Ġdo se -Ġsc roll -M B -in f -Ġsa uce -Ġland scape -red dit -ĠChampions hip -ĠRed dit -al id -Ġco in -Ġover s -Ġpost ing -ab out -Ġf el -and y -Ġb old -Ġfocus ing -e ffect -G R -Ġde emed -Ġrecommend ations -Ġste pped -Ġvot er -ĠDe ep -ĠInst agram -Ġmoder ate -ĠMary land -Ġrestrict ed -ĠM B -ĠCh all -Ġto b -Ġc ir -ĠO cc -ĠE ver -Ġcoll aps -IN FO -= - -ĠP ict -ĠAcc ount -n c -Ġo ught -Ġex port -Ġdr unk -( ' -Ġw ise -ĠM ort -ne cess -Ġan cest -ĠInc re -Ġfrequ ent -m ir -Ġinterpret ation -Ġdepend ent -Ġco ins -ĠB ol -V ideo -ĠJust in -Ġfat al -Ġcook ing -Ġconf usion -ip her -Ġcust ody -ĠMor gan -om ach -ĠGovern or -Ġrestaur ants -el ing -Ġacknowled ged -Ġthe r -Ġgen es -ch ing -He y -Ġtact ics -ĠMex ican -Ġv end -Ġhe s -qu er -Ġnot ing -ĠCamer on -Ġtarget ing -ro ck -Ġcred its -Ġemot ions -Ġrepresent atives -new s -Ġlegisl ative -Ġrem oving -Ġtweet ed -ĠCar ter -ĠF ixed -Ġfor cing -Ġspeak er -Ġm ales -ĠViet nam -l ined -Ġconcept s -Ġvo ices -o ir -ĠT rib -W he -ĠJer usalem -ĠS ant -Ġc ul -Ġl ady -ĠHaw ai -Ġar ts -ĠIn n -ĠMach ine -ĠEm peror -Ġsl ot -g ly -ĠPro cess -II I -Ġathlet es -ĠTem ple -ĠRep resent -Ġpres c -Ġt ons -Ġgold en -Ġp unch -ĠG R -iver pool -Ġen act -Ġlob by -Ġm os -Ġpick ing -Ġlif etime -Ġcogn itive -E ach -z o -Ġd ub -Ġcons ists -ol n -Ġf estival -am ous -Ġint ellig -w ords -ĠSm art -Ġde le -Ġl apt -Ġmag ical -ĠS in -b us -ur ities -igh th -ĠRub y -ĠS ure -ol ving -Ġj un -O ST -Ġimp osed -Ġast ron -Ġcor rel -ĠN S -ĠK it -ĠF uture -b urn -Ġimm une -oc us -Ġcour ses -ĠSt ring -Ġle an -Ġg host -Ġout comes -Ġexp ense -Ġevery day -Ġaccept able -A h -Ġequ ipped -Ġor ange -F R -ĠD utch -Th ough -ĠR ank -Q U -ĠRober ts -wh at -re nd -Ġdisapp ear -Ġsp awn -ĠL am -o is -Ġdes erve -Ġmin imal -Ġnerv ous -ĠW ould -Ġro ok -ĠV ancouver -Ġres ign -sh ire -ĠW orks -ĠB uild -Ġafford able -ĠG ary -ĠAren a -Ġh anging -Ġimpl ications -ĠS ong -Ġmain taining -Ġgu ards -C ON -Ġder ived -Ġexecut ed -Ġthe ories -Ġqu oted -ĠAnd re -og a -sel ess -in fo -ĠBel g -Ġt ears -ĠSur v -Ġbirth day -ig ious -im mer -Ġspect rum -Ġarchitect ure -Ġrec ruit -arm a -T able -Ġmon sters -ĠG ov -Ġdest ination -Ġattract ive -Ġf oss -ĠMore over -Ġpres ents -TH E -Ġrep ly -pt on -Ġc um -Ġdel ight -Ġaffect s -Ġdon ations -ĠT oy -ĠH im -M ENT -Ġover come -it ched -ĠFant asy -ĠH at -ĠBe ast -b ott -Ġinvestig ations -R un -Ġhun ting -d i -f und -Ġs essions -est yle -Ġport ray -oid s -Y eah -Ġcommun icate -Ġcom edy -ĠY ang -Ġbel t -ĠMar ine -Ġpredict ed -Pl ay -Ġimportant ly -Ġremark able -Ġelim inate -D avid -Ġb ind -V ID -Ġadvoc ates -ĠG aza -im p -D B -ĠN a -ĠSim ilar -I ES -Ġchar ity -v as -m ath -Ġâ ĸ -ok er -nd um -Ġcap s -ĠH al -2 000 -e an -Ġfle et -Ġrec re -R ight -Ġsleep ing -ij ing -k ind -Ġdesign ated -à ¤ -Ġanim ation -ke e -ĠInt rodu -Ġ/ > -Ġdelay ed -Ġtrem end -Ġcur ious -U se -Ġle ct -d am -Ġinnov ation -ĠPoint s -Ġload ing -Ġdisp ute -ct ic -ird s -ĠB Y -Ġn urs -ĠVal ue -ION S -ĠH um -Ġtem plate -m ers -Ġappear ances -ĠEnter tainment -Ġtransl ation -Ġsa ke -Ġbene ath -Ġin hib -Ġe uro -abet es -Ġstud ying -ĠM as -Ġper ceived -Ġexam ined -Ġe ager -Ġco aches -Ġim per -ch i -Ġprodu ces -" ). -ĠEvery one -Ġm unicip -Ġg irlfriend -Ġh ire -ĠV ice -Ġsu itable -op y -Ġin equ -ĠD uke -f ish -f irst -ĠO bs -Ġinter ior -ĠBru ce -ĠR y -Ġanal ys -Ġconsider able -Ġfore cast -Ġf ert -ors hip -ĠD rug -ĠA LL -: " -th ur -ĠM ail -Ġball ot -Ġinst antly -ĠCh annel -Ġp icks -Ġ198 9 -Ġt ent -ol i -Ġcivil ian -b ling -ell o -b u -Ġin ch -Ġlog o -Ġcooper ation -Ġwal ks -Ġinvest ments -Ġimp rison -ĠF estival -ĠK y -Ġleg ally -Ġg ri -ch arg -S l -Ġthreat ening -du ction -fl ow -Ġdismiss ed -ibr aries -c ap -e le -ĠMc G -ĠHar vard -ĠConserv ative -ĠC BS -p ng -Ġro ots -ĠH aving -umb led -ĠF un -\ / -ĠS earch -ple x -Ġdiscuss ing -Ġcontin u -ĠT ai -ĠW ik -F ree -f it -Ġref use -Ġmanag ing -Ġsy nd -ip edia -w alk -Ġprofession als -Ġguid ance -Ġunivers ities -Ġas semb -unt u -F inally -AS E -ĠAut o -ĠH ad -Ġann iversary -L D -ĠD ur -ĠUlt imate -ih ad -pro duct -Ġtrans it -Ġrest ore -Ġexpl aining -Ġass et -Ġtransfer red -Ġbur st -ap olis -ĠMag azine -ĠC ra -ĠB R -gg ed -ĠH E -M ich -b et -ĠL ady -yl um -erv es -Ġme ets -wh ite -L og -Ġcorrespond ing -Ġins isted -G G -Ġsurround ed -Ġt ens -Ġl ane -Ġco inc -h ome -Ġexist ed -ect ed -ĠDou ble -lam m -Ġske pt -ex p -Ġper ception -ie v -ĠBe ing -o ft -Ġadop t -. : -] ; -Wind ows -Ġsatell ite -AS H -Ġinf ant -d escription -ĠMe anwhile -c m -oc a -ĠT reat -act or -Ġtob acco -ĠN orm -em ption -Ġfl esh -Ġj e -o op -ĠHe aven -Ġbe ating -an im -Ġgather ing -Ġcult iv -G O -ab e -ĠJon athan -ĠSaf ety -Ġbad ly -pro t -Ġcho osing -Ġcontact ed -Ġqu it -Ġdist ur -Ġst ir -Ġto ken -D et -ĠP a -Ġfunction ality -00 3 -s ome -Ġlimit ations -Ġmet h -b uild -con fig -N T -re ll -ble m -ĠM om -Ġveter ans -ĠH u -Ġtrend s -are r -ĠG iven -ĠCa ption -m ay -AS T -Ġwond ering -ĠCl ark -n ormal -Ġsepar ated -Ġdes p -st ic -b rew -Ġrel ating -ĠN ik -ĠF arm -Ġenthus i -g ood -d eb -Ġactiv ist -Ġm art -Ġexplos ion -ĠEconom ic -L ink -Ġins ight -Ġconven ient -Ġcounter part -su pport -ĠV irt -ag en -ĠTenn essee -ĠSim on -ĠA ward -OC K -ĠF igure -Ġoverse as -Ġpr ide -ĠC as -n ote -m g -C urrent -Ġdispl ays -cont ent -Ġtravel ing -Ġhosp itals -ĠFin ancial -ĠP ast -Ġdefend ant -Ġstream ing -m ble -ĠBer lin -uk i -Ġdist ribut -Ġant ib -Ġch ocolate -ĠCast le -Ġinter rupt -ĠR ow -Ġconvers ion -Ġbug s -ĠR ather -li est -L Y -ĠJe an -com mon -ak h -Ġ1 30 -ot ton -ĠDe an -Ġam endment -Ġgame play -ĠWar ren -od a -Ġhigh lights -Ġir re -ĠNAT O -Ġball s -Ġdemand ing -U RE -ĠL uke -F igure -st op -on ia -z one -iz ers -ĠW R -Ġaward ed -Ġregul atory -ĠH art -ĠS N -pl ing -Ġs our -ĠP ixel -us ive -Ġf et -ĠS ent -Ġautom atic -Ġf er -vern ment -ĠKh an -T ON -f ather -Ġextraord inary -th rop -ĠP ython -ĠG PU -Ġsex ually -Ġdesk top -it ivity -ĠAnton io -Ġo rient -Ġe ars -ob by -ous es -vertis ements -Ġmanufacture rs -ic ient -min ute -Ġconv iction -Ġg arden -p ublic -Ġsatisf ied -f old -O K -Ġin hab -ĠTh ink -Ġprogram me -Ġst omach -Ġcoord in -Ġh oly -Ġth reshold -Ġr het -Ġser ial -Ġemploy ers -ĠEvery thing -ra h -Ġb other -Ġbr ands -Val ue -ĠT ed -ĠPlan et -Ġp ink -ĠFurther more -s a -P E -re ck -ĠUS D -ot te -Ġ& & -Ġland ed -g ets -Ġprodu cers -Ġhealth care -Ġdomin ant -Ġdest ro -Ġam ended -ch ron -Ġf its -ĠSy d -ĠAuthor ity -AT CH -Ġfight s -ĠL LC -Ġ-- - -ĠCor p -Ġtox ic -spe cific -ĠC orn -ĠChe l -Ġtele phone -ĠP ant -Ġmyster ious -aun ch -od ox -med ia -Ġwitness es -ag u -Ġquestion ed -ĠBre xit -ĠRem ember -ene z -Ġend orse -iat ric -ĠId ent -Ġridic ulous -1 10 -Ġpr ayer -Ġscient ist -Ġ19 50 -ĠA qu -Ġunder ground -ĠU FC -m are -ĠL ater -w ich -Ġsubsc rib -Ġhost s -Ġer r -Ġgr ants -ant om -Ġsum mon -ear ly -ĠC lear -ĠPr im -Ġsusp ension -Ġguarant eed -app er -Ġr ice -ĠSe an -ĠSh in -Ġrefere ndum -Ġfl ed -r ust -Ġ3 60 -ter y -Ġsh ocked -B R -ĠO il -ĠAll ah -Ġpart ly -Ġign or -Ġtrans mission -Ġhom osexual -ivers al -Ġhop efully -ãĤ ¤ -Ġless on -L eg -Ġ .. -Y et -t able -app ropri -re tt -Ġbo ards -Ġincor rect -Ġb acteria -ar u -am ac -Ġsn ap -.' " -Ġpar ad -t em -he art -Ġav ailability -Ġw isdom -Ġ( + -Ġpri est -ĠÂł ĠÂł -O pen -Ġsp an -Ġparam eter -Ġconv ince -Ġ( %) -r ac -Ġf o -Ġsafe ly -Ġconver ted -ĠOlymp ic -Ġres erve -Ġhe aling -ĠM ine -M ax -Ġin herent -ĠGra ham -Ġinteg rated -D em -Ġpip eline -Ġapp lying -Ġem bed -ĠCharl ie -Ġc ave -200 8 -Ġcons ensus -Ġre wards -P al -ĠHT ML -Ġpopular ity -look ing -ĠSw ord -ĠAr ts -' ) -Ġelect ron -clus ions -Ġinteg rity -Ġexclus ively -Ġgr ace -Ġtort ure -Ġburn ed -tw o -Ġ18 0 -P rodu -Ġent reprene -raph ics -Ġg ym -ric ane -ĠT am -Ġadministr ative -Ġmanufacture r -Ġ vel -ĠN i -Ġisol ated -ĠMedic ine -Ġback up -Ġpromot ing -Ġcommand er -Ġfle e -ĠRus sell -Ġforg otten -ĠMiss ouri -Ġres idence -m ons -Ġrese mb -Ġw and -Ġmeaning ful -P T -Ġb ol -Ġhe lic -Ġwealth y -Ġr ifle -str ong -row ing -pl an -as ury -âĢ¦ . -Ġexpand ing -ĠHam ilton -Ġrece ives -S I -eat ures -ĠAn im -RE E -P ut -Ġbrief ly -ri ve -Ġstim ul -Ġ`` ( -Ġ __ -Ġch ip -Ġha z -Ġpri ze -ĠTh ings -AC E -ul in -d ict -ok u -Ġassoci ate -ock ets -y outube -St ory -ateg ory -Ġm ild -ail ing -ĠY e -O rig -ĠK a -or ig -Ġpropag anda -Ġan onymous -Ġstrugg led -Ġout rage -AT ED -ĠBe ijing -r ary -Ġle ather -Ġworld s -Ġbroad er -12 5 -id al -ĠBet ter -Ġt ear -E xt -Ġpropos als -Ġit er -ĠSqu ad -Ġvol unt -m i -D id -ĠP u -p in -Ġspeak ers -Ġb orders -Ġfig ured -= ' -Ġsimultane ously -aed a -Ġcharg ing -Ġur ged -Ġcon j -25 6 -ĠG ordon -mer ce -Ġdocument ary -Sh are -it ol -ON E -ĠG arden -h att -ĠThom pson -ane ous -ap ore -Ġt anks -Ġless ons -tr ack -Ġout standing -Ġvolunte ers -Ġsp ray -Ġmanag ers -l arge -Ġcamp s -Ġart ificial -ĠR u -Ġb ags -th al -Ġcompat ible -ĠBl ade -Ġf ed -Ġarg ues -F I -Ġunf air -Ġcor n -Ġoff set -Ġdirect ions -Ġdisappoint ed -ĠCon vention -Ġview ing -M E -oc ity -Ġtown s -Ġlay ers -Ġro lled -Ġjump ed -Ġatt ribute -Ġun necess -inc oln -Ġsupp ose -ĠNet her -ch a -Ġbur ied -Ġsix th -B en -ress ing -OU R -Ġw ound -Ġcy cl -Ġmechan isms -Ġcongress ional -ĠE lement -Ġagre ements -Ġdec or -Ġclos est -ĠM it -Go ogle -} } -Ġm ixture -Ġflu id -S ign -ĠSch olar -Ġp ist -ask et -ab ling -Ġrac ing -he ro -ri el -ass y -Ġche aper -b en -Ġvert ical -amac are -ĠRead ing -g ments -Ġhelic op -Ġsacr ifice -ay a -p aren -V A -ĠL es -ĠStud io -Ġviol ations -ĠAn na -ac er -é ¾ -ĠR at -ĠBe ck -ĠD ick -ĠA CT -Ġcomp osition -Ġtext ure -ĠO wn -Ġsmart phone -ĠN A -Ġfor b -im port -Ġdef ending -il st -re r -Ġo h -ĠJere my -Ġbank ing -cept ions -Ġrespect ive -/ . -Ġdr inks -ĠW i -Ġb ands -ĠL iverpool -Ġg rip -ĠB uy -Ġopen ly -Ġreview ed -per t -Ġver ify -ĠCo le -ĠW ales -M O -Ġun pre -Ġshel ter -ĠIm perial -Ġgu i -ĠD ak -Ġsuggest ions -Ġexplicit ly -Ġsl ave -Ġblock chain -Ġcompet ing -Ġprom ising -S ON -Ġsoc cer -Ġconst itution -4 29 -Ġdist ract -ĠU ser -es ides -ĠMet hod -ĠTok yo -Ġaccompan ied -Cl ient -s ur -al og -Ġident ification -Ġinv asion -as ma -Ġindust ries -pp ers -Ġsub tle -ĠUn it -n atural -Ġsurv ived -Ġfl aw -ĺ ħ -ĠH oll -Ġdef icit -Ġtut orial -ĠCh ance -Ġarg uing -Ġcontem porary -Ġinteg ration -for ward -Ġt um -it is -Ġh iding -ĠD omin -ĠT an -ĠB uilding -ĠV in -Ġspokes person -ĠNot es -Ġemer ging -Ġprepar ation -Ġpro st -Ġsuspect s -Ġaut onom -D escription -Ġdeal t -ĠP ear -Ġstead y -Ġdecre ased -Ġso vere -ĠCl in -Ġgrad ually -ors es -ĠW AR -S erv -ãĤ ¢ -h r -Ġd irty -ĠB arn -ĠB C -Ġd il -Ġcal endar -Ġcompl iance -Ġch amber -b b -Ġpass enger -ate ful -ĠT itle -ĠSyd ney -ĠG ot -Ġdark ness -Ġdef ect -Ġpack ed -ass ion -Ġgod s -Ġh arsh -IC K -le ans -Ġalgorith m -Ġoxy gen -Ġvis its -Ġbl ade -Ġkil omet -ĠKent ucky -Ġkill er -P ack -enn y -Ġdiv ine -Ġnom ination -be ing -Ġeng ines -Ġc ats -Ġbuff er -ĠPh ill -Ġtra ff -AG E -Ġtong ue -Ġrad iation -ere r -m em -ĠExpl icit -é¾ į -Ġcou ples -Ġphys ics -ĠMc K -Ġpolit ically -aw ks -ĠBl oom -Ġwor ship -e ger -ut er -ĠF O -Ġmat hemat -Ġsent enced -Ġdis k -ĠM arg -Ġ/ * -P I -Ġoption al -Ġbab ies -Ġse eds -ĠScott ish -Ġth y -] ] -ĠHit ler -P H -ng th -Ġrec overed -ing e -Ġpow der -Ġl ips -Ġdesign er -Ġdis orders -Ġcour age -Ġch aos -" },{" -Ġcar rier -b ably -H igh -ĠR T -es ity -l en -Ġrout es -u ating -F il -N OT -w all -s burgh -Ġeng aging -ĠJava Script -ore r -li hood -Ġun ions -ĠF ederation -ĠTes la -Ġcomple tion -ĠT a -Ġprivile ge -ĠOr ange -Ġne ur -paren cy -Ġb ones -Ġtit led -Ġprosecut ors -ĠM E -Ġengine er -ĠUn iverse -ĠH ig -n ie -o ard -Ġheart s -ĠG re -uss ion -Ġmin istry -Ġpen et -ĠN ut -ĠO w -ĠX P -in stein -Ġbul k -S ystem -ic ism -ĠMarket able -Ġpre val -Ġpost er -Ġatt ending -ur able -Ġlicens ed -ĠG h -et ry -ĠTrad able -Ġbl ast -à ¤ -ĠTit an -ell ed -d ie -H ave -ĠFl ame -Ġprof ound -Ġparticip ating -Ġan ime -ĠE ss -Ġspec ify -Ġregard ed -ĠSpe ll -Ġs ons -own ed -Ġm erc -Ġexper imental -land o -h s -ĠDun geon -in os -Ġcomp ly -ĠSystem s -ar th -Ġse ized -l ocal -ĠGirl s -ud o -on ed -ĠF le -Ġconstruct ed -Ġhost ed -Ġsc ared -act ic -ĠIs lands -ĠM ORE -Ġbl ess -Ġblock ing -Ġch ips -Ġev ac -P s -Ġcorpor ation -Ġo x -Ġlight ing -Ġneighb ors -ĠU b -ar o -Ġbe ef -ĠU ber -F acebook -ar med -it ate -ĠR ating -ĠQu ick -Ġoccup ied -Ġaim s -ĠAdd itionally -ĠInt erest -Ġdram atically -Ġhe al -Ġpain ting -Ġengine ers -M M -ĠM ust -Ġquant ity -P aul -Ġearn ings -ĠPost s -st ra -ãĥ¼ ãĥ -Ġst ance -Ġdro pping -sc ript -Ġd ressed -M ake -Ġjust ify -ĠL td -Ġprompt ed -Ġscr ut -Ġspeed s -ĠGi ants -om er -ĠEd itor -Ġdescrib ing -ĠL ie -ment ed -Ġnow here -oc aly -Ġinst ruction -fort able -Ġent ities -Ġc m -ĠN atural -Ġinqu iry -Ġpress ed -iz ont -for ced -Ġra ises -ĠNet flix -ĠS ide -Ġout er -Ġamong st -im s -ows ki -Ġclim b -ne ver -Ġcomb ine -d ing -Ġcomp r -Ġsignific ance -Ġremem bered -ĠNev ada -ĠT el -ĠSc ar -ĠWar riors -ĠJ ane -Ġcou p -b as -Ġtermin al -, - -O H -Ġt ension -Ġw ings -ĠMy ster -�� �� -ĠUn like -val id -viron ments -ĠAl i -Ġn aked -book s -ĠM un -ĠG ulf -Ġd ensity -Ġdim in -Ġdesper ate -Ġpres idency -Ġ198 6 -h y -IN D -Ġun lock -im ens -Ġhand led -ĠE b -Ġdisapp eared -Ġgen re -Ġ198 8 -Ġdetermin ation -St ream -ik o -ap ters -Ġacknow ledge -J an -Ġcapital ism -P at -Ġ20 20 -Ġpain ful -Ġcur ve -Ġbom bs -st orm -ĠMet al -en cer -ĠF ig -ĠA aron -anc hes -Ġins piration -Ġexha ust -t ains -ash i -Ġdesc ript -Ġr itual -ĠChel sea -Ġpromot ion -ĠH ung -ĠW ard -iv a -ĠE T -Ġto ss -all ow -ĠFranc is -D ep -Ġhapp iness -ĠGl ass -Ġbet a -Ġstreng then -N E -o a -Ġbutt ons -ĠMur ray -Ġkick ed -Qu est -ĠT alk -ĠS everal -ĠZ ero -Ġdr one -ul k -Ġc am -ĠM obile -Ġprevent ing -Ġret ro -ĠA x -Ġcru el -Ġflo at -. ), -Ġfil ing -ĠGr ant -ĠB or -Ġr ib -Ġchampions hip -ĠM erc -Ġsty les -Ġc ake -Ġbuild s -ĠS elf -io x -Ġep ic -oy d -B el -ĠSt ew -. ( -ah u -ĠBe yond -Ġout s -Ġsol o -ĠT ree -Ġpres erve -Ġt ub -AR E -ro c -ĠIm pro -ĠW right -Ġbu nd -Ġtr aged -Ġoccas ional -b ian -Sec ond -r ons -Ġinter actions -form ed -s ing -Ġown s -Ġh ockey -Gener al -Ġlog ical -Ġexp end -Ġesc al -ĠGr iff -ĠC rown -ĠRes erve -Ġsto pping -Ġexc use -sec ond -Ġoper ated -Ġre aches -ĠMal ays -Ġpoll ution -ĠBrook lyn -Ġde lete -Ġhas h -Bl ock -ah a -âĢ ³ -Ġsh orter -p iece -> >> -ĠM ormon -t or -Ġpartic les -ĠB art -ry ption -Ġad min -Ġsqu ee -VID IA -Ġcreat or -iam eter -ic ular -N BC -Ġgrab bed -Ġn odd -Ġr ated -Ġrot ation -Ġgr asp -Ġexcess ive -ĠE C -ĠWh it -Ġinvent ory -ault s -ĠF B -Ġe cosystem -Ġbill ions -Ġvent ure -n amed -Ġdef ender -out e -Inst ead -ir able -W ar -Ġassum ption -Ġb ite -Ġearth qu -t ail -sp ace -Ġgif ts -boy s -Ġinev itable -Ġstruct ural -Ġbenef icial -Ġcompe lling -h ole -erv ation -Ġco at -o j -inc arn -ĠY ears -Ġdetermin ing -Ġrhet oric -Ġbound aries -Ġwh ites -A nt -add y -) - -ra ham -eter min -Ġhar vest -ĠCon c -Ġlapt op -ĠM atch -Ġenjoy ing -cc a -oll ar -Ġtri ps -Ġadd iction -ĠS ak -Ġpow ered -Ġc ous -ĠRuss ians -ie re -Ġret rie -qu ality -Ġdiff er -Ġking dom -ĠL aur -ĠCap itol -Ġcon clusions -ĠAl tern -ĠN av -Ġtrans parent -B ER -G roup -ĠCom plete -Ġinf er -Ġint rig -Ġins ane -R O -oph ob -is en -qu al -Mich ael -Ġm useum -ĠP ope -Ġres et -r ative -f ive -Ġagg reg -itte es -osit ory -Ġcar b -ĠRec ord -Ġdec ides -ĠF ix -Ġexcept ions -ĠCommission er -un s -ĠEnvironment al -Ġlegend ary -ist ence -Ġtun nel -k m -Ġins ult -Ġt roll -Ġsh ake -Ġdet ention -qu es -ĠCh rome -ĠF iles -Ġsub t -Ġprospect s -Ġpro l -re nder -pro of -Ġperform ances -St r -Ġh ref -ern ame -Ġachieve ment -Ġf ut -F ull -ĠLe ban -go ogle -ãĥ Ī -amp a -May be -Ġproject ed -ĠE mb -Ġcol leg -Ġa wards -Ġâ Ķ -G old -ĠBl ake -ĠR aj -if ting -Ġp ending -Ġinst inct -Ġdevelop ments -Con nect -ĠM and -ĠW ITH -ĠPhilipp ines -prof ile -Ġalt ogether -ĠB und -ĠT D -oo oo -amp ed -ip h -Ġste am -Ġold est -Ġdet ection -ul pt -Ġ ç -ĠWay ne -200 6 -f a -Ġcir cles -ĠF u -Ġdon ors -appropri ate -ĠDak ota -j amin -Ġmotiv ated -Ġpurch ases -ĠLouis iana -ĠS pl -Ġgl obe -Ġ10 5 -z ip -c all -Ġdepart ments -Ġsustain able -10 5 -ĠO P -if iers -Ġprevent ed -Ġinc omp -ĠComm ander -Ġdom inated -Ġ » -Ġinvest ed -Ġcomplex ity -Ġin cl -Ġens uring -Ġreal m -yn c -ĠInd ependent -r ained -ĠJ en -ĠFl ight -Ġat he -Ġspec ulation -ĠT E -oc ate -t ic -Ġpl aint -her ry -Ġto y -Ġ1 11 -Ġpl ates -st atus -ĠIs a -Ġdev oted -C op -ĠE S -25 5 -ur rency -M ain -Ġsl aves -Ġpe pper -Ġqu otes -Ġce iling -ĠF ish -Ġtrans formation -Ġfra ction -Ġadvant ages -Ġto ile -Ġstun ning -Ġmo ist -bre aking -s i -ĠL ocation -ĠMed ium -Ġtext s -Ġu gly -Ġb io -. âĢĶ -ĠB ased -Ġtr ains -ĠW ing -ĠAn cient -ĠRec ords -ĠH ope -Spe cial -ades h -ob i -[ / -Ġtempor arily -V er -h u -os er -Ġover night -Ġm amm -ĠTre asury -ĠV enezuel -ĠMeg a -Ġt ar -Ġexpect s -bl ack -or ph -\\ \\ -Ġaccept ance -Ġrad ar -s is -Ġjun ior -Ġfram es -Ġobserv ation -ac ies -P ower -ĠAdv anced -M ag -olog ically -ĠMe chan -Ġsent ences -Ġanaly sts -augh ters -force ment -Ġv ague -Ġcl ause -Ġdirect ors -Ġeval uate -Ġcabin et -M att -ĠClass ic -A ng -Ġcl er -ĠB uck -Ġresear cher -Ġ16 0 -Ġpoor ly -Ġexperien cing -ĠP ed -ĠMan hattan -Ġfre ed -Ġthem es -ad vant -Ġn in -Ġpra ise -10 4 -ĠLib ya -b est -Ġtrust ed -Ġce ase -Ġd ign -D irect -Ġbomb ing -Ġm igration -ĠSci ences -Ġmunicip al -ĠA verage -Ġgl ory -Ġreve aling -Ġare na -Ġuncertain ty -Ġbattle field -ia o -G od -Ġc inem -ra pe -el le -ap ons -Ġlist ing -Ġwa ited -Ġsp otted -ke ley -ĠAud io -e or -ard ing -idd ing -ig ma -ĠN eg -Ġl one -Ġ ---- -ex e -d eg -Ġtrans f -Ġwas h -Ġsl avery -Ġexpl oring -ĠW W -ats on -Ġen cl -l ies -ĠC reek -Ġwood en -Man ager -ĠBr and -um my -ĠAr thur -Ġbureau cr -Ġbl end -ar ians -F urther -Ġsupposed ly -Ġwind s -Ġ19 79 -Ġgrav ity -Ġanalys es -ĠTra vel -ĠV eter -Ġd umb -Ġaltern ate -g al -Ġconsum ed -Ġeffect iveness -.' ' -Ġpath s -ond a -L A -ĠStr ong -Ġen ables -Ġesc aped -Ġ" " -Ġ1 12 -Ġ198 3 -Ġsm iled -Ġtend ency -F ire -Ġp ars -ĠR oc -Ġl ake -Ġf itness -ĠA th -ĠH orn -Ġh ier -Ġimp ose -m other -Ġp ension -ic ut -bor ne -ic iary -. _ -ĠS U -Ġpol ar -is y -eng u -itial ized -AT A -w rite -Ġexerc ises -ĠD iamond -ot ypes -Ġharm ful -on z -Ġprint ing -st ory -Ġexpert ise -ĠG er -Ġtraged y -ĠF ly -Ġd ivid -amp ire -st ock -M em -Ġre ign -Ġun ve -Ġam end -ĠProp het -Ġmut ual -ĠF ac -Ġrepl acing -H ar -ĠCirc uit -Ġthro at -ĠSh ot -Ġbatter ies -Ġto ll -Ġaddress ing -ĠMedic aid -Ġp upp -ĠN ar -ol k -Ġequ ity -M R -ĠHis pan -ĠL arge -m id -D ev -Ġexp ed -Ġdem o -ĠMarsh all -erg us -Ġf iber -Ġdiv orce -ĠCre ate -Ġsl ower -ĠPark er -ĠStud ent -ĠTr aining -Ret urn -ĠT ru -Ġc ub -ĠRe ached -Ġpan ic -Ġqu arters -Ġre ct -Ġtreat ing -Ġr ats -ĠChristian ity -ol er -Ġsac red -Ġdecl are -ul ative -et ing -Ġdeliver ing -est one -Ġt el -ĠL arry -Ġmet a -ac cept -art z -ĠRog er -hand ed -Ġhead er -Ġtra pped -ĠCent ury -Ġkn ocked -ĠOx ford -Ġsurviv ors -b ot -Ġdemon stration -Ġd irt -Ġass ists -OM E -ĠD raft -ortun ate -fol io -pe red -ust ers -g t -ĠL ock -Ġjud icial -ver ted -Ġsec ured -out ing -ĠBook s -Ġhost ing -Ġlif ted -l ength -Ġj er -Ġwhe els -ĠR ange -umbn ails -Ġdiagn osis -te ch -ĠStew art -ĠP ract -Ġnation wide -Ġde ar -Ġoblig ations -Ġgrow s -Ġmand atory -Ġsusp icious -! ' -A pr -G reat -Ġmort gage -Ġprosecut or -Ġeditor ial -ĠK r -Ġprocess ed -ung le -Ġflex ibility -Ear lier -ĠC art -ĠS ug -Ġfoc uses -Ġstart up -Ġbre ach -ĠT ob -cy cle -ãĢ Į -ro se -Ġb izarre -ãĢ į -Ġveget ables -$ $ -Ġret reat -osh i -ĠSh op -ĠG round -ĠSt op -ĠHawai i -ĠA y -Per haps -ĠBe aut -uff er -enn a -Ġproduct ivity -F ixed -cont rol -Ġabs ent -ĠCamp aign -G reen -Ġident ifying -Ġreg ret -Ġpromot ed -ĠSe ven -Ġer u -ne ath -aug hed -ĠP in -ĠL iving -C ost -om atic -me ga -ĠN ig -oc y -Ġin box -Ġem pire -Ġhor izont -Ġbr anches -Ġmet aph -Act ive -ed i -ĠFil m -ĠS omething -Ġmod s -inc ial -ĠOrig inal -G en -Ġspir its -Ġear ning -H ist -Ġr iders -Ġsacr ific -M T -ĠV A -ĠS alt -Ġoccup ation -ĠM i -Ġdis g -lic t -Ġn it -Ġn odes -e em -ĠP ier -Ġhat red -ps y -ãĥ ī -Ġthe ater -Ġsophistic ated -Ġdef ended -Ġbes ides -Ġthorough ly -ĠMedic are -Ġbl amed -arent ly -Ġcry ing -F OR -pri v -Ġsing ing -ĠI l -Ġc ute -o ided -olit ical -ĠNe uro -å ¤ -Ġdon ation -ĠEag les -ĠG ive -T om -Ġsubstant ially -ĠLic ense -ĠJ a -Ġg rey -ĠAn imal -ĠE R -ĠU nd -Ġke en -Ġconclud e -ĠMississ ippi -Eng ine -ĠStud ios -P ress -o vers -ll ers -Ġ3 50 -ĠR angers -Ġr ou -ert o -E p -iss a -iv an -Ġse al -ĠReg ist -dis play -Ġwe aken -u um -ĠComm ons -ĠS ay -Ġcult ures -Ġl aughed -Ġsl ip -Ġtreat ments -iz able -m art -ĠR ice -Ġbe ast -Ġob esity -ĠLa ure -ig a -Wh ich -hold er -Ġelder ly -Ġp ays -Ġcompl ained -Ġc rop -Ġpro c -Ġexplos ive -ĠF an -ĠAr senal -A uthor -ef ul -Ġme als -Ġ( - -id ays -Ġimag ination -Ġann ually -Ġm s -as ures -H ead -ik h -m atic -Ġboy friend -ĠCom puter -Ġb ump -Ġsur ge -ĠCra ig -ĠKir k -D el -medi ate -Ġscen arios -ĠM ut -ĠSt ream -Ġcompet itors -Ù Ħ -ĠStan ford -ĠRes ources -az ed -b age -Ġorgan is -ĠRe lease -Ġsepar ately -Ġha bits -Ġmeasure ments -ĠCl ose -Ġaccomp any -Ġg ly -Ġt ang -ĠR ou -Ġplug in -Ġcon vey -ĠChall enge -oot s -j an -Ġcur s -ĠRel ations -ke eper -Ġapproach ing -p ing -Spe aking -Ġarrang ement -ĠV I -are ttes -Ġaffect ing -Ġperm its -b ecause -Ġu seless -ĠH us -!! !! -Ġdestro ying -Un fortunately -Ġfasc inating -S em -Ġelect oral -Ġtrans parency -ĠCh aos -Ġvolunte er -Ġstatist ical -Ġactiv ated -ro x -We b -H E -ĠHamp shire -is ive -M ap -Ġtr ash -ĠLaw rence -st ick -C r -Ġr ings -EX T -Ġoper ational -op es -D oes -ĠEv ans -Ġwitness ed -P ort -Ġlaunch ing -ec onom -w ear -ĠPart icip -um m -cul es -ĠR AM -ĠT un -Ġass ured -Ġb inary -Ġbet ray -Ġexpl oration -ĠF el -Ġad mission -it ated -S y -Ġav oided -ĠSim ulator -Ġcelebr ated -ĠElect ric -¥ ŀ -Ġcl uster -itzer land -he alth -L ine -ĠN ash -at on -Ġsp are -Ġenter prise -ĠD IS -clud es -Ġfl ights -Ġreg ards -ĠÃ Ĺ -h alf -Ġtr ucks -Ġcontact s -Ġunc ons -ĠCl imate -Ġimm ense -N EW -oc c -ect ive -Ġemb od -Ġpat rol -Ġbes ide -Ġv iable -Ġcre ep -Ġtrig gered -ver ning -Ġcompar able -q l -Ġg aining -ass es -Ġ( ); -ĠG rey -ĠM LS -s ized -Ġpros per -" ? -Ġpoll ing -Ġsh ar -ĠR C -Ġfire arm -or ient -Ġf ence -Ġvari ations -g iving -ĠP i -osp el -Ġpled ge -Ġc ure -Ġsp y -Ġviol ated -Ġr ushed -Ġstro ke -ĠBl og -sel s -ĠE c -,' ' -Ġp ale -ĠColl ins -ter ror -ĠCanad ians -Ġt une -Ġlabor atory -Ġn ons -t arian -Ġdis ability -ĠG am -Ġsing er -al g -ĠSen ior -Ġtrad ed -ĠWar rior -Ġinf ring -ĠFrank lin -Ġstr ain -ĠSwed ish -Ġsevent h -ĠB enn -ĠT ell -Ġsynd rome -Ġwond ered -id en -++ ++ -ig o -Ġpur ple -Ġjournal ism -Ġreb el -Ġf u -bl og -Ġinv ite -ren cies -ĠCont act -Is rael -ĠCont ent -Ġche er -Ġbed room -ĠEngine ering -ĠQue ens -Ġd well -ĠPlay Station -ĠD im -ĠCol on -l r -Ġoper ates -Ġmotiv ation -US A -ast ered -C ore -ĠTr uth -ol o -OS E -ĠMem ory -Ġpred ec -Ġan arch -Ġ19 20 -ĠY am -à ¨ -b id -Ġgr ateful -Ġexc itement -Ġtre asure -Ġlong est -ct ive -Ġdes erves -Ġreserv es -Ġcop s -ĠOtt awa -ĠEgypt ian -ank ed -Ġart if -Ġhypot hesis -: / -Ġpurch asing -Ġlove ly -H P -Ġdiv ide -Ġstrict ly -Ġquestion ing -Ġtaxp ayers -ĠJ oy -Ġroll s -ĠHe avy -Ġp orts -Ġmag netic -Ġinf lamm -Ġbr ush -t ics -â ĪĴ -Ġbott les -pp y -Ġp add -ãĤ ¯ -m illion -Ġdevast ating -Ġcomp iled -Ġmed ication -Ġtw elve -ĠPer ry -Sp ace -im b -y our -Ġle aked -ĠT ar -Ġun ity -Ġinfect ed -Ġtravel ed -ID E -ĠMc Donald -t xt -ĠPr inc -Ġinter ven -ĠTai wan -ĠP ow -Ġbe aring -ĠTh read -Ġz ones -iz ards -un ks -Ch apter -ll or -Ġ · -Ġw ounds -Ġdisc retion -Ġsucceed ed -ik ing -Ġicon ic -C all -Ġscreen ing -ĠM is -ict s -Ġmin isters -Ġsepar ation -Pl ayer -Ġb ip -Ġbel oved -Ġcount ing -ĠE ye -ar ound -ing ing -Ġtable t -Ġoff ence -in ance -h ave -ĠInf o -ĠNin ja -Ġprotect ive -ĠC ass -M ac -ĠQual ity -N orth -Ġ ic -ĠCub a -ĠChron icle -ĠPro perty -Ġfast est -ot os -ĠG erm -OW N -Ġbo om -ĠStan ley -ergus on -Ġcle ver -Ġent ers -m ode -ter ior -ĠS ens -Ġlin ear -AR K -Ġcomp aring -Ġpure ly -Ġsaf er -ĠPot ter -Ġc ups -R T -Ġgl uc -Ġatt ributed -Ġdu pl -ĠP ap -Ġprec ious -Ġp a -iction ary -ĠT ig -ĠTo o -ol utions -st an -Ġrob ots -Ġlob b -Ġstat ute -Ġprevent ion -w estern -16 0 -ĠAct ive -ĠMar ia -h al -N one -ell ar -ĠK B -ĠPart ners -ĠSing le -ĠFollow ing -ang o -ac ious -Ġth ou -Ġk g -Ġinflu ential -ĠFriend s -S ur -ain ted -Ġfor ums -Ġst arter -Ġcitizens hip -ĠE lection -on ge -ot ation -os ph -;; ;; -ut ical -p ur -ere n -Ġaccus ations -bit ious -ab bit -ĠOr d -Post ed -ir k -Ġsens itivity -ic he -ĠAm y -ĠF ab -Ġsum mit -Ġped est -Ġrub ber -Ġagric ultural -Ġcan cel -A E -Ġin aug -Ġcont am -Ġfirm ly -i w -st age -ĠK an -Ġt ier -Ġinv ention -Ġtransl ated -ĠR ules -B ox -Tw itter -ID S -Ġp izza -Ġdeb ug -ĠD rop -v s -Ġh orses -b ig -Ġb oring -Ġh ood -ĠMcC ain -at ched -ĠBro s -Ġsk ip -Ġess ay -st at -ĠLeg ends -Ġam munition -au c -Ġshoot er -Ġun h -Ġsuppl ied -Ġgener ic -ĠS K -ib an -yr ics -Ġ25 5 -Ġclim bing -Form er -Ġfl ip -Ġjump ing -Ġfrust ration -ĠTer ry -Ġneighborhood s -Ġmed ian -be an -Ġbr ains -Follow ing -Ġsh aped -Ġdraw s -Ġal tered -J ack -Ġrecip es -Ġsk illed -we alth -ach i -e lection -Ġbehavi ors -de als -ĠU ntil -F e -Ġdecl aration -mar ks -ĠBet ween -cel ona -Ġres on -Ġbub ble -Am ong -Ġim perial -G S -Ġfemin ist -200 5 -ĠK yle -Ġaccount ing -ĠTe le -ĠT yr -Ġconnect ing -Ġre hab -ĠP red -s im -Ġmeant ime -Ġphys ician -M W -ĠCamp bell -ĠBr andon -Ġcontribut ing -ĠR ule -ĠWe ight -ĠN ap -Ġinter active -Ġv ag -Ġhel met -ĠCom b -f our -Ġsh ipped -Ġcomple ting -ĠP D -PD ATE -Ġspread ing -Ġsc ary -erv ing -ĠG as -Ġfr ank -s chool -Ġrom antic -Ġstab il -R ob -Ġaccur ately -Ġac ute -ĠH ann -Ġsymbol s -Ġcivil ization -ĠA W -Ġlight ning -Ġcons iders -Ġven ue -Ġ × -Ġo ven -ĠS F -h is -Ġn u -ĠLear n -Ġpe oples -Ġst d -Ġsle e -Ġs lic -ĠStat istics -Ġcor ners -ĠB aker -Ġ: ) -ment ation -ol ver -Ġlaugh ing -ĠT odd -ond e -ĠH ills -Ġn uts -ĠW oman -pl ane -Ġl iver -ĠIn side -S orry -Ġagre es -Ġfund ament -ĠF isher -Ġa uction -Ġthread s -gl as -ĠBas ic -ĠN at -Ġlack ing -Ġceleb ration -j u -Ġs illy -E uro -Ġt att -ight y -cont rolled -T est -ĠSing h -Ġr age -Ġrh yth -o ffic -ĠPh antom -Ġhead lines -Ġrespond ing -ĠMor ning -Ġvit amin -Ġboot s -ĠS ite -al in -p i -Ġvir al -ĠU C -D ER -ĠSe x -Ġst ocks -c urrent -Ġch urches -ĠR are -ĠMur phy -Ġden ial -ĠG aming -Ġtou g -Ġn ick -Ġm akers -ĠRon ald -Ġgener ous -ĠD oc -ĠMor ris -Ġtransform ed -ĠN ormal -Ġ10 4 -ĠKick starter -ĠUp on -On line -ĠI RS -Ġw rap -Ġl oving -Ġarri ves -ĠD ue -Ġhe ter -ĠM ade -Ġrent al -Ġbelong s -Ġatt orneys -Ġcro ps -Ġmat ched -ul um -ol ine -10 9 -Ġdis par -Ġbuy ers -ĠCam bridge -Ġeth ics -rou ps -Ġjust ified -Ġmarg inal -Ġrespect ed -win ning -Ġnodd ed -ĠSer ge -ĠForm er -C raft -######## ######## -ĠWar ner -Ġd ash -et e -Ġent ert -ĠE scape -out heast -Ġkn ees -ĠB omb -Ġr ug -P ass -Ġatt itudes -go vernment -ĠPri or -Ġqual ities -Ġnot ification -ĠPh one -l ie -Ġanticip ated -ĠCom bat -ĠBar ry -Ġ198 2 -Us ers -on er -Ġcomput ing -ĠConnect icut -Ġless er -Ġpe ers -ĠC u -Ġtechn ically -Ġsub mission -ĠUn iversal -Ġman ually -our ge -Ġrespond ents -ĠB TC -ĠH ost -Ġf are -ĠB ird -Ġrece ipt -al so -Ġj ack -Ġagric ulture -Ġsk ull -Ġ! = -Ġpass ive -ĠC I -Ġsoc ieties -Ġremind ed -Ġinter ference -B uy -Ġâ ľ -g on -Ġscrut iny -ĠW itch -Ġconduct ing -Ġ ãĥ -Ġexch anges -ĠMit chell -Ġinhab it -Ġtw ist -B D -Ġwhere ver -group on -Ġj okes -ĠBen jamin -ĠR andom -fr ame -ĠL ions -Ġhighlight ed -ĠArk ansas -E nt -Ġp ile -Ġpre lim -g s -mind ed -Ġfel ony -ĠG A -ĠL uck -Ġpract ically -ĠB os -Ġact ress -D am -ĠB ou -Ġvis a -Ġembed ded -Ġhy brid -Ġear liest -Ġsoon er -s ocial -ĠH A -Ġste ep -Ġdis advant -Ġexplo it -ĠE gg -ĠUlt ra -Ġnecess ity -L ocal -ie ge -Ġd ated -Ġmass es -Ġsubsc ription -pl ess -Ġan onym -Ġpresum ably -Bl ue -The ir -asket ball -ĠPhil ip -Ġcom ed -load ed -r ane -Ġref lection -Ch ina -Ġext ends -Ġform ing -Ġund ers -200 1 -Ġgr at -Ġconcent rations -Ġins ulin -Ġsec ular -Ġwh ilst -Ġwin ners -Ad vertisements -Ġdeliber ately -ĠWork ing -Ġs ink -et ics -d ale -Ġmand ate -Ġg ram -Ġvac ation -Ġwarn ings -ri pp -ĠTH AT -Ġcomment ary -Ġint u -Ġa est -Ġreason ing -Ġbreak down -ĠZ ombie -Ġ-- > -ĠPolit ical -c ott -Ġthr ust -Ġtechn ological -Ġdec iding -Ġtraff icking -L ong -W elcome -pr ising -ĠCommun ications -Ġend ors -Ġsw ift -Ġmetab ol -co ins -res a -ĠHT TP -Ġen roll -ĠH appy -us r -int age -Ġ[ " -u ably -ĠM aterial -Ġrepe al -Se pt -k h -ĠMod i -Ġunder neath -ĠI L -sh ore -Ġdiagn osed -ace utical -Ġsh ower -au x -ĠSw itch -ĠStre ngth -Ġj ihad -n ational -Ġtra uma -uss y -on i -Ġcons olid -Ġcal ories -ĠF lynn -ag ged -16 8 -ĠP ink -Ġfulf ill -Ġch ains -Ġnot ably -ĠA V -L ife -ĠCh uck -m us -ĠUr ban -ĠH end -Ġdep osit -ĠS ad -Ġaff air -OR K -ie val -ĠF DA -Ġt rop -ĠOver all -Ġvirt ue -Ġsatisf action -au nd -Ġl un -ĠSw itzerland -ĠOper ation -pro cess -Ġsh ook -Ġcount ies -le ased -ĠCharl otte -1 12 -Ġtrans cript -Ġre dd -p ush -ĠHe y -ĠAn alysis -[ " -Ġaltern atives -ard less -Ġele ph -Ġpre jud -ĠLe af -H aving -ĠH ub -Ġexpress ions -ĠVol ume -Ġshock ing -ĠRed s -Ġread ily -Ġplan ets -ad ata -Ġcollaps ed -ĠMad rid -Ġir rit -i pper -ĠEn c -ĠW ire -Ġbu zz -ĠG P -ash a -Ġaccident ally -ur u -Ġfrust rated -ĠS A -Ġhung ry -ĠH uff -Ġlab els -ant o -ĠE P -Ġbar riers -) | -ĠBer keley -ĠJ ets -Ġp airs -ĠL an -J ames -ĠB ear -Ġhum or -ĠLiber ty -Ġmagn itude -Ġag ing -ĠM ason -Ġfriends hip -umb ling -Ġemer ge -Ġnewsp apers -Ġam bitious -ĠRich ards -atern al -Ġ198 1 -Ġcook ies -Ġsc ulpt -Ġpur suit -L ocation -Ġscript s -p c -Ġarrang ements -Ġd iameter -Ġl oses -am ation -Ġl iqu -ĠJ ake -aret te -Ġunderstand s -ĠZ en -v m -Ġappro ve -Ġw ip -Ġult ra -Ġint end -ĠD I -asc ular -Ġst ays -ĠK or -ĠK l -Ġinvest ing -L a -Ġbelie ving -b ad -m outh -Ġtaxp ayer -ãĥ ĥ -ĠQue bec -Ġl ap -ĠSw iss -d rop -Ġdr ain -ir i -et c -ft en -ĠN ex -Ġst raw -Ġscream ing -Ġcount ed -Ġdam aging -Ġamb assador -cent ury -Ġpro x -Ġarrest s -u v -il ateral -ĠCh arg -Ġpresc ribed -Ġindepend ently -Ġf ierce -ĠB aby -Ġb rave -Ġsu its -= > -Ġbas eline -ĠR ate -Ġis lands -Ġ( ( -g reen -ix els -Ġname ly -ĠVill age -th an -am y -V ersion -g mail -ential s -ĠS ud -ĠMel bourne -Ġarri ving -Ġquant um -e ff -rop olitan -T ri -Ġfun eral -ĠI R -ÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤ ÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤ -ĠC ob -it ably -Ġt urb -Ġcomb o -Re view -Ġdeploy ment -u ity -ĠB ott -Ġinv isible -Ġrender ing -Ġunl ocked -Ġa qu -ĠVlad imir -Ġp ad -ĠBr ain -ĠLeg acy -dr agon -ĠKurd ish -Ġsound ed -Ġdet ained -ĠD M -g ary -Ġd aughters -Ġdistur bing -uk a -ĠPar ad -Ġt ast -Ġunf ortunate -Ġu l -em in -Ġattend ance -tr l -Ġpar ks -ĠMem orial -ĠAl ice -oth y -gu ard -ĠD ise -ĠSh an -ĠFor um -R ich -Ġshif ted -ue z -Ġl ighter -ĠMag n -Ġc od -S ch -ham mad -P ub -3 50 -ĠP okemon -Ġprot otype -Ġun re -B ase -ĠStud ents -ĠRep ly -ĠCommun ist -Ġg au -ĠTy ler -I Z -Ġparticip ated -Ġsup rem -ĠDet ails -Ġvessel s -ro d -Ġt ribe -ke ep -Ġassum ptions -Ġp ound -Ġcr ude -ĠAv ailable -Ġswim ming -Ġin clusion -Ġadv ances -c ulation -Ġconserv ation -Ġover d -ĠBuff alo -Art icle -ed ge -Ġaw a -ĠMad ison -Ġsid ew -Ġcat ast -ĠK rist -uc le -ĠHigh way -ĠTer ror -Ġactiv ation -Ġuncons cious -ĠSat an -ĠSus an -ill ery -Ġarr anged -i op -Ġrum ors -ur ring -th ink -ĠKe ith -ĠK ind -Ġavoid ing -by n -n ut -ĠSpe aker -r us -n ames -Ġgu ilt -ĠOlymp ics -Ġsa il -ĠM es -lev ant -ĠColumb us -a ft -C ity -S outh -ĠHar vey -ĠP un -S everal -Ġment ally -Ġimp ress -m ount -ĠUb untu -âĢĶâĢĶâĢĶâĢĶ âĢĶâĢĶâĢĶâĢĶ -ĠSuper man -ĠMP s -Ġintent ions -ĠR acing -Ġlike lihood -Ġ2 40 -T otal -Ġto ys -ĠW atson -Ġur ge -L ear -ĠP aper -Ġoccur ring -ĠB eng -ĠC ert -Ġst ones -T im -ĠTw in -z b -ĠD ynam -Ġpolit ician -k ens -ĠEnter prise -UT ERS -Ġab ol -Ġref resh -Ġarbit rary -pe ction -Ġtrou bles -Ġ} ); -t v -Ġpil ots -Ġdist ribute -Ġaud it -Ġp ause -orig inal -Ġr ivals - £ -F ig -T L -ab il -ry ing -L in -ion ed -l on -Ġf ancy -Ġcr ashed -Ġt ract -Ġshe d -Ġcons ume -B ased -down load -in it -Ġvolt age -Int rodu -Ġcondem ned -ĠFin ance -res pect -Ġex cluded -Ġestablish ing -her ic -Ġher itage -Ġspect acular -Ġun st -ĠSnow den -ĠL ane -S an -Ġprotect ions -st ruction -inc inn -Ġmac ro -C ustom -ios ity -Ġes p -Ġfunction ing -Ġm ush -Ġp uzzle -Ġeth ical -M al -Ġgo verning -ĠF erguson -Ġrest ored -Ġst ressed -ĠCoun ter -ĠK as -cl ip -AN S -Ġse iz -U K -by ss -old own -ap i -Ġperman ently -oun ters -W est -Th rough -L ight -at oes -Ġne at -Ġc ord -ure r -Ġsevere ly -ĠA ven -Ġinter rog -Ġtri ple -G iven -N umber -Ġar ise -Ġs her -pl ant -Ġfl ower -ĠC ou -Ġat e -Ġnew er -b ul -Ġmean while -ĠL air -Ġadjust ment -ĠCop yright -Ġd ivers -i ological -Ġgam ers -o at -Ġhistor ically -Ġanal og -Ġlong time -Ġpres cription -ĠM ist -ĠHy per -ĠM aine -ĠDe ity -Ġmulti pl -ĠRe incarn -ĠH yd -ĠP ic -S il -r ants -ĠC ris -. ; -( { -epend ence -Ġrec y -ate ur -Ġqu ad -Ġgl ob -Ġcon ced -te am -Ġcapital ist -ĠL ot -Ġroy al -ĠCy ber -Ġblack s -met ic -ri v -ĠD anny -Ġsp o -ĠR O -Ġanim ated -rypt ed -ĠDep uty -Ġrend ered -F E -Ġstre ak -Ġcloud s -ĠDou g -~~~~ ~~~~ -Ġdisc our -ĠVe h -Ġpsych ology -ĠJ ourney -Ġcry stal -ĠFro st -Ġsuspic ion -Ġrel ate -or us -ĠC rypt -ĠN VIDIA -com ed -ut ing -incinn ati -Ġvulner ability -ost ic -Ġisol ation -Ġcool ing -ĠCoal ition -Ġ1 19 -F our -ĠDe al -Ġâ ī -se mble -ram ent -ĠBar celona -Ġ10 2 -Ġcoc aine -ocaly pse -F eb -ogen ic -Ġmut ation -Ġcrypt oc -ĠK el -ĠG it -a is -Ġs isters -AN K -Ġactiv ate -T er -Ġd read -yl on -Ġprop ri -A ust -ĠDef ault -Ġout door -Ġshe er -ce ive -Ġg ently -Ð ¾ -Pro gram -Ġâ ĨĴ -Ġve gan -ĠCr us -Ġrespons ibilities -ĠH R -OL D -Ġprev ents -Ġst iff -ĠW ere -Ġathlet ic -ĠSc ore -Ġ) : -Ġcolumn s -ĠL oc -av ailable -ĠF ram -ĠS essions -Ġcompan ion -Ġpack s -14 0 -ĠKn ights -Ġf art -Ġstream s -Ġsh ore -Ġapp eals -ĠPer formance -h aul -ĠSt ra -ĠN ag -10 3 -ĠTrans portation -B B -E v -z an -P ublic -Ġtw in -uls ion -M ult -Ġelect ro -Ġstat ue -ation ally -ĠN ort -Ġins pection -/ * -ig ue -Ġcomp assion -ĠT ales -ĠSte in -ĠSc reen -ĠB ug -ĠL ion -g irl -Ġwithdraw al -Ġobject ives -Ġblood y -Ġprelim inary -Ġj acket -Ġdim ensions -ĠC ool -ĠOcc up -Ġw reck -Ġdoub led -ank ing -Ġ19 75 -Ġglass es -ĠW ang -pro v -P ath -connect ed -ĠMult i -ĠNor way -agon ist -Ġfe ared -Ġtouch ing -Ġarg uably -¯¯¯¯ ¯¯¯¯ -ĠNC AA -che m -Ġsp at -ĠW WE -ĠC el -ig ger -Ġattack er -ĠJo in -ob ject -ett a -Ġelim inated -d et -Ġdest ruct -ĠLuc as -ct uary -18 0 -ĠBr ady -ĠBl ues -B ay -au kee -Ġtim eline -Ġdeleg ates -w ritten -uff icient -Ġsh apes -Cop yright -ou ble -serv ice -Ġp ione -Ġcolleg es -Ġrow s -Ġsp ite -Ġassess ed -3 60 -Ġle ase -Ġconfident ial -ck er -ĠMan ning -ĠV oice -Ġse aled -Ġcalcul ate -N O -ĠAss istant -Ġteen ager -ul ent -ather ine -Ġm ock -Ġd iamond -Ġf est -Ġsw itched -Ġres ume -ĠPu erto -Ġl anes -ir ation -ĠSimilar ly -Ġro d -ĠS el -ĠPal ace -ĠLim ited -e ous -Ġvar iant -Ġw ard -Ġ) ) -Sh ow -OO K -A lex -ĠN ep -br is -ĠWik ipedia -Ġexcept ional -Ġman ages -ĠD raw -Ag ain -Ġco pper -ut t -Ġex ports -Ġport folio -Ġelev ated -R ated -ĠOther wise -ĠT act -ĠShe l -ĠT X -" âĢĶ -Ġres ur -ĠW a -ven ant -Ġmon etary -pe ople -E mail -Ġfif ty -ĠS weet -ĠMalays ia -Ġconf using -ĠR io -ud a -uten ant -" ); -Ġpra ised -Ġvol umes -t urn -Ġm ature -Ġnon profit -Ġpassion ate -ĠPriv ate -Ġ10 3 -Ġdesc end -ç ¥ŀ -uff y -head ed -Whe ther -ri en -ze ch -be it -Ġch rom -ĠMc M -Ġd ancing -Ġe leg -ĠNot iced -11 5 -Ġadvoc acy -ENT S -amb ling -ĠMin or -ĠF inn -Ġprior ities -Ġthere of -ĠSt age -ĠRog ers -Ġsubst itute -ĠJ ar -ĠJeff erson -Ġlight ly -10 2 -ĠL isa -u its -ys ical -Ġshif ts -Ġd rones -Ġwork place -Ġres id -ens ed -ah n -Ġpref erences -ser ver -Ġdeb ates -d oc -ĠGod s -Ġhelicop ter -Ġhon our -Ġconsider ably -ed ed -ĠF emale -ĠAn ne -Ġre un -ĠF ace -ĠHall ow -ĠBud get -Ġcondem n -Ġt ender -Pro f -ocr atic -ĠTurn er -ĠAg ric -Ġ19 76 -Ġa pt -d isc -ĠF ighter -ĠA ur -Ġgar bage -in put -ĠK arl -ĠOl iver -ĠL anguage -k n -N on -ĠCl ar -Ġtrad itions -Ġad vertisement -ĠS or -Ġarch ive -Ġvill ages -7 50 -Ġimplement ing -w aukee -Ġdiet ary -Ġswitch ing -Rep ublic -Ġvel ocity -Ġc it -ĠA wards -Ġfin ancing -Ġlast ed -) ] -Ġrem inder -P erson -Ġprec ision -Ġdesign ers -ĠF ried -ĠB order -Ġtr agic -Ġw ield -Ġiniti atives -ĠT ank -w er -Ġjo ins -R o -in ery -Ġar row -Ġgener ating -found er -Ġsear ches -Ġrandom ly -A ccess -Ġb atch -Ġp osed -l at -Ġpursu ing -as a -Ġtest ified -form ing -ĠSh ar -w iki -ĠE ither -S ometimes -Ġsen ators -ĠJohn ny -ĠTal iban -ĠG PS -":" / -ãģ® å -Ġanaly zed -ĠRub io -ĠMove ment -op ard -ii i -St and -f ight -Ġign oring -i ang -ĠG N -so ever -ĠST AT -Ġref using -Ġswe at -Ġb ay -P ORT -ir med -ak y -Ġdis pro -Ġlabel ed -Ġ10 8 -H ello -Ġple asant -ab a -Ġtri umph -Ġab oard -Ġinc om -ĠC row -le tt -Ġfol k -Ġch ase -` ` -ĠBr us -Ġte ens -c ue -Ġter rain -h yd -il ight -OR Y -Su pport -ew s -ll i -rain ts -ĠC and -Ġab used -ach ment -l arg -B as -ĠC ancer -Ġ19 78 -Ġsupp orter -ac cess -ĠTer min -ĠT ampa -ĠAN Y -Ġnew est -ĠCrim inal -ed u -Ġ19 30 -Ġadm its -Ġend e -Ġfail ures -ur ate -ful ness -cy cl -ĠSub ject -Ġinf inite -th ree -W A -p it -ĠInst all -R ad -ili ation -G M -Ġcontin ent -Ġaccommod ate -ĠCl ay -Ġp up -ĠF unction -Ġham mer -ĠAlbert a -Ġrev ised -Ġminor ities -Ġmeasure ment -Con nell -Ġdis able -ĠM ix -In cre -Ġfor k -ĠR osen -Ġimpl ies -umb lr -AN G -Ġprote ins -Ġagg ression -Ġfacilit ate -S N -Ġilleg ally -u er -Ġacad em -Ġp uzz -ĠSh ift -p ay -oll o -Ġaud iences -B uild -Ġno ble -Ġsynt ax -â ĺħ -Ġbe am -ĠB ed -ĠA ld -Ġorig ins -v ideo -Ġ19 77 -ĠAss ault -Ġgar age -Te am -Ġver dict -Ġd war -ĠVirt ual -e vent -Ke ep -Ġsent iment -Ġwild life -sh irt -Ġb urg -Ġrecommend ation -rep resent -Ġgall ery -own ers -Ġsch olar -Ġconven ience -ĠSw ift -Ġconv inc -C ap -Ġwar fare -ĠVis ual -Ġconst itute -Ġab ort -ĠWe ather -ĠLook ing -ĠH em -Ġmart ial -Ġinc oming -et ition -Ġtoler ance -ĠCre ated -Ġfl ows -ĠE lder -Ġsoul s -Ġf oul -ĠP ain -ĠC AN -Ġ2 20 -b c -he nd -Ġgen ius -R eal -ĠW r -omet er -p ad -Ġlim iting -ĠS i -ĠL ore -ĠAd ventures -Ġvar ied -D isc -f in -ĠPerson al -Ch ris -Ġinv ented -Ġd ive -ĠR ise -Ġo z -ĠCom ics -Ġexp ose -ĠRe b -let ters -s ite -im ated -Ġh acking -Ġeduc ated -ĠNob ody -Ġdep ri -Ġincent ive -ãĤ · -Ġovers ight -Ġtrib es -ĠBelg ium -Ġlicens ing -our t -Produ ct -ah l -ĠG em -Ġspecial ist -Ġc ra -ann ers -ĠCor byn -Ġ19 73 -RE AD -Ġsum mar -Ġover look -ĠApp lication -Ġin appropriate -Ġdownload ed -Q ue -ĠB ears -Ġth umb -ĠChar acter -ĠReincarn ated -ĠS id -Ġdemonstr ates -s ky -ĠBloom berg -ĠAr ray -ĠRes ults -ĠFour th -ĠED T -ĠO scar -c end -Ġ10 6 -ĠN ULL -ĠH ERE -m atch -ĠBr un -Ġgluc ose -ie g -eg u -Ġcert ified -Ġrel ie -Ġhuman itarian -Ġpr ayers -K ing -Ġn an -h ou -10 8 -ul u -Ġrenew able -Ġdistingu ish -Ġd ense -ĠV ent -ĠPack age -ĠB oss -Ġedit ors -Ġm igr -T ra -ĠPet ers -ĠAr ctic -200 4 -ĠC ape -Ġloc ally -Ġlast ing -Ġhand y -. ). -P an -ĠR ES -Ind ex -Ġt ensions -Ġformer ly -Ġide ological -Ġsens ors -Ġdeal ers -Ġdef ines -S k -Ġproceed s -Ġpro xy -az ines -ĠB ash -ĠP ad -ĠC raft -eal ous -Ġshe ets -omet ry -J une -cl ock -T T -ĠThe atre -ĠB uzz -Ġch apters -Ġmill enn -Ġd ough -ĠCongress ional -Ġimag ined -av ior -Ġclin ic -Ġ19 45 -Ġhold er -ro ot -oles ter -Ġrest art -B N -ĠHam as -ĠJ ob -Ġor b -Ġr am -Ġdiscl ose -Ġtransl ate -Ġimm igrant -Ġannoy ing -Ġtreat y -an ium -ĠTe a -ĠLeg ion -Ġcrowd s -ĠB ec -ĠA er -oh yd -B ro -Look ing -Ġl bs -Ġagg ress -Ġse am -Ġinter cept -ĠM I -mer cial -act iv -ĠC it -Ġdim ension -Ġconsist ency -Ġr ushing -ĠDou glas -Ġtr im -Inst all -ick er -Ġsh y -10 6 -Ġment ions -pe lled -ĠT ak -c ost -Ġclass room -Ġfort une -dri ven -Ġun le -ĠWhe el -Ġinvest or -ĠM asters -k it -Ġassoci ations -ĠEv olution -op ing -us cript -Ġprov incial -ĠWal ter -av i -S O -Ġun limited -Eng lish -ĠC ards -ĠEb ola -ne red -Ġreven ge -Ġout right -um per -Ġf itting -ĠSol id -Ġform ally -Ġproblem atic -Ġhaz ard -Ġenc ryption -Ġstraight forward -ĠA K -Ġp se -ĠOr b -ĠCh amber -ĠM ak -Cont ents -Ġloyal ty -Ġl yrics -ĠSy m -Ġwel comed -Ġcook ed -Ġmon op -Ġn urse -Ġmis leading -Ġe ternal -Ġshif ting -Ġ+ = -V is -Ġinst itutional -ill ary -Ġp ant -VER T -ĠA CC -ĠEn h -Ġinc on -ĠRE UTERS -Ġdon ated -âĢ¦âĢ¦ âĢ¦âĢ¦ -In tern -Ġexhib it -Ġt ire -ĠR ic -ĠCh ampion -ĠMu hammad -N ING -ĠSoc cer -Ġmob ility -Ġvary ing -ĠM ovie -Ġl ord -o ak -F ield -Ġve ctor -us ions -Ġsc rap -Ġen abling -m ake -T or -. * -| | -ĠWe bsite -ĠN PC -Ġsocial ist -ĠBill y -ĠAdd itional -Ġc argo -Ġfar ms -ĠSo on -ĠPri ze -Ġmid night -Ġ9 00 -se en -ĠSp ot -Ġshe ep -Ġspons ored -ĠH i -ĠJ ump -Ġ19 67 -Micro soft -ĠAg ent -Ġch arts -d ir -Ġadj acent -Ġtr icks -Ġman ga -Ġex agger -/ > -foot ball -ĠF CC -G C -ĠT ier -and ra -OU ND -% ), -Ġfru its -V C -ĠA A -R ober -Ġmid st -â Ĺ -ank a -Ġlegisl ature -ĠNe il -Ġtour ists -" " -ĠWar ning -ĠNever theless -ĠOffic ial -ĠWh atever -Ġm old -Ġdraft ed -Ġsubst ances -Ġbre ed -Ġt ags -ĠT ask -Ġver b -Ġmanufact ured -com ments -ĠPol ish -Pro v -Ġdetermin es -Ob ama -k ers -Ġutter ly -Ġse ct -sc he -ĠG ates -ĠCh ap -Ġal uminum -Ġz ombie -ĠT ouch -ĠU P -Ġsatisf y -Ġpred omin -asc ript -Ġelabor ate -Ġ19 68 -Ġmeas uring -ĠV ari -any ahu -Ġs ir -ul ates -id ges -ick ets -ĠSp encer -T M -oub ted -Ġpre y -Ġinstall ing -ĠC ab -re ed -re ated -Su pp -Ġwr ist -ĠK erry -10 7 -ĠK le -ĠR achel -Ġc otton -ĠA RE -ĠE le -Cont rol -Ġload s -ĠD od -an as -b one -Ġclass ical -ĠReg ional -ĠInt eg -V M -Ġdes ires -Ġaut ism -support ed -ĠM essage -Ġcomp act -writ er -Ġ10 9 -ĠHur ricane -c ision -Ġcy cles -Ġdr ill -Ġcolle ague -Ġm aker -G erman -Ġmist aken -S un -ĠG ay -Ġwhat soever -Ġsell s -ĠA irl -l iv -ĠO ption -Ġsol ved -Ġse ctors -Ġhorizont al -Ġequ ation -ĠSk ill -ĠB io -g ement -ĠSn ap -ĠLeg al -Ġtradem ark -Ġmake up -Ġassemb led -Ġsa ves -ĠHallow een -ĠVer mont -ĠFR OM -Ġfar ming -ĠP odcast -accept able -ĠHig her -Ġas leep -ull ivan -Ġrefere n -ĠLe v -Ġbul lets -ok o -H C -Ġst airs -Ġmain tains -ĠL ower -ĠV i -Ġmar ine -Ġac res -Ġcoordin ator -ĠJ oh -Ġcounterpart s -ĠBrother s -Ġind ict -b ra -Ġch unk -Ġc ents -H ome -ĠMon th -Ġaccording ly -if les -ĠGerm ans -ĠSy n -H ub -Ġey eb -âĶĢâĶĢ âĶĢâĶĢ -Ġr anges -ĠHoll and -ĠRob ot -f c -M ike -Ġpl asma -Ġsw ap -Ġath lete -ĠR ams -,' " -Ġinfect ions -Ġcor rid -Ġv ib -Ġpat ches -Ġtradition ally -Ġrevel ation -Ġswe ep -Ġgl ance -Ġin ex -200 3 -ĠR aw -work ing -os ures -ĠD at -ĠLyn ch -Ġle verage -ĠRe id -Ġcorrel ation -ian ces -av ascript -Ġrep ository -ret ty -Ġ19 72 -24 0 -Ġo un -p ol -ĠRe ed -Ġtact ical -is ite -App le -ĠQu inn -Ġrap ed -ill o -Euro pe -Ġalgorith ms -ĠRod rig -i u -Ġill um -Ġf ame -Ġintrodu cing -Ġdel ays -ĠRaid ers -Ġwh istle -Ġnovel s -ĠRe ally -Ġder iv -Ġpublic ations -ĠNe ither -ĠCom merce -Ġa ston -l anguage -Not es -ĠR oth -ĠF ear -Ġm ate -Ġpar ade -ĠQ B -Ġman eu -ĠC incinnati -m itting -Ġwa ist -ĠR ew -Ġdisc ont -Ð ° -Ġst aring -Ġal ias -Ġsec urities -Ġtoile t -ĠJ edi -Ġun law -v ised -//// //// -] ( -ĠWe iss -Ġpre st -ĠComp an -Ġmem o -ĠGr ace -J uly -ĠEl ite -cent er -ĠSt ay -Ġgal axy -Ġto oth -ĠS ettings -Ġsubject ed -ãĤ ¦ -Ġline back -Ġretail ers -ĠW ant -Ġd angers -A ir -Ġvolunt ary -ew ay -Ġinterpret ed -ot ine -à § -Ġp el -Serv ice -ĠEvent ually -Ġcare ers -Ġthreat en -Ġmem or -ĠBrad ley -anc ies -s n -ĠUn known -N ational -Ġsh adows -ail and -ĠD ash -Every one -izz ard -M arch -= ( -Ġpull s -Ġstr anger -Ġback wards -ĠBern ard -imens ional -Ġch ron -Ġtheoret ical -k top -Ġw are -ĠInvest ig -ĠIn iti -ĠOper ations -o ven -oc ide -* / -Ġfl ames -ĠC ash -sh it -Ġc ab -ĠAn aly -ĠSe ah -Ġdefin ing -Ġorder ing -Ġimm un -Ġpers istent -AC H -Russ ian -m ans -Ġh ind -Ġphot ography - © -Ġh ug -Ġ10 7 -ĠH ence -i ots -ude au -Ġsubsid ies -Ġroutine ly -ĠDev ice -it ic -Ġdisg ust -land er -Ġ19 40 -Ġassign ment -ĠB esides -w ick -ĠD ust -us c -struct ed -11 1 -de velop -Ġf ond -Ġinter section -Ġdign ity -Ġcommission er -With out -re ach -Ġcart oon -Ġsc ales -ãĥ Ń -F IG -Ġsurve ys -ĠIndones ia -Ġart work -Ġun ch -Ġcy cling -un ct -au er -or ate -ĠOb viously -Ġcharacter ized -fe ld -Ġaff irm -Ġinn ings -Ġ é -Ġal iens -Ġcl oth -et ooth -ĠC ertain - § -Ġdig est -k now -ĠX L -Ġpredict ions -Ġd in -W AR -Ġafter math -Ex ample -ĠSu ccess -ĠTh r -IG N -Ġmin er -B us -Ġcl arity -heim er -ĠO UT -ĠS end -ĠCirc le -ĠD iet -Ġpron ounced -Ġcreat ors -Ġearthqu ake -atter y -ge ons -Ġo d -Ġlay ing -or p -U lt -pro ject -Ġunder min -Ġsequ el -S am -ĠDark ness -Ġre ception -b ull -Y S -ĠV ir -Ġsequ ences -ĠCo in -Ġout fit -ĠW ait -1 19 -Ġdel ivers -.... .. -Ġbl own -ĠE sc -ĠM ath -per m -ĠU l -Ġgl im -Ġfac ial -Ġgreen house -Ġto kens -/ - -ĠAnn ual -ĠON E -Ġteen age -ĠPhys ical -ĠL ang -ĠC elt -Ġsu ed -ivid ually -Ġpat ience -ch air -reg ular -Ġa ug -in v -ex cept -ĠL il -Ġn est -f d -s um -ĠCh ase -Russ ia -ĠJenn ifer -Ġoff season -Over all -F ore -Ġr iot -A ud -form er -Ġdefend ers -ĠC T -iot ic -rib ly -Ġautom ated -Ġpen is -Ġins ist -Ġdi agram -ĠS QL -ĠG arc -Ġw itch -cl ient -ier ra -am bers -Ġrec ount -f ar -V ery -oster one -Ġappreci ated -ĠPer fect -S ection -Ġd oses -oca ust -Ġcost ly -Ġg rams -ĠSh i -Ġwrest ling -Ġ19 71 -Ġtro phy -Ġn erve -ĠK az -ĠExper ience -Ġpled ged -Ġplay back -Ġcreat ivity -by e -Ġattack ers -Ġhold ers -ĠCo ach -ĠPh D -Ġtransf ers -Ġcol ored -ĠH indu -Ġd rown -Ġlist ened -ĠW A -ias m -P O -Ġappeal ing -Ġdiscl osed -ĠCh icken -ag ging -Ġple aded -Ġnav igation -ĠReturn s -Ġ[ [ -R OR -E A -Ġphotograp her -ĠR ider -ipp ers -Ġsl ice -Ġe rect -Ġhe d -iss ance -ĠVik ings -ur ious -Ġapp et -oubted ly -Ch ild -Ġauthent ic -o os -ĠM aking -Ġannoun cing -Ġb od -Ġmet er -ĠN ine -ĠR ogue -Ġwork force -Ġrenew ed -Ġorganis ations -ac s -P LE -Sh ort -Ġcomp ounds -ĠVis it -Ġen velop -ear th -Ġsupport ive -gg le -ĠBrus sels -ĠGu ild -Cre ate -RE L -Ġaver aged -Ġ19 69 -ri ages -Ġlength y -Ġforg ot -O kay -ĠE rd -Ġdeal er -Ġrec ession -D D -Ġdesper ately -Ġhun ger -Ġst icks -Ġm ph -ĠF aith -Ġintention ally -Ġdem ol -ue ller -ĠS ale -Ġde bris -s pring -Ġle ap ->> >> -Ġcontain ers -se lling -rane an -atter ing -Ġcomment ed -ĠC M -on ut -Ġwood s -es pecially -Ġorgan ize -iv ic -ĠWood s -ang a -s qu -Ġm aj -am on -Ġax is -Ġ19 74 -ĠDen mark -Ġwar rior -ĠP and -Ġout lined -ĠB O -ins ula -z illa -eb ook -Ġd are -Ġsear ched -Ġnav igate -S n -writ ing -Ġun ited -J apan -ĠHe brew -Ġfl ame -Ġrel ies -Ġcatch ing -ĠSh o -Ġimprison ment -Ġp ockets -Ġclos ure -ĠF am -t im -ade qu -Act ivity -Ġrecru iting -ĠW ATCH -ĠArgent ina -d est -Ġapolog ize -or o -Ġlack s -Ġtun ed -ĠGriff in -Ġinf amous -Ġcelebr ity -ss on -Ġ ---------------------------------------------------------------- -ĠIs is -ĠDis play -Ġcred ibility -Ġeconom ies -Ġhead line -ĠCow boys -Ġind ef -Ġl ately -Ġincent ives -but ton -ĠM ob -A ut -Ġres igned -ĠO m -c amp -Ġprof iles -Ġsche mes -olph ins -ay ed -Cl inton -en h -ĠY ahoo -Ġab st -Ġan k -su its -Ġw ished -ĠMar co -udd en -Ġsp here -ĠB ishop -Ġincorpor ated -ĠPl ant -11 4 -Ġh ated -p ic -Ġdon ate -Ġl ined -Ġbe ans -Ġsteal ing -Ġcost ume -Ġsher iff -Ġfor ty -Ġint act -Ġadapt ed -Ġtrave lling -b art -Ġnice ly -Ġdri ed -Ġsc al -os ity -NOT E -ĠB h -ĠBron cos -ĠI gn -Ġint imate -Ġchem istry -Ġopt imal -D eb -ĠGener ation -Ġ] , -ich i -ĠW ii -ĠYOU R -vent ions -W rite -Ġpop ul -un ning -ĠW or -V ol -Ġqu een -head s -K K -Ġanaly ze -op ic -ear chers -Ġd ot -leg raph -ast ically -Ġupgr ades -Ġca res -Ġext ending -Ġfree ze -Ġin ability -Ġorg ans -Ġpret end -Ġout let -11 3 -ol an -ĠM all -ul ing -t alk -Ġexpress ing -ĠAl ways -ĠBe gin -f iles -Ġlic enses -% % -ĠM itt -Ġfil ters -ĠMil waukee -G N -Ġunf old -M o -Ġnut rition -pp o -B o -Ġfound ing -Ġunder mine -Ġeas iest -ĠC zech -ĠM ack -Ġsexual ity -ĠN ixon -W in -ĠAr n -ĠK in -ãĤ £ -ic er -Ġfort un -Ġsurf aces -agh d -Ġcar riers -ĠP ART -ĠT ib -Ġinter val -Ġfrust rating -ĠSh ip -ĠAr med -ff e -Ġbo ats -ĠAb raham -in is -Ġsu ited -th read -i ov -ab ul -ĠVenezuel a -Ġto m -su per -Ġcast le -alth ough -iox ide -ec hes -Ġevolution ary -Ġnegoti ate -Ġconfront ed -Rem ember -Ġ17 0 -S uch -Ġ9 11 -m ult -ĠA byss -ur ry -ke es -spe c -ĠBarb ara -Ġbelong ing -Ġvill ain -ist ani -Ġaccount able -Ġport ions -ĠDe cl -U r -ĠK ate -g re -Ġmag azines -UC K -Ġregul ate -om on -ĠAl most -Ġover view -Ġsc ram -Ġl oot -ĠF itz -Ġcharacter istic -ĠSn ake -s ay -ĠR ico -Ġtra it -ĠJo ined -au cus -Ġadapt ation -ĠAirl ines -Ġarch ae -ĠI de -Ġb ikes -Ġliter ary -Ġinflu ences -ĠUs ed -C reat -Ġple a -ĠDef ence -ĠAss ass -Ġp ond -UL T -) " -Ġeval uated -Ġob taining -Ġdem ographic -Ġvig il -ale y -Ġsp ouse -ĠSeah awks -resp ons -ĠB elt -um atic -Ġr ises -run ner -ĠMichel le -Ġpot ent -r ace -ĠP AC -F ind -olester ol -IS S -ĠIntrodu ced -ress es -ign ment -O s -ĠT u -ĠDe x -ic ides -Ġspark ed -ĠLaur a -ĠBry ant -Ġsm iling -ĠNex us -Ġdefend ants -ĠCat al -Ġdis hes -sh aped -Ġpro long -m t -( $ -ãĢ Ĥ -Ġcalcul ations -ĠS ame -Ġp iv -H H -Ġcance lled -Ġgr in -Ġterrit ories -ist ically -C ome -ĠP arent -Pro ject -Ġneg lig -ĠPriv acy -Ġam mo -LE CT -olute ly -ĠEp ic -Ġmis under -w al -Apr il -m os -path y -ĠC arson -Ġalbum s -ĠE asy -Ġpist ol -< < -Ġ\ ( -t arget -hel p -Ġinter pre -cons cious -ĠH ousing -ĠJ oint -12 7 -Ġbe ers -s cience -ĠFire fox -effect ive -ĠC abin -ĠO kay -ĠApp lic -Ġspace craft -ĠS R -ve t -ĠStr ange -S B -Ġcor ps -iber al -e fficient -Ġpreval ence -Ġeconom ists -11 8 -Th read -ord able -OD E -ĠC ant -=- =- -if iable -ĠA round -Ġpo le -Ġwilling ness -CL A -ĠK id -Ġcomple ment -Ġsc attered -Ġin mates -Ġble eding -e very -Ġque ue -ĠTr ain -Ġh ij -Ġme lee -ple ted -Ġdig it -Ġg em -offic ial -Ġlif ting -Ð µ -Re qu -it utes -Ġpack aging -ĠWork ers -h ran -ĠLeban on -ol esc -Ġpun ished -ĠJ uan -Ġj am -ĠD ocument -Ġm apping -ic ates -Ġinev itably -Ġvan illa -ĠT on -Ġwat ches -Ġle agues -Ġiniti ated -deg ree -port ion -Ġrec alls -Ġru in -Ġm elt -I AN -Ġhe m -Ex p -Ġb aking -ĠCol omb -at ible -Ġrad ius -pl ug -ĠI F -et ically -Ġf ict -H ER -ĠT ap -atin um -Ġin k -Ġco h -ĠW izard -b oth -te x -Ġsp ends -ĠCurrent ly -ĠP it -Ġneur ons -ig nt -Ġr all -Ġbus es -b uilding -Ġadjust ments -Ġc ried -ibl ical -att ed -ĠZ ion -ĠM atter -Ġmed itation -ĠD ennis -Ġour s -ĠT ab -Ġrank ings -ort al -Ġad vers -Ġsur render -ĠG ob -ci um -om as -im eter -Ġmulti player -Ġhero in -Ġoptim istic -Ġindic ator -ĠBr ig -Ġgro cery -Ġapplic ant -ĠRock et -v id -Ex ception -p ent -Ġorgan izing -Ġenc ounters -ĠT OD -Ġjew el -S ave -ĠChrist ie -Ġhe ating -Ġl azy -ĠC P -Ġcous in -Con fig -Ġreg ener -Ġne arest -Ġachie ving -EN S -th row -ĠRich mond -ant le -200 2 -Ġan ten -b ird -13 3 -Ġn arc -r aint -un ny -ĠHispan ic -ourn aments -Ġprop he -ĠTh ailand -ĠT i -Ġinject ion -Ġinher it -rav is -Ġmed i -Ġwho ever -ĠDE BUG -G P -ĠH ud -C ard -p rom -Ġp or -Ġover head -L aw -Ġviol ate -Ġhe ated -Ġdescript ions -Ġachieve ments -ĠBe er -ĠQu ant -W as -Ġe ighth -ĠI v -Ġspecial ized -U PDATE -ĠD elta -P op -J ul -ĠAs k -oph y -Ġnews letters -ĠT ool -Ġg ard -ĠConf eder -ĠGM T -ĠAb bott -Ġimm unity -ĠV M -Is lam -Ġimpl icit -w d -Ġ19 44 -rav ity -omet ric -Ġsurv iving -ur ai -ĠPr ison -Ġr ust -ĠSk etch -Ġbe es -ĠThe ory -Ġmer it -T ex -ch at -Ġm im -Ġpast e -ĠK och -Ġignor ance -ĠSh oot -Ġbas ement -Un ited -ĠAd vis -he ight -Ġf oster -Ġdet ain -in formation -Ġne ural -' ; -Ġprov es -all ery -Ġinv itation -um bers -Ġc attle -Ġbicy cle -z i -Ġconsult ant -Ġap ology -ĠT iger -Ġ12 3 -99 9 -Ġind ividually -r t -ig ion -ĠBrazil ian -Ġdist urb -Ġentreprene urs -Ġfore sts -cer pt -pl ates -p her -clip se -Ġtw itter -Ġac ids -ograph ical -h um -ĠB ald -if ully -Ġcomp iler -ĠD A -Ġdon or -as i -Ġtrib al -l ash -ĠCon fig -Ġapplic ants -Ġsal aries -13 5 -Put in -ĠF ocus -ir s -Ġmisc onduct -ĠH az -Ġeat en -M obile -Mus lim -ĠMar cus -v iol -Ġfavor able -Ġst ub -ad in -ĠH ob -Ġfaith ful -Ġelectron ics -Ġvac uum -w ait -back ed -econom ic -d ist -Ġten ure -Ġsince re -ĠT ogether -ĠW ave -Ġprog ression -Ġden ying -Ġdist ress -br aska -th ird -Ġmix ing -Ġcolon ial -Ġpriv ately -Ġun rest -atern ity -Ġprem ises -ant i -greg ation -Ġlic ence -ĠH ind -ĠSam uel -Ġconvinc ing -ĠA ce -ĠR ust -ĠNet anyahu -Ġhand les -ĠP atch -orient ed -ah o -ĠG onz -Ġhack ers -claim er -Ġcustom s -ĠGr an -f ighters -Ġl uc -Ġman uscript -aren thood -Ġdev il -Ġwar riors -Ġoff enders -Will iam -Ġhol idays -Ġnight mare -Ġle ver -iff erent -St at -Ġexhib ition -put ed -ĠP ure -Ġal pha -Ġenthus iasm -ĠRepresent atives -E AR -ĠT yp -Ġwhe at -ĠAl f -Ġcor rection -Ġev angel -AT T -M iss -Ġs oup -Ġimpl ied -par am -Ġsex y -ĠL ux -Ġrep ublic -p atch -ab lish -Ġic ons -Ġfather s -ĠG ET -ĠCar ib -Ġregul ated -ĠCo hen -ĠBob by -Ġn er -Ġb ent -vent ory -ĠAl ong -ĠE ST -ĠWall ace -Ġmurd ers -r ise -ke ll -ĠCommon wealth -Ġn asty -et a -ĠM IT -Ġadminist ered -Ġgenuine ly -Ed itor -n ick -Ġhyd ro -**************** **************** -ĠB le -Ġfin es -Ġg orge -aus ible -r h -Ġapp le -ment ioned -Ġro pe -ot yp -H R -Ġdisappoint ing -Ġc age -n ik -Ġdoub ts -ĠF REE -print s -ĠM UST -Ġvend ors -ĠIn qu -Ġliber als -Ġcontract or -Ġup side -child ren -Ġtrick y -Ġregul ators -charg ed -l iter -Ġ *** -Ġreb ell -l ang -Ġloc als -Ġphys icians -Ġhe y -ar se -t m -ĠLe x -Ġbehavior al -success ful -F X -Ġbr ick -ov ic -Ġcon form -Ġreview ing -Ġins ights -Ġbi ology -ĠRem ove -ĠExt ra -Ġcomm itting -indu ced -ignt y -ig m -Ġat omic -Comm on -ĠE M -ĠP ere -ĠIt ems -e h -Ġpres erved -ĠH ood -Ġprison er -Ġbankrupt cy -Ġg ren -us hes -Ġexplo itation -Ġsign atures -Ġfin an -] ," -ĠM R -Ġme g -rem lin -Ġmusic ians -Ġselect ing -Ġexam ining -IN K -l ated -H i -Ġart ic -Ġp ets -Ġimp air -ĠM AN -Ġtable ts -in clude -R ange -Ġca ut -Ġlog s -Ġmount ing -Ġun aware -Ġdynam ics -ĠPalest ine -ĠQu arter -ĠPur ple -Ġm a -ĠIm port -Ġcollect ions -ci ation -Ġsuccess or -Ġcl one -Ġaim ing -Ġposs essed -Ġstick ing -Ġsh aking -Ġloc ate -ĠH ockey -T urn -17 0 -Ġfif teen -ĠHar rison -Ġcontinu ously -ĠT C -ĠVal ent -ĠRes cue -Ġby pass -am ount -Ġm ast -Ġprotect s -Ġart istic -Ġsomet ime -Ġsh oe -Ġshout ed -ific ant -et itive -ĠReg ister -ĠJ in -Ġconcent rated -ling ton -on ies -Ġgener ator -yr im -ĠAr men -Ġclear ing -id o -ĠT W -al ph -Ġlad ies -H ard -Ġdial og -Ġinput s -æ ľ -Ġpos es -Ġsl ots -ĠPrem ium -Ġle aks -Ġboss es -Ġ11 3 -c ourse -A cc -ĠNew ton -ĠAust ria -ĠM age -Ġte aches -ab ad -Ġwe ars -Ġc yl -Ġcur se -ĠS ales -ĠW ings -Ġp sy -Ġg aps -ĠIce land -ĠP interest -Ġland lord -Ġdefin itions -ĠK er -Ġsufficient ly -ĠP ence -ĠArch itect -Ġsur pass -Ġ11 4 -Ġsuper hero -ĠDise ase -Ġpri ests -ĠC ulture -Ġdefin itive -Ġsecret ly -ĠD ance -inst all -ch ief -ĠJess ica -W ould -Up dated -Ġlock er -ĠK ay -Ġmem orial -è ¦ -f at -Ġdis gu -Ġflav ors -ĠBase ball -ĠRes istance -Ġk icks -Ġen v -Ġteen agers -D ark -ĠC AR -Ġh alt -ĠL G -ĠGab riel -Ġfe ver -Ġs atur -Ġm all -Ġaffili ate -ĠS leep -ĠSpe cific -ĠV el -Ġj ar -ĠSac red -ĠEd wards -ĠA CL -Ġret ained -ĠG iant -Ġlim itation -in ces -Ġref usal -ĠT ale -ĠBut ler -Ġacc idents -ĠC SS -Ġimport ed -ĠCop y -Î ± -ER T -z el -Ġdiv isions -h ots -ĠAl b -ĠD S -Load er -W ashington -at isf -ĠCreat ive -\ . -ĠAut om -red ict -Ġrecept or -ĠCarl os -Met hod -ok a -Ġmal icious -Ġste pping -, [ -ĠD ad -Ġatt raction -ĠEffect s -ĠPir ate -ĠC er -ĠIndust ry -ĠR ud -Ġchar ter -Ġd ining -Ġins ists -Ġconfig ure -Ġ( # -ĠSim ple -ĠSc roll -UT C -17 5 -ĠK on -Ġmarket place -Ġ ãĤ -Ġref res -Ġg ates -er red -ĠP od -Ġbeh ave -Fr ank -n ode -Ġendors ed -he tt -as ive -ĠHom eland -Ġr ides -ĠLe ave -er ness -Ġflood ing -A FP -Ġris en -Ġcontin ually -Ġun anim -ĠCont ract -ĠP as -Ġgu ided -ĠCh ile -b d -Ġsu cc -pt ic -Ġcomm ittees -ĠL uther -ĠAny one -Ġs ab -12 4 -Ġp ixel -ĠB ak -ĠT ag -ĠBenn ett -En ter -sm all -ĠPresident ial -Ġp ul -Ġcontr ace -arch ive -Ġcoast al -ĠK ids -19 2 -âĢ ² -ick y -ING TON -Ġw olf -ĠSt alin -T ur -id get -am as -ĠUn less -Ġspons or -Ġmor ph -ĠCho ose -Ġrun ner -Ġun bel -Ġm ud -ĠMan a -Ġdub bed -Ġg odd -ure rs -wind ow -Ġrel ied -Ġcelebr ating -os c -Ġ13 5 -Ġlobb ying -Ġincom plete -Ġrestrict ion -Ġinc ap -it us -Ġexpect ation -ĠAp ollo -Ġint ens -Ġsyn c -G H -Ġmanip ulation -B Y -Ġspe ar -Ġbre asts -Ġvol can -il ia -M aterial -Ġform ats -ĠB ast -Ġparliament ary -Ġsn ake -Ġserv ants -ĠTr udeau -ĠGr im -ĠArab ic -ĠSC P -ĠBoy s -st ation -Ġprospect ive -ord e -in itialized -Ġb ored -AB LE -Ġaccess ed -Ġtax i -ĠShe ll -aid en -urs ed -in ates -ĠIns urance -ĠPet e -Sept ember -6 50 -Ġad ventures -ĠCo ver -Ġt ribute -Ġsk etch -Ġem power -Ġ Ø -ĠGl enn -ĠD aw -= \" -ĠPolit ics -Ġgu ides -Ġd ioxide -ĠG ore -ĠBr ight -ĠS ierra -Ġval ued -c ond -Ġpo inter -Se lect -Ġrisk y -Ġabsor b -im ages -Ġref uses -Ġbon uses -__ _ -Ġh ilar -ĠF eatures -2 20 -ĠCollect or -F oot -Ġ19 64 -cul us -Ġd awn -Ġwork out -ĠL O -Ġphilosoph ical -ĠSand y -ĠYou th -Ġl iable -A f -bl ue -Ġovert urn -less ness -ĠTrib une -ĠIn g -Ġfact ories -Ġcat ches -Ġpr one -Ġmat rix -Ġlog in -Ġin acc -Ġex ert -s ys -Ġneed le -ĠQ ur -Ġnot ified -ould er -t x -Ġremind s -Ġpublisher s -Ġn ort -Ġg it -Ġfl ies -ĠEm ily -Ġflow ing -ĠAl ien -ĠStr ateg -Ġhard est -Ġmod ification -AP I -ĠM Y -Ġcr ashes -st airs -n umber -Ġur ging -ch annel -ĠFal con -Ġinhabit ants -Ġterr ifying -Ġutil ize -Ġban ner -Ġcig arettes -Ġsens es -ĠHol mes -Ġpract ition -ĠPhill ips -ott o -Ġcomp ile -Mod el -ĠK o -Ġ[ ] -Americ ans -ĠTer ms -Ġmed ications -ĠAn a -Ġfundament ally -ĠNot ice -Ġwe aker -Ġ 0000 -Ġgar lic -Ġout break -Ġeconom ist -ĠB irth -Ġobst acles -ar cer -ĠOr thodox -Ġplace bo -ĠC rew -asp berry -ĠAng els -Ġdis charge -Ġdestruct ive -11 7 -ĠR ising -Ġd airy -l ate -Ġcoll ision -ĠTig ers -ean or -ocument ed -ĠIn valid -Ġd ont -ĠL iter -ĠV a -Ġhyd rogen -Ġvari ants -ĠBrown s -Ġ19 65 -Ġind igenous -Ġtrad es -Ġremain der -Ġswe pt -ĠImp act -Ġred ist -Ġun int -grad uate -ãĥ ķ -ĠW ILL -ãģ® ç -ĠCrit ical -Ġf isher -Ġv icious -Ġrevers ed -Y ear -ĠS ox -Ġshoot ings -Ġfil ming -Ġtouchdown s -ai res -m el -Ġgrand father -Ġaffect ion -ing le -Ġover ly -Add itional -Ġsup reme -ĠGr ad -Ġsport ing -Ġmer cy -ĠBrook s -ount y -Ġperform s -Ġtight ly -Ġdem ons -Ġkill ings -Ġfact ion -ĠNov a -aut s -Ġund oubtedly -ar in -Ġunder way -ra k -Ġl iv -ĠReg ion -Ġbrief ing -s ers -cl oud -ĠM ik -us p -Ġpred iction -az or -Ġport able -ĠG and -Ġpresent ing -Ġ10 80 - » -ush i -ĠSp ark -there um -Ġjust ification -ĠN y -Ġcontract ors -ming ham -ĠSt yle -å ħ -ĠChron icles -ĠPict ure -Ġprov ing -Ġw ives -set t -Ġmole cules -ĠFair y -Ġconsist ing -Ġp ier -al one -in ition -Ġn ucle -j son -Ġg otta -Ġmob il -Ġver bal -ar ium -Ġmon ument -uck ed -Ġ25 6 -T ech -mine craft -ĠTr ack -Ġt ile -Ġcompat ibility -as is -Ġs add -Ġinstruct ed -ĠM ueller -Ġle thal -Ġhorm one -Ġor che -el se -Ġske let -Ġentert aining -Ġminim ize -ag ain -Ġunder go -Ġconst raints -Ġcig arette -ĠIslam ist -Ġtravel s -ĠPant hers -l ings -C are -Ġlaw suits -ur as -Ġcry st -Ġlow ered -Ġaer ial -Ġcomb inations -Ġha un -Ġch a -Ġv ine -Ġquant ities -Ġlink ing -b ank -Ġso y -B ill -ĠAngel a -Ġrecip ient -ĠProt est -Ġs ocket -Ġsolid arity -Ġâ Ĩ -m ill -Ġvar ies -ĠPak istani -Dr agon -Ġun e -Ġhor izon -³³³³ ³³³³ -Ġprov inces -Ġfrank ly -Ġenact ed -not es -[ ' -Ġ19 2 -ocr acy -Ġendorse ment -Ġover time -Tr ue -L ab -lic ted -ĠD NC -Ġbe ats -ĠJam ie -15 2 -ĠIN T -Cont act -Ġaccount ed -h ash -ĠPack ers -p ires -Ġles bian -Ġamend ments -Ġhop eful -ĠFin land -Ġspot light -Ġconfig ured -Ġtrou bled -Ġg aze -ĠCal gary -Ġrel iability -Ġins urg -sw er -b uy -ĠSk in -Ġp ixels -Ġhand gun -Ġpar as -Ġcateg or -ĠE L -ĠRe x -Ind eed -Ġkind a -Ġconj unction -ĠBry an -ĠMan ufact -y ang -Pl us -S QL -ish ment -Ġdom inate -Ġn ail -Ġo ath -Ġeru pt -ĠF ine -it bart -ĠCh ip -ĠAb d -ĠN am -Ġbuy er -Ġdiss ent -Le aks -Cont in -Ġr ider -ĠSome one -Ġill usion -c in -ĠBoe ing -Ġin adequ -ov ation -i ants -Ġreb uild -4 50 -ĠDest iny -S W -ĠT ill -H it -ia z -ĠBang l -acher s -ĠRe form -Ġse gments -Ġsystem atic -d c -ĠConserv atives -Ġport al -h or -ĠDragon bound -Ġdrag ged -om o -Ġthe e -ad vert -ĠRep orts -ĠE t -Ġbarrel s -Aug ust -Ġcompar isons -Ġhe x -Ġan throp -" [ -bor ough -ab i -Ġpict ured -play ing -ĠAdd ress -ĠMir ror -Sm ith -Ġt ires -ĠN PR -AA AA -Ġclass ification -ĠTh an -ĠH arm -ĠR A -Ġreject ion -min ation -Ġr anged -ĠF alls -D I -H ost -ãĤ ´ -ĠEx ample -list ed -th irds -Ġsaf egu -br and -Ġprob able -Can ada -IT ION -ĠQ aeda -Ġch ick -Ġimport s -h it -l oc -W W -Ġble w -Ġany time -Ġwh oles -ik ed -Ġcal culation -cre ate -ĠO ri -Ġupgr aded -Ġapp ar -ut ory -ĠM ol -B rit -ĠJ ong -IN AL -ĠStart ing -Ġd ice -urt le -Ġre lying -cl osure -Ġprof itable -Ġsl aughter -ĠMan ual -c aster -Ġ" $ -Ġfe ather -ĠSim ply -ie ves -Ġdeter ior -ĠPC I -Ġst amp -Ġfl aws -Ġsh ade -ham mer -Ġpass port -Ġcont ing -am el -Ġobser vers -Ġneg lect -ĠR B -ĠBrother hood -Ġskept ical -f amily -us k -Ġemotion ally -â Ļ -ĠBet a -ason able -id ity -ĠM ul -Ġkick ing -ĠC arm -oll ah -VERT IS -ĠAt hen -Ġlad der -ĠBul let -å £ -00 01 -ĠWild life -ĠM ask -ĠN an -R ev -Ġun acceptable -leg al -Ġcrowd ed -ag i -ĠC ox -j e -Ġmor ality -Ġfu els -Ġc ables -Ġman kind -ĠCarib bean -Ġanch or -Ġby te -ĠO ften -ĠO z -Ġcraft ed -Ġhistor ian -ĠW u -Ġtow ers -ĠCitiz ens -Ġhel m -Ġcred entials -Ġsing ular -ĠJes se -Ġtack les -Ġcont empt -Ġa fore -ĠSh adows -Ġn il -Ġur gent -app le -bl ood -Ġv on -Ġoff line -Ġbreat he -Ġj umps -Ġirre levant -ox ic -om al -import ant -J im -Ġgl oves -arm ing -dep th -Ġtal ents -ook ie -ĠS B -Ġpal m -uff s -est a -IG H -Ġcan on -ĠVer izon -ĠP le -Ġcou pled -vel t -Ġfundra ising -ĠGet ting -ĠD LC -Ġmathemat ical -ĠH S -ĠCard inals -te lling -Ġspons ors -Ġ Ï -ĠBull s -op tion -Ġprop ose -Ġmem orable -Ġembr aced -Ġdecl ining -He alth -ed a -Ġ} ; -Ġsp am -m ile -Ġpit cher -ĠE ight -Ġcar ing -ut ic -ro le -Ġair line -ernand ez -ĠAth let -Ġcert ification -ux e -rig er -Ġem pir -Ġsens ation -Ġdis m -Ġb olt -Ġev olve -H ouse -Ġconsult ation -ĠD uty -Ġtou ches -ĠN athan -Ġf aint -h ad -" ( -ĠCons umer -ĠExt reme -Ġ12 7 -ĠHer m -ĠSac rament -iz oph -Ġanx ious -ul ously -Ġsoc ially -ĠU TC -Ġsol ving -ĠLet ter -Hist ory -ed uc -Pr ice -) ); -Ġrel oad -am ic -Ġp ork -Ġdisc ourse -Ġt ournaments -ai ro -ĠK ur -ĠCost a -Ġviol ating -Ġinterf ere -Ġrecre ational -uff le -Ġspe eches -Ġneed ing -Ġremem bers -Ġcred ited -n ia -f ocused -amer a -Ġb ru -um bs -ĠCub an -Ġpreced ing -Ġnons ense -ac ial -Ġsmart phones -ĠSt ories -S ports -ĠEmer gency -oun cing -ef ined -Ġb er -Ġconsult ing -Ġm asters -he astern -." [ -ĠRun ning -Ġsus cept -ĠF eng -Americ a -pr ises -st itial -ĠWeek ly -ĠGreat er -mod ules -if ter -G raphics -ul er -Ġwho lly -Ġsupp ress -Ġconce aled -Ġhapp ily -Ġaccept s -ĠEn joy -Ġr ivers -ĠEx cept -2 25 -ĠN HS -ĠMc Connell -Ġp ussy -fer red -ut able -Ġatt ain -Ġ> = -Ġdepos its -roph ic -Ġnot orious -ĠSh aw -il itation -Ġepid emic -all ic -Ġsmall est -ov ich -Ġaccess ories -per ties -Ġsur plus -ĠMe ch -Ġamb ig -ĠImm igration -Ġch im -ev al -Ġpract icing -ĠMyster y -Ġdom ains -ĠSil icon -app s -Ġkilomet ers -e a -ĠSm ash -Ġwarrant y -Ġn ost -s il -re v -J on -ĠDub lin -Ġtast es -Ġb out -g reat -er ror -Ġsw itches -ĠB apt -D O -ok i -Ġsour ced -pro du -Ġattach ment -ĠIss ue -ĠQuest ion -Jo in -Ġf itted -Ġunlaw ful -^ ^ -ere k -Ġauthent ication -Ġst ole -Ġaccount ability -l abel -S earch -Ġal beit -atic an -fund ed -ĠAdd ing -ĠI Q -Ġsub mar -l it -a que -ĠLear ning -Ġint eger -M aster -ĠCh rom -Ġprem ier -O p -ĠLi u -Ġbl essed -ĠGl obe -ĠResp onse -Ġlegit im -ĠMer kel -Ġdispos al - ´ -Ġgau ge -pe at -Ġindu ced -Ġquestion able -arth y -ĠV it -ĠF eed -U ntil -U t -worth y -R Y -ĠH erald -ĠHam mer -Ġmed al -ĠR ivers -ĠH ack -Ġclar ify -Ġtrack ed -Ġautonom ous -Ġten ant -ĠQ atar -er ie -Ġgr im -ĠMon itor -Ġresist ant -ĠSpe c -ĠWell s -N AS -14 8 -Ġmin ers -iot ics -Ġmiss es -11 6 -g ian -g it -ĠE yes -p res -Ġgrad uated -Ġang el -Ġsyn chron -Ġefficient ly -Ġtrans mitted -H arry -Ġglob ally -EN CE -ĠMont ana -r aged -ĠPre vention -Ġp iss -ĠL l -Ġshe lf -ĠB JP -ĠTest ament -ĠL ate -ik er -ĠH app -ĠJul ian -h all -Ġsp ont -Ġshut down -Ġincons istent -Ġsubscrib ers -Ġske leton -ĠNe braska -Ġins pire -ĠV oid -F eed -Ġang les -ĠSpr ings -Ġbench mark -Ġvacc ines -izoph ren -se xual -uff ed -Ġsh ine -ĠK ath -Ġgest ure -ine a -Ġr ip -Ġopp ression -Ġcons cience -b t -ĠL um -Ġinc idence -ĠF a -w r -Ġmin eral -ĠSp urs -alk y -Ġth under -Ġop io -Be ing -ĠPal m -Ġwas ted -Ġl b -i aries -ĠIniti ative -Ġcur ric -Ġmark er -ĠMc L -Ġext ensions -ĠP v -ĠAr ms -Ġoffer ings -Ġdef enses -Ġvend or -Ġcontrad ict -ĠCol in -Ġredd it -Ġper ipher -12 2 -Ġs ins -E dit -IC T -So ft -ĠSh ah -Ġadministr ator -ĠT rip -Ġporn ography -Ġtu ition -in ence -ĠPro gress -Ġcat alog -Ġsu ite -Ġh ike -Ġreprodu ctive -eng ine -Ġd rought -ĠNo ah -Ġ2 30 -Ġd ude -Ġrelax ed -Ġpart ition -Ġparticip ant -Ġtel esc -Ġfe as -ĠF F -own er -Ġswe eping -Ġl enses -Ġmatch up -ĠRe pl -ourn als -Ġcred ible -Ġgrand mother -Ġther mal -Ġsubscrib ing -Ġident ities -col m -U CT -Ġreluct ant -us ers -ĠC ort -Ġassist ed -OS S -ATION S -IS H -Ġpharm aceutical -ic able -ad ian -ĠSon ic -ĠF ury -ĠM ong -A H -ĠPsych ology -Ġph osph -Ġtreat s -Ń Ķ -Ġstead ily -ĠHell o -Ġrel ates -Ġcl ue -Ex pl -a uth -Ġrev ision -Ġe ld -os ion -Ġbr on -14 4 -ri kes -Ġmin es -Ġblank et -ĠF ail -el ed -ĠIm agine -ĠPl anned -a ic -Re quest -M ad -ĠHor se -ĠEag le -Ġcap ac -15 7 -Ġl ing -ĠN ice -ĠP arenthood -min ster -og s -ens itive -Not hing -Ġcar n -F in -ĠP E -Ġr ifles -ĠL P -S and -Ġgui Active -Ġtour ist -C NN -Ġunve iled -Ġpredec essor -} { -u ber -Ġoff shore -Ġopt ical -ĠR ot -ĠPear l -et on -Ġst ared -Ġfart her -at ility -cont in -ĠG y -ĠF oster -ĠC oc -ri ents -Ġdesign ing -ĠEconom y -ON G -W omen -ĠN ancy -er ver -Ġmas cul -Ġcasual ties -Ġ2 25 -ĠS ullivan -ĠCh oice -Ġa ster -w s -Ġhot els -Ġconsider ations -Ġcou ch -ĠSt rip -ĠG n -Ġmanip ulate -l ied -Ġsynt hetic -Ġassault ed -Ġoff enses -ĠDra ke -Ġim pe -Oct ober -ĠHer itage -h l -ĠBl air -Un like -Ġg rief -Ġ4 50 -Ġopt ed -Ġresign ation -il o -Ġver se -ĠT omb -Ġu pt -Ġa ired -ĠH ook -ĠML B -Ġassum es -out ed -ĠV ers -Ġinfer ior -Ġbund le -ĠD NS -ograp her -Ġmult ip -ĠSoul s -Ġillust rated -Ġtact ic -Ġdress ing -Ġdu o -Con f -Ġrel ent -Ġc ant -Ġscar ce -Ġcand y -ĠC F -Ġaffili ated -Ġspr int -yl an -ĠGarc ia -Ġj unk -Pr int -ex ec -C rit -Ġport rait -ir ies -ĠOF F -Ġdisp utes -W R -L ove -ãģ Ħ -ĠRe yn -Ġh ipp -op ath -Ġflo ors -ĠFe el -Ġwor ries -Ġsett lements -ĠP os -Ġmos que -Ġfin als -Ġcr ushed -ĠPro bably -ĠB ot -ĠM ans -ĠPer iod -Ġsovere ignty -Ġsell er -Ġap ost -Ġam ateur -Ġd orm -Ġconsum ing -Ġarm our -ĠRo ose -Ġint ensive -Ġelim inating -ĠSun ni -ĠAle ppo -j in -Ġadv ise -p al -ĠH alo -Ġdes cent -Ġsimpl er -Ġbo oth -ST R -L ater -ĠC ave -== = -Ġm ol -Ġf ist -Ġshot gun -su pp -Ġrob bery -E ffect -Ġobsc ure -ĠProf essional -Ġemb assy -Ġmilit ant -Ġinc arcer -Ġgener ates -Ġlaun ches -Ġadministr ators -Ġsh aft -Ġcirc ular -Ġfresh man -ĠW es -ĠJo el -ĠD rew -ĠDun can -ĠApp arently -s ight -ĠIntern al -ĠInd ividual -ĠF E -Ġb ore -ĠM t -Ġbroad ly -ĠO ptions -ount ain -ip es -ĠV ideos -20 4 -Ġh ills -Ġsim ulation -Ġdisappoint ment -it an -ĠLabor atory -Ġup ward -Ġbound ary -Ġdark er -h art -Ġdomin ance -C ong -ĠOr acle -ĠL ords -Ġscholars hip -ĠVin cent -ed e -ĠR ah -Ġencour ages -ro v -Ġqu o -Ġprem ise -ĠCris is -ĠHol ocaust -Ġrhyth m -Ġmet ric -cl ub -Ġtransport ed -Ġn od -ĠP ist -Ġancest ors -ĠFred er -th umbnails -ĠC E -ON D -Ph il -ven ge -ĠProduct s -cast le -Ġqual ifying -ĠK aren -VERTIS EMENT -Ġmight y -Ġexplan ations -Ġfix ing -D i -Ġdecl aring -Ġanonym ity -Ġju ven -ĠN ord -ĠDo om -ĠAct ually -O k -ph is -ĠDes ert -Ġ11 6 -I K -ĠF M -Ġinc omes -V EL -ok ers -Ġpe cul -Ġlight weight -g ue -Ġacc ent -Ġincre ment -ĠCh an -Ġcompl aining -ĠB aghd -Ġmidfield er -Ġover haul -Pro cess -ĠH ollow -ĠTit ans -Sm all -man uel -ĠUn ity -ĠEv ents -S ty -Ġdispro portion -n esty -en es -ĠC od -Ġdemonstr ations -ĠCrim son -ĠO H -Ġen rolled -Ġc el -ĠBre tt -Ġa ide -Ġhe els -Ġbroad band -Ġmark ing -Ġw izard -ĠN J -ĠChief s -Ġingred ient -Ġd ug -ĠSh ut -urch ase -end or -Ġfar mer -ĠGold man -12 9 -15 5 -Or der -Ġl ion -i ably -Ġst ain -ar ray -ilit ary -ĠFA Q -Ġexpl oded -ĠMcC arthy -ĠT weet -ĠG reens -ek ing -l n -ens en -Ġmotor cycle -Ġpartic le -Ġch olesterol -B ron -Ġst air -Ġox id -Ġdes irable -ib les -Ġthe or -for cing -Ġpromot ional -ov o -b oot -ĠBon us -raw ling -Ġshort age -ĠP sy -Ġrecru ited -Ġinf ants -Ġtest osterone -Ġded uct -Ġdistinct ive -Ġfirm ware -bu ilt -14 5 -Ġexpl ored -Ġfact ions -Ġv ide -Ġtatt oo -Ġfinan cially -Ġfat igue -Ġproceed ing -const itutional -Ġmis er -Ġch airs -gg ing -ipp le -Ġd ent -Ġdis reg -ç Ķ -st ant -ll o -b ps -aken ing -Ġab normal -ĠE RA -å£ « -ĠH BO -ĠM AR -Ġcon cess -Ġserv ant -Ġas pir -l av -ĠPan el -am o -Ġprec ip -Ġrecord ings -Ġproceed ed -Ġcol ony -ĠT ang -ab lo -Ġstri pped -Le ft -to o -Ġpot atoes -Ġfin est -% ). -Ġc rap -ĠZ ach -ab ases -ĠG oth -Ġbillion aire -w olf -Ġsan ction -S K -Ġlog ged -P o -ey ed -un al -Ġcr icket -Ġarm ies -Ġunc overed -Cl oud -ó n -Ġreb ounds -Ġm es -O per -P ac -Ġnation ally -Ġinsert ed -p ict -Ġgovern ance -Ð ¸ -Ġprivile ges -G ET -Ġfavor ites -im ity -Ġlo ver -the m -em pl -Ġgorge ous -An n -Ġsl ipped -Ġve to -B ob -Ġsl im -u cc -ĠF ame -udden ly -Ġden ies -ĠM aur -Ġdist ances -Ġw anna -t ar -ĠS ER -Ġâ Ī -Ġle mon -at hetic -Ġlit eral -Ġdistingu ished -Ġansw ering -G I -Ġrelig ions -ĠPhil os -ĠL ay -Ġcomp os -ire ments -ĠK os -ine z -roll ing -Ġyoung est -and ise -ĠB orn -Ġalt ar -am ina -ĠB oot -v oc -Ġdig ging -Ġpress ures -Ġl en -26 4 -Ġassass ination -ĠBir mingham -ĠMy th -Ġsovere ign -ĠArt ist -ĠPhot ograph -Ġdep icted -Ġdisp ens -orth y -Ġamb ul -int eg -ĠC ele -ĠTib et -Ġhier archy -Ġc u -Ġpre season -ĠPet erson -Ġcol ours -Ġworry ing -Ġback ers -ĠPal mer -ĠÎ ¼ -Ġcontribut or -Ġhear ings -Ġur ine -Ġ Ù -ourge ois -Sim ilar -ĠZ immer -s omething -ĠUS C -Ġstrength s -ĠF I -Ġlog ging -As ked -ĠTh ai -in qu -ĠW alt -Ġcrew s -it ism -3 01 -Ġshar ply -um ed -Ġred irect -r ators -In f -ĠWe apons -Ġte asp -19 99 -L ive -ĠEs pecially -ĠS ter -ĠVeter ans -Ġint ro -other apy -Ġmal ware -Ġbre eding -Ġmole cular -ĠR oute -ĠCom ment -oc hem -Ġa in -Se ason -Ġlineback er -Ä « -ĠEconom ics -es ar -ĠL ives -ĠEm ma -Ġk in -ĠTer rit -Ġpl anted -ot on -ĠBut ter -ĠSp ons -P ER -Ġdun geon -Ġsymb olic -Ġfil med -Ġdi ets -Ġconclud es -Ġcertain ty -ĠForm at -Ġstr angers -form at -ĠPh ase -Ġcop ied -Ġmet res -ld a -ĠUs ers -Ġdeliber ate -Ġwas hed -ĠL ance -im ation -Ġimpro per -ĠGen esis -ick r -ĠK ush -Ġreal ise -Ġembarrass ing -alk ing -b ucks -Ġver ified -Ġout line -year s -ĠIn come -20 2 -Ġz ombies -F inal -ĠMill enn -Ġmod ifications -ĠV ision -ĠM oses -ver b -iter ranean -ĠJ et -Ġnav al -ĠA gg -Ġur l -Ġvict ories -Ġnon etheless -Ġinj ust -ĠF act -ç ļ -Ġins ufficient -re view -face book -Ġnegoti ating -Ġguarant ees -im en -uten berg -Ġg ambling -Ġcon gr -Load ing -Ġnever theless -Ġpres idents -ĠIndust rial -Ġ11 8 -Ġp oured -ĠT ory -Ġ17 5 -Ġ: = -Sc ott -ange red -T ok -Ġorgan izers -M at -ĠG rowth -Ġad ul -Ġens ures -Ġ11 7 -é¾į å -Ġmass acre -Ġgr ades -be fore -AD VERTISEMENT -ĠSl ow -ĠM MA -âĢĶ " -ĠV atican -Q aeda -Ġo we -66 66 -ĠS orry -ĠGr ass -Ġbackground s -Ġexha usted -Ġcl an -Ġcomprom ised -ĠE lf -ĠIsa ac -ens on -In vest -IF A -Ġinterrupt ed -ãĥī ãĥ© -Ġtw isted -ĠDrag ons -M ode -ĠK remlin -Ġfert il -he res -ph an -ĠN ode -f ed -ĠOr c -Ġunw illing -C ent -Ġprior it -Ġgrad uates -Ġsubject ive -Ġiss uing -ĠL t -Ġview er -Ġw oke -Th us -bro ok -Ġdep ressed -Ġbr acket -ĠG or -ĠFight ing -Ġstri ker -Rep ort -ĠPortug al -Ġne o -w ed -19 9 -Ġflee ing -sh adow -ident ified -US E -Ste am -Ġstret ched -Ġrevel ations -art ed -ĠD w -Ġalign ment -est on -ĠJ ared -S ep -Ġblog s -up date -g om -r isk -Ġcl ash -ĠH our -Ġrun time -Ġunw anted -Ġsc am -Ġr ack -Ġen light -on est -ĠF err -Ġconv ictions -Ġp iano -Ġcirc ulation -ĠW elcome -Ġback lash -ĠW ade -Ġrece ivers -ot ive -J eff -Ġnetwork ing -ĠPre p -ĠExpl orer -Ġlect ure -Ġupload ed -ĠMe at -B LE -ĠNaz is -ĠSy nd -st ud -ro ots -ri ans -Ġportray ed -Ġ ?? -ĠBudd ha -s un -Rober t -ĠCom plex -Ġover see -Ġste alth -T itle -ĠJ obs -ĠK um -Ġappreci ation -ĠM OD -Ġbas ics -Ġcl ips -Ġnurs ing -Ġpropos ition -Ġreal ised -ĠNY C -Ġall ocated -ri um -ar an -ĠPro duction -ĠV ote -Ġsm ugg -Ġhun ter -az er -ĠCh anges -Ġfl uct -y on -Ar ray -Ġk its -W ater -Ġuncom mon -Ġrest ing -ell s -w ould -Ġpurs ued -Ġassert ion -omet own -ĠMos ul -ĠPl atform -io let -Ġshare holders -Ġtra ils -P ay -ĠEn forcement -ty pes -ĠAn onymous -Ġsatisf ying -il ogy -Ġ( ' -w ave -c ity -Ste ve -Ġconfront ation -ĠE ld -C apt -ah an -ht m -ĠC trl -ON S -2 30 -if a -hold ing -Ġdelic ate -Ġj aw -ĠGo ing -or um -S al -Ġd ull -ĠB eth -Ġpr isons -Ġe go -ĠEl sa -avor ite -ĠG ang -ĠN uclear -Ġsp ider -ats u -Ġsam pling -Ġabsor bed -ĠPh arm -iet h -Ġbuck et -ĠRec omm -O F -ĠF actory -AN CE -Ġb acter -H as -ĠObs erv -12 1 -Ġprem iere -De velop -Ġcur rencies -C ast -Ġaccompany ing -ĠNash ville -Ġfat ty -ĠBre nd -Ġloc ks -Ġcent ered -ĠU T -augh s -or ie -ĠAff ordable -v ance -D L -em et -Ġthr one -ĠBlu etooth -Ġn aming -if ts -AD E -Ġcorrect ed -Ġprompt ly -ĠST R -Ġgen ome -Ġcop e -Ġval ley -Ġround ed -ĠK end -al ion -p ers -Ġtour ism -Ġst ark -v l -Ġblow ing -ĠSche dule -st d -Ġunh appy -Ġlit igation -ced es -Ġand roid -Ġinteg ral -ere rs -ud ed -t ax -Ġre iter -ĠMot ors -oci ated -Ġwond ers -ĠAp ost -uck ing -ĠRoose velt -f ram -Ġyield s -Ġconstit utes -aw k -Int erest -Ġinter im -Ġbreak through -ĠC her -Ġpro sec -ĠD j -ĠM T -Res p -ĠP T -Ġs perm -ed it -B T -Lin ux -count ry -le ague -Ġd ick -Ġo ct -Ġinsert ing -Ġsc ra -ĠBrew ing -Ġ19 66 -Ġrun ners -Ġpl un -id y -ĠD ian -Ġdys function -Ġex clusion -Ġdis gr -Ġincorpor ate -Ġrecon c -Ġnom inated -ĠAr cher -d raw -achel or -Ġwrit ings -Ġshall ow -Ġh ast -ĠB MW -ĠR S -Ġth igh -Ġ19 63 -Ġl amb -Ġfav ored -ag le -Ġcool er -ĠH ours -ĠG U -ĠOrig in -Ġglim pse ----------------- ---- -L im -Ġche ek -Ġj ealous -- ' -Ġhar ness -ĠPo ison -Ġdis abilities -ne apolis -Ġout look -Ġnot ify -ĠIndian apolis -Ġab rupt -ns ic -Ġenc rypted -Ġfor fe -reat h -Ġr abb -Ġfound ations -Ġcompl iment -ĠInter view -ĠS we -Ġad olesc -Ġmon itors -ĠSacrament o -Ġtime ly -Ġcontem pl -Ġposition ed -Ġpost ers -ph ies -iov ascular -v oid -ĠFif th -Ġinvestig ative -OU N -Ġinteg rate -ĠIN C -ish a -ibl ings -ĠRe quest -ĠRodrig uez -Ġsl ides -ĠD X -Ġfemin ism -Ġdat as -Ġb end -ir us -ĠNig eria -F ox -Ch ange -Ġair plane -ĠLad en -Ġpublic ity -ixt y -Ġcommit ments -Ġaggreg ate -Ġdisplay ing -ĠAr row -Ġ12 2 -Ġrespect s -and roid -s ix -ĠSh a -Ġrest oration -) \ -W S -oy s -Ġillust rate -with out -12 6 -ĠâĶ Ĥ -Ġpick up -n els -Ġ .... -f ood -ĠF en -) ? -Ġphenomen a -Ġcompan ions -ĠW rite -Ġsp ill -Ġbr idges -ĠUp dated -ĠF o -Ġinsect s -ASH INGTON -Ġsc are -il tr -ĠZh ang -Ġsever ity -Ġind ul -14 9 -ĠCo ffee -Ġnorm s -Ġp ulse -ĠF T -Ġhorr ific -ĠDest roy -ĠJ SON -Ġo live -Ġdiscuss es -R est -E lect -ĠW inn -ĠSurv iv -ĠH ait -S ure -op ed -Ġro oted -ĠS ke -ĠBron ze -Ġl ol -Def ault -Ġcommod ity -red ited -Ġliber tarian -Ġforb idden -Ġgr an -à ¨ -Ġl ag -en z -dri ve -Ġmathemat ics -Ġw ires -Ġcrit ically -Ġcarb ohyd -ĠChance llor -ĠEd die -Ġban ning -ĠF ri -Ġcompl ications -et ric -ĠBangl adesh -Ġband width -St op -ĠOrig inally -Ġhalf way -yn asty -sh ine -Ġt ales -rit ies -av ier -Ġspin ning -ĠWH O -Ġneighbour hood -b ach -Ġcommer ce -ĠS le -B U -Ġentreprene ur -Ġpecul iar -ĠCom ments -f re -3 20 -IC S -Ġimag ery -ĠCan on -ĠElect ronic -sh ort -( ( -D ig -Ġcomm em -u ced -Ġincl ined -ĠSum mon -Ġcl iff -ĠMed iterranean -Ġpo etry -Ġprosper ity -ĠRe ce -Ġp ills -m ember -Ġfin ale -un c -ĠG ig -ä ½ -Ġl od -Ġback ward -- + -ĠFor ward -Ġth ri -s ure -Ġso ap -ĠF X -R ES -ĠSe xual -oul os -Ġfool ish -Ġright eous -Ġco ff -terror ism -ust ain -ot er -Ġab uses -ne xt -Ġab usive -Ġthere after -Ġprohib ition -ĠS UP -Ġd ip -Ġr ipped -Ġinher ited -Ġb ats -st ru -G T -Ġflaw ed -ph abet -Ġf og -do ors -Ġim aging -Ġdig its -ĠHung ary -Ġar rog -Ġteach ings -Ġprotocol s -ĠB anks -à ¸ -p ound -ĠC urt -." ) -. / -Ġex emption -end ix -ĠM ull -Ġimpro ves -ĠG amer -d imensional -I con -ĠMarg aret -St atus -d ates -Ġint ends -Ġdep ict -Ġpark ed -J oe -ĠMar ines -chn ology -! ). -Ġjud ged -Ġwe ights -R ay -Ġapart ments -he ster -Ġrein force -Ġoff ender -occ up -Ġs ore -e pt -ĠPH P -ĠB row -Ġauthor ization -ĠR isk -ĠDel aware -ĠQ U -Ġnot ifications -Ġsun light -Ġex clude -d at -Ġm esh -ĠSud an -Ġbelong ed -Ġsub way -Ġno on -ĠInter ior -ol ics -ĠL akers -Ġc oding -Dis claimer -Cal if -O ld -Ġdis l -???? ? -Ġconfir ms -Ġrecruit ment -Ġhom icide -Cons ider -ĠJeff rey -ft y -} ; -Ġobject ion -do ing -ĠLe o -W ant -Ġgl ow -ĠClar ke -ĠNorm an -Ġver ification -Ġpack et -ĠForm ula -Ġpl ag -es ville -Ġshout ing -Ġo v -ĠR EC -ĠB ub -Ġn inth -Ġener g -Ġvalid ity -Ġup s -j ack -Ġneighbor ing -ĠN ec -ew orks -ĠH ab -are z -Ġsp ine -Ġevent ual -ĠLe aders -ĠC arn -Ġprob ation -Ġrom ance -ms g -ĠMechan ical -ER Y -R ock -Ġpart isan -N ode -ass ets -min ent -Ġforeign ers -Ġtest ify -ĠUs ually -l ords -ĠG ren -ĠPow ell -BI L -Ġs r -Ġadd ict -Ġshell s -Ġs igh -ĠY ale -tern ity -Ġ7 50 -E U -ĠR ifle -Ġpat ron -em a -ĠB annon -an ity -Ġtrop ical -ĠV II -c ross -Every thing -ĠIS O -Ġhum ble -ass ing -ĠF IG -Ġupd ating -ys on -Ġcal cium -Ġcompet ent -Ġste ering -Pro t -ĠS Y -ĠFin als -ĠR ug -15 9 -13 7 -ĠG olf -Ġ12 6 -Ġaccommod ation -ĠHug hes -Ġaest hetic -art isan -ĠTw ilight -Ġpr ince -ĠAgric ulture -ĠDis co -Ġpreced ent -Ġtyp ing -author ized -O ption -ĠA ub -l ishes -ach t -m ag -P eter -ĠU FO -mont on -ĠL ith -Ġa rom -Ġsec uring -Ġconf ined -priv ate -Ġsw ords -Ġmark ers -Ġmetab olic -se lect -ĠCur se -ĠO t -g ressive -Ġinc umb -ĠS aga -Ġpr iced -Ġclear ance -Cont ent -Ġdr illing -Ġnot ices -Ġb ourgeois -Ġv est -Ġcook ie -ĠGuard ians -ry s -in yl -Ġ12 4 -Ġpl ausible -on gh -ĠOd in -Ġconcept ion -ĠY uk -ĠBaghd ad -ĠFl ag -Aust ral -ĠI BM -Ġintern ationally -ĠWiki Leaks -I ED -Ġc yn -Ġcho oses -ĠP ill -Ġcomb ining -Ġrad i -ĠMoh ammed -def ense -atch ing -Sub ject -ic iency -Fr ame -Ġ{ " -Ġche ss -Ġtim er -19 0 -Ġt in -Ġord inance -emet ery -Ġacc using -Ġnotice able -Ġcent res -Ġl id -ĠM ills -img ur -Ġz oom -erg ic -Ġcomp ression -pr im -f ind -Ġsur g -Ġp and -ĠK ee -ĠCh ad -cell ence -oy le -Ġsocial ism -ĠT ravis -ĠM Hz -Ġgu ild -ALL Y -ĠSub scribe -ĠRel ated -Ġoccur rence -itch ing -Ġfict ional -Ġcr ush -ĠE A -c od -m ix -ĠTri ple -Ġretrie ve -Ġstimul us -Ġpsych iat -ĠDo or -Ġhomosexual ity -Ġelement ary -Ġcell ular -id ian -ĠL aun -Ġintrig uing -Ġfo am -ĠB ass -id i -its u -Ġass ure -Ġcongr at -Ġbusiness man -ĠBo ost -cl ose -Ġl ied -Ġsc iences -ĠO mega -ĠG raphics -Ġ< = -sp oken -Ġconnect ivity -S aturday -ĠAven gers -Ġto ggle -Ġank le -Ġnational ist -mod el -ĠP ool -ophob ia -V ar -ĠM ons -ator ies -Ġaggress ively -C lear -For ge -act ers -Ġhed ge -Ġpip es -Ġbl unt -Ġs q -Ġremote ly -W ed -as ers -Ġref riger -Ġt iles -Ġresc ued -Ġcompr ised -ins ky -Ġman if -avan augh -Ġprol ifer -Ġal igned -x ml -Ġtri v -Ġcoord ination -ĠP ER -ĠQu ote -13 4 -b f -ĠS aw -Ġtermin ation -Ġ19 0 -Ġadd itions -Ġtri o -Ġproject ions -Ġpositive ly -Ġin clusive -Ġmem br -19 90 -old er -Ġpract iced -ink le -Ar ch -Ġstar ters -ari us -Ġinter mediate -ĠBen ef -ĠK iller -Ġinter ventions -ĠK il -ĠF lying -In v -Ġprem ature -Ġpsych iatric -Ġind ie -Ġcoll ar -ĠRain bow -af i -Ġdis ruption -ĠFO X -cast ing -Ġmis dem -c ro -Ġw ipe -ard on -Ġb ast -ĠTom my -ĠRepresent ative -Ġbell y -ĠP O -ĠBre itbart -13 2 -Ġmess aging -Sh ould -Ref erences -ĠG RE -ist ical -L P -ĠC av -ĠC razy -Ġintu itive -ke eping -ĠM oss -Ġdiscont in -ĠMod ule -Ġun related -ĠPract ice -ĠTrans port -Ġstatist ically -orn s -Ġs ized -p u -Ġca f -ĠWorld s -ĠRod gers -ĠL un -ĠCom ic -l iving -Ġc ared -Ġclim bed -) { -Ġconsist ed -Ġmed ieval -fol k -Ġh acked -Ġd ire -ĠHerm ione -Ġt ended -ce ans -D aniel -w ent -Ġlegisl ators -Ġred es -g ames -Ġg n -am iliar -Ġ+ + -gg y -th reat -Ġmag net -Ġper ceive -Ġz ip -Ġindict ment -Ġcrit ique -g ard -ĠSaf e -ĠC ream -Ġad vent -ob a -Ġv owed -ous ands -Ġsk i -Ġabort ions -u art -Ġstun ned -Ġadv ancing -Ġlack ed -Ġ\ " -Ġsch izophren -Ġeleg ant -Ġconf erences -Ġcance led -ĠHud son -ĠHop efully -Ġtr ump -Ġfrequ encies -Ġmet eor -ĠJun ior -ĠFle et -ĠMal colm -ĠT ools -Ġ ........ -Ġh obby -ĠEurope ans -Ġ15 00 -ĠInt o -Ġs way -ĠApp ro -ĠCom pl -Comm unity -Ġt ide -ĠSum mit -ä » -Ġinter vals -ĠE ther -Ġhabit at -ĠSteven s -lish ing -ĠDom ain -Ġtrig gers -Ġch asing -Ġchar m -ĠFl ower -it ored -Ġbless ing -Ġtext ures -F ive -Ġliqu or -R P -F IN -Ġ19 62 -C AR -Un known -Ġres il -ĠL ily -Ġabund ance -Ġpredict able -r ar -Ġbull shit -le en -che t -M or -M uch -ä ¹ -Ġemphas ized -Ġcr ust -Ġprim itive -Ġenjoy able -ĠPict ures -Ġteam mate -pl er -ĠT ol -ĠK ane -Ġsummon ed -th y -ram a -ĠH onda -Ġreal izing -Ġquick er -Ġconcent rate -cle ar -Ġ2 10 -ĠErd ogan -ar is -Ġrespond s -ĠB I -Ġelig ibility -Ġpus hes -ĠId aho -Ġagg rav -Ġru ins -ur ations -Ġb ans -Ġan at -sh are -Ġgr ind -h in -um en -Ġut ilities -ĠYan kees -Ġdat abases -ĠD D -Ġdispl aced -Ġdepend encies -Ġstim ulation -h un -h ouses -ĠP retty -ĠRaven s -ĠTOD AY -Ġassoci ates -Ġthe rape -cl ed -Ġde er -Ġrep airs -rent ice -Ġrecept ors -Ġrem ed -ĠC e -Ġmar riages -Ġball ots -ĠSold ier -Ġhilar ious -op l -13 8 -Ġinherent ly -Ġignor ant -Ġb ounce -ĠE aster -REL ATED -ĠCur rency -E V -ãĥ ŀ -ĠLe ad -Ġdece ased -B rien -ĠMus k -J S -Ġmer ge -heart ed -c reat -m itt -m und -ĠâĢ ĭ -ĠB ag -Ġproject ion -Ġj ava -ĠStand ards -ĠLeon ard -Ġcoc onut -ĠPop ulation -Ġtra ject -Ġimp ly -Ġcur iosity -ĠD B -ĠF resh -ĠP or -Ġheav ier -ne ys -gom ery -Ġdes erved -Ġphr ases -ĠG C -Ġye ast -d esc -De ath -Ġreb oot -Ġmet adata -IC AL -Ġrep ay -ĠInd ependence -Ġsubur ban -ical s -Ġat op -Ġall ocation -gener ation -ĠG ram -Ġmoist ure -Ġp ine -ĠLiber als -Ġa ides -Ġund erest -ĠBer ry -Ġcere mon -3 70 -ast rous -ĠPir ates -Ġt ense -ĠIndust ries -ĠApp eals -ĠN ear -Ġè£ı ç -Ġlo vers -ĠC AP -ĠC raw -Ġg iants -Ġeffic acy -E lement -ĠBeh avior -ĠToy ota -Ġint est -P riv -A I -Ġmaneu ver -Ġperfect ion -Ġb ang -p aper -r ill -Ge orge -b order -in ters -ĠS eth -Ġcl ues -ĠLe vi -ĠRe venue -14 7 -Ġv apor -Ġfortun ate -Ġthreat ens -Ġve t -Ġdepend ency -ers ed -art icle -ĠBl izzard -Ġch lor -Ġmin us -ĠB ills -Ġcryptoc urrency -Ġmetabol ism -ter ing -Ġp estic -step s -ĠTre asure -ract ed -ĠConst ant -Ġtem p -13 9 -ĠDet ective -ur ally -Ġrecover ing -Ġcort ex -Ġ14 4 -cl osed -Ġprejud ice -aun ted -Ġstorm s -ĠN OW -Ġmach inery -Add ress -Ġcompe lled -27 0 -Ġdesp air -b ane -Ġveget able -Ġbed s -Lear n -Ġcolor ful -Ġsp ike -Ġmarg ins -Ġsymp athy -Ġworks hop -ĠC BC -S at -Ġburn s -ĠG ender -Ġ12 9 -ĠC able -Ġdeb ts -ĠThe resa -Ġreflect ing -Ġa irst -Ġr im -ram id -Ġweakness es -W rit -ogg le -t i -ĠCh arge -Ġwe ighed -Ġ( . -Ġl aughter -Ġrou ter -ĠDemocr acy -D ear -Ġhas ht -Ġd y -Ġhint s -run ning -Ġfin ishes -ar us -M ass -res ult -asc us -Ġv intage -Ġcon qu -Ġwild ly -ac ist -Ġl ingu -Ġprot agonist -st rom -te enth -ĠSol o -m ac -f illed -Ġre nown -it ives -Ġmot ive -ĠAnt ar -ĠM ann -ĠAd just -Ġrock ets -Ġtrou bling -e i -Ġorgan isms -ass is -Christ ian -Ġ14 5 -ĠH ass -Ġsw all -Ġw ax -ĠSurv ival -V S -ĠM urd -v d -stand ard -Ġdrag ons -Ġacceler ation -r ational -f inal -Ġp aired -ĠE thereum -Ġinterf aces -Ġres ent -Ġartif acts -Å « -are l -Ġcompet itor -ĠNich olas -ĠSur face -c pp -ĠT ot -Ġeconom ically -Ġorgan ised -Ġen forced -in ho -Ġvar ieties -Ġab dom -ĠBa iley -id av -ĠSal v -p aid -Ġalt itude -ess ert -ĠG utenberg -are a -op oulos -Ġprofess ors -igg s -ĠF ate -he y -Ġ3 000 -D ist -Ġtw ins -c ill -ĠM aps -Ġtra ps -Ġwe ed -ĠK iss -Ġy oga -Ġrecip ients -ĠWest minster -Ġpool s -ĠWal mart -18 8 -ĠSchool s -att ack -ĠAR M -par agraph -W arning -j l -Ġself ish -anche z -ĠHe ights -F re -ĠS oph -Ġ -------------------------------- -t ml -33 3 -Ġraid s -Ġsatell ites -KE Y -Ġlast s -Ñ Ĥ -In s -ĠD ame -Ġunp redict -// / -gh ai -Ġart illery -Ġcru ise -Ġg el -ĠCabin et -Ġbl ows -ĠE sp -Ġprox imity -ot he -ĠSk ills -ĠU pper -ob o -ĠN DP -Ġenjoy s -Ġrepe ating -ĠConst ruction -ĠQuest ions -H illary -Ġu int -Ġprocess ors -ĠGib son -ĠMult iple -q a -ĠB om -ĠM iles -vent ional -Ġhur ts -s kin -ĠA IDS -Ġadvis ers -ĠR oot -Ġmethod ology -ĠD ale -Ġdet on -ĠKnow ledge -sequ ently -Ġ12 1 -Ġconnect s -C y -ĠD anger -Ġcontribut ors -ĠB ent -Ġbr ass -ĠGun s -int o -ĠFort une -Ġbro ker -bal ance -Ġlength s -Ġv ic -Ġaver aging -Ġappropri ately -ĠCamer a -Ġsand wich -ĠCD C -Ġcoord inate -Ġnav ig -Ġgood ness -l aim -Ġbra ke -Ġextrem ist -ĠW ake -ĠM end -ĠT iny -ĠC OL -ĠR F -ĠD ual -ĠW ine -C ase -Ġref ined -Ġl amp -L ead -Ġb apt -ĠCar b -ĠS add -ĠMin neapolis -PD F -Ear ly -ĠH idden -I ts -ĠT IME -Ġp ap -Ġcommission ed -ĠF ew -ĠCol ts -ĠB ren -Ġbot hered -Ġlike wise -Ex per -ĠSch w -c ry -n n -ĠM itch -im on -M G -b m -UM P -r ays -Ġregist ry -Ġ2 70 -ach ine -re lla -ant ing -00 000 -Ġru ined -sp ot -Ġt a -Ġmaxim ize -Ġincon ven -D ead -H uman -En abled -ĠMar ie -Ġch ill -ĠParad ise -Ġstar ring -ĠLat ino -ĠProt ocol -ĠE VER -Ġsuppl iers -m essage -ĠBro ck -Ġser um -âĸĪâĸĪ âĸĪâĸĪ -Ġen comp -Ġamb ition -ues e -Ġar rows -And rew -Ġanten na -Ġ19 61 -ĠB ark -Ġb ool -ãĤ ª -ĠSt orage -Ġrail way -Ġtoug her -ĠC ad -Ġwas hing -P y -' ] -em bed -ĠMem phis -ack le -Ġfam ously -ĠF ortunately -ov ies -Ġmind set -Ġsne ak -ĠD h -RA W -ĠSim pson -Ġliv est -Ġland mark -Ġc ement -L ow -Ġthr illed -ĠCour se -in el -Ġch uck -id ate -gl obal -Ġwh it -Ġ � -ad ays -s ki -ĠS V -Ġvir uses -30 6 -ĠResp ons -Ġthe aters -ĠBr anch -ĠGene va -ĠM K -Ġunbel iev -Ġcommun ist -Orig inal -ĠRe ceived -ĠTrans fer -ĠAr g -In put -ĠStr ategy -Ġpal ace -the ning -D ri -Ġsent encing -umbn ail -Ġp ins -re cy -Ġs iblings -Get ting -ĠB U -ĠNorth west -Ġprolong ed -ĠSak ura -C omb -ĠB our -Ġinadequ ate -ĠK ash -Ġus ername -ĠImpro ve -Ġbatt ling -ĠM AC -Ġcurric ulum -Ġs oda -ĠC annon -Ġsens ible -sp ons -De cember -Ġw icked -ĠP engu -Ġdict ators -ĠHe arts -og yn -Ġsimilar ities -ĠSt ats -Ġh ollow -it ations -": [ -Ġh over -ĠList en -s ch -S und -Ġc ad -ĠPar ks -Ġl ur -Ġhy pe -ĠL em -N AME -is ure -Fr iday -Ġshoot s -Ġclos es -Ġd b -ĠR idge -ĠDiff erent -Ġrepl ies -ĠBroad way -op ers -Ġint oler -ĠZe us -akes pe -Ġpropri etary -Ġrequest ing -Ġcontro llers -ĠM IN -im edia -be cca -Ġexp ans -Ġoil s -B ot -ĠCh and -Ġpr inter -Ġto pped -ĠP OL -ĠEar lier -S ocial -av in -Ġdecre ases -ĠSe b -Ġspecific ations -ĠBl ast -ĠK urt -Ġfre el -B rown -Ġdil ig -ro e -ĠPro blem -ĠQu ad -Ġdecent ral -ĠV ector -an ut -Ġplug ins -ĠGreg ory -Ġfuck ed -el ines -ĠAmb assador -t ake -Ġcle ans -ong yang -An onymous -st ro -" } -al ine -ĠO dd -ĠE ug -2 16 -Ġbo il -ĠP owers -Ġnurs es -Ob viously -ĠTechn ical -Ġexceed ed -OR S -Ġextrem ists -Ġtr aces -ex pl -Ġcom r -ĠS ach -) / -Ġm asks -Ġsc i -B on -Ġreg ression -we gian -Ġadvis or -it ures -ĠV o -ex ample -ĠInst ruct -Ġs iege -Ġredu ctions -pt r -Ġstat utory -Ġrem oves -Ġp uck -red its -Ġbe e -Ġsal ad -Ġpromot ions -ĠJosh ua -with standing -ET H -ĠCh a -im us -Ġexpend iture -aun ting -Ġdelight ed -Ġ15 5 -be h -Ġcar pet -ĠSp art -Ġj ungle -l ists -Ġbull ying -ĠNob el -ĠGl en -Ġreferen ced -Ġintrodu ces -se in -Ġcho pped -gl ass -ĠW rest -Ġneutral ity -Ġâ Ļ -Ġinvestig ator -Ġshel ves -Ġun constitutional -Ġreprodu ction -Ġmer chant -m ia -Ġmet rics -Ġexplos ives -ĠSon ia -Ġbod ily -Ġthick ness -Ġpredomin antly -ĠAb ility -Ġmon itored -IC H -Ġ] . -ĠMart inez -Ġvis ibility -Ġqu eries -Ġgen ocide -ĠWar fare -Qu ery -Ġstud ios -Ġemb ry -Ġcorrid or -Ġclean ed -com plete -ĠM H -Ġenroll ment -ING S -Ġimpact ed -Ġdis astrous -ĠY un -ĠCl aire -ĠBas ically -y t -uster ity -Ġindirect ly -w ik -Ġd od -ĠCar r -Ġam p -Ġprohib it -ĠIn itial -ĠR d -ij i -Ġeduc ate -c orn -i ott -ĠBeaut y -Ġdetect ive -ĠCon n -s ince -Ġst agger -Ġob ese -Ġb ree -olog ic -is se -walk er -Ġbl ades -Ġlaw ful -fun c -ĠBeh ind -Ġappet ite -Ġ( * -Ġt ennis -Ġoff spring -Ġj ets -Ġstruct ured -Ġafore mentioned -N ov -Ġsc aling -f ill -Ġst ew -Ġcur b -ĠStep han -ed In -S F -ob ic -é ŃĶ -ou g -ĠM M -Ġgen etically -ope z -13 6 -Ġu mb -anc ers -Ġcoh ort -Ġmerch andise -Ġimp osing -ĠLegisl ature -ĠArch ive -iv ia -ĠN aval -Ġoff ences -Ġmir acle -Ġsn apped -Ġf oes -Ġextensive ly -ĠR af -Ġc ater -ed ience -K it -ĠB in -Ġrecomm ends -ĠC ities -Ġrig id -ĠRE AD -ĠNob le -ĠT ian -Ġcertific ates -ant is -o iler -ĠBudd hist -d id -Ġsurvey ed -Ġdown ward -Ġprint s -ĠMot ion -ron ics -ĠS ans -oss ibly -u ctions -Ġcolon ies -ĠDan ish -un it -Ġsp oil -Ġadvis ory -ber ries -Pl an -Ġspecific ation -op hers -ĠRes ource -Ġsh irts -prising ly -commun ications -Ġtriv ial -Ġmention ing -ise xual -Ġsupp lements -Ġsuper vision -B P -v or -Ġw it -Ġco oldown -Ġplaint iff -ĠReview s -ĠS ri -ĠM int -ĠSug ar -Ġafter ward -ĠPri est -ĠInvest ment -og ene -ĠT aking -Ġstretch ing -Ġinflamm ation -ĠTe hran -Ġl ining -Ġfree zing -ĠEnt ity -Ġins piring -spe cial -pr ice -Ġsu e -ĠP orter -oun ge -ET A -ĠD erek -ĠLu is -u o -ym ph -Ġex terior -ih il -ĠAsh ley -in ator -Ġnut rients -ĠTh rones -Ġfin ances -ĠIn spect -Ġspe cially -ĠRequ ired -ĠP TS -ĠViol ence -oint ed -sh ots -Ġex cerpt -co on -IN S -ĠG ri -Ġrecogn ised -We ek -You ng -Ġv om -is le -ĠCur ry -ĠBudd h -Ġnot ebook -Ġd urable -/ ? -ĠG ad -ĠP upp -Ġforg ive -p ark -Ġpersonal ities -an alysis -cl amation -Ġelev ator -Ġware house -ĠR ole -un n -Ġillust ration -ĠSc an -Ġatmosp heric -Im port -AN C -rict ed -f u -01 0 -Ġar che -Ġreward ed -akespe are -Ġintern ally -ĠR BI -alk er -Ġeleph ant -ow itz -ĠP izza -Ġbip artisan -é s -Ġslow ed -ĠSt ark -Ġover ride -OU S -Ġ3 20 -undred s -ĠDe ck -ĠC ensus -be e -14 6 -ot or -Ġ ip -Ġu b -oc ations -ĠBut ton -r ice -Ġc ripp -ff f -Ġorig inated -Ġoverwhel med -app a -Ġfore most -âĢ ij -ĠL EG -re lease -eat ured -at ches -Ġre ps -Ġl ending -ĠRe ference -ĠCl ient -16 5 -vent h -Com plete -ĠPat rol -Ġsw orn -c am -Ġshut tle -ĠR alph -Ġh ometown -- , -on al -ĠB P -å ı -Ġpersu ade -ĠAlex and -Ġcomb ines -Ġv ivid -ĠL ag -Ġenc oding -Ġsal vation -w en -ĠRec overy -i ya -Un iversity -ĠB iden -Ġbud gets -ĠTex ans -f its -Ġhon ored -Ġp ython -T D -## # -cl one -Ġbl ink -ĠL iquid -Ġunemploy ed -Ġcl ashes -ĠCoun sel -Ġdirect ing -Ġpun ct -ĠFal cons -Ġsh ark -ĠDam ascus -Ġje ans -Ġemb ark -Ġse ize -Ġup wards -2 80 -ĠE z -ĠAny thing -Ġex otic -l ower -ĠCreat or -ĠU m -Ġsubur bs -ber ger -ĠW end -Ġm int -ĠX X -ĠD ro -Ġsuff ers -Ġher b -t ree -Ġfrag ile -Ġflood ed -ĠAl cohol -ole an -ny der -ĠK O -F ram -Ġ13 6 -Ġow ed -ĠMe lee -ĠH ash -Ġwh isk -Ġsu do -r r -Qu ick -app ro -Ġi i -ĠEx amples -he e -Ġpromot es -per ature -k ar -ĠHon or -Ġs odium -ĠL if -ros so -intend ent -Ġcorrespond ent -F ound -sec ret -Ġident ifies -ag ne -Ġl ou -ĠP P -Ġcoinc idence -m ove -Ġmilit ia -Ġinf iltr -ĠPrim ary -Ġpitch ing -ĠI b -ĠGO OD -ãĤ ¸ -ĠW izards -ir al -ĠVen us -R R -ĠâĢ ķ -ĠCase y -Ġsad ly -Ġadm ire -Ġembarrass ed -c b -M el -Ġtub es -Ġbeaut ifully -ĠQueens land -Bel ow -re z -qu et -ple asant -Ġ « -C amp -Ġdec isive -19 98 -ĠL amb -ut ton -h n -ĠJ agu -au nder -ĠC ord -Ġcl erk -Ġca ffe -Ġwip ed -Ġre im -ĠMount ains -Ġimprison ed -Ġdevelop s -ĠP ra -Ġmodel ing -Any one -ance l -ĠS it -Ġshield s -Ġl awn -Ġcard iovascular -Ġdemonstr ating -Ġpar se -ĠIsrael is -Ġeuro s -14 3 -Ġgl orious -ins ki -ec d -Ġcondition ing -Ġhel pless -Ġmicro sc -ĠHar bor -Ġst akes -Ġ2 60 -Ġun equ -ĠFl oyd -Ġd amp -Ġappar atus -ĠLaw s -Ġcoun ters -Ġindu ce -at able -ĠAh med -Ġsl am -N ovember -Ġpers ist -Ġim minent -á n -Ġsh red -Ġph ases -ĠEd monton -ĠArm strong -ĠMe et -ĠK itty -Ñ Ģ -c irc -ĠAd ult -Ġa rose -ĠX en -D an -g ow -Ġsuper f -ĠAd mir -Ġend ure -Ġkey word -yr us -Ġy arn -Ġpath way -ĠHop kins -mid t -Ġcens orship -d ependent -Ġinstruct or -S ources -Ġto e -Ġball oon -N ob -Ġsw ear -ĠCast ro -Ġgl oss -ĠK avanaugh -Ġremark ably -Ph otos -ĠN om -ĠS outheast -y ers -Ġvalid ation -Ġcann on -ĠVict ory -ĠPier re -Ġcaut ious -Aud io -Ġf etch -ĠG ift -ĠH yp -Ġrem edy -Z E -Ġsc ent -Ġbe ard -ĠR ut -- " -Ġpat ents -H y -Ġun just -Ġpot ato -Ġforth coming -Ġche f -ĠR ift -aff e -ĠR OM -ĠL aunch -Ġp ads -ĠNe o -Ġon set -Ġsquee ze -s afe -Ġpref ix -ĠT M -ĠN early -ĠClin ical -ĠM ental -ot iation -ĠUn ic -ant ry -ĠC ir -Ġep it -à ¦ -Ġextract ed -verse ly -ri ad -Ġstr ains -Ġto ps -Ġpo em -ĠRand y -ĠMap le -TH ER -up iter -ĠSS D -ļ é -Ġun con -per ing -Ġsle pt -in ers -Ġunder water -ĠEv idence -g one -20 5 -Ġhistor ians -Ġsynt hesis -Ġf rog -b asketball -Ġvibr ant -Ġsub ord -Ġ3 65 -ĠD ial -Ġcooper ate -HA HA -Ġgreet ed -15 8 -Ġj azz -Ġinto x -ĠWalk ing -Ġsuper visor -ĠF usion -ĠMer cedes -s end -H am -s d -n l -Ġtour s -ĠF IFA -Ġcul p -g d -30 4 -Ġple as -Ġillust rates -ĠColomb ia -Ġhighlight ing -ĠSum mary -Ġexp osing -ĠD ru -Ġir ony -r itional -ĠCar roll -ĠEll is -P ict -ĠR apt -Ġad apter -Ġun m -Ġcor pse -Ġceleb rities -D en -at um -ĠAp ocalypse -ĠW ag -lin ing -Ġhorm ones -R ub -ĠX i -ĠV aults -20 8 -alky rie -inos aur -Ġfeed s -v ity -Ġdefe ating -W ait -Ġemphas ize -ĠSteel ers -yr inth -le ys -ĠWhe never -Current ly -ĠCl ock -Ġcollect ively -any on -ĠJ P -Ġment ality -Ġdownload s -Ġsurround ings -ĠBarn es -Ġflags hip -Ġindic ators -Ġgra pp -Jan uary -ĠElement al -ĠAthen a -ib al -Ġs ights -Ġcap ita -ĠTreat y -Ġvo iced -ĠG az -let te -Ġy a -Ġexp ired -Leg end -H ot -n ature -Ġunst able -Ġ2 80 -à º -Com ment -AL E -Ġquest s -Ġhand ler -n is -Ġvers atile -Ġconce al -enge ance -ĠInter active -Ġobs essed -ĠDog s -Ġcr acked -S ound -s v -ĠD ylan -ro ads -f x -ĠCath olics -ĠH ag -Ġsl ammed -Ġgl owing -s ale -Ġtiss ues -ĠCh i -ne e -Ġc her -s ic -ur rection -Ġb acon -ul atory -) ." -Ġir regular -FOR M -ass ed -Ġintention al -Ġcompens ate -ĠSpe aking -ĠS ets -15 3 -Ġconvent ions -b ands -em ade -Ġe cc -ĠWin ston -ĠAssass in -ĠBelg ian -Ġdepend ence -Ġnic he -Ġb ark -ĠJ azz -Ġdisadvant age -Ġgas oline -Ġ16 5 -çļ Ħ -ess a -mod ule -ang ular -O Y -ĠTreat ment -it as -ol ation -ĠArn old -Ġfe ud -ĠN est -Ġthe atre -ew ater -Ġmin ors -olic y -ĠH aven -div ision -Ġtr unk -F ar -ĠP ull -Ġcapt uring -Ġ18 00 -ĠTe en -Ġex empl -Ġclin ics -ĠB urg -Ġsubst it -Ġpay load -ĠL av -ĠT roy -ĠW itness -Ġfrag ments -Ġpass words -Ġg ospel -ĠG in -Ġten ants -ol ith -S ix -Pre vious -ĠAg es -ĠDar win -Ġbl at -Ġem pathy -sm ith -b ag -ĠE cho -ĠC amb -ĠM add -ĠB oo -Ġred e -ĠBurn ing -Ġsmooth ly -ĠAd rian -ĠV ampire -ĠMon sters -ste am -Sty le -M a -re a -ĠD war -aly st -urs or -Ġelim ination -Ġcrypt o -ch t -ĠE ternal -âĢ¦ ] -ĠS orce -I ll -N ER -Ġu h -Con clusion -w age -Ġresp ir -Ġrem inis -het ical -Ġg y -Ġutil ized -ic idal -Ġ19 00 -Ġhun ters -ĠSw an -ĠRe act -Ġvis itor -ĠThanks giving -30 8 -Post s -Ġh ips -19 97 -om ers -Ġkn ocking -ĠVeh icle -Ġt il -Ġ13 8 -Ġm i -ĠInvest igation -ĠKen ya -Ġcas ino -Ġmot ives -Ġreg ain -re x -Ġweek ends -Ġstab bed -bor o -Ġexplo ited -ĠHA VE -ĠTe levision -c ock -Ġprepar ations -Ġende av -ĠRem ote -ĠM aker -ĠPro du -ĠEv an -Ġinform ational -ĠLouis ville -15 4 -ĠDream s -Ġpl ots -ĠRun ner -Ġhur ting -Ġacad emy -ĠMont gomery -n m -ĠL anc -ĠAl z -2 10 -el ong -Ġretail er -Ġar ising -Ġrebell ion -Ġbl onde -play ed -Ġinstrument al -C ross -Ġret ention -Ġtherape utic -Ġse as -Ġinfant ry -ĠCl int -Ġprompt ing -Ġbit ch -Ġst ems -ĠK ra -Ġthe sis -ĠB og -ru ed -Ġk ings -Ġcl ay -ific ent -ĠY ES -ĠTh ing -ĠCub s -vey ard -els h -in arily -ĠE y -ĠRoll ing -Ġev olving -Ind ia -Ġrecogn izes -Ġgrad uation -is ers -Ġfert ility -ĠMil an -Comm and -Ġbox ing -Ġ19 43 -Ġgl uten -ĠEm ir -Ġid ol -Ġcon ceived -ĠCre ation -Mer it -udd y -uss ions -ĠLie utenant -iet al -Ġunch anged -ĠSc ale -ĠCrime a -ball s -ator ial -Ġdepth s -Ġempir ical -Ġtrans m -Ġuns afe -miss ible -com fort -15 6 -Ġmechan ic -00 2 -l ins -Ġsm oked -P os -Ġslow ing -Ġl av -Tex as -Ġche ating -ĠMet ropolitan -eth yl -Ġdiscover ing -as se -Ġpen cil -ĠPy ongyang -Ġclos et -ĠShe et -ĠEnt ry -ou stic -Ġmy st -er ate -ari at -Ġminer als -Ġmusic ian -ĠP ul -ĠM az -24 9 -Ġper missions -Ġ iv -en ary -ick ers -ĠB ing -he a -en able -Ġgri ev -Ġassert ed -ĠColon el -Ġaff idav -w o -Ġse ated -ĠR ide -Ġpaint ings -ĠP ix -Ġ13 7 -ish i -umb ai -g otten -ĠEar l -Ġin ning -Ġc ensus -Ġtrave lled -ĠCons ult -18 5 -b ind -Ġsimpl icity -Ġoverlook ed -ĠHelp ful -Ġmon key -Ġoverwhelming ly -Bl ood -ĠFl int -ĠJ ama -ĠPres ent -ĠR age -ĠT A -pt ive -Ġturn out -w ald -ĠD olphins -ĠV PN -Ġon ion -Ġcraft ing -m ma -ĠMerc ury -Ġarr ange -Ġalert s -ĠO T -zb ollah -Ġg ases -ĠRichards on -s al -l ar -Ġfro st -Ġlower ing -Ġacc laim -Ġstart ups -ĠG ain -ess ment -Ġguard ian -äº º -ĠP ie -ĠL inks -Ġmer its -Ġaw ake -Ġparent al -Ġexceed s -Ġid le -ĠPil ot -Ġe Bay -ĠAc cept -ipe g -C am -ĠK ot -Ġtrad ers -olit ics -unk er -ĠP ale -os i -an mar -Ġ19 47 -ĠF ell -est ial -it ating -G F -ĠS r -if ted -Ġconnect or -ĠB one -ill es -2 60 -h ma -Ġoverl ap -ĠGit Hub -Ġclean er -ĠBapt ist -ĠW AS -Ġlung s -Ñ ģ -ĠB UT -Ġc ite -Ġpit ched -reat ment -Ġtro phies -ĠN u -38 6 -ĠPr ide -Ġattend ees -[ ] -17 9 -Ġspat ial -Ġpri zes -ĠRel igion -Ġshow case -ĠC ategory -vid ia -T arget -Pro perty -? , -Ġf usion -p ie -ĠU CLA -Ġsound track -Ġprin cess -ĠC aval -sh ould -Ġlim bs -Back ground -Ġlone ly -Ġc ores -ĠT ail -she et -Ġ13 2 -R a -ãĤ « -ĠB olt -Ġbook ed -Ġadmin ister -Ġequ als -w y -Ġobserv ing -ĠBar on -ĠAd obe -Ġv irgin -ĠSocial ist -M ove -gh azi -ĠLind a -2 12 -Ġbre wing -Ġmerch ants -bur se -Ġdiv or -Ġmet als -ĠN er -Ġsum s -ĠEn emy -Ġen vision -Ġgrant ing -ĠH oney -ĠSk yrim -Ġsoc io -gr aded -Ġselect ive -W ASHINGTON -Ġ19 48 -ĠSir ius -ĠG ross -act ivity -ĠI van -Ġfur ious -BS D -ĠPre vious -Ġrespons ive -Ġchar itable -Ġle aning -ĠP ew -Ġviol ates -\\\\ \\\\ -ĠCom ing -w ire -Ġpo et -Ġres olutions -comm and -ĠPortug uese -Ġnick name -Ġde af -Feb ruary -Ġrecogn ise -Ġentire ty -Ġseason al -pl aced -ĠTe legraph -Ġmicro phone -our ing -Ġgr ains -Ġgovern ed -Ġpost p -ĠW aters -in ement -Ġund ocumented -ĠCom cast -Ġf ox -Ġassault s -re on -man y -ĠJen kins -ĠAny way -Ġassess ments -Ġdown s -ĠM ouse -Ġsuper b -k t -ĠD ow -Ġtax ation -4 01 -Ġsm iles -Ġundert aken -Ġex h -Ġenthusi astic -Ġtw ent -Ġgovernment al -Ġautonom y -ĠTechn ologies -ĠCh ain -Ġpreval ent -f b -Ġnic otine -og ram -j ob -Ġawa iting -ĠMen u -Ġdep uties -k ov -ish ops -But ton -ĠShan ghai -Ġdies el -ĠD uck -R yan -ĠPC s -N F -j ury -ent e -Ġinacc urate -edd y -Wh atever -Ġshow c -ĠN ad -od us -et r -Ġplaint iffs -ĠW OR -ĠAss ange -Ġpriv at -Ġpremium s -Ġt am -UR L -Ġel ites -ĠR anger -otten ham -ĠH off -ĠAt hens -Ġdefin ite -Ġs ighed -Ġeven ly -2 11 -ĠAm ber -ak ia -Ġmail ing -Ġcr ashing -ĠConfeder ate -ru gged -W al -ĠDep ths -Ġjuven ile -Ġreact or -Introdu ction -ĠDel uxe -19 95 -ĠS anchez -ĠM ead -iv able -: - -ĠPlan ning -ĠT rap -qu in -ĠProt ect -ve red -In formation -Ġkid ney -inn amon -l as -Ġpolic ing -Ġtoler ate -ĠQ i -Ġbi ased -F ort -ĠK i -s ave -Ġprivile ged -Ġbe asts -ĠGl as -ĠC inem -Ġcome back -Sund ay -Ġext inction -h ops -Ġtrans mit -Ġdoub les -ĠFl at -16 7 -Ġdis puted -Ġinjust ice -f oo -V ict -role um -ĠJul ie -Con text -ĠR arity -iss ue -Comp onent -Ġcounsel ing -an ne -d ark -Ġobject ions -u ilt -Ġg ast -Ġpl ac -Ġun used -ãĥ ĩ -ĠT rial -ĠJ as -hed ral -ob b -Ġtempor al -ĠPR O -ĠN W -ĠAnn iversary -L arge -Ġther m -Ġd avid -Ġsystem ic -ĠSh ir -m ut -ĠNe pt -add ress -Ġscan ning -Ġunderstand able -Ġcan vas -C at -ĠZ oo -Ġang els -L O -ĠStat ement -ĠS ig -ov able -ĠA way -sh aring -ocr ats -st ated -Ġweigh ing -N or -w ild -B ey -Ġaston ishing -ĠReyn olds -Ġop ener -Ġtrain er -Ġsurg ical -p n -Ġadjust ing -whe el -Ġf rown -erv ative -Ġsusp end -With in -te in -Ġobst acle -Ġliber ties -ym es -Ġur anium -ans om -an ol -ub a -ĠL oss -Ġa rous -ĠHend erson -W ow -s pl -c ur -ĠÂ Ń -Ġtheir s -Dam age -Ġdownload ing -Ġdisc ern -ĠSt o -ĠFl a -Ġh ath -ĠA j -Ġun pleasant -Europe an -exp ensive -Ġscreens hot -ĠU V -Ġall ied -ĠPers ian -Ġmonop oly -Ġat om -ĠReds kins -"> < -Ġcan cell -Ġcinem a -13 1 -f air -ĠAlf red -Ġd uck -arg s -22 3 -ĠIS I -Ġsign aling -in ar -Ġlaugh s -Ġfor wards -Ġreck less -Ġlisten ers -at ivity -Ġvast ly -n ant -L ess -ĠHun ting -ĠScient ific -IT ED -Ġkn ight -ĠH TC -us a -t mp -Ġr ude -ĠLegend ary -Ġar ises -B ad -ĠCl aim -pe g -Ġreal ities -Th ink -Ġ ° -Ġro de -Ġstri ve -Ġan ecd -Ġshort s -Ġhypot hes -Ġcoord inated -ĠGand hi -ĠF PS -R ED -Ġsuscept ible -Ġshr ink -ĠCh art -Hel p -Ġ ion -de ep -rib es -ĠK ai -ĠCustom er -Sum mary -Ġc ough -w ife -Ġl end -Ġposition ing -Ġlot tery -ĠC anyon -Ġf ade -Ġbron ze -ĠKenn y -Ġbo asts -ĠEnh anced -rec ord -Ġemer gence -Ġa kin -ĠB ert -it ous -âĸ ij -Ġst ip -Ġexch anged -om ore -als h -Ġreserv oir -Ġstand point -W M -Ġiniti ate -Ġdec ay -Ġbrew ery -Ġter ribly -Ġmort al -lev ard -Ġrev is -N I -el o -Ġconf ess -ĠMS NBC -Ġsub missions -Cont roller -Ġ20 2 -ĠR uth -} ); -ĠAz ure -Ġ ." -20 6 -ĠMarket ing -Ġl aund -ien cies -Ġrenown ed -ĠT rou -ĠN GO -ble ms -Ġterr ified -Ġwar ns -Ġper t -Ġuns ure -4 80 -ale z -ult z -ĠOut side -Ġst yl -ĠUnder ground -Ġp anc -Ġd ictionary -Ġf oe -rim inal -ĠNor wegian -Ġj ailed -Ġm aternal -é e -ĠLu cy -c op -Ch o -Ġuns igned -ĠZe lda -ĠIns ider -ĠContin ued -Ġ13 3 -ĠNar uto -ĠMajor ity -16 9 -ĠW o -ãĤ ĵ -Ġpast or -Ġinform al -Ð ½ -an throp -jo in -ãģ Ĺ -it ational -N P -ĠWrit ing -f n -ĠB ever -19 5 -Ġy elling -Ġdr astically -Ġe ject -Ġne ut -Ġth rive -ĠFre qu -ou x -Ġpossess es -ĠSen ators -ĠD ES -ĠSh akespeare -ĠFran co -ĠL B -uch i -Ġinc arn -Ġfound ers -F unction -Ġbright ness -ĠB T -Ġwh ale -ĠThe ater -m ass -ĠD oll -S omething -Ġecho ed -ĠHe x -c rit -af ia -Ġgodd ess -Ġele ven -ĠPre view -ĠAur ora -Ġ4 01 -uls ive -ĠLog an -in burgh -ĠCent ers -ĠON LY -ĠA id -Ġparad ox -Ġh urd -ĠL C -D ue -c ourt -Ġoff ended -Ġeval uating -ĠMatthew s -Ġto mb -Ġpay roll -Ġextra ction -ĠH ands -if i -Ġsuper natural -ĠCOM M -] = -dog s -Ġ5 12 -ĠMe eting -Rich ard -ĠMax imum -Ġide als -Th ings -m and -ĠReg ardless -Ġhum ili -b uffer -L ittle -ĠD ani -ĠN ak -Ġliber ation -ĠA be -ĠO L -Ġstuff ed -ac a -ind a -raph ic -Ġmos qu -Ġcampaign ing -Ġoccup y -S qu -r ina -ĠW el -ĠV S -Ġphys ic -Ġp uls -r int -oad ed -ET F -ĠArch ives -Ġven ues -h ner -ĠTur bo -Ġl ust -Ġappeal ed -que z -il ib -ĠTim othy -Ġo mn -d ro -Ġobs ession -ĠSav age -19 96 -Gl obal -J es -2 14 -Ġsl iding -Ġdisapp ro -ĠMag ical -Ġvolunt arily -g b -ane y -Ġprop het -ĠRe in -ĠJul ia -ĠW orth -aur us -Ġb ounds -ie u -)) ) -Ġcro re -ĠCitiz en -S ky -Ġcolumn ist -Ġseek ers -ond o -IS A -ĠL ength -Ġnost alg -Ġnew com -Ġdet rim -ent ric -3 75 -ĠG E -Ġaut op -Ġacadem ics -App Data -ĠS hen -Ġid iot -ĠTrans it -Ġteasp oon -W il -K O -ĠCom edy -> , -Ġpop ulated -W D -Ġp igs -ĠO culus -Ġsymp athetic -Ġmar athon -19 8 -Ġseiz ure -s ided -Ġd op -irt ual -L and -ĠFl oor -osa urs -... ] -Ġl os -Ġsubsid iary -E Y -ĠPart s -ĠSt ef -ĠJud iciary -Ġ13 4 -Ġmir rors -Ġk et -t imes -Ġneuro log -Ġc av -ĠGu est -Ġtum or -sc ill -ĠLl oyd -E st -Ġcle arer -Ġstere otypes -Ġd ur -not hing -Red dit -Ġnegoti ated ----------------- -------- -23 5 -Ġfl own -ĠSe oul -ĠRes ident -ĠS CH -Ġdisappear ance -ĠV ince -g rown -Ġgrab s -r il -ĠInf inite -ĠTw enty -Ġpedest rian -Ġjer sey -ĠF ur -ĠInf inity -ĠEll iott -Ġment or -Ġmor ally -Ġob ey -sec ure -iff e -Ġantib iotics -ang led -ĠFre eman -ĠIntrodu ction -J un -Ġm arsh -ic ans -ĠEV ENTS -och ond -W all -icult y -Ġmisdem eanor -Ġl y -Th omas -ĠRes olution -Ġanim ations -ĠD ry -Ġinter course -ĠNew castle -ĠH og -ĠEqu ipment -17 7 -Ġterrit orial -Ġarch ives -20 3 -Fil ter -ĠMun ich -Ġcommand ed -ĠW and -Ġpit ches -ĠCro at -Ġrat ios -ĠM its -Ġaccum ulated -ĠSpecific ally -Ġgentle man -acer b -Ġp enn -Ġa ka -ĠF uk -Ġinterven e -ĠRef uge -ĠAlz heimer -Ġsuccess ion -oh an -d oes -L ord -Ġsepar at -Ġcorrespond ence -Ġsh iny -P rior -Ġs ulf -Ġmiser able -Ġded ication -( ). -Ġspecial ists -Ġdefect s -ĠC ult -ĠX ia -Ġje opard -ĠO re -Ab ility -Ġle ar -Ġamb itions -ĠB MI -ĠArab s -Ġ19 42 -Ġpres ervation -ific ate -Ġash amed -l oss -ĠRest aur -Ġrese mble -Ġen rich -ĠK N -ĠCl an -fl oat -Ġplay able -IT T -Ġharm ony -arr ison -ĠWe instein -w ere -Ġpoison ing -ĠCom put -ĠWord Press -m ajor -ĠVal ve -F an -ĠTh row -ĠRom ans -ĠDep ression -ad os -Ġtort ured -Ġbal ancing -bott om -Ġacqu iring -ĠMon te -ard i -Ġa ura -Ġ# # -ĠStand ing -ĠAtl as -C F -Ġintr ins -ĠBen ghazi -Ġcamp ing -Ġt apped -bl ade -st rous -ĠR abb -ĠW ritten -t ip -ĠNe igh -ster dam -ĠAll ow -ĠHe aling -ĠR hod -n um -Ġcaffe ine -ĠPer cent -Ġbo o -Ġapp les -30 5 -Ġwel coming -Ġappl aud -Ġa usterity - ± -ĠRe ality -ef e -å ® -Ġsu cks -Ġtab s -ĠPay Pal -Ġback pack -Ġgif ted -abul ary -ĠSc out -ir teen -Ġch in -Ġo mitted -Ġnegative ly -Ġaccess ing -ĠE arn -Ġambul ance -Ġhead phones -Ġ20 5 -ĠRef resh -p resident -ĠKit chen -ĠEnt ered -ĠS nyder -00 5 -om ical -Ġborrow ed -ĠN em -Ġav iation -Ġst all -rim ination -Ġuniform s -it ime -ĠSim mons -ener gy -ab lished -y y -qual ified -Ġrall ies -ĠSt uart -fl ight -Ġgang s -r ag -Ġv ault -lu x -ĠCom par -Ġdesign ation -20 9 -ĠJ os -d ollar -z ero -Ġwell s -30 3 -Ġconstitu ents -Ġhe ck -Ġc ows -Ġcommand ers -Ġdifferent ial -ĠC atherine -29 9 -Ġval ve -Ġbr ace -Ġperspect ives -c ert -f act -icular ly -ĠMc N -pl anes -Ġint ric -Ġpe as -ov an -Ġtoss ed -ret ch -ĠL opez -Ġunf amiliar -de ath -ĠA part -ĠCh ang -Ġrelie ved -rop he -Ġair ports -Ġfre ak -ut il -M ill -ĠCh in -ĠOw en -m ale -ĠBro ken -ĠWind s -ro b -r ising -Ġfire fighters -Ġauthor itarian -Ġ14 8 -Bit coin -ex ternal -Ġbrow sers -iche ver -or ian -Ġun b -Ġpo ke -ĠZ ot -M id -ĠPop ular -Ġco vert -Ġcont ributes -Ġ6 50 -Ġcont ention -G ate -Ġcons oles -Ġchrom os -ĠI X -Ġvis ually -ĠE isen -Ġjewel ry -Ġdeleg ation -Ġacceler ate -ĠR iley -Ġsl ope -Ġind oor -it ially -Ġhuge ly -Ġtun nels -Ġfin ed -Ġdirect ive -Ġfore head -ustom ed -Ġsk ate -Mus ic -g as -Ġrecogn izing -am bo -Ġover weight -ĠGr ade -Ù Ĭ -Ġsound ing -Ġlock ing -ĠR EM -St ore -Ġexc av -ĠLike wise -ĠL ights -Ġel bow -ĠSupp ly -w ic -Ġhands ome -19 94 -C oll -Ġadequ ately -ĠAssoci ate -Ġstri ps -Ġcrack down -Ġmar vel -ĠK un -Ġpass ages -@@ @@ -ĠT all -Ġthought ful -names e -Ġprost itution -bus iness -Ġball istic -person al -c ig -iz ational -R ound -ĠÂłĠÂł ĠÂłĠÂł -ĠCole man -Ġadm itting -ĠPl ug -Ġbit coins -ĠSu z -Ġfair ness -Ġsupp lier -Ġcatast rophic -ĠHel en -o qu -M arc -ĠArt icles -g ie -Ġend angered -Ġdest iny -ĠVol t -ol ia -ax is -Ġche at -Ġun ified -IC O -qu ote -30 2 -ĠS ed -Ġsupp ression -Ġanaly zing -Ġsqu at -Ġfig uring -Ġcoordin ates -Ġch unks -Ġ19 46 -Ġsub p -Ġw iki -ĠFor bes -ĠJ upiter -ĠE rik -im er -ĠCom mercial -\ ) -Ġlegitim acy -Ġd ental -ĠMe an -Ġdefic its -5 50 -Orig inally -ĠHor ror -Ġcontam ination -ll ah -Ġconf isc -ĠCl are -T B -ĠF ailed -an ed -Ġrul er -ĠCont roller -Ġfemin ists -F ix -g ay -20 7 -Ġr abbit -Th ird -ownt own -Ġgl ue -Ġvol atile -Ġsh ining -Ġf oll -Ġimp aired -Ġsup ers -æ Ī -Ġcl utch -ļé ĨĴ -Ġpro let -Ġ( ! -Ġy elled -ĠK iev -ĠEr n -ĠSh ock -K B -Ġsit uated -qu ery -ĠN as -Ġan nex -char acter -ĠHol iday -Ġautom ation -ĠJ ill -ĠRem astered -Ġl inem -Ġwild erness -ĠHor izon -ĠGu inea -A Z -Ġmain land -Ġsec recy -LE ASE -Ġp unk -ĠProv ince -( ), -Spe ed -Ġhand ing -ĠSeb ast -S ir -r ase -Ġj ournals -Ġcon gest -ĠT ut -ir rel -Ġschizophren ia -Ġmis ogyn -health y -I ron -Ġreact ed -- $ -25 2 -Ġpl ural -Ġpl um -Ġbarg ain -Ġground ed -f inder -Ġdis se -ĠL az -O OD -Ġat roc -F actory -Ġmin ions -Ġo ri -ĠB rave -ĠP RE -ĠMy anmar -ĠH od -Ġexped ition -Ġexpl ode -ĠCo ord -Ġext r -ĠB rief -ĠAD HD -Ġhard core -feed ing -Ġd ile -ĠF ruit -Ġvacc ination -ĠM ao -osp here -Ġcont ests -- | -Ġf ren -isp here -R om -ĠSh arp -ĠTre nd -Ġdis connect -âĢ¢ âĢ¢ -Ġper secution -Ear th -Ġhealth ier -38 4 -Ġc ob -ĠTr inity -OW S -AN N -Ġspecial ty -Ġg ru -Ġcooper ative -wh y -Start ing -ĠIss ues -st re -ens or -Ġ18 5 -Ad v -! ? -ĠRe vel -em ia -ĠH ulk -Ġcelebr ations -ĠS ou -ra ud -ĠKle in -Ġun real -con text -Ġpartners hips -Ġadop ting -t ical -Ġspl ash -ĠHe zbollah -c ategory -cycl op -xt on -ĠD ot -urd y -t z -Ġenvelop e -ĠN L -â ķ -Ġwhere in -Spe c -18 4 -Ġte lev -al iation -Ġmyth s -å ° -Ġrig orous -Ġcommun icating -Ġobser ver -Ġre he -ĠW ash -Ġapolog ized -ĠT in -Ġexpend itures -work ers -d ocument -Ġhes itate -ĠLen in -Ġunpredict able -Ġrenew al -cl er -ok ia -ĠCON T -Ġpost season -Tok ens -Ġex acerb -Ġbet ting -Ġ14 7 -Ġelev ation -W ood -ĠSol omon -19 4 -00 4 -out put -Ġredu nd -ĠM umbai -Ġp H -Ġreprodu ce -ĠD uration -MA X -Ġb og -C BS -ĠBal ance -ĠS gt -ĠRec ent -Ġc d -Ġpo pped -Ġincomp et -pro p -ay an -g uy -Pac ific -Ġty r -Ġ{ { -ĠMy stic -ĠD ana -Ġmast urb -Ġge ometry -à ¢ -ĠCor rect -Ġtraject ory -Ġdistract ed -Ġf oo -ĠW elsh -L uc -m ith -Ġrug by -Ġrespir atory -Ġtri angle -Ġ2 15 -Ġunder graduate -ĠSuper ior -ch anging -_ - -Ġright ly -Ġrefere e -Ġluc rative -Ġun authorized -Ġresemb les -ĠGN U -ĠDer by -Ġpath ways -ĠL ed -Ġend urance -Ġst int -Ġcollect or -F ast -Ġd ots -Ġnational s -ĠSec urities -Ġwh ip -Par am -Ġlearn s -M agic -Ġdetail ing -m oon -Ġbroadcast ing -Ġb aked -26 5 -hol m -ĠS ah -ĠHus sein -ĠCourt esy -17 4 -Ġ14 6 -Ġge ographic -pe ace -Ġjud ging -ĠS tern -B ur -Ġstory line -G un -ĠSt ick -24 5 -30 7 -ãĤ´ ãĥ³ -ĠAdminist rator -Ġbur nt -Ġp ave -ch oes -Ex ec -Ġcamp uses -Res ult -Ġmut ations -ĠCh arter -Ġcapt ures -Ġcomp ares -Ġbad ge -S cient -Ġer ad -ier y -o i -ett es -ĠE state -Ġst rap -Ġproud ly -Ġf ried -Ġwithd rawn -ĠV oy -ph ony -It ems -ĠP ierce -b ard -Ġann otation -ant on -ill on -Im pro -... ) -Ġhapp ier ----- -- -ad just -Ġstaff ers -Ġactiv ism -Ġper f -Ġal right -N eed -Ġcomm ence -Ġopio id -ĠAm anda -E s -ĠP ars -ĠK aw -W orks -24 8 -Ġind o -t c -end ant -ĠM oto -Ġlegal ization -OT E -Ġtask ed -Ġt sp -ĠACT IONS -16 6 -Ġrefres hing -ĠN R -ĠPere z -Ġinfring ement -S Y -List en -in ning -k u -Ġrot ate -pro gram -ar ah -Des ign -Ġ( £ -Ġst oring -Ġwar rants -Ġjud gement -ĠB rist -us ually -ph oto -ĠR an -ĠP ine -Ġoutrage ous -ĠValent ine -lu ence -ĠEvery body -Al tern -Ġrele vance -Ġtermin ated -Ġd essert -Ġfulf illed -Ġprosecut ed -ĠW ords -Ġm igrant -Ġcultiv ation -ÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤ ÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤ -idel ity -ĠV ern -ĠLog in -Ġmetaph or -ĠT ip -Ġrecru its -ĠP ig -rib ing -Ġenthusi asts -ex per -Ġfright ening -ĠH air -ans on -str ate -Ġh i -He ight -Ġown ing -n one -Ġdis like -Ġkn ives -pher d -Ġloud ly -ĠAP Is -Dis play -ĠL ac -ĠUS S -ab l -ver ages -J ew -Ġ17 2 -ĠHist orical -at oon -ĠPhys ics -in tern -Ġwarm th -Ġto pp -D M -Ġgun man -Ġem peror -od i -ãĥ £ -in atory -ĠR ib -Ġ13 1 -ĠSat urn -ĠSh ining -Ġw aking -Qu otes -Ġcomed ian -en berg - ½ -Ġbelie vers -Ġpaper work -c ustom -Ġle v -Ġl ament -Ġpour ing -22 2 -p olitical -ĠSupp lement -m aid -Ġcruel ty -Ġt read -ys ics -A w -rit es -Ġmod ifier -ĠP osition -Ad am -l b -ub s -Ġimper fect -Ġcl usters -ĠEngine er -ĠC herry -Ġinaug uration -ĠS au -Ġembod iment -ĠUn cle -Ġover r -Ġexplos ions -c ule -ĠPrinc eton -ĠAndre a -Ġincorrect ly -Ġearn est -Ġpil gr -ĠS print -Ġslee ve -Ġhe ars -ĠAm azing -Ġbrow sing -ag in -Ġhom eland -Ġha w -Ġd iving -ist ered -17 8 -Ġbarg aining -ĠArc ade -Ġdeleg ate -ters on -................................ ................................ -ĠJackson ville -27 5 -Ġst agn -Ġad am -ĠSher man -C B -Ġsub urb -ĠFood s -Ġconver ting -ĠAr ist -Ġch ambers -l ove -Ġam ino -ĠG an -Ġmad ness -m c -ĠUS E -def ined -Ġul tr -ind ust -Ġw olves -l ance -Add itionally -Ġcr acks -as ia -ĠRe ason -ĠP ump -Ġaccident al -ĠL aser -ĠR id -Ġinitial ized -ell i -Ġun named -Ġn oun -ĠPass ed -Ġhost age -ĠEth iop -sh irts -Ġun rel -ĠEmb assy -Ġ19 41 -Ġat oms -Ġpur ported -16 4 -ĠF i -Ġgall ons -ĠMon ica -Ġp g -en ment -Ġsort ed -ĠG ospel -Ġhe ights -Ġtr aced -Ġunder going -She ll -Ġs acks -Ġproport ions -Ġhall uc -F ont -ac et -Ġwar mer -ĠIN TER -Ġgrab bing -Pl ug -Ġreal ization -ĠBur ke -Ġen chant -AT ER -ĠSe ed -Ġabund ant -F M -Ġc ivic -V s -is i -Ġv ow -Ġre per -ĠPartners hip -Ġpenet ration -Ġax e -Ġsh attered -ĠZ ombies -Ġv inyl -ĠAl ert -e on -Ġoblig ed -ĠIll ust -ĠPl aza -ĠFront ier -Ġdavid jl -ĠSer ial -ĠH av -ĠNut rition -B i -Ġâĸ Ī -ĠJ ays -lin ux -Ġhur ry -Ġv oy -Ġhop eless -ĠSte alth -Ġ ãģ -ess ors -tt le -b org -ĠSaf ari -f ell -Ġw ary -d ue -ĠAb ove -H a -E LL -Ġnot or -ĠW on -T oo -Ġoccup ations -Ġposs essions -Ġinv iting -Ġpred ators -Ġacceler ated -Ġ15 7 -uter te -ĠC ube -e ast -acc ount -G ive -Ġtrans plant -red ients -id able -Ġscreens hots -ĠG und -ĠF S -Ġtravel ers -Ġsens ory -ĠF iat -ĠRock ets -İ ĭ -_ { -F riend -Ġchar ming -AL S -Ġenjoy ment -m ph -Ġ5 000 -ĠRE G -Ù Ĩ -b ia -Ġcomp ilation -ro st -ĠV P -ĠSch ne -201 9 -Ġcop ying -M ORE -ĠFl ore -f alls -2 15 -t otal -Ġdis ciples -d ouble -Ġexceed ing -Ġsm ashed -Ġconcept ual -ĠRom ania -ĠB rent -ĠI CE -ĠT ou -Ġg rap -Ġn ails -18 9 -ãĥ ĺ -Ġproc ure -e ur -Ġconfir ming -ĠC ec -aw i -ĠEd en -Ġn g -Ġengine ered -at ics -Ġhook ed -Ġdisgust ing -ĠMur der -ãĤ ¿ -L ibrary -Ġ16 8 -Al most -hem atic -Men u -ĠNot re -ĠJ ur -Ġkidn apped -Ġhack er -ĠJ ade -Ġcreep y -Ġdraw ings -ĠSpons or -Ġcycl ists -ĠGob lin -Ġoptim ized -Ġst aged -ĠMc D -bet ween -A ge -en o -S ex -ĠW ide -n ings -av is -Ġincap able -ĠK ob -Ġreward ing -ĠL one -oles cent -Ġcontract ed -Ġstick y -J ose -B all -f est -ĠIn put -ĠRec ently -Ġto mat -squ are -App lication -Ġnit rogen -Ġdupl icate -ĠRec on -ĠD ear -L ondon -Ġint ra -Ġd ock -Ġout reach -ĠM illion -Ġmamm als -am pton -V AL -Ġsn aps -Ġd os -ĠWh ole -ĠRead y -T ry -ĠWinn ipeg -ear ance -Ġinc urred -ren ched -ĠNS W -il ot -rain e -Ġc ube -g ot -Ġrun way -etermin ed -ĠHaw ks -Ġsurviv or -ĠW ish -ĠD in -ĠDE F -ĠV ault -18 7 -Ġmush rooms -Ġcris p -be y -ĠDisco very -Ġdevelopment al -Ġparad igm -Ġcha otic -ĠT su -Ġ3 33 -b ons -Ġbacter ial -Ġcomm its -Ġcos mic -Ġme ga -oc ative -ĠP aint -ophob ic -Ġv ain -Ġcar ved -ĠTh ief -ĠG ul -ows hip -Ġc ites -ĠEd inburgh -Ġdimin ished -Ġacknowled ges -ĠK ills -Ġmic row -ĠHer a -Ġsen iors -Ġwhere by -H op -at ron -Ġun available -ĠN ate -Ġ4 80 -Ġsl ated -ĠRe becca -ĠB attery -Ġgram mar -Ġhead set -Ġcurs or -Ġex cluding -any e -aunder ing -eb in -Ġfeas ible -ĠPub lishing -ĠLab s -ĠCl iff -ĠFerr ari -Ġp ac -vis ible -mark ed -pe ll -Ġpol ite -Ġstagger ing -ĠGal actic -Ġsuper st -Ġpar an -ĠOffic ers -ãĢ ģ -Ġspecific s -ul us -23 9 -ĠP aste -AM P -ĠPan ama -ĠDe lete -angu ard -rest rial -Ġhero ic -ĠD y -ا ÙĦ -Ġincumb ent -Ġcr unch -t ro -Ġsc oop -Ġblog ger -Ġsell ers -ure n -Ġmedic ines -ĠC aps -ĠAnim ation -ox y -Ġout ward -Ġinqu iries -22 9 -Ġpsych ologist -ĠS ask -ev il -Ġcontam inated -ãĤ ¨ -he rence -Ġbrand ed -ĠAbd ul -z h -Ġparagraph s -Ġmin s -Ġcor related -er b -Ġimp art -Ġmil estone -ĠSol utions -ot le -Ġunder cover -Ġmar ched -ĠCharg ers -f ax -ĠSec rets -Ġr uth -we ather -Ġfemin ine -Ġsh am -Ġprest igious -igg ins -Ġs ung -hist ory -ett le -gg ie -Ġout dated -ol and -Ġper ceptions -ĠS ession -ĠDod gers -u j -ĠE ND -D oc -Ġdefic iency -Gr and -ĠJ oker -Ġretro spect -Ġdiagn ostic -Ġharm less -Ġro gue -ĠA val -E qu -Ġtrans c -ĠRoberts on -ĠDep ending -ĠBurn s -iv o -Ġhost ility -F eatures -ĵ ĺ -Ġdis comfort -ĠL CD -spec ified -ĠEx pect -3 40 -Ġimper ative -ĠReg ular -Ch inese -Ġstate wide -Ġsy mm -Ġlo ops -Ġaut umn -N ick -Ġsh aping -Ġqu ot -Ġc herry -ĠCross ref -è¦ ļéĨĴ -Stand ard -he ed -ĠD ell -ĠViet namese -Ġo st -ĠV alkyrie -O A -Ass ad -Ġreb ound -ĠTra ffic -pl aces -æ ĺ -ĠB uc -17 2 -Ġshel ters -Ġins isting -ĠCertain ly -ĠKenn eth -ĠT CP -Ġpen al -ĠRe play -he ard -Ġdial ect -iz a -ĠF Y -it cher -ĠD L -Ġspir al -Ġquarterback s -Ġh ull -Ġgo ogle -Ġto dd -ĠSter ling -ĠPl ate -Ġsp ying -mb ol -ĠReal m -ĠPro ced -ĠCr ash -Ġtermin ate -Ġprotest ing -C enter -gu ided -Ġun cover -Ġboy cott -Ġreal izes -s ound -Ġpret ending -ĠV as -19 80 -Ġfram ed -Ġ13 9 -Ġdesc ended -Ġrehab ilitation -Ġborrow ing -ĠB uch -Ġbl ur -R on -ĠFro zen -en za -Ch ief -ĠP oor -Ġtransl ates -M IN -Ġ2 12 -J ECT -Ġerupt ed -Ġsuccess es -S EC -Ġpl ague -Ġg ems -d oms -Ġstret ches -ĠSp y -Ġstory telling -C redit -ĠP ush -Ġtra ction -Ġin effective -ĠL una -Ġt apes -Ġanaly tics -erc ise -Ġprogram mes -ĠCar bon -Ġbeh old -he avy -ĠConserv ation -ĠF IR -Ġs ack -ter min -ric ks -Ġhous ed -Ġunus ually -I ce -Ġexecut ing -ĠMor oc -ed ay -Ġed itions -Ġsm arter -ĠB A -Ġout law -Ġvan ished -ib a -AL SE -ĠSil va -23 8 -C ould -Ġphilos opher -Ġevac uated -Sec ret -14 2 -Ġvis as -ãĤ ¬ -ĠM alt -ĠClear ly -ĠN iger -ĠC airo -ĠF ist -3 80 -ĠX ML -aut o -it ant -Ġrein forced -Rec ord -ĠSurviv or -G Hz -Ġscrew s -parent s -Ġo ceans -ma res -Ġbra kes -vas ive -Ġhell o -ĠS IM -rim p -Ġo re -ĠArm our -24 7 -Ġterr ific -Ġt ones -14 1 -ĠMin utes -Ep isode -Ġcur ves -Ġinflamm atory -Ġbat ting -ĠBeaut iful -L ay -Ġunp op -v able -Ġr iots -ĠTact ics -b augh -ĠC ock -Ġorg asm -ĠS as -Ġconstruct or -et z -G ov -Ġant agon -Ġthe at -Ġde eds -ha o -c uts -ĠMc Cl -Ġu m -ĠScient ists -Ġgrass roots -ys sey -"] => -Ġsurf aced -Ġsh ades -Ġneighb ours -Ġad vertis -oy a -Ġmer ged -Up on -Ġg ad -Ġanticip ate -Any way -Ġsl ogan -Ġdis respect -I ran -ĠT B -act ed -Ġsubp oen -medi ately -OO OO -Ġwa iver -Ġvulner abilities -ott esville -ĠHuff ington -J osh -ĠD H -M onday -ĠEll en -K now -x on -it ems -22 8 -Ġf ills -ĠN ike -Ġcum ulative -and als -I r -Ġ ì -Ġfr iction -ig ator -Ġsc ans -ĠVi enna -ld om -Ġperform ers -P rim -Ġb idding -M ur -Ġlean ed -ĠPri x -al ks -Ġ[ âĢ¦] -ĠTw itch -ĠDevelop er -ĠG ir -Ġcall back -Ab stract -Ġacc ustomed -Ġfreed oms -ĠP G -ur acy -Ġl ump -is man -,, ,, -19 92 -ĠR ED -Ġwor m -M atch -ĠPl atinum -I J -ĠOwn er -Tri via -com pl -Ġnew born -Ġfant as -O wn -Ġ19 59 -Ġsymp ath -Ġub iqu -Ġoutput s -Ġal lev -Ġpr ag -K evin -Ġfav ors -Ġbur ial -Ġn urt -so lete -c ache -Ġ15 6 -Ġunl ocks -te chn -M aking -Ġcon quer -ad ic -æ ĸ -Ġel f -Ġelect orate -ĠKurd s -ĠSt ack -ĠSam urai -Ġâ ĺħ -Ġ{ } -ĠS aid -ĠFall out -Ġkind ness -ĠCustom s -ĠBou levard -Ġhelicop ters -ot ics -ĠVe get -com ment -Ġcritic ised -Ġpol ished -ĠRem ix -ĠC ultural -Ġrec ons -Ġdo i -at em -Sc reen -Ġbar red -Com ments -ĠGener ally -Ġsl ap -7 20 -V ari -p ine -Ġem pt -Ġh ats -ĠPlay ing -l ab -a verage -form s -ĠC otton -Ġcan s -ĠD ON -ĠSom alia -C rypt -ĠIncre ases -E ver -mod ern -Ġsur geon -3 000 -Ġrandom ized -================================ ================================ -B ern -im pl -ĠC OR -Ġpro claim -th ouse -Ġto es -Ġam ple -Ġpres erving -Ġdis bel -gr and -B esides -Ġsil k -ĠPat tern -h m -Ġenter prises -Ġaffidav it -ĠAdvis ory -Ġadvert ised -ĠRel igious -se ctions -psy ch -ĠField s -aw ays -Ġhasht ag -ĠNight mare -Ġv ampire -Ġfore nsic -rosso ver -n ar -Ġn avy -Ġvac ant -ĠD uel -Ġhall way -Ġface book -ident ally -ĠN RA -Ġm att -Ġhur ricane -ĠKir by -ĠP uzzle -Ġsk irt -ou st -du llah -Ġanal ogy -in ion -Ġtomat oes -ĠN V -ĠPe ak -ĠMe yer -Ġappoint ments -Ġm asc -Ġal ley -re hend -Ġchar ities -Ġund o -Ġdest inations -ĠTest ing -"> " -c ats -* . -Ġgest ures -gener al -Le ague -Ġpack ets -ĠInspect or -ĠBer g -Ġfraud ulent -Ġcritic ize -F un -Ġbl aming -nd ra -Ġsl ash -ĠE ston -Ġpropos ing -Ġwh ales -Ġtherap ist -Ġsub set -Ġle isure -EL D -ĠC VE -ĠAct ivity -Ġcul min -sh op -ĠD AY -is cher -ĠAdmir al -ĠAtt acks -Ġ19 58 -Ġmem oir -Ġfold ed -Ġsex ist -Ġ15 3 -ĠL I -Ġread ings -Ġembarrass ment -ĠEmploy ment -w art -ch in -Ġcontin uation -l ia -Rec ently -Ġd uel -Ġevac uation -ĠKash mir -Ġdis position -ĠR ig -Ġbol ts -Ġins urers -4 67 -M ex -Ġret aliation -Ġmis ery -Ġunre asonable -r aining -I mm -ĠP U -em er -Ġgen ital -ãĤ ³ -ĠC andy -Ġon ions -ĠP att -lin er -Ġconced ed -Ġf a -Ġfor c -ĠH ernandez -ĠGe off -deb ian -ĠTe ams -Ġc ries -Ġhome owners -23 7 -A BC -Ġst itch -Ġstat istic -Ġhead ers -ĠBi ology -Ġmot ors -ĠG EN -ĠL ip -Ġh ates -Ġhe el -S elf -i pl -ED IT -ort ing -Ġann ot -ĠSpe ech -old emort -ĠJ avascript -ĠLe Bron -Ġfoot print -Ġf n -Ġseiz ures -n as -h ide -Ġ19 54 -ĠBe e -ĠDecl aration -ĠKat ie -Ġreserv ations -N R -f emale -Ġsatur ated -Ġb iblical -Ġtroll s -Dev ice -ph otos -Ġdr ums -ãĥīãĥ© ãĤ´ãĥ³ -N ight -f ighter -ĠH ak -ri ber -Ġc ush -Ġdiscipl inary -ba um -ĠG H -ĠSch midt -ilib rium -Ġs ixty -ĠKush ner -ro ts -Ġp und -ĠR ac -Ġspr ings -Ġcon ve -Bus iness -F all -Ġqual ifications -Ġvers es -Ġnarc iss -ĠK oh -ĠW ow -ĠCharl ottesville -ed o -Ġinterrog ation -ĠW ool -36 5 -B rian -Ġâľ ĵ -Ġalleg es -ond s -id ation -ĠJack ie -y u -Ġl akes -Ġworth while -Ġcryst als -ĠJud a -Ġcomp rehend -Ġfl ush -Ġabsor ption -ĠO C -Ġfright ened -ĠCh ocolate -Mart in -Ġbu ys -Ġbu cks -Ġapp ell -ĠChampions hips -Ġlist ener -ĠDef ensive -Ġc z -ud s -ĠM ate -Ġre play -Ġdecor ated -Ġs unk -ĠV IP -ĠAn k -Ġ19 5 -aa aa -Nob ody -ĠMil k -ĠG ur -ĠM k -ĠS ara -Ġse ating -ĠW id -Tr ack -Ġemploy s -Ġgig antic -AP P -ãĤ § -in ventory -Ġtow el -at che -l asting -ĠT L -Ġlat ency -Ġkn e -B er -me aning -Ġup held -Ġplay ground -Ġm ant -S ide -Ġstere o -Ġnorth west -Ġexception ally -Ġr ays -Ġrec urring -D rive -Ġup right -Ġab duct -ĠMar athon -Ġgood bye -Ġal phabet -h p -Ġcourt room -ring ton -ot hing -T ag -Ġdiplom ats -Ġbar bar -ĠAqu a -18 3 -33 33 -Ġmat urity -Ġinst ability -ĠAp ache -Ġ= == -Ġfast ing -ĠGr id -Mod Loader -Ġ15 2 -A bs -ĠOper ating -ett i -Ġacqu aint -Don nell -ĠK em -ĠFor ge -Ġarm ored -M il -Ġphilos ophers -in vest -Pl ayers -â Ī -Ġmy riad -Ġcomr ades -R ot -Ġremember ing -Ġcorrespond s -Ġprogram mers -ĠLyn n -Ġo lig -Ġco herent -yn chron -ĠChem ical -Ġj ugg -p air -post s -E ye -ĠIn ner -Ġsem ester -ott est -ĠEmir ates -ric anes -or ously -m its -ĠW is -Ġd odge -l ocation -Ġf aded -Am azon -ĠPro ceed -ĠIN FO -j ournal -ĠTru ck -T en -Ġ2 17 -Ġstat utes -m obile -ĠT ypes -Rec omm -b uster -pe x -Ġleg ends -Ġhead ache -f aced -ĠWi Fi -if ty -ĠH ER -Ġcirc uits -ER ROR -22 6 -ol in -Ġcyl inder -osp ace -ik ers -P rem -Qu ant -Ġconflic ting -Ġslight est -Ġfor ged -ion age -Step hen -ĠK ub -ĠOpp ortun -ĠHe al -Ġbl o -Ġrul ers -Ġh uh -Ġsubmar ine -f y -ass er -Ġallow ance -ĠKas ich -ĠT as -ĠAustral ians -Forge ModLoader -ĠâĨ ij -ĠMat rix -am ins -Ġ12 00 -ĠAc qu -23 6 -D ocument -ĠBre aking -19 3 -ĠSub st -ĠRoll er -ĠPro perties -ĠN I -t ier -Ġcr ushing -Ġadvoc ating -Further more -keep ers -Ġsex ism -x d -Ġcall er -ĠS ense -chie ve -ĠT F -Ġfuel ed -Ġreminis cent -Ġobs ess -ur st -Ġup hold -ĠF ans -het ics -Ġâ Ĺ -ĠB ath -Ġbe verage -Ġo scill -25 4 -Ġpol es -Ġgrad ual -Ġex ting -ĠS uff -ĠS uddenly -Ġlik ing -Ġ19 49 -un ciation -am ination -ĠO mar -ĠL V -ĠCon sequently -Ġsynt hes -ĠG IF -Ġp ains -Ġinteract ing -u ously -inc re -Ġrum or -ĠScient ology -19 7 -ĠZ ig -Ġspe lling -ĠA SS -Ġexting u -ms on -Ġg h -Ġremark ed -ĠStrateg ic -ĠM ON -å ¥ -g ae -ĠWH AT -E ric -ĠCamp us -Ġmeth ane -Ġimag in -J UST -ĠAl m -X T -i q -ĠR SS -Ġwrong doing -att a -Ġbig ot -Ġdemonstr ators -ĠCal vin -ĠV illa -Ġmembr ane -ĠAw esome -Ġbenef ic -26 8 -Ġmagn ificent -ĠL ots -G reg -ĠBor is -Ġdetain ees -ĠH erman -Ġwhis pered -Ġa we -Prof essor -fund ing -Ġphys iological -ĠDest ruction -Ġlim b -Ġmanip ulated -Ġbub bles -Ġpse ud -Ġhyd ra -ĠBrist ol -Ġst ellar -ĠExp ansion -ĠK ell -ĠInterest ingly -Ġm ans -Ġdrag ging -Ġec ological -ĠF it -Ġg ent -Ġbenef ited -ĠHait i -Ġpoly g -ãĥ İ -Ġ20 30 -Ġpro w -Ġrecon struction -Ġwas t -Ġpsych ic -ĠGree ks -Hand ler -16 2 -ĠP ulse -Ġsol icit -Ġsy s -Ġinflu x -ĠG entle -per cent -Ġprolifer ation -Ġtax able -Ġdisreg ard -Ġesc aping -Ġg inger -Ġwith stand -Ġdevast ated -ĠD ew -ser ies -Ġinject ed -ela ide -Ġturn over -he at -Ļ Ĥ -H appy -ĠSil ent -ãĤ Ń -iv ism -Ġir rational -AM A -Ġre ef -r ub -Ġ16 2 -Ġbank ers -ĠEth ics -v v -Ġcritic isms -K n -18 6 -M ovie -ĠT ories -Ġno od -Ġdist ortion -F alse -od ore -Ġt asty -Res earch -ĠU ID -- ) -Ġdivor ced -ĠM U -ĠHay es -ĠIs n -ian i -ĠH Q -Ġ" # -ign ant -Ġtra umatic -ĠL ing -H un -Ġsab ot -on line -r andom -Ġren amed -ra red -K A -d ead -é t -ĠAss istance -Ġse af -++++ ++++ -Ġse ldom -ĠWeb b -Ġbo olean -u let -Ġref rain -ĠDI Y -ru le -Ġshut ting -Ġutil izing -load ing -ĠPar am -co al -oot er -Ġattract ing -ĠD ol -Ġher s -ag netic -ĠRe ach -im o -Ġdisc arded -ĠP ip -01 5 -ü r -Ġm ug -Im agine -C OL -Ġcurs ed -ĠSh ows -ĠCurt is -ĠSach s -spe aking -ĠV ista -ĠFram ework -ong o -Ġsub reddit -Ġcr us -ĠO val -R ow -g rowing -Ġinstall ment -Ġgl ac -ĠAdv ance -EC K -ĠLGBT Q -LE Y -Ġac et -Ġsuccess ive -ĠNic ole -Ġ19 57 -Qu ote -Ġcircumst ance -ack ets -Ġ14 2 -ort ium -Ġguess ed -ĠFr ame -Ġperpet rators -ĠAv iation -ĠBen ch -Ġhand c -A p -Ġ19 56 -25 9 -r and -Net Message -d in -urt les -h ig -ĠV III -ff iti -ĠSw ords -b ial -Ġkidn apping -dev ice -Ġb arn -ĠEl i -auc as -S end -Con structed -Ġ ½ -Ġneed les -Ġad vertisements -Ġv ou -Ġexhib ited -ĠFort ress -As k -B erry -TY PE -Ġcan cers -ump ing -ĠTerrit ory -Ġpr ud -Ġn as -Ġathe ist -Ġbal ances -ãģ Ł -ĠSh awn -& & -Ġland sc -ĠR GB -Ġpet ty -Ġex cellence -Ġtransl ations -Ġpar cel -ĠChe v -E ast -ĠOut put -im i -Ġamb ient -ĠTh reat -Ġvill ains -Ġ5 50 -IC A -Ġtall er -Ġle aking -c up -Ġpol ish -Ġinfect ious -ĠK C -Ġ@ @ -back ground -Ġbureaucr acy -ĠS ai -un less -it ious -ĠSky pe -At l -ID ENT -00 8 -Ġhyp ocr -Ġpit chers -Ġguess ing -ĠF INAL -Bet ween -Ġvill agers -Ġ25 2 -f ashion -ĠTun is -Be h -ĠEx c -ĠM ID -28 8 -ĠHas kell -19 6 -ĠN OR -Ġspec s -Ġinv ari -Ġgl ut -ĠC ars -Ġimp ulse -Ġhon ors -g el -Ġjurisd ictions -ĠBund le -ul as -Calif ornia -ĠIncre ase -Ġp ear -Ġsing les -Ġc ues -Ġunder went -ĠW S -Ġexagger ated -Ġdub ious -Ġfl ashing -L OG -) ]. -J ournal -t g -V an -ĠI stanbul -ĠIn sp -ĠFrank en -D raw -Ġsad ness -Ġiron ic -ĠF ry -x c -Ġ16 4 -is ch -W ay -ĠProtest ant -h orn -Ġun aff -ĠV iv -ill as -ĠProduct ions -ĠH ogan -Ġper imeter -ĠS isters -Ġspont aneous -Ġdown side -Ġdescend ants -Ġor n -w orm -Japan ese -Ġ19 55 -Ġ15 1 -ĠDo ing -els en -umb les -Ġrad ically -ĠDr um -ĠB ach -Ġli abilities -ĠO B -ĠElement ary -Ġmem e -yn es -Ġfinger print -ĠGr ab -Ġundert ake -Mem bers -ĠRead er -ĠSim s -g od -Ġhypot hetical -s cient -ĠA J -Ġchar ism -Ġad missions -ĠMiss ile -tr ade -Ġexerc ising -ĠBack ground -W ritten -Ġvoc als -whe ther -Ġv i -ĠW inner -Ġl itter -ĠSh ooting -ST EM -ãĤ ¡ -ĠA FL -Ġvari ability -Ġe ats -ĠD PS -b row -Ġeleph ants -Ġstr at -Ġ Å -Ġsett lers -Matt hew -Ġin advert -H I -ĠIM F -ĠGo al -Ġnerv es -John son -ey e -ablish ment -Th ursday -BIL ITY -H ad -am oto -het amine -ep s -Ġmit ochond -Ġcomp ressed -ĠTre vor -ĠAnim als -T ool -L ock -Ġtwe ak -Ġpin ch -Ġcancell ation -P ot -Ġfoc al -ĠAst ron -17 3 -ĠA SC -ĠO THER -umn i -Ġdem ise -d l -Ù ħ -Sem itism -Ġcr acking -Ġcollabor ative -Ġexpl ores -s ql -Ġher bs -Ġconfig urations -m is -ĠRes ult -ace y -ĠSm oke -Ġsan ct -el ia -Ġdeg ener -Ġdeep est -Ġscream ed -Ġn ap -Soft ware -ĠST AR -E F -ĠX in -spons ored -mans hip -23 3 -Ġprim aries -Ġfilter ing -Ġas semble -m il -ĠMy ers -b ows -Ġpun ched -M ic -Ġinnov ations -Ġfun c -and o -Ġfr acking -ĠV ul -о Ð -osh op -ĠIm mun -Ġsett ling -Ġadolesc ents -Ġreb uilding -Ġtransform ing -Ġpar ole -Ġhar bor -Ġbook ing -ot ional -onge vity -ĠY o -b ug -Ġemer ges -ĠMethod s -ĠCh u -P res -ĠDun geons -Ġtra iling -ĠR um -ĠH ugh -å¤ © -ĠE ra -ĠBatt les -Res ults -ĠTr ading -Ġvers a -c ss -ax ies -he et -Ġgre ed -19 89 -Ġgard ens -Ġconting ent -P ark -ĠLeaf s -h ook -ro be -Ġdiplom acy -ĠF uel -ĠInv asion -Ġupgr ading -M ale -Ġe lic -Ġrelent less -ĠCo venant -ap esh -ĠT rop -T y -pro duction -art y -Ġpun ches -ak o -cyclop edia -ĠR abbit -ĠHD MI -Ġ14 1 -Ġf oil -Item Image -ĠF G -Ġimplement ations -ĠP om -ixt ures -Ġaw ait -Ġ3 30 -am us -Ġumb rella -Ġfore see -se par -Ġcircum cision -Ġperipher al -S ay -ĠExper t -In c -Ġwithd rew -ĠAnd ers -f ried -Ġradio active -ĠOp ening -Ġboard ing -ĠN D -Ġover throw -Act iv -W P -ĠAct s -× Ļ -Ġmot ions -v ic -ĠM ighty -ĠDef ender -a er -Ġthank ful -ĠK illing -ĠBr is -mo il -Ġpredict ing -26 6 -ch oice -Ġkill ers -Ġinc ub -ĠChe st -ather ing -Ġpro claimed -fl ower -oss om -umbled ore -ĠCy cling -ĠOccup y -AG ES -P en -ĠY ug -Ġpack aged -Ġheight ened -c ot -st ack -C ond -Ġst amps -m age -Ġpersu aded -Ġens l -ĠCard inal -Ġsol itary -Ġpossess ing -ĠC ork -Ġev id -ĠT ay -Ġbl ues -Ġextrem ism -Ġlun ar -Ġcl own -Te chn -Ġfest ivals -ĠPv P -ĠL ar -Ġconsequ ently -p resent -Ġsom eday -ç İĭ -ĠMet eor -Ġtour ing -c ulture -Ġbe aches -S hip -c ause -ĠFl ood -ãĥ ¯ -Ġpur ity -th ose -Ġem ission -b olt -Ġch ord -ĠScript ure -L u -Ġ$ { -cre ated -Other s -25 8 -Ġelement al -Ġannoy ed -ĠA E -d an -ĠS ag -Res earchers -Ġfair y -âĢĵ âĢĵ -======== ==== -Sm art -GG GG -Ġskelet ons -Ġpup ils -link ed -Ġur gency -en abled -ĠF uck -Ġcoun cill -r ab -U AL -T I -Ġlif es -Ġconf essed -B ug -Ġharm on -ĠCON FIG -ĠNe utral -D ouble -Ġst aple -ĠSH A -Brit ish -ĠSN P -AT OR -oc o -Ġswing ing -ge x -ole on -pl ain -ĠMiss ing -ĠTro phy -v ari -ran ch -Ġ3 01 -4 40 -00000000 00000000 -Ġrest oring -Ġha ul -uc ing -ner g -Ġfut ures -Ġstrateg ist -quest ion -Ġlater al -ĠB ard -Ġs or -ĠRhod es -ĠD owntown -????? - -ĠL it -ĠB ened -Ġco il -st reet -ĠPort al -FI LE -ĠG ru -* , -23 1 -ne um -Ġsuck ed -Ġr apper -Ġtend encies -ĠLaure n -cell aneous -26 7 -Ġbrow se -Ġover c -head er -o ise -Ġbe et -ĠG le -St ay -Ġm um -Ġtyp ed -Ġdiscount s -T alk -ĠO g -ex isting -ĠS ell -u ph -C I -ĠAust rian -ĠW arm -Ġdismiss al -Ġaver ages -c amera -Ġalleg iance -L AN -=" # -Ġcomment ators -ĠSet ting -ĠMid west -Ġpharm ac -ĠEX P -Ġstain less -Ch icago -Ġt an -24 4 -Ġcountry side -ĠV ac -29 5 -Ġpin ned -Ġcr ises -Ġstandard ized -T ask -ĠJ ail -ĠD ocker -col ored -f orth -" }, -Ġpat rons -Ġsp ice -Ġm ourn -ĠM ood -Ġlaund ry -Ġequ ip -ĠM ole -y ll -ĠTH C -n ation -ĠSher lock -Ġiss u -ĠK re -ĠAmeric as -ĠA AA -Ġsystem atically -Ġcont ra -ĠS ally -Ġrational e -Ġcar riage -Ġpe aks -Ġcontrad iction -ens ation -ĠFail ure -Ġpro ps -Ġnames pace -Ġc ove -field s -ãĤ ĭ -Ġw ool -ĠC atch -Ġpresum ed -ĠD iana -r agon -ig i -Ġh amm -Ġst unt -ĠG UI -ĠObserv atory -ĠSh ore -Ġsmell s -ann ah -Ġcock pit -ĠD uterte -8 50 -Ġopp ressed -bre aker -ĠCont ribut -ĠPer u -ĠMons anto -ĠAtt empt -Ġcommand ing -Ġfr idge -ĠR in -ĠChe ss -ual ity -Ġo l -Republic an -ĠGl ory -ĠW IN -.... ... -ag ent -read ing -Ġin h -J ones -Ġcl icks -al an -Ġ[ ]; -ĠMaj esty -ĠC ed -op us -ate l -à ª -AR C -ĠEc uador -ãĥ ł -ĠK uro -Ġritual s -Ġcapt ive -Ġoun ce -Ġdisag reement -Ġsl og -f uel -P et -M ail -Ġexerc ised -Ġsol ic -Ġrain fall -Ġdev otion -ĠAss essment -Ġrob otic -opt ions -ĠR P -ĠFam ilies -ĠFl ames -Ġassign ments -00 7 -aked own -Ġvoc abulary -Re illy -Ġc aval -g ars -Ġsupp ressed -ĠS ET -ĠJohn s -Ġwar p -bro ken -Ġstat ues -Ġadvoc ated -Ġ2 75 -Ġper il -om orph -ĠF emin -per fect -Ġh atch -L ib -5 12 -Ġlif elong -3 13 -Ġche eks -Ġnum bered -ĠM ug -B ody -ra vel -We ight -ĠJ ak -ĠHe ath -Ġkiss ing -ĠJ UST -Ġw aving -u pload -Ġins ider -ĠPro gressive -ĠFil ter -tt a -ĠBe am -Ġviol ently -ip ation -Ġskept icism -Ġ19 18 -ĠAnn ie -ĠS I -Ġgen etics -Ġon board -at l -ĠFried man -ĠB ri -cept ive -Ġpir ate -ĠRep orter -27 8 -Ġmyth ology -Ġe clipse -Ġsk ins -Ġgly ph -ing ham -F iles -C our -w omen -Ġreg imes -Ġphotograp hed -K at -ĠMA X -Offic ials -Ġunexpected ly -Ġimpress ions -F ront -;;;; ;;;; -Ġsuprem acy -Ġs ang -Ġaggrav ated -Ġabrupt ly -ĠS ector -Ġexc uses -Ġcost ing -ide press -St ack -ĠR NA -ob il -Ġghost s -ld on -at ibility -Top ics -Ġreim burse -ĠH M -ĠDe g -Ġth ief -y et -ogen esis -le aning -ĠK ol -ĠB asketball -Ġf i -ĠSee ing -Ġrecy cling -Ġ[ - -Cong ress -Ġlect ures -P sy -Ġne p -Ġm aid -Ġori ented -A X -Ġrespect ful -re ne -fl ush -ĠUn loaded -re quest -gr id -ĠAltern atively -ĠHug o -Ġdec ree -ĠBuddh ism -and um -And roid -ĠCong o -ĠJoy ce -Ġacknowled ging -hes ive -ĠTom orrow -ĠH iro -th ren -ĠM aced -Ġho ax -ĠIncre ased -ĠPr adesh -W ild -____ __ -16 1 -Ġa unt -Ġdistribut ing -ĠT ucker -ĠSS L -ĠW olves -B uilding -ou lt -ĠLu o -ĠY as -ĠSp ir -ĠSh ape -ĠCamb od -ĠIP v -Ġm l -Ġext rad -39 0 -ĠPenn y -d ream -Ġstation ed -opt ional -ew orthy -. -ĠWorks hop -ĠRet ail -ĠAv atar -6 25 -N a -ĠV C -ĠSec ure -M Y -19 88 -oss ip -Ġpro state -Ġund en -Ġg amer -ĠCont ents -ĠWar hammer -ĠSent inel -3 10 -Ġse gregation -ĠF lex -ĠM AY -Ġdr ills -ĠDrug s -Islam ic -Ġsp ur -Ġca fe -Ġimag inary -Ġgu iding -Ġsw ings -ĠThe me -ob y -Ġn ud -Ġbe gging -Ġstr ongh -Ġreject ing -Ġpedest rians -ĠPro spect -R are -s le -Ġconcess ions -ĠConst itutional -Ġbe ams -Ġfib ers -p oon -Ġinstinct s -pro perty -ĠB IG -Sand ers -im ates -Ġco ating -Ġcorps es -ĠTR UE -check ed -Ġ16 6 -A sh -ĠJ S -ĠF iction -Ġcommun al -Ġener getic -oooo oooo -Ġnow adays -IL D -ib o -ĠSU V -R en -Ġdwell ing -Sil ver -Ġt ally -ĠM oving -Ġcow ard -Ġgener als -Ġhorn s -Ġcirc ulated -Ġrob bed -ĠUn limited -Ġharass ed -Ġinhib it -Ġcomp oser -ĠSpot ify -Ġspread s -3 64 -Ġsu icidal -Ġno ises -ĠSt ur -Ġs aga -ĠK ag -is o -Ġtheoret ically -M oney -Ġsimilar ity -Ġslic ed -ut ils -ing es -" - -Ġan th -Ġimp ed -Mod ule -Through out -Ġmen us -comm ittee -and i -ob j -in av -f ired -ĠAb dullah -Ġund ead -Ġfont s -H old -EN G -Ġsustain ability -Ġfl ick -Ġr azor -ĠF est -ĠChar acters -Ġword ing -Ġpopul ist -Ġcritic izing -Ġm use -v ine -Ġcard board -Ġkind ly -Ġfr inge -ĠThe ft -icult ural -Ġgovern ors -Ġ ���� -Ġ16 3 -Ġtime out -ĠA uth -Child ren -A U -Ġred emption -ĠAl ger -Ġ19 14 -Ġw aved -Ġastron auts -og rams -Ġsw amp -ĠFinn ish -Ġcand le -Ġton nes -ut m -Ġr ay -Ġsp un -Ġfear ful -art icles -Ġca us -or ically -ĠRequ ires -ĠG ol -Ġpop e -Ġinaug ural -Ġg le -AD A -ĠIS IL -ĠOff ensive -Ġwatch dog -Ġbal con -ent ity -ĠH oo -Ġgall on -AC C -Ġdoub ling -Ġimpl ication -ĠS ight -Ġdoct r ----- --- -Ġ\ \ -Ġm alt -R oll -Ġâī ¥ -Ġrec ap -add ing -u ces -ĠB end -fig ure -Ġtur key -Ġsoc ietal -ĠT ickets -Ġcommer cially -Ġsp icy -Ġ2 16 -ĠR amp -Ġsuperior ity -à ¯ -ĠTr acker -C arl -ĠC oy -ĠPatri ot -Ġconsult ed -Ġlist ings -Ġsle w -reens hot -ĠG one -Ġ[ ...] -30 9 -Ġh ottest -Ø ± -Ġrock y -ĠD iaz -Ġmass age -Ġpar aly -Ġp ony -A z -Ġcart ridge -ĠN Z -Ġsn ack -ĠLam ar -ple ment -ĠLes lie -Ġm ater -Ġsn ipp -24 6 -Ġjoint ly -ĠBris bane -ĠiP od -Ġpump ing -Ġgo at -ĠSh aron -eal ing -Ġcor on -Ġan omal -rah im -ĠConnect ion -Ġsculpt ure -Ġsched uling -ĠD addy -at hing -Ġeyeb rows -Ġcur ved -Ġsent iments -Ġdraft ing -D rop -( [ -Ġnom inal -ĠLeaders hip -ĠG row -Ġ17 6 -Ġconstruct ive -iv ation -Ġcorrupt ed -ger ald -ĠC ros -ĠChe ster -ĠL ap -ãģ ª -OT H -D ATA -Ġal mond -pro bably -I mp -Ġfe ast -ĠWar craft -F lor -Ġcheck point -Ġtrans cription -Ġ20 4 -Ġtwe aks -Ġrel ieve -S cience -Ġperform er -Z one -Ġtur moil -ig ated -hib it -ĠC afe -the med -Ġflu or -ben ch -Ġde com -ĠU nt -ĠBar rett -ĠF acts -Ġt asting -ĠPTS D -ĠSe al -ĠJuda ism -ĠDynam ic -ĠC ors -V e -ĠM ing -ĠTrans form -v on -ĠDef enders -ĠTact ical -ĠV on -ĠUn ivers -Ġdist orted -ĠB reath -?' " -Ġag on -ĠDead ly -Ġl an -ĠCy cle -orn ed -Ġrel iably -Ġgl or -ĠMon key -ãĥ ¡ -Ġad ren -Ġmicrow ave -ĠAl ban -irc raft -dig it -sm art -ĠD read -¯¯¯¯¯¯¯¯ ¯¯¯¯¯¯¯¯ -{ { -ĠRoc hester -Ġsimpl ified -Ġinf licted -Ġtake over -Ġyour selves -ad itional -Ġmus cular -K S -Ġing en -T ax -ĠFe ature -27 7 -Ġcru c -Ġcr ate -Ġun identified -Ġacclaim ed -ĠM anga -ĠFr ances -ĠNep al -ĠG erald -ĠKu wait -Ġsl ain -ĠHe b -ĠG oku -ãģ® æ -28 6 -M rs -ĠC ody -ĠSan ctuary -01 6 -Ġdism ant -Ġdatas et -ĠH ond -b uck -ĠPat terson -Ġpal ette -ĠG D -ic ol -ĠL odge -Ġplanet ary -ak in -ĠRegist ered -ab we -ĠPeters burg -Ġha iled -ĠP iece -S che -ĠDO J -Ġen umer -18 1 -ĠObs erver -ĠB old -f ounded -com merce -Ġexplo its -ĠF inding -UR N -ĠS ne -ĠAc id -ay ette -ĠVal ues -Ġdr astic -Ġarchitect ural -Ġ" . -× ķ -ump ed -Ġwra pping -Ġwid ow -ĠSl ayer -l ace -on ce -German y -av oid -Ġtem ples -P AR -à ´ -ĠLuc ifer -ĠFl ickr -l ov -for ces -Ġsc outing -Ġlou der -tes y -Ġbefore hand -Ä ĵ -ĠNe on -ĠW ol -ĠTyp ically -ĠPolit ico --+ -+ -Ġbuild er -Ġder ive -K ill -Ġp oker -Ġambig uous -Ġlif ts -Ġcy t -Ġrib s -ood le -ĠS ounds -h air -ĠSynd rome -t f -Ġproport ional -u id -Ġper taining -ĠKind le -ĠNeg ro -Ġreiter ated -ĠTon ight -oth s -ĠCorn ell -Ġo wing -Ġ20 8 -elf are -oc ating -ĠB irds -Sub scribe -Ġess ays -Ġburd ens -Ġillust rations -ar ious -ER AL -ĠCal cul -Ġx en -ĠLink edIn -ĠJ ung -Ġredes ign -Con nor -29 6 -Ġrevers al -ĠAd elaide -ĠL L -Ġs inking -Ġg um -US H -c apt -ĠGr imm -Ġfoot steps -ĠCB D -isp ers -Ġpro se -Wed nesday -ĠM ovies -ed in -Ġoverturn ed -Ġcontent ious -US B -~~~~~~~~ ~~~~~~~~ -ĠCo pper -Ġpoint less -N V -val ues -olph in -d ain -Ġdepos ited -ĠG W -Ġpreced ed -ĠCl a -ĠGo lem -ĠN im -ĠÎ ² -ĠEngine ers -m iddle -Ġfl att -oper ative -Ġcouncil s -imb abwe -el in -Ġstress ful -ĠL D -Ġres h -l ake -Ġwheel chair -ĠAltern ative -Ġoptim ize -oper ation -Ġpe ek -Ġones elf -ig il -Ġtrans itions -op athy -bl ank -Ġ16 9 -17 1 -________________________________ ________________________________ -Ġl aundering -En c -ĠD EC -Ġwork outs -Ġsp ikes -Ġdin osaurs -Ġdiscrim inatory -P ool -R ather -38 5 -R NA -tes ters -et o -ĠIdent ity -Ġve in -ĠBur ton -Ġarc ade -4 20 -Ult imately -ĠSad ly -à ° -p ill -Ġcub ic -ĠSpect rum -the se -st ates -Ġun official -h awks -ĠEVER Y -Ġrain bow -Ġincarcer ation -and ing -Ġsy ll -ĠEver ton -Ġ17 9 -ĠSer bia -Ġ18 9 -m eter -ĠMic key -Ġant iqu -Ġfact ual -ne ck -ĠN are -n orm -m ust -Ġhigh ways -Ġgl am -Ġdivid ing -ĠSquad ron -ĠMar tha -Ġbirth s -C over -//////// //////// -ĠW ong -Ph ot -ĠA LS -ri o -ĠNon etheless -ĠL emon -Ġ20 6 -ĠE E -Ġderiv ative -ĠWW II -v ote -Ġthere in -Ġsepar ating -44 6 -sy nc -ĠStre ets -Ġr att -Ġmunicip ality -ĠShort ly -Ġmon k -) ," -Ġscr ub -Ġoper atives -Ne ither -Pl ace -ĠLim it -F emale -ĠAct or -Char acter -Ġconstit uted -35 7 -Ġprotest ed -ĠSt raw -ĠHe ight -ild a -ĠTy ph -Ġflood s -Ġcos metic -W AY -pert ure -up on -t ons -ess ing -ĠP ocket -Ġro oft -ĠC aucas -Ġant idepress -Ġincomp atible -EC D -Ġoper a -ĠCont est -Ġgener ators -l ime -Def ense -19 87 -for um -Ġsav age -ĠHung arian -n z -Ġmet allic -Ġex pelled -Ġres idency -Ġdress es -66 6 -ĠC lement -f ires -C ategory -Ġge ek -al is -Ġc emetery -educ ated -Ġc rawl -ĠUn able -ĠT yson -ak is -Ġp ardon -ĠW ra -Ġstrengthen ed -ĠF ors -33 5 -ĠH C -ĠM ond -Ġvisual s -ĠBeat les -ett lement -Ġ ï -g ro -Ġb ash -Ġpo orest -Ġex cel -Ġaspir ations -ĠM unicip -ens ible -Ġceremon ies -Ġintimid ation -ĠCON TR -be ck -ĠK ap -as u -Ġtradem arks -ĠS ew -ĠComp etition -net work -ĠAr ri -ĠT et -Ro aming -W C -D at -Ġso b -Ġpair ing -Ġoverd ose -SA Y -ab er -Ġrev olt -ĠF ah -act ing -e q -est ation -F ight -ĠMar ks -27 3 -Ġ17 8 -R aw -ãģ ĭ -34 9 -bl ocks -Ġver ge -est ine -ĠPod esta -Ġinv asive -Ġprofound ly -ĠA o -e ach -Ġl est -inter pret -Ġshr inking -Ġerr one -Ġche es -ly s -ĠI vy -ĠDirect ory -Ġhint ed -V ICE -Ġcontact ing -ĠG ent -he i -Ġlabel ing -Ġmerc ury -ĠL ite -Ġexp ires -Ġdest abil -rit is -c u -Ġfeather s -Ġste er -Ġprogram med -ĠV ader -Go ing -ĠE lim -Ġy o -ĠMic he -Ġ20 3 -Ġslee ves -Ġb ully -ĠHum ans -36 8 -Ġcomp ress -ĠBan ner -AR S -Ġa while -Ġcal ib -Ġspons orship -ĠDiff iculty -ĠP apers -Ġident ifier -} . -Ġy og -ĠSh ia -Ġclean up -Ġvib e -int rodu -im ming -Austral ia -Ġout lines -ĠY outube -tr ain -ĠM akes -Ġde ported -Ġcent r -ĠD ug -ĠB oulder -ĠBuff y -Ġinj unction -ĠHar ley -ĠG roups -ĠD umbledore -ĠCl ara -Ġ" - -Ġsacrific ed -ep h -Sh adow -ib ling -Ġfreel ance -Ġevident ly -ph al -Ġret ains -M ir -Ġfin ite -d ar -ĠC ous -Ġrep aired -Ġperiod ic -Ġchampions hips -Ġaster oid -bl ind -Ġexpress ly -ĠAst ros -Ġsc aled -Ġge ographical -ĠRap ids -En joy -Ġel astic -ĠMoh amed -Mark et -be gin -Ġdisco vers -Ġtele communications -Ġscan ner -Ġen large -Ġsh arks -Ġpsy chedel -ĠRou ge -Ġsnap shot -is ine -X P -Ġpestic ides -ĠL SD -ĠDist ribution -re ally -Ġde gradation -Ġdisgu ise -Ġbi om -ĠEX T -Ġequ ations -Ġhaz ards -ĠComp ared -) * -Ġvirt ues -Ġeld ers -Ġenh ancing -ĠAc ross -er os -ang ling -Ġcomb ust -ucc i -Ġconc ussion -Ġcontrace ption -ĠK ang -Ġexpress es -Ġa ux -ĠP ione -Ġexhib its -Deb ug -OT AL -ĠAl ready -ĠWheel er -Ġexp ands -? : -Ġreconc iliation -Ġpir ates -Ġpur se -Ġdiscour age -Ġspect acle -R ank -Ġwra ps -ĠTh ought -Ġimp ending -O pp -ĠAng lo -ĠE UR -Ġscrew ed -ret ched -Ġencour agement -mod els -Ġconf use -mm m -ĠVit amin -âĸij âĸij -C ru -Ġkn ights -Ġdisc ard -Ġb ishops -ĠW ear -ĠGar rett -k an -ãĥ Ł -Ġmascul ine -cap ital -ĠA us -Ġfat ally -th anks -ĠA U -ĠG ut -12 00 -Ġ 00000000 -Ġsur rog -ĠBI OS -ra its -ĠWat ts -Ġresur rection -ĠElect oral -ĠT ips -4 000 -Ġnut rient -Ġdepict ing -Ġspr ink -Ġm uff -ĠL IM -ĠS ample -ps c -ib i -gener ated -Ġspec imens -Ġdiss atisf -Ġtail ored -Ġhold ings -ĠMonth ly -ĠE at -po ons -Ġne c -ĠC age -ĠLot us -ĠLan tern -Ġfront ier -Ġp ensions -Ġj oked -ĠHard y -=-=- =-=- -r ade -U ID -Ġr ails -Ġem it -Ġsl ate -Ġsm ug -Ġsp it -ĠCall s -ĠJac obs -f eat -ĠU E -Ġrest ruct -Ġregener ation -Ġenerg ies -ĠCon nor -OH N -ĠChe ese -Ġg er -Ġresur rect -man agement -N W -Ġpres ently -ĠBru ins -M ember -ĠM ang -id an -Ġboost ing -w yn -+ . -requ isite -ĠNY PD -ĠMe gan -ĠCond itions -Ġp ics -nes ium -ĠR ash -Ġ17 4 -ĠD ucks -Ġemb ro -z u -on ian -rel igious -Ġc raz -ĠAC A -ĠZ ucker -EM A -ĠPro s -We apon -ĠKn ox -ĠAr duino -Ġst ove -Ġheaven s -ĠP urchase -Ġher d -Ġfundra iser -Dig ital -5 000 -Ġprop onents -/ âĢĭ -Ġj elly -ĠVis a -Ġmon ks -Ġadvance ment -ĠW er -Ġ18 7 -e us -ert ility -Ġfet al -Ġ19 36 -L o -Ġout fits -Ġstair case -b omb -Ġcustom ized -cl air -T ree -Ġm apped -ĠConsider ing -ĠTor res -Ġmeth yl -Ġapprox imate -Ġdo om -ĠHans en -Ġc rossover -Ġstand alone -ä ¼ -Ġinv ites -Ġgra veyard -Ġh p -Donald Trump -Ġesc ort -G ar -Ġpredec essors -Ġh ay -Ġen zyme -ĠStra ight -vis ors -I ng -ane ously -ĠApp lied -Ġf ec -ĠDur ant -Ġout spoken -or b -Ġz eal -Ġdisgr ace -' ). -ĠChe ng -28 9 -ĠRen a -ĠSu icide -29 4 -Ġout raged -ĠNew man -ĠN vidia -ĠA ber -ĠB ers -Ġrecre ation -Wind ow -ĠD P -x e -Ġped oph -Ġfall out -ambo o -Ġpresent ations -ĠApp s -Ġh tml -3 45 -ĠX XX -Ġrub bing -ĠLe ather -Ġhum idity -se ys -est ablished -ĠUn its -64 6 -Ġrespect able -A uto -Ġthri ving -ĠInn ovation -ang s -Ext ra -reg ulation -29 8 -p ick -Ex amples -ĠC J -Att ack -Ġdr acon -L T -Ġstick er -re rs -Ġsun ny -I ss -reg ulated -d im -ĠAb stract -Ġhus bands -Off ice -om ination -it ars -AN GE -asc al -ĠK ris -ĠInf antry -Ġm alf -ĠA the -ĠR ally -bal anced -................ ........ -OU P -Ġmole cule -met ics -ĠSpl it -ĠInstruct ions -ĠN ights -c ards -Ġt ug -Ġcon e -å Ń -Ġt x -ĠDisc ussion -Ġcatast rophe -pp e -g io -Ġcommun ism -Ġhal ted -ĠGu ant -cle an -ĠSc hed -ĠK anye -Ġw ander -ĠSer iously -Ġ18 8 -enn ial -f ollow -product ive -ĠFl ow -ĠS ail -Ġc raw -Ġsim ulations -or u -ang les -ĠN olan -Ġmen stru -4 70 -Ġ20 7 -aj a -Ġcas ually -board ing -Ġ2 22 -ov y -ĠN umbers -um at -O E -28 7 -ĠCle mson -Ġcert s -Ġsl id -ĠT ribe -Ġto ast -Ġfort unes -Ġf als -ĠComm ittees -Ġg p -Ġf iery -ĠN ets -ĠAn ime -Pack age -ĠComp are -l aughter -in fect -Ġatroc ities -Ġjust ices -Ġins ults -ĠVern on -Ġsh aken -Ġperson a -est amp -36 7 -br ain -Ġexperiment ing -K en -ĠElect ronics -Ġ16 1 -dom ain -Ġgraph ical -b ishop -Ġwho pping -ĠEv angel -Ġadvertis ers -ĠSpe ar -Ġb ids -Ġdestro ys -ut z -Ġunders c -ĠAD D -Ġan ts -ĠC um -ipp les -ĠF ill -Ġgl anced -Ġind icted -ĠE ff -Ġmis con -ĠDes ktop -Ġab ide -ãĥ Ģ -ĠI o -ĠC oul -Ġcaps ule -ĠCh rys -M ON -Ġund es -ĠI RA -Ġc itation -Ġdict ate -ĠNet works -ĠConf lict -ĠSt uff -x a -is ec -ĠChem istry -Ġquarter ly -William s -an an -O pt -ĠAlexand ria -out heastern -ĠSpring field -ĠBlack s -Ġge ography -24 2 -Ġut most -ĠEx xon -ab outs -E VA -ĠEn able -ĠBar r -Ġdisag reed -ĠCy prus -Ġdement ia -Ġlab s -Ġubiqu itous -ĠLO VE -Ġconsolid ated -s r -Ġcream y -ĠTim ber -Reg ardless -ĠCert ificate -Ġ" ... -ogen ous -Capt ain -Ġinsult ing -ĠSor os -ĠInst r -ĠBulgar ia -bet ter -Ġsuck ing -ĠDavid son -at z -Ġcoll ateral -g if -Ġplag ued -ĠC ancel -ĠGard ner -R B -Ġsix teen -Rem ove -ur istic -c ook -R od -Ġcompr ising -f le -) âĢĶ -ĠVik ing -g rowth -agon al -Ġsr f -af ety -m ot -N early -st own -ĠF actor -Ġautom obile -Ġproced ural -m ask -amp ires -Ġdisapp ears -j ab -3 15 -Ġ19 51 -ne eded -Ġd aring -le ader -Ġp odium -Ġun healthy -Ġm und -Ġpy ramid -oc re -Ġkiss ed -Ġdream ed -ĠFant astic -ĠG ly -å Ĭ -Ġgreat ness -Ġsp ices -Ġmet ropolitan -Ġcomp uls -i ets -101 6 -ĠSh am -ĠP yr -fl ies -ĠMid night -Ġswall owed -Ġgen res -ĠL ucky -ĠRew ards -Ġdisp atch -ĠI PA -ĠApp ly -Ġa ven -al ities -3 12 -th ings -Ġ( ). -Ġm ates -ĠS z -ĠC OP -ol ate -O FF -Ġre charge -c aps -ĠYork er -ic one -Ġgal axies -ile aks -D ave -ĠP uzz -ĠCelt ic -ĠA FC -27 6 -ĠS ons -Ġaffirm ative -H or -Ġtutorial s -ĠC ITY -ĠR osa -ĠExt ension -Ser ies -Ġf ats -Ġr ab -l is -Ġun ic -Ġe ve -ĠSp in -Ġadul thood -ty p -Ġsect arian -Ġcheck out -ĠCy cl -S ingle -Ġmart yr -Ġch illing -88 8 -ou fl -Ġ] ; -Ġcongest ion -m k -ĠWhere as -Ġ19 38 -ur rencies -er ion -Ġbo ast -ĠPat ients -Ġch ap -ĠB D -real DonaldTrump -Ġexam ines -h ov -Ġstart ling -ĠBab ylon -w id -om ew -br ance -ĠOd yssey -w ig -Ġtor ch -ĠV ox -ĠMo z -ĠT roll -ĠAn s -Similar ly -ĠF ul -00 6 -Un less -ĠAl one -st ead -ĠPub lisher -r ights -t u -ĠDoes n -Ġprofession ally -Ġcl o -ic z -Ġste als -Ġ á -19 86 -Ġst urdy -ĠJoh ann -Ġmed als -Ġfil ings -ĠFr aser -d one -Ġmult inational -Ġf eder -Ġworth less -Ġp est -Yes terday -ank ind -Ġg ays -Ġb orne -ĠP OS -Pict ure -Ġpercent ages -25 1 -r ame -Ġpot ions -AM D -ĠLeban ese -Ġr ang -ĠL SU -ong s -Ġpen insula -ĠCl ause -AL K -oh a -ĠMac Book -Ġunanim ous -Ġl enders -Ġhang s -Ġfranch ises -ore rs -ĠUp dates -Ġisol ate -and ro -S oon -Ġdisrupt ive -ĠSur ve -Ġst itches -ĠSc orp -ĠDomin ion -Ġsupp lying -Ar g -Ġtur ret -ĠL uk -Ġbr ackets -* ) -ĠRevolution ary -ĠHon est -Ġnot icing -ĠSh annon -Ġafford ed -Ġth a -ĠJan et -! -- -ĠNare ndra -ĠPl ot -H ol -se ver -e enth -Ġobst ruction -Ġ10 24 -st aff -j as -or get -sc enes -l aughs -ĠF argo -cr ime -Ġorche str -Ġde let -ili ary -rie ved -Ġmilit ar -ĠGreen e -âĹ ı -ãģ ¦ -ĠGu ards -Ġunle ashed -ĠWe ber -Ġadjust able -Ġcal iber -Ġmotiv ations -Ġà ł -m Ah -ĠL anka -hand le -Ġp ent -ĠR av -ĠAng ular -ĠK au -umb ing -Ġphil anthrop -Ġde hyd -Ġtox icity -e er -ĠY ORK -w itz -å ¼ -ĠI E -commun ity -ĠA H -Ġret ali -Ġmass ively -ĠDani els -ĠD EL -Ġcar cin -Ur l -Ġrout ing -ĠNPC s -ĠR AF -ry ce -Ġwa ived -ĠGu atem -Every body -Ġco venant -Ġ17 3 -Ġrelax ing -Ġqu art -al most -Ġguard ed -ĠSold iers -ĠPL AY -Ġout going -L AND -Ġre write -ĠM OV -ĠIm per -ĠS olution -Ġphenomen al -Ġl ongevity -Ġimp at -ĠN issan -ir ie -Ġod or -ĠZ ar -ok s -Ġmilit ias -ĠSP EC -Ġtoler ated -ars er -ĠBrad ford -+ , -Ġsur real -s f -Can adian -Ġresemb lance -Ġcarbohyd rate -VI EW -Ġaccess ory -me al -larg est -ieg el -Some one -Ġtoug hest -os o -Ġfun nel -Ġcondemn ation -lu ent -Ġw ired -ĠSun set -Jes us -ĠP ST -ĠP ages -ĠTy coon -ĠP F -Ġselect ions -Ġ ठ-part isan -Ġhigh s -ĠR une -Ġcraft s -le ad -ĠParent s -Ġre claim -ek er -ĠAll ied -ae per -Ġlo oming -Ġbenefic iaries -ĠH ull -Stud ents -Jew ish -d j -Ġp act -tem plate -ĠOffic ials -ĠBay lor -Ġhe mp -Ġyouth s -ĠLevel s -ĠX iao -ĠC hes -Ġende avor -ĠRem oved -Ġhipp ocamp -H ell -ãĤ Ĭ -80 5 -Ġd inosaur -ĠWr ath -ĠIndones ian -Ġcalcul ator -ĠD ictionary -Ġ4 20 -ĠM AG -( _ -! , -t arians -Ġrestrict ing -rac use -Ġweek day -OU NT -Ġsh rugged -leg round -Ġb ald -ĠDo ctors -Ġt outed -ĠMax well -Ġ2 14 -Ġdiplom at -Ġrep ression -Ġconstitu ency -v ice -r anked -ĠNap oleon -g ang -ĠFore ver -t un -Ġbul b -ĠPD T -ĠC isco -V EN -Ġres umed -Ste ven -ĠManit oba -Ġfab ulous -ĠAg ents -19 84 -Ġam using -ĠMyster ies -Ġor thodox -fl oor -Ġquestion naire -Ġpenet rate -Ġfilm makers -ĠUn c -Ġst amped -Ġth irteen -Ġout field -Ġforward ed -Ġapp ra -Ġa ided -t ry -Ġunf ocused -ĠL iz -ĠWend y -ĠSc ene -Ch arg -Ġreject s -Ġleft ist -ĠProv idence -ĠBr id -reg n -Ġprophe cy -ĠL IVE -4 99 -Ġfor ge -ĠF ML -Ġintrins ic -ĠF rog -Ġw ont -ĠH olt -Ġfam ed -CL US -aeper nick -ĠH ate -ĠC ay -Ġregister ing -ort ality -rop y -ocaly ptic -a an -n av -Ġfasc ist -IF IED -Ġimpl icated -ĠRes ort -ĠChand ler -ĠBr ick -P in -ys c -Us age -ĠHel m -us ra -âĺħ âĺħ -ĠAb bas -Ġunanim ously -Ġke eper -Ġadd icted -?? ? -Ġhelm ets -Ġant ioxid -aps ed -80 8 -gi ene -Ġwa its -Ġmin ion -ra ved -ĠP orsche -Ġdream ing -Ġ17 1 -ĠC ain -Ġun for -ass o -ĠConfig uration -k un -hard t -Ġn ested -ĠL DS -L ES -Ġt ying -en os -Ġc ue -ĠMar qu -sk irts -Ġclick ed -Ġexp iration -ĠAccording ly -ĠW C -Ġbless ings -Ġaddict ive -ĠN arr -y x -ĠJagu ars -Ġrent s -ĠS iber -Ġt ipped -ous se -ĠFitz gerald -Ġhier arch -out ine -Ġwa velength -> . -ch id -ĠProcess ing -/ + -r anking -E asy -ĠConst ruct -Ġt et -ins ured -H UD -Ġqu oting -Ġcommun icated -in x -Ġin mate -Ġerect ed -ĠAbs olutely -ĠSure ly -Ġun im -ĠThr one -he id -Ġcl aws -Ġsuper star -ĠL enn -ĠWh is -U k -ab ol -Ġsk et -ĠN iet -Ġper ks -Ġaff inity -Ġopen ings -phas is -Ġdiscrim inate -T ip -v c -Ġgr inding -ĠJenn y -Ġast hma -hol es -ĠHom er -Ġreg isters -ĠGl ad -Ġcre ations -Ġlith ium -Ġappl ause -unt il -Just ice -ĠTur ks -Ġsc andals -Ġb ake -t ank -M ech -ĠMe ans -ĠM aid -Republic ans -is al -wind ows -ĠSant os -Ġveget ation -33 8 -t ri -Ġfl ux -ins ert -Ġclar ified -Ġmort g -ĠCh im -ĠT ort -Ġdiscl aim -met al -ĠAs ide -Ġindu ction -Ġinf l -Ġathe ists -amp h -Ġe ther -ĠV ital -ĠBu ilt -M ind -Ġweapon ry -S ET -Ġ18 6 -ad min -g am -cont ract -af a -Ġderiv atives -Ġsn acks -Ġch urn -E conom -Ġca pped -ĠUnder standing -ĠH ers -ĠI z -Ġd uct -I ENT -augh ty -Ġâľ Ķ -ĠN P -Ġsa iling -In itialized -Ġt ed -Ġreact ors -ĠL omb -Ġcho ke -ĠW orm -Ġadm iration -Ġsw ung -ens ibly -Ġr ash -ĠGo als -ĠImport ant -Sh ot -ĠR as -Ġtrain ers -ĠB un -Work ing -Ġhar med -ĠPand ora -ĠL TE -Ġmush room -ĠCH AR -ĠF ee -ĠM oy -B orn -ol iberal -ĠMart ial -Ġgentle men -Ġling ering -Offic ial -Ġgra ffiti -ĠN ames -D er -Ġqu int -ist rate -aze era -ĠNOT ICE -ĠFlore nce -Ġpay able -Ġdep icts -ĠSpe cies -He art -âĶĢâĶĢâĶĢâĶĢ âĶĢâĶĢâĶĢâĶĢ -Ġencl osed -Incre ases -D aily -ĠL is -Ġenact ment -ĠB acon -ĠSt eele -dem and -Ġ18 3 -Ġmouth s -Ġstr anded -Ġenhance ment -01 1 -ĠWh ats -Ġhe aled -en y -ĠR ab -Ġ3 40 -ĠLab yrinth -ro ach -ĠY osh -ĠCl ippers -Ġconcert s -Intern et -35 5 -Ġstick ers -Ġter med -ĠAx e -Ġgrand parents -Fr ance -ĠCl im -ĠU h -ul ic -Ġthr ill -cent ric -ĠOver view -ĠCond uct -Ġsubstant ive -Ġ18 2 -m ur -Ġstr ay -ĠCo ff -Ġrep etitive -ĠFor gotten -Ġqual ification -ew itness -ĠZ imbabwe -Ġsim ulated -ĠJ D -25 3 -ĠW are -Ġun sc -T imes -Ġsum mons -Ġdis connected -Ġ18 4 -ci us -ĠGu jar -od ka -Ġer ase -ĠTob acco -elect ed -Ġun cont -ĠShe pard -ĠL amp -Ġalert ed -Ġoper ative -arn a -u int -Ġneglig ence -ac ements -Ġsup ra -Ġprev ail -ĠSh ark -Ġbel ts -ãģ « -Ġt ighter -Engine ers -Ġin active -Ġexp onent -ĠWill ie -a ples -Ġhe ir -ĠH its -ian n -ĠS ays -Ġcurrent s -ĠBeng al -Ġar ist -B uffer -Ġbree ze -ĠWes ley -Col a -Ġpron oun -Ġde ed -ĠK ling -Ġof t -Ġinf lict -Ġpun ishing -Ġn m -ik u -OD UCT -01 4 -Ġsubsid y -ĠDE A -ĠHer bert -ĠJ al -B ank -Ġdef erred -Ġship ment -B ott -Ġal le -b earing -HT ML -Off line -Ġ2 13 -Ġscroll ing -Ġsc anned -ĠLib yan -ĠT OP -ch rom -d t -col umn -Psy NetMessage -Z ero -Ġtor so -0 50 -âķ IJ -Ġimp erson -ĠSchw artz -ud ic -Ġpiss ed -ĠS app -25 7 -ĠIS Ps -og l -Ġsuper vised -Ġad olescent -Ġatt ained -ĠDel ivery -ĠB unny -Ġ19 37 -Ġmini ature -Ġo s -Ġ3 70 -60 8 -ĠMour inho -Ġinn ate -Ġtem po -ĠN M -ĠFall en -00 9 -Ġprov ocative -Stream er -ĠBened ict -ĠBol she -Ġt urtle -ĠPC B -ĠEqu al -Direct or -ĠR end -Ġflu ids -Author ities -Ġcous ins -requ ency -ĠNeigh bor -s ets -sh ared -Char les -pass word -Ġg ears -Ġ2 11 -ĠHard ware -ri ka -Ġup stream -H om -Ġdisproportion ately -iv ities -Ġund efined -Ġelect rons -Ġcommem or -Event ually -Ġ> < -Ġir responsible -2 18 -ĠRe leased -ĠO VER -ĠI GN -ĠB read -st ellar -ĠS age -tt ed -dam age -ed ition -ĠPre c -Ġl ime -Ġconf inement -Ġcal orie -we apon -Ġdiff ering -ĠS ina -m ys -am d -Ġintric ate -k k -ĠP AT -ã o -st ones -lin ks -Ġr anch -Sem itic -Ġdifferent iate -ĠS inger -occup ied -Ġfort ress -c md -Ġinter ception -ĠAnk ara -Ġre pt -ĠSol itaire -Ġrem ake -p red -Ġd ared -aut ions -ĠB ACK -Run ning -Ġdebug ging -Ġgraph s -3 99 -ĠNig el -Ġb un -Ġpill ow -Ġprog ressed -fashion ed -Ġob edience -ER N -Ġrehe ars -C ell -t l -S her -Ġher ald -ĠPay ment -ĠC ory -ĠDe pt -Ġrep ent -ĠWe ak -uck land -Ġple asing -Ġshort ages -Ġjur ors -ĠK ab -q qa -Ant i -Ġw ow -ĠRC MP -Ġt sun -ĠS ic -Ġcomp rises -Ġsp ies -Ġprec inct -n u -Ġur ges -Ġtim ed -Ġstrip es -ĠB oots -Ġy en -Adv anced -Ġdisc rete -ĠArch angel -employ ment -D iff -Ġmon uments -Ġ20 9 -work er -Ġ19 6 -ĠI g -utter stock -T PS -J ac -Ġhomeless ness -Ġcomment ator -Ġrac ially -f ing -se ed -E le -ell ation -Ġeth anol -Ġpar ish -ĠD ong -ĠAw akening -Ġdev iation -ĠB earing -ĠTsu k -Ġrec ess -Ġl ymph -ĠCann abis -å ľ -ĠNEW S -Ġd ra -ĠStef an -ĠWr ong -ĠS AM -Ġloose ly -Ġinterpre ter -ĠPl ain -Go vernment -Ġbigot ry -Ġgren ades -ave z -pict ured -Ġmand ated -ĠMon k -ĠPed ro -Ġl ava -27 4 -Ġcyn ical -ĠScroll s -l ocks -M p -Ġcon gregation -orn ings -ph il -ĠI bid -Ġf erv -Ġdisapp earing -Ġarrog ant -sy n -ĠMa ver -ĠSu it -24 1 -Ġab bre -ack ers -P a -ĠY el -Whe never -Ġ23 5 -ĠV ine -ĠAn at -Ġext inct -LE T -Ġexecut able -V ERS -ox ide -D NA -ĠP rel -Ġresent ment -Ġcompr ise -ĠAv iv -Ġinter ceptions -Ġprol ific -IN A -ĠEr in -though t -2 19 -ĠPsychiat ry -un ky -chem ist -H o -ĠMcC oy -Ġbr icks -L os -ri ly -ĠUS SR -Ġr ud -Ġl aud -ĠW ise -ĠEmer ald -Ġrev ived -Ġdam ned -ĠRep air -id em -ct ica -Ġpatri arch -ĠN urs -me g -Ġcheap est -re ements -empt y -ĠCele br -Ġdepri vation -ch anted -ĠTh umbnails -E nergy -ĠEth an -ĠQ ing -Ġopp oses -W IND -v ik -ĠM au -ĠS UB -66 7 -G RE -ĠVol unte -nt on -C ook -å IJ -es que -Ġplum met -Ġsu ing -Ġpron ounce -Ġresist ing -ĠF ishing -ĠTri als -Ġy ell -Ġ3 10 -Ġin duct -Ġpersonal ized -oft en -R eb -EM BER -Ġview point -Ġexist ential -() ) -rem ove -MENT S -l asses -Ġev apor -Ġa isle -met a -Ġreflect ive -Ġentit lement -Ġdev ised -mus ic -asc ade -Ġwind ing -off set -Ġaccess ibility -ke red -Bet ter -ĠJohn ston -th inking -S now -ĠCroat ia -ĠAt omic -27 1 -34 8 -Ġtext book -ĠSix th -Ġ اÙĦ -Ġsl ider -ĠBur ger -b ol -S ync -Ġgrand children -Ġc erv -+ ) -Ġe ternity -Ġtweet ing -Ġspec ulative -Ġpiv otal -ĠW P -ĠT ER -ynam ic -Ġu pl -ĠC ats -per haps -Ġclass mates -Ġblat ant -' - -Ġl akh -ant ine -ĠB org -i om -/ ( -ĠAthlet ic -Ġs ar -OT A -ĠHoff man -Never theless -Ġad orable -Ġspawn ed -Ass ociated -ĠDom estic -Ġimpl ant -ĠLux em -ĠK ens -Ġp umps -ĠS AT -Att ributes -50 9 -av our -Ġcentral ized -ĠT N -Ġfresh ly -ĠA chieve -Ġouts iders -her ty -ĠRe e -ĠT owers -ĠD art -ak able -Ġm p -ĠHeaven ly -Ġr ipe -ĠCarol ine -ry an -Ġclass ics -Ġret iring -Ġ2 28 -Ġa h -Ġdeal ings -Ġpunch ing -ĠChap man -O ptions -max well -vol ume -Ġst al -Ġex ported -ĠQu ite -Ġnumer ical -B urn -F act -ĠKey stone -Ġtrend ing -Ġalter ing -ĠAfric ans -47 8 -ĠM N -ĠKn ock -Ġtempt ation -Ġprest ige -Over view -ĠTrad itional -ĠBah rain -Priv ate -ĠH OU -Ġbar r -ĠT at -C ube -US D -ĠGrand e -ĠG at -ĠFl o -Ġres ides -Ġind ec -vol ent -Ġperpet ual -ub es -Ġworld view -ĠQuant um -Ġfil tered -Ġen su -orget own -ERS ON -ĠM ild -37 9 -OT T -à ¥ -Ġvit amins -Ġrib bon -Ġsincere ly -ĠH in -Ġeight een -Ġcontradict ory -Ġgl aring -Ġexpect ancy -Ġcons pir -Ġmon strous -Ġ3 80 -re ci -Ġhand ic -Ġpump ed -Ġindic ative -Ġr app -Ġav ail -ĠLEG O -ĠMar ijuana -19 85 -ert on -Ġtwent ieth -################ ################ -ĠSw amp -Ġval uation -Ġaffili ates -adjust ed -ĠFac ility -26 2 -Ġenz ymes -itud inal -Ġimp rint -S ite -Ġinstall er -ĠT RA -m ology -lin ear -ĠCollect ive -ig ating -ĠT oken -Ġspec ulated -K N -ĠC ly -or ity -Ġdef er -Ġinspect ors -appro ved -R M -ĠSun s -Ġinform ing -ĠSy racuse -ib li -7 65 -Ġgl ove -Ġauthor ize -âĢ¦âĢ¦âĢ¦âĢ¦ âĢ¦âĢ¦âĢ¦âĢ¦ -ĠCru ise -Ġcontract ing -she ll -IF E -ĠJew el -p ract -ĠPhot oshop -ĠKnow ing -h arm -Ġattract ions -ad an -et us -01 8 -w agen -Al t -Ġmultip ly -Ġequ ilibrium -: { -ĠF ighters -ĠEd gar -Ġfour teen -Go vern -Ġmis use -Ġab using -Ġancest ry -ram er -64 4 -Ġwor ms -Ġthick er -ĠComb ine -Ġpeas ants -Ġv ind -Ġcon quest -Ġm ocked -Ġc innamon -ĠC ald -ĠGall up -Ġavoid ance -Ġincarn ation -ĠStr at -Ġt asted -ent a -ĠN eal -p ared -Ġtermin ology -ject ion -Scient ists -ĠIN S -ĠDe e -Ġdirect ories -R oad -ĠSh ap -br ight -ĠDirect ors -ĠCol umn -Ġb ob -Ġprefer ably -Ġgl itch -f urt -Ġe g -id is -C BC -Ġsur rendered -Ġtest ament -33 6 -ug gest -ĠN il -an other -Ġpat hetic -ĠDon na -Ġ2 18 -ĠA very -Ġwhis key -Ġf ixture -ĠCon quest -Ġbet s -O cc -ĠLe icester -] ." -Ġ) ); -Ġfl ashes -45 6 -Ġmask ed -ge bra -Ġcomput ed -che l -aud er -Ġdefe ats -ĠLiber ation -ĠOs ama -ĠV ive -Ch anges -Ch annel -Ġtar iffs -Ġm age -ĠS ax -Ġinadvert ently -ĠC RE -ĠRe aper -ink y -gr ading -Ġstere otyp -Ġcur l -ĠF ANT -Ġfram eworks -M om -ĠAn ch -Ġflav our -car bon -Ġperm itting -let cher -ĠMo zilla -ĠPark ing -ĠCh amp -Sc roll -Ġmurd erer -Ġrest ed -Ġow es -ĠP oss -AD D -IF F -res olution -ĠMin ing -Ġcompar ative -D im -Ġneighbour ing -ĠA ST -ĠT oxic -Ġbi ases -Ġgun fire -ur ous -ĠMom ent -19 83 -Ġper vasive -tt p -ĠNorm ally -r ir -S arah -ĠAlb any -Ġun sett -ĠS MS -ip ers -l ayer -ĠWh ites -up le -Ġtur bo -ĠLe eds -Ġthat s -ĠMin er -M ER -ĠRe ign -Ġper me -ĠBl itz -Ġ19 34 -Ġintimid ating -t ube -Ġecc entric -ab olic -box es -ĠAssoci ates -v otes -Ġsim ulate -um bo -aster y -Ġship ments -FF FF -an th -Ġseason ed -Ġexperiment ation -âĸ ł -law s -Me et -idd les -ant ics -R ating -IS IS -h ift -Ġfront s -b uf -01 7 -Ġun att -ĠD il -le ases -ĠGard ens -77 7 -t ouch -ve ll -45 8 -Ġ= ==== -s aving -Ġer osion -ĠQu in -Ġearn s -Ġaccomplish ment -ĠWe i -Ġ< [ -____ _ -Ġir rig -ĠT eddy -Ġconqu ered -ĠArm ored -Ġassert s -Ġmanip ulating -r é -Ġtranscript s -G allery -Ġplot ting -Ne il -Ġbetray al -load er -ĠS ul -Ġdispl acement -Ġroy alty -ĠW I -he it -ĠDev ices -alle l -Ġmunicipal ities -Ġcan al -St ars -ĠU AE -Ġ" âĢ¦ -ĠC U -ab ove -Ġreson ance -ĠguiActive Un -add ed -ĠBra ves -ĠI bn -Ġhere by -ĠB RE -Ġshare holder -ĠH ir -ĠJ i -Ġstrange ly -Ġadm ired -Ġpl ight -Ġb achelor -ĠP ole -cipl inary -T ony -ĠArmen ian -Ġun man -ĠZion ist -St age -isco ver -Ġautom otive -Ġs idelines -Ġsl ick -ĠRena issance -ĠF UN -Im ages -ĠH aj -Ġp ing -Ġshort cut -ĠBl vd -ĠLook s -Ġbur sts -Ġcl amp -Ġm ish -Ġsort ing -Ġpatri ot -Ġcorrect ness -ĠScand inav -ĠCaval iers -p ython -az ar -Ġ3 75 -ĠJa une -40 9 -Ġdetrim ental -Ġstab bing -Ġpoison ed -Ġf ountain -oc ent -or st -ĠMar i -Ġr ains -ĠO vers -ĠInst itution -ud get -AM Y -t ale -ĠK R -ĠPr ices -Ġhead aches -Ġlands l -ĠA ura -Bon us -ĠZ hao -ĠH ip -Ġhop s -ĠKurd istan -Ġexplo iting -ry n -Ġhypocr isy -op ening -Ġgun shot -Ġw ed -inter stitial -Inter stitial -Ġam en -Bre aking -Ġmarket ed -W ire -ĠC rowd -Contin ue -ĠK nown -ĠEffect ive -ore an -iz ons -Jose ph -Ġescal ation -us ername -Ġcur tain -AT ES -ĠP AR -ĠM iy -Ġcounter fe -l ene -Ġcont enders -d aily -ĠAs c -ĠPhill ip -most ly -Ġfil ename -he ne -Ġresemb ling -Ġst aging -ĠCh loe -Ġw iring -H on -ĠRen ew -ott age -ĠHy brid -m uch -Ġstro kes -Ġpolicy makers -AP TER -ĠArk ham -pl ot -Ġassist ants -Ġde port -ĠSe ga -Ġinflu enza -ĠC ursed -ĠK obe -Ġskin ny -Prov ider -ĠR ip -Ġincrement al -product s -B F -Ġd ome -ĠC redits -Ġlos ers -int s -ĠBet ty -ĠTal ent -ĠD AM -L v -E ss -Ġd ens -tem p -J udge -od ic -Ġ' ( -UR ES -ets k -V O -Ġretrie ved -Ġarchitect s -Ù ĩ -Ġeth ic -ĠSecond ary -st ocks -ad ia -Ġ3 25 -ĠOp inion -Ġsimultane ous -Ġd izz -ul p -Ġsmugg ling -ipp ery -R andom -f acing -ĠD as -Ġstock p -Ġdiscl osures -po inter -Ġcor al -ĠSe lection -ĠP ike -ival ent -Ġruth less -ĠR im -Ġensu ing -ĠExper iment -Ġcongress man -Ġbelie ver -Ġun specified -ĠM ord -Ġknowledge able -ĠV ERY -T X -Ġstra ps -Ġtur f -apesh ifter -Ġmar ital -Ġfl ock -ãģ Ĩ -26 3 -AM ES -ĠOpp osition -Ġtre asures -ĠG OD -Ġmodel ed -ĠWOR LD -Ġ( [ -ĠUs age -H F -Ġ$ ( -uss ed -Ġpione er -E ight -par se -b read -rit z -ĠMir anda -ĠK ant -++ ) -ore n -Ġprov oked -Ġbre eds -ĠIn cludes -ĠPast ebin -ĠFl ip -J ava -Ġbr ink -Ġrum ored -Ġun seen -Ġgar nered -ĠDef in -al ted -Ġtatt oos -Ġhes itation -is itions -ĠWe aver -ĠReport ing -Ġtherap ies -Ġconsult ants -Ġresid ual -ĠMal i -ĠRom a -i ago -ĠRes idents -ub i -Ġremed ies -Ġadapt ive -ĠAl ive -ĠBar cl -Ġwal lets -c rypt -etermin ation -ĠPel osi -Ġsl ipping -oton in -Ġall iances -pat rick -ir is -Ġor th -ĠPer kins -ĠDe V -ĠG ets -Ġdry ing -ge e -fore st -ĠFor get -ore m -33 9 -Ġvague ly -ĠD ion -ĠP orn -ĠH OW -Ġp neum -Ġrub ble -ĠT aste -enc ia -ĠG el -Ġd st -Ġ24 5 -ĠMoroc co -inf lamm -ĠTw ins -Ġb ots -d aughter -ĠB alk -Ġbre thren -Ġlog os -Ġgo bl -f ps -Ġsub division -Ġp awn -Ġsquee zed -Ġmor ale -ĠD W -' " -Ġkn ot -ook y -Ġdiv isive -Ġboost ed -ch y -ãĥ IJ -if act -Ġnewcom ers -ĠWrest ling -Ġsc outs -w olves -R at -Ġnin eteenth -ĠOs borne -St ats -Ġem powered -Ġpsych opath -ĠO EM -ugg age -ĠP K -ĠMoh ammad -P ak -Ġanarch ists -ĠExt ract -est hes -ĠStock holm -l oo -ĠG raph -Ġdeploy ing -ĠStr anger -ĠM old -Ġstaff er -Ġdiscount ed -uck le -ple ase -ĠLand ing -ÃŃ a -Ġ19 3 -Ġan te -Ġrep etition -Ġ+ /- -Ġpar ody -Ġlive ly -AA A -ĠHor us -Ġp its -ind ers -L OC -ĠVen ice -40 6 -ĠDis cover -â Ĩ -ellect ual -Ġp ens -Ġey el -ig uous -Im pl -Ġj oking -Ġinv al -ĠBel fast -Ġcredit ors -ĠSky walker -ov sky -Ġcease fire -Ġse als -is oft -) ). -ĠFel ix -IT S -Ġt resp -ĠBlock chain -ew are -ĠSch war -en ne -mount ed -ĠBe acon -les h -Ġimmense ly -Ġche ering -Em ploy -sc ene -ish ly -atche wan -ĠNic olas -Ġdr ained -ĠEx it -ĠAz erb -j un -Ġflo ated -u ania -De ep -Ġsuper v -Ġmyst ical -ĠD ollar -ĠApost le -ĠR EL -ĠProv ided -ĠB ucks -ãĥ ´ -cut ting -Ġenhance ments -ĠPengu ins -ĠIsa iah -Ġj erk -ĠW yn -Ġst alled -Ġcryptoc urrencies -ĠR oland -sing le -Ġl umin -ĠF ellow -ĠCap acity -ĠKaz akh -W N -Ġfin anced -38 9 -Ġt id -Ġcoll usion -ĠMy r -î Ģ -Sen ator -Ġped iatric -Ġneat ly -Ġsandwic hes -ĠArchitect ure -Ġt ucked -Ġbalcon y -Ġearthqu akes -qu ire -F uture -Ġhe fty -é Ĺ -Ġspecial izes -Ġstress es -Ġs ender -Ġmisunder standing -Ġep ile -Ġprov oke -ĠCol ors -Ġdis may -uk o -[ _ -58 6 -ne utral -Ġdon ating -ĠRand all -Mult i -Ġconvenient ly -ĠS ung -ĠC oca -Ġt ents -ĠAc celer -Ġpart nered -27 2 -ir ming -ĠB AS -s ometimes -Ġobject ed -ub ric -p osed -LC S -gr ass -Ġattribut able -V IS -Israel i -Ġrepe ats -ĠR M -v ag -ut a -in ous -Ġin ert -ĠMig uel -æ Ń -ĠHawai ian -B oard -Ġart ific -ĠAzerb ai -as io -ĠR ent -A IN -Ġappl iances -Ġnational ity -Ġass hole -ĠN eb -Ġnot ch -h ani -ĠBr ide -Av ailability -Ġintercept ed -Ġcontin ental -Ġsw elling -ĠPers pect -b ies -. < -ith metic -ĠL ara -Ġtempt ing -add r -Ġoversee ing -cl ad -ĠD V -ĠGing rich -Ġm un -ĠApp ropri -Ġalter ations -ĠPat reon -Ġha voc -Ġdiscipl ines -Ġnotor iously -aku ya -ier i -? ). -ĠW ent -Ġsil icon -Ġtre mb -Cont ainer -K nown -Ġmort ar -est e -ick a -Ar thur -ĠPre viously -ĠMart y -Ġsp arse -g ins -Ġin ward -ĠParticip ant -C opy -ĠM isc -Ġantib iotic -ĠRet ro -Ġel usive -Ġass ail -ĠBatt alion -ĠB ought -Ġdimin ish -ĠEuro pa -s ession -ĠDanger ous -ies el -Ġdisbel ief -Ġbl asts -ext reme -ĠBoy d -ĠProject s -ĠGu ys -Ġunder gone -Ġgr ill -ĠDw ight -Ġ19 7 -US ER -Ġfiles ystem -Ġcl ocks -T aylor -Ġwra pper -Ġfold ing -ous and -ĠPhilipp ine -ATION AL -ĠPer th -Ġas hes -Ġaccum ulate -ĠGate way -Sh op -orks hire -H an -ĠBar rel -ĠLe h -ĠX V -Ġwh im -Ġrep o -ĠC G -ĠM am -Ġincorpor ating -Ġbail out -Ġlingu istic -Ġdis integ -C LE -Ġcinem atic -ĠF iber -S yn -il ion -ĠCom pos -c hens -Ġne oc -Ġbo iled -F INE -on o -un cle -ik en -ĠB M -Î ¹ -Ġreceipt s -Ġdisp osed -ĠTh irty -ĠR ough -ĠA BS -Ġnot withstanding -oll en -# $ -Ġunrel iable -Ġbl oom -Ġmedi ocre -Ġtr am -ĠTas man -Ġsh akes -Ġmanifest o -ĠM W -Ġsatisf actory -Ġsh ores -Ġcomput ation -Ġassert ions -orm ons -ar ag -ab it -Dem ocrats -ĠL oot -ĠVol ks -ha ired -Ġgrav itational -S ing -ĠM iz -Ġthro ttle -Ġtyr anny -ĠView s -Ġrob ber -ĠMinor ity -Ġsh rine -sc ope -pur pose -Ġnucle us -our cing -ĠUS DA -ĠD HS -w ra -ĠBow ie -Sc ale -ĠB EL -x i -I ter -Ġ( ), -w right -Ġsail ors -ous ed -NAS A -ĠPro of -ĠMin eral -t oken -ĠF D -R ew -Ġe ll -6 30 -Ġchance llor -ĠG os -Ġamount ed -ĠRec re -ome z -ĠOpt im -ĠOl ive -Ġtrack er -ow ler -ĠUn ique -R oot -Ġmar itime -ĠQur an -ĠAd apt -Ġecosystem s -ĠRe peat -ĠS oy -ĠI MP -Ġgrad uating -and em -P ur -ĠRes et -ĠTr ick -ĠPh illy -ĠT ue -ĠMalays ian -Ġclim ax -Ġb ury -Ġcons pic -ĠSouth ampton -ĠFl owers -Ġesc orted -ĠEduc ational -ĠI RC -Ġbrut ally -e ating -Ġpill ar -ĠS ang -ĠJ ude -ar ling -ĠAm nesty -Ġrem inding -ĠAdminist rative -hes da -Ġfl ashed -ĠP BS -per ate -fe ature -Ġsw ipe -Ġgra ves -oult ry -26 1 -bre aks -ĠGu er -Ġsh rimp -ĠV oting -qu ist -Ġanaly tical -Ġtables poons -ĠS OU -Ġresear ched -Ġdisrupt ed -Ġj our -Ġrepl ica -Ġcart oons -b ians -} ) -c opy -G ot -ou ched -P UT -Ġsw arm -not ations -s aid -Ġreb uilt -Ġcollabor ate -Ġr aging -Ġn ar -Ġdem ographics -ĠD DR -Ġdist rust -oss ier -ĠK ro -Ġpump kin -Ġreg rets -Ġfatal ities -ĠL ens -ĠO le -p d -Ġpupp et -ĠOut look -ĠSt am -O l -F air -U U -Ġre written -Ä ± -Ġfasc inated -Ġve ctors -Ġtrib unal -u ay -ĠM ats -ĠCo ins -[ [ -Ġ18 1 -Ġrend ers -ĠK aepernick -Ġesp ionage -Ġsum m -Ġd itch -Acc ount -Ġspread sheet -Ġmut ant -p ast -40 7 -Ġd ye -Ġinit iation -Ġ4 000 -Ġpunish able -Ġth inner -ĠKh al -Ġinter medi -D un -ĠGoth am -Ġeager ly -Ġvag inal -p owers -V W -ĠWATCH ED -Ġpred ator -ams ung -Ġdispar ity -Ġ[ * -Ġam ph -Ġout skirts -ĠSpir its -Ġskelet al -Ð » -ĠR ear -Ġissu ance -ĠLog ic -re leased -Z Z -ĠB ound -Ent ry -Ġex its -is ol -ĠFound er -Ġw re -ĠGreen land -ĠM MO -t aker -IN C -ãģ ¾ -Ġhour ly -hen ko -Ġfantas ies -Ġdis ob -Ġdemol ition -ãĥ ĭ -Ġen listed -rat ulations -Ġmis guided -Ġens ured -Ġdiscour aged -m ort -Ġfl ank -Ġc ess -Ġreact s -ĠS ere -s ensitive -ĠSer pent -ass ad -Ġ24 7 -Ġcalm ly -b usters -Ġble ed -ĠSt ro -Ġamuse ment -ĠAntar ctica -Ġs cept -ĠG aw -a q -ason ic -Ġsp rawling -n ative -atur ated -ĠBattle field -IV ERS -E B -ĠG ems -ĠNorth western -ĠFil ms -ĠAut omatic -Ġappre hend -ãģ ¨ -Ġgui Name -Ġback end -Ġevid enced -ge ant -01 2 -ĠS iege -Ġexternal To -Ġunfocused Range -ĠguiActiveUn focused -Ġgui Icon -ĠexternalTo EVA -ĠexternalToEVA Only -F ri -ch ard -en aries -Ġchief s -Ġc f -ĠH UD -Ġcorro bor -Ġd B -ĠT aken -ĠPat ricia -ra il -ĠCh arm -ĠLiber tarian -rie ve -Person al -ĠO UR -ger ies -Ġdump ing -Ġneurolog ical -it imate -ĠClint ons -raft ed -ĠM olly -Ġtermin als -reg ister -Ġfl are -Ġenc oded -Ġautop sy -p el -m achine -Ġexempt ions -ĠRoy als -d istance -Ġdraft s -Ġl ame -ĠC unning -Ġsp ouses -ĠMark ets -ĠCar rier -Ġimp lying -ĠY ak -s id -Ġl oser -Ġvigil ant -Ġimpe achment -Ġaug mented -ĠEmploy ees -Ġunint ended -tern ally -ĠW att -Ġrecogn izable -ess im -æ Ŀ -Ġco ated -r ha -Ġlie utenant -ĠLegisl ation -pub lished -44 4 -01 3 -Ġide ally -ĠPass word -Ġsimpl ify -ĠMet a -ĠM RI -Ġple ading -organ ized -hand ler -Ġun ravel -cor rect -Ġ icy -Ġparan oid -Ġpass er -Ġinspect ions -of er -ĠHealth care -28 3 -ĠBr ut -iol a -for ge -ĠMed ieval -MS N -ie vers -ĠProgram ming -å ī -Ġ2 23 -m u -ĠC LE -ug a -Ġsho ppers -Ġinform ative -ĠPl ans -Ġsupplement ation -ĠT ests -ty ard -ocy tes -ĠVeg a -ĠGujar at -erman ent -Ex cept -ĠL OT -all a -ĠC umm -ĠO sw -Ġven om -ĠDeb t -ĠD OWN -Ġreun ion -Ġm uc -ĠRel ief -Ġge op -ĠðŁ ĺ -al ogue -An th -ech o -Ġcor ros -Ġrepl ication -ĠBl azing -ĠD aughter -Ġinf lic -ĠLind sey -Ù Ī -28 4 -Ex it -Ġgl oom -TA IN -Ġundermin ing -Ġadv ising -h idden -Ġover flow -Ġg or -urd ue -Ġe choes -enh agen -Ġimp uls -d rug -c ash -Ġas ync -Ġmir ac -at ts -p unk -Ġpiv ot -ĠLegisl ative -Ġblog gers -ĠCl aw -s burg -d yl -ĠRecomm end -Ġver te -Ġprohib iting -ĠPant her -Jon athan -Ġo min -Ġhate ful -28 1 -ĠOr che -ĠMurd och -down s -Ġas ymm -G ER -Al ways -Ġinform s -ĠW M -ĠP ony -ĠApp endix -ĠAr lington -J am -Ġmedic inal -ĠS lam -IT IES -Ġre aff -ĠR i -F G -S pring -b ool -Ġthigh s -Ġmark ings -ĠRa qqa -ĠL ak -p oll -ts ky -ĠMort y -ĠDef inition -Ġdeb unk -end ered -ĠLe one -a vers -Ġmortg ages -App arently -N ic -ha us -ĠTh ousands -au ld -Ġm ash -sh oot -Ġdi arr -Ġconscious ly -H ero -e as -ĠN aturally -ĠDestroy er -Ġdash board -serv ices -R og -Ġmillenn ials -Ġinv ade -- ( -Ġcomm issions -ĠA uckland -Ġbroadcast s -Ġfront al -Ġcr ank -ĠHist oric -Ġrum ours -CT V -Ġster il -Ġboost er -rock et -ãĤ ¼ -ut sche -ĠP I -Ġ2 33 -ĠProdu cer -ĠAnaly tics -Ġinval uable -Ġunint ention -ĠC Y -Ġscrut in -Ġg igg -Ġeng ulf -Ġprolet ariat -Ġh acks -ĠH ew -ar ak -ĠSl ime -ield ing -ag her -ĠEll iot -Ġtele com -Ġ2 19 -ult an -ĠAr bor -ĠSc outs -B an -Ġlifes pan -Ġbl asp -38 8 -Ġjud iciary -ĠContin ental -ask ing -Mc C -L ED -Ġbag gage -ĠSorce rer -Ġrem nants -ĠGriff ith -ets u -ĠSub aru -ĠPerson ality -des igned -ush ima -agn ar -Ġrec oil -Ġpass ions -\ ": -Ġte e -Ġabol ition -ĠCreat ing -j ac -Ġ19 4 -01 9 -Ġpill ars -ric hed -/ " -t k -Ġlive lihood -Ġro asted -ah on -ĠH utch -ass ert -Ġdivid end -Ġkn it -Ġd aunting -Ġdisturb ance -Ġsh ale -Ġcultiv ated -Ġrefriger ator -L B -ĠN ET -Ġcommercial s -Ġthink ers -45 5 -Ġch op -B road -Ġsuspic ions -Ġtag ged -l ifting -Ġsty lish -ĠShield s -Short ly -Ġt ails -A uth -ST E -ĠG AME -Ġse ism -ĠK is -olog ne -Ġcow ork -Ġforc ibly -Ġthy roid -ĠP B -AN E -mar ried -h orse -Ġpoly mer -ĠCh al -od or -DE BUG -ĠCon text -Ġbl iss -Ġpin point -ĠMat hemat -leg ram -ĠWeek end -Ġlab elled -Ġb art -it les -Ġest rogen -âĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶ âĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶ -" ' -Ġvis ibly -Ġouts ider -aid a -Are a -Ġdisse min -Ġdish onest -ĠCl osed -ĠBullet in -ĠRam sey -sw ord -ĠX I -our ced -S ame -34 6 -ĠRe pe -ĠK ou -c ake -em is -C ache -ĠMe aning -ĠEn light -onom y -Ġmanifest ation -sw orth -J ay -Ġch ore -ö r -D ream -Ġsanction ed -Ġcult urally -ĠA ra -N av -Ġthe ological -Ġstr ut -ĠV O -ĠHand book -Ġconstruct ing -Ġ ¶ -ĠBenef its -ĠPsych ological -s ac -å ¸ -p olicy -ĠMat ters -ĠReport ed -ĠBy te -Ġvit ro -ĠM aiden -Ġl am -ĠJenn ings -Ġgar ment -ĠRut gers -ĠStaff ord -ĠWell ington -Ġinter mitt -Ġn pm -Ġord eal -Ġplug ged -o oming -in ished -fram ework -Ġtim ber -Ġc ass -Ġ8 50 -il ess -ĠRed ux -7 68 -St re -Ġsurpass ed -w hel -Ġparalle ls -Ġve il -ĠG I -ĠR EST -Ġread iness -s ort -Ġmod ifying -ĠSl ate -ru ff -Ġmar ble -Ġinf rared -Ġaud itor -ĠFANT ASY -ĠP overty -ĠS PD -Ġ" ( -K y -RA Y -Ġexecut ions -ĠBever ly -ĠMarx ism -ĠBur st -ĠK ali -est ones -Clear ly -E ll -ãģ § -ĠProceed ings -T oken -IF IC -ñ a -Cent ral -ĠH aley -ĠD rama -Ġform ations -OR N -Book s -Ġdom inating -ĠFly ers -ĠCompan ion -Ġdiscipl ined -ĠYug oslav -ĠSpell s -Ġv engeance -Ġland lords -L en -ĠO gre -ano ia -Ġpier cing -Ġcon greg -Ġscore r -ob ia -Ġnic kel -ĠLear ns -Ġre jo -Ġmaster piece -Fl ash -Ġinhab ited -ĠOpen GL -ĠD ud -ĠI CO -Ġar ter -Ġpl ur -Ġmaster y -Ġlong standing -st ed -Ġw ines -Ġtelev ised -ĠSh rine -ĠBay ern -Ġâ ĵĺ -Ġencl osure -j ohn -Ġprophe ts -ĠRes urrection -ĠOrd ers -Ġun even -r als -Ġd wind -ĠL ah -ĠSl oven -37 8 -Ġins istence -aff le -ĠCl one -Ġhard ship -ĠCongress man -Ġple ad -Ġreview ers -Ġc ured -Ġ19 35 -as ley -f ake -ĠTh inking -yd ia -P ART -ĠD ota -o it -Ġwh ipped -Ġb ouncing -ĠHispan ics -com ings -Ġcann abin -ĠCh ambers -ĠZ ack -Option al -Ġco ats -Ġprow ess -ĠNort on -Ġplain ly -Ġfre ight -Ġinhib ition -Ġcl am -Ġ30 3 -ke f -ale igh -L uke -Ġpsych o -ator ium -M ED -Ġtreat ies -Ġind isc -Ġd c -OP S -Ġresil ient -ĠInter state -Ġsl ack -Ġmund ane -Ġestab lishes -35 9 -Ġstr ained -Ġn ond -S us -Ġcast e -ar ate -ie ving -Ġunfair ly -Ġpars er -on ial -urs ive -V ia -ĠOtt o -ĠAuthor ities -stro ke -K R -ĠMer cy -Ġfurn ished -Ġout set -Ġmet ic -19 82 -olith ic -ĠT ent -og ical -ĠA ircraft -Ġh ides -ĠBec ame -Ġeduc ators -re aching -Ġvol atility -Ġtodd ler -ĠNAS CAR -ĠTw elve -ĠHigh lights -Ġgra pe -Ġspl its -Ġpe asant -Ġre neg -ĠMS I -Tem p -st ars -Ġtre k -ĠHy de -b inding -Ġreal ism -Ġox ide -ĠH os -Ġmount s -Ġbit ing -Ġcollaps ing -Ġpost al -Ġmuse ums -Ġdet ached -Ġrespect ing -Ġmonop ol -Ġwork flow -ĠC ake -Tem plate -ĠOrgan isation -Ġpers istence -36 9 -C oming -B rad -Ġredund ant -ĠG TA -Ġb ending -Ġrev oked -Ġoff ending -Ġfram ing -Ġprint f -Comm un -mem bers -Out side -Ġconst rued -Ġc oded -F ORE -Ġch ast -Ch at -Ind ian -ĠY ard -? !" -ĠP orts -ĠX avier -ĠR ET -' ." -ĠBo at -iv ated -ich t -umer able -D s -ĠDun n -Ġcoff in -Ġsecure ly -ĠRapt ors -ĠB es -Install ation -Ġin ception -ĠHealth y -end ants -Ġpsych ologists -ĠShe ikh -c ultural -ĠBlack Berry -sh ift -F red -oc he -Ġc akes -ĠS EO -ĠG ian -ĠAs ians -og ging -e lement -Ġpund its -ĠV augh -ĠG avin -Ġh itter -Ġdrown ed -Ġch alk -ĠZ ika -Ġmeas les -80 2 -âĢ¦ .. -ĠAW S -] " -Ġdist ort -ĠM ast -Ġantib odies -ĠM ash -Mem ory -ĠUg anda -ĠPro b -Ġvom iting -ĠTurn s -Ġoccup ying -Ġev asion -ĠTher apy -Ġprom o -Ġelect r -Ġblue print -ĠD re -pr iced -ĠDep ot -Ġallev iate -ĠSom ali -m arg -n ine -Ġnostalg ia -ĠShe pherd -Ġcaval ry -Ġtor ped -ĠBlood y -x b -Ġs ank -Ġgo alt -report print -embed reportprint -clone embedreportprint -ĠIn itially -ĠF ischer -Ġnot eworthy -c ern -Ġin efficient -raw download -rawdownload cloneembedreportprint -c ation -ĠD ynasty -l ag -D ES -Ġdistinct ly -ĠEston ia -Ġopen ness -Ġg ossip -ru ck -W idth -ĠIb rahim -Ġpet roleum -Ġav atar -ĠH ed -ath a -ĠHog warts -Ġc aves -67 8 -Ġsafegu ard -ĠM og -iss on -ĠDur ham -sl aught -ĠGrad uate -Ġsub conscious -ĠEx cellent -ĠD um ----- - -Ġp iles -ĠW ORK -ĠG arn -ĠF ol -ĠAT M -Ġavoid s -ĠT ul -Ġble ak -EL Y -iv ist -light ly -P ers -ĠD ob -ĠL S -Ġins anity -Î µ -atal ie -En large -Ġtw ists -Ġfault y -Ġpir acy -Ġimp over -Ġrug ged -ĠF ashion -Ġs ands -' ? -sw ick -Ġn atives -Ġhe n -ĠNo ise -ãĥ Ĺ -Ġg reens -Ġfree zer -Ġd ynasty -ĠFather s -ĠNew ark -Ġarchae ological -Ġo t -ob ar -Ġblock ade -Ġall erg -L V -Ġdeb it -ĠR FC -ĠMil ton -ĠPress ure -Ġwill ingly -Ġdisproportion ate -Ġopp ressive -Ġdiamond s -Ġbelong ings -19 70 -Ġbell s -Ġimperial ism -Ġ2 27 -Ġexpl oding -ĠE clipse -Ġ19 19 -Ġr ant -Ġnom inations -34 7 -Ġpeace fully -ric a -ĠF UCK -Ġvib ration -mal ink -Ġro pes -ĠIv anka -ĠBrew ery -ĠBook er -ĠOw ens -go ers -Serv ices -ĠSn ape -Ġ19 1 -39 5 -Ġ2 99 -just ice -Ġb ri -Ġdisc s -Ġprom inently -Ġvul gar -Ġsk ipping -l ves -Ġtsun ami -37 4 -ĠU rug -ĠE id -rec ated -p hen -Ġfault s -ĠStart ed -9 50 -Ġp i -Ġdetect or -Ġbast ard -Ġvalid ated -Space Engineers -OUR CE -Ġ( ~ -Ġuns ur -Ġaff irmed -Ġfasc ism -Ġres olving -ĠCh avez -ĠC yn -Ġdet ract -L ost -Ġrig ged -Ġhom age -ĠBrun o -55 5 -ec a -Ġpress es -Ġhum our -Ġsp acing -Ġ' / -olk ien -C oun -OP ER -T re -S on -ĠCambod ia -ier re -m ong -o zy -Ġliquid ity -ĠSov iets -ĠFernand o -Ġ2 29 -Ġsl ug -ĠCatal an -elect ric -Ġsc enery -ĠH earth -Ġconst rained -Ġgoal ie -ĠGu idelines -ĠAm mo -ĠPear son -Ġtax ed -Ġfet us -Resp onse -ĠAlex is -th ia -G uy -Ġrecon struct -Ġextrem es -Ġconclud ing -ĠP eg -ook s -Ġded uctions -R ose -Ġground breaking -ĠT arg -ãĥ ģ -ĠRe ve -res ource -Ġmo ons -Ġelectrom agnetic -Ġamid st -ĠVik tor -N ESS -B ACK -Ġcomm ute -ĠAna heim -Ġfluct uations -6 40 -Ġnood les -ĠCop enhagen -ĠT ide -ĠGri zz -ĠS EE -Ġpip elines -Ġsc ars -end o -ag us -ĠE TF -/ # -ĠBec ome -44 8 -Ġvis c -ĠRecomm ended -Ġj umper -Ġcogn ition -Ġassass in -Ġwitness ing -ĠSet up -Ġl ac -v im -IS M -p ages -SS L -35 8 -Ġad ject -indust rial -l ore -cher y -Ġgl itter -Ġc alf -Flor ida -Ġspoil ers -Ġsucceed s -Ġch anting -Ġslog ans -ĠTr acy -Vis it -rol ogy -Ġm ornings -Ġline age -Ġs ip -Ġintense ly -Ġflour ish -ĠSle eping -ĠF em -or por -ĠK lan -ĠDar th -h ack -ĠNi elsen -Ġtum ors -Ġprocure ment -ĠY orkshire -Ġra ided -K Y -An na -Ġ// [ -ĠDis order -ĠMust ang -ĠW en -ĠTry ing -s q -Ġdeliver ies -Ġshut ter -Ġcere bral -Ġbip olar -ĠC N -l ass -j et -Ġdeb ating -> : -Ġe agle -gr ades -ĠD ixon -UG C -M AS -ĠDr aco -ĠMach ines -aff er -Ġem an - ² -pr on -ĠG ym -Ġcompar atively -ĠTrib unal -PR O -Ġle x -Ġfert ile -Ġdep ressing -Ġsuperf icial -ess ential -ĠHun ters -g p -Ġprom inence -L iber -ĠAn cest -ote chnology -Ġm ocking -ĠTra ff -ĸ ļ -Med ium -I raq -Ġpsychiat rist -Quant ity -ĠL ect -Ġno isy -5 20 -G Y -Ġsl apped -ĠM TV -Ġpar a -p ull -Mult iple -as her -Ġn our -ĠSe g -Spe ll -v ous -ord ial -Sen ior -ĠGold berg -ĠPl asma -ne ed -Ġmess enger -ere t -Ġteam ed -Ġliter acy -ĠLe ah -ĠD oyle -Ġem itted -U X -Ġev ade -Ġm aze -Ġwrong ly -ĠL ars -Ġstere otype -Ġpled ges -Ġarom a -ĠM ET -Ġac re -ĠO D -Ġf f -Ġbrew eries -ĠH ilton -und le -ĠK ak -ĠThank fully -ĠCan ucks -in ctions -ĠApp ears -Ġco er -Ġundermin ed -ro vers -And re -Ġbl aze -um ers -Ġfam ine -amp hetamine -ulk an -Am ount -Ġdesper ation -wik ipedia -develop ment -ĠCor inth -uss ia -Jack son -L I -N ative -R s -Oh io -ĠKath leen -F ortunately -Ġattend ant -ĠPre ferred -ĠDid n -ĠV s -M is -Ġrespond ent -Ġb oun -st able -Ġp aved -Ġunex pl -ĠChe ney -L M -ĠC ull -bl own -Ġconfront ing -oc ese -serv ing -W i -ĠLith uania -ann i -Ġst alk -h d -Ġv ener -AP H -ynchron ous -UR R -um ably -hist oric -H alf -H ay -Ġresil ience -spe ction -Ġabandon ing -O bs -ĠDeb bie -Ġgrad ient -ĠPl aint -ĠCan al -AR CH -Ġexpans ive -Ġfun g -Ġb ounced -U nd -Ġprec autions -Ġclar ification -Ġd agger -Ġgri ps -Ġ µ -ĠRiver a -ĠUnd ead -is ites -ĠFIR ST -ñ o -aud i -Ġhost ages -Ġcompl iant -Ġal umni -Se ven -Ġcyber security -e ither -Col lect -Ġinvari ably -ĠS oci -Ġlaw maker -Ġa le -ĠPerson ally -N azi -Ġcustom ization -ĠPro c -ĠSask atchewan -eat uring -Ġsp ared -Ġdiscontin ued -Ġcomput ational -ĠMotor ola -Ġsuprem acist -government al -Ġparad ise -ĠDown ing -ĠNik on -Ġcat alyst -ber ra -Tor onto -8 75 -bet a -ĠMac ron -Ġunreal istic -ve ctor -ĠVeh icles -it iveness -ĠR V -ĠCol bert -s in -o ji -ent in -ĠKr ish -hell o -ff ield -ok y -ĠT ate -Ġmap le -Ġa ids -chem ical -33 4 -n uts -ĠWar p -Ġx x -ĠRob b -umer ous -_- _ -ft ime -ĠV W -Ġw inger -ĠD ome -t ools -ĠP V -ĠGe orgetown -Ġg eared -Ġjihad ists -Ġc p -Ġster oids -M other -cler osis -ĠDR M -nes ia -Ġl inger -Ġimm ersive -ĠC OUN -Ġoutwe igh -ens ual -B and -Ġtransform s -mat ched -ps ons -ĠJud icial -f actor -Ġrefer ral -Ġodd ly -ĠW enger -B ring -ĠB ows -60 2 -IC LE -Ġl ions -ĠAcad emic -ĠTh orn -ĠRa ider -kef eller -St orage -L ower -ĠOr t -ĠEqu ality -AL T -ĠS OC -T ypes -Ġl yn -ĠAss et -co at -TP P -C VE -ĠPione er -app lication -Mod ern -ĠH K -En vironment -Al right -R ain -IP P -ĠShi ite -Ġm ound -ĠAb ilities -cond ition -St aff -Ġcompet ence -ĠM oor -ĠDi ablo -Ġwith held -Ġost ensibly -ĠB rom -Ġms g -Ġden omin -ĠRef erences -ĠF P -Ġplun ged -Ġp amph -m oving -cent ral -Ġdown right -Ġf ading -T al -T yp -ĠTh y -uk es -it he -Ġo ve -Ġbatt led -Ġseaf ood -Ġfig ur -ĠR D -c rop -Ġsqu ads -{ \ -à ¹ -ĠE h -Ġinterview ing -ĠQ in -Ġas piring -PL IC -Ġcla uses -ĠG ast -ĠN ir -Ġl uggage -Ġh ose -Ġsystem d -Ġdesc ending -ĠRev ised -ĠR ails -al ign -70 9 -33 7 -Ġf ug -charg ing -t ags -Ġut er -k ish -WAR NING -49 0 -prof its -Ġvoy age -Ġa ce -ĠV anguard -ĠT anks -ĠM uk -Ġ2 26 -S afe -Ar mor -Ġvolcan ic -Ġwom b -ĠM IL -Ġbegin ner -ĠRec ogn -ĠA AP -PL AY -) ! -Ġdetect ing -c n -Ġbre aches -Bas ically -ĠP ag -ĠMunicip al -ĠInd ie -ĠL af -ĠDis able -ĠOl son -Ġrest rained -Ġrul ings -Ġhum ane -ev ents -ĠCinem a -display Text -ĠH atch -action Date -onna issance -Ġassault ing -ĠL ug -CH AT -Ġvig orous -ĠPer se -Ġintoler ance -ĠSnap chat -ĠSh arks -Ġd ummy -ĠDi agn -ĠGu itar -im eters -40 3 -RE G -A x -Ġsepar ates -ĠMah m -Ġt v -j ah -O OL -C irc -ĠWinds or -uss ian -Ġintu ition -Ġdis dain -ĠDon ovan -Ġ2 21 -E mb -Ġcondem ning -Ġgener osity -zz y -Ġpant ies -ĠPre vent -Action Code -AN A -34 2 -external ActionCode -Ġspec ifying -Ġcryst all -J ere -Ġru pt -ĠApp rentice -Ġprof iling -Ð º -St rike -Ġsid eline -Ġoblig ated -Ġocc ult -Ġbureaucr atic -ant ically -rupt ed -neg ative -ĠEthiop ia -ĠC ivic -Ġins iders -el igible -ĠTV s -ĠB AR -ĠT I -i ologist -ĠA IR -Ġsubstit uted -Ar ab -ĠS aul -ĠY og -p rem -Ġbuild ers -Ġstation ary -Ġdoubt ful -Ġvig orously -Ġthr illing -Ph ysical -ĠCare y -ĠHyd ra -geon ing -ĠS ly -y ton -Ġborrow ers -ĠPark inson -Ġ ë -ĠJama ica -Ġsat ir -Ġinsurg ents -ĠF irm -Ġis ot -ĠK arn -our ning -ak ens -doc s -l ittle -ĠMon aco -CL ASS -Tur key -L y -ĠCon an -ass ic -Ġstar red -ĠPac ers -et ies -Ġt ipping -M oon -ĠR w -s ame -Ġcav ity -Ġgo of -ĠZ o -Sh ock -um mer -Ġemphas izes -Ġreg rett -Ġnovel ty -Ġen vy -ĠPass ive -r w -50 5 -Ġind ifferent -ĠR ica -ĠHim self -ĠFred die -Ġad ip -ä¸ Ģ -Ġbreak out -Ġhur ried -ĠHu ang -ĠD isk -Ġro aming -?????- ?????- -U V -ĠRick y -ĠS igma -Ġmarginal ized -Ġed its -Ġ30 4 -mem ory -Ġspec imen -29 3 -ãģ ¯ -Ġvert ically -Ġaud ition -ĠHe ck -Ġc aster -ĠHold ings -ad al -ĠC ron -ĠL iam -Ġdef lect -P ick -ĠDeb ug -RE F -Ġvers atility -ot hes -class ified -ĠMah ar -ĠH ort -C ounter -st asy -not iced -33 1 -ĠSh im -f uck -ĠB ie -Ġair ing -ĠPro tein -ĠHold ing -Ġspect ators -ili ated -ĠThat cher -n osis -ãĥ¼ ãĥ³ -Te le -B oston -ĠTem pl -st ay -Ġdecl arations -47 9 -Vol ume -ĠDesign er -ĠOver watch -id ae -Ġon wards -Ġn ets -ĠMan ila -part icularly -Ġpolit ic -o other -Ġport raits -Ġpave ment -c ffff -Ġs aints -Ġbegin ners -ES PN -Ġshort comings -âķIJ âķIJ -Ġcom et -ĠOrgan ic -qu el -Ġhospital ized -Bre ak -Ġpe el -dyl ib -asp x -ur ances -ĠT IM -P g -Ġread able -ĠMal ik -Ġm uzzle -Ġbench marks -d al -ĠV acc -ĠH icks -60 9 -ĠB iblical -he ng -Ġover load -ĠCivil ization -Ġimm oral -Ġf ries -ãĤ Ĵ -Ġreprodu ced -Ġform ulation -j ug -ire z -g ear -Ġco ached -Mp Server -ĠS J -ĠK w -In it -d eal -ĠO ro -ĠL oki -ĠSong s -Ġ23 2 -ĠLou ise -asion ally -Ġunc ond -olly wood -Ġprogress ives -ĠEn ough -ĠDo e -Ġwreck age -Ġbr ushed -ĠBase Type -Ġz oning -ish able -het ically -ĠC aucus -ĠH ue -Ġk arma -ĠSport ing -Ġtrad er -Ġseem ing -ĠCapt ure -4 30 -b ish -Ġt unes -Ġindo ors -ĠSp here -ĠD ancing -TER N -Ġno b -ĠG ST -m aps -Ġpe ppers -F it -Ġoverse es -ĠRabb i -ĠR uler -vert ising -off ice -xx x -Ġra ft -Ch anged -Ġtext books -L inks -ĠO mn -ãĢ ij -Ġinconven ience -ĠDon etsk -= ~ -Ġimplicit ly -Ġboost s -ĠB ones -ĠBo om -Cour tesy -Ġsens ational -AN Y -Ġgre edy -ed en -Ġinex per -ĠL er -ĠV ale -Ġtight en -ĠE AR -ĠN um -Ġancest or -S ent -ĠH orde -urg ical -all ah -Ġsa p -amb a -ĠSp read -tw itch -Ġgrand son -Ġfract ure -Ġmoder ator -ĠSe venth -ĠRe verse -Ġestim ation -Cho ose -Ġpar ach -Ġbar ric -ãĢ IJ -Ġcomp ass -Ġall ergic -âĢ ķ -OT HER -err illa -Ġw agon -Ġz inc -Ġrub bed -ĠFull er -ĠLuxem bourg -ĠHoo ver -Ġli ar -ĠEven ing -ĠCob b -est eem -Ġselect or -ĠB rawl -is ance -ĠE k -Ġtro op -Ġg uts -ĠApp eal -ĠTibet an -Ġrout ines -ĠM ent -Ġsummar ized -steam apps -Ġtr anqu -Ġ19 29 -or an -ĠAut hent -Ġg maxwell -Ġappre hens -Ġpo ems -Ġsa usage -ĠWeb ster -ur us -Ġthem ed -Ġl ounge -Ġcharg er -Sp oiler -Ġsp illed -h og -ĠSu nder -ĠA in -ĠAng ry -Ġdis qual -ĠFrequ ency -ĠEther net -Ġhel per -Per cent -Ġhorr ifying -Ġa il -ĠAll an -EE E -ĠCross ing -44 9 -Ġh olog -ĠPuzz les -ĠGo es -eren n -60 4 -ãģ ı -ĠRaf ael -Ġatt en -ĠE manuel -Ġup ro -ĠSus p -P sych -ĠTr ainer -ĠN ES -ĠHun ts -bec ue -Ġcounsel or -R ule -Ġtox ins -Ġb anners -r ifice -Ġgreet ing -Ġfren zy -Ġall ocate -Ġ* ) -ex pr -50 3 -ĠCh ick -ĠT orn -Ġconsolid ation -ĠF letcher -sw itch -fr ac -cl ips -ĠMcK in -ĠLun ar -Mon th -IT CH -Ġscholar ly -rap ed -39 8 -Ġ19 10 -Ġe greg -Ġin secure -Ġvict orious -cffff cc -Ġsing led -Ġel ves -ĠW ond -bur st -Ġcam oufl -ĠBL ACK -Ġcondition ed -ç ī -ans wered -Ġcompuls ory -asc ist -Ġpodcast s -ĠFrank furt -bn b -Ġne oliberal -ĠKey board -ĠBel le -w arm -Ġtrust s -Ġins ured -ĠBu cc -us able -60 7 -ĠPl ains -Ġ18 90 -Ġsabot age -Ġlod ged -f elt -Ġg a -ĠN arc -ĠSal em -Ġsevent y -ĠBl ank -p ocket -Ġwhis per -Ġm ating -om ics -ĠSal man -ĠK ad -Ġan gered -Ġcoll isions -Ġextraord inarily -Ġcoerc ion -G host -b irds -è Ģ -k ok -Ġper missible -avor able -Ġpo inters -Ġdiss ip -ac i -Ġtheat rical -ĠCos mic -Ġforget ting -Ġfinal ized -å¤ § -y out -l ibrary -Ġbo oming -ĠBel ieve -ĠTe acher -ĠL iv -ĠGOOD MAN -ĠDomin ican -OR ED -ĠPart ies -Ġprecip itation -ĠSl ot -R oy -ĠComb ined -Ġinteg rating -Ġch rome -Ġintest inal -ĠRe bell -Ġmatch ups -Ġblock buster -ĠLore n -ĠLe vy -Ġpre aching -ĠS ending -ĠPur pose -ra x -f if -Ġauthor itative -ĠP ET -ast ical -Ġdish on -Ġchat ting -Ġ"$ :/ -Connect ion -Ġrecre ate -Ġdel inqu -Ġbro th -ĠD irty -ĠAd min -z man -Ġscholars hips -Ġ25 3 -cont act -als a -7 67 -c reen -abb age -Ġ19 15 -Ġbl ended -Ġal armed -L anguage -35 6 -Ġbl ends -ĠCh anged -W olf -Ġhe pat -Creat ing -Ġper secut -Ġsweet ness -art e -Ġforfe iture -ĠRober to -im pro -N FL -ĠMag net -Det ailed -Ġinsign ificant -ĠPOL IT -ĠBB Q -ĠC PS -Ġse aw -amin er -m L -end if -f inals -Ġ26 5 -u ish -Ġ} ) -ĠPro blems -Ġem blem -Ġserious ness -Ġpars ing -Ġsubst itution -Ġpress ured -Ġrecy cled -ale b -Rub y -Ġprof iciency -Dri ver -ĠW ester -: ' -AF TA -Ġm antle -ĠClay ton -fl ag -Ġpractition er -c overed -ĠSt ruct -add afi -4 25 -ĠTown ship -ĠHyd ro -Lou is -34 3 -Ġcond o -ĠT ao -Ġutil ization -Ġnause a -ĠDem s -rid ges -p ause -Ġform ulas -Ġchall enger -37 6 -Ġdefect ive -ĠRail way -ĠPub Med -Ġyog urt -l bs -ĠNor folk -OP E -ĠMood y -Ġdistribut or -Ġscroll s -Ġextract s -St an -Ġv iability -Ġexp oses -Ġstar vation -ĠStep s -ĠD odd -f ew -ST D -33 2 -Ġclos ures -Ġcomplement ary -ĠS asha -ump y -Ġmon et -Ġartic ulate -ĠDo ct -k iller -Ġsc rim -Ġ2 64 -Ġprost itutes -Ġse vered -Ġattach ments -Ġcool ed -L ev -ĠF alk -f ail -Ġpolic eman -ĠD ag -Ġpray ed -ĠK ernel -Ġcl ut -Ġc ath -Ġan omaly -St orm -em aker -ĠBreak fast -ul i -o ire -J J -h z -Oper ation -ĠS ick -35 4 -ĠGuatem ala -R ate -Ġexp osures -f aces -ĠArch ae -ra f -ĠM ia -Ġ20 25 -Ġop aque -Ġdisgu ised -ĠHead quarters -S ah -Ġp ots -9 78 -ĠM alf -Ġfrown ed -Ġpoison ous -ĠCon vers -ee ks -Ġcr ab -." " -Ġtre ason -Ġr anc -Ġescal ating -Ġwar r -Ġmob s -Ġl amps -ĠSun shine -ĠBrun swick -Ph ones -Ġspe lled -ĠSk ip -Ġ20 50 -Ġ19 11 -ĠPl uto -ĠAm end -Ġme ats -38 7 -Ġst omp -ĠZh ou -ĠLevi athan -ĠHaz ard -ad v -ĠOr well -Ġal oud -Ġb umper -ĠAn arch -ub untu -ĠSer ious -f itting -ĠOption al -ĠCec il -RE AM -Ġser otonin -Ġcultiv ate -ag ogue -} \ -Ġmos ques -ĠSun ny -Ġre active -rev olution -ĠL up -ĠFed ora -Ġdefense man -ĠV ID -ist ine -Ġdrown ing -ĠBroad casting -Ġthr iller -ĠS cy -Ġacceler ating -Ġdirect s -od ied -b ike -d uration -Ġpain fully -R edd -Ġproduct ions -Ġg ag -Ġwh ist -Ġs ock -Ġinf initely -ĠConc ern -ĠCit adel -Ġlie u -Ġcand les -ogene ous -arg er -Ġheaven ly -inflamm atory -Per formance -C s -ruct ose -az aki -Ġp essim -Ġinf erence -Ġpow d -ĠZ oe -Ġpain ts -Ġd azz -pt a --------- --- -Ġins pir -ĠExper imental -ĠKn ife -reg or -b ors -Ġshow ers -rom eda -Ġs aint -Ġben ign -ĠJ iang -Ġenvision ed -Ġsh roud -IF T -H O -Ġsh uff -ĠI CC -Ġse greg -Ġrevis it -ighth ouse -L i -Ġsub strate -ĠSe as -ĠRew ard -ĠH ep -ĠBr ass -s bm -Ġelim inates -Ġst amina -ĠV AT -ĠLo an -Ġconst raint -Ġappropri ated -Ġp es -ĠA LE -r anging -Ġ40 4 -39 2 -Ġintellectual s -ach u -Ġrestruct uring -ĠLe vin -Ġrun es -Ġdelight ful -Ġcarbohyd rates -ĠMod els -ĠExp o -Ġtransport ing -all oc -Ġring ing -S amsung -Ġscarce ly -ĠURL s -ĠM AS -Ġprot otypes -Ġnarr ator -ĠCPU s -cd n -ĠBart on -Ġdecided ly -ĠSh u -ix ir -oc ious -ĠMy st -N intendo -Ġre use -Ġforg iven -F ew -in ical -n at -Ġseam less -ĠEv a -ĠE VE -ĠJ O -land ers -Ġso fter -neg ie -Ġtrans ient -Ġorb ital -Ġfulf il -ĠK om -Hop efully -Ġdynam ically -ĠHun ger -å Ľ -ĠArmen ia -el man -ber to -Ġp ige -ĠID s -lim it -Ġve ins -Ġso aring -p acks -Gold en -ĠCr ab -ist or -ĠR PM -Ġ$ $ -g ression -Ġjihad ist -Ġgam ble -Ġcare g -Ġinf lated -F ace -ĠFire arms -ĠEm manuel -â Ŀ -Ġsh ocks -gr ab -Ġspl end -ĠHP V -ab ortion -Ab ove -Ent ity -play ers -Ġcomm enced -ul ence -Ġfulfill ment -Ġembod iments -ĠW elfare -Ġha il -Ġ< @ -tt en -Ġcat cher -ĠJ azeera -Ġvolcan o -Ġstabil ize -ĠHand ler -Ġintens ified -ĠAb rams -Ġhum iliation -p aced -60 5 -ĠCent OS -Spe cific -Ġhe ed -ĠC AM -ĠGal ile -D ie -Ġabol ished -ĠThom son -ĠTe achers -ĠW ass -j ong -ĠIS BN -ĠAll ies -sh ake -å · -v ict -How ard -Ġde em -Ġexceed ingly -ĠSmart stocks -ib e -Ġdoor way -Ġcompet ed -ig mat -Ġnational ists -Ġg room -ĠKe en -Ġdispos able -de cl -ĠT olkien -ĠSche me -Ġb iod -Ġav id -ĠEl on -ag ar -ĠT SA -R oman -Ġartific ially -Ġadvis ors -X L -ĠInf erno -36 6 -Ġted ious -ĠPhot ography -ĠCar rie -Ġtro pe -ĠSand ra -Ġdec imal -Que en -ĠGund am -ĠO M -ote ch -N BA -Ġ19 32 -Ġent renched -ĠMar ion -Ġfr aternity -Lab our -Hen ry -Ġlat itude -E ither -Ġenh ances -ĠPot ential -Ġsh ines -id ad -Ġbread th -Ġcapac ities -ĠðŁ ĻĤ -ĠBron x -Ġsex es -Ġdifferent iation -Ġheavy weight -ĠT aj -d ra -Ġmigr ate -Ġexhaust ion -ĠR UN -els ius -ĠCu omo -Ġgu itars -Ġcl ones -ĠSom ew -ĠP ry ------------- - -Ġwarr anted -cy cles -Ġsalv age -Ġdis ks -R ANT -ĠNGO s -ĠMart ian -":[ {" -Ġadd icts -oj ure -il let -Ġamazing ly -art ments -p ixel -ĠGPU s -Lay out -è £ -ĠTam il -ĠBas il -Ġimpart ial -ĠSt ructure -f ork -b ryce -Ġr idge -ĠHamb urg -ri ous -Ġbl itz -cig arettes -Ġcan ned -40 2 -Ġiron ically -Ġcompassion ate -ĠHaw kins -. # -ĠCat hedral -Ġrall ied -in ternal -Ġqu ota -st akes -T EXT -m om -Ġcomple tes -Ġ23 8 -Ġsh rug -ãĥ ij -ĠN inth -Ġrev ise -ĠProv ider -Ġtre acher -Ġqu asi -ĠPR ES -Ġdep osition -Ġconfidential ity -iss ors -Ġim balance -Ġspan ning -Ġang ular -ĠC ul -commun ication -ĠNor a -ĠGen ius -op ter -Ġs acked -Sp ot -Ġfine ly -ĠCH R -28 2 -w aves -Pal est -ĠRo hing -N L -è ¿ -Ġsh itty -ĠSc alia -4 75 -Pro gress -Ġreferen cing -Ġclass rooms -ab ee -Ġs od -hes ion -70 8 -ĠZucker berg -ĠFin ish -ĠScot ia -ĠSav ior -ĠInstall ation -an tha -( - -Ġ30 2 -ĠP unk -Ġcr ater -yout u -Ġro ast -Ġinflu encing -Ġd up -ĠJ R -ĠG rav -Ġstat ure -Ġbath rooms -A side -W iki -me an -ĠZ ak -ĠOn es -ĠN ath -Ġhyper t -Ġcommence ment -C ivil -Ġmoder ately -Ġdistribut ors -Ġbreast feeding -Ġ9 80 -ĠS ik -ĠC ig -ĠAM ER -R IP -ĠCare er -ust ing -Ġmess ed -Ġe h -ĠJ ensen -/ $ -Ġblack mail -Ġconvers ions -Ġscientific ally -Ġmant ra -p aying -Ġiv ory -ĠCour ts -OU GH -aunt let -Ser ial -B row -ĠH undreds -3 23 -Ġpe e -Ġlin ux -Ġsub mer -ĠPrinc ipal -48 5 -ĠD SL -ĠCous ins -Ġdoctr ines -ĠAthlet ics -Ġ3 15 -ĠK arma -Ġatt ent -ur ger -Ġpresc ribe -Ġenc aps -ĠC ame -Ġsecret ive -ĠCr imes -d n -C lean -ĠEgypt ians -ĠCar penter -Ġ ll -H um -ĠMil o -Ġcapital ists -Ġbrief ed -T we -ĠBas in -elve t -M os -Ġplun ge -ĠKa iser -ĠFu j -ill in -Ġsafegu ards -Ġo ste -ĠOpportun ity -ĠM afia -ĠCall ing -ap a -ur ban -br ush -ill ard -c é -int elligence -ĠL ob -ĠDru id -Ġsm oother -Ġfoot ing -Ġmotor ists -arc ity -Ġmascul inity -Ġm ism -Ġabdom inal -ĠTa vern -ĠR oh -Ġesc apes -s igned -Anth ony -Ġsacrific ing -Ġintim acy -Ġan terior -ĠK od -Ġmot if -Ġg raz -Ġvisual ization -Ġguitar ist -ĠTro tsky -m agic -D ar -ĠMor i -Ġw ards -Ġtoile ts -l est -Ġtele port -ĠSund ays -ĠPl at -ET S -Ġe Sports -Pat rick -ĠK atherine -en ko -Ġhas sle -ĠM ick -gg les -Ġh ob -aint ain -Ġair borne -Ġsp ans -Ġch ili -Ġa perture -Ġvolunte ered -ĠInc ident -ĠF res -ĠVeter an -augh tered -ing o -Ġun insured -CL OSE -Ġf use -Ġer otic -Ġadvert ise -ra ising -Text ure -Ġatt ends -ĠRE AL -udd led -Ġsm oot -Ġ30 5 -ĠWill is -Ġbl ond -An alysis -ĠV T -on ica -Ġstrongh old -R F -N M -. >> -Ġprosper ous -Ġbo asted -29 2 -ĠManufact uring -PR ESS -g ren -Ġpharm acy -ĠRoc kefeller -k ai -Ġth umbs -ĠH ut -Ġmother board -Ġguard ians -ĠAl ter -ll ular -Ġsh ack -Ġwise ly -Ġback bone -erv a -Ġsu icides -ĠMcG regor -ij ah -E mer -ĠB rav -Ġdesign ate -P OST -produ ced -Ġcleans ing -irl wind -ex istent -ĠHum ph -ĠPay ne -Ġv ested -Å ¡ -Ġstring ent -ion a -Ġuns ub -Ġsum med -ĠHer cules -sub ject -ĠR agnar -ĠN os -Ġcharacter ization -Ġsav vy -ĠDaw son -ĠCas ino -Ġf ri -ĠBar rier -Ġmis information -Ġins ulation -Ġcorrid ors -Ġair planes -ĠNo ct -ah i -Ġ19 16 -k b -arm ac -Ġsh un -Ġsche ma -Ġhorr ified -Ġ23 9 -aund ers -N B -i ates -er ity -ĠSh ard -Ġr arity -Ġgroup ed -ĠGh ana -again st -ĠBi ological -ĠA ware -ow ell -Ï Ħ -ĠBe au -sh aw -H ack -ĠJul ius -US S -ol son -aun a -c ru -ĠMaur ice -ĠI k -Ġsequ encing -Ġradical s -Ġ( ?, -v irtual -Ġany ways -Ġreper c -Ġhand lers -Ġhes itant -é ĥ -ĠM F -ple mentation -ass ociated -Ġcampaign ed -ĠY ue -ut ations -ĠY oga -Ġsim mer -Ġro ds -Ġmel ody -Ġconv oy -v ideos -Ġscreen ed -N eg -ochem ical -Ġ( )) -Ġultr as -Ġant ip -ĠIsland ers -70 4 -Ġfet ish -Ġridic ulously -ĠK art -Ġmitochond rial -Ġinterf ering -Build er -Ġover fl -Ġac ne -ĠM ud -ĠK err -f lex -ĠPost al -ĠBalt ic -47 7 -ĠPers ons -our age -H B -ĠM use -ĠImm ortal -ĠDri ving -Ġpet itions -Ġsubsc ript -Ġs orce -ĠProcess or -ut on -S ony -Ġph on -Ġr aced -ĠAnth rop -Ġday time -ĠEx ercise -Add ing -Ġeng ages -ĠQual comm -Ġmir acles -Ġmem es -ĠDr ink -ĠOri oles -Ġhair s -ĠPol ar -ath om -Ġsl ippery -ĠR emy -Ġcar amel -ĠY EAR -Ġal k -I gn -a ution -ĠMer lin -ĠC ran -Ġap ologies -Ġ4 10 -Ġout ing -ĠMem ories -app ointed -Ġcount ered -u ld -pos ing -Ġfire wall -ĠW ast -ĠW et -work ed -se ller -Ġrepe aled -ere o -ass uming -BL IC -m ite -ĠCEO s -ĠChap el -ellig ent -________________ ________ -D og -Ġw art -Ġsubsc riber -s ports -Ġbe gged -ĠM V -Ġsem if -eth ical -Ġpre ach -Ġrev ital -Ġpun itive -Ġshort cuts -Ġinstit uted -ĠWars aw -Ġabdom en -ĠK ING -Ġsuper intendent -Ġf ry -ĠGe o -T OR -Ġcontrad ictions -apt ic -Ġlandsc apes -b ugs -Ġcl ust -Ġvol ley -c ribed -Ġt andem -Ġrob es -WH AT -Ġpromot er -Ġel oqu -review ed -ĠD K -ĠPl ato -Ġf ps -T ank -ĠDer rick -Ġpriorit ize -as per -ĠHond uras -ĠCom pleted -ne c -Ġm og -n ir -ĠMay o -DE F -st all -in ness -ĠVolks wagen -Ġprec aution -ĠM ell -i ak -ist ries -Ġ24 8 -Ġoverl apping -Sen ate -ĠEnh ance -res y -rac ial -OR TS -ĠM ormons -Str ong -ĠCo ch -Mex ico -ĠMad uro -Ġj ars -Ġcan e -W ik -oll a -iff erence -Ġphysic ist -ĠMag gie -Ġ28 5 -Ġdep iction -ĠMcL aren -J u -Ġsl ows -Ġcommission ers -ĠWill ow -ĠExpl os -hov ah -Ġtechn ician -Ġhom icides -ĠFl av -ĠTr uman -Ġ100 00 -u ctor -Ġsh ader -News letter -45 7 -Ġre ver -Ġhard ened -Ġwhere abouts -Ġrede velop -Ġcar bs -Ġtra vers -Ġsqu irrel -Ġfoll ower -Ġs ings -50 8 -Ġrabb its -emon ium -Ġdocument ing -Ġmisunder stood -) ' -R ick -gg ies -Ġprem ie -Ġsk ating -Ġpass ports -Ġf ists -aged don -H aw -AC P -0 80 -ĠThough ts -ĠCarl son -Ġpriest hood -h ua -Ġdun geons -ĠLo ans -Ġant is -Ġfamiliar ity -ĠS abb -op al -ĠIn k -st rike -Ġc ram -Ġlegal ized -Ġcu isine -Ġfib re -Tra vel -ĠMon ument -OD Y -eth y -Ġinter state -ĠP UR -em porary -ĠArab ian -develop ed -Ġsadd le -Ġg ithub -ĠOff er -ĠIS P -ro let -ĠSUP ER -ĠDen is -Ġmultipl ier -Ġstir red -Interest ingly -Ġcustom ary -Ġbill ed -he x -Ġmultipl ied -Ġfl ipping -ĠCros by -Ġfundament als -ia e -ĠPlay ed -ĠAt om -am azon -ĠFl am -ee z -activ ated -Ġtables poon -Ġliberal ism -ĠPal in -ĠP atel -N um -ĠT AM -Ġs urn -ĠRel oaded -Ġco ined -" ], -ĠCl ash -ĠAg u -Ġprag matic -ĠActiv ate -Ġ8 02 -Ġtrail ers -Ġsil hou -Ġprob es -Ġcirc us -ĠB ain -ĠLind say -ĠAb bey -Del ivery -Ġconcess ion -Ġgast ro -ĠSpr ite -Ä Ł -and el -Ġg imm -Ġaut obi -ĠT urtle -Ġwonder fully -ĠHar am -ĠWorld wide -ĠHand le -Ġtheor ists -Ġsle ek -ĠZh u -ograph ically -EG A -ĠOwn ers -ath s -ĠAntar ctic -n atal -=" " -fl ags -`` `` -Ġs ul -K h -Ġpot assium -Ġlinem an -Ġcere al -ĠSe asons -Ġ20 22 -Ġmat hematic -Ġastron omers -prof essional -Ġf ares -cknow led -Ġch i -Ġyoung sters -Ġmistaken ly -Ġhem isphere -ĠDiv inity -r one -Ġ" , -r ings -Ġattract s -v ana -å ¹ -C AP -Ġplay list -Ġpor ch -ãģ £ -Ġincorpor ates -Ġso ak -Ġassert ing -ĠTerror ism -ĠP ablo -J a -ces ter -Ġfear ing -ĠPr ayer -Ġescal ated -G W -Ġro be -ĠBright on -ac ists -ĠSym phony -ĠDwar f -ĠPar ade -ĠLe go -Ġinex pl -Ġl ords -le af -RA G -l iber -Ġcig ars -ĠJe hovah -60 6 -WIND OWS -ĠLiber ia -eb us -He avy -Ġl ubric -ĠR W -angu ages -Ġnarrow ed -com puter -ĠE mber -Ġmurder ing -Ġdown stream -ĠT uls -ĠT ables -Top ic -ĠAcc uracy -= / -l ost -ĠRe i -Ġprogress es -b ear -Ġestablish ments -Just in -ĠPe ach -ĠG omez -å ¿ -ĠTri angle -Id ent -ĠH ive -Res ources -Ġmix es -ĠAss uming -M u -Ġhyp oc -Ġs ane -ĠW an -id ious -Su ccess -Ġ io -Ang el -Ġdanger ously -ĠCreat ure -W ORK -: [ -ĠKat rina -List ener -M iller -ĠId lib -h ang -Ġcircum vent -h ref -Ġcel estial -ĠWe eks -ĠP ug -ĠDal ton -Ġsubpoen a -uk u -Ġpers isted -pe i -old ing -ĠDoc uments -ĠH ast -ĠC ENT -Ġprim er -Ġsyn onymous -Ġn ib -om bs -Ġnot ation -ĠD ish -ĠAt mosp -Ġforb id -ĠAN G -pat tern -l os -Ġproject iles -b rown -." , -ĠVen om -Ġfierce ly -ub lished -ĠU ran -ĠNic arag -4 10 -ĠC AL -OT OS -ĠMir acle -ĠEn chant -Ġguard ing -app end -Att ach -Ġlevel ed -Ġcond oms -ih ilation -64 9 -Ġnight mares -ĠTHE Y -ĠST ART -ĠK inn -Ġroomm ate -Ġhy giene -o pping -J ob -Ġl vl -ĠV ER -ĠKe eping -ab etic -Ġformat ting -eral a -Ġrev isions -Ġres urg -T el -ĠGood man -35 3 -p od -Ġind isp -ĠTrans lation -Ġg own -ĠM und -Ġc is -Ġby stand -col lect -ĠPun jab -act ively -ĠG amb -te ll -Ġimport ing -g encies -Ġloc om -ĠBr ill -H oly -ĠBer ger -Ġshow down -Ġrespond ers -IL Y -Ġt akedown -le ted -Ġmat tered -Ġpredict ive -Ġover lay -G PU -ĠV ick -Ġconvey ed -T ab -pe er -Sc an -Ġdefensive ly -v ae -Ġappro ving -Ġt iers -ĠV ia -quer ade -ĠSaud is -Ġdemol ished -ĠProp he -Ġmon o -Ġhospital ity -H AM -ĠAri el -M OD -ĠTor ah -Ġbl ah -ĠBel arus -erent ial -ĠT uc -Ġbank er -39 7 -Ġmosqu it -ĠScient ist -ĠMus ical -Ġh ust -Sh ift -Ġtor ment -Ġstand off -E duc -ĠF og -Ġampl ifier -Sh ape -Inst ance -ĠCrit ics -Ġda emon -H ouston -Ġmatt ress -ĠID F -Ġobsc ene -ĠA mer -hett i -Ġcomp iling -35 2 -vere tt -ĠRed uction -ist ration -ĠBl essed -ĠB achelor -3 16 -Ġpr ank -ĠVul can -dd ing -Ġm ourning -ĠQu int -ĠBl aster -test ing -Ġsed iment ->> > -ĠE ternity -ĠWH ERE -ĠM aze -Ġreact ing -ĠAl v -oms day -ĠC RA -Ġtransl ator -Ġbog us -at u -We bsite -oll s -Ġbapt ism -Ġs ibling -ĠAut umn -ve z -ãģ® é -gu ards -Ge org -assad ors -ĠFre ud -Ġcontin ents -ĠReg istry -Bern ie -ĸļ 士 -Ġtoler ant -ĠU W -Ġhor ribly -99 5 -ĠMID I -Ġimpat ient -oc ado -er i -ĠWor st -ĠNor ris -ĠTalk ing -Ġdef ends -ens able -Ġ20 21 -Ġanat omy -L ew -Ġdraw er -ĠCan berra -Ġpatri otic -é¾įå ĸļ士 -ĠAv g -AR M -Ġundis closed -Ġfare well -45 9 -b able -ĠAll ison -OL OG -Ġcon co -t ight -ĠAC PI -ĠM ines -l ich -ĠâĶ ľ -represent ed -200 000 -Ġenthusi ast -OT S -b il -ĠIng redients -Ġinvent or -ĠMy SQL -³³ Âł -ĠAB OUT -with in -Ġm k -B ul -ĠF ake -Ġdracon ian -W a -hel m -ĠTer ran -erv ille -Ġcommon place -SI ZE -Ġ" < -re place -ograph s -ĠSE LECT -inc ible -ĠMost ly -ĠShe ffield -ĠID E -ugg le -Ġcit ations -h urst -ĠUn ix -Ġunle ash -ĠP iper -ĠN ano -Ġsucc umb -Ġreluct ance -Ġ25 00 -ĠMer chant -Ġwire t -Ġcomb os -ĠBirth day -Ġchar coal -ĠU PS -ĠFair fax -Ġdrive way -ĠT ek -ĠP itch -ove re -Ġtechn icians -ĠAct ual -fl ation -ĠF iscal -ĠEm pty -an amo -Ġmag nesium -Ġsl ut -Ġgrow ers -Invest igators -( ): -ĠS atellite -ĠKe ynes -miss ive -l ane -Ġb orough -3 44 -ĠTE AM -ĠBet hesda -C V -h ower -ĠR AD -Ġch ant -ĠR iy -Ġcompos itions -Ġmild ly -Ġmedd ling -Ġag ility -ane ers -5 01 -Ġsyn th -ling er -29 1 -Ġex claimed -Part y -Ġcont amin -ĠMan or -ĠResp ond -Ġpra ising -Ġman ners -fle et -Sum mer -ĠLy nd -ĠDef initely -gr im -Ġbow ling -st ri -ç Ľ -y nt -Ġmand ates -D IV -Ġreconc ile -view s -ĠDam on -vet te -F lo -ĠGreat est -il on -ic ia -Ġportray al -Ġcush ion -50 4 -19 79 -oss al -App lic -sc ription -Ġmit igation -AT S -p ac -Ġer ased -Ġdefic iencies -ĠHolland e -ĠX u -Ġb red -Ġpregn ancies -f emin -Ġem ph -Ġpl anners -Ġout per -utter ing -Ġperpet rator -Ġm otto -ĠEll ison -ĠNE VER -Ġadmitted ly -AR I -ĠAzerbai jan -Ġmill isec -Ġcombust ion -ĠBott le -ĠL und -ĠP s -ĠD ress -Ġfabric ated -Ġbat tered -Ġs idel -ĠNot ting -Fore ign -ĠJer ome -0 20 -ĠAr bit -Ġkn ots -ĠR IGHT -M oving -ãģ Ļ -Ġsur geries -Ġcour thouse -Ġm astered -Ġhover ing -ĠBr an -ĠAl ison -Ġsaf est -m ilitary -Ġbull ied -Ġbar rage -Read er -ES E -ĠGe ographic -T ools -3 14 -ĠGe ek -ro th -gl ers -ĠF IN -Ï ģ -ĠA ston -al tern -48 8 -Ġveter in -G amer -Ġint el -ren ches -Sh ield -Ġam nesty -ĠB har -Ġp iled -Ġhonor able -ĠInst itutes -Ġso aked -Ġcom a -ĠE FF -34 1 -by tes -ĠG mail -le in -ĠCanad iens -m aterial -I l -Ġinstruct ors -ĠK Y -Ġconce ive -ub b -ĠP ossible -Ġeas ing -ĠChrist ina -Ġcar ic -ĠHD R -R OM -Ġsho vel -de lete -Ġp uff -ĠCh anging -Ġseam lessly -Att ribute -Ġacqu isitions -ak ery -ĠE F -Ġaut istic -ĠT akes -ĠPow der -ĠSt ir -5 10 -ĠBub ble -sett ings -ĠF owler -Ġmust ard -Ġmore over -Ġcopyright ed -ĠLED s -15 00 -æ ī -ĠH IS -en f -Ġcust od -ĠH uck -G i -Ġim g -An swer -C t -j ay -ĠInf rastructure -Ġfeder ally -L oc -Ġmicro bes -Ġover run -dd s -ot ent -adi ator ->>>> >>>> -Ġtorn ado -Ġadj ud -Ġintrig ued -Ġs i -ĠRevel ation -pro gress -Ġburgl ary -ĠSai yan -ĠK athy -Ġser pent -ĠAndre as -Ġcomp el -ess ler -ĠPl astic -ĠAd vent -ĠPos itive -ĠQ t -ĠHind us -reg istered -ular ity -Ġrighteous ness -Ġdemon ic -u itive -ĠB DS -ĠGre gg -c ia -ĠCrus ade -ĠSina i -W ARE -+ ( -Ġme ll -Ġder ail -y ards -A st -Ġnotice ably -ĠO ber -R am -Ġun noticed -Ġse q -av age -T s -Ġ6 40 -Ġconced e -Ġ] ) -F ill -Ġcapt ivity -ĠImprove ment -ĠCrus ader -ara oh -M AP -æ Ĺ -Ġstr ide -al ways -F ly -N it -Ġal gae -ĠCook ing -ĠDo ors -Mal ley -Ġpolic emen -ãģ į -Ġastron aut -access ible -49 5 -ĠR AW -cl iffe -udic rous -Ġdep ended -al ach -Ġvent ures -ra ke -Ġt its -ĠH ou -Ġcond om -ormon al -Ġind ent -Ġupload ing -Foot note -Import ant -Ġ27 1 -Ġmind ful -Ġcont ends -C ra -Ġcal ibr -ĠO ECD -plug in -F at -ĠIS S -ĠDynam ics -ans en -68 6 -' ), -Ġsp rite -Ġhand held -ĠH ipp -=~ =~ -Tr ust -Ġsem antics -ĠBund es -ĠRen o -ĠLiter ature -s ense -G ary -ĠA eg -ĠTr in -EE K -Ġcler ic -ĠSS H -Ġch rist -Ġinv ading -ib u -Ġen um -aur a -Ġal lege -ĠInc redible -B BC -Ġth ru -Ġsa iled -Ġem ulate -Ġin security -Ġc rou -Ġaccommod ations -Ġincompet ent -Ġsl ips -ĠEarth qu -s ama -IL LE -Ġi Phones -as aki -Ġby e -Ġar d -Ġext ras -Ġsl aughtered -Ġcrowd funding -res so -Ġfil ib -ĠER ROR -ĠT LS -e gg -ĠIt al -Ġen list -ĠCatal onia -ĠSc ots -Ġser geant -Ġdiss olve -N H -Ġstand ings -ri que -I Q -Ġbenef iciary -Ġaqu arium -You Tube -ĠPower Shell -Ġbright est -ĠWar rant -S old -Writ ing -Ġbegin nings -ĠRes erved -ĠLatin os -head ing -Ġ4 40 -Ġrooft op -AT ING -Ġ3 90 -VP N -G s -k ernel -turn ed -Ġprefer able -Ġturn overs -ĠH els -S a -ĠShin ji -ve h -ĠMOD ULE -V iol -Ġex iting -Ġj ab -ĠVan illa -Ġac ron -ĠG ap -ber n -A k -ĠMc Gu -Ġend lessly -ĠFar age -ĠNo el -V a -M K -Ġbr ute -ĠK ru -ĠES V -ĠOl ivia -âĢ ł -ĠK af -Ġtrust ing -Ġh ots -3 24 -Ġmal aria -Ġj son -Ġp ounding -ort ment -Count ry -Ġpostp oned -Ġunequ iv -? ), -ĠRo oney -udd ing -ĠLe ap -ur rence -sh apeshifter -ĠH AS -os ate -Ġca vern -Ġconserv atism -ĠB AD -Ġmile age -Ġarrest ing -V aults -Ġmix er -Dem ocratic -ĠB enson -Ġauth ored -8 000 -Ġpro active -ĠSpirit ual -t re -Ġincarcer ated -ĠS ort -Ġpe aked -Ġwield ing -re ciation -×Ļ × -P atch -ĠEm my -Ġex qu -tt o -ĠRat io -ĠP icks -ĠG ry -ph ant -Ġf ret -Ġeth n -Ġarch ived -% - -c ases -ĠBl aze -Ġim b -c v -y ss -im ony -Ġcount down -Ġaw akening -ĠTunis ia -ĠRe fer -ĠM J -Ġun natural -ĠCar negie -iz en -ĠN uggets -he ss -Ġev ils -64 7 -Ġintrodu ctory -l oving -ĠMcM ahon -Ġambig uity -L abel -ĠAlm ighty -Ġcolor ing -ĠCl aus -set ting -N ULL -ĠF avorite -ĠS IG -> ( -ĠSh iva -ĠMay er -Ġstorm ed -ĠCo verage -we apons -igh am -Ġun answered -Ġle ve -Ġc oy -c as -b ags -as ured -Se attle -ĠSant orum -ser ious -Ġcourage ous -ĠS oup -Ġconfisc ated -Ġ// / -Ġuncon ventional -Ġmom s -ĠRohing ya -ĠOrche stra -ĠPot ion -Ġdisc redit -ĠF IL -f ixed -ĠDe er -do i -ĠDim ension -Ġbureaucr ats -et een -Ġaction Group -oh m -Ġb umps -ĠUt ility -Ġsubmar ines -ren heit -re search -ĠShap iro -Ġsket ches -Ġde ceptive -ĠV il -es ame -ĠEss entially -Ġramp age -isk y -Ġmut tered -th ritis -Ġ23 6 -f et -b ars -Ġpup il -ĠTh ou -o S -s ong -Ġfract ured -Ġre vert -pict ure -Ġcrit erion -us her -Ġreperc ussions -ĠV intage -ĠSuper intendent -Offic ers -Ġflag ged -Ġbl ames -Ġin verse -ograp hers -Ġmakes hift -Ġdev oid -Ġfoss ils -ĠArist otle -ĠFund s -Ġde pleted -ĠFl u -ĠY uan -Ġw oes -Ġlip id -Ġsit u -requ isites -Ġfurn ish -ĠSam ar -Ġshame ful -Ġadverse ly -Ġad ept -Ġrem orse -Ġmurder ous -uck les -ĠE SL -Ġ3 14 -s ent -Ġred ef -ĠC ache -ĠP urs -ig ans -Ġ4 60 -Ġpres criptions -Ġf res -F uck -ocr ates -Tw enty -ĠWe ird -ĠT oggle -ĠC alled -itiz ens -Ġp oultry -Ġharvest ing -ãĤ¦ ãĤ¹ -Bott om -Ġcaution ed -t n -39 6 -ĠNik ki -Ġeval uations -Ġharass ing -Ġbind ings -ĠMon etary -Ġhit ters -Ġadvers ary -un ts -Ġset back -Ġenc rypt -ĠC ait -Ġl ows -eng es -ĠN orn -Ġbul bs -Ġbott led -ĠVoy ager -3 17 -Ġsp heres -p olitics -Ġsubt ract -Ġsens ations -Ġapp alling -Ġ3 16 -Ġenvironment ally -ĠST EM -Ġpub lishes -5 60 -Ġdilig ence -48 4 -Ġadv ises -Ġpet rol -Ġimag ining -Ġpatrol s -ĠInt eger -ĠAs hes -act us -ĠRad iant -ĠL T -it ability -ht aking -Set ting -Ġnu anced -ĠRe ef -ĠDevelop ers -N i -pie ces -99 0 -Lic ense -Ġlow ers -ĠOtt oman -3 27 -oo o -Ġqu itting -mark ets -Beh ind -Ġbas in -Ġdoc s -an ie -fl ash -ct l -Ġcivil ized -ĠFuk ushima -"] ," -ĠK S -ĠHonest ly -ar at -Ġconstruct s -ĠL ans -ĠD ire -ĠLI KE -ĠTrou ble -Ġwith holding -ĠOb livion -Ġsan ity -any a -Con st -Ġgro cer -ĠC elsius -Ġrecount ed -ĠW ife -B order -ate red -h appy -Ġspo iler -Ġlog ically -H all -Ġsucceed ing -Ġpoly morph -Ġax es -ĠShot gun -ĠS lim -ĠPrin ciples -ĠL eth -art a -Ġsc or -Sc reenshot -Ġrelax ation -#$ #$ -Ġdeter rent -idd y -Ġpower less -Ġles bians -Ġch ords -ĠEd ited -se lected -Ġseparat ists -000 2 -Ġair space -Ġturn around -Ġc unning -P ATH -P oly -Ġbomb ed -Ġt ion -x s -Ġwith hold -Ġw aged -ĠLiber ties -Fl ag -Ġcomfort ing -45 4 -ĠI ris -are rs -Ġr ag -Ġrel ocated -ĠGu arant -Ġstrateg ically -Ġgam ma -uber ty -ĠLock heed -g res -Ġgr illed -ĠLow e -st ats -ĠR ocks -Ġsens ing -Ġrent ing -ĠGe ological -ا Ø -ot rop -Ġse w -Ġimproper ly -48 6 -Ġâĸ ł -Ġstar ving -ĠB j -Disc ussion -3 28 -ĠCom bo -ĠFix es -N AT -Ġstri ving -th ora -Ġharvest ed -ĠP ing -Ġplay ful -Ġaven ues -Ġoccup ational -Ġw akes -ĠCou rier -Ġdrum mer -ĠBrow ser -ĠH outh -it u -Ġapp arel -p aste -Ġhun ted -ĠSecond ly -l ain -X Y -ĠP IN -ic ons -Ġcock tails -Ġs izable -Ġhurd les -est inal -ĠRecre ation -Ġe co -64 8 -ĠD ied -m int -Ġfinger prints -Ġdis pose -ĠBos nia -ts y -22 00 -Ġins pected -ĠF ou -Ġf uss -Ġamb ush -ĠR ak -Ġmanif ested -Pro secut -Ġsuff ice -ren ces -Ġcompens ated -ĠC yrus -Ġgen us -ĠWolver ine -ĠTrend s -Ġh ikes -ĠSe en -Ġen rol -C old -Ġpol itely -ĠSl av -ĠRu pert -Ġey ewitness -ĠAl to -Ġun comp -Ġposter ior -M ust -ĠHer z -Ġprogress ively -Ġ23 4 -Ġind ifference -ĠCunning ham -Ġacadem ia -Ġse wer -Ġast ounding -ĠA ES -r ather -Ġeld est -Ġclim bs -ĠAdd s -Ġout cry -Ġcont ag -ĠH ouses -Ġpe pt -ĠMel ania -interest ed -ĠU CH -ĠR oots -ĠHub bard -ĠT BD -ĠRoman ian -fil ename -St one -ĠIm pl -Ġchromos ome -C le -d x -Ġscram bled -ĠP t -Ġ24 2 -OP LE -Ġtremend ously -St reet -Ġcra ving -Ġbund led -ĠR G -p ipe -Ġinj uring -Ġarc ane -Part icip -ĠHero ic -st y -Ġto pping -ĠTemp est -rent ices -b h -Ġpar anoia -ĠUnic ode -Ġegreg ious -Ġ\ ' -ĠOsw ald -Ġgra vel -ĠSim psons -Ġbl and -ĠGuant anamo -Writ er -lin ers -ĠD ice -J C -Ġpar ity -Ġs ided -Ġ23 7 -ĠPyr rha -at ters -d k -F ine -comp an -Ġform ulated -ĠId ol -il ers -hem oth -ĠF av -Ġintr usion -Ġcar rots -ĠL ayer -ĠH acker -Ġ ---------------- -Ġmoder ation -é ģ -oc oc -Ġcharacter ize -ĠTe resa -Ġsocio economic -Ġper k -ĠParticip ation -tr aining -ĠPaul o -ph ys -Ġtrust worthy -Ġembod ied -ĠMer ch -c urrency -ĠPrior ity -Ġte asing -Ġabsor bing -Ġunf inished -ĠCompar ison -Ġdis ple -writ ers -Ġprofess ions -ĠPengu in -Ġang rily -ĠL INK -68 8 -ĠCor respond -Ġprev ailed -Ġcart el -l p -as ms -ĠRed emption -ĠIslam ists -effect s -d ose -ĠL atter -ĠHal ifax -Ġv as -ĠTop ics -ĠN amed -advert ising -zz a -IC ES -Ġret arded -ach able -ĠPupp et -ĠItem Level -Ġret ract -Ġident ifiable -A aron -ĠB uster -s ol -hel le -as semb -H ope -r anged -B a -ĠP urch -é Ģ -ĠSir i -Ġarri vals -Ġ19 12 -Ġshort ened -Ġ3 12 -Ġdiscrep ancy -ĠTem perature -ĠWal ton -Ġkind erg -p olit -Ġrem ix -Ġconnect ors -ãĥĺ ãĥ© -ĠKazakh stan -dom inated -Ġsu gars -im ble -ĠPan ic -ĠDem and -ĠCol ony -on en -ĠM ER -7 75 -ur ia -aza ar -ĠDeg ree -P ri -Ġsun shine -Ġ25 1 -Ġpsychedel ic -Ġdigit ally -ĠBra un -Ġsh immer -Ġsh ave -ĠTel esc -ĠAst ral -ĠVenezuel an -ĠO G -Ġc rawling -Int eg -ĠFe ather -Ġunfold ing -Ġappropri ation -Ġè£ı è -ĠMob ility -ĠN ey -- . -b ilt -L IN -ĠT ube -ĠCon versely -Ġkey boards -ĠC ao -Ġover th -Ġla ure ->> \ -ĠV iper -ach a -Off set -ĠR aleigh -ĠJ ae -J ordan -j p -Ġtotal itarian -Connect or -Ġobserv es -ĠSpart an -ĠIm mediately -ĠSc al -C ool -Ġt aps -Ġro ar -P ast -Ġch ars -ĠB ender -ĠShe ldon -Ġpain ter -Ġbe acon -ĠCreat ures -Ġdownt urn -Ġh inder -ĠAnd romeda -à Ľ -cc oli -ĠF itness -et rical -Ġutil izes -Ġsen ate -Ġen semble -Ġche ers -T W -Ġaff luent -k il -ry lic -ord ering -Com puter -Ġgru esome -ost ics -ĠUb isoft -ĠKel ley -Ġw rench -Ġbourgeois ie -IB LE -ĠPrest on -w orn -ar ist -reat ing -Ġst ained -ar ine -Ġsl ime -EN N -Ġche sts -Ġground water -ann ot -ĠTr ay -ĠLoc ke -ĠC TR -Ġd udes -ĠEx ternal -ĠDec oder -Ġpar amed -ĠMed line -80 9 -ĠD inner -rup al -g z -ĠG um -ĠDem o -j ee -Ġd h -ber man -arch s -Ġen qu -ĠEp stein -Ġdevast ation -Ġfriends hips -ĠAr d -Ġ23 1 -ĠRub in -ĠDist ance -Ġsp urred -Ġd ossier -Ġover looking -\\\\\\\\ \\\\\\\\ -Fore st -ĠCom es -\ ", -ĠIran ians -Ġf ixtures -L aughs -Ġcur ry -ĠKing ston -Ġsqu ash -Ġcat alogue -Ġabnormal ities -Ġdigest ive -.... ..... -Ġsubord inate -og ly -Ġ24 9 -M iddle -Ġmass ac -Ġburg ers -Ġdown stairs -Ġ19 31 -39 4 -ĠV G -Ġl asers -ĠS ikh -ĠAlex a -der ived -Ġcycl ist -ãģ® éŃĶ -onel iness -!!!! !!!! -Ġbuff s -leg ate -Ġrap ing -Ġrecomm ending -ro red -Ġmult icultural -un ique -Ġbusiness men -Ġune asy -ĠM AP -Ġdisp ersed -cipl ine -J ess -ĠK erala -å § -Ġabst raction -Sur v -U h -Ġprin ters -ij a -ow der -Ġanalog ous -ĠA SP -af er -Ġunfold ed -Ġlevel ing -Ġbre ached -ĠH earing -Ġn at -Ġtransl ating -crit ical -Ġant agonist -ĠYes terday -Ġfuzz y -w ash -m ere -Ġbe wild -ĠM ae -V irgin -ph rase -Ġsign aled -ĠH IGH -Ġprot ester -Ġgar ner -unk nown -Ġk ay -Ġabduct ed -Ġst alking -am n -Ġdes erving -ĠR iv -ĠJ orge -Ġscratch ing -ĠS aving -ip ing -Ġte ase -Ġmission ary -ĠMor row -T IME -P resent -Ġchem otherapy -tern ess -ĠH omes -ĠP urdue -Ġst aunch -ĠWhit ney -ĠTH ERE -Î ¼ -iat us -ĠErn est -ĠDe ploy -Ġcove ted -F ML -ĠDial ogue -Ġex ited -f ruit -Ġner d -":" "," -Ġv ivo -ru ly -4 60 -ĠAm en -rehens ible -Ġâ ĺ -D IR -Ġad herence -Ġche w -ĠCo ke -ĠSerge i -dig ital -ĠNe ck -g ently -enth al -/ ) -Ġwe ary -Ġgu ise -ĠConc ord -ĠOn ion -at cher -Ġb inge -ĠDirect ive -Ġman ned -ans k -Ġill usions -Ġbillion aires -38 3 -oly n -odynam ic -ĠWhe at -ĠA lic -Ġcol oured -ĠN AFTA -ab o -Ġmac ros -ind ependent -s weet -Ġsp ac -ĠK abul -Ġ Ä -em e -Ġdict ated -Ġsh outs -= { -Ġr ipping -ĠSh ay -ĠCr icket -direct ed -Ġanalys ed -ĠWAR RANT -ag ons -ĠBlaz ers -Ġche ered -Ġar ithmetic -ĠTan z -37 3 -ĠFl ags -Ġ29 5 -Ġw itches -ĠIn cluded -ĠG ained -ĠBl ades -G am -ĠSam antha -ĠAtl antis -ĠPr att -Ġspo iled -ĠI B -ĠRam irez -Pro bably -re ro -ĠN g -ĠWar lock -t p -Ġover he -Ġadministr ations -Ġt int -Ġreg iment -Ġpist ols -Ġblank ets -Ġep ist -Ġbowl s -Ġhydra ulic -Ġde an -Ġj ung -Ġasc end -70 5 -ĠSant iago -à ® -Ġun avoid -ĠSh aman -re b -Ġstem ming -99 8 -ĠM G -st icks -esthes ia -ER O -Ġmor bid -ĠGr ill -ĠP oe -any l -Ġdele ting -ĠSurve illance -Ġdirect ives -Ġiter ations -ĠR ox -ĠMil ky -F ather -Ġpat ented -44 7 -Ġprec ursor -Ġm aiden -ĠP hen -ĠVe gan -ĠPat ent -K elly -Redd itor -Ġn ods -Ġvent ilation -ĠSchwar z -Ġw izards -Ġomin ous -ĠHe ads -ĠB G -Ġl umber -ĠSp iel -Ġis Enabled -Ġancest ral -ĠSh ips -Ġwrest ler -ph i -Ġy uan -ĠRebell ion -Ġice berg -Ġmag ically -Ġdivers ion -ar ro -yth m -ĠR iders -ĠRob bie -ĠK ara -ĠMain tenance -ĠHer b -Ġhar ms -p acked -ĠFe instein -Ġmarry ing -Ġbl ending -ĠR ates -Ġ18 80 -Ġwr ink -ĠUn ch -ĠTor ch -desc ribed -Ġhuman oid -ilit ating -ĠCon v -ĠFe ld -IGH TS -Ġwhistlebl ower -ort mund -ets y -arre tt -ĠMon o -ĠI ke -ĠC NBC -ĠW AY -ĠMD MA -ĠIndividual s -Ġsupplement al -Ġpower house -ĠSt ru -F ocus -aph ael -ĠCol leg -att i -Z A -Ġp erenn -ĠSign ature -ĠRod ney -Ġcub es -idd led -ĠD ante -ĠIN V -iling ual -ĠC th -Ġso fa -Ġintimid ate -ĠR oe -ĠDi plom -ĠCount ries -ays on -Ġextrad ition -Ġdis abling -ĠCard iff -Ġmemor andum -ĠTr ace -Ġ?? ? -se ctor -ĠRou hani -ĠY ates -ĠFree ze -Ġbl adder -M otor -ĠProm ise -ant asy -Ġforesee able -ĠC ologne -cont ainer -ĠTre es -ĠG ors -ĠSin clair -Ġbar ring -key e -Ġsl ashed -ĠStat istical -é ĩ -Ġâĸ º -All ows -Ġhum ility -Ġdr illed -ĠF urn -44 3 -Ġse wage -Ġhome page -Ġcour tyard -Ġv ile -Ġsubsid iaries -aj o -direct ory -Ġam mon -V ers -charg es -Ġ} } -ĠCh ains -Ġ24 6 -n ob -Ġper cept -Ġg rit -Ġfisher men -ĠIraq is -ĠDIS TR -ĠF ULL -ĠEval uation -g raph -at ial -Ġcooper ating -Ġmel an -Ġenlight ened -Ġal i -t ailed -Ġsal ute -Ġweak est -ĠBull dogs -U A -ĠAll oy -Ġsem en -oc ene -ĠWilliam son -s pr -, âĢĶ -ĠG F -itt ens -Be at -ĠJ unk -iph ate -ĠFarm ers -ĠBit coins -ig ers -d h -ĠL oyal -p ayer -Ġentert ained -Ġpenn ed -Ġcoup on -Que ue -Ġweaken ing -c arry -Ġunderest imate -Ġshoot out -Ġcharism atic -ĠProced ure -Ġprud ent -in ances -Ġric hes -Ġcort ical -Ġstr ides -Ġd rib -ĠOil ers -5 40 -ĠPer form -ĠBang kok -Ġe uth -S ER -Ġsimpl istic -t ops -camp aign -Q uality -Ġimpover ished -ĠEisen hower -Ġaug ment -ĠH arden -Ġinterven ed -Ġlist ens -ĠK ok -Ġs age -Ġrub bish -ĠD ed -Ġm ull -pe lling -Ġvide ot -Produ ction -D J -m iah -Ġadapt ations -Ġmed ically -Ġboard ed -Ġarrog ance -Ġscra pped -Ġopp ress -FORM ATION -Ġj unction -4 15 -EE EE -S kill -Ġsub du -ĠSug gest -ĠP ett -Ġle tt -ĠMan ip -ĠC af -ĠCooper ation -T her -Ġreg ained -¶ æ -ref lect -Ġth ugs -ĠShel by -Ġdict ates -ĠWe iner -ĠH ale -Ġbatt leground -s child -Ġcond ol -h unt -osit ories -Ġacc uses -Fil ename -Ġsh ri -Ġmotiv ate -Ġreflect ions -N ull -ĠL obby -¥ µ -ĠS ATA -ĠBack up -Ñ ĥ -n in -ĠCor rection -Ġju icy -ut ra -ĠP ric -Ġrest raining -ĠAir bnb -ĠAr rest -Ġappropri ations -Ġsl opes -Ġmans laughter -Ġwork ings -ĠH uss -ĠF rey -Le ave -ĠHarm ony -ĠF eder -Ġ4 30 -Ġt rench -Ġglad ly -Ġbull pen -ĠG au -b ones -Ġgro ove -Ġpre text -ã ħĭ -Ġtransm itter -ĠComp onent -Ġunder age -ĠEm pires -T ile -Ġo y -ĠMar vin -ĠC AS -Ġbl oss -Ġrepl icated -ĠMar iners -Marc us -ĠBl ocks -Ġliber ated -Ġbutter fly -Fe el -Ġfer mentation -Ġyou tube -Ġoff end -ĠTer m -res ist -Ġcess ation -Ġinsurg ency -Ġb ir -ĠRa ise -59 5 -Ġhypothes es -50 2 -Ġpl aque -ocr at -Ġjack ets -ĠHuff Post -am ong -Ġconf er -48 7 -ĠL illy -Ġadapt ing -ĠF ay -Ġsh oved -ve c -Ġref ine -Ġg on -Ġgun men -z ai -ĠShut tle -ĠI zan -Ġ19 13 -Ġple thora -· · -Ġ5 10 -Ġp uberty -Ġ24 1 -ĠWe alth -ĠAl ma -ĠM EM -ĠAd ults -C as -pr ison -R ace -Ġwater proof -Ġathlet icism -Ġcapital ize -ĠJu ice -Ġillum inated -ĠP ascal -Ġirrit ation -ĠWitness es -ad le -ĠAst ro -Ġf ax -ĠEl vis -Prim ary -ĠL ich -ĠEl ves -Ġres iding -Ġst umble -3 19 -ĠP KK -Ġadvers aries -D OS -ĠR itual -Ġsm ear -Ġar son -ident al -Ġsc ant -Ġmon archy -Ġhal ftime -Ġresid ue -Ġind ign -ĠSh aun -ĠEl m -aur i -A ff -W ATCH -ĠLy on -hel ps -36 1 -Ġlobby ist -Ġdimin ishing -Ġout breaks -Ġgo ats -f avorite -ĠN ah -son ian -ĠBo oster -Ġsand box -ĠF are -ĠMalt a -Ġatt Rot -ĠM OR -ld e -Ġnavig ating -T ouch -Ġunt rue -ĠDis aster -Ġl udicrous -Pass word -ĠJ FK -blog spot -4 16 -ĠUN DER -ern al -Ġdelay ing -T OP -Ġimpl ants -ĠAV G -ĠH uge -att r -Ġjournal istic -ĠPe yton -ĠI A -R ap -go al -ĠProgram me -Ġsm ashing -w ives -print ln -ĠPl ague -in us -EE P -Ġcru iser -ĠPar ish -umin ium -Ġoccup ants -ĠJ ihad -m op -Ġp int -Ġhe ct -ĠMe cca -direct or -ĠFund ing -ĠM ixed -Ġst ag -T ier -Ġg ust -Ġbright ly -ors i -Ġup hill -R D -Ġles ions -ĠBund y -liv ious -Ġbi ologist -ĠFac ulty -ĠAuthor ization -Ġ24 4 -All ow -ï ¸ -ĠGi ul -Ġpert inent -ot aur -es se -ĠRo of -Ġunman ned -35 1 -ĠSh ak -ĠO rient -Ġend anger -D ir -Ġrepl en -ed ient -Ġtail or -Ġgad gets -Ġaud ible -âĺ Ĩ -N ice -Ġbomb ard -ĠR ape -Ġdef iance -ĠTW O -ĠFilip ino -Ġunaff ected -erv atives -Ġso ared -ĠBol ton -Ġcomprom ising -ĠBrew ers -R AL -ĠA HL -icy cle -Ġv ampires -Ġdi pped -oy er -ĠX III -Ġsidew ays -ĠW aste -ĠD iss -ĠâĶľ âĶĢâĶĢ -$ . -Ġhabit ats -ĠBe ef -tr uth -tr ained -spl it -R us -And y -ĠB ram -RE P -p id -è£ ħ -ĠMut ant -An im -ĠMar ina -Ġfut ile -hig hest -f requency -Ġepile psy -Ġcop ing -Ġconc ise -Ġtr acing -ĠS UN -pan el -ĠSoph ie -ĠCrow ley -ĠAd olf -ĠShoot er -Ġsh aky -ĠI G -ĠL ies -ĠBar ber -p kg -Ġupt ake -Ġpred atory -UL TS -/ ** -Ġintox icated -ĠWest brook -od der -he ment -Ġbas eman -AP D -st orage -ĠFif ty -ed itor -G EN -UT ION -ir ting -Ġse wing -r ift -Ġag ony -ĠS ands -Ġ25 4 -C ash -Ġl odge -Ġp unt -N atural -ĠIde as -Ġerrone ous -ĠSens or -ĠHann ity -Ġ19 21 -Ġm ould -ĠG on -kay a -Ġanonym ously -ĠK EY -Ġsim ulator -W inter -Ġstream ed -50 7 -? ", -Ġte ased -Ġco efficient -Ġwart ime -ĠTH R -' '. -ĠBank ing -mp ire -Ġf andom -Ġl ia -G a -Ġdown hill -Ġinterpre ting -Ind ividual -N orm -Ġjealous y -bit coin -Ġple asures -ĠToy s -ĠChev rolet -ĠAd visor -IZ E -Ġrecept ions -70 6 -C ro -Ġ26 2 -Ġcit rus -ir u -Review er -ject ed -U ES -an z -19 81 -ĠWork er -Ġcompl ied -ores cent -contin ental -T on -ĠPr ism -ĠShe ep -Ġ28 8 -n ox -ĠV og -O rd -Ġreal ms -te k -Ġirrig ation -Ġbicy cles -Ġelectron ically -p oly -t all -() ); -Ġaest hetics -ĠInteg rated -Expl ore -Ġd unk -47 6 -p ain -ĠJac ques -ĠD mit -Fram es -Ġreun ited -Ġhum id -D ro -P olitical -Ġyouth ful -Ġent ails -Ġmosqu ito -36 3 -spe cies -Ġcoord inating -ĠMay hem -ĠMagn us -M ount -Impro ved -ĠST ATE -ATT LE -Ġflow ed -Ġtack led -Ġfashion ed -Ġre organ -iv ari -f inger -Ġreluct antly -et ting -ĠV and -you ng -ĠGar land -Ġpresum ption -Ġamen ities -ĠPle asant -on ential -ĠO xy -Ġmor als -ĠY ah -Read y -Sim on -En h -D emon -Ġcl ich -Mon itor -ĠD U -Ġwel comes -Ġstand out -Ġdread ful -Ġban anas -Ġball oons -h ooting -bas ic -Ġsuff ix -Ġd uly -can o -Ch ain -at os -Ġgeop olitical -Ġ( & -ĠGem ini -ÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤ ÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤ -Ġacqu itted -L uck -prot ect -10 24 -Ġsc arcity -Ġmind fulness -ec ided -D N -pr ime -ĠPres idents -ĠVID EO -Ġ( âĪĴ -add ock -N OR -ĠP ru -p un -ĠL OL -)) )) -ĠL iqu -ĠS AS -Ġsty ling -Ġpunish ments -Ġnum b -Ġasc ertain -ĠRock ies -f lu -Th umbnail -Ġperpet rated -ĠSem i -Ġdis arm -ĠOld er -ĠEx ception -Ġexponent ially -ĠCommun ities -Ġabol ish -ĠPart ner -pt oms -Ġ7 77 -ĠFo ley -ĠC ases -Ġgre ase -ĠReb irth -G round -Ġ; ) -ĠDoct rine -ik ini -Y e -ĠBl ossom -Ġpers ists -b ill -Ġinf usion -Ġbud dies -9 11 -ĠPat ient -Ġdem os -Ġacquaint ance -ĠP aw -at ari -Ġx ml -Ġfasc ination -ĠSer ve -Ï Ĥ -br anded -Ġa z -Return s -Ġover shadow -Ġro am -Ġspeed y -n umbered -hel ial -Ġdisc iple -Ġass urances -g iven -pect ing -ĠN atalie -çĶ ° -Ġmosquit oes -rote in -Ġnumer ic -Ġindepend ents -Ġtrans itional -Ġreaction ary -ĠMech dragon -do ctor -Ġshort est -Ġsequ ential -ĠB ac -ĠAccount s -ãģ Į -ach y -ract ive -ĠReg iment -Ġbreat htaking -ffic iency -ĠB ates -Ġ3 11 -Ġward robe -ft s -ĠBer k -Sim ply -ĠRivers ide -iver ing -ident ial -lu cent -Ġen riched -ĠCon ver -ĠG iving -ãĥ Ļ -Ġlegal ize -ĠF TC -Ġfre aking -M ix -Ġter restrial -es ian -ci ents -W ing -LO AD -Ġled ge -ĠViol ent -ĠMet all -Ġ30 8 -Ġs outheastern -hett o -M eat -Ġslow down -Ġret reated -Jere my -end as -**** * -er ic -Ġre ins -opp able -ĠHuman ity -ear ances -rig an -C amera -Ġwa ivers -s oc -Ġalter ation -trans form -ĠC emetery -50 6 -Ġindef inite -Ġstim ulating -y g -60 3 -ĠS op -Ġdescript ive -Ph ase -ĠEd mund -Ġpneum onia -vent us -A mb -Ġlabor atories -ĠEx clusive -ug ar -W ere -Ġmalf unction -Ġhomosexual s -Ġ---- --- -un i -Ġturb ines -ĠEqu ity -D u -Ġmind ed -ĠR H -ĠBlack hawks -Ġfe ats -Ġ17 00 -re pl -36 2 -lad en -Ġindisp ensable -ly ss -tt i -Ġre el -Ġdiver ted -Ġlik eness -Ġsubscript ions -Ġfing ert -Ġfil thy -dest ruct -d raft -ĠBernard ino -l aunch -Ġper plex -ĠS UM -car b -Ġswe ater -ĠVent ure -ĠJ ag -ĠCele b -ĠV oters -Ġstead fast -Ġathlet ics -ĠHans on -ĠDr ac -Tr acker -Ġcomm end -ĠPres idency -ĠD ID -in formed -Ġweb page -P retty -Ġforce fully -ãĥĥ ãĤ¯ -Ġrel ocation -Ġsat ire -â ī -ĠSunder land -æ Ħ -V oice -???? ???? -Ġinform ant -Ġbow el -ĠUn iform -Ġ ..." -Ġpur ge -Ġpic nic -ĠU mb -ĠU PDATE -ĠSapp hire -ĠSt all -le arn -Ġobject ively -Ġob liter -Ġlooph ole -Ġjour neys -Ġo mission -Pro s -ĠSid ney -pl oma -Ġspray ed -Ġg uru -Ġtra itor -Ġtim et -Ġsn apping -ĠSe vent -urn al -ĠUk ip -Ġb owed -por al -l iberal -R os -Quest ions -i OS -Ġsummar ize -ST AT -Ġ18 50 -ap est -Ġl ender -ĠVari able -br inging -ĠL ORD -, ) -Ġcollaps es -x iety -ĠN ed -Y D -ĠSch a -Ġantib ody -Ġdis band -y re -ill usion -Ġro ver -s hed -ĠHiro sh -cc i -Ġcal am -ĠMort on -P interest -Ġ19 28 -ĠE uras -ord es -Ġf ences -ĠIn ventory -ĠVal encia -ĠU d -ĠT iff -Ġsqu e -Ġqu otation -Ġtroubles ome -er ker -QU EST -ĠKing doms -s outh -Ġle vy -Pr ince -ĠSt ing -Ġnick named -Ġapp e -Ġphot ographic -Ġcorp us -re ference -ĠT rog -U nt -) =( -ĠLat via -Ġactiv ating -Ġlicense e -Ġdispar ities -ĠNews letter -ãĥĥ ãĥĪ -Ġfree ing -ĠJe ep -ĠPer ception -ins k -Ġsil icone -ĠHay den -Le an -ĠSuz uki -ibr arian -66 8 -Ġsp or -Ġcorrel ations -ag hetti -Ġtu ber -ĠIP CC -il us -ĠV u -Ġwealth iest -ĠCarb uncle -an za -Ġfool ed -ĠZ ur -Ġd addy -ran o -il ian -Ġknock out -f man -requ ired -ĠWik ileaks -ĠD uffy -ON T -Ġins ol -ĠObject s -Ġb ou -ĠNord ic -ĠIns ert -sc an -Ġd ancers -Ġid iots -major ity -ĠNev ille -ĠFree BSD -Ġt art -pan ic -69 0 -Ġcoc oa -Ġsam pled -Ġlook up -Ind ust -Ġinject ions -gen re -Ġa u -Ġroad way -Ġgen itals -K ind -ĠEx aminer -ĠY az -F resh -Ġpar alysis -ĠAl uminum -Ġre ap -ok é -Ġsl oppy -ĠTun nel -pos ium -ner y -en ic -Ġher bal -ĠOut er -ĠBuild er -Ġinc ur -Ġide ologies -Ġback ups -cons uming -ĠDet ect -de ck -ĠKN OW -ĠG ret -ĠM IC -Ġtough ness -ĠEx hibit -Ġh ive -L es -ĠSCH OOL -ĠAt ari -ald e -ĠN ull -and estine -m ouse -Ġbrig ade -48 9 -Ġrev ol -ĠLaw son -ĠW ah -op oly -eb ted -ĠS aunders -Ġ3 13 -ĠW inc -Ġtab oo -ĠHel met -Ġw edge -ch ip -ĠT ina -b g -Ġinf uri -r n -Ġanomal ies -ĠSy nc -ĠEx am -ĠComm it -ĠDi ary -ĠALS O -ĠDe bor -omed ical -Ġcomprehens ion -6 55 -Ġempower ing -Ġ ire -Ġju ices -ĠE TH -ĠBox ing -=" / -Ġfacilit ated -p oke -ĠPars ons -ĠMod er -tra vel -Ġcivil izations -Ġliber tarians -Ġrun e -ĠCl arks -at hed -Ġcampaign ers -ĠDis patch -ĠFah renheit -ĠCap com --------- -- -Ġl ace -Ġdr aining -Ġl iner -ĠArt ificial -é n -t ask -] ). -ĠGM O -ĠOper ator -ord inary -ĠInf luence -ĠU ps -Ġpot ency -uss en -osp ons -ĠSw im -ĠDead line -Un ity -Ġcul inary -Ġenlight enment -Ġwe arer -Ġmin ed -Ġp ly -Ġinc est -ĠDVD s -W alk -B TC -Tr ade -Ġdev al -ib and -ĠOvers ight -Palest inian -Ġd art -Ġm ul -L R -Ġrem ovable -ĠReal ms -ì Ŀ -Ġmisc ar -ĠV ulkan -68 5 -è re -ĠS ap -Ġmer ging -ĠCar ly -che ster -Ġbr isk -Ġlux urious -ĠGener ator -Ġbit terness -Ġed ible -Ġ24 3 -T G -Ġrect angle -With No -bel ow -J enn -Ġdark est -Ġh itch -Ġdos age -Ġsc aven -ĠK eller -ĠIllust rated -Certain ly -ĠMaver icks -Marg inal -Ġdiarr hea -Ġenorm ously -Ġ9 99 -sh r -qu art -Ġadam ant -ĠM ew -Ġren ovation -Ġcerv ical -ĠPercent age -en ers -ĠKim ber -Ġflo ats -Ġde x -ĠW itcher -ĠSwan sea -d m -Ġsal ty -y ellow -Ġca pe -ĠDr ain -ĠPaul a -ĠTol edo -les i -Mag azine -ĠW ick -ĠM n -ĠA ck -ĠR iding -AS ON -Ġhom ophobic -AR P -Ġwand ered -C PU -ood oo -ĠP ipe -Ġtight ening -ĠBut t -3 18 -Ġdesert ed -S ession -Ġfacilit ating -J ump -Ġemer gencies -OW ER -Ġexhaust ive -ĠAF TER -Ġheart beat -ĠLab el -ack y -ĠCert ified -ilt ration -Z e -ĠU tt -Ġ13 00 -Ġpres ume -ĠDis p -Ġsur ged -Ġdoll s -Col umb -Ġchim pan -ĠR azor -Ġt icks -Ġcouncill or -Ġpilgr image -ĠReb els -ĠQ C -ĠA uction -x ia -ik k -b red -Ġinsert ion -Ġco arse -d B -SE E -ĠZ ap -ĠF oo -Ġcontem por -ĠQuarter ly -ot ions -ĠAl chemist -ĠT rey -ĠDu o -S weet -80 4 -ĠGi ov -Ġfun n -N in -h off -Ġram ifications -Ġ19 22 -ĠExper ts -az es -Ġgar ments -ar ial -ĠN ab -Ġ25 7 -ĠV ed -Ġhum orous -ĠPom pe -Ġn ylon -Ġlur king -ĠSerge y -ĠMatt is -Ġmisogyn y -ĠComp onents -ĠWatch ing -ĠF olk -ract ical -B ush -Ġt aped -Ġgroup ing -Ġbe ads -Ġ20 48 -Ġcon du -quer que -Read ing -Ġgriev ances -Ult ra -Ġend point -H ig -ĠSt atic -ĠScar borough -L ua -ĠMess i -a qu -ĠPsy Net -ĠR udd -Ġa venue -v p -J er -Ġsh ady -ĠRes ist -ĠArt emis -Ġcare less -Ġbro kers -Ġtemper ament -Ġ5 20 -T ags -ĠTurn ing -Ġut tered -Ġp edd -Ġimpro vised -Ġ: ( -Ġtab l -Ġpl ains -16 00 -press ure -ĠEss ence -marg in -friend s -ĠRest oration -Ġpoll ut -ĠPok er -ĠAugust ine -ĠC IS -ĠSE AL -or ama -Ġth wart -se ek -Ġp agan - º -cp u -Ġg arn -Ġass ortment -ĠI LCS -t ower -Recomm ended -Ġun born -ĠRandom Redditor -ĠRandomRedditor WithNo -Ġparaly zed -Ġeru ption -Ġinter sect -ĠSt oke -ĠS co -B ind -å ¾ -ĠP NG -ĠNeg ative -ĠNO AA -Le on -Ġall oy -ĠL ama -ĠD iversity -5 75 -Ġunderest imated -ĠSc or -Ġm ural -Ġb usted -so on -l if -Ġnone x -Ġall ergy -ĠUnder world -ĠR ays -ĠBl asio -Ġh rs -ĠD ir -Ġ3 27 -by ter -Ġrepl acements -Ġactiv ates -ri ved -M H -Ġp ans -ĠH I -Ġlong itudinal -Ġnu isance -al er -Ġsw ell -ĠS igned -s ci -ĠIs les -ĠA GA -Ġdef iant -Ġson ic -oc on -K C -ĠA im -t ie -ah ah -Ġm L -D X -Ġb isc -ĠBill board -ĠSY STEM -NE Y -ga ard -Ġdist ressed -former ly -Al an -Ġche fs -Ġopt ics -ĠC omet -ĠAM C -Ġredes igned -irm ation -Ġsight ings -38 2 -3 11 -ĠW B -Ġcont raction -ĠT OTAL -D ual -Ġstart led -Ġunderstand ably -Ġsung lasses -ETH OD -Ġd ocker -Ġsurf ing -ĠH EL -ĠSl ack -ton es -Ġsh alt -Vis ual -49 8 -Dep artment -c ussion -Ġunrest ricted -Ġt ad -Ġre name -employ ed -Ġeduc ating -Ġgrin ned -bed room -ĠActiv ities -ĠV elvet -ĠSW AT -Ġsh uffle -ig or -Ġsatur ation -F inding -c ream -ic ter -Ġv odka -tr acking -te c -Ġfore ground -iest a -Ġve hement -ĠEC B -ĠT ie -E y -Ġt urtles -ĠRail road -ĠKat z -ĠFram es -Ġmen ace -ĠFell owship -ĠEss ential -ugg ish -Ġdri p -ch witz -ĠKy oto -s b -ĠN ina -Param eter -Ġal arms -ĠCl aud -Ġpione ering -Ġchief ly -ĠSc ream -Col lection -Ġthank fully -ĠRonald o -åŃ IJ -st rip -ĠDisney land -com mercial -See ing -S oul -Ġevac uate -Ġc iv -ĠAs he -Ġdiv ides -ĠD agger -rehens ive -Ġber ries -ĠD F -Ġs ushi -Ġplur ality -W I -Ġdisadvant aged -Ġbatt alion -ob iles -45 1 -Ġcl ing -Ġunden iable -ĠL ounge -Ġha unt -p he -Ġquant ify -Ġdiff ered -Ġ[* ] -ĠV iz -c um -sl ave -Ġvide og -Ġqu ar -Ġbund les -ĠAl onso -t ackle -Ġneur onal -Ġlandsl ide -conf irmed -ĠDep th -Ġrenew ables -B ear -ĠMaced onia -Ġjer seys -Ġb unk -ĠSp awn -ĠControl s -ĠBuch anan -Ġrobot ics -Ġemphas izing -ĠTut orial -h yp -ist on -Ġmonument al -æ ° -ĠCar ry -Ġt bsp -en ance -H ill -art hed -Ġro tten -De an -Ġtw isting -Ġgood will -Ġimm ersion -L iving -Ġbr ushes -ĠC GI -ĠAt k -tr aditional -Ġph antom -ĠSt amina -Ġexpans ions -ĠMar in -Ġembark ed -ĠE g -int estinal -ĠPE OPLE -ĠBo oth -ĠApp alach -Ġreleg ated -V T -M IT -Ġmust er -Ġwithdraw ing -Ġmicrosc ope -ĠG athering -ĠC rescent -ĠArgent ine -ĠDec re -ĠDomin ic -Ġbud s -ant age -ĠI on -Ġwid ened -ONS ORED -ĠGl oves -iann opoulos -raz en -fe el -Ġrepay ment -Ġhind sight -ĠRE ALLY -ĠPist ol -ĠBra h -Ġwat ts -Ġsurv ives -Ġfl urry -iss y -Al ert -ĠUrug uay -Ph oenix -S low -ĠG rave -ĠF ir -Ġmanage able -Ġtar iff -ĠU DP -ĠPist ons -ĠNiger ian -Ġstrike outs -Ġcos metics -whel ming -f ab -c ape -pro xy -Ġre think -Ġover coming -sim ple -Ġw oo -Ġdistract ing -ĠSt anton -ĠTuls a -ĠD ock -65 9 -Ġdisc ord -ĠEm acs -ĠV es -ĠR OB -Ġreass uring -Ġcons ortium -Muslim s -3 21 -Ġprompt s -se i -ĠH itch -imp osed -ĠF ool -Ġindisc rim -wr ong -bu querque -D avis -! ] -Ġtim eless -ĠNE ED -Ġpestic ide -Ġrally ing -ĠCal der -Ġå ¤ -Ġx p -ĠUn le -ĠEx port -lu aj -B uff -) [ -Ġsq or -S audi -Ġis tg -Ġindul ge -pro c -Ġdisg usted -Ġcomp ounded -Ġn em -Ġschool ing -ĠC ure -process ing -S ol -Ġpro verb -it ized -ĠAlv arez -Ġscar f -Ġrect angular -re ve -Ġh ormonal -ĠSt ress -itiz en -Ġ4 25 -girl s -ĠNo ir -ĠR app -Ġmar ches -ch urch -ĠUs es -Ġ40 5 -ĠBer m -Ġord inances -ĠJud gment -Charg es -ĠZ in -Ġdust y -Ġstraw berries -Ġper ce -ĠTh ur -ĠDebor ah -net flix -ĠLam bert -Ġam used -ĠGu ang -Y OU -R GB -ĠC CTV -Ġf iat -r ang -Ġf ederation -ĠM ant -ĠB ust -ĠM are -respect ive -ĠM igration -ĠB IT -59 0 -Ġpatriot ism -Ġout lining -reg ion -ĠJos é -Ġbl asting -ĠEz ra -B s -Ġundermin es -ĠSm ooth -Ġcl ashed -rad io -Ġtransition ing -ĠBucc aneers -ĠOw l -Ġplug s -Ġh iatus -ĠPin ball -Ġm ig -ĠNut r -ĠWolf e -Ġinteg ers -Ġor bits -ĠEd win -ĠDirect X -b ite -Ġbl azing -v r -Ed ge -ĠP ID -ex it -ĠCom ed -ĠPath finder -ĠGu id -ĠSign s -ĠZ er -ĠAg enda -Ġreimburse ment -M esh -i Phone -ĠMar cos -ĠS ites -h ate -en burg -Ġs ockets -p end -Bat man -v ir -ĠSH OW -Ġprovision al -con n -ĠDeath s -AT IVE -Pro file -sy m -J A -Ġnin ja -inst alled -id ates -eb ra -ĠOm aha -Ġse izing -ĠBe asts -Ġsal ts -M ission -Gener ally -ĠTr ilogy -he on -leg ates -Ġd ime -Ġf aire -par able -G raph -Ġtotal ing -Ġdiagram s -ĠYan uk -ple t -ĠMe h -Ġmyth ical -ĠStep hens -aut ical -ochem istry -Ġkil ograms -Ġel bows -anc ock -ĠB CE -ĠPr ague -Ġimpro v -ĠDev in -Ġ" \ -par alle -Ġsuprem acists -ĠB illion -Ġreg imen -inn acle -Ġrequ isite -ang an -ĠBur lington -ain ment -ĠObject ive -oms ky -G V -Ġun ilateral -Ġt c -Ġh ires -ment al -Ġinvol untary -Ġtrans pl -ĠASC II - ¨ -Ev ents -Ġdoub ted -ĠKa plan -ĠCour age -ig on -ĠMan aging -ĠT art -Ġfalse hood -ĠV iolet -Ġair s -Ġfertil izer -Brit ain -Ġaqu atic -ou f -W ords -ĠHart ford -Ġeven ings -ĠV engeance -qu ite -G all -ĠP ret -Ġp df -ĠL M -ĠSo chi -ĠInter cept -9 20 -Ġprofit ability -ĠId le -ĠMac Donald -ĠEst ablishment -um sy -Ġgather ings -ĠN aj -Charl ie -Ġas cent -ĠProt ector -Ġal gebra -Ġbi os -for ums -EL S -Introdu ced -Ġ3 35 -Ġastron omy -Cont ribut -ĠPol ic -Pl atform -Ġcontain ment -w rap -Ġcoron ary -ĠJ elly -man ager -Ġheart breaking -c air -ĠChe ro -c gi -Med ical -ĠAccount ability -! !" -oph ile -Ġpsych otic -ĠRest rict -Ġequ itable -iss ues -Ġ19 05 -ĠN ek -c ised -ĠTr acking -Ġo zone -Ġcook er -ros is -Ġre open -Ġinf inity -ĠPharm aceutical -ens ional -Att empt -ĠR ory -Mar co -Ġawa its -H OW -t reated -Ġbol st -Ġreve red -Ġp ods -opp ers -00 10 -Ġampl itude -ric an -SP ONSORED -Ġtrou sers -Ġhal ves -ĠK aine -ĠCut ler -ĠA UTH -Ġsplend id -Ġprevent ive -ĠDud ley -if acts -umin ati -ĠY in -Ġad mon -ĠV ag -Ġin verted -Ġhast ily -ĠH ague -L yn -Ġled ger -Ġastron omical -get ting -Ġcirc a -ĠC ic -ĠTenn is -Lim ited -Ġd ru -ĠBY U -Ġtrave llers -Ġp ane -ĠInt ro -Ġpatient ly -Ġa iding -Ġlo os -ĠT ough -Ġ29 3 -Ġconsum es -Source File -Ġ"" " -Ġbond ing -Ġtil ted -Ġmenstru al -ĠCel estial -UL AR -Plug in -Ġrisk ing -N az -ĠRiy adh -Ġacc redited -Ġsk irm -é Ľ -Ġexam iner -Ġmess ing -Ġnear ing -ĠC hern -ĠBeck ham -Ġsw apped -Ġgo ose -K ay -Ġlo fty -ĠWal let -Ġ[ ' -Ġap ocalypse -Ġb amboo -ĠSP ACE -ĠEl ena -Ġ30 6 -ac ons -Ġtight ened -Ġadolesc ence -Ġrain y -Ġvandal ism -ĠNew town -Ġcon ject -c akes -Ġche ated -Ġmoder ators -par ams -E FF -Ġdece it -ĠST L -ĠTanz ania -ĠR I -Ġ19 23 -ĠEx ile -the l -Ġthe olog -Ġquir ky -ĠIr vine -Ġneed y -or is -U m -K a -Ġmail box -3 22 -Ġb os -ĠPet ra -K ING -Ġenlarg ed -O ften -Ġbad ass -Ġ3 43 -ĠPl aces -ĠC AD -Ġpr istine -Ġinterven ing -d irection -Ġl az -ĠD SM -Ġproject ing -ĠF unk -ag og -pay ment -n ov -Ġch atter -AR B -Ġexam inations -ĠHouse hold -ĠG us -F ord -4 14 -B oss -Ġmy stic -Ġle aps -ĠB av -ul z -b udget -Foot ball -Ġsubsid ized -Ġfirst hand -Ġcoinc ide -oc ular -Con n -ĠColl abor -Ġfool s -am ura -ah ar -r ists -Ġsw ollen -Ġexp ended -ĠP au -s up -Ġsp ar -Ġkey note -s uff -Ġunequ al -Ġprogress ing -str ings -ĠGamer gate -Dis ney -ĠEle ven -om nia -Ġscript ed -Ġear ners -bro ther -ĠEn abled -æ ³ -Ġlar vae -ĠL OC -m ess -Wil son -ĠTem plate -success fully -Ġparam ount -Ġcamoufl age -Ġbind s -ĠQu iet -ĠSh utterstock -r ush -Ġmasc ot -fort une -ĠCol t -ĠBe yon -hab i -Ġha irc -Ġ26 7 -ĠDe us -Ġtw itch -Ġconcent rating -Ġn ipples -c ible -Ġg ir -N Z -M ath -n ih -Requ ired -Ġp onder -ĠS AN -Ġwedd ings -Ġl oneliness -N ES -ĠMah jong -69 5 -add le -ĠGar ner -ĠC OUR -Br idge -Ġsp ree -ĠCald well -Ġbri bery -Ġ���� ���� -plug ins -Ġr acket -Ġchamp agne -vers ible -V ote -Ġmod ifiers -May or -6 80 -Ġassemb lies -ĠS ultan -ĠN ing -ĠLad ies -Ġsulf ur -Ġor bs -Ġ---- - -____ ___ -ĠJournal ism -Ġes ports -Ġl ush -Ġh ue -Ġspect ral -H onest -ãĥ ı -Ġbus hes -Ġrein forcement -Ġre opened -ĠWhe els -ĠM org -rie ving -Ġaux iliary -Ġj Query -ĠB AT -tes que -Ġver tex -p ure -f rey -ãĤ º -d os -Ġty ph -Ġc ull -Ġe q -Ġdec on -Ġtoss ing -Ġdispar ate -ĠBr igham -print f -led ged -Ġsu nd -Ġco zy -Ġhepat itis -per forming -Ġav al -ĠG G -f uture -Ġpet ertodd -ĠKos ovo -Ġmagn ets -Al ready -ĠEd ison -ĠCe res -ĠRA ID -Ġbrill iance -57 6 -Ġder ives -Ġhypert ension -ĠÎ Ķ -Ġlamb da -Ġfl air -Ġmission aries -Ġrap es -ĠSt arter -ĠMon ths -Ġdef y -Ġseism ic -ĠR aphael -Ġeuro zone -65 6 -z sche -Ġscr atched -Ġb ows -ĠLenn on -ĠGa ia -Ġdri pping -f acts -A le -Ġfrog s -ĠBre ast -ogene ity -ĠProsecut or -Ġampl ified -ĠHod g -ĠF n -Th ousands -ĠNI H -ĠMonitor ing -FT WARE -ĠPri ebus -ĠG rowing -hun ter -Ġdiagn ose -ĠM ald -ĠL R -Ġcrown ed -Ġburst ing -Ġdiss olution -j avascript -Ġuseful ness -ĠExec ution -: ( -ĠIv ory -a ah -Ġpersecut ed -viol ence -ist as -ĠCr ate -Ġimpuls es -ĠSp ani -ed es -Hand le -ĠZ erg -think able -Last ly -Ġspont aneously -Ġinconven ient -Ġdismiss ing -Ġpl otted -Ġeight y -Ġ7 37 -r ish -ĠThor nton -ath am -Ġsit com -V en -Rec ipe -t el -l und -Ġcle ars -ĠSas uke -Ġ25 8 -Ġopt ing -Ġen raged -est hetic -ĠA e -uch s -Pre p -Fl ow -Ġrun off -ĠE ating -ĠG iles -ĠAct ing -res ources -ib aba -Ġr pm -Ġske wed -ĠBl anc -ĠS akuya -Ġhot ter -Ġ19 24 -op ian -ck o -Ġcr umbling -Ġcapt ains -ĠAppropri ations -le aders -dro pping -an uts -Ġrevers ing -ĠP ose -ĠS ek -Sc ot -ĠIde a -c ise -ĠSloven ia -Ġ3 17 -Do ctor -Ġcro cod -ald i -Se a -ĠFar rell -Ġmerc enaries -ĠR NC -ĠGu ess -Ġp acing -M achine -Streamer Bot -ĠChar ity -Ġ29 8 -Ġcann ons -ĠTob y -TPP StreamerBot -ĠPass ion -cf g -Th om -Ġbad ges -ĠBern stein -. âĢĵ -ĠP OP -ĠCon j -Ġinitial ization -Ġbiod iversity -D ub -Ġfeud al -Ġdisclaim er -Ġc row -Ġign ition -ar f -S HA -Ġk Hz -h azard -ĠArt ists -oe uv -67 9 -ĠRud y -N ine -ĠRam adan -å ½ -itt o -Ġadren aline -C ert -Ġsmell ed -Ġimp unity -Ġag endas -ĠRe born -ĠCon cent -ĠSe ems -Ġo mega -ĠDust in -Ġback er -ĠSau ce -ĠBoy le -W IN -Ġsp ins -Ġpa uses -u pt -Ġshred ded -Ġstra pped -ĠCor ruption -Ġscr atches -Ġn i -Ġatt ire -ĠS AF -Factory Reloaded -ĠI PS -Ġ( % -Ġsem inar -f ocus -c ivil -Ġ18 60 -int osh -Ġcontin ual -Ġabbre vi -ĠS ok -oc obo -X M -Ġfr antic -Ġunavoid able -Ġar tery -Ġannot ations -b ath -Cl imate -Ġd ors -ĠSl ide -co ord -ĠRel oad -ĠL DL -ĠLove craft -Ġunim agin -Ġresemb led -Ġbarr acks -n p -Ġsurrog ate -Ġcategor ized -ãĤ © -Ġvacc inated -Ġdrain age -Ġind ist -ĠWhats App -Ġ18 70 -oler ance -inv oke -am orph -Ġrecon nect -Ġem anc -Ġblind ness -Ġ12 80 -intern et -c ollar -Ġalt ru -Ġab yss -ĠT RI -65 7 -Ġinf used -HE AD -Ġforest ry -ĠWood y -ĠC i -w i -s am -78 4 -hol iday -Ġmog ul -ĠF ees -ĠD EN -In ternal -ur bed -f usc -at om -ĠIll usion -Ġpoll ed -Ġfl ap -Ġco ax -L GBT -An aly -ĠSect ions -ĠCalif orn -em n -Ġh ither -ĠN IGHT -Ġn ailed -ĠPip eline -39 1 -o of -ĠPr imal -vere nd -Ġsl ashing -Ġret ri -avi our -Ġdepart ing -g il -IS C -Ġmid way -Ġultras ound -Ġbeh aving -ĠT ara -class es -V irtual -ĠColon ial -Ġstri pping -Ġorchestr ated -ĠGra ves -45 2 -ĠIron ically -ĠWrit ers -Ġl ends -ĠMan z -Ġra ven -Ġoxid ative -Ġ26 6 -EL F -act ually -asc ar -D raft -Ġfavour able -Ġhumili ating -Ġf idelity -ĠH of -ĠX uan -49 6 -Ġlay ered -at is -79 0 -Ġpay check -it on -K ar -ĠVM ware -ĠFar mer -Ġserv ic -gl omer -Ġsl ump -ĠFab ric -ĠD OC -est ing -Ġreass ure -Ġph yl -v olt -it ory -R ules -Ġoxid ation -Ġpri zed -Ġmist ress -ĠDj ango -WAR N -å ij -Ġenc ode -ĠFeed back -Ġstupid ity -I an -ĠYugoslav ia -× ¨ -ac l -UT E -19 77 -Ġqual ifies -Ġpuls es -pret ty -Ġfro ze -Ġs s -Iter ator -Ġur gently -Ġm ailed -ĠCh am -Ġsust aining -Ġbas il -Ġpupp ies -il ant -ĠP LEASE -l ap -ace ous -F ear -ĠMaster y -aut omatic -ĠT AG -Ġant im -ag les -47 3 -fram es -Ġwh ispers -ĠWho ever -Ġbra very -ĠUK IP -ract ions -"" " -Ġt ame -Ġpart ed -every thing -CON T -Ġind ebted -Ġadd r -re k -IR ED -Ġem inent -cl inton -Ġo usted -Ġreview er -Ġmelt down -Ġre arr -ĠY ao -the real -aby te -Ġst umbling -Ġbat ches -Ġ25 9 -Ġcontrace ptive -Ġprost itute -ens is -De cl -ĠSt rikes -M ilitary -ĠO ath -v acc -pp ings -05 2 -Ġpart Name -amp ing -Rep orts -K I -CH R -Ġsubt ly -sw ers -Bl ake -us ual -Ġcontest ants -Ġcart ridges -ĠGRE AT -Ġbl ush -ĠâĢ º -47 2 -Ġreason ed -ãĥ ¤ -paralle led -Ġd yn -ag ate -Ġnight ly -å Ĩ -55 6 -Ġsem antic -ĠAdv oc -Ġ !! -Ġdisag rees -ĠB W -V eh -Ġharm ing -Ġembr aces -Ġstri ves -Ġin land -ĠK ard -Ġhe ats -ĠGin ny -ut an -ern aut -yl ene -ĠE lev -J D -Ġh ars -ĠStar r -Ġsk ysc -Ġcollabor ators -Us ually -Ġrev olutions -ĠSTAT S -Ġdism antle -Ġconfident ly -Ġkin etic -Al i -Ġpercent ile -Ġextract ing -ill ian -est ead -Ġphysic ists -ĠMarsh al -Ġfell owship -Ġd ashed -ĠU R -ĠSi oux -ĠComp act -am ide -P ython -ĠLe igh -ĠPharm ac -ist rates -her ical -Ġf ue -ĠE min -Ġ( { -ĠNeighbor hood -Ġdisrupt ing -ĠD up -Ġg land -ĠSe v -ĠMar ian -arg on -ĠD und -Ġ< !-- -Ġstr and -Ġstadium s -z os -Ġpsych osis -ĠR ack -Ġbrilliant ly -ï¸ ı -Ġsubmer ged -ĠInst it -ĠCh ow -Ġc ages -ĠH ats -ĠU rs -Ġdil uted -us at -ien ne -ĠMembers hip -ĠBur k -Ġ ie -Ġarche type -D rug -ult on -ĠSp ock -ĠMcK ay -ĠDep end -F eatured -S oc -19 78 -ĠB ere -Ġrelent lessly -Ġcripp ling -Ġar thritis -çĶ Ł -ĠTrop ical -ĠBul g -ĠCher yl -Ġadm irable -Ġsub title -Over ride -Ġorig inating -ĠC CP -Ġsw ore -ĠSo le -ĠDis orders -3 29 -Ġprocess ion -Ġref urb -Ġimm ersed -requ ently -Ġskept ics -Ġcer amic -m itter -en stein -b elt -ĠT IT -b idden -Ġf ir -m ist -> ] -Ġwe ave -ĠParad ox -Ġentr usted -ĠBarcl ays -Ġnovel ist -og ie -80 6 -Ġnin ety -Ġdisag reements -@@@@ @@@@ -ĠAus chwitz -c ars -ĠL ET -t ub -arant ine -P OS -Ġback story -Ġcheer ful -ĠR ag -ek a -bi ased -Ġinexper ienced -ak ra -ĠW itt -t an -Ġrap ist -Ġplate au -ch al -ĠInqu is -exp ression -Ġc ipher -Ġsh aving -add en -re ly -( \ -ism a -ĠReg ulatory -CH AR -ily n -N VIDIA -G U -Ġmur m -la us -Christ opher -Ġcontract ual -ĠPro xy -ĠJa ime -ĠMethod ist -Ġstew ards -st a -per ia -Ġphys iology -Ġbump ed -Ġf ructose -Austral ian -ĠMet allic -ĠMas querade -ar b -Ġprom ul -Ġdown fall -Ġbut cher -Ġb our -ĠIN FORMATION -ĠB is -pect s -ad ena -Ġcontempl ating -ar oo -cent ered -ĠPe aks -Us ed -Ġmod em -Ġg enders -Ġ8 000 -37 1 -Ġm aternity -ĠR az -Ġrock ing -Ġhandgun s -ĠD ACA -Aut om -ĠN ile -Ġtum ult -ĠBenef it -ĠAppro ach -works hop -ĠLe aving -G er -inst ead -Ġvibr ations -Ġrep ositories -49 7 -ĠA unt -ĠJ ub -ĠExp edition -Al pha -Ġs ans -Ġoverd ue -Ġoverc rowd -Ġlegisl atures -Ġp aternal -ĠLeon ardo -Ġexp ressive -Ġdistract ions -Ġsil enced -tr ust -Ġb iking -Ġ5 60 -Ġpropri et -Ġimp osition -Ġcon glomer -Ġ= ================================================================ -ĠTe aching -ĠY ose -int ensive -T own -Ġtroll ing -ĠGr ac -ĠAS US -Y o -Ġspecial s -ĠNep h -ĠGod zilla -Dat abase -ĠHe gel -Ġ27 2 -19 76 -ĠGl oria -Ġdis emb -ĠInvestig ations -ĠB ane -ag ements -St range -Ġtre asury -ĠPl ays -Ġundes irable -Ġwid ening -Ġverb ally -Ġinf ancy -Ġcut ter -f ml -Ġ21 00 -prot otype -f ine -Ġdec riminal -Ġdysfunction al -Ġbes ie -ĠErn st -z eb -Ġnort heastern -Ġa ust -por ate -ĠMar lins -Ġsegreg ated -ew orld -ĠMa her -Ġtra verse -Ġmon astery -ur gy -G ear -s and -Com pl -ĠE MP -Ġpl ent -ĠMer cer -Ġ27 6 -TA BLE -Config uration -H undreds -Ġpr ic -Ġcollabor ating -ĠPar amount -ĠCumm ings -Ġ( < -Ġrecord er -Ġfl ats -Ġ4 16 -wh ose -Font Size -ĠOr bit -Y R -Ġwr ists -Ġb akery -) } -ĠB ounty -ĠLanc aster -Ġend ings -acc ording -ĠSal am -e asy -75 5 -ĠBur r -ĠBarn ett -onom ous -Un ion -Ġpreced ence -ĠScholars hip -ĠU X -Ġroll out -Ġbo on -al m -ĠCan ter -æ µ -Ġround ing -Ġcl ad -Ġv ap -ĠF eatured -is ations -Ġ5 40 -pol ice -Ġunsett ling -Ġdr ifting -ĠLum ia -ĠObama Care -ĠF avor -Hy per -ĠRoth schild -ĠMil iband -an aly -ĠJul iet -H u -Ġrec alling -a head -69 6 -Ġunf avorable -Ġd ances -O x -Ġleg ality -Ġ40 3 -rom ancer -Ġinqu ire -ĠM oves -\ "> -ĠVari ant -ĠMess iah -ĠL CS -ĠBah á -75 6 -Ġeyeb row -Ġ ¥ -ĠMc F -ĠFort y -M as -Ġpan icked -Ġtransform ations -q q -Ġrev olves -ring e -ĠA i -ax e -Ġon ward -ĠC FR -ĠB are -log in -Ġliqu ids -Ġde comp -second ary -il an -ĠCon vert -ami ya -Ġprosecut ing -Ġâī ¡ -ĠYork ers -ĠByr ne -sl ow -aw ei -J ean -Ġ26 9 -ĠSky dragon -Ġ é -ĠNicarag ua -ĠHuck abee -ĠHigh ly -Ġamph ib -ĠPast or -ĠL ets -Ġbl urred -Ġvisc eral -ĠC BO -Ġcollabor ated -z ig -Leg al -Ġapart heid -Ġbr id -Ġpres et -ĠD ET -ĠAM A -× Ķ -arch ing -auc uses -build er -Ġpo etic -Ġem ulator -ĠMole cular -Ġhon oring -ise um -Ġtract or -ĠCl uster -ĠCal m -ared evil -Ġsidew alks -Ġviol in -Ġgeneral ized -ĠAle c -Ġemb argo -Ġfast ball -ĠHT TPS -ĠL ack -ĠCh ill -ri ver -C hel -ĠSw arm -ĠLev ine -ro ying -L aunch -Ġkick er -Ġadd itive -ĠDe als -W idget -cont aining -Ġescal ate -ĠOP EN -Ġtwe aked -Ġst ash -Ġsp arks -ĠEs sex -ĠE cc -Ġconv ict -Ġblog ging -I ER -ĠH L -Ġmurd erers -75 9 -ĠH ib -Ġde pl -ĠJ ord -S ac -Ġdis sect -ĠHow e -os her -Ġcustom izable -ĠFran z -Ġat ro -Ä ĩ -Ġ000 4 -Ġout post -R oss -Ġglyph osate -ĠHast ings -ĠBE FORE -Ġsh ove -o pped -ĠSc ala -Ġam ulet -an ian -Ġexacerb ated -Ġe ater -47 1 -UM E -Ġpul p -izont al -ĠZ am -ĠAT I -imm une -aby tes -Ġunnecess arily -ĠC AT -ĠAx is -Ġvisual ize -à ī -ĠRad ical -f m -Doc uments -ĠFor rest -Ġcontext ual -ĠSy mbol -Ġtent ative -ĠDO ES -ĠGood s -Ġintermitt ent -} : -medi ated -Ġridic ule -Ġathe ism -Ġpath ogens -ĠM um -Ġre introdu -Ġ30 7 -i HUD -Ġflash light -Ġsw earing -Ġp engu -B u -Ġrot ated -ĠCr ane -Ġ() ); -Ġfashion able -Ġendors ing -46 3 -) [ -Ġingest ion -Ġcook s -Ġ9 50 -ot omy -ĠIm am -Ġk a -Ġte aser -ĠGhost s -ĠãĤ µ -19 69 -Ï ĥ -ub by -Ġconver ter -zan ne -end e -ĠPre par -ĠNic kel -ĠChim era -h im -ĠTyr ann -ĠSabb ath -ĠNich ols -Ġra pt -ih ar -Ġshe lling -Ġillum inate -Ġdent ist -ut or -ĠInteg ration -Ġwh ims -ĠLiter ary -Be aut -Ġp archment -ag ara -Br and -Ġder og -âĢ¦ ) -ĠNor se -Ġunw itting -Ġc uc -Ġborder line -Ġupset ting -Ġrec ourse -Ġd raped -ĠRad ar -Ġcold er -ĠPep si -im inary -], [ -65 8 -V i -ĠF rem -ĠP es -Ġveter inary -ĠT ED -ĠEp idem -n ova -k id -Ġdev out -o ct -j ad -M oh -ĠP AY -Ġge ometric -Ġ3 23 -Ġcircum ference -ich ick -19 75 -ĠY uri -ĠSh all -ĠH over -un in -S pr -Ġg raft -ĠHapp iness -Ġdisadvant ages -att acks -Ġhub s -ĠStar Craft -é ĸ -Ġgall eries -ĠKor ra -Ġgrocer ies -ĠGors uch -Ġrap ists -Ġfun gi -ĠTyph oon -V ector -ĠEm press -b attle -4 68 -Ġparas ite -ĠBom ber -S G -ex ist -ĠP f -Ġun se -Ġsurge ons -B irth -ĠUn sure -ĠPrint ed -ĠBehavior al -ĠA ster -Pak istan -Ġun ethical -Ġs v -ĠIo T -Ġlay outs -P ain -Ġconst ants -ĠL W -ĠB ake -Ġtow els -Ġdeterior ation -ĠBol ivia -Ġblind ed -ĠW arden -ĠMist ress -Ġon stage -Ġcl ans -ĠB EST -19 60 -Ġant ique -Ġrhet orical -ĠPer cy -ĠRw anda -, . -B ruce -Ġtra umat -ĠParliament ary -Ġfoot note -id ia -ĠLear ned -se eking -gen ic -Ġdim ensional -H ide -èĢ ħ -Ġintrig ue -in se -Ġle ases -Ġapp rentices -w ashing -Ġ19 26 -V ILLE -Ġsw oop -s cl -Ġbed rooms -on ics -ĠCr unch -comp atible -Ġincap ac -ĠYemen i -ash tra -z hou -d anger -Ġmanifest ations -ĠDem ons -AA F -Secret ary -ACT ED -L OD -Ġam y -ra per -eth nic -4 17 -Ġpos itives -Ġ27 3 -ĠRefuge es -Ġus b -ĠV ald -odd y -ĠMahm oud -As ia -Ġskull s -ĠEx odus -ĠComp et -ĠL IC -ĠM ansion -ĠA me -Ġconsolid ate -storm s -ont ent -99 6 -Ġcl en -Ġm ummy -fl at -75 8 -ĠV OL -oter ic -n en -ĠMin ute -S ov -Ġfin er -R h -ly cer -Ġreinforce ments -ĠJohann es -ĠGall agher -Ġgym n -S uddenly -Ġext ortion -k r -i ator -T a -Ġhippocamp us -N PR -ĠComput ing -Ġsquare ly -Ġmod elling -ĠFor ums -ĠL isp -ĠKrish na -Ġ3 24 -Ġr ushes -Ġens ued -Ġcre eping -on te -n ai -il ater -ĠHorn ets -Ġob livious -IN ST -55 9 -Ġjeopard y -Ġdistingu ishing -j ured -Ġbeg s -sim ilar -ph ot -5 30 -ĠPark way -Ġs inks -ĠHearth stone -ib ur -ĠBat on -Av oid -Ġd ancer -Ġmag istrate -ary n -Ġdisturb ances -ĠRom ero -Ġpar aph -Ġmis chief -âĸ ĵ -ĠSh aria -Ġur inary -r oute -iv as -f itted -Ġeject ed -ĠAl buquerque -Ġ4 70 -Ġirrit ated -ĠZ ip -ĠB iol -à į -Ġden ounce -Ġbin aries -ĠVer se -Ġopp os -ĠKend rick -ĠG PL -Ġsp ew -ĠEl ijah -ĠE as -Ġdr ifted -so far -Ġannoy ance -ĠB ET -47 4 -ĠSt rongh -it ates -ĠCogn itive -oph one -ĠIdent ification -ocr ine -connect ion -Ġbox er -ĠAS D -ĠAre as -Y ang -t ch -ull ah -Ġdece ive -Comb at -ep isode -cre te -W itness -Ġcondol ences -ht ar -Ġhe als -Ġbuck ets -ĠLA W -B lu -Ġsl ab -ĠOR DER -oc l -att on -ĠSteven son -ĠG inger -ĠFriend ly -ĠVander bilt -sp irit -ig l -ĠReg arding -ĠPR OG -Ġse aling -start ing -Ġcard inal -ĠV ec -ĠBe ir -Ġmillisec onds -we ak -per se -Ġster ile -ĠCont emporary -ĠPh ant -ĠCl o -Ġout p -Ġex iled -Ġ27 7 -Ġself ie -Ġman ic -Ġn ano -ter ms -Alex ander -Ġres olves -Ġmillenn ia -Ġexpl odes -Ġconst ellation -Ġadul tery -m otion -D OC -Ġbroad casters -Ġkinderg arten -ĠMay weather -ĠE co -ich o -Ġ28 7 -l aun -Ġm ute -Ġdisc reet -Ġpres chool -Ġpre empt -De lete -ĠFre ed -P i -H K -Ġblock er -ĠC umber -Ġw rought -d ating -Ġins urer -Ġquot as -Ġpre ached -Ġev iction -ĠReg ina -ĠP ens -Ġsevent een -ĠN ass -D ick -Ġfold s -Ġd otted -ĠA ad -Un iversal -Ġp izz -ĠG uru -Ġso ils -Ġno vice -ĠNe ander -Ġst ool -Ġdeton ated -ĠPik achu -ĠMass ive -IV ER -ĠAb del -Ġsubdu ed -Ġtall est -Ġprec arious -Ġa y -r ification -ĠOb j -c ale -Ġun question -cul osis -ad as -igr ated -D ays -Ġque ens -ĠGaz ette -ĠCol our -ĠBow man -ĠJ J -ï ve -Ġdomin ates -Stud ent -Ġm u -Ġback log -ĠElect ro -Tr uth -48 3 -Ġcond ensed -r ules -ĠCons piracy -Ġacron ym -hand led -ĠMat te -j ri -ĠImp ossible -l ude -cre ation -Ġwar med -ĠSl ave -Ġmis led -Ġfer ment -ĠK ah -ink i -ke leton -cy l -ĠKar in -Hun ter -Reg ister -ĠSur rey -Ġst ares -ĠW idth -ĠN ay -ĠSk i -Ġblack list -uck et -Ġexp ulsion -im et -Ġret weet -vant age -Fe ature -Ġtro opers -Ġhom ers -9 69 -Ġconting ency -ĠW TC -ĠBrew er -fore ign -W are -S olar -Ġund ue -RE C -ulner able -path ic -ĠBo ise -Ġ3 22 -Ġarous ed -ĠY ing -ä¸ į -uel ess -Ġp as -Ġmor p -Ġfl oral -Ex press -ud ging -k B -ĠGr anted -Ø ¯ -ĠMich a -ĠGoth ic -ĠSPEC IAL -ĠRic ardo -F ran -Ġadminister ing -6 20 -por a -Ġ ® -Ġcomprom ises -Ġb itten -Ac cept -Th irty -Ð ² -Ġmater ially -ĠTer r -ig matic -ch ains -Ġdo ve -stad t -Mar vel -FA ULT -Ġwind shield -Ġ3 36 -ad ier -Ġsw apping -Ġflaw less -ĠPred ator -ĠMiche le -Ġprop ulsion -ĠPsych ic -Ġassign ing -Ġfabric ation -Ġbar ley -l ust -Ġtow ering -Ġalter cation -ĠBent ley -Sp here -Ġtun a -ĠClass es -Fre edom -un er -L ady -v oice -Ġcool est -or r -Ġpal p -$ { -Ġhyster ia -ĠMet atron -p ants -Ġspawn ing -Exper ts -ĠInvest ors -ĠAn archy -Ġshr unk -ĠVict im -Ġ28 9 -Ġec stasy -ĠB inding -58 5 -ĠMel ody -57 8 -ot ally -ĠE tsy -lig a -Ġapplaud ed -Ġswe ating -Ġredist ributed -Ġpop corn -Ġsem inal -f ur -ĠNeuro science -R and -ĠO st -ĠMadd en -ĠIncre asing -ĠDaw kins -ĠSub way -Ġar sen -cons erv -B UR -Ġsp iked -ĠLy ft -ĠImper ium -ĠDrop box -Ġfav oured -Ġencomp asses -gh ost -Ġins pires -Ġbur geoning -ĠY oshi -ĠVert ical -ĠAud itor -Ġint ending -Ġfilib uster -Bl oom -f ac -ĠCav s -ign ing -Ġcowork ers -ĠBarb arian -rem ember -FL AG -Ġaudit ory -ason ry -Col lege -Ġmut ed -gem ony -ob in -ĠPsych o -9 68 -Ġlav ish -Ġhierarch ical -ĠDr one -ou k -Ġcripp led -ĠMax im -Sl ot -Ġqu iz -ĠV id -if ling -Ġarchae ologists -Ġabandon ment -d ial -le on -ĠF as -T ed -Ġr aspberry -Ġmaneu vers -Ġbehavi ours -Ġins ure -Ġrem od -Sw itch -h oe -Ġsp aced -Ġafford ability -ĠF ern -not ation -ĠBal anced -Ġoccup ies -en vironment -Ġneck lace -Ġsed an -F U -ĠBrav o -Ġab users -ĠAn ita -met adata -ĠG ithub -ait o -ĠF aster -ĠWass erman -ĠF lesh -Ġth orn -r arily -ĠMer ry -w ine -Ġpopul ace -ĠL ann -Ġrepair ing -Ġpsy che -Ġmod ulation -aw aru -âĢĭ âĢĭ -ari j -Ġdecor ations -Ġapolog ise -ĠG arg -app ly -Ġgive away -ĠFl an -ĠWy att -U ber -Ġauthor ised -ĠMor al -HAHA HAHA -activ ate -Ġtorped o -ĠF AR -Ġam assed -ĠA ram -ark in -ĠVict ims -st ab -Ġo m -ĠE CO -Ġopio ids -Ġpurpose ly -ĠV est -Ġer g -at an -ĠSur gery -Ġcorrect ing -ĠOrt iz -ĠBe et -Ġrev oke -Ġfre eway -ĠH iggins -F ail -ĠFar ms -ĠAT P -h ound -Ġp oking -ĠCommun ists -mon ster -iment ary -Ġunlock ing -Ġunf it -we ed -en ario -at ical -ĠEnlight enment -ĠN G -ĠComp ensation -de en -ĠWid ow -ĠCind y -ĠAfter wards -Ġ6 000 -ikh ail -ag ically -Ġrat ified -Ġcasual ty -H OME -p sey -f ee -Ġspark ling -Ġd é -Ġconcert ed -C atal -Ġcomp lying -ĠA res -ĠD ent -Sh ut -Ġsk im -ad minist -Ġhost ilities -ĠG ins -Ġ6 08 -Ġm uddy -ĠMc Int -ĠDec ay -5 25 -Ġconspic uous -ĠEx posure -Ġresc ind -Ġwear able -Ġ3 28 -our met -ah s -ĠRob ots -Ġe clips -inst ance -ĠRE PORT -ĠApp l -0 30 -ĠSk ies -01 00 -Ġfall acy -S ocket -ĠRece iver -Ġsol ves -ĠButter fly -ĠSho pping -ĠFI RE -65 4 -Med ic -Ġsing ers -ĠNeed less -'' '' -isher s -ĠD ive -58 8 -Ġselect ively -Ġcl umsy -88 9 -Ġpurch aser -ear ned -ard y -Ġbenef iting -eng lish -Ġyield ing -ĠP our -Ġspin ach -Ġdel ve -ĠC rom -6 10 -Ġexport ing -ĠMA KE -Ġ26 3 -Ġg rop -Ġenv oy -ĠInqu iry -ĠLu igi -d ry -ĠT uring -Thumbnail Image -ĠVar iety -Ġfac et -Ġfl uffy -Ġexcerpt s -Ġsh orth -ĠOl sen -CL UD -Ġrel iant -ĠUN C -T our -Ġbat hing -Comp any -Ġglobal ization -P red -ĠMalf oy -Ġh oc -j am -craft ed -ĠBond s -ĠKiss inger -Eng land -Ġorder ly -cat entry -Ġ26 1 -Ġexch anging -ĠInt ent -ĠAmend ments -D OM -Ġst out -³³³³³³³³ ³³³³³³³³ -ĠAir bus -Ġ27 8 -hy de -P oll -Item ThumbnailImage -Ġlooph oles -ĠPill ar -Ġexpl or -St retch -A part -Ġun married -Lim it -ĠTransform ers -Ġintellect ually -unct ure -18 00 -Ġd arn -B razil -Ġleft over -ber us -f red -Mine craft -3 26 -ĠForm s -Ġproof s -ĠDes igned -Ġindex es -ĠSupp ose -EM S -ĠL oving -ĠBon nie -im ating -OT US -Ġconduct or -Ġbehav ed -ĠF ren -Ġsy nerg -Ġmillenn ium -Ġcater ing -ĠL auder -W r -ĠY iannopoulos -ĠAT F -Ġensl aved -Ġawaken ed -D VD -ĠED ITION -ĠConc ert -ĠChall enger -ĠH aku -umer ic -Ġdep recated -ĠSH AR -4 12 -Ġdy stop -Ġtremb ling -Ġdread ed -ĠSp ac -p adding -Re pl -ĠG arrison -M ini -Ġun paralleled -am ar -URR ENT -w reck -c ertain -t al -ĠC LS -app ings -Ġsens ed -Ġf encing -ĠPas o -ĠDes k -Ġsc off -Ġcontem plate -ĠL iga -l iquid -75 7 -Ġapp rentice -ĠUCH IJ -5 70 -ĠTh ousand -ĠIll um -Ġchampion ed -ãĤ Į -Ġelect ors -Ġ3 98 -ĠH ancock -round ed -ĠJ OHN -Ġuns atisf -Ġqual ifier -ĠGad get -EN E -Ġdead liest -ĠPl ants -Ġ ions -Ġacc ents -Ġtwe aking -Ġsh aved -F REE -ĠCh aser -Again st -9 60 -Ġmeth amphetamine -Ġnormal ized -Ġ$ \ -ĠPre cision -ĠGu am -Ġch oked -ĠX II -ĠCast ing -Tor rent -Ġscal p -ĠJagu ar -w it -Ġsem ic -ix ie -ĠG ould -Ġconf ines -N usra -ĠL on -ĠJ ugg -y cle -ĠCod ec -E gypt -Ġrest rain -ĠAl iens -Ġch oking -ĠD unk -ĠBell a -ab c -Ġsl ang -Ġneuro trans -s av -Ġempower ment -â ĨĴ -Ġclim bers -ĠM im -ĠF ra -ros se -Cap ital -ĠCth ulhu -Inter face -Ġprof icient -ĠIN TO -Ġ3 18 -ront al -5 80 -ĠDes pair -K enn -Ġscrim mage -ĠCo at -as ions -Ġwall paper -ĠJ ol -Ġresurg ence -Ġant iv -ĠB alls -² ¾ -Ġbuff ers -Ġsub system -ĠSt ellar -ĠL ung -A IDS -Ġerad icate -Ġblat antly -Ġbehav es -ĠN un -Ġant ics -ex port -DE V -w b -Ġph p -ĠInteg rity -Ġexplore r -Ġrev olving -auth ored -g ans -Ġbas k -Ġas ynchronous -å į -TH ING -69 8 -G ene -ĠR acer -ĠN ico -iss ued -Ġser mon -p ossibly -Ġsize of -Ġentrepreneur ial -ox in -ĠMin erva -Ġpl atoon -n os -ri ks -A UT -ĠAval anche -ĠDes c -ij 士 -ĠP oc -Ġconf erred -Î » -Ġpat ched -F BI -66 2 -Ġfract ures -Ġdetect s -Ġded icate -Ġconstitu ent -Ġcos mos -W T -Ġswe ats -Ġspr ung -b ara -s olid -Ġuns us -Ġbul ky -ĠPhilipp e -ĠFen rir -Ġtherap ists -ore al -^^ ^^ -Ġtotal ed -Ġboo ze -ĠR PC -Prosecut ors -Ġdis eng -ĠSh ared -Ġmotor cycles -Ġinvent ions -Ġlett uce -ĠMer ge -ĠJ C -Ġspiritual ity -ĠWAR NING -Ġunl ucky -ĠT ess -Ġtong ues -ĠD UI -T umblr -Ġle ans -Ġinv aders -Ġcan opy -ĠHur ricanes -ĠB ret -ĠAP PLIC -id ine -ick le -Reg arding -Ġve ggies -Ġe jac -ju ven -F ish -D EM -ĠD ino -Th row -ĠCheck ing -be ard -( & -Ġj ails -Ġh r -trans fer -iv ating -Ġfle ets -ĠIm ag -ĠMc Donnell -Ġsnipp et -Is a -ĠCh att -ĠSt ain -ĠSet FontSize -ĠO y -ĠMathemat ics -49 4 -Ġelectro ly -ĠG ott -ĠBr as -B OOK -ĠF inger -d ump -Ġmut ants -Ġrent als -Ġinter tw -Ġc reek -ail a -Bro ther -ĠDisc ord -pe e -raw ler -Ġcar p -Ġ27 9 -ãĤ· ãĥ£ -rel ations -Ġcontr asts -Col umn -Ġrec onnaissance -Ġun know -Ġl ooting -Ġregul ates -Ġopt imum -ĠChero kee -ĠA ry -Lat est -Ġroad side -Ġd anced -ĠUnic orn -A cknowled -Ġuncont roll -ĠM US -at io -ch ance -ha ven -VAL UE -Ġfavour ites -Ġceremon ial -b inary -pe ed -wood s -EM P -Ġv ascular -Ġcontempl ated -Ġbar ren -ĠL IST -Y ellow -ospons ors -Ġwhisk y -ĠM amm -ĠDeV os -min imum -H ung -44 2 -P ic -ĠSnap dragon -77 6 -Ġcar ving -Ġund ecided -Ġadvantage ous -Ġpal ms -ĠA Q -Ġst arch -L oop -Ġpadd le -Ġfl aming -ĠHor izons -An imation -bo ost -Ġprob abilities -ĠM ish -Ġex odus -ĠEditor ial -Ġfung us -Ġdissent ing -ĠDel icious -rog ram -ĠD yn -d isk -t om -Ġfab rics -ĠC ove -ĠB ans -Ġsoft en -ĠCON S -Ġin eligible -Ġestim ating -ĠLex ington -pract ice -of i -Ġshe dding -ĠN ope -Ġbreat hed -ĠCorinth ians -y ne -ek i -B ull -Ġatt aching -reens hots -Ġanaly se -ĠK appa -Ġuns ustainable -Ġinter pol -ank y -he mer -Ġprot agonists -Ġform atted -ĠBry ce -ĠAch illes -ĠAb edin -sh ock -Ġb um -b os -qu a -ĠW arn -q t -ĠDi abetes -8 64 -ĠIn visible -Ġvan ish -Ġtrans mitting -Ġmur ky -ĠFe i -Ġawa ited -ĠJur assic -umm ies -Ġmen acing -g all -C ath -B uilt -ild o -ĠV otes -Ġon t -Ġmun itions -ĠFre em -ÃŃ n -Ġdec ency -lo pp -ie ved -ĠG ord -Ġun thinkable -ĠNews week -Ġ3 21 -He at -Ġpresent er -ji ang -Ġpl ank -ĠAval on -Ġben z -ĠR out -Ġslam ming -ĠD ai -ou ter -ĠCook ie -ĠAlic ia -ge y -Ġvan ity -Ġow l -á µ -t ested -ĠAw akens -Ġcan v -Ġblind ly -ĠRid ley -ĠEm ails -Requ ires -ĠSer bian -ograp hed -if rame -eter ia -Ġaltern ating -qu iet -Ġsoc iology -ĠUn lock -ĠCommun ism -Ġo ps -Ġatt ribution -Ġab duction -ĠAb ram -Ġsidel ined -ĠB OOK -Ġref ining -ĠFe eling -ĠOs lo -ĠPru itt -r ack -ang ible -Ġcaut iously -ĠM ARK -eed s -M ouse -ĠStep h -ĠP air -S ab -99 7 -ĠBa al -B ec -Ġcomm a -ĠP all -ĠG ael -Ġmisunder stand -ĠP esh -Order able -Ġdis mal -ĠSh iny -% " -Ġreal istically -Ġpat io -ĠG w -ĠVirt ue -Ġexhaust ing -wh atever -oph ys -y ip -4 18 -Ad just -ĠWa iting -ess on -ĠMaz da -ĠDo zens -Ġstream lined -Ġincompet ence -ĠM eth -Ġeth os -ON ES -Ġincent iv -Ġgr itty -ĠBut cher -Head er -Ġexp onential -à Ł -Ġcorrel ate -Ġcons ensual -s ounding -R ing -Orig in -Ġcon clusive -fe et -ac ly -ĠF ernandez -Buy able -Ġd ucks -aunt lets -Ġel ong -Ġ28 6 -Ġsim ul -G as -ĠK irst -Ġprot r -ĠRob o -ĠAo E -op ol -Ġpsych ologically -sp in -ilater ally -ĠCon rad -W ave -44 1 -ĠAd vertisement -ĠHarm on -ĠOri ental -is Special -Ġpresum ptive -Ġw il -ĠK ier -ne a -Ġp pm -Ġhar bour -ĠW ired -comp any -Ġcor oner -atur days -ĠP roud -ĠN EXT -ĠFl ake -val ued -ce iver -Ġfra ught -Ġc asing -Ġrun away -Ġg in -ĠLaure nt -ĠHar lem -ĠCur iosity -qu ished -Ġneuro science -ĠH ulu -Ġborrow er -Ġpetition er -ĠCo oldown -W ARD -Ġinv oking -conf idence -For ward -Ġst s -pop ulation -Delivery Date -Fil m -ĠC ov -quick Ship -quickShip Available -prim ary -isSpecial Orderable -inventory Quantity -channel Availability -BO X -ĠMulti player -ĠJen ner -77 8 -ĠM d -Ġ~ /. -M N -Ġchild ish -Ġantioxid ant -ĠChrom ebook -Ġ27 4 -Ġscreen play -Ġadvent urous -ĠRelations hip -respons ive -ming ton -Ġcorner stone -ĠF ey -F IR -Ġrook ies -ĠF eaturing -Ġorig inate -Ġelectro des -ant es -Ġscript ures -Ġgl ued -Ġdiscont ent -Ġaff licted -lay out -B rave -Ġm osa -ĠQuant ity -ĠH ik -w inner -H ours -Ġent ail -ĠCell s -olog ue -Ġv il -Ġpre acher -Ġdecor ative -d ifferent -Ġprejud ices -ĠSm oking -ĠNotting ham -so Type -Ġrhyth ms -ĠAl ph -bl ast -Ste el -ĠDaniel le -Ġstr ife -Ġrem atch -so DeliveryDate -ĠF ork -t rip -ol ulu -hes es -C G -ĠPOLIT ICO -ost a -ĠDr ift -é¾įå ¥ -é¾įå¥ ij士 -Ġvet ting -ĠJin ping -ĠRec ession -Min or -ĠF raud -enf ranch -Ġconven ed -ĠNA ACP -ĠMill ions -ĠFarm ing -ĠW oo -ĠFl are -rit o -imm igrant -Ġvac ancy -ĠHE AD -ĠV aj -eg al -ĠV igil -Stud y -Ġru ining -Ġr acks -Ġhe ater -ĠRand olph -ĠBr ush -ĠT ir -Ø ¨ -Ġc ov -% ] -Ġrecount s -ĠO PT -ĠM elt -Ġtr uce -Ġcas inos -Ġcrus ade -Ġcarn age -Ġstri pe -ĠK yl -Text ures -Ġ6 98 -Ġpro clamation -Ġgood ies -Ġ........ .. -pro claimed -P olit -Ġtop ical -Ġspecial ize -ĠA min -g m -Ġanch ored -Ġbear ings -s ample -ĠHigh land -ĠAut ism -Ġmerc enary -Ġinterview er -L ER -ĠSom ers -Ġembry o -ĠAss y -Ġ28 1 -ĠEd iting -ĠCh osen -6 60 -Ġp ci -ĠThunder bolt -BI LL -Ġchuck led -jri wal -h of -Ġearth ly -() { -ind ependence -Ġdisp ers -ĠV endor -ĠG areth -Ġp als -P enn -ĠSub mit -ic um -Th u -Ġcl andestine -Ġcann ibal -ĠCl erk -E Stream -gal itarian -âĻ ¥ -g ew -Ġhor rend -ĠL ov -ĠRe action -ocr in -Class ic -Ġecho ing -Ġdiscl osing -ĠIns ight -og un -ĠInc arn -upload s -pp erc -guy en -Ġ19 01 -ĠB ars -68 7 -Ġb ribes -ĠFres no -ur at -ĠRe ese -Ġintr usive -Ġgri pping -ĠBlue print -ĠR asm -un ia -man aged -ĠHeb do -Ġ3 45 -Ġdec oding -Ġpo ets -Ġj aws -ĠF IGHT -am eless -ĠMead ows -ĠHar baugh -Inter view -ĠH osp -ĠB RA -Ġdelet ion -m ob -W alker -ĠMoon light -ĠJ ed -ĠSoph ia -Ġus ur -Ġfortun ately -ĠPut ting -ĠF old -Ġsan itation -Ġpart isans -IS ON -B ow -ĠCON C -ĠRed uced -ĠS utton -Ġtouch screen -Ġembry os -âĢ¢âĢ¢ âĢ¢âĢ¢ -ĠK rug -com bat -ĠPet roleum -Ġam d -ĠCos mos -Ġpresc ribing -Ġconform ity -ours es -Ġplent iful -Ġdis illusion -ĠEc ology -itt al -Ġf anc -Ġassass inated -regn ancy -Ġperenn ial -ĠBul lets -Ġst ale -Ġc ached -ĠJud ith -ĠDise ases -All en -Ġl as -Ġsh ards -ĠSu arez -ĠFriend ship -inter face -ĠSupp orters -add ons -46 2 -ĠIm ran -ĠW im -Ġnew found -ĠM b -An imal -Ġd arling -and e -Ġrh y -ĠTw isted -pos al -yn ski -Var ious -× ľ -ĠK iw -uy omi -Ġwell being -ĠL au -an os -Ġunm ist -Ġmac OS -Ġrest room -ĠOl iv -ĠAir ways -Ġtimet able -9 80 -Ġrad ios -v oy -ias co -Ġcloud y -ĠDraw ing -Any thing -Sy ria -ĠH ert -st aking -Ġun checked -Ġb razen -ĠN RS -69 7 -onom ic -est ablish -Ġl eng -Ġdi agonal -ĠF ior -L air -ĠSt ard -Ġdef icient -jo ining -be am -Ġomn ip -Ġbl ender -Ġsun rise -Mo ore -ĠF ault -ĠCost ume -ĠM ub -Fl ags -an se -Ġpay out -ĠGovern ors -ĠD illon -ĠBan ana -N ar -Ġtra iled -Ġimperial ist -um ann -ats uki -4 35 -ĠRoad s -Ġsl ur -ĠIde ally -Ġt renches -C trl -Ġmir rored -ĠZ el -ĠC rest -Comp at -ĠRoll s -sc rib -ĠTra ils -omet ers -w inter -Ġimm ortality -il ated -Ġcontrad icts -un iversal -ill ions -ĠM ama -opt im -AT URE -Ġge o -et ter -ĠCar lo -4 24 -Ġcanon ical -ĠStrongh old -n ear -Ġperf ume -Ġorche stra -od iac -Ġup he -Ġreign ing -vers ive -Ġc aucuses -ĠD EM -Ġinsult ed -Ġ---- -- -ĠCr ush -Ġroot ing -ĠWra ith -Ġwh ore -Ġto fu -C md -ĠB ree -Ġ$ _ -Ġr ive -ĠAd vertising -Ġw att -ĠH O -Ġpersu asive -ĠParam eters -Ġobserv ational -ĠN CT -ĠMo j -ĠSal on -Ġtr unc -Ġexqu isite -ĠMar a -Ġpo op -ĠAN N -Ex c -ĠWonder ful -ĠT aco -Ġhome owner -ĠSmith sonian -orpor ated -mm mm -Ġlo af -ĠYam ato -ĠInd o -Ġcl inging -á s -Ġimm utable -h ub -Or ange -Ġfingert ips -ĠWood en -ĠK idd -ĠJ PM -ĠDam n -C ow -c odes -48 2 -Ġiniti ating -ĠEl k -ĠCut ting -Ġabsent ee -ĠV ance -ĠLil ith -G UI -Ġobsc ured -Ġdwar ves -ĠCh op -ĠB oko -Val ues -Ġmult imedia -Ġbrew ed -Reg ular -CRIP TION -ĠMort al -Ġa pex -Ġtravel er -Ġbo ils -Ġspray ing -Rep resent -ĠStars hip -4 28 -Ġdisappro val -Ġshadow y -Ġlament ed -ĠRe place -ĠFran ç -67 7 -d or -Ġunst oppable -Ġcoh orts -gy n -ĠClass ics -ĠAm ph -Ġsl uggish -ĠAdd iction -ĠPad res -Ġins cription -Ġin human -min us -ĠJere miah -at ars -Ter ror -ĠT os -ĠSh arma -ast a -c atch -Ġpl umbing -ĠTim bers -Sh ar -H al -ĠO sc -Ġcou pling -hum ans -Ġsp onge -Ġid ols -ĠSp a -ĠAdv ocate -ĠBe ats -lu a -Ġtick ing -Ġload er -ĠG ron -8 10 -Ġstim ulated -Ġside bar -ĠManufact urer -ore And -19 73 -Ġpra ises -ĠFl ores -dis able -ĠElect rical -ra ise -E th -Ġmigr ated -Ġlect urer -K ids -ĠCa vern -Ġk ettle -Ġgly c -ĠMand ela -ĠF ully -å§ « -FIN EST -Ġsquee zing -ĠRy der -amp oo -oreAnd Online -Inst oreAndOnline -Buyable InstoreAndOnline -Ġcommem orate -ĠRamp age -Aust in -ĠSh roud -ĠRu ins -9 15 -ĠK H -Ġwater front -ĠE SC -b aby -ĠC out -ĠEm blem -Ġequival ents -49 2 -Un ique -ĠNiet zsche -brow ser -Ġim itation -ĠWere wolf -ĠKir in -ac as -' ," -Ġà ¾ -Review ed -Ġc unt -Ġvo ic -ĠLen ovo -Ġbond ed -48 1 -Ġinhib itors -Ġendeav ors -ĠHav ana -ĠSt out -ĠJ olly -A ctor -*/ ( -Ġoccur rences -ĠT ens -Incre ased -ĠACT ION -Ġ ãĢĮ -ĠRank ings -ĠB reat -Ġ30 9 -D ou -Ġimpact ing -ĠDuc hess -pre fix -Q B -Ġsummon ing -Ġbest owed -ĠKe pler -ĠPOW ER -c ube -ĠK its -ĠG rip -Ġop ium -Ġrep utable -t oc -ich ael -ĠR ipple -Ġcaf é -ĠZ oom -ĠBur ma -Ġwa ive -Ġst alls -Ġdem eanor -inc erity -Ġfluor ide -ĠSH OULD -Par is -Ġlong ing -Ġpl at -Ġgross ly -Ġbull s -Ġshowc asing -ex pected -ĠG addafi -engine ering -Re peat -ĠK ut -Ġconce ivable -Ġtrim med -osc ope -ĠCand idate -ĠT ears -rol og -Lew is -S UP -Ġroad map -Ġsal iva -Ġtrump et -Jim my -Ġmirac ulous -Ġcolon ization -Ġam put -ĠGN OME -ate ch -D ifferent -ĠE LE -ĠGovern ments -ĠA head -ãħĭ ãħĭ -word press -L IB -ĠIn clude -ĠDor othy -0 45 -ĠColomb ian -Ġle ased -88 4 -Ġde grading -ĠDa isy -i ations -Ġbapt ized -Ġsurn ame -co x -Ġblink ed -ãĥ ¢ -Ġpoll en -Ġder mat -Ġre gex -ĠNich olson -ĠE ater -ç ľ -rad or -Ġnarrow er -Ġhur ricanes -Ġhalluc inations -r idden -ISS ION -ĠFire fly -Ġattain ment -Ġnom inate -Ġav ocado -ĠM eredith -Ġt s -Ġreve rence -Ġe uph -Ġcr ates -ĠT EXT -Ġ4 43 -Ġ3 19 -J SON -iqu ette -Ġshort stop -ic key -Ġpro pelled -Ġap i -ĠTh ieves -77 9 -Ġovers aw -Ġcol i -ĠNic ola -Ġover cl -ik awa -ĠC yr -Ġ38 4 -78 9 -ĠAll ows -10 27 -Det roit -TR Y -set up -ĠSocial ism -Sov iet -s usp -ĠAP R -ĠShut down -Ġal uminium -zb ek -ĠL over -GGGG GGGG -Ġdemocr acies -Ġ19 08 -ĠMer rill -ĠFranco is -gd ala -Ġtraff ickers -ĠT il -ĠGo at -Ġsp ed -ĠRes erv -Ġpro d -55 2 -Ġc ac -ĠUn iv -ĠSch we -Ġsw irling -ĠWild erness -ĠEgg s -Ġsadd ened -Ġarch aic -H yd -Ġexcess ively -B RE -Ġaer ospace -ĠVo ices -Cra ig -Ġign ited -In itially -ĠMc A -Ġhand set -Ġreform ing -Ġfrust rations -ĠDead pool -ĠBel ichick -ract or -ĠRagnar ok -ĠD rupal -ĠApp roximately -19 20 -ĠHub ble -arm or -ĠSar as -ĠJon as -Ġnostalg ic -Ġfeas ibility -Sah aran -Ġorb iting -Ġ9 70 -R u -Ġsh in -ĠInvestig ators -Ġinconsist encies -ĠP AN -B G -Ġgraz ing -Ġdetect ors -ĠStart up -ĠFun ny -ĠNa omi -Consider ing -Ġh og -ut f -ce mic -Ġfort ified -ĠFun ctions -Ġcod ec -nut rition -H at -" ! -micro soft -55 8 -ĠTh in -ĠA CE -Al ias -ĠO PS -p apers -P K -ãĢ İ -Ġimpro bable -N orthern -equ al -Ġlook out -Ġty res -ĠMod ified -ĠK op -Abs olutely -Ġbuild up -sil ver -Ġaud i -Ġgro tesque -ĠSab er -ĠPres byter -ON Y -Ġglac iers -ĠSho als -ĠK ass -ĠH RC -ĠNic ol -ĠL unch -ĠF oss -âĸ Ĵ -AD RA -ĠOne Plus -o ing -ground s -Ġincident al -Ġdatas ets -68 9 -ĠClarks on -Ġassemb ling -ĠCorrect ions -Ġdrink ers -Ġqual ifiers -Ġle ash -Ġunf ounded -ĠH undred -Ġkick off -T i -Ġrecon cil -ĠGr ants -ĠCompl iance -ĠDexter ity -Ġ19 06 -w arn -D allas -Max imum -n ard -av ia -be aut -ens itivity -tr ace -Ġpione ers -ĠF ract -ãĢ ı -Ġpre cept -Ġgloss y -ĠI EEE -Ac ross -Ġ6 80 -S leep -che on -Ġsatir ical -ĠMin otaur -ĠCla ude -Ġr é -ape go -Ġcar rot -ĠSem in -ino a -Ġz o -Ind ependent -Ġdiagn oses -ĠC ue -M AR -Ġrend ition -ĠK ik -Ġpath ology -Ġselect s -Link edIn -Ġass ay -ĠD res -Ġtext ual -post ed -IT AL -ĠM aul -N eal -Ġinter connected -Ġerr atic -ĠVir us -Ġ5 30 -Ġenvironmental ists -ĠP helps -Ġeng agements -ĠIN ST -Ġeconom ical -nox ious -Ġg earing -izz y -Ġfavor ably -ĠMcG ill -T erm -Ġh anged -Ġball park -ĠRe yes -Ġbe ware -ĠP sal -ĠMass acre -q i -Ġin accessible -acly sm -Ġfr ay -ill ac -Ġbitter ly -ĠCert ification -Mich igan -Ġir respective -al ore -Em pty -Ġendorse ments -Ġund et -f g -equ ipped -Ġmerc iless -ĠC ust -Ġimm ature -Ġvou cher -ĠBlack well -Ñ ı -h awk -dis ciplinary -ile e -ĠMak oto -ĠD ude -ãĥĩ ãĤ£ -Y ears -Ġin ver -Ġsh aman -ĠY ong -ip el -ell en -ĠCath y -br ids -Ġs arc -65 1 -N ear -Ġground work -Ġam az -Ġ4 15 -ĠHunting ton -hew s -ĠB ung -Ġarbit rarily -ĠW it -ĠAl berto -Ġdis qualified -best os -46 1 -Ġp c -Ġ28 4 -ro bat -Rob in -Ġh ugs -ĠTrans ition -ĠOcc asionally -Ġ3 26 -ĠWh ilst -ĠLe y -Ġspaces hip -cs v -Ġun successfully -ĠA u -le ck -ĠWing ed -ĠGrizz lies -. � -Ġne arer -ĠSorce ress -ĠInd igo -El se -8 40 -let es -Co ach -Ġup bringing -ĠK es -Ġseparat ist -Ġrac ists -Ġch ained -Ġabst inence -lear ning -Ġrein stated -Ġsymm etry -Ġremind ers -ĠChe vy -Ġm ont -Ġexempl ary -ĠT OR -Z X -Ġqual itative -ĠSt amp -ĠSav annah -ĠRoss i -Ġp aed -Ġdispens aries -ĠWall s -ĠCh ronic -Ġcompliment ary -ĠBeir ut -Ġ+ --- -igs list -Ġcrypt ographic -mas ters -ĠCap itals -Ġmax imal -Ġent ropy -Point s -Ġcombat ants -l ip -ĠGl ob -ĠB MC -ph ase -th ank -HT TP -Ġcomm uter -Ġ\( \ -.. / -ĠReg ener -ĠDO I -ĠActiv ision -Ġsl it -os al -RE M -Ġch ants -Y u -Ke ys -Bre xit -ĠFor ced -Ari zona -Ġsquad ron -IS O -ĠMal one -Ġ3 38 -Ġcontrast ing -Ġt idal -Ġlib el -Ġimpl anted -Ġupro ar -ĠC ater -Ġpropos itions -M anchester -ĠEuro s -it amin -G il -ĠEl ven -ĠSe ek -ĠB ai -Ġredevelop ment -ĠTown s -ĠL ub -! ", -al on -K rist -Ġmeas urable -Ġimagin able -Ġapost les -Y N -7 60 -Ġster oid -Ġspecific ity -ĠL ocated -ĠBeck er -ĠE du -ĠDiet ary -uts ch -ĠMar ilyn -Ġbl ister -ĠM EP -ĠK oz -ĠC MS -y ahoo -ĠCar ney -Ġbo asting -ĠC aleb -By te -read s -ad en -Pro blem -ĠWood ward -S we -S up -ĠK GB -Set up -Ġtac it -Ġret ribution -Ġd ues -ĠM ü -. ? -ä¸ Ń -p ots -Ġcame o -ĠP AL -educ ation -A my -like ly -g ling -Ġconstitution ally -ĠHam m -ĠSpe ak -Ġwid gets -br ate -Ġcra ppy -ĠI ter -Ġanticip ating -ĠB out -P ixel -ĠY ep -ĠLaur ie -Ġh ut -Ġbullet in -ĠSal vation -Ġch ats -ear able -Honest ly -AL TH -onse qu -c ult -isco very -ovy ch -Ġse lves -ĠSat oshi -S ounds -Ġconver gence -ĠRosen berg -19 74 -Ġnas al -Ġfull est -Ġfer ocious -x us -ist e -AM S -Ġlobb ied -Ġso othing -ĠGun n -t oday -0 24 -Ġinspir ational -ĠN BN -p b -g ewater -or ah -all owed -ĠCol iseum -Ġspecial izing -Ġinsane ly -ĠT ape -del ay -Ġt arn -ĠP ound -Ġmel anch -Ġdeploy ments -il and -Ġless en -Ġfur ry -ĠUE FA -Ġblood shed -ĠMe ier -ither ing -Ġhe irs -ĠJ aw -ax ter -ĠPublic ations -Ġal ters -int ention -ĠWinc hester -d etermination -ĠLif etime -th in -Mon ster -7 80 -Ġapprox imation -Ġsuper markets -ĠSecond s -or os -h uge -Ġb ribe -ĠLIM ITED -un ed -Ġmis interpret -ĠIn jury -Ġ3 67 -Ġthreshold s -ĠCarn ival -Ġgastro intestinal -Ġguid eline -Ġde ceived -f eatures -Ġpurported ly -ĠRon nie -ĠNew t -Ġsp acious -as us -Ġsuperhero es -ĠCyn thia -le gged -k amp -ch io -Ġth umbnail -ĠShir ley -ill ation -Ġshe ds -ĠZ y -E PA -Ġdam s -Ġy awn -n ah -ĠPe ggy -ĠE rie -ĠJu ventus -ĠF ountain -r x -don ald -al bum -ĠComp rehensive -Ġc aching -ĠU z -ulner ability -ĠPrinc iple -ĠJ ian -ing ers -cast s -ĠOs iris -ch art -t ile -ĠTiff any -ĠPatt on -ĠWh ip -Ġovers ized -J e -ĠCind erella -ĠB orders -ĠDa esh -M ah -Ġdog ma -Ġcommun ists -v u -Coun cil -Ġfresh water -Ġw ounding -Ġdeb acle -Ġyoung ster -Ġthread ed -ĠB ots -ĠSav ings -ãģ Ĥ -ol ing -oh o -Ġillum ination -M RI -Ġlo osen -tr ump -ag ency -ur ion -Ġmoment arily -ĠCh un -ĠBud apest -ĠAl ley -D isk -Ġaston ished -ĠCon quer -ĠAccount ing -h aving -ĠWe in -ĠAl right -Ġrev olver -Ġdel usion -Ġrelic s -Ġad herent -qu ant -Ġhand made -or io -Ġcomb ating -c oded -Ġquad ru -re th -N ik -ĠTrib al -ĠMyster ious -Ġin hal -ĠWin ning -ĠClass ification -ch anged -Ġun ab -Ġsc orn -icip ated -w l -ond uctor -Ġrein forcing -ĠChild hood -an ova -Ġadventure r -Ġdoctor al -ĠStrateg ies -Ġengulf ed -ĠEnc ounter -Ġl ashes -Crit ical -ric ular -ĠU TF -oci ation -check ing -ĠConsult ing -Run time -per iod -ĠAs gard -Ġdist illed -ĠPas adena -ĠD ying -ĠCOUN TY -Ġgran ite -Ġsm ack -Ġparach ute -ĠS UR -Virgin ia -ĠF urious -78 7 -ĠO kin -Ġcam el -ĠM bps -19 72 -ĠCh ao -ĠC yan -j oice -ef er -ĠW rap -ĠDeb ate -S eg -Ġfore arm -ĠIgn ore -Ġtim estamp -Ġprob ing -ĠNo on -ĠGra il -f en -Ġdorm ant -ĠFirst ly -ĠE ighth -ĠH UN -ĠDes ire -or as -Girl s -ĠDes mond -z ar -am ines -O AD -exec ute -Ġbo obs -ĠAT L -_ ( -Chel sea -Ġmasturb ation -ĠCo C -Ġdestroy er -ĠCh omsky -Ġsc atter -ĠAss ets -79 6 -ĠC argo -Ġrecept ive -ĠSc ope -Ġmarket ers -Ġlaun chers -Ġax le -ĠSE A -se q -ĠM off -f inding -ĠGib bs -Georg ia -extreme ly -N J -Ġlab orers -st als -Ġmed iation -ĠH edge -at own -Ġi od -des pite -v ill -J ane -ex istence -Ġcoinc ided -ĠUt ilities -ĠChe ap -Ġlog istical -Ġcul mination -ĠNic otine -p ak -F older -Ġrod ents -st uff -Ġlaw fully -Ġreper to -io ch -j j -Dial ogue -HH HH -lic tion -Look s -Ġ29 7 -Ġtur rets -ĠAb andon -Ġinc ess -ĠTraff ord -Ġcur led -Ġprefer ring -Ġprivat ization -Ġir resist -ĠP anda -ĠSh ake -ĠMc Gr -ãĥ Ħ -und ers -Ġdiscrim inated -Ġbart ender -I LE -Atl antic -Ġprop ensity -ĠW iz -ĠG im -con ference -Ġrein forces -G h -w agon -Ġe erie -F al -Ġhug ged -rac ist -R IC -F u -Ġf iller -ĠSt ub -Ġeng raved -ĠWrest le -Ġimagin ative -ĠPe er -ĠFact ors -an us -ĠDrac ula -mon itor -Ġrou ters -ib ia -ĠBoo lean -end ale -ĠSl aughter -ĠSh ack -R FC -ĠSpiel berg -S ax -ĠPH OTO -ĠCl over -ĠR ae -Dep ending -ĠMem or -ar am -Ġpier ced -Ġcur tains -v ale -ĠInqu isition -ĠP oke -Ġforecast ing -Ġcompl ains -S ense -ĠHer mes -isc overed -Ġb ible -ĠMor ph -Ġg erm -78 5 -D ON -Ġcon gen -Ġcr ane -ĠD PR -Ġrespect fully -R oom -ĠN aw -ĠDal ai -re ason -ĠAng us -Educ ation -ĠTitan ic -Ë ľ -Ġo val -un ited -Ġthird s -Ġmoist ur -ĠC PC -M iami -Ġtent acles -ĠPol aris -ex c -ex clusive -ĠPra irie -Ġcol ossal -ĠBl end -sur prisingly -ÃŃ s -Ġindo ctr -Ġbas al -ĠMP EG -und o -Spl it -Develop ment -Ġlan tern -19 71 -Ġprov ocation -Ġang uish -ĠB ind -ĠLe ia -duc ers -ipp y -conserv ancy -Ġinitial ize -ĠTw ice -ĠSu k -Ġpred ic -Ġdi ploma -Ġsoc iop -Ing redients -Ġhamm ered -ĠIr ma -Q aida -Ġglim ps -ĠB ian -Ġst acking -Ġf end -gov track -Ġun n -dem ocratic -ig ree -Ġ5 80 -Ġ29 4 -Ġstraw berry -ID ER -Ġcher ished -ĠH ots -Ġinfer red -Ġ8 08 -ĠS ocrates -O regon -ĠR oses -ĠFO IA -Ġins ensitive -Ġ40 8 -Recomm end -ĠSh ine -Ġpain staking -UG E -ĠHell er -ĠEnter prises -I OR -ad j -N RS -L G -Ġalien ated -Ġacknowled gement -ĠA UD -ĠRen eg -Ġvou chers -Ġ9 60 -Ġm oot -ĠDim ensions -Ġc abbage -B right -g at -ĠK lu -Ġlat ent -Ġz e -ĠM eng -Ġdis perse -Ġpand emonium -H Q -Ġvirt uous -ĠLoc ations -ee per -prov ided -Ġse ams -ĠW T -iz o -PR OV -Ġtit anium -Ġrecol lection -Ġcr an -Ġ7 80 -ĠN F -49 1 -64 2 -p acking -59 8 -text ure -Sp ider -fre edom -cipl ed -ĠTAM ADRA -âĻ ¦ -aut hent -ĠW ANT -r ified -Ġr ites -Ġuter us -k iss -Ġâī ¤ -Ġsk illet -Ġdis enfranch -ĠGa al -Comp an -Ġage ing -gu ide -B alt -Ġiter ator -Ġdiscretion ary -t ips -Ġprim ates -ĠTechn ique -ĠPay ments -az el -ĠR OCK -stant ial -0 60 -Ġd mg -ĠJack ets -ĠPlay off -Ġnurs ery -ĠSy mb -art on -Ġannex ation -Color ado -Ġco ils -ĠSh oes -âĦ¢ : -ĠRo z -COM PLE -ĠEve rest -ĠTri umph -J oy -G rid -à ¼ -process or -ĠPros per -ĠSever us -ĠSelect ed -r g -ĠTay yip -St ra -Ġski ing -Ġ? ) -Ġpe g -Tes la -Ġtime frame -Ġmaster mind -ĠN B -scient ific -ĠSh it -gener ic -IN TER -N UM -Ġst roll -ĠEn ix -ĠM MR -ĠE MS -m ovie -Ĥ ª -Ġminim izing -idd ling -Ġilleg itimate -Ġprot otyp -Ġpremature ly -Ġmanual s -obb ies -ĠCass idy -D EC -des ktop -Ġaer os -Ġscreen ings -Ġdeb ilitating -ĠGr ind -nature conservancy -Ġf ades -ter mination -assets adobe -F actor -Ġdefinitive ly -P oké -ap ult -ĠLaf ayette -C orn -ĠCor al -Ġstagn ant -T ue -Ġdissatisf action -G ender -Ġkid neys -ĠG ow -ĠDef eat -ĠAsh ton -Ġcart els -Ġfore closure -ĠExpl ore -stre ngth -ot in -Ġveterin arian -Ġf umble -Ġpar ap -ĠSt rait -r ils -Ġpr ick -ĠBerm uda -ĠAm munition -skin ned -Ġab ound -ĠB raz -Ġshar per -ĠAsc ension -Ġ9 78 -Ġpreview s -Ġcommun ion -ĠX Y -Ġph ony -Ġnewcom er -Ġ3 32 -." ," -Ġredist ribution -Prot ect -ĠSo f -K al -Ġlip stick -w orst -Ġtang led -Ġretrospect ive -int eger -Ġvolunte ering -Ġ19 07 -Ġ -------------------- -ic hen -Ġunve iling -Ġsen seless -Ġfisher ies -\ - -Ġh inges -Ġcalcul us -My th -Ġund efeated -Ġoptim izations -Ġdep ress -Ġbill board -ĠY ad -ĠPy ramid -Is n -I de -Ġleg ion -ĠK ramer -ent anyl -Ġpenet rating -ĠHaw th -ĠPR ODUCT -ĠGer ard -ĠP act -ĠIn cluding -ĠEl ias -ĠEl aine -vis ual -Ġhum ming -Ġcond esc -ĠF asc -ä¸ Ĭ -Ġe galitarian -Ġdev s -ĠD ahl -O ps -D H -ĠB ounce -id ated -ald o -Ġrepublic an -Ġh amb -ĠS ett -ograph ies -CH APTER -Ġtrans sexual -Ġsky rocket -ans wer -Ġmark up -Ø ª -Ġhero ine -Comp are -ĠT av -Be ast -Ġsuccess ors -Ġna ïve -ĠBuck ley -st ress -me at -Ġdownload able -Ġindex ed -Ġsc aff -ĠL ump -ĠHom o -Stud io -In sp -Ġr acked -far ious -ĠPet ty -Ex ternal -Ġ19 09 -W ars -com mit -put ers -Ġun ob -ĠEr r -ĠE G -ĠAl am -ĠSiber ia -ĠAtmosp heric -IS TER -ĠSatan ic -trans lation -ĠL oud -tra umatic -l ique -Ġreson ate -ĠWel ch -Ġspark ing -ĠT OM -t one -Ġout l -Ġhandc uffed -ĠSer ie -8 01 -Ġland marks -ĠRee ves -Ġsoft ened -Ġdazz ling -ĠW anted -month s -Mag ikarp -Ġunt reated -ĠBed ford -M i -ĠDynam o -O re -79 5 -Ġwrong ful -Ġl ured -Ġcort isol -Ġve x -d rawn -ile t -Download ha -ĠF action -Ġlab yrinth -Ġhij acked -w aters -er ick -Ġsuper iors -ĠRow ling -ĠGu inness -Ġt d -99 2 -Ġune arthed -Ġcentr if -Ġsham eless -P od -ĠF ib -Ġ icing -Ġpredict or -Ġ29 2 -fore station -con struct -C and -@ # -Ġag itated -Ġre pr -OV A -Ġkn itting -ĠLim a -Ġf odder -68 4 -ĠPerson a -k l -7 01 -Ġbreak up -á ¸ -Ġapp alled -Ġantidepress ants -ĠSus sex -Har ris -ĠTher mal -ee ee -U pload -Ġg ulf -Ġdoor step -ĠSh ank -L U -ĠM EN -ĠP ond -s orry -Ġmis fortune -n ance -Ġb ona -M ut -Ġde graded -ĠL OG -ĠN ess -an imal -Ġa version -und own -Ġsupplement ed -ĠC ups -Ġ50 4 -Ġdep rive -ĠSpark le -Å Ĥ -ĠMed itation -auth ors -ĠSab an -ĠN aked -air d -ĠMand arin -ĠScript ures -ĠPerson nel -ĠMahar ashtra -Ġ19 03 -ĠP ai -ĠMir age -omb at -Access ory -Ġfrag mented -T ogether -Ġbelie vable -ĠGl adiator -al igned -ĠSl ug -M AT -Ġconvert ible -ĠBour bon -amer on -ĠRe hab -nt ax -Ġpowd ered -pill ar -Ġsm oker -ĠMans on -ĠB F -5 11 -ĠGood ell -ĠD AR -m ud -g art -Ġob edient -ĠTrans mission -ĠDon ation -8 80 -Ġbother ing -Material s -ãĤ ± -dest roy -Ġfore going -Ġanarch ism -ĠK ry -ice ps -Ġl ittered -ĠSch iff -Ġanecd otal -un its -Ġf ian -ĠSt im -ĠS OME -ĠInv aders -Ġbehaviour al -ĠVent ures -Ġsub lime -Ġfru ition -ĠPen alty -Ġcorros ion -¶ ħ -Ġlik ened -Ġbesie ged -ween ey -ĠCre ep -Ġlinem en -mult i -ic ably -ud der -Ġvital ity -Ġshort fall -ĠP ants -ap ist -H idden -ĠDro ps -med ical -Ġpron unciation -ĠN RL -Ġinsight ful -J V -ĠBe ard -ĠCh ou -Ġchar ms -Ġb ins -Ġamb assadors -ĠS aturdays -Ġinhib itor -ĠFr anch -6 01 -', ' -ĠCon or -art ney -ĠX peria -g rave -be es -ĠProtest ants -Ġso aking -ĠM andal -Ġph ased -Ġ6 60 -Ġsc ams -Ġbuzz ing -ĠItal ians -ĠLoren zo -ĠJ A -Ġhes itated -Ġcl iffs -ĠG OT -ingu ishable -Ġk o -Ġinter ruption -Z ip -Lear ning -Ġundersc ores -ĠBl ink -K u -57 9 -ĠAut ob -I RE -Ġwater ing -Ġpast ry -8 20 -Ġvision ary -ĠTempl ar -awa ited -Ġpist on -Ġant id -current ly -Ġp ard -Ġw aging -Ġnob ility -ĠY us -Ġinject ing -f aith -ĠP ASS -å º -Ġret ake -ĠPR OC -Ġcat hedral -b ash -Ġwrest lers -Ġpartner ing -Ġn oses -Ġ3 58 -Trans form -am en -Ġb outs -ĠId eal -ĠConstant in -Ġse p -ĠMon arch -att en -ĠPe oples -mod ified -Ġmor atorium -Ġpen chant -Ġoffensive ly -Ġprox ies -ok ane -ĠTaiwan ese -ĠP oo -ĠH OME -us ional -Ġver bs -ĠO man -vis ory -Ġpersu asion -Ġmult it -Ġsc issors -G ay -ow ay -oph ysical -l us -gn u -Ġap ocalyptic -Ġabsurd ity -Ġplay book -Ġautobi ography -I UM -Ġsne aking -ĠSim ulation -pp s -ell ery -Plan et -Ġright fully -Ġn iece -ĠN EC -ĠIP O -ĠDis closure -lean or -ous y -ST ER -Ġ28 2 -Cru z -Ch all -64 3 -ĠSurv ive -ĠF atal -ĠAm id -ap o -We apons -D EN -7 70 -ĠGreen wald -Ġlin en -al os -Ġpollut ants -ĠPCI e -k at -Ġp aw -ĠK raft -C hem -ĠTermin ator -Ġre incarn -Ġ] [ -ĠSe eds -Ġsilhou ette -ĠSt ores -Ġgro oming -ĠD irection -ĠIs abel -ĠBr idges -ðŁ ij -E ED -ĠM orsi -Ġval ves -ĠRank ed -ĠPh arma -ĠOrgan izations -Ġpenet rated -ĠRod ham -ĠProt oss -Ġove rest -Ġex asper -ĠT J -Ġ 000000 -Ġtrick le -Ġbour bon -WH O -Ġw retched -Ġmicrosc opic -Ġcheck list -Ġad orned -R oyal -Ad minist -ĠRet irement -ĠHig hest -We ather -ile ge -Ġincre ments -ĠC osponsors -Ġmas se -ĠS inn -r f -Ġh ordes -as sembly -75 4 -ĠNat asha -ĠTY PE -ĠGEN ERAL -Ġarr anging -Ġ40 7 -l ator -Ġg lean -Ġdisc redited -Ġclin icians -UN E -Ġachie ves -ĠEm erson -com plex -= [ -Ġprincip ally -Ġfra il -p icked -Ġthan king -Ġre cl -ĠL AST -Ġsupp ressing -il ic -Ġantidepress ant -ĠLis bon -Ġth or -Ġsp a -Ġking doms -ĠPear ce -em o -Ġpl ung -Ġdiv est -Ġ ******************************** -b is -osp els -ad r -Sp irit -hall a -P ink -end ez -Ġresurrect ed -esc ape -ĠRosen stein -Ġge ological -Ġnecess ities -Ġcarn iv -ĠE lys -ĠBar ney -Ġ29 6 -dig y -ST ON -D OWN -Ġmil estones -Ġk er -Ġdismant ling -Ġre prim -Ġcross ings -19 45 -Ġpatri archy -Ġblasp hemy -Ġ3 59 -met ry -ĠOb esity -ĠDiff erences -bl ocking -ãĥķ ãĤ¡ -ich ita -ĠSab ha -ph alt -ĠCol o -ual a -effic ients -ĠMed ina -con sole -55 7 -ĠHann ibal -ĠHab it -ĠF ever -Ġthen ce -Ġsyn agogue -Ġessential s -Ġw ink -ĠTr ader -ID A -ĠSp oiler -ĠIceland ic -ĠHay ward -Ġpe ac -Ġmal ice -Ġflash back -Ġth w -Ġlay offs -L iquid -Ġtro oper -Ġh inge -ĠRead ers -Ph ill -ĠB auer -Cre ated -Ġaud its -ac compan -Ġunsus pecting -ier a -6666 6666 -Ġbro ch -Ġapprehend ed -ĠM alk -cer ning -ĠCod ex -O VER -M arsh -ĠD eng -ĠExp ression -Ġdisrespect ful -Ġasc ending -t ests -ĠPlaint iff -ster y -ĠAl ibaba -din and -ĠDem psey -Applic ations -mor al -Ġthrough put -Ġquar rel -Ġm ills -Ġhe mor -ĠC ASE -terror ist -st im -ifest yle -ro zen -CE PT -Ar k -u ci -lect ic -Ġirrit ating -she ets -A y -Ġrede emed -Ġhorn y -ĠTe ach -ĠS ear -dem ocracy -4 65 -ĠRest ore -Ġstand by -ĠP is -iff in -Ġsleep y -Ġextr ater -Ġcompl iments -Fram eworks -Ġinstall s -Ġb anging -sur face -found land -Ġmetaph ysical -Ġ28 3 -oul s -dev ices -Ar gs -ĠSac rifice -ĠMcC orm -es on -Cons ervative -ĠM ikhail -see ing -is ively -ĠRo oms -ĠGener ic -Ġenthusi astically -Ġgri pped -Ġcomed ic -ĠElectric ity -Ġgu errilla -Ġdec oration -ĠPerspect ive -Ġconsult ations -Ġun amb -Ġplag iar -Ġmagic ian -Ġe rection -ĠTour ism -or ied -ro xy -11 00 -T am -Ī è -Î ³ -× ª -ĠPred ators -Nit rome -Ġtelesc opes -project s -Ġun protected -Ġst ocked -ĠEnt reprene -nex pected -Ġwast ewater -V ill -Ġint imately -Ġi Cloud -ĠConst able -Ġspo of -Ġne farious -Ġfin s -Ġcens or -ĠMod es -ĠEs per -ar bon -Ġinter sections -Ġlaud ed -Ġphys i -Ġgener ously -ĠThe Nitrome -ĠTheNitrome Fan -Ġar isen -ĠÙ Ī -Ġg lands -ĠPav ilion -ĠGu pta -Ġuniform ly -Ġr amps -ri et -ĠWH EN -ĠVan essa -Ġrout ed -Ġlim p -ĠC PI -p ter -int uitive -Ġv aping -Ġexperiment ed -ĠOlymp us -ĠAm on -Ġsight ing -Ġinfiltr ate -ĠGentle man -Ġsign ings -ĠMe ow -ĠNav igation -che cks -4 33 -Ġel apsed -ĠBulg arian -esp ie -ĠS OM -d uring -Ġsp ills -anc a -ĠPly mouth -M AL -Ġdomest ically -ĠWater gate -ĠF AM -k illed -ed ited -ĠYour self -Ġsynchron ization -ĠPract ices -ST EP -Ġgen omes -ĠQ R -not ice -Ġloc ating -z in -Ġ3 29 -al cohol -Ġk itten -V o -Ġr inse -Ġgrapp le -ĠSc rew -ĠD ul -A IR -Ġle asing -ĠCaf é -Ġro ses -ĠRes pect -Ġmis lead -Ġperfect ed -Ġnud ity -Ġnon partisan -ĠCons umption -Report ing -Ġnu ances -Ġdeduct ible -ĠSh ots -Ġ3 77 -Ġæ ľ -ano oga -Ben ef -ĠB am -ĠS amp -if ix -Ġgal van -ĠMed als -rad ius -Ġno bles -Ġe aves -igr ate -K T -ĠHar bour -u ers -Ġrisk ed -re q -Ġneuro t -get table -ain a -Rom ney -Ġunder pin -Ġlo ft -ĠSub committee -ĠMong ol -b iz -Ġmanif ests -ass isted -ĠG aga -Ġsy nergy -Ġreligious ly -ĠPre f -ĠG erry -T AG -ĠCho i -4 66 -beh ind -ĠO u -Gold Magikarp -Ġhemor rh -R iver -Ġtend on -Ġinj ure -ĠF iona -Ġp ag -Ġag itation -|| || -ur an -ĠE SA -Ġest eem -Ġdod ging -Ġ4 12 -r ss -Ġce ases -ex cluding -Ġint akes -Ġinsert s -Ġemb old -ĠO ral -up uncture -4 11 -ĠUn ified -ĠDe le -Ġfurn ace -ĠCoy otes -ĠBr ach -L abor -Ġhand shake -Ġbru ises -Gr ade -éĹ ĺ -ĠGram my -ile en -St ates -ĠScandinav ian -ĠKard ash -8 66 -Ġeffort lessly -ĠDI RECT -ĠTH EN -ĠMe i -ert ation -19 68 -Ġgro in -w itch -Requ irements -98 5 -Ġroof s -Ġest ates -ĠH F -Ġha ha -Ġdense ly -ĠO CT -Ġpl astics -Ġincident ally -ĠTr acks -ĠTax es -Ġch anted -Ġforce ful -ĠBie ber -ĠK ahn -K ent -ĠC ot -lic ts -F ed -Ġhide ous -ĠVer d -ĠSynd icate -ĠIl legal -J et -ĠD AV -re asonable -c rew -Ġfundamental ist -Ġtruth ful -ĠJ ing -Ġl il -Ġdown ed -Ġen chanted -ĠPolic ies -ĠMcM aster -ĠH are -ides how -Ġpar ams -en cers -gorith m -Ġallow ances -Ġturb ulent -Ġcomplex ities -ĠK T -Ġ3 37 -ĠGen etic -F UN -D oug -t ick -Ġg igs -ument hal -Ġpatriarch al -Ġcal c -, ... -Ġc out -ĠGu an -Ġpath ological -ĠR ivals -Ġunder rated -Ġflu orescent -ĠJ iu -arna ev -ĠQu an -Ġ4 29 -Ġ ਠ-M ario -Con struct -ĠC itation -ĠR acial -ĠR SA -ĠF idel -Ġ3 95 -Person ally -C ause -à » -rad ical -in en -Ġvehement ly -ĠPap a -Ġintern ship -Ġfl akes -ĠRe ck -Luck ily -B ra -20 20 -rav ings -R N -W onder -Ser iously -Ġre usable -Ġpoll uted -ĠP eng -le igh -ind le -Ġcircuit ry -ĠMad onna -ĠB ART -Res idents -att ribute -Phil adelphia -Cl ub -Ġplan ner -Ġfr antically -Ġfaith fully -ĠTerrit ories -ĠL AT -ĠAnders en -an u -ĠP ARK -ĠS ora -i age -ĠPlay offs -ĠG CC -4 27 -Ġab norm -ĠL ever -Ġdisob edience -As ync -ĠShe a -V ert -Ġsk irts -ĠSaw yer -x p -Ġwors ening -Ġsc apego -ĠAng le -oth al -Ġtro ve -ĠSt y -ĠN guyen -mar ine -ide on -Dep ths -Bl og -ĠIll uminati -Ġtract s -Ġorgan ise -Ġo str -F s -Ġlever aging -ĠD aredevil -as ar -Ġl ang -Ġex termin -urs ions -ĠRom o -ãĤ¤ ãĥĪ -Ġcont ended -Ġencounter ing -ĠTable t -ĠAltern ate -sk ill -Ġswe ets -Ġco hesive -cap acity -Ġrep ud -Ġl izard -ro o -Ġpilgr ims -ĠR uff -ĠInstr ument -ĠLog o -uit ous -E H -Ġsales man -Ġank les -L ed -ĠPat ty -ud os -Own er -Ġdiscrep ancies -k j -M U -Ġuncond itional -Dragon Magazine -i ard -O ak -ĠConvers ation -be er -ĠOs aka -D elta -us ky -Ġsecret ion -Ġpl aza -Ġm ing -Ġde pletion -ĠM ous -ĠI TS -ĠH imal -ĠFle ming -Ġcyt ok -ĠH ick -Ġbat ters -ĠInt ellectual -6 75 -é r -IS ION -ĠQu entin -ĠCh apters -ih adi -Ġco aster -WAY S -ĠL izard -ĠY or -and ering -S kin -ha ust -ab by -Ġportray ing -Ġwield ed -d ash -Ġprop onent -Ġr ipple -Ġgrap hene -Ġfly er -Ġrec urrent -Ġdev ils -Ġwater fall -æĺ ¯ -go o -Text Color -Ġtam pering -IV ES -TR UMP -ĠAb el -ĠS AL -ĠHend ricks -ĠLu cius -b ots -Ġ40 96 -IST ORY -Gu est -ĠN X -in ant -Ben z -ĠLoad ed -ĠCle ver -t reatment -Ġta vern -Ġ3 39 -ĠT NT -ific antly -Tem perature -F el -Ġunder world -ĠJud ges -Ġ< + -Ġst ump -Ġoccup ancy -Ġab er -ĠF inder -) ", -ĠN unes -res et -in et -ect omy -Ġwell ness -ĠP eb -quart ered -and an -Ġneg atives -ĠTh iel -ĠCl ip -ĠL TD -Ġbl ight -Ġreperto ire -K yle -Ġqu er -ĠC es -Ġha pl -98 9 -ĠTh ames -isc opal -Des k -ivari ate -ĠEx cellence -found ation -Ġâ ĩ -X i -Ġmyster iously -esty les -Ġper ish -ĠEng els -ĠDE AD -09 0 -}} } -ĠUn real -Ġrest less -ID ES -orth odox -ĠInter mediate -Ġdin ners -ĠTr out -ĠSe ym -ĠHall s -og ged -Ġtraged ies -Ġdid nt -67 6 -Ġail ments -Ġobserv able -ĠV ide -ad apt -ĠD usk -Ġprofessional ism -ĠPres cott -ĠInd ies -p ox -ĠMe hran -W ide -Ġend emic -ĠPar an -B ird -Ġped als -ĠI U -ĠAdam ant -ĠH urt -Ġcorrel ates -urd en -Ġspons oring -cl imate -ĠUnivers ities -ĠK not -enn es -ĠDam ian -ĠAx el -S port -Ġbar b -ĠS no -sh own -ste en -ud ence -Ġnon violent -Ġhom ophobia -Ġbiom ass -ĠDet ail -Ġsrf N -ĠT une -accompan ied -I ENCE -Al bert -ĠMong o -z x -ĠCer berus -or bit -c ens -Ġsl ay -SH ARE -H Y -Ġb rawl -ĠPro be -Ġnonex istent -ĠClare nce -ĠBlack burn -Ġport als -ĠR ita -ĠRem ain -ĠLe vant -Ġtrick ed -ĠF erry -aver ing -ĠStraw berry -ĠAn swers -Ġhorrend ous -ĠA man -Supp lement -ĠT oad -Ġpe eled -Ġman oeuv -ĠU zbek -mond s -ĠH ector -Ġ40 2 -pe es -fix es -Ġd j -Ġres umes -Ġaccount ant -Ġadvers ity -Ġham pered -ĠL arson -Ġd oping -part s -H ur -Ġbe arded -Ġy r -ĠPlug in -å¥ ³ -Ġ/ ** -rol ley -Ġwaters hed -ĠSub mission -if lower -AS C -Ġcho ir -Ġsculpt ures -m A -incre asing -ai i -Ġsne akers -Ġconfront s -ĠEle phant -ĠEl ixir -Ġrec al -ĠT TL -w idget -ĠW ax -ĠGr ayson -Ġha irst -Ġhumili ated -ĠWAR N -app iness -ĠT TC -F uel -Ġpol io -Ġcomplex es -Ġbab e -ĠX IV -P F -). [ -P arts -Ġ4 35 -M eg -ĠY ards -ĠAL P -Ġy ells -Ġprin ces -Ġbull ies -ĠCapital ism -ex empt -FA Q -ĠSp onge -ĠAl a -Ġpleas antly -Ġbu f -Ġden ote -Ġunp ublished -Ġkne eling -asc a -Ġl apse -al ien -99 4 -Ġrefere es -ĠLaw yers -S anta -Ġpuzz ling -ĠProm etheus -ĠPh araoh -ĠDel ay -Ġfacilit ates -ĠC ES -Ġjew els -Ġbook let -ond ing -Ġpolar ization -ĠMor an -ĠSal ad -ĠS OS -ĠAdv ice -PH OTOS -IC AN -iat ures -ex press -ĠWonder land -ĠC ODE -ĠCL ASS -9 75 -Ġg rep -ĠD iesel -ĠGl ac -! ?" -Ġr m -o ine -disc rimination -ĠN urse -m allow -Ġv ortex -ĠCons ortium -Ġlarge Download -stra ight -augh lin -G rad -Ġpublic ized -ĠW aves -ĠRed d -Ġfest ivities -ĠM ane -ar ov -Ġfleet ing -ĠDr unk -ug en -C ele -Ġchromos omes -ĠD OT --+-+ -+-+ -Ġbus iest -ĠBe aver -Sy rian -ĠK yr -k as -ĠCross Ref -19 50 -76 01 -Ġrepe aling -ĠWin ners -ĠMac ro -ĠD OD -bl ance -S ort -64 1 -Ġmet re -ĠD irk -Ġgo ggles -Ġdraw backs -Ġcomplain ant -Ġauthor izing -Ġantit rust -oper ated -Ġm ah -Ġexagger ation -Am azing -ĠSer aph -Ġha ze -w ow -Ġextingu ished -Ġcan yon -ĠB osh -Ġv ents -Ġsc rape -Cor rect -4 26 -Ġav g -Dem and -ĠâĪ ¼ -Ġmicrobi ota -"} ]," -ĠSt ev -B io -ĠPlan es -Ġsuggest ive -Ġdec ipher -ĠRefuge e -ĠKe jriwal -ĠGreen peace -Ġdecl ass -ĠSound ers -Ġth o -Ġdec rypt -Ġbr ushing -ĠJane iro -ip op -S i -8 77 -ĠGeoff rey -Ġc pu -ĠHaz el -Ġview points -Ġcris py -ĠNot ification -Ġsold er -ĠMod est -ĠHem isphere -Ġcass ette -in cludes -Ġident ifiers -ĠC ALL -in cent -T odd -ĠSwe ep -Ġ3 34 -b oss -Ġsm ir -gin x -Ġtown ship -Ġg rieving -ĠMos que -Net flix -AS ED -ĠMillenn ials -oc om -19 67 -Ġbold ly -s leep -Ġes che -arij uana -Ġsw irl -ĠPen al -Ġneglig ent -ĠStephen son -K ER -ĠZ oro -ris is -Ġlocal ization -ĠSeym our -ĠAng lic -red itation -prot ection -ĠPa ige -Ġo mit -ĠR ousse -ĠT ub -Ġinv itations -t ty -Ġm oss -ph ysical -C redits -Ġan archy -Ġchild care -Ġl ull -ĠM ek -ĠL anguages -lat est -ĠSan ford -Ġus ability -Ġdiff use -ĠD ATA -Ġsp rites -ĠVeget a -ĠProm otion -ãĥ¼ ãĤ¯ -rict ing -z ee -Tur kish -ĠTD s -pro ven -57 1 -Ġsmug glers -707 10 -Ġreform ed -ĠLo is -Ġun fl -ĠWITH OUT -ĠReturn ing -ann ie -ĠTom as -Fr anc -ĠProf it -ĠSER V -ĠR umble -ik uman -es an -Ġt esters -Ġgad get -Ġbrace let -ĠF SA -comp onent -Ġparamed ics -Ġj an -ĠRem em -ĠSk inner -Ġl ov -ĠQu ake -rom a -Ġfl ask -Pr inc -Ġover power -Ġlod ging -ĠK KK -ret te -Ġabsor bs -w rote -Ġ ," -K ings -ĠH ail -ĠFall ing -xt ap -ĠHel ena -ire ns -L arry -Ġpamph let -ĠC PR -G ro -ĠHirosh ima -Ġhol istic -". [ -Ġdet achment -Ġas pire -Ġcompl icit -ĠGreen wood -Ġresp awn -ĠSt upid -ĠFin ished -f al -b ass -Ġab hor -Ġmock ery -ĠFe ast -VID EO -Ġcon sec -ĠHung ry -P ull -ĠH ust -it ance -? ãĢį -) -- -ĠPar allel -con v -4 69 -ha ar -w ant -P aper -m ins -ĠTor o -ĠTR UMP -ĠR ai -D W -ĠW icked -ĠL ep -Ġfun ky -Ġdetrim ent -ios is -ache v -Ġde grade -im ilation -Ġret ard -Ġfrag mentation -Ġcow boy -ĠY PG -ĠH AL -Parent s -ĠS ieg -ĠStra uss -ĠRub ber -× IJ -Fr ag -Ġp t -Ġoption ally -ĠZ IP -ĠTrans cript -ĠD well -88 2 -M erc -ĠM OT -ãĥ¯ ãĥ³ -Ġhun ts -Ġexec utes -In cludes -Ġacid ic -ĠRespons ibility -ĠD umb -we i -And erson -ĠJas per -ight on -abs olutely -Ad ult -Ġpl under -Mor ning -ĠT ours -ĠD ane -Î º -ĠT EST -ĠG ina -Ġcan ine -aw an -Ġsocial ists -ĠS oda -Ġimp etus -ĠSupplement ary -oli ath -ĠKinn ikuman -mitted ly -second s -Ġorganis ers -Ġdocument aries -Vari able -GRE EN -Ġres orts -Ġbr agging -Ġ3 68 -Art ist -w k -bl ers -Un common -ĠRet rieved -Ġhect ares -Ġtox in -r ank -Ġfaith s -ĠG raphic -Ġve c -ĠL IA -Af rican -Ġard ent -end iary -L ake -ĠD OS -cient ious -ĠOk awaru -ĠAll y -ĠTim eline -D ash -ĠI c -contin ue -Ġt idy -Ġinstinct ively -ĠP ossibly -ĠOut door -ĠWould n -Ġl ich -ĠBr ay -ĠA X -Ġà ī -Ġ+ # -\ ' -Direct ory -ab iding -Ġf eral -ic ative -but t -Ġper verse -S alt -Ġwar ped -Ġnin eteen -Ġcabin ets -Ġsrf Attach -ĠSl oan -Ġpower ing -reg ation -F light -se vere -Ġst ren -Ġc og -ap ache -Ġâ Ŀ -Ġcaf eteria -p aces -ĠGrim oire -uton ium -Ġr aining -Ġcir cling -Ġlineback ers -c redit -Ġrep atri -ĠCam den -lic ense -Ġly ric -Ġdescript or -Ġval leys -Ġre q -Ġback stage -ĠPro hibition -ĠK et -Op ening -S ym -æĸ ¹ -Ġserv ings -Ġoverse en -Ġaster oids -ĠMod s -ĠSpr inger -ĠCont ainer -è » -ĠM ens -Ġmult im -Ġfire fighter -pe c -Ġchlor ine -Ð ¼ -end i -Ġsp aring -Ġpolyg amy -ĠR N -ĠP ell -Ġt igers -Ġflash y -ĠMad ame -S word -Ġpref rontal -Ġpre requisite -uc a -Ġw ifi -Ġmiscon ception -Ġharsh ly -ĠStream ing -ot om -ĠGiul iani -foot ed -Ġtub ing -ind ividual -z ek -n uclear -m ol -Ġright ful -49 3 -Ġspecial ization -Ġpassion ately -ĠVel ocity -ĠAv ailability -T enn -Ġl atch -ĠSome body -Ġhel ium -cl aw -Ġdi pping -XX X -Ġinter personal -7 10 -Ġsub ter -Ġbi ologists -ĠLight ing -Ġopt ic -Ġden im -end on -ĠC orm -Ġ3 41 -ĠC oup -Ġfear less -Ġal ot -ĠCliff ord -ĠRun time -ĠProv ision -up dated -lene ck -Ġneur on -Ġgrad ing -ĠC t -sequ ence -in ia -con cept -Ġro aring -ri val -ĠCaucas ian -Ġmon og -key es -Ġappell ate -Ġlia ison -EStream Frame -ĠPl um -! . -Ġsp herical -Ġper ished -Ġbl ot -Ġben ches -Ġ4 11 -Ġpione ered -Ġhur led -Jenn ifer -ĠYose mite -Ch air -Ġreef s -Ġelect or -ĠAnt hem -65 2 -Ġun install -Ġimp ede -Ġbl inking -Ġgot o -Dec re -A ren -Ġstabil ization -ĠDis abled -ĠYanuk ovych -Ġoutlaw ed -ĠVent ura -ten ess -Ġplant ation -Ġy acht -ĠHu awei -Ġsol vent -Ġgr acious -Ġcur iously -Ġcapac itor -Ġc x -ĠRef lex -Ph ys -ĠC f -pt in -cons ervative -Ġinv ocation -c our -F N -ĠNew ly -H our -As ian -ĠLe ading -ĠAer ospace -An ne -Ġpre natal -Ġdeterior ating -H CR -ĠNorm andy -ol ini -ĠAm bro -9 10 -Ġset backs -ĠT RE -Ġs ig -ĠSc ourge -59 7 -79 8 -Game play -Ġm sec -M X -Ġprice y -ĠL LP -aker u -Ġover arching -ĠB ale -Ġworld ly -Cl ark -Ġscen ic -Ġdisl iked -ĠCont rolled -T ickets -ĠE W -ab ies -ĠPl enty -Non etheless -Ġart isan -Trans fer -ĠF amous -Ġinf ield -ble y -Ġunres olved -ĠML A -ãĤ Ĥ -Cor rection -Ġdemocr at -ĠMore no -ro cal -il ings -Ġsail or -Ġr ife -h ung -Ġtrop es -Ġsn atched -ĠL IN -ĠB ib -ES A -ĠPre v -ĠCam el -run time -Ġob noxious -4 37 -Ġsum mers -Ġunexpl ained -ĠWal ters -cal iber -Ġg ull -ĠEnd urance -ä½ ľ -Ġ3 47 -Ir ish -Ġaer obic -Ġcr amped -ĠHon olulu -à © -us erc -ec ast -AC Y -ĠQu ery -ãĤ¹ ãĥĪ -Bet a -Ġsuscept ibility -ĠSh iv -ĠLim baugh -Ġà ĸ -ĠN XT -ĠM uss -ĠBrit ons -ES CO -EG IN -Ġ% % -Ġsec ession -ĠPat ron -ĠLu a -n aires -ĠJPM organ -us b -ocy te -Ġcouncill ors -ĠLi ang -f arm -Ġnerv ously -Ġattract iveness -ĠK ov -j ump -Pl ot -Ġst ains -ĠStat ue -ĠApost les -he ter -ĠSUP PORT -Ġoverwhel m -Y ES -Ġ29 1 -d ensity -Ġtra pping -M it -Ġf ide -ĠPam ela -atl antic -Dam n -Ġp ts -OP A -Ġserv icing -Ġoverfl owing -ul o -ĠE rit -t icket -light ing -ĠH mm -ãĥ¼ ãĥ« -im oto -Ġchuck le -4 23 -ãģ ķ -sh ape -Ġque ues -Ġanch ors -ãĤ¼ ãĤ¦ãĤ¹ -F er -Ġaw oke -Ġ6 66 -h ands -Ġdiver gence -Ġ50 5 -T ips -Ġdep ot -Ġske w -ĠDel iver -op ot -Ġdiv ul -ĠE B -uns igned -ĠUn i -X box -Ġfor ks -Ġ7 02 -å ¯ -Ġpromot ers -ĠV apor -Ġlev ied -sl ot -Ġpig ment -Ġcyl inders -C RE -Ġsn atch -Ġperpet ually -Ġl icking -ĠFe et -ĠKra ken -ĠHold en -ĠCLS ID -m r -Ġproject or -Ġden otes -Ġchap el -ĠTor rent -b ler -R oute -ĠDef endant -ĠPublisher s -ĠM ales -ĠInn ov -ĠAg ility -rit er -ty mology -st ores -L ind -Ġf olly -ĠZur ich -B le -Ġnurt ure -Ġcoast line -uch in -D omin -Ġfri vol -ĠCons olid -res ults -M J -Ġphyl ogen -Ġha uled -ĠW iley -ĠJess ie -ĠPrep are -ĠE ps -Ġtreasure r -I AS -Ġcolon ists -Ġin und -ĠWW F -ĠCon verted -6 000 -out side -ĠApp earance -ĠRel ic -ĠM ister -s aw -Ġresult ant -Ġadject ive -ĠLaure l -ĠHind i -b da -Pe ace -Ġreb irth -Ġmembr anes -Ġforward ing -Ġcoll ided -ĠCar olyn -K ansas -5 99 -ĠSolid GoldMagikarp -Be ck -Ġstress ing -ĠGo o -ĠCooper ative -Ġf s -ĠAr chie -L iter -ĠK lopp -J erry -Ġfoot wear -War ren -Ġsc ree -h are -Under standing -P ed -Ġanth ology -ĠAnn ounce -M ega -Ġflu ent -Ġbond age -ĠDisc ount -il ial -C art -ĠNight mares -Sh am -ĠB oll -uss ie -H ttp -Atl anta -Ġun recogn -ĠB id -Ġunder grad -Ġforg iving -ĠGl over -AAAA AAAA -4 45 -V G -pa io -kill ers -Ġrespons ibly -Ġmobil ize -Ġeffect ed -ĠL umin -Ġk ale -Ġinfring ing -ann ounced -Ġf itt -b atch -ĠT ackle -ĠL ime -ĠAP P -uke mia -Ġrub y -Ġex oner -ĠCas ual -0 70 -Ġpel vic -Ġautom ate -ĠK ear -ĠCoast al -Ġcre ed -Ġbored om -ĠSt un -ri ott -Ĥ İ -Ġregener ate -Ġcomed ians -ĠOP ER -Sp ons -id ium -on is -L ocated -05 7 -Ġsusp ense -ĠD ating -C ass -Ġneoc ons -ĠShin zo -Ġaw oken -ch rist -ĠMess ages -att led -ĠSpr ay -ĠSp ice -C W -Ġshield ing -ĠG aul -Am id -Ġparam ilitary -Ġmult if -ĠTan ner -il k -Ġgodd amn -g ements -Ġbe friend -m obi -Ġ3 88 -fold er -acc a -Ġins in -g ap -N ev -fif th -Ġpsychiat ry -b anks -TH IS -Ġhar b -ac qu -Ġfac ade -ĠPower Point -80 3 -Ġbl uff -Sh ares -Ġfavor ing -El izabeth -Ãį Ãį -Ġr anger -77 2 -ĠAr che -h ak -ĠGen etics -ĠF EMA -Ġev olves -Ġest e -ĠP ets -ĠM é -ĠInterest ing -ĠCanter bury -ch apter -ĠStar fleet -Sp anish -Ġdraw back -ĠNor wich -9 70 -n orth -ag anda -Ġtransform ative -ram ids -bi ology -ad ay -Ġpropag ation -ĠGam ma -ĠDen ise -ĠCalcul ator -ent imes -ĠB ett -Ġapp endix -ĠHD D -AK ING -Ġst igmat -Ġhol ster -Ġord inarily -Ch ance -ĠCont rary -Ġad hesive -Ġgather s -6 12 -re au -ony ms -ew ays -Ġindu ces -Ġinterchange able -se m -Wh it -Ġtr ance -Ġincorpor ation -ĠExt ras -Fin ancial -Ġawkward ly -ĠStur geon -ĠH Y -Norm ally -ĠEnd ing -ĠAss ist -enc rypted -Ġsub jug -Ġn os -Ġfan atic -C ub -C U -?" . -Ġirre versible -å Ĥ -03 1 -ĠH AR -sp read -ul ia -= $ -Sc ope -L ots -Ġlif estyles -ol on -Ġf eds -Ġcongrat ulate -web kit -Ġindist inguishable -ĠSw ing -Ġcommand ments -qu ila -ab ella -m ethyl -ann abin -Ġo vere -Ġlob ster -ĠQU EST -ĠCONT IN -bern atorial -:::: :::: -ĠTra ve -ĠSam oa -AN I -75 2 -Ð ´ -userc ontent -ĠMod erate -y eah -ĠK itt -Ġwe e -Ġstuff ing -ĠInter vention -ĠD ign -Ġware houses -ĠF iji -Ġpel lets -Ġtake away -ĠT ABLE -ĠClass ical -col lection -Ġland fall -ĠMus cle -Ġsett les -ĠAD V -Ġ3 44 -L aura -Ġf ared -ĠPart ial -4 36 -oss ibility -ĠD aly -ĠT arant -ĠFu ji -am l -c ence -55 1 -ĠProced ures -ĠO CD -ĠU D -t in -Q UI -ach o -4 38 -Ġgl itches -Ġenchant ment -Ġcalcul ates -IR O -ĠH ua -alys es -ĠL ift -um o -Ġle apt -Ġhypothes ized -ĠGust av -it ans -VERS ION -æ ł -Rog er -Ġr and -ĠAd apter -Ġ3 31 -ĠPet ition -k ies -M ars -Ġunder cut -ze es -ĠLy ons -ĠDH CP -Miss ing -Ġretire es -Ġins idious -el i -> ) -. ãĢį -Ġfinal ists -ĠA ure -Ġacc user -Ġwas tes -ĠY s -ĠL ori -Ġconstitu encies -Ġsupp er -Ġmay hem -or ange -Ġmis placed -Ġmanager ial -Ġex ce -ĠCL I -Ġprim al -ĠL ent -Cry stal -h over -ĠN TS -end um -Ġd w -ĠAl c -n ostic -Ġpres erves -ĠTs arnaev -Ġtri pled -rel ative -Arc ade -k illing -ĠW EEK -ĠH anna -D ust -Com pleted -ģ « -Ġappro ves -ĠSur f -ĠLuther an -ven ants -Ġrobber ies -we ights -soft ware -at ana -ug al -Ġgrav y -ĠC ance -OLOG Y -ly ak -Ton ight -Ġunve il -Ġ19 04 -ĠMin ion -ent ious -st ice -pack ages -ĠG EAR -Ġg ol -ĠHutch inson -ĠProf ession -ĠG UN -ĠDiff erence -ĠTsuk uyomi -ĠLes bian -6 70 -Ġfug itive -ĠPlan etary --------------------------------- ------------------------ -Ġacc rued -Ġch icks -Ġsto pp -Ġblock ers -C od -Ġcomment ers -ĠSomew here -ĠPhot ographer -the me -Ġmay oral -w u -Ġanten nas -Ġrev amped -ĠSubject s -it é -im ura -Ġentr ances -liter ally -Ġten ets -ĠO MG -ĠMP H -ĠDon key -ĠOff ense -Ġ" + -Sn ap -ĠAF B -Ġan imate -ĠS od -His panic -Ġinconsist ency -D b -F Y -Ex port -Ġa pe -Ġpear l -ib el -ĠPAC s -Ġ{ \ -Ġact u -ĠHS BC -camp us -Ġpay off -Ġde ities -ĠN ato -ou ple -Ġcens ored -ĠCl ojure -Ġconf ounding -en i -Ġreck on -op he -Ġspot ting -Ġsign ifies -Ġprop el -Ġfest ive -S uggest -Ġpled ging -ĠB erman -Ġrebell ious -Ġovershadow ed -Ġinfiltr ated -j obs -67 2 -Ġscal able -Ġdomin ion -ĠNew foundland -ĠMead ow -Ġpart itions -AM I -Ġsupplement ary -str ument -Ġhair y -Ġperpet uate -Ġnuts hell -ĠPot ato -ĠHob bit -Ġcur ses -Flo at -Ġquiet er -Ġfuel ing -Ġcaps ules -ĠL ust -ĠH aunted -Exec utive -Ġchild birth -G re -Ġrad iant -å İ -Ġm alls -Ġin ept -ĠWarrant y -Ġspect ator -E h -t hens -Ġculmin ating -æ © -ary a -ãĤ ® -ilit arian -ĠOR IG -ĠSp ending -pt ives -ĠS iren -ĠRec ording -ay ne -Ġv im -Ġspr ang -T ang -ĠM FT -mor ning -ĠWe ed -m peg -cess ion -ĠCh ung -7 30 -w arning -56 2 -handed ly -P oor -P olitics -: # -Ġp ian -Ġfec es -ĠDocument ation -Ġban ished -Ġ3 99 -ĠAR C -Ġhe inous -J ake -ĠAm ir -way ne -v re -os henko -Ġnotebook s -Ġfound ational -Ġmarvel ous -ixt ape -Ġwithdraw als -Ġh orde -ĠD habi -is able -ĠK D -Ġcontag ious -ĠD ip -ĠAr rows -Ġpronoun s -Ġmorph ine -ĠB US -68 2 -Ġk osher -fin ished -ĠInstr uments -Ġf used -yd en -ĠSal mon -F ab -aff ected -K EN -C ENT -Dom ain -Ġpoke mon -ĠDr inking -G rowing -ĠInvestig ative -ĠA ether -em i -Ġtabl oid -Ġrep ro -ĠNot withstanding -ĠBers erker -Ġdram as -Ġclich é -Ġb ung -ĠU RI -ĠD os -0 44 -Ġpast ors -Ġl s -Ġac rylic -aun ts -Ed ward -Ġmajor ities -B ang -Ġfield ing -ĠRepl acement -ĠAl chemy -pp ard -ĠRome o -ĠSan ct -ĠLav rov -ib ble -Inst ruct -Ġimp ractical -ĠPlay boy -ce phal -Ġsw aps -Ġk an -ĠThe o -Ġillust rating -Ġdismant led -ĠTrans gender -ĠG uth -UG H -Ġtriumph ant -Ġencomp ass -Ġbook mark -udd in -j er -Ġpred icate -ES H -Ġwhen ce -ĠAB E -Ġnon profits -Se qu -Ġdi abetic -Ġp end -Ġheart felt -sh i -Ġinter acts -ĠTele com -Ġbombard ment -dep ending -ĠLow ry -ĠAd mission -ĠBl ooming -ust ration -ene gger -B rew -Ġmol ten -ĠNer d -P IN -âĸ Ģ -ave ment -Ġtou red -Ġco efficients -ĠTray von -ans son -Ġsand y -t old -fl ows -Ġpop ulous -ĠT inder -ĠBl iss -R achel -Min imum -Ġcontest ant -ĠRed uce -ĠMor se -ĠGrass ley -ĠClick er -Ġexp r -Ġs incerity -Ġmar qu -Ġelic it -ĠPro position -ĠDemon ic -Ġtac os -G reek -Ġpost war -Ġin sofar -ĠP ork -Ġ35 2 -doctor al -walk ing -Ġmid term -ĠSam my -sight ed -ĠTR ANS -ic i -AL D -ĠUS L -ĠF ISA -ĠAm pl -ĠAlex andra -ine lli -Tr ain -Ġsign ify -ĠVers us -Ġob fusc -Ġk h -Ġagg ro -ĠRen ault -Ġ3 48 -5 18 -ox icity -0 22 -ĠTw ist -Ġgoof y -D ynamic -Ġbrief ings -m ight -8 99 -Ġderog atory -T ro -Ġfor ging -ĠKor an -ĠMar ried -ĠBuc s -Ġpal ate -ĠCon version -m able -4 13 -Ġ( _ -Ġs iph -ĠN EO -col lege -Ġmarg inally -Ġfl irt -ĠTra ps -ĠP ace -é »Ĵ -Ġgoalt ender -Ġforb ids -Ġcler ks -ĠT ant -ĠRobb ins -ĠPrint ing -Ġpremie red -Ġmagn ification -ĠT G -ĠR ouse -ĠM ock -odynam ics -Ġpre clude -ism o -ĠPul itzer -Ġaval anche -ĠK odi -rib une -ĠL ena -Elect ric -Ġref inery -Ġend owed -Ġcounsel ors -Ġd olphin -ĠM ith -Ġarm oured -hib ited -Beg in -ĠP W -O il -ĠV or -ĠShar if -ĠFraz ier -est ate -Ġj ams -Pro xy -Ġband its -ĠPresbyter ian -ĠPrem iere -t iny -ĠCru el -Test ing -Ġhom er -ĠV ERS -ĠPro l -ĠDep osit -ĠCoff in -Ġsemin ars -Ġs ql -ĠDef endants -Altern atively -ĠR ats -ç « -ethy st -' > -Ġiss uer -58 9 -Ġch aired -ĠAccess ories -man ent -Ġmar row -ĠPrim ordial -C N -Ġlimit less -ĠCarn age -Ġund rafted -q v -IN ESS -on ew -Ġco hesion -98 7 -Ġne cks -Ġfootball er -ĠG ER -Ġdetect able -ĠSupport ing -ĠCS V -oc ally -k Hz -Ġund e -Ġsh one -Ġbud ding -tra k -Stand ing -ĠStar craft -ĠKem p -Ben ch -Ġthw arted -ĠGround s -ath i -L isa -Dial og -ĠS X -V ision -Ġingen ious -Ù IJ -Ġfost ering -ĠZ a -ĠIn gram -Ġ" @ -N aturally -6 16 -0 35 -ĠF AC -H mm -55 4 -Ġacceler ator -ĠV end -Ġsun screen -Ġtuber culosis -rav iolet -ĠFunction al -ĠEr rors -ed ar -19 66 -ĠSpect re -ĠRec ipes -88 5 -ĠM ankind -L iverpool -Ġ| -- -Ġsubst itutes -ĠX T -w ired -Ġinc o -ĠAf gh -E va -ic c -S ong -K night -Ġdilig ently -ĠBroad cast -A id -Ġaf ar -ĠH MS -aton in -ĠGr ateful -Ġfire place -ĠOm ni -e uro -ĠF RE -ĠSh ib -ĠDig est -t oggle -Ġheads ets -Ġdiff usion -ĠSqu irrel -ĠF N -Ġdark ened -out her -Ġsleep s -ĠX er -gun s -Ġset ups -Ġpars ed -Ġmamm oth -ĠCur ious -g ob -ĠFitz patrick -ĠEm il -im ov -........ ..... -ĠB enny -Second ly -Ġheart y -Ġcons on -st ained -Ġgal actic -cl ave -Ġplummet ed -Ġp ests -Ġsw at -Ġrefer rals -ĠLion el -h oly -Ġunder dog -ĠSl ater -ĠProv ide -ĠAm ar -ress or -å Į -ong a -Ġtim id -Ġp iety -ĠD ek -Ġsur ging -az o -Ġ6 10 -Ġdes ks -ĠSp okane -ĠAn field -Ġwars hips -ĠCob ra -Ġar ming -clus ively -ĠBad ge -ag ascar -ĠPR ESS -ĠMcK enzie -ĠFer dinand -burn ing -Af ee -Ġtyr ann -ĠI w -ĠBo one -100 7 -ĠRe pt -Ċ Âł -Ġcar avan -ĠD ill -ĠBundes liga -Ch uck -Ġheal er -ãĥ¼ãĥ Ĩ -ĠH obby -Ġneg ate -Ġcrit iques -section al -mop olitan -Ġd x -Ġouts ourcing -ĠC ipher -t ap -Sh arp -Ġup beat -Ġhang ar -Ġcru ising -ĠNi agara -Ġ3 42 -ill us -ĠS v -Ġsubt itles -Ġsqu ared -Ġbook store -Ġrevolution aries -ĠCarl ton -ab al -Ut ah -Ġdesp ise -ĠU M -cons ider -aid o -Ġc arts -ĠT urtles -Tr aining -Ġhonor ary - ¢ -Ġtri angles -4 22 -Ġreprint ed -Ġgrace ful -ĠMong olia -Ġdisrupt ions -ĠB oh -Ġ3 49 -Ġdr ains -Ġcons ulate -Ġb ends -Ġm afia -ur on -ĠF ulton -m isc -Ġren al -Ġin action -ck ing -Ġphot ons -Ġbru ised -ĠC odes -og i -Ġn ests -ĠLove ly -ĠLib re -ĠD aryl -Ġ# ## -S ys -. ," -Ġfree zes -est ablishment -and owski -Ġcum bers -ĠSt arg -ĠBom bs -Ġleg ions -Ġhand writing -Ġgr un -ĠC ah -sequ ent -Ġm oth -ĠMS M -Ins ert -F if -Ġmot el -Ġdex ter -ĠB ild -hearted ly -Ġpro pe -ĠText ure -ĠJ unction -ynt hesis -oc ard -ĠVer a -ĠBar th -Ġμ g -Ġl ashed -Ġ35 1 -ĠZ amb -ĠSt aples -ĠCort ex -ĠCork er -Ġcontinu um -ĠWR ITE -unt a -rid or -Ġde ems -0 33 -ĠG OLD -p as -Ġrep ressive -ãĥĨ ãĤ£ -Ġbaff led -Sc ar -Ġc rave -Ġ ______ -Ġentrepreneurs hip -ĠDirector ate -Ġ' [ -Ġv ines -Ġasc ended -ĠGR OUP -ĠGood bye -Ġdo gged -ãĥ´ ãĤ¡ -Man ufact -Ġunimagin able -ri ots -ier rez -Ġrel ativity -ĠCraft ing -ra ught -ud en -c ookie -Ġassass ins -Ġdissatisf ied -ac ci -Ġcondu it -Sp read -ĠR ican -n ice -izz le -Ġsc ares -ĠWH Y -ph ans -5 35 -Ġprot racted -ĠKrist en -5 36 -ĠSc rib -ĠNe h -Ġtwent ies -Ġpredic ament -Ġhandc uffs -Ġfruit ful -ĠU L -ĠLud wig -Ġatt est -ĠBre aker -Ġbi ologically -ĠDeal er -Ġrenov ations -f w -ess en -Al ice -ĠHen ri -Ġun ilaterally -ĠS idd -h ai -ĠSt retch -S ales -Ġcumbers ome -ĠJ avier -Ġtrend y -Ġrot ting -ĠChall enges -Ġscra ps -Ġfac ets -ĠVer onica -ĠVer ge -ĠS ana -Al ien -ĠR ih -Ġrad ial -ect ar -Ġ6 30 -cl i -Mar ie -Ġwild fire -ĠCat o -h ander -Ġwait ress -Ġch ops -ĠS ECTION -Ġblunt ly -ĠCat alog -n ian -stud y -Ġpat rolling -ĠT enth -nex us -ĠN ON -op sy -Ġsc athing -s ie -Ġdeterior ated -V B -Naz is -Ġdep ictions -Ġauthent icated -ĠCon ce -k rit -Ġpromul g -ĠL ONG -U FC -ĠVis itors -ĠRec all -Ġrehab ilit -ĠSL I -Ġglac ier -ĠB ite -Ġ50 3 -Ġvom it -Ġfer mented -ĠKh alid -Ġgrad ed -ĠMag icka -ĠIch igo -power ful -ic ators -75 3 -Ġsh rew -Ġ35 6 -Ġlegal izing -Ġall otted -ĠArch demon -ith ing -igg urat -V OL -Le od -Ġo ily -Ġindu cing -Ġamy gdala -Ġadm ins -ĠAcqu isition -C AN -Ġsche matic -Ġmo an -ĠCamer oon -Ġt ink -Ġmer ry -Ġbutter flies -ĠGo ff -Ġworks pace -ĠCor ona -Ġj avascript -ĠD olphin -ĠCant or -4 64 -to e -AP S -ĠAg ing -Ġpadd ed -ĠZ heng -ĠHe ld -Ġest ranged -Ġ7 70 -. } -ĠDun ham -Ġsm okes -Ġcap itals -und ai -Sh in -ĠFound ing -Ġent itle -Ġcenter piece -D iscover -Ġthere to -al ert -ĠN ou -ĠAnaly st -l c -F H -FI ELD -ĠP OV -gr ay -Ġar cs -ĠH OT -Ġr s -Ġoblig atory -ĠArchitect s -ĠS ven -ĠF EC -0 200 -Christ mas -ĠAlban ia -rat om -58 7 -Ġhard ships -Ġaut os -ĠCharg es -Ġap es -Ġ3 76 -wal let -Ġintox ication -Ġgobl in -Ġ5 70 -++++++++ ++++++++ -ĠYel p -ĠMag netic -ĠBr iggs -R ail -Ġspawn s -ĠW iggins -Ġshowc ased -Ġres orted -ub en -Ġwh ipping -Ġim itate -Ġdigest ion -ĠUS PS -ĠG est -Ġye a -ĠT ight -ind al -ic as -` . -C AST -'' ; -ĠF et -opath ic -In valid -Ġregrett ed -Ġbro ccoli -ĠSc ores -e ve -Ġpost ings -Ġaccum ulating -Ġneed less -elf th -Ġmay ors -Ġsc rib -Ġanecd otes -Ġbot ched -ĠRib bon -ĠConstant ine -i uses -ess es -Ġdev ise -Comp ared -Ġp udding -Ġg arg -Ġev oke -79 7 -Ġdet ox -9 09 -ĠPie ces -ĠMcC artney -Ġmet ast -ĠK rypt -P OR -Ġt ending -ĠMerch ants -Pro of -ĠV arg -ĠPort able -ãĥ¼ãĥĨ ãĤ£ -B rain -25 00 -Ġfol iage -Ø ¹ -Ġment ors -ĠA ires -Ġminimal ist -Ġing ested -ĠTro jan -ĠQ ian -inv olved -0 27 -Ġer oded -RA FT -Ġbl urry -M ob -Ġbuff et -ĠFn atic -ae a -KN OWN -ĠIn it -s afety -en um -ACT ION -ĠCrus her -ĠD ates -Ġ ................ -c alling -ak ov -Ġvent ured -Ġ5 55 -au ga -H art -ĠA ero -M AC -Ġthin ly -Ġar ra -ST ATE -ild e -ĠJac qu -ĠFem ales -Ġthe orem -Ġ3 46 -Ġsmart est -ĠPU BLIC -ĠK ron -ĠB its -ĠV essel -ĠTele phone -Ġdec ap -Ġadj unct -ĠS EN -mer ga -Ġred acted -Ġpre historic -Ġexplan atory -ĠRun s -ĠUtt ar -ĠM anny -ĠAUTH OR -ĠUnle ashed -ĠBow ling -be ans -79 3 -Ġunivers es -Ġsens it -ĠK ung -re peat -ctr l -Ġp aced -Ġfull er -Cl ock -Ġrec omb -ĠF aul -ĠB unker -Ġpool ed -Ġan a -ĠM outh -LL OW -hum ane -Ġbull do -ĠMicha els -f am -Ġwreck ed -Ġport rays -ĠWh ale -ĠH es -Ġguess es -ĠBrow se -ĠL APD -Ġconsequ ential -ĠInn ocent -ĠD RAG -Ġtrans gress -ĠO aks -Ġtri via -ĠRes on -ĠA DS --- + -ĠT oll -Ġgrasp ing -ĠTHE M -ĠT ags -ĠCon clusion -Ġpract icable -Ġho op -Ġunintention ally -Ġign ite -ĠM ov -ur ized -le hem -Ter min -Ġcolour ful -ĠLin ear -ĠEll ie -G y -Ġman power -Ġj s -Ġem oji -ĠSHAR ES -_ . -0000 7 -Ġsophistic ation -Ġunders core -Ġpract ise -Ġbl ob -op ens -Uk raine -Ke eping -Y C -J R -ult imate -Cl aim -Ġautom obiles -99 3 -ste el -Ġpart ing -ĠL ank -... ? -Ġ38 5 -Ġremem brance -Ġe ased -Ġcov ari -ĠS ind -Effect ive -Ġdisse mination -ĠMo ose -ĠCl apper -br ates -App ly -Ġinv is -Ġwors ened -âĢĶ - -Ġlegisl ator -ĠL ol -ĠRow e -Ġdealers hip -um ar -id ences -Ġinvestig ates -Ġc ascade -Ġbid der -ĠB EN -Iron ically -Ġpres iding -Ġd ing -Ġcontrad icted -Ġshut s -ĠF IX -Ġ3 66 -Dist rict -Ġsin ful -ĠChar isma -o ops -Ġtot ality -Ġrest itution -ĠOpt imus -ĠD ah -Ġcl ueless -urn ed -Ġnut rit -Ġland owners -Ġfl ushed -Ġbroad en -m ie -Ġprint ln -Ġn ig -ĠCorp us -J en -Ġprot o -ĠWik imedia -ĠPal o -C OR -Ġstory lines -Ġevangel icals -ĠDar rell -Ġrot or -ĠH W -sk illed -ery l -Ġbe gg -ĠBl umenthal -Ġwe aving -Ġdown wards -ĠJack et -ĠANG EL -Te chnology -Ġes oteric -alde hyde -Ġfur iously -Ġforeign er -We ak -CH O -ĠH ound -Exper ience -ĠPlay station -ĠM IA -ĠU ng -cl oth -ag all -Ġcal ming -iz ens -St ruct -ĠW itches -ĠCeleb ration -Ġ........ ...... -pt roller -ĠTC U -Ġb unny -ãĥ į -ut orial -Ġup scale -ĠSt a -ĠCol ossus -Ġchlor ide -ĠZ ac -ĠRe asons -ĠBrook ings -ĠWH ITE -][ / -ĠL ose -9 05 -Ġunders ide -ern els -Ġv ape -do zen -upp et -ĠST OP -mat ical -ĠStat ements -hed dar -P AC -Custom er -Ġmem os -ĠP J -end ars -ĠLim its -l augh -Ġstabil ized -ĠALE C -Y A -Up grade -al am -Ġtechn o -Ġan ew -fore seen -Ġcolleg iate -ĠPy ro -ĠD ism -Ġfront line -Ġammon ia -I U -Qu ite -John ny -ass in -G OP -ĠSt yles -ĠSovere ign -acter ial -5 49 -ĠR IP -ĠL ists -Ġ3 64 -ĠRece p -s ocket -ĠByr d -ĠCand le -An cient -Ġappell ant -en forcement -ace a -ans ki -Ġold s -88 6 -Ġsl urs -Ġem pires -Ġbuck le -Ġalien ation -ĠAber deen -Ġunic orn -Ġoverr iding -ĠL X -pp a -Ġdesp ised -ĠB ugs -ĠB ST -S outhern -5 33 -Ġhall mark -ĠPost er -Ġstem med -Ġprincip als -ĠT ECH -ĠSand wich -It aly -Ġche esy -ĠSet TextColor -ĠProt ective -ĠC ohn -J O -apt op -Re ason -Lead er -ĠUnder stand -ĠFr idays -ĠContin uous -Ġcl ipping -ĠR ye -Ġber th -tim er -ann is -re act -Ġbuff alo -ĠPar as -Ġ6 55 -Ġpres ided -ĠSun rise -Ġve ts -Ġcl oves -ĠMcC ull -Stre ngth -G AN -Ġill iter -ĠPric ing -l é -Ġresist or -Ġbr un -ĠSuff olk -Ñ ĭ -ĠL iver -Re leased -Ġwhat s -8 60 -ĠMe asures -Ġden ouncing -ĠRy zen -Ġsou ven -Ġcareg ivers -ch ini -ĠScar lett -Ġt rough -Cong ratulations -Ġtax is -ĠTrad ition -j it -Ġtable top -Ġhither to -Ġdis information -off ensive -h ra -ĠDISTR ICT -Ġcompl icate -chen ko -ĠRecon struction -Ġpalp able -Ġa usp -Ġ4 28 -Ġshowc ases -ĠPublic ation -know ledge -inn on -4 19 -Ġretri eval -and ers -Ġref ute -Ġinqu ired -g ur -Ġneg ativity -Ġcons erve -Ġafter life -Ġpres upp -ĠGill espie -Ġm t -ĠD N -T ap -Ġper pend -ĠS my -does n -Ġsp illing -Ġhyp ers -K ate -® , -ke pt -ĠP owered -Ġj a -ĠK lux -ard e -ab an -Ġ4 44 -Ġflatt ened -ĠImprove ments -urg a -ĠK und -Ġins cribed -Ġfac ult -Ġunpre pared -ĠCons umers -Ġsatisf ies -Ġpul monary -Ġinf iltration -Ġex ternally -Ġcongrat ulations -ag han -Ġair liner -Ġfl ung -Ġfly ers -G D -Ġsnipp ets -Ġrec ursive -Ġmaster ing -L ex -Ġovert ly -v g -Ġluck ily -Ġenc ro -ĠLanc et -ĠAbyss al -function al -Ġs ow -Ġsqu id -Ġnar ration -Ġn aughty -ĠHon our -ĠSpart ans -Ġsh atter -ĠTac oma -ĠCal ories -ĠR aces -Sub mit -Ġpurpose fully -w av -ĠY ok -F est -ĠG err -Met ro -Ġit iner -f amous -Ġ" { -in line -was her -Iss ue -ĠCL IENT -oz o -Vers ions -7 25 -ĠGl ock -Ġshield ed -ĠPC R -ENC Y -ĠWe ld -ĠSim pl -Ġredirect ed -ĠK ham -Ġ( > -Ġlab ou -Ġdi apers -ss l -Ġcell ar -organ isms -ore sc -ĠBer ks -did n -Sh ipping -C hest -Ġund one -Ġmillion aire -Ġc ords -ĠYoung er -appropri ately -Ġsequ els -u ve -ant icipated -Ġle wd -ĠSh irt -ĠDmit ry -V eter -Ġsl aying -ĠY ar -Ġcompl ication -I owa -ĠEric a -ĠBL M -g irlfriend -b odied -6 26 -19 63 -Ġintermedi ary -Ġcons olation -M ask -ĠSi em -ow an -Beg inning -Ġfix me -Ġculmin ated -Ġcon duc -ĠVolunte er -Ġpos itional -Ġgre ets -ĠDefin itions -Ġthink er -Ġingen uity -Ġfresh men -ĠMom ents -Ġ35 7 -ate urs -ĠFed Ex -s g -69 4 -Ġdwind ling -ĠBO X -sel age -Ġt mp -Ġst en -ĠS ut -Ġneighbourhood s -Ġclass mate -f ledged -Ġleft ists -Ġclim ates -ATH ER -ĠScy the -ul iffe -Ġs ag -Ġho pped -ĠF t -ĠE ck -ĠC K -ĠDo omsday -k ids -Ġgas ped -Ġmon iker -ĠL od -ĠC FL -t ions -r ums -fol ios -Ġm d -Ġunc anny -Ġtrans ports -ĠLab rador -Ġrail ways -Ġappl iance -ĠCTR L -æ Ģ -Pop ulation -ĠConfeder acy -Ġunb earable -Ġdors al -ĠIn form -op ted -ĠK ILL -Mar x -Ġhypoc ritical -q us -ĠN umerous -ĠGeorg ian -ĠAmbro se -ĠL och -Ġgu bernatorial -ĠX eon -ĠSupp orts -ens er -ee ly -ĠAven ger -19 65 -Ar my -Ġju xtap -Ġcho pping -ĠSpl ash -ĠS ustainable -ĠFin ch -Ġ18 61 -ict ive -at meal -ĠG ohan -Ġlights aber -ĠG PA -ug u -ĠRE PL -vari able -Ġher pes -Ġdesert s -ac iously -Ġsitu ational -week ly -ob l -Ġtext ile -ĠCorn wall -Ġcontrace ptives -ĠA ke -] - -ä¹ ĭ -: , -ĠW em -ĠB ihar -Ġ' . -Ġbe re -Ġanal ogue -ĠCook ies -Ġtake off -Whe el -Ġmaj estic -Ġcomm uting -0 23 -ĠCor pse -ass ment -min i -Ġgor illa -ĠAl as -ere e -Ġacquaint ances -ĠAd vantage -Ġspirit ually -Ġey ed -pm wiki -ĠE nder -Ġtrans lucent -Ġnight time -ĠIM AGES -5 45 -ĠK amp -ĠFre ak -Ġ ig -Port land -4 32 -ĠM ata -Ġmar ines -Ġh ors -ater asu -ĠAtt ribution -Ġ-------- - -Ġk ins -ĠBEL OW -++ + -Ġre eling -ol ed -Ġcl utter -ĠRel ative -Ġ4 27 -B US -Ġa vert -ĠChe ong -ĠA ble -ĠPry or -Develop er -Ġen cyclopedia -ĠUSA F -ĠG arry -Sp ain -Bl ocks -Ġexp osition -ĠGamer Gate -W OR -Ġstockp ile -Ġclot hed -ĠT one -ĠR ue -t umblr -Ġtreacher ous -Ġf rying -Ñ Į -ĠS ph -Ġrest raints -Ġemb odies -ĠG es -S afety -Ġnegoti ators -min ing -ĠAppalach ian -L OS -ĠJenn a -Ġpass ers -ç ĭ -sn ap -Ġshort en -creat or -Ġinn umerable -uther land -67 4 -ĠW OM -ĠAs cend -ĠArm ory -ĠTrans action -K ick -Ġsuit case -day Name -Ġwaste ful -mar riage -ĠMcC abe -ite ch -ĠO ss -Cl osure -ĠTreasure r -Ġindec ent -ĠD ull -Ġresid ences -19 59 -ĠS ettlement -Ham ilton -Ġself ies -ĠRank ing -ĠBark ley -ĠB ore -ĠW CS -ĠMar itime -ĠH uh -ĠForest ry -Ġcultiv ating -ĠBall ard -Ġg arrison -ĠSD L -9 30 -Ġnas cent -Ġirresist ible -Ġaw fully -\/ \/ -Ġequ ate -Ġanthrop ology -ĠSylv ia -Ġintest ine -Ġinnoc uous -cess ive -ag ra -ĠMet roid -G rant -8 55 -ģ ĸ -Ġ" _ -ãĥĥ ãĥī -Ġappra isal -ĠFred dy -04 6 -Ġ40 6 -Ġ18 30 -Ġd ocking -St atic -Ġp ont -ĠVolt age -ĠSt ead -ĠMort gage -ĠJon ah -Y L -CLASS IFIED -Ġas bestos -nik ov -Ġcoll agen -ĠOrb ital -P ocket -7 99 -Ġhy brids -inc hes -Ġinv oice -und y -Ġinequ alities -T rend -w ashed -B ALL -Ġluc id -ĠComment ary -Ġw itty -Br andon -Ġbru ising -Ġ6 20 -es cent -box ing -P OL -Ġ3 78 -R ect -Ġlic ences -ĠMcG ee -p ressed -D anny -Ġj ammed -ord inate -Ġle th -Ġdistingu ishes -ĠYam aha -IL S -ĠH ume -ĠC ategories -Rober ts -Ch art -Ġbeet le -ĠGra veyard -Ġ($ ) -o ÄŁ -Ġtw ilight -are lla -á ½ -Ġbooth s -ĠH HS -ĠFeld man -Ġexcav ation -Ġphilosoph ies -at ography -ĠGar age -te chnology -Ġunfor gettable -Ġver ifying -Ġsubord inates -E ls -Ġne b -G aming -EN A -ĠAchieve ment -it ters -ĠG abe -Ġd umps -for cer -Ġpo ignant -ĠM BA -ĠHe idi -ime i -Ġm ages -Ġliber ate -Ġcircum cised -ĠMer maid -ĠMat th -t ogether -ĠW ichita -Ġstore front -ĠAd in -V II -Four th -Ġexplore rs -W ER -Not able -Bro ok -m ens -F aith --------- - -ĠJ ou -¬ ¼ -Ġpine apple -Ġam alg -el n -ark able -ĠãĤµ ãĥ¼ãĥĨãĤ£ -ĠãĤµãĥ¼ãĥĨãĤ£ ãĥ¯ãĥ³ -Ġov arian -ĠE choes -Ġhairc ut -Ġp av -Ġch illed -anas ia -Ġsty led -Ġd ab -ni per -Ġminister ial -ĠD UP -T an -Ġsul ph -ĠD eter -ĠBo hem -od an -Ġeduc ator -â ĵĺ -sp ir -Ch icken -ĠE leanor -Ġqu i -Ġheav iest -Ġgrasp ed -U RA -Ġcro oked -Jess ica -pro blem -Ġpred etermined -Ġman iac -Ġbreath s -ĠLauder dale -Ġh obbies -y z -Cr ime -Ġcharism a -d L -Ġle aping -Ġk ittens -Ang elo -ĠJ ACK -ĠSu zanne -Ġhal ting -ENT ION -Ġswall owing -ĠEarthqu ake -Ġeight eenth -ĠN IC -ĠIN F -ĠCons cious -Ġparticular s -circ le -7 40 -Ġbene volent -Ġ7 47 -Ġ4 90 -Ġr undown -ĠVal erie -ĠB UR -Ġcivil isation -ĠS chn -W B -ot ide -intern ational -Ġj ohn -Ġ19 02 -Ġpe anuts -Ġflav ored -k us -Ġro ared -Ġcut off -é £ -Ġorn ament -Ġarchitect ures -Ġ3 69 -ol or -ĠWild e -ĠC RC -ĠAdjust ed -Ġprov oking -land ish -Ġrational ity -Ġjust ifies -Ġdisp el -Ġa meric -ĠPol es -Ø © -Ġen vis -ĠD oodle -ä½ ¿ -igs aw -auld ron -Techn ical -T een -up hem -ĠX iang -Ġdetract ors -ĠZ i -ĠJournal ists -Ġconduc ive -ĠVolunte ers -Ġs d -Know ing -Ġtrans missions -ĠPL AN -ĠL IB -Ġall uded -Ġob e -Ġd ope -ĠGold stein -Ġwavelength s -ĠDest ination -nd a -ug i -Ġattent ive -ĠLe an -ral tar -Ġman g -mb uds -ak ings -b ender -Ġacc ol -Ġcraw led -N OW -Min nesota -Ġflour ished -ĠZ up -ĠSuper visor -ĠOliv ier -Ex cellent -Ġwid en -D one -Ġw ig -Ġmiscon ceptions -Cor p -W an -Ġvener able -ĠNot ably -ĠKling on -an imate -Bo ost -ĠS AY -miss ing -ibli ography -mel on -Ġpay day -Ø ³ -bo le -Ġve iled -ĠAl phabet -It alian -Ġever lasting -ĠR IS -ĠC ree -rom pt -Ġh ating -Ġgrin ning -Ġge ographically -OS H -Ġwe eping -ĠÂłĠÂłĠÂłĠÂł ĠÂłĠÂłĠÂłĠÂł -Ġimpe cc -Let ter -Ġblo ated -PL A -ĠFe in -Ġper sever -Th under -Ġa ur -ĠR L -Ġpit falls -âĸ º -Ġpredomin ant -Ġ5 25 -7 18 -AP E -7 14 -Ġfarm land -ĠQ iao -Ġv iolet -ĠBah amas -Ġinflic ting -ĠE fficiency -Ġhome brew -Ġundert ook -Ġcur ly -ĠHard ing -man ia -59 6 -Ġtem pered -Ġhar rowing -ĠP ledge -ĠFranken stein -è ª -M otion -Ġpredict ably -ĠExpl osion -oc using -er d -col o -FF ER -Ġback field -ĠV IDE -ue bl -N arr -ĠArg ument -Ġgen omic -Ġbout ique -Ġbatt ed -ĠB inary -Ġg amb -ĠRh ythm -67 3 -Ġa float -ĠOlymp ia -Y ING -Ġend if -is in -Ġwin ters -Ġsc attering -I v -D istance -Ġtr u -ĠCom fort -Ġne xus -Ġair flow -ĠByz antine -p ayers -con i -ĠB etsy -D eal -ĠN ug -ĠContin ent -red ibly -Ġoptim izing -al beit -Ġec static -ĠPro to -ç · -iv ot -âĸ Ħ -em p -rou nder -Ġcl out -ĠI ST -66 3 -ĠDoll ars -ĠD AC -Ġsubsc ribed -Ġrehears al -Ġam ps -ĠSh ang -es m -Ġspr inkle -Ġassail ant -ĠO o -ĠCoin base -T act -Ġret ina -Ġn uns -R ON -att o -Ġj ug -ĠSV G -Ġb ikini -ĠFI LE -ĠFound ers -ep ort -ĠK P -Ġrest ores -ĠTh ick -Ġash ore -Ġappro vals -R ender -M AG -G raham -ĠCort ana -ãĥ³ ãĤ¸ -ss h -or ians -ars ity -ĠInsp ired -u pper -Ġsign alling -Ġreb uke -Ġfl ares -Ġdownt ime -Stud ies -Ġstagn ation -ĠSequ ence -Ġgr unt -Ġass ures -ĠPL A -59 2 -Ġintra ven -d epend -Sus an -ĠManz iel -Man ia -Cont ract -Ġsl ams -Ġcult ured -Ġcred itor -L IST -ĠH UM -ĠChatt anooga -serv ed -Ġclo aked -ĠF TP -p owder -ĠSt ella -uct ive -Ġcheap ly -ĠMU CH -ĠGalile o -Ġsu ites -spe ech -Ġdeliber ations -ĠCh ips -« ĺ -Bal ance -ĠWyn ne -ĠAk ron -Ass et -Ġhon oured -Ġed ged -Like wise -anim ous -ĠW age -ĠEz ek -ad vertisement -ĠRT X -ĠM AD -Ġmigr ating -ĠS QU -Ġ4 75 -Ed ited -Ġshorth and -ĠBas ics -Ġcro tch -ĠEV EN -Ġv m -effic iency -Ġcal ves -ĠF rie -ĠBrill iant -Ġstri kers -Ġrepent ance -Ġarter ies -r l -B ed -h ap -Ġcrypt ography -ĠSab res -Ġ4 14 -vi ks -ih ara -aps es -T alking -Ġintertw ined -Ġdoc ks -Ġalle le -ĠArt ifact -ĠH IM -t orn -ç ķ -Ġop acity -ĠE ly -os uke -Ġn ipple -Ġhand written -ĠV K -ĠChamber lain -ĠLa os -ig raph -g row -Ġtr illions -Ġdescend ant -ĠSail or -as uring -Ġce ilings -ĠWare house -f lying -ĠGl ow -Ġn ont -Ġmiscar riage -Ġrig s -Ġmin istries -Ġelabor ated -Ġdel usional -ĠHum ane -Ġ3 79 -n ets -Ġblack out -add ers -Ġn p -ĠT ire -ro sc -Ġsub div -Ġlink age -Ġchron ological -ĠHER O -Ġres ettlement -ĠVin yl -Ġpast oral -ĠMob il -ĠBar bar -Co oldown -ĠF ritz -c riminal -re pe -Ġbell ig -ĠBre ed -Ġ4 18 -Ġsem blance -ij k -Ġcur tail -Ġclin ch -cont ained -ĠProm pt -ast on -Ġw i -Ġpursu its -5 15 -ĠGl oss -Ġfl ips -Ġcoup ons -Ġcl oning -ĠLike ly -Rem oved -ĠQu artz -r ices -ĠSpe ars -Ġp ious -Ġdep reciation -ĠD are -oun ces -am az -O nt -Ġp innacle -d ocker -0 26 -ĠW yr -ĠPro per -Ë Ī -n il -By tes -Ġseek er -t rial -Ġunf olds -ĠMar se -Ġextravag ant -ĠSurviv ors -RED ACTED -ĠSpeed way -ĠCra igslist -sub mit -ĠGener ations -Ġup holding -Ġblood stream -ĠMiss ions -ĠL awn -Ġlim bo -ene i -H uh -ĠWild cats -pre p -ĠMark us -ĠFor bidden -rit ic -IN O -Ġexhib iting -requ ent -ch uk -Ġhabit ual -ĠComp atibility -Dr ag -RIP T -uj ah -GR OUND -Ġdelinqu ent -Ġburn er -Ġcontempor aries -Ġgimm ick -load s -Ġno zzle -p odcast -ĠW ak -ĠStat en -ĠK uh -ãģ ĵ -inter rupted -Ġinv incible -ĠBurn ett -cig arette -ĠPeb ble -ĠTem porary -ĠMar ino -58 2 -Ġwast eland -ident ly -T x -Ġr ite -ĠPan asonic -ĠM iddles -ĠHort on -ae us -Ġc uring -Ġm ats -Ġadj ourn -Ġfears ome -pe z -bo ats -Ġpro pell -Ġconflic ted -ĠAng er -Ġinsurg ent -K arl -Ġco ales -Ġsouth western -Ġdis su -ĠO vert -******** **** -Ġbox ed -ĠBr une -aa a -Ġgard ening -ĠEng el -tr acks -Ġpur ified -Ġplace holder -ĠL ikes -Ġd an -G ab -Ġe ct -ĠF aw -ĠEl iot -Ġ' , -otrop ic -ĠRu in -hed on -Ġca ul -Ġa ft -ĠCad illac -gh a -ass ian -ud eb -ĠT ick -Ġadjust s -AR GET -5 37 -isc he -ant y -ĠFried rich -ĠBl izz -ĠA OL -Camp aign -Ġmamm al -ĠVe il -ĠK ev -ĠMaur it -ĠDam ien -N ation -E astern -Ġ{ : -Ġ= ================================ -Ġstereotyp ical -Ġatt ic -ĠCy borg -requ ire -Ġaward ing -ĠPap ua -bt n -b ent -B oo -Ġ( = -ĠX ander -ĠSomers et -Ġcatch y -Ġcert ify -STR UCT -Ġit al -Ġt ides -ĠBr ands -G ray -comp etitive -Ġcur ator -ĠD G -omin ium -ĠGM Os -ci ating -ĠCarm en -ow ard -Balt imore -Ġr gb -C u -Ġwip es -spe ll -IT NESS -Ġsummar izes -ĠRe vis -Ġwhistlebl owers -ĠBre ach -Ġcro chet -k os -ews ki -Ġrep et -Ġcrim son -ĠKar achi -read able -dim ension -ĠI gor -ild ed -ĠZ ed -ĠKe ane -ĠCos metic -DE P -Ġretreat ing -ĠU A -ens ical -Ġd usk -ĠDick ens -Ġaren as -ĠPass age -level s -Ġcur v -P ope -Ġch ores -ĠEl ise -ĠComp ass -b ub -Ġmamm alian -ĠSans krit -ĠAN C -ĠCr ack -Q ual -L aun -amp unk -Ġlearn ers -Ġglam orous -Ġfur the -erm ott -c and -Gener ic -Ġnarr ated -Ġdisorder ly -ĠTrans actions -ĠDet ention -ĠR oku -Ä į -Ġunder statement -ĠS aur -ĠRodrig o -ĠAS AP -S in -Ġre joice -Method s -Ġelectro de -Ġworsh ipped -Ġid i -ĠPhys icians -Ġpop up -Ġde ft -ĠRem oval -ĠBu enos -ver bs -Ġfun k -ush a -rict ion -ore a -ĠBang alore -ĠKen obi -zz i -Ġnorm ative -Ġgobl ins -Ġcaf es -ĠUN CLASSIFIED -ĠF ired -S IGN -Ġs clerosis -ĠV oter -ĠSon ny -ĠExt end -ĠEV s -Ar senal -Ġp si -Ġwid est -ĠT us -Ġlo oms -Ġjust ifying -ĠGr anger -è ¯ -Ref er -58 3 -Ġflour ishing -ab re -Ġr ave -ĠCont ra -Ġ18 98 -Add s -Ġf ul -ĠCo oke -some one -= # -67 1 -Ġy ak -Ġar te -ĠMis cellaneous -ĠDet ection -ĠCl ancy -â ģ -ass ies -Ġval iant -ĠFemin ist -cor ruption -V el -P ear -Ġsucc inct -Ġquick est -k w -Ġsp itting -ĠL ibraries -åħ ī -ant z -D ad -ĠSpec ifications -rup ulous -and r -RES ULTS -Ġsnow ball -Ġpred is -ĠB axter -ĠNurs ing -ĠCh aff -s we -Ġout age -Ġnest ing -Ġnotor iety -tr igger -on ite -j on -Ġf ou -ook ed -ĠCelebr ity -re ality -Ġfat ig -Ġhug ging -Ġbother s -ĠPan zer -ĠCh andra -fig ured -Ġvol ts -ĠCloud s -Ġfee ble -ĠCur ve -ĠAs us -78 6 -abs or -ĠV ICE -ĠH ess -Ġmanufact ures -Ġgri zz -ĠPower ful -ac id -Ġsub sections -ĠKrug man -ĠAl ps -is u -Ġsequ est -ĠUlt ron -ĠT inker -ĠGo ose -Ġmism atch -Att orney -Ġmorph ology -ĠSix ers -ut tered -ĠE LECT -gr an -Rus sell -ĠG SL -Ġfort night -Ġ. ) -Ġapost le -pr one -el ist -Unt itled -ĠIm plementation -ist ors -Ġtank er -Ġpl ush -Ġattend ants -ĠT ik -ĠGreen wich -ĠY on -ĠSP L -cell s -unt led -S olution -ĠQu é -Ġvac ated -Ġupt ick -ĠMer idian -æ ĥ -ĠDr ill -9 25 -58 4 -Ġrenov ated -ĠKub rick -zy k -Ġl ousy -pp el -ohyd rate -ĠI zzy -lesi astical -CC C -ĠAj ax -Ġad apters -ĠPetra eus -Ġaffirm ation -ĠST OR -le ms -ad oes -ĠConstantin ople -Ġp onies -Ġl ighthouse -Ġadherent s -ĠBre es -omorph ic -Fight ing -Ġpl aster -ĠP VC -ĠOb st -Ġdear ly -ĠTo oth -icks on -Ġsh aming -P lex -A gg -ĠâĢ¦ " -Ġsub reddits -Ġpige on -ĠResident ial -ĠPass ing -Ġl um -ĠP ension -Ġpessim istic -Ġ4 32 -z inski -c ade -0 75 -Ġapolog ised -iy ah -Put ting -Ġgloom y -ĠLy me -=-=-=-=- =-=-=-=- -ĠT ome -ĠPsych iatric -ĠH IT -c ms -ap olog -Ġbreak er -Ġdeep en -Ġtheor ist -ĠHigh lands -Ġb aker -Ġst aples -Ġinterf ered -ĠAb ortion -jo ined -ch u -Ġform ulate -Ġvacc inations -Ġban ter -phe us -Ġoutfield er -ĠM eter -Ġ# #### -Ġ18 95 -Ġnarrow ing -ĠST ORY -f p -ĠC ST -ign ore -Ġproclaim ing -ĠR U -ĠB ALL -yn a -65 3 -Ġpos it -P RE -59 4 -ĠRegist rar -ĠPil grim -ic io -Ġpre tt -Ġlif eless -Ġ__ _ -Ne igh -ĠCh urches -orn o -Ġor cs -Ġkind red -ĠAud it -Ġmillenn ial -ĠPers ia -g ravity -ĠDis ability -ĠD ARK -W s -od on -Ġgrand daughter -ĠBro oke -ĠA DA -ER A -Ġpick ups -ĠWil kinson -ĠSh ards -ĠN K -Ġexp el -ĠKis lyak -Ġj argon -Ġpolar ized -ian e -Pub lisher -Ġreb utt -Ġapprehens ion -ĠK essler -Ġpr ism -F UL -19 64 -ĠL oll -ä ¿ -le thal -Å Ł -Ġg hetto -Ġb oulder -ĠSlow ly -ĠOsc ars -ĠInst ruction -ĠUl tr -ĠM oe -N ich -ĠP ATH -( * -ĠRE LEASE -un ing -rou se -en eg -Ġre imb -ĠDet ected -Do S -Ġster ling -Ġaggreg ation -ĠLone ly -ĠAtt end -hig her -Ġairst rike -ks on -SE LECT -Ġdef lation -ĠHer rera -C ole -rit ch -Ġadvis able -F ax -Ġwork around -Ġp id -mort em -ers en -Ġtyp o -Ġal um -78 2 -ĠJam al -script s -Ġcapt ives -ĠPres ence -ĠLie berman -angel o -Ġalcohol ism -ass i -Ġrec ite -Ġgap ing -Ġbask ets -ĠG ou -Brow ser -ne au -Ġcorrect ive -und a -sc oring -ĠX D -Ġfil ament -Ġdeep ening -ĠStain less -Int eger -Ġbu ggy -Ġten ancy -ĠMub arak -Ġt uple -ĠD roid -ĠS itting -Ġforfe it -ĠRasm ussen -ixt ies -es i -ĠKim mel -Ġmetic ulously -Ġap opt -ĠS eller -08 8 -ec ake -hem atically -T N -Ġmind less -Ġdig s -ĠAcc ord -ons ense -em ing -br ace -Ġe Book -ĠDist ribut -ĠInvest ments -w t -] ), -beh avior -56 3 -Ġbl inding -ĠPro testers -top ia -Ġreb orn -ĠKel vin -ĠDo ver -ĠD airy -ĠOut s -Ġ[ / -Ï Ģ -b p -ĠVan ity -ĠRec ap -ĠHOU SE -ĠF ACE -Ġ4 22 -69 2 -ĠAnt ioch -cook ed -Ġcoll ide -Ġa pr -Ġsle eper -ĠJar vis -Ġalternative ly -ĠLe aves -ĠM aw -Ġantiqu ity -ĠAdin ida -Ġab user -Poké mon -Ġass orted -ĠRev ision -ĠP iano -ĠG ideon -O cean -Ġsal on -Ġbust ling -ogn itive -ĠRah man -Ġwa iter -Ġpres ets -ĠO sh -ĠG HC -oper ator -Ġrept iles -Ġ4 13 -ĠG arr -ĠCh ak -Ġhas hes -Ġfail ings -Ġfolk lore -Ġab l -ĠC ena -ĠMac Arthur -ĠCOUR T -Ġperipher y -app ers -Ġreck oned -ĠInf lu -ĠC ET -Ġ3 72 -ĠDefin itive -ass ault -4 21 -Ġreservoir s -Ġd ives -ĠCo il -DA Q -Ġvivid ly -ĠR J -ĠBel lev -Ġec lectic -ĠShow down -ĠK M -ip ed -reet ings -ĠAs uka -L iberal -ĠÏ Ħ -Ġbystand ers -ĠGood win -uk ong -S it -ĠT rem -Ġcrim inally -ĠCirc us -ch rome -88 7 -Ġnan op -ĠOb i -ĠL OW -o gh -ĠAuth ors -ob yl -Ur ban -Ġt i -ĠWe ir -t rap -ag y -Ġparent heses -Ġout numbered -Ġcounter productive -ĠTob ias -ub is -P arser -ST AR -Ġsyn aptic -ĠG ears -Ġh iber -Ġdebunk ed -Ġex alted -aw atts -H OU -Ch urch -ĠPix ie -ĠU ri -ĠForm ation -ĠPred iction -C EO -Ġthro tt -ĠBrit ann -ĠMad agascar -ë ĭ -Ġbill boards -ĠRPG s -ĠBe es -complete ly -F IL -Ġdoes nt -ĠGreen berg -re ys -Ġsl ing -Ġempt ied -ĠPix ar -ĠDh arma -l uck -ingu ished -Ġend ot -Ġbab ys -05 9 -che st -r ats -Ġr idden -Ġbeet les -Ġillum inating -Ġfict itious -ĠProv incial -Ġ7 68 -Ġshe pherd -ĠR ender -Ġ18 96 -C rew -Ġmold ed -ĠXia omi -ĠSp iral -Ġdel im -Ġorgan ising -Ġho ops -ĠBe i -z hen -Ġfuck in -Ġdec ad -Ġun biased -am my -sw ing -Ġsmugg led -Ġk ios -ĠP ERSON -ĠInquis itor -Ġsnow y -Ġscrap ing -ĠBurg ess -P tr -ag ame -R W -Ġdro id -ĠL ys -ĠCass andra -Jac ob -Ġ35 4 -Ġpast ure -Ġfr anc -ĠScot ch -ĠEnd s -ĠI GF -def inition -Ġhyster ical -ĠBrown e -77 1 -Ġmobil ization -æ ķ -iqu eness -Th or -Ġspear headed -Ġembro iled -Ġconject ure -jud icial -Ch oice -Ġpaper back -P ir -Ġrec overs -ĠSur ge -ĠSh ogun -ĠPed iatrics -ãģ ł -Ġsweep s -ĠLabor atories -ĠP acks -al us -add in -Ġhead lights -g ra -Ev idence -COL OR -Ad min -Ĭ ± -Ġconco ct -s ufficient -Ġun marked -Ġrich ness -Ġdiss ertation -Ġseason ing -Ġg ib -ĠM ages -un ctions -ĠN id -che at -ĠTM Z -c itizens -ĠCatholic ism -n b -Ġdisemb ark -ĠPROG RAM -a ques -Ty ler -Or g -ĠSl ay -ĠN ero -ĠTown send -IN TON -te le -Ġmes mer -9 01 -Ġfire ball -ev idence -aff iliated -ĠFrench man -ĠAugust a -0 21 -Ġs led -Ġre used -ĠImmun ity -Ġwrest le -assemb led -Mar ia -Ġgun shots -ĠBarb ie -Ġcannabin oids -ĠTo ast -ĠK inder -IR D -Ġre juven -Ġg ore -Ġrupt ure -Ġbre aching -ĠCart oon -Ġ4 55 -ĠPale o -6 14 -Ġspe ars -ĠAm es -ab us -Mad ison -GR OUP -Ġab orted -y ah -Ġfel on -Ġcaus ation -Ġprep aid -Ġp itted -op lan -ĠShel ley -ĠRus so -ĠP agan -Ġwill fully -ĠCan aver -und rum -ĠSal ary -ĠAr paio -read er -ĠR ational -ĠOver se -ĠCa uses -Ġ* . -Ġw ob -Ke ith -ĠCons ent -man ac -77 3 -6 23 -Ġfate ful -et imes -Ġspir ited -ĠD ys -Ġhe gemony -Ġboy cot -ĠEn rique -em outh -Ġtim elines -ĠSah ara -ĠRel ax -ĠQuin cy -ĠLess ons -ĠE QU -SE A -N K -ĠCost co -Incre ase -Ġmotiv ating -ĠCh ong -am aru -ĠDiv ide -Ġped igree -ĠTasman ia -ĠPrel ude -L as -9 40 -57 4 -Ġch au -ĠSp iegel -un ic --- > -ĠPhil ips -ĠKaf ka -Ġuphe aval -Ġsent imental -Ġsa x -ĠAk ira -ser ial -Mat rix -Ġelect ing -Ġcomment er -ĠNeb ula -ple ts -ĠNad u -ĠAd ren -Ġen shr -ĠR AND -fin ancial -ĠCly de -uther ford -Ġsign age -Ġde line -Ġphosph ate -rovers ial -f ascist -ĠV all -ĠBeth lehem -Ġfor s -Ġeng lish -S olid -N ature -Ġv a -ĠGu ests -Ġtant al -Ġauto immune -;;;;;;;; ;;;; -ĠTot ally -ĠO v -Ġdef ences -ĠCoc onut -Ġtranqu il -Ġpl oy -Ġflav ours -ĠFl ask -ãĤ¨ ãĥ« -ĠWest on -ĠVol vo -8 70 -Ġmicro phones -ver bal -R PG -Ġi ii -; } -0 28 -Ġhead lined -Ġprim ed -Ġho ard -ĠSh ad -ĠEN TER -Ġtri angular -Ġcap it -l ik -ĠAn cients -Ġl ash -Ġconv ol -Ġcolon el -en emy -G ra -Ġpub s -ut ters -Ġassign s -ĠPen et -ĠMon strous -ĠBow en -il ver -H aunted -ĠD ing -start ed -pl in -Ġcontamin ants -ĠDO E -ff en -ĠTechn ician -R y -Ġrob bers -Ġhot line -ĠGuard iola -ĠKau fman -row er -ĠDres den -ĠAl pine -E lf -Ġf mt -ĠS ard -urs es -g pu -Un ix -Ġunequiv ocally -ĠCitizens hip -qu ad -m ire -ĠS weeney -B attery -6 15 -Ġpanc akes -Ġo ats -M aps -ĠCont rast -mbuds man -ĠE PS -Ġsub committee -Ġsour cing -Ġs izing -ĠBuff er -ĠMand atory -Ġmoder ates -ĠPattern s -ĠCh ocobo -ĠZ an -ĠSTAT ES -ĠJud ging -ĠIn her -* : -Ġb il -ĠY en -Ġexh ilar -oll ower -z ers -Ġsn ug -max imum -Ġdesp icable -ĠP ACK -ĠAn nex -Ġsarcast ic -Ġlate x -Ġt amp -ĠS ao -b ah -ĠRe verend -ĠChin atown -ĠA UT -d ocumented -ĠGA BA -ĠCan aan -ĠÙ ħ -Ġgovern s -pre v -E sc -ĠEst imates -OS P -Ġendeav our -ĠCl osing -omet ime -every one -Ġwor sen -Ġsc anners -Ġdev iations -ĠRobot ics -ĠCom pton -Ġsorce rer -Ġend ogenous -Ġem ulation -ĠPier cing -ĠA ph -ĠS ocket -Ġb ould -ĠO U -ĠBorder lands -Ġ18 63 -G ordon -ĠW TO -Ġrestrict s -Ġmosa ic -Ġmel odies -ç Ħ -T ar -Ġdis son -ĠProv ides -Ġ ...... -b ek -F IX -Ġbro om -ans hip -Do ctors -Ġner ds -ĠReg ions -na issance -Ġmet e -Ġcre pt -pl ings -Ġgirlfriend s -kn it -ig ent -ow e -Ġus hered -ĠB az -M obil -4 34 -ĠPres ents -orig in -Ġins omnia -ĠA ux -4 39 -ĠCh ili -irs ch -G AME -Ġgest ation -alg ia -rom ising -$ , -c row -ĠIn spection -at omic -Rel ations -J OHN -rom an -ĠClock work -ĠBak r -m one -M ET -Ġthirst y -Ġb c -Ġfacult ies -R um -Ġnu ance -ĠD arius -ple ting -fter s -etch up -Reg istration -ĠK E -R ah -Ġpref erential -ĠL ash -ĠH H -Val id -ĠN AV -Ġstar ve -ĠG ong -z ynski -ĠAct ress -Ġw ik -Ġun accompanied -lv l -Br ide -AD S -ĠCommand o -ĠVaugh n -Wal let -Ġho pping -ĠV ie -Ġcave ats -Ġal as -if led -ab use -66 1 -Ġib n -Ġg ul -Ġrob bing -t il -IL A -Ġmit igating -Ġapt ly -Ġty rant -Ġmid day -ĠGil more -ĠDe cker -Ġ§ § -part ial -Ex actly -Ġphen otype -Ġ[+ ] -ĠP lex -ĠI ps -vers ions -Ġe book -Ġch ic -g ross -":" "},{" -ĠSur prisingly -M organ -Ġresid ues -ĠConf ederation -in feld -Ġl yr -mod erate -Ġperpend icular -V K -Ġsynchron ized -Ġrefres hed -Ġad ore -ĠTor ment -ol ina -Ġ26 00 -Item Tracker -Ġp ies -ĠF AT -ĠR HP -0 48 -ĠRES P -ĠB J -all ows -P and -Ġunw elcome -ĠV oc -ĠBast ard -ĠO W -ĠL AR -ĠHeal er -Environment al -ĠKen yan -ĠTr ance -ĠP ats -Ġali ases -ĠGar field -Ġcampaign er -Ġadvance ments -ĠOkin awa -ĠC oh -ows ky -Ġstar ved -Ġsize able -Ġ: -) -Ġm RNA -Ġsusp ensions -ist ar -Scot land -Pr in --------------------------------- ---------------- -Ġ50 2 -Ġteasp oons -Ġ10 50 -Ġcoerc ive -ĠMason ic -edd ed -ĠPass enger -Ġl att -Ġbr aces -ĠSt eal -ĠNY T -ĠK ats -ĠCel est -ae z -T u -ĠCoul ter -ðŁ ĺ -Fl ickr -ĠWil mington -ith s -++ ; -Ġv ending -Ġneg ro -ĠPh i -ĠYellow stone -Call back -Ġsh ampoo -ĠSh ades -w at -Ġsuper human -Ġridic uled -Ġhol iest -om bo -Ġintern s -Ġh one -ĠPar agu -UR I -Ġd angling -ãĤ » -so v -ict ional -av ailability -Ġrev ocation -Ġd ow -in ic -ĠTHE IR -Ġis o -Ġout ings -ĠLeth al -Ġ) )) -Ġinacc ur -Ġout landish -Ġan us -let ico -id on -l ol -Ġun regulated -Ġsuccumb ed -Ġc uff -ĠWast eland -let al -Ġsub str -Ġcoff ers -Ġautom akers -ov i -ĠX ue -ĠDayton a -Ġjar ring -Ġf umes -Ġdisband ed -z ik -itt on -Ġstriking ly -Ġsp ores -Ad apter -.) : -ĠLynd on -ival ry -Ġor ally -Ġtumult uous -Ġdisple asure -Ġcon es -or rect -Ġappe ase -Ġder by -ĠTrip oli -ĠAl ess -Ġp oked -ĠGu ilty -v P -En ough -Ġorig inals -6 99 -Ġrabb i -Ġproverb ial -Ġpostp one -el ope -ĠMist y -Ġstaff ed -ĠUn employment -redit ary -Ġdilig ent -re comm -me asures -as in -8 25 -Ġpond s -Ġmm ol -ĠS AR -ĠC ARE -Ġ3 71 -Ġclen ched -ĠCors air -Ġcaric ature -z n -att ach -ĠSch ro -spe ak -p ainted -ĠS uc -ĠE NT -Ġcell ul -ĠP aid -di agn -WH ERE -Ġtext ed -B arn -Ġret racted -ĠRe ferred -S av -Ġup keep -Ġwork places -ĠTok ens -Ġampl ify -cl inical -Ġmult ic -mber g -Ġconvol uted -Reg ion -5 65 -ĠTop ic -Ġsn ail -Ġsal ine -Ġins urrection -ĠPet r -f orts -B AT -ĠNav ajo -Ġrud imentary -ĠLak sh -OND ON -Me asure -Ġtransform er -ĠGodd ard -Ġcoinc ides -ir in -R ex -ĠB ok -qu it -Ġshotgun s -Ġprolet arian -Ġsc orp -ĠAd a -5 14 -Ġsl ander -record ed -Ġemb ell -ris ome -Ġapolog izing -ĠMul cair -ĠGib raltar -Cl a -Ġall ot -ĠAtt ention -Ġ4 33 -le ave -Ġwh ine -ĠIss a -ĠFa ust -ĠBar ron -hen y -Ġvictim ized -J ews -Ġnurt uring -ett el -W inged -ĠSub tle -Ġflavor ful -ĠRep s -eng ed -call back -Ġdirection al -Ġcl asp -ĠDirect ions -plan et -icult ure -Hel per -ic ion -ac ia -Ġç ¥ŀ -Ġsur ges -Ġcan oe -ĠPrem iership -be en -Ġdef ied -ĠTro oper -Ġtrip od -Ġgas p -ĠE uph -ĠAd s -vern ight -high ly -R ole -Ġent angled -ĠZe it -6 18 -ĠRust y -Ġhaven s -ĠVaugh an -HA EL -ĠSER VICE -/ , -Ġstr icken -Ġdel usions -Ġb is -ĠH af -Ġgrat ification -Ġent icing -UN CH -Ad ams -ĠOL ED -ĠBeet le -Ġ18 99 -ĠSO FTWARE -ateg or -V L -ĠTot em -ĠG ators -AT URES -Ġimped ance -Reg istered -ĠC ary -ĠAer ial -on ne -en ium -Ġd red -ĠBe g -Ġconcurrent ly -Ġsuper power -ĠX an -j ew -imes ter -ĠDick inson -âĶ ģ -F la -Ġp ree -ĠRoll ins -© ¶æ -Ġden omination -ĠL ana -5 16 -Ġinc iting -sc ribed -j uries -ĠWond ers -app roximately -Ġsusp ending -Ġmountain ous -ĠL augh -oid al -N s -Det ect -) = -ĠL uthor -ĠSchwarz enegger -ĠMull er -ĠDev i -ec ycle -J ar -6 13 -ĠL ongh -B ah -ĠSP ORTS -n w -Ġref inement -Ġwater ways -Ġd iner -Bl ade -68 3 -F ac -Ġinitial s -Ġro g -Ġparan ormal -B UT -Ġ[ ( -ĠSw anson -ĠM esh -âĸ ¬ -Impro ve -ĠRad iation -ĠEst her -ĠE sk -ĠA ly -ik y -Ġir rad -ĠBuck ingham -Ġref ill -Ġ. _ -Re pe -CON CLUS -Ġdifferent iated -Ġchi rop -ĠAt kins -Pat tern -Ġexc ise -Ġcab al -N SA -ĠST A -ĠS IL -ĠPar aly -Ġr ye -ĠHow ell -ĠCount down -ness es -alys ed -Ġres ize -ãĤ ½ -Ġbudget ary -ĠStr as -w ang -Ġap iece -Ġprecinct s -Ġpe ach -Ġsky line -Ġ35 3 -pop ular -App earances -ĠMechan ics -ĠDev Online -S ullivan -Z en -Ġp u -op olis -5 44 -Ġde form -Ġcounter act -ĠL ange -Ġ4 17 -Con sole -77 4 -Ġnodd ing -Ġpopul ism -Ġhe p -Ġcoun selling -compl iance -U FF -Ġunden iably -Ġrail ing -ĠHor owitz -ĠSim one -ĠBung ie -Ġa k -ĠTal ks -x ff -fl ake -Cr ash -Ġsweat y -Ġban quet -ĠOFF IC -Ġinvent ive -Ġastron omer -ĠStam ford -ĠSc are -ĠGRE EN -olic ited -Ġr usher -Ġcent rist -ight ing -Ġsub class -Ġdis av -Ġdef und -ĠN anto -oci ate -m ast -Ġpac if -Ġm end -e ers -imm igration -ESS ION -Ġnumber ing -Ġlaugh able -ĠEnd ed -v iation -em ark -P itt -Ġmetic ulous -ĠL F -Ġcongrat ulated -ĠBir ch -Ġsway ed -Ġsemif inals -Ġhum ankind -m atter -ĠEqu ip -opa usal -S aid -ĠLay out -Ġvo icing -Ġth ug -Ġporn ographic -I PS -Ġmo aning -Ġgriev ance -Ġconf essions -esc al -TEXT URE -Aut hent -os aurus -P urchase -Ġreleg ation -al ter -ĠÂł Âł -Ġr iddled -Ġo gre -ĠLow ell -Occ up -E at -ĠHy der -ĠAdvis er -Com merce -H unt -ĠOr th -ĠComp etitive -ĠCL A -CD C -Ġsal ads -F le -Ġindustrial ized -` , -ĠO WN -Ġbec k -ĠPart icularly -oub t -Ġm M -ĠHuss ain -ĠChen nai -Ġ9 20 -Ġappoint ing -ĠCull en -,,,, ,,,, -Ġp ores -ver ified -Ġbi ochemical -em ate -Ġcoward ly -ĠHels inki -ĠEthiop ian -S OURCE -ER C -est ro -Ġbi otech -ĠS our -Ġbrew er -Bloom berg -Ġintens ify -Gl ass -an co -ĠF DR -gre SQL -ĠF ires -©¶æ ¥µ -ec o -100 1 -ĠHom eless -Ġinstant aneous -ĠH aste -ig el -D iamond -Ġp aving -Ġland fill -Ġd ads -h oun -: ] -Ġinc endiary -ĠLiving ston -ĠHil bert -ĠChe cks -st yles -in ators -ĠCl ive -ph rine -Ġchimpan zees -Ġp all -ĠJ M -ĠAad haar -ð Ŀ -Ġachie vable -dis abled -P ET -OOOO OOOO -M ot -Ġint angible -Ġbal let -ĠWe bs -ĠEst imated -Effect s -Ġb ailed -Josh ua -Ġturb ulence -Ġoccup ant -ĠDay light -Ġ36 1 -me et -Ġstat ically -Ġon look -Ġk i -il legal -Ġvel vet -Ġdehyd ration -Ġacqu ies -ĠRe z -ak ura -ĠU pton -at ro -Ġincomp rehensible -Ġback door -ĠRh ino -7 27 -Ġmath s -) + -Ġhe resy -Ġd f -ĠRoc he -ĠL ydia -Ġpanc reat -re ply -arre ll -Ġsolicit ation -Ġcirc adian -BI P -Ġfor ay -Ġcrypt ic -iz u -ime o -ĠTom ato -ĠH oms -ex amination -Ġqu arry -ĠVal iant -ĠJer icho -ĠIN CLUD -Ġ18 40 -5 19 -Ġres ists -Ġsnap shots -ĠSp ur -ĠAnt iqu -Log in -Ġbest selling -Ġant ic -ĠS utherland -ãĤ¢ ãĥ« -Ġ~ / -ĠP arm -è ĥ -P ages -int ensity -Ġimm obil -Ġ18 65 -zz o -Ġn ifty -Ġf entanyl -ĠPres ervation -op hen -Ġd arts -ĠD inosaur -po inters -ĠR ite -s uggest -aware ness -ĠSher idan -Ġst ances -Ġsor cery -Ġper jury -ĠNik ola -ie ver -Ġf iance -ĠJordan ian -ĠBall oon -Ġn ab -Ġk b -Ġhuman ities -ĠTan aka -hill ary -Ġconsult ancy -ĠZ ub -Ġrem ission -Ġconf id -CH Q -ĠF ug -Ġimpro vis -Y ep -/ _ -Ġunwilling ness -Ġport folios -05 5 -ĠInstruct or -aim an -Ġclaim ants -M bps -ĠBy e -re ceived -T weet -Ġind emn -ri z -am ara -N at -Ġeval uates -ĠL ur -ep ad -FO X -ĠTh ro -Ġrust y -Ġbed rock -ĠOp rah -J B -Ġmanip ulative -Ġwill ful -Ġrel apse -Ġext ant -The me -S ensor -ĠSt ability -go vern -Ġpo ppy -Ġkn ack -Ġins ulated -ĠT ile -ĠExt rem -Ġunt old -Ġconver ge -Ġref uel -ig roup -Ġdistort ions -Ġrav aged -Ġmechan ically -ĠRe illy -ĠN ose -ĠIncarn ation -ĠBeck y -abb ling -Ġt aco -Ġr ake -Ġmelanch oly -Ġillust rious -ĠDart mouth -Gu ide -ĠR azer -ĠBen z -Ult imate -ĠSur prise -Ġpage ant -off er -Who ever -Ġw iser -Ġchem ist -ĠHE LL -ĠBul k -Ġpl utonium -ĠCO VER -Ö ¼ -f ailed -Ġtire lessly -Ġinf ertility -ĠTr ident -ĠShow time -ĠC iv -V ice -requ ires -itt ance -Ġun controlled -interest ing -56 1 -Ġinnov ate -ateg ic -L ie -ĠS elling -U l -Ġsav ior -ĠT osh -Ġsw ast -P ASS -Ġr ink -Ġcard io -ĠI ro -ud i -Ġv antage -Ġv ans -ĠNi ño -+ = -Ġpropag ate -< ? -Ġmethod ological -204 39 -Ġtrig lycer -Ġing rained -ĠAn notations -arr anted -6 17 -ĠS odium -ĠA AC -techn ical -mult ipl -Ġ3 73 -å ĭ -Ġdec isively -Ġboost ers -Ġdessert s -ĠGren ade -Ġtest ifying -ĠSc ully -ID s -Ġlock down -ĠSc her -ĠR é -ĠWhit man -ĠRams ay -rem ote -Ġh ikers -ĠHy undai -Ġcons cientious -Ġcler ics -ĠSiber ian -ut i -is bury -Ġrel ayed -Ġqu artz -ĠC BI -seek ers -ull a -Ġweld ing -ĠSh al -ble acher -T ai -ĠSam son -Ġt umble -ĠInvest or -Ġsub contract -ĠShin ra -ow icz -j andro -d ad -Ġtermin ating -ĠNe ural -ä» £ -Ġleak age -ĠMid lands -ĠCaucas us -í ķ -c it -ll an -iv ably -ĠAlb ion -Ġ4 57 -Ġregist rations -Ġcomr ade -Ġclip board -0 47 -Ġdiscour aging -ĠO ops -Ad apt -Ġem path -n v -ĠPR OT -ĠDon n -ĠP ax -ĠB ayer -t is -Squ are -Ġfoot prints -part icip -ĠChile an -B rend -ind ucing -M agn -Ġclub house -ĠMagn um -Ġenc amp -ĠEth nic -uch a -ere y -Ġw atered -ĠCal ais -Ġcomplex ion -Ġsect s -Ġren ters -Ġbr as -oÄŁ an -Time out -Man agement -Ġinf ographic -P okemon -Cl ar -Ġloc ality -Ġfl ora -as el -P ont -Ġpop ulate -ĠO ng -Ġsubs istence -Ġa uctions -ĠMcA uliffe -ĠL OOK -br inger -Ġtit an -Ġmanif old -ĠâĹ ı -Ġcalibr ated -Ġcal iphate -ĠSH E -ĠCommission ers -ce ivable -j c -W inner -5 24 -Ġcond one -Other wise -Ġp iling -Ġem body -ĠCrime an -ut ics -ĠEx hibition -Ġ4 26 -e ering -Ġv ying -ĠH UGE -* =- -Ġprin cipled -à ¦ -Ġquir ks -ĠEdit ors -put ing -G ES -ĠF TA -ठ¾ -add on -ĠH AM -ĠFrie za -W oman -. $ -Ġc rib -ĠHer od -Ġtim ers -ĠSp aces -ĠMac intosh -at aka -Ġgl ide -Ġsmell ing -ĠB AL -Ġun su -Ġcond os -Ġbicy cl -ĠRev ival -55 3 -Ġjugg ling -H ug -ĠKardash ian -ĠBalk ans -mult iple -Ġnutrit ious -oc ry -19 00 -Ġinteg rates -Ġad joining -ĠF older -roll ment -ven ient -Ġu ber -y i -Ġwh iff -ĠJu ven -ĠB orough -net te -Ġb ilingual -ĠSp arks -ph thal -man ufact -Ġt outing -ĠPH I -Ke efe -Rew ard -Ġinf all -ĠTem per -typ ically -ĠNik ol -Ġregular s -Ġpseud onym -Ġexhib itions -Ġbl aster -Ġ40 9 -w arming -Ġrever ber -Ġrecip rocal -Ġ6 70 -ip ient -b ett -ĠBe gins -Ġit ching -ĠPh ar -Ass uming -Ġem itting -ĠML G -Ġbirth place -Ġt aunt -ĠL uffy -ĠAm it -Ġcir cled -ĠN ost -enn ett -Ġde forestation -ĠHist orically -ĠEvery day -Ġovert ake -79 2 -Ġn un -ĠLuc ia -Ġaccompan ies -ĠSe eking -ĠTr ash -an ism -R ogue -Ġnorth western -ĠSupplement al -ĠNY U -ĠF RI -ĠSat isf -x es -5 17 -Ġreass ured -Ġspor adic -Ġ7 01 -Ġmed ial -Ġcannabin oid -Ġbarbar ic -Ġep is -ĠExplos ive -ĠD ough -Ġuns olved -Support ed -Ġacknowled gment -sp awn -Ġkit chens -Ġ- = -talk ing -ic ist -ĠPeg asus -ĠPS U -Ġphot on -ĠAuthent ication -R G -@# & -76 2 -ĠCl air -Ġdi aper -Ġbr ist -ĠProsecut ors -ĠJ em -6 28 -ĠEvery where -ĠJean ne -equ ality -ãĥ© ãĥ³ -object s -ĠPel icans -Ġ39 2 -Ġbl u -b ys -ĠA go -Ġinstruction al -Ġdiscrim inating -ĠTR AN -ĠCorn el -ag os -Ġty re -Ġas piration -ĠBrid gewater -": - -! ". -ĠEn s -ĠCoc o -P ie -Ġdet ach -ĠC ouch -Ġphys ique -ĠOccup ations -osc opic -en ough -B uzz -App earance -Y P -Ġrac er -Ġcompl icity -r pm -T oy -Ġinterrupt s -ĠCat alyst -Ġut ilitarian -imp act -Ġsp aghetti -Ġp orous -Ġeste emed -Ġinc iner -ĠI OC -7 48 -Ġesp resso -ĠSm ile -abil ia -6 35 -Ġmathematic ian -Ġ4 24 -ĠK L -ĠH IP -Ġover heard -ĠT ud -ĠT ec -Ġqu izz -Ġfl attering -Ġcon n -âĢ İ -Ġatt aches -ĠR OS -ĠAC S -Ġt cp -ĠSh ame -sk ip -res pected -ĠTrin idad -gr ain -Ġfooth old -ĠUnch arted -ĠJul io -z l -av ored -ĠAn xiety -er rors -ĠCent auri -its ch -D addy -Ġclutch ing -ĠIm plement -ĠGut ierrez -Ġ7 60 -Ġtele portation -end ra -Ġrevers ible -st ros -Ad venture -08 3 -Ġliber ating -Ġas phalt -ĠSp end -AR DS -im sy -PR ES -ĠEmer ging -Ġwild fires -Ġtechn ologically -Ġem its -ĠART ICLE -Ġirregular ities -Ġcher ish -çī Ī -Ġst ink -ĠR ost -Econom ic -Ġcough ing -ĠMcC ann -pro perties -ilant ro -Ġreneg oti -Trans lation -Ġin quest -ĠGra pe -oot ers -gu i -ĠSwords man -ace ae -h itting -Ġr c -Ġexert ed -ĠS AP -it ent -Ġperil ous -Ġobsc urity -Ġassass inate -Ġab original -Ġresc uing -ĠSh attered -lock ing -all ion -Ch anging -ĠHar rington -ĠB ord -ĠAfgh ans -Jam ie -aret z -ĠAugust us -Ġ38 6 -8 30 -Ġj og -ok ingly -Tr igger -ĠH OR -Stat istics -Ġviewers hip -Ġadd itives -h ur -Ġmaxim izing -ĠR ove -ĠLou ie -ĠBuck et -ĠCHR IST -ou sel -Ġstre aks -ir ted -Ġt ert -Ġcolonial ism -Ġbur ying -y k -Cond ition -ĠDPR K -By Id -75 1 -âĹ ¼ -Ġwor risome -Ġvoc ational -sl ice -Ġsa ils -ĠCorrection al -95 4 -Ġt ul -K id -l uster -Ġfam ilial -ĠSp it -ĠEp iscopal -Specific ally -ĠVol cano -run s -q s -Ġve tted -Ġcram med -t rop -here r -Thank fully -Ġper cussion -Ġor anges -Ġround up -Ġ4 99 -x ious -Char acters -ĠZion ism -ĠR ao -ÃĽ ÃĽ -W F -Ġunintention al -ONE Y -Gr ab -Com mercial -Ġglut amate -ĠMcK enna -ru ciating -ning ton -ih u -Ch an -ĠSw ap -Ġleaf lets -Ġfunction ally -er ous -F arm -Ġcal oric -ĠLiter ally -con cert -Ġshe nan -Ġrep aid -ey es -Ġbas hing -ĠG orge -Ġcollabor ations -Ġun account -itch ie -Ġteam work -pp elin -Ġpip ing -Ġmin ced -Ġd iam -ri eg -Ġmasc ara -Ġsuck er -ĠMo ons -App s -ĠPe ck -Ġper v -ĠFl oat -o ley -ĠN ish -im ize -Ġarom atic -u in -end ish -! / -ĠB icycle -ĠAS IC -ile ged -ĠQuad ro -ios yn -Ġlock out -ĠW ink -SP EC -Attempt s -Ġseed ed -red o -ias is -Ġsn ag -ãĥķ ãĤ© -ãĤ ¶ -Ġground ing -Ġrelie ver -Ġfrivol ous -ĠG ifts -ĠF aces -Es pecially -Ġmicrobi ome -im ag -ĠSch l -ĠP les -ĠBle ach -ĠIr win -ĠE aton -ĠDisc iple -Ġmultipl ication -Ġcoer ced -Ġ4 19 -st h -E vil -B omb -Ġex orc -Ġstag gered -L ESS -Ġinert ia -ĠED IT -Ġgo b -Tr aditional -Ġclass y -Lear y -ĠP AGE -yr s -Ġtrans porter -Ġmat ured -Ġhij ab -Ġbi ome -Where as -Ġex termination -ĠT ues -ĠT akeru -ĠAud rey -er ial -ĠAd en -aff les -Ġnarciss istic -ĠB aird -UT F -I re -ĠCon nie -Ch amp -Ġwhis pering -ĠH att -D K -Ġdis infect -Ġdeduct ed -Ġpart ake -Ġdown grade -ĠEs ports -ĠContin uing -Ġdemocr atically -icro bial -itt a -Ġlim estone -Ġexempt ed -ĠFren zy -H erm -7 28 -Ġfled gling -Met a -765 61 -69 3 -% : -w ake -5 26 -ĠDis cipline -Ġvirgin ity -ĠLeg ions -ĠFrank ie -int ent -Ġrest rooms -ĠRou ter -da q -Ġobjection able -âĨ ij -w ark -ĠRah ul -g ain -activ ation -abs olute -ĠAccess ed -Ġ24 00 -ogg les -Ġsecond ly -ĠDEF ENSE -Ġpost age -wra pper -sh arp -7 29 -Ġcommun icates -Ġadd on -ĠMil itia -H ong -Ġsl umped -ĠJP EG -ĠI car -ad ish -68 1 -Ġmaj esty -ĠWolf gang -ĠEl astic -u per -Ġv iz -Ġunconscious ly -ĠST D -ĠS ass -Ġflower ing -ĠHel ic -ĠDra per -ĠAm ateur -Ġman ure -Ġdis ingen -ĠLe i -br ing -9 49 -Ġinhib ited -Ġhead quartered -Ġen igmatic -�� � -Ġred ress -R H -Ġratt led -Ġd iction -l io -ĠT BA -ĠSN AP -C alling -Ġfasc ists -ĠD ove -iew icz -0 36 -Ġco asts -ĠR ect -Ġ) ] -L ot -6 29 -ĠS EM -ĠPeters en -ĠExpl ain -ĠBo ards -ĠBe zos -ĠJ ournals -Ġ20 24 -p arser -Ġmist rust -Ġgr ate -ĠL ocked -bo a -S aint -g aming -Ġvow el -in ately -bl ow -All ah -Ġun matched -Ġb ordering -ĠExp end -n r -Or acle -rou ch -Ġcont iguous -ac us -Ġdist raught -58 1 -Ġanat omical -O X -ap ixel -8 33 -ĠPL US -Ġres usc -Ġab iding -57 3 -Ġvac ancies -Em ily -Ġhyp othal -ĠWer ner -ĠWe e -ĠDJ s -5 13 -Ġwitch craft -Ġac upuncture -ent ary -benef it -Product s -ĠP SP -ĠMP G -ĠJ inn -ĠJ arrett -Ġ4 45 -ĠIm aging -ĠP yth -Fin ish -Ġte x -Ġjuven iles -Ġhero ism -Ġdoubt less -ĠA ki -ĠT end -ĠPatri arch -Ġbit ters -ĠTele communications -it atively -ag na -Ġr g -ĠS OLD -Ġcomp ulsion -ĠN asa -ĠKath ryn -Ġmillion aires -Ġintrins ically -Ġbolst ered -time out -fl o -Ġtut or -p our -Stat ement -Ġ{ * -ĠRud olph -ĠKimber ly -rog ens -adi q -] + -Ġindign ation -Ġfract uring -ĠRe leases -ĠGr ain -pro tein -L ago -Ġvac ations -Ġboot ed -ĠTH REE -ĠH G -oresc ence -Ġt f -Ġso ar -iosyn cr -Ġgl ances -ĠSp oon -ĠJ ury -ĠCow boy -Ġcreat ively -Hig her -Ġsolic itor -Ġhaw k -ac io -89 6 -Ġsuperf lu -Ġbombs hell -ct ure -Ġbroker age -Ġraid ing -Ġf rench -Ġang led -Trans action -ĠGen ocide -u pe -ĠHait ian -57 2 -! : -Ġunwitting ly -iter ator -sc roll -Ġtall ied -Ġbi omedical -ĠC ARD -Ġe uphem -Ġbrain storm -a quin -K o -Mic helle -ĠR unes -ĠBall istic -ud ers -Ġmod esty -ĠiP ads -ĠEzek iel -Y E -Ġstars hip -Ġpower fully -Ġper l -ĠSh ade -ĠQu art -ĠE EG -Ġfisher man -OS ED -ĠTyp ical -df x -Ġmes hes -Ġet ched -worth iness -Ġtopp led -Ġ3 96 -or ius -We iss -Ġmy sql -ĠVal halla -Ù Ĵ -le asing -Ġrec omp -rap nel -S el -04 3 -Ġder ailed -ĠGu ides -IR T -Ġde human -ĠBritt any -" )) -Ġex claim -Ġb alk -Ġ8 40 -CLA IM -int el -L AB -Ġpe gged -Ġast roph -sm oking -Ġrig ging -Ġfix ation -Ġcat apult -ins ide -ĠC ascade -ĠBolshe vik -G aza -Dep th -Ġloud spe -Ġalmond s -me yer -l eness -j en -f resh -Ġunbeat en -ĠSqu id -ĠPres umably -Tim er -B W -Ġro sters -Ġell ipt -ĠHar riet -dat abase -ĠMut ual -ĠComm odore -uk ed -kn ife -ĠCOMM UN -h ya -Ġmel ts -arch ives -Ġrat ification -Ġmultip lying -Ġinter oper -Ġasc ert -w ings -ver ting -ĠScorp ion -ay e -ĠPorts mouth -ĠM TA -n it -iaz ep -Ġqu arantine -Ġslides how -Ġcent imeters -Ġsyn opsis -Ġsp ate -th irst -Ġnom inating -ĠMel vin -Pre view -Ġthro b -Ġgener ational -ĠRad ius -rest ling -put able -aw ar -N ECT -Ġunlaw fully -ĠRevel ations -Wik ipedia -sur v -Ġeye ing -ij n -ĠF W -Ġbr unt -Ġinter stellar -Ġcl itor -ĠCroat ian -ĠCh ic -ev a -ĠDis app -ĠA kin -iner ies -d ust -Interest ed -Ġgen esis -ĠE ucl -ö n -p icking -Ġmut ated -Ġdisappro ve -ĠHD L -Ġ6 25 -Ì ¶ -c ancer -Ġsqu ats -Ġle vers -Disc uss -= ] -D ex -ĠVIDE OS -A UD -Ġtrans act -ĠKin ect -ĠK uala -ĠC yp -7 47 -Ġsh attering -Ġarsen ic -ĠInt ake -ĠAngel o -ĠQu it -ĠK he -Ġ18 93 -M aker -0 29 -ĠPain ting -Dis able -9 16 -Ġanal ges -Ġtact ile -Ġprop hes -Ġd iced -ĠTravel s -ĠHe ader -ĠClub s -Ass istant -Ġinc rim -Ġd ips -Ġcruc ifix -ĠShan ahan -ĠInter pret -Ġ40 90 -al ogy -abb a -Ġsimul ac -hus band -S IM -Ġrecy cle -uc er -ed ged -Ġre naissance -ĠBomb ay -Cath olic -ĠL INE -ĠCl othing -re ports -Ġpl aus -Ġd ag -ĠM ace -Z I -Ġintr uder -ĠVeter inary -g ru -Ġsne aky -ĠS ie -ĠC innamon -P OSE -Ġcou rier -ĠC NS -Ġemanc ipation -s it -Ġplay through -ĠFac ilities -v irt -ĠG auntlet -Thom pson -Ġunbeliev ably -Param eters -Ġst itching -ign e -ĠTH ESE -Priv acy -Ġshenan igans -Ġvit ri -ĠVal id -59 1 -Ń · -ĠProt otype -ink a -SC P -ĠT id -è Ī -old ed -Ġindividual ity -Ġbark ing -Ġm ars -ĠW D -Ġ8 20 -Ġt ir -Ġsl apping -Ġdisgr untled -ĠAng ola -ri us -ĠTorn ado -ĠTh urs -Ġcapt cha -Ġang st -ĠP og -ĠAssass ins -ĠAd idas -Ġjoy ful -Ġwh ining -Emer gency -Ġphosph orus -Ġatt rition -oph on -ĠTimber wolves -ĠJ ah -ĠBr inging -ĠW ad -ĠEn sure -oh l -ĠX ie -omm el -c mp -Ġz ipper -Ġrel at -ĠCor ridor -m ilo -T ING -Av g -Ġcro pped -] } -Ġr aged -ĠLump ur -ĠGuer rero -our ke -N ut -Ġoff sets -og lu -dr m -Ġmort als -lat able -Ġdismiss ive -ä¸ ī -Ġthro ats -Ġchips et -ĠSpot light -Catal og -art ist -G b -Ġch illy -Ġst oked -Ġ3 74 -W ard -L atin -Ġf iasco -Ġble ach -Ġb rav -Enh anced -Ġin oc -ĠFior ina -_ > -Ġle ukemia -Ġel uc -Ġannoun cer -ĠLith uan -ĠArm ageddon -å ĩ -Len in -ĠR uk -Ġpe pp -ĠRom antic -ĠP IT -ĠInter stellar -ĠAt kinson -R aid -J s -Go al -C ourse -Ġvan ishing -es ley -ĠR ounds -Els a -59 3 -Ġredund ancy -ĠST AND -Ġprop hetic -Ġhabit able -ry u -Ġfaint ly -M ODE -Ġfl anked -IR C -Aw esome -Ġsp urious -ĠZ ah -ĠMS G -Ġsh ading -Ġmotiv ational -ĠSant ana -ĠS PR -Ġexc ruciating -om ial -ĠM iko -ĠLe opard -A byss -Ġ[ | -d irty -Ġbath s -Ġdem oral -and re -P B -Ġun ification -Ġsac rament -Ġ[ & -Ġpric eless -Ġgel atin -Ġeman ating -ĠAll aah -98 6 -Ġout burst -Ġer as -ĠX VI -ĠSP I -O tt -ĠLaz arus -PL IED -F lying -blog s -W isconsin -R aven -Ġreb ate -Ġcreep s -ĠSp an -ĠPain ter -ĠKir a -ĠAm os -ĠCor vette -Cons umer -ĠRec over -ck i -Ġpes ky -ĠIn vention -Compan ies -Ġchalleng ers -ad emic -ĠUkrain ians -ĠNeuro log -ĠFors aken -Ġent rants -Ġemb attled -Ġdef unct -ĠGlac ier -Ġpo isons -ĠH orses -m akes -ĠD irt -Ġ4 23 -hh h -ĠTrans formation -QUI RE -................ .. -Ġtrave ller -ĠSe xy -ĠK ern -ip olar -Ġransom ware -oooooooo oooooooo -E c -rub y -Prof essional -ĠOut break -arg ument -G rey -ĠFif a -ĠCH O -ĠFOR M -ĠAm trak -- [ -Ġcr adle -Ġantioxid ants -ãģ®å ® -7 36 -ĠNAS L -ĠContribut ions -Ind iana -ĠST EP -C SS -Ġsal ient -Ġall ocations -yr ights -Ġm ashed -ĠCut ter -Sex ual -Ġp ounded -Ġfan base -Ġc asc -ĠTrans parency -Ġanaly tic -ĠSummon er -× ŀ -ĠAD C -det ail -Ġvan quished -Ġcr abs -ar ie -Dest roy -ĠS ack -Ġtrans istor -Al abama -ĠK oen -ĠFisher ies -c one -Ġannex ed -ĠM GM -es a -Ġf aked -ĠCong ratulations -Ġhind ered -Ġcorrection al -ĠI TV -lee ve -Ġin appropriately -lic ks -Ġtresp ass -Ġp aws -Ġnegoti ator -ĠChrist ensen -lim its -ĠDian ne -Ġeleg ance -ĠContract s -an ke -Ob j -Ġvigil ance -Ġcast les -ĠN AD -ĠHol o -Ġemph atically -ĠTit us -ĠServ ing -ĠRich ie -ĠP igs -5 68 -Ġanim osity -ĠAtt ributes -ĠU riel -M Q -my ra -ĠApplic ant -Ġpsychiat rists -ĠV ij -ĠAb by -ag ree -P ush -Ġk Wh -hib a -Ġinc ite -ĠWe asley -ĠTax i -minist ic -hy per -ĠF arn -Ġ6 01 -ĠNation wide -F ake -95 2 -Ġma ize -Ġinteract ed -Ġtransition ed -Ġparas itic -Ġharm onic -Ġdec aying -Ġbas eless -ns ics -Ġtrans pired -Ġabund antly -ĠFore nsic -Ġtread mill -ĠJ av -ab and -Ġssh d -Ġfront man -ĠJak arta -oll er -dro ps -ĠSERV ICES -rompt u -oph ical -h ospital -bled on -6 45 -Ġmid range -ĠEV ENT -cul ated -raw led -Ġper ched -Ġover board -ĠPe el -ĠP wr -ĠCar th -ĠCOM PLE -co e -sh all -Ġdeter rence -M ETHOD -ĠAbs ent -M EN -Ġs ill -ĠLE VEL -Y ork -Ġsin ners -ĠOP EC -ĠN ur -ĠDesign s -se lection -Ġunw orthy -CH A -Ġstreng thens -88 3 -ed ly -Ġslic ing -Ġmal nutrition -Ġfilm making -ĠPol k -ur ated -Ġ4 21 -bre akers -!' " -Ġwet lands -ĠDisc rimination -Ġallow able -Ġste ered -ĠSic ily -S AM -Ġmust ache -Ġm ids -Ġcl ipped -Ġcirc ulate -Ġbr ittle -ĠBuild ings -ra ised -ĠRound up -Ġwealth ier -Ġoverw rite -Ġover powered -ĠGerr ard -s ites -PD ATED -Ġacute ly -ĠGam ble -Ġp im -ĠK us -Typ ically -De ploy -ĠMoroc can -p otion -com be -Ġvigil ante -Ġ36 3 -St ew -ĠB agg -Ġres ided -ĠSp o -Ġrem nant -Ġempt iness -br ainer -Ġout patient -pri ority -Ġle ptin -ĠPay ton -ĠGle aming -ĠS hed -ĠPol o -ĠMormon ism -rest ricted -arl ane -w x -Ġcreat ine -ĠAn on -ĠST UD -ĠJ UL -ĠT ee -5 28 -08 9 -Ġhat ched -Dis patch -ĠCompos ite -Ġ45 1 -p uff -ĠX COM -ĠOr n -ĠTH ANK -END ED -ĠAshe ville -Ġà ľ -Ġman go -ĠS lightly -world ly -ĠW ander -ĠExp and -ĠCh r -M ist -Ġorthodox y -ĠUN ESCO -reg ate -Else where -k ie -ir led -Ġtopp le -Ġadopt ive -ĠLeg s -d ress -ĠS agan -b are -ĠGl ou -Cr unch -Ġhelp ers -Ġchron ically -ĠH uma -1 0000 -Ġaccommod ating -äº Ķ -Ġwrink les -Ġdod ged -four th -Ġpre con -Ġcompress or -ĠK are -Ġev ict -ĠWar wick -im ar -Ġmodern ization -Ġband wagon -Ġref uted -Ġnet ted -ĠNa ples -ĠGen ie -per ors -Ġfield ed -Ġde re -ĠPar ables -le es -Ġtr out -asp ers -Ġn ihil -Ġhapp iest -Ġflo ppy -ĠLo ft -ĠHe ard -Ġun ison -Ġl ug -ĠRed mond -class ic -Supp orters -SH IP -G MT -Ġfue lled -ç IJ -Ġd d -ĠEmin em -Ġ18 97 -NY SE -Ġsecret aries -ĠF IA -ĠCanaver al -F avorite -Ġp omp -Ġdetain ee -ers hip -aim on -i our -ĠA pex -Ġplant ations -am ia -ac ion -R ust -Ġtow ed -ĠTru ly -5 77 -Ġshel tered -r ider -W o -Ġl air -ĠInt elligent -impro ve -m atically -Ġet iquette -ad ra -all o -ĠJun o -any thing -ĠStru ggle -ĠPred ict -ĠGr imes -ĠAMER ICA -ct x -ĠSit uation -W OOD -Ġsol uble -me ier -Ġintoler able -ang ering -Ġun interrupted -Ġtool tip -Ġinterrog ated -Ġgun ned -ĠSne ak -æŃ ¦ -Ġt ether -Ġcr umble -L ens -Ġclust ered -ĠSy l -ĠHas an -Ġdystop ian -w ana -Ġjoy stick -ĠTh ib -amm u -Tom orrow -5 46 -Ġoverc ame -Ġminim ized -cept or -Run ner -ENG TH -ĠBrend a -ĠAchieve ments -Ġtor ches -Ġrapp ort -ĠInvestig ator -ĠHand ling -rel ation -g rey -8 15 -Ġk cal -ĠComm ands -d q -Ġcur ls -Ġbe arer -Ġcyn icism -it ri -ĠUse ful -B ee -D CS -Ġab ras -P ract -BIL ITIES -7 12 -Ġdebug ger -Ġdebt or -ĠL ia -ĠK ers -Ġexacerb ate -ĠSt acy -ĠB land -ĠSc enes -Ġbranch ing -âĸĪâĸĪâĸĪâĸĪ âĸĪâĸĪâĸĪâĸĪ -ape ake -Ġs alsa -Ġmish and -ĠKon ami -ĠN ib -Ġanecd ote -Ġagree able -Ï ī -ĠNath aniel -ĠHe isman -ĠB eware -Ġ18 86 -spect ive -69 1 -5 22 -Ġinhib its -Ġhas hing -Ġ18 89 -å° Ĩ -v ich -P ure -Ġsolid ly -Ġaspir in -im aru -Ġstreet car -ĠU CS -ĠJ udd -Ġflash backs -p ins -Ġ14 40 -ĠUN HCR -ĠSym ptoms -T IT -5 38 -F ra -% ); -Ġo oz -Ġcur few -Ġcal med -Ġparticip ates -Te X -Ġnons ensical -Ġfull back -ĠDe L -mon key -h ari -Ġmetabol ites -Ġloot ed -ĠAL WAYS -ĠB CC -L t -oc het -B one -Ġveto ed -Ġg cc -ĠCL ICK -Ġ18 88 -s af -Ġstiff ness -Ġlow ly -ĠGe h -vers on -ors et -Ġun foreseen -Ġan esthesia -ĠOpt ical -Ġrecon structed -ĠT up -sh ows -NEW S -ĠNewsp aper -ĠA SA -ter a -N umbers -Ġinexpl icable -× ij -Ġhard ness -unt arily -ĠA cer -grad ient -ARD IS -Ġwood land -Ġmetaph ors -ĠWem bley -ĠPa vel -phil is -Ġre writing -Ġpercept ual -Ġ10 70 -worm s -ĠDown s -Ġunsur prisingly -Ġtag ging -fl ame -Ġlit res -Ġboun ces -ĠB abe -sh ut -Ġoverd oses -ĠShe ila -ĠCh au -ĠBl ess -Capt ure -ĠSign ificant -ĠSc ion -Ġ38 9 -ĠMc H -ĠTitan ium -ĠMe al -amed a -ag ents -agg ressive -B illy -76 3 -ĠS aying -DER R -it one -Coll ins -B ound -Ġbol ted -ĠDM CA -95 3 -Ġun iqueness -Ġep igen -un ci -ant am -Ġreck oning -ch airs -OG R -ĠSen egal -Ġ18 62 -re levant -Ġ ¯ -Ġpharm acies -ĠG eral -v ier -Y an -OR PG -Ġrab id -b ending -ĠUN ITED -Ġ4 65 -As sembly -Ġwe ep -Ġbe hest -ĠMother s -ĠJ ace -h id -Ġwh irlwind -ĠUN IVERS -Ġut opian -Ġkidn ap -Ph ilipp -K in -89 3 -Ġlivest ream -ĠM ISS -Ġsub versive -ĠTechn iques -ĠJUST ICE -ĠB ASE -Ġ38 7 -Ġassail ants -ĠHard core -Ġsprink led -ĠP se -é ļ -print ed -ĠH au -OR GE -ĠT OUR -Ġl aced -Ġit ch -G iving -Ġport ed -78 1 -//////////////// //////////////// -bre eding -Ġlog ger -ĠH OL -inn ie -First ly -Ġembry onic -Ġdeleg ated -p ai -O IL -Ġcentr ally -ĠR x -ĠSc outing -D utch -Ġhe reditary -ĠCru iser -s at -5 29 -ĠMar riott -other mal -Ġprohib itions -E arn -ĠSt ab -ĠColleg es -ĠBel ief -st retched -ĠL H -ĠEntity Item -C IA -Ġun rem -Ġlaure ate -Ġdenomin ations -sum mary -h ler -S pect -ĠK laus -ĠBe ans -Ġins ur -ĠPA X -Ġfield er -ĠV et -ĠSp arrow -z ie -ĠS Q -ĠMond ays -ĠOff line -ĠLer ner -ĠExt ensions -Ire land -Ġpatron age -Ġcontrast ed -ĠMan ia -h irt -Mos cow -Ġcondem ns -ĠAn ge -Ġcomp osing -ĠPe pe -ĠP addock -Ġheter ogeneity -Ġide ologically -Ġf ishes -Ġcur sing -ĠR utherford -ĠFlo ating -ĠAm elia -Te a -Syn opsis -Ġstun ts -Ġbe ad -Ġstock ing -ĠM ILL -ob ook -mass ive -\ < -Ġh ump -ĠPref erences -Engine Debug -ge ist -ĠNiet o -ome ver -ish y -eval uate -col onial -Altern ative -ĠGo Pro -ĠV ortex -ĠNET WORK -ans ky -Sec ure -ĠTh rust -Sn ake -Ġparcel s -Ġsam urai -Ġactress es -N ap -M F -ifer ation -Be er -5 23 -ĠI ly -oint ment -P ing -Ġstri ped -ĠMell on -oss ession -Ġneut ron -end ium -Ġa ph -ĠFlav oring -Ġ38 3 -Ġrespons iveness -ĠJ indal -ĠHitch cock -Den ver -ĠDRAG ON -sm anship -ĠDu pl -Ġs ly -Ġweb cam -ĠTw ain -ĠDar ling -ili ate -cons umer -D IT -Ġnames ake -Ġun orthodox -Ġfun er -ĠPL oS -ĠCONTR OL -ozy g -ogl obin -F ACE -ER G -ĠD ia -ĠF iesta -ce le -0 34 -Ġencl ave -âĸ¬ âĸ¬ -on ement -al ist -M and -Ġhome grown -ĠF ancy -Ġconcept ions -ĠCont ains -ure en -Ġreiter ate -Ġme ager -Ġinstall ments -Sp awn -6 27 -Ġphot oc -ĠCab rera -ĠRos enthal -ĠLans ing -is ner -Ġinvest s -ĠUFO s -EX P -Hard ware -Ġtr agically -Ġconced es -ie ft -ch am -bor gh -ĠSch r -ĠMel anie -ĠH oy -Ġvisit ation -Ġid iosyncr -Ġfract ions -Ġfore skin -ob os -Ġpo aching -ĠVI EW -Ġstimul ates -ĠG ork -can on -M IC -ĠNem esis -ĠInd ra -ĠDM V -Ġ5 29 -Ġinspect ing -Ġgrand ma -ĠW hedon -ĠSh ant -ĠP urg -ik an -ĠT eg -ĠCL R -z ac -Vict oria -ĠVer ify -ion ics -Ġpart ying -ĠM ou -col our -Ġtestim onies -l ations -Ġpress uring -hi ro -ac ers -Ġf id -ang ler -ĠCS I -Ġhere after -Ġdiss idents -report ing -iph any -che v -Ġsol itude -Ġl obe -Ġind is -Ġcred ential -re cent -ad ult -ĠNir vana -ĠFranch ise -L ayer -H yp -ĠBerks hire -Ġwill s -t if -Ġtot em -ĠJud ah -rep air -Inst ant -5 48 -Ġemb assies -Ġbott leneck -Ġb ount -Ġtyp ew -ĠAl vin -j ing -im ilar -R ush -Ġbr im -ĠHEL P -A im -] ' -Ġpass ively -Ġbound ed -ĠR ated -Ġcriminal ity -Ġbiom ark -Ġdisp atcher -ĠTow ards -Ġ+ ++ -right eous -f rog -ĠP anc -C arter -0 32 -æ© Ł -Ġult raviolet -ĠLic ensed -ĠT ata -ĠBl essing -ĠG AM -Ġchem ically -ĠSe af -ĠRE LE -ĠMerc enary -capital ist -Ġform ulations -Ġann ihilation -ĠVer b -ĠAr gon -Ġun loaded -Ġmorp hed -Ġconqu ering -back er -I ELD -Ġtheft s -Ġfront runner -ĠRoy ale -ĠFund amental -el ight -C hip -necess ary -ay n -ĠSl ip -Ġ4 48 -cern ed -P ause -Ġshock ingly -ĠAB V -Ġcomp osure -7 33 -ĠMotors port -ah ime -Mur ray -M ach -Ġgr ids -Ġdeb ian -Ġfurther more -Ġdexter ity -ĠCollect ions -os lov -il age -b j -ĠMont eneg -Ġstrut Connector -Ġmassac res -Ġbrief s -fet ched -uv ian -ol ition -Fail ure -emon ic -Ġfl ared -Ġclaim ant -Ġc ures -Ġgive aways -ĠSubst ance -al ions -Ġcr inge -ĠK ul -Ġarist ocracy -ĠUl ster -ol ated -h ousing -ĠM IS -Ġgl ared -ĠWil helm -ne eds -lam bda -build ers -ĠV IS -Ġradi ator -ĠGhost busters -Ġ4 36 -act ual -Ġher ds -ç a -watch ing -Ġcounter ing -Ch arge -Ġchar red -Ġwar heads -Ġiod ine -ĠM acy -04 1 -Ġdepart ures -ĠS ins -Ġdy ed -ĠConcept s -g ado -7 13 -Ġquot ations -Ġg ist -ĠChrist y -Ġant igen -ĠHem p -ĠD rawn -ĠB arg -ez vous -Ġp aternity -Ġar du -ĠAnch orage -ĠR ik -Ġover loaded -ĠUs ername -ĠTam my -ĠN au -ĠCell ular -Ġw aning -Ġrod ent -ĠWor cester -il ts -ĠT ad -Ġdwell ings -Ġbull ish -4 31 -Ġretali ate -Ġmig raine -ĠChev ron -CH ECK -Ġdon key -c rim -SP A -ĠAn alog -Ġmarqu ee -ĠHa as -B ir -ĠGD DR -ĠDownload s -Ġwill power -ĠFor th -ĠRecord ed -Ġimp ossibility -ĠLog ged -ĠFr anks -ĠR att -in itions -Ġclean ers -Ġsore ly -Ġflick ering -ĠEx amination -c atching -allow een -Ms g -Ġdun no -F a -Ġdys ph -c razy -.' '. -Ġmain line -Ġc s -Ġp tr -ĠW ally -ig un -95 1 -ĠBig foot -f ights -Ġretrie ving -J r -Ġdupl ication -ĠExpl an -Ġrel ational -Ġqu aint -Ġbisc uits -Ġad o -Ġsh udder -Ġantid ote -blood ed -ks h -Ġsa uces -Ġrein vest -Ġdispens ary -ĠD iver -Ġ9 000 -stud ent -Ġin separ -esc ap -Ġtodd lers -ĠGP IO -ĠAss ignment -head ers -Ġlack luster -Ġab ack -95 6 -Ġtool bar -7 45 -Ġo ust -Ġcontempl ation -ĠPRES IDENT -Ġ4 58 -==== == -Ġguarantee ing -ĠHe ist -ĠCann es -Ļ ½ -Ġcollabor ator -ĠAm p -Ġg ou -ĠSH ALL -st ories -78 3 -Ġmobil ized -Ġbro od -ĠL U -ĠðŁ ij -Ġref in -ĠAnthrop ology -v ind -ill i -Ġwarrant ies -ĠB abel -Ġsw ath -Ġc aches -Ġantagon ists -art ifacts -Ġhot ly -ĠSt arts -ĠG ö -z ag -!! !!! -Ġsc ourge -Ġcons piring -ru its -re verse -ĠShe en -ĠJes uit -ĠGiov anni -ad ies -Ġbutt ocks -ear cher -ac an -Ġvolley ball -Ġshroud ed -Ġscore board -b ats -ĠI PM -Ġass es -Ġde regulation -ĠTe legram -ĠReb oot -Ġ7 000 -ĠCan ary -Ġk ernels -ĠFranç ois -ĠD uff -ĠP on -ĠLe ica -ĠGar min -Ġor phans -ĠClaud ia -Ġcal endars -ĠLe ilan -ent o -R ocket -Ġbr unch -ĠHaw king -ain ers -Ġsens ibilities -Ġk W -ĠK and -Ġre claimed -Ġinteresting ly -× © -rom y -J M -ĠEnhance ment -b ush -Sk ip -Ġrapp ers -Ġg azing -p edia -ath lon -Rev olution -Ġsn ipers -Ġre verted -Ġconglomer ate -T erry -79 4 -Ġhars her -Ġdes olate -ĠHit man -Comm ission -Ġ( / -âĢ¦ ." -Com par -Ġampl ification -om inated -Ġreg ress -ĠColl ider -Ġinform ants -Ġg azed diff --git a/funasr_detach/models/whisper/utils/assets/gpt2/special_tokens_map.json b/funasr_detach/models/whisper/utils/assets/gpt2/special_tokens_map.json deleted file mode 100644 index 817762d631ad6f9c799f6b9dc713c46420e65546..0000000000000000000000000000000000000000 --- a/funasr_detach/models/whisper/utils/assets/gpt2/special_tokens_map.json +++ /dev/null @@ -1 +0,0 @@ -{"bos_token": "<|endoftext|>", "eos_token": "<|endoftext|>", "unk_token": "<|endoftext|>"} \ No newline at end of file diff --git a/funasr_detach/models/whisper/utils/assets/gpt2/tokenizer_config.json b/funasr_detach/models/whisper/utils/assets/gpt2/tokenizer_config.json deleted file mode 100644 index c92208ce39324aadabc239d73dd9bb3ec1cd45ca..0000000000000000000000000000000000000000 --- a/funasr_detach/models/whisper/utils/assets/gpt2/tokenizer_config.json +++ /dev/null @@ -1 +0,0 @@ -{"unk_token": "<|endoftext|>", "bos_token": "<|endoftext|>", "eos_token": "<|endoftext|>", "add_prefix_space": false, "model_max_length": 1024, "special_tokens_map_file": null, "name_or_path": "gpt2", "tokenizer_class": "GPT2Tokenizer"} \ No newline at end of file diff --git a/funasr_detach/models/whisper/utils/assets/gpt2/vocab.json b/funasr_detach/models/whisper/utils/assets/gpt2/vocab.json deleted file mode 100644 index 84ef7fb594b5c0979e48bdeddb60a0adef33df0b..0000000000000000000000000000000000000000 --- a/funasr_detach/models/whisper/utils/assets/gpt2/vocab.json +++ /dev/null @@ -1 +0,0 @@ -{"!":0,"\"":1,"#":2,"$":3,"%":4,"&":5,"'":6,"(":7,")":8,"*":9,"+":10,",":11,"-":12,".":13,"/":14,"0":15,"1":16,"2":17,"3":18,"4":19,"5":20,"6":21,"7":22,"8":23,"9":24,":":25,";":26,"<":27,"=":28,">":29,"?":30,"@":31,"A":32,"B":33,"C":34,"D":35,"E":36,"F":37,"G":38,"H":39,"I":40,"J":41,"K":42,"L":43,"M":44,"N":45,"O":46,"P":47,"Q":48,"R":49,"S":50,"T":51,"U":52,"V":53,"W":54,"X":55,"Y":56,"Z":57,"[":58,"\\":59,"]":60,"^":61,"_":62,"`":63,"a":64,"b":65,"c":66,"d":67,"e":68,"f":69,"g":70,"h":71,"i":72,"j":73,"k":74,"l":75,"m":76,"n":77,"o":78,"p":79,"q":80,"r":81,"s":82,"t":83,"u":84,"v":85,"w":86,"x":87,"y":88,"z":89,"{":90,"|":91,"}":92,"~":93,"¡":94,"¢":95,"£":96,"¤":97,"¥":98,"¦":99,"§":100,"¨":101,"©":102,"ª":103,"«":104,"¬":105,"®":106,"¯":107,"°":108,"±":109,"²":110,"³":111,"´":112,"µ":113,"¶":114,"·":115,"¸":116,"¹":117,"º":118,"»":119,"¼":120,"½":121,"¾":122,"¿":123,"À":124,"Á":125,"Â":126,"Ã":127,"Ä":128,"Å":129,"Æ":130,"Ç":131,"È":132,"É":133,"Ê":134,"Ë":135,"Ì":136,"Í":137,"Î":138,"Ï":139,"Ð":140,"Ñ":141,"Ò":142,"Ó":143,"Ô":144,"Õ":145,"Ö":146,"×":147,"Ø":148,"Ù":149,"Ú":150,"Û":151,"Ü":152,"Ý":153,"Þ":154,"ß":155,"à":156,"á":157,"â":158,"ã":159,"ä":160,"å":161,"æ":162,"ç":163,"è":164,"é":165,"ê":166,"ë":167,"ì":168,"í":169,"î":170,"ï":171,"ð":172,"ñ":173,"ò":174,"ó":175,"ô":176,"õ":177,"ö":178,"÷":179,"ø":180,"ù":181,"ú":182,"û":183,"ü":184,"ý":185,"þ":186,"ÿ":187,"Ā":188,"ā":189,"Ă":190,"ă":191,"Ą":192,"ą":193,"Ć":194,"ć":195,"Ĉ":196,"ĉ":197,"Ċ":198,"ċ":199,"Č":200,"č":201,"Ď":202,"ď":203,"Đ":204,"đ":205,"Ē":206,"ē":207,"Ĕ":208,"ĕ":209,"Ė":210,"ė":211,"Ę":212,"ę":213,"Ě":214,"ě":215,"Ĝ":216,"ĝ":217,"Ğ":218,"ğ":219,"Ġ":220,"ġ":221,"Ģ":222,"ģ":223,"Ĥ":224,"ĥ":225,"Ħ":226,"ħ":227,"Ĩ":228,"ĩ":229,"Ī":230,"ī":231,"Ĭ":232,"ĭ":233,"Į":234,"į":235,"İ":236,"ı":237,"IJ":238,"ij":239,"Ĵ":240,"ĵ":241,"Ķ":242,"ķ":243,"ĸ":244,"Ĺ":245,"ĺ":246,"Ļ":247,"ļ":248,"Ľ":249,"ľ":250,"Ŀ":251,"ŀ":252,"Ł":253,"ł":254,"Ń":255,"Ġt":256,"Ġa":257,"he":258,"in":259,"re":260,"on":261,"Ġthe":262,"er":263,"Ġs":264,"at":265,"Ġw":266,"Ġo":267,"en":268,"Ġc":269,"it":270,"is":271,"an":272,"or":273,"es":274,"Ġb":275,"ed":276,"Ġf":277,"ing":278,"Ġp":279,"ou":280,"Ġan":281,"al":282,"ar":283,"Ġto":284,"Ġm":285,"Ġof":286,"Ġin":287,"Ġd":288,"Ġh":289,"Ġand":290,"ic":291,"as":292,"le":293,"Ġth":294,"ion":295,"om":296,"ll":297,"ent":298,"Ġn":299,"Ġl":300,"st":301,"Ġre":302,"ve":303,"Ġe":304,"ro":305,"ly":306,"Ġbe":307,"Ġg":308,"ĠT":309,"ct":310,"ĠS":311,"id":312,"ot":313,"ĠI":314,"ut":315,"et":316,"ĠA":317,"Ġis":318,"Ġon":319,"im":320,"am":321,"ow":322,"ay":323,"ad":324,"se":325,"Ġthat":326,"ĠC":327,"ig":328,"Ġfor":329,"ac":330,"Ġy":331,"ver":332,"ur":333,"Ġu":334,"ld":335,"Ġst":336,"ĠM":337,"'s":338,"Ġhe":339,"Ġit":340,"ation":341,"ith":342,"ir":343,"ce":344,"Ġyou":345,"il":346,"ĠB":347,"Ġwh":348,"ol":349,"ĠP":350,"Ġwith":351,"Ġ1":352,"ter":353,"ch":354,"Ġas":355,"Ġwe":356,"Ġ(":357,"nd":358,"ill":359,"ĠD":360,"if":361,"Ġ2":362,"ag":363,"ers":364,"ke":365,"Ġ\"":366,"ĠH":367,"em":368,"Ġcon":369,"ĠW":370,"ĠR":371,"her":372,"Ġwas":373,"Ġr":374,"od":375,"ĠF":376,"ul":377,"ate":378,"Ġat":379,"ri":380,"pp":381,"ore":382,"ĠThe":383,"Ġse":384,"us":385,"Ġpro":386,"Ġha":387,"um":388,"Ġare":389,"Ġde":390,"ain":391,"and":392,"Ġor":393,"igh":394,"est":395,"ist":396,"ab":397,"rom":398,"ĠN":399,"th":400,"Ġcom":401,"ĠG":402,"un":403,"op":404,"00":405,"ĠL":406,"Ġnot":407,"ess":408,"Ġex":409,"Ġv":410,"res":411,"ĠE":412,"ew":413,"ity":414,"ant":415,"Ġby":416,"el":417,"os":418,"ort":419,"oc":420,"qu":421,"Ġfrom":422,"Ġhave":423,"Ġsu":424,"ive":425,"ould":426,"Ġsh":427,"Ġthis":428,"nt":429,"ra":430,"pe":431,"ight":432,"art":433,"ment":434,"Ġal":435,"ust":436,"end":437,"--":438,"all":439,"ĠO":440,"ack":441,"Ġch":442,"Ġle":443,"ies":444,"red":445,"ard":446,"âĢ":447,"out":448,"ĠJ":449,"Ġab":450,"ear":451,"iv":452,"ally":453,"our":454,"ost":455,"gh":456,"pt":457,"Ġpl":458,"ast":459,"Ġcan":460,"ak":461,"ome":462,"ud":463,"The":464,"Ġhis":465,"Ġdo":466,"Ġgo":467,"Ġhas":468,"ge":469,"'t":470,"ĠU":471,"rou":472,"Ġsa":473,"Ġj":474,"Ġbut":475,"Ġwor":476,"Ġall":477,"ect":478,"Ġk":479,"ame":480,"Ġwill":481,"ok":482,"Ġwhe":483,"Ġthey":484,"ide":485,"01":486,"ff":487,"ich":488,"pl":489,"ther":490,"Ġtr":491,"..":492,"Ġint":493,"ie":494,"ure":495,"age":496,"Ġne":497,"ial":498,"ap":499,"ine":500,"ice":501,"Ġme":502,"Ġout":503,"ans":504,"one":505,"ong":506,"ions":507,"Ġwho":508,"ĠK":509,"Ġup":510,"Ġtheir":511,"Ġad":512,"Ġ3":513,"Ġus":514,"ated":515,"ous":516,"Ġmore":517,"ue":518,"og":519,"ĠSt":520,"ind":521,"ike":522,"Ġso":523,"ime":524,"per":525,".\"":526,"ber":527,"iz":528,"act":529,"Ġone":530,"Ġsaid":531,"Ġ-":532,"are":533,"Ġyour":534,"cc":535,"ĠTh":536,"Ġcl":537,"ep":538,"ake":539,"able":540,"ip":541,"Ġcont":542,"Ġwhich":543,"ia":544,"Ġim":545,"Ġabout":546,"Ġwere":547,"very":548,"ub":549,"Ġhad":550,"Ġen":551,"Ġcomp":552,",\"":553,"ĠIn":554,"Ġun":555,"Ġag":556,"ire":557,"ace":558,"au":559,"ary":560,"Ġwould":561,"ass":562,"ry":563,"ĠâĢ":564,"cl":565,"ook":566,"ere":567,"so":568,"ĠV":569,"ign":570,"ib":571,"Ġoff":572,"Ġte":573,"ven":574,"ĠY":575,"ile":576,"ose":577,"ite":578,"orm":579,"Ġ201":580,"Ġres":581,"Ġman":582,"Ġper":583,"Ġother":584,"ord":585,"ult":586,"Ġbeen":587,"Ġlike":588,"ase":589,"ance":590,"ks":591,"ays":592,"own":593,"ence":594,"Ġdis":595,"ction":596,"Ġany":597,"Ġapp":598,"Ġsp":599,"int":600,"ress":601,"ations":602,"ail":603,"Ġ4":604,"ical":605,"Ġthem":606,"Ġher":607,"ount":608,"ĠCh":609,"Ġar":610,"Ġif":611,"Ġthere":612,"Ġpe":613,"Ġyear":614,"av":615,"Ġmy":616,"Ġsome":617,"Ġwhen":618,"ough":619,"ach":620,"Ġthan":621,"ru":622,"ond":623,"ick":624,"Ġover":625,"vel":626,"Ġqu":627,"ĊĊ":628,"Ġsc":629,"reat":630,"ree":631,"ĠIt":632,"ound":633,"port":634,"Ġalso":635,"Ġpart":636,"fter":637,"Ġkn":638,"Ġbec":639,"Ġtime":640,"ens":641,"Ġ5":642,"ople":643,"Ġwhat":644,"Ġno":645,"du":646,"mer":647,"ang":648,"Ġnew":649,"----":650,"Ġget":651,"ory":652,"ition":653,"ings":654,"Ġjust":655,"Ġinto":656,"Ġ0":657,"ents":658,"ove":659,"te":660,"Ġpeople":661,"Ġpre":662,"Ġits":663,"Ġrec":664,"Ġtw":665,"ian":666,"irst":667,"ark":668,"ors":669,"Ġwork":670,"ade":671,"ob":672,"Ġshe":673,"Ġour":674,"wn":675,"ink":676,"lic":677,"Ġ19":678,"ĠHe":679,"ish":680,"nder":681,"ause":682,"Ġhim":683,"ons":684,"Ġ[":685,"Ġro":686,"form":687,"ild":688,"ates":689,"vers":690,"Ġonly":691,"oll":692,"Ġspe":693,"ck":694,"ell":695,"amp":696,"Ġacc":697,"Ġbl":698,"ious":699,"urn":700,"ft":701,"ood":702,"Ġhow":703,"hed":704,"Ġ'":705,"Ġafter":706,"aw":707,"Ġatt":708,"ov":709,"ne":710,"Ġplay":711,"erv":712,"ict":713,"Ġcould":714,"itt":715,"Ġam":716,"Ġfirst":717,"Ġ6":718,"Ġact":719,"Ġ$":720,"ec":721,"hing":722,"ual":723,"ull":724,"Ġcomm":725,"oy":726,"old":727,"ces":728,"ater":729,"Ġfe":730,"Ġbet":731,"we":732,"iff":733,"Ġtwo":734,"ock":735,"Ġback":736,").":737,"ident":738,"Ġunder":739,"rough":740,"sel":741,"xt":742,"Ġmay":743,"round":744,"Ġpo":745,"ph":746,"iss":747,"Ġdes":748,"Ġmost":749,"Ġdid":750,"Ġadd":751,"ject":752,"Ġinc":753,"fore":754,"Ġpol":755,"ont":756,"Ġagain":757,"clud":758,"tern":759,"Ġknow":760,"Ġneed":761,"Ġcons":762,"Ġco":763,"Ġ.":764,"Ġwant":765,"Ġsee":766,"Ġ7":767,"ning":768,"iew":769,"ĠThis":770,"ced":771,"Ġeven":772,"Ġind":773,"ty":774,"ĠWe":775,"ath":776,"Ġthese":777,"Ġpr":778,"Ġuse":779,"Ġbecause":780,"Ġfl":781,"ng":782,"Ġnow":783,"ĠâĢĵ":784,"com":785,"ise":786,"Ġmake":787,"Ġthen":788,"ower":789,"Ġevery":790,"ĠUn":791,"Ġsec":792,"oss":793,"uch":794,"Ġem":795,"Ġ=":796,"ĠRe":797,"ied":798,"rit":799,"Ġinv":800,"lect":801,"Ġsupp":802,"ating":803,"Ġlook":804,"man":805,"pect":806,"Ġ8":807,"row":808,"Ġbu":809,"Ġwhere":810,"ific":811,"Ġyears":812,"ily":813,"Ġdiff":814,"Ġshould":815,"Ġrem":816,"Th":817,"In":818,"Ġev":819,"day":820,"'re":821,"rib":822,"Ġrel":823,"ss":824,"Ġdef":825,"Ġright":826,"Ġsy":827,"),":828,"les":829,"000":830,"hen":831,"Ġthrough":832,"ĠTr":833,"__":834,"Ġway":835,"Ġdon":836,"Ġ,":837,"Ġ10":838,"ased":839,"Ġass":840,"ublic":841,"Ġreg":842,"ĠAnd":843,"ix":844,"Ġvery":845,"Ġinclud":846,"other":847,"Ġimp":848,"oth":849,"Ġsub":850,"ĠâĢĶ":851,"Ġbeing":852,"arg":853,"ĠWh":854,"==":855,"ible":856,"Ġdoes":857,"ange":858,"ram":859,"Ġ9":860,"ert":861,"ps":862,"ited":863,"ational":864,"Ġbr":865,"Ġdown":866,"Ġmany":867,"aking":868,"Ġcall":869,"uring":870,"ities":871,"Ġph":872,"ics":873,"als":874,"Ġdec":875,"ative":876,"ener":877,"Ġbefore":878,"ility":879,"Ġwell":880,"Ġmuch":881,"erson":882,"Ġthose":883,"Ġsuch":884,"Ġke":885,"Ġend":886,"ĠBut":887,"ason":888,"ting":889,"Ġlong":890,"ef":891,"Ġthink":892,"ys":893,"Ġbel":894,"Ġsm":895,"its":896,"ax":897,"Ġown":898,"Ġprov":899,"Ġset":900,"ife":901,"ments":902,"ble":903,"ward":904,"Ġshow":905,"Ġpres":906,"ms":907,"omet":908,"Ġob":909,"Ġsay":910,"ĠSh":911,"ts":912,"ful":913,"Ġeff":914,"Ġgu":915,"Ġinst":916,"und":917,"ren":918,"cess":919,"Ġent":920,"ĠYou":921,"Ġgood":922,"Ġstart":923,"ince":924,"Ġmade":925,"tt":926,"stem":927,"olog":928,"up":929,"Ġ|":930,"ump":931,"Ġhel":932,"vern":933,"ular":934,"ually":935,"Ġac":936,"Ġmon":937,"Ġlast":938,"Ġ200":939,"10":940,"Ġstud":941,"ures":942,"ĠAr":943,"self":944,"ars":945,"meric":946,"ues":947,"cy":948,"Ġmin":949,"ollow":950,"Ġcol":951,"io":952,"Ġmod":953,"Ġcount":954,"ĠCom":955,"hes":956,"Ġfin":957,"air":958,"ier":959,"âĢĶ":960,"read":961,"ank":962,"atch":963,"ever":964,"Ġstr":965,"Ġpoint":966,"ork":967,"ĠNew":968,"Ġsur":969,"ool":970,"alk":971,"ement":972,"Ġused":973,"ract":974,"ween":975,"Ġsame":976,"oun":977,"ĠAl":978,"ci":979,"Ġdiffere":980,"Ġwhile":981,"--------":982,"Ġgame":983,"cept":984,"Ġsim":985,"...":986,"Ġinter":987,"ek":988,"Ġreport":989,"Ġprodu":990,"Ġstill":991,"led":992,"ah":993,"Ġhere":994,"Ġworld":995,"Ġthough":996,"Ġnum":997,"arch":998,"imes":999,"ale":1000,"ĠSe":1001,"ĠIf":1002,"//":1003,"ĠLe":1004,"Ġret":1005,"Ġref":1006,"Ġtrans":1007,"ner":1008,"ution":1009,"ters":1010,"Ġtake":1011,"ĠCl":1012,"Ġconf":1013,"way":1014,"ave":1015,"Ġgoing":1016,"Ġsl":1017,"ug":1018,"ĠAmeric":1019,"Ġspec":1020,"Ġhand":1021,"Ġbetween":1022,"ists":1023,"ĠDe":1024,"oot":1025,"It":1026,"Ġear":1027,"Ġagainst":1028,"Ġhigh":1029,"gan":1030,"az":1031,"ather":1032,"Ġexp":1033,"Ġop":1034,"Ġins":1035,"Ġgr":1036,"Ġhelp":1037,"Ġrequ":1038,"ets":1039,"ins":1040,"ĠPro":1041,"ism":1042,"Ġfound":1043,"land":1044,"ata":1045,"uss":1046,"ames":1047,"Ġperson":1048,"Ġgreat":1049,"pr":1050,"Ġsign":1051,"ĠAn":1052,"'ve":1053,"Ġsomet":1054,"Ġser":1055,"hip":1056,"Ġrun":1057,"Ġ:":1058,"Ġter":1059,"irect":1060,"Ġfollow":1061,"Ġdet":1062,"ices":1063,"Ġfind":1064,"12":1065,"Ġmem":1066,"Ġcr":1067,"ered":1068,"ex":1069,"Ġext":1070,"uth":1071,"ense":1072,"co":1073,"Ġteam":1074,"ving":1075,"ouse":1076,"ash":1077,"att":1078,"ved":1079,"Ġsystem":1080,"ĠAs":1081,"der":1082,"ives":1083,"min":1084,"Ġlead":1085,"ĠBl":1086,"cent":1087,"Ġaround":1088,"Ġgovern":1089,"Ġcur":1090,"velop":1091,"any":1092,"Ġcour":1093,"alth":1094,"ages":1095,"ize":1096,"Ġcar":1097,"ode":1098,"Ġlaw":1099,"Ġread":1100,"'m":1101,"con":1102,"Ġreal":1103,"Ġsupport":1104,"Ġ12":1105,"....":1106,"Ġreally":1107,"ness":1108,"Ġfact":1109,"Ġday":1110,"Ġboth":1111,"ying":1112,"Ġserv":1113,"ĠFor":1114,"Ġthree":1115,"Ġwom":1116,"Ġmed":1117,"ody":1118,"ĠThey":1119,"50":1120,"Ġexper":1121,"ton":1122,"Ġeach":1123,"akes":1124,"Ġche":1125,"Ġcre":1126,"ines":1127,"Ġrep":1128,"19":1129,"gg":1130,"illion":1131,"Ġgrou":1132,"ute":1133,"ik":1134,"We":1135,"get":1136,"ER":1137,"Ġmet":1138,"Ġsays":1139,"ox":1140,"Ġduring":1141,"ern":1142,"ized":1143,"ared":1144,"Ġfam":1145,"ically":1146,"Ġhapp":1147,"ĠIs":1148,"Ġchar":1149,"med":1150,"vent":1151,"Ġgener":1152,"ient":1153,"ple":1154,"iet":1155,"rent":1156,"11":1157,"ves":1158,"ption":1159,"Ġ20":1160,"formation":1161,"Ġcor":1162,"Ġoffic":1163,"ield":1164,"Ġtoo":1165,"ision":1166,"Ġinf":1167,"ĠZ":1168,"the":1169,"oad":1170,"Ġpublic":1171,"Ġprog":1172,"ric":1173,"**":1174,"Ġwar":1175,"Ġpower":1176,"view":1177,"Ġfew":1178,"Ġloc":1179,"Ġdifferent":1180,"Ġstate":1181,"Ġhead":1182,"'ll":1183,"Ġposs":1184,"Ġstat":1185,"ret":1186,"ants":1187,"Ġval":1188,"Ġiss":1189,"Ġcle":1190,"ivers":1191,"anc":1192,"Ġexpl":1193,"Ġanother":1194,"ĠQ":1195,"Ġav":1196,"thing":1197,"nce":1198,"Wh":1199,"Ġchild":1200,"Ġsince":1201,"ired":1202,"less":1203,"Ġlife":1204,"Ġdevelop":1205,"ittle":1206,"Ġdep":1207,"Ġpass":1208,"ãĥ":1209,"Ġturn":1210,"orn":1211,"This":1212,"bers":1213,"ross":1214,"ĠAd":1215,"Ġfr":1216,"Ġresp":1217,"Ġsecond":1218,"oh":1219,"Ġ/":1220,"Ġdisc":1221,"Ġ&":1222,"Ġsomething":1223,"Ġcomple":1224,"Ġed":1225,"Ġfil":1226,"Ġmonth":1227,"aj":1228,"uc":1229,"Ġgovernment":1230,"Ġwithout":1231,"Ġleg":1232,"Ġdist":1233,"Ġput":1234,"Ġquest":1235,"ann":1236,"Ġprot":1237,"20":1238,"Ġnever":1239,"ience":1240,"Ġlevel":1241,"Ġart":1242,"Ġthings":1243,"Ġmight":1244,"Ġeffect":1245,"Ġcontro":1246,"Ġcent":1247,"Ġ18":1248,"Ġallow":1249,"Ġbelie":1250,"chool":1251,"ott":1252,"Ġincre":1253,"Ġfeel":1254,"Ġresult":1255,"Ġlot":1256,"Ġfun":1257,"ote":1258,"Ġty":1259,"erest":1260,"Ġcontin":1261,"Ġusing":1262,"Ġbig":1263,"201":1264,"Ġask":1265,"Ġbest":1266,"Ġ)":1267,"IN":1268,"Ġopp":1269,"30":1270,"Ġnumber":1271,"iness":1272,"St":1273,"lease":1274,"Ġca":1275,"Ġmust":1276,"Ġdirect":1277,"Ġgl":1278,"Ġ<":1279,"Ġopen":1280,"Ġpost":1281,"Ġcome":1282,"Ġseem":1283,"ording":1284,"Ġweek":1285,"ately":1286,"ital":1287,"Ġel":1288,"riend":1289,"Ġfar":1290,"Ġtra":1291,"inal":1292,"Ġpri":1293,"ĠUS":1294,"Ġplace":1295,"Ġform":1296,"Ġtold":1297,"\":":1298,"ains":1299,"ature":1300,"ĠTrump":1301,"Ġstand":1302,"Ġ#":1303,"ider":1304,"ĠFr":1305,"Ġnext":1306,"Ġsoc":1307,"Ġpur":1308,"Ġlet":1309,"Ġlittle":1310,"Ġhum":1311,"Ġi":1312,"ron":1313,"15":1314,"Ġ15":1315,"Ġcommun":1316,"Ġmark":1317,"ĠThere":1318,"Ġwr":1319,"ĠThat":1320,"Ġinformation":1321,"ways":1322,"Ġbus":1323,"app":1324,"Ġinvest":1325,"me":1326,"Ġhard":1327,"ained":1328,"ead":1329,"Ġimport":1330,"Ġappro":1331,"Ġtest":1332,"Ġtri":1333,"Ġrest":1334,"osed":1335,"Ġfull":1336,"Ġcare":1337,"ĠSp":1338,"Ġcase":1339,"ON":1340,"Ġsk":1341,"Ġless":1342,"Ġ+":1343,"Ġpartic":1344,"ĠPl":1345,"ably":1346,"uck":1347,"ished":1348,"chn":1349,"be":1350,"Ġlist":1351,"ator":1352,"Ġtop":1353,"Ġadv":1354,"ĠBe":1355,"ruct":1356,"Ġdem":1357,"ration":1358,"ling":1359,"gy":1360,"reen":1361,"ger":1362,"Ġhome":1363,"Ġleft":1364,"Ġbetter":1365,"Ġdata":1366,"Ġ11":1367,"Ġattack":1368,"Ġproble":1369,"line":1370,"ards":1371,"Ġbeh":1372,"ral":1373,"ĠHow":1374,"ĠShe":1375,"arge":1376,"Ġ--":1377,"://":1378,"Ġbro":1379,"ĠPh":1380,"ats":1381,"Ġbuild":1382,"ww":1383,"ided":1384,"aim":1385,"ases":1386,"ency":1387,"Ġmain":1388,"ined":1389,"Ġincluding":1390,"Ġ{":1391,"Ġgot":1392,"Ġinterest":1393,"Ġkeep":1394,"ĠX":1395,"Ġeas":1396,"aining":1397,"Ġclass":1398,"âĢ¦":1399,"ĠNo":1400,"Ġvar":1401,"Ġsmall":1402,"ample":1403,"AT":1404,"Ġide":1405,"ĠSo":1406,"Ġrece":1407,"Ġpolit":1408,"Ġmov":1409,"Ġplan":1410,"Ġpercent":1411,"iving":1412,"Ġcamp":1413,"Ġpay":1414,"14":1415,"sc":1416,"ised":1417,"Ġunt":1418,"oney":1419,"ploy":1420,"====":1421,"Ġdidn":1422,"ĠInd":1423,"els":1424,"ertain":1425,"Ġpos":1426,"____":1427,"iver":1428,"Ġprocess":1429,"Ġprogram":1430,"ified":1431,"ĠRep":1432,"16":1433,"uro":1434,"ology":1435,"atter":1436,"ina":1437,"Ġname":1438,"ĠAll":1439,"Ġfour":1440,"Ġreturn":1441,"vious":1442,"bs":1443,"Ġcalled":1444,"Ġmove":1445,"ĠSc":1446,"ird":1447,"Ġgroup":1448,"Ġbre":1449,"Ġmen":1450,"Ġcap":1451,"ten":1452,"ee":1453,"Ġdri":1454,"leg":1455,"here":1456,"uthor":1457,"Ġpat":1458,"Ġcurrent":1459,"ides":1460,"Ġpop":1461,"to":1462,"ention":1463,"Ġalways":1464,"Ġmil":1465,"Ġwomen":1466,"Ġ16":1467,"Ġold":1468,"iven":1469,"raph":1470,"ĠOr":1471,"ror":1472,"ently":1473,"Ġnear":1474,"ĠEx":1475,"ream":1476,"sh":1477,"Ġ14":1478,"Ġfree":1479,"ission":1480,"stand":1481,"ĠCon":1482,"ality":1483,"used":1484,"13":1485,"Ġdesign":1486,"Ġchange":1487,"Ġchang":1488,"Ġbo":1489,"Ġvis":1490,"ember":1491,"Ġbook":1492,"ready":1493,"Ġkill":1494,"25":1495,"pped":1496,"Ġaway":1497,"Ġable":1498,"Ġcountry":1499,"Ġconst":1500,"arn":1501,"Ġorder":1502,"AR":1503,"ior":1504,"ium":1505,"orth":1506,"18":1507,"ailable":1508,"Ġsw":1509,"Ġmillion":1510,"Ġ13":1511,"atic":1512,"ted":1513,"ĠGo":1514,"Ġoper":1515,"eng":1516,"Ġthing":1517,"ajor":1518,"conom":1519,"ĠComm":1520,"Ġwhy":1521,"ured":1522,"ural":1523,"Ġschool":1524,"by":1525,"ĠMar":1526,"Ġaff":1527,"Ġdays":1528,"Ġann":1529,"ush":1530,"ane":1531,"If":1532,"eg":1533,"Ġprof":1534,"Ġhealth":1535,"outh":1536,"But":1537,"ional":1538,".,":1539,"Ġsol":1540,"Ġalready":1541,"Ġ30":1542,"Ġcharact":1543,"He":1544,"Ġfriend":1545,"ES":1546,"ians":1547,"icle":1548,"'d":1549,"ĠOn":1550,"Ġleast":1551,"Ġprom":1552,"Ġdr":1553,"Ġhist":1554,"ither":1555,"Ġest":1556,"iqu":1557,"17":1558,"son":1559,"Ġtell":1560,"Ġtalk":1561,"ohn":1562,"oint":1563,"lection":1564,"AN":1565,"Ġuntil":1566,"augh":1567,"Ġlater":1568,"Ġve":1569,"Ġview":1570,"ending":1571,"ived":1572,"Ġword":1573,"ware":1574,"Ġcost":1575,"Ġenough":1576,"Ġgive":1577,"ĠUnited":1578,"Ġtechn":1579,"arent":1580,"OR":1581,"Ġpar":1582,"ĠDr":1583,"Ġ2016":1584,"rist":1585,"ering":1586,"ĠÂ":1587,"Ġlarge":1588,"side":1589,"acy":1590,"ccess":1591,"Ġwin":1592,"Ġimportant":1593,"Ġ199":1594,"Ġdoesn":1595,"Ġ17":1596,"Ġbusiness":1597,"Ġclear":1598,"Ġrese":1599,"\",":1600,"ury":1601,"Ġequ":1602,"aster":1603,"alf":1604,"ĠAmerican":1605,"nect":1606,"Ġexpect":1607,"iversity":1608,"Ġocc":1609,"ĠFl":1610,"Ġkind":1611,"Ġmean":1612,"Ġpast":1613,"Ġdev":1614,"Ġbas":1615,"let":1616,"raft":1617,"Ġorgan":1618,"Ġdel":1619,"Ġperform":1620,"Ġstory":1621,"Ġseason":1622,"ĠCol":1623,"Ġclaim":1624,"Ġcame":1625,"Ġwithin":1626,"Ġline":1627,"Ġproject":1628,"ĠAt":1629,"Ġcontrol":1630,"ended":1631,"ĠSy":1632,"Ġair":1633,"ization":1634,"Ġ*":1635,"ley":1636,"Ġmoney":1637,"idd":1638,"You":1639,"for":1640,"Ġfamily":1641,"Ġmaking":1642,"Ġbit":1643,"Ġpolice":1644,"Ġhappen":1645,"Ġvers":1646,"ony":1647,"uff":1648,"ĠWhen":1649,"Ġsit":1650,"ideo":1651,"lf":1652,"ison":1653,"Ġsure":1654,"gin":1655,"Ġappear":1656,"Ġlight":1657,"Ġes":1658,"of":1659,"Ġwater":1660,"Ġtimes":1661,"not":1662,"Ġgrow":1663,"Ġcompany":1664,"ĠTe":1665,"ows":1666,"Ġmar":1667,"ource":1668,"iol":1669,"arm":1670,"br":1671,"Ġexample":1672,"Ġconc":1673,"Ġfore":1674,"ĠTo":1675,"pro":1676,"EN":1677,"ries":1678,"Ġ25":1679,"ĠCan":1680,"ney":1681,"Ġactually":1682,"Ġever":1683,"urity":1684,"aken":1685,"aps":1686,"Ġtax":1687,"Ġmajor":1688,"ama":1689,"Ġoften":1690,"eral":1691,"Ġhuman":1692,"Ġjob":1693,"ister":1694,"Ġavailable":1695,"ocr":1696,"enn":1697,"aid":1698,"ivid":1699,"Ġrecord":1700,"?\"":1701,"Ġsing":1702,"ĠAm":1703,"idence":1704,"Ġnews":1705,"ster":1706,"Ġeconom":1707,"Ġfollowing":1708,"ĠBr":1709,"ising":1710,"Ġhour":1711,"most":1712,"ument":1713,"Ġsex":1714,"Ġdesc":1715,"Ġbecome":1716,"ĠEd":1717,"Ġtook":1718,"Ġhaving":1719,"Ġproduct":1720,"ault":1721,"As":1722,"aring":1723,"Ġmeans":1724,"Ġhop":1725,"une":1726,"Ġcho":1727,"Ġcertain":1728,"Ġnon":1729,"Ġdeal":1730,"24":1731,"lement":1732,"oci":1733,"ene":1734,"Ġside":1735,"ĠPr":1736,"ĠMay":1737,"Ġreason":1738,"ued":1739,"ched":1740,"ulation":1741,"Ġelect":1742,"Ġofficial":1743,"Ġpossible":1744,"Ġhold":1745,"ands":1746,"ots":1747,"Ġcity":1748,"ories":1749,"Ġsever":1750,"Ġchildren":1751,"Ġonce":1752,"Ġactiv":1753,"ler":1754,"Ġnight":1755,"itions":1756,"ĠJohn":1757,"ape":1758,"play":1759,"Ġdone":1760,"Ġlim":1761,"Ġworking":1762,"ĠPres":1763,"orld":1764,"eb":1765,"ĠCo":1766,"Ġbody":1767,"ails":1768,"utes":1769,"ĠMr":1770,"Ġwhether":1771,"Ġauthor":1772,"rop":1773,"Ġproper":1774,"Ġseen":1775,");":1776,"Ġfac":1777,"ĠSu":1778,"Ġcond":1779,"iting":1780,"Ġcourse":1781,"Ġ}":1782,"----------------":1783,"aign":1784,"Ġevent":1785,"Ġeng":1786,"Ġpot":1787,"Ġintern":1788,"iam":1789,"Ġshort":1790,"empt":1791,"ãĤ":1792,"ĠGod":1793,"ilar":1794,"80":1795,"Ġorig":1796,"IS":1797,"ourn":1798,"ability":1799,"itive":1800,"Ġdam":1801,"Ġ100":1802,"Ġpress":1803,"Ġdoing":1804,"Ġprotect":1805,"ring":1806,"Ġthought":1807,"Ġquestion":1808,"rew":1809,"ĠWar":1810,"Ġseveral":1811,"ĠState":1812,"Ġgiven":1813,"Ġfund":1814,"ĠTw":1815,"Ġwent":1816,"ances":1817,"work":1818,"por":1819,"my":1820,"40":1821,"Ġarg":1822,"artment":1823,"ustom":1824,"Ġpolic":1825,"Ġmeet":1826,"Ġcreat":1827,"22":1828,"ĠStates":1829,"Ġgames":1830,"raw":1831,"uture":1832,"Ġunderstand":1833,"urs":1834,"ĠOb":1835,"lish":1836,"sy":1837,"Ġmakes":1838,"Ġwon":1839,"agon":1840,"Ġhtt":1841,"Ġlove":1842,"ential":1843,"Ġcomplete":1844,"par":1845,"ĠIm":1846,"AL":1847,"Ġaccount":1848,"Âł":1849,"ored":1850,"vert":1851,"Ġident":1852,"Ġ2015":1853,"Ġothers":1854,"ĠMin":1855,"iber":1856,"verage":1857,"There":1858,"itional":1859,"dd":1860,"Ġprob":1861,"Ġyoung":1862,"Ġalong":1863,"Ġaccording":1864,"Ġyet":1865,"Ġmembers":1866,"ĠWhat":1867,"oid":1868,"ĠMan":1869,"And":1870,"Ġamong":1871,"ai":1872,"Ġemploy":1873,"ĠRes":1874,"Ġ>":1875,"Ġinvol":1876,"Ġlow":1877,"af":1878,"ĠCar":1879,"Ġhig":1880,"ĠOne":1881,"ĠSec":1882,"ination":1883,"Ġlikely":1884,"Ġant":1885,"aged":1886,"ĠRuss":1887,"Ġben":1888,"Ġrele":1889,"For":1890,"back":1891,"ĠNot":1892,"Ġpresident":1893,"ball":1894,"Ġaccess":1895,"ividual":1896,"ĠDem":1897,"ĠEuro":1898,"60":1899,"Ġknown":1900,"irl":1901,"ĠGr":1902,"Ġearly":1903,"use":1904,"iety":1905,"âĢĵ":1906,"Ġfight":1907,"Ġsent":1908,"Ġtoday":1909,"Ġmarket":1910,"\".":1911,"Ġbased":1912,"Ġstrong":1913,"urther":1914,"Ġdeb":1915,"mber":1916,"Ġproblem":1917,"Ġdeath":1918,"Ġsocial":1919,"imate":1920,"AS":1921,"ortun":1922,"Ġcampaign":1923,"ery":1924,"Ch":1925,"Ġey":1926,"ially":1927,"Ġmus":1928,"wh":1929,"pos":1930,"Ġer":1931,"Ġsaf":1932,"Ġmonths":1933,"iron":1934,"Ġviol":1935,"Ġfive":1936,"Ġstre":1937,"Ġplayers":1938,"inc":1939,"ald":1940,"year":1941,"aun":1942,"Ġsuccess":1943,"Ġpresent":1944,"erence":1945,"Ġ2014":1946,"Ġsugg":1947,"Ġparticular":1948,"Ġtry":1949,"Ġsuggest":1950,"ĠChrist":1951,"ones":1952,"Ġpriv":1953,"23":1954,"Ġcrit":1955,"Ġland":1956,"Ġlocal":1957,"ify":1958,"29":1959,"Ġaut":1960,"ED":1961,"ĠGu":1962,"Ġmult":1963,"Ġpolitical":1964,"Ġasked":1965,"Ġformer":1966,"itter":1967,"ript":1968,"Ġclose":1969,"Ġpract":1970,"ĠYork":1971,"Ġgetting":1972,"Ġacross":1973,"Ġcomb":1974,"Ġbelieve":1975,"Ġz":1976,"Ġtoget":1977,"Ġtogether":1978,"ĠCent":1979,"irc":1980,"Ġindividual":1981,"ĠMc":1982,"27":1983,"isk":1984,"ĠEng":1985,"Ġface":1986,"Ġ24":1987,"Ġvalue":1988,"Ġarea":1989,"ev":1990,"Ġwrit":1991,"ĠPresident":1992,"Ġvot":1993,"Ġkey":1994,"Ġmom":1995,"put":1996,"Ġanything":1997,"Ġexperience":1998,"attle":1999,"Ġmind":2000,"aff":2001,"omm":2002,"Ġfuture":2003,"ged":2004,"Ġcut":2005,"Ġtot":2006,"itch":2007,"Ġvideo":2008,"Ġinvestig":2009,"Ġnet":2010,"ĠMy":2011,"rict":2012,"ien":2013,".)":2014,"Ġimpro":2015,"though":2016,"wards":2017,"Ġconnect":2018,"ĠMed":2019,"selves":2020,"ensive":2021,"mb":2022,"ober":2023,"ators":2024,"An":2025,"Ġ50":2026,"Ġredu":2027,"resent":2028,"Ġabove":2029,"Ġfre":2030,"ĠEurope":2031,"sw":2032,"Ġamount":2033,"ĠApp":2034,"Ġeither":2035,"Ġmilit":2036,"Ġanal":2037,"Ġfail":2038,"ĠEn":2039,"ales":2040,"Ġspecial":2041,"Ġblack":2042,"IT":2043,"cher":2044,"Ġlooking":2045,"Ġfire":2046,"yn":2047,"Ġalmost":2048,"oon":2049,"Ġstudy":2050,"Ġmiss":2051,"ches":2052,"rown":2053,"Ġtre":2054,"Ġcommunity":2055,"Ġmedia":2056,"Ġfood":2057,"Ġcomes":2058,"ĠUniversity":2059,"Ġsingle":2060,"What":2061,"uly":2062,"Ġhalf":2063,"ague":2064,"hod":2065,"ĠRepublic":2066,"Ġstarted":2067,"Ġquick":2068,"oto":2069,"book":2070,"Ġissue":2071,"itor":2072,"Ġelse":2073,"Ġconsider":2074,"26":2075,"rodu":2076,"Ġtaken":2077,"28":2078,"99":2079,"ĠWith":2080,"Ġtrue":2081,"Ġwa":2082,"Ġtrad":2083,"Ġago":2084,"Ġmess":2085,"ief":2086,"Ġadded":2087,"oke":2088,"Ġbad":2089,"Ġfav":2090,"33":2091,"Ġsimilar":2092,"ask":2093,"ĠDon":2094,"Ġcharacter":2095,"orts":2096,"ĠHouse":2097,"Ġreported":2098,"Ġtype":2099,"val":2100,"iod":2101,"ĠHowever":2102,"Ġtarg":2103,"Ġentire":2104,"pping":2105,"Ġhistory":2106,"Ġlive":2107,"ffic":2108,"........":2109,"ederal":2110,"Ġtrying":2111,"Ġdiscuss":2112,"ĠHar":2113,"aces":2114,"lished":2115,"Ġself":2116,"osp":2117,"rest":2118,"Ġroom":2119,"elt":2120,"Ġfall":2121,"olution":2122,"Ġet":2123,"Ġx":2124,"Ġisn":2125,"Ġidea":2126,"bo":2127,"Ġsound":2128,"ĠDep":2129,"Ġsomeone":2130,"cially":2131,"ully":2132,"Ġfoc":2133,"Ġobject":2134,"ift":2135,"aper":2136,"Ġplayer":2137,"Ġrather":2138,"Ġservice":2139,"ashing":2140,"ĠDo":2141,"ĠPart":2142,"rug":2143,"mon":2144,"ply":2145,"Ġmor":2146,"Ġnothing":2147,"Ġprovide":2148,"IC":2149,"ung":2150,"Ġparty":2151,"Ġexist":2152,"Ġmag":2153,"70":2154,"Ġrul":2155,"Ġhouse":2156,"Ġbehind":2157,"Ġhowever":2158,"ĠWorld":2159,"Ġsum":2160,"Ġapplic":2161,"Ġ;":2162,"Ġfunction":2163,"gr":2164,"ĠPol":2165,"Ġfront":2166,"200":2167,"Ġseries":2168,"Ġtem":2169,"Ġtyp":2170,"ills":2171,"Ġopt":2172,"Ġpoints":2173,"Ġbelow":2174,"itted":2175,"Ġspecific":2176,"Ġ2017":2177,"umb":2178,"Ġra":2179,"Ġprevious":2180,"Ġpret":2181,"reme":2182,"Ġcustom":2183,"Ġcourt":2184,"ĠMe":2185,"Ġrepl":2186,"Ġwhole":2187,"go":2188,"cer":2189,"Ġtreat":2190,"ĠAct":2191,"Ġprobably":2192,"Ġlearn":2193,"ender":2194,"ĠAss":2195,"Ġversion":2196,"now":2197,"Ġcheck":2198,"ĠCal":2199,"RE":2200,"minist":2201,"On":2202,"ources":2203,"Ġbenef":2204,"Ġdoc":2205,"Ġdeter":2206,"Ġenc":2207,"Ġsuper":2208,"Ġaddress":2209,"Ġvict":2210,"Ġ2013":2211,"Ġmeas":2212,"tr":2213,"Ġfield":2214,"When":2215,"Ġsignific":2216,"uge":2217,"Ġfeat":2218,"Ġcommon":2219,"load":2220,"Ġbegin":2221,"Ġbring":2222,"Ġaction":2223,"erman":2224,"Ġdescrib":2225,"Ġindust":2226,"Ġwanted":2227,"ried":2228,"ming":2229,"Ġattempt":2230,"45":2231,"fer":2232,"Ġdue":2233,"ression":2234,"##":2235,"Ġshall":2236,"Ġsix":2237,"oo":2238,"Ġstep":2239,"Ġpub":2240,"Ġhimself":2241,"Ġ23":2242,"Ġcop":2243,"Ġdest":2244,"Ġstop":2245,"AC":2246,"ibility":2247,"Ġlab":2248,"icult":2249,"Ġhours":2250,"Ġcreate":2251,"Ġfurther":2252,"ĠAmerica":2253,"ĠCity":2254,"Ġdou":2255,"head":2256,"ST":2257,"ĠNorth":2258,"cing":2259,"Ġnational":2260,"ule":2261,"ĠInst":2262,"Ġtaking":2263,"ĠQu":2264,"irt":2265,"Ġred":2266,"Ġresearch":2267,"viron":2268,"ĠGe":2269,"Ġbreak":2270,"ana":2271,"Ġspace":2272,"aterial":2273,"Ġrecent":2274,"ĠAb":2275,"Ġgeneral":2276,"Ġhit":2277,"Ġperiod":2278,"Ġeverything":2279,"ively":2280,"Ġphys":2281,"Ġsaying":2282,"anks":2283,"Ġcou":2284,"Ġcult":2285,"aced":2286,"eal":2287,"uation":2288,"Ġcoun":2289,"lu":2290,"Ġinclude":2291,"Ġposition":2292,"ĠAfter":2293,"ĠCanad":2294,"ĠEm":2295,"Ġimm":2296,"ĠRed":2297,"Ġpick":2298,"Ġcompl":2299,"Ġmatter":2300,"reg":2301,"ext":2302,"angu":2303,"isc":2304,"ole":2305,"aut":2306,"Ġcompet":2307,"eed":2308,"fect":2309,"Ġ21":2310,"ĠSen":2311,"ĠThese":2312,"asing":2313,"Ġcannot":2314,"Ġinit":2315,"Ġrelations":2316,"ached":2317,"Ġbar":2318,"Ġ40":2319,"ĠTH":2320,"Ġ2012":2321,"Ġvol":2322,"Ġground":2323,"Ġsecurity":2324,"Ġupd":2325,"ilt":2326,"35":2327,"Ġconcern":2328,"ĠJust":2329,"Ġwhite":2330,"Ġseems":2331,"ĠHer":2332,"pecially":2333,"ients":2334,"Ġannoun":2335,"Ġfig":2336,"ights":2337,"Ġstri":2338,"like":2339,"ids":2340,"Ġsus":2341,"Ġwatch":2342,"Ġâ":2343,"Ġwind":2344,"ĠCont":2345,"Ġitself":2346,"Ġmass":2347,"Al":2348,"yle":2349,"ique":2350,"ĠNational":2351,"Ġabs":2352,"Ġpack":2353,"Ġoutside":2354,"Ġanim":2355,"Ġpain":2356,"eter":2357,"Ġmanag":2358,"duct":2359,"ogn":2360,"Ġ]":2361,"ĠSept":2362,"sec":2363,"off":2364,"ĠJan":2365,"Ġfoot":2366,"ades":2367,"Ġthird":2368,"Ġmot":2369,"Ġevidence":2370,"inton":2371,"Ġthreat":2372,"apt":2373,"ples":2374,"cle":2375,"Ġlo":2376,"Ġdecl":2377,"Ġitem":2378,"medi":2379,"Ġrepresent":2380,"omb":2381,"amer":2382,"Ġsignificant":2383,"ograph":2384,"su":2385,"Ġcal":2386,"ires":2387,"0000":2388,"ID":2389,"AM":2390,"Ġsimply":2391,"Ġlonger":2392,"Ġfile":2393,"OT":2394,"che":2395,"So":2396,"ateg":2397,"org":2398,"ĠHis":2399,"Ġener":2400,"Ġdom":2401,"Ġupon":2402,"ili":2403,"\":\"":2404,"Ġthemselves":2405,"Ġcoming":2406,"Ġquite":2407,"Ġdifficult":2408,"ĠBar":2409,"ilities":2410,"rel":2411,"ends":2412,"cial":2413,"64":2414,"Ġwoman":2415,"rap":2416,"yr":2417,"Ġnecess":2418,"ips":2419,"Ġtext":2420,"Ġrequire":2421,"Ġmilitary":2422,"Ġreview":2423,"Ġrespons":2424,"75":2425,"Ġsubject":2426,"Ġinstead":2427,"Ġissues":2428,"Ġgen":2429,"\",\"":2430,"Ġminutes":2431,"Ġweap":2432,"ray":2433,"amed":2434,"time":2435,"bl":2436,"How":2437,"Ġcode":2438,"ĠSm":2439,"Ġhigher":2440,"ĠSte":2441,"ris":2442,"Ġpage":2443,"Ġstudents":2444,"ĠIntern":2445,"Ġmethod":2446,"ĠAug":2447,"ĠPer":2448,"ĠAg":2449,"Ġpolicy":2450,"ĠSw":2451,"Ġexec":2452,"Ġaccept":2453,"ume":2454,"ribut":2455,"Ġwords":2456,"Ġfinal":2457,"Ġchanges":2458,"ĠDemocr":2459,"Ġfriends":2460,"Ġrespect":2461,"Ġep":2462,"Ġcompan":2463,"ivil":2464,"Ġdamage":2465,"****":2466,"ogle":2467,"vironment":2468,"Ġneg":2469,"ental":2470,"Ġap":2471,"Ġtotal":2472,"ival":2473,"!\"":2474,"lim":2475,"Ġneeds":2476,"Ġagre":2477,"Ġdevelopment":2478,"Ġage":2479,"iple":2480,"21":2481,"Ġresults":2482,"ĠAf":2483,"Sh":2484,"Ġgun":2485,"ĠObama":2486,"roll":2487,"Ġ@":2488,"Ġrights":2489,"ĠBrit":2490,"Ġrunning":2491,"Ġwasn":2492,"Ġport":2493,"Ġrate":2494,"Ġpretty":2495,"Ġtarget":2496,"Ġsaw":2497,"Ġcirc":2498,"Ġworks":2499,"icro":2500,"alt":2501,"over":2502,"www":2503,"That":2504,"lier":2505,"Ġeveryone":2506,"ude":2507,"Ġpie":2508,"iddle":2509,"rael":2510,"Ġrad":2511,"Ġblock":2512,"Ġwalk":2513,"To":2514,"ãģ":2515,"nes":2516,"ĠAust":2517,"aul":2518,"rote":2519,"ĠSouth":2520,"ession":2521,"oph":2522,"Ġshows":2523,"Ġsite":2524,"Ġjo":2525,"Ġrisk":2526,"clus":2527,"lt":2528,"Ġinj":2529,"iding":2530,"ĠSpe":2531,"Ġchall":2532,"irm":2533,"Ġ22":2534,"itting":2535,"str":2536,"Ġhy":2537,"LE":2538,"key":2539,"Ġbegan":2540,"atur":2541,"ashington":2542,"lam":2543,"ĠDav":2544,"bit":2545,"Ġsize":2546,"ĠPar":2547,"38":2548,"ournal":2549,"face":2550,"Ġdecision":2551,"Ġlarg":2552,"Ġjud":2553,"rect":2554,"Ġcontinue":2555,"ĠOct":2556,"overed":2557,"ĠInt":2558,"========":2559,"Ġparent":2560,"ĠWill":2561,"Ġeasy":2562,"Ġdrug":2563,"anger":2564,"Ġsense":2565,"Ġdi":2566,"iday":2567,"Ġenergy":2568,"istic":2569,"Ġassoci":2570,"arter":2571,"obal":2572,"eks":2573,"ĠEl":2574,"urch":2575,"Ġgirl":2576,"oe":2577,"itle":2578,"Ġ28":2579,"ĠChe":2580,"Ġrequest":2581,"Ġsoon":2582,"Ġhost":2583,"ky":2584,"Ġstates":2585,"omes":2586,"Ġmaterial":2587,"lex":2588,"Ġmoment":2589,"Ġansw":2590,"onse":2591,"Ġespecially":2592,"Ġnorm":2593,"Ġservices":2594,"pite":2595,"ran":2596,"Ġrole":2597,"44":2598,"):":2599,"Ġcred":2600,"Cl":2601,"________":2602,"Ġmat":2603,"Ġlog":2604,"ĠClinton":2605,"OU":2606,"Ġoffice":2607,"Ġ26":2608,"Ġcharg":2609,"Ġtrack":2610,"ma":2611,"Ġheart":2612,"Ġball":2613,"Ġpersonal":2614,"Ġbuilding":2615,"na":2616,"set":2617,"body":2618,"ĠBlack":2619,"Ġincrease":2620,"itten":2621,"Ġneeded":2622,"36":2623,"32":2624,"=\"":2625,"Ġlost":2626,"Ġbecame":2627,"Ġgroups":2628,"ĠMus":2629,"Ġwrote":2630,"ĠPe":2631,"Ġprop":2632,"joy":2633,"é":2634,"ĠWhite":2635,"Ġdead":2636,".'":2637,"Ġhttp":2638,"Ġwebs":2639,"OS":2640,"Ġinside":2641,"Ġwrong":2642,"Ġstatement":2643,"Ġ...":2644,"yl":2645,"Ġfilm":2646,"Ġmusic":2647,"Ġshare":2648,"ification":2649,"Ġrelease":2650,"Ġforward":2651,"Ġstay":2652,"Ġcomput":2653,"itte":2654,"ser":2655,"Ġoriginal":2656,"Ġcard":2657,"Ġcand":2658,"Ġdiv":2659,"atural":2660,"Ġfavor":2661,"OM":2662,"Ġcases":2663,"uses":2664,"Ġsection":2665,"Ġleave":2666,"ging":2667,"oved":2668,"ĠWashington":2669,"39":2670,"ĠGl":2671,"Ġrequired":2672,"action":2673,"apan":2674,"oor":2675,"iter":2676,"ĠKing":2677,"Ġcountries":2678,"ĠGerman":2679,"lling":2680,"Ġ27":2681,"34":2682,"Ġquestions":2683,"Ġprim":2684,"Ġcell":2685,"Ġshoot":2686,"Ġanyone":2687,"ĠWest":2688,"Ġaffect":2689,"epend":2690,"Ġonline":2691,"ĠIsrael":2692,"ĠSeptember":2693,"Ġability":2694,"Ġcontent":2695,"ises":2696,"Ġreve":2697,"Ġlaun":2698,"Ġindic":2699,"Ġforce":2700,"cast":2701,"Ġsold":2702,"aving":2703,"fl":2704,"Ġsoft":2705,"Ġcompanies":2706,"ceed":2707,"Ġarticle":2708,"Ġaud":2709,"Ġrev":2710,"Ġeduc":2711,"Ġplaying":2712,"05":2713,"Ġheld":2714,"ctor":2715,"Ġreleased":2716,"Ġfederal":2717,"37":2718,"Ġadminist":2719,"Ġinterview":2720,"Ġinstall":2721,"Ġreceived":2722,"Ġsource":2723,"uk":2724,"Ph":2725,"Ġserious":2726,"Ġcreated":2727,"Ġcause":2728,"Ġimmedi":2729,"Ġdefin":2730,"uel":2731,"ĠDepartment":2732,"ctions":2733,"ĠCour":2734,"ĠNow":2735,"ze":2736,"ites":2737,"itution":2738,"Ġlate":2739,"Ġspeak":2740,"ners":2741,"Ġlegal":2742,"ari":2743,"ĠCor":2744,"Ġweeks":2745,"Ġmodel":2746,"Ġpred":2747,"Ġexact":2748,"BC":2749,"ĠBy":2750,"ING":2751,"osing":2752,"Ġtakes":2753,"Ġregard":2754,"Ġopportun":2755,"Ġprice":2756,"Ġ198":2757,"ĠApr":2758,"fully":2759,"Ġord":2760,"Ġproblems":2761,"ruction":2762,"ham":2763,"ĠCount":2764,"lege":2765,"Ġleaders":2766,"ET":2767,"lev":2768,"Ġdeep":2769,"ological":2770,"ese":2771,"haps":2772,"ĠSome":2773,"Ġpers":2774,"Ġcontract":2775,"Ġrelationship":2776,"sp":2777,"oud":2778,"Ġbase":2779,"48":2780,"mit":2781,"Ad":2782,"ancial":2783,"Ġconsum":2784,"Ġpotential":2785,"Ġlangu":2786,"rem":2787,"eth":2788,"Ġrelig":2789,"ressed":2790,"66":2791,"Ġlink":2792,"Ġlower":2793,"ayer":2794,"ĠJune":2795,"Ġfem":2796,"unt":2797,"erc":2798,"urd":2799,"Ġcontact":2800,"Ġill":2801,"Ġmother":2802,"Ġestab":2803,"htt":2804,"ĠMarch":2805,"ĠBro":2806,"ĠChina":2807,"Ġ29":2808,"Ġsqu":2809,"Ġprovided":2810,"Ġaverage":2811,"asons":2812,"Ġ2011":2813,"Ġexam":2814,"lin":2815,"55":2816,"ned":2817,"Ġperfect":2818,"Ġtou":2819,"alse":2820,"ux":2821,"Ġbuy":2822,"Ġshot":2823,"Ġcollect":2824,"Ġphot":2825,"Ġplayed":2826,"Ġsurpr":2827,"Ġofficials":2828,"Ġsimple":2829,"avy":2830,"Ġindustry":2831,"Ġhands":2832,"ground":2833,"Ġpull":2834,"Ġround":2835,"Ġuser":2836,"Ġrange":2837,"uary":2838,"Ġprivate":2839,"ops":2840,"ees":2841,"Ġways":2842,"ĠMich":2843,"Ġveh":2844,"Ġexcept":2845,"Ġterms":2846,"imum":2847,"pper":2848,"ION":2849,"ores":2850,"ĠDragon":2851,"oul":2852,"Ġden":2853,"Ġperformance":2854,"Ġbill":2855,"cil":2856,"47":2857,"Ġenvironment":2858,"Ġexc":2859,"add":2860,"Ġworth":2861,"Ġpict":2862,"Ġchance":2863,"Ġ2018":2864,"bor":2865,"Ġspeed":2866,"iction":2867,"Ġalleg":2868,"ĠJapan":2869,"atory":2870,"reet":2871,"Ġmatch":2872,"ĠII":2873,"Ġstru":2874,"order":2875,"Ġste":2876,"Ġliving":2877,"Ġstruct":2878,"ino":2879,"Ġsepar":2880,"hern":2881,"Ġresponse":2882,"Ġenjoy":2883,"Ġvia":2884,"AD":2885,"uments":2886,"acebook":2887,"Ġmember":2888,"ibr":2889,"izing":2890,"Ġtool":2891,"ĠMon":2892,"ĠWhile":2893,"hood":2894,"ĠAng":2895,"ĠDef":2896,"Ġoffer":2897,"Tr":2898,"aur":2899,"Ġturned":2900,"ĠJuly":2901,"down":2902,"anced":2903,"Ġrecently":2904,"ĠEar":2905,"Ġce":2906,"ĠStar":2907,"ĠCong":2908,"rought":2909,"Ġblood":2910,"Ġhope":2911,"Ġcomment":2912,"aint":2913,"Ġarri":2914,"iles":2915,"Ġparticip":2916,"ought":2917,"ription":2918,"08":2919,"49":2920,"Ġgave":2921,"Ġselect":2922,"Ġkilled":2923,"sych":2924,"Ġgoes":2925,"ij":2926,"Ġcoll":2927,"Ġimpact":2928,"atives":2929,"ĠSer":2930,"09":2931,"ĠAugust":2932,"Ġboy":2933,"de":2934,"ĠDes":2935,"Ġfelt":2936,"US":2937,"Ġexpected":2938,"Ġimage":2939,"ĠMark":2940,"ccording":2941,"oice":2942,"EC":2943,"ĠMag":2944,"ened":2945,"hold":2946,"ĠPost":2947,"Ġprevent":2948,"No":2949,"Ġinvolved":2950,"Ġeyes":2951,"Ġquickly":2952,"At":2953,"unk":2954,"Ġbehav":2955,"Ġur":2956,"Ġled":2957,"come":2958,"ey":2959,"Ġcandid":2960,"Ġearlier":2961,"Ġfocus":2962,"ety":2963,"Pro":2964,"ledge":2965,"ixed":2966,"illed":2967,"Ġpopular":2968,"AP":2969,"Ġsett":2970,"light":2971,"Ġvarious":2972,"inks":2973,"Ġlevels":2974,"Ġroad":2975,"ellig":2976,"ables":2977,"hel":2978,"ittee":2979,"ĠGener":2980,"ype":2981,"Ġheard":2982,"icles":2983,"Ġmis":2984,"Ġusers":2985,"ĠSan":2986,"Ġimprove":2987,"Ġfather":2988,"Ġsearch":2989,"They":2990,"vil":2991,"Ġprofess":2992,"Ġknew":2993,"Ġloss":2994,"Ġevents":2995,"65":2996,"Ġbillion":2997,"07":2998,"02":2999,"ĠNews":3000,"ĠAM":3001,"Ġcover":3002,"where":3003,"ension":3004,"Ġbott":3005,"Ġareas":3006,"ences":3007,"ope":3008,"ĠTwitter":3009,"ael":3010,"Ġgets":3011,"ĠGoogle":3012,"Ġsn":3013,"iant":3014,"Ġvote":3015,"Ġnearly":3016,"Ġincluded":3017,"Ġrecogn":3018,"zz":3019,"mm":3020,"aled":3021,"Ġhappened":3022,"04":3023,"Ġhot":3024,"Ġwhose":3025,"Ġcivil":3026,"Ġsuff":3027,"oes":3028,"itiz":3029,"ĠSyri":3030,"Ġrespond":3031,"Ġhon":3032,"Ġfeatures":3033,"Ġeconomic":3034,"ĠApril":3035,"rim":3036,"Ġtechnology":3037,"Ġoption":3038,"aging":3039,"Ġpurch":3040,"Re":3041,"Ġlat":3042,"chie":3043,"isl":3044,"Ġrecomm":3045,"uf":3046,"Ġtraining":3047,"Ġeffects":3048,"Ġfast":3049,"Ġ2010":3050,"Ġoccur":3051,"Ġwebsite":3052,"Ġemail":3053,"Ġsens":3054,"ech":3055,"Ġoil":3056,"Ġinflu":3057,"Ġcurrently":3058,"ĠSch":3059,"ĠAdd":3060,"Ġgoal":3061,"Ġscient":3062,"Ġconv":3063,"100":3064,"emy":3065,"Ġdecided":3066,"Ġtravel":3067,"Ġmention":3068,"LL":3069,"03":3070,"Ġelection":3071,"Ġphone":3072,"Ġlooks":3073,"Ġsituation":3074,"Ġcy":3075,"Ġhor":3076,"bed":3077,"ĠCourt":3078,"aily":3079,"aves":3080,"Ġquality":3081,"ĠComp":3082,"wise":3083,"Ġtable":3084,"Ġstaff":3085,"ĠWind":3086,"ett":3087,"Ġtried":3088,"idered":3089,"Ġaddition":3090,"Ġbox":3091,"Ġlack":3092,"arily":3093,"Ġwide":3094,"Ġmid":3095,"Ġboard":3096,"ysis":3097,"Ġanti":3098,"ha":3099,"Ġdig":3100,"ening":3101,"Ġdro":3102,"Con":3103,"68":3104,"Ġslow":3105,"based":3106,"sequ":3107,"Ġpath":3108,"Ex":3109,"aker":3110,"Ġworked":3111,"Ġpen":3112,"Ġengine":3113,"Ġlooked":3114,"ĠSuper":3115,"ĠServ":3116,"Ġvictim":3117,"Un":3118,"Ġproperty":3119,"Ġintrodu":3120,"Ġexecut":3121,"ĠPM":3122,"Le":3123,"Ġcolor":3124,"ĠMore":3125,"Ġ60":3126,"Ġnetwork":3127,"Ġdate":3128,"cul":3129,"idge":3130,"Ġextra":3131,"31":3132,"Ġsle":3133,"67":3134,"Ġwond":3135,"Ġreports":3136,"just":3137,"ĠAustral":3138,"Ġcapital":3139,"Ġens":3140,"Ġcommand":3141,"Ġallowed":3142,"Ġprep":3143,"Ġcapt":3144,"hib":3145,"Ġnumbers":3146,"chan":3147,"Ġfair":3148,"mp":3149,"oms":3150,"Ġreach":3151,"With":3152,"tain":3153,"Ġbroad":3154,"Ġcouple":3155,"ecause":3156,"lying":3157,"ĠFeb":3158,"Ġscreen":3159,"Ġlives":3160,"Ġprior":3161,"ĠCongress":3162,"Ar":3163,"Ġapproach":3164,"Ġemer":3165,"aries":3166,"ĠDis":3167,"serv":3168,"ĠNe":3169,"Ġbuilt":3170,"cies":3171,"Ġrepe":3172,"Ġrules":3173,"force":3174,"ĠPal":3175,"Ġfinancial":3176,"Ġconsidered":3177,"ĠChar":3178,"nces":3179,"ĠIS":3180,"Ġbrought":3181,"Ġbi":3182,"iers":3183,"ĠSim":3184,"OP":3185,"Ġproducts":3186,"Ġvisit":3187,"Ġdocument":3188,"Ġconduct":3189,"Ġcompletely":3190,"ining":3191,"ĠCalif":3192,"ibly":3193,"Ġwritten":3194,"ĠTV":3195,"ements":3196,"Ġdraw":3197,"One":3198,"Ġpublished":3199,"Ġsecret":3200,"rain":3201,"het":3202,"ĠFacebook":3203,"onday":3204,"ĠUp":3205,"Ġsexual":3206,"Ġthous":3207,"ĠPat":3208,"Ġess":3209,"Ġstandard":3210,"Ġarm":3211,"ges":3212,"ection":3213,"Ġfell":3214,"Ġforeign":3215,"ani":3216,"ĠFriday":3217,"Ġregular":3218,"inary":3219,"Ġincreased":3220,"Ġusually":3221,"Ġdemon":3222,"Ġdark":3223,"Ġadditional":3224,"rol":3225,"ĠOf":3226,"Ġproduction":3227,"!!":3228,"undred":3229,"Ġinternational":3230,"idents":3231,"ĠFree":3232,"roup":3233,"Ġrace":3234,"Ġmach":3235,"Ġhuge":3236,"All":3237,"lear":3238,"ovember":3239,"Ġtown":3240,"Ġattention":3241,"ĠOff":3242,"yond":3243,"ĠThen":3244,"field":3245,"Ġterror":3246,"raz":3247,"ĠBo":3248,"Ġmeeting":3249,"ĠPark":3250,"Ġarrest":3251,"Ġfear":3252,"Ġaw":3253,"ĠVal":3254,"oring":3255,"',":3256,"Ġextreme":3257,"arr":3258,"Ġworkers":3259,"After":3260,"Ġ31":3261,"net":3262,"ament":3263,"Ġdirectly":3264,"Ġpopulation":3265,"ube":3266,"ĠOctober":3267,"ĠIN":3268,"ĠJanuary":3269,"59":3270,"ĠDavid":3271,"Ġcross":3272,"cember":3273,"ĠFirst":3274,"Ġmessage":3275,"irit":3276,"Ġnation":3277,"Ġpoll":3278,"isions":3279,"Ġanswer":3280,"ny":3281,"isode":3282,"Ġcarry":3283,"ĠRussia":3284,"Ġhear":3285,"ength":3286,"roy":3287,"Ġnatural":3288,"inally":3289,"Ġdog":3290,"mitted":3291,"Ġtrade":3292,"Ġsubst":3293,"Ġmultiple":3294,"ĠAfric":3295,"Ġfans":3296,"Ġsort":3297,"Ġglobal":3298,"ication":3299,"ĠWed":3300,"ara":3301,"Ġachie":3302,"Ġlanguage":3303,"vey":3304,"Ġtal":3305,"Ġnecessary":3306,"Ġdetails":3307,"Ġsen":3308,"ĠSund":3309,"ĠReg":3310,"ĠRec":3311,"06":3312,"Ġsil":3313,"ressive":3314,"Ġmedical":3315,"unch":3316,"ornia":3317,"Ġund":3318,"fort":3319,"ocks":3320,"ĠMonday":3321,"uesday":3322,"craft":3323,"77":3324,"urt":3325,"Ġver":3326,"ĠHill":3327,"Ġreceive":3328,"Ġmorning":3329,"estern":3330,"Ġbank":3331,"Ġsat":3332,"irth":3333,"ĠHigh":3334,"Ġdevice":3335,"ĠTHE":3336,"ĠCenter":3337,"Ġsafe":3338,"Ġple":3339,"ĠCanada":3340,"Ġsystems":3341,"Ġassist":3342,"Ġsurv":3343,"Ġbattle":3344,"ĠSoc":3345,"vertis":3346,"She":3347,"Ġpaper":3348,"Ġgrowth":3349,"Ġcast":3350,"Sc":3351,"Ġplans":3352,"lled":3353,"Ġparts":3354,"Ġwall":3355,"Ġmovement":3356,"Ġpractice":3357,"imately":3358,"Ġdisplay":3359,"Ġsometimes":3360,"omp":3361,"ĠPaul":3362,"ĠYes":3363,"king":3364,"58":3365,"oly":3366,"Ġson":3367,"Ġavoid":3368,"okes":3369,"ĠJew":3370,"Ġtowards":3371,"asc":3372,"Ġ//":3373,"ĠKore":3374,"Ġtalking":3375,"Ġcorrect":3376,"Ġspent":3377,"icks":3378,"iable":3379,"eared":3380,"Ġterm":3381,"Ġwants":3382,"oming":3383,"Ġut":3384,"Ġdoub":3385,"Ġforces":3386,"Ġplease":3387,"69":3388,"ĠNovember":3389,"atform":3390,"ondon":3391,"Ġones":3392,"Ġimmediately":3393,"ĠRussian":3394,"ĠMet":3395,"Ġdeg":3396,"Ġparents":3397,"CH":3398,"ĠAmericans":3399,"aly":3400,"ĠMod":3401,"Ġshown":3402,"Ġconditions":3403,"Ġstuff":3404,"Ġreb":3405,"ĠYour":3406,"Ġincludes":3407,"nown":3408,"ĠSam":3409,"Ġexperien":3410,"mission":3411,"ĠEven":3412,"aught":3413,"Ġannounced":3414,"ĠRepublican":3415,"Ġdetermin":3416,"Ġdescribed":3417,"ĠCounty":3418,"()":3419,"Ġdoor":3420,"Ġchanged":3421,"Ġneigh":3422,"ĠHere":3423,"Ġclean":3424,"Ġpan":3425,"ĠDecember":3426,"ĠEuropean":3427,"iring":3428,"apter":3429,"Ġclub":3430,"ĠTuesday":3431,"Ġpaid":3432,"ĠNet":3433,"Ġattacks":3434,"Ġcharacters":3435,"Ġalone":3436,"Ġdirector":3437,"dom":3438,"Ġ35":3439,"Ġload":3440,"Ġrout":3441,"ĠCalifornia":3442,"Ġfinally":3443,"Ġrac":3444,"Ġcontr":3445,"Ġexactly":3446,"resh":3447,"pri":3448,"ĠIslam":3449,"Ġnature":3450,"Ġcareer":3451,"Ġlatest":3452,"Ġconvers":3453,"ĠSl":3454,"pose":3455,"cient":3456,"ĠInc":3457,"ivity":3458,"88":3459,"ĠAtt":3460,"ĠMor":3461,"nesday":3462,"Ġweight":3463,"ken":3464,"Ġnote":3465,"Ġteams":3466,"Ġ\\":3467,"airs":3468,"ĠGreen":3469,"Ġhundred":3470,"onent":3471,"Ġstreng":3472,"Ġconsist":3473,"icated":3474,"Ġregul":3475,"Ġlic":3476,"astic":3477,"Ġten":3478,"ursday":3479,"elligence":3480,"ously":3481,"ĠUK":3482,"BI":3483,"Ġcosts":3484,"Ġindepend":3485,"ĠAP":3486,"Ġnormal":3487,"Ġhom":3488,"Ġobvious":3489,"Ġswe":3490,"Ġstar":3491,"Ġready":3492,"acher":3493,"Ġimplement":3494,"gest":3495,"Ġsong":3496,"ĠGet":3497,"ĠLab":3498,"Ġinteresting":3499,"using":3500,"Ġgiving":3501,"ĠSunday":3502,"Ġetc":3503,"Ġmiddle":3504,"Ġremember":3505,"right":3506,"osition":3507,"utions":3508,"Ġmax":3509,"46":3510,"Ġyourself":3511,"Ġdemand":3512,"Ġtreatment":3513,"Ġdanger":3514,"ĠCons":3515,"Ġguy":3516,"ĠBritish":3517,"Ġphysical":3518,"Ġrelated":3519,"Ġremain":3520,"Ġcouldn":3521,"Ġrefer":3522,"Ġcitiz":3523,"box":3524,"ENT":3525,"board":3526,"Ġinn":3527,"IG":3528,"ero":3529,"ĠStreet":3530,"ospital":3531,"rench":3532,"chers":3533,"Ġstra":3534,"OL":3535,"ager":3536,"ĠAN":3537,"Ġeasily":3538,"IA":3539,"enge":3540,"iny":3541,"Ġclos":3542,"ocked":3543,"Ġuses":3544,"ĠCoun":3545,"Im":3546,"uild":3547,"??":3548,"more":3549,"Ġang":3550,"Ġwrite":3551,"olute":3552,"57":3553,"Ġleader":3554,"Ġreading":3555,"":3784,"Ġfigure":3785,"Ġdisapp":3786,"enty":3787,"Ġsoftware":3788,"Ġult":3789,"Ġofficers":3790,"New":3791,"Is":3792,"Ġremains":3793,"ĠIndia":3794,"Ġpsych":3795,"rief":3796,"Ġcat":3797,"esc":3798,"Ġobserv":3799,"Ġstage":3800,"ĠDark":3801,"Ġenter":3802,"change":3803,"Ġpassed":3804,"Ġdespite":3805,"ĠOut":3806,"Ġmovie":3807,"rs":3808,"Ġvoice":3809,"mine":3810,"ĠPlay":3811,"Ġtoward":3812,"ĠTer":3813,"Ġregion":3814,"Ġvalues":3815,"orters":3816,"Ġmount":3817,"Ġofficer":3818,"ĠOther":3819,"ban":3820,"Ġhous":3821,"wood":3822,"room":3823,"IV":3824,"ĠSun":3825,"see":3826,"ĠOver":3827,"rog":3828,"90":3829,"Ġlay":3830,"ĠTur":3831,"awn":3832,"Ġpressure":3833,"ĠSub":3834,"Ġbooks":3835,"edom":3836,"ĠSand":3837,"AA":3838,"ago":3839,"Ġreasons":3840,"ford":3841,"Ġactivity":3842,"UT":3843,"Now":3844,"ĠSenate":3845,"cell":3846,"night":3847,"Ġcalls":3848,"inter":3849,"Ġletter":3850,"ĠRob":3851,"ĠJe":3852,"Ġchoose":3853,"ĠLaw":3854,"Get":3855,"Be":3856,"Ġrob":3857,"Ġtypes":3858,"Ġplatform":3859,"Ġquarter":3860,"RA":3861,"ĠTime":3862,"Ġmaybe":3863,"ĠCr":3864,"95":3865,"pre":3866,"Ġmoving":3867,"Ġlif":3868,"Ġgold":3869,"Ġsom":3870,"Ġpatients":3871,"Ġtruth":3872,"ĠKe":3873,"urance":3874,"antly":3875,"mar":3876,"Ġcharge":3877,"ĠGreat":3878,"Ġcele":3879,"--------------------------------":3880,"Ġrock":3881,"roid":3882,"ancy":3883,"Ġcredit":3884,"aud":3885,"By":3886,"ĠEvery":3887,"Ġmoved":3888,"inger":3889,"ribution":3890,"Ġnames":3891,"Ġstraight":3892,"ĠHealth":3893,"ĠWell":3894,"Ġfeature":3895,"Ġrule":3896,"Ġsche":3897,"inated":3898,"ĠMichael":3899,"berg":3900,"41":3901,"iled":3902,"band":3903,"Ġclick":3904,"ĠAngel":3905,"onents":3906,"ÂŃ":3907,"ĠIraq":3908,"ĠSaturday":3909,"Ġaware":3910,"part":3911,"Ġpattern":3912,"OW":3913,"ĠLet":3914,"Ġgrad":3915,"igned":3916,"Ġassociated":3917,"Ġstyle":3918,"no":3919,"iation":3920,"aith":3921,"ilies":3922,"Ġstories":3923,"uration":3924,"Ġindividuals":3925,"ĠâĢ¦":3926,"miss":3927,"ĠAssoci":3928,"ishing":3929,"aby":3930,"Ġsummer":3931,"ĠBen":3932,"Ġ32":3933,"Ġarch":3934,"uty":3935,"ĠTexas":3936,"hol":3937,"Ġfully":3938,"Ġmill":3939,"Ġfollowed":3940,"ĠBill":3941,"ĠIndian":3942,"ĠSecret":3943,"ĠBel":3944,"ĠFebruary":3945,"Ġjobs":3946,"Ġseemed":3947,"ĠGovern":3948,"ipped":3949,"Ġreality":3950,"Ġlines":3951,"Ġpark":3952,"Ġmeasure":3953,"ĠOur":3954,"IM":3955,"Ġbrother":3956,"Ġgrowing":3957,"Ġban":3958,"Ġestim":3959,"Ġcry":3960,"ĠSchool":3961,"Ġmechan":3962,"ĠOF":3963,"ĠWindows":3964,"Ġrates":3965,"ĠOh":3966,"Ġpositive":3967,"Ġculture":3968,"istics":3969,"ica":3970,"Ġhar":3971,"ya":3972,"itely":3973,"ipp":3974,"Ġmap":3975,"encies":3976,"ĠWilliam":3977,"II":3978,"akers":3979,"56":3980,"ĠMart":3981,"ĠRem":3982,"Ġaltern":3983,"itude":3984,"Ġcoach":3985,"rowd":3986,"Don":3987,"Ġkids":3988,"Ġjournal":3989,"Ġcorpor":3990,"Ġfalse":3991,"Ġweb":3992,"Ġsleep":3993,"Ġcontain":3994,"Ġsto":3995,"Ġbed":3996,"iverse":3997,"ĠRich":3998,"ĠChinese":3999,"Ġpun":4000,"Ġmeant":4001,"known":4002,"Ġnotice":4003,"Ġfavorite":4004,"aven":4005,"Ġcondition":4006,"Ġpurpose":4007,"))":4008,"Ġorganization":4009,"Ġchalleng":4010,"Ġmanufact":4011,"Ġsusp":4012,"ĠAc":4013,"Ġcritic":4014,"unes":4015,"uclear":4016,"Ġmer":4017,"vention":4018,"Ġ80":4019,"Ġmist":4020,"ĠUs":4021,"ĠTor":4022,"http":4023,"olf":4024,"Ġlarger":4025,"Ġadvant":4026,"Ġresear":4027,"Ġactions":4028,"ml":4029,"Ġkept":4030,"Ġaim":4031,",'":4032,"col":4033,"Ġbenefits":4034,"ifying":4035,"Ġactual":4036,"ĠInternational":4037,"Ġvehicle":4038,"Ġchief":4039,"Ġefforts":4040,"ĠLeague":4041,"ĠMost":4042,"Ġwait":4043,"Ġadult":4044,"Ġoverall":4045,"Ġspeech":4046,"Ġhighly":4047,"Ġfemale":4048,"Ġerror":4049,"Ġeffective":4050,"54":4051,"Ġencour":4052,"well":4053,"Ġfailed":4054,"Ġconserv":4055,"Ġprograms":4056,"Ġtrou":4057,"Ġahead":4058,"500":4059,"vertisement":4060,"IP":4061,"ĠFound":4062,"pir":4063,"Ġ%":4064,"Ġcrime":4065,"ander":4066,"Ġlocation":4067,"ĠIran":4068,"Ġbehavior":4069,"azing":4070,"Ġrare":4071,"Ġemb":4072,"Ġcaused":4073,"Ġship":4074,"Ġactive":4075,"Ġcontribut":4076,"Ġgreen":4077,"Ġacqu":4078,"Ġreflect":4079,"venue":4080,"Ġfirm":4081,"Ġbirth":4082,"].":4083,"Ġclearly":4084,"Ġemot":4085,"Ġagency":4086,"riage":4087,"Ġmemory":4088,"98":4089,"SA":4090,"ĠSee":4091,"acing":4092,"CC":4093,"Ġbiggest":4094,"Ġrap":4095,"Ġbasic":4096,"Ġband":4097,"eat":4098,"Ġsuspect":4099,"ĠMac":4100,"Ġ90":4101,"mark":4102,"istan":4103,"Ġspread":4104,"ams":4105,"ki":4106,"asy":4107,"rav":4108,"ĠRober":4109,"Ġdemonstr":4110,"rated":4111,"Ġabsolute":4112,"Ġplaces":4113,"Ġimpl":4114,"ibrary":4115,"Ġcards":4116,"Ġdestroy":4117,"Ġvirt":4118,"vere":4119,"Ġappeared":4120,"yan":4121,"point":4122,"Ġbeg":4123,"Ġtemper":4124,"spe":4125,"anted":4126,"ears":4127,"ĠDirect":4128,"Ġlength":4129,"Ġblog":4130,"amb":4131,"Ġinteg":4132,"Ġresources":4133,"acc":4134,"iful":4135,"Ġspot":4136,"Ġforced":4137,"Ġthousands":4138,"ĠMinister":4139,"Ġqual":4140,"ĠFrench":4141,"atically":4142,"Ġgenerally":4143,"Ġdrink":4144,"Ġthus":4145,"IL":4146,"odes":4147,"Ġappropri":4148,"ĠRead":4149,"Ġwhom":4150,"Ġeye":4151,"Ġcollege":4152,"Ġ45":4153,"irection":4154,"Ġensure":4155,"Ġapparent":4156,"iders":4157,"Ġreligious":4158,"Ġminor":4159,"olic":4160,"Ġtro":4161,"ĠWhy":4162,"ribute":4163,"met":4164,"Ġprimary":4165,"Ġdeveloped":4166,"Ġpeace":4167,"Ġskin":4168,"ste":4169,"ava":4170,"Ġblue":4171,"Ġfamilies":4172,"Ġir":4173,"Ġapply":4174,"Ġinform":4175,"ĠSmith":4176,"CT":4177,"ii":4178,"Ġlimit":4179,"Ġresist":4180,"................":4181,"umn":4182,"Ġconflic":4183,"Ġtwe":4184,"udd":4185,"ĠTom":4186,"Ġliter":4187,"que":4188,"bon":4189,"Ġhair":4190,"Ġeventually":4191,"Ġpus":4192,"Ġhelped":4193,"Ġagg":4194,"orney":4195,"ĠApple":4196,"Ġfit":4197,"ĠSur":4198,"Ġprem":4199,"Ġsales":4200,"Ġseconds":4201,"Ġstrength":4202,"Ġfeeling":4203,"¿½":4204,"Ġtour":4205,"Ġknows":4206,"oom":4207,"Ġexerc":4208,"Ġsomew":4209,"�":4210,">>":4211,"Ġspokes":4212,"Ġideas":4213,"Ġregist":4214,"soft":4215,"ĠDel":4216,"ĠPC":4217,"Ġpropos":4218,"Ġlaunch":4219,"Ġbottom":4220,"TH":4221,"ĠPlease":4222,"vest":4223,"itz":4224,"ĠInter":4225,"Ġscript":4226,"Ġrat":4227,"arning":4228,"Ġil":4229,"ĠJer":4230,"ĠAre":4231,"Ġwhatever":4232,"oken":4233,"cience":4234,"Ġmode":4235,"Ġagree":4236,"Ġsources":4237,"Ġinitial":4238,"Ġrestrict":4239,"Ġwonder":4240,"usion":4241,"####":4242,"ĠSil":4243,"ville":4244,"Ġburn":4245,"tw":4246,"asion":4247,"Ġ£":4248,"Ġnor":4249,"uing":4250,"Ġreached":4251,"Ġsun":4252,"Ġcateg":4253,"igration":4254,"Ġcook":4255,"Ġpromot":4256,"Ġmale":4257,"Ġclimate":4258,"Ġfix":4259,"Ġalleged":4260,"UR":4261,"alled":4262,"Ġimages":4263,"Cont":4264,"ota":4265,"Ġschools":4266,"ios":4267,"Ġdrop":4268,"Ġstream":4269,"ĠMo":4270,"Ġpreviously":4271,"aling":4272,"Ġpet":4273,"Ġdouble":4274,"Ġ(@":4275,"annel":4276,"Ġdefault":4277,"ties":4278,"Ġrank":4279,"ĠDec":4280,"ĠCouncil":4281,"Ġweapon":4282,"Ġstock":4283,"Ġanaly":4284,"ĠStr":4285,"Ġpicture":4286,"ĠPolice":4287,"ference":4288,"Ġcentury":4289,"Ġcitizens":4290,"Ġonto":4291,"Ġexpand":4292,"Ġhero":4293,"ĠSol":4294,"Ġwild":4295,"Ġupdate":4296,"Ġcustomers":4297,"ront":4298,"def":4299,"Ġlik":4300,"Ġcriminal":4301,"ĠChristian":4302,"SP":4303,"76":4304,"Ġleaving":4305,"Ġotherwise":4306,"ĠDist":4307,"Ġbasis":4308,"52":4309,"53":4310,"icip":4311,"ĠBer":4312,"Ġrecommend":4313,"Ġfloor":4314,"Ġcrowd":4315,"oles":4316,"Ġ70":4317,"Ġcentral":4318,"ĠEv":4319,"Ġdream":4320,"Ġdownload":4321,"Ġconfir":4322,"ĠThom":4323,"Ġwindow":4324,"Ġhappens":4325,"Ġunit":4326,"Ġtend":4327,"Ġspl":4328,"Ġbecomes":4329,"Ġfighting":4330,"Ġpredict":4331,"ĠPress":4332,"ĠPower":4333,"Ġheavy":4334,"aked":4335,"Ġfan":4336,"orter":4337,"ategy":4338,"BA":4339,"izes":4340,"Ġspend":4341,"Here":4342,"Ġ2007":4343,"Ġadop":4344,"ĠHam":4345,"Ġfootball":4346,"ĠPort":4347,"oday":4348,"51":4349,"ampions":4350,"Ġtransfer":4351,"ht":4352,"Ġ38":4353,"term":4354,"acity":4355,"Ġbur":4356,"],":4357,"ternal":4358,"rig":4359,"but":4360,"Ġtherefore":4361,"ĠBecause":4362,"resp":4363,"rey":4364,"Ġmission":4365,"Some":4366,"Ġnoted":4367,"Ġassum":4368,"Ġdisease":4369,"Ġedit":4370,"Ġprogress":4371,"rd":4372,"ĠBrown":4373,"ocal":4374,"Ġadding":4375,"Ġraised":4376,"ĠAny":4377,"Ġtick":4378,"Ġseeing":4379,"ĠPeople":4380,"Ġagreement":4381,"Ġserver":4382,"Ġwat":4383,"Ġdebate":4384,"Ġsupposed":4385,"iling":4386,"Ġlargest":4387,"Ġsuccessful":4388,"ĠPri":4389,"ĠDemocratic":4390,"Ġjump":4391,"ĠSyria":4392,"Ġowners":4393,"Ġoffers":4394,"Ġshooting":4395,"Ġeffic":4396,"sey":4397,"Ġhaven":4398,"verse":4399,"tered":4400,"ĠLight":4401,"imal":4402,"ĠBig":4403,"Ġdefend":4404,"Ġbeat":4405,"Ġrecords":4406,"%)":4407,"Ġscen":4408,"Ġemployees":4409,"Ġdevices":4410,"hem":4411,"Ġcommer":4412,"ĠMex":4413,"Ġbenefit":4414,"ĠProf":4415,"Ġilleg":4416,"Ġsurface":4417,"ĠAlso":4418,"Ġharm":4419,"ingly":4420,"wide":4421,"ĠAlex":4422,"Ġshut":4423,"ĠCur":4424,"Ġlose":4425,"pm":4426,"Ġchallenge":4427,"semb":4428,"Ġstation":4429,"Ġintelligence":4430,"Ġaccur":4431,"ĠFlor":4432,"Ġrequires":4433,"ĠMal":4434,"bum":4435,"Ġhospital":4436,"Ġspirit":4437,"Ġoffered":4438,"Ġproduce":4439,"ĠCommun":4440,"Ġcreating":4441,"Ġcris":4442,"spect":4443,"Ġended":4444,"Ġdaily":4445,"Ġvoters":4446,"lands":4447,"ias":4448,"ih":4449,"ona":4450,"Ġsmart":4451,"ĠOffice":4452,"ĠLord":4453,"rial":4454,"ĠInternet":4455,"Ġcircum":4456,"Ġextremely":4457,"'.":4458,"Ġopinion":4459,"ĠMil":4460,"Ġgain":4461,"BS":4462,"ĠFin":4463,"yp":4464,"Ġuseful":4465,"Ġbudget":4466,"Ġcomfort":4467,"isf":4468,"Ġbackground":4469,"eline":4470,"Ġepisode":4471,"Ġenemy":4472,"Ġtrial":4473,"Ġestablish":4474,"date":4475,"ĠCap":4476,"Ġcontinues":4477,"Ġshowing":4478,"ĠUnion":4479,"with":4480,"Ġposted":4481,"ĠSystem":4482,"Ġeat":4483,"rian":4484,"Ġrise":4485,"ĠGermany":4486,"ils":4487,"Ġsigned":4488,"Ġvill":4489,"Ġgrand":4490,"mor":4491,"ĠEngland":4492,"Ġprojects":4493,"umber":4494,"Ġconference":4495,"za":4496,"Ġresponsible":4497,"ĠArab":4498,"Ġlearned":4499,"âĢĶâĢĶ":4500,"ipping":4501,"ĠGeorge":4502,"OC":4503,"Ġreturned":4504,"ĠAustralia":4505,"Ġbrief":4506,"Qu":4507,"Ġbrand":4508,"illing":4509,"abled":4510,"Ġhighest":4511,"Ġtrain":4512,"ĠCommission":4513,"while":4514,"Ġnom":4515,"ception":4516,"Ġmut":4517,"ĠBlue":4518,"Ġincident":4519,"vant":4520,"86":4521,"ĠID":4522,"Ġnuclear":4523,"74":4524,"ĠLike":4525,"ĠRE":4526,"ĠMicro":4527,"li":4528,"mail":4529,"Ġcharges":4530,"89":4531,"Ġadjust":4532,"ado":4533,"Ġearth":4534,"NA":4535,"Ġprices":4536,"PA":4537,"Ġdraft":4538,"Ġruns":4539,"Ġcandidate":4540,"enses":4541,"Ġmanagement":4542,"ĠPhil":4543,"ĠMiss":4544,"Ġteach":4545,"gram":4546,"Ġunderstanding":4547,"ait":4548,"icago":4549,"Add":4550,"ĠEp":4551,"secut":4552,"Ġseparate":4553,"Ġinstance":4554,"Ġeth":4555,"Ġunless":4556,"********":4557,"ĠFore":4558,"inate":4559,"Ġoperations":4560,"Sp":4561,"Ġfaith":4562,"gar":4563,"ĠChurch":4564,"ronic":4565,"Ġconfig":4566,"osure":4567,"Ġactivities":4568,"Ġtraditional":4569,"Ġ36":4570,"Ġdirection":4571,"Ġmachine":4572,"Ġsurround":4573,"Ġpush":4574,"unction":4575,"ĠEU":4576,"Ġeasier":4577,"Ġargument":4578,"GB":4579,"Ġmicro":4580,"Ġspending":4581,"izations":4582,"Ġtheory":4583,"adow":4584,"Ġcalling":4585,"ĠLast":4586,"Ġder":4587,"Ġinfluence":4588,"Ġcommit":4589,"Ġphoto":4590,"Ġunc":4591,"istry":4592,"gn":4593,"aste":4594,"acks":4595,"Ġdisp":4596,"ady":4597,"do":4598,"ĠGood":4599,"Ġ`":4600,"Ġwish":4601,"Ġrevealed":4602,"³³":4603,"lig":4604,"Ġenforce":4605,"ĠCommittee":4606,"Ġchem":4607,"Ġmiles":4608,"Ġinterested":4609,"Ġsolution":4610,"icy":4611,"inct":4612,"Ġ->":4613,"ĠDet":4614,"Ġremoved":4615,"Ġcompar":4616,"eah":4617,"Ġplant":4618,"ĠSince":4619,"Ġachieve":4620,"Ġadvantage":4621,"Ġslightly":4622,"bing":4623,"Ġplaced":4624,"under":4625,"2015":4626,"ĠMad":4627,"Ġtim":4628,"oses":4629,"Ġcru":4630,"ĠRock":4631,"Ġmostly":4632,"Ġnegative":4633,"Ġsetting":4634,"Ġproduced":4635,"Ġmur":4636,"Ġconnection":4637,"ĠMer":4638,"Ġdriver":4639,"Ġexecutive":4640,"Ġassault":4641,"Ġborn":4642,"ĠVer":4643,"tained":4644,"Ġstructure":4645,"Ġreduce":4646,"Ġdecades":4647,"Ġded":4648,"uke":4649,"ĠMany":4650,"idden":4651,"Ġleague":4652,"Se":4653,"Ġjoin":4654,"Ġdisco":4655,"Ġdie":4656,"cks":4657,"actions":4658,"Ġassess":4659,"agn":4660,"Ġgoals":4661,"ours":4662,"IR":4663,"Ġsenior":4664,"iller":4665,"mod":4666,"ipment":4667,"ocol":4668,"uy":4669,"ĠQue":4670,"Ġparties":4671,"irgin":4672,"Ġlearning":4673,"itable":4674,"Ġstreet":4675,"Ġcamera":4676,"App":4677,"Ġskills":4678,"bre":4679,"cious":4680,"Ġcelebr":4681,"ĠFranc":4682,"Ġexisting":4683,"Ġwilling":4684,"lor":4685,"Ġid":4686,"ĠSpace":4687,"Ġcritical":4688,"ĠLa":4689,"ortunately":4690,"Ġserve":4691,"Ġcold":4692,"Ġspecies":4693,"TS":4694,"Ġanimals":4695,"ĠBay":4696,"Ġolder":4697,"ĠUnder":4698,"estic":4699,"ĠTre":4700,"Ġteacher":4701,"Ġprefer":4702,"vis":4703,"Ġthread":4704,"ĠMatt":4705,"Ġmanager":4706,"ãĥ»":4707,"Ġprofessional":4708,"ĠVol":4709,"Ġnotes":4710,"These":4711,"ula":4712,"Ġfresh":4713,"ented":4714,"uzz":4715,"edy":4716,"clusion":4717,"ĠRel":4718,"Ġdoubt":4719,"EO":4720,"Ġopened":4721,"ĠBit":4722,"Advertisement":4723,"Ġguess":4724,"ĠUN":4725,"Ġsequ":4726,"Ġexplain":4727,"otten":4728,"Ġattract":4729,"aks":4730,"Ġstring":4731,"Ġcontext":4732,"ossible":4733,"ĠRepublicans":4734,"Ġsolid":4735,"Ġcities":4736,"Ġasking":4737,"Ġrandom":4738,"ups":4739,"uries":4740,"arant":4741,"dden":4742,"gl":4743,"ĠFlorida":4744,"Ġdepend":4745,"ĠScott":4746,"Ġ33":4747,"ĠiT":4748,"icon":4749,"Ġmentioned":4750,"Ġ2000":4751,"Ġclaimed":4752,"Ġdefinitely":4753,"ulf":4754,"Ġcore":4755,"Ġopening":4756,"ĠConst":4757,"which":4758,"ĠTra":4759,"AG":4760,"72":4761,"Ġbelieved":4762,"ada":4763,"Ġ48":4764,"ĠSecurity":4765,"yright":4766,"ĠPet":4767,"ĠLou":4768,"Ġholding":4769,"================":4770,"Ġice":4771,"Ġbrow":4772,"Ġauthorities":4773,"host":4774,"word":4775,"Ġscore":4776,"ĠDiv":4777,"Ġcells":4778,"Ġtransl":4779,"Ġneighbor":4780,"Ġremove":4781,"uct":4782,"Ġdistrict":4783,"ĠAccording":4784,"Ġworse":4785,"Ġconcerns":4786,"Ġpresidential":4787,"Ġpolicies":4788,"ĠHall":4789,"73":4790,"Ġhus":4791,"AY":4792,"Ġ2006":4793,"ĠJud":4794,"Ġindependent":4795,"ĠJustice":4796,"iliar":4797,"print":4798,"ighter":4799,"Ġprotection":4800,"zen":4801,"Ġsudden":4802,"house":4803,"ĠJes":4804,"PR":4805,"ĠInf":4806,"Ġbul":4807,"Ġ_":4808,"ĠService":4809,"ĠPR":4810,"Ġstrategy":4811,"ffect":4812,"Ġgirls":4813,"Ġmissing":4814,"oyal":4815,"ĠTeam":4816,"ulated":4817,"Ġdat":4818,"Ġpolitics":4819,"abor":4820,"According":4821,"Ġspell":4822,"Ġgraph":4823,"orthern":4824,"TC":4825,"Ab":4826,"Ġlabor":4827,"isher":4828,"Ġkick":4829,"ĠiTunes":4830,"Ġsteps":4831,"poses":4832,"Ġsmaller":4833,"En":4834,"bert":4835,"Ġroll":4836,"Ġresearchers":4837,"Ġclosed":4838,"Ġtransport":4839,"Ġlawy":4840,"________________":4841,"ĠChicago":4842,"Ġaspect":4843,"Ġnone":4844,"Ġmarriage":4845,"96":4846,"Ġelements":4847,"ĠFre":4848,"ĠSal":4849,"Ġdram":4850,"FC":4851,"top":4852,"equ":4853,"Ġhearing":4854,"Ġsupported":4855,"Ġtesting":4856,"cohol":4857,"Ġmassive":4858,"Ġstick":4859,"Ġguard":4860,"isco":4861,"phone":4862,"From":4863,"However":4864,"Ġborder":4865,"Ġcopy":4866,"ography":4867,"list":4868,"71":4869,"Ġowner":4870,"class":4871,"ruit":4872,"rate":4873,"ĠOnce":4874,"Ġdigital":4875,"Ġtask":4876,"ERS":4877,"Ġincred":4878,"tes":4879,"++":4880,"ĠFrance":4881,"Ġbreat":4882,"owl":4883,"Ġissued":4884,"ĠWestern":4885,"Ġdetect":4886,"Ġpartners":4887,"Ġshared":4888,"ĠCall":4889,"Ġcancer":4890,"ache":4891,"ribe":4892,"Ġexplained":4893,"Ġheat":4894,"{\"":4895,"Ġinvestment":4896,"ĠBook":4897,"Ġwood":4898,"Ġtools":4899,"ĠAlthough":4900,"Ġbelief":4901,"Ġcrisis":4902,"Ġge":4903,"ĠMP":4904,"Ġoperation":4905,"type":4906,"~~":4907,"ga":4908,"Ġcontains":4909,"anta":4910,"Ġexpress":4911,"ĠGroup":4912,"ĠJournal":4913,"ka":4914,"Ġamb":4915,"ĠUSA":4916,"Ġfinding":4917,"Ġfunding":4918,"how":4919,"Ġestablished":4920,"ideos":4921,"Ġdegree":4922,"Ġdangerous":4923,"anging":4924,"Ġfreedom":4925,"pport":4926,"outhern":4927,"Ġchurch":4928,"Ġcatch":4929,"ĠTwo":4930,"Ġpresence":4931,"ĠGuard":4932,"Up":4933,"Ġauthority":4934,"ĠProject":4935,"Ġbutton":4936,"Ġconsequ":4937,"Ġvalid":4938,"Ġweak":4939,"Ġstarts":4940,"Ġreference":4941,"ĠMem":4942,"\")":4943,"UN":4944,"orage":4945,"ĠOpen":4946,"Ġcollection":4947,"ym":4948,"gency":4949,"Ġbeautiful":4950,"ros":4951,"Ġtells":4952,"Ġwaiting":4953,"nel":4954,"Ġproviding":4955,"ĠDemocrats":4956,"Ġdaughter":4957,"Ġmaster":4958,"Ġpurposes":4959,"ĠJapanese":4960,"Ġequal":4961,"Ġturns":4962,"Ġdocuments":4963,"Ġwatching":4964,"Res":4965,"Ġran":4966,"2014":4967,"Ġreject":4968,"ĠKorea":4969,"Ġvictims":4970,"Level":4971,"erences":4972,"Ġwitness":4973,"Ġ34":4974,"Ġreform":4975,"coming":4976,"Ġoccup":4977,"Ġcaught":4978,"Ġtraffic":4979,"ading":4980,"Ġmodels":4981,"ario":4982,"Ġserved":4983,"Ġbatter":4984,"uate":4985,"ĠSecretary":4986,"Ġagreed":4987,"Ġtruly":4988,"ynam":4989,"ĠRet":4990,"Ġunits":4991,"ĠResearch":4992,"hand":4993,"azine":4994,"ĠMike":4995,"Ġvariety":4996,"otal":4997,"Ġamazing":4998,"Ġconfirmed":4999,"Ġentirely":5000,"Ġpurchase":5001,"Ġelement":5002,"Ġcash":5003,"Ġdetermine":5004,"De":5005,"Ġcars":5006,"ĠWall":5007,"âĸ":5008,"Ġviews":5009,"Ġdrugs":5010,"Ġdepartment":5011,"ĠStep":5012,"uit":5013,"Ġ39":5014,"asure":5015,"ĠClass":5016,"Ġcovered":5017,"ĠBank":5018,"Ġmere":5019,"uana":5020,"Ġmulti":5021,"Ġmix":5022,"Ġunlike":5023,"levision":5024,"Ġstopped":5025,"Ġsem":5026,"ĠGal":5027,"ules":5028,"Ġwel":5029,"ĠJohnson":5030,"la":5031,"Ġskill":5032,"Ġbecoming":5033,"rie":5034,"Ġappropriate":5035,"fe":5036,"ellow":5037,"ĠProt":5038,"ulate":5039,"ocation":5040,"Ġweekend":5041,"odies":5042,"Ġsites":5043,"Ġanimal":5044,"ĠTim":5045,"Ġscale":5046,"Ġcharged":5047,"Ġinstruct":5048,"illa":5049,"Ġmethods":5050,"Ġcert":5051,"Ġjudge":5052,"ĠHel":5053,"Ġdollars":5054,"Ġstanding":5055,"ĠSqu":5056,"Ġdebt":5057,"liam":5058,"Ġdriving":5059,"ĠSum":5060,"ĠEdition":5061,"Ġalbum":5062,"andon":5063,"IF":5064,"ĠUk":5065,"63":5066,"ader":5067,"Ġcommercial":5068,"esh":5069,"ĠGovernment":5070,"Ġdiscovered":5071,"Ġoutput":5072,"ĠHillary":5073,"ĠCarol":5074,"Ġ2005":5075,"Ġabuse":5076,"ancing":5077,"Ġswitch":5078,"Ġannual":5079,"Tw":5080,"Ġstated":5081,"agement":5082,"inner":5083,"Ġdemocr":5084,"Ġresidents":5085,"Ġallowing":5086,"Ġfactors":5087,"odd":5088,"Ġfuck":5089,"emies":5090,"Ġoccurred":5091,"oti":5092,"Ġnorth":5093,"ĠPublic":5094,"Ġinjury":5095,"Ġinsurance":5096,"CL":5097,"olly":5098,"ãĢ":5099,"Ġrepeated":5100,"Ġarms":5101,"anged":5102,"Ġconstruction":5103,"Ġfle":5104,"PU":5105,"icians":5106,"Ġforms":5107,"ĠMcC":5108,"antic":5109,"Ġmental":5110,"pire":5111,"Ġequipment":5112,"Ġfant":5113,"Ġdiscussion":5114,"Ġregarding":5115,"kin":5116,"arp":5117,"Ġchair":5118,"ogue":5119,"Ġproceed":5120,"ĠId":5121,"Our":5122,"Ġmurder":5123,"Man":5124,"Ġ49":5125,"asp":5126,"Ġsupply":5127,"Ġinput":5128,"Ġwealth":5129,"liament":5130,"Ġproced":5131,"orial":5132,"ĠStat":5133,"ĠNFL":5134,"hens":5135,"ĠInstitute":5136,"Ġputting":5137,"ournament":5138,"etic":5139,"Ġlocated":5140,"Ġkid":5141,"eria":5142,"run":5143,"Ġprinc":5144,"Ġ!":5145,"going":5146,"ĠBet":5147,"Ġclot":5148,"Ġtelling":5149,"Ġproposed":5150,"iot":5151,"orry":5152,"Ġfunds":5153,"gment":5154,"ĠLife":5155,"Ġbaby":5156,"ĠBack":5157,"Ġspoke":5158,"Image":5159,"Ġearn":5160,"ĠAT":5161,"gu":5162,"Ġexchange":5163,"ĠLin":5164,"oving":5165,"Ġpair":5166,"More":5167,"azon":5168,"Ġarrested":5169,"Ġkilling":5170,"can":5171,"ĠCard":5172,"yd":5173,"Ġidentified":5174,"Ġmobile":5175,"Ġthanks":5176,"onym":5177,"ĠForm":5178,"Ġhundreds":5179,"ĠChris":5180,"ĠCat":5181,"Ġtrend":5182,"hat":5183,"ĠAv":5184,"oman":5185,"Ġelectric":5186,"ĠWil":5187,"SE":5188,"Of":5189,"Ġrestaur":5190,"oted":5191,"Ġtrig":5192,"Ġnine":5193,"Ġbomb":5194,"Why":5195,"¯":5196,"Ġcoverage":5197,"Ġappeal":5198,"ĠRobert":5199,"ĠSup":5200,"Ġfinished":5201,"Ġflow":5202,"Ġdeliver":5203,"Ġcalcul":5204,"Ġphotos":5205,"Ġphil":5206,"Ġpieces":5207,"Ġappre":5208,"kes":5209,"Ġrough":5210,"Do":5211,"Ġpartner":5212,"Ġconcerned":5213,"Ġ37":5214,"ĠGen":5215,"Col":5216,"ctors":5217,"Ġ=>":5218,"state":5219,"Ġsuggested":5220,"ĠForce":5221,"CE":5222,"Ġherself":5223,"ĠPlan":5224,"works":5225,"ooth":5226,"rency":5227,"Ġcorner":5228,"Ġhusband":5229,"Ġinternet":5230,"ĠAut":5231,"ems":5232,"osen":5233,"ĠAtl":5234,"gen":5235,"Ġbalance":5236,"62":5237,"Ġsounds":5238,"text":5239,"Ġarr":5240,"oves":5241,"Ġmillions":5242,"Ġradio":5243,"Ġsatisf":5244,"ĠDam":5245,"Mr":5246,"Go":5247,"Spe":5248,"Ġcombat":5249,"rant":5250,"ĠGree":5251,"Ġfuel":5252,"Ġdistance":5253,"Ġtests":5254,"Ġdecre":5255,"ĠEr":5256,"Ġmanaged":5257,"DS":5258,"Ġtit":5259,"Ġmeasures":5260,"ĠLiber":5261,"Ġattend":5262,"ashed":5263,"ĠJose":5264,"ĠNight":5265,"dit":5266,"ĠNov":5267,"ĠEnd":5268,"outs":5269,"Ġgeneration":5270,"Ġadvoc":5271,"yth":5272,"Ġconversation":5273,"ĠSky":5274,"active":5275,"cel":5276,"rier":5277,"ĠFrank":5278,"Ġgender":5279,"Ġconcent":5280,"Ġcarried":5281,"anda":5282,"ĠVirgin":5283,"Ġarrived":5284,"icide":5285,"aded":5286,"Ġfailure":5287,"Ġminimum":5288,"lets":5289,"Ġworst":5290,"Ġkeeping":5291,"Ġintended":5292,"Ġillegal":5293,"Ġsubsc":5294,"Ġdetermined":5295,"Ġtrip":5296,"Yes":5297,"Ġraise":5298,"Ġ~":5299,"Ġfeels":5300,"Ġpackage":5301,"ĠJo":5302,"hi":5303,"2016":5304,"real":5305,"Ġfra":5306,"Ġsymb":5307,"Me":5308,"ucky":5309,"pret":5310,"ĠKh":5311,"ĠEdit":5312,"ĠWeb":5313,"emic":5314,"ĠColor":5315,"Ġjustice":5316,"Int":5317,"Ġfarm":5318,"cknow":5319,"\">":5320,"eless":5321,"Ġreduced":5322,"Ġ500":5323,"xx":5324,"ĠRad":5325,"ĠWood":5326,"Ġclin":5327,"Ġhyp":5328,"iler":5329,"ura":5330,"kins":5331,"85":5332,"61":5333,"ĠTheir":5334,"ĠMary":5335,"Ġsan":5336,"Ġnovel":5337,"ĠWho":5338,"Ġcapacity":5339,"Ġimpossible":5340,"Ġplays":5341,"Ġminister":5342,"ijuana":5343,"icate":5344,"ĠSet":5345,"Ġfram":5346,"Ġing":5347,"Ġcommunities":5348,"ĠFBI":5349,"ita":5350,"Ġbon":5351,"Ġstrateg":5352,"Ġinterests":5353,"lock":5354,"gers":5355,"mas":5356,"ĠAND":5357,"Ġconflict":5358,"Ġrequirements":5359,"Ġsac":5360,"Ġoperating":5361,"ini":5362,"related":5363,"Ġcommitted":5364,"Ġrelatively":5365,"Ġsouth":5366,"¯¯":5367,"Ġafford":5368,"Ġidentity":5369,"Ġdecisions":5370,"Ġaccused":5371,"place":5372,"Ġvictory":5373,"och":5374,"iat":5375,"Name":5376,"Com":5377,"tion":5378,"eds":5379,"Ġseek":5380,"Ġtight":5381,"ĠImages":5382,"Ġiniti":5383,"Ġhumans":5384,"Ġfamiliar":5385,"Ġaudience":5386,"Ġinternal":5387,"venture":5388,"Ġsides":5389,"ĠTO":5390,"Ġdim":5391,"Ġconclud":5392,"Ġappoint":5393,"Ġenforcement":5394,"ĠJim":5395,"ĠAssociation":5396,"Ġcircumst":5397,"ĠCanadian":5398,"Ġjoined":5399,"Ġdifferences":5400,"ĠLos":5401,"Ġprotest":5402,"Ġtwice":5403,"win":5404,"Ġglass":5405,"arsh":5406,"ĠArmy":5407,"Ġexpression":5408,"Ġdecide":5409,"Ġplanning":5410,"ania":5411,"Ġhandle":5412,"ĠMicrosoft":5413,"ĠNor":5414,"Ġmaximum":5415,"ĠRev":5416,"Ġsea":5417,"Ġeval":5418,"Ġhelps":5419,"ref":5420,"Ġbound":5421,"Ġmouth":5422,"Ġstandards":5423,"Ġclim":5424,"ĠCamp":5425,"ĠFox":5426,"cles":5427,"Ġarmy":5428,"ĠTechn":5429,"acking":5430,"xy":5431,"SS":5432,"Ġ42":5433,"Ġbug":5434,"ĠUkrain":5435,"ĠMax":5436,"ĠJones":5437,"ĠShow":5438,"lo":5439,"Ġplanet":5440,"Ġ75":5441,"Ġwinning":5442,"Ġfaster":5443,"Ġspect":5444,"Ġbroken":5445,"TR":5446,"Ġdefined":5447,"Ġhealthy":5448,"Ġcompetition":5449,"https":5450,"ĠIsland":5451,"ĠFe":5452,"Ġannounce":5453,"ĠCup":5454,"ĠInstead":5455,"Ġclient":5456,"Ġpossibly":5457,"section":5458,"ocket":5459,"look":5460,"Ġfinish":5461,"Ġcrew":5462,"Ġreserv":5463,"Ġeditor":5464,"Ġhate":5465,"Ġsale":5466,"Ġcontrovers":5467,"Ġpages":5468,"wing":5469,"Ġnumer":5470,"Ġopposition":5471,"Ġ2004":5472,"Ġrefuge":5473,"Ġflight":5474,"Ġapart":5475,"ĠLat":5476,"Americ":5477,"ĠAfrica":5478,"Ġapplications":5479,"ĠPalest":5480,"ĠBur":5481,"Ġgar":5482,"ĠSocial":5483,"Ġupgr":5484,"Ġshape":5485,"Ġspeaking":5486,"ansion":5487,"ao":5488,"ĠSn":5489,"Ġworry":5490,"ĠBritain":5491,"Please":5492,"roud":5493,"Ġhun":5494,"Ġintroduced":5495,"Ġdiet":5496,"Ind":5497,"ĠSecond":5498,"Ġfunctions":5499,"uts":5500,"ĠEach":5501,"ĠJeff":5502,"Ġstress":5503,"Ġaccounts":5504,"Ġguarant":5505,"ĠAnn":5506,"edia":5507,"Ġhonest":5508,"Ġtree":5509,"ĠAfrican":5510,"ĠBush":5511,"},":5512,"Ġsch":5513,"ĠOnly":5514,"Ġfif":5515,"igan":5516,"Ġexercise":5517,"ĠExp":5518,"Ġscientists":5519,"Ġlegislation":5520,"ĠWork":5521,"ĠSpr":5522,"ÃĤ":5523,"ĠHuman":5524,"Ġè":5525,"Ġsurvey":5526,"Ġrich":5527,"rip":5528,"Ġmaintain":5529,"Ġflo":5530,"Ġleadership":5531,"stream":5532,"ĠIslamic":5533,"Ġ01":5534,"ĠCollege":5535,"Ġmagic":5536,"ĠPrime":5537,"Ġfigures":5538,"2017":5539,"inder":5540,"xual":5541,"ĠDead":5542,"Ġabsolutely":5543,"Ġfourth":5544,"Ġpresented":5545,"respond":5546,"rible":5547,"Ġalcohol":5548,"ato":5549,"ĠDE":5550,"porary":5551,"Ġgrab":5552,"Ġvari":5553,"Ġquant":5554,"ĠPhoto":5555,"Ġplus":5556,"rick":5557,"arks":5558,"Ġalternative":5559,"Ġpil":5560,"Ġapprox":5561,"that":5562,"Ġobjects":5563,"ĠRo":5564,"ĠAndroid":5565,"Ġsignificantly":5566,"ĠRoad":5567,"kay":5568,"Read":5569,"avor":5570,"Ġacknow":5571,"ĠHD":5572,"ĠSing":5573,"Or":5574,"ĠMont":5575,"Ġuns":5576,"prof":5577,"Ġnegoti":5578,"ĠArch":5579,"iki":5580,"Ġtelevision":5581,"ĠJewish":5582,"Ġcommittee":5583,"Ġmotor":5584,"Ġappearance":5585,"Ġsitting":5586,"Ġstrike":5587,"ĠDown":5588,"comp":5589,"ĠHist":5590,"Ġfold":5591,"acement":5592,"ĠLouis":5593,"Ġbelong":5594,"ĠâĢ¢":5595,"Ġmort":5596,"Ġprepared":5597,"Ġ64":5598,"ĠMaster":5599,"Ġindeed":5600,"ĠDen":5601,"Ġrent":5602,"TA":5603,"ourney":5604,"arc":5605,"Su":5606,"97":5607,"Ġadvice":5608,"Ġchanging":5609,"Ġlisted":5610,"Ġlaunched":5611,"isation":5612,"ĠPeter":5613,"ishes":5614,"Ġlived":5615,"ĠMel":5616,"ĠSupreme":5617,"ĠFederal":5618,"Ġ);":5619,"ructure":5620,"Ġsets":5621,"Ġphilos":5622,"uous":5623,"ĠÂł":5624,"Ġapplied":5625,"ĠNOT":5626,"Ġhousing":5627,"ĠMount":5628,"Ġodd":5629,"Ġsust":5630,"DA":5631,"fficient":5632,"Ġ?":5633,"olved":5634,"Ġpowers":5635,"Ġthr":5636,"Ġremaining":5637,"ĠWater":5638,"LC":5639,"Ġcauses":5640,"ãģ®":5641,"Ġmanner":5642,"ads":5643,"Ġsuggests":5644,"Ġends":5645,"standing":5646,"fig":5647,"ĠDun":5648,"idth":5649,"Ġgay":5650,"Ġtermin":5651,"ĠAngeles":5652,"MS":5653,"Ġscientific":5654,"Ġcoal":5655,"apers":5656,"bar":5657,"ĠThomas":5658,"Ġsym":5659,"ĠRun":5660,"this":5661,"PC":5662,"igrants":5663,"Ġminute":5664,"ĠDistrict":5665,"cellent":5666,"Ġleaves":5667,"Ġcompleted":5668,"amin":5669,"Ġfocused":5670,"Ġmonitor":5671,"Ġvehicles":5672,"MA":5673,"ĠMass":5674,"ĠGrand":5675,"Ġaffected":5676,"itutional":5677,"Ġconstruct":5678,"Ġfollows":5679,"Ġton":5680,"reens":5681,"Ġhomes":5682,"ĠExt":5683,"ĠLevel":5684,"rast":5685,"ĠIr":5686,"Ġelim":5687,"Ġlargely":5688,"ĠJoe":5689,"Ġvotes":5690,"alls":5691,"Ġbusinesses":5692,"ĠFoundation":5693,"ĠCentral":5694,"Ġyards":5695,"Ġmaterials":5696,"ulner":5697,"Ġguide":5698,"Ġcloser":5699,"ums":5700,"Ġsports":5701,"eder":5702,"Just":5703,"Ġtaxes":5704,"84":5705,"ĠOld":5706,"Ġdecade":5707,"ola":5708,"Ġvir":5709,"Ġdropped":5710,"Ġdelay":5711,"itect":5712,"Ġsecure":5713,"stein":5714,"level":5715,"Ġtreated":5716,"Ġfiled":5717,"aine":5718,"Ġvan":5719,"Ġmir":5720,"Ġcolumn":5721,"icted":5722,"eper":5723,"Ġrot":5724,"Ġconsult":5725,"Ġentry":5726,"Ġmarijuana":5727,"ĠDou":5728,"Ġapparently":5729,"oking":5730,"clusive":5731,"Ġincreases":5732,"ano":5733,"Ġspecifically":5734,"Ġtele":5735,"ensions":5736,"Ġreligion":5737,"abilities":5738,"Ġframe":5739,"ĠNote":5740,"ĠLee":5741,"Ġhelping":5742,"Ġedge":5743,"oston":5744,"Ġorganizations":5745,"Ãĥ":5746,"ĠBoth":5747,"hips":5748,"Ġbigger":5749,"Ġboost":5750,"ĠStand":5751,"Ġrow":5752,"uls":5753,"abase":5754,"Ġrid":5755,"Let":5756,"aren":5757,"rave":5758,"Ġstret":5759,"PD":5760,"Ġvision":5761,"Ġwearing":5762,"Ġappreci":5763,"Ġaward":5764,"ĠUse":5765,"Ġfactor":5766,"war":5767,"ulations":5768,")(":5769,"Ġgod":5770,"Ġterrit":5771,"Ġparam":5772,"asts":5773,"87":5774,"Ġenemies":5775,"ĠGames":5776,"FF":5777,"Ġaccident":5778,"Well":5779,"ĠMartin":5780,"TER":5781,"Ġath":5782,"ĠHell":5783,"Ġforg":5784,"Ġveter":5785,"ĠMedic":5786,"free":5787,"Ġstars":5788,"Ġexpensive":5789,"Ġacad":5790,"rawn":5791,"ĠWhe":5792,"Ġlock":5793,"Ġformat":5794,"Ġsoldiers":5795,"sm":5796,"Ġagent":5797,"Ġresponsibility":5798,"ora":5799,"ĠScience":5800,"Ġrapid":5801,"Ġtough":5802,"ĠJesus":5803,"Ġbelieves":5804,"ML":5805,"Ġwear":5806,"lete":5807,"ÃĥÃĤ":5808,"ĠDri":5809,"Ġcommission":5810,"ĠBob":5811,"Oh":5812,"aped":5813,"Ġwarm":5814,"ÃĥÃĤÃĥÃĤ":5815,"Ġ2003":5816,"ortion":5817,"Ġhasn":5818,"uster":5819,"Ġunivers":5820,"ĠIll":5821,"Ġking":5822,"ologies":5823,"94":5824,"ĠTem":5825,"ĠMos":5826,"Ġpatient":5827,"ĠMexico":5828,"cean":5829,"ĠDeath":5830,"ĠSanders":5831,"you":5832,"ĠCast":5833,"ĠCompany":5834,"pty":5835,"Ġhappening":5836,"FP":5837,"ĠBattle":5838,"Ġbought":5839,"Am":5840,"Mod":5841,"Us":5842,"uters":5843,"ĠCre":5844,"ĠThose":5845,"Ġ44":5846,"iser":5847,"Ġsoul":5848,"ĠTop":5849,"ĠHarry":5850,"ĠAw":5851,"Ġseat":5852,"ffee":5853,"Ġrevolution":5854,"Ġ(\"":5855,"ĠDuring":5856,"ette":5857,"Ġring":5858,"Ġoffensive":5859,"Ġreturns":5860,"Ġvideos":5861,"Ġdiscl":5862,"Ġfamous":5863,"enced":5864,"ĠSign":5865,"ĠRiver":5866,"Ġ300":5867,"PM":5868,"ĠBus":5869,"ĠCH":5870,"Ġcandidates":5871,"arden":5872,"Ġpercentage":5873,"Ġvisual":5874,"Ġthank":5875,"Ġtrouble":5876,"nergy":5877,"Ġ2001":5878,"Ġprove":5879,"ashion":5880,"Ġenh":5881,"ĠLong":5882,"UM":5883,"Ġconnected":5884,"Ġpossibility":5885,"Over":5886,"Ġexpert":5887,"Ġlibrary":5888,"arts":5889,"ĠDirector":5890,"Ġfellow":5891,"92":5892,"irty":5893,"Ġdry":5894,"Ġsigns":5895,"ĠLove":5896,"Ġquiet":5897,"foot":5898,"Ġpure":5899,"ĠHun":5900,"Ġfilled":5901,"phas":5902,"ĠElect":5903,"endment":5904,"ĠExpl":5905,"Ġunable":5906,"ns":5907,"mo":5908,"Ġvast":5909,"obe":5910,"Ġidentify":5911,"apping":5912,"ĠCarolina":5913,"gress":5914,"Ġprote":5915,"Ġfish":5916,"Ġcircumstances":5917,"razy":5918,"ĠPhot":5919,"Ġbodies":5920,"ĠMur":5921,"Ġdeveloping":5922,"ĠAR":5923,"Ġexperienced":5924,"Ġsubstant":5925,"ĠBoard":5926,"esome":5927,"Ġdomestic":5928,"Ġcombined":5929,"ĠPut":5930,"Ġchemical":5931,"ĠChild":5932,"Ġpool":5933,"ĠCy":5934,"Ġegg":5935,"cons":5936,"sters":5937,"Ġhurt":5938,"Ġmarkets":5939,"Ġconservative":5940,"Ġsupporters":5941,"Ġagencies":5942,"idel":5943,"Ob":5944,"urb":5945,"Ġ43":5946,"ĠDefense":5947,"ye":5948,"ĠAp":5949,"dule":5950,"Ġtemperature":5951,"Ġconducted":5952,"ĠChief":5953,"Ġpulled":5954,"Ġfol":5955,"Last":5956,"onto":5957,"osis":5958,"VER":5959,"Des":5960,"ĠPan":5961,"First":5962,"Ġadvance":5963,"Ġlicense":5964,"rors":5965,"ĠJon":5966,"Ġimagine":5967,"Ġhell":5968,"Ġfixed":5969,"Ġincor":5970,"osite":5971,"ĠLog":5972,"icken":5973,"]:":5974,"Ġsurprise":5975,"hab":5976,"Ġcraft":5977,"olt":5978,"ĠJul":5979,"Ġdial":5980,"Ġrelevant":5981,"Ġentered":5982,"Ġleads":5983,"ĠAD":5984,"ĠClean":5985,"Ġpictures":5986,"essor":5987,"Ġalt":5988,"Ġpaying":5989,"Per":5990,"ĠMarket":5991,"Ġupdates":5992,"amily":5993,"ĠType":5994,"ĠHome":5995,"Ġ55":5996,"sembly":5997,"rome":5998,"83":5999,"Ġgreatest":6000,"Ġheight":6001,"Ġheav":6002,"aints":6003,"Ġlisten":6004,"aser":6005,"ĠSH":6006,"Ġcapable":6007,"acle":6008,"Ġperspect":6009,"inating":6010,"Ġoffering":6011,"rypt":6012,"ĠDevelop":6013,"abin":6014,"rc":6015,"Ġbright":6016,"alty":6017,"arrow":6018,"Ġsuppl":6019,"inding":6020,"acked":6021,"gypt":6022,"ĠAnother":6023,"pg":6024,"ĠVirginia":6025,"ĠLu":6026,"Ġplanned":6027,"Ġpit":6028,"Ġsweet":6029,"Type":6030,"ĠDi":6031,"Ġtypically":6032,"ĠFrancisco":6033,"Ġprospect":6034,"ĠDan":6035,"Ġteen":6036,"rees":6037,"Ġsched":6038,"Ġhol":6039,"Ġscr":6040,"Ġlots":6041,"life":6042,"Ġnewsp":6043,"Ġforget":6044,"ĠNone":6045,"ĠMiddle":6046,"ĠRyan":6047,"edd":6048,"Ġsevere":6049,"Ġsuit":6050,"ller":6051,"93":6052,"Ġcorrespond":6053,"Ġexplos":6054,"uations":6055,"Ġflag":6056,"game":6057,"rid":6058,"Ġprin":6059,"ĠData":6060,"Ġdeploy":6061,"ĠEnter":6062,"suit":6063,"ghan":6064,"ĠMen":6065,"Ġthoughts":6066,"Ġmatters":6067,"Ġadapt":6068,"ĠAri":6069,"Ġfill":6070,"Ġforth":6071,"Ġsam":6072,"Ġ41":6073,"Ġpayment":6074,"ĠHor":6075,"Ġspring":6076,"duc":6077,"Ġlosing":6078,"Ġbringing":6079,"FO":6080,"ala":6081,"Ġdistribution":6082,"hered":6083,"bour":6084,"ĠIsraeli":6085,"oma":6086,"Ġcombination":6087,"Ġplenty":6088,"VE":6089,"Can":6090,"ĠHaw":6091,"Ġperman":6092,"ĠSpecial":6093,"Ġtow":6094,"Ġseeking":6095,"Ġexamples":6096,"Ġclasses":6097,"cr":6098,"Ġbeer":6099,"Ġmoves":6100,"ĠIP":6101,"ĠKn":6102,"Ġpanel":6103,"Even":6104,"Ġproperly":6105,"Ġris":6106,"Ġplug":6107,"Ġestimated":6108,"Every":6109,"Ġdefensive":6110,"agraph":6111,"Ġpregn":6112,"Ġinstit":6113,"ĠVict":6114,"Ġvolume":6115,"Ġpositions":6116,"Ġlinks":6117,"ĠProgram":6118,"ĠWeek":6119,"agues":6120,"Ġtransform":6121,"ker":6122,"ĠCEO":6123,"Ġcas":6124,"Ġopponent":6125,"Ġtweet":6126,"ĠCode":6127,"Ġshop":6128,"Ġfly":6129,"Ġtalks":6130,"Ġbag":6131,"Phone":6132,"Ġaid":6133,"Ġplants":6134,"Ġ65":6135,"Ġattorney":6136,"arters":6137,"quest":6138,"ĠMagic":6139,"Ġbegins":6140,"Ġmyster":6141,"Ġenvironmental":6142,"Ġstorage":6143,"NN":6144,"Ġmarg":6145,"Ġske":6146,"Ġmetal":6147,"elly":6148,"Ġordered":6149,"Ġremained":6150,"Ġloved":6151,"Ġprompt":6152,"Ġupdated":6153,"Ġexperts":6154,"Ġwalking":6155,"Ġancient":6156,"Ġperformed":6157,"ATE":6158,"Ġneither":6159,"iency":6160,"Ġmanufacture":6161,"ĠPak":6162,"Ġselected":6163,"Ġmine":6164,"Ġultimately":6165,"Ġexplan":6166,"Ġlabel":6167,"ĠServices":6168,"ributed":6169,"Trump":6170,"Ġsyn":6171,"ĠUlt":6172,"SC":6173,"Ġmeat":6174,"Ġgiant":6175,"ĠWars":6176,"ĠON":6177,"Ġadm":6178,"Ġinterpret":6179,"Ġevening":6180,"Ġevil":6181,"ĠBoston":6182,"ĠWild":6183,"ĠÃ":6184,"ĠBitcoin":6185,"ĠAmazon":6186,"Dr":6187,"ĠInformation":6188,"Ġobviously":6189,"Ġadvanced":6190,"Photo":6191,"olar":6192,"Ġweather":6193,"Ġsymbol":6194,"Ġsole":6195,"Ġpotentially":6196,"oster":6197,"Ġoriginally":6198,"mun":6199,"300":6200,"aze":6201,"essions":6202,"Ġdeck":6203,"Ġstood":6204,"Ġyouth":6205,"ĠBern":6206,"Rep":6207,"ĠTest":6208,"Ġbasically":6209,"otic":6210,"Ġinvolve":6211,"olit":6212,"lyn":6213,"See":6214,"Ġaircraft":6215,"Ġconfirm":6216,"EW":6217,"Ġmessages":6218,"ĠRichard":6219,"Ġkit":6220,"Ġprohib":6221,"Ġvulner":6222,"isters":6223,"Ġexistence":6224,"Ġturning":6225,"ĠSP":6226,"Ġdesire":6227,"Ġflat":6228,"Ġment":6229,"season":6230,"anges":6231,"Ġneighborhood":6232,"ĠLake":6233,"ATION":6234,"Ġpointed":6235,"bur":6236,"Ġinnov":6237,"ucks":6238,"UL":6239,"Ġprofessor":6240,"Ġexpressed":6241,"AB":6242,"icious":6243,"Ġ2002":6244,"ĠDev":6245,"Ġsession":6246,"Ġbare":6247,"sen":6248,"Ġdiss":6249,"ĠCath":6250,"ĠPass":6251,"ĠPoint":6252,"Ġdoctor":6253,"orrow":6254,"ailed":6255,"ĠRub":6256,"ĠDC":6257,"ĠCharl":6258,"person":6259,"Ġwriter":6260,"ighters":6261,"ureau":6262,"Ġoblig":6263,"Ġrecorded":6264,"Ġbroke":6265,"Ġorders":6266,"ilty":6267,"Ġmotion":6268,"inity":6269,"law":6270,"adium":6271,"Ġimmigration":6272,"Ġcontrast":6273,"Ġbatt":6274,"Ġexcellent":6275,"Ġtechnical":6276,"ami":6277,"Ġtun":6278,"Ġcloud":6279,"ĠYear":6280,"geon":6281,"Ġcreation":6282,"Ġstrange":6283,"Ġauth":6284,"Ġfort":6285,"born":6286,"Ġextent":6287,"ĠToday":6288,"ĠClub":6289,"Ġrain":6290,"Ġsample":6291,"Ġaccepted":6292,"Ġtact":6293,"Ġfired":6294,"ĠSon":6295,"Ġstands":6296,"Ġboot":6297,"Ġ47":6298,"Ġstatements":6299,"Ġversions":6300,"Ġselling":6301,"ounded":6302,"Ġ1990":6303,"Ġweren":6304,"ĠWatch":6305,"Ġexperiment":6306,"Post":6307,"Ġretail":6308,"uled":6309,"Inst":6310,"unte":6311,"ãĥ¼":6312,"Ġdepart":6313,"Ġbond":6314,"ivery":6315,"ompl":6316,"Ġreaction":6317,"ĠSyrian":6318,"ĠPac":6319,"apped":6320,"aniel":6321,"DP":6322,"Ġresolution":6323,"Ġreact":6324,"Ġapproved":6325,"onom":6326,"mond":6327,"ĠOffic":6328,"---":6329,"Ġreplace":6330,"Ġtack":6331,"Ġsport":6332,"Ġchain":6333,"Ġemergency":6334,"rad":6335,"ĠPalestin":6336,"Ġ46":6337,"Ġautomatically":6338,"Ġroute":6339,"Ġpal":6340,"Ġbanks":6341,"ĠParis":6342,"ĠMedia":6343,"road":6344,"icing":6345,"ixt":6346,"isted":6347,"Ġgrew":6348,"Ġcoord":6349,"ĠWhere":6350,"omin":6351,"Ġsubs":6352,"��":6353,"Ġ±":6354,"Ġcorporate":6355,"Ġselection":6356,"noon":6357,"ĠReport":6358,"cs":6359,"cluding":6360,"orders":6361,"anche":6362,"ĠIts":6363,"Ġslowly":6364,"ĠEgypt":6365,"ĠAcc":6366,"Ġcolle":6367,"iques":6368,"EX":6369,"Ġattempts":6370,"url":6371,"ĠCross":6372,"Ġfindings":6373,"ĠSC":6374,"ĠOR":6375,"Ġindex":6376,"ensity":6377,"ĠWay":6378,"ĠLand":6379,"Ġshock":6380,"dis":6381,"Ġdynam":6382,"Ġcart":6383,"mosp":6384,"Since":6385,"iest":6386,"ĠBoy":6387,"Ġstorm":6388,"ĠContin":6389,"2013":6390,"hew":6391,"ilit":6392,"Ġessential":6393,"iquid":6394,"Other":6395,"ivered":6396,"Ġreasonable":6397,"Act":6398,"Ġsubsequ":6399,"ĠPack":6400,"ĠFort":6401,"Ġconsidering":6402,"Ġuniversity":6403,"log":6404,"Ġmarried":6405,"Ġillust":6406,"ĠTrue":6407,"£ı":6408,"Ġnumerous":6409,"rastructure":6410,"Ġseriously":6411,"Ġreferred":6412,"ua":6413,"Ġconsistent":6414,"onna":6415,"ĠReal":6416,"ruption":6417,"ciples":6418,"Ġfacts":6419,"91":6420,"otes":6421,"erg":6422,"Then":6423,"Ġaccompl":6424,"Note":6425,"Ġrevenue":6426,"Ġpassing":6427,"Ġmal":6428,"een":6429,"ĠYet":6430,"Ġgather":6431,"terday":6432,"ework":6433,"ĠAuthor":6434,"Pe":6435,"Ġoptim":6436,"Ġrub":6437,"Ġè£ı":6438,"Ġunknown":6439,"stone":6440,"Ġunion":6441,"olve":6442,"Ġopportunities":6443,"Ġbrowser":6444,"ĠWal":6445,"ĠCost":6446,"Ġreporting":6447,"sts":6448,"pet":6449,"Ġsand":6450,"Ġsuddenly":6451,"Ġsurprising":6452,"ĠVR":6453,"Ġsomewhat":6454,"ĠBas":6455,"ulture":6456,"izz":6457,"ĠCD":6458,"Ġchallenges":6459,"Ġsettings":6460,"Ġexperiences":6461,"ĠFull":6462,"Ġcann":6463,"Ġreceiving":6464,"EST":6465,"Ġjoint":6466,"Ġcultural":6467,"Ġast":6468,"82":6469,"astern":6470,"ceived":6471,"ĠCru":6472,"Ġbull":6473,"pired":6474,"amm":6475,"Ġfacing":6476,"power":6477,"Ġboss":6478,"ĠHol":6479,"Ġinstr":6480,"Ġincreasingly":6481,"Ġshift":6482,"Ġstreets":6483,"ĠWilliams":6484,"abb":6485,"Ġlie":6486,"Ġlaugh":6487,"ĠCa":6488,"PL":6489,"Ġadults":6490,"Ġcustomer":6491,"Ġobtained":6492,"Ġsupporting":6493,"html":6494,"fire":6495,"Ġdetailed":6496,"Ġpicked":6497,"ĠRight":6498,"lder":6499,"EE":6500,"stood":6501,"ĠKim":6502,"Ġwire":6503,"Ġsight":6504,"Ġdevelopers":6505,"Ġpersons":6506,"Ġsad":6507,"Ġcup":6508,"Ġwarning":6509,"Ġboys":6510,"long":6511,"Ġbird":6512,"fo":6513,"Ġwal":6514,"Ġobserved":6515,"Ġzone":6516,"iveness":6517,"Ġchannel":6518,"cript":6519,"Ġrefused":6520,"ĠAgain":6521,"Ġsuc":6522,"Ġspokesman":6523,"ĠRef":6524,"rite":6525,"ouston":6526,"ãĥ³":6527,"ĠSher":6528,"Ġacts":6529,"ĠName":6530,"Ġstruggle":6531,"arry":6532,"ometimes":6533,"Ġdiscrim":6534,"HT":6535,"Ġcategory":6536,"Ġrealize":6537,"Ġemployee":6538,"ĠAfghan":6539,"enger":6540,"Ġguns":6541,"ĠSteve":6542,"ĠMot":6543,"ĠOl":6544,"oked":6545,"Ġthick":6546,"Ġfairly":6547,"illy":6548,"Ġsurve":6549,"ĠMat":6550,"weight":6551,"âĶ":6552,"Ġtroops":6553,"Ġagents":6554,"Ġbattery":6555,"Ġmotiv":6556,"á":6557,"Sec":6558,"den":6559,"overy":6560,"LS":6561,"Ġflu":6562,"Ġconfident":6563,"ĠOper":6564,"Ġempty":6565,"Ġphen":6566,"Ġsector":6567,"Ġexcited":6568,"Ġremote":6569,"aph":6570,"oen":6571,"Ġdestroyed":6572,"Ġmoral":6573,"ĠHP":6574,"ĠRon":6575,"Ġdress":6576,"ĠBat":6577,"Ġlit":6578,"ĠMS":6579,"Ġaf":6580,"HL":6581,"rum":6582,"isms":6583,"Ġshouldn":6584,"Ġsympt":6585,"ĠToronto":6586,"hetic":6587,"Ġcarbon":6588,"Ġinstalled":6589,"Ġviolent":6590,"Ġsolar":6591,"ja":6592,"Ġpractices":6593,"Ġride":6594,"ĠPenn":6595,"Ġimproved":6596,"Ġaudio":6597,"Ġbehavi":6598,"ĠPS":6599,"Ġeating":6600,"Data":6601,"ĠReview":6602,"pass":6603,"claim":6604,"uated":6605,"angers":6606,"chen":6607,"Ġproperties":6608,"Ġanywhere":6609,"Another":6610,"Ġblow":6611,"ĠJackson":6612,"Ġproud":6613,"Ġplane":6614,"lines":6615,"Ġsquare":6616,"Ġproof":6617,"ansas":6618,"Ġtalked":6619,"makers":6620,"Ġsister":6621,"Ġholds":6622,"Ġresident":6623,"Ġ==":6624,"Ġresistance":6625,"Ġsplit":6626,"Ġprosecut":6627,"Ġconfidence":6628,"resents":6629,"Ġcuts":6630,"Ġexception":6631,"Ġzero":6632,"Getty":6633,"Ġcopyright":6634,"Ġtotally":6635,"ormal":6636,"ifications":6637,"ĠAustralian":6638,"Ġsick":6639,"Ġ150":6640,"Ġhousehold":6641,"Ġfees":6642,"Ġdrivers":6643,"ogen":6644,"ĠNY":6645,"Ġnecessarily":6646,"Ġregulations":6647,"earing":6648,"sl":6649,"Ġperspective":6650,"care":6651,"icial":6652,"His":6653,"Ġescape":6654,"Ġsurprised":6655,"ĠVan":6656,"urrent":6657,"Ġvac":6658,"81":6659,"ĠThus":6660,"Ġemphas":6661,"ĠChampions":6662,"ĠIce":6663,"Ġnarr":6664,"Ġheads":6665,"Ġcausing":6666,"bel":6667,"fortunately":6668,"ĠMa":6669,"Ġtargets":6670,"cipl":6671,"Ġafternoon":6672,"Ġadds":6673,"ĠMaybe":6674,"ĠFour":6675,"essed":6676,"plete":6677,"Ġusual":6678,"cho":6679,"ingu":6680,"Ġwithd":6681,"ĠEnergy":6682,"ĠEconom":6683,"OO":6684,"Ġarticles":6685,"Ġinjured":6686,"Ġmanage":6687,"Ġexplains":6688,"Ġdiagn":6689,"Rec":6690,"atures":6691,"Ġlinked":6692,"Ġdiscussed":6693,"Ġexplo":6694,"Ġoccasion":6695,"athan":6696,"Ġopposite":6697,"Ġfaces":6698,"Ġdenied":6699,"ĠKnight":6700,"Ġnut":6701,"Ġapproximately":6702,"Ġdisappoint":6703,"onymous":6704,"ĠBest":6705,"ĠLo":6706,"ĠHy":6707,"ĠAff":6708,"Ġvoting":6709,"anwhile":6710,"ĠIII":6711,"Ġinstitutions":6712,"agram":6713,"ĠDaily":6714,"Ġdrag":6715,"Ġnearby":6716,"Ġguilty":6717,"Ġconver":6718,"Pre":6719,"ship":6720,"Ġreward":6721,"Ġphilosoph":6722,"ĠSS":6723,"ugh":6724,"Ġapps":6725,"friend":6726,"Ġupper":6727,"Ġadvert":6728,"Ġsnow":6729,"Ġfrust":6730,"Ġourselves":6731,"Fr":6732,"ĠDie":6733,"ampion":6734,"Ġdismiss":6735,"Ġcere":6736,"Ġsignal":6737,"from":6738,"Ġ).":6739,"Ġ52":6740,"Ġcrimes":6741,"itors":6742,"estival":6743,"useum":6744,"Ġcouncil":6745,"ĠSaud":6746,"May":6747,"ĠGun":6748,"ician":6749,"ether":6750,"Ġsufficient":6751,"ĠHen":6752,"sole":6753,"Ġhistorical":6754,"ĠFar":6755,"ĠTurn":6756,"Ġpin":6757,"Ġsucceed":6758,"mat":6759,"lymp":6760,"Ġtradition":6761,"ĠOk":6762,"Ġcro":6763,"Ġdescription":6764,"alle":6765,"Ġsky":6766,"Te":6767,"Ġwidely":6768,"Ġwave":6769,"Ġdefinition":6770,"ĠJews":6771,"Ġcycle":6772,"Ġrefere":6773,"Ġbrings":6774,"usal":6775,"Ġalive":6776,"Ġfrequently":6777,"Ġintention":6778,"ĠControl":6779,"lv":6780,"ystem":6781,"Ġprivacy":6782,"gent":6783,"rence":6784,"ĠQuest":6785,"ĠChristmas":6786,"Ġrail":6787,"Ġcooper":6788,"Ġtested":6789,"ĠCapt":6790,"asks":6791,"Ġcomfortable":6792,"Ġdelivered":6793,"scape":6794,"Ġdepth":6795,"ĠGOP":6796,"Ġwrites":6797,"Ġassets":6798,"Ġsav":6799,"iments":6800,"Ġtransition":6801,"Ġartist":6802,"ĠLook":6803,"Ġlob":6804,"Ġcomponents":6805,"arity":6806,"Ġwalked":6807,"Ġroot":6808,"Ġparticipants":6809,"Ġnoticed":6810,"Ġresc":6811,"Ġnav":6812,"ĠAdminist":6813,"da":6814,"utral":6815,"plate":6816,"Ġimportance":6817,"Ġassert":6818,"iously":6819,"cription":6820,"Ġinjuries":6821,"ĠCheck":6822,"Ġregistered":6823,"Ġintent":6824,"Ġmissed":6825,"ographic":6826,"Ġsentence":6827,"ounter":6828,"Ġassistance":6829,"evin":6830,"Ġdatabase":6831,"Ġbuildings":6832,"Ġclassic":6833,"Ġthinks":6834,"ĠOhio":6835,"Pr":6836,"ugg":6837,"Ġfee":6838,"pan":6839,"Ġeffectively":6840,"Ġfacility":6841,"Ġbear":6842,"Ġchapter":6843,"Ġdogs":6844,"ĠColumb":6845,"Ġlatter":6846,"itial":6847,"Ġadmitted":6848,"TV":6849,"ĠGeorg":6850,"Ġposts":6851,"\\\\":6852,"Ġlawyer":6853,"Ġequival":6854,"Ġmand":6855,"Ġcontrolled":6856,"ĠWalk":6857,"ĠAndrew":6858,"Ġmenu":6859,"amental":6860,"Ġprotected":6861,"va":6862,"Ġadministr":6863,"oral":6864,"Ġrein":6865,"ĠSar":6866,"Ġamounts":6867,"Ġnative":6868,"ĠMoon":6869,"Ġrepresents":6870,"Ġabandon":6871,"Ġcarrying":6872,"Ġtank":6873,"mary":6874,"Ġdeclared":6875,"Tube":6876,"Ġhat":6877,"Ġpunish":6878,"ellect":6879,"mes":6880,"Ġuniverse":6881,"ĠRod":6882,"phy":6883,"Ġinfrastructure":6884,"Ġ51":6885,"Ġopposed":6886,"ownt":6887,"ca":6888,"ĠMake":6889,"Ġhardware":6890,"Ġcoffee":6891,"Rel":6892,"bal":6893,"world":6894,"ĠSaf":6895,"ĠSea":6896,"inals":6897,"Ġowned":6898,"Ġhall":6899,"ersion":6900,"Ġdescribe":6901,"ĠPot":6902,"Ġportion":6903,"Ġatmosp":6904,"Ġgovernments":6905,"Ġdepending":6906,"Ġoffense":6907,"Ġtrick":6908,"awa":6909,"ĠLine":6910,"ĠVis":6911,"ĠHard":6912,"ĠOrig":6913,"ĠClick":6914,"Ġdesk":6915,"ĠValley":6916,"ĠSov":6917,"Ġmovies":6918,"Ġremark":6919,"Ġmail":6920,"Ġconscious":6921,"Ġruling":6922,"ĠRights":6923,"Ġmedic":6924,"hent":6925,"ĠWomen":6926,"><":6927,"Ġreplaced":6928,"ĠPrem":6929,"ĠThanks":6930,"Ġrenew":6931,"ĠBall":6932,"iform":6933,"Ġshots":6934,"Comm":6935,"Ġarmed":6936,"Ġconstant":6937,"Ġtaste":6938,"Ġrealized":6939,"Ġbuff":6940,"Ġmo":6941,"Ġefficient":6942,"Most":6943,"oration":6944,"ifies":6945,"Ġcommunication":6946,"Ġflood":6947,"Ġconsequences":6948,"Ġanyway":6949,"igg":6950,"ĠGM":6951,"ĠThank":6952,"Ġiron":6953,"Ġevolution":6954,"ĠCop":6955,"twitter":6956,"Ġ95":6957,"Ġrelationships":6958,"adel":6959,"ĠYoung":6960,"Ġproposal":6961,"ayers":6962,"uilding":6963,"ĠHot":6964,"ORE":6965,"cos":6966,"Ġcollabor":6967,"PG":6968,"axy":6969,"Ġknowing":6970,"Ġsupports":6971,"owed":6972,"Ġcontrols":6973,"Ġmerely":6974,"umer":6975,"Ġathlet":6976,"Ġfashion":6977,"path":6978,"Ġgift":6979,"Ġera":6980,"AND":6981,"Ġkinds":6982,"ĠKorean":6983,"Ġlegit":6984,"ulous":6985,"Ġessentially":6986,"Ġtherap":6987,"nic":6988,"Ġsuffered":6989,"Ġhur":6990,"Ġpromise":6991,"Ġexcess":6992,"Ġoverw":6993,"Ġprime":6994,"ĠHouston":6995,"erry":6996,"ĠMs":6997,"RS":6998,"2012":6999,"Ġstores":7000,"ĠOlymp":7001,"Ġjourney":7002,"Although":7003,"Sub":7004,"ĠEduc":7005,"ĠChapter":7006,"Ġrequests":7007,"Ġconsumers":7008,"Ġtiny":7009,"Ġisol":7010,"ĠFair":7011,"ba":7012,"ĠYOU":7013,"Ġcrash":7014,"celer":7015,"Ġemotional":7016,"Ġgoods":7017,"Ġelected":7018,"Ġmoder":7019,"ĠLinux":7020,"Ġblocks":7021,"Ġisland":7022,"ĠSociety":7023,"Ġelections":7024,"Ġbroadcast":7025,"Ġcheap":7026,"Ġnations":7027,"Ġseasons":7028,"400":7029,"Ġwaste":7030,"ĠSat":7031,"Ġfields":7032,"employ":7033,"Ġprofile":7034,"Ġauthors":7035,"ALL":7036,"ĠGra":7037,"west":7038,"ĠTy":7039,"Ġdeaths":7040,"Ġvacc":7041,"Ġformed":7042,"Ġdu":7043,"Ġongoing":7044,"ĠMuslims":7045,"elf":7046,"igure":7047,"Ġassume":7048,"ĠUkraine":7049,"water":7050,"Ġcoast":7051,"Ġvoted":7052,"gor":7053,"ĠAS":7054,"ĠMichigan":7055,"aza":7056,"ĠArm":7057,"iro":7058,"Ġflex":7059,"asters":7060,"''":7061,"Ġwelcome":7062,"arl":7063,"Ġlocations":7064,"igation":7065,"ĠFil":7066,"Ġbuying":7067,"Ġarchitect":7068,"Ġharder":7069,"ĠCub":7070,"Ġinterface":7071,"Ġrestaurant":7072,"Ġdiscover":7073,"Ġexceed":7074,"Ġfavour":7075,"gery":7076,"Ġduty":7077,"Ġpitch":7078,"ador":7079,"ĠMach":7080,"boy":7081,"Ġresponded":7082,"Ġextended":7083,"hers":7084,"Many":7085,"raid":7086,"ifer":7087,"ĠIns":7088,"Ser":7089,"Ġmedium":7090,"she":7091,"ĠSports":7092,"Ġmagazine":7093,"utation":7094,"Ġlimits":7095,"ĠGall":7096,"Ġexternal":7097,"razil":7098,"Ġyounger":7099,"tle":7100,"Ġremind":7101,"ĠCON":7102,"Ġimmediate":7103,"Ġhidden":7104,"Ġvolunte":7105,"Ġsimpl":7106,"odcast":7107,"Ġphase":7108,"dr":7109,"Ġplot":7110,"Ġexposure":7111,"RI":7112,"ograp":7113,"vin":7114,"anish":7115,"ĠAcad":7116,"ĠEngine":7117,"Ġexpansion":7118,"ĠPay":7119,"Your":7120,"Ġpushed":7121,"ĠEll":7122,"ĠHead":7123,"Ġmarketing":7124,"ĠAC":7125,"ket":7126,"Ġhits":7127,"Ġgro":7128,"ĠAge":7129,"ĠScot":7130,"][":7131,"Ġstim":7132,"ĠiPhone":7133,"ĪĴ":7134,"Ġnarrow":7135,"ĠGetty":7136,"ĠTurkey":7137,"Ġperfectly":7138,"Ġenable":7139,"utch":7140,"Ġprecise":7141,"Ġregime":7142,"Ġshif":7143,"Ġcompens":7144,"gun":7145,"div":7146,"Ġchosen":7147,"ĠKen":7148,"Any":7149,"Ġtrees":7150,"Ġrecommended":7151,"ĠRen":7152,"uable":7153,"ĠHT":7154,"Follow":7155,"EG":7156,"ĠHand":7157,"ĠKenn":7158,"Ġarguments":7159,"Ġexists":7160,"Ġbike":7161,"ĠConserv":7162,"Ġbreaking":7163,"ĠGar":7164,"Ġcrazy":7165,"Ġvirtual":7166,"aylor":7167,"ixel":7168,"Ġ1980":7169,"Ġpermission":7170,"ĠSeries":7171,"Ġconsumer":7172,"Ġclosely":7173,"called":7174,"Ġ54":7175,"Ġhopes":7176,"Ġarray":7177,"ĠWin":7178,"ĠLabour":7179,"Ġspons":7180,"ĠIre":7181,"Ġpow":7182,"Ġreaders":7183,"Ġemployment":7184,"Ġcreature":7185,"Ġresulting":7186,"Ġaccurate":7187,"Ġmoments":7188,"Ġargued":7189,"Ġped":7190,"During":7191,"Ġ53":7192,"ĠTal":7193,"Ġsought":7194,"Ġsuffering":7195,"Ġicon":7196,"lee":7197,"Ġ($":7198,"alian":7199,"°":7200,"Ġpra":7201,"Ġbonus":7202,"(\"":7203,"ko":7204,"Ġacting":7205,"DE":7206,"fall":7207,"Ġcomparison":7208,"Ġsmooth":7209,"ĠNAS":7210,"upp":7211,"ĠJoseph":7212,"eping":7213,"ĠTake":7214,"ĠMid":7215,"Ġsending":7216,"fast":7217,"ĠFall":7218,"Ġdealing":7219,"user":7220,"ĠOrgan":7221,"Co":7222,"Ġattached":7223,"Ġsees":7224,"%.":7225,"Ġtypical":7226,"ART":7227,"Ġfinds":7228,"ĠAsia":7229,"umin":7230,"ĠCore":7231,"ĠEnt":7232,"inent":7233,"uce":7234,"ĠBlood":7235,"ĠNever":7236,"Ġemails":7237,"Ġhighlight":7238,"Ġconfront":7239,"atus":7240,"uted":7241,"Ġunus":7242,"Ġtopic":7243,"ĠAdam":7244,"Ġble":7245,"ati":7246,"Ġunderstood":7247,"Set":7248,"struct":7249,"TP":7250,"Ġmob":7251,"aa":7252,"ĠStart":7253,"pected":7254,"sell":7255,"Ġdedicated":7256,"ĠCA":7257,"uan":7258,"Ġsongs":7259,"escription":7260,"Ġtech":7261,"Ġrape":7262,"Ġaside":7263,"Ġgrant":7264,"Ġ56":7265,"sub":7266,"Ġargue":7267,"Ġcontaining":7268,"Ġschedule":7269,"Ġliberal":7270,"Ġpublicly":7271,"Ġheavily":7272,"ĠUt":7273,"iner":7274,"ĠSection":7275,"ĠCare":7276,"weet":7277,"ls":7278,"Dis":7279,"âĶĢ":7280,"ĠFollow":7281,"Back":7282,"ĠIT":7283,"Ġbes":7284,"ji":7285,"ĠHit":7286,"ested":7287,"Ġeverybody":7288,"ĠSwed":7289,"Ġfemin":7290,"Ġfacilities":7291,"Ġconven":7292,"Comp":7293,"ĠOS":7294,"core":7295,"Ġanx":7296,"Ġdivision":7297,"ĠCam":7298,"ĠStan":7299,"mates":7300,"Ġexplore":7301,"plom":7302,"Ġshares":7303,"pload":7304,"anes":7305,"Ġideal":7306,"eters":7307,"ĠBase":7308,"Ġplastic":7309,"Ġdistinct":7310,"ĠNetwork":7311,"ĠSeattle":7312,"Ġtrading":7313,"ensus":7314,"intend":7315,"Ġexhib":7316,"Ġinitially":7317,"ĠFood":7318,"Ġthousand":7319,"ĠBusiness":7320,"acter":7321,"Ġparagraph":7322,"Ġroughly":7323,"Ġwww":7324,"Ġcreative":7325,"ĠConf":7326,"Ġconsumption":7327,"Ġfilms":7328,"agan":7329,"Ġobtain":7330,"Ġtall":7331,"Ġtor":7332,"Ġacknowled":7333,"Ġgrown":7334,"alo":7335,"KE":7336,"Ġ400":7337,"enders":7338,"taining":7339,"UG":7340,"Ġsuicide":7341,"Ġwatched":7342,"ĠList":7343,"ali":7344,"rehens":7345,"Ġsurrounding":7346,"Ġpip":7347,"Ġflying":7348,"ĠJava":7349,"ordan":7350,"Ġserving":7351,"inations":7352,"post":7353,"Ġsho":7354,"Av":7355,"Ġjail":7356,"zy":7357,"Ġ1999":7358,"Ġ>":9609,"orous":9610,"Ġfirms":9611,"screen":9612,"una":9613,"Ġembarrass":9614,"ulse":9615,"Ġletting":9616,"Ġthrew":9617,"iley":9618,"Ġchannels":9619,"lan":9620,"ĠVegas":9621,"Ġsear":9622,"Ġfantastic":9623,"arre":9624,"uzzle":9625,"ĠDer":9626,"Those":9627,"Ġswing":9628,"Ġsheet":9629,"index":9630,"cover":9631,"ogan":9632,"Ġvariables":9633,"ĠTech":9634,"Ġspoken":9635,"achel":9636,"ĠDa":9637,"ĠMountain":9638,"Ġloaded":9639,"Ġfootage":9640,"version":9641,"Ġunl":9642,"ĠPhoenix":9643,"Ġthrowing":9644,"Ġfiring":9645,"Ġtracking":9646,"Ġwidth":9647,"Ġstruggling":9648,"rooms":9649,"otion":9650,"Ġmonthly":9651,"ĠServer":9652,"Ġeggs":9653,"open":9654,"MC":9655,"Ġ1993":9656,"Ġhired":9657,"Ġstayed":9658,"ĠAllen":9659,"Ġstro":9660,"Ġ98":9661,"step":9662,"ĠTurkish":9663,"Ġfabric":9664,"isting":9665,"ĠDom":9666,"Ġdates":9667,"Ġpron":9668,"Ġbasketball":9669,"Ġlucky":9670,"ĠArabia":9671,"Ġassumed":9672,"esty":9673,"Ġaffairs":9674,"Ġglad":9675,"ĠIndeed":9676,"ĠFA":9677,"ĠWord":9678,"Ġjoining":9679,"ifice":9680,"pread":9681,"irts":9682,"ĠSelect":9683,"Ġpopulations":9684,"aware":9685,"Ġnose":9686,"Ġcomplaints":9687,"start":9688,"Ġscoring":9689,"Thanks":9690,"Ġmining":9691,"Ġvisitors":9692,"SH":9693,"Ġdamaged":9694,"Ġcharacteristics":9695,"ĠPent":9696,"DC":9697,"Ġ83":9698,"ĠSix":9699,"rates":9700,"Ġflags":9701,"ĠBrew":9702,"dog":9703,"Mark":9704,"////":9705,"Ġexecution":9706,"Ġjoke":9707,"phones":9708,"Ġtestimony":9709,"Ġobst":9710,"QL":9711,"ĠCut":9712,"Ġstudied":9713,"ĠNintendo":9714,"icket":9715,"ĠNBC":9716,"Ġlad":9717,"ĠBra":9718,"ĠMoh":9719,"Ġkernel":9720,"Ġoverwhelming":9721,"Ġaged":9722,"Ġapplicable":9723,"ĠCond":9724,"Ġroads":9725,"ĠBlock":9726,"made":9727,"odge":9728,"Ġcommands":9729,"Ġoffices":9730,"veland":9731,"Ġtut":9732,"Ġreceiver":9733,"ĠFro":9734,"Ġshopping":9735,"ĠiP":9736,"ĠStre":9737,"ĠABC":9738,"Ġentertainment":9739,"ĠBow":9740,"orted":9741,"Mc":9742,"Ġreads":9743,"grad":9744,"ĠCollect":9745,"ĠâĪĴ":9746,"ĠCapital":9747,"ederation":9748,"Ġemployer":9749,"Ġinvolvement":9750,"Ġanxiety":9751,"alia":9752,"Ġroof":9753,"ĠAmong":9754,"ĠDemocrat":9755,"Ġstats":9756,"ĠVill":9757,"Ġconstitutional":9758,"Ġreferring":9759,"itty":9760,"Ġtackle":9761,"outube":9762,"Ġbacked":9763,"ĠHong":9764,"ĠBroad":9765,"Ġele":9766,"ĠOtt":9767,"Ġ1992":9768,"hour":9769,"achusetts":9770,"Cal":9771,"Ġdefeated":9772,"Ġ81":9773,"esp":9774,"Ġseemingly":9775,"was":9776,"ĠJenn":9777,"ĠKurd":9778,"Ġgene":9779,"Ġdiscount":9780,"Ret":9781,"ECT":9782,"();":9783,"Ġclubs":9784,"Ġsid":9785,"ĠMarsh":9786,"Check":9787,"Ġpp":9788,"ĠEag":9789,"idespread":9790,"Ġbeings":9791,"FT":9792,"Ġintroduction":9793,"ĠChange":9794,"ARD":9795,"Ġ110":9796,"adows":9797,"ierce":9798,"Ġmeal":9799,"author":9800,"ĠBang":9801,"lahoma":9802,"Ġranks":9803,"2011":9804,"????":9805,"max":9806,"Ġcollapse":9807,"Ġopens":9808,"Ġecho":9809,"Ġsoph":9810,"Ġracist":9811,"Ġenormous":9812,"Ġwaves":9813,"Ġtap":9814,"Ġcomprehensive":9815,".--":9816,"ĠRoy":9817,"Ġfarmers":9818,"Related":9819,"aired":9820,"rones":9821,"ĠCrim":9822,"Ġproportion":9823,"Ġdesigns":9824,"Ġnegotiations":9825,"Ġvirtually":9826,"ĠBatman":9827,"Ġwarn":9828,"Ġlegitimate":9829,"mate":9830,"Ġconvention":9831,",,":9832,"netic":9833,"ĠSD":9834,"Ġconsistently":9835,"Ġcompensation":9836,"Ġpunishment":9837,"Ġye":9838,"Ġtie":9839,"ĠBureau":9840,"irlf":9841,"ĠBu":9842,"ĠAren":9843,"ĠPhilipp":9844,"Ġknife":9845,"Ġmemories":9846,"ĠRoss":9847,"Ġangle":9848,"Ġ86":9849,"ĠThunder":9850,"Ġrend":9851,"ĠTour":9852,"Ġcounts":9853,"sung":9854,"ĠImp":9855,"Ġeducational":9856,"Ġaccessible":9857,"COM":9858,"Ġdrew":9859,"yer":9860,"Gl":9861,"amine":9862,"ORT":9863,"OB":9864,"IB":9865,"master":9866,"Ġtrials":9867,"ogy":9868,"har":9869,"ĠTrust":9870,"Ġpreferred":9871,"irlfriend":9872,"ĠNev":9873,"Ġbin":9874,"Ġcow":9875,"Page":9876,"Ġsignature":9877,"ĠBL":9878,"700":9879,"Ġretired":9880,"Ġbytes":9881,"Ġneighb":9882,"ĠLegend":9883,"Ġdevast":9884,"Ġsuspected":9885,"isons":9886,"ĠPokémon":9887,"scale":9888,"Ġcapabilities":9889,"Ġrevel":9890,"Ġcheese":9891,"dy":9892,"igrant":9893,"Ġfailing":9894,"bits":9895,"ĠHeroes":9896,"ĠGhost":9897,"ĠScient":9898,"Ġappointed":9899,"uri":9900,"Ġinstitution":9901,"Ġexpanded":9902,"greg":9903,"Ġmonitoring":9904,"Ġpodcast":9905,"Ġcoalition":9906,"Ġ96":9907,"Jo":9908,"Ġstolen":9909,"ĠSab":9910,"Ġstops":9911,"Ġholiday":9912,"Ġintr":9913,"Car":9914,"Black":9915,"ĠLGBT":9916,"Ġwarming":9917,"ĠAnderson":9918,"Ġ89":9919,"Ġproducer":9920,"Med":9921,"Ġaccuracy":9922,"ĠMarvel":9923,"izabeth":9924,"ĠPatrick":9925,"mony":9926,"Ġmini":9927,"acles":9928,"Ġovert":9929,"they":9930,"Ġmembership":9931,"ĠVen":9932,"Ġexch":9933,"Ġremoval":9934,"ĠDave":9935,"TY":9936,"mad":9937,"ĠFind":9938,"Ġadequ":9939,"Ġec":9940,"Ġteeth":9941,"Ġemotion":9942,"Ġperm":9943,"Ġsolely":9944,"db":9945,"Ġextraord":9946,"IGHT":9947,"cal":9948,"Ġguidelines":9949,"Ġdying":9950,"Ġsuspended":9951,"ĠPremier":9952,"ĠAnthony":9953,"elve":9954,"Ġdad":9955,"ĠEth":9956,"ĠFootball":9957,"Ġabandoned":9958,"Ġ<<":9959,"Ġmarch":9960,"Ġhorror":9961,"âĢ¦\"":9962,"Ġchildhood":9963,"Ġcampaigns":9964,"Ġlunch":9965,"ĠAlbert":9966,"block":9967,"âĸĪâĸĪ":9968,"ounding":9969,"Ġbone":9970,"organ":9971,"aders":9972,"ĠFlash":9973,"ĠDrive":9974,"Ġtonight":9975,"Ġwars":9976,"ĠFL":9977,"Ġformation":9978,"const":9979,"News":9980,"Ġcompe":9981,"orious":9982,"ĠStaff":9983,"Ġdiscussions":9984,"ĠProtection":9985,"ĠJam":9986,"Ġcriteria":9987,"Ġinstallation":9988,"Ġaccomplish":9989,"izza":9990,"Ġpublisher":9991,"Ġrescue":9992,"ĠTry":9993,"ULL":9994,"ĠSom":9995,"ĠHop":9996,"oret":9997,"ths":9998,"ordon":9999,"Ġpocket":10000,"ĠInv":10001,"Download":10002,"ĠCrime":10003,"Ġbene":10004,"ĠGuide":10005,"ĠAssembly":10006,"Ġparameters":10007,"IE":10008,"ĠAlexander":10009,"Ġconcert":10010,"ĠSche":10011,"Ġshoes":10012,"Ġvisiting":10013,"Ġrecall":10014,"Ġbub":10015,"Ġrural":10016,"Ġconcrete":10017,"ĠRos":10018,"Next":10019,"Russ":10020,"Ġloans":10021,"ĠShield":10022,"Ġtrem":10023,"hemat":10024,"kg":10025,"ĠHarris":10026,"isition":10027,"ĠMove":10028,"ĠFC":10029,"Ġfate":10030,"ĠCho":10031,"Ġtired":10032,"Ġprincipal":10033,"hist":10034,"iences":10035,"athy":10036,"Ġsevent":10037,"Ġmood":10038,"Ġstrategic":10039,"Ġdiseases":10040,"Ġforum":10041,"Ġtempor":10042,"Ġheadquarters":10043,"Par":10044,"ige":10045,"flix":10046,"Ġguitar":10047,"Ġ94":10048,"Only":10049,"Ġreleases":10050,"roph":10051,"================================":10052,"Ġ600":10053,"ĠContinue":10054,"igate":10055,"ĠCrit":10056,"system":10057,"Ġdisabled":10058,"Ġunexpected":10059,"ithub":10060,"Ġunclear":10061,"ĠEst":10062,"Ġcontrad":10063,"Ġstrategies":10064,"ventures":10065,"Ġpassage":10066,"AME":10067,"Ġimproving":10068,"Ġreveals":10069,"Ġdecrease":10070,"ova":10071,"Ġannoy":10072,"ĠShort":10073,"ĠLibrary":10074,"Ġcyber":10075,"nell":10076,"ĠHur":10077,"ĠCB":10078,"Ġphotograp":10079,"UI":10080,"Ġsed":10081,"Ge":10082,"Ġ87":10083,"Ġdiverse":10084,"Ġencouraged":10085,"Ġconspiracy":10086,"Ġbirds":10087,"Ġoperator":10088,"Ġhandful":10089,"Ġclassified":10090,"?)":10091,"Ġdramatic":10092,"Ġinvestigators":10093,"ito":10094,"Ġwidespread":10095,"ĠRoom":10096,"----------------------------------------------------------------":10097,"Ġcollective":10098,"Ġjournalist":10099,"String":10100,"Ġtemperatures":10101,"ila":10102,"Ġguid":10103,"Ġinspect":10104,"Ġmissile":10105,"ĠMayor":10106,"Ġmanual":10107,"Ġsimultane":10108,"Ġratings":10109,"Ġsuck":10110,"Ġ97":10111,"Ġuniversal":10112,"Ġpharm":10113,"Ġdisrupt":10114,"iano":10115,"AV":10116,"Ġft":10117,"Ġstatist":10118,"olds":10119,"ĠWalker":10120,"php":10121,"Ġundert":10122,"ĠLas":10123,"ishop":10124,"ntil":10125,"reshold":10126,"ĠWhether":10127,"Ms":10128,"Ġdeny":10129,"ĠCloud":10130,"Ġprovider":10131,"Ġsurviv":10132,"ĠUpdate":10133,"has":10134,"Ġmistakes":10135,"charge":10136,"pled":10137,"rity":10138,"Ġnode":10139,"ĠMassachusetts":10140,"ools":10141,"lication":10142,"Ġfails":10143,"emale":10144,"ori":10145,"backs":10146,"Ġshirt":10147,"Ġ''":10148,"ĠNAT":10149,"Ġwaters":10150,"elson":10151,"Ġease":10152,"Ġscar":10153,"Ġcontents":10154,"mind":10155,"Ġcontribution":10156,"Ġshr":10157,"Ġhanded":10158,"Ġstability":10159,"Ġtrave":10160,"Em":10161,"Ġmirror":10162,"123":10163,"Ġweigh":10164,"Ġfiction":10165,"ouver":10166,"istant":10167,"rition":10168,"ĠFed":10169,"Ġphysically":10170,"Ġstake":10171,"ĠArticle":10172,"ĠArc":10173,"ĠLewis":10174,"ĠMind":10175,"Ġdemonstrate":10176,"Ġprofits":10177,"vision":10178,"omic":10179,"olid":10180,"Ġbattles":10181,"Ġdrives":10182,"Ġeastern":10183,"ĠSony":10184,"!!!":10185,"aration":10186,"vard":10187,"ĠGL":10188,"portation":10189,"Ġ92":10190,"Ġlawmakers":10191,"Ġprotecting":10192,"ĠEPA":10193,"Ġyeah":10194,"Ġshame":10195,"olph":10196,"even":10197,"xit":10198,"Ġattach":10199,"Ġrepresenting":10200,"Ġobs":10201,"ĠUtah":10202,"iffs":10203,"ĠFreedom":10204,"ó":10205,"AK":10206,"Ġincidents":10207,"itage":10208,"Ġviewers":10209,"cd":10210,"Ġmouse":10211,"Ġclar":10212,"Ġaccordance":10213,"Ġbot":10214,"cor":10215,"ĠSummer":10216,"held":10217,"Ġinnocent":10218,"Ġinitiative":10219,"ols":10220,"________________________________":10221,"Ġspots":10222,"pace":10223,"Ġconventional":10224,"Ġcorporations":10225,"Ġblocked":10226,"HD":10227,"attered":10228,"Ġrefers":10229,"Ġbuck":10230,"ĠDigital":10231,"120":10232,"Ġtopics":10233,"TF":10234,"Äģ":10235,"brid":10236,"reement":10237,"Ġunderlying":10238,"ĠMember":10239,"Ġinvestigating":10240,"Ġpregnancy":10241,"Ġtouchdown":10242,"ĠBand":10243,"ĠCaller":10244,"Ġinstances":10245,"PP":10246,"wa":10247,"Good":10248,"Ġ1991":10249,"ĠCold":10250,"Ġfears":10251,"Ġremarks":10252,"ĨĴ":10253,"atal":10254,"Ġmit":10255,"Ġexperiments":10256,"ipt":10257,"Color":10258,"indu":10259,"Update":10260,"Ġ93":10261,"Ag":10262,"Ġå":10263,"ancouver":10264,"Both":10265,"Ġjudges":10266,"Object":10267,"Ġstere":10268,"umbn":10269,"Ġparticipation":10270,"ĠStars":10271,"ĠJere":10272,"Ġweekly":10273,"ĠBan":10274,"Ġconversations":10275,"ĠPitt":10276,"uz":10277,"ĠIndiana":10278,"ĠKick":10279,"Ġinfection":10280,"Ġheroes":10281,"Ġsettled":10282,"Ġstrip":10283,"Ġhal":10284,"Ġdump":10285,"ĠSci":10286,"Ġles":10287,"Ġreferences":10288,"ĠURL":10289,"ĠBridge":10290,"Ġwanting":10291,"Force":10292,"Ġexclus":10293,"Meanwhile":10294,"mn":10295,"Ġgentle":10296,"maker":10297,"senal":10298,"ĠGro":10299,"ouri":10300,"ĠRain":10301,"ĠAlliance":10302,"Ġlift":10303,"ela":10304,"SD":10305,"ĠCleveland":10306,"Ġranked":10307,"Ġstadium":10308,"Ġdeadly":10309,"ä¸":10310,"Ġriding":10311,"aria":10312,"ĠArmor":10313,"Ġdocumentation":10314,"ĠGreece":10315,"reek":10316,"Ġlens":10317,"ĠSa":10318,"Ġgross":10319,"ĠEmer":10320,"agers":10321,"ĠDub":10322,"ĠRh":10323,"ĠAMD":10324,"Ġarrival":10325,"Ġdesert":10326,"Ġsupplement":10327,"ĠResp":10328,"Ġknee":10329,"Ġmargin":10330,"font":10331,"ogg":10332,"2010":10333,"ĠPir":10334,"ĠProm":10335,"ivals":10336,"Ġintake":10337,"Ġdifferently":10338,"ugs":10339,"Ġbits":10340,"cluded":10341,"Ġsearching":10342,"ĠDu":10343,"umble":10344,"Ġfunctional":10345,"ĠBaltimore":10346,"ĠCould":10347,"Ġdesired":10348,"Ġcircuit":10349,"ĠLyn":10350,"ĠGO":10351,"ĠFalse":10352,"repre":10353,"':":10354,"alties":10355,"Ġminim":10356,"Ġdrove":10357,"ĠShould":10358,"Ġhip":10359,"Ġpros":10360,"Ġutility":10361,"ĠNature":10362,"ĠMode":10363,"President":10364,"opp":10365,"rat":10366,"formance":10367,"Ġconcentration":10368,"Ġfont":10369,"ĠBud":10370,"Ġamid":10371,"Ġrevers":10372,"ĠML":10373,"Bar":10374,"Ġinteraction":10375,"Ġjurisd":10376,"Ġspells":10377,"dep":10378,"fil":10379,"Ġcivilians":10380,"utter":10381,"ĠCooper":10382,"ĠBelow":10383,"Ġentrance":10384,"Ġconvert":10385,"Ġcontroversy":10386,"owered":10387,"Ġcontrary":10388,"Ġarc":10389,"ĠExecutive":10390,"ĠOfficer":10391,"Ġpackages":10392,"Ġprogressive":10393,"width":10394,"Ġreserved":10395,"vol":10396,"ĠSamsung":10397,"Ġprinted":10398,"Ġcenters":10399,"Ġintroduce":10400,"ĠKennedy":10401,"Ġodds":10402,"Ġsurely":10403,"Ġindependence":10404,"Ġpassengers":10405,"reprene":10406,"ĠBeh":10407,"Ġloves":10408,"ĠESPN":10409,"Ġfacilit":10410,"Ġidentical":10411,"Ġdoct":10412,"Ġpartnership":10413,"conf":10414,"ĠHide":10415,"Ġconfused":10416,"ĠCow":10417,"Men":10418,"Ġwrest":10419,"ĠIraqi":10420,"Ġholes":10421,"ĠStudies":10422,"Ġpregnant":10423,"hard":10424,"Ġsignals":10425,"IX":10426,"Ġpulling":10427,"Ġgraduate":10428,"Ġnominee":10429,"Date":10430,"Ġpermitted":10431,"ĠâĤ¬":10432,"ĠOklahoma":10433,"Start":10434,"Ġauthorized":10435,"Ġalarm":10436,"ĠCos":10437,"van":10438,"Ġgenerations":10439,"cular":10440,"Ġdragon":10441,"ĠSoftware":10442,"ĠEdward":10443,"Ġcontroller":10444,"Sen":10445,"gered":10446,"ĠVik":10447,"Ġapproached":10448,"Thank":10449,"Ġcance":10450,"Ġformula":10451,"ĠSmall":10452,"Ġweakness":10453,"Ġramp":10454,"itudes":10455,"jud":10456,"Ġbrilliant":10457,"Ġaccus":10458,"source":10459,"Ġ800":10460,"ĠEvil":10461,"Sw":10462,"Ġhomeless":10463,"week":10464,"iens":10465,"rics":10466,"ĠThird":10467,"TO":10468,"Ġorganic":10469,"Ġpresentation":10470,"agh":10471,"ĠDownload":10472,"vation":10473,"Ġassembly":10474,"orable":10475,"holders":10476,"ĠBernie":10477,"ĠHelp":10478,"Ġtong":10479,"ĠFight":10480,"Ġbeach":10481,"Book":10482,"ĠLic":10483,"Ġrush":10484,"ĠRound":10485,"oup":10486,"ĠMarx":10487,"Ġcalculated":10488,"ĠDevil":10489,"ĠSarah":10490,"Ġoccasionally":10491,"Ġbullet":10492,"Available":10493,"gate":10494,"Ġ91":10495,"Ġhosp":10496,"Ġpromises":10497,"ĠHIV":10498,"ĠStadium":10499,"ĠStock":10500,"ĠCorporation":10501,"gage":10502,"NG":10503,"ĠCredit":10504,"Ġsne":10505,"ibl":10506,"Ġaccum":10507,"such":10508,"Ġterrorists":10509,"Ġconsciousness":10510,"ĠZh":10511,"Ġdrama":10512,"oola":10513,"piration":10514,"Ġlabour":10515,"ĠNin":10516,"Ġutter":10517,"Ġdemocratic":10518,"Ġassass":10519,"ilation":10520,"Ġgest":10521,"Ġabroad":10522,"Ġmetab":10523,"Ġsorts":10524,"Ġflav":10525,"UB":10526,"Ġmg":10527,"ĠNothing":10528,"ĠOd":10529,"Ġmusical":10530,"2009":10531,"Ġdrops":10532,"ocated":10533,"ateral":10534,"000000":10535,"Ġgre":10536,"Ġequality":10537,"Ġburden":10538,"Ġvig":10539,"ĠLeader":10540,"------------":10541,"Ġceremony":10542,"Ġfighter":10543,"Ġactors":10544,"Ġæ":10545,"aman":10546,"Fi":10547,"Ġalign":10548,"puter":10549,"Ġelder":10550,"ĠNSA":10551,"Ġrepresentation":10552,"ĠOntario":10553,"ITH":10554,"usalem":10555,"Ġharassment":10556,"itzer":10557,"Ġsymp":10558,"Ġboxes":10559,"ĠDR":10560,"Ġmanifest":10561,"atre":10562,"Ġ^":10563,"Ġdies":10564,"leton":10565,"Ġmissions":10566,"ethe":10567,"Ġresolve":10568,"Ġfollowers":10569,"Ġasc":10570,"Ġkm":10571,"lord":10572,"ammed":10573,"Ġsilent":10574,"ĠAssociated":10575,"Ġtiming":10576,"Ġprisoners":10577,"ĠKings":10578,"ĠFive":10579,"Ġtower":10580,"Ġapproaches":10581,"Ġprecisely":10582,"Ġbureau":10583,"ĠMother":10584,"ĠIss":10585,"Ġkeyboard":10586,"itual":10587,"Ġfunded":10588,"Ġstaying":10589,"Ġpsychological":10590,"Ġmile":10591,"ĠLeon":10592,"ĠBarb":10593,"will":10594,"Ġwider":10595,"ĠAtlantic":10596,"Ġtill":10597,"ĠRome":10598,"rot":10599,"Ġaccompan":10600,"Ġflour":10601,"aco":10602,"World":10603,"ĠExpress":10604,"ĠYu":10605,"Cor":10606,"Ġpleased":10607,"party":10608,"Ġpointing":10609,"Ġinflation":10610,"Ġroy":10611,"Ġ),":10612,"ainer":10613,"Ġwedding":10614,"ormon":10615,"Ġrequiring":10616,"Ġqualified":10617,"Ġsegment":10618,"END":10619,"Ġsizes":10620,"eals":10621,"Ġcorrupt":10622,"assador":10623,"Ġceleb":10624,"Ġdreams":10625,"ĠMess":10626,"Ġchecking":10627,"ĠVersion":10628,"Ġpreparing":10629,"Ġactively":10630,"ĠDiff":10631,"Ġlux":10632,"ĠWinter":10633,"acteria":10634,"ĠNE":10635,"Ġdeputy":10636,"Ġtransgender":10637,"Ġsummary":10638,"Ġinher":10639,"eries":10640,"char":10641,"ĠYan":10642,"Ġknock":10643,"ĠPath":10644,"Ġlip":10645,"roller":10646,"Ġimpression":10647,"Ġcelebrate":10648,"Ġslide":10649,"Ġguests":10650,"Ġclip":10651,"FS":10652,"Ġsavings":10653,"Ġcaptain":10654,"Ġlegacy":10655,"ĠDenver":10656,"Ġwounded":10657,"taboola":10658,"ACT":10659,"Ġpursue":10660,"Ġoxy":10661,"Ġq":10662,"Ġsemi":10663,"ĠNeed":10664,"ĠAffairs":10665,"Ġobsc":10666,"Ġchecked":10667,"Ġdual":10668,"Code":10669,"ĠMD":10670,"lem":10671,"ulty":10672,"Ġ©":10673,"ĠElizabeth":10674,"Ġcenturies":10675,"arded":10676,"src":10677,"Ġevident":10678,"ennis":10679,"atin":10680,"Ġunemployment":10681,"ĠMario":10682,"Ġintim":10683,"Christ":10684,"Ġbiological":10685,"Ġsoldier":10686,"ĠAdded":10687,"Ġmath":10688,"ĠGil":10689,"Ġbias":10690,"Ġdating":10691,"ĠOcean":10692,"Ġmice":10693,"Mus":10694,"hire":10695,"ĠTes":10696,"Server":10697,"limited":10698,"Size":10699,"Ġmeters":10700,"Ġrocket":10701,"essee":10702,"Ġcertificate":10703,"ĠIranian":10704,"ASS":10705,"Ġgrid":10706,"Dec":10707,"Ġrolling":10708,"commun":10709,"ĠSweden":10710,"bury":10711,"Ġtissue":10712,"Ġracism":10713,"ĠLocal":10714,"Ġmystery":10715,"Ġexamine":10716,"Ġstem":10717,"Ġsits":10718,"Ġhoped":10719,"oting":10720,"Ġdialogue":10721,"Ġpersu":10722,"Watch":10723,"lay":10724,"MAN":10725,"Ġchronic":10726,"ĠPortland":10727,"market":10728,"ĠSEC":10729,"Ġparallel":10730,"Ġscandal":10731,"Ġcarries":10732,"Ġphenomenon":10733,"human":10734,"acker":10735,"ĠOx":10736,"Ġretirement":10737,"tainment":10738,"ovie":10739,"ĠGear":10740,"Ġduties":10741,"Ġdose":10742,"Ġscroll":10743,"MB":10744,"inf":10745,"Ġsauce":10746,"Ġlandscape":10747,"reddit":10748,"ĠChampionship":10749,"ĠReddit":10750,"alid":10751,"Ġcoin":10752,"Ġovers":10753,"Ġposting":10754,"about":10755,"Ġfel":10756,"andy":10757,"Ġbold":10758,"Ġfocusing":10759,"effect":10760,"GR":10761,"Ġdeemed":10762,"Ġrecommendations":10763,"Ġstepped":10764,"Ġvoter":10765,"ĠDeep":10766,"ĠInstagram":10767,"Ġmoderate":10768,"ĠMaryland":10769,"Ġrestricted":10770,"ĠMB":10771,"ĠChall":10772,"Ġtob":10773,"Ġcir":10774,"ĠOcc":10775,"ĠEver":10776,"Ġcollaps":10777,"INFO":10778,"=-":10779,"ĠPict":10780,"ĠAccount":10781,"nc":10782,"Ġought":10783,"Ġexport":10784,"Ġdrunk":10785,"('":10786,"Ġwise":10787,"ĠMort":10788,"necess":10789,"Ġancest":10790,"ĠIncre":10791,"Ġfrequent":10792,"mir":10793,"Ġinterpretation":10794,"Ġdependent":10795,"Ġcoins":10796,"ĠBol":10797,"Video":10798,"ĠJustin":10799,"Ġfatal":10800,"Ġcooking":10801,"Ġconfusion":10802,"ipher":10803,"Ġcustody":10804,"ĠMorgan":10805,"omach":10806,"ĠGovernor":10807,"Ġrestaurants":10808,"eling":10809,"Ġacknowledged":10810,"Ġther":10811,"Ġgenes":10812,"ching":10813,"Hey":10814,"Ġtactics":10815,"ĠMexican":10816,"Ġvend":10817,"Ġhes":10818,"quer":10819,"Ġnoting":10820,"ĠCameron":10821,"Ġtargeting":10822,"rock":10823,"Ġcredits":10824,"Ġemotions":10825,"Ġrepresentatives":10826,"news":10827,"Ġlegislative":10828,"Ġremoving":10829,"Ġtweeted":10830,"ĠCarter":10831,"ĠFixed":10832,"Ġforcing":10833,"Ġspeaker":10834,"Ġmales":10835,"ĠVietnam":10836,"lined":10837,"Ġconcepts":10838,"Ġvoices":10839,"oir":10840,"ĠTrib":10841,"Whe":10842,"ĠJerusalem":10843,"ĠSant":10844,"Ġcul":10845,"Ġlady":10846,"ĠHawai":10847,"Ġarts":10848,"ĠInn":10849,"ĠMachine":10850,"ĠEmperor":10851,"Ġslot":10852,"gly":10853,"ĠProcess":10854,"III":10855,"Ġathletes":10856,"ĠTemple":10857,"ĠRepresent":10858,"Ġpresc":10859,"Ġtons":10860,"Ġgolden":10861,"Ġpunch":10862,"ĠGR":10863,"iverpool":10864,"Ġenact":10865,"Ġlobby":10866,"Ġmos":10867,"Ġpicking":10868,"Ġlifetime":10869,"Ġcognitive":10870,"Each":10871,"zo":10872,"Ġdub":10873,"Ġconsists":10874,"oln":10875,"Ġfestival":10876,"amous":10877,"Ġintellig":10878,"words":10879,"ĠSmart":10880,"Ġdele":10881,"Ġlapt":10882,"Ġmagical":10883,"ĠSin":10884,"bus":10885,"urities":10886,"ighth":10887,"ĠRuby":10888,"ĠSure":10889,"olving":10890,"Ġjun":10891,"OST":10892,"Ġimposed":10893,"Ġastron":10894,"Ġcorrel":10895,"ĠNS":10896,"ĠKit":10897,"ĠFuture":10898,"burn":10899,"Ġimmune":10900,"ocus":10901,"Ġcourses":10902,"ĠString":10903,"Ġlean":10904,"Ġghost":10905,"Ġoutcomes":10906,"Ġexpense":10907,"Ġeveryday":10908,"Ġacceptable":10909,"Ah":10910,"Ġequipped":10911,"Ġorange":10912,"FR":10913,"ĠDutch":10914,"Though":10915,"ĠRank":10916,"QU":10917,"ĠRoberts":10918,"what":10919,"rend":10920,"Ġdisappear":10921,"Ġspawn":10922,"ĠLam":10923,"ois":10924,"Ġdeserve":10925,"Ġminimal":10926,"Ġnervous":10927,"ĠWould":10928,"Ġrook":10929,"ĠVancouver":10930,"Ġresign":10931,"shire":10932,"ĠWorks":10933,"ĠBuild":10934,"Ġaffordable":10935,"ĠGary":10936,"ĠArena":10937,"Ġhanging":10938,"Ġimplications":10939,"ĠSong":10940,"Ġmaintaining":10941,"Ġguards":10942,"CON":10943,"Ġderived":10944,"Ġexecuted":10945,"Ġtheories":10946,"Ġquoted":10947,"ĠAndre":10948,"oga":10949,"seless":10950,"info":10951,"ĠBelg":10952,"Ġtears":10953,"ĠSurv":10954,"Ġbirthday":10955,"igious":10956,"immer":10957,"Ġspectrum":10958,"Ġarchitecture":10959,"Ġrecruit":10960,"arma":10961,"Table":10962,"Ġmonsters":10963,"ĠGov":10964,"Ġdestination":10965,"Ġattractive":10966,"Ġfoss":10967,"ĠMoreover":10968,"Ġpresents":10969,"THE":10970,"Ġreply":10971,"pton":10972,"Ġcum":10973,"Ġdelight":10974,"Ġaffects":10975,"Ġdonations":10976,"ĠToy":10977,"ĠHim":10978,"MENT":10979,"Ġovercome":10980,"itched":10981,"ĠFantasy":10982,"ĠHat":10983,"ĠBeast":10984,"bott":10985,"Ġinvestigations":10986,"Run":10987,"Ġhunting":10988,"di":10989,"fund":10990,"Ġsessions":10991,"estyle":10992,"Ġportray":10993,"oids":10994,"Yeah":10995,"Ġcommunicate":10996,"Ġcomedy":10997,"ĠYang":10998,"Ġbelt":10999,"ĠMarine":11000,"Ġpredicted":11001,"Play":11002,"Ġimportantly":11003,"Ġremarkable":11004,"Ġeliminate":11005,"David":11006,"Ġbind":11007,"VID":11008,"Ġadvocates":11009,"ĠGaza":11010,"imp":11011,"DB":11012,"ĠNa":11013,"ĠSimilar":11014,"IES":11015,"Ġcharity":11016,"vas":11017,"math":11018,"Ġâĸ":11019,"oker":11020,"ndum":11021,"Ġcaps":11022,"ĠHal":11023,"2000":11024,"ean":11025,"Ġfleet":11026,"Ġrecre":11027,"Right":11028,"Ġsleeping":11029,"ijing":11030,"kind":11031,"Ġdesignated":11032,"ä":11033,"Ġanimation":11034,"kee":11035,"ĠIntrodu":11036,"Ġ/>":11037,"Ġdelayed":11038,"Ġtremend":11039,"Ġcurious":11040,"Use":11041,"Ġlect":11042,"dam":11043,"Ġinnovation":11044,"ĠPoints":11045,"Ġloading":11046,"Ġdispute":11047,"ctic":11048,"irds":11049,"ĠBY":11050,"Ġnurs":11051,"ĠValue":11052,"IONS":11053,"ĠHum":11054,"Ġtemplate":11055,"mers":11056,"Ġappearances":11057,"ĠEntertainment":11058,"Ġtranslation":11059,"Ġsake":11060,"Ġbeneath":11061,"Ġinhib":11062,"Ġeuro":11063,"abetes":11064,"Ġstudying":11065,"ĠMas":11066,"Ġperceived":11067,"Ġexamined":11068,"Ġeager":11069,"Ġcoaches":11070,"Ġimper":11071,"chi":11072,"Ġproduces":11073,"\").":11074,"ĠEveryone":11075,"Ġmunicip":11076,"Ġgirlfriend":11077,"Ġhire":11078,"ĠVice":11079,"Ġsuitable":11080,"opy":11081,"Ġinequ":11082,"ĠDuke":11083,"fish":11084,"first":11085,"ĠObs":11086,"Ġinterior":11087,"ĠBruce":11088,"ĠRy":11089,"Ġanalys":11090,"Ġconsiderable":11091,"Ġforecast":11092,"Ġfert":11093,"orship":11094,"ĠDrug":11095,"ĠALL":11096,":\"":11097,"thur":11098,"ĠMail":11099,"Ġballot":11100,"Ġinstantly":11101,"ĠChannel":11102,"Ġpicks":11103,"Ġ1989":11104,"Ġtent":11105,"oli":11106,"Ġcivilian":11107,"bling":11108,"ello":11109,"bu":11110,"Ġinch":11111,"Ġlogo":11112,"Ġcooperation":11113,"Ġwalks":11114,"Ġinvestments":11115,"Ġimprison":11116,"ĠFestival":11117,"ĠKy":11118,"Ġlegally":11119,"Ġgri":11120,"charg":11121,"Sl":11122,"Ġthreatening":11123,"duction":11124,"flow":11125,"Ġdismissed":11126,"ibraries":11127,"cap":11128,"ele":11129,"ĠMcG":11130,"ĠHarvard":11131,"ĠConservative":11132,"ĠCBS":11133,"png":11134,"Ġroots":11135,"ĠHaving":11136,"umbled":11137,"ĠFun":11138,"\\/":11139,"ĠSearch":11140,"plex":11141,"Ġdiscussing":11142,"Ġcontinu":11143,"ĠTai":11144,"ĠWik":11145,"Free":11146,"fit":11147,"Ġrefuse":11148,"Ġmanaging":11149,"Ġsynd":11150,"ipedia":11151,"walk":11152,"Ġprofessionals":11153,"Ġguidance":11154,"Ġuniversities":11155,"Ġassemb":11156,"untu":11157,"Finally":11158,"ASE":11159,"ĠAuto":11160,"ĠHad":11161,"Ġanniversary":11162,"LD":11163,"ĠDur":11164,"ĠUltimate":11165,"ihad":11166,"product":11167,"Ġtransit":11168,"Ġrestore":11169,"Ġexplaining":11170,"Ġasset":11171,"Ġtransferred":11172,"Ġburst":11173,"apolis":11174,"ĠMagazine":11175,"ĠCra":11176,"ĠBR":11177,"gged":11178,"ĠHE":11179,"Mich":11180,"bet":11181,"ĠLady":11182,"ylum":11183,"erves":11184,"Ġmeets":11185,"white":11186,"Log":11187,"Ġcorresponding":11188,"Ġinsisted":11189,"GG":11190,"Ġsurrounded":11191,"Ġtens":11192,"Ġlane":11193,"Ġcoinc":11194,"home":11195,"Ġexisted":11196,"ected":11197,"ĠDouble":11198,"lamm":11199,"Ġskept":11200,"exp":11201,"Ġperception":11202,"iev":11203,"ĠBeing":11204,"oft":11205,"Ġadopt":11206,".:":11207,"];":11208,"Windows":11209,"Ġsatellite":11210,"ASH":11211,"Ġinfant":11212,"description":11213,"ĠMeanwhile":11214,"cm":11215,"oca":11216,"ĠTreat":11217,"actor":11218,"Ġtobacco":11219,"ĠNorm":11220,"emption":11221,"Ġflesh":11222,"Ġje":11223,"oop":11224,"ĠHeaven":11225,"Ġbeating":11226,"anim":11227,"Ġgathering":11228,"Ġcultiv":11229,"GO":11230,"abe":11231,"ĠJonathan":11232,"ĠSafety":11233,"Ġbadly":11234,"prot":11235,"Ġchoosing":11236,"Ġcontacted":11237,"Ġquit":11238,"Ġdistur":11239,"Ġstir":11240,"Ġtoken":11241,"Det":11242,"ĠPa":11243,"Ġfunctionality":11244,"003":11245,"some":11246,"Ġlimitations":11247,"Ġmeth":11248,"build":11249,"config":11250,"NT":11251,"rell":11252,"blem":11253,"ĠMom":11254,"Ġveterans":11255,"ĠHu":11256,"Ġtrends":11257,"arer":11258,"ĠGiven":11259,"ĠCaption":11260,"may":11261,"AST":11262,"Ġwondering":11263,"ĠClark":11264,"normal":11265,"Ġseparated":11266,"Ġdesp":11267,"stic":11268,"brew":11269,"Ġrelating":11270,"ĠNik":11271,"ĠFarm":11272,"Ġenthusi":11273,"good":11274,"deb":11275,"Ġactivist":11276,"Ġmart":11277,"Ġexplosion":11278,"ĠEconomic":11279,"Link":11280,"Ġinsight":11281,"Ġconvenient":11282,"Ġcounterpart":11283,"support":11284,"ĠVirt":11285,"agen":11286,"ĠTennessee":11287,"ĠSimon":11288,"ĠAward":11289,"OCK":11290,"ĠFigure":11291,"Ġoverseas":11292,"Ġpride":11293,"ĠCas":11294,"note":11295,"mg":11296,"Current":11297,"Ġdisplays":11298,"content":11299,"Ġtraveling":11300,"Ġhospitals":11301,"ĠFinancial":11302,"ĠPast":11303,"Ġdefendant":11304,"Ġstreaming":11305,"mble":11306,"ĠBerlin":11307,"uki":11308,"Ġdistribut":11309,"Ġantib":11310,"Ġchocolate":11311,"ĠCastle":11312,"Ġinterrupt":11313,"ĠRow":11314,"Ġconversion":11315,"Ġbugs":11316,"ĠRather":11317,"liest":11318,"LY":11319,"ĠJean":11320,"common":11321,"akh":11322,"Ġ130":11323,"otton":11324,"ĠDean":11325,"Ġamendment":11326,"Ġgameplay":11327,"ĠWarren":11328,"oda":11329,"Ġhighlights":11330,"Ġirre":11331,"ĠNATO":11332,"Ġballs":11333,"Ġdemanding":11334,"URE":11335,"ĠLuke":11336,"Figure":11337,"stop":11338,"onia":11339,"zone":11340,"izers":11341,"ĠWR":11342,"Ġawarded":11343,"Ġregulatory":11344,"ĠHart":11345,"ĠSN":11346,"pling":11347,"Ġsour":11348,"ĠPixel":11349,"usive":11350,"Ġfet":11351,"ĠSent":11352,"Ġautomatic":11353,"Ġfer":11354,"vernment":11355,"ĠKhan":11356,"TON":11357,"father":11358,"Ġextraordinary":11359,"throp":11360,"ĠPython":11361,"ĠGPU":11362,"Ġsexually":11363,"Ġdesktop":11364,"itivity":11365,"ĠAntonio":11366,"Ġorient":11367,"Ġears":11368,"obby":11369,"ouses":11370,"vertisements":11371,"Ġmanufacturers":11372,"icient":11373,"minute":11374,"Ġconviction":11375,"Ġgarden":11376,"public":11377,"Ġsatisfied":11378,"fold":11379,"OK":11380,"Ġinhab":11381,"ĠThink":11382,"Ġprogramme":11383,"Ġstomach":11384,"Ġcoordin":11385,"Ġholy":11386,"Ġthreshold":11387,"Ġrhet":11388,"Ġserial":11389,"Ġemployers":11390,"ĠEverything":11391,"rah":11392,"Ġbother":11393,"Ġbrands":11394,"Value":11395,"ĠTed":11396,"ĠPlanet":11397,"Ġpink":11398,"ĠFurthermore":11399,"sa":11400,"PE":11401,"reck":11402,"ĠUSD":11403,"otte":11404,"Ġ&&":11405,"Ġlanded":11406,"gets":11407,"Ġproducers":11408,"Ġhealthcare":11409,"Ġdominant":11410,"Ġdestro":11411,"Ġamended":11412,"chron":11413,"Ġfits":11414,"ĠSyd":11415,"ĠAuthority":11416,"ATCH":11417,"Ġfights":11418,"ĠLLC":11419,"Ġ---":11420,"ĠCorp":11421,"Ġtoxic":11422,"specific":11423,"ĠCorn":11424,"ĠChel":11425,"Ġtelephone":11426,"ĠPant":11427,"Ġmysterious":11428,"aunch":11429,"odox":11430,"media":11431,"Ġwitnesses":11432,"agu":11433,"Ġquestioned":11434,"ĠBrexit":11435,"ĠRemember":11436,"enez":11437,"Ġendorse":11438,"iatric":11439,"ĠIdent":11440,"Ġridiculous":11441,"110":11442,"Ġprayer":11443,"Ġscientist":11444,"Ġ1950":11445,"ĠAqu":11446,"Ġunderground":11447,"ĠUFC":11448,"mare":11449,"ĠLater":11450,"wich":11451,"Ġsubscrib":11452,"Ġhosts":11453,"Ġerr":11454,"Ġgrants":11455,"antom":11456,"Ġsummon":11457,"early":11458,"ĠClear":11459,"ĠPrim":11460,"Ġsuspension":11461,"Ġguaranteed":11462,"apper":11463,"Ġrice":11464,"ĠSean":11465,"ĠShin":11466,"Ġreferendum":11467,"Ġfled":11468,"rust":11469,"Ġ360":11470,"tery":11471,"Ġshocked":11472,"BR":11473,"ĠOil":11474,"ĠAllah":11475,"Ġpartly":11476,"Ġignor":11477,"Ġtransmission":11478,"Ġhomosexual":11479,"iversal":11480,"Ġhopefully":11481,"ãĤ¤":11482,"Ġlesson":11483,"Leg":11484,"Ġ..":11485,"Yet":11486,"table":11487,"appropri":11488,"rett":11489,"Ġboards":11490,"Ġincorrect":11491,"Ġbacteria":11492,"aru":11493,"amac":11494,"Ġsnap":11495,".'\"":11496,"Ġparad":11497,"tem":11498,"heart":11499,"Ġavailability":11500,"Ġwisdom":11501,"Ġ(+":11502,"Ġpriest":11503,"ĠÂłĠÂł":11504,"Open":11505,"Ġspan":11506,"Ġparameter":11507,"Ġconvince":11508,"Ġ(%)":11509,"rac":11510,"Ġfo":11511,"Ġsafely":11512,"Ġconverted":11513,"ĠOlympic":11514,"Ġreserve":11515,"Ġhealing":11516,"ĠMine":11517,"Max":11518,"Ġinherent":11519,"ĠGraham":11520,"Ġintegrated":11521,"Dem":11522,"Ġpipeline":11523,"Ġapplying":11524,"Ġembed":11525,"ĠCharlie":11526,"Ġcave":11527,"2008":11528,"Ġconsensus":11529,"Ġrewards":11530,"Pal":11531,"ĠHTML":11532,"Ġpopularity":11533,"looking":11534,"ĠSword":11535,"ĠArts":11536,"')":11537,"Ġelectron":11538,"clusions":11539,"Ġintegrity":11540,"Ġexclusively":11541,"Ġgrace":11542,"Ġtorture":11543,"Ġburned":11544,"two":11545,"Ġ180":11546,"Produ":11547,"Ġentreprene":11548,"raphics":11549,"Ġgym":11550,"ricane":11551,"ĠTam":11552,"Ġadministrative":11553,"Ġmanufacturer":11554,"Ġvel":11555,"ĠNi":11556,"Ġisolated":11557,"ĠMedicine":11558,"Ġbackup":11559,"Ġpromoting":11560,"Ġcommander":11561,"Ġflee":11562,"ĠRussell":11563,"Ġforgotten":11564,"ĠMissouri":11565,"Ġresidence":11566,"mons":11567,"Ġresemb":11568,"Ġwand":11569,"Ġmeaningful":11570,"PT":11571,"Ġbol":11572,"Ġhelic":11573,"Ġwealthy":11574,"Ġrifle":11575,"strong":11576,"rowing":11577,"plan":11578,"asury":11579,"âĢ¦.":11580,"Ġexpanding":11581,"ĠHamilton":11582,"Ġreceives":11583,"SI":11584,"eatures":11585,"ĠAnim":11586,"REE":11587,"Put":11588,"Ġbriefly":11589,"rive":11590,"Ġstimul":11591,"Ġ``(":11592,"Ġ__":11593,"Ġchip":11594,"Ġhaz":11595,"Ġprize":11596,"ĠThings":11597,"ACE":11598,"ulin":11599,"dict":11600,"oku":11601,"Ġassociate":11602,"ockets":11603,"youtube":11604,"Story":11605,"ategory":11606,"Ġmild":11607,"ailing":11608,"ĠYe":11609,"Orig":11610,"ĠKa":11611,"orig":11612,"Ġpropaganda":11613,"Ġanonymous":11614,"Ġstruggled":11615,"Ġoutrage":11616,"ATED":11617,"ĠBeijing":11618,"rary":11619,"Ġleather":11620,"Ġworlds":11621,"Ġbroader":11622,"125":11623,"idal":11624,"ĠBetter":11625,"Ġtear":11626,"Ext":11627,"Ġproposals":11628,"Ġiter":11629,"ĠSquad":11630,"Ġvolunt":11631,"mi":11632,"Did":11633,"ĠPu":11634,"pin":11635,"Ġspeakers":11636,"Ġborders":11637,"Ġfigured":11638,"='":11639,"Ġsimultaneously":11640,"aeda":11641,"Ġcharging":11642,"Ġurged":11643,"Ġconj":11644,"256":11645,"ĠGordon":11646,"merce":11647,"Ġdocumentary":11648,"Share":11649,"itol":11650,"ONE":11651,"ĠGarden":11652,"hatt":11653,"ĠThompson":11654,"aneous":11655,"apore":11656,"Ġtanks":11657,"Ġlessons":11658,"track":11659,"Ġoutstanding":11660,"Ġvolunteers":11661,"Ġspray":11662,"Ġmanagers":11663,"large":11664,"Ġcamps":11665,"Ġartificial":11666,"ĠRu":11667,"Ġbags":11668,"thal":11669,"Ġcompatible":11670,"ĠBlade":11671,"Ġfed":11672,"Ġargues":11673,"FI":11674,"Ġunfair":11675,"Ġcorn":11676,"Ġoffset":11677,"Ġdirections":11678,"Ġdisappointed":11679,"ĠConvention":11680,"Ġviewing":11681,"ME":11682,"ocity":11683,"Ġtowns":11684,"Ġlayers":11685,"Ġrolled":11686,"Ġjumped":11687,"Ġattribute":11688,"Ġunnecess":11689,"incoln":11690,"Ġsuppose":11691,"ĠNether":11692,"cha":11693,"Ġburied":11694,"Ġsixth":11695,"Ben":11696,"ressing":11697,"OUR":11698,"Ġwound":11699,"Ġcycl":11700,"Ġmechanisms":11701,"Ġcongressional":11702,"ĠElement":11703,"Ġagreements":11704,"Ġdecor":11705,"Ġclosest":11706,"ĠMit":11707,"Google":11708,"}}":11709,"Ġmixture":11710,"Ġfluid":11711,"Sign":11712,"ĠScholar":11713,"Ġpist":11714,"asket":11715,"abling":11716,"Ġracing":11717,"hero":11718,"riel":11719,"assy":11720,"Ġcheaper":11721,"ben":11722,"Ġvertical":11723,"amacare":11724,"ĠReading":11725,"gments":11726,"Ġhelicop":11727,"Ġsacrifice":11728,"aya":11729,"paren":11730,"VA":11731,"ĠLes":11732,"ĠStudio":11733,"Ġviolations":11734,"ĠAnna":11735,"acer":11736,"é¾":11737,"ĠRat":11738,"ĠBeck":11739,"ĠDick":11740,"ĠACT":11741,"Ġcomposition":11742,"Ġtexture":11743,"ĠOwn":11744,"Ġsmartphone":11745,"ĠNA":11746,"Ġforb":11747,"import":11748,"Ġdefending":11749,"ilst":11750,"rer":11751,"Ġoh":11752,"ĠJeremy":11753,"Ġbanking":11754,"ceptions":11755,"Ġrespective":11756,"/.":11757,"Ġdrinks":11758,"ĠWi":11759,"Ġbands":11760,"ĠLiverpool":11761,"Ġgrip":11762,"ĠBuy":11763,"Ġopenly":11764,"Ġreviewed":11765,"pert":11766,"Ġverify":11767,"ĠCole":11768,"ĠWales":11769,"MO":11770,"Ġunpre":11771,"Ġshelter":11772,"ĠImperial":11773,"Ġgui":11774,"ĠDak":11775,"Ġsuggestions":11776,"Ġexplicitly":11777,"Ġslave":11778,"Ġblockchain":11779,"Ġcompeting":11780,"Ġpromising":11781,"SON":11782,"Ġsoccer":11783,"Ġconstitution":11784,"429":11785,"Ġdistract":11786,"ĠUser":11787,"esides":11788,"ĠMethod":11789,"ĠTokyo":11790,"Ġaccompanied":11791,"Client":11792,"sur":11793,"alog":11794,"Ġidentification":11795,"Ġinvasion":11796,"asma":11797,"Ġindustries":11798,"ppers":11799,"Ġsubtle":11800,"ĠUnit":11801,"natural":11802,"Ġsurvived":11803,"Ġflaw":11804,"ĺħ":11805,"ĠHoll":11806,"Ġdeficit":11807,"Ġtutorial":11808,"ĠChance":11809,"Ġarguing":11810,"Ġcontemporary":11811,"Ġintegration":11812,"forward":11813,"Ġtum":11814,"itis":11815,"Ġhiding":11816,"ĠDomin":11817,"ĠTan":11818,"ĠBuilding":11819,"ĠVin":11820,"Ġspokesperson":11821,"ĠNotes":11822,"Ġemerging":11823,"Ġpreparation":11824,"Ġprost":11825,"Ġsuspects":11826,"Ġautonom":11827,"Description":11828,"Ġdealt":11829,"ĠPear":11830,"Ġsteady":11831,"Ġdecreased":11832,"Ġsovere":11833,"ĠClin":11834,"Ġgradually":11835,"orses":11836,"ĠWAR":11837,"Serv":11838,"ãĤ¢":11839,"hr":11840,"Ġdirty":11841,"ĠBarn":11842,"ĠBC":11843,"Ġdil":11844,"Ġcalendar":11845,"Ġcompliance":11846,"Ġchamber":11847,"bb":11848,"Ġpassenger":11849,"ateful":11850,"ĠTitle":11851,"ĠSydney":11852,"ĠGot":11853,"Ġdarkness":11854,"Ġdefect":11855,"Ġpacked":11856,"assion":11857,"Ġgods":11858,"Ġharsh":11859,"ICK":11860,"leans":11861,"Ġalgorithm":11862,"Ġoxygen":11863,"Ġvisits":11864,"Ġblade":11865,"Ġkilomet":11866,"ĠKentucky":11867,"Ġkiller":11868,"Pack":11869,"enny":11870,"Ġdivine":11871,"Ġnomination":11872,"being":11873,"Ġengines":11874,"Ġcats":11875,"Ġbuffer":11876,"ĠPhill":11877,"Ġtraff":11878,"AGE":11879,"Ġtongue":11880,"Ġradiation":11881,"erer":11882,"mem":11883,"ĠExplicit":11884,"é¾į":11885,"Ġcouples":11886,"Ġphysics":11887,"ĠMcK":11888,"Ġpolitically":11889,"awks":11890,"ĠBloom":11891,"Ġworship":11892,"eger":11893,"uter":11894,"ĠFO":11895,"Ġmathemat":11896,"Ġsentenced":11897,"Ġdisk":11898,"ĠMarg":11899,"Ġ/*":11900,"PI":11901,"Ġoptional":11902,"Ġbabies":11903,"Ġseeds":11904,"ĠScottish":11905,"Ġthy":11906,"]]":11907,"ĠHitler":11908,"PH":11909,"ngth":11910,"Ġrecovered":11911,"inge":11912,"Ġpowder":11913,"Ġlips":11914,"Ġdesigner":11915,"Ġdisorders":11916,"Ġcourage":11917,"Ġchaos":11918,"\"},{\"":11919,"Ġcarrier":11920,"bably":11921,"High":11922,"ĠRT":11923,"esity":11924,"len":11925,"Ġroutes":11926,"uating":11927,"Fil":11928,"NOT":11929,"wall":11930,"sburgh":11931,"Ġengaging":11932,"ĠJavaScript":11933,"orer":11934,"lihood":11935,"Ġunions":11936,"ĠFederation":11937,"ĠTesla":11938,"Ġcompletion":11939,"ĠTa":11940,"Ġprivilege":11941,"ĠOrange":11942,"Ġneur":11943,"parency":11944,"Ġbones":11945,"Ġtitled":11946,"Ġprosecutors":11947,"ĠME":11948,"Ġengineer":11949,"ĠUniverse":11950,"ĠHig":11951,"nie":11952,"oard":11953,"Ġhearts":11954,"ĠGre":11955,"ussion":11956,"Ġministry":11957,"Ġpenet":11958,"ĠNut":11959,"ĠOw":11960,"ĠXP":11961,"instein":11962,"Ġbulk":11963,"System":11964,"icism":11965,"ĠMarketable":11966,"Ġpreval":11967,"Ġposter":11968,"Ġattending":11969,"urable":11970,"Ġlicensed":11971,"ĠGh":11972,"etry":11973,"ĠTradable":11974,"Ġblast":11975,"à¤":11976,"ĠTitan":11977,"elled":11978,"die":11979,"Have":11980,"ĠFlame":11981,"Ġprofound":11982,"Ġparticipating":11983,"Ġanime":11984,"ĠEss":11985,"Ġspecify":11986,"Ġregarded":11987,"ĠSpell":11988,"Ġsons":11989,"owned":11990,"Ġmerc":11991,"Ġexperimental":11992,"lando":11993,"hs":11994,"ĠDungeon":11995,"inos":11996,"Ġcomply":11997,"ĠSystems":11998,"arth":11999,"Ġseized":12000,"local":12001,"ĠGirls":12002,"udo":12003,"oned":12004,"ĠFle":12005,"Ġconstructed":12006,"Ġhosted":12007,"Ġscared":12008,"actic":12009,"ĠIslands":12010,"ĠMORE":12011,"Ġbless":12012,"Ġblocking":12013,"Ġchips":12014,"Ġevac":12015,"Ps":12016,"Ġcorporation":12017,"Ġox":12018,"Ġlighting":12019,"Ġneighbors":12020,"ĠUb":12021,"aro":12022,"Ġbeef":12023,"ĠUber":12024,"Facebook":12025,"armed":12026,"itate":12027,"ĠRating":12028,"ĠQuick":12029,"Ġoccupied":12030,"Ġaims":12031,"ĠAdditionally":12032,"ĠInterest":12033,"Ġdramatically":12034,"Ġheal":12035,"Ġpainting":12036,"Ġengineers":12037,"MM":12038,"ĠMust":12039,"Ġquantity":12040,"Paul":12041,"Ġearnings":12042,"ĠPosts":12043,"stra":12044,"ãĥ¼ãĥ":12045,"Ġstance":12046,"Ġdropping":12047,"script":12048,"Ġdressed":12049,"Make":12050,"Ġjustify":12051,"ĠLtd":12052,"Ġprompted":12053,"Ġscrut":12054,"Ġspeeds":12055,"ĠGiants":12056,"omer":12057,"ĠEditor":12058,"Ġdescribing":12059,"ĠLie":12060,"mented":12061,"Ġnowhere":12062,"ocaly":12063,"Ġinstruction":12064,"fortable":12065,"Ġentities":12066,"Ġcm":12067,"ĠNatural":12068,"Ġinquiry":12069,"Ġpressed":12070,"izont":12071,"forced":12072,"Ġraises":12073,"ĠNetflix":12074,"ĠSide":12075,"Ġouter":12076,"Ġamongst":12077,"ims":12078,"owski":12079,"Ġclimb":12080,"never":12081,"Ġcombine":12082,"ding":12083,"Ġcompr":12084,"Ġsignificance":12085,"Ġremembered":12086,"ĠNevada":12087,"ĠTel":12088,"ĠScar":12089,"ĠWarriors":12090,"ĠJane":12091,"Ġcoup":12092,"bas":12093,"Ġterminal":12094,",-":12095,"OH":12096,"Ġtension":12097,"Ġwings":12098,"ĠMyster":12099,"����":12100,"ĠUnlike":12101,"valid":12102,"vironments":12103,"ĠAli":12104,"Ġnaked":12105,"books":12106,"ĠMun":12107,"ĠGulf":12108,"Ġdensity":12109,"Ġdimin":12110,"Ġdesperate":12111,"Ġpresidency":12112,"Ġ1986":12113,"hy":12114,"IND":12115,"Ġunlock":12116,"imens":12117,"Ġhandled":12118,"ĠEb":12119,"Ġdisappeared":12120,"Ġgenre":12121,"Ġ1988":12122,"Ġdetermination":12123,"Stream":12124,"iko":12125,"apters":12126,"Ġacknowledge":12127,"Jan":12128,"Ġcapitalism":12129,"Pat":12130,"Ġ2020":12131,"Ġpainful":12132,"Ġcurve":12133,"Ġbombs":12134,"storm":12135,"ĠMetal":12136,"encer":12137,"ĠFig":12138,"ĠAaron":12139,"anches":12140,"Ġinspiration":12141,"Ġexhaust":12142,"tains":12143,"ashi":12144,"Ġdescript":12145,"Ġritual":12146,"ĠChelsea":12147,"Ġpromotion":12148,"ĠHung":12149,"ĠWard":12150,"iva":12151,"ĠET":12152,"Ġtoss":12153,"allow":12154,"ĠFrancis":12155,"Dep":12156,"Ġhappiness":12157,"ĠGlass":12158,"Ġbeta":12159,"Ġstrengthen":12160,"NE":12161,"oa":12162,"Ġbuttons":12163,"ĠMurray":12164,"Ġkicked":12165,"Quest":12166,"ĠTalk":12167,"ĠSeveral":12168,"ĠZero":12169,"Ġdrone":12170,"ulk":12171,"Ġcam":12172,"ĠMobile":12173,"Ġpreventing":12174,"Ġretro":12175,"ĠAx":12176,"Ġcruel":12177,"Ġfloat":12178,".),":12179,"Ġfiling":12180,"ĠGrant":12181,"ĠBor":12182,"Ġrib":12183,"Ġchampionship":12184,"ĠMerc":12185,"Ġstyles":12186,"Ġcake":12187,"Ġbuilds":12188,"ĠSelf":12189,"iox":12190,"Ġepic":12191,"oyd":12192,"Bel":12193,"ĠStew":12194,".(":12195,"ahu":12196,"ĠBeyond":12197,"Ġouts":12198,"Ġsolo":12199,"ĠTree":12200,"Ġpreserve":12201,"Ġtub":12202,"ARE":12203,"roc":12204,"ĠImpro":12205,"ĠWright":12206,"Ġbund":12207,"Ġtraged":12208,"Ġoccasional":12209,"bian":12210,"Second":12211,"rons":12212,"Ġinteractions":12213,"formed":12214,"sing":12215,"Ġowns":12216,"Ġhockey":12217,"General":12218,"Ġlogical":12219,"Ġexpend":12220,"Ġescal":12221,"ĠGriff":12222,"ĠCrown":12223,"ĠReserve":12224,"Ġstopping":12225,"Ġexcuse":12226,"second":12227,"Ġoperated":12228,"Ġreaches":12229,"ĠMalays":12230,"Ġpollution":12231,"ĠBrooklyn":12232,"Ġdelete":12233,"Ġhash":12234,"Block":12235,"aha":12236,"âĢ³":12237,"Ġshorter":12238,"piece":12239,">>>":13163,"ĠMormon":13164,"tor":13165,"Ġparticles":13166,"ĠBart":13167,"ryption":13168,"Ġadmin":13169,"Ġsquee":13170,"VIDIA":13171,"Ġcreator":13172,"iameter":13173,"icular":13174,"NBC":13175,"Ġgrabbed":13176,"Ġnodd":13177,"Ġrated":13178,"Ġrotation":13179,"Ġgrasp":13180,"Ġexcessive":13181,"ĠEC":13182,"ĠWhit":13183,"Ġinventory":13184,"aults":13185,"ĠFB":13186,"Ġecosystem":13187,"Ġbillions":13188,"Ġventure":13189,"named":13190,"Ġdefender":13191,"oute":13192,"Instead":13193,"irable":13194,"War":13195,"Ġassumption":13196,"Ġbite":13197,"Ġearthqu":13198,"tail":13199,"space":13200,"Ġgifts":13201,"boys":13202,"Ġinevitable":13203,"Ġstructural":13204,"Ġbeneficial":13205,"Ġcompelling":13206,"hole":13207,"ervation":13208,"Ġcoat":13209,"oj":13210,"incarn":13211,"ĠYears":13212,"Ġdetermining":13213,"Ġrhetoric":13214,"Ġboundaries":13215,"Ġwhites":13216,"Ant":13217,"addy":13218,")-":13219,"raham":13220,"etermin":13221,"Ġharvest":13222,"ĠConc":13223,"Ġlaptop":13224,"ĠMatch":13225,"Ġenjoying":13226,"cca":13227,"ollar":13228,"Ġtrips":13229,"Ġaddiction":13230,"ĠSak":13231,"Ġpowered":13232,"Ġcous":13233,"ĠRussians":13234,"iere":13235,"Ġretrie":13236,"quality":13237,"Ġdiffer":13238,"Ġkingdom":13239,"ĠLaur":13240,"ĠCapitol":13241,"Ġconclusions":13242,"ĠAltern":13243,"ĠNav":13244,"Ġtransparent":13245,"BER":13246,"Group":13247,"ĠComplete":13248,"Ġinfer":13249,"Ġintrig":13250,"Ġinsane":13251,"RO":13252,"ophob":13253,"isen":13254,"qual":13255,"Michael":13256,"Ġmuseum":13257,"ĠPope":13258,"Ġreset":13259,"rative":13260,"five":13261,"Ġaggreg":13262,"ittees":13263,"ository":13264,"Ġcarb":13265,"ĠRecord":13266,"Ġdecides":13267,"ĠFix":13268,"Ġexceptions":13269,"ĠCommissioner":13270,"uns":13271,"ĠEnvironmental":13272,"Ġlegendary":13273,"istence":13274,"Ġtunnel":13275,"km":13276,"Ġinsult":13277,"Ġtroll":13278,"Ġshake":13279,"Ġdetention":13280,"ques":13281,"ĠChrome":13282,"ĠFiles":13283,"Ġsubt":13284,"Ġprospects":13285,"Ġprol":13286,"render":13287,"proof":13288,"Ġperformances":13289,"Str":13290,"Ġhref":13291,"ername":13292,"Ġachievement":13293,"Ġfut":13294,"Full":13295,"ĠLeban":13296,"google":13297,"ãĥĪ":13298,"ampa":13299,"Maybe":13300,"Ġprojected":13301,"ĠEmb":13302,"Ġcolleg":13303,"Ġawards":13304,"ĠâĶ":13305,"Gold":13306,"ĠBlake":13307,"ĠRaj":13308,"ifting":13309,"Ġpending":13310,"Ġinstinct":13311,"Ġdevelopments":13312,"Connect":13313,"ĠMand":13314,"ĠWITH":13315,"ĠPhilippines":13316,"profile":13317,"Ġaltogether":13318,"ĠBund":13319,"ĠTD":13320,"oooo":13321,"amped":13322,"iph":13323,"Ġsteam":13324,"Ġoldest":13325,"Ġdetection":13326,"ulpt":13327,"Ġç":13328,"ĠWayne":13329,"2006":13330,"fa":13331,"Ġcircles":13332,"ĠFu":13333,"Ġdonors":13334,"appropriate":13335,"ĠDakota":13336,"jamin":13337,"Ġmotivated":13338,"Ġpurchases":13339,"ĠLouisiana":13340,"ĠSpl":13341,"Ġglobe":13342,"Ġ105":13343,"zip":13344,"call":13345,"Ġdepartments":13346,"Ġsustainable":13347,"105":13348,"ĠOP":13349,"ifiers":13350,"Ġprevented":13351,"Ġincomp":13352,"ĠCommander":13353,"Ġdominated":13354,"Ġ»":13355,"Ġinvested":13356,"Ġcomplexity":13357,"Ġincl":13358,"Ġensuring":13359,"Ġrealm":13360,"ync":13361,"ĠIndependent":13362,"rained":13363,"ĠJen":13364,"ĠFlight":13365,"Ġathe":13366,"Ġspeculation":13367,"ĠTE":13368,"ocate":13369,"tic":13370,"Ġplaint":13371,"herry":13372,"Ġtoy":13373,"Ġ111":13374,"Ġplates":13375,"status":13376,"ĠIsa":13377,"Ġdevoted":13378,"Cop":13379,"ĠES":13380,"255":13381,"urrency":13382,"Main":13383,"Ġslaves":13384,"Ġpepper":13385,"Ġquotes":13386,"Ġceiling":13387,"ĠFish":13388,"Ġtransformation":13389,"Ġfraction":13390,"Ġadvantages":13391,"Ġtoile":13392,"Ġstunning":13393,"Ġmoist":13394,"breaking":13395,"si":13396,"ĠLocation":13397,"ĠMedium":13398,"Ġtexts":13399,"Ġugly":13400,"Ġbio":13401,".âĢĶ":13402,"ĠBased":13403,"Ġtrains":13404,"ĠWing":13405,"ĠAncient":13406,"ĠRecords":13407,"ĠHope":13408,"Special":13409,"adesh":13410,"obi":13411,"[/":13412,"Ġtemporarily":13413,"Ver":13414,"hu":13415,"oser":13416,"Ġovernight":13417,"Ġmamm":13418,"ĠTreasury":13419,"ĠVenezuel":13420,"ĠMega":13421,"Ġtar":13422,"Ġexpects":13423,"black":13424,"orph":13425,"\\\\\\\\":13426,"Ġacceptance":13427,"Ġradar":13428,"sis":13429,"Ġjunior":13430,"Ġframes":13431,"Ġobservation":13432,"acies":13433,"Power":13434,"ĠAdvanced":13435,"Mag":13436,"ologically":13437,"ĠMechan":13438,"Ġsentences":13439,"Ġanalysts":13440,"aughters":13441,"forcement":13442,"Ġvague":13443,"Ġclause":13444,"Ġdirectors":13445,"Ġevaluate":13446,"Ġcabinet":13447,"Matt":13448,"ĠClassic":13449,"Ang":13450,"Ġcler":13451,"ĠBuck":13452,"Ġresearcher":13453,"Ġ160":13454,"Ġpoorly":13455,"Ġexperiencing":13456,"ĠPed":13457,"ĠManhattan":13458,"Ġfreed":13459,"Ġthemes":13460,"advant":13461,"Ġnin":13462,"Ġpraise":13463,"104":13464,"ĠLibya":13465,"best":13466,"Ġtrusted":13467,"Ġcease":13468,"Ġdign":13469,"Direct":13470,"Ġbombing":13471,"Ġmigration":13472,"ĠSciences":13473,"Ġmunicipal":13474,"ĠAverage":13475,"Ġglory":13476,"Ġrevealing":13477,"Ġarena":13478,"Ġuncertainty":13479,"Ġbattlefield":13480,"iao":13481,"God":13482,"Ġcinem":13483,"rape":13484,"elle":13485,"apons":13486,"Ġlisting":13487,"Ġwaited":13488,"Ġspotted":13489,"keley":13490,"ĠAudio":13491,"eor":13492,"arding":13493,"idding":13494,"igma":13495,"ĠNeg":13496,"Ġlone":13497,"Ġ----":13498,"exe":13499,"deg":13500,"Ġtransf":13501,"Ġwash":13502,"Ġslavery":13503,"Ġexploring":13504,"ĠWW":13505,"atson":13506,"Ġencl":13507,"lies":13508,"ĠCreek":13509,"Ġwooden":13510,"Manager":13511,"ĠBrand":13512,"ummy":13513,"ĠArthur":13514,"Ġbureaucr":13515,"Ġblend":13516,"arians":13517,"Further":13518,"Ġsupposedly":13519,"Ġwinds":13520,"Ġ1979":13521,"Ġgravity":13522,"Ġanalyses":13523,"ĠTravel":13524,"ĠVeter":13525,"Ġdumb":13526,"Ġalternate":13527,"gal":13528,"Ġconsumed":13529,"Ġeffectiveness":13530,".''":13531,"Ġpaths":13532,"onda":13533,"LA":13534,"ĠStrong":13535,"Ġenables":13536,"Ġescaped":13537,"Ġ\"\"":13538,"Ġ112":13539,"Ġ1983":13540,"Ġsmiled":13541,"Ġtendency":13542,"Fire":13543,"Ġpars":13544,"ĠRoc":13545,"Ġlake":13546,"Ġfitness":13547,"ĠAth":13548,"ĠHorn":13549,"Ġhier":13550,"Ġimpose":13551,"mother":13552,"Ġpension":13553,"icut":13554,"borne":13555,"iciary":13556,"._":13557,"ĠSU":13558,"Ġpolar":13559,"isy":13560,"engu":13561,"itialized":13562,"ATA":13563,"write":13564,"Ġexercises":13565,"ĠDiamond":13566,"otypes":13567,"Ġharmful":13568,"onz":13569,"Ġprinting":13570,"story":13571,"Ġexpertise":13572,"ĠGer":13573,"Ġtragedy":13574,"ĠFly":13575,"Ġdivid":13576,"ampire":13577,"stock":13578,"Mem":13579,"Ġreign":13580,"Ġunve":13581,"Ġamend":13582,"ĠProphet":13583,"Ġmutual":13584,"ĠFac":13585,"Ġreplacing":13586,"Har":13587,"ĠCircuit":13588,"Ġthroat":13589,"ĠShot":13590,"Ġbatteries":13591,"Ġtoll":13592,"Ġaddressing":13593,"ĠMedicaid":13594,"Ġpupp":13595,"ĠNar":13596,"olk":13597,"Ġequity":13598,"MR":13599,"ĠHispan":13600,"ĠLarge":13601,"mid":13602,"Dev":13603,"Ġexped":13604,"Ġdemo":13605,"ĠMarshall":13606,"ergus":13607,"Ġfiber":13608,"Ġdivorce":13609,"ĠCreate":13610,"Ġslower":13611,"ĠParker":13612,"ĠStudent":13613,"ĠTraining":13614,"Return":13615,"ĠTru":13616,"Ġcub":13617,"ĠReached":13618,"Ġpanic":13619,"Ġquarters":13620,"Ġrect":13621,"Ġtreating":13622,"Ġrats":13623,"ĠChristianity":13624,"oler":13625,"Ġsacred":13626,"Ġdeclare":13627,"ulative":13628,"eting":13629,"Ġdelivering":13630,"estone":13631,"Ġtel":13632,"ĠLarry":13633,"Ġmeta":13634,"accept":13635,"artz":13636,"ĠRoger":13637,"handed":13638,"Ġheader":13639,"Ġtrapped":13640,"ĠCentury":13641,"Ġknocked":13642,"ĠOxford":13643,"Ġsurvivors":13644,"bot":13645,"Ġdemonstration":13646,"Ġdirt":13647,"Ġassists":13648,"OME":13649,"ĠDraft":13650,"ortunate":13651,"folio":13652,"pered":13653,"usters":13654,"gt":13655,"ĠLock":13656,"Ġjudicial":13657,"verted":13658,"Ġsecured":13659,"outing":13660,"ĠBooks":13661,"Ġhosting":13662,"Ġlifted":13663,"length":13664,"Ġjer":13665,"Ġwheels":13666,"ĠRange":13667,"umbnails":13668,"Ġdiagnosis":13669,"tech":13670,"ĠStewart":13671,"ĠPract":13672,"Ġnationwide":13673,"Ġdear":13674,"Ġobligations":13675,"Ġgrows":13676,"Ġmandatory":13677,"Ġsuspicious":13678,"!'":13679,"Apr":13680,"Great":13681,"Ġmortgage":13682,"Ġprosecutor":13683,"Ġeditorial":13684,"ĠKr":13685,"Ġprocessed":13686,"ungle":13687,"Ġflexibility":13688,"Earlier":13689,"ĠCart":13690,"ĠSug":13691,"Ġfocuses":13692,"Ġstartup":13693,"Ġbreach":13694,"ĠTob":13695,"cycle":13696,"ãĢĮ":13697,"rose":13698,"Ġbizarre":13699,"ãĢį":13700,"Ġvegetables":13701,"$$":13702,"Ġretreat":13703,"oshi":13704,"ĠShop":13705,"ĠGround":13706,"ĠStop":13707,"ĠHawaii":13708,"ĠAy":13709,"Perhaps":13710,"ĠBeaut":13711,"uffer":13712,"enna":13713,"Ġproductivity":13714,"Fixed":13715,"control":13716,"Ġabsent":13717,"ĠCampaign":13718,"Green":13719,"Ġidentifying":13720,"Ġregret":13721,"Ġpromoted":13722,"ĠSeven":13723,"Ġeru":13724,"neath":13725,"aughed":13726,"ĠPin":13727,"ĠLiving":13728,"Cost":13729,"omatic":13730,"mega":13731,"ĠNig":13732,"ocy":13733,"Ġinbox":13734,"Ġempire":13735,"Ġhorizont":13736,"Ġbranches":13737,"Ġmetaph":13738,"Active":13739,"edi":13740,"ĠFilm":13741,"ĠSomething":13742,"Ġmods":13743,"incial":13744,"ĠOriginal":13745,"Gen":13746,"Ġspirits":13747,"Ġearning":13748,"Hist":13749,"Ġriders":13750,"Ġsacrific":13751,"MT":13752,"ĠVA":13753,"ĠSalt":13754,"Ġoccupation":13755,"ĠMi":13756,"Ġdisg":13757,"lict":13758,"Ġnit":13759,"Ġnodes":13760,"eem":13761,"ĠPier":13762,"Ġhatred":13763,"psy":13764,"ãĥī":13765,"Ġtheater":13766,"Ġsophisticated":13767,"Ġdefended":13768,"Ġbesides":13769,"Ġthoroughly":13770,"ĠMedicare":13771,"Ġblamed":13772,"arently":13773,"Ġcrying":13774,"FOR":13775,"priv":13776,"Ġsinging":13777,"ĠIl":13778,"Ġcute":13779,"oided":13780,"olitical":13781,"ĠNeuro":13782,"å¤":13783,"Ġdonation":13784,"ĠEagles":13785,"ĠGive":13786,"Tom":13787,"Ġsubstantially":13788,"ĠLicense":13789,"ĠJa":13790,"Ġgrey":13791,"ĠAnimal":13792,"ĠER":13793,"ĠUnd":13794,"Ġkeen":13795,"Ġconclude":13796,"ĠMississippi":13797,"Engine":13798,"ĠStudios":13799,"Press":13800,"overs":13801,"llers":13802,"Ġ350":13803,"ĠRangers":13804,"Ġrou":13805,"erto":13806,"Ep":13807,"issa":13808,"ivan":13809,"Ġseal":13810,"ĠRegist":13811,"display":13812,"Ġweaken":13813,"uum":13814,"ĠCommons":13815,"ĠSay":13816,"Ġcultures":13817,"Ġlaughed":13818,"Ġslip":13819,"Ġtreatments":13820,"izable":13821,"mart":13822,"ĠRice":13823,"Ġbeast":13824,"Ġobesity":13825,"ĠLaure":13826,"iga":13827,"Which":13828,"holder":13829,"Ġelderly":13830,"Ġpays":13831,"Ġcomplained":13832,"Ġcrop":13833,"Ġproc":13834,"Ġexplosive":13835,"ĠFan":13836,"ĠArsenal":13837,"Author":13838,"eful":13839,"Ġmeals":13840,"Ġ(-":13841,"idays":13842,"Ġimagination":13843,"Ġannually":13844,"Ġms":13845,"asures":13846,"Head":13847,"ikh":13848,"matic":13849,"Ġboyfriend":13850,"ĠComputer":13851,"Ġbump":13852,"Ġsurge":13853,"ĠCraig":13854,"ĠKirk":13855,"Del":13856,"mediate":13857,"Ġscenarios":13858,"ĠMut":13859,"ĠStream":13860,"Ġcompetitors":13861,"ÙĦ":13862,"ĠStanford":13863,"ĠResources":13864,"azed":13865,"bage":13866,"Ġorganis":13867,"ĠRelease":13868,"Ġseparately":13869,"Ġhabits":13870,"Ġmeasurements":13871,"ĠClose":13872,"Ġaccompany":13873,"Ġgly":13874,"Ġtang":13875,"ĠRou":13876,"Ġplugin":13877,"Ġconvey":13878,"ĠChallenge":13879,"oots":13880,"jan":13881,"Ġcurs":13882,"ĠRelations":13883,"keeper":13884,"Ġapproaching":13885,"ping":13886,"Speaking":13887,"Ġarrangement":13888,"ĠVI":13889,"arettes":13890,"Ġaffecting":13891,"Ġpermits":13892,"because":13893,"Ġuseless":13894,"ĠHus":13895,"!!!!":13896,"Ġdestroying":13897,"Unfortunately":13898,"Ġfascinating":13899,"Sem":13900,"Ġelectoral":13901,"Ġtransparency":13902,"ĠChaos":13903,"Ġvolunteer":13904,"Ġstatistical":13905,"Ġactivated":13906,"rox":13907,"Web":13908,"HE":13909,"ĠHampshire":13910,"isive":13911,"Map":13912,"Ġtrash":13913,"ĠLawrence":13914,"stick":13915,"Cr":13916,"Ġrings":13917,"EXT":13918,"Ġoperational":13919,"opes":13920,"Does":13921,"ĠEvans":13922,"Ġwitnessed":13923,"Port":13924,"Ġlaunching":13925,"econom":13926,"wear":13927,"ĠParticip":13928,"umm":13929,"cules":13930,"ĠRAM":13931,"ĠTun":13932,"Ġassured":13933,"Ġbinary":13934,"Ġbetray":13935,"Ġexploration":13936,"ĠFel":13937,"Ġadmission":13938,"itated":13939,"Sy":13940,"Ġavoided":13941,"ĠSimulator":13942,"Ġcelebrated":13943,"ĠElectric":13944,"¥ŀ":13945,"Ġcluster":13946,"itzerland":13947,"health":13948,"Line":13949,"ĠNash":13950,"aton":13951,"Ġspare":13952,"Ġenterprise":13953,"ĠDIS":13954,"cludes":13955,"Ġflights":13956,"Ġregards":13957,"ĠÃĹ":13958,"half":13959,"Ġtrucks":13960,"Ġcontacts":13961,"Ġuncons":13962,"ĠClimate":13963,"Ġimmense":13964,"NEW":13965,"occ":13966,"ective":13967,"Ġembod":13968,"Ġpatrol":13969,"Ġbeside":13970,"Ġviable":13971,"Ġcreep":13972,"Ġtriggered":13973,"verning":13974,"Ġcomparable":13975,"ql":13976,"Ġgaining":13977,"asses":13978,"Ġ();":13979,"ĠGrey":13980,"ĠMLS":13981,"sized":13982,"Ġprosper":13983,"\"?":13984,"Ġpolling":13985,"Ġshar":13986,"ĠRC":13987,"Ġfirearm":13988,"orient":13989,"Ġfence":13990,"Ġvariations":13991,"giving":13992,"ĠPi":13993,"ospel":13994,"Ġpledge":13995,"Ġcure":13996,"Ġspy":13997,"Ġviolated":13998,"Ġrushed":13999,"Ġstroke":14000,"ĠBlog":14001,"sels":14002,"ĠEc":14003,",''":14004,"Ġpale":14005,"ĠCollins":14006,"terror":14007,"ĠCanadians":14008,"Ġtune":14009,"Ġlaboratory":14010,"Ġnons":14011,"tarian":14012,"Ġdisability":14013,"ĠGam":14014,"Ġsinger":14015,"alg":14016,"ĠSenior":14017,"Ġtraded":14018,"ĠWarrior":14019,"Ġinfring":14020,"ĠFranklin":14021,"Ġstrain":14022,"ĠSwedish":14023,"Ġseventh":14024,"ĠBenn":14025,"ĠTell":14026,"Ġsyndrome":14027,"Ġwondered":14028,"iden":14029,"++++":14030,"igo":14031,"Ġpurple":14032,"Ġjournalism":14033,"Ġrebel":14034,"Ġfu":14035,"blog":14036,"Ġinvite":14037,"rencies":14038,"ĠContact":14039,"Israel":14040,"ĠContent":14041,"Ġcheer":14042,"Ġbedroom":14043,"ĠEngineering":14044,"ĠQueens":14045,"Ġdwell":14046,"ĠPlayStation":14047,"ĠDim":14048,"ĠColon":14049,"lr":14050,"Ġoperates":14051,"Ġmotivation":14052,"USA":14053,"astered":14054,"Core":14055,"ĠTruth":14056,"olo":14057,"OSE":14058,"ĠMemory":14059,"Ġpredec":14060,"Ġanarch":14061,"Ġ1920":14062,"ĠYam":14063,"è":14064,"bid":14065,"Ġgrateful":14066,"Ġexcitement":14067,"Ġtreasure":14068,"Ġlongest":14069,"ctive":14070,"Ġdeserves":14071,"Ġreserves":14072,"Ġcops":14073,"ĠOttawa":14074,"ĠEgyptian":14075,"anked":14076,"Ġartif":14077,"Ġhypothesis":14078,":/":14079,"Ġpurchasing":14080,"Ġlovely":14081,"HP":14082,"Ġdivide":14083,"Ġstrictly":14084,"Ġquestioning":14085,"Ġtaxpayers":14086,"ĠJoy":14087,"Ġrolls":14088,"ĠHeavy":14089,"Ġports":14090,"Ġmagnetic":14091,"Ġinflamm":14092,"Ġbrush":14093,"tics":14094,"âĪĴ":14095,"Ġbottles":14096,"ppy":14097,"Ġpadd":14098,"ãĤ¯":14099,"million":14100,"Ġdevastating":14101,"Ġcompiled":14102,"Ġmedication":14103,"Ġtwelve":14104,"ĠPerry":14105,"Space":14106,"imb":14107,"your":14108,"Ġleaked":14109,"ĠTar":14110,"Ġunity":14111,"Ġinfected":14112,"Ġtraveled":14113,"IDE":14114,"ĠMcDonald":14115,"txt":14116,"ĠPrinc":14117,"Ġinterven":14118,"ĠTaiwan":14119,"ĠPow":14120,"Ġbearing":14121,"ĠThread":14122,"Ġzones":14123,"izards":14124,"unks":14125,"Chapter":14126,"llor":14127,"Ġ·":14128,"Ġwounds":14129,"Ġdiscretion":14130,"Ġsucceeded":14131,"iking":14132,"Ġiconic":14133,"Call":14134,"Ġscreening":14135,"ĠMis":14136,"icts":14137,"Ġministers":14138,"Ġseparation":14139,"Player":14140,"Ġbip":14141,"Ġbeloved":14142,"Ġcounting":14143,"ĠEye":14144,"around":14145,"inging":14146,"Ġtablet":14147,"Ġoffence":14148,"inance":14149,"have":14150,"ĠInfo":14151,"ĠNinja":14152,"Ġprotective":14153,"ĠCass":14154,"Mac":14155,"ĠQuality":14156,"North":14157,"Ġic":14158,"ĠCuba":14159,"ĠChronicle":14160,"ĠProperty":14161,"Ġfastest":14162,"otos":14163,"ĠGerm":14164,"OWN":14165,"Ġboom":14166,"ĠStanley":14167,"erguson":14168,"Ġclever":14169,"Ġenters":14170,"mode":14171,"terior":14172,"ĠSens":14173,"Ġlinear":14174,"ARK":14175,"Ġcomparing":14176,"Ġpurely":14177,"Ġsafer":14178,"ĠPotter":14179,"Ġcups":14180,"RT":14181,"Ġgluc":14182,"Ġattributed":14183,"Ġdupl":14184,"ĠPap":14185,"Ġprecious":14186,"Ġpa":14187,"ictionary":14188,"ĠTig":14189,"ĠToo":14190,"olutions":14191,"stan":14192,"Ġrobots":14193,"Ġlobb":14194,"Ġstatute":14195,"Ġprevention":14196,"western":14197,"160":14198,"ĠActive":14199,"ĠMaria":14200,"hal":14201,"None":14202,"ellar":14203,"ĠKB":14204,"ĠPartners":14205,"ĠSingle":14206,"ĠFollowing":14207,"ango":14208,"acious":14209,"Ġthou":14210,"Ġkg":14211,"Ġinfluential":14212,"ĠFriends":14213,"Sur":14214,"ainted":14215,"Ġforums":14216,"Ġstarter":14217,"Ġcitizenship":14218,"ĠElection":14219,"onge":14220,"otation":14221,"osph":14222,";;;;":14223,"utical":14224,"pur":14225,"eren":14226,"Ġaccusations":14227,"bitious":14228,"abbit":14229,"ĠOrd":14230,"Posted":14231,"irk":14232,"Ġsensitivity":14233,"iche":14234,"ĠAmy":14235,"ĠFab":14236,"Ġsummit":14237,"Ġpedest":14238,"Ġrubber":14239,"Ġagricultural":14240,"Ġcancel":14241,"AE":14242,"Ġinaug":14243,"Ġcontam":14244,"Ġfirmly":14245,"iw":14246,"stage":14247,"ĠKan":14248,"Ġtier":14249,"Ġinvention":14250,"Ġtranslated":14251,"ĠRules":14252,"Box":14253,"Twitter":14254,"IDS":14255,"Ġpizza":14256,"Ġdebug":14257,"ĠDrop":14258,"vs":14259,"Ġhorses":14260,"big":14261,"Ġboring":14262,"Ġhood":14263,"ĠMcCain":14264,"atched":14265,"ĠBros":14266,"Ġskip":14267,"Ġessay":14268,"stat":14269,"ĠLegends":14270,"Ġammunition":14271,"auc":14272,"Ġshooter":14273,"Ġunh":14274,"Ġsupplied":14275,"Ġgeneric":14276,"ĠSK":14277,"iban":14278,"yrics":14279,"Ġ255":14280,"Ġclimbing":14281,"Former":14282,"Ġflip":14283,"Ġjumping":14284,"Ġfrustration":14285,"ĠTerry":14286,"Ġneighborhoods":14287,"Ġmedian":14288,"bean":14289,"Ġbrains":14290,"Following":14291,"Ġshaped":14292,"Ġdraws":14293,"Ġaltered":14294,"Jack":14295,"Ġrecipes":14296,"Ġskilled":14297,"wealth":14298,"achi":14299,"election":14300,"Ġbehaviors":14301,"deals":14302,"ĠUntil":14303,"Fe":14304,"Ġdeclaration":14305,"marks":14306,"ĠBetween":14307,"celona":14308,"Ġreson":14309,"Ġbubble":14310,"Among":14311,"Ġimperial":14312,"GS":14313,"Ġfeminist":14314,"2005":14315,"ĠKyle":14316,"Ġaccounting":14317,"ĠTele":14318,"ĠTyr":14319,"Ġconnecting":14320,"Ġrehab":14321,"ĠPred":14322,"sim":14323,"Ġmeantime":14324,"Ġphysician":14325,"MW":14326,"ĠCampbell":14327,"ĠBrandon":14328,"Ġcontributing":14329,"ĠRule":14330,"ĠWeight":14331,"ĠNap":14332,"Ġinteractive":14333,"Ġvag":14334,"Ġhelmet":14335,"ĠComb":14336,"four":14337,"Ġshipped":14338,"Ġcompleting":14339,"ĠPD":14340,"PDATE":14341,"Ġspreading":14342,"Ġscary":14343,"erving":14344,"ĠGas":14345,"Ġfrank":14346,"school":14347,"Ġromantic":14348,"Ġstabil":14349,"Rob":14350,"Ġaccurately":14351,"Ġacute":14352,"ĠHann":14353,"Ġsymbols":14354,"Ġcivilization":14355,"ĠAW":14356,"Ġlightning":14357,"Ġconsiders":14358,"Ġvenue":14359,"Ġ×":14360,"Ġoven":14361,"ĠSF":14362,"his":14363,"Ġnu":14364,"ĠLearn":14365,"Ġpeoples":14366,"Ġstd":14367,"Ġslee":14368,"Ġslic":14369,"ĠStatistics":14370,"Ġcorners":14371,"ĠBaker":14372,"Ġ:)":14373,"mentation":14374,"olver":14375,"Ġlaughing":14376,"ĠTodd":14377,"onde":14378,"ĠHills":14379,"Ġnuts":14380,"ĠWoman":14381,"plane":14382,"Ġliver":14383,"ĠInside":14384,"Sorry":14385,"Ġagrees":14386,"Ġfundament":14387,"ĠFisher":14388,"Ġauction":14389,"Ġthreads":14390,"glas":14391,"ĠBasic":14392,"ĠNat":14393,"Ġlacking":14394,"Ġcelebration":14395,"ju":14396,"Ġsilly":14397,"Euro":14398,"Ġtatt":14399,"ighty":14400,"controlled":14401,"Test":14402,"ĠSingh":14403,"Ġrage":14404,"Ġrhyth":14405,"offic":14406,"ĠPhantom":14407,"Ġheadlines":14408,"Ġresponding":14409,"ĠMorning":14410,"Ġvitamin":14411,"Ġboots":14412,"ĠSite":14413,"alin":14414,"pi":14415,"Ġviral":14416,"ĠUC":14417,"DER":14418,"ĠSex":14419,"Ġstocks":14420,"current":14421,"Ġchurches":14422,"ĠRare":14423,"ĠMurphy":14424,"Ġdenial":14425,"ĠGaming":14426,"Ġtoug":14427,"Ġnick":14428,"Ġmakers":14429,"ĠRonald":14430,"Ġgenerous":14431,"ĠDoc":14432,"ĠMorris":14433,"Ġtransformed":14434,"ĠNormal":14435,"Ġ104":14436,"ĠKickstarter":14437,"ĠUpon":14438,"Online":14439,"ĠIRS":14440,"Ġwrap":14441,"Ġloving":14442,"Ġarrives":14443,"ĠDue":14444,"Ġheter":14445,"ĠMade":14446,"Ġrental":14447,"Ġbelongs":14448,"Ġattorneys":14449,"Ġcrops":14450,"Ġmatched":14451,"ulum":14452,"oline":14453,"109":14454,"Ġdispar":14455,"Ġbuyers":14456,"ĠCambridge":14457,"Ġethics":14458,"roups":14459,"Ġjustified":14460,"Ġmarginal":14461,"Ġrespected":14462,"winning":14463,"Ġnodded":14464,"ĠSerge":14465,"ĠFormer":14466,"Craft":14467,"################":14468,"ĠWarner":14469,"Ġdash":14470,"ete":14471,"Ġentert":14472,"ĠEscape":14473,"outheast":14474,"Ġknees":14475,"ĠBomb":14476,"Ġrug":14477,"Pass":14478,"Ġattitudes":14479,"government":14480,"ĠPrior":14481,"Ġqualities":14482,"Ġnotification":14483,"ĠPhone":14484,"lie":14485,"Ġanticipated":14486,"ĠCombat":14487,"ĠBarry":14488,"Ġ1982":14489,"Users":14490,"oner":14491,"Ġcomputing":14492,"ĠConnecticut":14493,"Ġlesser":14494,"Ġpeers":14495,"ĠCu":14496,"Ġtechnically":14497,"Ġsubmission":14498,"ĠUniversal":14499,"Ġmanually":14500,"ourge":14501,"Ġrespondents":14502,"ĠBTC":14503,"ĠHost":14504,"Ġfare":14505,"ĠBird":14506,"Ġreceipt":14507,"also":14508,"Ġjack":14509,"Ġagriculture":14510,"Ġskull":14511,"Ġ!=":14512,"Ġpassive":14513,"ĠCI":14514,"Ġsocieties":14515,"Ġreminded":14516,"Ġinterference":14517,"Buy":14518,"Ġâľ":14519,"gon":14520,"Ġscrutiny":14521,"ĠWitch":14522,"Ġconducting":14523,"Ġãĥ":14524,"Ġexchanges":14525,"ĠMitchell":14526,"Ġinhabit":14527,"Ġtwist":14528,"BD":14529,"Ġwherever":14530,"groupon":14531,"Ġjokes":14532,"ĠBenjamin":14533,"ĠRandom":14534,"frame":14535,"ĠLions":14536,"Ġhighlighted":14537,"ĠArkansas":14538,"Ent":14539,"Ġpile":14540,"Ġprelim":14541,"gs":14542,"minded":14543,"Ġfelony":14544,"ĠGA":14545,"ĠLuck":14546,"Ġpractically":14547,"ĠBos":14548,"Ġactress":14549,"Dam":14550,"ĠBou":14551,"Ġvisa":14552,"Ġembedded":14553,"Ġhybrid":14554,"Ġearliest":14555,"Ġsooner":14556,"social":14557,"ĠHA":14558,"Ġsteep":14559,"Ġdisadvant":14560,"Ġexploit":14561,"ĠEgg":14562,"ĠUltra":14563,"Ġnecessity":14564,"Local":14565,"iege":14566,"Ġdated":14567,"Ġmasses":14568,"Ġsubscription":14569,"pless":14570,"Ġanonym":14571,"Ġpresumably":14572,"Blue":14573,"Their":14574,"asketball":14575,"ĠPhilip":14576,"Ġcomed":14577,"loaded":14578,"rane":14579,"Ġreflection":14580,"China":14581,"Ġextends":14582,"Ġforming":14583,"Ġunders":14584,"2001":14585,"Ġgrat":14586,"Ġconcentrations":14587,"Ġinsulin":14588,"Ġsecular":14589,"Ġwhilst":14590,"Ġwinners":14591,"Advertisements":14592,"Ġdeliberately":14593,"ĠWorking":14594,"Ġsink":14595,"etics":14596,"dale":14597,"Ġmandate":14598,"Ġgram":14599,"Ġvacation":14600,"Ġwarnings":14601,"ripp":14602,"ĠTHAT":14603,"Ġcommentary":14604,"Ġintu":14605,"Ġaest":14606,"Ġreasoning":14607,"Ġbreakdown":14608,"ĠZombie":14609,"Ġ-->":14610,"ĠPolitical":14611,"cott":14612,"Ġthrust":14613,"Ġtechnological":14614,"Ġdeciding":14615,"Ġtrafficking":14616,"Long":14617,"Welcome":14618,"prising":14619,"ĠCommunications":14620,"Ġendors":14621,"Ġswift":14622,"Ġmetabol":14623,"coins":14624,"resa":14625,"ĠHTTP":14626,"Ġenroll":14627,"ĠHappy":14628,"usr":14629,"intage":14630,"Ġ[\"":14631,"uably":14632,"ĠMaterial":14633,"Ġrepeal":14634,"Sept":14635,"kh":14636,"ĠModi":14637,"Ġunderneath":14638,"ĠIL":14639,"shore":14640,"Ġdiagnosed":14641,"aceutical":14642,"Ġshower":14643,"aux":14644,"ĠSwitch":14645,"ĠStrength":14646,"Ġjihad":14647,"national":14648,"Ġtrauma":14649,"ussy":14650,"oni":14651,"Ġconsolid":14652,"Ġcalories":14653,"ĠFlynn":14654,"agged":14655,"168":14656,"ĠPink":14657,"Ġfulfill":14658,"Ġchains":14659,"Ġnotably":14660,"ĠAV":14661,"Life":14662,"ĠChuck":14663,"mus":14664,"ĠUrban":14665,"ĠHend":14666,"Ġdeposit":14667,"ĠSad":14668,"Ġaffair":14669,"ORK":14670,"ieval":14671,"ĠFDA":14672,"Ġtrop":14673,"ĠOverall":14674,"Ġvirtue":14675,"Ġsatisfaction":14676,"aund":14677,"Ġlun":14678,"ĠSwitzerland":14679,"ĠOperation":14680,"process":14681,"Ġshook":14682,"Ġcounties":14683,"leased":14684,"ĠCharlotte":14685,"112":14686,"Ġtranscript":14687,"Ġredd":14688,"push":14689,"ĠHey":14690,"ĠAnalysis":14691,"[\"":14692,"Ġalternatives":14693,"ardless":14694,"Ġeleph":14695,"Ġprejud":14696,"ĠLeaf":14697,"Having":14698,"ĠHub":14699,"Ġexpressions":14700,"ĠVolume":14701,"Ġshocking":14702,"ĠReds":14703,"Ġreadily":14704,"Ġplanets":14705,"adata":14706,"Ġcollapsed":14707,"ĠMadrid":14708,"Ġirrit":14709,"ipper":14710,"ĠEnc":14711,"ĠWire":14712,"Ġbuzz":14713,"ĠGP":14714,"asha":14715,"Ġaccidentally":14716,"uru":14717,"Ġfrustrated":14718,"ĠSA":14719,"Ġhungry":14720,"ĠHuff":14721,"Ġlabels":14722,"anto":14723,"ĠEP":14724,"Ġbarriers":14725,")|":14726,"ĠBerkeley":14727,"ĠJets":14728,"Ġpairs":14729,"ĠLan":14730,"James":14731,"ĠBear":14732,"Ġhumor":14733,"ĠLiberty":14734,"Ġmagnitude":14735,"Ġaging":14736,"ĠMason":14737,"Ġfriendship":14738,"umbling":14739,"Ġemerge":14740,"Ġnewspapers":14741,"Ġambitious":14742,"ĠRichards":14743,"aternal":14744,"Ġ1981":14745,"Ġcookies":14746,"Ġsculpt":14747,"Ġpursuit":14748,"Location":14749,"Ġscripts":14750,"pc":14751,"Ġarrangements":14752,"Ġdiameter":14753,"Ġloses":14754,"amation":14755,"Ġliqu":14756,"ĠJake":14757,"arette":14758,"Ġunderstands":14759,"ĠZen":14760,"vm":14761,"Ġapprove":14762,"Ġwip":14763,"Ġultra":14764,"Ġintend":14765,"ĠDI":14766,"ascular":14767,"Ġstays":14768,"ĠKor":14769,"ĠKl":14770,"Ġinvesting":14771,"La":14772,"Ġbelieving":14773,"bad":14774,"mouth":14775,"Ġtaxpayer":14776,"ãĥĥ":14777,"ĠQuebec":14778,"Ġlap":14779,"ĠSwiss":14780,"drop":14781,"Ġdrain":14782,"iri":14783,"etc":14784,"ften":14785,"ĠNex":14786,"Ġstraw":14787,"Ġscreaming":14788,"Ġcounted":14789,"Ġdamaging":14790,"Ġambassador":14791,"century":14792,"Ġprox":14793,"Ġarrests":14794,"uv":14795,"ilateral":14796,"ĠCharg":14797,"Ġprescribed":14798,"Ġindependently":14799,"Ġfierce":14800,"ĠBaby":14801,"Ġbrave":14802,"Ġsuits":14803,"=>":14804,"Ġbaseline":14805,"ĠRate":14806,"Ġislands":14807,"Ġ((":14808,"green":14809,"ixels":14810,"Ġnamely":14811,"ĠVillage":14812,"than":14813,"amy":14814,"Version":14815,"gmail":14816,"entials":14817,"ĠSud":14818,"ĠMelbourne":14819,"Ġarriving":14820,"Ġquantum":14821,"eff":14822,"ropolitan":14823,"Tri":14824,"Ġfuneral":14825,"ĠIR":14826,"ÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤ":14827,"ĠCob":14828,"itably":14829,"Ġturb":14830,"Ġcombo":14831,"Review":14832,"Ġdeployment":14833,"uity":14834,"ĠBott":14835,"Ġinvisible":14836,"Ġrendering":14837,"Ġunlocked":14838,"Ġaqu":14839,"ĠVladimir":14840,"Ġpad":14841,"ĠBrain":14842,"ĠLegacy":14843,"dragon":14844,"ĠKurdish":14845,"Ġsounded":14846,"Ġdetained":14847,"ĠDM":14848,"gary":14849,"Ġdaughters":14850,"Ġdisturbing":14851,"uka":14852,"ĠParad":14853,"Ġtast":14854,"Ġunfortunate":14855,"Ġul":14856,"emin":14857,"Ġattendance":14858,"trl":14859,"Ġparks":14860,"ĠMemorial":14861,"ĠAlice":14862,"othy":14863,"guard":14864,"ĠDise":14865,"ĠShan":14866,"ĠForum":14867,"Rich":14868,"Ġshifted":14869,"uez":14870,"Ġlighter":14871,"ĠMagn":14872,"Ġcod":14873,"Sch":14874,"hammad":14875,"Pub":14876,"350":14877,"ĠPokemon":14878,"Ġprototype":14879,"Ġunre":14880,"Base":14881,"ĠStudents":14882,"ĠReply":14883,"ĠCommunist":14884,"Ġgau":14885,"ĠTyler":14886,"IZ":14887,"Ġparticipated":14888,"Ġsuprem":14889,"ĠDetails":14890,"Ġvessels":14891,"rod":14892,"Ġtribe":14893,"keep":14894,"Ġassumptions":14895,"Ġpound":14896,"Ġcrude":14897,"ĠAvailable":14898,"Ġswimming":14899,"Ġinclusion":14900,"Ġadvances":14901,"culation":14902,"Ġconservation":14903,"Ġoverd":14904,"ĠBuffalo":14905,"Article":14906,"edge":14907,"Ġawa":14908,"ĠMadison":14909,"Ġsidew":14910,"Ġcatast":14911,"ĠKrist":14912,"ucle":14913,"ĠHighway":14914,"ĠTerror":14915,"Ġactivation":14916,"Ġunconscious":14917,"ĠSatan":14918,"ĠSusan":14919,"illery":14920,"Ġarranged":14921,"iop":14922,"Ġrumors":14923,"urring":14924,"think":14925,"ĠKeith":14926,"ĠKind":14927,"Ġavoiding":14928,"byn":14929,"nut":14930,"ĠSpeaker":14931,"rus":14932,"names":14933,"Ġguilt":14934,"ĠOlympics":14935,"Ġsail":14936,"ĠMes":14937,"levant":14938,"ĠColumbus":14939,"aft":14940,"City":14941,"South":14942,"ĠHarvey":14943,"ĠPun":14944,"Several":14945,"Ġmentally":14946,"Ġimpress":14947,"mount":14948,"ĠUbuntu":14949,"âĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶ":14950,"ĠSuperman":14951,"ĠMPs":14952,"Ġintentions":14953,"ĠRacing":14954,"Ġlikelihood":14955,"Ġ240":14956,"Total":14957,"Ġtoys":14958,"ĠWatson":14959,"Ġurge":14960,"Lear":14961,"ĠPaper":14962,"Ġoccurring":14963,"ĠBeng":14964,"ĠCert":14965,"Ġstones":14966,"Tim":14967,"ĠTwin":14968,"zb":14969,"ĠDynam":14970,"Ġpolitician":14971,"kens":14972,"ĠEnterprise":14973,"UTERS":14974,"Ġabol":14975,"Ġrefresh":14976,"Ġarbitrary":14977,"pection":14978,"Ġtroubles":14979,"Ġ});":14980,"tv":14981,"Ġpilots":14982,"Ġdistribute":14983,"Ġaudit":14984,"Ġpause":14985,"original":14986,"Ġrivals":14987,"£":14988,"Fig":14989,"TL":14990,"abil":14991,"rying":14992,"Lin":14993,"ioned":14994,"lon":14995,"Ġfancy":14996,"Ġcrashed":14997,"Ġtract":14998,"Ġshed":14999,"Ġconsume":15000,"Based":15001,"download":15002,"init":15003,"Ġvoltage":15004,"Introdu":15005,"Ġcondemned":15006,"ĠFinance":15007,"respect":15008,"Ġexcluded":15009,"Ġestablishing":15010,"heric":15011,"Ġheritage":15012,"Ġspectacular":15013,"Ġunst":15014,"ĠSnowden":15015,"ĠLane":15016,"San":15017,"Ġprotections":15018,"struction":15019,"incinn":15020,"Ġmacro":15021,"Custom":15022,"iosity":15023,"Ġesp":15024,"Ġfunctioning":15025,"Ġmush":15026,"Ġpuzzle":15027,"Ġethical":15028,"Mal":15029,"Ġgoverning":15030,"ĠFerguson":15031,"Ġrestored":15032,"Ġstressed":15033,"ĠCounter":15034,"ĠKas":15035,"clip":15036,"ANS":15037,"Ġseiz":15038,"UK":15039,"byss":15040,"oldown":15041,"api":15042,"Ġpermanently":15043,"ounters":15044,"West":15045,"Through":15046,"Light":15047,"atoes":15048,"Ġneat":15049,"Ġcord":15050,"urer":15051,"Ġseverely":15052,"ĠAven":15053,"Ġinterrog":15054,"Ġtriple":15055,"Given":15056,"Number":15057,"Ġarise":15058,"Ġsher":15059,"plant":15060,"Ġflower":15061,"ĠCou":15062,"Ġate":15063,"Ġnewer":15064,"bul":15065,"Ġmeanwhile":15066,"ĠLair":15067,"Ġadjustment":15068,"ĠCopyright":15069,"Ġdivers":15070,"iological":15071,"Ġgamers":15072,"oat":15073,"Ġhistorically":15074,"Ġanalog":15075,"Ġlongtime":15076,"Ġprescription":15077,"ĠMist":15078,"ĠHyper":15079,"ĠMaine":15080,"ĠDeity":15081,"Ġmultipl":15082,"ĠReincarn":15083,"ĠHyd":15084,"ĠPic":15085,"Sil":15086,"rants":15087,"ĠCris":15088,".;":15089,"({":15090,"ependence":15091,"Ġrecy":15092,"ateur":15093,"Ġquad":15094,"Ġglob":15095,"Ġconced":15096,"team":15097,"Ġcapitalist":15098,"ĠLot":15099,"Ġroyal":15100,"ĠCyber":15101,"Ġblacks":15102,"metic":15103,"riv":15104,"ĠDanny":15105,"Ġspo":15106,"ĠRO":15107,"Ġanimated":15108,"rypted":15109,"ĠDeputy":15110,"Ġrendered":15111,"FE":15112,"Ġstreak":15113,"Ġclouds":15114,"ĠDoug":15115,"~~~~~~~~":15116,"Ġdiscour":15117,"ĠVeh":15118,"Ġpsychology":15119,"ĠJourney":15120,"Ġcrystal":15121,"ĠFrost":15122,"Ġsuspicion":15123,"Ġrelate":15124,"orus":15125,"ĠCrypt":15126,"ĠNVIDIA":15127,"comed":15128,"uting":15129,"incinnati":15130,"Ġvulnerability":15131,"ostic":15132,"Ġisolation":15133,"Ġcooling":15134,"ĠCoalition":15135,"Ġ119":15136,"Four":15137,"ĠDeal":15138,"Ġâī":15139,"semble":15140,"rament":15141,"ĠBarcelona":15142,"Ġ102":15143,"Ġcocaine":15144,"ocalypse":15145,"Feb":15146,"ogenic":15147,"Ġmutation":15148,"Ġcryptoc":15149,"ĠKel":15150,"ĠGit":15151,"ais":15152,"Ġsisters":15153,"ANK":15154,"Ġactivate":15155,"Ter":15156,"Ġdread":15157,"ylon":15158,"Ġpropri":15159,"Aust":15160,"ĠDefault":15161,"Ġoutdoor":15162,"Ġsheer":15163,"ceive":15164,"Ġgently":15165,"о":15166,"Program":15167,"ĠâĨĴ":15168,"Ġvegan":15169,"ĠCrus":15170,"Ġresponsibilities":15171,"ĠHR":15172,"OLD":15173,"Ġprevents":15174,"Ġstiff":15175,"ĠWere":15176,"Ġathletic":15177,"ĠScore":15178,"Ġ):":15179,"Ġcolumns":15180,"ĠLoc":15181,"available":15182,"ĠFram":15183,"ĠSessions":15184,"Ġcompanion":15185,"Ġpacks":15186,"140":15187,"ĠKnights":15188,"Ġfart":15189,"Ġstreams":15190,"Ġshore":15191,"Ġappeals":15192,"ĠPerformance":15193,"haul":15194,"ĠStra":15195,"ĠNag":15196,"103":15197,"ĠTransportation":15198,"BB":15199,"Ev":15200,"zan":15201,"Public":15202,"Ġtwin":15203,"ulsion":15204,"Mult":15205,"Ġelectro":15206,"Ġstatue":15207,"ationally":15208,"ĠNort":15209,"Ġinspection":15210,"/*":15211,"igue":15212,"Ġcompassion":15213,"ĠTales":15214,"ĠStein":15215,"ĠScreen":15216,"ĠBug":15217,"ĠLion":15218,"girl":15219,"Ġwithdrawal":15220,"Ġobjectives":15221,"Ġbloody":15222,"Ġpreliminary":15223,"Ġjacket":15224,"Ġdimensions":15225,"ĠCool":15226,"ĠOccup":15227,"Ġwreck":15228,"Ġdoubled":15229,"anking":15230,"Ġ1975":15231,"Ġglasses":15232,"ĠWang":15233,"prov":15234,"Path":15235,"connected":15236,"ĠMulti":15237,"ĠNorway":15238,"agonist":15239,"Ġfeared":15240,"Ġtouching":15241,"Ġarguably":15242,"¯¯¯¯¯¯¯¯":15243,"ĠNCAA":15244,"chem":15245,"Ġspat":15246,"ĠWWE":15247,"ĠCel":15248,"igger":15249,"Ġattacker":15250,"ĠJoin":15251,"object":15252,"etta":15253,"Ġeliminated":15254,"det":15255,"Ġdestruct":15256,"ĠLucas":15257,"ctuary":15258,"180":15259,"ĠBrady":15260,"ĠBlues":15261,"Bay":15262,"aukee":15263,"Ġtimeline":15264,"Ġdelegates":15265,"written":15266,"ufficient":15267,"Ġshapes":15268,"Copyright":15269,"ouble":15270,"service":15271,"Ġpione":15272,"Ġcolleges":15273,"Ġrows":15274,"Ġspite":15275,"Ġassessed":15276,"360":15277,"Ġlease":15278,"Ġconfidential":15279,"cker":15280,"ĠManning":15281,"ĠVoice":15282,"Ġsealed":15283,"Ġcalculate":15284,"NO":15285,"ĠAssistant":15286,"Ġteenager":15287,"ulent":15288,"atherine":15289,"Ġmock":15290,"Ġdiamond":15291,"Ġfest":15292,"Ġswitched":15293,"Ġresume":15294,"ĠPuerto":15295,"Ġlanes":15296,"iration":15297,"ĠSimilarly":15298,"Ġrod":15299,"ĠSel":15300,"ĠPalace":15301,"ĠLimited":15302,"eous":15303,"Ġvariant":15304,"Ġward":15305,"Ġ))":15306,"Show":15307,"OOK":15308,"Alex":15309,"ĠNep":15310,"bris":15311,"ĠWikipedia":15312,"Ġexceptional":15313,"Ġmanages":15314,"ĠDraw":15315,"Again":15316,"Ġcopper":15317,"utt":15318,"Ġexports":15319,"Ġportfolio":15320,"Ġelevated":15321,"Rated":15322,"ĠOtherwise":15323,"ĠTact":15324,"ĠShel":15325,"ĠTX":15326,"\"âĢĶ":15327,"Ġresur":15328,"ĠWa":15329,"venant":15330,"Ġmonetary":15331,"people":15332,"Email":15333,"Ġfifty":15334,"ĠSweet":15335,"ĠMalaysia":15336,"Ġconfusing":15337,"ĠRio":15338,"uda":15339,"utenant":15340,"\");":15341,"Ġpraised":15342,"Ġvolumes":15343,"turn":15344,"Ġmature":15345,"Ġnonprofit":15346,"Ġpassionate":15347,"ĠPrivate":15348,"Ġ103":15349,"Ġdescend":15350,"ç¥ŀ":15351,"uffy":15352,"headed":15353,"Whether":15354,"rien":15355,"zech":15356,"beit":15357,"Ġchrom":15358,"ĠMcM":15359,"Ġdancing":15360,"Ġeleg":15361,"ĠNoticed":15362,"115":15363,"Ġadvocacy":15364,"ENTS":15365,"ambling":15366,"ĠMinor":15367,"ĠFinn":15368,"Ġpriorities":15369,"Ġthereof":15370,"ĠStage":15371,"ĠRogers":15372,"Ġsubstitute":15373,"ĠJar":15374,"ĠJefferson":15375,"Ġlightly":15376,"102":15377,"ĠLisa":15378,"uits":15379,"ysical":15380,"Ġshifts":15381,"Ġdrones":15382,"Ġworkplace":15383,"Ġresid":15384,"ensed":15385,"ahn":15386,"Ġpreferences":15387,"server":15388,"Ġdebates":15389,"doc":15390,"ĠGods":15391,"Ġhelicopter":15392,"Ġhonour":15393,"Ġconsiderably":15394,"eded":15395,"ĠFemale":15396,"ĠAnne":15397,"Ġreun":15398,"ĠFace":15399,"ĠHallow":15400,"ĠBudget":15401,"Ġcondemn":15402,"Ġtender":15403,"Prof":15404,"ocratic":15405,"ĠTurner":15406,"ĠAgric":15407,"Ġ1976":15408,"Ġapt":15409,"disc":15410,"ĠFighter":15411,"ĠAur":15412,"Ġgarbage":15413,"input":15414,"ĠKarl":15415,"ĠOliver":15416,"ĠLanguage":15417,"kn":15418,"Non":15419,"ĠClar":15420,"Ġtraditions":15421,"Ġadvertisement":15422,"ĠSor":15423,"Ġarchive":15424,"Ġvillages":15425,"750":15426,"Ġimplementing":15427,"waukee":15428,"Ġdietary":15429,"Ġswitching":15430,"Republic":15431,"Ġvelocity":15432,"Ġcit":15433,"ĠAwards":15434,"Ġfinancing":15435,"Ġlasted":15436,")]":15437,"Ġreminder":15438,"Person":15439,"Ġprecision":15440,"Ġdesigners":15441,"ĠFried":15442,"ĠBorder":15443,"Ġtragic":15444,"Ġwield":15445,"Ġinitiatives":15446,"ĠTank":15447,"wer":15448,"Ġjoins":15449,"Ro":15450,"inery":15451,"Ġarrow":15452,"Ġgenerating":15453,"founder":15454,"Ġsearches":15455,"Ġrandomly":15456,"Access":15457,"Ġbatch":15458,"Ġposed":15459,"lat":15460,"Ġpursuing":15461,"asa":15462,"Ġtestified":15463,"forming":15464,"ĠShar":15465,"wiki":15466,"ĠEither":15467,"Sometimes":15468,"Ġsenators":15469,"ĠJohnny":15470,"ĠTaliban":15471,"ĠGPS":15472,"\":\"/":15473,"ãģ®å":15474,"Ġanalyzed":15475,"ĠRubio":15476,"ĠMovement":15477,"opard":15478,"iii":15479,"Stand":15480,"fight":15481,"Ġignoring":15482,"iang":15483,"ĠGN":15484,"soever":15485,"ĠSTAT":15486,"Ġrefusing":15487,"Ġsweat":15488,"Ġbay":15489,"PORT":15490,"irmed":15491,"aky":15492,"Ġdispro":15493,"Ġlabeled":15494,"Ġ108":15495,"Hello":15496,"Ġpleasant":15497,"aba":15498,"Ġtriumph":15499,"Ġaboard":15500,"Ġincom":15501,"ĠCrow":15502,"lett":15503,"Ġfolk":15504,"Ġchase":15505,"``":15506,"ĠBrus":15507,"Ġteens":15508,"cue":15509,"Ġterrain":15510,"hyd":15511,"ilight":15512,"ORY":15513,"Support":15514,"ews":15515,"lli":15516,"raints":15517,"ĠCand":15518,"Ġabused":15519,"achment":15520,"larg":15521,"Bas":15522,"ĠCancer":15523,"Ġ1978":15524,"Ġsupporter":15525,"access":15526,"ĠTermin":15527,"ĠTampa":15528,"ĠANY":15529,"Ġnewest":15530,"ĠCriminal":15531,"edu":15532,"Ġ1930":15533,"Ġadmits":15534,"Ġende":15535,"Ġfailures":15536,"urate":15537,"fulness":15538,"cycl":15539,"ĠSubject":15540,"Ġinfinite":15541,"three":15542,"WA":15543,"pit":15544,"ĠInstall":15545,"Rad":15546,"iliation":15547,"GM":15548,"Ġcontinent":15549,"Ġaccommodate":15550,"ĠClay":15551,"Ġpup":15552,"ĠFunction":15553,"Ġhammer":15554,"ĠAlberta":15555,"Ġrevised":15556,"Ġminorities":15557,"Ġmeasurement":15558,"Connell":15559,"Ġdisable":15560,"ĠMix":15561,"Incre":15562,"Ġfork":15563,"ĠRosen":15564,"Ġimplies":15565,"umblr":15566,"ANG":15567,"Ġproteins":15568,"Ġaggression":15569,"Ġfacilitate":15570,"SN":15571,"Ġillegally":15572,"uer":15573,"Ġacadem":15574,"Ġpuzz":15575,"ĠShift":15576,"pay":15577,"ollo":15578,"Ġaudiences":15579,"Build":15580,"Ġnoble":15581,"Ġsyntax":15582,"âĺħ":15583,"Ġbeam":15584,"ĠBed":15585,"ĠAld":15586,"Ġorigins":15587,"video":15588,"Ġ1977":15589,"ĠAssault":15590,"Ġgarage":15591,"Team":15592,"Ġverdict":15593,"Ġdwar":15594,"ĠVirtual":15595,"event":15596,"Keep":15597,"Ġsentiment":15598,"Ġwildlife":15599,"shirt":15600,"Ġburg":15601,"Ġrecommendation":15602,"represent":15603,"Ġgallery":15604,"owners":15605,"Ġscholar":15606,"Ġconvenience":15607,"ĠSwift":15608,"Ġconvinc":15609,"Cap":15610,"Ġwarfare":15611,"ĠVisual":15612,"Ġconstitute":15613,"Ġabort":15614,"ĠWeather":15615,"ĠLooking":15616,"ĠHem":15617,"Ġmartial":15618,"Ġincoming":15619,"etition":15620,"Ġtolerance":15621,"ĠCreated":15622,"Ġflows":15623,"ĠElder":15624,"Ġsouls":15625,"Ġfoul":15626,"ĠPain":15627,"ĠCAN":15628,"Ġ220":15629,"bc":15630,"hend":15631,"Ġgenius":15632,"Real":15633,"ĠWr":15634,"ometer":15635,"pad":15636,"Ġlimiting":15637,"ĠSi":15638,"ĠLore":15639,"ĠAdventures":15640,"Ġvaried":15641,"Disc":15642,"fin":15643,"ĠPersonal":15644,"Chris":15645,"Ġinvented":15646,"Ġdive":15647,"ĠRise":15648,"Ġoz":15649,"ĠComics":15650,"Ġexpose":15651,"ĠReb":15652,"letters":15653,"site":15654,"imated":15655,"Ġhacking":15656,"Ġeducated":15657,"ĠNobody":15658,"Ġdepri":15659,"Ġincentive":15660,"ãĤ·":15661,"Ġoversight":15662,"Ġtribes":15663,"ĠBelgium":15664,"Ġlicensing":15665,"ourt":15666,"Product":15667,"ahl":15668,"ĠGem":15669,"Ġspecialist":15670,"Ġcra":15671,"anners":15672,"ĠCorbyn":15673,"Ġ1973":15674,"READ":15675,"Ġsummar":15676,"Ġoverlook":15677,"ĠApplication":15678,"Ġinappropriate":15679,"Ġdownloaded":15680,"Que":15681,"ĠBears":15682,"Ġthumb":15683,"ĠCharacter":15684,"ĠReincarnated":15685,"ĠSid":15686,"Ġdemonstrates":15687,"sky":15688,"ĠBloomberg":15689,"ĠArray":15690,"ĠResults":15691,"ĠFourth":15692,"ĠEDT":15693,"ĠOscar":15694,"cend":15695,"Ġ106":15696,"ĠNULL":15697,"ĠHERE":15698,"match":15699,"ĠBrun":15700,"Ġglucose":15701,"ieg":15702,"egu":15703,"Ġcertified":15704,"Ġrelie":15705,"Ġhumanitarian":15706,"Ġprayers":15707,"King":15708,"Ġnan":15709,"hou":15710,"108":15711,"ulu":15712,"Ġrenewable":15713,"Ġdistinguish":15714,"Ġdense":15715,"ĠVent":15716,"ĠPackage":15717,"ĠBoss":15718,"Ġeditors":15719,"Ġmigr":15720,"Tra":15721,"ĠPeters":15722,"ĠArctic":15723,"2004":15724,"ĠCape":15725,"Ġlocally":15726,"Ġlasting":15727,"Ġhandy":15728,".).":15729,"Pan":15730,"ĠRES":15731,"Index":15732,"Ġtensions":15733,"Ġformerly":15734,"Ġideological":15735,"Ġsensors":15736,"Ġdealers":15737,"Ġdefines":15738,"Sk":15739,"Ġproceeds":15740,"Ġproxy":15741,"azines":15742,"ĠBash":15743,"ĠPad":15744,"ĠCraft":15745,"ealous":15746,"Ġsheets":15747,"ometry":15748,"June":15749,"clock":15750,"TT":15751,"ĠTheatre":15752,"ĠBuzz":15753,"Ġchapters":15754,"Ġmillenn":15755,"Ġdough":15756,"ĠCongressional":15757,"Ġimagined":15758,"avior":15759,"Ġclinic":15760,"Ġ1945":15761,"Ġholder":15762,"root":15763,"olester":15764,"Ġrestart":15765,"BN":15766,"ĠHamas":15767,"ĠJob":15768,"Ġorb":15769,"Ġram":15770,"Ġdisclose":15771,"Ġtranslate":15772,"Ġimmigrant":15773,"Ġannoying":15774,"Ġtreaty":15775,"anium":15776,"ĠTea":15777,"ĠLegion":15778,"Ġcrowds":15779,"ĠBec":15780,"ĠAer":15781,"ohyd":15782,"Bro":15783,"Looking":15784,"Ġlbs":15785,"Ġaggress":15786,"Ġseam":15787,"Ġintercept":15788,"ĠMI":15789,"mercial":15790,"activ":15791,"ĠCit":15792,"Ġdimension":15793,"Ġconsistency":15794,"Ġrushing":15795,"ĠDouglas":15796,"Ġtrim":15797,"Install":15798,"icker":15799,"Ġshy":15800,"106":15801,"Ġmentions":15802,"pelled":15803,"ĠTak":15804,"cost":15805,"Ġclassroom":15806,"Ġfortune":15807,"driven":15808,"Ġunle":15809,"ĠWheel":15810,"Ġinvestor":15811,"ĠMasters":15812,"kit":15813,"Ġassociations":15814,"ĠEvolution":15815,"oping":15816,"uscript":15817,"Ġprovincial":15818,"ĠWalter":15819,"avi":15820,"SO":15821,"Ġunlimited":15822,"English":15823,"ĠCards":15824,"ĠEbola":15825,"nered":15826,"Ġrevenge":15827,"Ġoutright":15828,"umper":15829,"Ġfitting":15830,"ĠSolid":15831,"Ġformally":15832,"Ġproblematic":15833,"Ġhazard":15834,"Ġencryption":15835,"Ġstraightforward":15836,"ĠAK":15837,"Ġpse":15838,"ĠOrb":15839,"ĠChamber":15840,"ĠMak":15841,"Contents":15842,"Ġloyalty":15843,"Ġlyrics":15844,"ĠSym":15845,"Ġwelcomed":15846,"Ġcooked":15847,"Ġmonop":15848,"Ġnurse":15849,"Ġmisleading":15850,"Ġeternal":15851,"Ġshifting":15852,"Ġ+=":15853,"Vis":15854,"Ġinstitutional":15855,"illary":15856,"Ġpant":15857,"VERT":15858,"ĠACC":15859,"ĠEnh":15860,"Ġincon":15861,"ĠREUTERS":15862,"Ġdonated":15863,"âĢ¦âĢ¦âĢ¦âĢ¦":15864,"Intern":15865,"Ġexhibit":15866,"Ġtire":15867,"ĠRic":15868,"ĠChampion":15869,"ĠMuhammad":15870,"NING":15871,"ĠSoccer":15872,"Ġmobility":15873,"Ġvarying":15874,"ĠMovie":15875,"Ġlord":15876,"oak":15877,"Field":15878,"Ġvector":15879,"usions":15880,"Ġscrap":15881,"Ġenabling":15882,"make":15883,"Tor":15884,".*":15885,"||":15886,"ĠWebsite":15887,"ĠNPC":15888,"Ġsocialist":15889,"ĠBilly":15890,"ĠAdditional":15891,"Ġcargo":15892,"Ġfarms":15893,"ĠSoon":15894,"ĠPrize":15895,"Ġmidnight":15896,"Ġ900":15897,"seen":15898,"ĠSpot":15899,"Ġsheep":15900,"Ġsponsored":15901,"ĠHi":15902,"ĠJump":15903,"Ġ1967":15904,"Microsoft":15905,"ĠAgent":15906,"Ġcharts":15907,"dir":15908,"Ġadjacent":15909,"Ġtricks":15910,"Ġmanga":15911,"Ġexagger":15912,"/>":15913,"football":15914,"ĠFCC":15915,"GC":15916,"ĠTier":15917,"andra":15918,"OUND":15919,"%),":15920,"Ġfruits":15921,"VC":15922,"ĠAA":15923,"Rober":15924,"Ġmidst":15925,"âĹ":15926,"anka":15927,"Ġlegislature":15928,"ĠNeil":15929,"Ġtourists":15930,"\"\"":15931,"ĠWarning":15932,"ĠNevertheless":15933,"ĠOfficial":15934,"ĠWhatever":15935,"Ġmold":15936,"Ġdrafted":15937,"Ġsubstances":15938,"Ġbreed":15939,"Ġtags":15940,"ĠTask":15941,"Ġverb":15942,"Ġmanufactured":15943,"comments":15944,"ĠPolish":15945,"Prov":15946,"Ġdetermines":15947,"Obama":15948,"kers":15949,"Ġutterly":15950,"Ġsect":15951,"sche":15952,"ĠGates":15953,"ĠChap":15954,"Ġaluminum":15955,"Ġzombie":15956,"ĠTouch":15957,"ĠUP":15958,"Ġsatisfy":15959,"Ġpredomin":15960,"ascript":15961,"Ġelaborate":15962,"Ġ1968":15963,"Ġmeasuring":15964,"ĠVari":15965,"anyahu":15966,"Ġsir":15967,"ulates":15968,"idges":15969,"ickets":15970,"ĠSpencer":15971,"TM":15972,"oubted":15973,"Ġprey":15974,"Ġinstalling":15975,"ĠCab":15976,"reed":15977,"reated":15978,"Supp":15979,"Ġwrist":15980,"ĠKerry":15981,"107":15982,"ĠKle":15983,"ĠRachel":15984,"Ġcotton":15985,"ĠARE":15986,"ĠEle":15987,"Control":15988,"Ġloads":15989,"ĠDod":15990,"anas":15991,"bone":15992,"Ġclassical":15993,"ĠRegional":15994,"ĠInteg":15995,"VM":15996,"Ġdesires":15997,"Ġautism":15998,"supported":15999,"ĠMessage":16000,"Ġcompact":16001,"writer":16002,"Ġ109":16003,"ĠHurricane":16004,"cision":16005,"Ġcycles":16006,"Ġdrill":16007,"Ġcolleague":16008,"Ġmaker":16009,"German":16010,"Ġmistaken":16011,"Sun":16012,"ĠGay":16013,"Ġwhatsoever":16014,"Ġsells":16015,"ĠAirl":16016,"liv":16017,"ĠOption":16018,"Ġsolved":16019,"Ġsectors":16020,"Ġhorizontal":16021,"Ġequation":16022,"ĠSkill":16023,"ĠBio":16024,"gement":16025,"ĠSnap":16026,"ĠLegal":16027,"Ġtrademark":16028,"Ġmakeup":16029,"Ġassembled":16030,"Ġsaves":16031,"ĠHalloween":16032,"ĠVermont":16033,"ĠFROM":16034,"Ġfarming":16035,"ĠPodcast":16036,"acceptable":16037,"ĠHigher":16038,"Ġasleep":16039,"ullivan":16040,"Ġreferen":16041,"ĠLev":16042,"Ġbullets":16043,"oko":16044,"HC":16045,"Ġstairs":16046,"Ġmaintains":16047,"ĠLower":16048,"ĠVi":16049,"Ġmarine":16050,"Ġacres":16051,"Ġcoordinator":16052,"ĠJoh":16053,"Ġcounterparts":16054,"ĠBrothers":16055,"Ġindict":16056,"bra":16057,"Ġchunk":16058,"Ġcents":16059,"Home":16060,"ĠMonth":16061,"Ġaccordingly":16062,"ifles":16063,"ĠGermans":16064,"ĠSyn":16065,"Hub":16066,"Ġeyeb":16067,"âĶĢâĶĢâĶĢâĶĢ":16068,"Ġranges":16069,"ĠHolland":16070,"ĠRobot":16071,"fc":16072,"Mike":16073,"Ġplasma":16074,"Ġswap":16075,"Ġathlete":16076,"ĠRams":16077,",'\"":16078,"Ġinfections":16079,"Ġcorrid":16080,"Ġvib":16081,"Ġpatches":16082,"Ġtraditionally":16083,"Ġrevelation":16084,"Ġsweep":16085,"Ġglance":16086,"Ġinex":16087,"2003":16088,"ĠRaw":16089,"working":16090,"osures":16091,"ĠDat":16092,"ĠLynch":16093,"Ġleverage":16094,"ĠReid":16095,"Ġcorrelation":16096,"iances":16097,"avascript":16098,"Ġrepository":16099,"retty":16100,"Ġ1972":16101,"240":16102,"Ġoun":16103,"pol":16104,"ĠReed":16105,"Ġtactical":16106,"isite":16107,"Apple":16108,"ĠQuinn":16109,"Ġraped":16110,"illo":16111,"Europe":16112,"Ġalgorithms":16113,"ĠRodrig":16114,"iu":16115,"Ġillum":16116,"Ġfame":16117,"Ġintroducing":16118,"Ġdelays":16119,"ĠRaiders":16120,"Ġwhistle":16121,"Ġnovels":16122,"ĠReally":16123,"Ġderiv":16124,"Ġpublications":16125,"ĠNeither":16126,"ĠCommerce":16127,"Ġaston":16128,"language":16129,"Notes":16130,"ĠRoth":16131,"ĠFear":16132,"Ġmate":16133,"Ġparade":16134,"ĠQB":16135,"Ġmaneu":16136,"ĠCincinnati":16137,"mitting":16138,"Ġwaist":16139,"ĠRew":16140,"Ġdiscont":16141,"а":16142,"Ġstaring":16143,"Ġalias":16144,"Ġsecurities":16145,"Ġtoilet":16146,"ĠJedi":16147,"Ġunlaw":16148,"vised":16149,"////////":16150,"](":16151,"ĠWeiss":16152,"Ġprest":16153,"ĠCompan":16154,"Ġmemo":16155,"ĠGrace":16156,"July":16157,"ĠElite":16158,"center":16159,"ĠStay":16160,"Ġgalaxy":16161,"Ġtooth":16162,"ĠSettings":16163,"Ġsubjected":16164,"ãĤ¦":16165,"Ġlineback":16166,"Ġretailers":16167,"ĠWant":16168,"Ġdangers":16169,"Air":16170,"Ġvoluntary":16171,"eway":16172,"Ġinterpreted":16173,"otine":16174,"ç":16175,"Ġpel":16176,"Service":16177,"ĠEventually":16178,"Ġcareers":16179,"Ġthreaten":16180,"Ġmemor":16181,"ĠBradley":16182,"ancies":16183,"sn":16184,"ĠUnknown":16185,"National":16186,"Ġshadows":16187,"ailand":16188,"ĠDash":16189,"Everyone":16190,"izzard":16191,"March":16192,"=(":16193,"Ġpulls":16194,"Ġstranger":16195,"Ġbackwards":16196,"ĠBernard":16197,"imensional":16198,"Ġchron":16199,"Ġtheoretical":16200,"ktop":16201,"Ġware":16202,"ĠInvestig":16203,"ĠIniti":16204,"ĠOperations":16205,"oven":16206,"ocide":16207,"*/":16208,"Ġflames":16209,"ĠCash":16210,"shit":16211,"Ġcab":16212,"ĠAnaly":16213,"ĠSeah":16214,"Ġdefining":16215,"Ġordering":16216,"Ġimmun":16217,"Ġpersistent":16218,"ACH":16219,"Russian":16220,"mans":16221,"Ġhind":16222,"Ġphotography":16223,"©":16224,"Ġhug":16225,"Ġ107":16226,"ĠHence":16227,"iots":16228,"udeau":16229,"Ġsubsidies":16230,"Ġroutinely":16231,"ĠDevice":16232,"itic":16233,"Ġdisgust":16234,"lander":16235,"Ġ1940":16236,"Ġassignment":16237,"ĠBesides":16238,"wick":16239,"ĠDust":16240,"usc":16241,"structed":16242,"111":16243,"develop":16244,"Ġfond":16245,"Ġintersection":16246,"Ġdignity":16247,"Ġcommissioner":16248,"Without":16249,"reach":16250,"Ġcartoon":16251,"Ġscales":16252,"ãĥŃ":16253,"FIG":16254,"Ġsurveys":16255,"ĠIndonesia":16256,"Ġartwork":16257,"Ġunch":16258,"Ġcycling":16259,"unct":16260,"auer":16261,"orate":16262,"ĠObviously":16263,"Ġcharacterized":16264,"feld":16265,"Ġaffirm":16266,"Ġinnings":16267,"Ġé":16268,"Ġaliens":16269,"Ġcloth":16270,"etooth":16271,"ĠCertain":16272,"§":16273,"Ġdigest":16274,"know":16275,"ĠXL":16276,"Ġpredictions":16277,"Ġdin":16278,"WAR":16279,"Ġaftermath":16280,"Example":16281,"ĠSuccess":16282,"ĠThr":16283,"IGN":16284,"Ġminer":16285,"Bus":16286,"Ġclarity":16287,"heimer":16288,"ĠOUT":16289,"ĠSend":16290,"ĠCircle":16291,"ĠDiet":16292,"Ġpronounced":16293,"Ġcreators":16294,"Ġearthquake":16295,"attery":16296,"geons":16297,"Ġod":16298,"Ġlaying":16299,"orp":16300,"Ult":16301,"project":16302,"Ġundermin":16303,"Ġsequel":16304,"Sam":16305,"ĠDarkness":16306,"Ġreception":16307,"bull":16308,"YS":16309,"ĠVir":16310,"Ġsequences":16311,"ĠCoin":16312,"Ġoutfit":16313,"ĠWait":16314,"119":16315,"Ġdelivers":16316,"......":16317,"Ġblown":16318,"ĠEsc":16319,"ĠMath":16320,"perm":16321,"ĠUl":16322,"Ġglim":16323,"Ġfacial":16324,"Ġgreenhouse":16325,"Ġtokens":16326,"/-":16327,"ĠAnnual":16328,"ĠONE":16329,"Ġteenage":16330,"ĠPhysical":16331,"ĠLang":16332,"ĠCelt":16333,"Ġsued":16334,"ividually":16335,"Ġpatience":16336,"chair":16337,"regular":16338,"Ġaug":16339,"inv":16340,"except":16341,"ĠLil":16342,"Ġnest":16343,"fd":16344,"sum":16345,"ĠChase":16346,"Russia":16347,"ĠJennifer":16348,"Ġoffseason":16349,"Overall":16350,"Fore":16351,"Ġriot":16352,"Aud":16353,"former":16354,"Ġdefenders":16355,"ĠCT":16356,"iotic":16357,"ribly":16358,"Ġautomated":16359,"Ġpenis":16360,"Ġinsist":16361,"Ġdiagram":16362,"ĠSQL":16363,"ĠGarc":16364,"Ġwitch":16365,"client":16366,"ierra":16367,"ambers":16368,"Ġrecount":16369,"far":16370,"Very":16371,"osterone":16372,"Ġappreciated":16373,"ĠPerfect":16374,"Section":16375,"Ġdoses":16376,"ocaust":16377,"Ġcostly":16378,"Ġgrams":16379,"ĠShi":16380,"Ġwrestling":16381,"Ġ1971":16382,"Ġtrophy":16383,"Ġnerve":16384,"ĠKaz":16385,"ĠExperience":16386,"Ġpledged":16387,"Ġplayback":16388,"Ġcreativity":16389,"bye":16390,"Ġattackers":16391,"Ġholders":16392,"ĠCoach":16393,"ĠPhD":16394,"Ġtransfers":16395,"Ġcolored":16396,"ĠHindu":16397,"Ġdrown":16398,"Ġlistened":16399,"ĠWA":16400,"iasm":16401,"PO":16402,"Ġappealing":16403,"Ġdisclosed":16404,"ĠChicken":16405,"agging":16406,"Ġpleaded":16407,"Ġnavigation":16408,"ĠReturns":16409,"Ġ[[":16410,"ROR":16411,"EA":16412,"Ġphotographer":16413,"ĠRider":16414,"ippers":16415,"Ġslice":16416,"Ġerect":16417,"Ġhed":16418,"issance":16419,"ĠVikings":16420,"urious":16421,"Ġappet":16422,"oubtedly":16423,"Child":16424,"Ġauthentic":16425,"oos":16426,"ĠMaking":16427,"Ġannouncing":16428,"Ġbod":16429,"Ġmeter":16430,"ĠNine":16431,"ĠRogue":16432,"Ġworkforce":16433,"Ġrenewed":16434,"Ġorganisations":16435,"acs":16436,"PLE":16437,"Short":16438,"Ġcompounds":16439,"ĠVisit":16440,"Ġenvelop":16441,"earth":16442,"Ġsupportive":16443,"ggle":16444,"ĠBrussels":16445,"ĠGuild":16446,"Create":16447,"REL":16448,"Ġaveraged":16449,"Ġ1969":16450,"riages":16451,"Ġlengthy":16452,"Ġforgot":16453,"Okay":16454,"ĠErd":16455,"Ġdealer":16456,"Ġrecession":16457,"DD":16458,"Ġdesperately":16459,"Ġhunger":16460,"Ġsticks":16461,"Ġmph":16462,"ĠFaith":16463,"Ġintentionally":16464,"Ġdemol":16465,"ueller":16466,"ĠSale":16467,"Ġdebris":16468,"spring":16469,"Ġleap":16470,">>>>":16471,"Ġcontainers":16472,"selling":16473,"ranean":16474,"attering":16475,"Ġcommented":16476,"ĠCM":16477,"onut":16478,"Ġwoods":16479,"especially":16480,"Ġorganize":16481,"ivic":16482,"ĠWoods":16483,"anga":16484,"squ":16485,"Ġmaj":16486,"amon":16487,"Ġaxis":16488,"Ġ1974":16489,"ĠDenmark":16490,"Ġwarrior":16491,"ĠPand":16492,"Ġoutlined":16493,"ĠBO":16494,"insula":16495,"zilla":16496,"ebook":16497,"Ġdare":16498,"Ġsearched":16499,"Ġnavigate":16500,"Sn":16501,"writing":16502,"Ġunited":16503,"Japan":16504,"ĠHebrew":16505,"Ġflame":16506,"Ġrelies":16507,"Ġcatching":16508,"ĠSho":16509,"Ġimprisonment":16510,"Ġpockets":16511,"Ġclosure":16512,"ĠFam":16513,"tim":16514,"adequ":16515,"Activity":16516,"Ġrecruiting":16517,"ĠWATCH":16518,"ĠArgentina":16519,"dest":16520,"Ġapologize":16521,"oro":16522,"Ġlacks":16523,"Ġtuned":16524,"ĠGriffin":16525,"Ġinfamous":16526,"Ġcelebrity":16527,"sson":16528,"Ġ----------------------------------------------------------------":16529,"ĠIsis":16530,"ĠDisplay":16531,"Ġcredibility":16532,"Ġeconomies":16533,"Ġheadline":16534,"ĠCowboys":16535,"Ġindef":16536,"Ġlately":16537,"Ġincentives":16538,"button":16539,"ĠMob":16540,"Aut":16541,"Ġresigned":16542,"ĠOm":16543,"camp":16544,"Ġprofiles":16545,"Ġschemes":16546,"olphins":16547,"ayed":16548,"Clinton":16549,"enh":16550,"ĠYahoo":16551,"Ġabst":16552,"Ġank":16553,"suits":16554,"Ġwished":16555,"ĠMarco":16556,"udden":16557,"Ġsphere":16558,"ĠBishop":16559,"Ġincorporated":16560,"ĠPlant":16561,"114":16562,"Ġhated":16563,"pic":16564,"Ġdonate":16565,"Ġlined":16566,"Ġbeans":16567,"Ġstealing":16568,"Ġcostume":16569,"Ġsheriff":16570,"Ġforty":16571,"Ġintact":16572,"Ġadapted":16573,"Ġtravelling":16574,"bart":16575,"Ġnicely":16576,"Ġdried":16577,"Ġscal":16578,"osity":16579,"NOTE":16580,"ĠBh":16581,"ĠBroncos":16582,"ĠIgn":16583,"Ġintimate":16584,"Ġchemistry":16585,"Ġoptimal":16586,"Deb":16587,"ĠGeneration":16588,"Ġ],":16589,"ichi":16590,"ĠWii":16591,"ĠYOUR":16592,"ventions":16593,"Write":16594,"Ġpopul":16595,"unning":16596,"ĠWor":16597,"Vol":16598,"Ġqueen":16599,"heads":16600,"KK":16601,"Ġanalyze":16602,"opic":16603,"earchers":16604,"Ġdot":16605,"legraph":16606,"astically":16607,"Ġupgrades":16608,"Ġcares":16609,"Ġextending":16610,"Ġfreeze":16611,"Ġinability":16612,"Ġorgans":16613,"Ġpretend":16614,"Ġoutlet":16615,"113":16616,"olan":16617,"ĠMall":16618,"uling":16619,"talk":16620,"Ġexpressing":16621,"ĠAlways":16622,"ĠBegin":16623,"files":16624,"Ġlicenses":16625,"%%":16626,"ĠMitt":16627,"Ġfilters":16628,"ĠMilwaukee":16629,"GN":16630,"Ġunfold":16631,"Mo":16632,"Ġnutrition":16633,"ppo":16634,"Bo":16635,"Ġfounding":16636,"Ġundermine":16637,"Ġeasiest":16638,"ĠCzech":16639,"ĠMack":16640,"Ġsexuality":16641,"ĠNixon":16642,"Win":16643,"ĠArn":16644,"ĠKin":16645,"ãĤ£":16646,"icer":16647,"Ġfortun":16648,"Ġsurfaces":16649,"aghd":16650,"Ġcarriers":16651,"ĠPART":16652,"ĠTib":16653,"Ġinterval":16654,"Ġfrustrating":16655,"ĠShip":16656,"ĠArmed":16657,"ffe":16658,"Ġboats":16659,"ĠAbraham":16660,"inis":16661,"Ġsuited":16662,"thread":16663,"iov":16664,"abul":16665,"ĠVenezuela":16666,"Ġtom":16667,"super":16668,"Ġcastle":16669,"although":16670,"ioxide":16671,"eches":16672,"Ġevolutionary":16673,"Ġnegotiate":16674,"Ġconfronted":16675,"Remember":16676,"Ġ170":16677,"Such":16678,"Ġ911":16679,"mult":16680,"ĠAbyss":16681,"urry":16682,"kees":16683,"spec":16684,"ĠBarbara":16685,"Ġbelonging":16686,"Ġvillain":16687,"istani":16688,"Ġaccountable":16689,"Ġportions":16690,"ĠDecl":16691,"Ur":16692,"ĠKate":16693,"gre":16694,"Ġmagazines":16695,"UCK":16696,"Ġregulate":16697,"omon":16698,"ĠAlmost":16699,"Ġoverview":16700,"Ġscram":16701,"Ġloot":16702,"ĠFitz":16703,"Ġcharacteristic":16704,"ĠSnake":16705,"say":16706,"ĠRico":16707,"Ġtrait":16708,"ĠJoined":16709,"aucus":16710,"Ġadaptation":16711,"ĠAirlines":16712,"Ġarchae":16713,"ĠIde":16714,"Ġbikes":16715,"Ġliterary":16716,"Ġinfluences":16717,"ĠUsed":16718,"Creat":16719,"Ġplea":16720,"ĠDefence":16721,"ĠAssass":16722,"Ġpond":16723,"ULT":16724,")\"":16725,"Ġevaluated":16726,"Ġobtaining":16727,"Ġdemographic":16728,"Ġvigil":16729,"aley":16730,"Ġspouse":16731,"ĠSeahawks":16732,"respons":16733,"ĠBelt":16734,"umatic":16735,"Ġrises":16736,"runner":16737,"ĠMichelle":16738,"Ġpotent":16739,"race":16740,"ĠPAC":16741,"Find":16742,"olesterol":16743,"ISS":16744,"ĠIntroduced":16745,"resses":16746,"ignment":16747,"Os":16748,"ĠTu":16749,"ĠDex":16750,"icides":16751,"Ġsparked":16752,"ĠLaura":16753,"ĠBryant":16754,"Ġsmiling":16755,"ĠNexus":16756,"Ġdefendants":16757,"ĠCatal":16758,"Ġdishes":16759,"shaped":16760,"Ġprolong":16761,"mt":16762,"($":16763,"ãĢĤ":16764,"Ġcalculations":16765,"ĠSame":16766,"Ġpiv":16767,"HH":16768,"Ġcancelled":16769,"Ġgrin":16770,"Ġterritories":16771,"istically":16772,"Come":16773,"ĠParent":16774,"Project":16775,"Ġneglig":16776,"ĠPrivacy":16777,"Ġammo":16778,"LECT":16779,"olutely":16780,"ĠEpic":16781,"Ġmisunder":16782,"wal":16783,"April":16784,"mos":16785,"pathy":16786,"ĠCarson":16787,"Ġalbums":16788,"ĠEasy":16789,"Ġpistol":16790,"<<":16791,"Ġ\\(":16792,"target":16793,"help":16794,"Ġinterpre":16795,"conscious":16796,"ĠHousing":16797,"ĠJoint":16798,"127":16799,"Ġbeers":16800,"science":16801,"ĠFirefox":16802,"effective":16803,"ĠCabin":16804,"ĠOkay":16805,"ĠApplic":16806,"Ġspacecraft":16807,"ĠSR":16808,"vet":16809,"ĠStrange":16810,"SB":16811,"Ġcorps":16812,"iberal":16813,"efficient":16814,"Ġprevalence":16815,"Ġeconomists":16816,"118":16817,"Thread":16818,"ordable":16819,"ODE":16820,"ĠCant":16821,"=-=-":16822,"ifiable":16823,"ĠAround":16824,"Ġpole":16825,"Ġwillingness":16826,"CLA":16827,"ĠKid":16828,"Ġcomplement":16829,"Ġscattered":16830,"Ġinmates":16831,"Ġbleeding":16832,"every":16833,"Ġqueue":16834,"ĠTrain":16835,"Ġhij":16836,"Ġmelee":16837,"pleted":16838,"Ġdigit":16839,"Ġgem":16840,"official":16841,"Ġlifting":16842,"е":16843,"Requ":16844,"itutes":16845,"Ġpackaging":16846,"ĠWorkers":16847,"hran":16848,"ĠLebanon":16849,"olesc":16850,"Ġpunished":16851,"ĠJuan":16852,"Ġjam":16853,"ĠDocument":16854,"Ġmapping":16855,"icates":16856,"Ġinevitably":16857,"Ġvanilla":16858,"ĠTon":16859,"Ġwatches":16860,"Ġleagues":16861,"Ġinitiated":16862,"degree":16863,"portion":16864,"Ġrecalls":16865,"Ġruin":16866,"Ġmelt":16867,"IAN":16868,"Ġhem":16869,"Exp":16870,"Ġbaking":16871,"ĠColomb":16872,"atible":16873,"Ġradius":16874,"plug":16875,"ĠIF":16876,"etically":16877,"Ġfict":16878,"HER":16879,"ĠTap":16880,"atinum":16881,"Ġink":16882,"Ġcoh":16883,"ĠWizard":16884,"both":16885,"tex":16886,"Ġspends":16887,"ĠCurrently":16888,"ĠPit":16889,"Ġneurons":16890,"ignt":16891,"Ġrall":16892,"Ġbuses":16893,"building":16894,"Ġadjustments":16895,"Ġcried":16896,"iblical":16897,"atted":16898,"ĠZion":16899,"ĠMatter":16900,"Ġmeditation":16901,"ĠDennis":16902,"Ġours":16903,"ĠTab":16904,"Ġrankings":16905,"ortal":16906,"Ġadvers":16907,"Ġsurrender":16908,"ĠGob":16909,"cium":16910,"omas":16911,"imeter":16912,"Ġmultiplayer":16913,"Ġheroin":16914,"Ġoptimistic":16915,"Ġindicator":16916,"ĠBrig":16917,"Ġgrocery":16918,"Ġapplicant":16919,"ĠRocket":16920,"vid":16921,"Exception":16922,"pent":16923,"Ġorganizing":16924,"Ġencounters":16925,"ĠTOD":16926,"Ġjewel":16927,"Save":16928,"ĠChristie":16929,"Ġheating":16930,"Ġlazy":16931,"ĠCP":16932,"Ġcousin":16933,"Config":16934,"Ġregener":16935,"Ġnearest":16936,"Ġachieving":16937,"ENS":16938,"throw":16939,"ĠRichmond":16940,"antle":16941,"2002":16942,"Ġanten":16943,"bird":16944,"133":16945,"Ġnarc":16946,"raint":16947,"unny":16948,"ĠHispanic":16949,"ournaments":16950,"Ġprophe":16951,"ĠThailand":16952,"ĠTi":16953,"Ġinjection":16954,"Ġinherit":16955,"ravis":16956,"Ġmedi":16957,"Ġwhoever":16958,"ĠDEBUG":16959,"GP":16960,"ĠHud":16961,"Card":16962,"prom":16963,"Ġpor":16964,"Ġoverhead":16965,"Law":16966,"Ġviolate":16967,"Ġheated":16968,"Ġdescriptions":16969,"Ġachievements":16970,"ĠBeer":16971,"ĠQuant":16972,"Was":16973,"Ġeighth":16974,"ĠIv":16975,"Ġspecialized":16976,"UPDATE":16977,"ĠDelta":16978,"Pop":16979,"Jul":16980,"ĠAsk":16981,"ophy":16982,"Ġnewsletters":16983,"ĠTool":16984,"Ġgard":16985,"ĠConfeder":16986,"ĠGMT":16987,"ĠAbbott":16988,"Ġimmunity":16989,"ĠVM":16990,"Islam":16991,"Ġimplicit":16992,"wd":16993,"Ġ1944":16994,"ravity":16995,"ometric":16996,"Ġsurviving":16997,"urai":16998,"ĠPrison":16999,"Ġrust":17000,"ĠSketch":17001,"Ġbees":17002,"ĠTheory":17003,"Ġmerit":17004,"Tex":17005,"chat":17006,"Ġmim":17007,"Ġpaste":17008,"ĠKoch":17009,"Ġignorance":17010,"ĠShoot":17011,"Ġbasement":17012,"United":17013,"ĠAdvis":17014,"height":17015,"Ġfoster":17016,"Ġdetain":17017,"information":17018,"Ġneural":17019,"';":17020,"Ġproves":17021,"allery":17022,"Ġinvitation":17023,"umbers":17024,"Ġcattle":17025,"Ġbicycle":17026,"zi":17027,"Ġconsultant":17028,"Ġapology":17029,"ĠTiger":17030,"Ġ123":17031,"999":17032,"Ġindividually":17033,"rt":17034,"igion":17035,"ĠBrazilian":17036,"Ġdisturb":17037,"Ġentrepreneurs":17038,"Ġforests":17039,"cerpt":17040,"plates":17041,"pher":17042,"clipse":17043,"Ġtwitter":17044,"Ġacids":17045,"ographical":17046,"hum":17047,"ĠBald":17048,"ifully":17049,"Ġcompiler":17050,"ĠDA":17051,"Ġdonor":17052,"asi":17053,"Ġtribal":17054,"lash":17055,"ĠConfig":17056,"Ġapplicants":17057,"Ġsalaries":17058,"135":17059,"Putin":17060,"ĠFocus":17061,"irs":17062,"Ġmisconduct":17063,"ĠHaz":17064,"Ġeaten":17065,"Mobile":17066,"Muslim":17067,"ĠMarcus":17068,"viol":17069,"Ġfavorable":17070,"Ġstub":17071,"adin":17072,"ĠHob":17073,"Ġfaithful":17074,"Ġelectronics":17075,"Ġvacuum":17076,"wait":17077,"backed":17078,"economic":17079,"dist":17080,"Ġtenure":17081,"Ġsincere":17082,"ĠTogether":17083,"ĠWave":17084,"Ġprogression":17085,"Ġdenying":17086,"Ġdistress":17087,"braska":17088,"third":17089,"Ġmixing":17090,"Ġcolonial":17091,"Ġprivately":17092,"Ġunrest":17093,"aternity":17094,"Ġpremises":17095,"anti":17096,"gregation":17097,"Ġlicence":17098,"ĠHind":17099,"ĠSamuel":17100,"Ġconvincing":17101,"ĠAce":17102,"ĠRust":17103,"ĠNetanyahu":17104,"Ġhandles":17105,"ĠPatch":17106,"oriented":17107,"aho":17108,"ĠGonz":17109,"Ġhackers":17110,"claimer":17111,"Ġcustoms":17112,"ĠGran":17113,"fighters":17114,"Ġluc":17115,"Ġmanuscript":17116,"arenthood":17117,"Ġdevil":17118,"Ġwarriors":17119,"Ġoffenders":17120,"William":17121,"Ġholidays":17122,"Ġnightmare":17123,"Ġlever":17124,"ifferent":17125,"Stat":17126,"Ġexhibition":17127,"puted":17128,"ĠPure":17129,"Ġalpha":17130,"Ġenthusiasm":17131,"ĠRepresentatives":17132,"EAR":17133,"ĠTyp":17134,"Ġwheat":17135,"ĠAlf":17136,"Ġcorrection":17137,"Ġevangel":17138,"ATT":17139,"Miss":17140,"Ġsoup":17141,"Ġimplied":17142,"param":17143,"Ġsexy":17144,"ĠLux":17145,"Ġrepublic":17146,"patch":17147,"ablish":17148,"Ġicons":17149,"Ġfathers":17150,"ĠGET":17151,"ĠCarib":17152,"Ġregulated":17153,"ĠCohen":17154,"ĠBobby":17155,"Ġner":17156,"Ġbent":17157,"ventory":17158,"ĠAlong":17159,"ĠEST":17160,"ĠWallace":17161,"Ġmurders":17162,"rise":17163,"kell":17164,"ĠCommonwealth":17165,"Ġnasty":17166,"eta":17167,"ĠMIT":17168,"Ġadministered":17169,"Ġgenuinely":17170,"Editor":17171,"nick":17172,"Ġhydro":17173,"********************************":17174,"ĠBle":17175,"Ġfines":17176,"Ġgorge":17177,"ausible":17178,"rh":17179,"Ġapple":17180,"mentioned":17181,"Ġrope":17182,"otyp":17183,"HR":17184,"Ġdisappointing":17185,"Ġcage":17186,"nik":17187,"Ġdoubts":17188,"ĠFREE":17189,"prints":17190,"ĠMUST":17191,"Ġvendors":17192,"ĠInqu":17193,"Ġliberals":17194,"Ġcontractor":17195,"Ġupside":17196,"children":17197,"Ġtricky":17198,"Ġregulators":17199,"charged":17200,"liter":17201,"Ġ***":17202,"Ġrebell":17203,"lang":17204,"Ġlocals":17205,"Ġphysicians":17206,"Ġhey":17207,"arse":17208,"tm":17209,"ĠLex":17210,"Ġbehavioral":17211,"successful":17212,"FX":17213,"Ġbrick":17214,"ovic":17215,"Ġconform":17216,"Ġreviewing":17217,"Ġinsights":17218,"Ġbiology":17219,"ĠRemove":17220,"ĠExtra":17221,"Ġcommitting":17222,"induced":17223,"ignty":17224,"igm":17225,"Ġatomic":17226,"Common":17227,"ĠEM":17228,"ĠPere":17229,"ĠItems":17230,"eh":17231,"Ġpreserved":17232,"ĠHood":17233,"Ġprisoner":17234,"Ġbankruptcy":17235,"Ġgren":17236,"ushes":17237,"Ġexploitation":17238,"Ġsignatures":17239,"Ġfinan":17240,"],\"":17241,"ĠMR":17242,"Ġmeg":17243,"remlin":17244,"Ġmusicians":17245,"Ġselecting":17246,"Ġexamining":17247,"INK":17248,"lated":17249,"Hi":17250,"Ġartic":17251,"Ġpets":17252,"Ġimpair":17253,"ĠMAN":17254,"Ġtablets":17255,"include":17256,"Range":17257,"Ġcaut":17258,"Ġlogs":17259,"Ġmounting":17260,"Ġunaware":17261,"Ġdynamics":17262,"ĠPalestine":17263,"ĠQuarter":17264,"ĠPurple":17265,"Ġma":17266,"ĠImport":17267,"Ġcollections":17268,"ciation":17269,"Ġsuccessor":17270,"Ġclone":17271,"Ġaiming":17272,"Ġpossessed":17273,"Ġsticking":17274,"Ġshaking":17275,"Ġlocate":17276,"ĠHockey":17277,"Turn":17278,"170":17279,"Ġfifteen":17280,"ĠHarrison":17281,"Ġcontinuously":17282,"ĠTC":17283,"ĠValent":17284,"ĠRescue":17285,"Ġbypass":17286,"amount":17287,"Ġmast":17288,"Ġprotects":17289,"Ġartistic":17290,"Ġsometime":17291,"Ġshoe":17292,"Ġshouted":17293,"ificant":17294,"etitive":17295,"ĠRegister":17296,"ĠJin":17297,"Ġconcentrated":17298,"lington":17299,"onies":17300,"Ġgenerator":17301,"yrim":17302,"ĠArmen":17303,"Ġclearing":17304,"ido":17305,"ĠTW":17306,"alph":17307,"Ġladies":17308,"Hard":17309,"Ġdialog":17310,"Ġinputs":17311,"æľ":17312,"Ġposes":17313,"Ġslots":17314,"ĠPremium":17315,"Ġleaks":17316,"Ġbosses":17317,"Ġ113":17318,"course":17319,"Acc":17320,"ĠNewton":17321,"ĠAustria":17322,"ĠMage":17323,"Ġteaches":17324,"abad":17325,"Ġwears":17326,"Ġcyl":17327,"Ġcurse":17328,"ĠSales":17329,"ĠWings":17330,"Ġpsy":17331,"Ġgaps":17332,"ĠIceland":17333,"ĠPinterest":17334,"Ġlandlord":17335,"Ġdefinitions":17336,"ĠKer":17337,"Ġsufficiently":17338,"ĠPence":17339,"ĠArchitect":17340,"Ġsurpass":17341,"Ġ114":17342,"Ġsuperhero":17343,"ĠDisease":17344,"Ġpriests":17345,"ĠCulture":17346,"Ġdefinitive":17347,"Ġsecretly":17348,"ĠDance":17349,"install":17350,"chief":17351,"ĠJessica":17352,"Would":17353,"Updated":17354,"Ġlocker":17355,"ĠKay":17356,"Ġmemorial":17357,"è¦":17358,"fat":17359,"Ġdisgu":17360,"Ġflavors":17361,"ĠBaseball":17362,"ĠResistance":17363,"Ġkicks":17364,"Ġenv":17365,"Ġteenagers":17366,"Dark":17367,"ĠCAR":17368,"Ġhalt":17369,"ĠLG":17370,"ĠGabriel":17371,"Ġfever":17372,"Ġsatur":17373,"Ġmall":17374,"Ġaffiliate":17375,"ĠSleep":17376,"ĠSpecific":17377,"ĠVel":17378,"Ġjar":17379,"ĠSacred":17380,"ĠEdwards":17381,"ĠACL":17382,"Ġretained":17383,"ĠGiant":17384,"Ġlimitation":17385,"inces":17386,"Ġrefusal":17387,"ĠTale":17388,"ĠButler":17389,"Ġaccidents":17390,"ĠCSS":17391,"Ġimported":17392,"ĠCopy":17393,"α":17394,"ERT":17395,"zel":17396,"Ġdivisions":17397,"hots":17398,"ĠAlb":17399,"ĠDS":17400,"Loader":17401,"Washington":17402,"atisf":17403,"ĠCreative":17404,"\\.":17405,"ĠAutom":17406,"redict":17407,"Ġreceptor":17408,"ĠCarlos":17409,"Method":17410,"oka":17411,"Ġmalicious":17412,"Ġstepping":17413,",[":17414,"ĠDad":17415,"Ġattraction":17416,"ĠEffects":17417,"ĠPirate":17418,"ĠCer":17419,"ĠIndustry":17420,"ĠRud":17421,"Ġcharter":17422,"Ġdining":17423,"Ġinsists":17424,"Ġconfigure":17425,"Ġ(#":17426,"ĠSimple":17427,"ĠScroll":17428,"UTC":17429,"175":17430,"ĠKon":17431,"Ġmarketplace":17432,"ĠãĤ":17433,"Ġrefres":17434,"Ġgates":17435,"erred":17436,"ĠPod":17437,"Ġbehave":17438,"Frank":17439,"node":17440,"Ġendorsed":17441,"hett":17442,"asive":17443,"ĠHomeland":17444,"Ġrides":17445,"ĠLeave":17446,"erness":17447,"Ġflooding":17448,"AFP":17449,"Ġrisen":17450,"Ġcontinually":17451,"Ġunanim":17452,"ĠContract":17453,"ĠPas":17454,"Ġguided":17455,"ĠChile":17456,"bd":17457,"Ġsucc":17458,"ptic":17459,"Ġcommittees":17460,"ĠLuther":17461,"ĠAnyone":17462,"Ġsab":17463,"124":17464,"Ġpixel":17465,"ĠBak":17466,"ĠTag":17467,"ĠBennett":17468,"Enter":17469,"small":17470,"ĠPresidential":17471,"Ġpul":17472,"Ġcontrace":17473,"archive":17474,"Ġcoastal":17475,"ĠKids":17476,"192":17477,"âĢ²":17478,"icky":17479,"INGTON":17480,"Ġwolf":17481,"ĠStalin":17482,"Tur":17483,"idget":17484,"amas":17485,"ĠUnless":17486,"Ġsponsor":17487,"Ġmorph":17488,"ĠChoose":17489,"Ġrunner":17490,"Ġunbel":17491,"Ġmud":17492,"ĠMana":17493,"Ġdubbed":17494,"Ġgodd":17495,"urers":17496,"window":17497,"Ġrelied":17498,"Ġcelebrating":17499,"osc":17500,"Ġ135":17501,"Ġlobbying":17502,"Ġincomplete":17503,"Ġrestriction":17504,"Ġincap":17505,"itus":17506,"Ġexpectation":17507,"ĠApollo":17508,"Ġintens":17509,"Ġsync":17510,"GH":17511,"Ġmanipulation":17512,"BY":17513,"Ġspear":17514,"Ġbreasts":17515,"Ġvolcan":17516,"ilia":17517,"Material":17518,"Ġformats":17519,"ĠBast":17520,"Ġparliamentary":17521,"Ġsnake":17522,"Ġservants":17523,"ĠTrudeau":17524,"ĠGrim":17525,"ĠArabic":17526,"ĠSCP":17527,"ĠBoys":17528,"station":17529,"Ġprospective":17530,"orde":17531,"initialized":17532,"Ġbored":17533,"ABLE":17534,"Ġaccessed":17535,"Ġtaxi":17536,"ĠShell":17537,"aiden":17538,"ursed":17539,"inates":17540,"ĠInsurance":17541,"ĠPete":17542,"September":17543,"650":17544,"Ġadventures":17545,"ĠCover":17546,"Ġtribute":17547,"Ġsketch":17548,"Ġempower":17549,"ĠØ":17550,"ĠGlenn":17551,"ĠDaw":17552,"=\\\"":17553,"ĠPolitics":17554,"Ġguides":17555,"Ġdioxide":17556,"ĠGore":17557,"ĠBright":17558,"ĠSierra":17559,"Ġvalued":17560,"cond":17561,"Ġpointer":17562,"Select":17563,"Ġrisky":17564,"Ġabsorb":17565,"images":17566,"Ġrefuses":17567,"Ġbonuses":17568,"___":17569,"Ġhilar":17570,"ĠFeatures":17571,"220":17572,"ĠCollector":17573,"Foot":17574,"Ġ1964":17575,"culus":17576,"Ġdawn":17577,"Ġworkout":17578,"ĠLO":17579,"Ġphilosophical":17580,"ĠSandy":17581,"ĠYouth":17582,"Ġliable":17583,"Af":17584,"blue":17585,"Ġoverturn":17586,"lessness":17587,"ĠTribune":17588,"ĠIng":17589,"Ġfactories":17590,"Ġcatches":17591,"Ġprone":17592,"Ġmatrix":17593,"Ġlogin":17594,"Ġinacc":17595,"Ġexert":17596,"sys":17597,"Ġneedle":17598,"ĠQur":17599,"Ġnotified":17600,"oulder":17601,"tx":17602,"Ġreminds":17603,"Ġpublishers":17604,"Ġnort":17605,"Ġgit":17606,"Ġflies":17607,"ĠEmily":17608,"Ġflowing":17609,"ĠAlien":17610,"ĠStrateg":17611,"Ġhardest":17612,"Ġmodification":17613,"API":17614,"ĠMY":17615,"Ġcrashes":17616,"stairs":17617,"number":17618,"Ġurging":17619,"channel":17620,"ĠFalcon":17621,"Ġinhabitants":17622,"Ġterrifying":17623,"Ġutilize":17624,"Ġbanner":17625,"Ġcigarettes":17626,"Ġsenses":17627,"ĠHolmes":17628,"Ġpractition":17629,"ĠPhillips":17630,"otto":17631,"Ġcompile":17632,"Model":17633,"ĠKo":17634,"Ġ[]":17635,"Americans":17636,"ĠTerms":17637,"Ġmedications":17638,"ĠAna":17639,"Ġfundamentally":17640,"ĠNotice":17641,"Ġweaker":17642,"Ġ0000":17643,"Ġgarlic":17644,"Ġoutbreak":17645,"Ġeconomist":17646,"ĠBirth":17647,"Ġobstacles":17648,"arcer":17649,"ĠOrthodox":17650,"Ġplacebo":17651,"ĠCrew":17652,"aspberry":17653,"ĠAngels":17654,"Ġdischarge":17655,"Ġdestructive":17656,"117":17657,"ĠRising":17658,"Ġdairy":17659,"late":17660,"Ġcollision":17661,"ĠTigers":17662,"eanor":17663,"ocumented":17664,"ĠInvalid":17665,"Ġdont":17666,"ĠLiter":17667,"ĠVa":17668,"Ġhydrogen":17669,"Ġvariants":17670,"ĠBrowns":17671,"Ġ1965":17672,"Ġindigenous":17673,"Ġtrades":17674,"Ġremainder":17675,"Ġswept":17676,"ĠImpact":17677,"Ġredist":17678,"Ġunint":17679,"graduate":17680,"ãĥķ":17681,"ĠWILL":17682,"ãģ®ç":17683,"ĠCritical":17684,"Ġfisher":17685,"Ġvicious":17686,"Ġreversed":17687,"Year":17688,"ĠSox":17689,"Ġshootings":17690,"Ġfilming":17691,"Ġtouchdowns":17692,"aires":17693,"mel":17694,"Ġgrandfather":17695,"Ġaffection":17696,"ingle":17697,"Ġoverly":17698,"Additional":17699,"Ġsupreme":17700,"ĠGrad":17701,"Ġsporting":17702,"Ġmercy":17703,"ĠBrooks":17704,"ounty":17705,"Ġperforms":17706,"Ġtightly":17707,"Ġdemons":17708,"Ġkillings":17709,"Ġfaction":17710,"ĠNova":17711,"auts":17712,"Ġundoubtedly":17713,"arin":17714,"Ġunderway":17715,"rak":17716,"Ġliv":17717,"ĠRegion":17718,"Ġbriefing":17719,"sers":17720,"cloud":17721,"ĠMik":17722,"usp":17723,"Ġprediction":17724,"azor":17725,"Ġportable":17726,"ĠGand":17727,"Ġpresenting":17728,"Ġ1080":17729,"»":17730,"ushi":17731,"ĠSpark":17732,"thereum":17733,"Ġjustification":17734,"ĠNy":17735,"Ġcontractors":17736,"mingham":17737,"ĠStyle":17738,"åħ":17739,"ĠChronicles":17740,"ĠPicture":17741,"Ġproving":17742,"Ġwives":17743,"sett":17744,"Ġmolecules":17745,"ĠFairy":17746,"Ġconsisting":17747,"Ġpier":17748,"alone":17749,"inition":17750,"Ġnucle":17751,"json":17752,"Ġgotta":17753,"Ġmobil":17754,"Ġverbal":17755,"arium":17756,"Ġmonument":17757,"ucked":17758,"Ġ256":17759,"Tech":17760,"minecraft":17761,"ĠTrack":17762,"Ġtile":17763,"Ġcompatibility":17764,"asis":17765,"Ġsadd":17766,"Ġinstructed":17767,"ĠMueller":17768,"Ġlethal":17769,"Ġhormone":17770,"Ġorche":17771,"else":17772,"Ġskelet":17773,"Ġentertaining":17774,"Ġminimize":17775,"again":17776,"Ġundergo":17777,"Ġconstraints":17778,"Ġcigarette":17779,"ĠIslamist":17780,"Ġtravels":17781,"ĠPanthers":17782,"lings":17783,"Care":17784,"Ġlawsuits":17785,"uras":17786,"Ġcryst":17787,"Ġlowered":17788,"Ġaerial":17789,"Ġcombinations":17790,"Ġhaun":17791,"Ġcha":17792,"Ġvine":17793,"Ġquantities":17794,"Ġlinking":17795,"bank":17796,"Ġsoy":17797,"Bill":17798,"ĠAngela":17799,"Ġrecipient":17800,"ĠProtest":17801,"Ġsocket":17802,"Ġsolidarity":17803,"ĠâĨ":17804,"mill":17805,"Ġvaries":17806,"ĠPakistani":17807,"Dragon":17808,"Ġune":17809,"Ġhorizon":17810,"³³³³³³³³":17811,"Ġprovinces":17812,"Ġfrankly":17813,"Ġenacted":17814,"notes":17815,"['":17816,"Ġ192":17817,"ocracy":17818,"Ġendorsement":17819,"Ġovertime":17820,"True":17821,"Lab":17822,"licted":17823,"ĠDNC":17824,"Ġbeats":17825,"ĠJamie":17826,"152":17827,"ĠINT":17828,"Contact":17829,"Ġaccounted":17830,"hash":17831,"ĠPackers":17832,"pires":17833,"Ġlesbian":17834,"Ġamendments":17835,"Ġhopeful":17836,"ĠFinland":17837,"Ġspotlight":17838,"Ġconfigured":17839,"Ġtroubled":17840,"Ġgaze":17841,"ĠCalgary":17842,"Ġreliability":17843,"Ġinsurg":17844,"swer":17845,"buy":17846,"ĠSkin":17847,"Ġpixels":17848,"Ġhandgun":17849,"Ġparas":17850,"Ġcategor":17851,"ĠEL":17852,"ĠRex":17853,"Indeed":17854,"Ġkinda":17855,"Ġconjunction":17856,"ĠBryan":17857,"ĠManufact":17858,"yang":17859,"Plus":17860,"SQL":17861,"ishment":17862,"Ġdominate":17863,"Ġnail":17864,"Ġoath":17865,"Ġerupt":17866,"ĠFine":17867,"itbart":17868,"ĠChip":17869,"ĠAbd":17870,"ĠNam":17871,"Ġbuyer":17872,"Ġdissent":17873,"Leaks":17874,"Contin":17875,"Ġrider":17876,"ĠSomeone":17877,"Ġillusion":17878,"cin":17879,"ĠBoeing":17880,"Ġinadequ":17881,"ovation":17882,"iants":17883,"Ġrebuild":17884,"450":17885,"ĠDestiny":17886,"SW":17887,"ĠTill":17888,"Hit":17889,"iaz":17890,"ĠBangl":17891,"achers":17892,"ĠReform":17893,"Ġsegments":17894,"Ġsystematic":17895,"dc":17896,"ĠConservatives":17897,"Ġportal":17898,"hor":17899,"ĠDragonbound":17900,"Ġdragged":17901,"omo":17902,"Ġthee":17903,"advert":17904,"ĠReports":17905,"ĠEt":17906,"Ġbarrels":17907,"August":17908,"Ġcomparisons":17909,"Ġhex":17910,"Ġanthrop":17911,"\"[":17912,"borough":17913,"abi":17914,"Ġpictured":17915,"playing":17916,"ĠAddress":17917,"ĠMirror":17918,"Smith":17919,"Ġtires":17920,"ĠNPR":17921,"AAAA":17922,"Ġclassification":17923,"ĠThan":17924,"ĠHarm":17925,"ĠRA":17926,"Ġrejection":17927,"mination":17928,"Ġranged":17929,"ĠFalls":17930,"DI":17931,"Host":17932,"ãĤ´":17933,"ĠExample":17934,"listed":17935,"thirds":17936,"Ġsafegu":17937,"brand":17938,"Ġprobable":17939,"Canada":17940,"ITION":17941,"ĠQaeda":17942,"Ġchick":17943,"Ġimports":17944,"hit":17945,"loc":17946,"WW":17947,"Ġblew":17948,"Ġanytime":17949,"Ġwholes":17950,"iked":17951,"Ġcalculation":17952,"create":17953,"ĠOri":17954,"Ġupgraded":17955,"Ġappar":17956,"utory":17957,"ĠMol":17958,"Brit":17959,"ĠJong":17960,"INAL":17961,"ĠStarting":17962,"Ġdice":17963,"urtle":17964,"Ġrelying":17965,"closure":17966,"Ġprofitable":17967,"Ġslaughter":17968,"ĠManual":17969,"caster":17970,"Ġ\"$":17971,"Ġfeather":17972,"ĠSimply":17973,"ieves":17974,"Ġdeterior":17975,"ĠPCI":17976,"Ġstamp":17977,"Ġflaws":17978,"Ġshade":17979,"hammer":17980,"Ġpassport":17981,"Ġconting":17982,"amel":17983,"Ġobservers":17984,"Ġneglect":17985,"ĠRB":17986,"ĠBrotherhood":17987,"Ġskeptical":17988,"family":17989,"usk":17990,"Ġemotionally":17991,"âĻ":17992,"ĠBeta":17993,"asonable":17994,"idity":17995,"ĠMul":17996,"Ġkicking":17997,"ĠCarm":17998,"ollah":17999,"VERTIS":18000,"ĠAthen":18001,"Ġladder":18002,"ĠBullet":18003,"å£":18004,"0001":18005,"ĠWildlife":18006,"ĠMask":18007,"ĠNan":18008,"Rev":18009,"Ġunacceptable":18010,"legal":18011,"Ġcrowded":18012,"agi":18013,"ĠCox":18014,"je":18015,"Ġmorality":18016,"Ġfuels":18017,"Ġcables":18018,"Ġmankind":18019,"ĠCaribbean":18020,"Ġanchor":18021,"Ġbyte":18022,"ĠOften":18023,"ĠOz":18024,"Ġcrafted":18025,"Ġhistorian":18026,"ĠWu":18027,"Ġtowers":18028,"ĠCitizens":18029,"Ġhelm":18030,"Ġcredentials":18031,"Ġsingular":18032,"ĠJesse":18033,"Ġtackles":18034,"Ġcontempt":18035,"Ġafore":18036,"ĠShadows":18037,"Ġnil":18038,"Ġurgent":18039,"apple":18040,"blood":18041,"Ġvon":18042,"Ġoffline":18043,"Ġbreathe":18044,"Ġjumps":18045,"Ġirrelevant":18046,"oxic":18047,"omal":18048,"important":18049,"Jim":18050,"Ġgloves":18051,"arming":18052,"depth":18053,"Ġtalents":18054,"ookie":18055,"ĠSB":18056,"Ġpalm":18057,"uffs":18058,"esta":18059,"IGH":18060,"Ġcanon":18061,"ĠVerizon":18062,"ĠPle":18063,"Ġcoupled":18064,"velt":18065,"Ġfundraising":18066,"ĠGetting":18067,"ĠDLC":18068,"Ġmathematical":18069,"ĠHS":18070,"ĠCardinals":18071,"telling":18072,"Ġsponsors":18073,"ĠÏ":18074,"ĠBulls":18075,"option":18076,"Ġpropose":18077,"Ġmemorable":18078,"Ġembraced":18079,"Ġdeclining":18080,"Health":18081,"eda":18082,"Ġ};":18083,"Ġspam":18084,"mile":18085,"Ġpitcher":18086,"ĠEight":18087,"Ġcaring":18088,"utic":18089,"role":18090,"Ġairline":18091,"ernandez":18092,"ĠAthlet":18093,"Ġcertification":18094,"uxe":18095,"riger":18096,"Ġempir":18097,"Ġsensation":18098,"Ġdism":18099,"Ġbolt":18100,"Ġevolve":18101,"House":18102,"Ġconsultation":18103,"ĠDuty":18104,"Ġtouches":18105,"ĠNathan":18106,"Ġfaint":18107,"had":18108,"\"(":18109,"ĠConsumer":18110,"ĠExtreme":18111,"Ġ127":18112,"ĠHerm":18113,"ĠSacrament":18114,"izoph":18115,"Ġanxious":18116,"ulously":18117,"Ġsocially":18118,"ĠUTC":18119,"Ġsolving":18120,"ĠLetter":18121,"History":18122,"educ":18123,"Price":18124,"));":18125,"Ġreload":18126,"amic":18127,"Ġpork":18128,"Ġdiscourse":18129,"Ġtournaments":18130,"airo":18131,"ĠKur":18132,"ĠCosta":18133,"Ġviolating":18134,"Ġinterfere":18135,"Ġrecreational":18136,"uffle":18137,"Ġspeeches":18138,"Ġneeding":18139,"Ġremembers":18140,"Ġcredited":18141,"nia":18142,"focused":18143,"amera":18144,"Ġbru":18145,"umbs":18146,"ĠCuban":18147,"Ġpreceding":18148,"Ġnonsense":18149,"acial":18150,"Ġsmartphones":18151,"ĠStories":18152,"Sports":18153,"ĠEmergency":18154,"ouncing":18155,"efined":18156,"Ġber":18157,"Ġconsulting":18158,"Ġmasters":18159,"heastern":18160,".\"[":18161,"ĠRunning":18162,"Ġsuscept":18163,"ĠFeng":18164,"America":18165,"prises":18166,"stitial":18167,"ĠWeekly":18168,"ĠGreater":18169,"modules":18170,"ifter":18171,"Graphics":18172,"uler":18173,"Ġwholly":18174,"Ġsuppress":18175,"Ġconcealed":18176,"Ġhappily":18177,"Ġaccepts":18178,"ĠEnjoy":18179,"Ġrivers":18180,"ĠExcept":18181,"225":18182,"ĠNHS":18183,"ĠMcConnell":18184,"Ġpussy":18185,"ferred":18186,"utable":18187,"Ġattain":18188,"Ġ>=":18189,"Ġdeposits":18190,"rophic":18191,"Ġnotorious":18192,"ĠShaw":18193,"ilitation":18194,"Ġepidemic":18195,"allic":18196,"Ġsmallest":18197,"ovich":18198,"Ġaccessories":18199,"perties":18200,"Ġsurplus":18201,"ĠMech":18202,"Ġambig":18203,"ĠImmigration":18204,"Ġchim":18205,"eval":18206,"Ġpracticing":18207,"ĠMystery":18208,"Ġdomains":18209,"ĠSilicon":18210,"apps":18211,"Ġkilometers":18212,"ea":18213,"ĠSmash":18214,"Ġwarranty":18215,"Ġnost":18216,"sil":18217,"rev":18218,"Jon":18219,"ĠDublin":18220,"Ġtastes":18221,"Ġbout":18222,"great":18223,"error":18224,"Ġswitches":18225,"ĠBapt":18226,"DO":18227,"oki":18228,"Ġsourced":18229,"produ":18230,"Ġattachment":18231,"ĠIssue":18232,"ĠQuestion":18233,"Join":18234,"Ġfitted":18235,"Ġunlawful":18236,"^^":18237,"erek":18238,"Ġauthentication":18239,"Ġstole":18240,"Ġaccountability":18241,"label":18242,"Search":18243,"Ġalbeit":18244,"atican":18245,"funded":18246,"ĠAdding":18247,"ĠIQ":18248,"Ġsubmar":18249,"lit":18250,"aque":18251,"ĠLearning":18252,"Ġinteger":18253,"Master":18254,"ĠChrom":18255,"Ġpremier":18256,"Op":18257,"ĠLiu":18258,"Ġblessed":18259,"ĠGlobe":18260,"ĠResponse":18261,"Ġlegitim":18262,"ĠMerkel":18263,"Ġdisposal":18264,"´":18265,"Ġgauge":18266,"peat":18267,"Ġinduced":18268,"Ġquestionable":18269,"arthy":18270,"ĠVit":18271,"ĠFeed":18272,"Until":18273,"Ut":18274,"worthy":18275,"RY":18276,"ĠHerald":18277,"ĠHammer":18278,"Ġmedal":18279,"ĠRivers":18280,"ĠHack":18281,"Ġclarify":18282,"Ġtracked":18283,"Ġautonomous":18284,"Ġtenant":18285,"ĠQatar":18286,"erie":18287,"Ġgrim":18288,"ĠMonitor":18289,"Ġresistant":18290,"ĠSpec":18291,"ĠWells":18292,"NAS":18293,"148":18294,"Ġminers":18295,"iotics":18296,"Ġmisses":18297,"116":18298,"gian":18299,"git":18300,"ĠEyes":18301,"pres":18302,"Ġgraduated":18303,"Ġangel":18304,"Ġsynchron":18305,"Ġefficiently":18306,"Ġtransmitted":18307,"Harry":18308,"Ġglobally":18309,"ENCE":18310,"ĠMontana":18311,"raged":18312,"ĠPrevention":18313,"Ġpiss":18314,"ĠLl":18315,"Ġshelf":18316,"ĠBJP":18317,"ĠTestament":18318,"ĠLate":18319,"iker":18320,"ĠHapp":18321,"ĠJulian":18322,"hall":18323,"Ġspont":18324,"Ġshutdown":18325,"Ġinconsistent":18326,"Ġsubscribers":18327,"Ġskeleton":18328,"ĠNebraska":18329,"Ġinspire":18330,"ĠVoid":18331,"Feed":18332,"Ġangles":18333,"ĠSprings":18334,"Ġbenchmark":18335,"Ġvaccines":18336,"izophren":18337,"sexual":18338,"uffed":18339,"Ġshine":18340,"ĠKath":18341,"Ġgesture":18342,"inea":18343,"Ġrip":18344,"Ġoppression":18345,"Ġconscience":18346,"bt":18347,"ĠLum":18348,"Ġincidence":18349,"ĠFa":18350,"wr":18351,"Ġmineral":18352,"ĠSpurs":18353,"alky":18354,"Ġthunder":18355,"Ġopio":18356,"Being":18357,"ĠPalm":18358,"Ġwasted":18359,"Ġlb":18360,"iaries":18361,"ĠInitiative":18362,"Ġcurric":18363,"Ġmarker":18364,"ĠMcL":18365,"Ġextensions":18366,"ĠPv":18367,"ĠArms":18368,"Ġofferings":18369,"Ġdefenses":18370,"Ġvendor":18371,"Ġcontradict":18372,"ĠColin":18373,"Ġreddit":18374,"Ġperipher":18375,"122":18376,"Ġsins":18377,"Edit":18378,"ICT":18379,"Soft":18380,"ĠShah":18381,"Ġadministrator":18382,"ĠTrip":18383,"Ġpornography":18384,"Ġtuition":18385,"inence":18386,"ĠProgress":18387,"Ġcatalog":18388,"Ġsuite":18389,"Ġhike":18390,"Ġreproductive":18391,"engine":18392,"Ġdrought":18393,"ĠNoah":18394,"Ġ230":18395,"Ġdude":18396,"Ġrelaxed":18397,"Ġpartition":18398,"Ġparticipant":18399,"Ġtelesc":18400,"Ġfeas":18401,"ĠFF":18402,"owner":18403,"Ġsweeping":18404,"Ġlenses":18405,"Ġmatchup":18406,"ĠRepl":18407,"ournals":18408,"Ġcredible":18409,"Ġgrandmother":18410,"Ġthermal":18411,"Ġsubscribing":18412,"Ġidentities":18413,"colm":18414,"UCT":18415,"Ġreluctant":18416,"users":18417,"ĠCort":18418,"Ġassisted":18419,"OSS":18420,"ATIONS":18421,"ISH":18422,"Ġpharmaceutical":18423,"icable":18424,"adian":18425,"ĠSonic":18426,"ĠFury":18427,"ĠMong":18428,"AH":18429,"ĠPsychology":18430,"Ġphosph":18431,"Ġtreats":18432,"ŃĶ":18433,"Ġsteadily":18434,"ĠHello":18435,"Ġrelates":18436,"Ġclue":18437,"Expl":18438,"auth":18439,"Ġrevision":18440,"Ġeld":18441,"osion":18442,"Ġbron":18443,"144":18444,"rikes":18445,"Ġmines":18446,"Ġblanket":18447,"ĠFail":18448,"eled":18449,"ĠImagine":18450,"ĠPlanned":18451,"aic":18452,"Request":18453,"Mad":18454,"ĠHorse":18455,"ĠEagle":18456,"Ġcapac":18457,"157":18458,"Ġling":18459,"ĠNice":18460,"ĠParenthood":18461,"minster":18462,"ogs":18463,"ensitive":18464,"Nothing":18465,"Ġcarn":18466,"Fin":18467,"ĠPE":18468,"Ġrifles":18469,"ĠLP":18470,"Sand":18471,"ĠguiActive":18472,"Ġtourist":18473,"CNN":18474,"Ġunveiled":18475,"Ġpredecessor":18476,"}{":18477,"uber":18478,"Ġoffshore":18479,"Ġoptical":18480,"ĠRot":18481,"ĠPearl":18482,"eton":18483,"Ġstared":18484,"Ġfarther":18485,"atility":18486,"contin":18487,"ĠGy":18488,"ĠFoster":18489,"ĠCoc":18490,"rients":18491,"Ġdesigning":18492,"ĠEconomy":18493,"ONG":18494,"Women":18495,"ĠNancy":18496,"erver":18497,"Ġmascul":18498,"Ġcasualties":18499,"Ġ225":18500,"ĠSullivan":18501,"ĠChoice":18502,"Ġaster":18503,"ws":18504,"Ġhotels":18505,"Ġconsiderations":18506,"Ġcouch":18507,"ĠStrip":18508,"ĠGn":18509,"Ġmanipulate":18510,"lied":18511,"Ġsynthetic":18512,"Ġassaulted":18513,"Ġoffenses":18514,"ĠDrake":18515,"Ġimpe":18516,"October":18517,"ĠHeritage":18518,"hl":18519,"ĠBlair":18520,"Unlike":18521,"Ġgrief":18522,"Ġ450":18523,"Ġopted":18524,"Ġresignation":18525,"ilo":18526,"Ġverse":18527,"ĠTomb":18528,"Ġupt":18529,"Ġaired":18530,"ĠHook":18531,"ĠMLB":18532,"Ġassumes":18533,"outed":18534,"ĠVers":18535,"Ġinferior":18536,"Ġbundle":18537,"ĠDNS":18538,"ographer":18539,"Ġmultip":18540,"ĠSouls":18541,"Ġillustrated":18542,"Ġtactic":18543,"Ġdressing":18544,"Ġduo":18545,"Conf":18546,"Ġrelent":18547,"Ġcant":18548,"Ġscarce":18549,"Ġcandy":18550,"ĠCF":18551,"Ġaffiliated":18552,"Ġsprint":18553,"ylan":18554,"ĠGarcia":18555,"Ġjunk":18556,"Print":18557,"exec":18558,"Crit":18559,"Ġportrait":18560,"iries":18561,"ĠOFF":18562,"Ġdisputes":18563,"WR":18564,"Love":18565,"ãģĦ":18566,"ĠReyn":18567,"Ġhipp":18568,"opath":18569,"Ġfloors":18570,"ĠFeel":18571,"Ġworries":18572,"Ġsettlements":18573,"ĠPos":18574,"Ġmosque":18575,"Ġfinals":18576,"Ġcrushed":18577,"ĠProbably":18578,"ĠBot":18579,"ĠMans":18580,"ĠPeriod":18581,"Ġsovereignty":18582,"Ġseller":18583,"Ġapost":18584,"Ġamateur":18585,"Ġdorm":18586,"Ġconsuming":18587,"Ġarmour":18588,"ĠRoose":18589,"Ġintensive":18590,"Ġeliminating":18591,"ĠSunni":18592,"ĠAleppo":18593,"jin":18594,"Ġadvise":18595,"pal":18596,"ĠHalo":18597,"Ġdescent":18598,"Ġsimpler":18599,"Ġbooth":18600,"STR":18601,"Later":18602,"ĠCave":18603,"===":18604,"Ġmol":18605,"Ġfist":18606,"Ġshotgun":18607,"supp":18608,"Ġrobbery":18609,"Effect":18610,"Ġobscure":18611,"ĠProfessional":18612,"Ġembassy":18613,"Ġmilitant":18614,"Ġincarcer":18615,"Ġgenerates":18616,"Ġlaunches":18617,"Ġadministrators":18618,"Ġshaft":18619,"Ġcircular":18620,"Ġfreshman":18621,"ĠWes":18622,"ĠJoel":18623,"ĠDrew":18624,"ĠDuncan":18625,"ĠApparently":18626,"sight":18627,"ĠInternal":18628,"ĠIndividual":18629,"ĠFE":18630,"Ġbore":18631,"ĠMt":18632,"Ġbroadly":18633,"ĠOptions":18634,"ountain":18635,"ipes":18636,"ĠVideos":18637,"204":18638,"Ġhills":18639,"Ġsimulation":18640,"Ġdisappointment":18641,"itan":18642,"ĠLaboratory":18643,"Ġupward":18644,"Ġboundary":18645,"Ġdarker":18646,"hart":18647,"Ġdominance":18648,"Cong":18649,"ĠOracle":18650,"ĠLords":18651,"Ġscholarship":18652,"ĠVincent":18653,"ede":18654,"ĠRah":18655,"Ġencourages":18656,"rov":18657,"Ġquo":18658,"Ġpremise":18659,"ĠCrisis":18660,"ĠHolocaust":18661,"Ġrhythm":18662,"Ġmetric":18663,"club":18664,"Ġtransported":18665,"Ġnod":18666,"ĠPist":18667,"Ġancestors":18668,"ĠFreder":18669,"thumbnails":18670,"ĠCE":18671,"OND":18672,"Phil":18673,"venge":18674,"ĠProducts":18675,"castle":18676,"Ġqualifying":18677,"ĠKaren":18678,"VERTISEMENT":18679,"Ġmighty":18680,"Ġexplanations":18681,"Ġfixing":18682,"Di":18683,"Ġdeclaring":18684,"Ġanonymity":18685,"Ġjuven":18686,"ĠNord":18687,"ĠDoom":18688,"ĠActually":18689,"Ok":18690,"phis":18691,"ĠDesert":18692,"Ġ116":18693,"IK":18694,"ĠFM":18695,"Ġincomes":18696,"VEL":18697,"okers":18698,"Ġpecul":18699,"Ġlightweight":18700,"gue":18701,"Ġaccent":18702,"Ġincrement":18703,"ĠChan":18704,"Ġcomplaining":18705,"ĠBaghd":18706,"Ġmidfielder":18707,"Ġoverhaul":18708,"Process":18709,"ĠHollow":18710,"ĠTitans":18711,"Small":18712,"manuel":18713,"ĠUnity":18714,"ĠEvents":18715,"Sty":18716,"Ġdisproportion":18717,"nesty":18718,"enes":18719,"ĠCod":18720,"Ġdemonstrations":18721,"ĠCrimson":18722,"ĠOH":18723,"Ġenrolled":18724,"Ġcel":18725,"ĠBrett":18726,"Ġaide":18727,"Ġheels":18728,"Ġbroadband":18729,"Ġmarking":18730,"Ġwizard":18731,"ĠNJ":18732,"ĠChiefs":18733,"Ġingredient":18734,"Ġdug":18735,"ĠShut":18736,"urchase":18737,"endor":18738,"Ġfarmer":18739,"ĠGoldman":18740,"129":18741,"155":18742,"Order":18743,"Ġlion":18744,"iably":18745,"Ġstain":18746,"array":18747,"ilitary":18748,"ĠFAQ":18749,"Ġexploded":18750,"ĠMcCarthy":18751,"ĠTweet":18752,"ĠGreens":18753,"eking":18754,"ln":18755,"ensen":18756,"Ġmotorcycle":18757,"Ġparticle":18758,"Ġcholesterol":18759,"Bron":18760,"Ġstair":18761,"Ġoxid":18762,"Ġdesirable":18763,"ibles":18764,"Ġtheor":18765,"forcing":18766,"Ġpromotional":18767,"ovo":18768,"boot":18769,"ĠBonus":18770,"rawling":18771,"Ġshortage":18772,"ĠPsy":18773,"Ġrecruited":18774,"Ġinfants":18775,"Ġtestosterone":18776,"Ġdeduct":18777,"Ġdistinctive":18778,"Ġfirmware":18779,"built":18780,"145":18781,"Ġexplored":18782,"Ġfactions":18783,"Ġvide":18784,"Ġtattoo":18785,"Ġfinancially":18786,"Ġfatigue":18787,"Ġproceeding":18788,"constitutional":18789,"Ġmiser":18790,"Ġchairs":18791,"gging":18792,"ipple":18793,"Ġdent":18794,"Ġdisreg":18795,"çĶ":18796,"stant":18797,"llo":18798,"bps":18799,"akening":18800,"Ġabnormal":18801,"ĠERA":18802,"士":18803,"ĠHBO":18804,"ĠMAR":18805,"Ġconcess":18806,"Ġservant":18807,"Ġaspir":18808,"lav":18809,"ĠPanel":18810,"amo":18811,"Ġprecip":18812,"Ġrecordings":18813,"Ġproceeded":18814,"Ġcolony":18815,"ĠTang":18816,"ablo":18817,"Ġstripped":18818,"Left":18819,"too":18820,"Ġpotatoes":18821,"Ġfinest":18822,"%).":18823,"Ġcrap":18824,"ĠZach":18825,"abases":18826,"ĠGoth":18827,"Ġbillionaire":18828,"wolf":18829,"Ġsanction":18830,"SK":18831,"Ġlogged":18832,"Po":18833,"eyed":18834,"unal":18835,"Ġcricket":18836,"Ġarmies":18837,"Ġuncovered":18838,"Cloud":18839,"ón":18840,"Ġrebounds":18841,"Ġmes":18842,"Oper":18843,"Pac":18844,"Ġnationally":18845,"Ġinserted":18846,"pict":18847,"Ġgovernance":18848,"и":18849,"Ġprivileges":18850,"GET":18851,"Ġfavorites":18852,"imity":18853,"Ġlover":18854,"them":18855,"empl":18856,"Ġgorgeous":18857,"Ann":18858,"Ġslipped":18859,"Ġveto":18860,"Bob":18861,"Ġslim":18862,"ucc":18863,"ĠFame":18864,"uddenly":18865,"Ġdenies":18866,"ĠMaur":18867,"Ġdistances":18868,"Ġwanna":18869,"tar":18870,"ĠSER":18871,"ĠâĪ":18872,"Ġlemon":18873,"athetic":18874,"Ġliteral":18875,"Ġdistinguished":18876,"Ġanswering":18877,"GI":18878,"Ġreligions":18879,"ĠPhilos":18880,"ĠLay":18881,"Ġcompos":18882,"irements":18883,"ĠKos":18884,"inez":18885,"rolling":18886,"Ġyoungest":18887,"andise":18888,"ĠBorn":18889,"Ġaltar":18890,"amina":18891,"ĠBoot":18892,"voc":18893,"Ġdigging":18894,"Ġpressures":18895,"Ġlen":18896,"264":18897,"Ġassassination":18898,"ĠBirmingham":18899,"ĠMyth":18900,"Ġsovereign":18901,"ĠArtist":18902,"ĠPhotograph":18903,"Ġdepicted":18904,"Ġdispens":18905,"orthy":18906,"Ġambul":18907,"integ":18908,"ĠCele":18909,"ĠTibet":18910,"Ġhierarchy":18911,"Ġcu":18912,"Ġpreseason":18913,"ĠPeterson":18914,"Ġcolours":18915,"Ġworrying":18916,"Ġbackers":18917,"ĠPalmer":18918,"Ġμ":18919,"Ġcontributor":18920,"Ġhearings":18921,"Ġurine":18922,"ĠÙ":18923,"ourgeois":18924,"Similar":18925,"ĠZimmer":18926,"something":18927,"ĠUSC":18928,"Ġstrengths":18929,"ĠFI":18930,"Ġlogging":18931,"Asked":18932,"ĠThai":18933,"inqu":18934,"ĠWalt":18935,"Ġcrews":18936,"itism":18937,"301":18938,"Ġsharply":18939,"umed":18940,"Ġredirect":18941,"rators":18942,"Inf":18943,"ĠWeapons":18944,"Ġteasp":18945,"1999":18946,"Live":18947,"ĠEspecially":18948,"ĠSter":18949,"ĠVeterans":18950,"Ġintro":18951,"otherapy":18952,"Ġmalware":18953,"Ġbreeding":18954,"Ġmolecular":18955,"ĠRoute":18956,"ĠComment":18957,"ochem":18958,"Ġain":18959,"Season":18960,"Ġlinebacker":18961,"Ä«":18962,"ĠEconomics":18963,"esar":18964,"ĠLives":18965,"ĠEmma":18966,"Ġkin":18967,"ĠTerrit":18968,"Ġplanted":18969,"oton":18970,"ĠButter":18971,"ĠSpons":18972,"PER":18973,"Ġdungeon":18974,"Ġsymbolic":18975,"Ġfilmed":18976,"Ġdiets":18977,"Ġconcludes":18978,"Ġcertainty":18979,"ĠFormat":18980,"Ġstrangers":18981,"format":18982,"ĠPhase":18983,"Ġcopied":18984,"Ġmetres":18985,"lda":18986,"ĠUsers":18987,"Ġdeliberate":18988,"Ġwashed":18989,"ĠLance":18990,"imation":18991,"Ġimproper":18992,"ĠGenesis":18993,"ickr":18994,"ĠKush":18995,"Ġrealise":18996,"Ġembarrassing":18997,"alking":18998,"bucks":18999,"Ġverified":19000,"Ġoutline":19001,"years":19002,"ĠIncome":19003,"202":19004,"Ġzombies":19005,"Final":19006,"ĠMillenn":19007,"Ġmodifications":19008,"ĠVision":19009,"ĠMoses":19010,"verb":19011,"iterranean":19012,"ĠJet":19013,"Ġnaval":19014,"ĠAgg":19015,"Ġurl":19016,"Ġvictories":19017,"Ġnonetheless":19018,"Ġinjust":19019,"ĠFact":19020,"çļ":19021,"Ġinsufficient":19022,"review":19023,"facebook":19024,"Ġnegotiating":19025,"Ġguarantees":19026,"imen":19027,"utenberg":19028,"Ġgambling":19029,"Ġcongr":19030,"Loading":19031,"Ġnevertheless":19032,"Ġpresidents":19033,"ĠIndustrial":19034,"Ġ118":19035,"Ġpoured":19036,"ĠTory":19037,"Ġ175":19038,"Ġ:=":19039,"Scott":19040,"angered":19041,"Tok":19042,"Ġorganizers":19043,"Mat":19044,"ĠGrowth":19045,"Ġadul":19046,"Ġensures":19047,"Ġ117":19048,"é¾įå":19049,"Ġmassacre":19050,"Ġgrades":19051,"before":19052,"ADVERTISEMENT":19053,"ĠSlow":19054,"ĠMMA":19055,"âĢĶ\"":19056,"ĠVatican":19057,"Qaeda":19058,"Ġowe":19059,"6666":19060,"ĠSorry":19061,"ĠGrass":19062,"Ġbackgrounds":19063,"Ġexhausted":19064,"Ġclan":19065,"Ġcompromised":19066,"ĠElf":19067,"ĠIsaac":19068,"enson":19069,"Invest":19070,"IFA":19071,"Ġinterrupted":19072,"ãĥīãĥ©":19073,"Ġtwisted":19074,"ĠDragons":19075,"Mode":19076,"ĠKremlin":19077,"Ġfertil":19078,"heres":19079,"phan":19080,"ĠNode":19081,"fed":19082,"ĠOrc":19083,"Ġunwilling":19084,"Cent":19085,"Ġpriorit":19086,"Ġgraduates":19087,"Ġsubjective":19088,"Ġissuing":19089,"ĠLt":19090,"Ġviewer":19091,"Ġwoke":19092,"Thus":19093,"brook":19094,"Ġdepressed":19095,"Ġbracket":19096,"ĠGor":19097,"ĠFighting":19098,"Ġstriker":19099,"Report":19100,"ĠPortugal":19101,"Ġneo":19102,"wed":19103,"199":19104,"Ġfleeing":19105,"shadow":19106,"identified":19107,"USE":19108,"Steam":19109,"Ġstretched":19110,"Ġrevelations":19111,"arted":19112,"ĠDw":19113,"Ġalignment":19114,"eston":19115,"ĠJared":19116,"Sep":19117,"Ġblogs":19118,"update":19119,"gom":19120,"risk":19121,"Ġclash":19122,"ĠHour":19123,"Ġruntime":19124,"Ġunwanted":19125,"Ġscam":19126,"Ġrack":19127,"Ġenlight":19128,"onest":19129,"ĠFerr":19130,"Ġconvictions":19131,"Ġpiano":19132,"Ġcirculation":19133,"ĠWelcome":19134,"Ġbacklash":19135,"ĠWade":19136,"Ġreceivers":19137,"otive":19138,"Jeff":19139,"Ġnetworking":19140,"ĠPrep":19141,"ĠExplorer":19142,"Ġlecture":19143,"Ġuploaded":19144,"ĠMeat":19145,"BLE":19146,"ĠNazis":19147,"ĠSynd":19148,"stud":19149,"roots":19150,"rians":19151,"Ġportrayed":19152,"Ġ??":19153,"ĠBuddha":19154,"sun":19155,"Robert":19156,"ĠComplex":19157,"Ġoversee":19158,"Ġstealth":19159,"Title":19160,"ĠJobs":19161,"ĠKum":19162,"Ġappreciation":19163,"ĠMOD":19164,"Ġbasics":19165,"Ġclips":19166,"Ġnursing":19167,"Ġproposition":19168,"Ġrealised":19169,"ĠNYC":19170,"Ġallocated":19171,"rium":19172,"aran":19173,"ĠProduction":19174,"ĠVote":19175,"Ġsmugg":19176,"Ġhunter":19177,"azer":19178,"ĠChanges":19179,"Ġfluct":19180,"yon":19181,"Array":19182,"Ġkits":19183,"Water":19184,"Ġuncommon":19185,"Ġresting":19186,"ells":19187,"would":19188,"Ġpursued":19189,"Ġassertion":19190,"ometown":19191,"ĠMosul":19192,"ĠPlatform":19193,"iolet":19194,"Ġshareholders":19195,"Ġtrails":19196,"Pay":19197,"ĠEnforcement":19198,"types":19199,"ĠAnonymous":19200,"Ġsatisfying":19201,"ilogy":19202,"Ġ('":19203,"wave":19204,"city":19205,"Steve":19206,"Ġconfrontation":19207,"ĠEld":19208,"Capt":19209,"ahan":19210,"htm":19211,"ĠCtrl":19212,"ONS":19213,"230":19214,"ifa":19215,"holding":19216,"Ġdelicate":19217,"Ġjaw":19218,"ĠGoing":19219,"orum":19220,"Sal":19221,"Ġdull":19222,"ĠBeth":19223,"Ġprisons":19224,"Ġego":19225,"ĠElsa":19226,"avorite":19227,"ĠGang":19228,"ĠNuclear":19229,"Ġspider":19230,"atsu":19231,"Ġsampling":19232,"Ġabsorbed":19233,"ĠPharm":19234,"ieth":19235,"Ġbucket":19236,"ĠRecomm":19237,"OF":19238,"ĠFactory":19239,"ANCE":19240,"Ġbacter":19241,"Has":19242,"ĠObserv":19243,"121":19244,"Ġpremiere":19245,"Develop":19246,"Ġcurrencies":19247,"Cast":19248,"Ġaccompanying":19249,"ĠNashville":19250,"Ġfatty":19251,"ĠBrend":19252,"Ġlocks":19253,"Ġcentered":19254,"ĠUT":19255,"aughs":19256,"orie":19257,"ĠAffordable":19258,"vance":19259,"DL":19260,"emet":19261,"Ġthrone":19262,"ĠBluetooth":19263,"Ġnaming":19264,"ifts":19265,"ADE":19266,"Ġcorrected":19267,"Ġpromptly":19268,"ĠSTR":19269,"Ġgenome":19270,"Ġcope":19271,"Ġvalley":19272,"Ġrounded":19273,"ĠKend":19274,"alion":19275,"pers":19276,"Ġtourism":19277,"Ġstark":19278,"vl":19279,"Ġblowing":19280,"ĠSchedule":19281,"std":19282,"Ġunhappy":19283,"Ġlitigation":19284,"cedes":19285,"Ġandroid":19286,"Ġintegral":19287,"erers":19288,"uded":19289,"tax":19290,"Ġreiter":19291,"ĠMotors":19292,"ociated":19293,"Ġwonders":19294,"ĠApost":19295,"ucking":19296,"ĠRoosevelt":19297,"fram":19298,"Ġyields":19299,"Ġconstitutes":19300,"awk":19301,"Interest":19302,"Ġinterim":19303,"Ġbreakthrough":19304,"ĠCher":19305,"Ġprosec":19306,"ĠDj":19307,"ĠMT":19308,"Resp":19309,"ĠPT":19310,"Ġsperm":19311,"edit":19312,"BT":19313,"Linux":19314,"country":19315,"league":19316,"Ġdick":19317,"Ġoct":19318,"Ġinserting":19319,"Ġscra":19320,"ĠBrewing":19321,"Ġ1966":19322,"Ġrunners":19323,"Ġplun":19324,"idy":19325,"ĠDian":19326,"Ġdysfunction":19327,"Ġexclusion":19328,"Ġdisgr":19329,"Ġincorporate":19330,"Ġreconc":19331,"Ġnominated":19332,"ĠArcher":19333,"draw":19334,"achelor":19335,"Ġwritings":19336,"Ġshallow":19337,"Ġhast":19338,"ĠBMW":19339,"ĠRS":19340,"Ġthigh":19341,"Ġ1963":19342,"Ġlamb":19343,"Ġfavored":19344,"agle":19345,"Ġcooler":19346,"ĠHours":19347,"ĠGU":19348,"ĠOrigin":19349,"Ġglimpse":19350,"--------------------":19351,"Lim":19352,"Ġcheek":19353,"Ġjealous":19354,"-'":19355,"Ġharness":19356,"ĠPoison":19357,"Ġdisabilities":19358,"neapolis":19359,"Ġoutlook":19360,"Ġnotify":19361,"ĠIndianapolis":19362,"Ġabrupt":19363,"nsic":19364,"Ġencrypted":19365,"Ġforfe":19366,"reath":19367,"Ġrabb":19368,"Ġfoundations":19369,"Ġcompliment":19370,"ĠInterview":19371,"ĠSwe":19372,"Ġadolesc":19373,"Ġmonitors":19374,"ĠSacramento":19375,"Ġtimely":19376,"Ġcontempl":19377,"Ġpositioned":19378,"Ġposters":19379,"phies":19380,"iovascular":19381,"void":19382,"ĠFifth":19383,"Ġinvestigative":19384,"OUN":19385,"Ġintegrate":19386,"ĠINC":19387,"isha":19388,"iblings":19389,"ĠRequest":19390,"ĠRodriguez":19391,"Ġslides":19392,"ĠDX":19393,"Ġfeminism":19394,"Ġdatas":19395,"Ġbend":19396,"irus":19397,"ĠNigeria":19398,"Fox":19399,"Change":19400,"Ġairplane":19401,"ĠLaden":19402,"Ġpublicity":19403,"ixty":19404,"Ġcommitments":19405,"Ġaggregate":19406,"Ġdisplaying":19407,"ĠArrow":19408,"Ġ122":19409,"Ġrespects":19410,"android":19411,"six":19412,"ĠSha":19413,"Ġrestoration":19414,")\\":19415,"WS":19416,"oys":19417,"Ġillustrate":19418,"without":19419,"126":19420,"ĠâĶĤ":19421,"Ġpickup":19422,"nels":19423,"Ġ....":19424,"food":19425,"ĠFen":19426,")?":19427,"Ġphenomena":19428,"Ġcompanions":19429,"ĠWrite":19430,"Ġspill":19431,"Ġbridges":19432,"ĠUpdated":19433,"ĠFo":19434,"Ġinsects":19435,"ASHINGTON":19436,"Ġscare":19437,"iltr":19438,"ĠZhang":19439,"Ġseverity":19440,"Ġindul":19441,"149":19442,"ĠCoffee":19443,"Ġnorms":19444,"Ġpulse":19445,"ĠFT":19446,"Ġhorrific":19447,"ĠDestroy":19448,"ĠJSON":19449,"Ġolive":19450,"Ġdiscusses":19451,"Rest":19452,"Elect":19453,"ĠWinn":19454,"ĠSurviv":19455,"ĠHait":19456,"Sure":19457,"oped":19458,"Ġrooted":19459,"ĠSke":19460,"ĠBronze":19461,"Ġlol":19462,"Default":19463,"Ġcommodity":19464,"redited":19465,"Ġlibertarian":19466,"Ġforbidden":19467,"Ġgran":19468,"à¨":19469,"Ġlag":19470,"enz":19471,"drive":19472,"Ġmathematics":19473,"Ġwires":19474,"Ġcritically":19475,"Ġcarbohyd":19476,"ĠChancellor":19477,"ĠEddie":19478,"Ġbanning":19479,"ĠFri":19480,"Ġcomplications":19481,"etric":19482,"ĠBangladesh":19483,"Ġbandwidth":19484,"Stop":19485,"ĠOriginally":19486,"Ġhalfway":19487,"ynasty":19488,"shine":19489,"Ġtales":19490,"rities":19491,"avier":19492,"Ġspinning":19493,"ĠWHO":19494,"Ġneighbourhood":19495,"bach":19496,"Ġcommerce":19497,"ĠSle":19498,"BU":19499,"Ġentrepreneur":19500,"Ġpeculiar":19501,"ĠComments":19502,"fre":19503,"320":19504,"ICS":19505,"Ġimagery":19506,"ĠCanon":19507,"ĠElectronic":19508,"short":19509,"((":19510,"Dig":19511,"Ġcommem":19512,"uced":19513,"Ġinclined":19514,"ĠSummon":19515,"Ġcliff":19516,"ĠMediterranean":19517,"Ġpoetry":19518,"Ġprosperity":19519,"ĠRece":19520,"Ġpills":19521,"member":19522,"Ġfinale":19523,"unc":19524,"ĠGig":19525,"ä½":19526,"Ġlod":19527,"Ġbackward":19528,"-+":19529,"ĠForward":19530,"Ġthri":19531,"sure":19532,"Ġsoap":19533,"ĠFX":19534,"RES":19535,"ĠSexual":19536,"oulos":19537,"Ġfoolish":19538,"Ġrighteous":19539,"Ġcoff":19540,"terrorism":19541,"ustain":19542,"oter":19543,"Ġabuses":19544,"next":19545,"Ġabusive":19546,"Ġthereafter":19547,"Ġprohibition":19548,"ĠSUP":19549,"Ġdip":19550,"Ġripped":19551,"Ġinherited":19552,"Ġbats":19553,"stru":19554,"GT":19555,"Ġflawed":19556,"phabet":19557,"Ġfog":19558,"doors":19559,"Ġimaging":19560,"Ġdigits":19561,"ĠHungary":19562,"Ġarrog":19563,"Ġteachings":19564,"Ġprotocols":19565,"ĠBanks":19566,"à¸":19567,"pound":19568,"ĠCurt":19569,".\")":19570,"./":19571,"Ġexemption":19572,"endix":19573,"ĠMull":19574,"Ġimproves":19575,"ĠGamer":19576,"dimensional":19577,"Icon":19578,"ĠMargaret":19579,"Status":19580,"dates":19581,"Ġintends":19582,"Ġdepict":19583,"Ġparked":19584,"Joe":19585,"ĠMarines":19586,"chnology":19587,"!).":19588,"Ġjudged":19589,"Ġweights":19590,"Ray":19591,"Ġapartments":19592,"hester":19593,"Ġreinforce":19594,"Ġoffender":19595,"occup":19596,"Ġsore":19597,"ept":19598,"ĠPHP":19599,"ĠBrow":19600,"Ġauthorization":19601,"ĠRisk":19602,"ĠDelaware":19603,"ĠQU":19604,"Ġnotifications":19605,"Ġsunlight":19606,"Ġexclude":19607,"dat":19608,"Ġmesh":19609,"ĠSudan":19610,"Ġbelonged":19611,"Ġsubway":19612,"Ġnoon":19613,"ĠInterior":19614,"olics":19615,"ĠLakers":19616,"Ġcoding":19617,"Disclaimer":19618,"Calif":19619,"Old":19620,"Ġdisl":19621,"?????":19622,"Ġconfirms":19623,"Ġrecruitment":19624,"Ġhomicide":19625,"Consider":19626,"ĠJeffrey":19627,"fty":19628,"};":19629,"Ġobjection":19630,"doing":19631,"ĠLeo":19632,"Want":19633,"Ġglow":19634,"ĠClarke":19635,"ĠNorman":19636,"Ġverification":19637,"Ġpacket":19638,"ĠFormula":19639,"Ġplag":19640,"esville":19641,"Ġshouting":19642,"Ġov":19643,"ĠREC":19644,"ĠBub":19645,"Ġninth":19646,"Ġenerg":19647,"Ġvalidity":19648,"Ġups":19649,"jack":19650,"Ġneighboring":19651,"ĠNec":19652,"eworks":19653,"ĠHab":19654,"arez":19655,"Ġspine":19656,"Ġeventual":19657,"ĠLeaders":19658,"ĠCarn":19659,"Ġprobation":19660,"Ġromance":19661,"msg":19662,"ĠMechanical":19663,"ERY":19664,"Rock":19665,"Ġpartisan":19666,"Node":19667,"assets":19668,"minent":19669,"Ġforeigners":19670,"Ġtestify":19671,"ĠUsually":19672,"lords":19673,"ĠGren":19674,"ĠPowell":19675,"BIL":19676,"Ġsr":19677,"Ġaddict":19678,"Ġshells":19679,"Ġsigh":19680,"ĠYale":19681,"ternity":19682,"Ġ750":19683,"EU":19684,"ĠRifle":19685,"Ġpatron":19686,"ema":19687,"ĠBannon":19688,"anity":19689,"Ġtropical":19690,"ĠVII":19691,"cross":19692,"Everything":19693,"ĠISO":19694,"Ġhumble":19695,"assing":19696,"ĠFIG":19697,"Ġupdating":19698,"yson":19699,"Ġcalcium":19700,"Ġcompetent":19701,"Ġsteering":19702,"Prot":19703,"ĠSY":19704,"ĠFinals":19705,"ĠRug":19706,"159":19707,"137":19708,"ĠGolf":19709,"Ġ126":19710,"Ġaccommodation":19711,"ĠHughes":19712,"Ġaesthetic":19713,"artisan":19714,"ĠTwilight":19715,"Ġprince":19716,"ĠAgriculture":19717,"ĠDisco":19718,"Ġprecedent":19719,"Ġtyping":19720,"authorized":19721,"Option":19722,"ĠAub":19723,"lishes":19724,"acht":19725,"mag":19726,"Peter":19727,"ĠUFO":19728,"monton":19729,"ĠLith":19730,"Ġarom":19731,"Ġsecuring":19732,"Ġconfined":19733,"private":19734,"Ġswords":19735,"Ġmarkers":19736,"Ġmetabolic":19737,"select":19738,"ĠCurse":19739,"ĠOt":19740,"gressive":19741,"Ġincumb":19742,"ĠSaga":19743,"Ġpriced":19744,"Ġclearance":19745,"Content":19746,"Ġdrilling":19747,"Ġnotices":19748,"Ġbourgeois":19749,"Ġvest":19750,"Ġcookie":19751,"ĠGuardians":19752,"rys":19753,"inyl":19754,"Ġ124":19755,"Ġplausible":19756,"ongh":19757,"ĠOdin":19758,"Ġconception":19759,"ĠYuk":19760,"ĠBaghdad":19761,"ĠFlag":19762,"Austral":19763,"ĠIBM":19764,"Ġinternationally":19765,"ĠWikiLeaks":19766,"IED":19767,"Ġcyn":19768,"Ġchooses":19769,"ĠPill":19770,"Ġcombining":19771,"Ġradi":19772,"ĠMohammed":19773,"defense":19774,"atching":19775,"Subject":19776,"iciency":19777,"Frame":19778,"Ġ{\"":19779,"Ġchess":19780,"Ġtimer":19781,"190":19782,"Ġtin":19783,"Ġordinance":19784,"emetery":19785,"Ġaccusing":19786,"Ġnoticeable":19787,"Ġcentres":19788,"Ġlid":19789,"ĠMills":19790,"imgur":19791,"Ġzoom":19792,"ergic":19793,"Ġcompression":19794,"prim":19795,"find":19796,"Ġsurg":19797,"Ġpand":19798,"ĠKee":19799,"ĠChad":19800,"cellence":19801,"oyle":19802,"Ġsocialism":19803,"ĠTravis":19804,"ĠMHz":19805,"Ġguild":19806,"ALLY":19807,"ĠSubscribe":19808,"ĠRelated":19809,"Ġoccurrence":19810,"itching":19811,"Ġfictional":19812,"Ġcrush":19813,"ĠEA":19814,"cod":19815,"mix":19816,"ĠTriple":19817,"Ġretrieve":19818,"Ġstimulus":19819,"Ġpsychiat":19820,"ĠDoor":19821,"Ġhomosexuality":19822,"Ġelementary":19823,"Ġcellular":19824,"idian":19825,"ĠLaun":19826,"Ġintriguing":19827,"Ġfoam":19828,"ĠBass":19829,"idi":19830,"itsu":19831,"Ġassure":19832,"Ġcongrat":19833,"Ġbusinessman":19834,"ĠBoost":19835,"close":19836,"Ġlied":19837,"Ġsciences":19838,"ĠOmega":19839,"ĠGraphics":19840,"Ġ<=":19841,"spoken":19842,"Ġconnectivity":19843,"Saturday":19844,"ĠAvengers":19845,"Ġtoggle":19846,"Ġankle":19847,"Ġnationalist":19848,"model":19849,"ĠPool":19850,"ophobia":19851,"Var":19852,"ĠMons":19853,"atories":19854,"Ġaggressively":19855,"Clear":19856,"Forge":19857,"acters":19858,"Ġhedge":19859,"Ġpipes":19860,"Ġblunt":19861,"Ġsq":19862,"Ġremotely":19863,"Wed":19864,"asers":19865,"Ġrefriger":19866,"Ġtiles":19867,"Ġrescued":19868,"Ġcomprised":19869,"insky":19870,"Ġmanif":19871,"avanaugh":19872,"Ġprolifer":19873,"Ġaligned":19874,"xml":19875,"Ġtriv":19876,"Ġcoordination":19877,"ĠPER":19878,"ĠQuote":19879,"134":19880,"bf":19881,"ĠSaw":19882,"Ġtermination":19883,"Ġ190":19884,"Ġadditions":19885,"Ġtrio":19886,"Ġprojections":19887,"Ġpositively":19888,"Ġinclusive":19889,"Ġmembr":19890,"1990":19891,"older":19892,"Ġpracticed":19893,"inkle":19894,"Arch":19895,"Ġstarters":19896,"arius":19897,"Ġintermediate":19898,"ĠBenef":19899,"ĠKiller":19900,"Ġinterventions":19901,"ĠKil":19902,"ĠFlying":19903,"Inv":19904,"Ġpremature":19905,"Ġpsychiatric":19906,"Ġindie":19907,"Ġcollar":19908,"ĠRainbow":19909,"afi":19910,"Ġdisruption":19911,"ĠFOX":19912,"casting":19913,"Ġmisdem":19914,"cro":19915,"Ġwipe":19916,"ardon":19917,"Ġbast":19918,"ĠTommy":19919,"ĠRepresentative":19920,"Ġbelly":19921,"ĠPO":19922,"ĠBreitbart":19923,"132":19924,"Ġmessaging":19925,"Should":19926,"References":19927,"ĠGRE":19928,"istical":19929,"LP":19930,"ĠCav":19931,"ĠCrazy":19932,"Ġintuitive":19933,"keeping":19934,"ĠMoss":19935,"Ġdiscontin":19936,"ĠModule":19937,"Ġunrelated":19938,"ĠPractice":19939,"ĠTransport":19940,"Ġstatistically":19941,"orns":19942,"Ġsized":19943,"pu":19944,"Ġcaf":19945,"ĠWorlds":19946,"ĠRodgers":19947,"ĠLun":19948,"ĠComic":19949,"living":19950,"Ġcared":19951,"Ġclimbed":19952,"){":19953,"Ġconsisted":19954,"Ġmedieval":19955,"folk":19956,"Ġhacked":19957,"Ġdire":19958,"ĠHermione":19959,"Ġtended":19960,"ceans":19961,"Daniel":19962,"went":19963,"Ġlegislators":19964,"Ġredes":19965,"games":19966,"Ġgn":19967,"amiliar":19968,"Ġ++":19969,"ggy":19970,"threat":19971,"Ġmagnet":19972,"Ġperceive":19973,"Ġzip":19974,"Ġindictment":19975,"Ġcritique":19976,"gard":19977,"ĠSafe":19978,"ĠCream":19979,"Ġadvent":19980,"oba":19981,"Ġvowed":19982,"ousands":19983,"Ġski":19984,"Ġabortions":19985,"uart":19986,"Ġstunned":19987,"Ġadvancing":19988,"Ġlacked":19989,"Ġ\\\"":19990,"Ġschizophren":19991,"Ġelegant":19992,"Ġconferences":19993,"Ġcanceled":19994,"ĠHudson":19995,"ĠHopefully":19996,"Ġtrump":19997,"Ġfrequencies":19998,"Ġmeteor":19999,"ĠJunior":20000,"ĠFleet":20001,"ĠMalcolm":20002,"ĠTools":20003,"Ġ........":20004,"Ġhobby":20005,"ĠEuropeans":20006,"Ġ1500":20007,"ĠInto":20008,"Ġsway":20009,"ĠAppro":20010,"ĠCompl":20011,"Community":20012,"Ġtide":20013,"ĠSummit":20014,"ä»":20015,"Ġintervals":20016,"ĠEther":20017,"Ġhabitat":20018,"ĠStevens":20019,"lishing":20020,"ĠDomain":20021,"Ġtriggers":20022,"Ġchasing":20023,"Ġcharm":20024,"ĠFlower":20025,"itored":20026,"Ġblessing":20027,"Ġtextures":20028,"Five":20029,"Ġliquor":20030,"RP":20031,"FIN":20032,"Ġ1962":20033,"CAR":20034,"Unknown":20035,"Ġresil":20036,"ĠLily":20037,"Ġabundance":20038,"Ġpredictable":20039,"rar":20040,"Ġbullshit":20041,"leen":20042,"chet":20043,"Mor":20044,"Much":20045,"ä¹":20046,"Ġemphasized":20047,"Ġcrust":20048,"Ġprimitive":20049,"Ġenjoyable":20050,"ĠPictures":20051,"Ġteammate":20052,"pler":20053,"ĠTol":20054,"ĠKane":20055,"Ġsummoned":20056,"thy":20057,"rama":20058,"ĠHonda":20059,"Ġrealizing":20060,"Ġquicker":20061,"Ġconcentrate":20062,"clear":20063,"Ġ210":20064,"ĠErdogan":20065,"aris":20066,"Ġresponds":20067,"ĠBI":20068,"Ġeligibility":20069,"Ġpushes":20070,"ĠIdaho":20071,"Ġaggrav":20072,"Ġruins":20073,"urations":20074,"Ġbans":20075,"Ġanat":20076,"share":20077,"Ġgrind":20078,"hin":20079,"umen":20080,"Ġutilities":20081,"ĠYankees":20082,"Ġdatabases":20083,"ĠDD":20084,"Ġdisplaced":20085,"Ġdependencies":20086,"Ġstimulation":20087,"hun":20088,"houses":20089,"ĠPretty":20090,"ĠRavens":20091,"ĠTODAY":20092,"Ġassociates":20093,"Ġtherape":20094,"cled":20095,"Ġdeer":20096,"Ġrepairs":20097,"rentice":20098,"Ġreceptors":20099,"Ġremed":20100,"ĠCe":20101,"Ġmarriages":20102,"Ġballots":20103,"ĠSoldier":20104,"Ġhilarious":20105,"opl":20106,"138":20107,"Ġinherently":20108,"Ġignorant":20109,"Ġbounce":20110,"ĠEaster":20111,"RELATED":20112,"ĠCurrency":20113,"EV":20114,"ãĥŀ":20115,"ĠLead":20116,"Ġdeceased":20117,"Brien":20118,"ĠMusk":20119,"JS":20120,"Ġmerge":20121,"hearted":20122,"creat":20123,"mitt":20124,"mund":20125,"ĠâĢĭ":20126,"ĠBag":20127,"Ġprojection":20128,"Ġjava":20129,"ĠStandards":20130,"ĠLeonard":20131,"Ġcoconut":20132,"ĠPopulation":20133,"Ġtraject":20134,"Ġimply":20135,"Ġcuriosity":20136,"ĠDB":20137,"ĠFresh":20138,"ĠPor":20139,"Ġheavier":20140,"neys":20141,"gomery":20142,"Ġdeserved":20143,"Ġphrases":20144,"ĠGC":20145,"Ġyeast":20146,"desc":20147,"Death":20148,"Ġreboot":20149,"Ġmetadata":20150,"ICAL":20151,"Ġrepay":20152,"ĠIndependence":20153,"Ġsuburban":20154,"icals":20155,"Ġatop":20156,"Ġallocation":20157,"generation":20158,"ĠGram":20159,"Ġmoisture":20160,"Ġpine":20161,"ĠLiberals":20162,"Ġaides":20163,"Ġunderest":20164,"ĠBerry":20165,"Ġceremon":20166,"370":20167,"astrous":20168,"ĠPirates":20169,"Ġtense":20170,"ĠIndustries":20171,"ĠAppeals":20172,"ĠNear":20173,"Ġè£ıç":20174,"Ġlovers":20175,"ĠCAP":20176,"ĠCraw":20177,"Ġgiants":20178,"Ġefficacy":20179,"Element":20180,"ĠBehavior":20181,"ĠToyota":20182,"Ġintest":20183,"Priv":20184,"AI":20185,"Ġmaneuver":20186,"Ġperfection":20187,"Ġbang":20188,"paper":20189,"rill":20190,"George":20191,"border":20192,"inters":20193,"ĠSeth":20194,"Ġclues":20195,"ĠLevi":20196,"ĠRevenue":20197,"147":20198,"Ġvapor":20199,"Ġfortunate":20200,"Ġthreatens":20201,"Ġvet":20202,"Ġdependency":20203,"ersed":20204,"article":20205,"ĠBlizzard":20206,"Ġchlor":20207,"Ġminus":20208,"ĠBills":20209,"Ġcryptocurrency":20210,"Ġmetabolism":20211,"tering":20212,"Ġpestic":20213,"steps":20214,"ĠTreasure":20215,"racted":20216,"ĠConstant":20217,"Ġtemp":20218,"139":20219,"ĠDetective":20220,"urally":20221,"Ġrecovering":20222,"Ġcortex":20223,"Ġ144":20224,"closed":20225,"Ġprejudice":20226,"aunted":20227,"Ġstorms":20228,"ĠNOW":20229,"Ġmachinery":20230,"Address":20231,"Ġcompelled":20232,"270":20233,"Ġdespair":20234,"bane":20235,"Ġvegetable":20236,"Ġbeds":20237,"Learn":20238,"Ġcolorful":20239,"Ġspike":20240,"Ġmargins":20241,"Ġsympathy":20242,"Ġworkshop":20243,"ĠCBC":20244,"Sat":20245,"Ġburns":20246,"ĠGender":20247,"Ġ129":20248,"ĠCable":20249,"Ġdebts":20250,"ĠTheresa":20251,"Ġreflecting":20252,"Ġairst":20253,"Ġrim":20254,"ramid":20255,"Ġweaknesses":20256,"Writ":20257,"oggle":20258,"ti":20259,"ĠCharge":20260,"Ġweighed":20261,"Ġ(.":20262,"Ġlaughter":20263,"Ġrouter":20264,"ĠDemocracy":20265,"Dear":20266,"Ġhasht":20267,"Ġdy":20268,"Ġhints":20269,"running":20270,"Ġfinishes":20271,"arus":20272,"Mass":20273,"result":20274,"ascus":20275,"Ġvintage":20276,"Ġconqu":20277,"Ġwildly":20278,"acist":20279,"Ġlingu":20280,"Ġprotagonist":20281,"strom":20282,"teenth":20283,"ĠSolo":20284,"mac":20285,"filled":20286,"Ġrenown":20287,"itives":20288,"Ġmotive":20289,"ĠAntar":20290,"ĠMann":20291,"ĠAdjust":20292,"Ġrockets":20293,"Ġtroubling":20294,"ei":20295,"Ġorganisms":20296,"assis":20297,"Christian":20298,"Ġ145":20299,"ĠHass":20300,"Ġswall":20301,"Ġwax":20302,"ĠSurvival":20303,"VS":20304,"ĠMurd":20305,"vd":20306,"standard":20307,"Ġdragons":20308,"Ġacceleration":20309,"rational":20310,"final":20311,"Ġpaired":20312,"ĠEthereum":20313,"Ġinterfaces":20314,"Ġresent":20315,"Ġartifacts":20316,"Å«":20317,"arel":20318,"Ġcompetitor":20319,"ĠNicholas":20320,"ĠSurface":20321,"cpp":20322,"ĠTot":20323,"Ġeconomically":20324,"Ġorganised":20325,"Ġenforced":20326,"inho":20327,"Ġvarieties":20328,"Ġabdom":20329,"ĠBailey":20330,"idav":20331,"ĠSalv":20332,"paid":20333,"Ġaltitude":20334,"essert":20335,"ĠGutenberg":20336,"area":20337,"opoulos":20338,"Ġprofessors":20339,"iggs":20340,"ĠFate":20341,"hey":20342,"Ġ3000":20343,"Dist":20344,"Ġtwins":20345,"cill":20346,"ĠMaps":20347,"Ġtraps":20348,"Ġweed":20349,"ĠKiss":20350,"Ġyoga":20351,"Ġrecipients":20352,"ĠWestminster":20353,"Ġpools":20354,"ĠWalmart":20355,"188":20356,"ĠSchools":20357,"attack":20358,"ĠARM":20359,"paragraph":20360,"Warning":20361,"jl":20362,"Ġselfish":20363,"anchez":20364,"ĠHeights":20365,"Fre":20366,"ĠSoph":20367,"Ġ--------------------------------":20368,"tml":20369,"333":20370,"Ġraids":20371,"Ġsatellites":20372,"KEY":20373,"Ġlasts":20374,"ÑĤ":20375,"Ins":20376,"ĠDame":20377,"Ġunpredict":20378,"///":20379,"ghai":20380,"Ġartillery":20381,"Ġcruise":20382,"Ġgel":20383,"ĠCabinet":20384,"Ġblows":20385,"ĠEsp":20386,"Ġproximity":20387,"othe":20388,"ĠSkills":20389,"ĠUpper":20390,"obo":20391,"ĠNDP":20392,"Ġenjoys":20393,"Ġrepeating":20394,"ĠConstruction":20395,"ĠQuestions":20396,"Hillary":20397,"Ġuint":20398,"Ġprocessors":20399,"ĠGibson":20400,"ĠMultiple":20401,"qa":20402,"ĠBom":20403,"ĠMiles":20404,"ventional":20405,"Ġhurts":20406,"skin":20407,"ĠAIDS":20408,"Ġadvisers":20409,"ĠRoot":20410,"Ġmethodology":20411,"ĠDale":20412,"Ġdeton":20413,"ĠKnowledge":20414,"sequently":20415,"Ġ121":20416,"Ġconnects":20417,"Cy":20418,"ĠDanger":20419,"Ġcontributors":20420,"ĠBent":20421,"Ġbrass":20422,"ĠGuns":20423,"into":20424,"ĠFortune":20425,"Ġbroker":20426,"balance":20427,"Ġlengths":20428,"Ġvic":20429,"Ġaveraging":20430,"Ġappropriately":20431,"ĠCamera":20432,"Ġsandwich":20433,"ĠCDC":20434,"Ġcoordinate":20435,"Ġnavig":20436,"Ġgoodness":20437,"laim":20438,"Ġbrake":20439,"Ġextremist":20440,"ĠWake":20441,"ĠMend":20442,"ĠTiny":20443,"ĠCOL":20444,"ĠRF":20445,"ĠDual":20446,"ĠWine":20447,"Case":20448,"Ġrefined":20449,"Ġlamp":20450,"Lead":20451,"Ġbapt":20452,"ĠCarb":20453,"ĠSadd":20454,"ĠMinneapolis":20455,"PDF":20456,"Early":20457,"ĠHidden":20458,"Its":20459,"ĠTIME":20460,"Ġpap":20461,"Ġcommissioned":20462,"ĠFew":20463,"ĠColts":20464,"ĠBren":20465,"Ġbothered":20466,"Ġlikewise":20467,"Exper":20468,"ĠSchw":20469,"cry":20470,"nn":20471,"ĠMitch":20472,"imon":20473,"MG":20474,"bm":20475,"UMP":20476,"rays":20477,"Ġregistry":20478,"Ġ270":20479,"achine":20480,"rella":20481,"anting":20482,"00000":20483,"Ġruined":20484,"spot":20485,"Ġta":20486,"Ġmaximize":20487,"Ġinconven":20488,"Dead":20489,"Human":20490,"Enabled":20491,"ĠMarie":20492,"Ġchill":20493,"ĠParadise":20494,"Ġstarring":20495,"ĠLatino":20496,"ĠProtocol":20497,"ĠEVER":20498,"Ġsuppliers":20499,"message":20500,"ĠBrock":20501,"Ġserum":20502,"âĸĪâĸĪâĸĪâĸĪ":20503,"Ġencomp":20504,"Ġambition":20505,"uese":20506,"Ġarrows":20507,"Andrew":20508,"Ġantenna":20509,"Ġ1961":20510,"ĠBark":20511,"Ġbool":20512,"ãĤª":20513,"ĠStorage":20514,"Ġrailway":20515,"Ġtougher":20516,"ĠCad":20517,"Ġwashing":20518,"Py":20519,"']":20520,"embed":20521,"ĠMemphis":20522,"ackle":20523,"Ġfamously":20524,"ĠFortunately":20525,"ovies":20526,"Ġmindset":20527,"Ġsneak":20528,"ĠDh":20529,"RAW":20530,"ĠSimpson":20531,"Ġlivest":20532,"Ġlandmark":20533,"Ġcement":20534,"Low":20535,"Ġthrilled":20536,"ĠCourse":20537,"inel":20538,"Ġchuck":20539,"idate":20540,"global":20541,"Ġwhit":20542,"Ġ�":20543,"adays":20544,"ski":20545,"ĠSV":20546,"Ġviruses":20547,"306":20548,"ĠRespons":20549,"Ġtheaters":20550,"ĠBranch":20551,"ĠGeneva":20552,"ĠMK":20553,"Ġunbeliev":20554,"Ġcommunist":20555,"Original":20556,"ĠReceived":20557,"ĠTransfer":20558,"ĠArg":20559,"Input":20560,"ĠStrategy":20561,"Ġpalace":20562,"thening":20563,"Dri":20564,"Ġsentencing":20565,"umbnail":20566,"Ġpins":20567,"recy":20568,"Ġsiblings":20569,"Getting":20570,"ĠBU":20571,"ĠNorthwest":20572,"Ġprolonged":20573,"ĠSakura":20574,"Comb":20575,"ĠBour":20576,"Ġinadequate":20577,"ĠKash":20578,"Ġusername":20579,"ĠImprove":20580,"Ġbattling":20581,"ĠMAC":20582,"Ġcurriculum":20583,"Ġsoda":20584,"ĠCannon":20585,"Ġsensible":20586,"spons":20587,"December":20588,"Ġwicked":20589,"ĠPengu":20590,"Ġdictators":20591,"ĠHearts":20592,"ogyn":20593,"Ġsimilarities":20594,"ĠStats":20595,"Ġhollow":20596,"itations":20597,"\":[":20598,"Ġhover":20599,"ĠListen":20600,"sch":20601,"Sund":20602,"Ġcad":20603,"ĠParks":20604,"Ġlur":20605,"Ġhype":20606,"ĠLem":20607,"NAME":20608,"isure":20609,"Friday":20610,"Ġshoots":20611,"Ġcloses":20612,"Ġdb":20613,"ĠRidge":20614,"ĠDifferent":20615,"Ġreplies":20616,"ĠBroadway":20617,"opers":20618,"Ġintoler":20619,"ĠZeus":20620,"akespe":20621,"Ġproprietary":20622,"Ġrequesting":20623,"Ġcontrollers":20624,"ĠMIN":20625,"imedia":20626,"becca":20627,"Ġexpans":20628,"Ġoils":20629,"Bot":20630,"ĠChand":20631,"Ġprinter":20632,"Ġtopped":20633,"ĠPOL":20634,"ĠEarlier":20635,"Social":20636,"avin":20637,"Ġdecreases":20638,"ĠSeb":20639,"Ġspecifications":20640,"ĠBlast":20641,"ĠKurt":20642,"Ġfreel":20643,"Brown":20644,"Ġdilig":20645,"roe":20646,"ĠProblem":20647,"ĠQuad":20648,"Ġdecentral":20649,"ĠVector":20650,"anut":20651,"Ġplugins":20652,"ĠGregory":20653,"Ġfucked":20654,"elines":20655,"ĠAmbassador":20656,"take":20657,"Ġcleans":20658,"ongyang":20659,"Anonymous":20660,"stro":20661,"\"}":20662,"aline":20663,"ĠOdd":20664,"ĠEug":20665,"216":20666,"Ġboil":20667,"ĠPowers":20668,"Ġnurses":20669,"Obviously":20670,"ĠTechnical":20671,"Ġexceeded":20672,"ORS":20673,"Ġextremists":20674,"Ġtraces":20675,"expl":20676,"Ġcomr":20677,"ĠSach":20678,")/":20679,"Ġmasks":20680,"Ġsci":20681,"Bon":20682,"Ġregression":20683,"wegian":20684,"Ġadvisor":20685,"itures":20686,"ĠVo":20687,"example":20688,"ĠInstruct":20689,"Ġsiege":20690,"Ġreductions":20691,"ptr":20692,"Ġstatutory":20693,"Ġremoves":20694,"Ġpuck":20695,"redits":20696,"Ġbee":20697,"Ġsalad":20698,"Ġpromotions":20699,"ĠJoshua":20700,"withstanding":20701,"ETH":20702,"ĠCha":20703,"imus":20704,"Ġexpenditure":20705,"aunting":20706,"Ġdelighted":20707,"Ġ155":20708,"beh":20709,"Ġcarpet":20710,"ĠSpart":20711,"Ġjungle":20712,"lists":20713,"Ġbullying":20714,"ĠNobel":20715,"ĠGlen":20716,"Ġreferenced":20717,"Ġintroduces":20718,"sein":20719,"Ġchopped":20720,"glass":20721,"ĠWrest":20722,"Ġneutrality":20723,"ĠâĻ":20724,"Ġinvestigator":20725,"Ġshelves":20726,"Ġunconstitutional":20727,"Ġreproduction":20728,"Ġmerchant":20729,"mia":20730,"Ġmetrics":20731,"Ġexplosives":20732,"ĠSonia":20733,"Ġbodily":20734,"Ġthickness":20735,"Ġpredominantly":20736,"ĠAbility":20737,"Ġmonitored":20738,"ICH":20739,"Ġ].":20740,"ĠMartinez":20741,"Ġvisibility":20742,"Ġqueries":20743,"Ġgenocide":20744,"ĠWarfare":20745,"Query":20746,"Ġstudios":20747,"Ġembry":20748,"Ġcorridor":20749,"Ġcleaned":20750,"complete":20751,"ĠMH":20752,"Ġenrollment":20753,"INGS":20754,"Ġimpacted":20755,"Ġdisastrous":20756,"ĠYun":20757,"ĠClaire":20758,"ĠBasically":20759,"yt":20760,"usterity":20761,"Ġindirectly":20762,"wik":20763,"Ġdod":20764,"ĠCarr":20765,"Ġamp":20766,"Ġprohibit":20767,"ĠInitial":20768,"ĠRd":20769,"iji":20770,"Ġeducate":20771,"corn":20772,"iott":20773,"ĠBeauty":20774,"Ġdetective":20775,"ĠConn":20776,"since":20777,"Ġstagger":20778,"Ġobese":20779,"Ġbree":20780,"ologic":20781,"isse":20782,"walker":20783,"Ġblades":20784,"Ġlawful":20785,"func":20786,"ĠBehind":20787,"Ġappetite":20788,"Ġ(*":20789,"Ġtennis":20790,"Ġoffspring":20791,"Ġjets":20792,"Ġstructured":20793,"Ġaforementioned":20794,"Nov":20795,"Ġscaling":20796,"fill":20797,"Ġstew":20798,"Ġcurb":20799,"ĠStephan":20800,"edIn":20801,"SF":20802,"obic":20803,"éŃĶ":20804,"oug":20805,"ĠMM":20806,"Ġgenetically":20807,"opez":20808,"136":20809,"Ġumb":20810,"ancers":20811,"Ġcohort":20812,"Ġmerchandise":20813,"Ġimposing":20814,"ĠLegislature":20815,"ĠArchive":20816,"ivia":20817,"ĠNaval":20818,"Ġoffences":20819,"Ġmiracle":20820,"Ġsnapped":20821,"Ġfoes":20822,"Ġextensively":20823,"ĠRaf":20824,"Ġcater":20825,"edience":20826,"Kit":20827,"ĠBin":20828,"Ġrecommends":20829,"ĠCities":20830,"Ġrigid":20831,"ĠREAD":20832,"ĠNoble":20833,"ĠTian":20834,"Ġcertificates":20835,"antis":20836,"oiler":20837,"ĠBuddhist":20838,"did":20839,"Ġsurveyed":20840,"Ġdownward":20841,"Ġprints":20842,"ĠMotion":20843,"ronics":20844,"ĠSans":20845,"ossibly":20846,"uctions":20847,"Ġcolonies":20848,"ĠDanish":20849,"unit":20850,"Ġspoil":20851,"Ġadvisory":20852,"berries":20853,"Plan":20854,"Ġspecification":20855,"ophers":20856,"ĠResource":20857,"Ġshirts":20858,"prisingly":20859,"communications":20860,"Ġtrivial":20861,"Ġmentioning":20862,"isexual":20863,"Ġsupplements":20864,"Ġsupervision":20865,"BP":20866,"vor":20867,"Ġwit":20868,"Ġcooldown":20869,"Ġplaintiff":20870,"ĠReviews":20871,"ĠSri":20872,"ĠMint":20873,"ĠSugar":20874,"Ġafterward":20875,"ĠPriest":20876,"ĠInvestment":20877,"ogene":20878,"ĠTaking":20879,"Ġstretching":20880,"Ġinflammation":20881,"ĠTehran":20882,"Ġlining":20883,"Ġfreezing":20884,"ĠEntity":20885,"Ġinspiring":20886,"special":20887,"price":20888,"Ġsue":20889,"ĠPorter":20890,"ounge":20891,"ETA":20892,"ĠDerek":20893,"ĠLuis":20894,"uo":20895,"ymph":20896,"Ġexterior":20897,"ihil":20898,"ĠAshley":20899,"inator":20900,"Ġnutrients":20901,"ĠThrones":20902,"Ġfinances":20903,"ĠInspect":20904,"Ġspecially":20905,"ĠRequired":20906,"ĠPTS":20907,"ĠViolence":20908,"ointed":20909,"shots":20910,"Ġexcerpt":20911,"coon":20912,"INS":20913,"ĠGri":20914,"Ġrecognised":20915,"Week":20916,"Young":20917,"Ġvom":20918,"isle":20919,"ĠCurry":20920,"ĠBuddh":20921,"Ġnotebook":20922,"Ġdurable":20923,"/?":20924,"ĠGad":20925,"ĠPupp":20926,"Ġforgive":20927,"park":20928,"Ġpersonalities":20929,"analysis":20930,"clamation":20931,"Ġelevator":20932,"Ġwarehouse":20933,"ĠRole":20934,"unn":20935,"Ġillustration":20936,"ĠScan":20937,"Ġatmospheric":20938,"Import":20939,"ANC":20940,"ricted":20941,"fu":20942,"010":20943,"Ġarche":20944,"Ġrewarded":20945,"akespeare":20946,"Ġinternally":20947,"ĠRBI":20948,"alker":20949,"Ġelephant":20950,"owitz":20951,"ĠPizza":20952,"Ġbipartisan":20953,"és":20954,"Ġslowed":20955,"ĠStark":20956,"Ġoverride":20957,"OUS":20958,"Ġ320":20959,"undreds":20960,"ĠDeck":20961,"ĠCensus":20962,"bee":20963,"146":20964,"otor":20965,"Ġip":20966,"Ġub":20967,"ocations":20968,"ĠButton":20969,"rice":20970,"Ġcripp":20971,"fff":20972,"Ġoriginated":20973,"Ġoverwhelmed":20974,"appa":20975,"Ġforemost":20976,"âĢij":20977,"ĠLEG":20978,"release":20979,"eatured":20980,"atches":20981,"Ġreps":20982,"Ġlending":20983,"ĠReference":20984,"ĠClient":20985,"165":20986,"venth":20987,"Complete":20988,"ĠPatrol":20989,"Ġsworn":20990,"cam":20991,"Ġshuttle":20992,"ĠRalph":20993,"Ġhometown":20994,"-,":20995,"onal":20996,"ĠBP":20997,"åı":20998,"Ġpersuade":20999,"ĠAlexand":21000,"Ġcombines":21001,"Ġvivid":21002,"ĠLag":21003,"Ġencoding":21004,"Ġsalvation":21005,"wen":21006,"ĠRecovery":21007,"iya":21008,"University":21009,"ĠBiden":21010,"Ġbudgets":21011,"ĠTexans":21012,"fits":21013,"Ġhonored":21014,"Ġpython":21015,"TD":21016,"###":21017,"clone":21018,"Ġblink":21019,"ĠLiquid":21020,"Ġunemployed":21021,"Ġclashes":21022,"ĠCounsel":21023,"Ġdirecting":21024,"Ġpunct":21025,"ĠFalcons":21026,"Ġshark":21027,"ĠDamascus":21028,"Ġjeans":21029,"Ġembark":21030,"Ġseize":21031,"Ġupwards":21032,"280":21033,"ĠEz":21034,"ĠAnything":21035,"Ġexotic":21036,"lower":21037,"ĠCreator":21038,"ĠUm":21039,"Ġsuburbs":21040,"berger":21041,"ĠWend":21042,"Ġmint":21043,"ĠXX":21044,"ĠDro":21045,"Ġsuffers":21046,"Ġherb":21047,"tree":21048,"Ġfragile":21049,"Ġflooded":21050,"ĠAlcohol":21051,"olean":21052,"nyder":21053,"ĠKO":21054,"Fram":21055,"Ġ136":21056,"Ġowed":21057,"ĠMelee":21058,"ĠHash":21059,"Ġwhisk":21060,"Ġsudo":21061,"rr":21062,"Quick":21063,"appro":21064,"Ġii":21065,"ĠExamples":21066,"hee":21067,"Ġpromotes":21068,"perature":21069,"kar":21070,"ĠHonor":21071,"Ġsodium":21072,"ĠLif":21073,"rosso":21074,"intendent":21075,"Ġcorrespondent":21076,"Found":21077,"secret":21078,"Ġidentifies":21079,"agne":21080,"Ġlou":21081,"ĠPP":21082,"Ġcoincidence":21083,"move":21084,"Ġmilitia":21085,"Ġinfiltr":21086,"ĠPrimary":21087,"Ġpitching":21088,"ĠIb":21089,"ĠGOOD":21090,"ãĤ¸":21091,"ĠWizards":21092,"iral":21093,"ĠVenus":21094,"RR":21095,"ĠâĢķ":21096,"ĠCasey":21097,"Ġsadly":21098,"Ġadmire":21099,"Ġembarrassed":21100,"cb":21101,"Mel":21102,"Ġtubes":21103,"Ġbeautifully":21104,"ĠQueensland":21105,"Below":21106,"rez":21107,"quet":21108,"pleasant":21109,"Ġ«":21110,"Camp":21111,"Ġdecisive":21112,"1998":21113,"ĠLamb":21114,"utton":21115,"hn":21116,"ĠJagu":21117,"aunder":21118,"ĠCord":21119,"Ġclerk":21120,"Ġcaffe":21121,"Ġwiped":21122,"Ġreim":21123,"ĠMountains":21124,"Ġimprisoned":21125,"Ġdevelops":21126,"ĠPra":21127,"Ġmodeling":21128,"Anyone":21129,"ancel":21130,"ĠSit":21131,"Ġshields":21132,"Ġlawn":21133,"Ġcardiovascular":21134,"Ġdemonstrating":21135,"Ġparse":21136,"ĠIsraelis":21137,"Ġeuros":21138,"143":21139,"Ġglorious":21140,"inski":21141,"ecd":21142,"Ġconditioning":21143,"Ġhelpless":21144,"Ġmicrosc":21145,"ĠHarbor":21146,"Ġstakes":21147,"Ġ260":21148,"Ġunequ":21149,"ĠFloyd":21150,"Ġdamp":21151,"Ġapparatus":21152,"ĠLaws":21153,"Ġcounters":21154,"Ġinduce":21155,"atable":21156,"ĠAhmed":21157,"Ġslam":21158,"November":21159,"Ġpersist":21160,"Ġimminent":21161,"án":21162,"Ġshred":21163,"Ġphases":21164,"ĠEdmonton":21165,"ĠArmstrong":21166,"ĠMeet":21167,"ĠKitty":21168,"ÑĢ":21169,"circ":21170,"ĠAdult":21171,"Ġarose":21172,"ĠXen":21173,"Dan":21174,"gow":21175,"Ġsuperf":21176,"ĠAdmir":21177,"Ġendure":21178,"Ġkeyword":21179,"yrus":21180,"Ġyarn":21181,"Ġpathway":21182,"ĠHopkins":21183,"midt":21184,"Ġcensorship":21185,"dependent":21186,"Ġinstructor":21187,"Sources":21188,"Ġtoe":21189,"Ġballoon":21190,"Nob":21191,"Ġswear":21192,"ĠCastro":21193,"Ġgloss":21194,"ĠKavanaugh":21195,"Ġremarkably":21196,"Photos":21197,"ĠNom":21198,"ĠSoutheast":21199,"yers":21200,"Ġvalidation":21201,"Ġcannon":21202,"ĠVictory":21203,"ĠPierre":21204,"Ġcautious":21205,"Audio":21206,"Ġfetch":21207,"ĠGift":21208,"ĠHyp":21209,"Ġremedy":21210,"ZE":21211,"Ġscent":21212,"Ġbeard":21213,"ĠRut":21214,"-\"":21215,"Ġpatents":21216,"Hy":21217,"Ġunjust":21218,"Ġpotato":21219,"Ġforthcoming":21220,"Ġchef":21221,"ĠRift":21222,"affe":21223,"ĠROM":21224,"ĠLaunch":21225,"Ġpads":21226,"ĠNeo":21227,"Ġonset":21228,"Ġsqueeze":21229,"safe":21230,"Ġprefix":21231,"ĠTM":21232,"ĠNearly":21233,"ĠClinical":21234,"ĠMental":21235,"otiation":21236,"ĠUnic":21237,"antry":21238,"ĠCir":21239,"Ġepit":21240,"æ":21241,"Ġextracted":21242,"versely":21243,"riad":21244,"Ġstrains":21245,"Ġtops":21246,"Ġpoem":21247,"ĠRandy":21248,"ĠMaple":21249,"THER":21250,"upiter":21251,"ĠSSD":21252,"ļé":21253,"Ġuncon":21254,"pering":21255,"Ġslept":21256,"iners":21257,"Ġunderwater":21258,"ĠEvidence":21259,"gone":21260,"205":21261,"Ġhistorians":21262,"Ġsynthesis":21263,"Ġfrog":21264,"basketball":21265,"Ġvibrant":21266,"Ġsubord":21267,"Ġ365":21268,"ĠDial":21269,"Ġcooperate":21270,"HAHA":21271,"Ġgreeted":21272,"158":21273,"Ġjazz":21274,"Ġintox":21275,"ĠWalking":21276,"Ġsupervisor":21277,"ĠFusion":21278,"ĠMercedes":21279,"send":21280,"Ham":21281,"sd":21282,"nl":21283,"Ġtours":21284,"ĠFIFA":21285,"Ġculp":21286,"gd":21287,"304":21288,"Ġpleas":21289,"Ġillustrates":21290,"ĠColombia":21291,"Ġhighlighting":21292,"ĠSummary":21293,"Ġexposing":21294,"ĠDru":21295,"Ġirony":21296,"ritional":21297,"ĠCarroll":21298,"ĠEllis":21299,"Pict":21300,"ĠRapt":21301,"Ġadapter":21302,"Ġunm":21303,"Ġcorpse":21304,"Ġcelebrities":21305,"Den":21306,"atum":21307,"ĠApocalypse":21308,"ĠWag":21309,"lining":21310,"Ġhormones":21311,"Rub":21312,"ĠXi":21313,"ĠVaults":21314,"208":21315,"alkyrie":21316,"inosaur":21317,"Ġfeeds":21318,"vity":21319,"Ġdefeating":21320,"Wait":21321,"Ġemphasize":21322,"ĠSteelers":21323,"yrinth":21324,"leys":21325,"ĠWhenever":21326,"Currently":21327,"ĠClock":21328,"Ġcollectively":21329,"anyon":21330,"ĠJP":21331,"Ġmentality":21332,"Ġdownloads":21333,"Ġsurroundings":21334,"ĠBarnes":21335,"Ġflagship":21336,"Ġindicators":21337,"Ġgrapp":21338,"January":21339,"ĠElemental":21340,"ĠAthena":21341,"ibal":21342,"Ġsights":21343,"Ġcapita":21344,"ĠTreaty":21345,"Ġvoiced":21346,"ĠGaz":21347,"lette":21348,"Ġya":21349,"Ġexpired":21350,"Legend":21351,"Hot":21352,"nature":21353,"Ġunstable":21354,"Ġ280":21355,"ú":21356,"Comment":21357,"ALE":21358,"Ġquests":21359,"Ġhandler":21360,"nis":21361,"Ġversatile":21362,"Ġconceal":21363,"engeance":21364,"ĠInteractive":21365,"Ġobsessed":21366,"ĠDogs":21367,"Ġcracked":21368,"Sound":21369,"sv":21370,"ĠDylan":21371,"roads":21372,"fx":21373,"ĠCatholics":21374,"ĠHag":21375,"Ġslammed":21376,"Ġglowing":21377,"sale":21378,"Ġtissues":21379,"ĠChi":21380,"nee":21381,"Ġcher":21382,"sic":21383,"urrection":21384,"Ġbacon":21385,"ulatory":21386,").\"":21387,"Ġirregular":21388,"FORM":21389,"assed":21390,"Ġintentional":21391,"Ġcompensate":21392,"ĠSpeaking":21393,"ĠSets":21394,"153":21395,"Ġconventions":21396,"bands":21397,"emade":21398,"Ġecc":21399,"ĠWinston":21400,"ĠAssassin":21401,"ĠBelgian":21402,"Ġdependence":21403,"Ġniche":21404,"Ġbark":21405,"ĠJazz":21406,"Ġdisadvantage":21407,"Ġgasoline":21408,"Ġ165":21409,"çļĦ":21410,"essa":21411,"module":21412,"angular":21413,"OY":21414,"ĠTreatment":21415,"itas":21416,"olation":21417,"ĠArnold":21418,"Ġfeud":21419,"ĠNest":21420,"Ġtheatre":21421,"ewater":21422,"Ġminors":21423,"olicy":21424,"ĠHaven":21425,"division":21426,"Ġtrunk":21427,"Far":21428,"ĠPull":21429,"Ġcapturing":21430,"Ġ1800":21431,"ĠTeen":21432,"Ġexempl":21433,"Ġclinics":21434,"ĠBurg":21435,"Ġsubstit":21436,"Ġpayload":21437,"ĠLav":21438,"ĠTroy":21439,"ĠWitness":21440,"Ġfragments":21441,"Ġpasswords":21442,"Ġgospel":21443,"ĠGin":21444,"Ġtenants":21445,"olith":21446,"Six":21447,"Previous":21448,"ĠAges":21449,"ĠDarwin":21450,"Ġblat":21451,"Ġempathy":21452,"smith":21453,"bag":21454,"ĠEcho":21455,"ĠCamb":21456,"ĠMadd":21457,"ĠBoo":21458,"Ġrede":21459,"ĠBurning":21460,"Ġsmoothly":21461,"ĠAdrian":21462,"ĠVampire":21463,"ĠMonsters":21464,"steam":21465,"Style":21466,"Ma":21467,"rea":21468,"ĠDwar":21469,"alyst":21470,"ursor":21471,"Ġelimination":21472,"Ġcrypto":21473,"cht":21474,"ĠEternal":21475,"âĢ¦]":21476,"ĠSorce":21477,"Ill":21478,"NER":21479,"Ġuh":21480,"Conclusion":21481,"wage":21482,"Ġrespir":21483,"Ġreminis":21484,"hetical":21485,"Ġgy":21486,"Ġutilized":21487,"icidal":21488,"Ġ1900":21489,"Ġhunters":21490,"ĠSwan":21491,"ĠReact":21492,"Ġvisitor":21493,"ĠThanksgiving":21494,"308":21495,"Posts":21496,"Ġhips":21497,"1997":21498,"omers":21499,"Ġknocking":21500,"ĠVehicle":21501,"Ġtil":21502,"Ġ138":21503,"Ġmi":21504,"ĠInvestigation":21505,"ĠKenya":21506,"Ġcasino":21507,"Ġmotives":21508,"Ġregain":21509,"rex":21510,"Ġweekends":21511,"Ġstabbed":21512,"boro":21513,"Ġexploited":21514,"ĠHAVE":21515,"ĠTelevision":21516,"cock":21517,"Ġpreparations":21518,"Ġendeav":21519,"ĠRemote":21520,"ĠMaker":21521,"ĠProdu":21522,"ĠEvan":21523,"Ġinformational":21524,"ĠLouisville":21525,"154":21526,"ĠDreams":21527,"Ġplots":21528,"ĠRunner":21529,"Ġhurting":21530,"Ġacademy":21531,"ĠMontgomery":21532,"nm":21533,"ĠLanc":21534,"ĠAlz":21535,"210":21536,"elong":21537,"Ġretailer":21538,"Ġarising":21539,"Ġrebellion":21540,"Ġblonde":21541,"played":21542,"Ġinstrumental":21543,"Cross":21544,"Ġretention":21545,"Ġtherapeutic":21546,"Ġseas":21547,"Ġinfantry":21548,"ĠClint":21549,"Ġprompting":21550,"Ġbitch":21551,"Ġstems":21552,"ĠKra":21553,"Ġthesis":21554,"ĠBog":21555,"rued":21556,"Ġkings":21557,"Ġclay":21558,"ificent":21559,"ĠYES":21560,"ĠThing":21561,"ĠCubs":21562,"veyard":21563,"elsh":21564,"inarily":21565,"ĠEy":21566,"ĠRolling":21567,"Ġevolving":21568,"India":21569,"Ġrecognizes":21570,"Ġgraduation":21571,"isers":21572,"Ġfertility":21573,"ĠMilan":21574,"Command":21575,"Ġboxing":21576,"Ġ1943":21577,"Ġgluten":21578,"ĠEmir":21579,"Ġidol":21580,"Ġconceived":21581,"ĠCreation":21582,"Merit":21583,"uddy":21584,"ussions":21585,"ĠLieutenant":21586,"ietal":21587,"Ġunchanged":21588,"ĠScale":21589,"ĠCrimea":21590,"balls":21591,"atorial":21592,"Ġdepths":21593,"Ġempirical":21594,"Ġtransm":21595,"Ġunsafe":21596,"missible":21597,"comfort":21598,"156":21599,"Ġmechanic":21600,"002":21601,"lins":21602,"Ġsmoked":21603,"Pos":21604,"Ġslowing":21605,"Ġlav":21606,"Texas":21607,"Ġcheating":21608,"ĠMetropolitan":21609,"ethyl":21610,"Ġdiscovering":21611,"asse":21612,"Ġpencil":21613,"ĠPyongyang":21614,"Ġcloset":21615,"ĠSheet":21616,"ĠEntry":21617,"oustic":21618,"Ġmyst":21619,"erate":21620,"ariat":21621,"Ġminerals":21622,"Ġmusician":21623,"ĠPul":21624,"ĠMaz":21625,"249":21626,"Ġpermissions":21627,"Ġiv":21628,"enary":21629,"ickers":21630,"ĠBing":21631,"hea":21632,"enable":21633,"Ġgriev":21634,"Ġasserted":21635,"ĠColonel":21636,"Ġaffidav":21637,"wo":21638,"Ġseated":21639,"ĠRide":21640,"Ġpaintings":21641,"ĠPix":21642,"Ġ137":21643,"ishi":21644,"umbai":21645,"gotten":21646,"ĠEarl":21647,"Ġinning":21648,"Ġcensus":21649,"Ġtravelled":21650,"ĠConsult":21651,"185":21652,"bind":21653,"Ġsimplicity":21654,"Ġoverlooked":21655,"ĠHelpful":21656,"Ġmonkey":21657,"Ġoverwhelmingly":21658,"Blood":21659,"ĠFlint":21660,"ĠJama":21661,"ĠPresent":21662,"ĠRage":21663,"ĠTA":21664,"ptive":21665,"Ġturnout":21666,"wald":21667,"ĠDolphins":21668,"ĠVPN":21669,"Ġonion":21670,"Ġcrafting":21671,"mma":21672,"ĠMercury":21673,"Ġarrange":21674,"Ġalerts":21675,"ĠOT":21676,"zbollah":21677,"Ġgases":21678,"ĠRichardson":21679,"sal":21680,"lar":21681,"Ġfrost":21682,"Ġlowering":21683,"Ġacclaim":21684,"Ġstartups":21685,"ĠGain":21686,"essment":21687,"Ġguardian":21688,"人":21689,"ĠPie":21690,"ĠLinks":21691,"Ġmerits":21692,"Ġawake":21693,"Ġparental":21694,"Ġexceeds":21695,"Ġidle":21696,"ĠPilot":21697,"ĠeBay":21698,"ĠAccept":21699,"ipeg":21700,"Cam":21701,"ĠKot":21702,"Ġtraders":21703,"olitics":21704,"unker":21705,"ĠPale":21706,"osi":21707,"anmar":21708,"Ġ1947":21709,"ĠFell":21710,"estial":21711,"itating":21712,"GF":21713,"ĠSr":21714,"ifted":21715,"Ġconnector":21716,"ĠBone":21717,"illes":21718,"260":21719,"hma":21720,"Ġoverlap":21721,"ĠGitHub":21722,"Ġcleaner":21723,"ĠBaptist":21724,"ĠWAS":21725,"Ġlungs":21726,"Ñģ":21727,"ĠBUT":21728,"Ġcite":21729,"Ġpitched":21730,"reatment":21731,"Ġtrophies":21732,"ĠNu":21733,"386":21734,"ĠPride":21735,"Ġattendees":21736,"[]":21737,"179":21738,"Ġspatial":21739,"Ġprizes":21740,"ĠReligion":21741,"Ġshowcase":21742,"ĠCategory":21743,"vidia":21744,"Target":21745,"Property":21746,"?,":21747,"Ġfusion":21748,"pie":21749,"ĠUCLA":21750,"Ġsoundtrack":21751,"Ġprincess":21752,"ĠCaval":21753,"should":21754,"Ġlimbs":21755,"Background":21756,"Ġlonely":21757,"Ġcores":21758,"ĠTail":21759,"sheet":21760,"Ġ132":21761,"Ra":21762,"ãĤ«":21763,"ĠBolt":21764,"Ġbooked":21765,"Ġadminister":21766,"Ġequals":21767,"wy":21768,"Ġobserving":21769,"ĠBaron":21770,"ĠAdobe":21771,"Ġvirgin":21772,"ĠSocialist":21773,"Move":21774,"ghazi":21775,"ĠLinda":21776,"212":21777,"Ġbrewing":21778,"Ġmerchants":21779,"burse":21780,"Ġdivor":21781,"Ġmetals":21782,"ĠNer":21783,"Ġsums":21784,"ĠEnemy":21785,"Ġenvision":21786,"Ġgranting":21787,"ĠHoney":21788,"ĠSkyrim":21789,"Ġsocio":21790,"graded":21791,"Ġselective":21792,"WASHINGTON":21793,"Ġ1948":21794,"ĠSirius":21795,"ĠGross":21796,"activity":21797,"ĠIvan":21798,"Ġfurious":21799,"BSD":21800,"ĠPrevious":21801,"Ġresponsive":21802,"Ġcharitable":21803,"Ġleaning":21804,"ĠPew":21805,"Ġviolates":21806,"\\\\\\\\\\\\\\\\":21807,"ĠComing":21808,"wire":21809,"Ġpoet":21810,"Ġresolutions":21811,"command":21812,"ĠPortuguese":21813,"Ġnickname":21814,"Ġdeaf":21815,"February":21816,"Ġrecognise":21817,"Ġentirety":21818,"Ġseasonal":21819,"placed":21820,"ĠTelegraph":21821,"Ġmicrophone":21822,"ouring":21823,"Ġgrains":21824,"Ġgoverned":21825,"Ġpostp":21826,"ĠWaters":21827,"inement":21828,"Ġundocumented":21829,"ĠComcast":21830,"Ġfox":21831,"Ġassaults":21832,"reon":21833,"many":21834,"ĠJenkins":21835,"ĠAnyway":21836,"Ġassessments":21837,"Ġdowns":21838,"ĠMouse":21839,"Ġsuperb":21840,"kt":21841,"ĠDow":21842,"Ġtaxation":21843,"401":21844,"Ġsmiles":21845,"Ġundertaken":21846,"Ġexh":21847,"Ġenthusiastic":21848,"Ġtwent":21849,"Ġgovernmental":21850,"Ġautonomy":21851,"ĠTechnologies":21852,"ĠChain":21853,"Ġprevalent":21854,"fb":21855,"Ġnicotine":21856,"ogram":21857,"job":21858,"Ġawaiting":21859,"ĠMenu":21860,"Ġdeputies":21861,"kov":21862,"ishops":21863,"Button":21864,"ĠShanghai":21865,"Ġdiesel":21866,"ĠDuck":21867,"Ryan":21868,"ĠPCs":21869,"NF":21870,"jury":21871,"ente":21872,"Ġinaccurate":21873,"eddy":21874,"Whatever":21875,"Ġshowc":21876,"ĠNad":21877,"odus":21878,"etr":21879,"Ġplaintiffs":21880,"ĠWOR":21881,"ĠAssange":21882,"Ġprivat":21883,"Ġpremiums":21884,"Ġtam":21885,"URL":21886,"Ġelites":21887,"ĠRanger":21888,"ottenham":21889,"ĠHoff":21890,"ĠAthens":21891,"Ġdefinite":21892,"Ġsighed":21893,"Ġevenly":21894,"211":21895,"ĠAmber":21896,"akia":21897,"Ġmailing":21898,"Ġcrashing":21899,"ĠConfederate":21900,"rugged":21901,"Wal":21902,"ĠDepths":21903,"Ġjuvenile":21904,"Ġreactor":21905,"Introduction":21906,"ĠDeluxe":21907,"1995":21908,"ĠSanchez":21909,"ĠMead":21910,"ivable":21911,":-":21912,"ĠPlanning":21913,"ĠTrap":21914,"quin":21915,"ĠProtect":21916,"vered":21917,"Information":21918,"Ġkidney":21919,"innamon":21920,"las":21921,"Ġpolicing":21922,"Ġtolerate":21923,"ĠQi":21924,"Ġbiased":21925,"Fort":21926,"ĠKi":21927,"save":21928,"Ġprivileged":21929,"Ġbeasts":21930,"ĠGlas":21931,"ĠCinem":21932,"Ġcomeback":21933,"Sunday":21934,"Ġextinction":21935,"hops":21936,"Ġtransmit":21937,"Ġdoubles":21938,"ĠFlat":21939,"167":21940,"Ġdisputed":21941,"Ġinjustice":21942,"foo":21943,"Vict":21944,"roleum":21945,"ĠJulie":21946,"Context":21947,"ĠRarity":21948,"issue":21949,"Component":21950,"Ġcounseling":21951,"anne":21952,"dark":21953,"Ġobjections":21954,"uilt":21955,"Ġgast":21956,"Ġplac":21957,"Ġunused":21958,"ãĥĩ":21959,"ĠTrial":21960,"ĠJas":21961,"hedral":21962,"obb":21963,"Ġtemporal":21964,"ĠPRO":21965,"ĠNW":21966,"ĠAnniversary":21967,"Large":21968,"Ġtherm":21969,"Ġdavid":21970,"Ġsystemic":21971,"ĠShir":21972,"mut":21973,"ĠNept":21974,"address":21975,"Ġscanning":21976,"Ġunderstandable":21977,"Ġcanvas":21978,"Cat":21979,"ĠZoo":21980,"Ġangels":21981,"LO":21982,"ĠStatement":21983,"ĠSig":21984,"ovable":21985,"ĠAway":21986,"sharing":21987,"ocrats":21988,"stated":21989,"Ġweighing":21990,"Nor":21991,"wild":21992,"Bey":21993,"Ġastonishing":21994,"ĠReynolds":21995,"Ġopener":21996,"Ġtrainer":21997,"Ġsurgical":21998,"pn":21999,"Ġadjusting":22000,"wheel":22001,"Ġfrown":22002,"ervative":22003,"Ġsuspend":22004,"Within":22005,"tein":22006,"Ġobstacle":22007,"Ġliberties":22008,"ymes":22009,"Ġuranium":22010,"ansom":22011,"anol":22012,"uba":22013,"ĠLoss":22014,"Ġarous":22015,"ĠHenderson":22016,"Wow":22017,"spl":22018,"cur":22019,"ĠÂŃ":22020,"Ġtheirs":22021,"Damage":22022,"Ġdownloading":22023,"Ġdiscern":22024,"ĠSto":22025,"ĠFla":22026,"Ġhath":22027,"ĠAj":22028,"Ġunpleasant":22029,"European":22030,"expensive":22031,"Ġscreenshot":22032,"ĠUV":22033,"Ġallied":22034,"ĠPersian":22035,"Ġmonopoly":22036,"Ġatom":22037,"ĠRedskins":22038,"\"><":22039,"Ġcancell":22040,"Ġcinema":22041,"131":22042,"fair":22043,"ĠAlfred":22044,"Ġduck":22045,"args":22046,"223":22047,"ĠISI":22048,"Ġsignaling":22049,"inar":22050,"Ġlaughs":22051,"Ġforwards":22052,"Ġreckless":22053,"Ġlisteners":22054,"ativity":22055,"Ġvastly":22056,"nant":22057,"Less":22058,"ĠHunting":22059,"ĠScientific":22060,"ITED":22061,"Ġknight":22062,"ĠHTC":22063,"usa":22064,"tmp":22065,"Ġrude":22066,"ĠLegendary":22067,"Ġarises":22068,"Bad":22069,"ĠClaim":22070,"peg":22071,"Ġrealities":22072,"Think":22073,"Ġ°":22074,"Ġrode":22075,"Ġstrive":22076,"Ġanecd":22077,"Ġshorts":22078,"Ġhypothes":22079,"Ġcoordinated":22080,"ĠGandhi":22081,"ĠFPS":22082,"RED":22083,"Ġsusceptible":22084,"Ġshrink":22085,"ĠChart":22086,"Help":22087,"Ġion":22088,"deep":22089,"ribes":22090,"ĠKai":22091,"ĠCustomer":22092,"Summary":22093,"Ġcough":22094,"wife":22095,"Ġlend":22096,"Ġpositioning":22097,"Ġlottery":22098,"ĠCanyon":22099,"Ġfade":22100,"Ġbronze":22101,"ĠKenny":22102,"Ġboasts":22103,"ĠEnhanced":22104,"record":22105,"Ġemergence":22106,"Ġakin":22107,"ĠBert":22108,"itous":22109,"âĸij":22110,"Ġstip":22111,"Ġexchanged":22112,"omore":22113,"alsh":22114,"Ġreservoir":22115,"Ġstandpoint":22116,"WM":22117,"Ġinitiate":22118,"Ġdecay":22119,"Ġbrewery":22120,"Ġterribly":22121,"Ġmortal":22122,"levard":22123,"Ġrevis":22124,"NI":22125,"elo":22126,"Ġconfess":22127,"ĠMSNBC":22128,"Ġsubmissions":22129,"Controller":22130,"Ġ202":22131,"ĠRuth":22132,"});":22133,"ĠAzure":22134,"Ġ.\"":22135,"206":22136,"ĠMarketing":22137,"Ġlaund":22138,"iencies":22139,"Ġrenowned":22140,"ĠTrou":22141,"ĠNGO":22142,"blems":22143,"Ġterrified":22144,"Ġwarns":22145,"Ġpert":22146,"Ġunsure":22147,"480":22148,"alez":22149,"ultz":22150,"ĠOutside":22151,"Ġstyl":22152,"ĠUnderground":22153,"Ġpanc":22154,"Ġdictionary":22155,"Ġfoe":22156,"riminal":22157,"ĠNorwegian":22158,"Ġjailed":22159,"Ġmaternal":22160,"ée":22161,"ĠLucy":22162,"cop":22163,"Cho":22164,"Ġunsigned":22165,"ĠZelda":22166,"ĠInsider":22167,"ĠContinued":22168,"Ġ133":22169,"ĠNaruto":22170,"ĠMajority":22171,"169":22172,"ĠWo":22173,"ãĤĵ":22174,"Ġpastor":22175,"Ġinformal":22176,"н":22177,"anthrop":22178,"join":22179,"ãģĹ":22180,"itational":22181,"NP":22182,"ĠWriting":22183,"fn":22184,"ĠBever":22185,"195":22186,"Ġyelling":22187,"Ġdrastically":22188,"Ġeject":22189,"Ġneut":22190,"Ġthrive":22191,"ĠFrequ":22192,"oux":22193,"Ġpossesses":22194,"ĠSenators":22195,"ĠDES":22196,"ĠShakespeare":22197,"ĠFranco":22198,"ĠLB":22199,"uchi":22200,"Ġincarn":22201,"Ġfounders":22202,"Function":22203,"Ġbrightness":22204,"ĠBT":22205,"Ġwhale":22206,"ĠTheater":22207,"mass":22208,"ĠDoll":22209,"Something":22210,"Ġechoed":22211,"ĠHex":22212,"crit":22213,"afia":22214,"Ġgoddess":22215,"Ġeleven":22216,"ĠPreview":22217,"ĠAurora":22218,"Ġ401":22219,"ulsive":22220,"ĠLogan":22221,"inburgh":22222,"ĠCenters":22223,"ĠONLY":22224,"ĠAid":22225,"Ġparadox":22226,"Ġhurd":22227,"ĠLC":22228,"Due":22229,"court":22230,"Ġoffended":22231,"Ġevaluating":22232,"ĠMatthews":22233,"Ġtomb":22234,"Ġpayroll":22235,"Ġextraction":22236,"ĠHands":22237,"ifi":22238,"Ġsupernatural":22239,"ĠCOMM":22240,"]=":22241,"dogs":22242,"Ġ512":22243,"ĠMeeting":22244,"Richard":22245,"ĠMaximum":22246,"Ġideals":22247,"Things":22248,"mand":22249,"ĠRegardless":22250,"Ġhumili":22251,"buffer":22252,"Little":22253,"ĠDani":22254,"ĠNak":22255,"Ġliberation":22256,"ĠAbe":22257,"ĠOL":22258,"Ġstuffed":22259,"aca":22260,"inda":22261,"raphic":22262,"Ġmosqu":22263,"Ġcampaigning":22264,"Ġoccupy":22265,"Squ":22266,"rina":22267,"ĠWel":22268,"ĠVS":22269,"Ġphysic":22270,"Ġpuls":22271,"rint":22272,"oaded":22273,"ETF":22274,"ĠArchives":22275,"Ġvenues":22276,"hner":22277,"ĠTurbo":22278,"Ġlust":22279,"Ġappealed":22280,"quez":22281,"ilib":22282,"ĠTimothy":22283,"Ġomn":22284,"dro":22285,"Ġobsession":22286,"ĠSavage":22287,"1996":22288,"Global":22289,"Jes":22290,"214":22291,"Ġsliding":22292,"Ġdisappro":22293,"ĠMagical":22294,"Ġvoluntarily":22295,"gb":22296,"aney":22297,"Ġprophet":22298,"ĠRein":22299,"ĠJulia":22300,"ĠWorth":22301,"aurus":22302,"Ġbounds":22303,"ieu":22304,")))":22305,"Ġcrore":22306,"ĠCitizen":22307,"Sky":22308,"Ġcolumnist":22309,"Ġseekers":22310,"ondo":22311,"ISA":22312,"ĠLength":22313,"Ġnostalg":22314,"Ġnewcom":22315,"Ġdetrim":22316,"entric":22317,"375":22318,"ĠGE":22319,"Ġautop":22320,"Ġacademics":22321,"AppData":22322,"ĠShen":22323,"Ġidiot":22324,"ĠTransit":22325,"Ġteaspoon":22326,"Wil":22327,"KO":22328,"ĠComedy":22329,">,":22330,"Ġpopulated":22331,"WD":22332,"Ġpigs":22333,"ĠOculus":22334,"Ġsympathetic":22335,"Ġmarathon":22336,"198":22337,"Ġseizure":22338,"sided":22339,"Ġdop":22340,"irtual":22341,"Land":22342,"ĠFloor":22343,"osaurs":22344,"...]":22345,"Ġlos":22346,"Ġsubsidiary":22347,"EY":22348,"ĠParts":22349,"ĠStef":22350,"ĠJudiciary":22351,"Ġ134":22352,"Ġmirrors":22353,"Ġket":22354,"times":22355,"Ġneurolog":22356,"Ġcav":22357,"ĠGuest":22358,"Ġtumor":22359,"scill":22360,"ĠLloyd":22361,"Est":22362,"Ġclearer":22363,"Ġstereotypes":22364,"Ġdur":22365,"nothing":22366,"Reddit":22367,"Ġnegotiated":22368,"------------------------":22369,"235":22370,"Ġflown":22371,"ĠSeoul":22372,"ĠResident":22373,"ĠSCH":22374,"Ġdisappearance":22375,"ĠVince":22376,"grown":22377,"Ġgrabs":22378,"ril":22379,"ĠInfinite":22380,"ĠTwenty":22381,"Ġpedestrian":22382,"Ġjersey":22383,"ĠFur":22384,"ĠInfinity":22385,"ĠElliott":22386,"Ġmentor":22387,"Ġmorally":22388,"Ġobey":22389,"secure":22390,"iffe":22391,"Ġantibiotics":22392,"angled":22393,"ĠFreeman":22394,"ĠIntroduction":22395,"Jun":22396,"Ġmarsh":22397,"icans":22398,"ĠEVENTS":22399,"ochond":22400,"Wall":22401,"iculty":22402,"Ġmisdemeanor":22403,"Ġly":22404,"Thomas":22405,"ĠResolution":22406,"Ġanimations":22407,"ĠDry":22408,"Ġintercourse":22409,"ĠNewcastle":22410,"ĠHog":22411,"ĠEquipment":22412,"177":22413,"Ġterritorial":22414,"Ġarchives":22415,"203":22416,"Filter":22417,"ĠMunich":22418,"Ġcommanded":22419,"ĠWand":22420,"Ġpitches":22421,"ĠCroat":22422,"Ġratios":22423,"ĠMits":22424,"Ġaccumulated":22425,"ĠSpecifically":22426,"Ġgentleman":22427,"acerb":22428,"Ġpenn":22429,"Ġaka":22430,"ĠFuk":22431,"Ġintervene":22432,"ĠRefuge":22433,"ĠAlzheimer":22434,"Ġsuccession":22435,"ohan":22436,"does":22437,"Lord":22438,"Ġseparat":22439,"Ġcorrespondence":22440,"Ġshiny":22441,"Prior":22442,"Ġsulf":22443,"Ġmiserable":22444,"Ġdedication":22445,"().":22446,"Ġspecialists":22447,"Ġdefects":22448,"ĠCult":22449,"ĠXia":22450,"Ġjeopard":22451,"ĠOre":22452,"Ability":22453,"Ġlear":22454,"Ġambitions":22455,"ĠBMI":22456,"ĠArabs":22457,"Ġ1942":22458,"Ġpreservation":22459,"ificate":22460,"Ġashamed":22461,"loss":22462,"ĠRestaur":22463,"Ġresemble":22464,"Ġenrich":22465,"ĠKN":22466,"ĠClan":22467,"float":22468,"Ġplayable":22469,"ITT":22470,"Ġharmony":22471,"arrison":22472,"ĠWeinstein":22473,"were":22474,"Ġpoisoning":22475,"ĠComput":22476,"ĠWordPress":22477,"major":22478,"ĠValve":22479,"Fan":22480,"ĠThrow":22481,"ĠRomans":22482,"ĠDepression":22483,"ados":22484,"Ġtortured":22485,"Ġbalancing":22486,"bottom":22487,"Ġacquiring":22488,"ĠMonte":22489,"ardi":22490,"Ġaura":22491,"Ġ##":22492,"ĠStanding":22493,"ĠAtlas":22494,"CF":22495,"Ġintrins":22496,"ĠBenghazi":22497,"Ġcamping":22498,"Ġtapped":22499,"blade":22500,"strous":22501,"ĠRabb":22502,"ĠWritten":22503,"tip":22504,"ĠNeigh":22505,"sterdam":22506,"ĠAllow":22507,"ĠHealing":22508,"ĠRhod":22509,"num":22510,"Ġcaffeine":22511,"ĠPercent":22512,"Ġboo":22513,"Ġapples":22514,"305":22515,"Ġwelcoming":22516,"Ġapplaud":22517,"Ġausterity":22518,"±":22519,"ĠReality":22520,"efe":22521,"å®":22522,"Ġsucks":22523,"Ġtabs":22524,"ĠPayPal":22525,"Ġbackpack":22526,"Ġgifted":22527,"abulary":22528,"ĠScout":22529,"irteen":22530,"Ġchin":22531,"Ġomitted":22532,"Ġnegatively":22533,"Ġaccessing":22534,"ĠEarn":22535,"Ġambulance":22536,"Ġheadphones":22537,"Ġ205":22538,"ĠRefresh":22539,"president":22540,"ĠKitchen":22541,"ĠEntered":22542,"ĠSnyder":22543,"005":22544,"omical":22545,"Ġborrowed":22546,"ĠNem":22547,"Ġaviation":22548,"Ġstall":22549,"rimination":22550,"Ġuniforms":22551,"itime":22552,"ĠSimmons":22553,"energy":22554,"ablished":22555,"yy":22556,"qualified":22557,"Ġrallies":22558,"ĠStuart":22559,"flight":22560,"Ġgangs":22561,"rag":22562,"Ġvault":22563,"lux":22564,"ĠCompar":22565,"Ġdesignation":22566,"209":22567,"ĠJos":22568,"dollar":22569,"zero":22570,"Ġwells":22571,"303":22572,"Ġconstituents":22573,"Ġheck":22574,"Ġcows":22575,"Ġcommanders":22576,"Ġdifferential":22577,"ĠCatherine":22578,"299":22579,"Ġvalve":22580,"Ġbrace":22581,"Ġperspectives":22582,"cert":22583,"fact":22584,"icularly":22585,"ĠMcN":22586,"planes":22587,"Ġintric":22588,"Ġpeas":22589,"ovan":22590,"Ġtossed":22591,"retch":22592,"ĠLopez":22593,"Ġunfamiliar":22594,"death":22595,"ĠApart":22596,"ĠChang":22597,"Ġrelieved":22598,"rophe":22599,"Ġairports":22600,"Ġfreak":22601,"util":22602,"Mill":22603,"ĠChin":22604,"ĠOwen":22605,"male":22606,"ĠBroken":22607,"ĠWinds":22608,"rob":22609,"rising":22610,"Ġfirefighters":22611,"Ġauthoritarian":22612,"Ġ148":22613,"Bitcoin":22614,"external":22615,"Ġbrowsers":22616,"ichever":22617,"orian":22618,"Ġunb":22619,"Ġpoke":22620,"ĠZot":22621,"Mid":22622,"ĠPopular":22623,"Ġcovert":22624,"Ġcontributes":22625,"Ġ650":22626,"Ġcontention":22627,"Gate":22628,"Ġconsoles":22629,"Ġchromos":22630,"ĠIX":22631,"Ġvisually":22632,"ĠEisen":22633,"Ġjewelry":22634,"Ġdelegation":22635,"Ġaccelerate":22636,"ĠRiley":22637,"Ġslope":22638,"Ġindoor":22639,"itially":22640,"Ġhugely":22641,"Ġtunnels":22642,"Ġfined":22643,"Ġdirective":22644,"Ġforehead":22645,"ustomed":22646,"Ġskate":22647,"Music":22648,"gas":22649,"Ġrecognizing":22650,"ambo":22651,"Ġoverweight":22652,"ĠGrade":22653,"ÙĬ":22654,"Ġsounding":22655,"Ġlocking":22656,"ĠREM":22657,"Store":22658,"Ġexcav":22659,"ĠLikewise":22660,"ĠLights":22661,"Ġelbow":22662,"ĠSupply":22663,"wic":22664,"Ġhandsome":22665,"1994":22666,"Coll":22667,"Ġadequately":22668,"ĠAssociate":22669,"Ġstrips":22670,"Ġcrackdown":22671,"Ġmarvel":22672,"ĠKun":22673,"Ġpassages":22674,"@@@@":22675,"ĠTall":22676,"Ġthoughtful":22677,"namese":22678,"Ġprostitution":22679,"business":22680,"Ġballistic":22681,"personal":22682,"cig":22683,"izational":22684,"Round":22685,"ĠÂłĠÂłĠÂłĠÂł":22686,"ĠColeman":22687,"Ġadmitting":22688,"ĠPlug":22689,"Ġbitcoins":22690,"ĠSuz":22691,"Ġfairness":22692,"Ġsupplier":22693,"Ġcatastrophic":22694,"ĠHelen":22695,"oqu":22696,"Marc":22697,"ĠArticles":22698,"gie":22699,"Ġendangered":22700,"Ġdestiny":22701,"ĠVolt":22702,"olia":22703,"axis":22704,"Ġcheat":22705,"Ġunified":22706,"ICO":22707,"quote":22708,"302":22709,"ĠSed":22710,"Ġsuppression":22711,"Ġanalyzing":22712,"Ġsquat":22713,"Ġfiguring":22714,"Ġcoordinates":22715,"Ġchunks":22716,"Ġ1946":22717,"Ġsubp":22718,"Ġwiki":22719,"ĠForbes":22720,"ĠJupiter":22721,"ĠErik":22722,"imer":22723,"ĠCommercial":22724,"\\)":22725,"Ġlegitimacy":22726,"Ġdental":22727,"ĠMean":22728,"Ġdeficits":22729,"550":22730,"Originally":22731,"ĠHorror":22732,"Ġcontamination":22733,"llah":22734,"Ġconfisc":22735,"ĠClare":22736,"TB":22737,"ĠFailed":22738,"aned":22739,"Ġruler":22740,"ĠController":22741,"Ġfeminists":22742,"Fix":22743,"gay":22744,"207":22745,"Ġrabbit":22746,"Third":22747,"owntown":22748,"Ġglue":22749,"Ġvolatile":22750,"Ġshining":22751,"Ġfoll":22752,"Ġimpaired":22753,"Ġsupers":22754,"æĪ":22755,"Ġclutch":22756,"ļéĨĴ":22757,"Ġprolet":22758,"Ġ(!":22759,"Ġyelled":22760,"ĠKiev":22761,"ĠErn":22762,"ĠShock":22763,"KB":22764,"Ġsituated":22765,"query":22766,"ĠNas":22767,"Ġannex":22768,"character":22769,"ĠHoliday":22770,"Ġautomation":22771,"ĠJill":22772,"ĠRemastered":22773,"Ġlinem":22774,"Ġwilderness":22775,"ĠHorizon":22776,"ĠGuinea":22777,"AZ":22778,"Ġmainland":22779,"Ġsecrecy":22780,"LEASE":22781,"Ġpunk":22782,"ĠProvince":22783,"(),":22784,"Speed":22785,"Ġhanding":22786,"ĠSebast":22787,"Sir":22788,"rase":22789,"Ġjournals":22790,"Ġcongest":22791,"ĠTut":22792,"irrel":22793,"Ġschizophrenia":22794,"Ġmisogyn":22795,"healthy":22796,"Iron":22797,"Ġreacted":22798,"-$":22799,"252":22800,"Ġplural":22801,"Ġplum":22802,"Ġbargain":22803,"Ġgrounded":22804,"finder":22805,"Ġdisse":22806,"ĠLaz":22807,"OOD":22808,"Ġatroc":22809,"Factory":22810,"Ġminions":22811,"Ġori":22812,"ĠBrave":22813,"ĠPRE":22814,"ĠMyanmar":22815,"ĠHod":22816,"Ġexpedition":22817,"Ġexplode":22818,"ĠCoord":22819,"Ġextr":22820,"ĠBrief":22821,"ĠADHD":22822,"Ġhardcore":22823,"feeding":22824,"Ġdile":22825,"ĠFruit":22826,"Ġvaccination":22827,"ĠMao":22828,"osphere":22829,"Ġcontests":22830,"-|":22831,"Ġfren":22832,"isphere":22833,"Rom":22834,"ĠSharp":22835,"ĠTrend":22836,"Ġdisconnect":22837,"âĢ¢âĢ¢":22838,"Ġpersecution":22839,"Earth":22840,"Ġhealthier":22841,"384":22842,"Ġcob":22843,"ĠTrinity":22844,"OWS":22845,"ANN":22846,"Ġspecialty":22847,"Ġgru":22848,"Ġcooperative":22849,"why":22850,"Starting":22851,"ĠIssues":22852,"stre":22853,"ensor":22854,"Ġ185":22855,"Adv":22856,"!?":22857,"ĠRevel":22858,"emia":22859,"ĠHulk":22860,"Ġcelebrations":22861,"ĠSou":22862,"raud":22863,"ĠKlein":22864,"Ġunreal":22865,"context":22866,"Ġpartnerships":22867,"Ġadopting":22868,"tical":22869,"Ġsplash":22870,"ĠHezbollah":22871,"category":22872,"cyclop":22873,"xton":22874,"ĠDot":22875,"urdy":22876,"tz":22877,"Ġenvelope":22878,"ĠNL":22879,"âķ":22880,"Ġwherein":22881,"Spec":22882,"184":22883,"Ġtelev":22884,"aliation":22885,"Ġmyths":22886,"å°":22887,"Ġrigorous":22888,"Ġcommunicating":22889,"Ġobserver":22890,"Ġrehe":22891,"ĠWash":22892,"Ġapologized":22893,"ĠTin":22894,"Ġexpenditures":22895,"workers":22896,"document":22897,"Ġhesitate":22898,"ĠLenin":22899,"Ġunpredictable":22900,"Ġrenewal":22901,"cler":22902,"okia":22903,"ĠCONT":22904,"Ġpostseason":22905,"Tokens":22906,"Ġexacerb":22907,"Ġbetting":22908,"Ġ147":22909,"Ġelevation":22910,"Wood":22911,"ĠSolomon":22912,"194":22913,"004":22914,"output":22915,"Ġredund":22916,"ĠMumbai":22917,"ĠpH":22918,"Ġreproduce":22919,"ĠDuration":22920,"MAX":22921,"Ġbog":22922,"CBS":22923,"ĠBalance":22924,"ĠSgt":22925,"ĠRecent":22926,"Ġcd":22927,"Ġpopped":22928,"Ġincompet":22929,"prop":22930,"ayan":22931,"guy":22932,"Pacific":22933,"Ġtyr":22934,"Ġ{{":22935,"ĠMystic":22936,"ĠDana":22937,"Ġmasturb":22938,"Ġgeometry":22939,"â":22940,"ĠCorrect":22941,"Ġtrajectory":22942,"Ġdistracted":22943,"Ġfoo":22944,"ĠWelsh":22945,"Luc":22946,"mith":22947,"Ġrugby":22948,"Ġrespiratory":22949,"Ġtriangle":22950,"Ġ215":22951,"Ġundergraduate":22952,"ĠSuperior":22953,"changing":22954,"_-":22955,"Ġrightly":22956,"Ġreferee":22957,"Ġlucrative":22958,"Ġunauthorized":22959,"Ġresembles":22960,"ĠGNU":22961,"ĠDerby":22962,"Ġpathways":22963,"ĠLed":22964,"Ġendurance":22965,"Ġstint":22966,"Ġcollector":22967,"Fast":22968,"Ġdots":22969,"Ġnationals":22970,"ĠSecurities":22971,"Ġwhip":22972,"Param":22973,"Ġlearns":22974,"Magic":22975,"Ġdetailing":22976,"moon":22977,"Ġbroadcasting":22978,"Ġbaked":22979,"265":22980,"holm":22981,"ĠSah":22982,"ĠHussein":22983,"ĠCourtesy":22984,"174":22985,"Ġ146":22986,"Ġgeographic":22987,"peace":22988,"Ġjudging":22989,"ĠStern":22990,"Bur":22991,"Ġstoryline":22992,"Gun":22993,"ĠStick":22994,"245":22995,"307":22996,"ãĤ´ãĥ³":22997,"ĠAdministrator":22998,"Ġburnt":22999,"Ġpave":23000,"choes":23001,"Exec":23002,"Ġcampuses":23003,"Result":23004,"Ġmutations":23005,"ĠCharter":23006,"Ġcaptures":23007,"Ġcompares":23008,"Ġbadge":23009,"Scient":23010,"Ġerad":23011,"iery":23012,"oi":23013,"ettes":23014,"ĠEstate":23015,"Ġstrap":23016,"Ġproudly":23017,"Ġfried":23018,"Ġwithdrawn":23019,"ĠVoy":23020,"phony":23021,"Items":23022,"ĠPierce":23023,"bard":23024,"Ġannotation":23025,"anton":23026,"illon":23027,"Impro":23028,"...)":23029,"Ġhappier":23030,"------":23031,"adjust":23032,"Ġstaffers":23033,"Ġactivism":23034,"Ġperf":23035,"Ġalright":23036,"Need":23037,"Ġcommence":23038,"Ġopioid":23039,"ĠAmanda":23040,"Es":23041,"ĠPars":23042,"ĠKaw":23043,"Works":23044,"248":23045,"Ġindo":23046,"tc":23047,"endant":23048,"ĠMoto":23049,"Ġlegalization":23050,"OTE":23051,"Ġtasked":23052,"Ġtsp":23053,"ĠACTIONS":23054,"166":23055,"Ġrefreshing":23056,"ĠNR":23057,"ĠPerez":23058,"Ġinfringement":23059,"SY":23060,"Listen":23061,"inning":23062,"ku":23063,"Ġrotate":23064,"program":23065,"arah":23066,"Design":23067,"Ġ(£":23068,"Ġstoring":23069,"Ġwarrants":23070,"Ġjudgement":23071,"ĠBrist":23072,"usually":23073,"photo":23074,"ĠRan":23075,"ĠPine":23076,"Ġoutrageous":23077,"ĠValentine":23078,"luence":23079,"ĠEverybody":23080,"Altern":23081,"Ġrelevance":23082,"Ġterminated":23083,"Ġdessert":23084,"Ġfulfilled":23085,"Ġprosecuted":23086,"ĠWords":23087,"Ġmigrant":23088,"Ġcultivation":23089,"ÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤ":23090,"idelity":23091,"ĠVern":23092,"ĠLogin":23093,"Ġmetaphor":23094,"ĠTip":23095,"Ġrecruits":23096,"ĠPig":23097,"ribing":23098,"Ġenthusiasts":23099,"exper":23100,"Ġfrightening":23101,"ĠHair":23102,"anson":23103,"strate":23104,"Ġhi":23105,"Height":23106,"Ġowning":23107,"none":23108,"Ġdislike":23109,"Ġknives":23110,"pherd":23111,"Ġloudly":23112,"ĠAPIs":23113,"Display":23114,"ĠLac":23115,"ĠUSS":23116,"abl":23117,"verages":23118,"Jew":23119,"Ġ172":23120,"ĠHistorical":23121,"atoon":23122,"ĠPhysics":23123,"intern":23124,"Ġwarmth":23125,"Ġtopp":23126,"DM":23127,"Ġgunman":23128,"Ġemperor":23129,"odi":23130,"ãĥ£":23131,"inatory":23132,"ĠRib":23133,"Ġ131":23134,"ĠSaturn":23135,"ĠShining":23136,"Ġwaking":23137,"Quotes":23138,"Ġcomedian":23139,"enberg":23140,"½":23141,"Ġbelievers":23142,"Ġpaperwork":23143,"custom":23144,"Ġlev":23145,"Ġlament":23146,"Ġpouring":23147,"222":23148,"political":23149,"ĠSupplement":23150,"maid":23151,"Ġcruelty":23152,"Ġtread":23153,"ysics":23154,"Aw":23155,"rites":23156,"Ġmodifier":23157,"ĠPosition":23158,"Adam":23159,"lb":23160,"ubs":23161,"Ġimperfect":23162,"Ġclusters":23163,"ĠEngineer":23164,"ĠCherry":23165,"Ġinauguration":23166,"ĠSau":23167,"Ġembodiment":23168,"ĠUncle":23169,"Ġoverr":23170,"Ġexplosions":23171,"cule":23172,"ĠPrinceton":23173,"ĠAndrea":23174,"Ġincorrectly":23175,"Ġearnest":23176,"Ġpilgr":23177,"ĠSprint":23178,"Ġsleeve":23179,"Ġhears":23180,"ĠAmazing":23181,"Ġbrowsing":23182,"agin":23183,"Ġhomeland":23184,"Ġhaw":23185,"Ġdiving":23186,"istered":23187,"178":23188,"Ġbargaining":23189,"ĠArcade":23190,"Ġdelegate":23191,"terson":23192,"................................................................":23193,"ĠJacksonville":23194,"275":23195,"Ġstagn":23196,"Ġadam":23197,"ĠSherman":23198,"CB":23199,"Ġsuburb":23200,"ĠFoods":23201,"Ġconverting":23202,"ĠArist":23203,"Ġchambers":23204,"love":23205,"Ġamino":23206,"ĠGan":23207,"Ġmadness":23208,"mc":23209,"ĠUSE":23210,"defined":23211,"Ġultr":23212,"indust":23213,"Ġwolves":23214,"lance":23215,"Additionally":23216,"Ġcracks":23217,"asia":23218,"ĠReason":23219,"ĠPump":23220,"Ġaccidental":23221,"ĠLaser":23222,"ĠRid":23223,"Ġinitialized":23224,"elli":23225,"Ġunnamed":23226,"Ġnoun":23227,"ĠPassed":23228,"Ġhostage":23229,"ĠEthiop":23230,"shirts":23231,"Ġunrel":23232,"ĠEmbassy":23233,"Ġ1941":23234,"Ġatoms":23235,"Ġpurported":23236,"164":23237,"ĠFi":23238,"Ġgallons":23239,"ĠMonica":23240,"Ġpg":23241,"enment":23242,"Ġsorted":23243,"ĠGospel":23244,"Ġheights":23245,"Ġtraced":23246,"Ġundergoing":23247,"Shell":23248,"Ġsacks":23249,"Ġproportions":23250,"Ġhalluc":23251,"Font":23252,"acet":23253,"Ġwarmer":23254,"ĠINTER":23255,"Ġgrabbing":23256,"Plug":23257,"Ġrealization":23258,"ĠBurke":23259,"Ġenchant":23260,"ATER":23261,"ĠSeed":23262,"Ġabundant":23263,"FM":23264,"Ġcivic":23265,"Vs":23266,"isi":23267,"Ġvow":23268,"Ġreper":23269,"ĠPartnership":23270,"Ġpenetration":23271,"Ġaxe":23272,"Ġshattered":23273,"ĠZombies":23274,"Ġvinyl":23275,"ĠAlert":23276,"eon":23277,"Ġobliged":23278,"ĠIllust":23279,"ĠPlaza":23280,"ĠFrontier":23281,"Ġdavidjl":23282,"ĠSerial":23283,"ĠHav":23284,"ĠNutrition":23285,"Bi":23286,"ĠâĸĪ":23287,"ĠJays":23288,"linux":23289,"Ġhurry":23290,"Ġvoy":23291,"Ġhopeless":23292,"ĠStealth":23293,"Ġãģ":23294,"essors":23295,"ttle":23296,"borg":23297,"ĠSafari":23298,"fell":23299,"Ġwary":23300,"due":23301,"ĠAbove":23302,"Ha":23303,"ELL":23304,"Ġnotor":23305,"ĠWon":23306,"Too":23307,"Ġoccupations":23308,"Ġpossessions":23309,"Ġinviting":23310,"Ġpredators":23311,"Ġaccelerated":23312,"Ġ157":23313,"uterte":23314,"ĠCube":23315,"east":23316,"account":23317,"Give":23318,"Ġtransplant":23319,"redients":23320,"idable":23321,"Ġscreenshots":23322,"ĠGund":23323,"ĠFS":23324,"Ġtravelers":23325,"Ġsensory":23326,"ĠFiat":23327,"ĠRockets":23328,"İĭ":23329,"_{":23330,"Friend":23331,"Ġcharming":23332,"ALS":23333,"Ġenjoyment":23334,"mph":23335,"Ġ5000":23336,"ĠREG":23337,"ÙĨ":23338,"bia":23339,"Ġcompilation":23340,"rost":23341,"ĠVP":23342,"ĠSchne":23343,"2019":23344,"Ġcopying":23345,"MORE":23346,"ĠFlore":23347,"falls":23348,"215":23349,"total":23350,"Ġdisciples":23351,"double":23352,"Ġexceeding":23353,"Ġsmashed":23354,"Ġconceptual":23355,"ĠRomania":23356,"ĠBrent":23357,"ĠICE":23358,"ĠTou":23359,"Ġgrap":23360,"Ġnails":23361,"189":23362,"ãĥĺ":23363,"Ġprocure":23364,"eur":23365,"Ġconfirming":23366,"ĠCec":23367,"awi":23368,"ĠEden":23369,"Ġng":23370,"Ġengineered":23371,"atics":23372,"Ġhooked":23373,"Ġdisgusting":23374,"ĠMurder":23375,"ãĤ¿":23376,"Library":23377,"Ġ168":23378,"Almost":23379,"hematic":23380,"Menu":23381,"ĠNotre":23382,"ĠJur":23383,"Ġkidnapped":23384,"Ġhacker":23385,"ĠJade":23386,"Ġcreepy":23387,"Ġdrawings":23388,"ĠSponsor":23389,"Ġcyclists":23390,"ĠGoblin":23391,"Ġoptimized":23392,"Ġstaged":23393,"ĠMcD":23394,"between":23395,"Age":23396,"eno":23397,"Sex":23398,"ĠWide":23399,"nings":23400,"avis":23401,"Ġincapable":23402,"ĠKob":23403,"Ġrewarding":23404,"ĠLone":23405,"olescent":23406,"Ġcontracted":23407,"Ġsticky":23408,"Jose":23409,"Ball":23410,"fest":23411,"ĠInput":23412,"ĠRecently":23413,"Ġtomat":23414,"square":23415,"Application":23416,"Ġnitrogen":23417,"Ġduplicate":23418,"ĠRecon":23419,"ĠDear":23420,"London":23421,"Ġintra":23422,"Ġdock":23423,"Ġoutreach":23424,"ĠMillion":23425,"Ġmammals":23426,"ampton":23427,"VAL":23428,"Ġsnaps":23429,"Ġdos":23430,"ĠWhole":23431,"ĠReady":23432,"Try":23433,"ĠWinnipeg":23434,"earance":23435,"Ġincurred":23436,"renched":23437,"ĠNSW":23438,"ilot":23439,"raine":23440,"Ġcube":23441,"got":23442,"Ġrunway":23443,"etermined":23444,"ĠHawks":23445,"Ġsurvivor":23446,"ĠWish":23447,"ĠDin":23448,"ĠDEF":23449,"ĠVault":23450,"187":23451,"Ġmushrooms":23452,"Ġcrisp":23453,"bey":23454,"ĠDiscovery":23455,"Ġdevelopmental":23456,"Ġparadigm":23457,"Ġchaotic":23458,"ĠTsu":23459,"Ġ333":23460,"bons":23461,"Ġbacterial":23462,"Ġcommits":23463,"Ġcosmic":23464,"Ġmega":23465,"ocative":23466,"ĠPaint":23467,"ophobic":23468,"Ġvain":23469,"Ġcarved":23470,"ĠThief":23471,"ĠGul":23472,"owship":23473,"Ġcites":23474,"ĠEdinburgh":23475,"Ġdiminished":23476,"Ġacknowledges":23477,"ĠKills":23478,"Ġmicrow":23479,"ĠHera":23480,"Ġseniors":23481,"Ġwhereby":23482,"Hop":23483,"atron":23484,"Ġunavailable":23485,"ĠNate":23486,"Ġ480":23487,"Ġslated":23488,"ĠRebecca":23489,"ĠBattery":23490,"Ġgrammar":23491,"Ġheadset":23492,"Ġcursor":23493,"Ġexcluding":23494,"anye":23495,"aundering":23496,"ebin":23497,"Ġfeasible":23498,"ĠPublishing":23499,"ĠLabs":23500,"ĠCliff":23501,"ĠFerrari":23502,"Ġpac":23503,"visible":23504,"marked":23505,"pell":23506,"Ġpolite":23507,"Ġstaggering":23508,"ĠGalactic":23509,"Ġsuperst":23510,"Ġparan":23511,"ĠOfficers":23512,"ãĢģ":23513,"Ġspecifics":23514,"ulus":23515,"239":23516,"ĠPaste":23517,"AMP":23518,"ĠPanama":23519,"ĠDelete":23520,"anguard":23521,"restrial":23522,"Ġheroic":23523,"ĠDy":23524,"اÙĦ":23525,"Ġincumbent":23526,"Ġcrunch":23527,"tro":23528,"Ġscoop":23529,"Ġblogger":23530,"Ġsellers":23531,"uren":23532,"Ġmedicines":23533,"ĠCaps":23534,"ĠAnimation":23535,"oxy":23536,"Ġoutward":23537,"Ġinquiries":23538,"229":23539,"Ġpsychologist":23540,"ĠSask":23541,"evil":23542,"Ġcontaminated":23543,"ãĤ¨":23544,"herence":23545,"Ġbranded":23546,"ĠAbdul":23547,"zh":23548,"Ġparagraphs":23549,"Ġmins":23550,"Ġcorrelated":23551,"erb":23552,"Ġimpart":23553,"Ġmilestone":23554,"ĠSolutions":23555,"otle":23556,"Ġundercover":23557,"Ġmarched":23558,"ĠChargers":23559,"fax":23560,"ĠSecrets":23561,"Ġruth":23562,"weather":23563,"Ġfeminine":23564,"Ġsham":23565,"Ġprestigious":23566,"iggins":23567,"Ġsung":23568,"history":23569,"ettle":23570,"ggie":23571,"Ġoutdated":23572,"oland":23573,"Ġperceptions":23574,"ĠSession":23575,"ĠDodgers":23576,"uj":23577,"ĠEND":23578,"Doc":23579,"Ġdeficiency":23580,"Grand":23581,"ĠJoker":23582,"Ġretrospect":23583,"Ġdiagnostic":23584,"Ġharmless":23585,"Ġrogue":23586,"ĠAval":23587,"Equ":23588,"Ġtransc":23589,"ĠRobertson":23590,"ĠDepending":23591,"ĠBurns":23592,"ivo":23593,"Ġhostility":23594,"Features":23595,"ĵĺ":23596,"Ġdiscomfort":23597,"ĠLCD":23598,"specified":23599,"ĠExpect":23600,"340":23601,"Ġimperative":23602,"ĠRegular":23603,"Chinese":23604,"Ġstatewide":23605,"Ġsymm":23606,"Ġloops":23607,"Ġautumn":23608,"Nick":23609,"Ġshaping":23610,"Ġquot":23611,"Ġcherry":23612,"ĠCrossref":23613,"è¦ļéĨĴ":23614,"Standard":23615,"heed":23616,"ĠDell":23617,"ĠVietnamese":23618,"Ġost":23619,"ĠValkyrie":23620,"OA":23621,"Assad":23622,"Ġrebound":23623,"ĠTraffic":23624,"places":23625,"æĺ":23626,"ĠBuc":23627,"172":23628,"Ġshelters":23629,"Ġinsisting":23630,"ĠCertainly":23631,"ĠKenneth":23632,"ĠTCP":23633,"Ġpenal":23634,"ĠReplay":23635,"heard":23636,"Ġdialect":23637,"iza":23638,"ĠFY":23639,"itcher":23640,"ĠDL":23641,"Ġspiral":23642,"Ġquarterbacks":23643,"Ġhull":23644,"Ġgoogle":23645,"Ġtodd":23646,"ĠSterling":23647,"ĠPlate":23648,"Ġspying":23649,"mbol":23650,"ĠRealm":23651,"ĠProced":23652,"ĠCrash":23653,"Ġterminate":23654,"Ġprotesting":23655,"Center":23656,"guided":23657,"Ġuncover":23658,"Ġboycott":23659,"Ġrealizes":23660,"sound":23661,"Ġpretending":23662,"ĠVas":23663,"1980":23664,"Ġframed":23665,"Ġ139":23666,"Ġdescended":23667,"Ġrehabilitation":23668,"Ġborrowing":23669,"ĠBuch":23670,"Ġblur":23671,"Ron":23672,"ĠFrozen":23673,"enza":23674,"Chief":23675,"ĠPoor":23676,"Ġtranslates":23677,"MIN":23678,"Ġ212":23679,"JECT":23680,"Ġerupted":23681,"Ġsuccesses":23682,"SEC":23683,"Ġplague":23684,"Ġgems":23685,"doms":23686,"Ġstretches":23687,"ĠSpy":23688,"Ġstorytelling":23689,"Credit":23690,"ĠPush":23691,"Ġtraction":23692,"Ġineffective":23693,"ĠLuna":23694,"Ġtapes":23695,"Ġanalytics":23696,"ercise":23697,"Ġprogrammes":23698,"ĠCarbon":23699,"Ġbehold":23700,"heavy":23701,"ĠConservation":23702,"ĠFIR":23703,"Ġsack":23704,"termin":23705,"ricks":23706,"Ġhoused":23707,"Ġunusually":23708,"Ice":23709,"Ġexecuting":23710,"ĠMoroc":23711,"eday":23712,"Ġeditions":23713,"Ġsmarter":23714,"ĠBA":23715,"Ġoutlaw":23716,"Ġvanished":23717,"iba":23718,"ALSE":23719,"ĠSilva":23720,"238":23721,"Could":23722,"Ġphilosopher":23723,"Ġevacuated":23724,"Secret":23725,"142":23726,"Ġvisas":23727,"ãĤ¬":23728,"ĠMalt":23729,"ĠClearly":23730,"ĠNiger":23731,"ĠCairo":23732,"ĠFist":23733,"380":23734,"ĠXML":23735,"auto":23736,"itant":23737,"Ġreinforced":23738,"Record":23739,"ĠSurvivor":23740,"GHz":23741,"Ġscrews":23742,"parents":23743,"Ġoceans":23744,"mares":23745,"Ġbrakes":23746,"vasive":23747,"Ġhello":23748,"ĠSIM":23749,"rimp":23750,"Ġore":23751,"ĠArmour":23752,"247":23753,"Ġterrific":23754,"Ġtones":23755,"141":23756,"ĠMinutes":23757,"Episode":23758,"Ġcurves":23759,"Ġinflammatory":23760,"Ġbatting":23761,"ĠBeautiful":23762,"Lay":23763,"Ġunpop":23764,"vable":23765,"Ġriots":23766,"ĠTactics":23767,"baugh":23768,"ĠCock":23769,"Ġorgasm":23770,"ĠSas":23771,"Ġconstructor":23772,"etz":23773,"Gov":23774,"Ġantagon":23775,"Ġtheat":23776,"Ġdeeds":23777,"hao":23778,"cuts":23779,"ĠMcCl":23780,"Ġum":23781,"ĠScientists":23782,"Ġgrassroots":23783,"yssey":23784,"\"]=>":23785,"Ġsurfaced":23786,"Ġshades":23787,"Ġneighbours":23788,"Ġadvertis":23789,"oya":23790,"Ġmerged":23791,"Upon":23792,"Ġgad":23793,"Ġanticipate":23794,"Anyway":23795,"Ġslogan":23796,"Ġdisrespect":23797,"Iran":23798,"ĠTB":23799,"acted":23800,"Ġsubpoen":23801,"mediately":23802,"OOOO":23803,"Ġwaiver":23804,"Ġvulnerabilities":23805,"ottesville":23806,"ĠHuffington":23807,"Josh":23808,"ĠDH":23809,"Monday":23810,"ĠEllen":23811,"Know":23812,"xon":23813,"items":23814,"228":23815,"Ġfills":23816,"ĠNike":23817,"Ġcumulative":23818,"andals":23819,"Ir":23820,"Ġì":23821,"Ġfriction":23822,"igator":23823,"Ġscans":23824,"ĠVienna":23825,"ldom":23826,"Ġperformers":23827,"Prim":23828,"Ġbidding":23829,"Mur":23830,"Ġleaned":23831,"ĠPrix":23832,"alks":23833,"Ġ[âĢ¦]":23834,"ĠTwitch":23835,"ĠDeveloper":23836,"ĠGir":23837,"Ġcallback":23838,"Abstract":23839,"Ġaccustomed":23840,"Ġfreedoms":23841,"ĠPG":23842,"uracy":23843,"Ġlump":23844,"isman":23845,",,,,":23846,"1992":23847,"ĠRED":23848,"Ġworm":23849,"Match":23850,"ĠPlatinum":23851,"IJ":23852,"ĠOwner":23853,"Trivia":23854,"compl":23855,"Ġnewborn":23856,"Ġfantas":23857,"Own":23858,"Ġ1959":23859,"Ġsympath":23860,"Ġubiqu":23861,"Ġoutputs":23862,"Ġallev":23863,"Ġprag":23864,"Kevin":23865,"Ġfavors":23866,"Ġburial":23867,"Ġnurt":23868,"solete":23869,"cache":23870,"Ġ156":23871,"Ġunlocks":23872,"techn":23873,"Making":23874,"Ġconquer":23875,"adic":23876,"æĸ":23877,"Ġelf":23878,"Ġelectorate":23879,"ĠKurds":23880,"ĠStack":23881,"ĠSamurai":23882,"Ġâĺħ":23883,"Ġ{}":23884,"ĠSaid":23885,"ĠFallout":23886,"Ġkindness":23887,"ĠCustoms":23888,"ĠBoulevard":23889,"Ġhelicopters":23890,"otics":23891,"ĠVeget":23892,"comment":23893,"Ġcriticised":23894,"Ġpolished":23895,"ĠRemix":23896,"ĠCultural":23897,"Ġrecons":23898,"Ġdoi":23899,"atem":23900,"Screen":23901,"Ġbarred":23902,"Comments":23903,"ĠGenerally":23904,"Ġslap":23905,"720":23906,"Vari":23907,"pine":23908,"Ġempt":23909,"Ġhats":23910,"ĠPlaying":23911,"lab":23912,"average":23913,"forms":23914,"ĠCotton":23915,"Ġcans":23916,"ĠDON":23917,"ĠSomalia":23918,"Crypt":23919,"ĠIncreases":23920,"Ever":23921,"modern":23922,"Ġsurgeon":23923,"3000":23924,"Ġrandomized":23925,"================================================================":23926,"Bern":23927,"impl":23928,"ĠCOR":23929,"Ġproclaim":23930,"thouse":23931,"Ġtoes":23932,"Ġample":23933,"Ġpreserving":23934,"Ġdisbel":23935,"grand":23936,"Besides":23937,"Ġsilk":23938,"ĠPattern":23939,"hm":23940,"Ġenterprises":23941,"Ġaffidavit":23942,"ĠAdvisory":23943,"Ġadvertised":23944,"ĠReligious":23945,"sections":23946,"psych":23947,"ĠFields":23948,"aways":23949,"Ġhashtag":23950,"ĠNightmare":23951,"Ġvampire":23952,"Ġforensic":23953,"rossover":23954,"nar":23955,"Ġnavy":23956,"Ġvacant":23957,"ĠDuel":23958,"Ġhallway":23959,"Ġfacebook":23960,"identally":23961,"ĠNRA":23962,"Ġmatt":23963,"Ġhurricane":23964,"ĠKirby":23965,"ĠPuzzle":23966,"Ġskirt":23967,"oust":23968,"dullah":23969,"Ġanalogy":23970,"inion":23971,"Ġtomatoes":23972,"ĠNV":23973,"ĠPeak":23974,"ĠMeyer":23975,"Ġappointments":23976,"Ġmasc":23977,"Ġalley":23978,"rehend":23979,"Ġcharities":23980,"Ġundo":23981,"Ġdestinations":23982,"ĠTesting":23983,"\">\"":24618,"cats":24619,"*.":24620,"Ġgestures":24621,"general":24622,"League":24623,"Ġpackets":24624,"ĠInspector":24625,"ĠBerg":24626,"Ġfraudulent":24627,"Ġcriticize":24628,"Fun":24629,"Ġblaming":24630,"ndra":24631,"Ġslash":24632,"ĠEston":24633,"Ġproposing":24634,"Ġwhales":24635,"Ġtherapist":24636,"Ġsubset":24637,"Ġleisure":24638,"ELD":24639,"ĠCVE":24640,"ĠActivity":24641,"Ġculmin":24642,"shop":24643,"ĠDAY":24644,"ischer":24645,"ĠAdmiral":24646,"ĠAttacks":24647,"Ġ1958":24648,"Ġmemoir":24649,"Ġfolded":24650,"Ġsexist":24651,"Ġ153":24652,"ĠLI":24653,"Ġreadings":24654,"Ġembarrassment":24655,"ĠEmployment":24656,"wart":24657,"chin":24658,"Ġcontinuation":24659,"lia":24660,"Recently":24661,"Ġduel":24662,"Ġevacuation":24663,"ĠKashmir":24664,"Ġdisposition":24665,"ĠRig":24666,"Ġbolts":24667,"Ġinsurers":24668,"467":24669,"Mex":24670,"Ġretaliation":24671,"Ġmisery":24672,"Ġunreasonable":24673,"raining":24674,"Imm":24675,"ĠPU":24676,"emer":24677,"Ġgenital":24678,"ãĤ³":24679,"ĠCandy":24680,"Ġonions":24681,"ĠPatt":24682,"liner":24683,"Ġconceded":24684,"Ġfa":24685,"Ġforc":24686,"ĠHernandez":24687,"ĠGeoff":24688,"debian":24689,"ĠTeams":24690,"Ġcries":24691,"Ġhomeowners":24692,"237":24693,"ABC":24694,"Ġstitch":24695,"Ġstatistic":24696,"Ġheaders":24697,"ĠBiology":24698,"Ġmotors":24699,"ĠGEN":24700,"ĠLip":24701,"Ġhates":24702,"Ġheel":24703,"Self":24704,"ipl":24705,"EDIT":24706,"orting":24707,"Ġannot":24708,"ĠSpeech":24709,"oldemort":24710,"ĠJavascript":24711,"ĠLeBron":24712,"Ġfootprint":24713,"Ġfn":24714,"Ġseizures":24715,"nas":24716,"hide":24717,"Ġ1954":24718,"ĠBee":24719,"ĠDeclaration":24720,"ĠKatie":24721,"Ġreservations":24722,"NR":24723,"female":24724,"Ġsaturated":24725,"Ġbiblical":24726,"Ġtrolls":24727,"Device":24728,"photos":24729,"Ġdrums":24730,"ãĥīãĥ©ãĤ´ãĥ³":24731,"Night":24732,"fighter":24733,"ĠHak":24734,"riber":24735,"Ġcush":24736,"Ġdisciplinary":24737,"baum":24738,"ĠGH":24739,"ĠSchmidt":24740,"ilibrium":24741,"Ġsixty":24742,"ĠKushner":24743,"rots":24744,"Ġpund":24745,"ĠRac":24746,"Ġsprings":24747,"Ġconve":24748,"Business":24749,"Fall":24750,"Ġqualifications":24751,"Ġverses":24752,"Ġnarciss":24753,"ĠKoh":24754,"ĠWow":24755,"ĠCharlottesville":24756,"edo":24757,"Ġinterrogation":24758,"ĠWool":24759,"365":24760,"Brian":24761,"Ġâľĵ":24762,"Ġalleges":24763,"onds":24764,"idation":24765,"ĠJackie":24766,"yu":24767,"Ġlakes":24768,"Ġworthwhile":24769,"Ġcrystals":24770,"ĠJuda":24771,"Ġcomprehend":24772,"Ġflush":24773,"Ġabsorption":24774,"ĠOC":24775,"Ġfrightened":24776,"ĠChocolate":24777,"Martin":24778,"Ġbuys":24779,"Ġbucks":24780,"Ġappell":24781,"ĠChampionships":24782,"Ġlistener":24783,"ĠDefensive":24784,"Ġcz":24785,"uds":24786,"ĠMate":24787,"Ġreplay":24788,"Ġdecorated":24789,"Ġsunk":24790,"ĠVIP":24791,"ĠAnk":24792,"Ġ195":24793,"aaaa":24794,"Nobody":24795,"ĠMilk":24796,"ĠGur":24797,"ĠMk":24798,"ĠSara":24799,"Ġseating":24800,"ĠWid":24801,"Track":24802,"Ġemploys":24803,"Ġgigantic":24804,"APP":24805,"ãĤ§":24806,"inventory":24807,"Ġtowel":24808,"atche":24809,"lasting":24810,"ĠTL":24811,"Ġlatency":24812,"Ġkne":24813,"Ber":24814,"meaning":24815,"Ġupheld":24816,"Ġplayground":24817,"Ġmant":24818,"Side":24819,"Ġstereo":24820,"Ġnorthwest":24821,"Ġexceptionally":24822,"Ġrays":24823,"Ġrecurring":24824,"Drive":24825,"Ġupright":24826,"Ġabduct":24827,"ĠMarathon":24828,"Ġgoodbye":24829,"Ġalphabet":24830,"hp":24831,"Ġcourtroom":24832,"rington":24833,"othing":24834,"Tag":24835,"Ġdiplomats":24836,"Ġbarbar":24837,"ĠAqua":24838,"183":24839,"3333":24840,"Ġmaturity":24841,"Ġinstability":24842,"ĠApache":24843,"Ġ===":24844,"Ġfasting":24845,"ĠGrid":24846,"ModLoader":24847,"Ġ152":24848,"Abs":24849,"ĠOperating":24850,"etti":24851,"Ġacquaint":24852,"Donnell":24853,"ĠKem":24854,"ĠForge":24855,"Ġarmored":24856,"Mil":24857,"Ġphilosophers":24858,"invest":24859,"Players":24860,"âĪ":24861,"Ġmyriad":24862,"Ġcomrades":24863,"Rot":24864,"Ġremembering":24865,"Ġcorresponds":24866,"Ġprogrammers":24867,"ĠLynn":24868,"Ġolig":24869,"Ġcoherent":24870,"ynchron":24871,"ĠChemical":24872,"Ġjugg":24873,"pair":24874,"posts":24875,"Eye":24876,"ĠInner":24877,"Ġsemester":24878,"ottest":24879,"ĠEmirates":24880,"ricanes":24881,"orously":24882,"mits":24883,"ĠWis":24884,"Ġdodge":24885,"location":24886,"Ġfaded":24887,"Amazon":24888,"ĠProceed":24889,"ĠINFO":24890,"journal":24891,"ĠTruck":24892,"Ten":24893,"Ġ217":24894,"Ġstatutes":24895,"mobile":24896,"ĠTypes":24897,"Recomm":24898,"buster":24899,"pex":24900,"Ġlegends":24901,"Ġheadache":24902,"faced":24903,"ĠWiFi":24904,"ifty":24905,"ĠHER":24906,"Ġcircuits":24907,"ERROR":24908,"226":24909,"olin":24910,"Ġcylinder":24911,"ospace":24912,"ikers":24913,"Prem":24914,"Quant":24915,"Ġconflicting":24916,"Ġslightest":24917,"Ġforged":24918,"ionage":24919,"Stephen":24920,"ĠKub":24921,"ĠOpportun":24922,"ĠHeal":24923,"Ġblo":24924,"Ġrulers":24925,"Ġhuh":24926,"Ġsubmarine":24927,"fy":24928,"asser":24929,"Ġallowance":24930,"ĠKasich":24931,"ĠTas":24932,"ĠAustralians":24933,"ForgeModLoader":24934,"ĠâĨij":24935,"ĠMatrix":24936,"amins":24937,"Ġ1200":24938,"ĠAcqu":24939,"236":24940,"Document":24941,"ĠBreaking":24942,"193":24943,"ĠSubst":24944,"ĠRoller":24945,"ĠProperties":24946,"ĠNI":24947,"tier":24948,"Ġcrushing":24949,"Ġadvocating":24950,"Furthermore":24951,"keepers":24952,"Ġsexism":24953,"xd":24954,"Ġcaller":24955,"ĠSense":24956,"chieve":24957,"ĠTF":24958,"Ġfueled":24959,"Ġreminiscent":24960,"Ġobsess":24961,"urst":24962,"Ġuphold":24963,"ĠFans":24964,"hetics":24965,"ĠâĹ":24966,"ĠBath":24967,"Ġbeverage":24968,"Ġoscill":24969,"254":24970,"Ġpoles":24971,"Ġgradual":24972,"Ġexting":24973,"ĠSuff":24974,"ĠSuddenly":24975,"Ġliking":24976,"Ġ1949":24977,"unciation":24978,"amination":24979,"ĠOmar":24980,"ĠLV":24981,"ĠConsequently":24982,"Ġsynthes":24983,"ĠGIF":24984,"Ġpains":24985,"Ġinteracting":24986,"uously":24987,"incre":24988,"Ġrumor":24989,"ĠScientology":24990,"197":24991,"ĠZig":24992,"Ġspelling":24993,"ĠASS":24994,"Ġextingu":24995,"mson":24996,"Ġgh":24997,"Ġremarked":24998,"ĠStrategic":24999,"ĠMON":25000,"å¥":25001,"gae":25002,"ĠWHAT":25003,"Eric":25004,"ĠCampus":25005,"Ġmethane":25006,"Ġimagin":25007,"JUST":25008,"ĠAlm":25009,"XT":25010,"iq":25011,"ĠRSS":25012,"Ġwrongdoing":25013,"atta":25014,"Ġbigot":25015,"Ġdemonstrators":25016,"ĠCalvin":25017,"ĠVilla":25018,"Ġmembrane":25019,"ĠAwesome":25020,"Ġbenefic":25021,"268":25022,"Ġmagnificent":25023,"ĠLots":25024,"Greg":25025,"ĠBoris":25026,"Ġdetainees":25027,"ĠHerman":25028,"Ġwhispered":25029,"Ġawe":25030,"Professor":25031,"funding":25032,"Ġphysiological":25033,"ĠDestruction":25034,"Ġlimb":25035,"Ġmanipulated":25036,"Ġbubbles":25037,"Ġpseud":25038,"Ġhydra":25039,"ĠBristol":25040,"Ġstellar":25041,"ĠExpansion":25042,"ĠKell":25043,"ĠInterestingly":25044,"Ġmans":25045,"Ġdragging":25046,"Ġecological":25047,"ĠFit":25048,"Ġgent":25049,"Ġbenefited":25050,"ĠHaiti":25051,"Ġpolyg":25052,"ãĥİ":25053,"Ġ2030":25054,"Ġprow":25055,"Ġreconstruction":25056,"Ġwast":25057,"Ġpsychic":25058,"ĠGreeks":25059,"Handler":25060,"162":25061,"ĠPulse":25062,"Ġsolicit":25063,"Ġsys":25064,"Ġinflux":25065,"ĠGentle":25066,"percent":25067,"Ġproliferation":25068,"Ġtaxable":25069,"Ġdisregard":25070,"Ġescaping":25071,"Ġginger":25072,"Ġwithstand":25073,"Ġdevastated":25074,"ĠDew":25075,"series":25076,"Ġinjected":25077,"elaide":25078,"Ġturnover":25079,"heat":25080,"ĻĤ":25081,"Happy":25082,"ĠSilent":25083,"ãĤŃ":25084,"ivism":25085,"Ġirrational":25086,"AMA":25087,"Ġreef":25088,"rub":25089,"Ġ162":25090,"Ġbankers":25091,"ĠEthics":25092,"vv":25093,"Ġcriticisms":25094,"Kn":25095,"186":25096,"Movie":25097,"ĠTories":25098,"Ġnood":25099,"Ġdistortion":25100,"False":25101,"odore":25102,"Ġtasty":25103,"Research":25104,"ĠUID":25105,"-)":25106,"Ġdivorced":25107,"ĠMU":25108,"ĠHayes":25109,"ĠIsn":25110,"iani":25111,"ĠHQ":25112,"Ġ\"#":25113,"ignant":25114,"Ġtraumatic":25115,"ĠLing":25116,"Hun":25117,"Ġsabot":25118,"online":25119,"random":25120,"Ġrenamed":25121,"rared":25122,"KA":25123,"dead":25124,"ét":25125,"ĠAssistance":25126,"Ġseaf":25127,"++++++++":25128,"Ġseldom":25129,"ĠWebb":25130,"Ġboolean":25131,"ulet":25132,"Ġrefrain":25133,"ĠDIY":25134,"rule":25135,"Ġshutting":25136,"Ġutilizing":25137,"loading":25138,"ĠParam":25139,"coal":25140,"ooter":25141,"Ġattracting":25142,"ĠDol":25143,"Ġhers":25144,"agnetic":25145,"ĠReach":25146,"imo":25147,"Ġdiscarded":25148,"ĠPip":25149,"015":25150,"ür":25151,"Ġmug":25152,"Imagine":25153,"COL":25154,"Ġcursed":25155,"ĠShows":25156,"ĠCurtis":25157,"ĠSachs":25158,"speaking":25159,"ĠVista":25160,"ĠFramework":25161,"ongo":25162,"Ġsubreddit":25163,"Ġcrus":25164,"ĠOval":25165,"Row":25166,"growing":25167,"Ġinstallment":25168,"Ġglac":25169,"ĠAdvance":25170,"ECK":25171,"ĠLGBTQ":25172,"LEY":25173,"Ġacet":25174,"Ġsuccessive":25175,"ĠNicole":25176,"Ġ1957":25177,"Quote":25178,"Ġcircumstance":25179,"ackets":25180,"Ġ142":25181,"ortium":25182,"Ġguessed":25183,"ĠFrame":25184,"Ġperpetrators":25185,"ĠAviation":25186,"ĠBench":25187,"Ġhandc":25188,"Ap":25189,"Ġ1956":25190,"259":25191,"rand":25192,"NetMessage":25193,"din":25194,"urtles":25195,"hig":25196,"ĠVIII":25197,"ffiti":25198,"ĠSwords":25199,"bial":25200,"Ġkidnapping":25201,"device":25202,"Ġbarn":25203,"ĠEli":25204,"aucas":25205,"Send":25206,"Constructed":25207,"Ġ½":25208,"Ġneedles":25209,"Ġadvertisements":25210,"Ġvou":25211,"Ġexhibited":25212,"ĠFortress":25213,"Ask":25214,"Berry":25215,"TYPE":25216,"Ġcancers":25217,"umping":25218,"ĠTerritory":25219,"Ġprud":25220,"Ġnas":25221,"Ġatheist":25222,"Ġbalances":25223,"ãģŁ":25224,"ĠShawn":25225,"&&":25226,"Ġlandsc":25227,"ĠRGB":25228,"Ġpetty":25229,"Ġexcellence":25230,"Ġtranslations":25231,"Ġparcel":25232,"ĠChev":25233,"East":25234,"ĠOutput":25235,"imi":25236,"Ġambient":25237,"ĠThreat":25238,"Ġvillains":25239,"Ġ550":25240,"ICA":25241,"Ġtaller":25242,"Ġleaking":25243,"cup":25244,"Ġpolish":25245,"Ġinfectious":25246,"ĠKC":25247,"Ġ@@":25248,"background":25249,"Ġbureaucracy":25250,"ĠSai":25251,"unless":25252,"itious":25253,"ĠSkype":25254,"Atl":25255,"IDENT":25256,"008":25257,"Ġhypocr":25258,"Ġpitchers":25259,"Ġguessing":25260,"ĠFINAL":25261,"Between":25262,"Ġvillagers":25263,"Ġ252":25264,"fashion":25265,"ĠTunis":25266,"Beh":25267,"ĠExc":25268,"ĠMID":25269,"288":25270,"ĠHaskell":25271,"196":25272,"ĠNOR":25273,"Ġspecs":25274,"Ġinvari":25275,"Ġglut":25276,"ĠCars":25277,"Ġimpulse":25278,"Ġhonors":25279,"gel":25280,"Ġjurisdictions":25281,"ĠBundle":25282,"ulas":25283,"California":25284,"ĠIncrease":25285,"Ġpear":25286,"Ġsingles":25287,"Ġcues":25288,"Ġunderwent":25289,"ĠWS":25290,"Ġexaggerated":25291,"Ġdubious":25292,"Ġflashing":25293,"LOG":25294,")].":25295,"Journal":25296,"tg":25297,"Van":25298,"ĠIstanbul":25299,"ĠInsp":25300,"ĠFranken":25301,"Draw":25302,"Ġsadness":25303,"Ġironic":25304,"ĠFry":25305,"xc":25306,"Ġ164":25307,"isch":25308,"Way":25309,"ĠProtestant":25310,"horn":25311,"Ġunaff":25312,"ĠViv":25313,"illas":25314,"ĠProductions":25315,"ĠHogan":25316,"Ġperimeter":25317,"ĠSisters":25318,"Ġspontaneous":25319,"Ġdownside":25320,"Ġdescendants":25321,"Ġorn":25322,"worm":25323,"Japanese":25324,"Ġ1955":25325,"Ġ151":25326,"ĠDoing":25327,"elsen":25328,"umbles":25329,"Ġradically":25330,"ĠDrum":25331,"ĠBach":25332,"Ġliabilities":25333,"ĠOB":25334,"ĠElementary":25335,"Ġmeme":25336,"ynes":25337,"Ġfingerprint":25338,"ĠGrab":25339,"Ġundertake":25340,"Members":25341,"ĠReader":25342,"ĠSims":25343,"god":25344,"Ġhypothetical":25345,"scient":25346,"ĠAJ":25347,"Ġcharism":25348,"Ġadmissions":25349,"ĠMissile":25350,"trade":25351,"Ġexercising":25352,"ĠBackground":25353,"Written":25354,"Ġvocals":25355,"whether":25356,"Ġvi":25357,"ĠWinner":25358,"Ġlitter":25359,"ĠShooting":25360,"STEM":25361,"ãĤ¡":25362,"ĠAFL":25363,"Ġvariability":25364,"Ġeats":25365,"ĠDPS":25366,"brow":25367,"Ġelephants":25368,"Ġstrat":25369,"ĠÅ":25370,"Ġsettlers":25371,"Matthew":25372,"Ġinadvert":25373,"HI":25374,"ĠIMF":25375,"ĠGoal":25376,"Ġnerves":25377,"Johnson":25378,"eye":25379,"ablishment":25380,"Thursday":25381,"BILITY":25382,"Had":25383,"amoto":25384,"hetamine":25385,"eps":25386,"Ġmitochond":25387,"Ġcompressed":25388,"ĠTrevor":25389,"ĠAnimals":25390,"Tool":25391,"Lock":25392,"Ġtweak":25393,"Ġpinch":25394,"Ġcancellation":25395,"Pot":25396,"Ġfocal":25397,"ĠAstron":25398,"173":25399,"ĠASC":25400,"ĠOTHER":25401,"umni":25402,"Ġdemise":25403,"dl":25404,"Ùħ":25405,"Semitism":25406,"Ġcracking":25407,"Ġcollaborative":25408,"Ġexplores":25409,"sql":25410,"Ġherbs":25411,"Ġconfigurations":25412,"mis":25413,"ĠResult":25414,"acey":25415,"ĠSmoke":25416,"Ġsanct":25417,"elia":25418,"Ġdegener":25419,"Ġdeepest":25420,"Ġscreamed":25421,"Ġnap":25422,"Software":25423,"ĠSTAR":25424,"EF":25425,"ĠXin":25426,"sponsored":25427,"manship":25428,"233":25429,"Ġprimaries":25430,"Ġfiltering":25431,"Ġassemble":25432,"mil":25433,"ĠMyers":25434,"bows":25435,"Ġpunched":25436,"Mic":25437,"Ġinnovations":25438,"Ġfunc":25439,"ando":25440,"Ġfracking":25441,"ĠVul":25442,"оÐ":25443,"oshop":25444,"ĠImmun":25445,"Ġsettling":25446,"Ġadolescents":25447,"Ġrebuilding":25448,"Ġtransforming":25449,"Ġparole":25450,"Ġharbor":25451,"Ġbooking":25452,"otional":25453,"ongevity":25454,"ĠYo":25455,"bug":25456,"Ġemerges":25457,"ĠMethods":25458,"ĠChu":25459,"Pres":25460,"ĠDungeons":25461,"Ġtrailing":25462,"ĠRum":25463,"ĠHugh":25464,"天":25465,"ĠEra":25466,"ĠBattles":25467,"Results":25468,"ĠTrading":25469,"Ġversa":25470,"css":25471,"axies":25472,"heet":25473,"Ġgreed":25474,"1989":25475,"Ġgardens":25476,"Ġcontingent":25477,"Park":25478,"ĠLeafs":25479,"hook":25480,"robe":25481,"Ġdiplomacy":25482,"ĠFuel":25483,"ĠInvasion":25484,"Ġupgrading":25485,"Male":25486,"Ġelic":25487,"Ġrelentless":25488,"ĠCovenant":25489,"apesh":25490,"ĠTrop":25491,"Ty":25492,"production":25493,"arty":25494,"Ġpunches":25495,"ako":25496,"cyclopedia":25497,"ĠRabbit":25498,"ĠHDMI":25499,"Ġ141":25500,"Ġfoil":25501,"ItemImage":25502,"ĠFG":25503,"Ġimplementations":25504,"ĠPom":25505,"ixtures":25506,"Ġawait":25507,"Ġ330":25508,"amus":25509,"Ġumbrella":25510,"Ġforesee":25511,"separ":25512,"Ġcircumcision":25513,"Ġperipheral":25514,"Say":25515,"ĠExpert":25516,"Inc":25517,"Ġwithdrew":25518,"ĠAnders":25519,"fried":25520,"Ġradioactive":25521,"ĠOpening":25522,"Ġboarding":25523,"ĠND":25524,"Ġoverthrow":25525,"Activ":25526,"WP":25527,"ĠActs":25528,"×Ļ":25529,"Ġmotions":25530,"vic":25531,"ĠMighty":25532,"ĠDefender":25533,"aer":25534,"Ġthankful":25535,"ĠKilling":25536,"ĠBris":25537,"moil":25538,"Ġpredicting":25539,"266":25540,"choice":25541,"Ġkillers":25542,"Ġincub":25543,"ĠChest":25544,"athering":25545,"Ġproclaimed":25546,"flower":25547,"ossom":25548,"umbledore":25549,"ĠCycling":25550,"ĠOccupy":25551,"AGES":25552,"Pen":25553,"ĠYug":25554,"Ġpackaged":25555,"Ġheightened":25556,"cot":25557,"stack":25558,"Cond":25559,"Ġstamps":25560,"mage":25561,"Ġpersuaded":25562,"Ġensl":25563,"ĠCardinal":25564,"Ġsolitary":25565,"Ġpossessing":25566,"ĠCork":25567,"Ġevid":25568,"ĠTay":25569,"Ġblues":25570,"Ġextremism":25571,"Ġlunar":25572,"Ġclown":25573,"Techn":25574,"Ġfestivals":25575,"ĠPvP":25576,"ĠLar":25577,"Ġconsequently":25578,"present":25579,"Ġsomeday":25580,"çİĭ":25581,"ĠMeteor":25582,"Ġtouring":25583,"culture":25584,"Ġbeaches":25585,"Ship":25586,"cause":25587,"ĠFlood":25588,"ãĥ¯":25589,"Ġpurity":25590,"those":25591,"Ġemission":25592,"bolt":25593,"Ġchord":25594,"ĠScripture":25595,"Lu":25596,"Ġ${":25597,"created":25598,"Others":25599,"258":25600,"Ġelemental":25601,"Ġannoyed":25602,"ĠAE":25603,"dan":25604,"ĠSag":25605,"Researchers":25606,"Ġfairy":25607,"âĢĵâĢĵ":25608,"============":25609,"Smart":25610,"GGGG":25611,"Ġskeletons":25612,"Ġpupils":25613,"linked":25614,"Ġurgency":25615,"enabled":25616,"ĠFuck":25617,"Ġcouncill":25618,"rab":25619,"UAL":25620,"TI":25621,"Ġlifes":25622,"Ġconfessed":25623,"Bug":25624,"Ġharmon":25625,"ĠCONFIG":25626,"ĠNeutral":25627,"Double":25628,"Ġstaple":25629,"ĠSHA":25630,"British":25631,"ĠSNP":25632,"ATOR":25633,"oco":25634,"Ġswinging":25635,"gex":25636,"oleon":25637,"plain":25638,"ĠMissing":25639,"ĠTrophy":25640,"vari":25641,"ranch":25642,"Ġ301":25643,"440":25644,"0000000000000000":25645,"Ġrestoring":25646,"Ġhaul":25647,"ucing":25648,"nerg":25649,"Ġfutures":25650,"Ġstrategist":25651,"question":25652,"Ġlateral":25653,"ĠBard":25654,"Ġsor":25655,"ĠRhodes":25656,"ĠDowntown":25657,"?????-":25658,"ĠLit":25659,"ĠBened":25660,"Ġcoil":25661,"street":25662,"ĠPortal":25663,"FILE":25664,"ĠGru":25665,"*,":25666,"231":25667,"neum":25668,"Ġsucked":25669,"Ġrapper":25670,"Ġtendencies":25671,"ĠLauren":25672,"cellaneous":25673,"267":25674,"Ġbrowse":25675,"Ġoverc":25676,"header":25677,"oise":25678,"Ġbeet":25679,"ĠGle":25680,"Stay":25681,"Ġmum":25682,"Ġtyped":25683,"Ġdiscounts":25684,"Talk":25685,"ĠOg":25686,"existing":25687,"ĠSell":25688,"uph":25689,"CI":25690,"ĠAustrian":25691,"ĠWarm":25692,"Ġdismissal":25693,"Ġaverages":25694,"camera":25695,"Ġallegiance":25696,"LAN":25697,"=\"#":25698,"Ġcommentators":25699,"ĠSetting":25700,"ĠMidwest":25701,"Ġpharmac":25702,"ĠEXP":25703,"Ġstainless":25704,"Chicago":25705,"Ġtan":25706,"244":25707,"Ġcountryside":25708,"ĠVac":25709,"295":25710,"Ġpinned":25711,"Ġcrises":25712,"Ġstandardized":25713,"Task":25714,"ĠJail":25715,"ĠDocker":25716,"colored":25717,"forth":25718,"\"},":25719,"Ġpatrons":25720,"Ġspice":25721,"Ġmourn":25722,"ĠMood":25723,"Ġlaundry":25724,"Ġequip":25725,"ĠMole":25726,"yll":25727,"ĠTHC":25728,"nation":25729,"ĠSherlock":25730,"Ġissu":25731,"ĠKre":25732,"ĠAmericas":25733,"ĠAAA":25734,"Ġsystematically":25735,"Ġcontra":25736,"ĠSally":25737,"Ġrationale":25738,"Ġcarriage":25739,"Ġpeaks":25740,"Ġcontradiction":25741,"ensation":25742,"ĠFailure":25743,"Ġprops":25744,"Ġnamespace":25745,"Ġcove":25746,"fields":25747,"ãĤĭ":25748,"Ġwool":25749,"ĠCatch":25750,"Ġpresumed":25751,"ĠDiana":25752,"ragon":25753,"igi":25754,"Ġhamm":25755,"Ġstunt":25756,"ĠGUI":25757,"ĠObservatory":25758,"ĠShore":25759,"Ġsmells":25760,"annah":25761,"Ġcockpit":25762,"ĠDuterte":25763,"850":25764,"Ġoppressed":25765,"breaker":25766,"ĠContribut":25767,"ĠPeru":25768,"ĠMonsanto":25769,"ĠAttempt":25770,"Ġcommanding":25771,"Ġfridge":25772,"ĠRin":25773,"ĠChess":25774,"uality":25775,"Ġol":25776,"Republican":25777,"ĠGlory":25778,"ĠWIN":25779,".......":25780,"agent":25781,"reading":25782,"Ġinh":25783,"Jones":25784,"Ġclicks":25785,"alan":25786,"Ġ[];":25787,"ĠMajesty":25788,"ĠCed":25789,"opus":25790,"atel":25791,"ê":25792,"ARC":25793,"ĠEcuador":25794,"ãĥł":25795,"ĠKuro":25796,"Ġrituals":25797,"Ġcaptive":25798,"Ġounce":25799,"Ġdisagreement":25800,"Ġslog":25801,"fuel":25802,"Pet":25803,"Mail":25804,"Ġexercised":25805,"Ġsolic":25806,"Ġrainfall":25807,"Ġdevotion":25808,"ĠAssessment":25809,"Ġrobotic":25810,"options":25811,"ĠRP":25812,"ĠFamilies":25813,"ĠFlames":25814,"Ġassignments":25815,"007":25816,"akedown":25817,"Ġvocabulary":25818,"Reilly":25819,"Ġcaval":25820,"gars":25821,"Ġsuppressed":25822,"ĠSET":25823,"ĠJohns":25824,"Ġwarp":25825,"broken":25826,"Ġstatues":25827,"Ġadvocated":25828,"Ġ275":25829,"Ġperil":25830,"omorph":25831,"ĠFemin":25832,"perfect":25833,"Ġhatch":25834,"Lib":25835,"512":25836,"Ġlifelong":25837,"313":25838,"Ġcheeks":25839,"Ġnumbered":25840,"ĠMug":25841,"Body":25842,"ravel":25843,"Weight":25844,"ĠJak":25845,"ĠHeath":25846,"Ġkissing":25847,"ĠJUST":25848,"Ġwaving":25849,"upload":25850,"Ġinsider":25851,"ĠProgressive":25852,"ĠFilter":25853,"tta":25854,"ĠBeam":25855,"Ġviolently":25856,"ipation":25857,"Ġskepticism":25858,"Ġ1918":25859,"ĠAnnie":25860,"ĠSI":25861,"Ġgenetics":25862,"Ġonboard":25863,"atl":25864,"ĠFriedman":25865,"ĠBri":25866,"ceptive":25867,"Ġpirate":25868,"ĠReporter":25869,"278":25870,"Ġmythology":25871,"Ġeclipse":25872,"Ġskins":25873,"Ġglyph":25874,"ingham":25875,"Files":25876,"Cour":25877,"women":25878,"Ġregimes":25879,"Ġphotographed":25880,"Kat":25881,"ĠMAX":25882,"Officials":25883,"Ġunexpectedly":25884,"Ġimpressions":25885,"Front":25886,";;;;;;;;":25887,"Ġsupremacy":25888,"Ġsang":25889,"Ġaggravated":25890,"Ġabruptly":25891,"ĠSector":25892,"Ġexcuses":25893,"Ġcosting":25894,"idepress":25895,"Stack":25896,"ĠRNA":25897,"obil":25898,"Ġghosts":25899,"ldon":25900,"atibility":25901,"Topics":25902,"Ġreimburse":25903,"ĠHM":25904,"ĠDeg":25905,"Ġthief":25906,"yet":25907,"ogenesis":25908,"leaning":25909,"ĠKol":25910,"ĠBasketball":25911,"Ġfi":25912,"ĠSeeing":25913,"Ġrecycling":25914,"Ġ[-":25915,"Congress":25916,"Ġlectures":25917,"Psy":25918,"Ġnep":25919,"Ġmaid":25920,"Ġoriented":25921,"AX":25922,"Ġrespectful":25923,"rene":25924,"flush":25925,"ĠUnloaded":25926,"request":25927,"grid":25928,"ĠAlternatively":25929,"ĠHugo":25930,"Ġdecree":25931,"ĠBuddhism":25932,"andum":25933,"Android":25934,"ĠCongo":25935,"ĠJoyce":25936,"Ġacknowledging":25937,"hesive":25938,"ĠTomorrow":25939,"ĠHiro":25940,"thren":25941,"ĠMaced":25942,"Ġhoax":25943,"ĠIncreased":25944,"ĠPradesh":25945,"Wild":25946,"______":25947,"161":25948,"Ġaunt":25949,"Ġdistributing":25950,"ĠTucker":25951,"ĠSSL":25952,"ĠWolves":25953,"Building":25954,"oult":25955,"ĠLuo":25956,"ĠYas":25957,"ĠSpir":25958,"ĠShape":25959,"ĠCambod":25960,"ĠIPv":25961,"Ġml":25962,"Ġextrad":25963,"390":25964,"ĠPenny":25965,"dream":25966,"Ġstationed":25967,"optional":25968,"eworthy":25969,".":26700,"ĠWorkshop":26701,"ĠRetail":26702,"ĠAvatar":26703,"625":26704,"Na":26705,"ĠVC":26706,"ĠSecure":26707,"MY":26708,"1988":26709,"ossip":26710,"Ġprostate":26711,"Ġunden":26712,"Ġgamer":26713,"ĠContents":26714,"ĠWarhammer":26715,"ĠSentinel":26716,"310":26717,"Ġsegregation":26718,"ĠFlex":26719,"ĠMAY":26720,"Ġdrills":26721,"ĠDrugs":26722,"Islamic":26723,"Ġspur":26724,"Ġcafe":26725,"Ġimaginary":26726,"Ġguiding":26727,"Ġswings":26728,"ĠTheme":26729,"oby":26730,"Ġnud":26731,"Ġbegging":26732,"Ġstrongh":26733,"Ġrejecting":26734,"Ġpedestrians":26735,"ĠProspect":26736,"Rare":26737,"sle":26738,"Ġconcessions":26739,"ĠConstitutional":26740,"Ġbeams":26741,"Ġfibers":26742,"poon":26743,"Ġinstincts":26744,"property":26745,"ĠBIG":26746,"Sanders":26747,"imates":26748,"Ġcoating":26749,"Ġcorpses":26750,"ĠTRUE":26751,"checked":26752,"Ġ166":26753,"Ash":26754,"ĠJS":26755,"ĠFiction":26756,"Ġcommunal":26757,"Ġenergetic":26758,"oooooooo":26759,"Ġnowadays":26760,"ILD":26761,"ibo":26762,"ĠSUV":26763,"Ren":26764,"Ġdwelling":26765,"Silver":26766,"Ġtally":26767,"ĠMoving":26768,"Ġcoward":26769,"Ġgenerals":26770,"Ġhorns":26771,"Ġcirculated":26772,"Ġrobbed":26773,"ĠUnlimited":26774,"Ġharassed":26775,"Ġinhibit":26776,"Ġcomposer":26777,"ĠSpotify":26778,"Ġspreads":26779,"364":26780,"Ġsuicidal":26781,"Ġnoises":26782,"ĠStur":26783,"Ġsaga":26784,"ĠKag":26785,"iso":26786,"Ġtheoretically":26787,"Money":26788,"Ġsimilarity":26789,"Ġsliced":26790,"utils":26791,"inges":26792,"\"-":26793,"Ġanth":26794,"Ġimped":26795,"Module":26796,"Throughout":26797,"Ġmenus":26798,"committee":26799,"andi":26800,"obj":26801,"inav":26802,"fired":26803,"ĠAbdullah":26804,"Ġundead":26805,"Ġfonts":26806,"Hold":26807,"ENG":26808,"Ġsustainability":26809,"Ġflick":26810,"Ġrazor":26811,"ĠFest":26812,"ĠCharacters":26813,"Ġwording":26814,"Ġpopulist":26815,"Ġcriticizing":26816,"Ġmuse":26817,"vine":26818,"Ġcardboard":26819,"Ġkindly":26820,"Ġfringe":26821,"ĠTheft":26822,"icultural":26823,"Ġgovernors":26824,"Ġ����":26825,"Ġ163":26826,"Ġtimeout":26827,"ĠAuth":26828,"Children":26829,"AU":26830,"Ġredemption":26831,"ĠAlger":26832,"Ġ1914":26833,"Ġwaved":26834,"Ġastronauts":26835,"ograms":26836,"Ġswamp":26837,"ĠFinnish":26838,"Ġcandle":26839,"Ġtonnes":26840,"utm":26841,"Ġray":26842,"Ġspun":26843,"Ġfearful":26844,"articles":26845,"Ġcaus":26846,"orically":26847,"ĠRequires":26848,"ĠGol":26849,"Ġpope":26850,"Ġinaugural":26851,"Ġgle":26852,"ADA":26853,"ĠISIL":26854,"ĠOffensive":26855,"Ġwatchdog":26856,"Ġbalcon":26857,"entity":26858,"ĠHoo":26859,"Ġgallon":26860,"ACC":26861,"Ġdoubling":26862,"Ġimplication":26863,"ĠSight":26864,"Ġdoctr":26865,"-------":26866,"Ġ\\\\":26867,"Ġmalt":26868,"Roll":26869,"Ġâī¥":26870,"Ġrecap":26871,"adding":26872,"uces":26873,"ĠBend":26874,"figure":26875,"Ġturkey":26876,"Ġsocietal":26877,"ĠTickets":26878,"Ġcommercially":26879,"Ġspicy":26880,"Ġ216":26881,"ĠRamp":26882,"Ġsuperiority":26883,"ï":26884,"ĠTracker":26885,"Carl":26886,"ĠCoy":26887,"ĠPatriot":26888,"Ġconsulted":26889,"Ġlistings":26890,"Ġslew":26891,"reenshot":26892,"ĠGone":26893,"Ġ[...]":26894,"309":26895,"Ġhottest":26896,"ر":26897,"Ġrocky":26898,"ĠDiaz":26899,"Ġmassage":26900,"Ġparaly":26901,"Ġpony":26902,"Az":26903,"Ġcartridge":26904,"ĠNZ":26905,"Ġsnack":26906,"ĠLamar":26907,"plement":26908,"ĠLeslie":26909,"Ġmater":26910,"Ġsnipp":26911,"246":26912,"Ġjointly":26913,"ĠBrisbane":26914,"ĠiPod":26915,"Ġpumping":26916,"Ġgoat":26917,"ĠSharon":26918,"ealing":26919,"Ġcoron":26920,"Ġanomal":26921,"rahim":26922,"ĠConnection":26923,"Ġsculpture":26924,"Ġscheduling":26925,"ĠDaddy":26926,"athing":26927,"Ġeyebrows":26928,"Ġcurved":26929,"Ġsentiments":26930,"Ġdrafting":26931,"Drop":26932,"([":26933,"Ġnominal":26934,"ĠLeadership":26935,"ĠGrow":26936,"Ġ176":26937,"Ġconstructive":26938,"ivation":26939,"Ġcorrupted":26940,"gerald":26941,"ĠCros":26942,"ĠChester":26943,"ĠLap":26944,"ãģª":26945,"OTH":26946,"DATA":26947,"Ġalmond":26948,"probably":26949,"Imp":26950,"Ġfeast":26951,"ĠWarcraft":26952,"Flor":26953,"Ġcheckpoint":26954,"Ġtranscription":26955,"Ġ204":26956,"Ġtweaks":26957,"Ġrelieve":26958,"Science":26959,"Ġperformer":26960,"Zone":26961,"Ġturmoil":26962,"igated":26963,"hibit":26964,"ĠCafe":26965,"themed":26966,"Ġfluor":26967,"bench":26968,"Ġdecom":26969,"ĠUnt":26970,"ĠBarrett":26971,"ĠFacts":26972,"Ġtasting":26973,"ĠPTSD":26974,"ĠSeal":26975,"ĠJudaism":26976,"ĠDynamic":26977,"ĠCors":26978,"Ve":26979,"ĠMing":26980,"ĠTransform":26981,"von":26982,"ĠDefenders":26983,"ĠTactical":26984,"ĠVon":26985,"ĠUnivers":26986,"Ġdistorted":26987,"ĠBreath":26988,"?'\"":26989,"Ġagon":26990,"ĠDeadly":26991,"Ġlan":26992,"ĠCycle":26993,"orned":26994,"Ġreliably":26995,"Ġglor":26996,"ĠMonkey":26997,"ãĥ¡":26998,"Ġadren":26999,"Ġmicrowave":27000,"ĠAlban":27001,"ircraft":27002,"digit":27003,"smart":27004,"ĠDread":27005,"¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯":27006,"{{":27007,"ĠRochester":27008,"Ġsimplified":27009,"Ġinflicted":27010,"Ġtakeover":27011,"Ġyourselves":27012,"aditional":27013,"Ġmuscular":27014,"KS":27015,"Ġingen":27016,"Tax":27017,"ĠFeature":27018,"277":27019,"Ġcruc":27020,"Ġcrate":27021,"Ġunidentified":27022,"Ġacclaimed":27023,"ĠManga":27024,"ĠFrances":27025,"ĠNepal":27026,"ĠGerald":27027,"ĠKuwait":27028,"Ġslain":27029,"ĠHeb":27030,"ĠGoku":27031,"ãģ®æ":27032,"286":27033,"Mrs":27034,"ĠCody":27035,"ĠSanctuary":27036,"016":27037,"Ġdismant":27038,"Ġdataset":27039,"ĠHond":27040,"buck":27041,"ĠPatterson":27042,"Ġpalette":27043,"ĠGD":27044,"icol":27045,"ĠLodge":27046,"Ġplanetary":27047,"akin":27048,"ĠRegistered":27049,"abwe":27050,"ĠPetersburg":27051,"Ġhailed":27052,"ĠPiece":27053,"Sche":27054,"ĠDOJ":27055,"Ġenumer":27056,"181":27057,"ĠObserver":27058,"ĠBold":27059,"founded":27060,"commerce":27061,"Ġexploits":27062,"ĠFinding":27063,"URN":27064,"ĠSne":27065,"ĠAcid":27066,"ayette":27067,"ĠValues":27068,"Ġdrastic":27069,"Ġarchitectural":27070,"Ġ\".":27071,"×ķ":27072,"umped":27073,"Ġwrapping":27074,"Ġwidow":27075,"ĠSlayer":27076,"lace":27077,"once":27078,"Germany":27079,"avoid":27080,"Ġtemples":27081,"PAR":27082,"ô":27083,"ĠLucifer":27084,"ĠFlickr":27085,"lov":27086,"forces":27087,"Ġscouting":27088,"Ġlouder":27089,"tesy":27090,"Ġbeforehand":27091,"Äĵ":27092,"ĠNeon":27093,"ĠWol":27094,"ĠTypically":27095,"ĠPolitico":27096,"-+-+":27097,"Ġbuilder":27098,"Ġderive":27099,"Kill":27100,"Ġpoker":27101,"Ġambiguous":27102,"Ġlifts":27103,"Ġcyt":27104,"Ġribs":27105,"oodle":27106,"ĠSounds":27107,"hair":27108,"ĠSyndrome":27109,"tf":27110,"Ġproportional":27111,"uid":27112,"Ġpertaining":27113,"ĠKindle":27114,"ĠNegro":27115,"Ġreiterated":27116,"ĠTonight":27117,"oths":27118,"ĠCornell":27119,"Ġowing":27120,"Ġ208":27121,"elfare":27122,"ocating":27123,"ĠBirds":27124,"Subscribe":27125,"Ġessays":27126,"Ġburdens":27127,"Ġillustrations":27128,"arious":27129,"ERAL":27130,"ĠCalcul":27131,"Ġxen":27132,"ĠLinkedIn":27133,"ĠJung":27134,"Ġredesign":27135,"Connor":27136,"296":27137,"Ġreversal":27138,"ĠAdelaide":27139,"ĠLL":27140,"Ġsinking":27141,"Ġgum":27142,"USH":27143,"capt":27144,"ĠGrimm":27145,"Ġfootsteps":27146,"ĠCBD":27147,"ispers":27148,"Ġprose":27149,"Wednesday":27150,"ĠMovies":27151,"edin":27152,"Ġoverturned":27153,"Ġcontentious":27154,"USB":27155,"~~~~~~~~~~~~~~~~":27156,"ĠCopper":27157,"Ġpointless":27158,"NV":27159,"values":27160,"olphin":27161,"dain":27162,"Ġdeposited":27163,"ĠGW":27164,"Ġpreceded":27165,"ĠCla":27166,"ĠGolem":27167,"ĠNim":27168,"Ġβ":27169,"ĠEngineers":27170,"middle":27171,"Ġflatt":27172,"operative":27173,"Ġcouncils":27174,"imbabwe":27175,"elin":27176,"Ġstressful":27177,"ĠLD":27178,"Ġresh":27179,"lake":27180,"Ġwheelchair":27181,"ĠAlternative":27182,"Ġoptimize":27183,"operation":27184,"Ġpeek":27185,"Ġoneself":27186,"igil":27187,"Ġtransitions":27188,"opathy":27189,"blank":27190,"Ġ169":27191,"171":27192,"________________________________________________________________":27193,"Ġlaundering":27194,"Enc":27195,"ĠDEC":27196,"Ġworkouts":27197,"Ġspikes":27198,"Ġdinosaurs":27199,"Ġdiscriminatory":27200,"Pool":27201,"Rather":27202,"385":27203,"RNA":27204,"testers":27205,"eto":27206,"ĠIdentity":27207,"Ġvein":27208,"ĠBurton":27209,"Ġarcade":27210,"420":27211,"Ultimately":27212,"ĠSadly":27213,"ð":27214,"pill":27215,"Ġcubic":27216,"ĠSpectrum":27217,"these":27218,"states":27219,"Ġunofficial":27220,"hawks":27221,"ĠEVERY":27222,"Ġrainbow":27223,"Ġincarceration":27224,"anding":27225,"Ġsyll":27226,"ĠEverton":27227,"Ġ179":27228,"ĠSerbia":27229,"Ġ189":27230,"meter":27231,"ĠMickey":27232,"Ġantiqu":27233,"Ġfactual":27234,"neck":27235,"ĠNare":27236,"norm":27237,"must":27238,"Ġhighways":27239,"Ġglam":27240,"Ġdividing":27241,"ĠSquadron":27242,"ĠMartha":27243,"Ġbirths":27244,"Cover":27245,"////////////////":27246,"ĠWong":27247,"Phot":27248,"ĠALS":27249,"rio":27250,"ĠNonetheless":27251,"ĠLemon":27252,"Ġ206":27253,"ĠEE":27254,"Ġderivative":27255,"ĠWWII":27256,"vote":27257,"Ġtherein":27258,"Ġseparating":27259,"446":27260,"sync":27261,"ĠStreets":27262,"Ġratt":27263,"Ġmunicipality":27264,"ĠShortly":27265,"Ġmonk":27266,"),\"":27267,"Ġscrub":27268,"Ġoperatives":27269,"Neither":27270,"Place":27271,"ĠLimit":27272,"Female":27273,"ĠActor":27274,"Character":27275,"Ġconstituted":27276,"357":27277,"Ġprotested":27278,"ĠStraw":27279,"ĠHeight":27280,"ilda":27281,"ĠTyph":27282,"Ġfloods":27283,"Ġcosmetic":27284,"WAY":27285,"perture":27286,"upon":27287,"tons":27288,"essing":27289,"ĠPocket":27290,"Ġrooft":27291,"ĠCaucas":27292,"Ġantidepress":27293,"Ġincompatible":27294,"ECD":27295,"Ġopera":27296,"ĠContest":27297,"Ġgenerators":27298,"lime":27299,"Defense":27300,"1987":27301,"forum":27302,"Ġsavage":27303,"ĠHungarian":27304,"nz":27305,"Ġmetallic":27306,"Ġexpelled":27307,"Ġresidency":27308,"Ġdresses":27309,"666":27310,"ĠClement":27311,"fires":27312,"Category":27313,"Ġgeek":27314,"alis":27315,"Ġcemetery":27316,"educated":27317,"Ġcrawl":27318,"ĠUnable":27319,"ĠTyson":27320,"akis":27321,"Ġpardon":27322,"ĠWra":27323,"Ġstrengthened":27324,"ĠFors":27325,"335":27326,"ĠHC":27327,"ĠMond":27328,"Ġvisuals":27329,"ĠBeatles":27330,"ettlement":27331,"Ġï":27332,"gro":27333,"Ġbash":27334,"Ġpoorest":27335,"Ġexcel":27336,"Ġaspirations":27337,"ĠMunicip":27338,"ensible":27339,"Ġceremonies":27340,"Ġintimidation":27341,"ĠCONTR":27342,"beck":27343,"ĠKap":27344,"asu":27345,"Ġtrademarks":27346,"ĠSew":27347,"ĠCompetition":27348,"network":27349,"ĠArri":27350,"ĠTet":27351,"Roaming":27352,"WC":27353,"Dat":27354,"Ġsob":27355,"Ġpairing":27356,"Ġoverdose":27357,"SAY":27358,"aber":27359,"Ġrevolt":27360,"ĠFah":27361,"acting":27362,"eq":27363,"estation":27364,"Fight":27365,"ĠMarks":27366,"273":27367,"Ġ178":27368,"Raw":27369,"ãģĭ":27370,"349":27371,"blocks":27372,"Ġverge":27373,"estine":27374,"ĠPodesta":27375,"Ġinvasive":27376,"Ġprofoundly":27377,"ĠAo":27378,"each":27379,"Ġlest":27380,"interpret":27381,"Ġshrinking":27382,"Ġerrone":27383,"Ġchees":27384,"lys":27385,"ĠIvy":27386,"ĠDirectory":27387,"Ġhinted":27388,"VICE":27389,"Ġcontacting":27390,"ĠGent":27391,"hei":27392,"Ġlabeling":27393,"Ġmercury":27394,"ĠLite":27395,"Ġexpires":27396,"Ġdestabil":27397,"ritis":27398,"cu":27399,"Ġfeathers":27400,"Ġsteer":27401,"Ġprogrammed":27402,"ĠVader":27403,"Going":27404,"ĠElim":27405,"Ġyo":27406,"ĠMiche":27407,"Ġ203":27408,"Ġsleeves":27409,"Ġbully":27410,"ĠHumans":27411,"368":27412,"Ġcompress":27413,"ĠBanner":27414,"ARS":27415,"Ġawhile":27416,"Ġcalib":27417,"Ġsponsorship":27418,"ĠDifficulty":27419,"ĠPapers":27420,"Ġidentifier":27421,"}.":27422,"Ġyog":27423,"ĠShia":27424,"Ġcleanup":27425,"Ġvibe":27426,"introdu":27427,"imming":27428,"Australia":27429,"Ġoutlines":27430,"ĠYoutube":27431,"train":27432,"ĠMakes":27433,"Ġdeported":27434,"Ġcentr":27435,"ĠDug":27436,"ĠBoulder":27437,"ĠBuffy":27438,"Ġinjunction":27439,"ĠHarley":27440,"ĠGroups":27441,"ĠDumbledore":27442,"ĠClara":27443,"Ġ\"-":27444,"Ġsacrificed":27445,"eph":27446,"Shadow":27447,"ibling":27448,"Ġfreelance":27449,"Ġevidently":27450,"phal":27451,"Ġretains":27452,"Mir":27453,"Ġfinite":27454,"dar":27455,"ĠCous":27456,"Ġrepaired":27457,"Ġperiodic":27458,"Ġchampionships":27459,"Ġasteroid":27460,"blind":27461,"Ġexpressly":27462,"ĠAstros":27463,"Ġscaled":27464,"Ġgeographical":27465,"ĠRapids":27466,"Enjoy":27467,"Ġelastic":27468,"ĠMohamed":27469,"Market":27470,"begin":27471,"Ġdiscovers":27472,"Ġtelecommunications":27473,"Ġscanner":27474,"Ġenlarge":27475,"Ġsharks":27476,"Ġpsychedel":27477,"ĠRouge":27478,"Ġsnapshot":27479,"isine":27480,"XP":27481,"Ġpesticides":27482,"ĠLSD":27483,"ĠDistribution":27484,"really":27485,"Ġdegradation":27486,"Ġdisguise":27487,"Ġbiom":27488,"ĠEXT":27489,"Ġequations":27490,"Ġhazards":27491,"ĠCompared":27492,")*":27493,"Ġvirtues":27494,"Ġelders":27495,"Ġenhancing":27496,"ĠAcross":27497,"eros":27498,"angling":27499,"Ġcombust":27500,"ucci":27501,"Ġconcussion":27502,"Ġcontraception":27503,"ĠKang":27504,"Ġexpresses":27505,"Ġaux":27506,"ĠPione":27507,"Ġexhibits":27508,"Debug":27509,"OTAL":27510,"ĠAlready":27511,"ĠWheeler":27512,"Ġexpands":27513,"?:":27514,"Ġreconciliation":27515,"Ġpirates":27516,"Ġpurse":27517,"Ġdiscourage":27518,"Ġspectacle":27519,"Rank":27520,"Ġwraps":27521,"ĠThought":27522,"Ġimpending":27523,"Opp":27524,"ĠAnglo":27525,"ĠEUR":27526,"Ġscrewed":27527,"retched":27528,"Ġencouragement":27529,"models":27530,"Ġconfuse":27531,"mmm":27532,"ĠVitamin":27533,"âĸijâĸij":27534,"Cru":27535,"Ġknights":27536,"Ġdiscard":27537,"Ġbishops":27538,"ĠWear":27539,"ĠGarrett":27540,"kan":27541,"ãĥŁ":27542,"Ġmasculine":27543,"capital":27544,"ĠAus":27545,"Ġfatally":27546,"thanks":27547,"ĠAU":27548,"ĠGut":27549,"1200":27550,"Ġ00000000":27551,"Ġsurrog":27552,"ĠBIOS":27553,"raits":27554,"ĠWatts":27555,"Ġresurrection":27556,"ĠElectoral":27557,"ĠTips":27558,"4000":27559,"Ġnutrient":27560,"Ġdepicting":27561,"Ġsprink":27562,"Ġmuff":27563,"ĠLIM":27564,"ĠSample":27565,"psc":27566,"ibi":27567,"generated":27568,"Ġspecimens":27569,"Ġdissatisf":27570,"Ġtailored":27571,"Ġholdings":27572,"ĠMonthly":27573,"ĠEat":27574,"poons":27575,"Ġnec":27576,"ĠCage":27577,"ĠLotus":27578,"ĠLantern":27579,"Ġfrontier":27580,"Ġpensions":27581,"Ġjoked":27582,"ĠHardy":27583,"=-=-=-=-":27584,"rade":27585,"UID":27586,"Ġrails":27587,"Ġemit":27588,"Ġslate":27589,"Ġsmug":27590,"Ġspit":27591,"ĠCalls":27592,"ĠJacobs":27593,"feat":27594,"ĠUE":27595,"Ġrestruct":27596,"Ġregeneration":27597,"Ġenergies":27598,"ĠConnor":27599,"OHN":27600,"ĠCheese":27601,"Ġger":27602,"Ġresurrect":27603,"management":27604,"NW":27605,"Ġpresently":27606,"ĠBruins":27607,"Member":27608,"ĠMang":27609,"idan":27610,"Ġboosting":27611,"wyn":27612,"+.":27613,"requisite":27614,"ĠNYPD":27615,"ĠMegan":27616,"ĠConditions":27617,"Ġpics":27618,"nesium":27619,"ĠRash":27620,"Ġ174":27621,"ĠDucks":27622,"Ġembro":27623,"zu":27624,"onian":27625,"religious":27626,"Ġcraz":27627,"ĠACA":27628,"ĠZucker":27629,"EMA":27630,"ĠPros":27631,"Weapon":27632,"ĠKnox":27633,"ĠArduino":27634,"Ġstove":27635,"Ġheavens":27636,"ĠPurchase":27637,"Ġherd":27638,"Ġfundraiser":27639,"Digital":27640,"5000":27641,"Ġproponents":27642,"/âĢĭ":27643,"Ġjelly":27644,"ĠVisa":27645,"Ġmonks":27646,"Ġadvancement":27647,"ĠWer":27648,"Ġ187":27649,"eus":27650,"ertility":27651,"Ġfetal":27652,"Ġ1936":27653,"Lo":27654,"Ġoutfits":27655,"Ġstaircase":27656,"bomb":27657,"Ġcustomized":27658,"clair":27659,"Tree":27660,"Ġmapped":27661,"ĠConsidering":27662,"ĠTorres":27663,"Ġmethyl":27664,"Ġapproximate":27665,"Ġdoom":27666,"ĠHansen":27667,"Ġcrossover":27668,"Ġstandalone":27669,"ä¼":27670,"Ġinvites":27671,"Ġgraveyard":27672,"Ġhp":27673,"DonaldTrump":27674,"Ġescort":27675,"Gar":27676,"Ġpredecessors":27677,"Ġhay":27678,"Ġenzyme":27679,"ĠStraight":27680,"visors":27681,"Ing":27682,"aneously":27683,"ĠApplied":27684,"Ġfec":27685,"ĠDurant":27686,"Ġoutspoken":27687,"orb":27688,"Ġzeal":27689,"Ġdisgrace":27690,"').":27691,"ĠCheng":27692,"289":27693,"ĠRena":27694,"ĠSuicide":27695,"294":27696,"Ġoutraged":27697,"ĠNewman":27698,"ĠNvidia":27699,"ĠAber":27700,"ĠBers":27701,"Ġrecreation":27702,"Window":27703,"ĠDP":27704,"xe":27705,"Ġpedoph":27706,"Ġfallout":27707,"amboo":27708,"Ġpresentations":27709,"ĠApps":27710,"Ġhtml":27711,"345":27712,"ĠXXX":27713,"Ġrubbing":27714,"ĠLeather":27715,"Ġhumidity":27716,"seys":27717,"established":27718,"ĠUnits":27719,"646":27720,"Ġrespectable":27721,"Auto":27722,"Ġthriving":27723,"ĠInnovation":27724,"angs":27725,"Extra":27726,"regulation":27727,"298":27728,"pick":27729,"Examples":27730,"ĠCJ":27731,"Attack":27732,"Ġdracon":27733,"LT":27734,"Ġsticker":27735,"rers":27736,"Ġsunny":27737,"Iss":27738,"regulated":27739,"dim":27740,"ĠAbstract":27741,"Ġhusbands":27742,"Office":27743,"omination":27744,"itars":27745,"ANGE":27746,"ascal":27747,"ĠKris":27748,"ĠInfantry":27749,"Ġmalf":27750,"ĠAthe":27751,"ĠRally":27752,"balanced":27753,"........................":27754,"OUP":27755,"Ġmolecule":27756,"metics":27757,"ĠSplit":27758,"ĠInstructions":27759,"ĠNights":27760,"cards":27761,"Ġtug":27762,"Ġcone":27763,"åŃ":27764,"Ġtx":27765,"ĠDiscussion":27766,"Ġcatastrophe":27767,"ppe":27768,"gio":27769,"Ġcommunism":27770,"Ġhalted":27771,"ĠGuant":27772,"clean":27773,"ĠSched":27774,"ĠKanye":27775,"Ġwander":27776,"ĠSeriously":27777,"Ġ188":27778,"ennial":27779,"follow":27780,"productive":27781,"ĠFlow":27782,"ĠSail":27783,"Ġcraw":27784,"Ġsimulations":27785,"oru":27786,"angles":27787,"ĠNolan":27788,"Ġmenstru":27789,"470":27790,"Ġ207":27791,"aja":27792,"Ġcasually":27793,"boarding":27794,"Ġ222":27795,"ovy":27796,"ĠNumbers":27797,"umat":27798,"OE":27799,"287":27800,"ĠClemson":27801,"Ġcerts":27802,"Ġslid":27803,"ĠTribe":27804,"Ġtoast":27805,"Ġfortunes":27806,"Ġfals":27807,"ĠCommittees":27808,"Ġgp":27809,"Ġfiery":27810,"ĠNets":27811,"ĠAnime":27812,"Package":27813,"ĠCompare":27814,"laughter":27815,"infect":27816,"Ġatrocities":27817,"Ġjustices":27818,"Ġinsults":27819,"ĠVernon":27820,"Ġshaken":27821,"Ġpersona":27822,"estamp":27823,"367":27824,"brain":27825,"Ġexperimenting":27826,"Ken":27827,"ĠElectronics":27828,"Ġ161":27829,"domain":27830,"Ġgraphical":27831,"bishop":27832,"Ġwhopping":27833,"ĠEvangel":27834,"Ġadvertisers":27835,"ĠSpear":27836,"Ġbids":27837,"Ġdestroys":27838,"utz":27839,"Ġundersc":27840,"ĠADD":27841,"Ġants":27842,"ĠCum":27843,"ipples":27844,"ĠFill":27845,"Ġglanced":27846,"Ġindicted":27847,"ĠEff":27848,"Ġmiscon":27849,"ĠDesktop":27850,"Ġabide":27851,"ãĥĢ":27852,"ĠIo":27853,"ĠCoul":27854,"Ġcapsule":27855,"ĠChrys":27856,"MON":27857,"Ġundes":27858,"ĠIRA":27859,"Ġcitation":27860,"Ġdictate":27861,"ĠNetworks":27862,"ĠConflict":27863,"ĠStuff":27864,"xa":27865,"isec":27866,"ĠChemistry":27867,"Ġquarterly":27868,"Williams":27869,"anan":27870,"Opt":27871,"ĠAlexandria":27872,"outheastern":27873,"ĠSpringfield":27874,"ĠBlacks":27875,"Ġgeography":27876,"242":27877,"Ġutmost":27878,"ĠExxon":27879,"abouts":27880,"EVA":27881,"ĠEnable":27882,"ĠBarr":27883,"Ġdisagreed":27884,"ĠCyprus":27885,"Ġdementia":27886,"Ġlabs":27887,"Ġubiquitous":27888,"ĠLOVE":27889,"Ġconsolidated":27890,"sr":27891,"Ġcreamy":27892,"ĠTimber":27893,"Regardless":27894,"ĠCertificate":27895,"Ġ\"...":27896,"ogenous":27897,"Captain":27898,"Ġinsulting":27899,"ĠSoros":27900,"ĠInstr":27901,"ĠBulgaria":27902,"better":27903,"Ġsucking":27904,"ĠDavidson":27905,"atz":27906,"Ġcollateral":27907,"gif":27908,"Ġplagued":27909,"ĠCancel":27910,"ĠGardner":27911,"RB":27912,"Ġsixteen":27913,"Remove":27914,"uristic":27915,"cook":27916,"Rod":27917,"Ġcomprising":27918,"fle":27919,")âĢĶ":27920,"ĠViking":27921,"growth":27922,"agonal":27923,"Ġsrf":27924,"afety":27925,"mot":27926,"Nearly":27927,"stown":27928,"ĠFactor":27929,"Ġautomobile":27930,"Ġprocedural":27931,"mask":27932,"ampires":27933,"Ġdisappears":27934,"jab":27935,"315":27936,"Ġ1951":27937,"needed":27938,"Ġdaring":27939,"leader":27940,"Ġpodium":27941,"Ġunhealthy":27942,"Ġmund":27943,"Ġpyramid":27944,"ocre":27945,"Ġkissed":27946,"Ġdreamed":27947,"ĠFantastic":27948,"ĠGly":27949,"åĬ":27950,"Ġgreatness":27951,"Ġspices":27952,"Ġmetropolitan":27953,"Ġcompuls":27954,"iets":27955,"1016":27956,"ĠSham":27957,"ĠPyr":27958,"flies":27959,"ĠMidnight":27960,"Ġswallowed":27961,"Ġgenres":27962,"ĠLucky":27963,"ĠRewards":27964,"Ġdispatch":27965,"ĠIPA":27966,"ĠApply":27967,"Ġaven":27968,"alities":27969,"312":27970,"things":27971,"Ġ().":27972,"Ġmates":27973,"ĠSz":27974,"ĠCOP":27975,"olate":27976,"OFF":27977,"Ġrecharge":27978,"caps":27979,"ĠYorker":27980,"icone":27981,"Ġgalaxies":27982,"ileaks":27983,"Dave":27984,"ĠPuzz":27985,"ĠCeltic":27986,"ĠAFC":27987,"276":27988,"ĠSons":27989,"Ġaffirmative":27990,"Hor":27991,"Ġtutorials":27992,"ĠCITY":27993,"ĠRosa":27994,"ĠExtension":27995,"Series":27996,"Ġfats":27997,"Ġrab":27998,"lis":27999,"Ġunic":28000,"Ġeve":28001,"ĠSpin":28002,"Ġadulthood":28003,"typ":28004,"Ġsectarian":28005,"Ġcheckout":28006,"ĠCycl":28007,"Single":28008,"Ġmartyr":28009,"Ġchilling":28010,"888":28011,"oufl":28012,"Ġ];":28013,"Ġcongestion":28014,"mk":28015,"ĠWhereas":28016,"Ġ1938":28017,"urrencies":28018,"erion":28019,"Ġboast":28020,"ĠPatients":28021,"Ġchap":28022,"ĠBD":28023,"realDonaldTrump":28024,"Ġexamines":28025,"hov":28026,"Ġstartling":28027,"ĠBabylon":28028,"wid":28029,"omew":28030,"brance":28031,"ĠOdyssey":28032,"wig":28033,"Ġtorch":28034,"ĠVox":28035,"ĠMoz":28036,"ĠTroll":28037,"ĠAns":28038,"Similarly":28039,"ĠFul":28040,"006":28041,"Unless":28042,"ĠAlone":28043,"stead":28044,"ĠPublisher":28045,"rights":28046,"tu":28047,"ĠDoesn":28048,"Ġprofessionally":28049,"Ġclo":28050,"icz":28051,"Ġsteals":28052,"Ġá":28053,"1986":28054,"Ġsturdy":28055,"ĠJohann":28056,"Ġmedals":28057,"Ġfilings":28058,"ĠFraser":28059,"done":28060,"Ġmultinational":28061,"Ġfeder":28062,"Ġworthless":28063,"Ġpest":28064,"Yesterday":28065,"ankind":28066,"Ġgays":28067,"Ġborne":28068,"ĠPOS":28069,"Picture":28070,"Ġpercentages":28071,"251":28072,"rame":28073,"Ġpotions":28074,"AMD":28075,"ĠLebanese":28076,"Ġrang":28077,"ĠLSU":28078,"ongs":28079,"Ġpeninsula":28080,"ĠClause":28081,"ALK":28082,"oha":28083,"ĠMacBook":28084,"Ġunanimous":28085,"Ġlenders":28086,"Ġhangs":28087,"Ġfranchises":28088,"orers":28089,"ĠUpdates":28090,"Ġisolate":28091,"andro":28092,"Soon":28093,"Ġdisruptive":28094,"ĠSurve":28095,"Ġstitches":28096,"ĠScorp":28097,"ĠDominion":28098,"Ġsupplying":28099,"Arg":28100,"Ġturret":28101,"ĠLuk":28102,"Ġbrackets":28103,"*)":28104,"ĠRevolutionary":28105,"ĠHonest":28106,"Ġnoticing":28107,"ĠShannon":28108,"Ġafforded":28109,"Ġtha":28110,"ĠJanet":28111,"!--":28112,"ĠNarendra":28113,"ĠPlot":28114,"Hol":28115,"sever":28116,"eenth":28117,"Ġobstruction":28118,"Ġ1024":28119,"staff":28120,"jas":28121,"orget":28122,"scenes":28123,"laughs":28124,"ĠFargo":28125,"crime":28126,"Ġorchestr":28127,"Ġdelet":28128,"iliary":28129,"rieved":28130,"Ġmilitar":28131,"ĠGreene":28132,"âĹı":28133,"ãģ¦":28134,"ĠGuards":28135,"Ġunleashed":28136,"ĠWeber":28137,"Ġadjustable":28138,"Ġcaliber":28139,"Ġmotivations":28140,"ĠÃł":28141,"mAh":28142,"ĠLanka":28143,"handle":28144,"Ġpent":28145,"ĠRav":28146,"ĠAngular":28147,"ĠKau":28148,"umbing":28149,"Ġphilanthrop":28150,"Ġdehyd":28151,"Ġtoxicity":28152,"eer":28153,"ĠYORK":28154,"witz":28155,"å¼":28156,"ĠIE":28157,"community":28158,"ĠAH":28159,"Ġretali":28160,"Ġmassively":28161,"ĠDaniels":28162,"ĠDEL":28163,"Ġcarcin":28164,"Url":28165,"Ġrouting":28166,"ĠNPCs":28167,"ĠRAF":28168,"ryce":28169,"Ġwaived":28170,"ĠGuatem":28171,"Everybody":28172,"Ġcovenant":28173,"Ġ173":28174,"Ġrelaxing":28175,"Ġquart":28176,"almost":28177,"Ġguarded":28178,"ĠSoldiers":28179,"ĠPLAY":28180,"Ġoutgoing":28181,"LAND":28182,"Ġrewrite":28183,"ĠMOV":28184,"ĠImper":28185,"ĠSolution":28186,"Ġphenomenal":28187,"Ġlongevity":28188,"Ġimpat":28189,"ĠNissan":28190,"irie":28191,"Ġodor":28192,"ĠZar":28193,"oks":28194,"Ġmilitias":28195,"ĠSPEC":28196,"Ġtolerated":28197,"arser":28198,"ĠBradford":28199,"+,":28200,"Ġsurreal":28201,"sf":28202,"Canadian":28203,"Ġresemblance":28204,"Ġcarbohydrate":28205,"VIEW":28206,"Ġaccessory":28207,"meal":28208,"largest":28209,"iegel":28210,"Someone":28211,"Ġtoughest":28212,"oso":28213,"Ġfunnel":28214,"Ġcondemnation":28215,"luent":28216,"Ġwired":28217,"ĠSunset":28218,"Jesus":28219,"ĠPST":28220,"ĠPages":28221,"ĠTycoon":28222,"ĠPF":28223,"Ġselections":28224,"Ġà¤":28225,"partisan":28226,"Ġhighs":28227,"ĠRune":28228,"Ġcrafts":28229,"lead":28230,"ĠParents":28231,"Ġreclaim":28232,"eker":28233,"ĠAllied":28234,"aeper":28235,"Ġlooming":28236,"Ġbeneficiaries":28237,"ĠHull":28238,"Students":28239,"Jewish":28240,"dj":28241,"Ġpact":28242,"template":28243,"ĠOfficials":28244,"ĠBaylor":28245,"Ġhemp":28246,"Ġyouths":28247,"ĠLevels":28248,"ĠXiao":28249,"ĠChes":28250,"Ġendeavor":28251,"ĠRemoved":28252,"Ġhippocamp":28253,"Hell":28254,"ãĤĬ":28255,"805":28256,"Ġdinosaur":28257,"ĠWrath":28258,"ĠIndonesian":28259,"Ġcalculator":28260,"ĠDictionary":28261,"Ġ420":28262,"ĠMAG":28263,"(_":28264,"!,":28265,"tarians":28266,"Ġrestricting":28267,"racuse":28268,"Ġweekday":28269,"OUNT":28270,"Ġshrugged":28271,"leground":28272,"Ġbald":28273,"ĠDoctors":28274,"Ġtouted":28275,"ĠMaxwell":28276,"Ġ214":28277,"Ġdiplomat":28278,"Ġrepression":28279,"Ġconstituency":28280,"vice":28281,"ranked":28282,"ĠNapoleon":28283,"gang":28284,"ĠForever":28285,"tun":28286,"Ġbulb":28287,"ĠPDT":28288,"ĠCisco":28289,"VEN":28290,"Ġresumed":28291,"Steven":28292,"ĠManitoba":28293,"Ġfabulous":28294,"ĠAgents":28295,"1984":28296,"Ġamusing":28297,"ĠMysteries":28298,"Ġorthodox":28299,"floor":28300,"Ġquestionnaire":28301,"Ġpenetrate":28302,"Ġfilmmakers":28303,"ĠUnc":28304,"Ġstamped":28305,"Ġthirteen":28306,"Ġoutfield":28307,"Ġforwarded":28308,"Ġappra":28309,"Ġaided":28310,"try":28311,"Ġunfocused":28312,"ĠLiz":28313,"ĠWendy":28314,"ĠScene":28315,"Charg":28316,"Ġrejects":28317,"Ġleftist":28318,"ĠProvidence":28319,"ĠBrid":28320,"regn":28321,"Ġprophecy":28322,"ĠLIVE":28323,"499":28324,"Ġforge":28325,"ĠFML":28326,"Ġintrinsic":28327,"ĠFrog":28328,"Ġwont":28329,"ĠHolt":28330,"Ġfamed":28331,"CLUS":28332,"aepernick":28333,"ĠHate":28334,"ĠCay":28335,"Ġregistering":28336,"ortality":28337,"ropy":28338,"ocalyptic":28339,"aan":28340,"nav":28341,"Ġfascist":28342,"IFIED":28343,"Ġimplicated":28344,"ĠResort":28345,"ĠChandler":28346,"ĠBrick":28347,"Pin":28348,"ysc":28349,"Usage":28350,"ĠHelm":28351,"usra":28352,"âĺħâĺħ":28353,"ĠAbbas":28354,"Ġunanimously":28355,"Ġkeeper":28356,"Ġaddicted":28357,"???":28358,"Ġhelmets":28359,"Ġantioxid":28360,"apsed":28361,"808":28362,"giene":28363,"Ġwaits":28364,"Ġminion":28365,"raved":28366,"ĠPorsche":28367,"Ġdreaming":28368,"Ġ171":28369,"ĠCain":28370,"Ġunfor":28371,"asso":28372,"ĠConfiguration":28373,"kun":28374,"hardt":28375,"Ġnested":28376,"ĠLDS":28377,"LES":28378,"Ġtying":28379,"enos":28380,"Ġcue":28381,"ĠMarqu":28382,"skirts":28383,"Ġclicked":28384,"Ġexpiration":28385,"ĠAccordingly":28386,"ĠWC":28387,"Ġblessings":28388,"Ġaddictive":28389,"ĠNarr":28390,"yx":28391,"ĠJaguars":28392,"Ġrents":28393,"ĠSiber":28394,"Ġtipped":28395,"ousse":28396,"ĠFitzgerald":28397,"Ġhierarch":28398,"outine":28399,"Ġwavelength":28400,">.":28401,"chid":28402,"ĠProcessing":28403,"/+":28404,"ranking":28405,"Easy":28406,"ĠConstruct":28407,"Ġtet":28408,"insured":28409,"HUD":28410,"Ġquoting":28411,"Ġcommunicated":28412,"inx":28413,"Ġinmate":28414,"Ġerected":28415,"ĠAbsolutely":28416,"ĠSurely":28417,"Ġunim":28418,"ĠThrone":28419,"heid":28420,"Ġclaws":28421,"Ġsuperstar":28422,"ĠLenn":28423,"ĠWhis":28424,"Uk":28425,"abol":28426,"Ġsket":28427,"ĠNiet":28428,"Ġperks":28429,"Ġaffinity":28430,"Ġopenings":28431,"phasis":28432,"Ġdiscriminate":28433,"Tip":28434,"vc":28435,"Ġgrinding":28436,"ĠJenny":28437,"Ġasthma":28438,"holes":28439,"ĠHomer":28440,"Ġregisters":28441,"ĠGlad":28442,"Ġcreations":28443,"Ġlithium":28444,"Ġapplause":28445,"until":28446,"Justice":28447,"ĠTurks":28448,"Ġscandals":28449,"Ġbake":28450,"tank":28451,"Mech":28452,"ĠMeans":28453,"ĠMaid":28454,"Republicans":28455,"isal":28456,"windows":28457,"ĠSantos":28458,"Ġvegetation":28459,"338":28460,"tri":28461,"Ġflux":28462,"insert":28463,"Ġclarified":28464,"Ġmortg":28465,"ĠChim":28466,"ĠTort":28467,"Ġdisclaim":28468,"metal":28469,"ĠAside":28470,"Ġinduction":28471,"Ġinfl":28472,"Ġatheists":28473,"amph":28474,"Ġether":28475,"ĠVital":28476,"ĠBuilt":28477,"Mind":28478,"Ġweaponry":28479,"SET":28480,"Ġ186":28481,"admin":28482,"gam":28483,"contract":28484,"afa":28485,"Ġderivatives":28486,"Ġsnacks":28487,"Ġchurn":28488,"Econom":28489,"Ġcapped":28490,"ĠUnderstanding":28491,"ĠHers":28492,"ĠIz":28493,"Ġduct":28494,"IENT":28495,"aughty":28496,"ĠâľĶ":28497,"ĠNP":28498,"Ġsailing":28499,"Initialized":28500,"Ġted":28501,"Ġreactors":28502,"ĠLomb":28503,"Ġchoke":28504,"ĠWorm":28505,"Ġadmiration":28506,"Ġswung":28507,"ensibly":28508,"Ġrash":28509,"ĠGoals":28510,"ĠImportant":28511,"Shot":28512,"ĠRas":28513,"Ġtrainers":28514,"ĠBun":28515,"Working":28516,"Ġharmed":28517,"ĠPandora":28518,"ĠLTE":28519,"Ġmushroom":28520,"ĠCHAR":28521,"ĠFee":28522,"ĠMoy":28523,"Born":28524,"oliberal":28525,"ĠMartial":28526,"Ġgentlemen":28527,"Ġlingering":28528,"Official":28529,"Ġgraffiti":28530,"ĠNames":28531,"Der":28532,"Ġquint":28533,"istrate":28534,"azeera":28535,"ĠNOTICE":28536,"ĠFlorence":28537,"Ġpayable":28538,"Ġdepicts":28539,"ĠSpecies":28540,"Heart":28541,"âĶĢâĶĢâĶĢâĶĢâĶĢâĶĢâĶĢâĶĢ":28542,"Ġenclosed":28543,"Increases":28544,"Daily":28545,"ĠLis":28546,"Ġenactment":28547,"ĠBacon":28548,"ĠSteele":28549,"demand":28550,"Ġ183":28551,"Ġmouths":28552,"Ġstranded":28553,"Ġenhancement":28554,"011":28555,"ĠWhats":28556,"Ġhealed":28557,"eny":28558,"ĠRab":28559,"Ġ340":28560,"ĠLabyrinth":28561,"roach":28562,"ĠYosh":28563,"ĠClippers":28564,"Ġconcerts":28565,"Internet":28566,"355":28567,"Ġstickers":28568,"Ġtermed":28569,"ĠAxe":28570,"Ġgrandparents":28571,"France":28572,"ĠClim":28573,"ĠUh":28574,"ulic":28575,"Ġthrill":28576,"centric":28577,"ĠOverview":28578,"ĠConduct":28579,"Ġsubstantive":28580,"Ġ182":28581,"mur":28582,"Ġstray":28583,"ĠCoff":28584,"Ġrepetitive":28585,"ĠForgotten":28586,"Ġqualification":28587,"ewitness":28588,"ĠZimbabwe":28589,"Ġsimulated":28590,"ĠJD":28591,"253":28592,"ĠWare":28593,"Ġunsc":28594,"Times":28595,"Ġsummons":28596,"Ġdisconnected":28597,"Ġ184":28598,"cius":28599,"ĠGujar":28600,"odka":28601,"Ġerase":28602,"ĠTobacco":28603,"elected":28604,"Ġuncont":28605,"ĠShepard":28606,"ĠLamp":28607,"Ġalerted":28608,"Ġoperative":28609,"arna":28610,"uint":28611,"Ġnegligence":28612,"acements":28613,"Ġsupra":28614,"Ġprevail":28615,"ĠShark":28616,"Ġbelts":28617,"ãģ«":28618,"Ġtighter":28619,"Engineers":28620,"Ġinactive":28621,"Ġexponent":28622,"ĠWillie":28623,"aples":28624,"Ġheir":28625,"ĠHits":28626,"iann":28627,"ĠSays":28628,"Ġcurrents":28629,"ĠBengal":28630,"Ġarist":28631,"Buffer":28632,"Ġbreeze":28633,"ĠWesley":28634,"Cola":28635,"Ġpronoun":28636,"Ġdeed":28637,"ĠKling":28638,"Ġoft":28639,"Ġinflict":28640,"Ġpunishing":28641,"Ġnm":28642,"iku":28643,"ODUCT":28644,"014":28645,"Ġsubsidy":28646,"ĠDEA":28647,"ĠHerbert":28648,"ĠJal":28649,"Bank":28650,"Ġdeferred":28651,"Ġshipment":28652,"Bott":28653,"Ġalle":28654,"bearing":28655,"HTML":28656,"Offline":28657,"Ġ213":28658,"Ġscrolling":28659,"Ġscanned":28660,"ĠLibyan":28661,"ĠTOP":28662,"chrom":28663,"dt":28664,"column":28665,"PsyNetMessage":28666,"Zero":28667,"Ġtorso":28668,"050":28669,"âķIJ":28670,"Ġimperson":28671,"ĠSchwartz":28672,"udic":28673,"Ġpissed":28674,"ĠSapp":28675,"257":28676,"ĠISPs":28677,"ogl":28678,"Ġsupervised":28679,"Ġadolescent":28680,"Ġattained":28681,"ĠDelivery":28682,"ĠBunny":28683,"Ġ1937":28684,"Ġminiature":28685,"Ġos":28686,"Ġ370":28687,"608":28688,"ĠMourinho":28689,"Ġinnate":28690,"Ġtempo":28691,"ĠNM":28692,"ĠFallen":28693,"009":28694,"Ġprovocative":28695,"Streamer":28696,"ĠBenedict":28697,"ĠBolshe":28698,"Ġturtle":28699,"ĠPCB":28700,"ĠEqual":28701,"Director":28702,"ĠRend":28703,"Ġfluids":28704,"Authorities":28705,"Ġcousins":28706,"requency":28707,"ĠNeighbor":28708,"sets":28709,"shared":28710,"Charles":28711,"password":28712,"Ġgears":28713,"Ġ211":28714,"ĠHardware":28715,"rika":28716,"Ġupstream":28717,"Hom":28718,"Ġdisproportionately":28719,"ivities":28720,"Ġundefined":28721,"Ġelectrons":28722,"Ġcommemor":28723,"Eventually":28724,"Ġ><":28725,"Ġirresponsible":28726,"218":28727,"ĠReleased":28728,"ĠOVER":28729,"ĠIGN":28730,"ĠBread":28731,"stellar":28732,"ĠSage":28733,"tted":28734,"damage":28735,"edition":28736,"ĠPrec":28737,"Ġlime":28738,"Ġconfinement":28739,"Ġcalorie":28740,"weapon":28741,"Ġdiffering":28742,"ĠSina":28743,"mys":28744,"amd":28745,"Ġintricate":28746,"kk":28747,"ĠPAT":28748,"ão":28749,"stones":28750,"links":28751,"Ġranch":28752,"Semitic":28753,"Ġdifferentiate":28754,"ĠSinger":28755,"occupied":28756,"Ġfortress":28757,"cmd":28758,"Ġinterception":28759,"ĠAnkara":28760,"Ġrept":28761,"ĠSolitaire":28762,"Ġremake":28763,"pred":28764,"Ġdared":28765,"autions":28766,"ĠBACK":28767,"Running":28768,"Ġdebugging":28769,"Ġgraphs":28770,"399":28771,"ĠNigel":28772,"Ġbun":28773,"Ġpillow":28774,"Ġprogressed":28775,"fashioned":28776,"Ġobedience":28777,"ERN":28778,"Ġrehears":28779,"Cell":28780,"tl":28781,"Sher":28782,"Ġherald":28783,"ĠPayment":28784,"ĠCory":28785,"ĠDept":28786,"Ġrepent":28787,"ĠWeak":28788,"uckland":28789,"Ġpleasing":28790,"Ġshortages":28791,"Ġjurors":28792,"ĠKab":28793,"qqa":28794,"Anti":28795,"Ġwow":28796,"ĠRCMP":28797,"Ġtsun":28798,"ĠSic":28799,"Ġcomprises":28800,"Ġspies":28801,"Ġprecinct":28802,"nu":28803,"Ġurges":28804,"Ġtimed":28805,"Ġstripes":28806,"ĠBoots":28807,"Ġyen":28808,"Advanced":28809,"Ġdiscrete":28810,"ĠArchangel":28811,"employment":28812,"Diff":28813,"Ġmonuments":28814,"Ġ209":28815,"worker":28816,"Ġ196":28817,"ĠIg":28818,"utterstock":28819,"TPS":28820,"Jac":28821,"Ġhomelessness":28822,"Ġcommentator":28823,"Ġracially":28824,"fing":28825,"seed":28826,"Ele":28827,"ellation":28828,"Ġethanol":28829,"Ġparish":28830,"ĠDong":28831,"ĠAwakening":28832,"Ġdeviation":28833,"ĠBearing":28834,"ĠTsuk":28835,"Ġrecess":28836,"Ġlymph":28837,"ĠCannabis":28838,"åľ":28839,"ĠNEWS":28840,"Ġdra":28841,"ĠStefan":28842,"ĠWrong":28843,"ĠSAM":28844,"Ġloosely":28845,"Ġinterpreter":28846,"ĠPlain":28847,"Government":28848,"Ġbigotry":28849,"Ġgrenades":28850,"avez":28851,"pictured":28852,"Ġmandated":28853,"ĠMonk":28854,"ĠPedro":28855,"Ġlava":28856,"274":28857,"Ġcynical":28858,"ĠScrolls":28859,"locks":28860,"Mp":28861,"Ġcongregation":28862,"ornings":28863,"phil":28864,"ĠIbid":28865,"Ġferv":28866,"Ġdisappearing":28867,"Ġarrogant":28868,"syn":28869,"ĠMaver":28870,"ĠSuit":28871,"241":28872,"Ġabbre":28873,"ackers":28874,"Pa":28875,"ĠYel":28876,"Whenever":28877,"Ġ235":28878,"ĠVine":28879,"ĠAnat":28880,"Ġextinct":28881,"LET":28882,"Ġexecutable":28883,"VERS":28884,"oxide":28885,"DNA":28886,"ĠPrel":28887,"Ġresentment":28888,"Ġcomprise":28889,"ĠAviv":28890,"Ġinterceptions":28891,"Ġprolific":28892,"INA":28893,"ĠErin":28894,"thought":28895,"219":28896,"ĠPsychiatry":28897,"unky":28898,"chemist":28899,"Ho":28900,"ĠMcCoy":28901,"Ġbricks":28902,"Los":28903,"rily":28904,"ĠUSSR":28905,"Ġrud":28906,"Ġlaud":28907,"ĠWise":28908,"ĠEmerald":28909,"Ġrevived":28910,"Ġdamned":28911,"ĠRepair":28912,"idem":28913,"ctica":28914,"Ġpatriarch":28915,"ĠNurs":28916,"meg":28917,"Ġcheapest":28918,"reements":28919,"empty":28920,"ĠCelebr":28921,"Ġdeprivation":28922,"chanted":28923,"ĠThumbnails":28924,"Energy":28925,"ĠEthan":28926,"ĠQing":28927,"Ġopposes":28928,"WIND":28929,"vik":28930,"ĠMau":28931,"ĠSUB":28932,"667":28933,"GRE":28934,"ĠVolunte":28935,"nton":28936,"Cook":28937,"åIJ":28938,"esque":28939,"Ġplummet":28940,"Ġsuing":28941,"Ġpronounce":28942,"Ġresisting":28943,"ĠFishing":28944,"ĠTrials":28945,"Ġyell":28946,"Ġ310":28947,"Ġinduct":28948,"Ġpersonalized":28949,"often":28950,"Reb":28951,"EMBER":28952,"Ġviewpoint":28953,"Ġexistential":28954,"())":28955,"remove":28956,"MENTS":28957,"lasses":28958,"Ġevapor":28959,"Ġaisle":28960,"meta":28961,"Ġreflective":28962,"Ġentitlement":28963,"Ġdevised":28964,"music":28965,"ascade":28966,"Ġwinding":28967,"offset":28968,"Ġaccessibility":28969,"kered":28970,"Better":28971,"ĠJohnston":28972,"thinking":28973,"Snow":28974,"ĠCroatia":28975,"ĠAtomic":28976,"271":28977,"348":28978,"Ġtextbook":28979,"ĠSixth":28980,"ĠاÙĦ":28981,"Ġslider":28982,"ĠBurger":28983,"bol":28984,"Sync":28985,"Ġgrandchildren":28986,"Ġcerv":28987,"+)":28988,"Ġeternity":28989,"Ġtweeting":28990,"Ġspeculative":28991,"Ġpivotal":28992,"ĠWP":28993,"ĠTER":28994,"ynamic":28995,"Ġupl":28996,"ĠCats":28997,"perhaps":28998,"Ġclassmates":28999,"Ġblatant":29000,"'-":29001,"Ġlakh":29002,"antine":29003,"ĠBorg":29004,"iom":29005,"/(":29006,"ĠAthletic":29007,"Ġsar":29008,"OTA":29009,"ĠHoffman":29010,"Nevertheless":29011,"Ġadorable":29012,"Ġspawned":29013,"Associated":29014,"ĠDomestic":29015,"Ġimplant":29016,"ĠLuxem":29017,"ĠKens":29018,"Ġpumps":29019,"ĠSAT":29020,"Attributes":29021,"509":29022,"avour":29023,"Ġcentralized":29024,"ĠTN":29025,"Ġfreshly":29026,"ĠAchieve":29027,"Ġoutsiders":29028,"herty":29029,"ĠRee":29030,"ĠTowers":29031,"ĠDart":29032,"akable":29033,"Ġmp":29034,"ĠHeavenly":29035,"Ġripe":29036,"ĠCaroline":29037,"ryan":29038,"Ġclassics":29039,"Ġretiring":29040,"Ġ228":29041,"Ġah":29042,"Ġdealings":29043,"Ġpunching":29044,"ĠChapman":29045,"Options":29046,"maxwell":29047,"volume":29048,"Ġstal":29049,"Ġexported":29050,"ĠQuite":29051,"Ġnumerical":29052,"Burn":29053,"Fact":29054,"ĠKeystone":29055,"Ġtrending":29056,"Ġaltering":29057,"ĠAfricans":29058,"478":29059,"ĠMN":29060,"ĠKnock":29061,"Ġtemptation":29062,"Ġprestige":29063,"Overview":29064,"ĠTraditional":29065,"ĠBahrain":29066,"Private":29067,"ĠHOU":29068,"Ġbarr":29069,"ĠTat":29070,"Cube":29071,"USD":29072,"ĠGrande":29073,"ĠGat":29074,"ĠFlo":29075,"Ġresides":29076,"Ġindec":29077,"volent":29078,"Ġperpetual":29079,"ubes":29080,"Ġworldview":29081,"ĠQuantum":29082,"Ġfiltered":29083,"Ġensu":29084,"orgetown":29085,"ERSON":29086,"ĠMild":29087,"379":29088,"OTT":29089,"Ã¥":29090,"Ġvitamins":29091,"Ġribbon":29092,"Ġsincerely":29093,"ĠHin":29094,"Ġeighteen":29095,"Ġcontradictory":29096,"Ġglaring":29097,"Ġexpectancy":29098,"Ġconspir":29099,"Ġmonstrous":29100,"Ġ380":29101,"reci":29102,"Ġhandic":29103,"Ġpumped":29104,"Ġindicative":29105,"Ġrapp":29106,"Ġavail":29107,"ĠLEGO":29108,"ĠMarijuana":29109,"1985":29110,"erton":29111,"Ġtwentieth":29112,"################################":29113,"ĠSwamp":29114,"Ġvaluation":29115,"Ġaffiliates":29116,"adjusted":29117,"ĠFacility":29118,"262":29119,"Ġenzymes":29120,"itudinal":29121,"Ġimprint":29122,"Site":29123,"Ġinstaller":29124,"ĠTRA":29125,"mology":29126,"linear":29127,"ĠCollective":29128,"igating":29129,"ĠToken":29130,"Ġspeculated":29131,"KN":29132,"ĠCly":29133,"ority":29134,"Ġdefer":29135,"Ġinspectors":29136,"approved":29137,"RM":29138,"ĠSuns":29139,"Ġinforming":29140,"ĠSyracuse":29141,"ibli":29142,"765":29143,"Ġglove":29144,"Ġauthorize":29145,"âĢ¦âĢ¦âĢ¦âĢ¦âĢ¦âĢ¦âĢ¦âĢ¦":29146,"ĠCruise":29147,"Ġcontracting":29148,"shell":29149,"IFE":29150,"ĠJewel":29151,"pract":29152,"ĠPhotoshop":29153,"ĠKnowing":29154,"harm":29155,"Ġattractions":29156,"adan":29157,"etus":29158,"018":29159,"wagen":29160,"Alt":29161,"Ġmultiply":29162,"Ġequilibrium":29163,":{":29164,"ĠFighters":29165,"ĠEdgar":29166,"Ġfourteen":29167,"Govern":29168,"Ġmisuse":29169,"Ġabusing":29170,"Ġancestry":29171,"ramer":29172,"644":29173,"Ġworms":29174,"Ġthicker":29175,"ĠCombine":29176,"Ġpeasants":29177,"Ġvind":29178,"Ġconquest":29179,"Ġmocked":29180,"Ġcinnamon":29181,"ĠCald":29182,"ĠGallup":29183,"Ġavoidance":29184,"Ġincarnation":29185,"ĠStrat":29186,"Ġtasted":29187,"enta":29188,"ĠNeal":29189,"pared":29190,"Ġterminology":29191,"jection":29192,"Scientists":29193,"ĠINS":29194,"ĠDee":29195,"Ġdirectories":29196,"Road":29197,"ĠShap":29198,"bright":29199,"ĠDirectors":29200,"ĠColumn":29201,"Ġbob":29202,"Ġpreferably":29203,"Ġglitch":29204,"furt":29205,"Ġeg":29206,"idis":29207,"CBC":29208,"Ġsurrendered":29209,"Ġtestament":29210,"336":29211,"uggest":29212,"ĠNil":29213,"another":29214,"Ġpathetic":29215,"ĠDonna":29216,"Ġ218":29217,"ĠAvery":29218,"Ġwhiskey":29219,"Ġfixture":29220,"ĠConquest":29221,"Ġbets":29222,"Occ":29223,"ĠLeicester":29224,"].\"":29225,"Ġ));":29226,"Ġflashes":29227,"456":29228,"Ġmasked":29229,"gebra":29230,"Ġcomputed":29231,"chel":29232,"auder":29233,"Ġdefeats":29234,"ĠLiberation":29235,"ĠOsama":29236,"ĠVive":29237,"Changes":29238,"Channel":29239,"Ġtariffs":29240,"Ġmage":29241,"ĠSax":29242,"Ġinadvertently":29243,"ĠCRE":29244,"ĠReaper":29245,"inky":29246,"grading":29247,"Ġstereotyp":29248,"Ġcurl":29249,"ĠFANT":29250,"Ġframeworks":29251,"Mom":29252,"ĠAnch":29253,"Ġflavour":29254,"carbon":29255,"Ġpermitting":29256,"letcher":29257,"ĠMozilla":29258,"ĠParking":29259,"ĠChamp":29260,"Scroll":29261,"Ġmurderer":29262,"Ġrested":29263,"Ġowes":29264,"ĠPoss":29265,"ADD":29266,"IFF":29267,"resolution":29268,"ĠMining":29269,"Ġcomparative":29270,"Dim":29271,"Ġneighbouring":29272,"ĠAST":29273,"ĠToxic":29274,"Ġbiases":29275,"Ġgunfire":29276,"urous":29277,"ĠMoment":29278,"1983":29279,"Ġpervasive":29280,"ttp":29281,"ĠNormally":29282,"rir":29283,"Sarah":29284,"ĠAlbany":29285,"Ġunsett":29286,"ĠSMS":29287,"ipers":29288,"layer":29289,"ĠWhites":29290,"uple":29291,"Ġturbo":29292,"ĠLeeds":29293,"Ġthats":29294,"ĠMiner":29295,"MER":29296,"ĠReign":29297,"Ġperme":29298,"ĠBlitz":29299,"Ġ1934":29300,"Ġintimidating":29301,"tube":29302,"Ġeccentric":29303,"abolic":29304,"boxes":29305,"ĠAssociates":29306,"votes":29307,"Ġsimulate":29308,"umbo":29309,"astery":29310,"Ġshipments":29311,"FFFF":29312,"anth":29313,"Ġseasoned":29314,"Ġexperimentation":29315,"âĸł":29316,"laws":29317,"Meet":29318,"iddles":29319,"antics":29320,"Rating":29321,"ISIS":29322,"hift":29323,"Ġfronts":29324,"buf":29325,"017":29326,"Ġunatt":29327,"ĠDil":29328,"leases":29329,"ĠGardens":29330,"777":29331,"touch":29332,"vell":29333,"458":29334,"Ġ=====":29335,"saving":29336,"Ġerosion":29337,"ĠQuin":29338,"Ġearns":29339,"Ġaccomplishment":29340,"ĠWei":29341,"Ġ<[":29342,"_____":29343,"Ġirrig":29344,"ĠTeddy":29345,"Ġconquered":29346,"ĠArmored":29347,"Ġasserts":29348,"Ġmanipulating":29349,"ré":29350,"Ġtranscripts":29351,"Gallery":29352,"Ġplotting":29353,"Neil":29354,"Ġbetrayal":29355,"loader":29356,"ĠSul":29357,"Ġdisplacement":29358,"Ġroyalty":29359,"ĠWI":29360,"heit":29361,"ĠDevices":29362,"allel":29363,"Ġmunicipalities":29364,"Ġcanal":29365,"Stars":29366,"ĠUAE":29367,"Ġ\"âĢ¦":29368,"ĠCU":29369,"above":29370,"Ġresonance":29371,"ĠguiActiveUn":29372,"added":29373,"ĠBraves":29374,"ĠIbn":29375,"Ġhereby":29376,"ĠBRE":29377,"Ġshareholder":29378,"ĠHir":29379,"ĠJi":29380,"Ġstrangely":29381,"Ġadmired":29382,"Ġplight":29383,"Ġbachelor":29384,"ĠPole":29385,"ciplinary":29386,"Tony":29387,"ĠArmenian":29388,"Ġunman":29389,"ĠZionist":29390,"Stage":29391,"iscover":29392,"Ġautomotive":29393,"Ġsidelines":29394,"Ġslick":29395,"ĠRenaissance":29396,"ĠFUN":29397,"Images":29398,"ĠHaj":29399,"Ġping":29400,"Ġshortcut":29401,"ĠBlvd":29402,"ĠLooks":29403,"Ġbursts":29404,"Ġclamp":29405,"Ġmish":29406,"Ġsorting":29407,"Ġpatriot":29408,"Ġcorrectness":29409,"ĠScandinav":29410,"ĠCavaliers":29411,"python":29412,"azar":29413,"Ġ375":29414,"ĠJaune":29415,"409":29416,"Ġdetrimental":29417,"Ġstabbing":29418,"Ġpoisoned":29419,"Ġfountain":29420,"ocent":29421,"orst":29422,"ĠMari":29423,"Ġrains":29424,"ĠOvers":29425,"ĠInstitution":29426,"udget":29427,"AMY":29428,"tale":29429,"ĠKR":29430,"ĠPrices":29431,"Ġheadaches":29432,"Ġlandsl":29433,"ĠAura":29434,"Bonus":29435,"ĠZhao":29436,"ĠHip":29437,"Ġhops":29438,"ĠKurdistan":29439,"Ġexploiting":29440,"ryn":29441,"Ġhypocrisy":29442,"opening":29443,"Ġgunshot":29444,"Ġwed":29445,"interstitial":29446,"Interstitial":29447,"Ġamen":29448,"Breaking":29449,"Ġmarketed":29450,"Wire":29451,"ĠCrowd":29452,"Continue":29453,"ĠKnown":29454,"ĠEffective":29455,"orean":29456,"izons":29457,"Joseph":29458,"Ġescalation":29459,"username":29460,"Ġcurtain":29461,"ATES":29462,"ĠPAR":29463,"ĠMiy":29464,"Ġcounterfe":29465,"lene":29466,"Ġcontenders":29467,"daily":29468,"ĠAsc":29469,"ĠPhillip":29470,"mostly":29471,"Ġfilename":29472,"hene":29473,"Ġresembling":29474,"Ġstaging":29475,"ĠChloe":29476,"Ġwiring":29477,"Hon":29478,"ĠRenew":29479,"ottage":29480,"ĠHybrid":29481,"much":29482,"Ġstrokes":29483,"Ġpolicymakers":29484,"APTER":29485,"ĠArkham":29486,"plot":29487,"Ġassistants":29488,"Ġdeport":29489,"ĠSega":29490,"Ġinfluenza":29491,"ĠCursed":29492,"ĠKobe":29493,"Ġskinny":29494,"Provider":29495,"ĠRip":29496,"Ġincremental":29497,"products":29498,"BF":29499,"Ġdome":29500,"ĠCredits":29501,"Ġlosers":29502,"ints":29503,"ĠBetty":29504,"ĠTalent":29505,"ĠDAM":29506,"Lv":29507,"Ess":29508,"Ġdens":29509,"temp":29510,"Judge":29511,"odic":29512,"Ġ'(":29513,"URES":29514,"etsk":29515,"VO":29516,"Ġretrieved":29517,"Ġarchitects":29518,"Ùĩ":29519,"Ġethic":29520,"ĠSecondary":29521,"stocks":29522,"adia":29523,"Ġ325":29524,"ĠOpinion":29525,"Ġsimultaneous":29526,"Ġdizz":29527,"ulp":29528,"Ġsmuggling":29529,"ippery":29530,"Random":29531,"facing":29532,"ĠDas":29533,"Ġstockp":29534,"Ġdisclosures":29535,"pointer":29536,"Ġcoral":29537,"ĠSelection":29538,"ĠPike":29539,"ivalent":29540,"Ġruthless":29541,"ĠRim":29542,"Ġensuing":29543,"ĠExperiment":29544,"Ġcongressman":29545,"Ġbeliever":29546,"Ġunspecified":29547,"ĠMord":29548,"Ġknowledgeable":29549,"ĠVERY":29550,"TX":29551,"Ġstraps":29552,"Ġturf":29553,"apeshifter":29554,"Ġmarital":29555,"Ġflock":29556,"ãģĨ":29557,"263":29558,"AMES":29559,"ĠOpposition":29560,"Ġtreasures":29561,"ĠGOD":29562,"Ġmodeled":29563,"ĠWORLD":29564,"Ġ([":29565,"ĠUsage":29566,"HF":29567,"Ġ$(":29568,"ussed":29569,"Ġpioneer":29570,"Eight":29571,"parse":29572,"bread":29573,"ritz":29574,"ĠMiranda":29575,"ĠKant":29576,"++)":29577,"oren":29578,"Ġprovoked":29579,"Ġbreeds":29580,"ĠIncludes":29581,"ĠPastebin":29582,"ĠFlip":29583,"Java":29584,"Ġbrink":29585,"Ġrumored":29586,"Ġunseen":29587,"Ġgarnered":29588,"ĠDefin":29589,"alted":29590,"Ġtattoos":29591,"Ġhesitation":29592,"isitions":29593,"ĠWeaver":29594,"ĠReporting":29595,"Ġtherapies":29596,"Ġconsultants":29597,"Ġresidual":29598,"ĠMali":29599,"ĠRoma":29600,"iago":29601,"ĠResidents":29602,"ubi":29603,"Ġremedies":29604,"Ġadaptive":29605,"ĠAlive":29606,"ĠBarcl":29607,"Ġwallets":29608,"crypt":29609,"etermination":29610,"ĠPelosi":29611,"Ġslipping":29612,"otonin":29613,"Ġalliances":29614,"patrick":29615,"iris":29616,"Ġorth":29617,"ĠPerkins":29618,"ĠDeV":29619,"ĠGets":29620,"Ġdrying":29621,"gee":29622,"forest":29623,"ĠForget":29624,"orem":29625,"339":29626,"Ġvaguely":29627,"ĠDion":29628,"ĠPorn":29629,"ĠHOW":29630,"Ġpneum":29631,"Ġrubble":29632,"ĠTaste":29633,"encia":29634,"ĠGel":29635,"Ġdst":29636,"Ġ245":29637,"ĠMorocco":29638,"inflamm":29639,"ĠTwins":29640,"Ġbots":29641,"daughter":29642,"ĠBalk":29643,"Ġbrethren":29644,"Ġlogos":29645,"Ġgobl":29646,"fps":29647,"Ġsubdivision":29648,"Ġpawn":29649,"Ġsqueezed":29650,"Ġmorale":29651,"ĠDW":29652,"'\"":29653,"Ġknot":29654,"ooky":29655,"Ġdivisive":29656,"Ġboosted":29657,"chy":29658,"ãĥIJ":29659,"ifact":29660,"Ġnewcomers":29661,"ĠWrestling":29662,"Ġscouts":29663,"wolves":29664,"Rat":29665,"Ġnineteenth":29666,"ĠOsborne":29667,"Stats":29668,"Ġempowered":29669,"Ġpsychopath":29670,"ĠOEM":29671,"uggage":29672,"ĠPK":29673,"ĠMohammad":29674,"Pak":29675,"Ġanarchists":29676,"ĠExtract":29677,"esthes":29678,"ĠStockholm":29679,"loo":29680,"ĠGraph":29681,"Ġdeploying":29682,"ĠStranger":29683,"ĠMold":29684,"Ġstaffer":29685,"Ġdiscounted":29686,"uckle":29687,"please":29688,"ĠLanding":29689,"ÃŃa":29690,"Ġ193":29691,"Ġante":29692,"Ġrepetition":29693,"Ġ+/-":29694,"Ġparody":29695,"Ġlively":29696,"AAA":29697,"ĠHorus":29698,"Ġpits":29699,"inders":29700,"LOC":29701,"ĠVenice":29702,"406":29703,"ĠDiscover":29704,"âĨ":29705,"ellectual":29706,"Ġpens":29707,"Ġeyel":29708,"iguous":29709,"Impl":29710,"Ġjoking":29711,"Ġinval":29712,"ĠBelfast":29713,"Ġcreditors":29714,"ĠSkywalker":29715,"ovsky":29716,"Ġceasefire":29717,"Ġseals":29718,"isoft":29719,")).":29720,"ĠFelix":29721,"ITS":29722,"Ġtresp":29723,"ĠBlockchain":29724,"eware":29725,"ĠSchwar":29726,"enne":29727,"mounted":29728,"ĠBeacon":29729,"lesh":29730,"Ġimmensely":29731,"Ġcheering":29732,"Employ":29733,"scene":29734,"ishly":29735,"atchewan":29736,"ĠNicolas":29737,"Ġdrained":29738,"ĠExit":29739,"ĠAzerb":29740,"jun":29741,"Ġfloated":29742,"uania":29743,"Deep":29744,"Ġsuperv":29745,"Ġmystical":29746,"ĠDollar":29747,"ĠApostle":29748,"ĠREL":29749,"ĠProvided":29750,"ĠBucks":29751,"ãĥ´":29752,"cutting":29753,"Ġenhancements":29754,"ĠPenguins":29755,"ĠIsaiah":29756,"Ġjerk":29757,"ĠWyn":29758,"Ġstalled":29759,"Ġcryptocurrencies":29760,"ĠRoland":29761,"single":29762,"Ġlumin":29763,"ĠFellow":29764,"ĠCapacity":29765,"ĠKazakh":29766,"WN":29767,"Ġfinanced":29768,"389":29769,"Ġtid":29770,"Ġcollusion":29771,"ĠMyr":29772,"îĢ":29773,"Senator":29774,"Ġpediatric":29775,"Ġneatly":29776,"Ġsandwiches":29777,"ĠArchitecture":29778,"Ġtucked":29779,"Ġbalcony":29780,"Ġearthquakes":29781,"quire":29782,"Future":29783,"Ġhefty":29784,"éĹ":29785,"Ġspecializes":29786,"Ġstresses":29787,"Ġsender":29788,"Ġmisunderstanding":29789,"Ġepile":29790,"Ġprovoke":29791,"ĠColors":29792,"Ġdismay":29793,"uko":29794,"[_":29795,"586":29796,"neutral":29797,"Ġdonating":29798,"ĠRandall":29799,"Multi":29800,"Ġconveniently":29801,"ĠSung":29802,"ĠCoca":29803,"Ġtents":29804,"ĠAcceler":29805,"Ġpartnered":29806,"272":29807,"irming":29808,"ĠBAS":29809,"sometimes":29810,"Ġobjected":29811,"ubric":29812,"posed":29813,"LCS":29814,"grass":29815,"Ġattributable":29816,"VIS":29817,"Israeli":29818,"Ġrepeats":29819,"ĠRM":29820,"vag":29821,"uta":29822,"inous":29823,"Ġinert":29824,"ĠMiguel":29825,"æŃ":29826,"ĠHawaiian":29827,"Board":29828,"Ġartific":29829,"ĠAzerbai":29830,"asio":29831,"ĠRent":29832,"AIN":29833,"Ġappliances":29834,"Ġnationality":29835,"Ġasshole":29836,"ĠNeb":29837,"Ġnotch":29838,"hani":29839,"ĠBride":29840,"Availability":29841,"Ġintercepted":29842,"Ġcontinental":29843,"Ġswelling":29844,"ĠPerspect":29845,"bies":29846,".<":29847,"ithmetic":29848,"ĠLara":29849,"Ġtempting":29850,"addr":29851,"Ġoverseeing":29852,"clad":29853,"ĠDV":29854,"ĠGingrich":29855,"Ġmun":29856,"ĠAppropri":29857,"Ġalterations":29858,"ĠPatreon":29859,"Ġhavoc":29860,"Ġdisciplines":29861,"Ġnotoriously":29862,"akuya":29863,"ieri":29864,"?).":29865,"ĠWent":29866,"Ġsilicon":29867,"Ġtremb":29868,"Container":29869,"Known":29870,"Ġmortar":29871,"este":29872,"icka":29873,"Arthur":29874,"ĠPreviously":29875,"ĠMarty":29876,"Ġsparse":29877,"gins":29878,"Ġinward":29879,"ĠParticipant":29880,"Copy":29881,"ĠMisc":29882,"Ġantibiotic":29883,"ĠRetro":29884,"Ġelusive":29885,"Ġassail":29886,"ĠBattalion":29887,"ĠBought":29888,"Ġdiminish":29889,"ĠEuropa":29890,"session":29891,"ĠDangerous":29892,"iesel":29893,"Ġdisbelief":29894,"Ġblasts":29895,"extreme":29896,"ĠBoyd":29897,"ĠProjects":29898,"ĠGuys":29899,"Ġundergone":29900,"Ġgrill":29901,"ĠDwight":29902,"Ġ197":29903,"USER":29904,"Ġfilesystem":29905,"Ġclocks":29906,"Taylor":29907,"Ġwrapper":29908,"Ġfolding":29909,"ousand":29910,"ĠPhilippine":29911,"ATIONAL":29912,"ĠPerth":29913,"Ġashes":29914,"Ġaccumulate":29915,"ĠGateway":29916,"Shop":29917,"orkshire":29918,"Han":29919,"ĠBarrel":29920,"ĠLeh":29921,"ĠXV":29922,"Ġwhim":29923,"Ġrepo":29924,"ĠCG":29925,"ĠMam":29926,"Ġincorporating":29927,"Ġbailout":29928,"Ġlinguistic":29929,"Ġdisinteg":29930,"CLE":29931,"Ġcinematic":29932,"ĠFiber":29933,"Syn":29934,"ilion":29935,"ĠCompos":29936,"chens":29937,"Ġneoc":29938,"Ġboiled":29939,"FINE":29940,"ono":29941,"uncle":29942,"iken":29943,"ĠBM":29944,"ι":29945,"Ġreceipts":29946,"Ġdisposed":29947,"ĠThirty":29948,"ĠRough":29949,"ĠABS":29950,"Ġnotwithstanding":29951,"ollen":29952,"#$":29953,"Ġunreliable":29954,"Ġbloom":29955,"Ġmediocre":29956,"Ġtram":29957,"ĠTasman":29958,"Ġshakes":29959,"Ġmanifesto":29960,"ĠMW":29961,"Ġsatisfactory":29962,"Ġshores":29963,"Ġcomputation":29964,"Ġassertions":29965,"ormons":29966,"arag":29967,"abit":29968,"Democrats":29969,"ĠLoot":29970,"ĠVolks":29971,"haired":29972,"Ġgravitational":29973,"Sing":29974,"ĠMiz":29975,"Ġthrottle":29976,"Ġtyranny":29977,"ĠViews":29978,"Ġrobber":29979,"ĠMinority":29980,"Ġshrine":29981,"scope":29982,"purpose":29983,"Ġnucleus":29984,"ourcing":29985,"ĠUSDA":29986,"ĠDHS":29987,"wra":29988,"ĠBowie":29989,"Scale":29990,"ĠBEL":29991,"xi":29992,"Iter":29993,"Ġ(),":29994,"wright":29995,"Ġsailors":29996,"oused":29997,"NASA":29998,"ĠProof":29999,"ĠMineral":30000,"token":30001,"ĠFD":30002,"Rew":30003,"Ġell":30004,"630":30005,"Ġchancellor":30006,"ĠGos":30007,"Ġamounted":30008,"ĠRecre":30009,"omez":30010,"ĠOptim":30011,"ĠOlive":30012,"Ġtracker":30013,"owler":30014,"ĠUnique":30015,"Root":30016,"Ġmaritime":30017,"ĠQuran":30018,"ĠAdapt":30019,"Ġecosystems":30020,"ĠRepeat":30021,"ĠSoy":30022,"ĠIMP":30023,"Ġgraduating":30024,"andem":30025,"Pur":30026,"ĠReset":30027,"ĠTrick":30028,"ĠPhilly":30029,"ĠTue":30030,"ĠMalaysian":30031,"Ġclimax":30032,"Ġbury":30033,"Ġconspic":30034,"ĠSouthampton":30035,"ĠFlowers":30036,"Ġescorted":30037,"ĠEducational":30038,"ĠIRC":30039,"Ġbrutally":30040,"eating":30041,"Ġpillar":30042,"ĠSang":30043,"ĠJude":30044,"arling":30045,"ĠAmnesty":30046,"Ġreminding":30047,"ĠAdministrative":30048,"hesda":30049,"Ġflashed":30050,"ĠPBS":30051,"perate":30052,"feature":30053,"Ġswipe":30054,"Ġgraves":30055,"oultry":30056,"261":30057,"breaks":30058,"ĠGuer":30059,"Ġshrimp":30060,"ĠVoting":30061,"quist":30062,"Ġanalytical":30063,"Ġtablespoons":30064,"ĠSOU":30065,"Ġresearched":30066,"Ġdisrupted":30067,"Ġjour":30068,"Ġreplica":30069,"Ġcartoons":30070,"bians":30071,"})":30072,"copy":30073,"Got":30074,"ouched":30075,"PUT":30076,"Ġswarm":30077,"notations":30078,"said":30079,"Ġrebuilt":30080,"Ġcollaborate":30081,"Ġraging":30082,"Ġnar":30083,"Ġdemographics":30084,"ĠDDR":30085,"Ġdistrust":30086,"ossier":30087,"ĠKro":30088,"Ġpumpkin":30089,"Ġregrets":30090,"Ġfatalities":30091,"ĠLens":30092,"ĠOle":30093,"pd":30094,"Ġpuppet":30095,"ĠOutlook":30096,"ĠStam":30097,"Ol":30098,"Fair":30099,"UU":30100,"Ġrewritten":30101,"ı":30102,"Ġfascinated":30103,"Ġvectors":30104,"Ġtribunal":30105,"uay":30106,"ĠMats":30107,"ĠCoins":30108,"[[":30109,"Ġ181":30110,"Ġrenders":30111,"ĠKaepernick":30112,"Ġespionage":30113,"Ġsumm":30114,"Ġditch":30115,"Account":30116,"Ġspreadsheet":30117,"Ġmutant":30118,"past":30119,"407":30120,"Ġdye":30121,"Ġinitiation":30122,"Ġ4000":30123,"Ġpunishable":30124,"Ġthinner":30125,"ĠKhal":30126,"Ġintermedi":30127,"Dun":30128,"ĠGotham":30129,"Ġeagerly":30130,"Ġvaginal":30131,"powers":30132,"VW":30133,"ĠWATCHED":30134,"Ġpredator":30135,"amsung":30136,"Ġdisparity":30137,"Ġ[*":30138,"Ġamph":30139,"Ġoutskirts":30140,"ĠSpirits":30141,"Ġskeletal":30142,"л":30143,"ĠRear":30144,"Ġissuance":30145,"ĠLogic":30146,"released":30147,"ZZ":30148,"ĠBound":30149,"Entry":30150,"Ġexits":30151,"isol":30152,"ĠFounder":30153,"Ġwre":30154,"ĠGreenland":30155,"ĠMMO":30156,"taker":30157,"INC":30158,"ãģ¾":30159,"Ġhourly":30160,"henko":30161,"Ġfantasies":30162,"Ġdisob":30163,"Ġdemolition":30164,"ãĥĭ":30165,"Ġenlisted":30166,"ratulations":30167,"Ġmisguided":30168,"Ġensured":30169,"Ġdiscouraged":30170,"mort":30171,"Ġflank":30172,"Ġcess":30173,"Ġreacts":30174,"ĠSere":30175,"sensitive":30176,"ĠSerpent":30177,"assad":30178,"Ġ247":30179,"Ġcalmly":30180,"busters":30181,"Ġbleed":30182,"ĠStro":30183,"Ġamusement":30184,"ĠAntarctica":30185,"Ġscept":30186,"ĠGaw":30187,"aq":30188,"asonic":30189,"Ġsprawling":30190,"native":30191,"aturated":30192,"ĠBattlefield":30193,"IVERS":30194,"EB":30195,"ĠGems":30196,"ĠNorthwestern":30197,"ĠFilms":30198,"ĠAutomatic":30199,"Ġapprehend":30200,"ãģ¨":30201,"ĠguiName":30202,"Ġbackend":30203,"Ġevidenced":30204,"geant":30205,"012":30206,"ĠSiege":30207,"ĠexternalTo":30208,"ĠunfocusedRange":30209,"ĠguiActiveUnfocused":30210,"ĠguiIcon":30211,"ĠexternalToEVA":30212,"ĠexternalToEVAOnly":30213,"Fri":30214,"chard":30215,"enaries":30216,"Ġchiefs":30217,"Ġcf":30218,"ĠHUD":30219,"Ġcorrobor":30220,"ĠdB":30221,"ĠTaken":30222,"ĠPatricia":30223,"rail":30224,"ĠCharm":30225,"ĠLibertarian":30226,"rieve":30227,"Personal":30228,"ĠOUR":30229,"geries":30230,"Ġdumping":30231,"Ġneurological":30232,"itimate":30233,"ĠClintons":30234,"rafted":30235,"ĠMolly":30236,"Ġterminals":30237,"register":30238,"Ġflare":30239,"Ġencoded":30240,"Ġautopsy":30241,"pel":30242,"machine":30243,"Ġexemptions":30244,"ĠRoyals":30245,"distance":30246,"Ġdrafts":30247,"Ġlame":30248,"ĠCunning":30249,"Ġspouses":30250,"ĠMarkets":30251,"ĠCarrier":30252,"Ġimplying":30253,"ĠYak":30254,"sid":30255,"Ġloser":30256,"Ġvigilant":30257,"Ġimpeachment":30258,"Ġaugmented":30259,"ĠEmployees":30260,"Ġunintended":30261,"ternally":30262,"ĠWatt":30263,"Ġrecognizable":30264,"essim":30265,"æĿ":30266,"Ġcoated":30267,"rha":30268,"Ġlieutenant":30269,"ĠLegislation":30270,"published":30271,"444":30272,"013":30273,"Ġideally":30274,"ĠPassword":30275,"Ġsimplify":30276,"ĠMeta":30277,"ĠMRI":30278,"Ġpleading":30279,"organized":30280,"handler":30281,"Ġunravel":30282,"correct":30283,"Ġicy":30284,"Ġparanoid":30285,"Ġpasser":30286,"Ġinspections":30287,"ofer":30288,"ĠHealthcare":30289,"283":30290,"ĠBrut":30291,"iola":30292,"forge":30293,"ĠMedieval":30294,"MSN":30295,"ievers":30296,"ĠProgramming":30297,"åī":30298,"Ġ223":30299,"mu":30300,"ĠCLE":30301,"uga":30302,"Ġshoppers":30303,"Ġinformative":30304,"ĠPlans":30305,"Ġsupplementation":30306,"ĠTests":30307,"tyard":30308,"ocytes":30309,"ĠVega":30310,"ĠGujarat":30311,"ermanent":30312,"Except":30313,"ĠLOT":30314,"alla":30315,"ĠCumm":30316,"ĠOsw":30317,"Ġvenom":30318,"ĠDebt":30319,"ĠDOWN":30320,"Ġreunion":30321,"Ġmuc":30322,"ĠRelief":30323,"Ġgeop":30324,"ĠðŁĺ":30325,"alogue":30326,"Anth":30327,"echo":30328,"Ġcorros":30329,"Ġreplication":30330,"ĠBlazing":30331,"ĠDaughter":30332,"Ġinflic":30333,"ĠLindsey":30334,"ÙĪ":30335,"284":30336,"Exit":30337,"Ġgloom":30338,"TAIN":30339,"Ġundermining":30340,"Ġadvising":30341,"hidden":30342,"Ġoverflow":30343,"Ġgor":30344,"urdue":30345,"Ġechoes":30346,"enhagen":30347,"Ġimpuls":30348,"drug":30349,"cash":30350,"Ġasync":30351,"Ġmirac":30352,"atts":30353,"punk":30354,"Ġpivot":30355,"ĠLegislative":30356,"Ġbloggers":30357,"ĠClaw":30358,"sburg":30359,"dyl":30360,"ĠRecommend":30361,"Ġverte":30362,"Ġprohibiting":30363,"ĠPanther":30364,"Jonathan":30365,"Ġomin":30366,"Ġhateful":30367,"281":30368,"ĠOrche":30369,"ĠMurdoch":30370,"downs":30371,"Ġasymm":30372,"GER":30373,"Always":30374,"Ġinforms":30375,"ĠWM":30376,"ĠPony":30377,"ĠAppendix":30378,"ĠArlington":30379,"Jam":30380,"Ġmedicinal":30381,"ĠSlam":30382,"ITIES":30383,"Ġreaff":30384,"ĠRi":30385,"FG":30386,"Spring":30387,"bool":30388,"Ġthighs":30389,"Ġmarkings":30390,"ĠRaqqa":30391,"ĠLak":30392,"poll":30393,"tsky":30394,"ĠMorty":30395,"ĠDefinition":30396,"Ġdebunk":30397,"endered":30398,"ĠLeone":30399,"avers":30400,"Ġmortgages":30401,"Apparently":30402,"Nic":30403,"haus":30404,"ĠThousands":30405,"auld":30406,"Ġmash":30407,"shoot":30408,"Ġdiarr":30409,"Ġconsciously":30410,"Hero":30411,"eas":30412,"ĠNaturally":30413,"ĠDestroyer":30414,"Ġdashboard":30415,"services":30416,"Rog":30417,"Ġmillennials":30418,"Ġinvade":30419,"-(":30420,"Ġcommissions":30421,"ĠAuckland":30422,"Ġbroadcasts":30423,"Ġfrontal":30424,"Ġcrank":30425,"ĠHistoric":30426,"Ġrumours":30427,"CTV":30428,"Ġsteril":30429,"Ġbooster":30430,"rocket":30431,"ãĤ¼":30432,"utsche":30433,"ĠPI":30434,"Ġ233":30435,"ĠProducer":30436,"ĠAnalytics":30437,"Ġinvaluable":30438,"Ġunintention":30439,"ĠCY":30440,"Ġscrutin":30441,"Ġgigg":30442,"Ġengulf":30443,"Ġproletariat":30444,"Ġhacks":30445,"ĠHew":30446,"arak":30447,"ĠSlime":30448,"ielding":30449,"agher":30450,"ĠElliot":30451,"Ġtelecom":30452,"Ġ219":30453,"ultan":30454,"ĠArbor":30455,"ĠScouts":30456,"Ban":30457,"Ġlifespan":30458,"Ġblasp":30459,"388":30460,"Ġjudiciary":30461,"ĠContinental":30462,"asking":30463,"McC":30464,"LED":30465,"Ġbaggage":30466,"ĠSorcerer":30467,"Ġremnants":30468,"ĠGriffith":30469,"etsu":30470,"ĠSubaru":30471,"ĠPersonality":30472,"designed":30473,"ushima":30474,"agnar":30475,"Ġrecoil":30476,"Ġpassions":30477,"\\\":":30478,"Ġtee":30479,"Ġabolition":30480,"ĠCreating":30481,"jac":30482,"Ġ194":30483,"019":30484,"Ġpillars":30485,"riched":30486,"/\"":30487,"tk":30488,"Ġlivelihood":30489,"Ġroasted":30490,"ahon":30491,"ĠHutch":30492,"assert":30493,"Ġdividend":30494,"Ġknit":30495,"Ġdaunting":30496,"Ġdisturbance":30497,"Ġshale":30498,"Ġcultivated":30499,"Ġrefrigerator":30500,"LB":30501,"ĠNET":30502,"Ġcommercials":30503,"Ġthinkers":30504,"455":30505,"Ġchop":30506,"Broad":30507,"Ġsuspicions":30508,"Ġtagged":30509,"lifting":30510,"Ġstylish":30511,"ĠShields":30512,"Shortly":30513,"Ġtails":30514,"Auth":30515,"STE":30516,"ĠGAME":30517,"Ġseism":30518,"ĠKis":30519,"ologne":30520,"Ġcowork":30521,"Ġforcibly":30522,"Ġthyroid":30523,"ĠPB":30524,"ANE":30525,"married":30526,"horse":30527,"Ġpolymer":30528,"ĠChal":30529,"odor":30530,"DEBUG":30531,"ĠContext":30532,"Ġbliss":30533,"Ġpinpoint":30534,"ĠMathemat":30535,"legram":30536,"ĠWeekend":30537,"Ġlabelled":30538,"Ġbart":30539,"itles":30540,"Ġestrogen":30541,"âĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶ":30542,"\"'":30543,"Ġvisibly":30544,"Ġoutsider":30545,"aida":30546,"Area":30547,"Ġdissemin":30548,"Ġdishonest":30549,"ĠClosed":30550,"ĠBulletin":30551,"ĠRamsey":30552,"sword":30553,"ĠXI":30554,"ourced":30555,"Same":30556,"346":30557,"ĠRepe":30558,"ĠKou":30559,"cake":30560,"emis":30561,"Cache":30562,"ĠMeaning":30563,"ĠEnlight":30564,"onomy":30565,"Ġmanifestation":30566,"sworth":30567,"Jay":30568,"Ġchore":30569,"ör":30570,"Dream":30571,"Ġsanctioned":30572,"Ġculturally":30573,"ĠAra":30574,"Nav":30575,"Ġtheological":30576,"Ġstrut":30577,"ĠVO":30578,"ĠHandbook":30579,"Ġconstructing":30580,"Ġ¶":30581,"ĠBenefits":30582,"ĠPsychological":30583,"sac":30584,"å¸":30585,"policy":30586,"ĠMatters":30587,"ĠReported":30588,"ĠByte":30589,"Ġvitro":30590,"ĠMaiden":30591,"Ġlam":30592,"ĠJennings":30593,"Ġgarment":30594,"ĠRutgers":30595,"ĠStafford":30596,"ĠWellington":30597,"Ġintermitt":30598,"Ġnpm":30599,"Ġordeal":30600,"Ġplugged":30601,"ooming":30602,"inished":30603,"framework":30604,"Ġtimber":30605,"Ġcass":30606,"Ġ850":30607,"iless":30608,"ĠRedux":30609,"768":30610,"Stre":30611,"Ġsurpassed":30612,"whel":30613,"Ġparallels":30614,"Ġveil":30615,"ĠGI":30616,"ĠREST":30617,"Ġreadiness":30618,"sort":30619,"Ġmodifying":30620,"ĠSlate":30621,"ruff":30622,"Ġmarble":30623,"Ġinfrared":30624,"Ġauditor":30625,"ĠFANTASY":30626,"ĠPoverty":30627,"ĠSPD":30628,"Ġ\"(":30629,"Ky":30630,"RAY":30631,"Ġexecutions":30632,"ĠBeverly":30633,"ĠMarxism":30634,"ĠBurst":30635,"ĠKali":30636,"estones":30637,"Clearly":30638,"Ell":30639,"ãģ§":30640,"ĠProceedings":30641,"Token":30642,"IFIC":30643,"ña":30644,"Central":30645,"ĠHaley":30646,"ĠDrama":30647,"Ġformations":30648,"ORN":30649,"Books":30650,"Ġdominating":30651,"ĠFlyers":30652,"ĠCompanion":30653,"Ġdisciplined":30654,"ĠYugoslav":30655,"ĠSpells":30656,"Ġvengeance":30657,"Ġlandlords":30658,"Len":30659,"ĠOgre":30660,"anoia":30661,"Ġpiercing":30662,"Ġcongreg":30663,"Ġscorer":30664,"obia":30665,"Ġnickel":30666,"ĠLearns":30667,"Ġrejo":30668,"Ġmasterpiece":30669,"Flash":30670,"Ġinhabited":30671,"ĠOpenGL":30672,"ĠDud":30673,"ĠICO":30674,"Ġarter":30675,"Ġplur":30676,"Ġmastery":30677,"Ġlongstanding":30678,"sted":30679,"Ġwines":30680,"Ġtelevised":30681,"ĠShrine":30682,"ĠBayern":30683,"Ġâĵĺ":30684,"Ġenclosure":30685,"john":30686,"Ġprophets":30687,"ĠResurrection":30688,"ĠOrders":30689,"Ġuneven":30690,"rals":30691,"Ġdwind":30692,"ĠLah":30693,"ĠSloven":30694,"378":30695,"Ġinsistence":30696,"affle":30697,"ĠClone":30698,"Ġhardship":30699,"ĠCongressman":30700,"Ġplead":30701,"Ġreviewers":30702,"Ġcured":30703,"Ġ1935":30704,"asley":30705,"fake":30706,"ĠThinking":30707,"ydia":30708,"PART":30709,"ĠDota":30710,"oit":30711,"Ġwhipped":30712,"Ġbouncing":30713,"ĠHispanics":30714,"comings":30715,"Ġcannabin":30716,"ĠChambers":30717,"ĠZack":30718,"Optional":30719,"Ġcoats":30720,"Ġprowess":30721,"ĠNorton":30722,"Ġplainly":30723,"Ġfreight":30724,"Ġinhibition":30725,"Ġclam":30726,"Ġ303":30727,"kef":30728,"aleigh":30729,"Luke":30730,"Ġpsycho":30731,"atorium":30732,"MED":30733,"Ġtreaties":30734,"Ġindisc":30735,"Ġdc":30736,"OPS":30737,"Ġresilient":30738,"ĠInterstate":30739,"Ġslack":30740,"Ġmundane":30741,"Ġestablishes":30742,"359":30743,"Ġstrained":30744,"Ġnond":30745,"Sus":30746,"Ġcaste":30747,"arate":30748,"ieving":30749,"Ġunfairly":30750,"Ġparser":30751,"onial":30752,"ursive":30753,"Via":30754,"ĠOtto":30755,"ĠAuthorities":30756,"stroke":30757,"KR":30758,"ĠMercy":30759,"Ġfurnished":30760,"Ġoutset":30761,"Ġmetic":30762,"1982":30763,"olithic":30764,"ĠTent":30765,"ogical":30766,"ĠAircraft":30767,"Ġhides":30768,"ĠBecame":30769,"Ġeducators":30770,"reaching":30771,"Ġvolatility":30772,"Ġtoddler":30773,"ĠNASCAR":30774,"ĠTwelve":30775,"ĠHighlights":30776,"Ġgrape":30777,"Ġsplits":30778,"Ġpeasant":30779,"Ġreneg":30780,"ĠMSI":30781,"Temp":30782,"stars":30783,"Ġtrek":30784,"ĠHyde":30785,"binding":30786,"Ġrealism":30787,"Ġoxide":30788,"ĠHos":30789,"Ġmounts":30790,"Ġbiting":30791,"Ġcollapsing":30792,"Ġpostal":30793,"Ġmuseums":30794,"Ġdetached":30795,"Ġrespecting":30796,"Ġmonopol":30797,"Ġworkflow":30798,"ĠCake":30799,"Template":30800,"ĠOrganisation":30801,"Ġpersistence":30802,"369":30803,"Coming":30804,"Brad":30805,"Ġredundant":30806,"ĠGTA":30807,"Ġbending":30808,"Ġrevoked":30809,"Ġoffending":30810,"Ġframing":30811,"Ġprintf":30812,"Commun":30813,"members":30814,"Outside":30815,"Ġconstrued":30816,"Ġcoded":30817,"FORE":30818,"Ġchast":30819,"Chat":30820,"Indian":30821,"ĠYard":30822,"?!\"":30823,"ĠPorts":30824,"ĠXavier":30825,"ĠRET":30826,"'.\"":30827,"ĠBoat":30828,"ivated":30829,"icht":30830,"umerable":30831,"Ds":30832,"ĠDunn":30833,"Ġcoffin":30834,"Ġsecurely":30835,"ĠRaptors":30836,"ĠBes":30837,"Installation":30838,"Ġinception":30839,"ĠHealthy":30840,"endants":30841,"Ġpsychologists":30842,"ĠSheikh":30843,"cultural":30844,"ĠBlackBerry":30845,"shift":30846,"Fred":30847,"oche":30848,"Ġcakes":30849,"ĠSEO":30850,"ĠGian":30851,"ĠAsians":30852,"ogging":30853,"element":30854,"Ġpundits":30855,"ĠVaugh":30856,"ĠGavin":30857,"Ġhitter":30858,"Ġdrowned":30859,"Ġchalk":30860,"ĠZika":30861,"Ġmeasles":30862,"802":30863,"âĢ¦..":30864,"ĠAWS":30865,"]\"":30866,"Ġdistort":30867,"ĠMast":30868,"Ġantibodies":30869,"ĠMash":30870,"Memory":30871,"ĠUganda":30872,"ĠProb":30873,"Ġvomiting":30874,"ĠTurns":30875,"Ġoccupying":30876,"Ġevasion":30877,"ĠTherapy":30878,"Ġpromo":30879,"Ġelectr":30880,"Ġblueprint":30881,"ĠDre":30882,"priced":30883,"ĠDepot":30884,"Ġalleviate":30885,"ĠSomali":30886,"marg":30887,"nine":30888,"Ġnostalgia":30889,"ĠShepherd":30890,"Ġcavalry":30891,"Ġtorped":30892,"ĠBloody":30893,"xb":30894,"Ġsank":30895,"Ġgoalt":30896,"reportprint":30897,"embedreportprint":30898,"cloneembedreportprint":30899,"ĠInitially":30900,"ĠFischer":30901,"Ġnoteworthy":30902,"cern":30903,"Ġinefficient":30904,"rawdownload":30905,"rawdownloadcloneembedreportprint":30906,"cation":30907,"ĠDynasty":30908,"lag":30909,"DES":30910,"Ġdistinctly":30911,"ĠEstonia":30912,"Ġopenness":30913,"Ġgossip":30914,"ruck":30915,"Width":30916,"ĠIbrahim":30917,"Ġpetroleum":30918,"Ġavatar":30919,"ĠHed":30920,"atha":30921,"ĠHogwarts":30922,"Ġcaves":30923,"678":30924,"Ġsafeguard":30925,"ĠMog":30926,"isson":30927,"ĠDurham":30928,"slaught":30929,"ĠGraduate":30930,"Ġsubconscious":30931,"ĠExcellent":30932,"ĠDum":30933,"-----":30934,"Ġpiles":30935,"ĠWORK":30936,"ĠGarn":30937,"ĠFol":30938,"ĠATM":30939,"Ġavoids":30940,"ĠTul":30941,"Ġbleak":30942,"ELY":30943,"ivist":30944,"lightly":30945,"Pers":30946,"ĠDob":30947,"ĠLS":30948,"Ġinsanity":30949,"ε":30950,"atalie":30951,"Enlarge":30952,"Ġtwists":30953,"Ġfaulty":30954,"Ġpiracy":30955,"Ġimpover":30956,"Ġrugged":30957,"ĠFashion":30958,"Ġsands":30959,"'?":30960,"swick":30961,"Ġnatives":30962,"Ġhen":30963,"ĠNoise":30964,"ãĥĹ":30965,"Ġgreens":30966,"Ġfreezer":30967,"Ġdynasty":30968,"ĠFathers":30969,"ĠNewark":30970,"Ġarchaeological":30971,"Ġot":30972,"obar":30973,"Ġblockade":30974,"Ġallerg":30975,"LV":30976,"Ġdebit":30977,"ĠRFC":30978,"ĠMilton":30979,"ĠPressure":30980,"Ġwillingly":30981,"Ġdisproportionate":30982,"Ġoppressive":30983,"Ġdiamonds":30984,"Ġbelongings":30985,"1970":30986,"Ġbells":30987,"Ġimperialism":30988,"Ġ227":30989,"Ġexploding":30990,"ĠEclipse":30991,"Ġ1919":30992,"Ġrant":30993,"Ġnominations":30994,"347":30995,"Ġpeacefully":30996,"rica":30997,"ĠFUCK":30998,"Ġvibration":30999,"malink":31000,"Ġropes":31001,"ĠIvanka":31002,"ĠBrewery":31003,"ĠBooker":31004,"ĠOwens":31005,"goers":31006,"Services":31007,"ĠSnape":31008,"Ġ191":31009,"395":31010,"Ġ299":31011,"justice":31012,"Ġbri":31013,"Ġdiscs":31014,"Ġprominently":31015,"Ġvulgar":31016,"Ġskipping":31017,"lves":31018,"Ġtsunami":31019,"374":31020,"ĠUrug":31021,"ĠEid":31022,"recated":31023,"phen":31024,"Ġfaults":31025,"ĠStarted":31026,"950":31027,"Ġpi":31028,"Ġdetector":31029,"Ġbastard":31030,"Ġvalidated":31031,"SpaceEngineers":31032,"OURCE":31033,"Ġ(~":31034,"Ġunsur":31035,"Ġaffirmed":31036,"Ġfascism":31037,"Ġresolving":31038,"ĠChavez":31039,"ĠCyn":31040,"Ġdetract":31041,"Lost":31042,"Ġrigged":31043,"Ġhomage":31044,"ĠBruno":31045,"555":31046,"eca":31047,"Ġpresses":31048,"Ġhumour":31049,"Ġspacing":31050,"Ġ'/":31051,"olkien":31052,"Coun":31053,"OPER":31054,"Tre":31055,"Son":31056,"ĠCambodia":31057,"ierre":31058,"mong":31059,"ozy":31060,"Ġliquidity":31061,"ĠSoviets":31062,"ĠFernando":31063,"Ġ229":31064,"Ġslug":31065,"ĠCatalan":31066,"electric":31067,"Ġscenery":31068,"ĠHearth":31069,"Ġconstrained":31070,"Ġgoalie":31071,"ĠGuidelines":31072,"ĠAmmo":31073,"ĠPearson":31074,"Ġtaxed":31075,"Ġfetus":31076,"Response":31077,"ĠAlexis":31078,"thia":31079,"Guy":31080,"Ġreconstruct":31081,"Ġextremes":31082,"Ġconcluding":31083,"ĠPeg":31084,"ooks":31085,"Ġdeductions":31086,"Rose":31087,"Ġgroundbreaking":31088,"ĠTarg":31089,"ãĥģ":31090,"ĠReve":31091,"resource":31092,"Ġmoons":31093,"Ġelectromagnetic":31094,"Ġamidst":31095,"ĠViktor":31096,"NESS":31097,"BACK":31098,"Ġcommute":31099,"ĠAnaheim":31100,"Ġfluctuations":31101,"640":31102,"Ġnoodles":31103,"ĠCopenhagen":31104,"ĠTide":31105,"ĠGrizz":31106,"ĠSEE":31107,"Ġpipelines":31108,"Ġscars":31109,"endo":31110,"agus":31111,"ĠETF":31112,"/#":31113,"ĠBecome":31114,"448":31115,"Ġvisc":31116,"ĠRecommended":31117,"Ġjumper":31118,"Ġcognition":31119,"Ġassassin":31120,"Ġwitnessing":31121,"ĠSetup":31122,"Ġlac":31123,"vim":31124,"ISM":31125,"pages":31126,"SSL":31127,"358":31128,"Ġadject":31129,"industrial":31130,"lore":31131,"chery":31132,"Ġglitter":31133,"Ġcalf":31134,"Florida":31135,"Ġspoilers":31136,"Ġsucceeds":31137,"Ġchanting":31138,"Ġslogans":31139,"ĠTracy":31140,"Visit":31141,"rology":31142,"Ġmornings":31143,"Ġlineage":31144,"Ġsip":31145,"Ġintensely":31146,"Ġflourish":31147,"ĠSleeping":31148,"ĠFem":31149,"orpor":31150,"ĠKlan":31151,"ĠDarth":31152,"hack":31153,"ĠNielsen":31154,"Ġtumors":31155,"Ġprocurement":31156,"ĠYorkshire":31157,"Ġraided":31158,"KY":31159,"Anna":31160,"Ġ//[":31161,"ĠDisorder":31162,"ĠMustang":31163,"ĠWen":31164,"ĠTrying":31165,"sq":31166,"Ġdeliveries":31167,"Ġshutter":31168,"Ġcerebral":31169,"Ġbipolar":31170,"ĠCN":31171,"lass":31172,"jet":31173,"Ġdebating":31174,">:":31175,"Ġeagle":31176,"grades":31177,"ĠDixon":31178,"UGC":31179,"MAS":31180,"ĠDraco":31181,"ĠMachines":31182,"affer":31183,"Ġeman":31184,"²":31185,"pron":31186,"ĠGym":31187,"Ġcomparatively":31188,"ĠTribunal":31189,"PRO":31190,"Ġlex":31191,"Ġfertile":31192,"Ġdepressing":31193,"Ġsuperficial":31194,"essential":31195,"ĠHunters":31196,"gp":31197,"Ġprominence":31198,"Liber":31199,"ĠAncest":31200,"otechnology":31201,"Ġmocking":31202,"ĠTraff":31203,"ĸļ":31204,"Medium":31205,"Iraq":31206,"Ġpsychiatrist":31207,"Quantity":31208,"ĠLect":31209,"Ġnoisy":31210,"520":31211,"GY":31212,"Ġslapped":31213,"ĠMTV":31214,"Ġpara":31215,"pull":31216,"Multiple":31217,"asher":31218,"Ġnour":31219,"ĠSeg":31220,"Spell":31221,"vous":31222,"ordial":31223,"Senior":31224,"ĠGoldberg":31225,"ĠPlasma":31226,"need":31227,"Ġmessenger":31228,"eret":31229,"Ġteamed":31230,"Ġliteracy":31231,"ĠLeah":31232,"ĠDoyle":31233,"Ġemitted":31234,"UX":31235,"Ġevade":31236,"Ġmaze":31237,"Ġwrongly":31238,"ĠLars":31239,"Ġstereotype":31240,"Ġpledges":31241,"Ġaroma":31242,"ĠMET":31243,"Ġacre":31244,"ĠOD":31245,"Ġff":31246,"Ġbreweries":31247,"ĠHilton":31248,"undle":31249,"ĠKak":31250,"ĠThankfully":31251,"ĠCanucks":31252,"inctions":31253,"ĠAppears":31254,"Ġcoer":31255,"Ġundermined":31256,"rovers":31257,"Andre":31258,"Ġblaze":31259,"umers":31260,"Ġfamine":31261,"amphetamine":31262,"ulkan":31263,"Amount":31264,"Ġdesperation":31265,"wikipedia":31266,"development":31267,"ĠCorinth":31268,"ussia":31269,"Jackson":31270,"LI":31271,"Native":31272,"Rs":31273,"Ohio":31274,"ĠKathleen":31275,"Fortunately":31276,"Ġattendant":31277,"ĠPreferred":31278,"ĠDidn":31279,"ĠVs":31280,"Mis":31281,"Ġrespondent":31282,"Ġboun":31283,"stable":31284,"Ġpaved":31285,"Ġunexpl":31286,"ĠCheney":31287,"LM":31288,"ĠCull":31289,"blown":31290,"Ġconfronting":31291,"ocese":31292,"serving":31293,"Wi":31294,"ĠLithuania":31295,"anni":31296,"Ġstalk":31297,"hd":31298,"Ġvener":31299,"APH":31300,"ynchronous":31301,"URR":31302,"umably":31303,"historic":31304,"Half":31305,"Hay":31306,"Ġresilience":31307,"spection":31308,"Ġabandoning":31309,"Obs":31310,"ĠDebbie":31311,"Ġgradient":31312,"ĠPlaint":31313,"ĠCanal":31314,"ARCH":31315,"Ġexpansive":31316,"Ġfung":31317,"Ġbounced":31318,"Und":31319,"Ġprecautions":31320,"Ġclarification":31321,"Ġdagger":31322,"Ġgrips":31323,"Ġµ":31324,"ĠRivera":31325,"ĠUndead":31326,"isites":31327,"ĠFIRST":31328,"ño":31329,"audi":31330,"Ġhostages":31331,"Ġcompliant":31332,"Ġalumni":31333,"Seven":31334,"Ġcybersecurity":31335,"either":31336,"Collect":31337,"Ġinvariably":31338,"ĠSoci":31339,"Ġlawmaker":31340,"Ġale":31341,"ĠPersonally":31342,"Nazi":31343,"Ġcustomization":31344,"ĠProc":31345,"ĠSaskatchewan":31346,"eaturing":31347,"Ġspared":31348,"Ġdiscontinued":31349,"Ġcomputational":31350,"ĠMotorola":31351,"Ġsupremacist":31352,"governmental":31353,"Ġparadise":31354,"ĠDowning":31355,"ĠNikon":31356,"Ġcatalyst":31357,"berra":31358,"Toronto":31359,"875":31360,"beta":31361,"ĠMacron":31362,"Ġunrealistic":31363,"vector":31364,"ĠVehicles":31365,"itiveness":31366,"ĠRV":31367,"ĠColbert":31368,"sin":31369,"oji":31370,"entin":31371,"ĠKrish":31372,"hello":31373,"ffield":31374,"oky":31375,"ĠTate":31376,"Ġmaple":31377,"Ġaids":31378,"chemical":31379,"334":31380,"nuts":31381,"ĠWarp":31382,"Ġxx":31383,"ĠRobb":31384,"umerous":31385,"_-_":31386,"ftime":31387,"ĠVW":31388,"Ġwinger":31389,"ĠDome":31390,"tools":31391,"ĠPV":31392,"ĠGeorgetown":31393,"Ġgeared":31394,"Ġjihadists":31395,"Ġcp":31396,"Ġsteroids":31397,"Mother":31398,"clerosis":31399,"ĠDRM":31400,"nesia":31401,"Ġlinger":31402,"Ġimmersive":31403,"ĠCOUN":31404,"Ġoutweigh":31405,"ensual":31406,"Band":31407,"Ġtransforms":31408,"matched":31409,"psons":31410,"ĠJudicial":31411,"factor":31412,"Ġreferral":31413,"Ġoddly":31414,"ĠWenger":31415,"Bring":31416,"ĠBows":31417,"602":31418,"ICLE":31419,"Ġlions":31420,"ĠAcademic":31421,"ĠThorn":31422,"ĠRaider":31423,"kefeller":31424,"Storage":31425,"Lower":31426,"ĠOrt":31427,"ĠEquality":31428,"ALT":31429,"ĠSOC":31430,"Types":31431,"Ġlyn":31432,"ĠAsset":31433,"coat":31434,"TPP":31435,"CVE":31436,"ĠPioneer":31437,"application":31438,"Modern":31439,"ĠHK":31440,"Environment":31441,"Alright":31442,"Rain":31443,"IPP":31444,"ĠShiite":31445,"Ġmound":31446,"ĠAbilities":31447,"condition":31448,"Staff":31449,"Ġcompetence":31450,"ĠMoor":31451,"ĠDiablo":31452,"Ġwithheld":31453,"Ġostensibly":31454,"ĠBrom":31455,"Ġmsg":31456,"Ġdenomin":31457,"ĠReferences":31458,"ĠFP":31459,"Ġplunged":31460,"Ġpamph":31461,"moving":31462,"central":31463,"Ġdownright":31464,"Ġfading":31465,"Tal":31466,"Typ":31467,"ĠThy":31468,"ukes":31469,"ithe":31470,"Ġove":31471,"Ġbattled":31472,"Ġseafood":31473,"Ġfigur":31474,"ĠRD":31475,"crop":31476,"Ġsquads":31477,"{\\":31478,"à¹":31479,"ĠEh":31480,"Ġinterviewing":31481,"ĠQin":31482,"Ġaspiring":31483,"PLIC":31484,"Ġclauses":31485,"ĠGast":31486,"ĠNir":31487,"Ġluggage":31488,"Ġhose":31489,"Ġsystemd":31490,"Ġdescending":31491,"ĠRevised":31492,"ĠRails":31493,"align":31494,"709":31495,"337":31496,"Ġfug":31497,"charging":31498,"tags":31499,"Ġuter":31500,"kish":31501,"WARNING":31502,"490":31503,"profits":31504,"Ġvoyage":31505,"Ġace":31506,"ĠVanguard":31507,"ĠTanks":31508,"ĠMuk":31509,"Ġ226":31510,"Safe":31511,"Armor":31512,"Ġvolcanic":31513,"Ġwomb":31514,"ĠMIL":31515,"Ġbeginner":31516,"ĠRecogn":31517,"ĠAAP":31518,"PLAY":31519,")!":31520,"Ġdetecting":31521,"cn":31522,"Ġbreaches":31523,"Basically":31524,"ĠPag":31525,"ĠMunicipal":31526,"ĠIndie":31527,"ĠLaf":31528,"ĠDisable":31529,"ĠOlson":31530,"Ġrestrained":31531,"Ġrulings":31532,"Ġhumane":31533,"events":31534,"ĠCinema":31535,"displayText":31536,"ĠHatch":31537,"actionDate":31538,"onnaissance":31539,"Ġassaulting":31540,"ĠLug":31541,"CHAT":31542,"Ġvigorous":31543,"ĠPerse":31544,"Ġintolerance":31545,"ĠSnapchat":31546,"ĠSharks":31547,"Ġdummy":31548,"ĠDiagn":31549,"ĠGuitar":31550,"imeters":31551,"403":31552,"REG":31553,"Ax":31554,"Ġseparates":31555,"ĠMahm":31556,"Ġtv":31557,"jah":31558,"OOL":31559,"Circ":31560,"ĠWindsor":31561,"ussian":31562,"Ġintuition":31563,"Ġdisdain":31564,"ĠDonovan":31565,"Ġ221":31566,"Emb":31567,"Ġcondemning":31568,"Ġgenerosity":31569,"zzy":31570,"Ġpanties":31571,"ĠPrevent":31572,"ActionCode":31573,"ANA":31574,"342":31575,"externalActionCode":31576,"Ġspecifying":31577,"Ġcrystall":31578,"Jere":31579,"Ġrupt":31580,"ĠApprentice":31581,"Ġprofiling":31582,"к":31583,"Strike":31584,"Ġsideline":31585,"Ġobligated":31586,"Ġoccult":31587,"Ġbureaucratic":31588,"antically":31589,"rupted":31590,"negative":31591,"ĠEthiopia":31592,"ĠCivic":31593,"Ġinsiders":31594,"eligible":31595,"ĠTVs":31596,"ĠBAR":31597,"ĠTI":31598,"iologist":31599,"ĠAIR":31600,"Ġsubstituted":31601,"Arab":31602,"ĠSaul":31603,"ĠYog":31604,"prem":31605,"Ġbuilders":31606,"Ġstationary":31607,"Ġdoubtful":31608,"Ġvigorously":31609,"Ġthrilling":31610,"Physical":31611,"ĠCarey":31612,"ĠHydra":31613,"geoning":31614,"ĠSly":31615,"yton":31616,"Ġborrowers":31617,"ĠParkinson":31618,"Ġë":31619,"ĠJamaica":31620,"Ġsatir":31621,"Ġinsurgents":31622,"ĠFirm":31623,"Ġisot":31624,"ĠKarn":31625,"ourning":31626,"akens":31627,"docs":31628,"little":31629,"ĠMonaco":31630,"CLASS":31631,"Turkey":31632,"Ly":31633,"ĠConan":31634,"assic":31635,"Ġstarred":31636,"ĠPacers":31637,"eties":31638,"Ġtipping":31639,"Moon":31640,"ĠRw":31641,"same":31642,"Ġcavity":31643,"Ġgoof":31644,"ĠZo":31645,"Shock":31646,"ummer":31647,"Ġemphasizes":31648,"Ġregrett":31649,"Ġnovelty":31650,"Ġenvy":31651,"ĠPassive":31652,"rw":31653,"505":31654,"Ġindifferent":31655,"ĠRica":31656,"ĠHimself":31657,"ĠFreddie":31658,"Ġadip":31659,"ä¸Ģ":31660,"Ġbreakout":31661,"Ġhurried":31662,"ĠHuang":31663,"ĠDisk":31664,"Ġroaming":31665,"?????-?????-":31666,"UV":31667,"ĠRicky":31668,"ĠSigma":31669,"Ġmarginalized":31670,"Ġedits":31671,"Ġ304":31672,"memory":31673,"Ġspecimen":31674,"293":31675,"ãģ¯":31676,"Ġvertically":31677,"Ġaudition":31678,"ĠHeck":31679,"Ġcaster":31680,"ĠHoldings":31681,"adal":31682,"ĠCron":31683,"ĠLiam":31684,"Ġdeflect":31685,"Pick":31686,"ĠDebug":31687,"REF":31688,"Ġversatility":31689,"othes":31690,"classified":31691,"ĠMahar":31692,"ĠHort":31693,"Counter":31694,"stasy":31695,"noticed":31696,"331":31697,"ĠShim":31698,"fuck":31699,"ĠBie":31700,"Ġairing":31701,"ĠProtein":31702,"ĠHolding":31703,"Ġspectators":31704,"iliated":31705,"ĠThatcher":31706,"nosis":31707,"ãĥ¼ãĥ³":31708,"Tele":31709,"Boston":31710,"ĠTempl":31711,"stay":31712,"Ġdeclarations":31713,"479":31714,"Volume":31715,"ĠDesigner":31716,"ĠOverwatch":31717,"idae":31718,"Ġonwards":31719,"Ġnets":31720,"ĠManila":31721,"particularly":31722,"Ġpolitic":31723,"oother":31724,"Ġportraits":31725,"Ġpavement":31726,"cffff":31727,"Ġsaints":31728,"Ġbeginners":31729,"ESPN":31730,"Ġshortcomings":31731,"âķIJâķIJ":31732,"Ġcomet":31733,"ĠOrganic":31734,"quel":31735,"Ġhospitalized":31736,"Break":31737,"Ġpeel":31738,"dylib":31739,"aspx":31740,"urances":31741,"ĠTIM":31742,"Pg":31743,"Ġreadable":31744,"ĠMalik":31745,"Ġmuzzle":31746,"Ġbenchmarks":31747,"dal":31748,"ĠVacc":31749,"ĠHicks":31750,"609":31751,"ĠBiblical":31752,"heng":31753,"Ġoverload":31754,"ĠCivilization":31755,"Ġimmoral":31756,"Ġfries":31757,"ãĤĴ":31758,"Ġreproduced":31759,"Ġformulation":31760,"jug":31761,"irez":31762,"gear":31763,"Ġcoached":31764,"MpServer":31765,"ĠSJ":31766,"ĠKw":31767,"Init":31768,"deal":31769,"ĠOro":31770,"ĠLoki":31771,"ĠSongs":31772,"Ġ232":31773,"ĠLouise":31774,"asionally":31775,"Ġuncond":31776,"ollywood":31777,"Ġprogressives":31778,"ĠEnough":31779,"ĠDoe":31780,"Ġwreckage":31781,"Ġbrushed":31782,"ĠBaseType":31783,"Ġzoning":31784,"ishable":31785,"hetically":31786,"ĠCaucus":31787,"ĠHue":31788,"Ġkarma":31789,"ĠSporting":31790,"Ġtrader":31791,"Ġseeming":31792,"ĠCapture":31793,"430":31794,"bish":31795,"Ġtunes":31796,"Ġindoors":31797,"ĠSphere":31798,"ĠDancing":31799,"TERN":31800,"Ġnob":31801,"ĠGST":31802,"maps":31803,"Ġpeppers":31804,"Fit":31805,"Ġoversees":31806,"ĠRabbi":31807,"ĠRuler":31808,"vertising":31809,"office":31810,"xxx":31811,"Ġraft":31812,"Changed":31813,"Ġtextbooks":31814,"Links":31815,"ĠOmn":31816,"ãĢij":31817,"Ġinconvenience":31818,"ĠDonetsk":31819,"=~":31820,"Ġimplicitly":31821,"Ġboosts":31822,"ĠBones":31823,"ĠBoom":31824,"Courtesy":31825,"Ġsensational":31826,"ANY":31827,"Ġgreedy":31828,"eden":31829,"Ġinexper":31830,"ĠLer":31831,"ĠVale":31832,"Ġtighten":31833,"ĠEAR":31834,"ĠNum":31835,"Ġancestor":31836,"Sent":31837,"ĠHorde":31838,"urgical":31839,"allah":31840,"Ġsap":31841,"amba":31842,"ĠSpread":31843,"twitch":31844,"Ġgrandson":31845,"Ġfracture":31846,"Ġmoderator":31847,"ĠSeventh":31848,"ĠReverse":31849,"Ġestimation":31850,"Choose":31851,"Ġparach":31852,"Ġbarric":31853,"ãĢIJ":31854,"Ġcompass":31855,"Ġallergic":31856,"âĢķ":31857,"OTHER":31858,"errilla":31859,"Ġwagon":31860,"Ġzinc":31861,"Ġrubbed":31862,"ĠFuller":31863,"ĠLuxembourg":31864,"ĠHoover":31865,"Ġliar":31866,"ĠEvening":31867,"ĠCobb":31868,"esteem":31869,"Ġselector":31870,"ĠBrawl":31871,"isance":31872,"ĠEk":31873,"Ġtroop":31874,"Ġguts":31875,"ĠAppeal":31876,"ĠTibetan":31877,"Ġroutines":31878,"ĠMent":31879,"Ġsummarized":31880,"steamapps":31881,"Ġtranqu":31882,"Ġ1929":31883,"oran":31884,"ĠAuthent":31885,"Ġgmaxwell":31886,"Ġapprehens":31887,"Ġpoems":31888,"Ġsausage":31889,"ĠWebster":31890,"urus":31891,"Ġthemed":31892,"Ġlounge":31893,"Ġcharger":31894,"Spoiler":31895,"Ġspilled":31896,"hog":31897,"ĠSunder":31898,"ĠAin":31899,"ĠAngry":31900,"Ġdisqual":31901,"ĠFrequency":31902,"ĠEthernet":31903,"Ġhelper":31904,"Percent":31905,"Ġhorrifying":31906,"Ġail":31907,"ĠAllan":31908,"EEE":31909,"ĠCrossing":31910,"449":31911,"Ġholog":31912,"ĠPuzzles":31913,"ĠGoes":31914,"erenn":31915,"604":31916,"ãģı":31917,"ĠRafael":31918,"Ġatten":31919,"ĠEmanuel":31920,"Ġupro":31921,"ĠSusp":31922,"Psych":31923,"ĠTrainer":31924,"ĠNES":31925,"ĠHunts":31926,"becue":31927,"Ġcounselor":31928,"Rule":31929,"Ġtoxins":31930,"Ġbanners":31931,"rifice":31932,"Ġgreeting":31933,"Ġfrenzy":31934,"Ġallocate":31935,"Ġ*)":31936,"expr":31937,"503":31938,"ĠChick":31939,"ĠTorn":31940,"Ġconsolidation":31941,"ĠFletcher":31942,"switch":31943,"frac":31944,"clips":31945,"ĠMcKin":31946,"ĠLunar":31947,"Month":31948,"ITCH":31949,"Ġscholarly":31950,"raped":31951,"398":31952,"Ġ1910":31953,"Ġegreg":31954,"Ġinsecure":31955,"Ġvictorious":31956,"cffffcc":31957,"Ġsingled":31958,"Ġelves":31959,"ĠWond":31960,"burst":31961,"Ġcamoufl":31962,"ĠBLACK":31963,"Ġconditioned":31964,"çī":31965,"answered":31966,"Ġcompulsory":31967,"ascist":31968,"Ġpodcasts":31969,"ĠFrankfurt":31970,"bnb":31971,"Ġneoliberal":31972,"ĠKeyboard":31973,"ĠBelle":31974,"warm":31975,"Ġtrusts":31976,"Ġinsured":31977,"ĠBucc":31978,"usable":31979,"607":31980,"ĠPlains":31981,"Ġ1890":31982,"Ġsabotage":31983,"Ġlodged":31984,"felt":31985,"Ġga":31986,"ĠNarc":31987,"ĠSalem":31988,"Ġseventy":31989,"ĠBlank":31990,"pocket":31991,"Ġwhisper":31992,"Ġmating":31993,"omics":31994,"ĠSalman":31995,"ĠKad":31996,"Ġangered":31997,"Ġcollisions":31998,"Ġextraordinarily":31999,"Ġcoercion":32000,"Ghost":32001,"birds":32002,"èĢ":32003,"kok":32004,"Ġpermissible":32005,"avorable":32006,"Ġpointers":32007,"Ġdissip":32008,"aci":32009,"Ġtheatrical":32010,"ĠCosmic":32011,"Ġforgetting":32012,"Ġfinalized":32013,"大":32014,"yout":32015,"library":32016,"Ġbooming":32017,"ĠBelieve":32018,"ĠTeacher":32019,"ĠLiv":32020,"ĠGOODMAN":32021,"ĠDominican":32022,"ORED":32023,"ĠParties":32024,"Ġprecipitation":32025,"ĠSlot":32026,"Roy":32027,"ĠCombined":32028,"Ġintegrating":32029,"Ġchrome":32030,"Ġintestinal":32031,"ĠRebell":32032,"Ġmatchups":32033,"Ġblockbuster":32034,"ĠLoren":32035,"ĠLevy":32036,"Ġpreaching":32037,"ĠSending":32038,"ĠPurpose":32039,"rax":32040,"fif":32041,"Ġauthoritative":32042,"ĠPET":32043,"astical":32044,"Ġdishon":32045,"Ġchatting":32046,"Ġ\"$:/":32047,"Connection":32048,"Ġrecreate":32049,"Ġdelinqu":32050,"Ġbroth":32051,"ĠDirty":32052,"ĠAdmin":32053,"zman":32054,"Ġscholarships":32055,"Ġ253":32056,"contact":32057,"alsa":32058,"767":32059,"creen":32060,"abbage":32061,"Ġ1915":32062,"Ġblended":32063,"Ġalarmed":32064,"Language":32065,"356":32066,"Ġblends":32067,"ĠChanged":32068,"Wolf":32069,"Ġhepat":32070,"Creating":32071,"Ġpersecut":32072,"Ġsweetness":32073,"arte":32074,"Ġforfeiture":32075,"ĠRoberto":32076,"impro":32077,"NFL":32078,"ĠMagnet":32079,"Detailed":32080,"Ġinsignificant":32081,"ĠPOLIT":32082,"ĠBBQ":32083,"ĠCPS":32084,"Ġseaw":32085,"aminer":32086,"mL":32087,"endif":32088,"finals":32089,"Ġ265":32090,"uish":32091,"Ġ})":32092,"ĠProblems":32093,"Ġemblem":32094,"Ġseriousness":32095,"Ġparsing":32096,"Ġsubstitution":32097,"Ġpressured":32098,"Ġrecycled":32099,"aleb":32100,"Ruby":32101,"Ġproficiency":32102,"Driver":32103,"ĠWester":32104,":'":32105,"AFTA":32106,"Ġmantle":32107,"ĠClayton":32108,"flag":32109,"Ġpractitioner":32110,"covered":32111,"ĠStruct":32112,"addafi":32113,"425":32114,"ĠTownship":32115,"ĠHydro":32116,"Louis":32117,"343":32118,"Ġcondo":32119,"ĠTao":32120,"Ġutilization":32121,"Ġnausea":32122,"ĠDems":32123,"ridges":32124,"pause":32125,"Ġformulas":32126,"Ġchallenger":32127,"376":32128,"Ġdefective":32129,"ĠRailway":32130,"ĠPubMed":32131,"Ġyogurt":32132,"lbs":32133,"ĠNorfolk":32134,"OPE":32135,"ĠMoody":32136,"Ġdistributor":32137,"Ġscrolls":32138,"Ġextracts":32139,"Stan":32140,"Ġviability":32141,"Ġexposes":32142,"Ġstarvation":32143,"ĠSteps":32144,"ĠDodd":32145,"few":32146,"STD":32147,"332":32148,"Ġclosures":32149,"Ġcomplementary":32150,"ĠSasha":32151,"umpy":32152,"Ġmonet":32153,"Ġarticulate":32154,"ĠDoct":32155,"killer":32156,"Ġscrim":32157,"Ġ264":32158,"Ġprostitutes":32159,"Ġsevered":32160,"Ġattachments":32161,"Ġcooled":32162,"Lev":32163,"ĠFalk":32164,"fail":32165,"Ġpoliceman":32166,"ĠDag":32167,"Ġprayed":32168,"ĠKernel":32169,"Ġclut":32170,"Ġcath":32171,"Ġanomaly":32172,"Storm":32173,"emaker":32174,"ĠBreakfast":32175,"uli":32176,"oire":32177,"JJ":32178,"hz":32179,"Operation":32180,"ĠSick":32181,"354":32182,"ĠGuatemala":32183,"Rate":32184,"Ġexposures":32185,"faces":32186,"ĠArchae":32187,"raf":32188,"ĠMia":32189,"Ġ2025":32190,"Ġopaque":32191,"Ġdisguised":32192,"ĠHeadquarters":32193,"Sah":32194,"Ġpots":32195,"978":32196,"ĠMalf":32197,"Ġfrowned":32198,"Ġpoisonous":32199,"ĠConvers":32200,"eeks":32201,"Ġcrab":32202,".\"\"":32203,"Ġtreason":32204,"Ġranc":32205,"Ġescalating":32206,"Ġwarr":32207,"Ġmobs":32208,"Ġlamps":32209,"ĠSunshine":32210,"ĠBrunswick":32211,"Phones":32212,"Ġspelled":32213,"ĠSkip":32214,"Ġ2050":32215,"Ġ1911":32216,"ĠPluto":32217,"ĠAmend":32218,"Ġmeats":32219,"387":32220,"Ġstomp":32221,"ĠZhou":32222,"ĠLeviathan":32223,"ĠHazard":32224,"adv":32225,"ĠOrwell":32226,"Ġaloud":32227,"Ġbumper":32228,"ĠAnarch":32229,"ubuntu":32230,"ĠSerious":32231,"fitting":32232,"ĠOptional":32233,"ĠCecil":32234,"REAM":32235,"Ġserotonin":32236,"Ġcultivate":32237,"agogue":32238,"}\\":32239,"Ġmosques":32240,"ĠSunny":32241,"Ġreactive":32242,"revolution":32243,"ĠLup":32244,"ĠFedora":32245,"Ġdefenseman":32246,"ĠVID":32247,"istine":32248,"Ġdrowning":32249,"ĠBroadcasting":32250,"Ġthriller":32251,"ĠScy":32252,"Ġaccelerating":32253,"Ġdirects":32254,"odied":32255,"bike":32256,"duration":32257,"Ġpainfully":32258,"Redd":32259,"Ġproductions":32260,"Ġgag":32261,"Ġwhist":32262,"Ġsock":32263,"Ġinfinitely":32264,"ĠConcern":32265,"ĠCitadel":32266,"Ġlieu":32267,"Ġcandles":32268,"ogeneous":32269,"arger":32270,"Ġheavenly":32271,"inflammatory":32272,"Performance":32273,"Cs":32274,"ructose":32275,"azaki":32276,"Ġpessim":32277,"Ġinference":32278,"Ġpowd":32279,"ĠZoe":32280,"Ġpaints":32281,"Ġdazz":32282,"pta":32283,"-----------":32284,"Ġinspir":32285,"ĠExperimental":32286,"ĠKnife":32287,"regor":32288,"bors":32289,"Ġshowers":32290,"romeda":32291,"Ġsaint":32292,"Ġbenign":32293,"ĠJiang":32294,"Ġenvisioned":32295,"Ġshroud":32296,"IFT":32297,"HO":32298,"Ġshuff":32299,"ĠICC":32300,"Ġsegreg":32301,"Ġrevisit":32302,"ighthouse":32303,"Li":32304,"Ġsubstrate":32305,"ĠSeas":32306,"ĠReward":32307,"ĠHep":32308,"ĠBrass":32309,"sbm":32310,"Ġeliminates":32311,"Ġstamina":32312,"ĠVAT":32313,"ĠLoan":32314,"Ġconstraint":32315,"Ġappropriated":32316,"Ġpes":32317,"ĠALE":32318,"ranging":32319,"Ġ404":32320,"392":32321,"Ġintellectuals":32322,"achu":32323,"Ġrestructuring":32324,"ĠLevin":32325,"Ġrunes":32326,"Ġdelightful":32327,"Ġcarbohydrates":32328,"ĠModels":32329,"ĠExpo":32330,"Ġtransporting":32331,"alloc":32332,"Ġringing":32333,"Samsung":32334,"Ġscarcely":32335,"ĠURLs":32336,"ĠMAS":32337,"Ġprototypes":32338,"Ġnarrator":32339,"ĠCPUs":32340,"cdn":32341,"ĠBarton":32342,"Ġdecidedly":32343,"ĠShu":32344,"ixir":32345,"ocious":32346,"ĠMyst":32347,"Nintendo":32348,"Ġreuse":32349,"Ġforgiven":32350,"Few":32351,"inical":32352,"nat":32353,"Ġseamless":32354,"ĠEva":32355,"ĠEVE":32356,"ĠJO":32357,"landers":32358,"Ġsofter":32359,"negie":32360,"Ġtransient":32361,"Ġorbital":32362,"Ġfulfil":32363,"ĠKom":32364,"Hopefully":32365,"Ġdynamically":32366,"ĠHunger":32367,"åĽ":32368,"ĠArmenia":32369,"elman":32370,"berto":32371,"Ġpige":32372,"ĠIDs":32373,"limit":32374,"Ġveins":32375,"Ġsoaring":32376,"packs":32377,"Golden":32378,"ĠCrab":32379,"istor":32380,"ĠRPM":32381,"Ġ$$":32382,"gression":32383,"Ġjihadist":32384,"Ġgamble":32385,"Ġcareg":32386,"Ġinflated":32387,"Face":32388,"ĠFirearms":32389,"ĠEmmanuel":32390,"âĿ":32391,"Ġshocks":32392,"grab":32393,"Ġsplend":32394,"ĠHPV":32395,"abortion":32396,"Above":32397,"Entity":32398,"players":32399,"Ġcommenced":32400,"ulence":32401,"Ġfulfillment":32402,"Ġembodiments":32403,"ĠWelfare":32404,"Ġhail":32405,"Ġ<@":32406,"tten":32407,"Ġcatcher":32408,"ĠJazeera":32409,"Ġvolcano":32410,"Ġstabilize":32411,"ĠHandler":32412,"Ġintensified":32413,"ĠAbrams":32414,"Ġhumiliation":32415,"paced":32416,"605":32417,"ĠCentOS":32418,"Specific":32419,"Ġheed":32420,"ĠCAM":32421,"ĠGalile":32422,"Die":32423,"Ġabolished":32424,"ĠThomson":32425,"ĠTeachers":32426,"ĠWass":32427,"jong":32428,"ĠISBN":32429,"ĠAllies":32430,"shake":32431,"å·":32432,"vict":32433,"Howard":32434,"Ġdeem":32435,"Ġexceedingly":32436,"ĠSmartstocks":32437,"ibe":32438,"Ġdoorway":32439,"Ġcompeted":32440,"igmat":32441,"Ġnationalists":32442,"Ġgroom":32443,"ĠKeen":32444,"Ġdisposable":32445,"decl":32446,"ĠTolkien":32447,"ĠScheme":32448,"Ġbiod":32449,"Ġavid":32450,"ĠElon":32451,"agar":32452,"ĠTSA":32453,"Roman":32454,"Ġartificially":32455,"Ġadvisors":32456,"XL":32457,"ĠInferno":32458,"366":32459,"Ġtedious":32460,"ĠPhotography":32461,"ĠCarrie":32462,"Ġtrope":32463,"ĠSandra":32464,"Ġdecimal":32465,"Queen":32466,"ĠGundam":32467,"ĠOM":32468,"otech":32469,"NBA":32470,"Ġ1932":32471,"Ġentrenched":32472,"ĠMarion":32473,"Ġfraternity":32474,"Labour":32475,"Henry":32476,"Ġlatitude":32477,"Either":32478,"Ġenhances":32479,"ĠPotential":32480,"Ġshines":32481,"idad":32482,"Ġbreadth":32483,"Ġcapacities":32484,"ĠðŁĻĤ":32485,"ĠBronx":32486,"Ġsexes":32487,"Ġdifferentiation":32488,"Ġheavyweight":32489,"ĠTaj":32490,"dra":32491,"Ġmigrate":32492,"Ġexhaustion":32493,"ĠRUN":32494,"elsius":32495,"ĠCuomo":32496,"Ġguitars":32497,"Ġclones":32498,"ĠSomew":32499,"ĠPry":32500,"-------------":32501,"Ġwarranted":32502,"cycles":32503,"Ġsalvage":32504,"Ġdisks":32505,"RANT":32506,"ĠNGOs":32507,"ĠMartian":32508,"\":[{\"":32509,"Ġaddicts":32510,"ojure":32511,"illet":32512,"Ġamazingly":32513,"artments":32514,"pixel":32515,"ĠGPUs":32516,"Layout":32517,"è£":32518,"ĠTamil":32519,"ĠBasil":32520,"Ġimpartial":32521,"ĠStructure":32522,"fork":32523,"bryce":32524,"Ġridge":32525,"ĠHamburg":32526,"rious":32527,"Ġblitz":32528,"cigarettes":32529,"Ġcanned":32530,"402":32531,"Ġironically":32532,"Ġcompassionate":32533,"ĠHawkins":32534,".#":32535,"ĠCathedral":32536,"Ġrallied":32537,"internal":32538,"Ġquota":32539,"stakes":32540,"TEXT":32541,"mom":32542,"Ġcompletes":32543,"Ġ238":32544,"Ġshrug":32545,"ãĥij":32546,"ĠNinth":32547,"Ġrevise":32548,"ĠProvider":32549,"Ġtreacher":32550,"Ġquasi":32551,"ĠPRES":32552,"Ġdeposition":32553,"Ġconfidentiality":32554,"issors":32555,"Ġimbalance":32556,"Ġspanning":32557,"Ġangular":32558,"ĠCul":32559,"communication":32560,"ĠNora":32561,"ĠGenius":32562,"opter":32563,"Ġsacked":32564,"Spot":32565,"Ġfinely":32566,"ĠCHR":32567,"282":32568,"waves":32569,"Palest":32570,"ĠRohing":32571,"NL":32572,"è¿":32573,"Ġshitty":32574,"ĠScalia":32575,"475":32576,"Progress":32577,"Ġreferencing":32578,"Ġclassrooms":32579,"abee":32580,"Ġsod":32581,"hesion":32582,"708":32583,"ĠZuckerberg":32584,"ĠFinish":32585,"ĠScotia":32586,"ĠSavior":32587,"ĠInstallation":32588,"antha":32589,"(-":32590,"Ġ302":32591,"ĠPunk":32592,"Ġcrater":32593,"youtu":32594,"Ġroast":32595,"Ġinfluencing":32596,"Ġdup":32597,"ĠJR":32598,"ĠGrav":32599,"Ġstature":32600,"Ġbathrooms":32601,"Aside":32602,"Wiki":32603,"mean":32604,"ĠZak":32605,"ĠOnes":32606,"ĠNath":32607,"Ġhypert":32608,"Ġcommencement":32609,"Civil":32610,"Ġmoderately":32611,"Ġdistributors":32612,"Ġbreastfeeding":32613,"Ġ980":32614,"ĠSik":32615,"ĠCig":32616,"ĠAMER":32617,"RIP":32618,"ĠCareer":32619,"usting":32620,"Ġmessed":32621,"Ġeh":32622,"ĠJensen":32623,"/$":32624,"Ġblackmail":32625,"Ġconversions":32626,"Ġscientifically":32627,"Ġmantra":32628,"paying":32629,"Ġivory":32630,"ĠCourts":32631,"OUGH":32632,"auntlet":32633,"Serial":32634,"Brow":32635,"ĠHundreds":32636,"323":32637,"Ġpee":32638,"Ġlinux":32639,"Ġsubmer":32640,"ĠPrincipal":32641,"485":32642,"ĠDSL":32643,"ĠCousins":32644,"Ġdoctrines":32645,"ĠAthletics":32646,"Ġ315":32647,"ĠKarma":32648,"Ġattent":32649,"urger":32650,"Ġprescribe":32651,"Ġencaps":32652,"ĠCame":32653,"Ġsecretive":32654,"ĠCrimes":32655,"dn":32656,"Clean":32657,"ĠEgyptians":32658,"ĠCarpenter":32659,"Ġll":32660,"Hum":32661,"ĠMilo":32662,"Ġcapitalists":32663,"Ġbriefed":32664,"Twe":32665,"ĠBasin":32666,"elvet":32667,"Mos":32668,"Ġplunge":32669,"ĠKaiser":32670,"ĠFuj":32671,"illin":32672,"Ġsafeguards":32673,"Ġoste":32674,"ĠOpportunity":32675,"ĠMafia":32676,"ĠCalling":32677,"apa":32678,"urban":32679,"brush":32680,"illard":32681,"cé":32682,"intelligence":32683,"ĠLob":32684,"ĠDruid":32685,"Ġsmoother":32686,"Ġfooting":32687,"Ġmotorists":32688,"arcity":32689,"Ġmasculinity":32690,"Ġmism":32691,"Ġabdominal":32692,"ĠTavern":32693,"ĠRoh":32694,"Ġescapes":32695,"signed":32696,"Anthony":32697,"Ġsacrificing":32698,"Ġintimacy":32699,"Ġanterior":32700,"ĠKod":32701,"Ġmotif":32702,"Ġgraz":32703,"Ġvisualization":32704,"Ġguitarist":32705,"ĠTrotsky":32706,"magic":32707,"Dar":32708,"ĠMori":32709,"Ġwards":32710,"Ġtoilets":32711,"lest":32712,"Ġteleport":32713,"ĠSundays":32714,"ĠPlat":32715,"ETS":32716,"ĠeSports":32717,"Patrick":32718,"ĠKatherine":32719,"enko":32720,"Ġhassle":32721,"ĠMick":32722,"ggles":32723,"Ġhob":32724,"aintain":32725,"Ġairborne":32726,"Ġspans":32727,"Ġchili":32728,"Ġaperture":32729,"Ġvolunteered":32730,"ĠIncident":32731,"ĠFres":32732,"ĠVeteran":32733,"aughtered":32734,"ingo":32735,"Ġuninsured":32736,"CLOSE":32737,"Ġfuse":32738,"Ġerotic":32739,"Ġadvertise":32740,"raising":32741,"Texture":32742,"Ġattends":32743,"ĠREAL":32744,"uddled":32745,"Ġsmoot":32746,"Ġ305":32747,"ĠWillis":32748,"Ġblond":32749,"Analysis":32750,"ĠVT":32751,"onica":32752,"Ġstronghold":32753,"RF":32754,"NM":32755,".>>":32756,"Ġprosperous":32757,"Ġboasted":32758,"292":32759,"ĠManufacturing":32760,"PRESS":32761,"gren":32762,"Ġpharmacy":32763,"ĠRockefeller":32764,"kai":32765,"Ġthumbs":32766,"ĠHut":32767,"Ġmotherboard":32768,"Ġguardians":32769,"ĠAlter":32770,"llular":32771,"Ġshack":32772,"Ġwisely":32773,"Ġbackbone":32774,"erva":32775,"Ġsuicides":32776,"ĠMcGregor":32777,"ijah":32778,"Emer":32779,"ĠBrav":32780,"Ġdesignate":32781,"POST":32782,"produced":32783,"Ġcleansing":32784,"irlwind":32785,"existent":32786,"ĠHumph":32787,"ĠPayne":32788,"Ġvested":32789,"Å¡":32790,"Ġstringent":32791,"iona":32792,"Ġunsub":32793,"Ġsummed":32794,"ĠHercules":32795,"subject":32796,"ĠRagnar":32797,"ĠNos":32798,"Ġcharacterization":32799,"Ġsavvy":32800,"ĠDawson":32801,"ĠCasino":32802,"Ġfri":32803,"ĠBarrier":32804,"Ġmisinformation":32805,"Ġinsulation":32806,"Ġcorridors":32807,"Ġairplanes":32808,"ĠNoct":32809,"ahi":32810,"Ġ1916":32811,"kb":32812,"armac":32813,"Ġshun":32814,"Ġschema":32815,"Ġhorrified":32816,"Ġ239":32817,"aunders":32818,"NB":32819,"iates":32820,"erity":32821,"ĠShard":32822,"Ġrarity":32823,"Ġgrouped":32824,"ĠGhana":32825,"against":32826,"ĠBiological":32827,"ĠAware":32828,"owell":32829,"ÏĦ":32830,"ĠBeau":32831,"shaw":32832,"Hack":32833,"ĠJulius":32834,"USS":32835,"olson":32836,"auna":32837,"cru":32838,"ĠMaurice":32839,"ĠIk":32840,"Ġsequencing":32841,"Ġradicals":32842,"Ġ(?,":32843,"virtual":32844,"Ġanyways":32845,"Ġreperc":32846,"Ġhandlers":32847,"Ġhesitant":32848,"éĥ":32849,"ĠMF":32850,"plementation":32851,"associated":32852,"Ġcampaigned":32853,"ĠYue":32854,"utations":32855,"ĠYoga":32856,"Ġsimmer":32857,"Ġrods":32858,"Ġmelody":32859,"Ġconvoy":32860,"videos":32861,"Ġscreened":32862,"Neg":32863,"ochemical":32864,"Ġ())":32865,"Ġultras":32866,"Ġantip":32867,"ĠIslanders":32868,"704":32869,"Ġfetish":32870,"Ġridiculously":32871,"ĠKart":32872,"Ġmitochondrial":32873,"Ġinterfering":32874,"Builder":32875,"Ġoverfl":32876,"Ġacne":32877,"ĠMud":32878,"ĠKerr":32879,"flex":32880,"ĠPostal":32881,"ĠBaltic":32882,"477":32883,"ĠPersons":32884,"ourage":32885,"HB":32886,"ĠMuse":32887,"ĠImmortal":32888,"ĠDriving":32889,"Ġpetitions":32890,"Ġsubscript":32891,"Ġsorce":32892,"ĠProcessor":32893,"uton":32894,"Sony":32895,"Ġphon":32896,"Ġraced":32897,"ĠAnthrop":32898,"Ġdaytime":32899,"ĠExercise":32900,"Adding":32901,"Ġengages":32902,"ĠQualcomm":32903,"Ġmiracles":32904,"Ġmemes":32905,"ĠDrink":32906,"ĠOrioles":32907,"Ġhairs":32908,"ĠPolar":32909,"athom":32910,"Ġslippery":32911,"ĠRemy":32912,"Ġcaramel":32913,"ĠYEAR":32914,"Ġalk":32915,"Ign":32916,"aution":32917,"ĠMerlin":32918,"ĠCran":32919,"Ġapologies":32920,"Ġ410":32921,"Ġouting":32922,"ĠMemories":32923,"appointed":32924,"Ġcountered":32925,"uld":32926,"posing":32927,"Ġfirewall":32928,"ĠWast":32929,"ĠWet":32930,"worked":32931,"seller":32932,"Ġrepealed":32933,"ereo":32934,"assuming":32935,"BLIC":32936,"mite":32937,"ĠCEOs":32938,"ĠChapel":32939,"elligent":32940,"________________________":32941,"Dog":32942,"Ġwart":32943,"Ġsubscriber":32944,"sports":32945,"Ġbegged":32946,"ĠMV":32947,"Ġsemif":32948,"ethical":32949,"Ġpreach":32950,"Ġrevital":32951,"Ġpunitive":32952,"Ġshortcuts":32953,"Ġinstituted":32954,"ĠWarsaw":32955,"Ġabdomen":32956,"ĠKING":32957,"Ġsuperintendent":32958,"Ġfry":32959,"ĠGeo":32960,"TOR":32961,"Ġcontradictions":32962,"aptic":32963,"Ġlandscapes":32964,"bugs":32965,"Ġclust":32966,"Ġvolley":32967,"cribed":32968,"Ġtandem":32969,"Ġrobes":32970,"WHAT":32971,"Ġpromoter":32972,"Ġeloqu":32973,"reviewed":32974,"ĠDK":32975,"ĠPlato":32976,"Ġfps":32977,"Tank":32978,"ĠDerrick":32979,"Ġprioritize":32980,"asper":32981,"ĠHonduras":32982,"ĠCompleted":32983,"nec":32984,"Ġmog":32985,"nir":32986,"ĠMayo":32987,"DEF":32988,"stall":32989,"inness":32990,"ĠVolkswagen":32991,"Ġprecaution":32992,"ĠMell":32993,"iak":32994,"istries":32995,"Ġ248":32996,"Ġoverlapping":32997,"Senate":32998,"ĠEnhance":32999,"resy":33000,"racial":33001,"ORTS":33002,"ĠMormons":33003,"Strong":33004,"ĠCoch":33005,"Mexico":33006,"ĠMaduro":33007,"Ġjars":33008,"Ġcane":33009,"Wik":33010,"olla":33011,"ifference":33012,"Ġphysicist":33013,"ĠMaggie":33014,"Ġ285":33015,"Ġdepiction":33016,"ĠMcLaren":33017,"Ju":33018,"Ġslows":33019,"Ġcommissioners":33020,"ĠWillow":33021,"ĠExplos":33022,"hovah":33023,"Ġtechnician":33024,"Ġhomicides":33025,"ĠFlav":33026,"ĠTruman":33027,"Ġ10000":33028,"uctor":33029,"Ġshader":33030,"Newsletter":33031,"457":33032,"Ġrever":33033,"Ġhardened":33034,"Ġwhereabouts":33035,"Ġredevelop":33036,"Ġcarbs":33037,"Ġtravers":33038,"Ġsquirrel":33039,"Ġfollower":33040,"Ġsings":33041,"508":33042,"Ġrabbits":33043,"emonium":33044,"Ġdocumenting":33045,"Ġmisunderstood":33046,")'":33047,"Rick":33048,"ggies":33049,"Ġpremie":33050,"Ġskating":33051,"Ġpassports":33052,"Ġfists":33053,"ageddon":33054,"Haw":33055,"ACP":33056,"080":33057,"ĠThoughts":33058,"ĠCarlson":33059,"Ġpriesthood":33060,"hua":33061,"Ġdungeons":33062,"ĠLoans":33063,"Ġantis":33064,"Ġfamiliarity":33065,"ĠSabb":33066,"opal":33067,"ĠInk":33068,"strike":33069,"Ġcram":33070,"Ġlegalized":33071,"Ġcuisine":33072,"Ġfibre":33073,"Travel":33074,"ĠMonument":33075,"ODY":33076,"ethy":33077,"Ġinterstate":33078,"ĠPUR":33079,"emporary":33080,"ĠArabian":33081,"developed":33082,"Ġsaddle":33083,"Ġgithub":33084,"ĠOffer":33085,"ĠISP":33086,"rolet":33087,"ĠSUPER":33088,"ĠDenis":33089,"Ġmultiplier":33090,"Ġstirred":33091,"Interestingly":33092,"Ġcustomary":33093,"Ġbilled":33094,"hex":33095,"Ġmultiplied":33096,"Ġflipping":33097,"ĠCrosby":33098,"Ġfundamentals":33099,"iae":33100,"ĠPlayed":33101,"ĠAtom":33102,"amazon":33103,"ĠFlam":33104,"eez":33105,"activated":33106,"Ġtablespoon":33107,"Ġliberalism":33108,"ĠPalin":33109,"ĠPatel":33110,"Num":33111,"ĠTAM":33112,"Ġsurn":33113,"ĠReloaded":33114,"Ġcoined":33115,"\"],":33116,"ĠClash":33117,"ĠAgu":33118,"Ġpragmatic":33119,"ĠActivate":33120,"Ġ802":33121,"Ġtrailers":33122,"Ġsilhou":33123,"Ġprobes":33124,"Ġcircus":33125,"ĠBain":33126,"ĠLindsay":33127,"ĠAbbey":33128,"Delivery":33129,"Ġconcession":33130,"Ġgastro":33131,"ĠSprite":33132,"ÄŁ":33133,"andel":33134,"Ġgimm":33135,"Ġautobi":33136,"ĠTurtle":33137,"Ġwonderfully":33138,"ĠHaram":33139,"ĠWorldwide":33140,"ĠHandle":33141,"Ġtheorists":33142,"Ġsleek":33143,"ĠZhu":33144,"ographically":33145,"EGA":33146,"ĠOwners":33147,"aths":33148,"ĠAntarctic":33149,"natal":33150,"=\"\"":33151,"flags":33152,"````":33153,"Ġsul":33154,"Kh":33155,"Ġpotassium":33156,"Ġlineman":33157,"Ġcereal":33158,"ĠSeasons":33159,"Ġ2022":33160,"Ġmathematic":33161,"Ġastronomers":33162,"professional":33163,"Ġfares":33164,"cknowled":33165,"Ġchi":33166,"Ġyoungsters":33167,"Ġmistakenly":33168,"Ġhemisphere":33169,"ĠDivinity":33170,"rone":33171,"Ġ\",":33172,"rings":33173,"Ġattracts":33174,"vana":33175,"å¹":33176,"CAP":33177,"Ġplaylist":33178,"Ġporch":33179,"ãģ£":33180,"Ġincorporates":33181,"Ġsoak":33182,"Ġasserting":33183,"ĠTerrorism":33184,"ĠPablo":33185,"Ja":33186,"cester":33187,"Ġfearing":33188,"ĠPrayer":33189,"Ġescalated":33190,"GW":33191,"Ġrobe":33192,"ĠBrighton":33193,"acists":33194,"ĠSymphony":33195,"ĠDwarf":33196,"ĠParade":33197,"ĠLego":33198,"Ġinexpl":33199,"Ġlords":33200,"leaf":33201,"RAG":33202,"liber":33203,"Ġcigars":33204,"ĠJehovah":33205,"606":33206,"WINDOWS":33207,"ĠLiberia":33208,"ebus":33209,"Heavy":33210,"Ġlubric":33211,"ĠRW":33212,"anguages":33213,"Ġnarrowed":33214,"computer":33215,"ĠEmber":33216,"Ġmurdering":33217,"Ġdownstream":33218,"ĠTuls":33219,"ĠTables":33220,"Topic":33221,"ĠAccuracy":33222,"=/":33223,"lost":33224,"ĠRei":33225,"Ġprogresses":33226,"bear":33227,"Ġestablishments":33228,"Justin":33229,"ĠPeach":33230,"ĠGomez":33231,"å¿":33232,"ĠTriangle":33233,"Ident":33234,"ĠHive":33235,"Resources":33236,"Ġmixes":33237,"ĠAssuming":33238,"Mu":33239,"Ġhypoc":33240,"Ġsane":33241,"ĠWan":33242,"idious":33243,"Success":33244,"Ġio":33245,"Angel":33246,"Ġdangerously":33247,"ĠCreature":33248,"WORK":33249,":[":33250,"ĠKatrina":33251,"Listener":33252,"Miller":33253,"ĠIdlib":33254,"hang":33255,"Ġcircumvent":33256,"href":33257,"Ġcelestial":33258,"ĠWeeks":33259,"ĠPug":33260,"ĠDalton":33261,"Ġsubpoena":33262,"uku":33263,"Ġpersisted":33264,"pei":33265,"olding":33266,"ĠDocuments":33267,"ĠHast":33268,"ĠCENT":33269,"Ġprimer":33270,"Ġsynonymous":33271,"Ġnib":33272,"ombs":33273,"Ġnotation":33274,"ĠDish":33275,"ĠAtmosp":33276,"Ġforbid":33277,"ĠANG":33278,"pattern":33279,"los":33280,"Ġprojectiles":33281,"brown":33282,".\",":33283,"ĠVenom":33284,"Ġfiercely":33285,"ublished":33286,"ĠUran":33287,"ĠNicarag":33288,"410":33289,"ĠCAL":33290,"OTOS":33291,"ĠMiracle":33292,"ĠEnchant":33293,"Ġguarding":33294,"append":33295,"Attach":33296,"Ġleveled":33297,"Ġcondoms":33298,"ihilation":33299,"649":33300,"Ġnightmares":33301,"ĠTHEY":33302,"ĠSTART":33303,"ĠKinn":33304,"Ġroommate":33305,"Ġhygiene":33306,"opping":33307,"Job":33308,"Ġlvl":33309,"ĠVER":33310,"ĠKeeping":33311,"abetic":33312,"Ġformatting":33313,"erala":33314,"Ġrevisions":33315,"Ġresurg":33316,"Tel":33317,"ĠGoodman":33318,"353":33319,"pod":33320,"Ġindisp":33321,"ĠTranslation":33322,"Ġgown":33323,"ĠMund":33324,"Ġcis":33325,"Ġbystand":33326,"collect":33327,"ĠPunjab":33328,"actively":33329,"ĠGamb":33330,"tell":33331,"Ġimporting":33332,"gencies":33333,"Ġlocom":33334,"ĠBrill":33335,"Holy":33336,"ĠBerger":33337,"Ġshowdown":33338,"Ġresponders":33339,"ILY":33340,"Ġtakedown":33341,"leted":33342,"Ġmattered":33343,"Ġpredictive":33344,"Ġoverlay":33345,"GPU":33346,"ĠVick":33347,"Ġconveyed":33348,"Tab":33349,"peer":33350,"Scan":33351,"Ġdefensively":33352,"vae":33353,"Ġapproving":33354,"Ġtiers":33355,"ĠVia":33356,"querade":33357,"ĠSaudis":33358,"Ġdemolished":33359,"ĠProphe":33360,"Ġmono":33361,"Ġhospitality":33362,"HAM":33363,"ĠAriel":33364,"MOD":33365,"ĠTorah":33366,"Ġblah":33367,"ĠBelarus":33368,"erential":33369,"ĠTuc":33370,"Ġbanker":33371,"397":33372,"Ġmosquit":33373,"ĠScientist":33374,"ĠMusical":33375,"Ġhust":33376,"Shift":33377,"Ġtorment":33378,"Ġstandoff":33379,"Educ":33380,"ĠFog":33381,"Ġamplifier":33382,"Shape":33383,"Instance":33384,"ĠCritics":33385,"Ġdaemon":33386,"Houston":33387,"Ġmattress":33388,"ĠIDF":33389,"Ġobscene":33390,"ĠAmer":33391,"hetti":33392,"Ġcompiling":33393,"352":33394,"verett":33395,"ĠReduction":33396,"istration":33397,"ĠBlessed":33398,"ĠBachelor":33399,"316":33400,"Ġprank":33401,"ĠVulcan":33402,"dding":33403,"Ġmourning":33404,"ĠQuint":33405,"ĠBlaster":33406,"testing":33407,"Ġsediment":33408,">>>":33409,"ĠEternity":33410,"ĠWHERE":33411,"ĠMaze":33412,"Ġreacting":33413,"ĠAlv":33414,"omsday":33415,"ĠCRA":33416,"Ġtranslator":33417,"Ġbogus":33418,"atu":33419,"Website":33420,"olls":33421,"Ġbaptism":33422,"Ġsibling":33423,"ĠAutumn":33424,"vez":33425,"ãģ®é":33426,"guards":33427,"Georg":33428,"assadors":33429,"ĠFreud":33430,"Ġcontinents":33431,"ĠRegistry":33432,"Bernie":33433,"ĸļ士":33434,"Ġtolerant":33435,"ĠUW":33436,"Ġhorribly":33437,"995":33438,"ĠMIDI":33439,"Ġimpatient":33440,"ocado":33441,"eri":33442,"ĠWorst":33443,"ĠNorris":33444,"ĠTalking":33445,"Ġdefends":33446,"ensable":33447,"Ġ2021":33448,"Ġanatomy":33449,"Lew":33450,"Ġdrawer":33451,"ĠCanberra":33452,"Ġpatriotic":33453,"é¾įåĸļ士":33454,"ĠAvg":33455,"ARM":33456,"Ġundisclosed":33457,"Ġfarewell":33458,"459":33459,"bable":33460,"ĠAllison":33461,"OLOG":33462,"Ġconco":33463,"tight":33464,"ĠACPI":33465,"ĠMines":33466,"lich":33467,"ĠâĶľ":33468,"represented":33469,"200000":33470,"Ġenthusiast":33471,"OTS":33472,"bil":33473,"ĠIngredients":33474,"Ġinventor":33475,"ĠMySQL":33476,"³³³":33477,"ĠABOUT":33478,"within":33479,"Ġmk":33480,"Bul":33481,"ĠFake":33482,"Ġdraconian":33483,"Wa":33484,"helm":33485,"ĠTerran":33486,"erville":33487,"Ġcommonplace":33488,"SIZE":33489,"Ġ\"<":33490,"replace":33491,"ographs":33492,"ĠSELECT":33493,"incible":33494,"ĠMostly":33495,"ĠSheffield":33496,"ĠIDE":33497,"uggle":33498,"Ġcitations":33499,"hurst":33500,"ĠUnix":33501,"Ġunleash":33502,"ĠPiper":33503,"ĠNano":33504,"Ġsuccumb":33505,"Ġreluctance":33506,"Ġ2500":33507,"ĠMerchant":33508,"Ġwiret":33509,"Ġcombos":33510,"ĠBirthday":33511,"Ġcharcoal":33512,"ĠUPS":33513,"ĠFairfax":33514,"Ġdriveway":33515,"ĠTek":33516,"ĠPitch":33517,"overe":33518,"Ġtechnicians":33519,"ĠActual":33520,"flation":33521,"ĠFiscal":33522,"ĠEmpty":33523,"anamo":33524,"Ġmagnesium":33525,"Ġslut":33526,"Ġgrowers":33527,"Investigators":33528,"():":33529,"ĠSatellite":33530,"ĠKeynes":33531,"missive":33532,"lane":33533,"Ġborough":33534,"344":33535,"ĠTEAM":33536,"ĠBethesda":33537,"CV":33538,"hower":33539,"ĠRAD":33540,"Ġchant":33541,"ĠRiy":33542,"Ġcompositions":33543,"Ġmildly":33544,"Ġmeddling":33545,"Ġagility":33546,"aneers":33547,"501":33548,"Ġsynth":33549,"linger":33550,"291":33551,"Ġexclaimed":33552,"Party":33553,"Ġcontamin":33554,"ĠManor":33555,"ĠRespond":33556,"Ġpraising":33557,"Ġmanners":33558,"fleet":33559,"Summer":33560,"ĠLynd":33561,"ĠDefinitely":33562,"grim":33563,"Ġbowling":33564,"stri":33565,"çĽ":33566,"ynt":33567,"Ġmandates":33568,"DIV":33569,"Ġreconcile":33570,"views":33571,"ĠDamon":33572,"vette":33573,"Flo":33574,"ĠGreatest":33575,"ilon":33576,"icia":33577,"Ġportrayal":33578,"Ġcushion":33579,"504":33580,"1979":33581,"ossal":33582,"Applic":33583,"scription":33584,"Ġmitigation":33585,"ATS":33586,"pac":33587,"Ġerased":33588,"Ġdeficiencies":33589,"ĠHollande":33590,"ĠXu":33591,"Ġbred":33592,"Ġpregnancies":33593,"femin":33594,"Ġemph":33595,"Ġplanners":33596,"Ġoutper":33597,"uttering":33598,"Ġperpetrator":33599,"Ġmotto":33600,"ĠEllison":33601,"ĠNEVER":33602,"Ġadmittedly":33603,"ARI":33604,"ĠAzerbaijan":33605,"Ġmillisec":33606,"Ġcombustion":33607,"ĠBottle":33608,"ĠLund":33609,"ĠPs":33610,"ĠDress":33611,"Ġfabricated":33612,"Ġbattered":33613,"Ġsidel":33614,"ĠNotting":33615,"Foreign":33616,"ĠJerome":33617,"020":33618,"ĠArbit":33619,"Ġknots":33620,"ĠRIGHT":33621,"Moving":33622,"ãģĻ":33623,"Ġsurgeries":33624,"Ġcourthouse":33625,"Ġmastered":33626,"Ġhovering":33627,"ĠBran":33628,"ĠAlison":33629,"Ġsafest":33630,"military":33631,"Ġbullied":33632,"Ġbarrage":33633,"Reader":33634,"ESE":33635,"ĠGeographic":33636,"Tools":33637,"314":33638,"ĠGeek":33639,"roth":33640,"glers":33641,"ĠFIN":33642,"Ïģ":33643,"ĠAston":33644,"altern":33645,"488":33646,"Ġveterin":33647,"Gamer":33648,"Ġintel":33649,"renches":33650,"Shield":33651,"Ġamnesty":33652,"ĠBhar":33653,"Ġpiled":33654,"Ġhonorable":33655,"ĠInstitutes":33656,"Ġsoaked":33657,"Ġcoma":33658,"ĠEFF":33659,"341":33660,"bytes":33661,"ĠGmail":33662,"lein":33663,"ĠCanadiens":33664,"material":33665,"Il":33666,"Ġinstructors":33667,"ĠKY":33668,"Ġconceive":33669,"ubb":33670,"ĠPossible":33671,"Ġeasing":33672,"ĠChristina":33673,"Ġcaric":33674,"ĠHDR":33675,"ROM":33676,"Ġshovel":33677,"delete":33678,"Ġpuff":33679,"ĠChanging":33680,"Ġseamlessly":33681,"Attribute":33682,"Ġacquisitions":33683,"akery":33684,"ĠEF":33685,"Ġautistic":33686,"ĠTakes":33687,"ĠPowder":33688,"ĠStir":33689,"510":33690,"ĠBubble":33691,"settings":33692,"ĠFowler":33693,"Ġmustard":33694,"Ġmoreover":33695,"Ġcopyrighted":33696,"ĠLEDs":33697,"1500":33698,"æī":33699,"ĠHIS":33700,"enf":33701,"Ġcustod":33702,"ĠHuck":33703,"Gi":33704,"Ġimg":33705,"Answer":33706,"Ct":33707,"jay":33708,"ĠInfrastructure":33709,"Ġfederally":33710,"Loc":33711,"Ġmicrobes":33712,"Ġoverrun":33713,"dds":33714,"otent":33715,"adiator":33716,">>>>>>>>":33717,"Ġtornado":33718,"Ġadjud":33719,"Ġintrigued":33720,"Ġsi":33721,"ĠRevelation":33722,"progress":33723,"Ġburglary":33724,"ĠSaiyan":33725,"ĠKathy":33726,"Ġserpent":33727,"ĠAndreas":33728,"Ġcompel":33729,"essler":33730,"ĠPlastic":33731,"ĠAdvent":33732,"ĠPositive":33733,"ĠQt":33734,"ĠHindus":33735,"registered":33736,"ularity":33737,"Ġrighteousness":33738,"Ġdemonic":33739,"uitive":33740,"ĠBDS":33741,"ĠGregg":33742,"cia":33743,"ĠCrusade":33744,"ĠSinai":33745,"WARE":33746,"+(":33747,"Ġmell":33748,"Ġderail":33749,"yards":33750,"Ast":33751,"Ġnoticeably":33752,"ĠOber":33753,"Ram":33754,"Ġunnoticed":33755,"Ġseq":33756,"avage":33757,"Ts":33758,"Ġ640":33759,"Ġconcede":33760,"Ġ])":33761,"Fill":33762,"Ġcaptivity":33763,"ĠImprovement":33764,"ĠCrusader":33765,"araoh":33766,"MAP":33767,"æĹ":33768,"Ġstride":33769,"always":33770,"Fly":33771,"Nit":33772,"Ġalgae":33773,"ĠCooking":33774,"ĠDoors":33775,"Malley":33776,"Ġpolicemen":33777,"ãģį":33778,"Ġastronaut":33779,"accessible":33780,"495":33781,"ĠRAW":33782,"cliffe":33783,"udicrous":33784,"Ġdepended":33785,"alach":33786,"Ġventures":33787,"rake":33788,"Ġtits":33789,"ĠHou":33790,"Ġcondom":33791,"ormonal":33792,"Ġindent":33793,"Ġuploading":33794,"Footnote":33795,"Important":33796,"Ġ271":33797,"Ġmindful":33798,"Ġcontends":33799,"Cra":33800,"Ġcalibr":33801,"ĠOECD":33802,"plugin":33803,"Fat":33804,"ĠISS":33805,"ĠDynamics":33806,"ansen":33807,"686":33808,"'),":33809,"Ġsprite":33810,"Ġhandheld":33811,"ĠHipp":33812,"=~=~":33813,"Trust":33814,"Ġsemantics":33815,"ĠBundes":33816,"ĠReno":33817,"ĠLiterature":33818,"sense":33819,"Gary":33820,"ĠAeg":33821,"ĠTrin":33822,"EEK":33823,"Ġcleric":33824,"ĠSSH":33825,"Ġchrist":33826,"Ġinvading":33827,"ibu":33828,"Ġenum":33829,"aura":33830,"Ġallege":33831,"ĠIncredible":33832,"BBC":33833,"Ġthru":33834,"Ġsailed":33835,"Ġemulate":33836,"Ġinsecurity":33837,"Ġcrou":33838,"Ġaccommodations":33839,"Ġincompetent":33840,"Ġslips":33841,"ĠEarthqu":33842,"sama":33843,"ILLE":33844,"ĠiPhones":33845,"asaki":33846,"Ġbye":33847,"Ġard":33848,"Ġextras":33849,"Ġslaughtered":33850,"Ġcrowdfunding":33851,"resso":33852,"Ġfilib":33853,"ĠERROR":33854,"ĠTLS":33855,"egg":33856,"ĠItal":33857,"Ġenlist":33858,"ĠCatalonia":33859,"ĠScots":33860,"Ġsergeant":33861,"Ġdissolve":33862,"NH":33863,"Ġstandings":33864,"rique":33865,"IQ":33866,"Ġbeneficiary":33867,"Ġaquarium":33868,"YouTube":33869,"ĠPowerShell":33870,"Ġbrightest":33871,"ĠWarrant":33872,"Sold":33873,"Writing":33874,"Ġbeginnings":33875,"ĠReserved":33876,"ĠLatinos":33877,"heading":33878,"Ġ440":33879,"Ġrooftop":33880,"ATING":33881,"Ġ390":33882,"VPN":33883,"Gs":33884,"kernel":33885,"turned":33886,"Ġpreferable":33887,"Ġturnovers":33888,"ĠHels":33889,"Sa":33890,"ĠShinji":33891,"veh":33892,"ĠMODULE":33893,"Viol":33894,"Ġexiting":33895,"Ġjab":33896,"ĠVanilla":33897,"Ġacron":33898,"ĠGap":33899,"bern":33900,"Ak":33901,"ĠMcGu":33902,"Ġendlessly":33903,"ĠFarage":33904,"ĠNoel":33905,"Va":33906,"MK":33907,"Ġbrute":33908,"ĠKru":33909,"ĠESV":33910,"ĠOlivia":33911,"âĢł":33912,"ĠKaf":33913,"Ġtrusting":33914,"Ġhots":33915,"324":33916,"Ġmalaria":33917,"Ġjson":33918,"Ġpounding":33919,"ortment":33920,"Country":33921,"Ġpostponed":33922,"Ġunequiv":33923,"?),":33924,"ĠRooney":33925,"udding":33926,"ĠLeap":33927,"urrence":33928,"shapeshifter":33929,"ĠHAS":33930,"osate":33931,"Ġcavern":33932,"Ġconservatism":33933,"ĠBAD":33934,"Ġmileage":33935,"Ġarresting":33936,"Vaults":33937,"Ġmixer":33938,"Democratic":33939,"ĠBenson":33940,"Ġauthored":33941,"8000":33942,"Ġproactive":33943,"ĠSpiritual":33944,"tre":33945,"Ġincarcerated":33946,"ĠSort":33947,"Ġpeaked":33948,"Ġwielding":33949,"reciation":33950,"×Ļ×":33951,"Patch":33952,"ĠEmmy":33953,"Ġexqu":33954,"tto":33955,"ĠRatio":33956,"ĠPicks":33957,"ĠGry":33958,"phant":33959,"Ġfret":33960,"Ġethn":33961,"Ġarchived":33962,"%-":33963,"cases":33964,"ĠBlaze":33965,"Ġimb":33966,"cv":33967,"yss":33968,"imony":33969,"Ġcountdown":33970,"Ġawakening":33971,"ĠTunisia":33972,"ĠRefer":33973,"ĠMJ":33974,"Ġunnatural":33975,"ĠCarnegie":33976,"izen":33977,"ĠNuggets":33978,"hess":33979,"Ġevils":33980,"647":33981,"Ġintroductory":33982,"loving":33983,"ĠMcMahon":33984,"Ġambiguity":33985,"Label":33986,"ĠAlmighty":33987,"Ġcoloring":33988,"ĠClaus":33989,"setting":33990,"NULL":33991,"ĠFavorite":33992,"ĠSIG":33993,">(":33994,"ĠShiva":33995,"ĠMayer":33996,"Ġstormed":33997,"ĠCoverage":33998,"weapons":33999,"igham":34000,"Ġunanswered":34001,"Ġleve":34002,"Ġcoy":34003,"cas":34004,"bags":34005,"asured":34006,"Seattle":34007,"ĠSantorum":34008,"serious":34009,"Ġcourageous":34010,"ĠSoup":34011,"Ġconfiscated":34012,"Ġ///":34013,"Ġunconventional":34014,"Ġmoms":34015,"ĠRohingya":34016,"ĠOrchestra":34017,"ĠPotion":34018,"Ġdiscredit":34019,"ĠFIL":34020,"fixed":34021,"ĠDeer":34022,"doi":34023,"ĠDimension":34024,"Ġbureaucrats":34025,"eteen":34026,"ĠactionGroup":34027,"ohm":34028,"Ġbumps":34029,"ĠUtility":34030,"Ġsubmarines":34031,"renheit":34032,"research":34033,"ĠShapiro":34034,"Ġsketches":34035,"Ġdeceptive":34036,"ĠVil":34037,"esame":34038,"ĠEssentially":34039,"Ġrampage":34040,"isky":34041,"Ġmuttered":34042,"thritis":34043,"Ġ236":34044,"fet":34045,"bars":34046,"Ġpupil":34047,"ĠThou":34048,"oS":34049,"song":34050,"Ġfractured":34051,"Ġrevert":34052,"picture":34053,"Ġcriterion":34054,"usher":34055,"Ġrepercussions":34056,"ĠVintage":34057,"ĠSuperintendent":34058,"Officers":34059,"Ġflagged":34060,"Ġblames":34061,"Ġinverse":34062,"ographers":34063,"Ġmakeshift":34064,"Ġdevoid":34065,"Ġfossils":34066,"ĠAristotle":34067,"ĠFunds":34068,"Ġdepleted":34069,"ĠFlu":34070,"ĠYuan":34071,"Ġwoes":34072,"Ġlipid":34073,"Ġsitu":34074,"requisites":34075,"Ġfurnish":34076,"ĠSamar":34077,"Ġshameful":34078,"Ġadversely":34079,"Ġadept":34080,"Ġremorse":34081,"Ġmurderous":34082,"uckles":34083,"ĠESL":34084,"Ġ314":34085,"sent":34086,"Ġredef":34087,"ĠCache":34088,"ĠPurs":34089,"igans":34090,"Ġ460":34091,"Ġprescriptions":34092,"Ġfres":34093,"Fuck":34094,"ocrates":34095,"Twenty":34096,"ĠWeird":34097,"ĠToggle":34098,"ĠCalled":34099,"itizens":34100,"Ġpoultry":34101,"Ġharvesting":34102,"ãĤ¦ãĤ¹":34103,"Bottom":34104,"Ġcautioned":34105,"tn":34106,"396":34107,"ĠNikki":34108,"Ġevaluations":34109,"Ġharassing":34110,"Ġbindings":34111,"ĠMonetary":34112,"Ġhitters":34113,"Ġadversary":34114,"unts":34115,"Ġsetback":34116,"Ġencrypt":34117,"ĠCait":34118,"Ġlows":34119,"enges":34120,"ĠNorn":34121,"Ġbulbs":34122,"Ġbottled":34123,"ĠVoyager":34124,"317":34125,"Ġspheres":34126,"politics":34127,"Ġsubtract":34128,"Ġsensations":34129,"Ġappalling":34130,"Ġ316":34131,"Ġenvironmentally":34132,"ĠSTEM":34133,"Ġpublishes":34134,"560":34135,"Ġdiligence":34136,"484":34137,"Ġadvises":34138,"Ġpetrol":34139,"Ġimagining":34140,"Ġpatrols":34141,"ĠInteger":34142,"ĠAshes":34143,"actus":34144,"ĠRadiant":34145,"ĠLT":34146,"itability":34147,"htaking":34148,"Setting":34149,"Ġnuanced":34150,"ĠReef":34151,"ĠDevelopers":34152,"Ni":34153,"pieces":34154,"990":34155,"License":34156,"Ġlowers":34157,"ĠOttoman":34158,"327":34159,"ooo":34160,"Ġquitting":34161,"markets":34162,"Behind":34163,"Ġbasin":34164,"Ġdocs":34165,"anie":34166,"flash":34167,"ctl":34168,"Ġcivilized":34169,"ĠFukushima":34170,"\"],\"":34171,"ĠKS":34172,"ĠHonestly":34173,"arat":34174,"Ġconstructs":34175,"ĠLans":34176,"ĠDire":34177,"ĠLIKE":34178,"ĠTrouble":34179,"Ġwithholding":34180,"ĠOblivion":34181,"Ġsanity":34182,"anya":34183,"Const":34184,"Ġgrocer":34185,"ĠCelsius":34186,"Ġrecounted":34187,"ĠWife":34188,"Border":34189,"atered":34190,"happy":34191,"Ġspoiler":34192,"Ġlogically":34193,"Hall":34194,"Ġsucceeding":34195,"Ġpolymorph":34196,"Ġaxes":34197,"ĠShotgun":34198,"ĠSlim":34199,"ĠPrinciples":34200,"ĠLeth":34201,"arta":34202,"Ġscor":34203,"Screenshot":34204,"Ġrelaxation":34205,"#$#$":34206,"Ġdeterrent":34207,"iddy":34208,"Ġpowerless":34209,"Ġlesbians":34210,"Ġchords":34211,"ĠEdited":34212,"selected":34213,"Ġseparatists":34214,"0002":34215,"Ġairspace":34216,"Ġturnaround":34217,"Ġcunning":34218,"PATH":34219,"Poly":34220,"Ġbombed":34221,"Ġtion":34222,"xs":34223,"Ġwithhold":34224,"Ġwaged":34225,"ĠLiberties":34226,"Flag":34227,"Ġcomforting":34228,"454":34229,"ĠIris":34230,"arers":34231,"Ġrag":34232,"Ġrelocated":34233,"ĠGuarant":34234,"Ġstrategically":34235,"Ġgamma":34236,"uberty":34237,"ĠLockheed":34238,"gres":34239,"Ġgrilled":34240,"ĠLowe":34241,"stats":34242,"ĠRocks":34243,"Ġsensing":34244,"Ġrenting":34245,"ĠGeological":34246,"اØ":34247,"otrop":34248,"Ġsew":34249,"Ġimproperly":34250,"486":34251,"Ġâĸł":34252,"Ġstarving":34253,"ĠBj":34254,"Discussion":34255,"328":34256,"ĠCombo":34257,"ĠFixes":34258,"NAT":34259,"Ġstriving":34260,"thora":34261,"Ġharvested":34262,"ĠPing":34263,"Ġplayful":34264,"Ġavenues":34265,"Ġoccupational":34266,"Ġwakes":34267,"ĠCourier":34268,"Ġdrummer":34269,"ĠBrowser":34270,"ĠHouth":34271,"itu":34272,"Ġapparel":34273,"paste":34274,"Ġhunted":34275,"ĠSecondly":34276,"lain":34277,"XY":34278,"ĠPIN":34279,"icons":34280,"Ġcocktails":34281,"Ġsizable":34282,"Ġhurdles":34283,"estinal":34284,"ĠRecreation":34285,"Ġeco":34286,"648":34287,"ĠDied":34288,"mint":34289,"Ġfingerprints":34290,"Ġdispose":34291,"ĠBosnia":34292,"tsy":34293,"2200":34294,"Ġinspected":34295,"ĠFou":34296,"Ġfuss":34297,"Ġambush":34298,"ĠRak":34299,"Ġmanifested":34300,"Prosecut":34301,"Ġsuffice":34302,"rences":34303,"Ġcompensated":34304,"ĠCyrus":34305,"Ġgenus":34306,"ĠWolverine":34307,"ĠTrends":34308,"Ġhikes":34309,"ĠSeen":34310,"Ġenrol":34311,"Cold":34312,"Ġpolitely":34313,"ĠSlav":34314,"ĠRupert":34315,"Ġeyewitness":34316,"ĠAlto":34317,"Ġuncomp":34318,"Ġposterior":34319,"Must":34320,"ĠHerz":34321,"Ġprogressively":34322,"Ġ234":34323,"Ġindifference":34324,"ĠCunningham":34325,"Ġacademia":34326,"Ġsewer":34327,"Ġastounding":34328,"ĠAES":34329,"rather":34330,"Ġeldest":34331,"Ġclimbs":34332,"ĠAdds":34333,"Ġoutcry":34334,"Ġcontag":34335,"ĠHouses":34336,"Ġpept":34337,"ĠMelania":34338,"interested":34339,"ĠUCH":34340,"ĠRoots":34341,"ĠHubbard":34342,"ĠTBD":34343,"ĠRomanian":34344,"filename":34345,"Stone":34346,"ĠImpl":34347,"Ġchromosome":34348,"Cle":34349,"dx":34350,"Ġscrambled":34351,"ĠPt":34352,"Ġ242":34353,"OPLE":34354,"Ġtremendously":34355,"Street":34356,"Ġcraving":34357,"Ġbundled":34358,"ĠRG":34359,"pipe":34360,"Ġinjuring":34361,"Ġarcane":34362,"Particip":34363,"ĠHeroic":34364,"sty":34365,"Ġtopping":34366,"ĠTempest":34367,"rentices":34368,"bh":34369,"Ġparanoia":34370,"ĠUnicode":34371,"Ġegregious":34372,"Ġ\\'":34373,"ĠOswald":34374,"Ġgravel":34375,"ĠSimpsons":34376,"Ġbland":34377,"ĠGuantanamo":34378,"Writer":34379,"liners":34380,"ĠDice":34381,"JC":34382,"Ġparity":34383,"Ġsided":34384,"Ġ237":34385,"ĠPyrrha":34386,"atters":34387,"dk":34388,"Fine":34389,"compan":34390,"Ġformulated":34391,"ĠIdol":34392,"ilers":34393,"hemoth":34394,"ĠFav":34395,"Ġintrusion":34396,"Ġcarrots":34397,"ĠLayer":34398,"ĠHacker":34399,"Ġ----------------":34400,"Ġmoderation":34401,"éģ":34402,"ococ":34403,"Ġcharacterize":34404,"ĠTeresa":34405,"Ġsocioeconomic":34406,"Ġperk":34407,"ĠParticipation":34408,"training":34409,"ĠPaulo":34410,"phys":34411,"Ġtrustworthy":34412,"Ġembodied":34413,"ĠMerch":34414,"currency":34415,"ĠPriority":34416,"Ġteasing":34417,"Ġabsorbing":34418,"Ġunfinished":34419,"ĠComparison":34420,"Ġdisple":34421,"writers":34422,"Ġprofessions":34423,"ĠPenguin":34424,"Ġangrily":34425,"ĠLINK":34426,"688":34427,"ĠCorrespond":34428,"Ġprevailed":34429,"Ġcartel":34430,"lp":34431,"asms":34432,"ĠRedemption":34433,"ĠIslamists":34434,"effects":34435,"dose":34436,"ĠLatter":34437,"ĠHalifax":34438,"Ġvas":34439,"ĠTopics":34440,"ĠNamed":34441,"advertising":34442,"zza":34443,"ICES":34444,"Ġretarded":34445,"achable":34446,"ĠPuppet":34447,"ĠItemLevel":34448,"Ġretract":34449,"Ġidentifiable":34450,"Aaron":34451,"ĠBuster":34452,"sol":34453,"helle":34454,"assemb":34455,"Hope":34456,"ranged":34457,"Ba":34458,"ĠPurch":34459,"éĢ":34460,"ĠSiri":34461,"Ġarrivals":34462,"Ġ1912":34463,"Ġshortened":34464,"Ġ312":34465,"Ġdiscrepancy":34466,"ĠTemperature":34467,"ĠWalton":34468,"Ġkinderg":34469,"polit":34470,"Ġremix":34471,"Ġconnectors":34472,"ãĥĺãĥ©":34473,"ĠKazakhstan":34474,"dominated":34475,"Ġsugars":34476,"imble":34477,"ĠPanic":34478,"ĠDemand":34479,"ĠColony":34480,"onen":34481,"ĠMER":34482,"775":34483,"uria":34484,"azaar":34485,"ĠDegree":34486,"Pri":34487,"Ġsunshine":34488,"Ġ251":34489,"Ġpsychedelic":34490,"Ġdigitally":34491,"ĠBraun":34492,"Ġshimmer":34493,"Ġshave":34494,"ĠTelesc":34495,"ĠAstral":34496,"ĠVenezuelan":34497,"ĠOG":34498,"Ġcrawling":34499,"Integ":34500,"ĠFeather":34501,"Ġunfolding":34502,"Ġappropriation":34503,"Ġè£ıè":34504,"ĠMobility":34505,"ĠNey":34506,"-.":34507,"bilt":34508,"LIN":34509,"ĠTube":34510,"ĠConversely":34511,"Ġkeyboards":34512,"ĠCao":34513,"Ġoverth":34514,"Ġlaure":34515,">>\\":34516,"ĠViper":34517,"acha":34518,"Offset":34519,"ĠRaleigh":34520,"ĠJae":34521,"Jordan":34522,"jp":34523,"Ġtotalitarian":34524,"Connector":34525,"Ġobserves":34526,"ĠSpartan":34527,"ĠImmediately":34528,"ĠScal":34529,"Cool":34530,"Ġtaps":34531,"Ġroar":34532,"Past":34533,"Ġchars":34534,"ĠBender":34535,"ĠSheldon":34536,"Ġpainter":34537,"Ġbeacon":34538,"ĠCreatures":34539,"Ġdownturn":34540,"Ġhinder":34541,"ĠAndromeda":34542,"ÃĽ":34543,"ccoli":34544,"ĠFitness":34545,"etrical":34546,"Ġutilizes":34547,"Ġsenate":34548,"Ġensemble":34549,"Ġcheers":34550,"TW":34551,"Ġaffluent":34552,"kil":34553,"rylic":34554,"ordering":34555,"Computer":34556,"Ġgruesome":34557,"ostics":34558,"ĠUbisoft":34559,"ĠKelley":34560,"Ġwrench":34561,"Ġbourgeoisie":34562,"IBLE":34563,"ĠPreston":34564,"worn":34565,"arist":34566,"reating":34567,"Ġstained":34568,"arine":34569,"Ġslime":34570,"ENN":34571,"Ġchests":34572,"Ġgroundwater":34573,"annot":34574,"ĠTray":34575,"ĠLocke":34576,"ĠCTR":34577,"Ġdudes":34578,"ĠExternal":34579,"ĠDecoder":34580,"Ġparamed":34581,"ĠMedline":34582,"809":34583,"ĠDinner":34584,"rupal":34585,"gz":34586,"ĠGum":34587,"ĠDemo":34588,"jee":34589,"Ġdh":34590,"berman":34591,"archs":34592,"Ġenqu":34593,"ĠEpstein":34594,"Ġdevastation":34595,"Ġfriendships":34596,"ĠArd":34597,"Ġ231":34598,"ĠRubin":34599,"ĠDistance":34600,"Ġspurred":34601,"Ġdossier":34602,"Ġoverlooking":34603,"\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\":34604,"Forest":34605,"ĠComes":34606,"\\\",":34607,"ĠIranians":34608,"Ġfixtures":34609,"Laughs":34610,"Ġcurry":34611,"ĠKingston":34612,"Ġsquash":34613,"Ġcatalogue":34614,"Ġabnormalities":34615,"Ġdigestive":34616,".........":34617,"Ġsubordinate":34618,"ogly":34619,"Ġ249":34620,"Middle":34621,"Ġmassac":34622,"Ġburgers":34623,"Ġdownstairs":34624,"Ġ1931":34625,"394":34626,"ĠVG":34627,"Ġlasers":34628,"ĠSikh":34629,"ĠAlexa":34630,"derived":34631,"Ġcyclist":34632,"ãģ®éŃĶ":34633,"oneliness":34634,"!!!!!!!!":34635,"Ġbuffs":34636,"legate":34637,"Ġraping":34638,"Ġrecommending":34639,"rored":34640,"Ġmulticultural":34641,"unique":34642,"Ġbusinessmen":34643,"Ġuneasy":34644,"ĠMAP":34645,"Ġdispersed":34646,"cipline":34647,"Jess":34648,"ĠKerala":34649,"å§":34650,"Ġabstraction":34651,"Surv":34652,"Uh":34653,"Ġprinters":34654,"ija":34655,"owder":34656,"Ġanalogous":34657,"ĠASP":34658,"afer":34659,"Ġunfolded":34660,"Ġleveling":34661,"Ġbreached":34662,"ĠHearing":34663,"Ġnat":34664,"Ġtranslating":34665,"critical":34666,"Ġantagonist":34667,"ĠYesterday":34668,"Ġfuzzy":34669,"wash":34670,"mere":34671,"Ġbewild":34672,"ĠMae":34673,"Virgin":34674,"phrase":34675,"Ġsignaled":34676,"ĠHIGH":34677,"Ġprotester":34678,"Ġgarner":34679,"unknown":34680,"Ġkay":34681,"Ġabducted":34682,"Ġstalking":34683,"amn":34684,"Ġdeserving":34685,"ĠRiv":34686,"ĠJorge":34687,"Ġscratching":34688,"ĠSaving":34689,"iping":34690,"Ġtease":34691,"Ġmissionary":34692,"ĠMorrow":34693,"TIME":34694,"Present":34695,"Ġchemotherapy":34696,"terness":34697,"ĠHomes":34698,"ĠPurdue":34699,"Ġstaunch":34700,"ĠWhitney":34701,"ĠTHERE":34702,"μ":34703,"iatus":34704,"ĠErnest":34705,"ĠDeploy":34706,"Ġcoveted":34707,"FML":34708,"ĠDialogue":34709,"Ġexited":34710,"fruit":34711,"Ġnerd":34712,"\":\"\",\"":34713,"Ġvivo":34714,"ruly":34715,"460":34716,"ĠAmen":34717,"rehensible":34718,"Ġâĺ":34719,"DIR":34720,"Ġadherence":34721,"Ġchew":34722,"ĠCoke":34723,"ĠSergei":34724,"digital":34725,"ĠNeck":34726,"gently":34727,"enthal":34728,"/)":34729,"Ġweary":34730,"Ġguise":34731,"ĠConcord":34732,"ĠOnion":34733,"atcher":34734,"Ġbinge":34735,"ĠDirective":34736,"Ġmanned":34737,"ansk":34738,"Ġillusions":34739,"Ġbillionaires":34740,"383":34741,"olyn":34742,"odynamic":34743,"ĠWheat":34744,"ĠAlic":34745,"Ġcoloured":34746,"ĠNAFTA":34747,"abo":34748,"Ġmacros":34749,"independent":34750,"sweet":34751,"Ġspac":34752,"ĠKabul":34753,"ĠÄ":34754,"eme":34755,"Ġdictated":34756,"Ġshouts":34757,"={":34758,"Ġripping":34759,"ĠShay":34760,"ĠCricket":34761,"directed":34762,"Ġanalysed":34763,"ĠWARRANT":34764,"agons":34765,"ĠBlazers":34766,"Ġcheered":34767,"Ġarithmetic":34768,"ĠTanz":34769,"373":34770,"ĠFlags":34771,"Ġ295":34772,"Ġwitches":34773,"ĠIncluded":34774,"ĠGained":34775,"ĠBlades":34776,"Gam":34777,"ĠSamantha":34778,"ĠAtlantis":34779,"ĠPratt":34780,"Ġspoiled":34781,"ĠIB":34782,"ĠRamirez":34783,"Probably":34784,"rero":34785,"ĠNg":34786,"ĠWarlock":34787,"tp":34788,"Ġoverhe":34789,"Ġadministrations":34790,"Ġtint":34791,"Ġregiment":34792,"Ġpistols":34793,"Ġblankets":34794,"Ġepist":34795,"Ġbowls":34796,"Ġhydraulic":34797,"Ġdean":34798,"Ġjung":34799,"Ġascend":34800,"705":34801,"ĠSantiago":34802,"î":34803,"Ġunavoid":34804,"ĠShaman":34805,"reb":34806,"Ġstemming":34807,"998":34808,"ĠMG":34809,"sticks":34810,"esthesia":34811,"ERO":34812,"Ġmorbid":34813,"ĠGrill":34814,"ĠPoe":34815,"anyl":34816,"Ġdeleting":34817,"ĠSurveillance":34818,"Ġdirectives":34819,"Ġiterations":34820,"ĠRox":34821,"ĠMilky":34822,"Father":34823,"Ġpatented":34824,"447":34825,"Ġprecursor":34826,"Ġmaiden":34827,"ĠPhen":34828,"ĠVegan":34829,"ĠPatent":34830,"Kelly":34831,"Redditor":34832,"Ġnods":34833,"Ġventilation":34834,"ĠSchwarz":34835,"Ġwizards":34836,"Ġominous":34837,"ĠHeads":34838,"ĠBG":34839,"Ġlumber":34840,"ĠSpiel":34841,"ĠisEnabled":34842,"Ġancestral":34843,"ĠShips":34844,"Ġwrestler":34845,"phi":34846,"Ġyuan":34847,"ĠRebellion":34848,"Ġiceberg":34849,"Ġmagically":34850,"Ġdiversion":34851,"arro":34852,"ythm":34853,"ĠRiders":34854,"ĠRobbie":34855,"ĠKara":34856,"ĠMaintenance":34857,"ĠHerb":34858,"Ġharms":34859,"packed":34860,"ĠFeinstein":34861,"Ġmarrying":34862,"Ġblending":34863,"ĠRates":34864,"Ġ1880":34865,"Ġwrink":34866,"ĠUnch":34867,"ĠTorch":34868,"described":34869,"Ġhumanoid":34870,"ilitating":34871,"ĠConv":34872,"ĠFeld":34873,"IGHTS":34874,"Ġwhistleblower":34875,"ortmund":34876,"etsy":34877,"arrett":34878,"ĠMono":34879,"ĠIke":34880,"ĠCNBC":34881,"ĠWAY":34882,"ĠMDMA":34883,"ĠIndividuals":34884,"Ġsupplemental":34885,"Ġpowerhouse":34886,"ĠStru":34887,"Focus":34888,"aphael":34889,"ĠColleg":34890,"atti":34891,"ZA":34892,"Ġperenn":34893,"ĠSignature":34894,"ĠRodney":34895,"Ġcubes":34896,"iddled":34897,"ĠDante":34898,"ĠINV":34899,"ilingual":34900,"ĠCth":34901,"Ġsofa":34902,"Ġintimidate":34903,"ĠRoe":34904,"ĠDiplom":34905,"ĠCountries":34906,"ayson":34907,"Ġextradition":34908,"Ġdisabling":34909,"ĠCardiff":34910,"Ġmemorandum":34911,"ĠTrace":34912,"Ġ???":34913,"sector":34914,"ĠRouhani":34915,"ĠYates":34916,"ĠFreeze":34917,"Ġbladder":34918,"Motor":34919,"ĠPromise":34920,"antasy":34921,"Ġforeseeable":34922,"ĠCologne":34923,"container":34924,"ĠTrees":34925,"ĠGors":34926,"ĠSinclair":34927,"Ġbarring":34928,"keye":34929,"Ġslashed":34930,"ĠStatistical":34931,"éĩ":34932,"Ġâĸº":34933,"Allows":34934,"Ġhumility":34935,"Ġdrilled":34936,"ĠFurn":34937,"443":34938,"Ġsewage":34939,"Ġhomepage":34940,"Ġcourtyard":34941,"Ġvile":34942,"Ġsubsidiaries":34943,"ajo":34944,"directory":34945,"Ġammon":34946,"Vers":34947,"charges":34948,"Ġ}}":34949,"ĠChains":34950,"Ġ246":34951,"nob":34952,"Ġpercept":34953,"Ġgrit":34954,"Ġfishermen":34955,"ĠIraqis":34956,"ĠDISTR":34957,"ĠFULL":34958,"ĠEvaluation":34959,"graph":34960,"atial":34961,"Ġcooperating":34962,"Ġmelan":34963,"Ġenlightened":34964,"Ġali":34965,"tailed":34966,"Ġsalute":34967,"Ġweakest":34968,"ĠBulldogs":34969,"UA":34970,"ĠAlloy":34971,"Ġsemen":34972,"ocene":34973,"ĠWilliamson":34974,"spr":34975,",âĢĶ":34976,"ĠGF":34977,"ittens":34978,"Beat":34979,"ĠJunk":34980,"iphate":34981,"ĠFarmers":34982,"ĠBitcoins":34983,"igers":34984,"dh":34985,"ĠLoyal":34986,"payer":34987,"Ġentertained":34988,"Ġpenned":34989,"Ġcoupon":34990,"Queue":34991,"Ġweakening":34992,"carry":34993,"Ġunderestimate":34994,"Ġshootout":34995,"Ġcharismatic":34996,"ĠProcedure":34997,"Ġprudent":34998,"inances":34999,"Ġriches":35000,"Ġcortical":35001,"Ġstrides":35002,"Ġdrib":35003,"ĠOilers":35004,"540":35005,"ĠPerform":35006,"ĠBangkok":35007,"Ġeuth":35008,"SER":35009,"Ġsimplistic":35010,"tops":35011,"campaign":35012,"Quality":35013,"Ġimpoverished":35014,"ĠEisenhower":35015,"Ġaugment":35016,"ĠHarden":35017,"Ġintervened":35018,"Ġlistens":35019,"ĠKok":35020,"Ġsage":35021,"Ġrubbish":35022,"ĠDed":35023,"Ġmull":35024,"pelling":35025,"Ġvideot":35026,"Production":35027,"DJ":35028,"miah":35029,"Ġadaptations":35030,"Ġmedically":35031,"Ġboarded":35032,"Ġarrogance":35033,"Ġscrapped":35034,"Ġoppress":35035,"FORMATION":35036,"Ġjunction":35037,"415":35038,"EEEE":35039,"Skill":35040,"Ġsubdu":35041,"ĠSuggest":35042,"ĠPett":35043,"Ġlett":35044,"ĠManip":35045,"ĠCaf":35046,"ĠCooperation":35047,"Ther":35048,"Ġregained":35049,"¶æ":35050,"reflect":35051,"Ġthugs":35052,"ĠShelby":35053,"Ġdictates":35054,"ĠWeiner":35055,"ĠHale":35056,"Ġbattleground":35057,"schild":35058,"Ġcondol":35059,"hunt":35060,"ositories":35061,"Ġaccuses":35062,"Filename":35063,"Ġshri":35064,"Ġmotivate":35065,"Ġreflections":35066,"Null":35067,"ĠLobby":35068,"¥µ":35069,"ĠSATA":35070,"ĠBackup":35071,"Ñĥ":35072,"nin":35073,"ĠCorrection":35074,"Ġjuicy":35075,"utra":35076,"ĠPric":35077,"Ġrestraining":35078,"ĠAirbnb":35079,"ĠArrest":35080,"Ġappropriations":35081,"Ġslopes":35082,"Ġmanslaughter":35083,"Ġworkings":35084,"ĠHuss":35085,"ĠFrey":35086,"Leave":35087,"ĠHarmony":35088,"ĠFeder":35089,"Ġ430":35090,"Ġtrench":35091,"Ġgladly":35092,"Ġbullpen":35093,"ĠGau":35094,"bones":35095,"Ġgroove":35096,"Ġpretext":35097,"ãħĭ":35098,"Ġtransmitter":35099,"ĠComponent":35100,"Ġunderage":35101,"ĠEmpires":35102,"Tile":35103,"Ġoy":35104,"ĠMarvin":35105,"ĠCAS":35106,"Ġbloss":35107,"Ġreplicated":35108,"ĠMariners":35109,"Marcus":35110,"ĠBlocks":35111,"Ġliberated":35112,"Ġbutterfly":35113,"Feel":35114,"Ġfermentation":35115,"Ġyoutube":35116,"Ġoffend":35117,"ĠTerm":35118,"resist":35119,"Ġcessation":35120,"Ġinsurgency":35121,"Ġbir":35122,"ĠRaise":35123,"595":35124,"Ġhypotheses":35125,"502":35126,"Ġplaque":35127,"ocrat":35128,"Ġjackets":35129,"ĠHuffPost":35130,"among":35131,"Ġconfer":35132,"487":35133,"ĠLilly":35134,"Ġadapting":35135,"ĠFay":35136,"Ġshoved":35137,"vec":35138,"Ġrefine":35139,"Ġgon":35140,"Ġgunmen":35141,"zai":35142,"ĠShuttle":35143,"ĠIzan":35144,"Ġ1913":35145,"Ġplethora":35146,"··":35147,"Ġ510":35148,"Ġpuberty":35149,"Ġ241":35150,"ĠWealth":35151,"ĠAlma":35152,"ĠMEM":35153,"ĠAdults":35154,"Cas":35155,"prison":35156,"Race":35157,"Ġwaterproof":35158,"Ġathleticism":35159,"Ġcapitalize":35160,"ĠJuice":35161,"Ġilluminated":35162,"ĠPascal":35163,"Ġirritation":35164,"ĠWitnesses":35165,"adle":35166,"ĠAstro":35167,"Ġfax":35168,"ĠElvis":35169,"Primary":35170,"ĠLich":35171,"ĠElves":35172,"Ġresiding":35173,"Ġstumble":35174,"319":35175,"ĠPKK":35176,"Ġadversaries":35177,"DOS":35178,"ĠRitual":35179,"Ġsmear":35180,"Ġarson":35181,"idental":35182,"Ġscant":35183,"Ġmonarchy":35184,"Ġhalftime":35185,"Ġresidue":35186,"Ġindign":35187,"ĠShaun":35188,"ĠElm":35189,"auri":35190,"Aff":35191,"WATCH":35192,"ĠLyon":35193,"helps":35194,"361":35195,"Ġlobbyist":35196,"Ġdiminishing":35197,"Ġoutbreaks":35198,"Ġgoats":35199,"favorite":35200,"ĠNah":35201,"sonian":35202,"ĠBooster":35203,"Ġsandbox":35204,"ĠFare":35205,"ĠMalta":35206,"ĠattRot":35207,"ĠMOR":35208,"lde":35209,"Ġnavigating":35210,"Touch":35211,"Ġuntrue":35212,"ĠDisaster":35213,"Ġludicrous":35214,"Password":35215,"ĠJFK":35216,"blogspot":35217,"416":35218,"ĠUNDER":35219,"ernal":35220,"Ġdelaying":35221,"TOP":35222,"Ġimplants":35223,"ĠAVG":35224,"ĠHuge":35225,"attr":35226,"Ġjournalistic":35227,"ĠPeyton":35228,"ĠIA":35229,"Rap":35230,"goal":35231,"ĠProgramme":35232,"Ġsmashing":35233,"wives":35234,"println":35235,"ĠPlague":35236,"inus":35237,"EEP":35238,"Ġcruiser":35239,"ĠParish":35240,"uminium":35241,"Ġoccupants":35242,"ĠJihad":35243,"mop":35244,"Ġpint":35245,"Ġhect":35246,"ĠMecca":35247,"director":35248,"ĠFunding":35249,"ĠMixed":35250,"Ġstag":35251,"Tier":35252,"Ġgust":35253,"Ġbrightly":35254,"orsi":35255,"Ġuphill":35256,"RD":35257,"Ġlesions":35258,"ĠBundy":35259,"livious":35260,"Ġbiologist":35261,"ĠFaculty":35262,"ĠAuthorization":35263,"Ġ244":35264,"Allow":35265,"ï¸":35266,"ĠGiul":35267,"Ġpertinent":35268,"otaur":35269,"esse":35270,"ĠRoof":35271,"Ġunmanned":35272,"351":35273,"ĠShak":35274,"ĠOrient":35275,"Ġendanger":35276,"Dir":35277,"Ġreplen":35278,"edient":35279,"Ġtailor":35280,"Ġgadgets":35281,"Ġaudible":35282,"âĺĨ":35283,"Nice":35284,"Ġbombard":35285,"ĠRape":35286,"Ġdefiance":35287,"ĠTWO":35288,"ĠFilipino":35289,"Ġunaffected":35290,"ervatives":35291,"Ġsoared":35292,"ĠBolton":35293,"Ġcompromising":35294,"ĠBrewers":35295,"RAL":35296,"ĠAHL":35297,"icycle":35298,"Ġvampires":35299,"Ġdipped":35300,"oyer":35301,"ĠXIII":35302,"Ġsideways":35303,"ĠWaste":35304,"ĠDiss":35305,"ĠâĶľâĶĢâĶĢ":35306,"$.":35307,"Ġhabitats":35308,"ĠBeef":35309,"truth":35310,"trained":35311,"split":35312,"Rus":35313,"Andy":35314,"ĠBram":35315,"REP":35316,"pid":35317,"è£ħ":35318,"ĠMutant":35319,"Anim":35320,"ĠMarina":35321,"Ġfutile":35322,"highest":35323,"frequency":35324,"Ġepilepsy":35325,"Ġcoping":35326,"Ġconcise":35327,"Ġtracing":35328,"ĠSUN":35329,"panel":35330,"ĠSophie":35331,"ĠCrowley":35332,"ĠAdolf":35333,"ĠShooter":35334,"Ġshaky":35335,"ĠIG":35336,"ĠLies":35337,"ĠBarber":35338,"pkg":35339,"Ġuptake":35340,"Ġpredatory":35341,"ULTS":35342,"/**":35343,"Ġintoxicated":35344,"ĠWestbrook":35345,"odder":35346,"hement":35347,"Ġbaseman":35348,"APD":35349,"storage":35350,"ĠFifty":35351,"editor":35352,"GEN":35353,"UTION":35354,"irting":35355,"Ġsewing":35356,"rift":35357,"Ġagony":35358,"ĠSands":35359,"Ġ254":35360,"Cash":35361,"Ġlodge":35362,"Ġpunt":35363,"Natural":35364,"ĠIdeas":35365,"Ġerroneous":35366,"ĠSensor":35367,"ĠHannity":35368,"Ġ1921":35369,"Ġmould":35370,"ĠGon":35371,"kaya":35372,"Ġanonymously":35373,"ĠKEY":35374,"Ġsimulator":35375,"Winter":35376,"Ġstreamed":35377,"507":35378,"?\",":35379,"Ġteased":35380,"Ġcoefficient":35381,"Ġwartime":35382,"ĠTHR":35383,"''.":35384,"ĠBanking":35385,"mpire":35386,"Ġfandom":35387,"Ġlia":35388,"Ga":35389,"Ġdownhill":35390,"Ġinterpreting":35391,"Individual":35392,"Norm":35393,"Ġjealousy":35394,"bitcoin":35395,"Ġpleasures":35396,"ĠToys":35397,"ĠChevrolet":35398,"ĠAdvisor":35399,"IZE":35400,"Ġreceptions":35401,"706":35402,"Cro":35403,"Ġ262":35404,"Ġcitrus":35405,"iru":35406,"Reviewer":35407,"jected":35408,"UES":35409,"anz":35410,"1981":35411,"ĠWorker":35412,"Ġcomplied":35413,"orescent":35414,"continental":35415,"Ton":35416,"ĠPrism":35417,"ĠSheep":35418,"Ġ288":35419,"nox":35420,"ĠVog":35421,"Ord":35422,"Ġrealms":35423,"tek":35424,"Ġirrigation":35425,"Ġbicycles":35426,"Ġelectronically":35427,"poly":35428,"tall":35429,"());":35430,"Ġaesthetics":35431,"ĠIntegrated":35432,"Explore":35433,"Ġdunk":35434,"476":35435,"pain":35436,"ĠJacques":35437,"ĠDmit":35438,"Frames":35439,"Ġreunited":35440,"Ġhumid":35441,"Dro":35442,"Political":35443,"Ġyouthful":35444,"Ġentails":35445,"Ġmosquito":35446,"363":35447,"species":35448,"Ġcoordinating":35449,"ĠMayhem":35450,"ĠMagnus":35451,"Mount":35452,"Improved":35453,"ĠSTATE":35454,"ATTLE":35455,"Ġflowed":35456,"Ġtackled":35457,"Ġfashioned":35458,"Ġreorgan":35459,"ivari":35460,"finger":35461,"Ġreluctantly":35462,"etting":35463,"ĠVand":35464,"young":35465,"ĠGarland":35466,"Ġpresumption":35467,"Ġamenities":35468,"ĠPleasant":35469,"onential":35470,"ĠOxy":35471,"Ġmorals":35472,"ĠYah":35473,"Ready":35474,"Simon":35475,"Enh":35476,"Demon":35477,"Ġclich":35478,"Monitor":35479,"ĠDU":35480,"Ġwelcomes":35481,"Ġstandout":35482,"Ġdreadful":35483,"Ġbananas":35484,"Ġballoons":35485,"hooting":35486,"basic":35487,"Ġsuffix":35488,"Ġduly":35489,"cano":35490,"Chain":35491,"atos":35492,"Ġgeopolitical":35493,"Ġ(&":35494,"ĠGemini":35495,"ÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤ":35496,"Ġacquitted":35497,"Luck":35498,"protect":35499,"1024":35500,"Ġscarcity":35501,"Ġmindfulness":35502,"ecided":35503,"DN":35504,"prime":35505,"ĠPresidents":35506,"ĠVIDEO":35507,"Ġ(âĪĴ":35508,"addock":35509,"NOR":35510,"ĠPru":35511,"pun":35512,"ĠLOL":35513,"))))":35514,"ĠLiqu":35515,"ĠSAS":35516,"Ġstyling":35517,"Ġpunishments":35518,"Ġnumb":35519,"Ġascertain":35520,"ĠRockies":35521,"flu":35522,"Thumbnail":35523,"Ġperpetrated":35524,"ĠSemi":35525,"Ġdisarm":35526,"ĠOlder":35527,"ĠException":35528,"Ġexponentially":35529,"ĠCommunities":35530,"Ġabolish":35531,"ĠPartner":35532,"ptoms":35533,"Ġ777":35534,"ĠFoley":35535,"ĠCases":35536,"Ġgrease":35537,"ĠRebirth":35538,"Ground":35539,"Ġ;)":35540,"ĠDoctrine":35541,"ikini":35542,"Ye":35543,"ĠBlossom":35544,"Ġpersists":35545,"bill":35546,"Ġinfusion":35547,"Ġbuddies":35548,"911":35549,"ĠPatient":35550,"Ġdemos":35551,"Ġacquaintance":35552,"ĠPaw":35553,"atari":35554,"Ġxml":35555,"Ġfascination":35556,"ĠServe":35557,"ÏĤ":35558,"branded":35559,"Ġaz":35560,"Returns":35561,"Ġovershadow":35562,"Ġroam":35563,"Ġspeedy":35564,"numbered":35565,"helial":35566,"Ġdisciple":35567,"Ġassurances":35568,"given":35569,"pecting":35570,"ĠNatalie":35571,"çĶ°":35572,"Ġmosquitoes":35573,"rotein":35574,"Ġnumeric":35575,"Ġindependents":35576,"Ġtransitional":35577,"Ġreactionary":35578,"ĠMechdragon":35579,"doctor":35580,"Ġshortest":35581,"Ġsequential":35582,"ĠBac":35583,"ĠAccounts":35584,"ãģĮ":35585,"achy":35586,"ractive":35587,"ĠRegiment":35588,"Ġbreathtaking":35589,"fficiency":35590,"ĠBates":35591,"Ġ311":35592,"Ġwardrobe":35593,"fts":35594,"ĠBerk":35595,"Simply":35596,"ĠRiverside":35597,"ivering":35598,"idential":35599,"lucent":35600,"Ġenriched":35601,"ĠConver":35602,"ĠGiving":35603,"ãĥĻ":35604,"Ġlegalize":35605,"ĠFTC":35606,"Ġfreaking":35607,"Mix":35608,"Ġterrestrial":35609,"esian":35610,"cients":35611,"Wing":35612,"LOAD":35613,"Ġledge":35614,"ĠViolent":35615,"ĠMetall":35616,"Ġ308":35617,"Ġsoutheastern":35618,"hetto":35619,"Meat":35620,"Ġslowdown":35621,"Ġretreated":35622,"Jeremy":35623,"endas":35624,"*****":35625,"eric":35626,"Ġreins":35627,"oppable":35628,"ĠHumanity":35629,"earances":35630,"rigan":35631,"Camera":35632,"Ġwaivers":35633,"soc":35634,"Ġalteration":35635,"transform":35636,"ĠCemetery":35637,"506":35638,"Ġindefinite":35639,"Ġstimulating":35640,"yg":35641,"603":35642,"ĠSop":35643,"Ġdescriptive":35644,"Phase":35645,"ĠEdmund":35646,"Ġpneumonia":35647,"ventus":35648,"Amb":35649,"Ġlaboratories":35650,"ĠExclusive":35651,"ugar":35652,"Were":35653,"Ġmalfunction":35654,"Ġhomosexuals":35655,"Ġ-------":35656,"uni":35657,"Ġturbines":35658,"ĠEquity":35659,"Du":35660,"Ġminded":35661,"ĠRH":35662,"ĠBlackhawks":35663,"Ġfeats":35664,"Ġ1700":35665,"repl":35666,"362":35667,"laden":35668,"Ġindispensable":35669,"lyss":35670,"tti":35671,"Ġreel":35672,"Ġdiverted":35673,"Ġlikeness":35674,"Ġsubscriptions":35675,"Ġfingert":35676,"Ġfilthy":35677,"destruct":35678,"draft":35679,"ĠBernardino":35680,"launch":35681,"Ġperplex":35682,"ĠSUM":35683,"carb":35684,"Ġsweater":35685,"ĠVenture":35686,"ĠJag":35687,"ĠCeleb":35688,"ĠVoters":35689,"Ġsteadfast":35690,"Ġathletics":35691,"ĠHanson":35692,"ĠDrac":35693,"Tracker":35694,"Ġcommend":35695,"ĠPresidency":35696,"ĠDID":35697,"informed":35698,"Ġwebpage":35699,"Pretty":35700,"Ġforcefully":35701,"ãĥĥãĤ¯":35702,"Ġrelocation":35703,"Ġsatire":35704,"âī":35705,"ĠSunderland":35706,"æĦ":35707,"Voice":35708,"????????":35709,"Ġinformant":35710,"Ġbowel":35711,"ĠUniform":35712,"Ġ...\"":35713,"Ġpurge":35714,"Ġpicnic":35715,"ĠUmb":35716,"ĠUPDATE":35717,"ĠSapphire":35718,"ĠStall":35719,"learn":35720,"Ġobjectively":35721,"Ġobliter":35722,"Ġloophole":35723,"Ġjourneys":35724,"Ġomission":35725,"Pros":35726,"ĠSidney":35727,"ploma":35728,"Ġsprayed":35729,"Ġguru":35730,"Ġtraitor":35731,"Ġtimet":35732,"Ġsnapping":35733,"ĠSevent":35734,"urnal":35735,"ĠUkip":35736,"Ġbowed":35737,"poral":35738,"liberal":35739,"Ros":35740,"Questions":35741,"iOS":35742,"Ġsummarize":35743,"STAT":35744,"Ġ1850":35745,"apest":35746,"Ġlender":35747,"ĠVariable":35748,"bringing":35749,"ĠLORD":35750,",)":35751,"Ġcollapses":35752,"xiety":35753,"ĠNed":35754,"YD":35755,"ĠScha":35756,"Ġantibody":35757,"Ġdisband":35758,"yre":35759,"illusion":35760,"Ġrover":35761,"shed":35762,"ĠHirosh":35763,"cci":35764,"Ġcalam":35765,"ĠMorton":35766,"Pinterest":35767,"Ġ1928":35768,"ĠEuras":35769,"ordes":35770,"Ġfences":35771,"ĠInventory":35772,"ĠValencia":35773,"ĠUd":35774,"ĠTiff":35775,"Ġsque":35776,"Ġquotation":35777,"Ġtroublesome":35778,"erker":35779,"QUEST":35780,"ĠKingdoms":35781,"south":35782,"Ġlevy":35783,"Prince":35784,"ĠSting":35785,"Ġnicknamed":35786,"Ġappe":35787,"Ġphotographic":35788,"Ġcorpus":35789,"reference":35790,"ĠTrog":35791,"Unt":35792,")=(":35793,"ĠLatvia":35794,"Ġactivating":35795,"Ġlicensee":35796,"Ġdisparities":35797,"ĠNewsletter":35798,"ãĥĥãĥĪ":35799,"Ġfreeing":35800,"ĠJeep":35801,"ĠPerception":35802,"insk":35803,"Ġsilicone":35804,"ĠHayden":35805,"Lean":35806,"ĠSuzuki":35807,"ibrarian":35808,"668":35809,"Ġspor":35810,"Ġcorrelations":35811,"aghetti":35812,"Ġtuber":35813,"ĠIPCC":35814,"ilus":35815,"ĠVu":35816,"Ġwealthiest":35817,"ĠCarbuncle":35818,"anza":35819,"Ġfooled":35820,"ĠZur":35821,"Ġdaddy":35822,"rano":35823,"ilian":35824,"Ġknockout":35825,"fman":35826,"required":35827,"ĠWikileaks":35828,"ĠDuffy":35829,"ONT":35830,"Ġinsol":35831,"ĠObjects":35832,"Ġbou":35833,"ĠNordic":35834,"ĠInsert":35835,"scan":35836,"Ġdancers":35837,"Ġidiots":35838,"majority":35839,"ĠNeville":35840,"ĠFreeBSD":35841,"Ġtart":35842,"panic":35843,"690":35844,"Ġcocoa":35845,"Ġsampled":35846,"Ġlookup":35847,"Indust":35848,"Ġinjections":35849,"genre":35850,"Ġau":35851,"Ġroadway":35852,"Ġgenitals":35853,"Kind":35854,"ĠExaminer":35855,"ĠYaz":35856,"Fresh":35857,"Ġparalysis":35858,"ĠAluminum":35859,"Ġreap":35860,"oké":35861,"Ġsloppy":35862,"ĠTunnel":35863,"posium":35864,"nery":35865,"enic":35866,"Ġherbal":35867,"ĠOuter":35868,"ĠBuilder":35869,"Ġincur":35870,"Ġideologies":35871,"Ġbackups":35872,"consuming":35873,"ĠDetect":35874,"deck":35875,"ĠKNOW":35876,"ĠGret":35877,"ĠMIC":35878,"Ġtoughness":35879,"ĠExhibit":35880,"Ġhive":35881,"Les":35882,"ĠSCHOOL":35883,"ĠAtari":35884,"alde":35885,"ĠNull":35886,"andestine":35887,"mouse":35888,"Ġbrigade":35889,"489":35890,"Ġrevol":35891,"ĠLawson":35892,"ĠWah":35893,"opoly":35894,"ebted":35895,"ĠSaunders":35896,"Ġ313":35897,"ĠWinc":35898,"Ġtaboo":35899,"ĠHelmet":35900,"Ġwedge":35901,"chip":35902,"ĠTina":35903,"bg":35904,"Ġinfuri":35905,"rn":35906,"Ġanomalies":35907,"ĠSync":35908,"ĠExam":35909,"ĠCommit":35910,"ĠDiary":35911,"ĠALSO":35912,"ĠDebor":35913,"omedical":35914,"Ġcomprehension":35915,"655":35916,"Ġempowering":35917,"Ġire":35918,"Ġjuices":35919,"ĠETH":35920,"ĠBoxing":35921,"=\"/":35922,"Ġfacilitated":35923,"poke":35924,"ĠParsons":35925,"ĠModer":35926,"travel":35927,"Ġcivilizations":35928,"Ġlibertarians":35929,"Ġrune":35930,"ĠClarks":35931,"athed":35932,"Ġcampaigners":35933,"ĠDispatch":35934,"ĠFahrenheit":35935,"ĠCapcom":35936,"----------":35937,"Ġlace":35938,"Ġdraining":35939,"Ġliner":35940,"ĠArtificial":35941,"én":35942,"task":35943,"]).":35944,"ĠGMO":35945,"ĠOperator":35946,"ordinary":35947,"ĠInfluence":35948,"ĠUps":35949,"Ġpotency":35950,"ussen":35951,"ospons":35952,"ĠSwim":35953,"ĠDeadline":35954,"Unity":35955,"Ġculinary":35956,"Ġenlightenment":35957,"Ġwearer":35958,"Ġmined":35959,"Ġply":35960,"Ġincest":35961,"ĠDVDs":35962,"Walk":35963,"BTC":35964,"Trade":35965,"Ġdeval":35966,"iband":35967,"ĠOversight":35968,"Palestinian":35969,"Ġdart":35970,"Ġmul":35971,"LR":35972,"Ġremovable":35973,"ĠRealms":35974,"ìĿ":35975,"Ġmiscar":35976,"ĠVulkan":35977,"685":35978,"ère":35979,"ĠSap":35980,"Ġmerging":35981,"ĠCarly":35982,"chester":35983,"Ġbrisk":35984,"Ġluxurious":35985,"ĠGenerator":35986,"Ġbitterness":35987,"Ġedible":35988,"Ġ243":35989,"TG":35990,"Ġrectangle":35991,"WithNo":35992,"below":35993,"Jenn":35994,"Ġdarkest":35995,"Ġhitch":35996,"Ġdosage":35997,"Ġscaven":35998,"ĠKeller":35999,"ĠIllustrated":36000,"Certainly":36001,"ĠMavericks":36002,"Marginal":36003,"Ġdiarrhea":36004,"Ġenormously":36005,"Ġ999":36006,"shr":36007,"quart":36008,"Ġadamant":36009,"ĠMew":36010,"Ġrenovation":36011,"Ġcervical":36012,"ĠPercentage":36013,"eners":36014,"ĠKimber":36015,"Ġfloats":36016,"Ġdex":36017,"ĠWitcher":36018,"ĠSwansea":36019,"dm":36020,"Ġsalty":36021,"yellow":36022,"Ġcape":36023,"ĠDrain":36024,"ĠPaula":36025,"ĠToledo":36026,"lesi":36027,"Magazine":36028,"ĠWick":36029,"ĠMn":36030,"ĠAck":36031,"ĠRiding":36032,"ASON":36033,"Ġhomophobic":36034,"ARP":36035,"Ġwandered":36036,"CPU":36037,"oodoo":36038,"ĠPipe":36039,"Ġtightening":36040,"ĠButt":36041,"318":36042,"Ġdeserted":36043,"Session":36044,"Ġfacilitating":36045,"Jump":36046,"Ġemergencies":36047,"OWER":36048,"Ġexhaustive":36049,"ĠAFTER":36050,"Ġheartbeat":36051,"ĠLabel":36052,"acky":36053,"ĠCertified":36054,"iltration":36055,"Ze":36056,"ĠUtt":36057,"Ġ1300":36058,"Ġpresume":36059,"ĠDisp":36060,"Ġsurged":36061,"Ġdolls":36062,"Columb":36063,"Ġchimpan":36064,"ĠRazor":36065,"Ġticks":36066,"Ġcouncillor":36067,"Ġpilgrimage":36068,"ĠRebels":36069,"ĠQC":36070,"ĠAuction":36071,"xia":36072,"ikk":36073,"bred":36074,"Ġinsertion":36075,"Ġcoarse":36076,"dB":36077,"SEE":36078,"ĠZap":36079,"ĠFoo":36080,"Ġcontempor":36081,"ĠQuarterly":36082,"otions":36083,"ĠAlchemist":36084,"ĠTrey":36085,"ĠDuo":36086,"Sweet":36087,"804":36088,"ĠGiov":36089,"Ġfunn":36090,"Nin":36091,"hoff":36092,"Ġramifications":36093,"Ġ1922":36094,"ĠExperts":36095,"azes":36096,"Ġgarments":36097,"arial":36098,"ĠNab":36099,"Ġ257":36100,"ĠVed":36101,"Ġhumorous":36102,"ĠPompe":36103,"Ġnylon":36104,"Ġlurking":36105,"ĠSergey":36106,"ĠMattis":36107,"Ġmisogyny":36108,"ĠComponents":36109,"ĠWatching":36110,"ĠFolk":36111,"ractical":36112,"Bush":36113,"Ġtaped":36114,"Ġgrouping":36115,"Ġbeads":36116,"Ġ2048":36117,"Ġcondu":36118,"querque":36119,"Reading":36120,"Ġgrievances":36121,"Ultra":36122,"Ġendpoint":36123,"Hig":36124,"ĠStatic":36125,"ĠScarborough":36126,"Lua":36127,"ĠMessi":36128,"aqu":36129,"ĠPsyNet":36130,"ĠRudd":36131,"Ġavenue":36132,"vp":36133,"Jer":36134,"Ġshady":36135,"ĠResist":36136,"ĠArtemis":36137,"Ġcareless":36138,"Ġbrokers":36139,"Ġtemperament":36140,"Ġ520":36141,"Tags":36142,"ĠTurning":36143,"Ġuttered":36144,"Ġpedd":36145,"Ġimprovised":36146,"Ġ:(":36147,"Ġtabl":36148,"Ġplains":36149,"1600":36150,"pressure":36151,"ĠEssence":36152,"margin":36153,"friends":36154,"ĠRestoration":36155,"Ġpollut":36156,"ĠPoker":36157,"ĠAugustine":36158,"ĠCIS":36159,"ĠSEAL":36160,"orama":36161,"Ġthwart":36162,"seek":36163,"Ġpagan":36164,"º":36165,"cpu":36166,"Ġgarn":36167,"Ġassortment":36168,"ĠILCS":36169,"tower":36170,"Recommended":36171,"Ġunborn":36172,"ĠRandomRedditor":36173,"ĠRandomRedditorWithNo":36174,"Ġparalyzed":36175,"Ġeruption":36176,"Ġintersect":36177,"ĠStoke":36178,"ĠSco":36179,"Bind":36180,"å¾":36181,"ĠPNG":36182,"ĠNegative":36183,"ĠNOAA":36184,"Leon":36185,"Ġalloy":36186,"ĠLama":36187,"ĠDiversity":36188,"575":36189,"Ġunderestimated":36190,"ĠScor":36191,"Ġmural":36192,"Ġbusted":36193,"soon":36194,"lif":36195,"Ġnonex":36196,"Ġallergy":36197,"ĠUnderworld":36198,"ĠRays":36199,"ĠBlasio":36200,"Ġhrs":36201,"ĠDir":36202,"Ġ327":36203,"byter":36204,"Ġreplacements":36205,"Ġactivates":36206,"rived":36207,"MH":36208,"Ġpans":36209,"ĠHI":36210,"Ġlongitudinal":36211,"Ġnuisance":36212,"aler":36213,"Ġswell":36214,"ĠSigned":36215,"sci":36216,"ĠIsles":36217,"ĠAGA":36218,"Ġdefiant":36219,"Ġsonic":36220,"ocon":36221,"KC":36222,"ĠAim":36223,"tie":36224,"ahah":36225,"ĠmL":36226,"DX":36227,"Ġbisc":36228,"ĠBillboard":36229,"ĠSYSTEM":36230,"NEY":36231,"gaard":36232,"Ġdistressed":36233,"formerly":36234,"Alan":36235,"Ġchefs":36236,"Ġoptics":36237,"ĠComet":36238,"ĠAMC":36239,"Ġredesigned":36240,"irmation":36241,"Ġsightings":36242,"382":36243,"311":36244,"ĠWB":36245,"Ġcontraction":36246,"ĠTOTAL":36247,"Dual":36248,"Ġstartled":36249,"Ġunderstandably":36250,"Ġsunglasses":36251,"ETHOD":36252,"Ġdocker":36253,"Ġsurfing":36254,"ĠHEL":36255,"ĠSlack":36256,"tones":36257,"Ġshalt":36258,"Visual":36259,"498":36260,"Department":36261,"cussion":36262,"Ġunrestricted":36263,"Ġtad":36264,"Ġrename":36265,"employed":36266,"Ġeducating":36267,"Ġgrinned":36268,"bedroom":36269,"ĠActivities":36270,"ĠVelvet":36271,"ĠSWAT":36272,"Ġshuffle":36273,"igor":36274,"Ġsaturation":36275,"Finding":36276,"cream":36277,"icter":36278,"Ġvodka":36279,"tracking":36280,"tec":36281,"Ġforeground":36282,"iesta":36283,"Ġvehement":36284,"ĠECB":36285,"ĠTie":36286,"Ey":36287,"Ġturtles":36288,"ĠRailroad":36289,"ĠKatz":36290,"ĠFrames":36291,"Ġmenace":36292,"ĠFellowship":36293,"ĠEssential":36294,"uggish":36295,"Ġdrip":36296,"chwitz":36297,"ĠKyoto":36298,"sb":36299,"ĠNina":36300,"Parameter":36301,"Ġalarms":36302,"ĠClaud":36303,"Ġpioneering":36304,"Ġchiefly":36305,"ĠScream":36306,"Collection":36307,"Ġthankfully":36308,"ĠRonaldo":36309,"åŃIJ":36310,"strip":36311,"ĠDisneyland":36312,"commercial":36313,"Seeing":36314,"Soul":36315,"Ġevacuate":36316,"Ġciv":36317,"ĠAshe":36318,"Ġdivides":36319,"ĠDagger":36320,"rehensive":36321,"Ġberries":36322,"ĠDF":36323,"Ġsushi":36324,"Ġplurality":36325,"WI":36326,"Ġdisadvantaged":36327,"Ġbattalion":36328,"obiles":36329,"451":36330,"Ġcling":36331,"Ġundeniable":36332,"ĠLounge":36333,"Ġhaunt":36334,"phe":36335,"Ġquantify":36336,"Ġdiffered":36337,"Ġ[*]":36338,"ĠViz":36339,"cum":36340,"slave":36341,"Ġvideog":36342,"Ġquar":36343,"Ġbundles":36344,"ĠAlonso":36345,"tackle":36346,"Ġneuronal":36347,"Ġlandslide":36348,"confirmed":36349,"ĠDepth":36350,"Ġrenewables":36351,"Bear":36352,"ĠMacedonia":36353,"Ġjerseys":36354,"Ġbunk":36355,"ĠSpawn":36356,"ĠControls":36357,"ĠBuchanan":36358,"Ġrobotics":36359,"Ġemphasizing":36360,"ĠTutorial":36361,"hyp":36362,"iston":36363,"Ġmonumental":36364,"æ°":36365,"ĠCarry":36366,"Ġtbsp":36367,"enance":36368,"Hill":36369,"arthed":36370,"Ġrotten":36371,"Dean":36372,"Ġtwisting":36373,"Ġgoodwill":36374,"Ġimmersion":36375,"Living":36376,"Ġbrushes":36377,"ĠCGI":36378,"ĠAtk":36379,"traditional":36380,"Ġphantom":36381,"ĠStamina":36382,"Ġexpansions":36383,"ĠMarin":36384,"Ġembarked":36385,"ĠEg":36386,"intestinal":36387,"ĠPEOPLE":36388,"ĠBooth":36389,"ĠAppalach":36390,"Ġrelegated":36391,"VT":36392,"MIT":36393,"Ġmuster":36394,"Ġwithdrawing":36395,"Ġmicroscope":36396,"ĠGathering":36397,"ĠCrescent":36398,"ĠArgentine":36399,"ĠDecre":36400,"ĠDominic":36401,"Ġbuds":36402,"antage":36403,"ĠIon":36404,"Ġwidened":36405,"ONSORED":36406,"ĠGloves":36407,"iannopoulos":36408,"razen":36409,"feel":36410,"Ġrepayment":36411,"Ġhindsight":36412,"ĠREALLY":36413,"ĠPistol":36414,"ĠBrah":36415,"Ġwatts":36416,"Ġsurvives":36417,"Ġflurry":36418,"issy":36419,"Alert":36420,"ĠUruguay":36421,"Phoenix":36422,"Slow":36423,"ĠGrave":36424,"ĠFir":36425,"Ġmanageable":36426,"Ġtariff":36427,"ĠUDP":36428,"ĠPistons":36429,"ĠNigerian":36430,"Ġstrikeouts":36431,"Ġcosmetics":36432,"whelming":36433,"fab":36434,"cape":36435,"proxy":36436,"Ġrethink":36437,"Ġovercoming":36438,"simple":36439,"Ġwoo":36440,"Ġdistracting":36441,"ĠStanton":36442,"ĠTulsa":36443,"ĠDock":36444,"659":36445,"Ġdiscord":36446,"ĠEmacs":36447,"ĠVes":36448,"ĠROB":36449,"Ġreassuring":36450,"Ġconsortium":36451,"Muslims":36452,"321":36453,"Ġprompts":36454,"sei":36455,"ĠHitch":36456,"imposed":36457,"ĠFool":36458,"Ġindiscrim":36459,"wrong":36460,"buquerque":36461,"Davis":36462,"!]":36463,"Ġtimeless":36464,"ĠNEED":36465,"Ġpesticide":36466,"Ġrallying":36467,"ĠCalder":36468,"Ġå¤":36469,"Ġxp":36470,"ĠUnle":36471,"ĠExport":36472,"luaj":36473,"Buff":36474,")[":36937,"Ġsqor":36938,"Saudi":36939,"Ġistg":36940,"Ġindulge":36941,"proc":36942,"Ġdisgusted":36943,"Ġcompounded":36944,"Ġnem":36945,"Ġschooling":36946,"ĠCure":36947,"processing":36948,"Sol":36949,"Ġproverb":36950,"itized":36951,"ĠAlvarez":36952,"Ġscarf":36953,"Ġrectangular":36954,"reve":36955,"Ġhormonal":36956,"ĠStress":36957,"itizen":36958,"Ġ425":36959,"girls":36960,"ĠNoir":36961,"ĠRapp":36962,"Ġmarches":36963,"church":36964,"ĠUses":36965,"Ġ405":36966,"ĠBerm":36967,"Ġordinances":36968,"ĠJudgment":36969,"Charges":36970,"ĠZin":36971,"Ġdusty":36972,"Ġstrawberries":36973,"Ġperce":36974,"ĠThur":36975,"ĠDeborah":36976,"netflix":36977,"ĠLambert":36978,"Ġamused":36979,"ĠGuang":36980,"YOU":36981,"RGB":36982,"ĠCCTV":36983,"Ġfiat":36984,"rang":36985,"Ġfederation":36986,"ĠMant":36987,"ĠBust":36988,"ĠMare":36989,"respective":36990,"ĠMigration":36991,"ĠBIT":36992,"590":36993,"Ġpatriotism":36994,"Ġoutlining":36995,"region":36996,"ĠJosé":36997,"Ġblasting":36998,"ĠEzra":36999,"Bs":37000,"Ġundermines":37001,"ĠSmooth":37002,"Ġclashed":37003,"radio":37004,"Ġtransitioning":37005,"ĠBuccaneers":37006,"ĠOwl":37007,"Ġplugs":37008,"Ġhiatus":37009,"ĠPinball":37010,"Ġmig":37011,"ĠNutr":37012,"ĠWolfe":37013,"Ġintegers":37014,"Ġorbits":37015,"ĠEdwin":37016,"ĠDirectX":37017,"bite":37018,"Ġblazing":37019,"vr":37020,"Edge":37021,"ĠPID":37022,"exit":37023,"ĠComed":37024,"ĠPathfinder":37025,"ĠGuid":37026,"ĠSigns":37027,"ĠZer":37028,"ĠAgenda":37029,"Ġreimbursement":37030,"Mesh":37031,"iPhone":37032,"ĠMarcos":37033,"ĠSites":37034,"hate":37035,"enburg":37036,"Ġsockets":37037,"pend":37038,"Batman":37039,"vir":37040,"ĠSHOW":37041,"Ġprovisional":37042,"conn":37043,"ĠDeaths":37044,"ATIVE":37045,"Profile":37046,"sym":37047,"JA":37048,"Ġninja":37049,"installed":37050,"idates":37051,"ebra":37052,"ĠOmaha":37053,"Ġseizing":37054,"ĠBeasts":37055,"Ġsalts":37056,"Mission":37057,"Generally":37058,"ĠTrilogy":37059,"heon":37060,"legates":37061,"Ġdime":37062,"Ġfaire":37063,"parable":37064,"Graph":37065,"Ġtotaling":37066,"Ġdiagrams":37067,"ĠYanuk":37068,"plet":37069,"ĠMeh":37070,"Ġmythical":37071,"ĠStephens":37072,"autical":37073,"ochemistry":37074,"Ġkilograms":37075,"Ġelbows":37076,"ancock":37077,"ĠBCE":37078,"ĠPrague":37079,"Ġimprov":37080,"ĠDevin":37081,"Ġ\"\\":37082,"paralle":37083,"Ġsupremacists":37084,"ĠBillion":37085,"Ġregimen":37086,"innacle":37087,"Ġrequisite":37088,"angan":37089,"ĠBurlington":37090,"ainment":37091,"ĠObjective":37092,"omsky":37093,"GV":37094,"Ġunilateral":37095,"Ġtc":37096,"Ġhires":37097,"mental":37098,"Ġinvoluntary":37099,"Ġtranspl":37100,"ĠASCII":37101,"¨":37102,"Events":37103,"Ġdoubted":37104,"ĠKaplan":37105,"ĠCourage":37106,"igon":37107,"ĠManaging":37108,"ĠTart":37109,"Ġfalsehood":37110,"ĠViolet":37111,"Ġairs":37112,"Ġfertilizer":37113,"Britain":37114,"Ġaquatic":37115,"ouf":37116,"Words":37117,"ĠHartford":37118,"Ġevenings":37119,"ĠVengeance":37120,"quite":37121,"Gall":37122,"ĠPret":37123,"Ġpdf":37124,"ĠLM":37125,"ĠSochi":37126,"ĠIntercept":37127,"920":37128,"Ġprofitability":37129,"ĠIdle":37130,"ĠMacDonald":37131,"ĠEstablishment":37132,"umsy":37133,"Ġgatherings":37134,"ĠNaj":37135,"Charlie":37136,"Ġascent":37137,"ĠProtector":37138,"Ġalgebra":37139,"Ġbios":37140,"forums":37141,"ELS":37142,"Introduced":37143,"Ġ335":37144,"Ġastronomy":37145,"Contribut":37146,"ĠPolic":37147,"Platform":37148,"Ġcontainment":37149,"wrap":37150,"Ġcoronary":37151,"ĠJelly":37152,"manager":37153,"Ġheartbreaking":37154,"cair":37155,"ĠChero":37156,"cgi":37157,"Medical":37158,"ĠAccountability":37159,"!!\"":37160,"ophile":37161,"Ġpsychotic":37162,"ĠRestrict":37163,"Ġequitable":37164,"issues":37165,"Ġ1905":37166,"ĠNek":37167,"cised":37168,"ĠTracking":37169,"Ġozone":37170,"Ġcooker":37171,"rosis":37172,"Ġreopen":37173,"Ġinfinity":37174,"ĠPharmaceutical":37175,"ensional":37176,"Attempt":37177,"ĠRory":37178,"Marco":37179,"Ġawaits":37180,"HOW":37181,"treated":37182,"Ġbolst":37183,"Ġrevered":37184,"Ġpods":37185,"oppers":37186,"0010":37187,"Ġamplitude":37188,"rican":37189,"SPONSORED":37190,"Ġtrousers":37191,"Ġhalves":37192,"ĠKaine":37193,"ĠCutler":37194,"ĠAUTH":37195,"Ġsplendid":37196,"Ġpreventive":37197,"ĠDudley":37198,"ifacts":37199,"uminati":37200,"ĠYin":37201,"Ġadmon":37202,"ĠVag":37203,"Ġinverted":37204,"Ġhastily":37205,"ĠHague":37206,"Lyn":37207,"Ġledger":37208,"Ġastronomical":37209,"getting":37210,"Ġcirca":37211,"ĠCic":37212,"ĠTennis":37213,"Limited":37214,"Ġdru":37215,"ĠBYU":37216,"Ġtravellers":37217,"Ġpane":37218,"ĠIntro":37219,"Ġpatiently":37220,"Ġaiding":37221,"Ġloos":37222,"ĠTough":37223,"Ġ293":37224,"Ġconsumes":37225,"SourceFile":37226,"Ġ\"\"\"":37227,"Ġbonding":37228,"Ġtilted":37229,"Ġmenstrual":37230,"ĠCelestial":37231,"ULAR":37232,"Plugin":37233,"Ġrisking":37234,"Naz":37235,"ĠRiyadh":37236,"Ġaccredited":37237,"Ġskirm":37238,"éĽ":37239,"Ġexaminer":37240,"Ġmessing":37241,"Ġnearing":37242,"ĠChern":37243,"ĠBeckham":37244,"Ġswapped":37245,"Ġgoose":37246,"Kay":37247,"Ġlofty":37248,"ĠWallet":37249,"Ġ['":37250,"Ġapocalypse":37251,"Ġbamboo":37252,"ĠSPACE":37253,"ĠElena":37254,"Ġ306":37255,"acons":37256,"Ġtightened":37257,"Ġadolescence":37258,"Ġrainy":37259,"Ġvandalism":37260,"ĠNewtown":37261,"Ġconject":37262,"cakes":37263,"Ġcheated":37264,"Ġmoderators":37265,"params":37266,"EFF":37267,"Ġdeceit":37268,"ĠSTL":37269,"ĠTanzania":37270,"ĠRI":37271,"Ġ1923":37272,"ĠExile":37273,"thel":37274,"Ġtheolog":37275,"Ġquirky":37276,"ĠIrvine":37277,"Ġneedy":37278,"oris":37279,"Um":37280,"Ka":37281,"Ġmailbox":37282,"322":37283,"Ġbos":37284,"ĠPetra":37285,"KING":37286,"Ġenlarged":37287,"Often":37288,"Ġbadass":37289,"Ġ343":37290,"ĠPlaces":37291,"ĠCAD":37292,"Ġpristine":37293,"Ġintervening":37294,"direction":37295,"Ġlaz":37296,"ĠDSM":37297,"Ġprojecting":37298,"ĠFunk":37299,"agog":37300,"payment":37301,"nov":37302,"Ġchatter":37303,"ARB":37304,"Ġexaminations":37305,"ĠHousehold":37306,"ĠGus":37307,"Ford":37308,"414":37309,"Boss":37310,"Ġmystic":37311,"Ġleaps":37312,"ĠBav":37313,"ulz":37314,"budget":37315,"Football":37316,"Ġsubsidized":37317,"Ġfirsthand":37318,"Ġcoincide":37319,"ocular":37320,"Conn":37321,"ĠCollabor":37322,"Ġfools":37323,"amura":37324,"ahar":37325,"rists":37326,"Ġswollen":37327,"Ġexpended":37328,"ĠPau":37329,"sup":37330,"Ġspar":37331,"Ġkeynote":37332,"suff":37333,"Ġunequal":37334,"Ġprogressing":37335,"strings":37336,"ĠGamergate":37337,"Disney":37338,"ĠEleven":37339,"omnia":37340,"Ġscripted":37341,"Ġearners":37342,"brother":37343,"ĠEnabled":37344,"æ³":37345,"Ġlarvae":37346,"ĠLOC":37347,"mess":37348,"Wilson":37349,"ĠTemplate":37350,"successfully":37351,"Ġparamount":37352,"Ġcamouflage":37353,"Ġbinds":37354,"ĠQuiet":37355,"ĠShutterstock":37356,"rush":37357,"Ġmascot":37358,"fortune":37359,"ĠColt":37360,"ĠBeyon":37361,"habi":37362,"Ġhairc":37363,"Ġ267":37364,"ĠDeus":37365,"Ġtwitch":37366,"Ġconcentrating":37367,"Ġnipples":37368,"cible":37369,"Ġgir":37370,"NZ":37371,"Math":37372,"nih":37373,"Required":37374,"Ġponder":37375,"ĠSAN":37376,"Ġweddings":37377,"Ġloneliness":37378,"NES":37379,"ĠMahjong":37380,"695":37381,"addle":37382,"ĠGarner":37383,"ĠCOUR":37384,"Bridge":37385,"Ġspree":37386,"ĠCaldwell":37387,"Ġbribery":37388,"Ġ��������":37389,"plugins":37390,"Ġracket":37391,"Ġchampagne":37392,"versible":37393,"Vote":37394,"Ġmodifiers":37395,"Mayor":37396,"680":37397,"Ġassemblies":37398,"ĠSultan":37399,"ĠNing":37400,"ĠLadies":37401,"Ġsulfur":37402,"Ġorbs":37403,"Ġ-----":37404,"_______":37405,"ĠJournalism":37406,"Ġesports":37407,"Ġlush":37408,"Ġhue":37409,"Ġspectral":37410,"Honest":37411,"ãĥı":37412,"Ġbushes":37413,"Ġreinforcement":37414,"Ġreopened":37415,"ĠWheels":37416,"ĠMorg":37417,"rieving":37418,"Ġauxiliary":37419,"ĠjQuery":37420,"ĠBAT":37421,"tesque":37422,"Ġvertex":37423,"pure":37424,"frey":37425,"ãĤº":37426,"dos":37427,"Ġtyph":37428,"Ġcull":37429,"Ġeq":37430,"Ġdecon":37431,"Ġtossing":37432,"Ġdisparate":37433,"ĠBrigham":37434,"printf":37435,"ledged":37436,"Ġsund":37437,"Ġcozy":37438,"Ġhepatitis":37439,"performing":37440,"Ġaval":37441,"ĠGG":37442,"future":37443,"Ġpetertodd":37444,"ĠKosovo":37445,"Ġmagnets":37446,"Already":37447,"ĠEdison":37448,"ĠCeres":37449,"ĠRAID":37450,"Ġbrilliance":37451,"576":37452,"Ġderives":37453,"Ġhypertension":37454,"ĠÎĶ":37455,"Ġlambda":37456,"Ġflair":37457,"Ġmissionaries":37458,"Ġrapes":37459,"ĠStarter":37460,"ĠMonths":37461,"Ġdefy":37462,"Ġseismic":37463,"ĠRaphael":37464,"Ġeurozone":37465,"656":37466,"zsche":37467,"Ġscratched":37468,"Ġbows":37469,"ĠLennon":37470,"ĠGaia":37471,"Ġdripping":37472,"facts":37473,"Ale":37474,"Ġfrogs":37475,"ĠBreast":37476,"ogeneity":37477,"ĠProsecutor":37478,"Ġamplified":37479,"ĠHodg":37480,"ĠFn":37481,"Thousands":37482,"ĠNIH":37483,"ĠMonitoring":37484,"FTWARE":37485,"ĠPriebus":37486,"ĠGrowing":37487,"hunter":37488,"Ġdiagnose":37489,"ĠMald":37490,"ĠLR":37491,"Ġcrowned":37492,"Ġbursting":37493,"Ġdissolution":37494,"javascript":37495,"Ġusefulness":37496,"ĠExecution":37497,":(":37498,"ĠIvory":37499,"aah":37500,"Ġpersecuted":37501,"violence":37502,"istas":37503,"ĠCrate":37504,"Ġimpulses":37505,"ĠSpani":37506,"edes":37507,"Handle":37508,"ĠZerg":37509,"thinkable":37510,"Lastly":37511,"Ġspontaneously":37512,"Ġinconvenient":37513,"Ġdismissing":37514,"Ġplotted":37515,"Ġeighty":37516,"Ġ737":37517,"rish":37518,"ĠThornton":37519,"atham":37520,"Ġsitcom":37521,"Ven":37522,"Recipe":37523,"tel":37524,"lund":37525,"Ġclears":37526,"ĠSasuke":37527,"Ġ258":37528,"Ġopting":37529,"Ġenraged":37530,"esthetic":37531,"ĠAe":37532,"uchs":37533,"Prep":37534,"Flow":37535,"Ġrunoff":37536,"ĠEating":37537,"ĠGiles":37538,"ĠActing":37539,"resources":37540,"ibaba":37541,"Ġrpm":37542,"Ġskewed":37543,"ĠBlanc":37544,"ĠSakuya":37545,"Ġhotter":37546,"Ġ1924":37547,"opian":37548,"cko":37549,"Ġcrumbling":37550,"Ġcaptains":37551,"ĠAppropriations":37552,"leaders":37553,"dropping":37554,"anuts":37555,"Ġreversing":37556,"ĠPose":37557,"ĠSek":37558,"Scot":37559,"ĠIdea":37560,"cise":37561,"ĠSlovenia":37562,"Ġ317":37563,"Doctor":37564,"Ġcrocod":37565,"aldi":37566,"Sea":37567,"ĠFarrell":37568,"Ġmercenaries":37569,"ĠRNC":37570,"ĠGuess":37571,"Ġpacing":37572,"Machine":37573,"StreamerBot":37574,"ĠCharity":37575,"Ġ298":37576,"Ġcannons":37577,"ĠToby":37578,"TPPStreamerBot":37579,"ĠPassion":37580,"cfg":37581,"Thom":37582,"Ġbadges":37583,"ĠBernstein":37584,".âĢĵ":37585,"ĠPOP":37586,"ĠConj":37587,"Ġinitialization":37588,"Ġbiodiversity":37589,"Dub":37590,"Ġfeudal":37591,"Ġdisclaimer":37592,"Ġcrow":37593,"Ġignition":37594,"arf":37595,"SHA":37596,"ĠkHz":37597,"hazard":37598,"ĠArtists":37599,"oeuv":37600,"679":37601,"ĠRudy":37602,"Nine":37603,"ĠRamadan":37604,"å½":37605,"itto":37606,"Ġadrenaline":37607,"Cert":37608,"Ġsmelled":37609,"Ġimpunity":37610,"Ġagendas":37611,"ĠReborn":37612,"ĠConcent":37613,"ĠSeems":37614,"Ġomega":37615,"ĠDustin":37616,"Ġbacker":37617,"ĠSauce":37618,"ĠBoyle":37619,"WIN":37620,"Ġspins":37621,"Ġpauses":37622,"upt":37623,"Ġshredded":37624,"Ġstrapped":37625,"ĠCorruption":37626,"Ġscratches":37627,"Ġni":37628,"Ġattire":37629,"ĠSAF":37630,"FactoryReloaded":37631,"ĠIPS":37632,"Ġ(%":37633,"Ġseminar":37634,"focus":37635,"civil":37636,"Ġ1860":37637,"intosh":37638,"Ġcontinual":37639,"Ġabbrevi":37640,"ĠSok":37641,"ocobo":37642,"XM":37643,"Ġfrantic":37644,"Ġunavoidable":37645,"Ġartery":37646,"Ġannotations":37647,"bath":37648,"Climate":37649,"Ġdors":37650,"ĠSlide":37651,"coord":37652,"ĠReload":37653,"ĠLDL":37654,"ĠLovecraft":37655,"Ġunimagin":37656,"Ġresembled":37657,"Ġbarracks":37658,"np":37659,"Ġsurrogate":37660,"Ġcategorized":37661,"ãĤ©":37662,"Ġvaccinated":37663,"Ġdrainage":37664,"Ġindist":37665,"ĠWhatsApp":37666,"Ġ1870":37667,"olerance":37668,"invoke":37669,"amorph":37670,"Ġreconnect":37671,"Ġemanc":37672,"Ġblindness":37673,"Ġ1280":37674,"internet":37675,"collar":37676,"Ġaltru":37677,"Ġabyss":37678,"ĠTRI":37679,"657":37680,"Ġinfused":37681,"HEAD":37682,"Ġforestry":37683,"ĠWoody":37684,"ĠCi":37685,"wi":37686,"sam":37687,"784":37688,"holiday":37689,"Ġmogul":37690,"ĠFees":37691,"ĠDEN":37692,"Internal":37693,"urbed":37694,"fusc":37695,"atom":37696,"ĠIllusion":37697,"Ġpolled":37698,"Ġflap":37699,"Ġcoax":37700,"LGBT":37701,"Analy":37702,"ĠSections":37703,"ĠCaliforn":37704,"emn":37705,"Ġhither":37706,"ĠNIGHT":37707,"Ġnailed":37708,"ĠPipeline":37709,"391":37710,"oof":37711,"ĠPrimal":37712,"verend":37713,"Ġslashing":37714,"Ġretri":37715,"aviour":37716,"Ġdeparting":37717,"gil":37718,"ISC":37719,"Ġmidway":37720,"Ġultrasound":37721,"Ġbehaving":37722,"ĠTara":37723,"classes":37724,"Virtual":37725,"ĠColonial":37726,"Ġstripping":37727,"Ġorchestrated":37728,"ĠGraves":37729,"452":37730,"ĠIronically":37731,"ĠWriters":37732,"Ġlends":37733,"ĠManz":37734,"Ġraven":37735,"Ġoxidative":37736,"Ġ266":37737,"ELF":37738,"actually":37739,"ascar":37740,"Draft":37741,"Ġfavourable":37742,"Ġhumiliating":37743,"Ġfidelity":37744,"ĠHof":37745,"ĠXuan":37746,"496":37747,"Ġlayered":37748,"atis":37749,"790":37750,"Ġpaycheck":37751,"iton":37752,"Kar":37753,"ĠVMware":37754,"ĠFarmer":37755,"Ġservic":37756,"glomer":37757,"Ġslump":37758,"ĠFabric":37759,"ĠDOC":37760,"esting":37761,"Ġreassure":37762,"Ġphyl":37763,"volt":37764,"itory":37765,"Rules":37766,"Ġoxidation":37767,"Ġprized":37768,"Ġmistress":37769,"ĠDjango":37770,"WARN":37771,"åij":37772,"Ġencode":37773,"ĠFeedback":37774,"Ġstupidity":37775,"Ian":37776,"ĠYugoslavia":37777,"ר":37778,"acl":37779,"UTE":37780,"1977":37781,"Ġqualifies":37782,"Ġpulses":37783,"pretty":37784,"Ġfroze":37785,"Ġss":37786,"Iterator":37787,"Ġurgently":37788,"Ġmailed":37789,"ĠCham":37790,"Ġsustaining":37791,"Ġbasil":37792,"Ġpuppies":37793,"ilant":37794,"ĠPLEASE":37795,"lap":37796,"aceous":37797,"Fear":37798,"ĠMastery":37799,"automatic":37800,"ĠTAG":37801,"Ġantim":37802,"agles":37803,"473":37804,"frames":37805,"Ġwhispers":37806,"ĠWhoever":37807,"Ġbravery":37808,"ĠUKIP":37809,"ractions":37810,"\"\"\"":37811,"Ġtame":37812,"Ġparted":37813,"everything":37814,"CONT":37815,"Ġindebted":37816,"Ġaddr":37817,"rek":37818,"IRED":37819,"Ġeminent":37820,"clinton":37821,"Ġousted":37822,"Ġreviewer":37823,"Ġmeltdown":37824,"Ġrearr":37825,"ĠYao":37826,"thereal":37827,"abyte":37828,"Ġstumbling":37829,"Ġbatches":37830,"Ġ259":37831,"Ġcontraceptive":37832,"Ġprostitute":37833,"ensis":37834,"Decl":37835,"ĠStrikes":37836,"Military":37837,"ĠOath":37838,"vacc":37839,"ppings":37840,"052":37841,"ĠpartName":37842,"amping":37843,"Reports":37844,"KI":37845,"CHR":37846,"Ġsubtly":37847,"swers":37848,"Blake":37849,"usual":37850,"Ġcontestants":37851,"Ġcartridges":37852,"ĠGREAT":37853,"Ġblush":37854,"ĠâĢº":37855,"472":37856,"Ġreasoned":37857,"ãĥ¤":37858,"paralleled":37859,"Ġdyn":37860,"agate":37861,"Ġnightly":37862,"åĨ":37863,"556":37864,"Ġsemantic":37865,"ĠAdvoc":37866,"Ġ!!":37867,"Ġdisagrees":37868,"ĠBW":37869,"Veh":37870,"Ġharming":37871,"Ġembraces":37872,"Ġstrives":37873,"Ġinland":37874,"ĠKard":37875,"Ġheats":37876,"ĠGinny":37877,"utan":37878,"ernaut":37879,"ylene":37880,"ĠElev":37881,"JD":37882,"Ġhars":37883,"ĠStarr":37884,"Ġskysc":37885,"Ġcollaborators":37886,"Usually":37887,"Ġrevolutions":37888,"ĠSTATS":37889,"Ġdismantle":37890,"Ġconfidently":37891,"Ġkinetic":37892,"Ali":37893,"Ġpercentile":37894,"Ġextracting":37895,"illian":37896,"estead":37897,"Ġphysicists":37898,"ĠMarshal":37899,"Ġfellowship":37900,"Ġdashed":37901,"ĠUR":37902,"ĠSioux":37903,"ĠCompact":37904,"amide":37905,"Python":37906,"ĠLeigh":37907,"ĠPharmac":37908,"istrates":37909,"herical":37910,"Ġfue":37911,"ĠEmin":37912,"Ġ({":37913,"ĠNeighborhood":37914,"Ġdisrupting":37915,"ĠDup":37916,"Ġgland":37917,"ĠSev":37918,"ĠMarian":37919,"argon":37920,"ĠDund":37921,"Ġ":46904,"ĠPhilips":46905,"ĠKafka":46906,"Ġupheaval":46907,"Ġsentimental":46908,"Ġsax":46909,"ĠAkira":46910,"serial":46911,"Matrix":46912,"Ġelecting":46913,"Ġcommenter":46914,"ĠNebula":46915,"plets":46916,"ĠNadu":46917,"ĠAdren":46918,"Ġenshr":46919,"ĠRAND":46920,"financial":46921,"ĠClyde":46922,"utherford":46923,"Ġsignage":46924,"Ġdeline":46925,"Ġphosphate":46926,"roversial":46927,"fascist":46928,"ĠVall":46929,"ĠBethlehem":46930,"Ġfors":46931,"Ġenglish":46932,"Solid":46933,"Nature":46934,"Ġva":46935,"ĠGuests":46936,"Ġtantal":46937,"Ġautoimmune":46938,";;;;;;;;;;;;":46939,"ĠTotally":46940,"ĠOv":46941,"Ġdefences":46942,"ĠCoconut":46943,"Ġtranquil":46944,"Ġploy":46945,"Ġflavours":46946,"ĠFlask":46947,"ãĤ¨ãĥ«":46948,"ĠWeston":46949,"ĠVolvo":46950,"870":46951,"Ġmicrophones":46952,"verbal":46953,"RPG":46954,"Ġiii":46955,";}":46956,"028":46957,"Ġheadlined":46958,"Ġprimed":46959,"Ġhoard":46960,"ĠShad":46961,"ĠENTER":46962,"Ġtriangular":46963,"Ġcapit":46964,"lik":46965,"ĠAncients":46966,"Ġlash":46967,"Ġconvol":46968,"Ġcolonel":46969,"enemy":46970,"Gra":46971,"Ġpubs":46972,"utters":46973,"Ġassigns":46974,"ĠPenet":46975,"ĠMonstrous":46976,"ĠBowen":46977,"ilver":46978,"Haunted":46979,"ĠDing":46980,"started":46981,"plin":46982,"Ġcontaminants":46983,"ĠDOE":46984,"ffen":46985,"ĠTechnician":46986,"Ry":46987,"Ġrobbers":46988,"Ġhotline":46989,"ĠGuardiola":46990,"ĠKaufman":46991,"rower":46992,"ĠDresden":46993,"ĠAlpine":46994,"Elf":46995,"Ġfmt":46996,"ĠSard":46997,"urses":46998,"gpu":46999,"Unix":47000,"Ġunequivocally":47001,"ĠCitizenship":47002,"quad":47003,"mire":47004,"ĠSweeney":47005,"Battery":47006,"615":47007,"Ġpancakes":47008,"Ġoats":47009,"Maps":47010,"ĠContrast":47011,"mbudsman":47012,"ĠEPS":47013,"Ġsubcommittee":47014,"Ġsourcing":47015,"Ġsizing":47016,"ĠBuffer":47017,"ĠMandatory":47018,"Ġmoderates":47019,"ĠPatterns":47020,"ĠChocobo":47021,"ĠZan":47022,"ĠSTATES":47023,"ĠJudging":47024,"ĠInher":47025,"*:":47026,"Ġbil":47027,"ĠYen":47028,"Ġexhilar":47029,"ollower":47030,"zers":47031,"Ġsnug":47032,"maximum":47033,"Ġdespicable":47034,"ĠPACK":47035,"ĠAnnex":47036,"Ġsarcastic":47037,"Ġlatex":47038,"Ġtamp":47039,"ĠSao":47040,"bah":47041,"ĠReverend":47042,"ĠChinatown":47043,"ĠAUT":47044,"documented":47045,"ĠGABA":47046,"ĠCanaan":47047,"ĠÙħ":47048,"Ġgoverns":47049,"prev":47050,"Esc":47051,"ĠEstimates":47052,"OSP":47053,"Ġendeavour":47054,"ĠClosing":47055,"ometime":47056,"everyone":47057,"Ġworsen":47058,"Ġscanners":47059,"Ġdeviations":47060,"ĠRobotics":47061,"ĠCompton":47062,"Ġsorcerer":47063,"Ġendogenous":47064,"Ġemulation":47065,"ĠPiercing":47066,"ĠAph":47067,"ĠSocket":47068,"Ġbould":47069,"ĠOU":47070,"ĠBorderlands":47071,"Ġ1863":47072,"Gordon":47073,"ĠWTO":47074,"Ġrestricts":47075,"Ġmosaic":47076,"Ġmelodies":47077,"çĦ":47078,"Tar":47079,"Ġdisson":47080,"ĠProvides":47081,"Ġ......":47082,"bek":47083,"FIX":47084,"Ġbroom":47085,"anship":47086,"Doctors":47087,"Ġnerds":47088,"ĠRegions":47089,"naissance":47090,"Ġmete":47091,"Ġcrept":47092,"plings":47093,"Ġgirlfriends":47094,"knit":47095,"igent":47096,"owe":47097,"Ġushered":47098,"ĠBaz":47099,"Mobil":47100,"434":47101,"ĠPresents":47102,"origin":47103,"Ġinsomnia":47104,"ĠAux":47105,"439":47106,"ĠChili":47107,"irsch":47108,"GAME":47109,"Ġgestation":47110,"algia":47111,"romising":47112,"$,":47113,"crow":47114,"ĠInspection":47115,"atomic":47116,"Relations":47117,"JOHN":47118,"roman":47119,"ĠClockwork":47120,"ĠBakr":47121,"mone":47122,"MET":47123,"Ġthirsty":47124,"Ġbc":47125,"Ġfaculties":47126,"Rum":47127,"Ġnuance":47128,"ĠDarius":47129,"pleting":47130,"fters":47131,"etchup":47132,"Registration":47133,"ĠKE":47134,"Rah":47135,"Ġpreferential":47136,"ĠLash":47137,"ĠHH":47138,"Valid":47139,"ĠNAV":47140,"Ġstarve":47141,"ĠGong":47142,"zynski":47143,"ĠActress":47144,"Ġwik":47145,"Ġunaccompanied":47146,"lvl":47147,"Bride":47148,"ADS":47149,"ĠCommando":47150,"ĠVaughn":47151,"Wallet":47152,"Ġhopping":47153,"ĠVie":47154,"Ġcaveats":47155,"Ġalas":47156,"ifled":47157,"abuse":47158,"661":47159,"Ġibn":47160,"Ġgul":47161,"Ġrobbing":47162,"til":47163,"ILA":47164,"Ġmitigating":47165,"Ġaptly":47166,"Ġtyrant":47167,"Ġmidday":47168,"ĠGilmore":47169,"ĠDecker":47170,"Ġ§§":47171,"partial":47172,"Exactly":47173,"Ġphenotype":47174,"Ġ[+]":47175,"ĠPlex":47176,"ĠIps":47177,"versions":47178,"Ġebook":47179,"Ġchic":47180,"gross":47181,"\":\"\"},{\"":47182,"ĠSurprisingly":47183,"Morgan":47184,"Ġresidues":47185,"ĠConfederation":47186,"infeld":47187,"Ġlyr":47188,"moderate":47189,"Ġperpendicular":47190,"VK":47191,"Ġsynchronized":47192,"Ġrefreshed":47193,"Ġadore":47194,"ĠTorment":47195,"olina":47196,"Ġ2600":47197,"ItemTracker":47198,"Ġpies":47199,"ĠFAT":47200,"ĠRHP":47201,"048":47202,"ĠRESP":47203,"ĠBJ":47204,"allows":47205,"Pand":47206,"Ġunwelcome":47207,"ĠVoc":47208,"ĠBastard":47209,"ĠOW":47210,"ĠLAR":47211,"ĠHealer":47212,"Environmental":47213,"ĠKenyan":47214,"ĠTrance":47215,"ĠPats":47216,"Ġaliases":47217,"ĠGarfield":47218,"Ġcampaigner":47219,"Ġadvancements":47220,"ĠOkinawa":47221,"ĠCoh":47222,"owsky":47223,"Ġstarved":47224,"Ġsizeable":47225,"Ġ:-)":47226,"ĠmRNA":47227,"Ġsuspensions":47228,"istar":47229,"Scotland":47230,"Prin":47231,"------------------------------------------------":47232,"Ġ502":47233,"Ġteaspoons":47234,"Ġ1050":47235,"Ġcoercive":47236,"ĠMasonic":47237,"edded":47238,"ĠPassenger":47239,"Ġlatt":47240,"Ġbraces":47241,"ĠSteal":47242,"ĠNYT":47243,"ĠKats":47244,"ĠCelest":47245,"aez":47246,"Tu":47247,"ĠCoulter":47248,"ðŁĺ":47249,"Flickr":47250,"ĠWilmington":47251,"iths":47252,"++;":47253,"Ġvending":47254,"Ġnegro":47255,"ĠPhi":47256,"ĠYellowstone":47257,"Callback":47258,"Ġshampoo":47259,"ĠShades":47260,"wat":47261,"Ġsuperhuman":47262,"Ġridiculed":47263,"Ġholiest":47264,"ombo":47265,"Ġinterns":47266,"Ġhone":47267,"ĠParagu":47268,"URI":47269,"Ġdangling":47270,"ãĤ»":47271,"sov":47272,"ictional":47273,"availability":47274,"Ġrevocation":47275,"Ġdow":47276,"inic":47277,"ĠTHEIR":47278,"Ġiso":47279,"Ġoutings":47280,"ĠLethal":47281,"Ġ)))":47282,"Ġinaccur":47283,"Ġoutlandish":47284,"Ġanus":47285,"letico":47286,"idon":47287,"lol":47288,"Ġunregulated":47289,"Ġsuccumbed":47290,"Ġcuff":47291,"ĠWasteland":47292,"letal":47293,"Ġsubstr":47294,"Ġcoffers":47295,"Ġautomakers":47296,"ovi":47297,"ĠXue":47298,"ĠDaytona":47299,"Ġjarring":47300,"Ġfumes":47301,"Ġdisbanded":47302,"zik":47303,"itton":47304,"Ġstrikingly":47305,"Ġspores":47306,"Adapter":47307,".):":47308,"ĠLyndon":47309,"ivalry":47310,"Ġorally":47311,"Ġtumultuous":47312,"Ġdispleasure":47313,"Ġcones":47314,"orrect":47315,"Ġappease":47316,"Ġderby":47317,"ĠTripoli":47318,"ĠAless":47319,"Ġpoked":47320,"ĠGuilty":47321,"vP":47322,"Enough":47323,"Ġoriginals":47324,"699":47325,"Ġrabbi":47326,"Ġproverbial":47327,"Ġpostpone":47328,"elope":47329,"ĠMisty":47330,"Ġstaffed":47331,"ĠUnemployment":47332,"reditary":47333,"Ġdiligent":47334,"recomm":47335,"measures":47336,"asin":47337,"825":47338,"Ġponds":47339,"Ġmmol":47340,"ĠSAR":47341,"ĠCARE":47342,"Ġ371":47343,"Ġclenched":47344,"ĠCorsair":47345,"Ġcaricature":47346,"zn":47347,"attach":47348,"ĠSchro":47349,"speak":47350,"painted":47351,"ĠSuc":47352,"ĠENT":47353,"Ġcellul":47354,"ĠPaid":47355,"diagn":47356,"WHERE":47357,"Ġtexted":47358,"Barn":47359,"Ġretracted":47360,"ĠReferred":47361,"Sav":47362,"Ġupkeep":47363,"Ġworkplaces":47364,"ĠTokens":47365,"Ġamplify":47366,"clinical":47367,"Ġmultic":47368,"mberg":47369,"Ġconvoluted":47370,"Region":47371,"565":47372,"ĠTopic":47373,"Ġsnail":47374,"Ġsaline":47375,"Ġinsurrection":47376,"ĠPetr":47377,"forts":47378,"BAT":47379,"ĠNavajo":47380,"Ġrudimentary":47381,"ĠLaksh":47382,"ONDON":47383,"Measure":47384,"Ġtransformer":47385,"ĠGoddard":47386,"Ġcoincides":47387,"irin":47388,"Rex":47389,"ĠBok":47390,"quit":47391,"Ġshotguns":47392,"Ġproletarian":47393,"Ġscorp":47394,"ĠAda":47395,"514":47396,"Ġslander":47397,"recorded":47398,"Ġembell":47399,"risome":47400,"Ġapologizing":47401,"ĠMulcair":47402,"ĠGibraltar":47403,"Cla":47404,"Ġallot":47405,"ĠAttention":47406,"Ġ433":47407,"leave":47408,"Ġwhine":47409,"ĠIssa":47410,"ĠFaust":47411,"ĠBarron":47412,"heny":47413,"Ġvictimized":47414,"Jews":47415,"Ġnurturing":47416,"ettel":47417,"Winged":47418,"ĠSubtle":47419,"Ġflavorful":47420,"ĠReps":47421,"enged":47422,"callback":47423,"Ġdirectional":47424,"Ġclasp":47425,"ĠDirections":47426,"planet":47427,"iculture":47428,"Helper":47429,"icion":47430,"acia":47431,"Ġç¥ŀ":47432,"Ġsurges":47433,"Ġcanoe":47434,"ĠPremiership":47435,"been":47436,"Ġdefied":47437,"ĠTrooper":47438,"Ġtripod":47439,"Ġgasp":47440,"ĠEuph":47441,"ĠAds":47442,"vernight":47443,"highly":47444,"Role":47445,"Ġentangled":47446,"ĠZeit":47447,"618":47448,"ĠRusty":47449,"Ġhavens":47450,"ĠVaughan":47451,"HAEL":47452,"ĠSERVICE":47453,"/,":47454,"Ġstricken":47455,"Ġdelusions":47456,"Ġbis":47457,"ĠHaf":47458,"Ġgratification":47459,"Ġenticing":47460,"UNCH":47461,"Adams":47462,"ĠOLED":47463,"ĠBeetle":47464,"Ġ1899":47465,"ĠSOFTWARE":47466,"ategor":47467,"VL":47468,"ĠTotem":47469,"ĠGators":47470,"ATURES":47471,"Ġimpedance":47472,"Registered":47473,"ĠCary":47474,"ĠAerial":47475,"onne":47476,"enium":47477,"Ġdred":47478,"ĠBeg":47479,"Ġconcurrently":47480,"Ġsuperpower":47481,"ĠXan":47482,"jew":47483,"imester":47484,"ĠDickinson":47485,"âĶģ":47486,"Fla":47487,"Ġpree":47488,"ĠRollins":47489,"©¶æ":47490,"Ġdenomination":47491,"ĠLana":47492,"516":47493,"Ġinciting":47494,"scribed":47495,"juries":47496,"ĠWonders":47497,"approximately":47498,"Ġsuspending":47499,"Ġmountainous":47500,"ĠLaugh":47501,"oidal":47502,"Ns":47503,"Detect":47504,")=":47505,"ĠLuthor":47506,"ĠSchwarzenegger":47507,"ĠMuller":47508,"ĠDevi":47509,"ecycle":47510,"Jar":47511,"613":47512,"ĠLongh":47513,"Bah":47514,"ĠSPORTS":47515,"nw":47516,"Ġrefinement":47517,"Ġwaterways":47518,"Ġdiner":47519,"Blade":47520,"683":47521,"Fac":47522,"Ġinitials":47523,"Ġrog":47524,"Ġparanormal":47525,"BUT":47526,"Ġ[(":47527,"ĠSwanson":47528,"ĠMesh":47529,"âĸ¬":47530,"Improve":47531,"ĠRadiation":47532,"ĠEsther":47533,"ĠEsk":47534,"ĠAly":47535,"iky":47536,"Ġirrad":47537,"ĠBuckingham":47538,"Ġrefill":47539,"Ġ._":47540,"Repe":47541,"CONCLUS":47542,"Ġdifferentiated":47543,"Ġchirop":47544,"ĠAtkins":47545,"Pattern":47546,"Ġexcise":47547,"Ġcabal":47548,"NSA":47549,"ĠSTA":47550,"ĠSIL":47551,"ĠParaly":47552,"Ġrye":47553,"ĠHowell":47554,"ĠCountdown":47555,"nesses":47556,"alysed":47557,"Ġresize":47558,"ãĤ½":47559,"Ġbudgetary":47560,"ĠStras":47561,"wang":47562,"Ġapiece":47563,"Ġprecincts":47564,"Ġpeach":47565,"Ġskyline":47566,"Ġ353":47567,"popular":47568,"Appearances":47569,"ĠMechanics":47570,"ĠDevOnline":47571,"Sullivan":47572,"Zen":47573,"Ġpu":47574,"opolis":47575,"544":47576,"Ġdeform":47577,"Ġcounteract":47578,"ĠLange":47579,"Ġ417":47580,"Console":47581,"774":47582,"Ġnodding":47583,"Ġpopulism":47584,"Ġhep":47585,"Ġcounselling":47586,"compliance":47587,"UFF":47588,"Ġundeniably":47589,"Ġrailing":47590,"ĠHorowitz":47591,"ĠSimone":47592,"ĠBungie":47593,"Ġak":47594,"ĠTalks":47595,"xff":47596,"flake":47597,"Crash":47598,"Ġsweaty":47599,"Ġbanquet":47600,"ĠOFFIC":47601,"Ġinventive":47602,"Ġastronomer":47603,"ĠStamford":47604,"ĠScare":47605,"ĠGREEN":47606,"olicited":47607,"Ġrusher":47608,"Ġcentrist":47609,"ighting":47610,"Ġsubclass":47611,"Ġdisav":47612,"Ġdefund":47613,"ĠNanto":47614,"ociate":47615,"mast":47616,"Ġpacif":47617,"Ġmend":47618,"eers":47619,"immigration":47620,"ESSION":47621,"Ġnumbering":47622,"Ġlaughable":47623,"ĠEnded":47624,"viation":47625,"emark":47626,"Pitt":47627,"Ġmeticulous":47628,"ĠLF":47629,"Ġcongratulated":47630,"ĠBirch":47631,"Ġswayed":47632,"Ġsemifinals":47633,"Ġhumankind":47634,"matter":47635,"ĠEquip":47636,"opausal":47637,"Said":47638,"ĠLayout":47639,"Ġvoicing":47640,"Ġthug":47641,"Ġpornographic":47642,"IPS":47643,"Ġmoaning":47644,"Ġgrievance":47645,"Ġconfessions":47646,"escal":47647,"TEXTURE":47648,"Authent":47649,"osaurus":47650,"Purchase":47651,"Ġrelegation":47652,"alter":47653,"Ġ³³":47654,"Ġriddled":47655,"Ġogre":47656,"ĠLowell":47657,"Occup":47658,"Eat":47659,"ĠHyder":47660,"ĠAdviser":47661,"Commerce":47662,"Hunt":47663,"ĠOrth":47664,"ĠCompetitive":47665,"ĠCLA":47666,"CDC":47667,"Ġsalads":47668,"Fle":47669,"Ġindustrialized":47670,"`,":47671,"ĠOWN":47672,"Ġbeck":47673,"ĠParticularly":47674,"oubt":47675,"ĠmM":47676,"ĠHussain":47677,"ĠChennai":47678,"Ġ920":47679,"Ġappointing":47680,"ĠCullen":47681,",,,,,,,,":47682,"Ġpores":47683,"verified":47684,"Ġbiochemical":47685,"emate":47686,"Ġcowardly":47687,"ĠHelsinki":47688,"ĠEthiopian":47689,"SOURCE":47690,"ERC":47691,"estro":47692,"Ġbiotech":47693,"ĠSour":47694,"Ġbrewer":47695,"Bloomberg":47696,"Ġintensify":47697,"Glass":47698,"anco":47699,"ĠFDR":47700,"greSQL":47701,"ĠFires":47702,"©¶æ¥µ":47703,"eco":47704,"1001":47705,"ĠHomeless":47706,"Ġinstantaneous":47707,"ĠHaste":47708,"igel":47709,"Diamond":47710,"Ġpaving":47711,"Ġlandfill":47712,"Ġdads":47713,"houn":47714,":]":47715,"Ġincendiary":47716,"ĠLivingston":47717,"ĠHilbert":47718,"ĠChecks":47719,"styles":47720,"inators":47721,"ĠClive":47722,"phrine":47723,"Ġchimpanzees":47724,"Ġpall":47725,"ĠJM":47726,"ĠAadhaar":47727,"ðĿ":47728,"Ġachievable":47729,"disabled":47730,"PET":47731,"OOOOOOOO":47732,"Mot":47733,"Ġintangible":47734,"Ġballet":47735,"ĠWebs":47736,"ĠEstimated":47737,"Effects":47738,"Ġbailed":47739,"Joshua":47740,"Ġturbulence":47741,"Ġoccupant":47742,"ĠDaylight":47743,"Ġ361":47744,"meet":47745,"Ġstatically":47746,"Ġonlook":47747,"Ġki":47748,"illegal":47749,"Ġvelvet":47750,"Ġdehydration":47751,"Ġacquies":47752,"ĠRez":47753,"akura":47754,"ĠUpton":47755,"atro":47756,"Ġincomprehensible":47757,"Ġbackdoor":47758,"ĠRhino":47759,"727":47760,"Ġmaths":47761,")+":47762,"Ġheresy":47763,"Ġdf":47764,"ĠRoche":47765,"ĠLydia":47766,"Ġpancreat":47767,"reply":47768,"arrell":47769,"Ġsolicitation":47770,"Ġcircadian":47771,"BIP":47772,"Ġforay":47773,"Ġcryptic":47774,"izu":47775,"imeo":47776,"ĠTomato":47777,"ĠHoms":47778,"examination":47779,"Ġquarry":47780,"ĠValiant":47781,"ĠJericho":47782,"ĠINCLUD":47783,"Ġ1840":47784,"519":47785,"Ġresists":47786,"Ġsnapshots":47787,"ĠSpur":47788,"ĠAntiqu":47789,"Login":47790,"Ġbestselling":47791,"Ġantic":47792,"ĠSutherland":47793,"ãĤ¢ãĥ«":47794,"Ġ~/":47795,"ĠParm":47796,"èĥ":47797,"Pages":47798,"intensity":47799,"Ġimmobil":47800,"Ġ1865":47801,"zzo":47802,"Ġnifty":47803,"Ġfentanyl":47804,"ĠPreservation":47805,"ophen":47806,"Ġdarts":47807,"ĠDinosaur":47808,"pointers":47809,"ĠRite":47810,"suggest":47811,"awareness":47812,"ĠSheridan":47813,"Ġstances":47814,"Ġsorcery":47815,"Ġperjury":47816,"ĠNikola":47817,"iever":47818,"Ġfiance":47819,"ĠJordanian":47820,"ĠBalloon":47821,"Ġnab":47822,"Ġkb":47823,"Ġhumanities":47824,"ĠTanaka":47825,"hillary":47826,"Ġconsultancy":47827,"ĠZub":47828,"Ġremission":47829,"Ġconfid":47830,"CHQ":47831,"ĠFug":47832,"Ġimprovis":47833,"Yep":47834,"/_":47835,"Ġunwillingness":47836,"Ġportfolios":47837,"055":47838,"ĠInstructor":47839,"aiman":47840,"Ġclaimants":47841,"Mbps":47842,"ĠBye":47843,"received":47844,"Tweet":47845,"Ġindemn":47846,"riz":47847,"amara":47848,"Nat":47849,"Ġevaluates":47850,"ĠLur":47851,"epad":47852,"FOX":47853,"ĠThro":47854,"Ġrusty":47855,"Ġbedrock":47856,"ĠOprah":47857,"JB":47858,"Ġmanipulative":47859,"Ġwillful":47860,"Ġrelapse":47861,"Ġextant":47862,"Theme":47863,"Sensor":47864,"ĠStability":47865,"govern":47866,"Ġpoppy":47867,"Ġknack":47868,"Ġinsulated":47869,"ĠTile":47870,"ĠExtrem":47871,"Ġuntold":47872,"Ġconverge":47873,"Ġrefuel":47874,"igroup":47875,"Ġdistortions":47876,"Ġravaged":47877,"Ġmechanically":47878,"ĠReilly":47879,"ĠNose":47880,"ĠIncarnation":47881,"ĠBecky":47882,"abbling":47883,"Ġtaco":47884,"Ġrake":47885,"Ġmelancholy":47886,"Ġillustrious":47887,"ĠDartmouth":47888,"Guide":47889,"ĠRazer":47890,"ĠBenz":47891,"Ultimate":47892,"ĠSurprise":47893,"Ġpageant":47894,"offer":47895,"Whoever":47896,"Ġwiser":47897,"Ġchemist":47898,"ĠHELL":47899,"ĠBulk":47900,"Ġplutonium":47901,"ĠCOVER":47902,"Ö¼":47903,"failed":47904,"Ġtirelessly":47905,"Ġinfertility":47906,"ĠTrident":47907,"ĠShowtime":47908,"ĠCiv":47909,"Vice":47910,"requires":47911,"ittance":47912,"Ġuncontrolled":47913,"interesting":47914,"561":47915,"Ġinnovate":47916,"ategic":47917,"Lie":47918,"ĠSelling":47919,"Ul":47920,"Ġsavior":47921,"ĠTosh":47922,"Ġswast":47923,"PASS":47924,"Ġrink":47925,"Ġcardio":47926,"ĠIro":47927,"udi":47928,"Ġvantage":47929,"Ġvans":47930,"ĠNiño":47931,"+=":47932,"Ġpropagate":47933,"":49029,"Ġleukemia":49030,"Ġeluc":49031,"Ġannouncer":49032,"ĠLithuan":49033,"ĠArmageddon":49034,"åĩ":49035,"Lenin":49036,"ĠRuk":49037,"Ġpepp":49038,"ĠRomantic":49039,"ĠPIT":49040,"ĠInterstellar":49041,"ĠAtkinson":49042,"Raid":49043,"Js":49044,"Goal":49045,"Course":49046,"Ġvanishing":49047,"esley":49048,"ĠRounds":49049,"Elsa":49050,"593":49051,"Ġredundancy":49052,"ĠSTAND":49053,"Ġprophetic":49054,"Ġhabitable":49055,"ryu":49056,"Ġfaintly":49057,"MODE":49058,"Ġflanked":49059,"IRC":49060,"Awesome":49061,"Ġspurious":49062,"ĠZah":49063,"ĠMSG":49064,"Ġshading":49065,"Ġmotivational":49066,"ĠSantana":49067,"ĠSPR":49068,"Ġexcruciating":49069,"omial":49070,"ĠMiko":49071,"ĠLeopard":49072,"Abyss":49073,"Ġ[|":49074,"dirty":49075,"Ġbaths":49076,"Ġdemoral":49077,"andre":49078,"PB":49079,"Ġunification":49080,"Ġsacrament":49081,"Ġ[&":49082,"Ġpriceless":49083,"Ġgelatin":49084,"Ġemanating":49085,"ĠAllaah":49086,"986":49087,"Ġoutburst":49088,"Ġeras":49089,"ĠXVI":49090,"ĠSPI":49091,"Ott":49092,"ĠLazarus":49093,"PLIED":49094,"Flying":49095,"blogs":49096,"Wisconsin":49097,"Raven":49098,"Ġrebate":49099,"Ġcreeps":49100,"ĠSpan":49101,"ĠPainter":49102,"ĠKira":49103,"ĠAmos":49104,"ĠCorvette":49105,"Consumer":49106,"ĠRecover":49107,"cki":49108,"Ġpesky":49109,"ĠInvention":49110,"Companies":49111,"Ġchallengers":49112,"ademic":49113,"ĠUkrainians":49114,"ĠNeurolog":49115,"ĠForsaken":49116,"Ġentrants":49117,"Ġembattled":49118,"Ġdefunct":49119,"ĠGlacier":49120,"Ġpoisons":49121,"ĠHorses":49122,"makes":49123,"ĠDirt":49124,"Ġ423":49125,"hhh":49126,"ĠTransformation":49127,"QUIRE":49128,"..................":49129,"Ġtraveller":49130,"ĠSexy":49131,"ĠKern":49132,"ipolar":49133,"Ġransomware":49134,"oooooooooooooooo":49135,"Ec":49136,"ruby":49137,"Professional":49138,"ĠOutbreak":49139,"argument":49140,"Grey":49141,"ĠFifa":49142,"ĠCHO":49143,"ĠFORM":49144,"ĠAmtrak":49145,"-[":49146,"Ġcradle":49147,"Ġantioxidants":49148,"ãģ®å®":49149,"736":49150,"ĠNASL":49151,"ĠContributions":49152,"Indiana":49153,"ĠSTEP":49154,"CSS":49155,"Ġsalient":49156,"Ġallocations":49157,"yrights":49158,"Ġmashed":49159,"ĠCutter":49160,"Sexual":49161,"Ġpounded":49162,"Ġfanbase":49163,"Ġcasc":49164,"ĠTransparency":49165,"Ġanalytic":49166,"ĠSummoner":49167,"×ŀ":49168,"ĠADC":49169,"detail":49170,"Ġvanquished":49171,"Ġcrabs":49172,"arie":49173,"Destroy":49174,"ĠSack":49175,"Ġtransistor":49176,"Alabama":49177,"ĠKoen":49178,"ĠFisheries":49179,"cone":49180,"Ġannexed":49181,"ĠMGM":49182,"esa":49183,"Ġfaked":49184,"ĠCongratulations":49185,"Ġhindered":49186,"Ġcorrectional":49187,"ĠITV":49188,"leeve":49189,"Ġinappropriately":49190,"licks":49191,"Ġtrespass":49192,"Ġpaws":49193,"Ġnegotiator":49194,"ĠChristensen":49195,"limits":49196,"ĠDianne":49197,"Ġelegance":49198,"ĠContracts":49199,"anke":49200,"Obj":49201,"Ġvigilance":49202,"Ġcastles":49203,"ĠNAD":49204,"ĠHolo":49205,"Ġemphatically":49206,"ĠTitus":49207,"ĠServing":49208,"ĠRichie":49209,"ĠPigs":49210,"568":49211,"Ġanimosity":49212,"ĠAttributes":49213,"ĠUriel":49214,"MQ":49215,"myra":49216,"ĠApplicant":49217,"Ġpsychiatrists":49218,"ĠVij":49219,"ĠAbby":49220,"agree":49221,"Push":49222,"ĠkWh":49223,"hiba":49224,"Ġincite":49225,"ĠWeasley":49226,"ĠTaxi":49227,"ministic":49228,"hyper":49229,"ĠFarn":49230,"Ġ601":49231,"ĠNationwide":49232,"Fake":49233,"952":49234,"Ġmaize":49235,"Ġinteracted":49236,"Ġtransitioned":49237,"Ġparasitic":49238,"Ġharmonic":49239,"Ġdecaying":49240,"Ġbaseless":49241,"nsics":49242,"Ġtranspired":49243,"Ġabundantly":49244,"ĠForensic":49245,"Ġtreadmill":49246,"ĠJav":49247,"aband":49248,"Ġsshd":49249,"Ġfrontman":49250,"ĠJakarta":49251,"oller":49252,"drops":49253,"ĠSERVICES":49254,"romptu":49255,"ophical":49256,"hospital":49257,"bledon":49258,"645":49259,"Ġmidrange":49260,"ĠEVENT":49261,"culated":49262,"rawled":49263,"Ġperched":49264,"Ġoverboard":49265,"ĠPeel":49266,"ĠPwr":49267,"ĠCarth":49268,"ĠCOMPLE":49269,"coe":49270,"shall":49271,"Ġdeterrence":49272,"METHOD":49273,"ĠAbsent":49274,"MEN":49275,"Ġsill":49276,"ĠLEVEL":49277,"York":49278,"Ġsinners":49279,"ĠOPEC":49280,"ĠNur":49281,"ĠDesigns":49282,"selection":49283,"Ġunworthy":49284,"CHA":49285,"Ġstrengthens":49286,"883":49287,"edly":49288,"Ġslicing":49289,"Ġmalnutrition":49290,"Ġfilmmaking":49291,"ĠPolk":49292,"urated":49293,"Ġ421":49294,"breakers":49295,"!'\"":49296,"Ġwetlands":49297,"ĠDiscrimination":49298,"Ġallowable":49299,"Ġsteered":49300,"ĠSicily":49301,"SAM":49302,"Ġmustache":49303,"Ġmids":49304,"Ġclipped":49305,"Ġcirculate":49306,"Ġbrittle":49307,"ĠBuildings":49308,"raised":49309,"ĠRoundup":49310,"Ġwealthier":49311,"Ġoverwrite":49312,"Ġoverpowered":49313,"ĠGerrard":49314,"sites":49315,"PDATED":49316,"Ġacutely":49317,"ĠGamble":49318,"Ġpim":49319,"ĠKus":49320,"Typically":49321,"Deploy":49322,"ĠMoroccan":49323,"potion":49324,"combe":49325,"Ġvigilante":49326,"Ġ363":49327,"Stew":49328,"ĠBagg":49329,"Ġresided":49330,"ĠSpo":49331,"Ġremnant":49332,"Ġemptiness":49333,"brainer":49334,"Ġoutpatient":49335,"priority":49336,"Ġleptin":49337,"ĠPayton":49338,"ĠGleaming":49339,"ĠShed":49340,"ĠPolo":49341,"ĠMormonism":49342,"restricted":49343,"arlane":49344,"wx":49345,"Ġcreatine":49346,"ĠAnon":49347,"ĠSTUD":49348,"ĠJUL":49349,"ĠTee":49350,"528":49351,"089":49352,"Ġhatched":49353,"Dispatch":49354,"ĠComposite":49355,"Ġ451":49356,"puff":49357,"ĠXCOM":49358,"ĠOrn":49359,"ĠTHANK":49360,"ENDED":49361,"ĠAsheville":49362,"ĠÃľ":49363,"Ġmango":49364,"ĠSlightly":49365,"worldly":49366,"ĠWander":49367,"ĠExpand":49368,"ĠChr":49369,"Mist":49370,"Ġorthodoxy":49371,"ĠUNESCO":49372,"regate":49373,"Elsewhere":49374,"kie":49375,"irled":49376,"Ġtopple":49377,"Ġadoptive":49378,"ĠLegs":49379,"dress":49380,"ĠSagan":49381,"bare":49382,"ĠGlou":49383,"Crunch":49384,"Ġhelpers":49385,"Ġchronically":49386,"ĠHuma":49387,"10000":49388,"Ġaccommodating":49389,"äºĶ":49390,"Ġwrinkles":49391,"Ġdodged":49392,"fourth":49393,"Ġprecon":49394,"Ġcompressor":49395,"ĠKare":49396,"Ġevict":49397,"ĠWarwick":49398,"imar":49399,"Ġmodernization":49400,"Ġbandwagon":49401,"Ġrefuted":49402,"Ġnetted":49403,"ĠNaples":49404,"ĠGenie":49405,"perors":49406,"Ġfielded":49407,"Ġdere":49408,"ĠParables":49409,"lees":49410,"Ġtrout":49411,"aspers":49412,"Ġnihil":49413,"Ġhappiest":49414,"Ġfloppy":49415,"ĠLoft":49416,"ĠHeard":49417,"Ġunison":49418,"Ġlug":49419,"ĠRedmond":49420,"classic":49421,"Supporters":49422,"SHIP":49423,"GMT":49424,"Ġfuelled":49425,"çIJ":49426,"Ġdd":49427,"ĠEminem":49428,"Ġ1897":49429,"NYSE":49430,"Ġsecretaries":49431,"ĠFIA":49432,"ĠCanaveral":49433,"Favorite":49434,"Ġpomp":49435,"Ġdetainee":49436,"ership":49437,"aimon":49438,"iour":49439,"ĠApex":49440,"Ġplantations":49441,"amia":49442,"acion":49443,"Rust":49444,"Ġtowed":49445,"ĠTruly":49446,"577":49447,"Ġsheltered":49448,"rider":49449,"Wo":49450,"Ġlair":49451,"ĠIntelligent":49452,"improve":49453,"matically":49454,"Ġetiquette":49455,"adra":49456,"allo":49457,"ĠJuno":49458,"anything":49459,"ĠStruggle":49460,"ĠPredict":49461,"ĠGrimes":49462,"ĠAMERICA":49463,"ctx":49464,"ĠSituation":49465,"WOOD":49466,"Ġsoluble":49467,"meier":49468,"Ġintolerable":49469,"angering":49470,"Ġuninterrupted":49471,"Ġtooltip":49472,"Ġinterrogated":49473,"Ġgunned":49474,"ĠSneak":49475,"æѦ":49476,"Ġtether":49477,"Ġcrumble":49478,"Lens":49479,"Ġclustered":49480,"ĠSyl":49481,"ĠHasan":49482,"Ġdystopian":49483,"wana":49484,"Ġjoystick":49485,"ĠThib":49486,"ammu":49487,"Tomorrow":49488,"546":49489,"Ġovercame":49490,"Ġminimized":49491,"ceptor":49492,"Runner":49493,"ENGTH":49494,"ĠBrenda":49495,"ĠAchievements":49496,"Ġtorches":49497,"Ġrapport":49498,"ĠInvestigator":49499,"ĠHandling":49500,"relation":49501,"grey":49502,"815":49503,"Ġkcal":49504,"ĠCommands":49505,"dq":49506,"Ġcurls":49507,"Ġbearer":49508,"Ġcynicism":49509,"itri":49510,"ĠUseful":49511,"Bee":49512,"DCS":49513,"Ġabras":49514,"Pract":49515,"BILITIES":49516,"712":49517,"Ġdebugger":49518,"Ġdebtor":49519,"ĠLia":49520,"ĠKers":49521,"Ġexacerbate":49522,"ĠStacy":49523,"ĠBland":49524,"ĠScenes":49525,"Ġbranching":49526,"âĸĪâĸĪâĸĪâĸĪâĸĪâĸĪâĸĪâĸĪ":49527,"apeake":49528,"Ġsalsa":49529,"Ġmishand":49530,"ĠKonami":49531,"ĠNib":49532,"Ġanecdote":49533,"Ġagreeable":49534,"Ïī":49535,"ĠNathaniel":49536,"ĠHeisman":49537,"ĠBeware":49538,"Ġ1886":49539,"spective":49540,"691":49541,"522":49542,"Ġinhibits":49543,"Ġhashing":49544,"Ġ1889":49545,"å°Ĩ":49546,"vich":49547,"Pure":49548,"Ġsolidly":49549,"Ġaspirin":49550,"imaru":49551,"Ġstreetcar":49552,"ĠUCS":49553,"ĠJudd":49554,"Ġflashbacks":49555,"pins":49556,"Ġ1440":49557,"ĠUNHCR":49558,"ĠSymptoms":49559,"TIT":49560,"538":49561,"Fra":49562,"%);":49563,"Ġooz":49564,"Ġcurfew":49565,"Ġcalmed":49566,"Ġparticipates":49567,"TeX":49568,"Ġnonsensical":49569,"Ġfullback":49570,"ĠDeL":49571,"monkey":49572,"hari":49573,"Ġmetabolites":49574,"Ġlooted":49575,"ĠALWAYS":49576,"ĠBCC":49577,"Lt":49578,"ochet":49579,"Bone":49580,"Ġvetoed":49581,"Ġgcc":49582,"ĠCLICK":49583,"Ġ1888":49584,"saf":49585,"Ġstiffness":49586,"Ġlowly":49587,"ĠGeh":49588,"verson":49589,"orset":49590,"Ġunforeseen":49591,"Ġanesthesia":49592,"ĠOptical":49593,"Ġreconstructed":49594,"ĠTup":49595,"shows":49596,"NEWS":49597,"ĠNewspaper":49598,"ĠASA":49599,"tera":49600,"Numbers":49601,"Ġinexplicable":49602,"×ij":49603,"Ġhardness":49604,"untarily":49605,"ĠAcer":49606,"gradient":49607,"ARDIS":49608,"Ġwoodland":49609,"Ġmetaphors":49610,"ĠWembley":49611,"ĠPavel":49612,"philis":49613,"Ġrewriting":49614,"Ġperceptual":49615,"Ġ1070":49616,"worms":49617,"ĠDowns":49618,"Ġunsurprisingly":49619,"Ġtagging":49620,"flame":49621,"Ġlitres":49622,"Ġbounces":49623,"ĠBabe":49624,"shut":49625,"Ġoverdoses":49626,"ĠSheila":49627,"ĠChau":49628,"ĠBless":49629,"Capture":49630,"ĠSignificant":49631,"ĠScion":49632,"Ġ389":49633,"ĠMcH":49634,"ĠTitanium":49635,"ĠMeal":49636,"ameda":49637,"agents":49638,"aggressive":49639,"Billy":49640,"763":49641,"ĠSaying":49642,"DERR":49643,"itone":49644,"Collins":49645,"Bound":49646,"Ġbolted":49647,"ĠDMCA":49648,"953":49649,"Ġuniqueness":49650,"Ġepigen":49651,"unci":49652,"antam":49653,"Ġreckoning":49654,"chairs":49655,"OGR":49656,"ĠSenegal":49657,"Ġ1862":49658,"relevant":49659,"Ġ¯":49660,"Ġpharmacies":49661,"ĠGeral":49662,"vier":49663,"Yan":49664,"ORPG":49665,"Ġrabid":49666,"bending":49667,"ĠUNITED":49668,"Ġ465":49669,"Assembly":49670,"Ġweep":49671,"Ġbehest":49672,"ĠMothers":49673,"ĠJace":49674,"hid":49675,"Ġwhirlwind":49676,"ĠUNIVERS":49677,"Ġutopian":49678,"Ġkidnap":49679,"Philipp":49680,"Kin":49681,"893":49682,"Ġlivestream":49683,"ĠMISS":49684,"Ġsubversive":49685,"ĠTechniques":49686,"ĠJUSTICE":49687,"ĠBASE":49688,"Ġ387":49689,"Ġassailants":49690,"ĠHardcore":49691,"Ġsprinkled":49692,"ĠPse":49693,"éļ":49694,"printed":49695,"ĠHau":49696,"ORGE":49697,"ĠTOUR":49698,"Ġlaced":49699,"Ġitch":49700,"Giving":49701,"Ġported":49702,"781":49703,"////////////////////////////////":49704,"breeding":49705,"Ġlogger":49706,"ĠHOL":49707,"innie":49708,"Firstly":49709,"Ġembryonic":49710,"Ġdelegated":49711,"pai":49712,"OIL":49713,"Ġcentrally":49714,"ĠRx":49715,"ĠScouting":49716,"Dutch":49717,"Ġhereditary":49718,"ĠCruiser":49719,"sat":49720,"529":49721,"ĠMarriott":49722,"othermal":49723,"Ġprohibitions":49724,"Earn":49725,"ĠStab":49726,"ĠColleges":49727,"ĠBelief":49728,"stretched":49729,"ĠLH":49730,"ĠEntityItem":49731,"CIA":49732,"Ġunrem":49733,"Ġlaureate":49734,"Ġdenominations":49735,"summary":49736,"hler":49737,"Spect":49738,"ĠKlaus":49739,"ĠBeans":49740,"Ġinsur":49741,"ĠPAX":49742,"Ġfielder":49743,"ĠVet":49744,"ĠSparrow":49745,"zie":49746,"ĠSQ":49747,"ĠMondays":49748,"ĠOffline":49749,"ĠLerner":49750,"ĠExtensions":49751,"Ireland":49752,"Ġpatronage":49753,"Ġcontrasted":49754,"ĠMania":49755,"hirt":49756,"Moscow":49757,"Ġcondemns":49758,"ĠAnge":49759,"Ġcomposing":49760,"ĠPepe":49761,"ĠPaddock":49762,"Ġheterogeneity":49763,"Ġideologically":49764,"Ġfishes":49765,"Ġcursing":49766,"ĠRutherford":49767,"ĠFloating":49768,"ĠAmelia":49769,"Tea":49770,"Synopsis":49771,"Ġstunts":49772,"Ġbead":49773,"Ġstocking":49774,"ĠMILL":49775,"obook":49776,"massive":49777,"\\<":49778,"Ġhump":49779,"ĠPreferences":49780,"EngineDebug":49781,"geist":49782,"ĠNieto":49783,"omever":49784,"ishy":49785,"evaluate":49786,"colonial":49787,"Alternative":49788,"ĠGoPro":49789,"ĠVortex":49790,"ĠNETWORK":49791,"ansky":49792,"Secure":49793,"ĠThrust":49794,"Snake":49795,"Ġparcels":49796,"Ġsamurai":49797,"Ġactresses":49798,"Nap":49799,"MF":49800,"iferation":49801,"Beer":49802,"523":49803,"ĠIly":49804,"ointment":49805,"Ping":49806,"Ġstriped":49807,"ĠMellon":49808,"ossession":49809,"Ġneutron":49810,"endium":49811,"Ġaph":49812,"ĠFlavoring":49813,"Ġ383":49814,"Ġresponsiveness":49815,"ĠJindal":49816,"ĠHitchcock":49817,"Denver":49818,"ĠDRAGON":49819,"smanship":49820,"ĠDupl":49821,"Ġsly":49822,"Ġwebcam":49823,"ĠTwain":49824,"ĠDarling":49825,"iliate":49826,"consumer":49827,"DIT":49828,"Ġnamesake":49829,"Ġunorthodox":49830,"Ġfuner":49831,"ĠPLoS":49832,"ĠCONTROL":49833,"ozyg":49834,"oglobin":49835,"FACE":49836,"ERG":49837,"ĠDia":49838,"ĠFiesta":49839,"cele":49840,"034":49841,"Ġenclave":49842,"âĸ¬âĸ¬":49843,"onement":49844,"alist":49845,"Mand":49846,"Ġhomegrown":49847,"ĠFancy":49848,"Ġconceptions":49849,"ĠContains":49850,"ureen":49851,"Ġreiterate":49852,"Ġmeager":49853,"Ġinstallments":49854,"Spawn":49855,"627":49856,"Ġphotoc":49857,"ĠCabrera":49858,"ĠRosenthal":49859,"ĠLansing":49860,"isner":49861,"Ġinvests":49862,"ĠUFOs":49863,"EXP":49864,"Hardware":49865,"Ġtragically":49866,"Ġconcedes":49867,"ieft":49868,"cham":49869,"borgh":49870,"ĠSchr":49871,"ĠMelanie":49872,"ĠHoy":49873,"Ġvisitation":49874,"Ġidiosyncr":49875,"Ġfractions":49876,"Ġforeskin":49877,"obos":49878,"Ġpoaching":49879,"ĠVIEW":49880,"Ġstimulates":49881,"ĠGork":49882,"canon":49883,"MIC":49884,"ĠNemesis":49885,"ĠIndra":49886,"ĠDMV":49887,"Ġ529":49888,"Ġinspecting":49889,"Ġgrandma":49890,"ĠWhedon":49891,"ĠShant":49892,"ĠPurg":49893,"ikan":49894,"ĠTeg":49895,"ĠCLR":49896,"zac":49897,"Victoria":49898,"ĠVerify":49899,"ionics":49900,"Ġpartying":49901,"ĠMou":49902,"colour":49903,"Ġtestimonies":49904,"lations":49905,"Ġpressuring":49906,"hiro":49907,"acers":49908,"Ġfid":49909,"angler":49910,"ĠCSI":49911,"Ġhereafter":49912,"Ġdissidents":49913,"reporting":49914,"iphany":49915,"chev":49916,"Ġsolitude":49917,"Ġlobe":49918,"Ġindis":49919,"Ġcredential":49920,"recent":49921,"adult":49922,"ĠNirvana":49923,"ĠFranchise":49924,"Layer":49925,"Hyp":49926,"ĠBerkshire":49927,"Ġwills":49928,"tif":49929,"Ġtotem":49930,"ĠJudah":49931,"repair":49932,"Instant":49933,"548":49934,"Ġembassies":49935,"Ġbottleneck":49936,"Ġbount":49937,"Ġtypew":49938,"ĠAlvin":49939,"jing":49940,"imilar":49941,"Rush":49942,"Ġbrim":49943,"ĠHELP":49944,"Aim":49945,"]'":49946,"Ġpassively":49947,"Ġbounded":49948,"ĠRated":49949,"Ġcriminality":49950,"Ġbiomark":49951,"Ġdispatcher":49952,"ĠTowards":49953,"Ġ+++":49954,"righteous":49955,"frog":49956,"ĠPanc":49957,"Carter":49958,"032":49959,"æ©Ł":49960,"Ġultraviolet":49961,"ĠLicensed":49962,"ĠTata":49963,"ĠBlessing":49964,"ĠGAM":49965,"Ġchemically":49966,"ĠSeaf":49967,"ĠRELE":49968,"ĠMercenary":49969,"capitalist":49970,"Ġformulations":49971,"Ġannihilation":49972,"ĠVerb":49973,"ĠArgon":49974,"Ġunloaded":49975,"Ġmorphed":49976,"Ġconquering":49977,"backer":49978,"IELD":49979,"Ġthefts":49980,"Ġfrontrunner":49981,"ĠRoyale":49982,"ĠFundamental":49983,"elight":49984,"Chip":49985,"necessary":49986,"ayn":49987,"ĠSlip":49988,"Ġ448":49989,"cerned":49990,"Pause":49991,"Ġshockingly":49992,"ĠABV":49993,"Ġcomposure":49994,"733":49995,"ĠMotorsport":49996,"ahime":49997,"Murray":49998,"Mach":49999,"Ġgrids":50000,"Ġdebian":50001,"Ġfurthermore":50002,"Ġdexterity":50003,"ĠCollections":50004,"oslov":50005,"ilage":50006,"bj":50007,"ĠMonteneg":50008,"ĠstrutConnector":50009,"Ġmassacres":50010,"Ġbriefs":50011,"fetched":50012,"uvian":50013,"olition":50014,"Failure":50015,"emonic":50016,"Ġflared":50017,"Ġclaimant":50018,"Ġcures":50019,"Ġgiveaways":50020,"ĠSubstance":50021,"alions":50022,"Ġcringe":50023,"ĠKul":50024,"Ġaristocracy":50025,"ĠUlster":50026,"olated":50027,"housing":50028,"ĠMIS":50029,"Ġglared":50030,"ĠWilhelm":50031,"needs":50032,"lambda":50033,"builders":50034,"ĠVIS":50035,"Ġradiator":50036,"ĠGhostbusters":50037,"Ġ436":50038,"actual":50039,"Ġherds":50040,"ça":50041,"watching":50042,"Ġcountering":50043,"Charge":50044,"Ġcharred":50045,"Ġwarheads":50046,"Ġiodine":50047,"ĠMacy":50048,"041":50049,"Ġdepartures":50050,"ĠSins":50051,"Ġdyed":50052,"ĠConcepts":50053,"gado":50054,"713":50055,"Ġquotations":50056,"Ġgist":50057,"ĠChristy":50058,"Ġantigen":50059,"ĠHemp":50060,"ĠDrawn":50061,"ĠBarg":50062,"ezvous":50063,"Ġpaternity":50064,"Ġardu":50065,"ĠAnchorage":50066,"ĠRik":50067,"Ġoverloaded":50068,"ĠUsername":50069,"ĠTammy":50070,"ĠNau":50071,"ĠCellular":50072,"Ġwaning":50073,"Ġrodent":50074,"ĠWorcester":50075,"ilts":50076,"ĠTad":50077,"Ġdwellings":50078,"Ġbullish":50079,"431":50080,"Ġretaliate":50081,"Ġmigraine":50082,"ĠChevron":50083,"CHECK":50084,"Ġdonkey":50085,"crim":50086,"SPA":50087,"ĠAnalog":50088,"Ġmarquee":50089,"ĠHaas":50090,"Bir":50091,"ĠGDDR":50092,"ĠDownloads":50093,"Ġwillpower":50094,"ĠForth":50095,"ĠRecorded":50096,"Ġimpossibility":50097,"ĠLogged":50098,"ĠFranks":50099,"ĠRatt":50100,"initions":50101,"Ġcleaners":50102,"Ġsorely":50103,"Ġflickering":50104,"ĠExamination":50105,"catching":50106,"alloween":50107,"Msg":50108,"Ġdunno":50109,"Fa":50110,"Ġdysph":50111,"crazy":50112,".''.":50113,"Ġmainline":50114,"Ġcs":50115,"Ġptr":50116,"ĠWally":50117,"igun":50118,"951":50119,"ĠBigfoot":50120,"fights":50121,"Ġretrieving":50122,"Jr":50123,"Ġduplication":50124,"ĠExplan":50125,"Ġrelational":50126,"Ġquaint":50127,"Ġbiscuits":50128,"Ġado":50129,"Ġshudder":50130,"Ġantidote":50131,"blooded":50132,"ksh":50133,"Ġsauces":50134,"Ġreinvest":50135,"Ġdispensary":50136,"ĠDiver":50137,"Ġ9000":50138,"student":50139,"Ġinsepar":50140,"escap":50141,"Ġtoddlers":50142,"ĠGPIO":50143,"ĠAssignment":50144,"headers":50145,"Ġlackluster":50146,"Ġaback":50147,"956":50148,"Ġtoolbar":50149,"745":50150,"Ġoust":50151,"Ġcontemplation":50152,"ĠPRESIDENT":50153,"Ġ458":50154,"======":50155,"Ġguaranteeing":50156,"ĠHeist":50157,"ĠCannes":50158,"Ļ½":50159,"Ġcollaborator":50160,"ĠAmp":50161,"Ġgou":50162,"ĠSHALL":50163,"stories":50164,"783":50165,"Ġmobilized":50166,"Ġbrood":50167,"ĠLU":50168,"ĠðŁij":50169,"Ġrefin":50170,"ĠAnthropology":50171,"vind":50172,"illi":50173,"Ġwarranties":50174,"ĠBabel":50175,"Ġswath":50176,"Ġcaches":50177,"Ġantagonists":50178,"artifacts":50179,"Ġhotly":50180,"ĠStarts":50181,"ĠGö":50182,"zag":50183,"!!!!!":50184,"Ġscourge":50185,"Ġconspiring":50186,"ruits":50187,"reverse":50188,"ĠSheen":50189,"ĠJesuit":50190,"ĠGiovanni":50191,"adies":50192,"Ġbuttocks":50193,"earcher":50194,"acan":50195,"Ġvolleyball":50196,"Ġshrouded":50197,"Ġscoreboard":50198,"bats":50199,"ĠIPM":50200,"Ġasses":50201,"Ġderegulation":50202,"ĠTelegram":50203,"ĠReboot":50204,"Ġ7000":50205,"ĠCanary":50206,"Ġkernels":50207,"ĠFrançois":50208,"ĠDuff":50209,"ĠPon":50210,"ĠLeica":50211,"ĠGarmin":50212,"Ġorphans":50213,"ĠClaudia":50214,"Ġcalendars":50215,"ĠLeilan":50216,"ento":50217,"Rocket":50218,"Ġbrunch":50219,"ĠHawking":50220,"ainers":50221,"Ġsensibilities":50222,"ĠkW":50223,"ĠKand":50224,"Ġreclaimed":50225,"Ġinterestingly":50226,"ש":50227,"romy":50228,"JM":50229,"ĠEnhancement":50230,"bush":50231,"Skip":50232,"Ġrappers":50233,"Ġgazing":50234,"pedia":50235,"athlon":50236,"Revolution":50237,"Ġsnipers":50238,"Ġreverted":50239,"Ġconglomerate":50240,"Terry":50241,"794":50242,"Ġharsher":50243,"Ġdesolate":50244,"ĠHitman":50245,"Commission":50246,"Ġ(/":50247,"âĢ¦.\"":50248,"Compar":50249,"Ġamplification":50250,"ominated":50251,"Ġregress":50252,"ĠCollider":50253,"Ġinformants":50254,"Ġgazed":50255,"<|endoftext|>":50256} \ No newline at end of file diff --git a/funasr_detach/models/whisper/utils/assets/mel_filters.npz b/funasr_detach/models/whisper/utils/assets/mel_filters.npz deleted file mode 100644 index 9793b084efeb26bf24cb2ba1f30d0989b2c94aa1..0000000000000000000000000000000000000000 --- a/funasr_detach/models/whisper/utils/assets/mel_filters.npz +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:dd2cc75e70e36fcbdd8ffbc2499062f30094093e6bf2cbafa9859f59972b420b -size 2048 diff --git a/funasr_detach/models/whisper/utils/assets/multilingual/added_tokens.json b/funasr_detach/models/whisper/utils/assets/multilingual/added_tokens.json deleted file mode 100644 index 5c509999d68b2eb119d8a4d1218376910beac339..0000000000000000000000000000000000000000 --- a/funasr_detach/models/whisper/utils/assets/multilingual/added_tokens.json +++ /dev/null @@ -1 +0,0 @@ -{"<|endoftext|>": 50257} diff --git a/funasr_detach/models/whisper/utils/assets/multilingual/merges.txt b/funasr_detach/models/whisper/utils/assets/multilingual/merges.txt deleted file mode 100644 index afde9f4fa8bcf6d4762c5780977825666dc49eac..0000000000000000000000000000000000000000 --- a/funasr_detach/models/whisper/utils/assets/multilingual/merges.txt +++ /dev/null @@ -1,50000 +0,0 @@ -Ġ t -Ġ a -Ġt h -i n -e r -Ġ w -Ġ s -o u -Ġth e -r e -o n -a t -e n -Ġ c -i t -i s -Ġ b -n d -Ġ d -Ġ m -Ġ h -Ġ o -in g -e s -Ġ p -Ġt o -a n -Ġ f -o r -l l -Ġ I -Ġ l -Ġ y -a r -Ġ g -Ġy ou -e d -Ġa nd -Ġ in -Ġo f -a s -Ġ n -o m -i c -Ġth at -u s -e t -v e -a l -o w -l e -Ġ is -Ġ e -Ġ it -o t -' s -Ġb e -i on -Ġ T -Ġw h -Ġ A -en t -Ġ S -Ġ re -a y -Ġw e -Ġ on -er e -Ġh a -u t -a c -i d -i g -o s -k e -v er -i m -Ġ Ð -ĠT h -a m -a ll -Ġf or -e l -c h -r o -Ġth is -Ġs t -Ġ W -Ġ u -a d -ou t -i r -l d -c t -Ġ k -i f -Ġg o -. . -Ð ¾ -it h -l y -h t -q u -Ġ - -Ġd o -Ġ j -Ġha ve -Ġ B -Ġa n -Ġw ith -Ġa re -Ġ r -Ġd e -Ġs e -Ġs o -Ġ v -s t -i ll -u r -Ġl i -Ġ M -es t -o d -all y -' t -us t -Ġa s -Ġ C -c e -Ġm e -Ð ° -Ð µ -i l -Ġ H -Ġw as -t er -t h -Ġc an -an t -Ġc om -ou r -ig ht -Ġ Y -at ion -ĠA nd -o l -Ġs h -Ñ Ĥ -o p -s e -Ġn ot -ĠS o -Ġn e -u n -Ġa b -Ġli ke -Ġa t -Ġ D -i e -Ġh e -Ġc on -Ġc h -o re -Ġa l -Ġo r -Ġ qu -Ġ O -om e -r a -u l -Ġ N -p p -Ġyou r -ou ld -Ġ P -Ġf r -g e -er s -' re -Ð ¸ -Ġthe y -Ġwh at -us e -Ġa ll -ĠTh e -Ġ L -es s -e m -Ġk n -Ġj ust -ar t -Ġp ro -ver y -u m -Ġl o -Ġ ì -Ġm y -o k -Ġe x -a b -Ġth ere -Ġb ut -Ġkn ow -Ġs u -Ġ G -Ñ ģ -Ġ E -Ġm a -о Ð -Ġ en -Ġab out -ĠI t -is t -Ġw or -r i -in d -Ġon e -at e -a nd -in k -Ġl e -or t -' m -Ġ F -ic h -Ñ Ģ -id e -Ġg et -Ġ out -.. . -Ġw ill -ã ģ -i ve -Ð ½ -Ġfr om -a in -ĠW e -Ġu p -p e -re s -c a -Ġ R -Ġ if -Ġp l -Ġd on -ac k -Ġ 1 -Ġ " -Ġt r -Ġ us -ĠW h -it y -Ġ J -ĠY ou -Ġh ere -h er -Ġs ome -ou g -a k -ar d -Ġgo ing -Ġu n -m ent -Ġth ink -Ġp e -en d -Ġ ( -ca use -Ġt im -as t -à © -Ġ our -Ġw ant -am e -i es -Ġ ë -u d -in e -Ġre ally -Ġt e -Ġse e -c i -Ġb y -s o -u re -os e -Ġ [ -a re -Ġm ore -a h -on e -c k -op le -а Ð -Ġthe n -Ġth ing -Ġthe m -v en -ou nd -os t -on g -e ct -Ġr ight -a g -Ġin t -Ġpe ople -Ġwh en -ou s -p l -Ġtim e -Ġ im -Ġwh o -Ġ 2 -a p -Ġbe cause -h ing -Ġn o -ic e -Ġlo ok -Ġh as -Ġw ould -Ġh ow -ac t -Ġf e -n t -oug h -Ġp r -ĠB ut -Ġs ay -Ñ ĥ -Ġn ow -Ġm an -Ġ very -Ġwor k -i z -Ġ K -i v -it t -Ġa r -e p -Ġc l -Ġwh ich -Ġc o -an s -' ve -Ġs a -f f -' ll -Ġan y -Ġa ct -Ġy e -b er -ac h -a ge -p er -Ġal so -f er -Ġthe se -Ġa d -е Ð -th er -ac e -ic k -a ke -re at -i re -u e -Ġa g -Ġ U -u ch -ion s -r y -0 0 -n a -Ġd id -Ġqu e -Ġha d -Ġe very -ĠH e -Ġl a -Ġw ay -Ġs p -b le -ĠTh is -as s -Ġthe ir -it e -Ġne ed -Ġp art -Ġw ere -Ġb ack -i p -ow n -om et -b e -as e -Ġma ke -ir st -i a -en ce -an g -an k -Ġg ot -Ġp re -Ġcon t -Ġo ther -p t -ĠTh at -o g -Ġgo od -Ġint o -al k -Ġbe en -Ġa m -Ġo ver -u ally -Ġ â -ì Ŀ -Ġu nd -h e -w ay -Ġg r -Ñ Į -Ġd if -Ġp er -Ñ ı -ĠI n -Ġt w -on d -ar s -in t -or m -Ġl ot -Ġwh ere -Ġ à -Ġ V -Ġs omet -Ð » -en s -Ġg u -Ġa c -u g -Ñ ĭ -Ä ± -Ġf irst -re e -Ġh is -itt le -Ġim p -Ġm o -a v -Ġl ittle -ĠWh at -Ġm uch -Ġ z -Ġ ê -ab le -ĠÐ ¿ -Ġp o -Ġcom p -n e -Ġd is -Ġl et -an ce -Ġh er -Ġthing s -Ġst art -ul t -Ġa pp -Ġre s -Ġf o -Ġc ould -Ġin ter -Ġth ose -Ġd es -Ġwe ll -Ġtw o -Ġk ind -x t -res s -el y -à ¤ -Ġb r -Ġth r -ĠÐ ² -Ġ i -is h -Ġdif fer -Ġ ro -ĠS t -Ġsomet hing -Ġt ake -Ġb o -y s -Ġsh e -Ġt alk -l o -Ñ ĩ -Ġe ven -Ð º -ã Ģ -ĠÐ ½ -Ġb u -ĠI f -Ġd own -ĠC h -ad e -ation s -Ġ use -or d -Ġof f -Ġact ually -Ġs pe -d u -at ed -at er -os s -n ing -à ¼ -Ġdo es -Ġ Ñģ -Ġne w -Ġb et -ve l -c ess -p le -Ġha pp -t ing -on na -Ġ es -Ġd ay -Ġon ly -ig n -k ay -s el -ent s -ou nt -i ld -i le -Ġs c -Ġh im -Ġag ain -v ing -Ġg onna -Ġcom m -Ġh el -ot her -Ġ ke -ic al -Ġ 3 -Ġe l -Ġthr ough -Ġcom e -ar k -d ay -i er -à ³ -Ġth an -ĠThe y -Ġm ay -Ġs er -í ķ -Ġc all -Ġdiffer ent -Ġsh ould -ĠTh ere -ar y -ĠN ow -ã Ĥ -th ing -w e -or y -f ter -Ġp ut -or s -i al -ë ĭ -Ġund er -Ġin c -ĠY e -u b -f orm -Ġv ide -à ¸ -ver s -Ġfe el -à ¡ -od y -f t -f ore -Ġe m -g et -Ġsa id -it ion -Ġre c -i ous -at ch -Ġtr y -Ġhel p -Ġsh ow -Ð ´ -Ġb it -u ll -Ð ² -ÑĤ о -g r -Ġpl ay -if e -a il -ĠYe ah -Ġqu est -Ġman y -Ġp ers -Ġg reat -Ã Ń -Ġ est -n g -Ġâ Ļ -t y -l a -ĠO h -Ġ × -à ® -ĠB e -ad y -Ġm ost -ct ion -ĠN o -Ġdo ing -Ġbe ing -Ġto o -c es -Ġb l -. " -Ġre m -is s -on s -> > -r u -w n -on t -i b -e ll -Ġs m -ot h -u al -Ġ >> -Ġp h -l es -o c -f ul -Ġse c -is e -Ġad d -ig h -er t -Ġs ame -â Ģ -Ġme an -Ġf ind -e k -Ġen d -- - -Ð ¼ -Ġst ill -a z -Ġ ' -Ġm in -Ġye ars -ur n -Ġar ound -sel f -Ġw r -b s -oug ht -ĠâĻ ª -Ġf l -an ge -Ġa fter -Ġpo int -m er -v ed -Ġl ong -o y -ä ¸ -Ġc r -way s -Ġs y -Ġt ra -Ġ2 0 -a ve -Ġch e -Ġ ent -Ġbe fore -p h -Ġat t -i an -i ly -Ġpers on -Ġb ig -Ġs ch -Ġre al -Ġne xt -Ġlo ve -Ġvide o -ĠL et -Ġf in -Ġma k -i ble -Ġto day -er m -ĠA l -ow er -an n -i x -Ġp ar -Ġst ud -à ¶ -Ġimp ort -t e -Ġg ive -v es -Ġd ie -Ġde c -Ġte ll -ĠÐ º -Ñģ ÑĤ -Ġwh y -ic ally -ic t -re d -Ġb as -Ġsu re -Ġbe l -at ing -Ġt ak -Ġs et -Ġl ife -Ġdid n -Ø § -o b -u nd -at h -Ġo p -ĠÐ ¾ -a it -Ġwor ld -Ġsu pp -i o -Ġc our -ĠÐ ¸ -w ard -е н -Ġal ways -u p -Ġha nd -ĠH ow -ci al -Ġcon s -Ġ Ñ -Ġin d -Ġ 4 -ĠA s -Ġf un -j ect -Ġimport ant -Ġs ur -e w -at es -Ġ 5 -Ġd i -Ġm ade -Ġin s -Ġas k -Ġ et -Ġn um -Ġc ar -ĠO kay -Ġs im -i k -Ġl ast -ĠG o -Ġm us -Ġre l -ul ar -´ ì -ĠWe ll -pe ct -ĠTh ank -Ġth ree -à £ -ã ĥ -Ġin v -Ġg en -l ic -Ġhapp en -ë Ĭ -i en -e ver -оР² -Ġst r -ĠA ll -Ġin st -Ġâ Ģ -Ġde f -Ġs l -Ġm ight -un g -Ġye ar -Ġo wn -Ġke ep -b ody -d er -Ġ ÑĤ -ĠÐ ´ -Ġan other -Ġm od -Ġe v -Ġgu ys -Ġab le -ã o -qu e -id ent -ĠY es -Ġit s -Ġpl ace -Ġpro du -ar n -ĠÐ ¼ -Ġre p -Ġex per -Ġf am -it ies -if ic -Ġh igh -i ed -o ol -ie w -е ÑĤ -re n -Ġdon e -Ġ ... -ëĬ Ķ -st em -ĠS e -Ġbet ter -c ome -Ġd el -Ġt y -Ġu m -Ġh o -ĠA n -Ġm on -ing s -Ġs k -Ġo b -c om -ble m -op e -st and -' d -ment s -Ġe le -ĠI s -Ġd a -Ġre g -le ase -i ke -al s -iz e -ê ° -Ġc are -Ġne ver -ìĿ ´ -es e -Ġm et -ol og -ĠWh en -u ck -е ÑĢ -Ġ é -Ġd at -à § -Ġex am -il ity -Ġd et -c ri -Ġus ed -ĠD o -Ġtr ans -e g -t en -Ñ İ -c us -Ġsec ond -Ġb est -Ġh ard -Ġ ide -Ġpro blem -ê ³ -ĠU n -Ñ ħ -Ġ Î -Ġw atch -ĠS h -at ter -Ġpre t -Ġd er -Ġcour se -Å Ł -at ive -ic s -Ġquest ion -ut e -ì Ĺ -ĠF or -at her -Ġc ol -i end -Ġ í -Ġ Z -Ġdoes n -ar ch -Ġinter est -Ġp ol -Ġc or -i ence -Ġp res -Ġe ach -Ġsy stem -Ġf act -i el -ab ly -Ġ er -Ġr un -Ġì Ŀ -Ġto p -n er -Ġth ought -Ġe as -i ent -Ġc re -Ñ Ī -Ġcomm un -y e -re ady -ll ow -Ġevery thing -om m -Ġm ed -ļ Ķ -Ġc ount -it s -Ġcom pl -h ip -Ù Ħ -o ok -Ġto get -Ġtoget her -am p -Ġg ame -Ġal ready -аР» -Ġcall ed -al e -Å Ĥ -ĠM y -Ġunder stand -Ġd r -Ġm om -it ed -оР» -Ġus ing -z y -Ġnum ber -ãĢ ģ -c ed -Ġc le -н о -ëĭ ¤ -in ce -Ġlook ing -Ġpret ty -Ġpro b -ĠS he -Ġ ve -Ġget ting -Ġwe ek -Ġe ff -u ff -a ir -u es -er n -Ġ Q -ou p -ent ion -Ġs ide -оР¼ -Ġfor m -Ġb us -Ġas s -Ġ ed -as on -we en -âĢ ¦ -Ġt urn -Ġc ur -Ġco ll -Ġd ire -ĠG od -Ġ1 0 -Ġe qu -ĠÐ ± -Ġop en -Ġsu ch -ir d -аРº -Ġe ar -Ä Ļ -g an -Ġpart ic -Ġfr iend -Ġex p -Ġex t -Ġh ome -Ġw ater -ĠO n -ÑĤ ÑĮ -or k -Ġп ÑĢ -Ġmo ve -n ess -en se -h o -Ġch ar -c o -in s -Ġb oth -Ġ1 9 -Ġg ra -Ġbet ween -á » -Ġì ķ -as h -ĠR e -a i -al th -u res -em ber -Ġa v -Ġ ver -à ª -one y -Ġth ank -Ġmay be -u c -im e -ê³ ł -Ġa way -Ġn ame -ou se -Ġac c -Ġmus ic -Ġch ange -Ġp ass -g er -Ġbu ild -Ġv al -in ess -an y -Ġfe w -´ ë -t a -Ġl ist -à ¥ -Ġo ld -Ġì ŀ -Ġs ort -Ġme m -Ġc a -ce pt -Ġgen er -Ġye ah -Ġwh ile -Ġany thing -r ic -gr am -Ġe in -c y -ur ing -ĠD e -Ġp ower -Ġcom ing -Ġwor d -Ġ- - -Ġbel ie -Ġf ound -t o -Ð ¿ -Ġme ans -Ġin form -Ġ Ø -Ġ Ñĩ -Ġsm all -00 0 -Ġc ame -Ġ íķ -w h -Ġwork ing -Ġexam ple -Ġp os -Ġde p -ê ² -ä º -ot e -Ġde m -ì § -t s -Ġv ar -a ut -Ġt ri -ch n -Ġhe ad -Ġwho le -× Ļ -z e -Ġtry ing -Ġt em -Ġc ou -et s -Ġ 6 -Ġf il -vel op -Ġc ase -à ¯ -Ġprob ably -Ġo kay -Ġpl an -Ġs it -Ġsch ool -ĠTh en -¸ ë -m e -Ġpro cess -Ġf ar -Ġre ad -Ġp oss -Ġb re -Ġso l -ic ht -Ġsupp ort -ĠT o -ert ain -Ġstart ed -Ġc ap -Ġle ft -Ġdat a -Ġtim es -еР» -Ġwant ed -а н -Ġtalk ing -Ġis t -Ġha ving -um p -Ġcont in -Ġsu b -ĠÐ · -p r -ëĭ Ī -in a -Å ¼ -Ġc reat -od e -× ķ -æ ĺ -! ! -Ġt erm -is m -оР´ -ĠBe cause -Ġw ent -id er -Ġpro v -Ġch ild -Ġd en -Ġl ight -b r -³ о -o h -Ġbo ok -Ġ Ù -ut ion -ĠJ ust -en e -Ġf our -Ġv is -ê° Ģ -Ġh ope -Ġmak ing -ĠL e -ì ķ -Ġo pp -a u -Ġm oney -Ġpro gram -à ¨ -Ġst and -I N -Ġs ign -Ġle arn -à ł -ĠD on -Ġte am -Ġн а -l ud -Ġre st -ic es -æ ľ -Ġ ÑĢ -Ġa ut -Ġle ad -ation al -d e -g y -Ġn ice -Ġd as -Ġd ist -Ġh um -ĠO ne -æ Ī -Ġcom es -Ġj o -Ġc ent -Ġex pl -Ġm ark -re en -l ed -g in -ì ļĶ -Ġle vel -Ġcon f -us h -Ġde velop -Ġt est -en g -v ious -at ure -еР¼ -re t -Ġj e -Ġst uff -Ġcl ass -ow s -Ġê · -Ġs i -Ġl es -ro p -ç ļ -Ġp or -Ġw ar -ìĹ IJ -Ġevery one -Ġg e -Ġche ck -ot t -Ġs ing -Ġar t -Ġfo llow -Ġ20 1 -ĠF r -a is -ì ĸ -Î ± -å ° -Ġà ł -im es -Ġre t -Ġch ang -Ġp ub -Ġin f -Ġte chn -ad a -iv es -Ġbe h -æĺ ¯ -Ġlook s -ãĢ Ĥ -Ð · -ĠWh y -çļ Ħ -Ġen ough -Ġb ra -it ch -ä » -Ġad v -Ð ± -Ġwith out -w er -mer ic -d en -Ġcompl et -Ġide a -ter s -o ck -Ġdef in -Ġe ver -Ġg l -Ġon ce -Ġbr ing -Ġsay ing -Ġan s -Ġhe ar -n ect -Ġl ess -g o -re am -ad o -ì ŀ -Ġm ind -ent e -Ġf ull -Ġb ad -Ġw om -Ġsome one -Ġd u -Ġw on -Ġcont ro -ort un -Ġhe alth -Ġch o -ĠA r -Ġcon c -Ġinform ation -Ġst op -at t -at ely -ä ½ -Ġgr oup -Ġ Ñĥ -Ġqu ite -Ġres p -E R -ug ht -ê ¸ -m an -iz ed -ĠB r -Ġrem ember -Ġfam ily -Ġbus iness -a w -Ġspe c -Ġa u -ĠO r -Ä ħ -Ġse en -Ġl ar -Ġ 7 -g g -b ers -Ġd ra -Ġmon th -Ġsay s -Ġis s -Ġli ve -Ġl ine -Ġmom ent -Ġex c -el s -Ġs ound -Ġco ol -Ġlo c -Ġc ertain -Ġd ri -о ÑĤ -am es -Ġm ust -n y -и ÑĤ -Ġk id -Ġinc lud -ìĿ Ħ -at or -Ä Ł -h a -are d -Ġse em -Ð ¹ -ì Ħ -Ġel se -Ġì ł -ir l -Ġ 8 -Ġv o -Ġquest ions -in es -e e -æĪ ij -ü r -ĠA meric -Ġst ory -Ġser v -ver n -ag es -l and -ĠâĢ ĵ -er a -ĠC an -Ġp op -et her -Ġn a -Ġor der -Ġmak es -Ġs ince -c on -ct or -Ġth ough -Ġprodu ct -л и -Ġle g -Ġme et -al f -Ñģ Ñı -un ch -it er -o ve -×ķ × -i et -аР¼ -it al -Ġsu per -l ing -Ġp ay -Ġpar a -Ġj ob -ĠH ere -Ġs w -k s -pt ion -m a -Ġbelie ve -¬ ë -Ġw ait -оР¹ -Ġun t -Ġqu ick -h r -ĠÑ į -ĠP ro -Ġm en -à ¹ -Ġday s -Ġgo es -Ġspe ak -ĠA t -em ent -Ġm iss -Ġa w -Ġdes ign -Ġpro ject -о ÑĢ -i j -ant s -at s -ĠCh r -Ġ 9 -Ġc ut -Ġre qu -Ġн е -ĠN ot -as ter -Ġm ill -Ġpartic ular -Ġp ie -Ġstud ents -Ġf ive -ou n -ĠN e -Ġg i -Ġp as -Ġf ree -ĠS p -l ich -Ġpro f -Ġen g -Ġpr ot -ĠL ike -os ed -Ġcon nect -a pp -Ġë § -it ing -Ġb lo -Ġl os -ist s -Ġexper ience -re nt -Ġst ay -Ġfo od -t on -ru ct -Ġh ist -v iew -in ing -m ost -i vers -b o -ãģ Ħ -ĠT r -g en -Ġp lease -Ġcommun ity -Ġc e -A N -n o -Ġb ody -Ġh our -Ġ vers -á º -c er -Ġê ° -Ġre ason -ĠR ight -Ġl ater -Ï Ħ -Ġh ouse -Ġ X -оР½ -Ġst ate -f ic -å ¤ -Å Ľ -iel d -Ġp ri -Ġp ast -Ġw alk -olog y -er ing -an na -Ġt er -Ġho ld -Ġor gan -b en -Î ¿ -ó n -Ġeff ect -Ġyour self -Ġpl us -a j -and o -ur al -Ġro om -le ct -ê² Į -? " -s ide -Ġbe come -Ñ Ĩ -Ġ  -o od -Ġcon st -Ġn ight -ut es -Ð ¶ -Ġbre ak -Ġp ain -Ġst ep -ire d -Ġnot hing -Ġunt il -Ñ ĸ -аР² -Ù Ĭ -Ġd uring -ì§ Ģ -l ess -o ll -н Ñĭ -Î ¹ -f ect -i ver -ı Ħ -ith er -y ing -Ġbe gin -×Ļ × -iv id -Ġà § -Ġs al -Ġt a -Ġp ot -Ġ $ -Ġm ar -Ġcle ar -Ġf ace -Ġgr ow -Ġ * -Ġins ide -Ġfriend s -Ġle ave -en n -Ġeas y -Ġare a -al ity -ou d -Ġe at -Ù Ĩ -Ġp ur -or n -Ġsa w -Ġans wer -Ġfr ont -Ġbe aut -¼ ë -Ġm atter -Ġs on -ĠN ew -Ġres ult -id es -ch e -Ġf ut -p s -Ġfo cus -Ġinterest ing -å ¥ -Ġa p -" . -Ġcre ate -о Ñģ -Ġp ress -r oss -Ġp ick -l ine -Ġto ok -ĠM ay -r ow -Ġ ich -ĺ ë -Ġre f -Ġm or -r act -are nt -A R -Ġex act -Ġsp ace -w ork -н и -Ġb ir -Ġde v -Ð ³ -Ġto ld -Ġpub lic -ci ally -Ġv iew -ĠHe y -m ed -ll o -c c -Ġf ac -Ġcou ple -Ġhe art -l er -Ġre ady -Ġal most -ar ing -Ġh alf -ĠM e -av or -i que -Ġchar ac -Ġpr act -O N -an e -Ġ il -н а -Ġv i -l ish -he ad -Ġle ast -Ġbas ically -as ed -r ight -Ġy et -Ġtak ing -Ġcount ry -Ġw in -Ġis n -Ġposs ible -Ġc am -Ġinc re -Ġp at -Ġw anna -Ġcons ider -Ġab s -Ġwith in -Ġhum an -Ġthink ing -Ġo h -¡ ľ -Ġqu i -as es -Ġ 0 -it ely -ä¸ į -Ġk ill -Ġm il -Ġinv est -is ter -Ġsu c -ion al -el f -Ġwh ether -Ġcontro l -Ġagain st -ot s -ëĭĪ ëĭ¤ -i or -Ġpres ent -Ġ ا -Ġwatch ing -u be -er v -Ġn icht -Ġgo vern -ĠTh ese -Ġ : -u it -ug h -Ġwork s -o o -Ġw ir -Ġa ir -ĠT e -аР· -is ion -wh ere -Ġto t -j oy -ì ĭ -Ġv ol -ĠÐ µ -Ġcl ose -ĠA d -Ñ ī -in ed -Ġun a -Ġê· ¸ë -° ë -or ry -Ġb ro -Ġfil m -if t -2 0 -Ġty pe -Ġhappen ed -ĠA m -Ġg irl -ĠA re -ward s -Ġp our -Ġcol or -el t -а Ñģ -Ġs ense -le x -ĠW ith -us s -ri b -Ġre se -Ġn orm -Ġfut ure -Ġde al -end ing -e y -Ġ x -er o -ĠC l -u k -Ġwhat ever -sel ves -Ġyou ng -ì Ĭ -ĠM ar -ĠChr ist -Ġgu ess -Ġper form -Ġen er -r on -Ġh it -Ġw ond -Ġdire ct -ĠE very -Ġof ten -Ġf a -Ġal ong -Ġcl ick -ĠL ook -Ġsit u -Ġhapp y -e ad -Ġag o -Ġen c -Ġmy self -Ġco ver -оР± -Ġm id -Ġc ost -Ġt en -ĠS ch -Ġex pect -Ġwas n -Ġstr ong -if ul -Ġopp ortun -in al -y le -Ġsh are -Ġtr ue -Ġapp ro -Ġch all -Ġmin utes -Ġch ann -Ġë Ĥ -Î µ -l i -Ġm ess -or ies -pe cially -Ġwr ong -Ġy es -Ġì Ĺ -ir on -Ġall ow -Ġsu bs -Ġf ore -Ġf ight -Ġso cial -Ġc ra -an a -Ġa ff -Ġ ess -Ġway s -Ġsh ort -Ġf all -Ġla w -ĠWh o -Ġen joy -Ġc al -Ġac cess -f e -Ġn on -Ġac ross -er y -vious ly -ĠE x -id ed -Ġl ink -ĠP r -Ġterm s -ac es -Ġl and -az ing -Ġ1 5 -Ġm ult -Ġspe cial -å Ģ -iv ing -ìĿ Ģ -Ġty p -Ġst e -Ġ Ä -Ġfor ward -å ı -Ġf re -å¥ ½ -Ġrese arch -௠į -а ÑĤ -Ġma in -Ġrec ord -Ġh u -Ġdefin itely -Ġe ither -Ġlist en -Ġke y -Ġmark et -ĠÑĩ ÑĤо -iz ation -Ġvide os -Ġgu y -Ġf ig -Ġst ra -ĠP l -ull y -am os -Ġm ention -Ġs ong -Ġinter n -r al -ur s -Ġh on -Ġval ue -Ġb ar -c le -оР¶ -Ä ĩ -ľ ë -Ġz u -и м -ä½ ł -Ġsing le -Ġa uch -cus s -Ġget s -Ġsomet imes -å ¾ -am b -m m -c ing -Ġper fect -ĠB l -out h -ì ł -Ġs ci -p ar -Ġre d -Ġp ost -Ġm ot -Ġele ct -ĠE u -it ive -ĠS ome -Ġdes cri -Ġcur rent -é s -Ġt re -ĠE n -Ġm it -E N -Ī ë -i um -Ġhe ard -Ġsim ple -l ar -Ġevery body -il ar -Ġneed s -Ġdif fic -ĠGo od -um ent -c ent -Ġo per -а ÑĤÑĮ -et y -Ġbl ack -Ġgi ven -on es -Ġwe l -é Ģ -Ġìķ Ħ -Ġ3 0 -A T -Ġst at -ou ch -ĠM r -а ÑĢ -Ġsh o -Ġcon d -× Ķ -m y -Ġchild ren -Ġe u -еР´ -ìķ Ħ -ter n -Ġu h -Ġh ar -Ġpr om -Ġp ull -re w -Ġcomp any -Ġbeaut iful -ust om -íķ ĺ -к и -Ġst re -Ġam azing -ri es -Ġsuc cess -Ġm ach -n ot -Ġdis cuss -Ġn at -¦ ¬ -Ġun e -Ġdiffic ult -Ġr is -Î ½ -Ġc amp -Ġbu y -ä¸ Ģ -Ġma g -p o -ĠY our -Ġbeh ind -ic a -ı n -ĠO K -Ġl ang -Ġwom en -Ġen v -Ġre ce -Ġchann el -i ally -u le -Ġ1 2 -th ers -Ġb ott -Ġrep ort -ent ly -f ully -T he -Ġs ent -Ġev ent -Ġener gy -l t -Ġword s -ar r -d le -Ġa head -ard s -Ø ± -äº Ĩ -Ġto ol -con om -е Ñģ -Ġexact ly -Ġf avor -Ġl ow -Ġpro per -Ġìŀ Ī -Ġ ! -Ġrel ations -Ġm as -Ġkid s -Ġent ire -ud e -Ù ħ -ĠWh ere -Ġon es -Ġc ity -ol ut -Ġs ix -ab ility -ö r -il i -ĠE s -Ġhapp ens -ain s -Ġmod el -Ġp ict -Ġes pecially -Ġ1 00 -k t -Ġso on -b y -ro du -Ġan n -Ġsubs cri -ĠQ u -Ġav ail -im ent -Ġv oc -k a -Ġ2 00 -ap er -ĠI nd -Ġì § -h or -į ° -j or -и л -Ġs qu -A U -ar ning -ĠÐ ³ -I S -ĠÐ » -еР¹ -y es -å ħ -ĠÐ Ĵ -Ġor ig -оР³Ð¾ -Ġask ed -il t -оР³ -Ġcontin ue -Ġì ĺ -r am -Ġo thers -E S -oh n -Ġl ay -Ġbas ed -Ġp u -Ġapp e -Ġl im -Ġpro p -Ģ ë -m in -Ġh ot -ĠL a -Ġf ast -Ġprot ect -Ġam ount -Ġa qu -Ġf und -Ġc ustom -Ġc ult -Ġhand s -Ġha ven -Ġa ud -Ġout side -ĠA fter -ap s -Ġan im -pl oy -Ġh at -ĠF irst -Ġt reat -Ġe p -Ġm ater -Ġbuild ing -Ġë ° -å IJ -ìĦ ľ -z a -ught er -ĠP e -ne y -et er -at ic -Ġed uc -ê¸ ° -Ġmo v -ĵ ¤ -am a -r ation -Ġs n -Ù Ī -Ġs um -Ġph ot -ĠÐ Ŀ -Ġ . -æľ ī -Ġfin ish -itt ing -å ® -Ġlar ge -Ġì ĸ -Ġwh ite -ar a -Ġma is -ĠH i -Ġd am -Ġا ÙĦ -Ġbo x -ĠHe llo -Ġs le -Ġo pt -ri ed -¥ ¼ -Ġact iv -Ġn ão -ĠC om -Ġplay ing -T h -Ġavail able -Ġp ort -å Ī -ĠA h -Ġl as -Ġear ly -Ġwond er -± ° -Ġ1 8 -c ul -Ġfun ction -Ġmor ning -ll e -i ents -u x -Ġc ir -it ions -Ġde ep -Ġpol it -y or -m p -ak ing -Į ë -ĠM an -Ġmill ion -Ġ / -Ġind ivid -Ġp an -Ġgovern ment -Ġwr ite -ĠT od -am ent -Ġ Ï -Ġw ind -ĠE ng -ch en -W h -ì ľ -Ġ ident -ãģ § -v ent -ur ch -Ġh y -Ġy a -Ġtr ad -Ġrelations hip -à º -Ġd ou -O R -Ġs we -Ġne g -in ation -Ġte xt -i pp -Ġf ine -á s -ĠD r -ĠC ome -Ġmonth s -, " -ен и -Ġhour s -Ġp od -ir t -Ġinv ol -Ġcoll ect -Ġau f -Ġp a -Ġhist ory -m b -if y -Ġ ? -Ġbel ow -as ure -ab y -Ġlang u -Ġan t -Ġcom b -at o -Ġex ist -Ġë ĭ -Ġtak es -Ġcharac ter -a ff -Ġf ield -Ġe conom -ie f -Ġpie ce -å ľ -Ġre ach -Ġê ² -on y -Ġmater ial -Ġd ig -Ġph ys -Ġimp ro -Ġsim ilar -I C -Ġn et -y n -Ġpos ition -à Ł -Ġb ene -re ad -Ġle arning -um e -Ġcle an -ÑĤо ÑĢ -Ġco ok -Ġseem s -Ġo l -ĠU S -ĠJ es -Ġ à® -ent ial -ivers ity -ac y -Ġ Ñı -olut ely -re ct -ĠP lease -Ġrep res -Ġt ouch -m en -ĠÐ ° -i ón -ĠThank s -Ġan g -Ġma jor -Ġit self -ill s -" , -i ans -Ġsc reen -Ġh or -Ġknow n -Ġenv iron -Ġfin al -Ġfig ure -ĠT w -Ġe yes -Ġim ag -Ġsee ing -Ġha ir -re m -Ġapp lic -end s -p ut -Ġnew s -Ġcomplet ely -ugh s -Ġkn ew -if ied -ĠJ e -ĠD id -Ġsitu ation -Ġf lo -m s -Ġph one -Ġb all -d o -Ġp arent -Ġs orry -ur y -и н -ip s -аР´ -Ġinst ead -Ġhu ge -Ġt u -Ġ ãģ -ĠG r -Ġdet ail -ĠÐ Ł -Ġindivid ual -Ġf ire -Ġcl os -Ġw er -un e -Ġrun ning -Ġcon vers -Ġrec omm -Ġcom o -Ġsome body -ĠJ ohn -ĠìĿ ´ -ĠO ur -pl es -ĠP h -Ġan al -Ġ5 0 -Ġof fer -Ġ < -ition al -g est -Ġv ous -l et -ic y -Ġfeel ing -L E -r os -Ġth ird -оРº -Ġser ies -ĠAn y -is ed -o ld -Ġdra w -Ġserv ice -Ġcan not -b al -ãģ Ĩ -Ġli ving -ı m -Ġdiffer ence -Ġopportun ity -Ġne ar -or th -k en -Ġloc al -Ø ª -ĠC on -Ġob ject -Ġd ass -ãģ Ļ -IJ × -Ġquick ly -ra ph -Ġiss ues -éĢ Ļ -ĠAmeric an -Ġpre p -en ces -Ġprof ess -ll ing -o f -Ġfo ot -b re -Ġus ually -Ġgener al -d a -an ces -Ġd est -Ġo cc -Ġmem bers -Ġd ans -Ġequ al -z t -Ġbe com -Ġmo ving -Ġspec ific -ÃŃ a -Ġf ur -Ġne cess -Ġcomm on -Ġatt ack -ĠÑį ÑĤо -ĠTod ay -Ġun s -ĠG u -i od -Ġacc ount -Ġgra nd -Ġs elf -ĠE l -Ġt ast -Ġcont ent -Ġc u -Ħ ë -ĠMay be -ĠJes us -ore s -p ort -© ´ -Ġg ives -Ġnorm al -ÑĢ Ñĥ -Ġimp act -ä r -Ġd ies -Ġl ab -s h -i os -ĠP res -ĠU nd -ĠO f -Ġfin ally -Ġdo ll -Ġvoc ê -p ly -ĠA g -Ġtak en -Ġgr ound -f ort -Ġg ave -ĠIn st -Ġl ost -Ġwork ed -Ġl iter -Ġiss ue -Ġind ust -Ġret urn -Ġhappen ing -Ġwant s -и в -Ġproblem s -ĠC ar -Ŀ ¼ -ĠAl so -Ġs ize -Ġob viously -ĠS u -ĠS c -Ġrecomm end -our ces -ast ic -.. .. -Ġm i -l ier -ĠE ven -ci a -Ġh ur -v a -Ġm ass -Ġwould n -un t -ck s -Ġf elt -os p -l ight -ол ÑĮ -n ie -Ġbott om -Ġб Ñĭ -ore d -is on -Ġgr ad -Ġum a -Ġv a -Ġì Ĥ -ress ion -ul ation -I D -id ence -Ġb ur -Ġg one -l u -ìĸ ´ì -Ġre du -Ġj a -ìĿ ĺ -it a -Ġso ft -Ġç a -ic o -er al -à ± -a f -Ġpoint s -g u -Ġd é -ap t -a x -ĠAl right -Ġcam era -Ġa ch -Ġп о -Ġse ver -5 0 -Ġs ie -Ï ģ -Ġm al -Ġcomp ut -Ġmid dle -Ġcould n -m ing -Ġì ĭ -ĠH is -Ġg ames -Ġint rodu -Ġc ell -p or -Ġsle ep -Ġë ³ -id ing -Ġ ou -Ġde g -Ġdr ink -Ġenviron ment -ĠUn ited -Ġtalk ed -Ġcho ose -Ġj our -e ge -ĠM in -Ġint e -Ġr ather -Ġoff ic -к а -ac hing -Ġmention ed -Ġf ill -Ġtr ack -Ġn ie -Ġ ut -Ġв Ñĭ -ib ility -Ġv ac -Ġr ad -Ġp ack -Ġs end -ĠD as -ĠA b -Ġeng ine -ãģ Ĺ -Ġcomp et -à ´ -Ġв Ñģ -Ġdo or -Ġlong er -å° į -Ġlangu age -Ġext ra -pl ay -Ġwe bs -um b -ro om -ç ľ -Ġbegin ning -Ġre fer -A M -n en -ig her -f ace -er c -Ġfor get -Ġcom ment -еРº -л Ñı -r or -ż e -ĠG e -Ġd ark -Ġany one -ant e -g es -ìĬ µ -Ñ ij -b ed -j e -ruct ure -Ġpr im -id a -è ¦ -ãģ ¾ -Ġm ix -Ġstart ing -ĠìĿ ´ë -Ġprov ide -act ion -Ġm other -Ġper iod -Ġst ick -ĠYou T -Ġtechn ology -ê ¹ -Ġb ed -Ġg iving -Ġexpl ain -z en -im ate -Ġrepres ent -lo ad -ĠHow ever -Ġli ves -ut h -ir it -og n -Ġli k -Ġresp ons -Ġpri v -Ġto m -ç ão -i am -Ġexc ited -Ġc ard -gr ound -Ġ× Ķ -Ġs ens -Ġte ach -id o -h od -Ġep is -Ġwel come -Ġw all -ä ¹ -Ġch ance -h en -ĠÐ ¡ -ĠÄ ij -Ġsim ply -ĠÑĤ ак -r ing -j a -b ook -Ġsever al -st e -Ġcreat ed -Ġо ÑĤ -Ġp ush -= = -Ġh igher -u f -our ce -o ke -Ġon line -Ġre le -Ġt on -ens ive -Ġfavor ite -Ñĥ д -Ġlook ed -Ġv on -âĢ Ķ -Ġf ür -Ġbut ton -Ġb ill -Ġchang es -! " -Ġsl ow -ab les -Ġde ath -and s -ate g -Ġthem selves -ãģ £ -Ġc op -ãģ ® -Ġperson al -ug hing -Ġ1 1 -g ar -ad es -Ġneed ed -Ġstud y -ag ed -ÑģÑĤ в -in o -Ġdis c -k i -Ġadd ress -× ¨ -itt en -es ome -ĠÐ ¶ -¤ ë -ur a -Ġm u -Ġcontin u -f or -Ġm atch -ãģ ¦ -Ġstra ight -IJ ë -n ers -Ġdo g -Ġde b -ĠC O -Ġo s -g ed -c ame -Ġcor rect -et te -ĠSe e -Ġinclud ing -ĠEu ro -est er -Ġj ump -ĠWh ich -Ġк ак -s on -y a -IN G -Ġe ine -os h -en cy -Ġmed ia -Ġsubscri be -é Ĥ -Ġpr in -Ġha b -ĠP er -ĠW as -Ġp age -it or -Ġto wards -Ġtri ed -en ge -art ment -Ġvar i -Ġp aper -Ġpict ure -Ġvers ion -Ġbr ought -w are -ĠSt ates -Ġs ich -led ge -Ġper cent -Ġgo d -e c -ĠC omm -Ġdec ided -Ġse lect -íķ ľ -) . -ur ity -Ġfur ther -Ġcom ments -le ment -Ġd ream -Ġcent er -m i -Ġc as -Ġwom an -Ġro ad -Ġf ail -Ġbe came -l us -il ities -ãģ ¯ -ĠC o -Ġman age -Ġrec ogn -Ġact ion -Ġbene f -Ġear lier -× ľ -Ġspe ed -Ġm ent -Ġso ci -Ġsho ot -u i -Ġà ¤ -Ġapp ly -v o -x im -Ġca use -Ġsur pr -Ġha ben -D I -Ġf ather -ĠNe xt -ĠYouT ube -Ġc ode -Ġro le -g ress -Ġg reen -et t -Ġbu ilt -Ġfl ow -Ġb ase -Ġtra ining -Ġr ound -ĠW ill -Ġp ath -ĠR o -Ġinterest ed -ìĸ ´ -Ġres pect -Ġchang ed -iss ion -Ġstud ent -og raph -Ġappro ach -Ġshow s -å° ± -Ġt ar -Ġcr it -Ġg lo -ìĬµ ëĭĪëĭ¤ -Ġde ad -ĠPres ident -Ġth ous -Ġb al -st er -e x -Ġabs olutely -Ġm ic -Ġpract ice -Ġqu ality -Ġl ower -og le -Ġse par -b all -med i -Ġre view -ĠA pp -Ġo k -âĢ ĭ -Ġexper ien -Ġconc ern -ent ially -m ore -ĠJ o -ap an -ĠI ch -ist ic -Ġf air -Ġwebs ite -i res -ĠB y -Ġtra vel -Ġris k -Ġm ir -Ġbo ard -Ġs en -Ġparent s -ĠW ow -Ġfe ed -Ġsa ve -Ġser ious -Ġin it -E L -und red -A S -Ġv an -or row -Ġwor th -Ġse arch -Ġ1 6 -Ġpart s -ÑģÑĤ ÑĮ -Ġcomp an -Ġmov ie -Ġmet hod -Ġ ill -Ġw ish -d y -Ġit em -Ġmin us -ang er -Ġvo ice -Ġsk in -Ġare as -Ġe ight -Ġo bs -Ġ , -аР¹ -Ġo il -Ġc y -Ġb aby -s y -Ġem ploy -ĠK e -Ġpl aces -Ġf ix -Ġest á -ãģ ¨ -iv ed -Ġlot s -Ġse ason -un k -al t -Ġt able -ĠÐ ¢ -à ¢ -Ġatt ention -ãģ ª -ĠH er -Ġa ge -Ġp ra -b ack -c il -Ġnet work -r it -Ġdo c -Ġare n -ig en -Ġë Ħ -Ø ¯ -end er -Ġtot al -Ġpr ice -Ġcra zy -ì ļ -i qu -th ough -Y ou -Ù ĩ -ãĤ ĵ -Ï ħ -Ġs at -Ġb i -ĠD ie -Ġsh a -Ġthank s -u h -Ġst age -аР¶ -ĠF l -Ġle av -Ġbo y -Ġa f -ö n -ĠG et -Ġac cept -Ġent er -Ġt ur -Ġsi ÄĻ -Ġhon est -ãĢ Į -Ġs am -Ġre pl -g ing -Ġdevelop ment -ĠA ct -or a -ãĢ į -ä ¾ -Ġknow s -Ġim age -ĠL ord -и ÑĤÑĮ -Ġweek s -Ġse x -Ķ ë -Ġh undred -Ġsound s -Ġlearn ed -Ġb ud -ĠÑģ ÑĤ -Ġinc red -â Ļ -Ġn os -Ġd rop -Ġb en -ĠÐ ĺ -Ġsa fe -at a -Ġf uck -so ci -Ġd an -Ġcr oss -1 0 -m o -ver t -Ġ1 7 -z ie -å ķ -Ġd om -ĠB o -Ġset ting -Ġinvol ved -ar ily -Ġs ind -Ġs us -Ġwor ry -et h -ê¹ Į -Ġs un -Ġh ier -Ġcertain ly -ou l -ort s -ĠE r -ĠU m -Ġca us -Ġnat ural -Ġà ¼ -Ġc ry -ĠSe c -Ġs om -æ ² -Ġeduc ation -а еÑĤ -Ġmult ip -Ġal one -Ġe ye -Ġr ate -ĠEuro pe -è ¿ -m on -Ġf it -iz ing -pp ed -Ġpress ure -th e -и Ñģ -it es -ĠA f -re ci -att le -Ġserv ices -ĠGo ogle -é ģ -Ġc ases -Ġdri ve -Ġchall eng -u z -ĠM o -ìľ ¼ë -v al -åĢ ĭ -Ġf ol -Ġì ¢ -ff ic -Ġr a -Ġs in -Ġbl ue -Ġaff ect -Ġm is -Ġsh ot -Ġо б -as ing -Ġsign ific -ĠC he -Ġê ³ -Ġpos itive -ì £ -Ġw ie -Ġ4 0 -ord ing -ĠFr om -ê µ -Ġbra nd -Ġtr ust -Ġp le -Ġcommun ic -Ġwe ight -Ġask ing -Ġta x -ĠJ apan -ãģ Ł -Ġíķ ĺ -op s -Ï Ĥ -Ġput ting -Ġro ll -ĠAmeric a -re g -ŀ × -at ures -ens ion -ĠS omet -Ġorig inal -p ing -Ġ ÅŁ -Ġproduct s -ãĥ ¼ -Ġcont act -ol ution -Ġgo al -Ġp ow -Ġperform ance -Ġblo od -at ors -ĠM ich -Ġtem per -ĠD an -Ġsu gg -ÑĤ и -Ġim m -Ġoff ice -Ġar ri -Ġcom fort -ĠÐ Ķ -Ġsugg est -Ġpl at -Ĥ ĺ -1 9 -Ġo m -Ġse ven -ĠC ent -ill e -Ġcon cept -Ġb ag -ü n -ive ly -Ġd iv -m os -æ ī -Ġfeel s -Ġ ir -ak es -le y -Ġpartic ip -ĠÐ ļ -f l -j ust -Ġs il -ĠP a -A L -Ġgot ta -Ġf an -Ġchall enge -Ġcompan ies -ĠPe ople -< / -оР· -Ġp en -is ing -Ġa us -em ic -am ente -Ġmeet ing -Ġvis it -Ġsupp osed -ĠOn ce -д а -or ld -3 0 -U S -Ġvi ol -Ġnot ice -ĠÐ IJ -h an -p ed -ì ĺ -h h -Ġtr ou -Ġmin ute -ĠP ar -r ay -Ġt it -Ġup d -Ġblo ck -Ġd ue -a ur -Ġfor ce -Ġcou n -ĠâĢ Ķ -Ġtyp es -ë § -Ġl ate -Ġimpro ve -Ġì Ī -Ġa ve -ul es -c l -am ed -Ġaw esome -ĠO k -Ġv ot -Ġmach ine -Ġfollow ing -Ġme asure -ac ión -u el -ch an -Ġab ility -Ġt out -Ġide as -Ġincre ase -Ġen s -ĠÑ ħ -Ġë ª -Ġj est -ĠÐ ľ -Ġtr uth -h y -Ġsp end -Ġsci ence -et e -Ġ1 4 -Ġepis ode -Ġal g -end ed -ãģ ĵ -ar i -ll a -Ġf ish -Ġthr ow -m it -å ¹ -Ġcir c -ĠC al -Ġt our -Ġdire ction -Ġno ch -еР² -é n -Ġcount ries -Ġindust ry -in y -ic le -Ġfe et -I t -Ġlead ers -et zt -Ġst aff -ç Ķ -Ġpur p -it o -? ! -ĠJ a -Ġst ore -et ic -ĠCh ina -Ġë IJ -ĠUn iversity -Ġ # -Ġdec ision -Ġach ie -Ġact ual -u ly -Ġse ction -Ġresult s -Ġst ar -Ġm ist -ib ly -Ġd ad -Ġnum bers -om b -è ª -ĠS pe -Ġm er -Ġ2 5 -Ġaut om -Ġco ld -Ø ¨ -Ħ ľ -ag er -ĠT V -ĠS ie -ĠH ave -Ġ że -ug g -ain ed -Ġup on -Ġlo g -Ġcomplet e -Ġbra in -ag ing -ĠM us -o ver -Ġeas ier -Ġinte gr -Ġm ás -Ġturn ed -Ġst ri -iv al -Ġhe av -ĠT H -Ġwr iting -ÑĢ а -åľ ¨ -å¤ § -Ġcl a -d ing -Ġtell ing -и д -ic ated -ä» ¥ -ac ht -ãģ Ĥ -h aps -ĠSt e -Ġres ources -Ġd ann -Ġpart y -Ġ ÏĦ -Ġsa f -is es -t re -o int -Ġknow ledge -Ġany more -Ġf ly -Ġma int -и к -å ij -Ġse ll -la ughs -ĠY ork -Ġb ien -Ġo d -Ġeas ily -Ġr ange -Ġo ption -Ø ¹ -Ġapp reci -oc r -Ġdet erm -Ñ Ħ -Ġmean ing -Ġs ite -Ġdis co -ver age -Ġl ose -Ġinst all -Ġem ot -ant ly -ä t -Ġt amb -ĠW ar -ĠH o -ĠG en -em y -еР· -ĠP ol -Ġmess age -Ġnot e -Į Ģ -Ġh et -Ġim medi -Ġav o -Ġbook s -Ġbecom es -res h -è s -as ons -Ġhim self -ut s -Ġj u -Ġaw are -Ġrequ ire -Ġsystem s -ĠH ar -Ġam ong -Ġh om -Ġb reat -Ġwe ird -Ġë ¶ -Î » -Ø © -if f -or ing -Ġplat form -ĠT ake -Ġhelp s -ut ions -Ġfor g -Ġl uck -ĠEng lish -Ġwe b -Ġneg ative -Ġt ut -Ġab ove -ng th -Ġê ±° -Ġst ories -Ġlo ad -Ġback ground -Ġsw itch -g a -Ġprin ci -Ġfin an -Ġvar ious -Ġl Ãł -Ġkind s -ain ing -Ġn ature -ĠÐ ŀ -c z -Ġpr ay -Ġg ar -ir m -Ġ & -Ġì ĥ -n s -ĠR ep -ĠF e -Ġre v -ra nd -Ġlike ly -Ġunderstand ing -ı r -ãģ ĭ -Ġf al -Ġ1 3 -ÑĨ и -Ġsu d -Ġbr other -Ġpl ant -Ġthrough out -w ise -p re -Ġcult ure -ĠÙ ħ -Ġwonder ful -Ġa h -pp er -Ġso ld -Ġstart s -Ġwr itten -Î ¯ -n i -Ġ×Ķ × -ĠD av -Ġu lt -Ġar m -Ġro ck -Ġwe ar -ë į° -an o -ra g -Ġsqu are -ан и -c ast -le br -Ġliter ally -Ġplay ed -Ġhe at -on se -r ict -Ġins p -id s -Ġpop ular -ë ıĦ -Ġc atch -Ġm ount -Ġj ud -Wh at -еР± -R A -a ud -к о -Ġsur face -Ġcon v -Ġpie ces -O h -æ Ģ -Ġst yle -pp ing -Ġread ing -Ġconvers ation -оР¿ -ä¾ Ĩ -ĠAg ain -Ġb ank -t ime -Ñĥ ÑĤ -er ve -ĠG reat -Ġcap t -аР± -ay s -ĠF in -ific ation -Ġä r -а Ñİ -Ġe gg -ĠW el -Ġtar get -ul a -ch es -an i -O O -ic ious -n ow -Ï ĥ -bo ard -Ġg ente -Ġd ro -ĠE t -Ġd in -Ġc os -Ġaut hor -Ø ³ -Ġo ch -Ġem ail -Ġsp irit -Ġs itting -m as -Ġstre ngth -Ġbig ger -ĠW ait -Ġm at -Ġpol ice -ress ed -Ġwait ing -is hing -Ġdoll ars -ho od -s s -Ġimag ine -in i -Ġm es -Ġdis e -id ge -ab or -Ġp et -Ġh op -ĠK ing -Ġcomput er -Ġgo ld -Ġn u -Ġf ing -) , -Ġsec urity -ru ction -Ġsol ution -e xt -Ġp atter -ick en -ure d -Ġstand ard -ìĭ ľ -Ġdou ble -Î · -Ġw ife -is a -Ġdirect ly -ac ed -Ġb unch -Ġ ¿ -ал ÑĮ -Ġreg ard -Ġswe et -Ġun ique -ĠâĻ « -Ġtra in -ĠG erm -Î ¬ -R E -Ġbeh av -Ġpre d -ì ĥ -s et -Ġdescri ption -é e -Ġc at -å ĵ -Ġcoll ege -ì Ľ -Ġapplic ation -ĠS en -as k -Ġc red -ub lic -Ġmultip le -Ġn i -Ġpres ident -Ġadd ed -Ġro b -Ġaqu i -Ġh osp -Ġtool s -Ġg un -Ġbas ic -Ġl ines -Ġst ructure -ĠR uss -Ġtot ally -Ġbig gest -Ġe en -Ġar g -Ġ× ľ -Ġp ark -ĠD es -Ġce lebr -Ġf ait -ен ÑĮ -Ġsu ff -Ġreg ular -¨ ë -Ġm ine -ĠK ore -Ġpre vious -Ġp i -Ġse g -Ġpol icy -Ġк о -ĠTr ump -Ġvac c -ó w -ĠS y -и Ñĩ -it ter -Ġpolit ical -r as -Ġal s -ел ÑĮ -Ġsha pe -an z -Ġon to -Ġar ch -Ġam b -ag ram -ĠS m -ct ions -Ġjo in -b or -å Ľ -Ġfr ame -ł ĩ -Ġcho ice -௠ģ -Ñĥ Ñİ -ĠC or -ĠS w -I T -Ġt end -ĠE ar -Ġto r -Ġev ents -Ġcla im -ĠD a -ĠM ark -Ġgroup s -Ġe ating -ĠW orld -Ġrec ently -Ġtast e -Ġsur v -à ¤ -Ġsk ills -Ġи з -itt ed -Ġsh op -ìĿ ´ì -Ġest ab -ĠëĤ ĺ -Ġsecond s -ĠTh ose -ĠE nt -Ġì Ħ -ers on -Ġto wn -Ġc and -Ġopt ions -Ġ ing -V ID -Ġenc our -Ġr é -âĻ ª -Ġent re -Ġmove ment -ĠB en -Ġbir th -Ġwh e -Ġh ang -ĠE m -ig e -ro ll -Ġun f -ì Ĥ -Ġr id -Ġsp read -Ġh ost -al d -ĠE d -Ġcons um -U N -Ġop in -it ar -ĠM ed -Ġsub ject -Ġp al -Ġcar ry -Ġag ree -ĠWh ile -Ġcare er -Ġsci ent -Ġsud den -Ġf ile -z i -Ġex cept -é º -Ġpot ential -ĠAn other -Ġcomp lex -ĠS im -end o -Ġr ais -Ġphys ical -Ġd ate -ak er -ĠC ol -Ġpower ful -Ġmem ber -ra p -Ġsp ot -Ġs ource -Ġf em -é m -Ġem p -j i -iet y -Ġinf lu -Ġd ry -Ġlo ck -Ġz ero -ĠU h -Ġr out -Ġpor que -Ġ2 4 -Ġt al -Ġfol ks -Ġla unch -Ġcomp on -ĠWel come -Ġk ann -ä n -ĠÑį ÑĤ -e es -ĠÙ Ī -Ġany way -Ġaud ience -äº º -Ġsl ight -on a -Ġu r -Ġrel ig -Ġext rem -ı z -ĠM a -Î ¼ -Ġà ¶ -Ġall ows -Ġf at -ĠF ace -Ġn ational -Ġinter view -ĠM c -é t -Ġc ute -el a -Ġsec ret -ĠW est -ĠD ep -Ġex erc -Ġhist or -Ġpri or -Ġ6 0 -av a -ac her -y ond -ĠH a -Ġest e -in ary -ĠN orth -on st -Ġsm art -am s -ал и -Ġd ar -er ed -Ġfun ny -ĠO b -ĠBl ack -Ġrel ated -ĠB u -Ġsome where -ĠR em -n es -ment e -ĠRe ally -Ġcreat ing -Ġfam il -Ġsoci ety -Ġg el -Ġtrans form -Ä ĥ -Ġinclud e -Ġh ol -l ike -k o -air s -Ġп од -Ġpers pect -Ġb es -Ġparticular ly -Ġshow ing -ĠP art -Ġqu al -lo ck -Ġreal ity -ho ld -ict ion -o on -Ġv ir -ãģ « -it ary -Ġdr ug -Ġfe ature -Ġre asons -Ġ× © -Ġwr ote -Ġf ant -Ġb and -Ù ĥ -en a -ke y -Ġear th -d om -Ġfe atures -Ġflo or -Ġspeak ing -Ġt ip -ĠA ust -Ġst ock -Ġch urch -Ġr ac -ìľ¼ë ¡ľ -ภĻ -ãĤ Į -k y -Ġresp onse -Û Į -ul ations -Ġsl ide -Ġgrad u -ci ous -Ġme ant -Ġ == -Ġ× IJ× -ã ħ -Ġkind a -Ġsc ene -Ġm uit -Ġê° Ģ -r ast -re st -Ġplay ers -w a -Ġbro ad -Ġtom orrow -oc ol -ĠÑģ в -ĠB ar -ı k -Ġse a -Ġrem ove -Ġrem ind -ом Ñĥ -ĠS ince -Ġave c -ce ll -и Ñħ -Ġdoc ument -Ġê·¸ë Ł -Ġne igh -be at -Ġp Ã¥ -Ġas pect -Ġd ed -lish ed -il s -Ġour selves -u ce -Ġhe y -ĠпÑĢ о -ent y -Ġas soci -ad os -um ber -Ġ ] -éĤ £ -no v -Ġì Ļ -Ñĥ Ñĩ -Ġcond ition -ëĬĶ ëį° -Ġval ues -Ġsc en -min ist -Ġc ast -Ġgrow ing -Ġus er -Ġresp ond -l im -é r -y m -çľ ĭ -os es -sy ch -ĠÑĢ аз -Ġappe ar -Ġpro gress -eng th -Ġj ak -ĠD is -Ġpat ients -ĠS er -Ġg as -è re -ìĸ´ì ļĶ -Ġre ci -ìĿ ¸ -Ġs ca -ep end -Ñģ к -аР¿ -Ġb atter -Ġve h -ð Ł -Ġac com -Ġbe at -Ġpain t -Ġcont rib -Ġs ad -Æ ° -al es -Ġt ree -b a -Ġb orn -ic ed -à® ķ -b and -Ġme chan -ĠD et -Ġcap ital -Ġdel iver -Ġfe ar -ŀ ĺ -ĠS outh -Ġb ought -Ġst ress -Ġv or -? ? -i h -ìķ ¼ -Ġer a -ìĿ´ ë -а Ñı -is ions -iv ity -Ġhelp ed -Ġass ist -Ġplay er -r an -Ġimmedi ately -Ġmo ved -c ie -ê ± -Ġann oun -å ¿ -ìŀ IJ -Ġprodu ction -Ġsum mer -Ġt un -Ġprogram s -G H -al ing -ir a -el ess -. ) -Ġa verage -è¦ ģ -Ġgl ass -om an -if ically -Ġëĭ ¤ -ĠC ong -ĠV er -Ġtr ick -Ġbe gan -Ġv ill -ê ±° -h ow -æ Ń -Ġt ill -Ġ9 0 -ber t -Ġê ¸ -Ġtemper ature -à ² -๠Ī -Ġgra ph -Ġê· ¸ -Ġr ot -Ġmo b -A Y -a el -Ġre pe -Ġdev ice -Ġ19 9 -Ġte le -Ġke pt -p a -æ ĸ -ver se -Ġst ream -е Ñĩ -ess ion -Ġstr ugg -z z -Ġdeg ree -Ġhelp ing -Ġsm ell -Ġper haps -p ro -Ġcont ext -Ġi k -Ġп еÑĢ -Ġcal cul -éº ¼ -b ing -Ġreal ize -l am -ĠCh ar -y t -ĠìĿ ´ì -Ġd anger -ĠI m -a a -Ġlo ved -Ġpurp ose -Ġfinish ed -Ġpe ace -Ġo t -Ġglo bal -Ï Ģ -Ġab er -ĸ Ī -Ġcharac ters -Ġn ur -Ġdam age -Ġem er -Ġpre c -ĠW ir -Ġinst it -ij × -Ġallow ed -b on -Ġto d -еР³Ð¾ -Ġj etzt -Ġmed ic -Ġsmall er -ce ed -Ġlevel s -Ġint ell -W e -Ġse m -Ġcurrent ly -Ġmod ern -Ġcont ract -Ġdetail s -ortun ately -O S -Ġst ates -Ġad just -ant age -e z -ĠV ery -Ġsc ale -Ġre lease -Ġf az -Ġ ic -it ude -A C -ĠP at -id en -Ń IJ -Ġpre fer -olog ical -ĠFace book -Ġê° Ļ -Ġ .. -ĠM ake -Ġко ÑĤоÑĢ -ĠDav id -ĠAf ric -Ġmod e -ĠC ity -Ġsh all -ĠÑ Ħ -im in -Ġз а -r om -u a -Ġbe yond -Ġdist rib -к Ñĥ -ĠDo es -Ġv ict -r ate -Ġv ai -Ġsuccess ful -Ġh ous -ah a -est s -ĠE st -Ġdisco ver -Ġthere fore -ch a -Ġc up -Ġpop ulation -ĠI l -s c -Ġsp ent -re l -Ġuse ful -Ġt ab -æ Ŀ -Ġ Å -Ġìł ľ -Ġcon se -Ġqu ant -ay a -Ġb on -åı ¯ -ĠCh in -Ġê² ĥ -ound s -е ÑĪ -ell e -Ġ ice -2 1 -Ġk ick -ä¸ ĭ -Ġstep s -Ġton ight -нÑĭ й -ren ch -. ' -Ġgra b -Ġimp lement -ĠìĪ ĺ -Ġmiss ion -Ġclear ly -Ġappreci ate -è Ģ -Ġf resh -ar m -ĠTw o -Ġex ec -Ġproject s -Ġcommun ities -ri ble -Ġreg ion -Ġfre qu -ro y -Ġhow ever -Ġpart ners -an c -Ġmin im -Ġl at -Ġfamil ies -Ġev idence -Ġp un -ra ft -Ġl oss -Ġma p -Ġany body -Ġchang ing -Ġr ules -Ġorgan ization -Ġess entially -ĠR ed -Ġele ment -æ Ĺ -Ġv irt -r at -Ġpr int -and er -are n -em os -ο Ïħ -Ġcond itions -ab e -Ġd ance -и ÑĢ -Ġd os -о Ñĩ -ĠQ ue -Ġwalk ing -Ġt ro -Ġ id -Ġadd itional -Ġfull y -Ġf ans -Ġadd ition -Ġlik ed -Ġü ber -Ġb ow -d i -Ġm aster -o ff -) : -m ber -Ġë ¬ -å ¯ -åĪ ° -la use -Ġo der -Ġsaf ety -Ġre act -à® ¿ -b t -Ġdis app -Ġgirl s -S t -ĠA ng -Ġfa ith -Ġturn s -Ġt ight -Ġm outh -am i -z er -Ġwe ap -Ġб Ñĥд -Ġhosp ital -ra id -Ġmic ro -ĠSt ate -ĠM ost -ag n -Ġdec ide -Ġpat ient -Ġcor ner -Ġdi ed -N o -ĠSt ud -re nd -em pt -Ġli e -Ġl if -ĠBe fore -t ó -ĠSu per -Ġbe ll -6 0 -Ġpriv ate -ĠPa ul -Ġg ib -Ġag re -´ì Ħľ -Ġs ig -Ġinvest ig -Ñı ÑĤ -en ing -Ġdist ance -Ġwar m -Ġdig ital -å¾ Ī -in er -Ġp and -ĠCO VID -Ð ³Ð¾ -g n -Ġr ace -Ġpr oud -Ġte aching -Ġ ÑĤо -ìŀ ¥ -ĠAll ah -I n -Ġw ood -Ġcol ors -Ġw ird -u j -id ad -Ġcustom ers -Ġconnect ed -Ġlay er -Ġachie ve -Ġperspect ive -ĠC oll -Ù Ĥ -Ġcl oud -!! ! -Ġend ed -łĩ ê²Į -Ġmanage ment -Ġr ich -Ġsub st -Ġrem o -Ġser ve -Ġres ist -Ġthought s -Ġgrow th -ili ar -Ġright s -Ġchar ge -Ġcons ist -Ġwer den -Ġem b -and om -Ġhur t -Ġk an -i as -л о -Ġsh it -Ġbe g -Ġrece ived -it ation -Ġme at -Ġis so -ff ee -Ġfam ous -Ġcomfort able -I L -ĠB ye -èª ª -åĢ ij -oth es -Ġmed ical -Ġenjoy ed -Ġhealth y -Ġw y -c ies -Ġeff ort -Ġdo ctor -Ġmil itary -L AU -Ġg ro -Ġb attle -Ġf ed -Ġcap ac -Ġaf raid -iv il -ĠвÑģ е -Ġl ength -ys is -Ġbe i -¤ í -Ġorgan iz -or g -in c -Ġinter act -ĠChin ese -Ġacc ording -Ġincred ible -Ġkill ed -Ġda ughter -ĠÏ Ģ -Ñĭ в -Ġschool s -Ġ « -ll er -Ġshould n -n al -Ġcr is -Ġch icken -Ġf aster -Ġextrem ely -Ġopp os -Ġn ous -Ġ + -ri a -Ġfinan cial -Ġexc iting -Ġjour ney -×Ļ× Ŀ -ł ë -Ġdis play -Ġmem ory -Ġheav y -н е -Ġpass ed -ÑĢ и -il es -Ġp sych -Ġspec ifically -Ġeng age -Ġl ed -or ge -ĠD em -ord er -Ġ8 0 -Ġcre am -ester day -Ġed ge -Ġп ол -Ġbu ll -Ġind ic -Ġk tó -Ġhope fully -um ents -ag en -н ого -Ġh ate -ch t -8 0 -Ġeff ic -Ġì§ Ģ -Ġintern et -Ġbud get -Ġproper ty -id ay -Ġì ļ -Ġм ож -ol a -Ġshow ed -ĠM on -Ġthous and -A P -Ġpo or -us ed -ĠJ ack -Ġs Ã¥ -ĥ ½ -Ġes c -Ġsoft ware -Ġqu ar -ĠØ ¨ -Ġnecess arily -om en -i y -Ġevent ually -ish ed -Ġbr ight -E D -Ġs pl -Ġdem and -Ġth reat -Ġs ir -Ġrele ased -ck et -ĠâĢ « -Ġrequ ired -Ġv ote -ì ¹ -à® ¤ -Ġdevelop ed -ĠìĤ ¬ -at ory -Ġd ir -ca pe -Ġslight ly -à ¬ -๠ī -re et -Ġdise ase -Ġcour t -Ġitem s -ĠEar th -ÑģÑĤ и -ж е -ì ² -Ġchalleng es -ĠBr it -Ġdesign ed -1 2 -Ġhear ing -Ġlisten ing -z o -ĠÑģ л -ãģ§ ãģĻ -Ġper o -Ġwe aring -pl ic -Ġch em -Ġbal ance -Ġb a -Ġrece ive -im a -Ġsignific ant -Ġм Ñĭ -an ch -ĠC r -ĠC oun -ê¸ Ī -Ġjo bs -Ġoffic ial -Ġper m -om s -Ġopportun ities -Ġover all -Ġh us -od es -Ġn ation -ĠR eg -Ġor d -Ġrest aur -Ġì Ĩ -Ġm el -v in -Ġw enn -Ġk ön -æ ĥ -Ġopin ion -ãĤ Ĥ -è ¬ -ĠSomet imes -ç Ĥ -Ñī е -as c -O U -Ġ20 20 -Ġdel icious -ig er -Ġìķ Ī -o le -Ġhand le -Ġc it -Ġíķ ľ -Ġf ör -o oth -Ġnecess ary -Ġind epend -æ Ħ -ist en -h am -Ġé t -ãĥ ³ -Ġmult i -Ï Į -? ) -Ġcamp us -Ġtop ic -Ġr ain -Ġpan el -ĠS am -Ġlar ger -aud ience -Ġpa id -Ġeconom ic -ol t -Ġstre et -ĠC ont -Ġdri ving -Ġìł Ģ -Ġh ay -Ġprofess ional -ĠIn tern -å ¸ -Ġin put -Ġc ateg -Ġc ro -Ġ ll -E T -Ñĭ й -* * -ĠZ e -B LE -Ġì ¤ -re es -ĠÐ ¯ -ed e -ier t -Ġfo ld -Ġd ur -ĠN ational -Ġìĸ ´ë -an ced -Ġfa ire -ut ed -Ġk ing -Ġw ild -o i -up beat -Ġpre vent -i us -Ġà ¨ -Ġw ide -Ġr ing -Ġtit le -Ġstand ing -Ġal though -Ġh i -Ġsa uce -Ġs ides -Ġanim als -il ing -at ives -ìĹIJ ìĦľ -ĠO ver -Ġdes p -Ġconsider ed -ar ies -i ers -Ġein en -Ġs ister -Ġë ķ -ĠS ure -ãĤ ĭ -ri end -a ign -Ġsh own -Ġs ac -Ġs ont -Ġcent ury -Ġt ien -ĠÎ º -ĠS T -åķ Ĭ -Ġold er -ie m -Ġtr uly -ĠS i -Ġwind ow -iqu es -ar io -æ² Ĵ -Ġloc ation -Î º -Ġì ľ -v i -ag ue -ĠS orry -Ġdis p -Ġhe ll -Ġà ī -Ġtr ade -Ġcrit ical -Ġê ± -Ġn amed -Ġprep ared -ĠH ouse -al u -Ġt ough -Ġtri p -Ġs and -c el -ü z -ĠP ut -Ġap art -is f -v is -Ġli br -a ven -Ġv ie -Ġeffect ive -ภ² -Ġmag n -Ġmuit o -Ġê µ -h al -Ġlim it -Ġn ine -Ġwill ing -ı ÅŁ -s p -еР³ -h i -Ġal t -ĠJ an -Ġorig in -ĠU s -Ġele ments -Ġus es -Ġhelp ful -Ġfl at -Ġfam iliar -ĠP ark -Ġc ore -Ġclos er -Ġact ive -Ġad minist -C E -нÑĭ е -ç Ħ -Ġrel ative -Ġment al -Ġr andom -Ġpart ner -Ġut il -ph one -Ġr ule -w w -Ġìł ķ -Ġsch on -Ġco ffee -H A -Ġconnect ion -Ġun it -la ughing -l og -Ġapp l -л а -us ic -ĠB ra -Ġany where -AU DI -Ġsepar ate -bo x -Ġd ivid -Ġtest ing -Ġs ick -Ġwer en -ä» ĸ -Ġ׾ × -Ġadv antage -Ġtrans fer -' . -Ġë ¹ -Ġfind ing -н ой -Ġì¢ ĭ -Ġfor t -Ġeconom y -Ġl ack -Ġleav ing -Ġd im -å İ -ĠR es -Ø Ń -Ġdiscuss ion -еР¿ -Ġg es -du ct -Ġch ain -Ġus ers -e ch -ÅĤ a -Ġdis h -Ġcare ful -Ġte acher -Ġopt im -Ġfl u -at ically -Ġref lect -Ġtreat ment -e ed -i ÄĻ -à ¹ -à® ¾ -Ġequ ip -Ġplan ning -Ġsol ve -ãģ Ŀ -ĠT om -Ġavo id -Ġp ou -Ġgreat er -l in -O L -ĠL u -ĠM ore -Ġatt ract -ê n -un a -Ġphot o -er ation -Ġplan et -Ġcop y -Ġvis ual -ir ing -Ġintern ational -Ġla ughing -Ġth ick -Ġhold ing -Ġbring ing -Ġlet ter -Ġb urn -Ġeffect s -it é -our s -O T -ê me -ĠSch ool -×ķ× ª -rop ri -l ig -α ι -Ġad ult -Ġsu gar -Ġr ide -Ġhigh light -Ġno body -Ġ2 1 -Ġch at -ĠпÑĢ и -Ġin nov -ung en -Ġatt ach -ed om -å Ĭ -y l -Ġleg al -Ġr ice -Ġcoll abor -k ing -d own -æ Ļ -ãĤ Ĭ -Ġi h -ĠA c -ous ly -Ġr ap -Ġsol id -Ġgener ally -Ġpatter n -al i -à¸ Ń -Ġtrans l -in ter -a ult -Ġë ¨ -Ġexp ress -Ġexam ples -Ġch ose -Ġtell s -ÃŃ s -ain t -ĠT ell -ĠMich ael -æ ¨ -ĠN umber -Ġt ap -Ġexper iment -Ġbenef it -Ġì ° -Ġse qu -Ġexp ensive -Ġgener ation -ĠM any -Ġadd ing -Ġk il -Ġcamp aign -ĠA nt -ra w -omm en -Ġs oul -j o -ĠAct ually -am m -ê² ł -Ġma xim -Ġsal t -Ġc ru -Ġcall ing -ãģ Į -Ġbas is -b an -Ġkeep ing -ĠM or -ed s -ì Ĩ -Ġto do -ам и -н Ñı -Ġli ved -ĠD u -ãĤ ī -å® ¶ -for ce -å¹ ´ -fer ence -al a -Ġocc ur -s k -Ġrec ent -Ġc ars -Ġtrad itional -ent le -² Ī -Ġhel d -Ġn ach -ĠCent er -er en -Ġb in -Ù ģ -Ġcomm e -Ġre ve -Ġìĺ ¤ -Ġexpect ed -ab il -Ġfocus ed -o v -Ġi P -or ial -i ro -Ġet c -am ing -ĠS on -Ġy esterday -Ġstr ate -ĠÑ Ĩ -Ġë ı -p es -Ġactiv ity -Ġadv ice -Ġopen ing -f in -Ġre la -é ĸ -Ġinst ance -ĠEvery one -b l -p en -Ġvis ion -ĠA lex -if orn -Ġt ick -H e -Ġstrate gy -Ġk om -P E -ĠG l -Ġelect ric -1 5 -Ġda ily -Ġhus band -Ġst ation -Ġanal ysis -yn am -Ġatt empt -Ġbill ion -v ant -Ġfor th -Ġm ath -al y -Ġbehav ior -ĠM as -k an -ĠD ay -Ġbl ess -Ġg ut -ĠH igh -o x -Ġd ress -Ġj ed -è ¯ -å ĸ -Ġexperien ces -ist a -Ġfight ing -å · -ĠÑģ к -Ġmost ly -a use -Ġpict ures -ен ÑĤ -Ġm ad -Ġmod els -ÑĪ е -ĠC ount -Å Ħ -ÅĤ o -ep t -O M -ĠA N -Ġtrou ble -4 0 -Ġb ird -ul ate -Ġm ur -Ġprodu ce -Ġmar ried -b it -Ġthe ory -í ĺ -Ġlead er -ĠL ast -A A -è µ -Ġim ages -Ġexp and -ĠP or -Ġpur ch -ĠS an -ĠChrist mas -ĠAust ral -Ġw id -ĠM iss -Ġknow ing -Ġz e -s hip -k u -Ñħ од -ĠInst agram -ĠInd ia -Ġest a -ĠCal iforn -Ġ7 0 -Ġdra g -Ġbr ush -Ġn ames -A nd -Ġy o -ill a -Ġsch ed -Ġdest roy -ye ar -Ġv amos -Ġ ÙĦ -ç a -Ġforg ot -и е -Ġra ise -re me -íķ ´ -ĠG ive -Ġcont ain -ra b -Ġg ift -ĠÑģ п -Ġrequ est -Ġsh ut -Ġdeg rees -Ġbenef its -Ñĭ е -Ġstud ies -Ġend s -Ġevery where -Ġher o -op h -er ry -Ġmaterial s -en ed -N A -å į -Ġmu y -Ġwor se -ä» Ģ -ĠM ad -Ġdec isions -ion e -Ġfore ign -la ughter -i ber -ени Ñı -ãħ ĭ -Ġreal ized -Ġ ign -Ġwe ak -ĠÎ ¼ -Ġsca red -Ġass um -A K -ï ¿ -ï¿ ½ -Ġcover ed -ĠS at -Ġо н -Ġindividual s -Ġcomp ared -1 1 -ĠAd d -ic les -Ġc ert -r ar -Ġbr ief -Ġactiv ities -Ġf ab -b ar -Ġa st -ĠO ther -Ġclass es -Ġo g -Ġmiss ing -ãģ ł -é Ŀ -w ers -× © -Ġintrodu ce -Ġequ ation -ãģ¾ ãģĻ -Ġn om -Ġpain ting -us hing -ĠA P -Ġencour age -Ġsh ip -itt ee -iver se -ot a -n am -ãĥ » -Ġexerc ise -ĠÐ Ń -Ġn as -Ġthous ands -ĠCaliforn ia -Ġs es -Ġr ow -ŀ Ī -Ġpand emic -Ġsk ill -b el -Ġdire ctor -Ġmil k -Ġn ut -Ġmot ion -Ġcl osed -è ¨ -Ġcred it -ah r -Ġche ese -Ġal tern -im ately -Ġs ust -ĠT ra -Ġgl ad -Ġhigh ly -Ġw a -Ġredu ce -Ġb le -ad or -in ated -ion es -ci ent -Ġdep ending -Ġsh aring -Ġca ught -ra el -Ġme hr -Ġpass ion -ç Ľ -Ġr u -Ġfar m -T I -av es -ĠR ob -ĠB ro -Ġmot iv -ret ch -ru pt -ĠB ig -Ġall e -Ġet t -ub s -ĠJapan ese -ĠH all -и ли -AUDI BLE -ç ¬ -Ġcell s -ik a -el ine -il er -Ġì £ -Ġsk y -IN AUDIBLE -end e -ap ter -Ġp in -Ġg ather -h ol -le ction -Ġsy n -Ġpl ug -r ound -Ġun iversity -h ib -Ġfant astic -k n -Ġho le -ĠRem ember -in ct -ak s -C H -Ġbro ken -Ġstr ateg -Ġal ive -Ġt ank -Ġc art -r ated -r ie -ĠSt ep -ĠEvery thing -Ġb ound -Ġso bre -Ġcustom er -¡ Į -ur g -ĠB ill -L a -wh at -Ġre action -Ġs ession -Ġpl ans -ĠìĿ´ë łĩê²Į -Ġdown load -ì Ļ -u er -Ġc ab -Ġinst r -if ying -ĠN ice -Ġteam s -ı l -Ġgo als -is ch -Ġtrans port -Ġanim al -Ġcost s -Ġcall s -Ġse hr -ì Ī -ri an -Ġd ial -Ġwe ather -๠Ģ -Ġв оÑĤ -ĠPl ay -Ġsh ared -Ġsm ooth -ab a -Ġleav es -à® © -Ġconc ent -Ġsh ift -ĠëIJ ĺ -ĠGo vern -Ġdem onst -Ġbut ter -ĠìĹ ¬ -Ġsat isf -Īë ¬ -Ġrecogn ize -ĠF rench -Ġvol ume -ä nd -Ñĥ м -Ġì§ Ħ -ĠKe ep -ow a -ipp ed -ÑģÑĤ ÑĢ -Ġdet ect -ĠÏ ĥ -Ġl ift -Ġcl othes -ĠSt op -à µ -m et -Ġcl in -Ġar r -f riend -Ġst uck -Y e -h and -um a -Ġsc ri -Ġfuck ing -ct ors -× ª -Ġjo ining -Ġc ette -ĠØ £ -ĠWh ite -Ġi hr -Î Ń -ãģ Ń -Ġinclud ed -ess o -Ġac ad -b um -Ġs ab -Ġд лÑı -è¿ Ļ -uf act -ĠRep ublic -r im -Ġye llow -Ġlim ited -T ER -ĠT y -Ġnot es -v est -и з -al ed -Ġph ase -and a -ĠM om -R I -Ġim mer -m al -Ġin j -Ġy ang -ud ible -аР³ -Ġset t -Ġmag ic -Ġens ure -Ġsp ring -Ġsh ock -Ġwhe el -ог да -ãĤ Ī -Ġcan cer -Ġro ot -Ð IJ -gen cy -Ġë į -i i -Ġout put -Ġcomm it -Ġwork ers -ìķĦ ìļĶ -ĠÑģ ам -ve y -Ġpe u -Ġc ivil -is c -Ġbr ings -ÑĢ ав -an ia -Ä ģ -c raft -mb ol -Ġintell ig -b i -ac ing -y ou -Ġbecom ing -ĠD er -em a -å°± æĺ¯ -Ġing red -Ġcomm and -Ġupd ate -Ġpre m -Ġopen ed -Ħ ¤ -ени е -Ġg ard -Ġstat ement -Ġsc rew -Ġpr ote -Ġc ards -Ġt ask -Ġeven ing -Ġst itch -in en -ĠB er -m ark -ĠD ad -Ġе ÑģÑĤÑĮ -Ġ× ŀ× -ìĹ Ī -Ġb an -Ġcl im -Ġfre edom -Ġnorm ally -еÑģ ÑĮ -å ¦ -Ġprov ided -Ġìŀ IJ -ĠìķĦ ëĭĪ -ĠK im -ied er -ìĿ Į -Ġcit iz -Ġb ike -Ġb ak -Ġno ise -Ġcl imate -iz es -å¾ Į -Ġincre asing -ĠTH E -Ġli qu -Ġperson ally -e f -res p -Ġleg s -ind er -Ġp ed -Ġë§ İ -Ġdep end -Ġvar iety -ĠIs rael -Ġwas h -å Ĩ -Ġqu iet -ĠJ ames -ĠJ ew -Ġfore ver -ĠI nt -Ġcoun ter -ur ance -ĠAny way -ca re -ĠOn ly -ci ón -ad i -ĠE v -ëĭĪ ê¹Į -ĠÎ ± -Ġslow ly -Ġо д -Ġnot iced -ier en -Ġfe ll -ĠÐ ij -Ġm ême -Ġwhen ever -! ) -ĠH y -å ¼ -ord s -us ion -ĠSt ar -Ġí ĺ -ĠM ac -ä¸ Ĭ -i ven -Ġìĭ ľ -ĠìĹ Ĩ -ĠT ur -Ġg er -r is -Ġve z -Ġл Ñİ -Ġvers us -ا Ø -ocol ate -Ġplan e -Ġz o -Ġsu it -Th is -Ġn erv -ĠA cc -Ñĥ ж -ìĤ ¬ -n h -em e -Ġa uss -Ġme as -Ġtr ès -Ï ī -Ñģ ли -ĠAr t -ĠSec ond -олÑĮ ко -ch o -it ect -е ÑģÑĤ -Ġb oss -Ġinc ome -ł ¤ -Ġsh ad -Ġapp ropri -ĠM al -op t -Ġart ist -Ġplay s -oth ers -ĠIn ter -Ġvir us -Ġh ung -Ġconst ant -Ġscri pt -Ġsn ow -ul f -k et -Ġdev ices -Ġmet al -ight s -ìĦ ¸ -Ġsal es -Ġve get -Ġcollect ion -Ġv ia -k er -Ġgot ten -O W -i én -Ġacc ur -Ġw ave -ult y -ĠA ir -Ġlead ing -ic ing -Ġcent ral -ĠChrist ian -f r -ĠAl though -Ġsong s -Ġf if -нÑĭ Ñħ -Ġbel ong -oss ible -ì ° -Ġphot os -is l -Ġrela x -s a -US IC -ê · -Ġman ufact -ĠTw itter -Ġdanger ous -Ġhy d -le ar -i ant -ĠâĢ ¦ -Ġsudden ly -Ġla ugh -Ġang le -ĠG ot -Ġwor ried -о е -Ġp ap -ĠM art -en o -Ġbatter y -Ġп оÑģ -Ġlight s -Ġar ms -ĠA bs -m es -âĢ ĵ -use um -Ġte a -ĠM ic -Ġfor mer -ograph y -Ġapplic ations -ĠD ire -çĦ ¶ -Ġfeed back -itch en -yor um -u ed -ig t -Æ° á» -os ition -ĠD el -Ġíķ ĺë -ĠB ack -ad s -Ġpr ime -ì£ ¼ -ì£ ł -× ij -Ġm ut -] . -ĠÐ Ĺ -lo c -k in -Ġexper t -Ġal right -ung s -Ġsupp ly -Ġleaders hip -ĠF ra -Ġtyp ically -Ġs el -Ġtre es -Ġ2 2 -h ar -Ġwor st -Ġbus y -ant o -ĠU p -ĠB as -Ġpresent ation -Ġstr ange -Ġth in -ÑĤ е -Ġveh icle -Ġд о -cell ent -7 0 -Ġt ired -Ġcris is -Ġt iny -as y -Ġr an -é ĩ -Ġfor ces -Ġо Ñĩ -Ġident ify -Ġass ess -иÑĤ е -S E -Ġcreat ive -ç Ł -Ġdep artment -Ġinit ial -æĪij åĢij -ĠD am -ak t -v ere -Ġinf ect -Ġp ump -Ạ¡ -Ġv iel -Ġr are -Ġd ot -ash ion -em pl -Ġf lex -Ġk on -Ġtr uck -Ġle ct -Ġpl astic -la w -Ġlik es -Ġr ough -ĠM AT -í ŀĪ -Ġcomm er -Ġas se -Ġc ake -Ġact ions -Ġad m -Ġother wise -ĠHe alth -Ġcoll e -à¹Ģ ภ-Ġr ub -å¾ Ĺ -æ Ķ -Ġsc r -Ġz um -ĠH im -Ġch amp -Ġconcern ed -Ġ5 00 -Ġpl ate -ĠO ut -Ġdon c -Ġequip ment -Ġta ught -ll ed -Ġí Ļ -iv a -Ġmot or - » -Ġgu ide -å ī -Ġstop ped -Ġr at -Ġlab or -Ġa im -Ġprep are -ĠÑ Ī -Ġshoot ing -ann ed -cri pt -Ġen emy -Ġdep ends -Ġn av -Ġb er -Ġland s -Ġun ivers -i u -Ġfact or -ok ing -Ġcar bon -b ut -ĠL ove -el d -ĠÎ µ -Ġg a -Ġé s -Ġbre ad -Ġvol t -í Ĭ -Ġwas te -Ġkeep s -æī Ģ -Ġst or -Ġhon or -Ġun less -Ġcol um -Ġë ĮĢ -Ġpl ants -Ye ah -Ġinclud es -ä¸ Ń -Ġo x -Ġpe ut -ë§ Į -ìĥ ģ -ist ry -ภ± -ĠDep artment -ant a -Ġfing er -Ġst retch -Ġsy mbol -Ġneigh bor -æ ¬ -ê° Ħ -~ ~ -ĠÑĤ Ñĭ -ĠA ber -k es -Ġmass ive -ĠC H -ĠS al -× ł -ãĤ Ĵ -Ġd ynam -ach e -ĠP re -Ġmon itor -ent ed -E O -Ġrais ed -ist ics -Ú © -Ġv ou -it en -¡ ° -Ġbusiness es -Ġe arn -Ġmob ile -id ade -Ġha be -y r -l ict -Ġcon duct -Ġfed eral -Ġw o -b u -Ġn one -Ġteach ers -ĠاÙĦ Ø -éģ ĵ -id ents -ا ÙĦ -Ġtre nd -еР¶ -Ġal bum -Ġm ich -b ased -ภµ -Ġtrans ition -Ġн о -õ es -h ost -ed y -ĠPro f -p an -ij n -Ġcapac ity -und o -Ġ× ij× -Ġbreat h -Ġм ен -Ġm ü -í Ļ -ĠA ut -hing ton -Ġn or -Ġg ain -po int -Y es -ĠØ ª -ĠN a -Ã¥ r -Ġi ç -ĠM ary -Ġsp in -Ġant i -åIJ § -Ġsome how -Ġlaw s -Ġmom ents -Ġg re -Ġmo ves -ĠW ould -Ġpred ict -Ġv ra -Ġ201 9 -¶ Ħ -Ġfund ament -2 5 -Ġp ure -Ġw ow -Ġis land -Ġinvest ment -Ġb ath -ĠY a -Ġhard er -Ġt ips -å Ĺ -Ġelect ron -ĠB ob -Ġb ond -od ies -ĠA ug -Ġgib t -Ġch air -Ġtw ice -w ood -Ġcl ar -Ġmas k -Ġhonest ly -Ġ201 8 -t ies -' , -Ġp ens -Ġsurpr ised -Ġcommunic ation -ãģ£ ãģ¦ -Ġsp r -Ġwh ose -Ġst ars -× IJ× -ĠâĢ ĭ -Ġproper ly -Ġg rew -os ing -Ġdi vers -A D -Ġem pt -Ġexp ression -Ạ¿ -ĠP al -ãģ Ĭ -Ġjust ice -Ġp air -w o -Ġse at -or ter -Ġlink s -ĠM er -Ġre nd -но е -up id -ĠH el -ĠM arch -ĠL o -Ñģ ÑĮ -Ġhas n -Ġev alu -ãģ ı -å¤ © -il os -Ġfund ing -Ġv en -u an -ĠM aster -ĠO l -ĠF re -Ġy ap -ĠS ir -s ch -Ġmist ake -am an -Ġdin ner -ĠWas hington -Ġorganiz ations -Ġж е -av ing -Ġv ÃŃ -Ġbirth day -Ġbe ar -ĠÙ ģ -Ġaff ord -Ġre ven -Ġrelationship s -r ough -ĠT ime -Ġt ag -ĠS un -u ary -ĠP o -c ar -ab ilities -Ġpr ison -Ġl ic -ìł ķ -id den -Ġspec ies -é » -Ġf irm -Ġsc ore -Ġd it -Ġspe ct -Ġp el -Ġcompl icated -æ¨ £ -Ġr ank -Ġoppos ite -Ġpick ed -Ġк он -el er -Ġm ig -ĠS l -ĠN et -Ġne ck -ĠFr ance -Ġtechn ical -ภ¡ -Ġmil es -Ġprim ary -Ġse in -s es -Ġla ughs -b ra -ÅĽ ci -ri age -Ġn ic -et ers -Ġà ª -olog ies -ĠI S -r ad -ud o -ı nd -m ar -Ġex ch -Ġcompet ition -Ġauss i -ĠS erv -Ġre nt -Ġch ocolate -Ġw ieder -Ġnear ly -Ġspe ech -Ġun c -Ġpar am -ĠBrit ish -Ġrem ain -ภģ -ur t -ĠØ ¹ -Ġcr ack -ail s -Ġprom ise -Ġpay ing -i ÃŁ -Ġad apt -ал а -Ġmov ies -Ġw ire -Ł ¬ -æľ ĥ -Ġter rible -Ġs ó -Ġperfect ly -åij ¢ -ord in -Ġj á -Ġimp ossible -ĠTh ree -Ġn h -Ġtur ning -r um -ĠB el -ig g -Ġrespons ible -и й -Ġincred ibly -w i -ian o -Ġhum ans -Ġà ĩ -Ġsetting s -Ġj oy -o ot -Ġdeal ing -ill ed -Ġsur round -Ġfollow ed -Ġposs ibly -Ġinit i -st en -Ġpr os -Ġcand id -Ġass ign -Ġviol ence -W ell -Ġr ise -P S -Ġtamb ém -Ġë ĵ¤ -i ance -y an -Ġaud io -ĠB et -ĠAmeric ans -ĠAs s -is chen -ìŀ ħ -Ġult imately -Ġpol ic -Ġmajor ity -éĢĻ åĢĭ -ĠFin ally -er ap -Ġgu ard -ĠMAT T -Ġbr own -м и -Ġch a -ĠHo ly -Ġnerv ous -ipp ing -ÄĻ d -ĠS a -ĵ ľë -¶ Ģ -l ie -çľ Ł -Ġn uc -ĠA pr -é Ľ -ĠKore a -eg o -ĠCan ada -Ġkön nen -Ġcomp ar -Ġg anz -ĠM ais -Ġthem e -Ġk i -Ġdraw ing -az on -ĠO ff -t t -ĠW ind -Ġtod os -Ġob vious -на Ñı -I M -ĠÐ ł -we ll -Ġbl ow -Ġho ok -Ġcir cle -Ġë³ ´ -Ġarch itect -ĠK r -Ġc ó -Ġprotect ion -eg a -å ĩ -Ġwatch ed -Ġans wers -Ġdi et -iv o -Ġpow der -Ġyour s -Ġhigh est -çĤ º -F F -å º -Ġbo ys -ö yle -Ġl unch -è¬ Ŀ -ĠI I -Ġset s -Ġmo le -Û ģ -Ġwin ter -Ġluck y -Ġrespons ibility -Ġsign al -Ġwond ering -Ġa x -Ġcook ing -ов оÑĢ -le g -Ġп оÑĤ -Ġsurpr ise -Ġdem ocr -Ġlo op -Ġj ag -Ġcur ious -Ġmarket ing -Ð Ŀ -ar on -ĠApp le -Ġvirt ual -Ġ19 8 -no on -ĠM et -оÑģ ÑĤо -об Ñĭ -it u -ĠA w -Ġbu ying -Ġrestaur ant -ĠB ud -Ġdou bt -Ġgr ant -Ġver d -Ġc ash -Ġfac ulty -Th at -ĠE in -å¤ ļ -Ġw ed -it ness -ĠM ag -n el -Ġn arr -Ġacc ident -Ġmed ium -em ents -Ġcr ow -n ight -ìĿ ¼ -ä¹ Ł -Ġlibr ary -аÑİ ÑĤ -Ġtamb ién -Ġrefer ence -Ġfour th -h ouse -v ention -Ġfill ed -ĠC our -ib r -Ġn g -Ġdevelop ing -Ġprov ides -Ġpo ll -Ġtra ffic -arent ly -à® Ł -Ġform s -Ġcl ient -Ġg entle -Ġmus s -ĠCong ress -ĠInd ian -ce an -Ġp il -Ġc zy -st ood -ut y -Ġn ä -Ġsp ending -Ġconst ruction -ina udible -Ġë§ Ī -Īë¬ ´ -Ġìĥ Ŀ -om a -os en -ag o -Ġlar gest -ãħĭ ãħĭ -Ġun iverse -b es -os a -Ġе го -Ġd ude -ĠM AR -Ġind eed -ε ι -Ġman aged -ĠSh ould -S o -Ġappl ied -Ġfair ly -ĠD en -Ġanal y -Ġconst antly -Ñģ п -H ow -ĠS ay -en cies -ĠP C -Ġegg s -à® ° -Ġet h -ĠEnt ão -in ar -i ot -Ġc z -ĠEurope an -ãģ Ī -ĠA M -Ġc á -Ġrad io -§ Į -Ġh ide -ä» Ĭ -ĠSt art -Ġcl ub -ĠH ope -Ġeff orts -lus ion -Ġc ities -h one -Ġreach ed -Ġgu id -ro id -Ġhar m -Ġcut ting -Ġb ul -1 8 -i est -ĠMe x -Ġ iron -çŁ ¥ -Ġafter noon -Ġha ll -Ġpr zy -Ġg osh -Ġinflu ence -Ġв ид -Ġincre ased -ĠMin ister -Ġdis ci -ĠP eter -Ġver t -Ġmen u -Ġse lling -ur ally -Ġqu ote -Ġ ¡ -Ġcontin ues -mp re -ĠÅŁ ey -it ution -Ġна Ñģ -c les -ĠGerm an -c zy -ĠÐ £ -B e -Ġk itchen -ĠT ry -i pe -Ġic on -ar p -Ġprov iding -ĠTr ans -Ġtechn ique -Ġh är -Ġinf rast -Ġsus p -ü ck -ic ip -ĠÐ ķ -Ġc in -ìĸ ´ë -Ġpr z -Ġcompon ent -Ġby e -ĠB ible -iz er -C h -Ġsol utions -Ġaccom pl -Ġ201 6 -I E -ĠT a -Ġass ume -Ġliqu id -Ġë¨ ¹ -Ġquar ter -Ġfem ale -ĠTh ink -Ġstat us -it ute -Ġco ach -Ġre in -Ġcomb ination -è · -ĠT er -Ġobject s -Ġdist rict -Ġmake up -Ġmur der -w as -f en -Ġbow l -Ġpub lished -Ġsp orts -ãģ ¡ -Ġident ity -Ġseem ed -Ġact ing -л Ñİ -ri x -Ġup load -Ġh ast -Ġbo at -ĠM od -ri o -Ġ = -Ġcy cle -¯ ¸ -Ġl oud -ust ed -com ing -Ġ201 7 -Ġon t -Ġleg isl -Ġst ruct -ĠSomet hing -Ġconf lict -Ġu pper -Ġman ager -Ġm ort -Ġf ra -ĠÄ ° -ĠM ike -ĠW ork -Ġn ó -ph ere -ĠìĤ ¬ë -ĠL and -Ġfil ter -Ġprom ot -æ ° -æĻ Ĥ -ķ ¼ -Ġrecord ing -× Ŀ -Ġassoci ated -Ġf uel -und er -Ġele ction -Ġemploy ees -ĠCom p -ÑĢÑĥ г -ĠW o -ro l -Ġsa ved -ĠH on -ĠV i -åĪ Ĩ -ac a -p ret -Ġw et -Ġst upid -Ġl ad -Ġf est -Ġw ake -Ġи н -Ġgreat est -ĠJ im -Ġserious ly -Ġì ¹ -Ġfeel ings -Ġ3 00 -i ation -Ġbeaut y -Ġìŀ ĺ -Ġs an -ĵ ł -Ġ- ( -Ġcons cious -Ġд ел -b ye -ç Ļ -M an -Ġlet s -Ġsho es -y d -ä¹ Ī -Ġdisapp e -ĠCount y -ĠSc ott -Ġbut t -Ġaqu ÃŃ -Ġconf ig -resp ond -LAU GH -© ëĭĪëĭ¤ -Ġdivid ed -Ġac qu -Ġz one -Ġk omm -a ção -ì§ ľ -c ut -Ġ2 3 -Ġmaxim um -ro g -Ġrun s -Ġcompon ents -Ġarri ved -Ġconf ident -ÑĢ ов -Ġhe ight -Ġpro ced -E M -ĠÐŃ ÑĤо -ĠM en -Ġtalk s -Ġconf idence -ĠChr is -Ġlead s -Ġn ose -f all -b b -ĠNot hing -is er -Ġindepend ent -Ġmin or -Ġsy m -l en -ci ence -Ġf ashion -Ġsex ual -Ġb un -h ere -Ġso il -Ġdies e -Ġsh ap -Ġempt y -Ġjour nal -ag on -ĠThe ir -Ġweek end -ÃŃ t -Ġer ror -Ġn ar -à ¸ -è © -an cy -Ġìķ Ĭ -Ġfore st -Ġha cer -Ġmiss ed -ãģ ķ -åı¯ 以 -Ġev il -Ġstor age -Ġsing ing -in ha -Ġkn ock -Ġimp ress -ĠоÑĩ енÑĮ -ĠGo ld -ĠS ur -ĠP ort -åİ » -ĠL ond -Ġfaz er -ot y -ot o -Ġan x -ĠWill iam -Ġexist ing -pl ace -ĠC D -Î ³ -ĠColl ege -l or -ĠE ast -s en -f ach -o ft -Ġexperien ced -Ġlo ves -im m -Ġpo ly -Ġes se -ì ¤ -ĠG rand -è § -ch er -Ġvict im -ĠG es -л ÑĮ -v ision -Ġt all -Ġl ens -Ġз на -ĠB oth -Ġì ² -Ġsust ain -Ġarg ument -Ġfact ors -Ġautom atically -Ġfr uit -Ġli ber -Ġa le -ĠP ress -ĠB a -ĠÐ ³Ð¾ -Ġhundred s -th at -ĠR ich -Ġreci pe -ĠI T -è ĩ -Ạ¥ -Ġdescri be -Ġdri ver -ĠO ct -ĠM at -д е -Ġme al -Ġlat est -Ġth erap -Ġcomp are -ĠAm azon -Ġì¢ Ģ -ĠRuss ia -Ġstr ing -Ġk a -ĠComm un -Ġd ia -I s -Ġmill ions -Ġcor por -Ġcor respond -Ġfix ed -ĠJo e -Ù İ -Ġview s -Ġr iver -Ġstud io -ig ger -Ġfl avor -Ġpres ence -Ġun its -Ġsa ving -av our -Ġp esso -or ith -Ġh ers -ĠN at -as ion -ĠFr ank -о ÑĪ -ÅĤ y -í Ħ -Ġein em -Ġfun ctions -um an -Ġn orth -Ġìł Ħ -Ġhor se -v id -Ġple asure -а ÑĪ -é es -ind a -Ġt ail -Ġexpl ore -S T -Ġcommer cial -ĠD uring -ar l -] : -f it -Ġr ates -æ ³ -M USIC -Ġhous ing -Ġein er -Ġsitu ations -æ ĭ -Ġdec re -Ġappropri ate -ен но -% . -Ġb ac -Ġw at -ens ity -ä h -kn own -it z -Ġemot ional -erv ation -Ġbl ind -1 6 -í ĥ -大 家 -Ġjo ined -Ġloc ated -ĠÑģ м -ad as -ber g -Ġd ess -Ġde ar -ed en -c os -Ġad opt -1 00 -ow e -ĠChe ck -ism o -Ġsim pl -Ġang ry -Ġмен Ñı -ĠC am -Ġp ad -Ġatt end -Ġsam ple -æĹ ¥ -Ġì Ľ -ĠI N -ul ous -ĠS ar -ĠSh ow -Ġinfrast ructure -ĠAug ust -Ġless on -Ġn iet -æ İ -Ġfo i -Ġbro ke -t r -ç ķ -Ġ4 5 -Ġg ew -Ñĥ п -at i -Ġmaint ain -Ġart ists -ing er -æĿ ¥ -er ved -I A -Ġequ als -Ġoper ation -ill y -ĠëĤ ´ -Ġcrow d -Ġintern al -Ġtest s -ĠR ock -ĠC ons -ĠëĦ Ī무 -w ar -Ġs ou -Ġch art -ĠJ une -ĠApr il -g ent -Ġv ent -Ġqu and -ĠKore an -im o -ç ī -id ers -Ġmount ain -ÑģÑĤ ав -æľ Ī -ij k -Ġdiscover ed -ĠS und -ĠS il -Ġso lo - ´ -Ġsch ol -ĠE ach -ç µ -Ġb are -Ġí Į -ĠvÃŃ de -Ġingred ients -ĠIt s -Ŀ¼ ê³ł -Ġì Ĭ -Ï į -ĠLe e -Ġsc ary -Ġprinci p -Ġspirit ual -ì ħ -ĠH old -æ²Ĵ æľī -Ġdef ine -ĠL es -ĠN or -ĠE nd -Ġbl og -ĠG reen -аеÑĤ ÑģÑı -p art -el es -äº ĭ -ĠUnd er -Ġpart e -Ġ3 5 -Ġse ctor -ĠS ept -Ġaut h -à® ® -om in -Ġcl ients -Ġc i -ĠFr iday -er as -Ġtw e -ul ated -Ġcult ural -ĠÑģв о -Ġëį Ķ -Ġà º -Ġpar ce -à® ² -Ġtrad ition -Ġjud ge -ĠGen eral -Ġdeterm ine -ĠIs n -ĠP L -ne ath -Ġmatter s -íķ ´ì -! ] -а Ñħ -Ġpo ol -Ġvari able -Ġvacc ine -Ġcaus ed -Ġw est -ĠY ep -f ast -Ġph ilos -hor a -Ġcontinu ed -Ġunf ortunately -ãģ į -æ ķ -Ġfl ight -Ġw rap -Ġhu h -ĠAbs olutely -Ġp ink -Ġrem ains -Ġn é -Ġf le -ĠS ol -Ġlos ing -Ġalg orith -Ġrequ ires -Ġfound ation -ĠB ur -Ġprofess ion -ĠM id -Ġë ŃIJ -c an -ĠM il -Ġyoung er -Ġappe ars -ter m -íķĺ ê³ł -ac le -ĠLond on -Ġengine ering -ภ¢ -Ġadv ent -ìĦ¸ ìļĶ -Ġê¸ ° -ĠM aj -ÑĢ ем -ing u -ĠU K -u ro -s pe -Ġt ent -Ġreport ed -ĠA L -H ey -Ġë§ IJ -Ġd ent -ĠAustral ia -ĠJan uary -³ ´ -ag ues -ars h -r ig -Ġtien e -ภ£ -Î ® -Ġmach en -un te -Ñĥ Ñģ -Ġelect r -Ġtut orial -Ġpl aced -ĠìĿ´ ê±° -ĠCoun cil -í ĸĪ -°ë ¦¬ -ah ren -Ġê·¸ë ŀĺ -Ġpro ve -f ol -Ġqu er -Ġche ap -ĠF ather -ĠP ower -ĵ ľ -Ġpur s -Ġes p -ĠB re -ê¸ °ë -om as -æĥ ³ -ил ÑĮ -Ġge ht -os ter -ê³ ¼ -Ġfil es -ĠÐ § -be ll -Ġwh om -Ġë ĺ -Ġex cellent -Ġdat ab -Ġg ö -Ġì§Ħ ì§ľ -Ġbelie f -j et -Ġj ack -Ġsw im -ri al -um in -a uc -Ġso ll -Ġess ential -íķĺ ëĬĶ -Ġev ol -cha ft -ain e -th let -Ġinc or -Ġreport s -Ġdefin ition -ke l -Ġcirc um -Ġprodu ced -Ġ× Ľ -ant ic -n et -Ġa ward -Ġd urch -Ġtrans p -Ġm ale -¦ ¬ë -Ġmo on -ĠGe orge -Ġfly ing -i ó -Ġs ources -Ġpl enty -ĠDem ocr -R O -Ġ 00 -Ġsec ure -ĠB ir -ra in -Ġz ur -Ġeffic ient -Ġrepe at -Ġmethod s -Ġcal m -Ġdiscuss ed -ĠìŀĪ ëĬĶ -Ġser ver -an ie -ĠInst ead -Ġide al -Ġcon ven -Ġhop ing -ĠT or -Ġdep th -Ġhe aven -EN CE -Ġhab it -gr ad -Ġfl ag -Ġin e -Ġk h -ĠL I -Ġfac ing -ĠA U -ĠT im -Ġg em -ĠJ ul -Ġel a -iz za -Ġfe llow -Ġqu el -Ġsp oke -Ġcitiz ens -u ge -é ĥ½ -Ġp ages -Ġf asc -Ġrelig ious -at en -Ġch apter -ĠV al -Ġcons ult -ĠM ill -g l -op er -Ġinf in -Ġmar riage -Ġmedic ine -Ġд в -Ġdog s -Ġinstr ument -ĠEx act -á n -Ġ20 21 -Ġf er -Ġwe alth -Ġgr ade -Ñĭ Ñħ -Ġcr ime -Ġth read -Ġess a -Ġw ine -co hol -ph a -ภĩ -og ue -Ġins urance -arr ator -ĠSept ember -Ġv id -ĠSp irit -Ġg est -ĠRuss ian -Ġproper ties -Ġart icle -Ġunder neath -y er -Ġjo int -Ġrelative ly -Ġin ch -Ġdesp ite -ĠG ree -Ġclass ic -Ġsupport ing -Ġinst ruct -lus ive -Ġdi agn -æ Ĭ -Ġadminist ration -аб оÑĤ -ĠO pen -æīĢ 以 -Ġп ок -Ġdoll ar -Ġconse qu -o ber -ĠGerm any -Ġter r -ĠQ U -ĠÐ ĵ -ç ¾ -Ġstrong er -É Ļ -ĠÙ Ĭ -ĠiP hone -Ġfab ric -ü h -Ġen em -æ ¯ -Ġsub t -E E -ond e -Ġcre w -Ġremo ved -Ġl ady -Ġpot entially -ĠÐĿ о -y al -Ġsym pt -Ġar my -Ġintrodu ced -t es -Ġaspect s -1 4 -ĠL ou -Ġ ) -Ġde ploy -p et -Ġh an -ĠW atch -Ġweap ons -Ġph en -Ġreg ister -Ġein fach -Ġsp ort -Ġbr idge -Ġin ner -Ġminim um -Ġw itness -Ġes o -Ġvill age -Ġown er -¦¬ ê³ł -Ġsc ream -il ed -Ġp itch -b ru -Ġadv ance -ä¸į æĺ¯ -Ġsupp ose -ĠAt t -еÑĤ ÑģÑı -Ġdiffer ences -ak ed -Ġinter pret -à ¦ -iend o -Ġabs ol -ĠбÑĥд еÑĤ -Ġë ² -Ġtri al -Ġthink s -ly ing -cept ion -ĠAfric an -Ġchem ical -Ġta pe -Ġconvers ations -Ġdistrib ution -t i -ĠA I -Ġfl ash -Ġunder stood -ĠGovern ment -å° ı -! ? -ĠS k -ê± °ë -ri er -T S -ĠAcc ording -Ñİ ÑĤ -Ġsp ons -ÑĤ обÑĭ -Ġval u -ere m -icht ig -Ġresist ance -ĠG al -ger y -Ġbeg ins -Ġadv anced -Ġrele vant -Ġpolit ics -ĠF am -Ġç ok -ĠN ever -ill ing -Ġfoot ball -и и -ĠI D -ĠAfric a -Ġfing ers -Ġб олÑĮ -Ġà ¡ -Ġcl ip -ĠL at -ãĤ Ħ -Ġì§Ģ ê¸Ī -es se -Ġvo or -Ġas ide -æ ŀ -Ġto ward -Ġb at -Ġval id -ĠM ens -Ġcomplet ed -ı ÄŁ -Ġpod cast -ĠB on -Û Ĵ -ĠJ uly -il a -Ġpack age -Ġpull ed -ch ar -ĠM el -o is -Ġs outh -Ġë Ķ -Ġimport ance -Ġp ushing -Ġis ol -Ġstand s -c ill -ä ¼ -Ġ ðŁ -or i -ê° ģ -Ġhom es -Ġconcern s -Ġb iz -å ½ -b ie -Ġb is -Ġge ar -ĠM S -Ġh un -ĠM att -Ạ£ -se y -ĠSec ret -Ġod d -ĠM ax -oll y -f ord -ĠS H -Ġrepl ace -Ġnav ig -Ġin i -и Ñı -Ġgi ant -Ġma nd -ĠH app -TI ON -g un -iam o -ìŀħ ëĭĪëĭ¤ -Ġg ap -Ġê tre -Ġclass room -Ġhy p -ak i -è ® -is ters -ack s -ĠÑģ о -Ġb ug -Ġgra v -am in -Ġevery day -Ġì ¡° -Ġgard en -ce mber -Ġest o -åĹ İ -Ø ¬ -Ł ° -å ģ -Ġr om -Ġìłľ ê°Ģ -Ġfall ing -Ġfa ult -ell y -Ġch est -Ġл и -Ġpot ato -Ġbuild ings -Ġoper ating -Ġp are -w r -D on -ĠF our -Ġv ul -Ġl á -Ġfr ust -ĠD ann -ol es -ny a -Ġì ¶ -ĠÑĢ аÑģ -× Ľ -Ġa ÃŃ -w ord -Ġweap on -Ġob t -ĠF all -ĠSte ve -Ġmix ed -Ġp ode -ĠA S -ĠL eg -Ġdes c -Ġspl it -Ġemer gency -ĠS ing -Ġprof it -Ġtyp ical -ĠDon c -Ġannoun ce -ĠTe x -Ġsac r -tern al -Ġcomm ittee -ig o -Ġdi am -ph as -Ġdef e -ĠProf ess -Ġdec l -Ñĥ ÑĢ -2 2 -ol f -ĠM ond -u y -Ġa y -Ġl em -Ġlove ly -ĠC ould -Ġgu ar -H H -Ġcare fully -ĠL isten -Ġк ÑĢ -Ġyou th -ĠThere fore -Ġdream s -ĠJe ff -? ] -Ġë Ī -D A -Ġb odies -au x -Ġtechn iques -Ġmechan ism -× ĵ -Ġо ни -Ġdes ire -à ® -ĠV o -qu es -ĠÑĥ же -ĠWho a -ĠG ame -Ġh al -an ish -Ġpract ices -5 00 -Ġsort s -up s -ate ful -Ġhers elf -Ġgu itar -Ġprop os -Ġsit es -Ġbe ach -Ġ× ¢ -ç¬ ¬ -н Ñĥ -Ġdr am -ĠNo ve -V E -r ant -Ġpl ot -ĠìŬ 기 -ĠC a -Ġestab lished -Ġ201 5 -Ġinsp ired -Ġannoun ced -ä¸ ª -ĠÑĤ ÑĢ -Ġ2 6 -Ġv oy -Ġte ch -ìł ģ -Ġprocess es -ont o -ĠP an -Ġrap id -ist an -Ġ19 7 -Ġrelig ion -Ġ2 8 -Ġsm ile -Ġb ab -Ġ Ú© -ĠV ir -Ġsched ule -Ġexec ut -Ġpr on -Ñ į -ĠÐĿ Ñĥ -m usic -ìĽ IJ -Ġg an -ìĭ ł -Ġdef ault -Ġbe m -Ù ī -Ġfor ced -ĠOb viously -Ġst one -Ġt ie -Ġdrink ing -Ġser ved -C ause -Ġcon ference -ĠExact ly -ãĥ Ī -ł ľ -ìĻ Ģ -ĠR a -Ġf ake -Ġdif f -ãģ © -Ġchalleng ing -Ġì¤ ij -Ï ĩ -ä»Ģ 麼 -Ġintellig ence -re te -Ġstud ying -Ġapp oint -Ġt an -Ġи м -Ġcur ve -ĠTe am -ĠA z -Ġз д -ĠMus ic -f ield -ir ation -Ġfail ed -Ġno vel -Ġdifferent ly -Ġes cape -ĠY o -ĠOct ober -ı yor -Ġdescri bed -Ġcon vert -ac ement -Ġhot el -is ation -Ġsu is -ãģ ij -å ŃIJ -æĢ İ -Ġwalk ed -2 00 -Ġneighbor hood -is p -ĠL os -Ġh idden -Ġ2 7 -л е -Ġph r -ĠIs land -ĠSt reet -end a -hip s -os ure -Ġdefin ed -ภ§ -Ġv ida -Ġlab el -ĠEvery body -Ġjo ke -ia o -ا ÙĨ -Ġa thlet -... " -ĠF ire -D o -Ġdef ense -Ġent ertain -á t -Ġpolic ies -Ġal cohol -ĠEng ine -Ġg al -ĠJ ud -Ġvol unte -ick s -et a -ag t -Ġ× ķ -Ġm ö -1 3 -Ġenc oun -Ġe h -Ġor ange -Ġabs or -Ġsp aces -ĠNove mber -êµ ¬ -i at -Ġt am -ck now -Ġst orm -ĠDire ctor -Ġpre gn -ĠìĿ ¼ -Ġо п -Ġres ource -Ġb ard -ne w -ĠDe cember -u its -Ġwe il -Ġconst ruct -s i -n ic -Ġfl our -Ġrest rict -ü t -Ġentire ly -Ġbreak ing -ent lich -Ġtw enty -Ġcaus es -Ġele v -ĠS pr -ĠIntern et -Ġk iss -Ġoper ations -s zy -Ġë Ĭ -Ġscient ists -Ġgr own -Ġown ers -out s -Ġcour ses -Ġus ual -Ġin n -Ġtrans m -ñ o -Ġnu est -к ов -Ġcateg ory -ĠL ife -ĠPl us -Ġat mos -wh ile -Ġrecord s -Ġde ÄŁ -ëĭ¤ ê³ł -ĠìĤ¬ë ŀ -Ġrequire ments -in n -Ġimm ig -Ġdeep er -ç ´ -Ġapp s -Ġcolle agues -ż y -Ġoff ers -Ġt á -Ġcolum n -la ud -I R -ĠM s -Ġexch ange -l as -ĠL aw -ĠJ on -is se -ro gen -Ġmo i -× Ĺ -Ġs ending -Ġhe llo -е е -ÅĽ Äĩ -Ġsuc ceed -Ġsuff ering -Ġad vert -Ġì£ ¼ -çŁ¥ éģĵ -Ġrec o -ın ı -Ġк ом -all ey -Ġfail ure -ie j -Ġëķ Į -Ġdrug s -Ġcu ando -Ġìĸ´ë ĸ -ĠAb out -Ġqu ando -9 0 -ĠF ed -1 7 -S h -in ho -ĠSund ay -ĠPh il -Ġacad emic -ĠIn c -Ġmaint en -åĩ º -Ġre ward -er d -Ġcomm itted -ìĬ ¤ -г ÑĢ -Ġstand ards -Ġk al -Ġint ention -ĠZ h -Ġa cknow -ä ¿ -Ġ== = -og y -å § -Ġfilm s -is k -Ġte eth -Ġstrugg le -r d -u en -Ġdis s -ĠD ar -am y -Ġenem ies -Ġve loc -ĠC all -um bs -иÑĤ елÑĮ -Ġo cean -é d -ìļ ° -Ġtre m -ient o -еÑĪ ÑĮ -ffic ient -Ġbott le -Ġinstit ution -est y -ĠH an -h ab -ëĬ ĺ -Ġar rest -éĤ Ħ -Ġlet ters -oun ce -í Į -A n -Ġcreat es -Ġcl ock -Ġdeb t -Ġan cient -ific ations -g i -B ut -ĠT u -k l -Ġb order -Ġo ok -ĠB ay -est a -Ġë³ ´ì -Ġw ra -pre ne -Ġê² Į -ang le -Ġbelie ved -ien cy -ak a -Ġcrit ic -Ġb omb -Ġha m -ĠÐ Ľ -êµ Ń -ĠGu ys -ros oft -Ġcr im -et ch -AR R -Ġs ight -и на -Ġa in -á» ij -is che -Ġau x -Ġnum er -Ġsurv ive -A ll -B C -Ġs z -Ł ¬ë -Ġj am -ĠCour t -Ġall es -Ġtr igger -Ð ŀ -Ġform at -Ġdec ades -Ġc es -Ġsign s -Ġrob ot -ĠCh urch -Ġa z -Ġs oup -ĠTex as -ut en -ĠÑĩ ÑĤобÑĭ -Ġneigh b -ĸ ×Ķ -Ġcommunic ate -Å ¡ -Ġel imin -Ġfrequ ency -her n -id os -Ġem phas -Ġmess ages -Ġg ender -ĠW enn -Ġв о -Ġpr ices -ol o -Ġп он -w ing -ĠF il -а ем -ĠC ur -Ġfal se -Ġfield s -Ġs é -2 4 -Ġm ac -u ÅŁ -Ġlay ers -Ġadv oc -w an -Ġk ar -ĠÅ ŀ -Ġdec or -Ġwall s -o e -iss ions -Ġres ol -× ¢ -ĠCar ol -ĠV ide -le ep -ĠY OU -Ġfl ip -Ġsur gery -Ġch op -U R -. , -Ġag ency -Ġwant ing -Ġsol ar -Ġhor iz -ĠAd am -Ġstay ing -ol ic -Ġgr ateful -Ġrem ark -Ġtechn ologies -Ġprote in -å¿ ĥ -д ел -ĠM ont -Ġshould er -Ġz a -re y -ĠO oh -Ġst y -ic ar -оÑĤ ÑĢ -Ġrout e -ĠT urn -Ġb om -Ġdeb ate -Ġposs ibility -Ġíķ ´ì -ap a -Ġinv ent -ür lich -Ġprof ile -Ġsen ior -pp y -v as -Ġm undo -ate ver -Ġapp arently -en er -× IJ -ç Ń -Ġprec is -Ġal ign -Ġkn ife -ĠRo bert -å ĭ -Ġfo ol -Ġinv ite -us ing -Ġcircum st -Ġcapt ure -Ġd ough -ĠS and -Ġse u -ĠNew s -Ġb ite -Ġne ut -w ide -Ġlect ure -Ġëĺ IJ -Ġorigin ally -Ġcho ices -ĠG ar -Ġver se -Ġl it -Ġ19 6 -íķ ł -Ġmeas ures -ç ões -w ater -ri ve -Ġz ijn -í ģ -ĠB us -Ġhe b -е Ñħ -ĠK ar -ĠN ão -Ġkill ing -à® ª -Ġmir ror -m od -Ġm ol -Ġcre ation -Ġest im -Ġatmos phere -Ġg am -Ġt ables -is i -ĠL ittle -Ġt as -ĠE le -é l -Ġscen es -Ġt one -Ġaffect ed -ĠAU DI -ĠBr own -I f -ĠÙ ĩ -ĠDan iel -羣 çļĦ -qu er -ch i -íķ ĺë -Ġmist akes -Ġs la -ãĤ ¤ -Ġent r -Ġе Ñģли -Ġsh out -Ġport ion -Ñ Ĺ -Ġpre viously -á» Ļ -ĠпÑĢ ед -оÑģ ÑĮ -Ġhead s -ç İ -å Ń -åľ ĭ -Ġgr ass -ภ° -cri be -Ġqu é -ĠSp anish -Ġoffer ed -ĠбÑĭ ло -ĠCl oud -Ġve ctor -ĠH uh -Ġk ad -if ts -ĠÎ ½ -Ġhung ry -Ð ¡ -Ġpar all -AN D -ĠvÃŃde o -iz z -Ġocc up -Ġí Ķ -Ġsee k -h es -Ġdo ors -Ġhous es -Ġconsider ing -Ġgradu ate -Ġf ulf -è ¡Į -è £ -Ġext reme -Ġflow ers -it ate -ĠP ri -Ġfundament al -Ñĩ аÑģ -è¯ ´ -Ġtext ure -į ĺ -ĠAN D -à® ± -ĠT em -Ġn ada -ì§ Ħ -Ġcelebr ate -um s -Ġp ill -Ġи ли -go ing -Ġh ip -Ġsupport ed -Ġper man -Ġagre ement -Ġty m -Ġë ij -ĵ¤ ìĿ´ -Ġpurch ase -í Ķ -ĠPl an -eg en -Ġrec over -P U -ĠMic rosoft -du c -Ġhol es -Ġdro pped -Ġp ig -Ġend ing -Ġattack s -be c -Ġre n -Ġr app -Ġìļ °ë¦¬ -Ġter ror -Ġ× Ļ -Ġed it -Ġa o -. -Ġhero es -ĠB oston -Ġdepend ent -Ġmotiv ation -fl ix -Ġse am -ки е -Ġdra in -od ed -Ġgu ilty -ĠJ enn -ing en -Ġgrant ed -ĠK elly -ĠS av -ĠUn cle -ĠHon estly -EL I -Ġnavig ate -Ġbless ed -c ore -Ġear ning -Ġsign als -Ġdis k -ial s -Ġag es -æ ħ -Ġpartic le -ĠÑĩ еÑĢ -Ġcan n -Ġt ier -Ġstat ements -ê³ł ìļĶ -ĠëķĮ문 ìĹIJ -ĠCh o -Ġpol ar -an ç -ĠK enn -ĠN i -ĠF ight -or gan -é ķ -ĠCh a -ĠS ÃŃ -ãĥ ª -Ġs lic -Ġcert ific -Ġtempl ate -ĠFed eral -Ġconsider ation -Ġexpl o -ĠM ain -ĠN E -Ġalong side -Ġd ressed -ĠP oint -Ġenviron ments -Ġpró xim -Ġda ar -Ġprom pt -Ġpurs ue -Ġentertain ment -Ġth roat -Ġproblem a -Ġm art -ì ¼ -Ġprov ider -Ø Į -Ġ× Ĺ -int e -m aking -Ġstro ke -Ġtiss ue -U n -Ġpre cious -ĠAr ts -ink ing -ĠÐŀ н -Ġи Ñģ -n ah -ĠÐķ Ñģли -Ġcor ners -Ġtrick y -in ch -l ijk -Ġpress ing -le vel -AN G -Ġrad iation -ìĦ ł -Ġconf ront -Ġv et -Ġrepresent ative -Ġprop ag -Ġcra p -ĠDe c -Ġr amp -еп еÑĢÑĮ -u és -ess en -cri ption -Ġb ills -ĠMatth ew -Ġan ime -ấ t -Ġlow est -h as -sc reen -og rap -ал о -int on -ĠJ ah -èĢ ħ -it Ãł -Ġk ay -Ġrot ation -ĠW ere -abe i -Ġtri als -Ġle ver -ight y -Ġsp oon -Ġh unt -c ling -Ġdis m -ĠболÑĮ ÑĪ -Ġass ault -Ġíĺ ķ -Ġweek ly -Ġm ismo -Ġgen etic -ul pt -ĠStud ent -Ġreal istic -Ġauthent ic -æī ĵ -ast a -Ġarrest ed -Ġguid elines -Ġ×ľ× IJ -Ġд ав -ĠCom ing -f ür -Ġrequ ests -ĥ IJ -Ġanaly ze -Ġinter ess -Ġh alt -ĠO per -on om -Ġd uck -Ġwith d -s er -ĠÏ Į -ĠHist ory -Ġyout ube -ãĤ į -Ġsab er -w alk -f ont -Ġover view -3 9 -ü y -ett i -Ġfro zen -Ġf lesh -ÄŁ i -ĠP M -ĠìĻ Ģ -é ¢ -ÑĨи и -Ġê¸ °ë -íģ ¬ -Ġpr ose -oo oo -r ates -W S -Ġautom atic -Ġcollect ing -Å ij -Ġneighb ors -» . -ĠEx pl -Ġcir cul -co ver -we g -Ġstick s -Ġe ller -Ġw ww -Ġd orm -ĠEx per -Ġstat istics -Ġemail s -Ġgra ve -im iz -H S -Ġu it -, ' -Ġlas er -è ī -ĠÑĤ ем -Ñĭ ÑĪ -Ñī Ñij -Ġgen au -Ġtien en -Ġmed itation -ĠOr gan -Ġest imate -Ġë¬ ´ì -l ets -Ġn Ãły -Ġmind set -Ġres on -Ġm és -Ġnumer ous -Ġvie lleicht -ĠTh ird -u ous -ĠDe ad -ан д -H N -Ġrac ing -Ġag ents -ĠU t -Ġte ar -ĠH P -Ġchem istry -Ġsurv ival -æĸ ° -Ġconvin ced -Ġ ; -Ġreg ulations -ĠE S -åĴ Į -3 00 -Ġen se -Ġì µ -Ġd ict -G A -Ġah ÃŃ -åĭ ķ -Ġte j -Ġо ÑģÑĤ -ĠE lect -Ġintellect ual -Ġbi as -Ġbur den -çĤ ¹ -Ġìĸ´ëĸ » -Ġche er -Ġso ph -Ġportfol io -ub a -Ġest os -T V -F or -Ġas h -Ġkom mer -Ġcollect ive -Ġw rest -ĠJ etzt -ĠW at -re ich -Ġprim er -act ive -Ġm ie -ick ed -Ġhun ting -Ġtest im -Ġcompass ion -ĠØ ± -Ġbr ut -Ġsal ad -об Ñīе -Ġsol ving -Ġflo ating -ç · -Ġattract ive -ÙĪ ÙĦ -Ġper d -if fer -Ġsc ulpt -hh h -ĠWe ek -Ġent hus -Ġn ad -Ġmer ch -ĠíĻ ķ -Ġm ile -好 äºĨ -ĠÎ ¸ -ĠëĤ ĺë -éĩ į -3 8 -Ġch ains -ĠAl most -Ġtick ets -r in -ĠC C -Ġdistrib uted -abet es -Ġtemper atures -Ġg ained -Ġflex ibility -Ġscream ing -Ġab road -un o -Ġentreprene urs -ĠNet work -ĠCanad ian -Ġpre v -Ġs ö -ĠÑĤеб Ñı -ĠP oke -ĠP od -ĠTur key -çı¾ åľ¨ -Ġabst ract -Ġsn ake -ĠAm y -ĠëĬIJëĤ Į -Ġbra ve -ĠìŀĪ ìĸ´ìļĶ -ĠK al -Ġ200 7 -á rio -Ġmark ed -gin es -Ġall oc -ON G -Ġscient ist -Ġes ca -Ġrac ism -× ij× -ĠS ams -ĠP enn -Ġload s -Ġà® ¨ -ü ber -M e -ix ò -Ġper ò -an ne -Ġexp ressed -м еÑĢ -Ġmo et -Ġret urning -n ia -Ġexp on -P ro -Ġlo yal -M L -Ġl amp -Ġsh y -Ġcomp osition -ĠL y -Ġmagn etic -Ġprem ier -Ġmeasure d -Ġsumm ary -Ġattack ed -Ġfin ishing -Ð Ĺ -ç ¥ -Ġs its -Ġhyd rogen -Ġma i -ĠDeuts ch -as ı -Ġobt ain -v ie -Ġso it -Ġë° Ķ -Ġl ane -Ġconse gu -в о -Ġe ase -ak in -ĠF a -Ġunt uk -Ġbur st -Ġc um -al ım -ú blic -id i -ĠRoy al -ĠK on -Ġcommon ly -Ġremo ving -Ġj ur -il ib -Ġan ch -íĸ ī -Æ°á» £ -ĠÐľ Ñĭ -ĠAn th -ĠS Ã¥ -Ġinter rupt -Ġst ere -ĠO S -ony m -ter y -ĠMar ia -ê² ĥ -Ġexpl oring -Ġtransp arent -Ġf ate -ĠJ ung -Ġgr up -Ġdark er -ĠD oug -Ġman e -æĶ ¾ -ạ i -d ri -lo ok -ĠDes ign -Ġtut aj -Ġhorizont al -re on -ort e -ĠCor rect -ĠSte ven -Ġv ine -0 2 -i Äĩ -Ġsie mpre -ĠK ey -åĥ ı -ĠG ames -Ġna ar -Ġshock ed -el ve -ĠR ose -ìĭ ¬ -Ġstop ping -oh l -ĠM ix -Ġsuff ered -Ġsig ma -Ġweak ness -ĠO w -ี à¹Ī -I F -Ġà® ħ -ad ed -ĠNet flix -an es -Ġrem ained -ir y -Ġr ip -ell t -Ġsil ent -Ġpro ven -Ġtox ic -Ġal umin -Ġmulti pl -al and -Ġ3 4 -0 6 -ĠB ru -Ġìłķ ë§IJ -J ust -b oy -Ġsho e -Ġcreat ure -Ġhead ed -ĠоÑĤ к -æ ± -Ġess ence -Ġremark able -Ġnú mer -Ġd rew -Ġpu zzle -ĠLibr ary -ĠF u -ash es -k k -ĠI st -¦ ° -ĠB ry -Ġc eremony -Ġà® İ -Ġc ri -e qu -ãĤ ¢ -Ġpri ze -Ġdim ensions -og ram -Ġle ather -Ġpop ulations -u um -Ġve gan -Ñı д -Ġcó mo -å Ħ -Ġstri p -å £ -Ġvac ation -ħ ķ -Ġme als -ili pp -Ġ ents -ar am -ric ht -Ġgra in -ĠSp ain -Ġche ek -ĠA ff -I ON -ĠBr ing -Ġ3 8 -iel en -ul u -ĠболÑĮ ÑĪе -Ġannounce ment -ĠÑĤ ÑĥÑĤ -ĠPro phet -ard o -3 7 -Ġw oke -Ġtransl ation -ĠN OT -ĠC L -Ġd Ã¼ÅŁ -ÑĨ Ñĸ -ac er -ĠL oc -Ġper ception -N O -Ġdies en -L ook -he art -av ed -Ġbound ary -Ġfl ows -Ñij м -Ġarg uments -Ġelect ions -ı s -Ġhe ck -Ġsuit able -Ġf iber -ĠSt ra -x y -ĠH um -Ġmonth ly -u per -Ġgol f -Ġl ately -ĠG ard -ĠR en -ĠA st -ĠF ant -аÑģ Ñģ -Ġobs er -ë ¡ľ -Ġeas iest -į Ķë -Ġwebs ites -p ol -Ġco con -Ġà® ĩ -ĠV eg -Ġwalk s -Ġint ro -Ġdirect ed -ĠAn na -Ġëĵ¤ ìĸ´ -ĠEaster n -ĠS aint -ĠB ow -Ġro ast -ĠU RL -Ġjed en -ur as -aj a -Ġse mi -Ġrapid ly -Ġtarget s -ĠCont rol -Ġb ah -Ġref lection -Ġcreat ivity -hold ers -Ġìĺ ¬ë -Ġamong st -Ġfeed ing -ÑįÑĤ омÑĥ -Ġвид е -Ġë§Įë ĵ¤ -ĠSm art -Ġrel iable -Ġvez es -Ġ× ¨ -ch uckles -az ione -ĠWilliam s -Ġa ç -Ġsle e -е Ñī -Ġtim eline -Ġthor ough -á» į -ĠO t -ạ n -Ġimag ination -Ġmechan ics -r ist -Ġclaim ed -ÏĦ η -ê te -ĠHur ry -ĠiP ad -Ġconst ru -ĠC la -ĠAl s -ä¼ ļ -ut z -Ġcult ures -Ġìĸ´ëĸ» ê²Į -Ġbelong s -Ġy er -ĠDoes n -Ġge omet -Ġb id -Ġfo am -Ġh ob -ĠBrit ain -Ġsubst ance -Ġann iversary -ĠëĦ Ī -Ġnot ed -Ġgovern or -Ġstock s -3 1 -Ġdi ye -ìĬ ¤ë -Ġre b -z el -Ġmultip ly -Ġoper ator -Ħ¤ ìļĶ -Ġwat ers -Ġd är -Ġuns er -ĠEliz abeth -é« ĺ -Ġincreasing ly -ĠG ro -Ġen gines -ir s -Ø « -Ġtre asure -P C -in ction -ir i -Ġacc um -Ġvari ation -Ġp om -Ġtit les -ĠF est -ó s -Ġeld er -ny m -r un -Ñı в -Ġinnov ative -Ġnom bre -Ġco inc -Ġfr anch -Ġent onces -Ġnicht s -Ġexc lusive -ĠChe ers -ĠB i -u je -æŃ ¡ -Ġp ok -ĠP rem -Ġrock et -ELI PE -Ġhosp itals -ri um -Ġjust e -Ġham mer -Ġquant um -Ġrespons es -ll y -end i -Ġact ively -Ġfr idge -i ate -l ong -Ġqu em -Ġdeath s -Ġsuper ior -ck en -ìĿ´ì ĹIJ -kt op -Ġgather ed -£ ¨ -Ġd azu -Ġreci pes -Ġbu zz -c en -Ġany time -ons ense -Ġcirc les -Ġsol ved -Ġìĭ ł -Ġcoron avirus -ĠLu ke -Ġbu bb -Ġcont empor -r zy -ĠJ ane -Ġд ом -Ġscrew s -Ġhy brid -Ġcas ual -Ġsel bst -be ing -ĠÄ IJ -ĠCol umb -ĠÑħ оÑĩ -Ġbu cket -Ġevalu ate -Ġid ol -Ġrep utation -ĠìĨ Įë -ÙĪ ر -Ġhe cho -Ġpo em -Ġsubject s -pl ant -ĠBe h -ĠSpe aking -Ġbatter ies -Ġfollow ers -ö l -Ġg ently -Ġsi xt -Ġparam eter -Ġik ke -ĠT our -ĠD J -ot te -ĠJ ahren -Ġprepar ation -Ġд Ñĥм -Ġ8 00 -c op -ik ing -Ġë¬ ¸ -Ġн Ñĥ -Ġл еÑĤ -åIJ Į -ĠI de -Ġì¡° ê¸Ī -Ġla ughter -Ġmole cules -ĠR est -Ġobs erved -d zie -Ġadvert ising -ert o -Ġmo ins -ĠM IT -Ġexc it -Ġt um -Ġty l -Ġinvest ed -Ġph arm -Ġunex pected -Ġph i -oty pe -we ise -Ġge ç -jour d -Ġhors es -n Äħ -= " -ĠS M -Ġf ib -Ġcl ips -çķ ¶ -å¦Ĥ æŀľ -Ġreg ime -Ġrot ate -r ou -n ik -Ġarm or -ðŁ ĺ -еÑĢ а -åº ¦ -ĠO ch -Ġr ichtig -üz el -ane ously -m ek -éĮ ¯ -ĠX iao -Ġexist ed -w orth -ãģ£ ãģ¨ -Ġna ught -Ġhe iÃŁt -ĠB al -Ġres id -iv ot -om atic -Ġh ired -Ġgrad ually -Ġon ions -Ġcomp at -Ġint im -Ġj ew -Ġcontrib ution -ĠI re -ac ji -Ġsl ice -Ġimm un -ĠR us -Ġgr ows -ĠSimilar ly -Ġhard est -Ġst ruck -Ġmeasure ment -... ] -th ey -Ġìł Ģë -Ġsne ak -Ġappl ies -Ġн ем -æ ĵ -×ij ר -ĠЧ ÑĤо -Ġout ro -Ġinnoc ent -Ġm og -ĠSams ung -Ġmer cy -Ġhand ling -Ġinter vention -id ays -g ot -Ġcur ric -Ġbound aries -Ġconf using -Ŀ¼ ëĬĶ -æ ĩ -Ġstitch es -ÃŃ vel -Ġtun nel -it ä -Ġg ost -im y -Ġcz as -Ġm é -Ġcat al -ĠSim on -ĠLI AM -m ic -ĠÐ ¤ -Ġey el -is as -ĠC PU -ĠD ou -Ġnä ch -Ġinfin ity -Ġr if -ĠPe ace -ĠC u -Ġminim al -Ġlisten ed -Ġpo le -hal b -Ġload ed -Ġste ady -ĠBes ides -ê m -Ġl ap -Ġco op -Ġfriends hip -w orld -Ġge h -Ġtyl ko -ĠLa ura -Ġsurround ed -ĠE vent -Ġch ap -ĠW onder -bre ak -Ġdro ve -Ġbroad er -Ġch i -F i -Ġge hen -Ġwest ern -Ġintellig ent -Ġpers ist -Ġfound ed -ãģĵ ãģ¨ -Ġhistor ic -Ġfr Ã¥ -cks Ã¥ -Ġhand y -Ġsy mp -Ġr ows -Ġnut ri -b ur -ĠLe on -Ġsist ema -Ġext ensive -ĠÑĥ в -í ı -Ġnight s -Ġcá c -Ġcount ing -ĠM ust -all ow -еÑģ Ñģ -M om -Ġнад о -Ġbar rel -ãĥ ŀ -AR D -Ġinstall ation -Ġin sect -Ġëħ ¸ë -uj Äħ -ĠÄij i -Ġpack ed -Ġf iction -N ow -ĠY ay -Ġper t -r ons -und e -ach es -Ġsty les -Ġapr ès -ok u -ĠV ice -ın ız -com m -Ġassign ed -Ġinteract ions -Ġac ab -F ELIPE -Ġresc ue -Ġindust ries -ĠAnd y -Ġpra ise -Ġfl ame -Ġsn ack -í Ĥ -ç ģ -Ġsw o -rend er -Ġbo ards -ĠÑĤ ом -en ne -Ġpast a -Ġdev il -ĠF el -Ġhat te -Ġcoll eg -e h -ì » -ãģĵ ãģ® -Ġproduct ive -for ward -и п -Ġsmart phone -Ġinv is -Ġb um -Ġwho a -ìŀ Ħ -Ġo cksÃ¥ -ĠL ang -ĠSy ria -Ġses i -ί α -Ġappro val -4 8 -Ġод ин -Ġë ĸ -ĠH arr -ĠAd minist -Ġ× ¤ -ĠDe an -f i -Ġcitiz en -Ġsh ark -0 5 -Ġbo il -Ġindic ate -å ¡ -A re -Ġlay out -Ġref r -ĠPac ific -AA AA -ĠAustral ian -g ression -V oice -ал ÑģÑı -Ġshel ter -T o -au pt -Ġevalu ation -ap or -Ġcur rency -Ġм ного -ig os -ãģ ° -Ġo ct -Ġro yal -è ³ -as il -ĠChild ren -Ġr ien -Ġë ĵľë -Ġbar rier -Ġej emplo -Ġe k -N D -es p -ен а -Ġp ic -Ġkill er -Ġintegr ate -Ġfew er -Ġdis abilities -Ġ .... -Ġtri angle -Ġfe es -Ġwid ely -em i -Ġoverwhel ming -Ġz omb -Ġb ere -Ġho od -ĠA ye -ĠHar vard -e v -ĠÏĦ οÏħ -Ġcup s -ĠA uch -z ona -Ġ199 0 -Ġwe iÃŁ -Ġcr unch -æ ¥ -Ġз ав -Ġmeas uring -Ġst ations -ĠStep hen -Ġshort ly -Ġsig ning -Ġcom edy -om o -Ġsuggest ions -Ġsign ature -ĠпÑĢ ив -Ġdis order -as ka -Ġworld s -Ġprecis ely -n orm -ra v -ĠC ivil -In ter -ĠC ertain -Ġinj ured -Ġsuggest s -ĠGold en -Ġcy ber -ĠØ ´ -Ġtempor ary -Ġco oper -Ġvot ed -Ġ ought -ấ y -x ual -Ġpan els -Ġ9 5 -Ġhands ome -ĠпÑĢ ов -Ġper mit -Ġke in -Ġbad ly -Ġnot ifications -iz a -ĠNot ice -Ġinc lusive -Ġanswer ing -Ġí Ĺ -u ld -íħ Į -Ġnow adays -Ġ3 7 -Ġb olt -Ġstat ic -ĠH op -Ġav ant -aj o -Ġ맼 ìŀĪ -Ġfif ty -ĠF inal -Ġsc ores -ĠT ap -Ġcy l -Ġconv ince -Ġany ways -od a -Ġìķ ¼ -Ġser ves -ĠÑĤак ой -ĠZo om -Ġsaving s -ul o -Ġs outhern -view er -Ġho je -Ġse ja -Ġrepresent ing -Īë įĺ -l ik -ĠSome body -Ġbe ast -Ġstick ing -Ġins ist -Ġtal ented -Ġexplain ing -Ġatt orney -éĥ ¨ -Ġst airs -ĠD og -í ĭ -Ġc ig -Ġshap ed -Ġs ons -Ïģ ι -ut t -Ġì Ķ -Ġpar ad -ìĿ¸ë į° -Ġh orn -ĠJ our -ann o -Ġworld wide -åĬ Ľ -Ġparticip ation -¦ Ħ -Ġm ów -Ġburn ed -Ġwrit ers -all ah -ĠF und -Ġcle ver -ĠLe ute -b in -Ġbe ating -f oot -ĠìĽ IJ -ĠStud io -Ġv ag -be y -r ze -Ġoppos ition -Ġж из -w ho -Ġê± ´ -Ġtr ace -Ġд енÑĮ -Ġep id -Ġges ch -ĠN ar -ĠB E -Ñĥ й -ĠS ign -ed ly -Ġcl ay -Ġinst antly -Ġgather ing -ĠGal axy -Ġb ored -ĠBudd h -c é -Ġm am -Ġsl ope -Ġëĭ¤ ìĿĮ -Ġsch ön -Ġp ir -ge f -am er -Ġh ö -Ġcolle ague -Ġpres ents -ad ium -Ġà® µ -Ġfal ar -be ep -Ġdri ed -ism s -Ġro pe -Ġworks hop -Ġest ud -Ġb ands -Ġthem es -åħ ¬ -ÙĬ ر -åIJ İ -Ġremind er -ÑĤ Ñĥ -ĠB h -Ġcocon ut -ĠÑģ ÑĤо -ĠCh annel -Ġimmig ration -ä s -.. ... -ä¸ » -çĻ ½ -st op -Ġк аÑĢ -Ġco ins -ĠÑĩ аÑģ -Ġdest ruction -l ined -Ġbar riers -ant ine -Ġprint ed -Ġcongrat ulations -ĠHe art -Ġin qu -th a -Ġhard ly -ĠA ven -Ġt inha -ĠS ony -ĠN F -Ġgradu ates -Ġsque eze -ere my -ÏĦ ι -Ġep ic -ĠJ u -Ġol m -ĠLa ughter -Ġbelief s -ĠC ru -ĠTr ue -ĠS oul -owe en -Ġrom antic -Ġз в -Ġan os -ĠY up -éĺ ¿ -d im -Ġin fer -Ġз ам -Ġso c -uk a -Ġprec ise -Ġdro pping -Ġcl ue -Ġer rors -char ge -ĠP u -omet er -Ġlamb da -ac ional -ĠD ong -Ġcham ber -Ġthank ful -ĠN u -ĠHaw ai -Ġinf o -Ġactiv ate -ĠQ ual -Ġqu ed -Ñĥ лÑĮ -Ġcl oth -åĸ ľ -Ġw ichtig -5 5 -Ġot ra -ograp her -Ġcur ios -Ġ19 80 -Ġemp res -d ess -e ur -Ġcl uster -ar ter -ob ile -ĠY an -ĠAd v -Ġdiscipl ine -Ġìłķ ëıĦ -ĠPl ace -ĠSe lect -T E -ĠбÑĭ ла -Ġwh is -Ġb ay -ĠD or -en cing -Ġrep et -Ġf icar -p ad -Ġf og -u yor -Ġsn ap -ib t -Ġso bie -Ġappoint ment -ĠR y -Ġce iling -our se -Ġwr ites -ĠAfghan istan -Ġm os -az e -Ġpen al -Ġcry stal -IC E -ê° IJ -é Ł -ĠTes la -Ġthe ories -Ġappe al -Ġnewsp aper -Ġcook ies -æ © -ĠاÙĦ ÙĦ -Ġma j -ĠGet ting -k ommen -ĠHe aven -ell s -Ġdiv ine -Ä « -Ġa kt -Ġhop es -ĠCh en -we gen -** * -ĠFra ge -Ġн и -ภ¹ -min ister -nes ota -wh ich -Ġexpl icit -Ġverd ad -Ġgradu ated -ĠPh ilipp -Q L -ĠM I -Ġdev ot -Ġc ure -Ġclos est -Ġà Ħ -Ġsex y -ãģ Ľ -ĠDe ath -ok o -ug u -ĠAn ne -itar ian -es a -ег од -ĠD ur -Ġ 000 -ze it -Ġtour nament -Ġmel hor -ภª -Ġin du -Ġf law -Ġw ars -ĠM ind -ĠI ron -ÑĤ ак -ĠV R -Ġs iz -ĠS outhern -Ġê·¸ëŁ ¬ë -Ġaw ak -Ġìķ ŀ -Ġc ube -believ able -if all -d is -Ġabandon ed -m ind -Ġpar l -Ġclass ical -è ĭ -á»Ļ t -ĠAut o -ĠB or -ç © -4 00 -ĠSoci ety -Ġsubt le -Ġmiss ions -Ġremember ed -ĠE ither -Ġda für -OR D -Ġint ensity -ES IN -ĠC up -Ġrare ly -Ġto ys -ĠChar lie -á» Ł -Ġgla ube -Ġround s -T IN -Ġcap ability -Ġderiv ative -Ġrefer ring -Ġd Ã¥ -ĠT ALI -Ġcott on -Ġcon fer -Ġcolum ns -Ġliber al -Ġnun ca -Ġμ ε -Ġind o -ib en -ĠBe ispiel -Ġê·¸ë łĩ -ĠÑĥ Ñĩ -Ġh oy -Ġfr y -ĠScott ish -è Ĭ -Ġc iv -Ġconserv ative -Ġair pl -Ġs ar -r us -Ġinvest ments -Ġinfin ite -Ġà® ķ -ĠTALI ESIN -ĠG ary -ue ll -Ġа к -ĠC ir -Ġrit ual -Ġ>> > -Ġtem pt -ĠTe ch -ĠPoke mon -Ġimprove ments -Ġsp are -Ġtransl ate -Ġson ra -ĠFil m -w ort -Ġм и -Ġperiod s -Ġje alous -ãģĦ ãģĦ -Ġt ir -M I -Ġconduct ed -ĠìķĪë ħķ -0 9 -ĠPol it -ĠWhere as -Ġmoist ure -Ġs ins -Ġk ap -ĠÑį к -Ġben im -Ġelimin ate -Ġathlet es -ĠMan ager -Ġfeature d -ap ore -äº Ľ -Ġë° ľ -Ġper f -ĠTh us -Ġdeb ut -об ÑĢ -Ġse ñ -Ġmyster ious -w ords -Ķ ê°Ģ -Ġcheck s -Ġvolunte er -Ġwas hing -ĠMar vel -ĠA B -iss ors -! ' -ĠF ull -ye on -Ġwe igh -ĠJO HN -Ġv os -Ġproced ures -Ġaddress ed -ĠBer lin -put er -ĠB an -Ġmedic ation -Ġdr one -ĠÑĥ б -ĠJe an -Ġcap s -Ġdisappoint ed -Ġw ore -Ġêµ Ń -Ġorgan ize -ĠHall oween -Ġfant asy -y ard -Ġnos otros -Ġjump ed -Ġphot ography -ĠN ame -re c -A B -Ġbless ing -ĠSh ut -Ġbit ter -p op -ãģĿ ãĤĮ -Ġde i -Ġfulf ill -çIJ Ĩ -Ġden gan -Ġbe lo -ĠMean while -Ġdep ois -Ġdi abetes -Ġbu nd -ĠZe aland -Ġdig est -Ġt ires -Ġdo d -ag ne -ế t -Ġpe el -Ġз аб -Ġn odes -Ġtrend s -ĠSw itch -ĠA ward -ĠOr ig -ĠH al -Ġest as -Ġ3 60 -Ġsim ult -Ġcom ic -Ġm Ãł -Ġbal anced -ĠPrin cess -Ġkilomet ers -á» © -Ġpart ir -ì¤ ij -so ft -ĠV iew -Ġbi ological -in st -4 4 -Ġman era -Ġcompreh ensive -ĠS ab -Ġcr imes -y ers -ĠComp any -ĠPh ot -Ġpou co -i ac -Ġbe im -in ate -Ġsub sequ -ĠMay or -Ġcent uries -è res -ìŀĸ ìķĦìļĶ -Ġê·¸ëŁ ¼ -ĠFra u -ĠO H -Ġëģ Ŀ -ĠN ah -ĠSer ies -Ġover night -íĴ Ī -ĠâĢ ¢ -Ġtra ve -atter ed -Ġwar ri -ĠGru nd -ĠInd ones -Ġsc ra -ob y -ĠBro ok -Ġcur s -Ġë ¸ -Ġexpl ains -ram atic -Ġparticip ating -Ġmin ut -Ġcontract s -Ġg egen -Ġdisappe ared -ĠS N -Ġrob ust -ap h -Ġsh rim -Ġdev ast -c ope -Ġme ets -Ġpeace ful -m ate -Ġwe ld -Ġ× ª -d on -Ñĥ ÑĤÑĮ -Ġregister ed -ĠN ik -j in -Ġc av -Ġe cht -io x -Ġflow ing -но ÑģÑĤи -Ġto e -Ġent ity -ов а -f its -ĠPat rick -ÑĤ ÑĢ -Ġle verage -Ġcor rel -i ah -Ġstr ings -ist inct -Ġg ue -arch y -Ġteng o -ım ız -Ġor bit -ä¸ º -Ġе ÑīÑij -ca ke -Ġ׾ ×Ķ -ĠMin nesota -Ġbra ke -ow ie -Ġcra w -ê¸°ë ¥¼ -Ġprogram me -ĠÑģл ÑĥÑĩ -åı ª -ien ces -ĠO ui -ĠP ers -im iento -ĠIn vest -Ġsl ower -æĻĤ åĢĻ -ĠB eth -Ġnur se -ĠSpr ing -S p -Ġun employ -д и -Ġgen ius -ĠA aron -Ġê·¸ëŁ ¬ -Ġe i -ãģĹ ãĤĩ -Ġtank s -Ġau jourd -Ġcomplex ity -ĠÑĢ еÑĪ -Ġold est -Ġlet z -åħ ¥ -Ġphenomen on -pr int -ĠBund es -it at -ê» ĺ -Ġ4 2 -ĠW i -Ġinc om -Ġg ek -Ġembr ace -Ġt ies -out e -Ġd ose -ĠF riends -Ñĭ ÑĤ -егод нÑı -Ġor g -Ħë ¡ľ -ó g -Ġex ceed -Ġgod s -Ġê±° ìĺĪìļĶ -Ġsoci et -ĠUn ivers -it ät -Ġword en -Ġsm oking -Ġint ens -ab ul -em ia -è ij -4 7 -f ly -Ġ200 6 -ĠSer iously -Ġprze z -æ ¼ -c re -Ġn an -Ġmod es -ов аÑĤÑĮ -ĠH ang -em en -Ġbenefic ial -Ġvot ers -ĠBro ad -Ġb ent -W ow -Ġm ul -åĵ ¥ -ĠU C -Ġdam aged -ĠUk raine -Ġw ipe -Ġst ones -Ġman agers -Ġr ab -ÑģÑĤÑĢ о -l at -Ġde ce -Ġgraph ic -Ġf oss -Ġdisag ree -ĠAm en -Ġsec rets -ho le -ink le -Ġfortun ate -Ġì ± -ìľ Ħ -èIJ ¬ -Ġhab its -Ġbur ied -Ġh in -Ġvirt ually -ol as -ĠR P -ĠT ab -l ow -Ġsacr ific -Ġestim ated -ol n -Ù ĭ -c ur -ĠFe el -Ġcast le -Ġus eless -Ġdis g -ĠJac ob -Ġga an -Ġup side -Ġpare ce -ãĥ³ ãĥ -Ġsh ipping -ĠC R -Ġdis rupt -ac ter -UN D -f u -å® Į -ĠP ick -ĠChar l -ĠB ull -Ġenter prise -Ġpunish ment -ack ing -Ġfr action -Ġtab let -Ġch ord -Ġsimilar ly -åħ¶ 實 -ĠTor onto -Ġcour ts -ÄŁ l -esz cze -Ġpron oun -ĠS ister -ĠM P -Ġgreat ly -ĠD ank -ic op -Ġgar bage -Ġresol ve -ĠS af -ĠG un -Ġcomp ound -Ġë° ° -ĠMus ik -âĻ « -Ġcha os -ĠWhen ever -Ġe uros -Ġor chest -Ġrefr iger -al an -ภ· -ĠAm azing -Ġp ud -ag an -Ġj eszcze -is y -Ġaccur acy -ĠA ma -is ode -ë ĮĢ -Ġinterpret ation -ĠL iber -æ · -c am -Ġevol ved -ĠK ay -ÑĨ Ñĭ -Ġcreat or -it as -Ġal arm -Ġcelebr ation -z ent -Ġfun cion -Ġo v -umb ling -Ġ % -ภĪ -Ġrestrict ions -Ġн ав -ĠK inder -Ġban ana -ÑĮ Ñı -Ġdiam eter -Ġnor thern -ur ers -ĠP as -æĪij çļĦ -Ġwork force -Ġj ung -Ġguar ante -Ġequ ilib -Ġsu ite -Ġeu ro -Ġdel iber -S te -Ġdownt own -Ġch in -Ġc odes -ed ia -Ġshe ep -res hold -wn ie -ó b -Ġunder lying -l ia -j er -ÏĢ ÏĮ -ç Ŀ -th rop -Ġz ap -Ġvac uum -ĠH ab -Ġwra pped -ì ¢ -Ġinvent ory -м а -Ġco ord -Ġpl ates -Ġsy mm -T e -ĠwÅĤa ÅĽnie -Ġreach es -Ġlon ely -S cript -le e -ess er -Ġê± ¸ -ĠGes ch -ĠMo ving -Ġré p -ĠV ill -åIJ Ī -ĠR achel -Ġtem os -ON E -Ġstra in -Ġang el -Ġf Ã¥ -T r -Ġach o -Ġhighlight s -ĠW er -ĠCar l -Ġbl ur -Ġreg ards - · -ил ÑģÑı -Ġrec re -ĠY ani -U CK -ł ¸ -Ġelectr ons -ĠSp iel -Ġv ed -Ú ¾ -Ġbe am -Ġid iot -ë ĵ¤ -на Ñĩ -id d -Ġsk i -it ative -Ġhyp othes -ãģ§ãģĻ ãģŃ -ent er -ĠìķĦëĭĪ ë -Ġih re -Ġpre view -ang el -Ġdem on -Ġd us -Ġd ic -ĠK om -LE Y -... ! -Ġsie ht -ĠSon ic -Ġten ho -an as -Ġdig it -ĠMa ar -Ġunder grad -oun cer -uff y -Ġconvers ion -Ġdis connect -Ġe cho -om er -Ġcurric ulum -Ġper ché -Ġw and -.. ? -Ġroll ed -Ġentreprene ur -Ġtheore t -ĠÑī о -Ġins ights -Ġzus ammen -o in -ret t -p rodu -Ġvisit ors -e ous -Ġgrand mother -Ġhum or -Ġн иÑħ -zen ia -ins on -Ġres et -Ġbase ball -Ġmatch ing -ëĭ¤ ê°Ģ -Ġpun to -ì ¡ -Ġre de -Ġaddress ing -Ġfore cast -ĠB ol -Ġcol ored -Ġdocument ation -Ġexpect ation -ĠNor thern -Ġcre o -Ġà® ļ -f on -Ġuns ere -U M -Ġcop ies -Ġexpand ed -Ġveter ans -ĠAl m -Ġво обÑīе -Ġpsych ological -Ġnos so -Ġpay ments -im eters -Ġ-- > -ĠJenn ifer -Ġvolunte ers -os se -or ious -ĠбÑĭ ли -è Ĥ -ĠEs s -w s -ĠB C -ĠI C -W oman -Ġv ont -Ġeth nic -EN N -им о -Ġlo b -Ġou i -c s -Ġre he -Ġìł ģ -Ġch ick -ús ica -Ġk ont -ĠDist rict -Ġp ile -Ġа в -ей ÑģÑĤв -Ġ £ -Ġiss ued -Ġком п -Ġpros per -Ġprof ound -ĠDe ar -Ġãģ ĵ -Ġfund ed -Ġb isa -ŀ ĺë -× Ł -ĠìĿ ĺ -Ġtw elve -ĠChamp ions -éĿŀ 常 -Ñģ л -Ġ200 5 -p m -Ġon de -Ġdiff é -ĠCh all -Ġdifficult ies -Ġgar age -Ġd á -ün k -Ġë¬ ¼ -Ġtr an -Ġsubm itted -z w -ÙĪ ا -Ġar k -ĠìĦ ± -Ġgrocer y -он а -i ere -Ġa est -Ġexhib ition -Ġr és -Ġconsist ency -Ġcook ie -н ей -Ġrepl acement -æ² ¹ -ĠS em -ĠìĤ¬ ìļ© -8 00 -Ġgen es -Ġtrans action -ĠE L -Ġdur ante -ib les -ĠE at -t ail -iss ance -Ġto ss -Ġsurv ived -Ġoff ices -Ġsupport ive -Wh ere -Ġtout es -Ġë§ ī -Ġj okes -ier on -ap ers -Ġm ature -ĠM arsh -Ġs ido -k ind -Ġreal mente -ĠChe f -Ġquel que -Ġjud ges -e ft -ER S -Ġj et -Ġpers ons -è » -iz ations -ri k -Ġsh ops -ĠW y -Ġele g -qu è -qu oi -Ġjug a -Ġíķľë ²Ī -ĠQuest ion -ĠGlo bal -Ġìķ½ ê°Ħ -ĠSt ation -æİ ¥ -ĠOh io -Ġstick y -Ġst ressed -Ġg ün -Ġí Ŀ -ÑģÑĤ Ñĥп -é ¡Į -ĠPh D -im mer -Ġment or -Ġinv ented -Ġre un -Ġine vit -Ġpol ÃŃt -Ġexec ute -ĠSt ory -Ġout standing -Ġgu er -ĠR ain -Ġch oses -ĠT it -ĠÑģ еÑĢ -ĠSing apore -ĠN one -Ġch ronic -°ë į° -Ġe go -æł · -ES T -ãģĤ ãĤĬ -ĠW ang -ĠN AT -Ġa ug -Ġdes ktop -Ġetern al -ĠìĤ¬ ìĭ¤ -ĠConst itution -ìĤ ¬ë -×Ļ× ľ -p res -ĠТ Ñĭ -Ġinter f -Ġlist s -Ġfight s -ft en -ĠI owa -Ġmotiv ated -ĠH osp -Ġelse where -Ġpath s -Ġinst ances -B l -r ange -á» ± -ĠS it -man a -Ġìĭľ ìŀij -Ġm ình -ans as -Ġs na -Ġphilos oph -Ġpas se -Æ°á» Ŀi -ak h -ent al -Ġih n -ru ctor -Ġв аÑĪ -Ġgener ous -Ġp ivot -п ол -Ġjam ais -Ġcom ent -ĠL ew -od zi -ĠX box -Ġв од -Ġcons ent -ī ìŀ¥ -Ġdis par -l ass -ĠGovern or -Be ifall -Ġê° ľ -Ġbelo ved -׳ ×ķ -se ll -Ġhon ored -le h -Ġw äre -un ting -Ġfra ud -ĠR AM -ê± ¸ -Ġkill s -Ġeconom ics -0 4 -п еÑĢ -Ġco isas -Ġи гÑĢ -ÃŃ m -Ġmö chte -Ġìµ ľ -Ġstim ul -Ġfast est -l v -Ġg én -ĠS ounds -Ġ19 70 -Ġhome work -spe aking -Ġencour aging -Ġqu ery -Ġre vers -pro fit -Ġd y -Ġìŀ ij -ëĬĶëį° ìļĶ -Ġso ap -ĠG all -ĠC N -ĠAn s -Ġf ic -ank s -Ġdess ert -ĠìłĢ íĿ¬ -ĠM aking -Ġcome ç -ê³ Ħ -Ġassoci ation -D ad -he e -Ġh ogy -Ġap ro -Ġinvis ible -Americ an -í İ -Ġvi be -Ġem issions -Ġadvoc ate -Ġkick ed -Ġ vel -Ġsum mar -Ġfre aking -ch ron -Ġpin ch -Ġwszyst k -isc al -Ġpro ved -Ġmind ful -Ġt ä -Ġno ises -Ġisol ated -Ġcross ed -Ġê° ķ -Ġvo ilÃł -Ġch ore -ĠR A -C om -Ġrelax ed -at ro -Ġpre vention -Voice over -O D -ĠCo vid -Ġsepar ation -Ġ- [ -иÑĩ его -çĻ ¼ -ĠS D -ble ep -Ġindepend ence -Ġpart ial -Ġalgorith ms -ĠAny one -Ġassoci ate -h um -ic ular -Ġb ạn -Ġbatt les -G ood -App lause -Ġbast ante -Ġadv ant -ĠS weet -Ġref used -ãĤ ¸ -ĠÑĤеб е -pl et -Ġencour aged -åĵ ¦ -Ġmir acle -ĠB un -ĠV ar -rim ination -e lect -ĠM ult -Ġdeliver ing -e ing -Ġc m -ne hmen -ĠL ine -Ġë§ Į -en ced -ĠS ound -ĠCont in -ij d -UN G -k le -Ġth reshold -Ġcomp act -ad t -Ġto es -ĠP ur -own ed -ment ed -Ġdes igning -Ġvacc inated -Ġexha ust -Ġbas ics -Ġcons ists -ĠGu y -ac zy -Ġm ÃŃ -w on -å® ³ -Ġ8 5 -æ Ĥ -Ġm um -Ġign or -Ġprint ing -ac ular -p ow -Ġexpand ing -Ġg ir -ĠC ab -íĺ ¸ -ÑĤÑĮ ÑģÑı -ĠìĹ¬ëŁ¬ë ¶Ħ -Ġang les -Ġterm inal -ĠW on -ĠInter esting -Ġcross ing -Ġbond s -Ġpu eden -Ġor b -lar ın -Ġcreep y -Ġnutr ition -Ġall ies -Ġwire less -Ġdes ired -Ġcomp ute -ĠAri zona -ĠBeaut iful -Ġprodu ces -Ġnuest ro -t ed -Ġel igible -ĠÑģ оз -ic ial -ĠH ero -Ġcons ume -Ġrob ots -Ġpurch ased -c ción -Ġ iz -ượ c -ίν αι -ĠØ£ ÙĨ -Ġshad ows -ĠMed ia -Ġprin cess -Ġk lar -Ġwood en -Ġus ar -Ġg üzel -Ġsl ot -r ade -Ġë Ĵ -Ġhar mon -Ġingred ient -ors hip -ek i -Ġgrand father -Ġexcit ement -Ġpolit icians -.. ! -Ġout s -Ġsepar ately -ĠÑı к -ĠW elt -ĠP ow -j an -Ġorient ation -åı ĭ -L C -age m -ÛĮ Úº -åIJ Ĺ -Ġbran ches -ad en -rent e -ĠI hr -as m -Ġest ão -ĠN ic -Ġsla ve -Ġcomp ress -c rowd -Ġclim bing -ĠMan agement -ĠB ah -Ġpan ic -Ġk or -Ġcool ing -Ġb ind -Ġз ад -Ġr ack -Ġent it -Ġs ends -Ġyour selves -d es -ĠMuslim s -Ġí ļ -ism a -cy cle -un kt -ĠC ore -Ġinj uries -Ġident ical -ка Ñı -ĠDeutsch land -Ġе е -is an -Ġtr uc -let on -Ġback up -Ġult ra -Ġab und -ille urs -Ġby ÅĤo -åħ ĥ -ort ed -Ġearth qu -Ġк л -Ġobs ervation -Ġmainten ant -el en -Ġsett led -Ġp ela -ĠE conom -Ġ Õ -Ġste ering -ĠAL L -ĠC her -Ġpat ience -ĠS now -Ġb or -Ġworth y -Ġcá i -Ġ× § -Ġκ α -d og -ĠK aren -ill es -Î ² -Ġagric ulture -×ķ× Ł -ĠSe an -Ġsens ors -íķ ´ë -ag h -Ġpublic ly -Ġpe ux -ĠAlex ander -Ġprior it -Ġla zy -ard on -atter ing -Ġcost ume -س ت -è¿ ĺ -Ġun w -Ð Ľ -Ġthick ness -qu ito -g unt -ist as -ne ys -ĠëIJĺ ê²Į -ĠBr asil -Ġto ken -Ġaff ili -l on -Ġf Ã¥r -ĠBe ach -Ġw itch -ĠSe ven -Ġp ant -λ λ -Ġcapt ain -å Ŀ -Ġve ut -Ġpou voir -ac z -ĠBar b -Ġut ility -Ġcontempor ary -Ġobt ained -Ġpainting s -e ar -Ġpe an -ĠO g -Ġc ust -л ем -Ĥ ĺë -ĠIs so -Ġac onte -ĠTe le -ĠAss istant -à ī -íĸĪ ìĬµëĭĪëĭ¤ -Ġcount s -Ġbu ck -ĠDe ep -Ġtack le -Ġh arsh -Ġdec ides -éĹ ľ -. âĢĭ -éĤ Ĭ -ĠAng el -Ġlay ing -Ġcal ories -Ġcontro lling -Ġadvant ages -ĠÑįÑĤ ой -Ġappro aching -Ġthreat s -ak an -em atic -m ann -ê³ µ -m umbles -ac ió -Ġmaint aining -Ġfound er -l ah -f ight -Ġadm itted -âĢ¦ . -ķ Į -ab ol -Ġus age -Ġn onsense -ĠPal est -Ġcont re -ĠDemocr atic -ĠE R -j ekt -Ġar bit -Ġг ол -ĠMich elle -ich er -es h -ĠP ho -к ом -4 9 -ĠEner gy -ο Ïį -Ġc ents -Ġref ers -Ġg ospel -ĠSh a -ĠSh are -×Ļ× ł -Ġclin ic -ĠëĦ £ -Ġequ ality -ug s -Ġsh ed -Ġplan es -Ġtout e -re ck -Ġstra nd -Ġbi ology -Ġle ague -ĠP ok -Ġnúmer o -ĠCo ast -Ġconsist ently -Ġnuc le -OO OO -Ġob jet -Ġch or -Ġg inger -Ġd abei -Ġcoop eration -à¯į . -nt en -ç ¤ -l Ãł -ìĸ ij -r ado -Ġpass ive -Ġglo ves -Ġunder ground -Ġlog ical -Ġk et -Ġfunction ality -¸ë ¦¬ -Ġport al -ell er -×Ļ× ¨ -ĠT ed -ĠG re -IJ ľ -Ġperson nel -Ġemer ging -ĠF ür -Ġmeant ime -usal em -ĠC lear -Ġtra pped -Ġìļ ° -Ġdis pl -Ġmet tre -Ġmun icip -Ġwithd raw -Ġsp at -un es -Ġaccess ibility -æĪij 们 -Ġap are -Ġpros pect -Ġн аз -Ġcop per -ĠP RO -Ïħ ÏĦ -Ġattack ing -ĠV in -ĠSt one -Ġinvestig ate -st yle -ĠÎ » -ë ¡Ŀ -ë§ Ī -Ġins pect -Ġli ver -ал иÑģÑĮ -Ġser a -hal ten -em an -Ġmin istry -' ' -Ġd ots -ãħĭãħĭ ãħĭãħĭ -Ñĥ ÑģÑĤ -ĠJ ak -AK E -Ġg aps -uck er -ĠинÑĤеÑĢ еÑģ -ĠEm ily -Ġinter val -Ġt ender -ĠTechn ology -g ame -Ġtri b -ÙĦ ا -ĠDevelop ment -Ùħ ا -Ġwr ist -Ġf ires -Ġtarget ed -ìł IJ -Ġso d -íļ Į -Ġoldu ÄŁ -Ġse asons -vent ions -Ġн его -Ġsomet ime -ли в -n é -Ġt ú -ĠDe us -Ġexec ution -á p -ĠCh ange -ĠInd eed -Ġreg ulation -ĠH ung -é is -Ġwish es -Ġj azz -Ġstruct ural -Ġblow ing -Ġby Äĩ -Ġtherm al -ph ant -ÑĢÑĥ з -ан ÑĤ -ĠP ull -Ġconf usion -нÑĭ ми -Ġscen arios -ìłģ ìľ¼ë¡ľ -Ġд еÑĤ -Ġtatto o -Ġaut re -Ġhe ating -Ġtreat ing -Ġпон им -Ġexc lus -ĠL OL -we ar -ag le -Ġzur ück -Ġr ational -s u -Ġdet er -ĠN ative -à®ķ ள -ach ed -Ġ ãĥ -ĠEnt onces -Ġhor a -ìĿ´ìĹIJ ìļĶ -Ġl ite -à « -Ġsix th -Ġбол ее -act or -Ġpsych ology -çĽ ¸ -Ġdem ands -Ġpe er -Ġnew ly -ĠWW E -Don ald -ĠBo x -Ġp ine -Ġload ing -ĠN ico -Ġs ÅĤ -omm e -AR T -Ġrecru it -Ġbug s -arent s -ĠпÑĢ об -ĠIn side -ipp er -d ramatic -Ġplan ets -ord e -Ġy oga -ch ild -ĠMar ie -Ġãģ Ĥ -ĠB L -Ġfil med -Ġref resh -Ġtomato es -Ġf et -Qu é -Ġ !! -ĠëĤ ´ë -r ine -Ġinteract ive -s al -ann ah -pe z -ç¶ ĵ -Ġunderstand s -ĠTok yo -Ġlibr aries -Ġread er -ij IJ -o z -ĠEnd e -ĠF lo -Ġm ild -Ġpo etry -Ġж ив -æĦ Ľ -Ġbeh ave -Ġdo en -ĠSus an -p age -ra ham -Ġcommunic ations -Ġtun ing -Ġp ac -Ġanx ious -I O -M ark -Ġhi ç -book s -Ġp iss -Ġen abled -achel or -ĠF OR -Ġé c -ĠT R -il st -h at -ĠìĿ Į -Ġty ch -Ġj ar -Ġbuild s -ĠAr gent -Ġinter medi -Ġl ou -Ġa ra -Ġassign ment -Ġcabin et -Ġretire ment -ãģ » -Ġdis abled -ric a -Ġa wards -Ġbo ots -Ġacknow led -Ġth y -Ġêµ ¬ -Ġsy nd -ни й -il ton -Ġprob l -ĠF al -Ġverd ade -Ġ7 00 -ĠLe arning -oc us -Ġpal ace -N ot -t ain -c m -Ġmagn et -inc oln -Ġfig uring -ĠL yn -ĠB oss -ĠV O -Ġdiagn osis -Ġequ ipped -w atch -in os -ad ers -Ġsh elf -Ġorgan is -Ġn od -Ġk ız -pp ers -Ġrest ore -Ġart ic -ĠVo ice -ı yorum -ê² © -Ġspread ing -Ġh ips -Ġw ard -ure au -Ġinter section -6 6 -Ġ3 9 -ç ³ -Ġwait ed -ì ´ -hh hh -Ġd ys -ĠE N -Ġb atch -Ġca f -Ġmark er -大家 好 -or able -ó ria -Ġste pped -Ġcelebr ating -ан а -Ġwor n -ĠF ol -Ġpl a -Ġattempt s -Ġtwe et -Ġr ust -g ence -í Ĩµ -Ġre vel -Ġre cept -en ess -Ġ( ( -ãĥ¼ ãĥ -! âĢĭ -ĠìĨ IJ -Ġinfluen ced -и ж -Ġкон еÑĩно -Ġcolleg es -ion i -Ġs ag -An n -ol ar -Ġexpress ions -Ġsu its -Ġowners hip -el and -pie ce -æĢİ ä¹Ī -Ġdesp ués -Ġt el -Ġins ult -Ġêµ īìŀ¥ -ĠSm all -ĠF R -ok a -ber ries -ĠAnt on -ел Ñı -Ñı Ñģ -Ġval ve -act s -Ġwood s -à® £ -Ġcult iv -Ġf á -ãģ¨ ãģĦãģĨ -Ġche ers -Ġassum ption -Ġfit ness -ÃŃ cul -Ġpod r -Ġwe it -ĠH ind -Ġd ign -Ġз н -Ġsqu ad -Ġdest ro -c ere -sh irt -imm t -eng ers -Ġs ä -k ÅĤad -Ġ ÈĻ -Ġocc as -Ġì¤ Ħ -Ġprocess or -ĠD M -ĠDad dy -Ġsoon er -Ġstraight forward -Ġdepart ments -ĠChr ome -Ġwork place -ĠPy thon -Ġm eng -ĠD AN -ĠI ce -ĠëĪ Ī -ĠG i -Ġh iring -Ġland ed -Ġdemocr atic -ied z -ãģĺ ãĤĥ -Ġse v -ic ia -Ġespe cial -ĠN ous -Ġh ät -Ġb ou -per t -ies z -åij Ģ -Ġv il -ÅĽ li -Ġî n -Ġloss es -éķ · -Ġto ast -Ġreal m -ĠAust in -ĠIn formation -Ġres ume -Ġch ase -Ġsal ary -Ġë¶ Ħ -ли Ñĩ -ĠÑģл ед -ĠFur ther -Ġcar ing -Ġv ig -Ġval or -è¿Ļ 个 -ĠÑĩ а -Ġanalyt ics -Ġglo be -ĠM AN -Ġn el -ìĿ´ì ķ¼ -Ł ¼ -Ġo y -íķĺ ìĦ¸ìļĶ -j en -Ġtrou bles -ah aha -Ġchurch es -u et -Ġmeasure ments -b il -ì ½ -if ully -ин Ñĥ -ĠWil son -¦ ´ -ĠíĮ Į -Ġì° ¨ -Ġp úblic -ĠJer usalem -Ġn ails -Ġsp ine -Ġhe mos -Ġz n -qu is -ĠLe ben -Ġrefer ences -IT H -i per -ĠÑģеб Ñı -ì ģ -ĠW a -st ate -§ Ŀ -åħ ± -ĠGen er -Ġact ress -ĠEn joy -๠ĥ -Ġ× Ĵ -Ġinfect ed -Ġsh aking -Ġn ick -ภ¸ -Ġf ot -Ġaccompl ished -u ke -Ġshe ets -Ġf ence -Ġnurs ing -Ġintrodu cing -Ġfe at -O ne -T O -Ġcl ubs -ĠBru ce -on ge -ch ange -ĠBat man -åı ° -ĠOffic er -Ġhyd ro -Ġsupp lement -Ġc ela -Ġlong est -Ġcompet ing -Ġcon he -g iving -Ġbra ins -Ġlo ans -Ġw age -ĠCl inton -Ġs Äĥ -ane ous -Ġl ord -ÑĢÑĥ ж -Ġqu iz -Ġst iff -ĠL GB -s z -M E -m are -th ere -Ġn är -ĠM and -l ast -Ġd ag -Ġhalf way -ĠB and -Ġëĭ¤ ìĭľ -ĠA ren -Ġi le -P N -ent o -Ġalg um -Ġsoc cer -Ġblock ed -ĠJon athan -Ġse w -ĠTest ament -Ġv ale -Ġbehav i -å§ ĭ -Ġcon na -IC H -Ġaud iences -m l -amm ad -ĠìĤ ´ì -I GH -Ġr aces -em ed -Ġm á»Ļt -à ¯ -Ġover s -Ġdecl ared -Ġs ana -ĠU na -ĠÑĢ е -uck s -Ġp airs -Ġan ge -N e -Ġup s -av y -ø r -ree k -Ġbehav iors -Ġreflect ed -Ġprior ities -Ġcon du -Ġret reat -Ġexp enses -Ġë´ IJ -Ġtri ple -Ġêµīìŀ¥ íŀĪ -ä lt -Ġind igenous -Ġmin ing -Ġaccept able -Ġru in -C A -u ine -Ġpip eline -ct ic -ê t -ĠвÑģ его -Ġb oun -ĠDig ital -ĠBo om -ÑĨ е -Ġл ÑĥÑĩ -Ġas c -ĮĢë ¡ľ -ĠGood bye -Ġrend er -ene z -ar re -ĠTH AT -b our -ic ión -ãĤ Ń -E very -Ġw ires -ĠPar liament -n ung -ate ur -ĠS ave -ĠPh ys -Ġam or -ĠE ve -Ġfr ight -Ġgam ma -Ġmic ros -m itt -ĠC ode -ĠBe y -pl ed -ĠиÑģп олÑĮз -ç Ĺ -ìĥ ī -å¥ ¹ -Ġmon et -ĠJah re -Ġlux ury -Ġde af -Ġbet ray -Ġê² ° -и ки -Ġdefe ated -Ġunder t -Ġwe g -Ġcool er -ãģķ ãĤĵ -iam i -éĤĦ æľī -ĠJess ica -ĠJ oy -Ġsoph istic -ени и -ðĿ ĺ -Ġch ili -ĠTy pe -Ġprote ins -Ġpresent ing -al ia -ìļ ¸ -ĠMaj or -Ġmolec ule -um er -Ġcoll apse -ĠAny ways -ĠMount ain -ant ed -ãĢ IJ -Ġвиде о -æ° ´ -A ud -Ġcon qu -Ġvo ll -Ġkn it -Ġmem br -ĠMark et -Ġd ari -Ġcalcul ated -г и -Ġshrim p -ĠM u -ĠпÑĢ оÑĤ -Ġìĺģ ìĥģ -Ġproduct ivity -Ġcogn itive -ĠHe b -ict ions -ê² ½ -Ġcr é -f ör -Ġpray ing -ash i -ĠT ik -ó r -w en -ÑĮ Ñİ -ix o -Ġ( " -ĠÑĤ ел -Ġìĸ´ëĸ ¤ -ĠпеÑĢ ед -ĠD rive -ãĢ ij -ĠE qu -Ġequilib rium -Ġdescri bes -не е -4 2 -ĠCur rent -y y -Ġabsor b -Ġsold ier -d ers -Ġtestim ony -Ġdec line -ľë ¡ľ -g age -Ġinsp ire -la pping -Ġspin ning -Ġsla very -Ġfac ial -Ġtrad itions -ári os -ĠHosp ital -Ġn est -ĠëĪ Ħ -Ġto i -Ġfe ars -ìħ ¨ -ĠM uh -Ġgradu ation -Ġimpact ed -Ġa unt -ĠLet s -Ġalumin um -Ġdomin ant -ĠDav is -ĠNav y -Ġcom pt -op les -Ġest ava -è ¥ -Ġsc al -Ġpres erve -ĠO pp -Ġpract ically -Ġmagn itude -Ġf itting -Ġcoordin ate -Ġfurn iture -ĠFam il -Ġexplos ion -Ġdocument ary -ĠS cript -Ġport ray -m at -Ġschedul ed -Ġdynam ics -ph y -ak y -ĠU I -C he -Ġcontinu ously -ĠPro v -å° ij -Ñĥ з -ra h -Ġger ne -pro of -Ġsecret ary -ĠPat reon -sc ream -ĠK ids -á»ĵ i -Ġk g -Ġuncertain ty -Ġк ажд -Ġmit ig -Ġread s -å· ² -ĠR u -Ġpri est -Ġн ед -Ġlimit ations -Ġflo at -6 00 -ĠT oy -ĠJim my -Ġoff ensive -en i -ĠX i -Ġeye br -ĠTur k -Ġaccident ally -Ġoh ne -ĠS aud -9 5 -ĠD utch -ан Ñģ -ĠSe attle -Ġëĵ ± -che ck -k ÄĻ -Ġcontrib utions -Ġbes ide -Ġqu indi -Ġfle w -æĹ ¶ -Ø° ا -ĠL O -Ġwa ist -ĠE V -Ġhol idays -j on -Ġmis under -Ñı н -Ġb out -Ġd imin -Ạ½ -ó l -ĠGr ace -Ġinput s -Ġden y -Ġform ing -ĠB ild -Ġad equ -Ġfol k -Ġreject ed -se mb -Ġfrust rated -op en -ĠBet ter -il on -Ġtow el -Ġdifferent ial -Ġsac red -Ġsa il -éĩ Į -ent imes -Ġgentle man -Ġicon ic -Ġcomp aring -Ġs agt -Ġtext s -Ġgrand ma -Ġroll s -Ġcont ents -ä¸į 好 -оÑģ Ñģ -Ġsusp ension -ro it -¦ ¼ -Ġasse z -Ġd ort -ĠM ath -ĠVict or -ĠJava Script -ä¸į å°į -Ġen han -Å Ļ -ĠB ush -Ġpromot ion -Ġk in -Ġmon sters -ĠColor ado -ĠÎ ² -íķ´ì ļĶ -æŃ £ -iffer ent -Ġn aked -Ġpro d -et ics -ĠW oman -Ġtreat ments -Ġest oy -v é -Ġlif ting -Ġy apt -ĠRo ber -Ġì¹ ľ -Ġsubst itute -ak u -r idge -Ġê± °ë -Ġrespond ed -Ġb é -ĠEngine er -Ġtransfer red -ë ² -Ġha ber -o op -ĠW E -Ġv est -Ġfor ty -ĠD S -Ġ200 4 -Ġco aching -n om -ĠB ab -Ġn ossa -ĠJ ake -Ġg y -Ġde leg -Ġìŀ ł -ĠкÑĢ аÑģ -Ġstand point -Ġdis ad -Ġart work -A d -ill o -ĠÄij ược -ĠPr om -ĠL ib -Ġcritic ism -Ġcontact s -ÑĢ ам -Ġachieve ment -ÐĶ а -Ġdiss ol -ĠVeg as -Ġstream s -ĠK ent -ĠعÙĦ Ùī -Ġrad ius -Ġsu cks -ĠA ch -Ġf i -ou st -ĠлÑİд и -Ġpal ette -ĠH az -ĠAnth ony -Ġtem a -ĠC os -Ġsa fer -α ÏĤ -Ġcont rad -Ġma ior -Ġinfl ation -ĠSil ver -Ġatt ending -íķľ íħĮ -art o -Ġapplaud ing -Ġcomput ing -ĠH at -æ » -k now -mak ers -Ġcon oc -Ġeduc ated -Ġmod ified -Ġinc lusion -ment al -ŀ IJ -is ia -ĠÏĢ οÏħ -Ġa un -ĠIre land -Ġk ö -Ġcompl iance -Ġinsp iring -иÑĤелÑĮ но -Ġdisp os -ì° ¨ -Ġw ip -r ical -raw d -Ġt res -Ġmob il -olut ions -B O -Ġb ounce -Ġassum ed -ĠMed ical -Ġf iscal -Ġng Æ°á»Ŀi -ition ally -Ġst olen -ĠB M -Ġmechanism s -ε ί -Ġqual ified -Ġìŀ IJë -ught ers -ĠH IV -ĠL ots -Ġser vers -Ġcar r -ĠT ogether -Ġattract ed -Ġk r -æĪij æĺ¯ -th ur -in in -ĠH alf -È Ľ -ĠP ap -Ġremind ed -AL L -Ġhel met -Ġbott les -Ġprofess ors -Ġse ine -ÅĤ Äħ -ãĥ ı -Ġê±° ìķ¼ -Ġ×¢ ׾ -f un -ĠB ird -Ġfight er -ĠëĶ °ë -ĠT ool -Ġt in -ino is -ë ¶Ħ -×Ļ× Ł -ĠC AR -åIJ į -irst y -Ġout door -ĠN S -ãħ İ -ff en -Ġl ud -H ello -Ġroll er -ie le -ĠPol and -Ġap a -ex p -Ġcertific ate -ĠT own -аÑİÑĤ ÑģÑı -ild e -Ġdeterm in -P R -Ġfree ze -Ġmain stream -Ġobject ives -b lo -Ġtak ie -åĵĪ åĵĪ -Ġë°Ķë ¡ľ -el et -ĠI V -ĠF ast -Ġd ere -em p -ĠD ra -ĠìŀĪ ìĹĪ -Ġdisc rimination -Ġε ίναι -ne cess -æ ® -ıģ ı -Ġpost ing -wi ÅĽcie -Ġl ub -Ġol ive -Ġr im -Ġmodel ing -Ġa ño -ĠPak istan -Ġover l -Ġinf lam -N E -ìĹIJ ê²Į -Ġatt ended -Ġdeal t -ĠAl t -ĠL incoln -Ġaw ake -Ġfil ters -ĠWith in -czy wiÅĽcie -Ġs û -ĠJohn ny -Ġintegr ity -Ġisol ation -ĠE asy -ĠпÑĢ ин -ĠAl ice -Ġsm iling -en ix -, ... -Î ¶ -Ġbeg un -Ġjew el -Ġconvention al -Ġstat ist -Ġhand ed -Ġir re -Ġpro hib -Ġsatell ite -é¦ Ļ -ĠInd ust -Ġtra ged -Ġtra va -Ġih m -Ġcru el -ĠAg ora -ĠD oc -Ġz ones -Ġm all -Ġtr ay -×ķ× ł -Ġir rit -Ġk ans -ĠBe at -ud ge -ie lle -Ġtrust ed -Ġb ikes -ĠÑĥ п -ĠM ember -w ick -Ġcreat ors -Ġher itage -ind istinct -Ġres ur -enn en -C ome -Ġf iring -ĠBu eno -ĠТ о -ik an -ett es -Ġk es -Ġtri ps -Ġdivor ce -ĠK l -Ġcons ol -ke ep -기 ê°Ģ -ĠRep ort -Ġhost ing -Ġdiam ond -Ġcompl ic -Ġhel icop -Ġdep uis -d s -ĠCh an -Ñı л -Ġsc issors -il ation -Ġprop ortion -ER E -ĠÙĪ اÙĦ -int a -Ġmuch as -u ation -it is -æĬ Ĭ -Ñı Ñī -Ġni in -Ġemphas ize -uel a -Ġprodu cers -Ġr ze -änd er -ET H -æ º -Ġconst itu -åĽ ½ -Ġperform ances -ist le -go v -ĠL iter -Ġincorpor ate -Ġeduc ate -ĠN in -ì ª½ -Ùĩ Ùħ -el eration -×ķ× ij -Ġya ÅŁ -or ous -ĠC as -Ġgr ants -ëĬ ¥ -am el -Ġê·¸ë łĩê²Į -ĠE ste -Ñħод иÑĤ -ĠпоÑģ ле -Ġg ent -Ġfocus es -al ities -ĠR h -ë ³´ -æ° ij -ĠD ance -r r -Ġam er -Ġutil ize -Ġl ÃŃ -ĠAm ong -Ġpregn ancy -Ġlo ops -ал оÑģÑĮ -ĠM oh -Ġcatch ing -Ġglo b -Ġa jud -Ġ[ ? -ĠAn al -lo oking -Ġsurf aces -Ġprogress ive -Ġvir al -0 8 -Î ¾ -K A -Ġ ży -Ġpick s -ann on -Ġbul k -ĠR oss -Ġdescri bing -ĠG el -Ġloc ally -Ġend less -Ġmass age -Ġclean ed -Ġtravel ed -ен Ñĭ -Ġsent iment -ig ma -ĠN as -Ġchemical s -Ġright eous -ĠMag ic -Ġrel ates -Ġtruck s -Ġ19 60 -åĪ ¥ -Ġapp et -Ġsn acks -ĠSum mer -Ġy üz -Ġpr is -ĠMex ican -Ġtransp aren -Ġminor ity -Ġver te -Ġl assen -4 6 -л ек -é p -ĠÑĦ илÑĮ -Ġi yi -Ġsp an -íķĺ ì§Ģ -Ġind icated -qu ar -Ġscholars hip -ĠLGB T -Ġhistor ically -ó ÅĤ -Ġmin ist -Ġpen et -ĠR ap -Ġcons ervation -çĽ ´ -ĠH oney -ĠBe i -id el -Ġrespons ibilities -Ġmess y -ĠEx cept -OR E -Ġiniti atives -Ġjun ior -Ġdesign ers -Ġexpl oration -Ġspons or -Ġmob ility -Ġint eg -land o -Ġb ark -Ġindic ates -à ¶ -Ġemploy er -å® ī -Ġcous in -Ġbo iling -Ġch rom -Ġç al -Ġper pet -Ġcont ained -Ġpark s -Ð « -ĠEngine ering -P lease -ĠStart ing -her o -Ġlaw yers -è¥ ¿ -Ġz d -Ġfranch ise -ra ge -Ġint uit -ĠG L -re ach -ĠE lle -Ġnh Æ° -ĠN ord -Ġbe an -0 7 -Ġple asant -å½ ĵ -v iron -Ġgrad ient -z us -ĠE M -Ġess ay -ìĹIJ ìļĶ -ế n -n u -á» « -ĠÃī s -Ġden omin -ĠGirl s -Ġperson nes -ĠاÙĦØ £ -b ild -ĠSt at -Ġcompl iment -ĠK ate -Ġoptim al -Ġh id -د ÙĬ -Ġquick er -w all -E n -IN E -?? ? -ì² ´ -ĠA ction -å Ł -Ġpenal ty -ĠK az -' ? -Ġc ried -Ġcan vas -ft e -Ġexc lud -¸ë ¡ľ -Ġemphas is -Ġen zy -ĠH ou -Ġoverse as -ÃŃ amos -å¸ « -ö glich -Ġhead phones -c n -ĠA ge -Ġa kan -Ġcharacter istic -íķĺë ©´ -get s -Ġë¶ Ī -Ġr ival -Ġb orders -em ente -em ás -Ġy ol -Ġcom pe -end ers -ınd an -Ġmö glich -Ġbubb les -nat ural -Ġar med -Ġel abor -ĠìĿ´ë ²Ī -Ġwash ed -οÏħ με -è« ĭ -Ġfl avors -Ġexist e -Ġpre st -ĠThe ma -оп ÑĢоÑģ -er on -U E -er i -Ġconc er -Ġa ixò -åħ © -Ġprotect ive -Ġзна Ñİ -ĠëĤ ł -ĠII I -Ġme er -ĠSh op -ll i -ĠOr der -ĠM Y -ĠG host -ãĤĤ ãģĨ -ad el -Ġst ole -Ġrele asing -ĠCom ment -Ġtra ins -ë ªħ -Ġw issen -ens ed -Ġdesc end -Ġf ier -Ġrad i -Ġpers u -ç ¢ -Ġм н -ĠD est -Ġwor ries -it et -b as -Ġst ab -n ame -or ic -ĠCl ose -Ġalum ni -ĠS elf -ff e -it ating -ather ine -ĠRight s -Ġell os -Ġwar rant -Ġn erve -Ġveget able -ĠTe il -Ġê°Ļ ìĿ´ -R Y -Ġsustain ability -Ġste ht -Ġbr id -ada ÅŁ -Ġt v -Ġdur ation -Ġpesso a -Ġmet rics -Ġad am -c as -аÑĢ и -Ġev ident -Ġdisplay ed -Ø§Ø ¦ -Ġre ck -ĠBudd ha -Ġde le -ĠDie go -os ph -Ġb la -ĠM ik -ul ator -Ġ200 1 -Ġpromot ing -y ch -ĠE X -Ġlast ly -Ġout line -Ġspir its -Ġve ux -Ġsubt ract -ĠÅŁ imdi -Ġp ins -Ġbur ger -Ġmol to -Ġhab ÃŃa -Ġë° ĺ -ig u -er st -Ġn en -Ġbac on -it ious -Ġcar ries -Ġprom ises -nd e -ĠLe ft -ĠL im -æ £ -Ġ4 4 -Ġcare ers -Ġì£ ¼ë -Ġspeed s -qu é -m ad -mark et -is me -Ġ200 3 -Ġre cess -ĠJ UD -Ġrac ist -ĠSch l -Ġpar ler -Ġot ros -ish es -Ġconvert ed -aa aa -ани и -ĠAr k -ĠCh ance -Ġelement ary -ε ν -ink s -Inter viewer -Ġfre ely -al ah -Ġëĭ¤ë ¥¸ -Ġrequest ed -Ġtor que -no ÅĽci -ou red -ĠSt aff -Ġst ain -ĠAl an -Ġv ere -ĠW inter -Ġdef ect -ied y -Ġbe ats -Ġh á -um n -o ons -it udes -Ġse it -o ly -Ġres erv -Ġext r -Ġphys ician -vis or -Ġhand ful -ĠN ations -Ġì¢ĭ ìĿĢ -uc cess -Ġup stairs -ĠSqu are -Ġhe in -ĠSe ason -ol is -Ġpr ince -Ġdef ensive -ç ½ -Ġм еÑģÑĤ -Ñĸ й -Ġا ÙĨ -um ble -ê¹Į ìļĶ -Ġass ass -Ġcirc ular -Ġqual ities -Ġh mm -Ġbl own -ĠL iz -ĠK ur -ĠS A -Ġfind ings -Ġcol ours -Ġde lle -ĠI R -ĠA th -ĠD ub -ĠO x -ĠØ ® -Ġpo ckets -Ġgr ill -Ġswitch ing -Ġprefer red -ĠW ales -Ġex emplo -Ġchop ped -Ġvacc ination -Ġne uro -Ġspec ify -iv os -Ġser á -Ġz ie -Ġà® ® -Ġresult ing -ĠU gh -Ġmess ed -C D -Ġpa ar -Ġcom er -Ġcou ch -ĠFest ival -Ġ4 9 -v ous -z ens -ç¨ ® -ĠKenn edy -ĠT s -Ġë³´ì Ĺ -Ġdemonst ration -Ġun to -Ġfrust rating -Ġlabor atory -Ġe gy -Ġbeaut ifully -Ġìŀ ¬ë -Ġal gu -Ġö yle -ä½ł çľĭ -ĠP H -Ġfort une -Ġclean er -ĠRob in -Ġsa us -ĠG eld -Ġk at -o bs -Ġol ur -Ġm att -Ġquest a -Ġsuggest ion -en cer -о ÑģÑĤ -Ġrad ar -Ġìŀ ¡ -ish a -à® ¨ -ãĤĵ ãģª -j es -Ġve el -ìĤ ° -Ġauth ors -ãĢ İ -pl an -Ġcollabor ative -Ġinst inct -Ġfar ming -au ge -E du -Ġmembers hip -Ġsimult aneously -Ġb ake -Ġk ä -Ġlect ures -Ñĩ еÑģ -Ġprend re -Ġcoll aps -ĠS aya -ĠF ut -Ġy og -ĠR ather -ر ÙĬ -Ġcamp s -ол од -Ġsim ulation -ĠM ak -La ughs -Ġgre y -Ġsent ences -y en -ĠUn less -J e -ĠSat an -ĠÑĤак же -ĠN A -Ġbr on -Ġ? ] -Ġsoul s -Ġlight ning -Ġimag ined -Ġczy li -ps ilon -et ta -Ġbelie ving -Ġstrong est -ĠC ON -Ġquel ques -Ġimmig rants -Ġwall et -éĢĻ æĺ¯ -ĠJer sey -Ġimplic ations -Ġfor b -ãĢ ı -Ġun believable -Ø§Ø ¡ -Ġoper ational -ü s -ĠG M -Ġê·¸ëŁ °ëį° -Ġgrac ias -Ġent end -ĠReg ard -ro b -ĠÑĤ еÑħ -è ı -ĠRev olution -Ġwa ar -ĠB iz -th eless -Ġspons ored -qu ier -ĠìĿ ¼ë -Ġte k -ĠëIJ ł -ig keit -ĠL uck -ĠCertain ly -Ġto ll -Ġн иÑĩего -ĠM oney -ĠÑģ ÑĤоÑĢ -ĠDou ble -ĠW olf -Ġch unk -ά ν -it és -on ing -M ar -Ġgrand es -Ġcollect ions -ĠEurop a -Ġа ÑĢ -ĠâĢĭâĢĭ âĢĭ -Ġê·¸ëŁ¬ë ©´ -Ġоб ÑĬ -Ġãģ ª -Ġìĭľ ê°Ħ -ĠC ustom -Ġì² ĺ -Ñĸ лÑĮ -Ġindivid ually -í Ĺ -Ġdo zen -Ġo we -ĠVict oria -åı¯ èĥ½ -Ġbe et -ur b -Ġanal og -i ção -Ĥ ľ -so ever -Ġmod o -Ġsubscri bed -ìŀ ¬ -Ġent ities -çī ĩ -Ġclos et -Ġrespond ing -Ġprin ter -ĠStep han -Ġby ÅĤ -ĠD om -ĠF ern -ĠP ier -ĠwiÄĻ c -Ġh ence -Ġmod ules -ãĥ ¬ -ĠëĶ ± -ĠDann y -ĠÑģеб е -Ġv ad -ĠìĹ Ħ -Ġs ous -Ġsp here -B Y -ĠP ed -ign ed -Ġwhe at -Ġund ers -Ġevol ve -Ġdec lar -Ġlight ly -Ġident ifying -æĦı æĢĿ -Ġlegend ary -Ġgen uine -Ġgr ind -ĠU ne -ge ben -Ġb icy -Ġjump s -Ġprov ince -zi ÄĻ -Ġ×IJ× ł×Ļ -Ġh oc -Ġб л -ĠGr ad -Ġreven ge -ĠاÙĦ ت -o oh -æĭ ľ -аÑĨи и -å¹ ³ -Ġelect ro -ĠëIJ IJ -ãģ§ ãģ¯ -Ġf als -ri el -ok er -ĠEx cellent -ĠMor gan -Ġbr ick -Ġsubstant ial -Ġpoll ution -ĠT ür -ĠEv et -Ġl ung -ãģ ĸ -×Ļ× © -omm es -Ġreal izing -Ġhum ble -ĠL ock -Ġb od -Ġìĸ ¸ -Ġpe ers -uz z -Ġembed ded -Ġclar o -Ġag greg -Ġemploy ers -ĠR aj -Ġãģ ¨ -ĠY i -Ġje u -at ers -Ġstri kes -n os -aut res -d r -op her -ĠApp arently -íĺ Ħ -Ġinf ant -ا ب -ÑĤ Ñĭ -í Ľ -Ú ¯ -Ġred es -acaÄŁ ım -ĠDA VID -ĠCh icken -Ġperspect ives -Ġview er -Ġsh ar -ĠпÑĢо из -lig t -er os -it able -ил оÑģÑĮ -Ġdif ÃŃ -´ë į° -Ġret ired -Ġthat s -zen ie -be iten -Ġmy cket -ĠR ab -Ġinflam m -ì° ® -Ġd um -Ġdad dy -æľ Ł -Ġimm ers -Ġplay list -௠Ĩ -Ġtra um -Ġref use -st ep -à® ļ -c up -Ġpop s -r imin -ay ım -Ġa ld -Ġun necess -Ġd ah -ĠIr ish -Ġcomp r -la ÅŁ -T P -Ġtransl ated -S c -ce ÄŁim -´ IJ -Ġd rei -ĠлÑİд ей -Ġqu iero -Ġhe le -z lich -Ġapp les -Ġdistrict s -Ġcred its -Ġas p -Ġëĭ ¨ -or al -å½ ± -Ġste pping -ĠV a -Ġg ains -6 5 -Ġnuest ra -ed ay -ass ador -ĠL ind -Ġcrop s -ci endo -ig ue -Ġb ana -A m -Ġp ent -Ġadd iction -Ġpack aging -ä d -ª ¨ -Ġper què -Ġcampaign s -Ġste ep -Ġne ue -Ġembarrass ed -Ġdist inction -it zer -åij Ĭ -Ġregist ration -Ġll am -ĠAlm ighty -li est -Ġu z -n ak -ç º -Ġter az -iam ente -Ġtrans actions -Ġc ôt -Ġswitch ed -Ġcom bo -Ġpray ers -Ġintern ship -Ġaddress es -Ġchar ity -ĠW OO -Ġb ait -è¿ ĩ -Ġ � -Ġf ica -ĠTy ler -ar u -Ġat oms -ĠLe vel -ĠпоÑĤ ом -Ġf ame -ul k -Ġteach es -Ġre build -ед ÑĮ -ĠIndones ia -ush i -ĠSh ort -Ġens uring -f s -e le -Ġmargin al -Ġconclud e -am t -Ġver ify -ĠMc Donald -Ġsk al -Ġrec onst -ĠM ann -Ġbas ement -Ġtransform ed -Ġoccasion ally -z one -ĠD ans -Ġкак ой -Ġdiagn osed -ĠÏĦ α -Ġcomm ands -Ġpresident ial -Ġab b -Ġbrack et -ĠL em -Ã¥ ng -Ġfavor ites -Ġrev ol -ĠíĬ ¹ -Ġhar ass -é ħ -Ġcle ans -st änd -Ġknock ed -Ġpe oples -Ġmusic ians -Ġmut ual -ĠC old -8 8 -ze j -at ie -ĠHon or -Ġobs essed -ĠM USIC -ĠBre ak -ú ng -Ġmod ify -Ġs öyle -Ġ×ŀ ×Ķ -ĠOn line -f o -ĠMill er -Ġlik ing -Ġin hab -Ġgrat itude -ĠJour nal -arn ess -J ohn -ĠG it -åī Ľ -Ġsin cere -ĠS ci -ĠE li -Ġsymbol s -Ġman ually -ε ÏĤ -Ġв Ñĸд -ĠF at -Ġlab els -Ġsophistic ated -ump s -Ġrele ases -Ġ4 7 -ĠO M -ê°Ģ ë -ĠB ien -ĠRe f -è¨ ĺ -ĠSt a -ĠE gg -Ġindic ator -ps on -Ġnas ıl -R ight -Ġcon vey -Ġkn ot -Ġconnect s -ul as -Ġpre ced -Ġine quality -am iento -Ġrep ly -O Y -Ġdism iss -ĠëIJ ľ -çĦ ¡ -ĠÑħоÑĢоÑĪ о -Ġm éd -Ġrandom ly -ĠO nt -u ard -Ġpull s -ĠÑĤ епеÑĢÑĮ -ĠNe ed -ĠSo ft -Ġstrength s -Ġgo ed -um en -æŃ » -Ġíİ ¸ -Ġд об -Ġclar ity -ĠA i -Ġball oon -ĠP and -ĠìķĦ ëĭ -Ġsh iny -Ġsmall est -on ia -h ill -ot ing -Ġe ing -Ġmere ly -Ġse us -Ġн еп -Ġí Ĩµ -Ġgu ides -Ġspecial ist -Ġste ak -ãĤĪ ãģĨ -Ġmig ration -que le -Ġru ined -Ġpu pp -å¥ ³ -Ġk end -ang an -Ġpal m -Ġunf air -Ġz m -ĠD V -ch ester -и Ñİ -Ġo oh -er g -AT H -° © -åĵ ª -r ison -Ġinvol ving -Ġpart ly -anç ais -Ġv ow -Ġprom inent -Ġcry st -ib a -Ġdes erves -Ġover t -Ġsens it -ĠWh e -Ġtight en -Ġintim id -Ġal iment -w ill -Ġstrength en -ĠT an -åı Ī -ãģĹ ãģ¾ãģĻ -on i -ĠM un -Ġpro ph -Ġrehe ars -ĠK le -Ġve ces -Ġwonder ed -ok i -Ġsens es -´ì ĭ -Æ°á» Ľ -ĠÈĻ i -Ġmuch os -Ġwatch es -ortun ate -ĠJ uan -ìŀĸ ìķĦ -ÑĢ е -e i -ion en -Ġexperiment al -Ġda ughters -ภĽ -Ġment ally -bec ca -aw are -ìĦ Ŀ -Ġwhat soever -Ġen ables -ĠL ow -o id -ภĬ -ó d -Ø º -Ġconstruct ed -ĠLad ies -Ġaccus ed -Ġа н -D an -Ġsp awn -Ġcontain ers -Ġart istic -ı p -Ġdisc l -Ġaut res -in as -ĠN ation -Ġn ag -be an -w he -ľë ıĦ -ĠSe oul -Ġíı ¬ -ĠN ich -Ġcomp lement -Ġinter ven -ĠMod el -ĠOr ange -nam on -Ġcalcul ation -se e -Ġusted es -Ġle b -Ġdo ct -Ñĸ н -Ġf oster -Ġel astic -ĠAh h -Ġa ce -ĠP ink -ĠJ eg -Ġde er -ãģĹ ãģĦ -s is -Ġjak o -ĠEm ma -ÑģÑĤв енно -Ġport rait -Ġmak er -Ġa ument -ÑĢ об -Ġairpl ane -Ġtransparen cy -Ġadjust ment -ĠCD C -ç on -Ġupload ed -Ġд ейÑģÑĤв -Ġго ÑĤов -Ġit er -Ġcur se -ô n -mer ce -ar an -Ġle ak -çµ IJ -Ġabs ence -Ñģ кий -Ġread ers -al er -Ġbene ath -ang o -h etic -Ġfin ns -Ġpo op -Ġdu plic -H i -ig s -olog ically -op p -Ġd izer -ĠAll en -Ġgl i -Ġacc eleration -Ġvit amin -ãĥ Ń -v ä -ĠAc cess -à® Ļ -r ás -Ġappreci ated -Ġn ah -Ġpos ter -Ġt ale -Ġhighlight ed -æĸ ĩ -ż eli -Ġblock chain -Ġmic row -Ġcin ema -ĠCh ang -ĠSe arch -ust ers -ĠZ ero -ĠDiv ision -ÑĢ аÑģ -Ġsca re -Ġj elly -ĠAdminist ration -S O -Ġl ined -Ġê° Ħ -Ġge ben -Ġso da -Ġwin ners -³ ¼ -Ù Ĵ -ĠAm b -åķı é¡Į -å Ķ -Ġpe g -å· ± -4 3 -Ġra us -Ġre wards -Ġinc lus -Ġhigh way -Ġha h -Ġmultipl ied -Ġs ẽ -Ġdisci ples -Ġn ing -Ġdress ing -Ġattrib utes -ĠM osc -ĠGree ce -Ġse k -ĠLe arn -Ġj us -rend re -Ġperson ne -pl ete -Ġpl acing -Ġl uego -ill ance -Ġоб Ñī -Ġprov ision -Ġl ion -t ra -bo ards -Ġbehavi our -he y -Ġsubscri ption -Ġprot agon -ãĥ £ -Ġvar a -ĠÅŁ u -Ġha ha -Ġteas poon -æ Ł -av oir -Ġcrypt o -ĠÑģÑĤ аÑĢ -ĠSt ore -ab s -ĠStud ents -Ġla und -int o -Ġapproach ed -° ľ -ÑĥÑİ Ñī -ĠL abor -ot es -iat ric -Ġgro ÃŁ -ut ive -Ġи д -ĠG ib -Ġpl acement -ĠdifÃŃ cil -Ġf rog -ĠвÑģе Ñħ -ĠJ r -az ed -Ñĥ Ñī -Ġê ¼ -fr ame -а еÑĪÑĮ -Ġlock down -åij ³ -Ġmed i -Ġ×Ķ× ŀ× -ени й -em ale -ì¢ ħ -ater al -Ġdist ant -Ġbe ars -Ġjournal ist -è§ £ -ĠMarsh all -ĠIh nen -uet ooth -b ag -ĠÄij ã -ĠHigh ness -Ġì° į -и ка -ĠW u -ĠFr an -Ġp eng -Ġf on -Ġhypothes is -ĠÑĢ Ñĥ -Ġl y -× ļ -ìĽ Ķ -ĠRad io -ภŀ -D av -Ġembarrass ing -ĠìŀĪ ìĸ´ -Ġcast ing -Ġc age -ĠP sych -ĠìĿ¼ ëĭ¨ -ĠÅ ¾ -im b -Ġdirect ors -S H -ĠÏĦη ν -á»ģ u -Ġkon uÅŁ -Ġoption al -quar ters -ik er -ĠS ant -Ġvers es -ë ¶Ģ -Ġo lar -ĠÏ ĩ -ãĥ ķ -Ġγ ια -ĠI mm -Ġcontrovers ial -Ġer sten -Ġreci p -ĠChristian ity -Ġê´ ľ -ord on -×ķ× © -Ġsl ash -ĠP f -Ñĥд ÑĮ -×ķ× Ŀ -ĠPer ry -Ġm amy -Ġbackground s -Ġà®İ ன -Ġpend ant -ĠColumb ia -Ġin verse -ĠÑĩеÑĢ ез -Ġs v -Ġdig ging -4 1 -ch em -Ġnavig ation -ĠSh in -ĠFr ont -P D -Ġbe aring -ĠW asser -Ġw ax -ĠCH RIS -ch ing -Ġpress ed -E l -ĠD al -ons in -Ġb inding -Ñģк ой -po ons -Ġmo ck -are st -к ÑĢа -M M -Ġcor rupt -st orm -Ġref res -ĠCo ach -ll ä -ĠTH IS -Ġpar ag -Ġìĵ ° -p ool -Ġbill ions -Ġê¹ Ģ -gr oup -Ġwel coming -cell ence -ĠDu ke -ê¸ ´ -Ġprim era -ìł ¸ -Ġp ond -Ġstat ue -Ġêµ ¬ë -Ġh atch -Ġinstrument al -Ġresident ial -ì» ¤ -Ġaccept ing -osh i -d ate -ĠìĶ ¨ -Ġplant ed -Ġj oking -Ġì Ħľ -Ġh ated -ĠÑĢаÑģ Ñģк -Ġsle pt -Ġpack ages -Ġisland s -es en -ÄŁ ı -Ġdi agon -ĠO sc -Ġmes h -Ġsc ales -ar ity -ĠDef ense -ãģ¡ ãĤĩ -ĠLew is -ĠÑģ егоднÑı -Ġfl ies -uin ely -ĠCons ider -Ġst ark -he w -ĠAs ÃŃ -³ ´ë -Ġprop ose -Ġíķĺë ©´ -od o -ĠNorm ally -Ġhe eft -ĠHarr is -g ro -ĠBlo od -b ase -Ġi OS -Ġtouch es -Ġinsp ir -Ġ× ĵ -Ġb inary -Ġì¶ Ķ -Ġser ial -Ġ ion -Ġunemploy ment -Ġodd s -ĠF ab -ĠF BI -BR UN -Ġweight s -ν ο -at ile -Ġnurs es -Ġinvolve ment -ĠíĶ ¼ -Ġgovern ance -Ġâ Ĥ¬ -ÑĢÑĥ п -ier ra -íĺ ķ -ĠJ erry -Ġbe ard -Ġsal vation -ĠAl ong -g entle -ĠK i -b ol -ĠPl at -Ġhas ht -è¿ ij -Ġw are -Ġpart ie -y cz -Ġint r -F ih -n ent -Ġche at -il en -Ġë ¯ -or ie -Ġfá cil -et ric -Ġaffect ing -unci ation -Ġaff airs -Ġbe e -Ġview ing -Ġor ang -ĠL an -ĠС ÑĤ -ä¸ ĸ -ĠM es -ĥ ģ -er ie -Ġes pa -Ġinter pre -Ġposs ess -Ġpure ly -rit o -f ound -as ma -ìłģ ìĿ¸ -Ġexam ine -ĠÑĥ м -Ġbes ch -ĠTom orrow -ĠB lock -Ġvari ant -Ġprefer ence -Ġcoach es -Ġmedic ations -Ġíĺ Ħ -Ġemp ire -ë Ħ¤ -ĠIll inois -Ġcris py -Ġth ì -Ġbe es -7 7 -Ġgl ow -è º -ĠStud ies -åIJ Ħ -ĠChall enge -Ġunlike ly -Ð § -ıy orsun -DI E -Ġminim ize -iz ard -Ġú n -Ġencont rar -ĠK ill -å » -Ġvan illa -ĠGr ant -ĠG T -se a -Ġs ought -в од -Ġnä m -ĠA unt -OW N -Ġpump kin -st ellen -Ġr ag -ег да -Ġstory t -Ġfor um -æ© Ł -Ġestab a -uch e -Ġcon gress -ĠRe y -Ġdram atically -ĠSp ort -ĠYe llow -Ġê³Ħ ìĨį -Ġdisg usting -ĠRe cent -Ġacqu ired -Ġc ables -çĶ ļ -d in -Ġv isto -Ġcommunic ating -ÑģÑĤав лÑı -еÑģ ÑĤо -ãĥ»ãĥ» ãĥ» -Ġré g -Ġso cks -Ġpro ces -be cause -Ġut ter -Ġcoloc ar -Ġnew est -Ġgr amm -è¡ ¨ -ä¸į çŁ¥éģĵ -Ġsh ifting -Ġcar rier -ĠÑģк оÑĢ -ĠSch w -Ġexec uted -Ġmaint ained -ĠÏ Ĩ -ĠM oses -Ġdis se -Ġhor r -ãĢ ľ -Ġr ally -Ġall em -ĠEvent ually -Ġdi yor -lv ania -Ġsch nell -Ġê³ ¼ -Ġë§ ¤ -Ġstrugg les -l ate -Ġclar ify -é ment -Ġmulti plic -иб о -Ġjour n -Ġfra gr -Ġsurprising ly -Ġdesper ate -5 2 -Ġs ul -ĠRe ad -ĠF ried -Ġm ond -w oo -Ġorgan izing -ãģĹãĤĩ ãģĨ -ĠSo on -Ġв опÑĢоÑģ -ĠN ur -ĠÐĹ Ð´ -Ġsp ider -е ÑģÑı -Ġtutorial s -Ġnutri ents -or er -Ġcoe fficient -Ġarrange ment -Ġpr icing -n an -y u -B L -Ġtri be -ĠHow ard -un ks -Ġnew er -Ġprov in -Ġpred iction -h os -Ġol sun -ĠAr ound -Ġv ier -ĠÑģÑĤоÑĢ он -Ġv alley -ĠE la -if i -Ġgal axy -Ġtran qu -Ġad vers -ĠTem ple -iff s -ig ence -èĩª å·± -Ġkön nte -ĠÄij ó -D id -Ġphotograph s -ĠA WS -ÑĨи Ñı -Ġgu ards -Ġappoint ed -ĠG il -Ġм ом -Ġc od -ĠUn like -Ġeven ly -isc onsin -Ġest ou -Ġm nie -ĠEx ec -ĠM V -ĠE ine -ä¿ ¡ -ĠRog er -ĠF ac -ĠL ist -Ġf uer -аеÑĤ е -om ed -Ġattract ion -èī ² -Ġter rain -ĠD rop -Ġcorpor ations -Ġsci ences -Ġthr one -ãģĦ ãģŁ -Ġa j -ĠR ot -çī ¹ -Ġsupp orters -ĠB ere -H ere -Ġdifer entes -Ġsignific ance -Ïĥ η -æĪij 覺å¾Ĺ -Ġcl amp -Ġë ĮĢë -Ġfab ulous -re z -æĮ ģ -Ġassum ptions -ut her -w id -p ot -è¿ İ -Ġy an -ul in -ÑĢ Ñĭв -ĠSl ow -ĠPenn sy -Ġíķ ´ìĦľ -Ġme io -Ġwealth y -ĠE ight -Ġpul se -Ġfr iction -id ity -ĠH oll -i yorum -Ġsound ed -ĠC arr -Ġfor k -â ĺ -ĠP A -Ġcons pir -Ġc oding -r t -ĠTy p -Ġìĸ ij -Ġп ог -Ġmis er -ĠÑģм оÑĤÑĢ -ĠSw eden -Ġolar ak -ĠZh ang -ĠCh i -ĠT itan -Ġscreen ing -ĠSp ider -ĠÅŀ imdi -Ġobst acles -lar a -Ġchalleng ed -p se -T ON -á» ¥ -ĠP i -Ġlag i -ie urs -Ġhur ting -Ġneg lect -Ġgener ating -Ġyoung est -Ġaud it -ĠÑĢ ез -Ïģ ά -Ġdon ate -ĠPD F -Ġvis its -Ġcru ise -P P -as er -Ġw sp -back s -iv als -ãģĨ ãĤĵ -Ġde ve -Ġprop ort -Ġc ath -ĠE ffect -Ġwind s -ĠìĻ Ķ -Ġchart s -Ġs ama -Ġautom ation -Ġпок а -Ġol an -Ġbo ats -Ġca fe -Ġden ied -ĠM ama -Ġblock ing -ĠTh or -Ġphenomen al -Ġstake holders -Ġun os -Ñĥ еÑĤ -ĠAb raham -ãģ§ ãĤĤ -Ġdetect ion -Ġjur is -Ġpower ed -z ial -Ġwel fare -Ġup grad -Ġmoż na -ĠC ase -c ular -Ķ ìĿ´ -ãĥ ģ -ĠGu ess -Ġcy cles -ä¾ ĭ -çµ ¦ -ro ck -um i -Ġel ite -Ġqu è -åł ± -ÑĤ ом -Ġsh ore -gun ta -Ġk u -Ġfaith ful -ĠJ eremy -a id -à · -ug al -å°į åķĬ -ĠV el -Ġvra i -st ell -¨ ¸ -Ġk ol -è ½ -Ġquant o -Ġз аÑĢ -Ġ200 2 -es y -Ġres erve -Ġмом енÑĤ -Ġdeploy ed -Ġdefin ing -Ġsa u -Ġga at -" ) -Ġtrans mit -Ġpubl ishing -Ġrank ing -Ġoff ense -Ġ4 6 -p in -ĠT aking -Ġentit led -Ġgen uinely -Ġvari ations -Ġfind e -Ġt au -Ġunf ortunate -ĠR ah -port s -Ġc Å -Ġmon key -Ġbr ac -we i -l ung -Ġart if -Ġsy rup -ĠÐĶ ав -Ġlift ed -Ġche z -ĠAd vent -ĠSt ock -Ġdo l -м ен -иÑĪ ÑĮ -Ġy n -g io -d et -Ġdes se -Ġg ri -ĠChair man -ç ħ -Ġcu enta -an im -Ġcra b -Ġesc al -Ġpremi ère -ĠGe f -Ġd ining -Ġsevent h -Ġch asing -ĠT ower -Ġbrut al -Ġfundament ally -ãģ¨ ãģĨ -л ениÑı -st age -Ġacqu is -Ġcyl inder -Ġcomm ander -m em -ĠU V -ha ppy -Ġe psilon -Ġinv itation -Ġfar mer -ch air -Ġdest iny -Ġso vere -ĠHeb rew -Ġserv ant -Ġbe w -Ġg ast -ut ies -Ġadministr ative -ĠComm and -é ta -Ġnit rogen -ê· ¼ -Ġab i -Ġvill ain -Ġblank et -ĠS end -Ġbeat en -² Ħ -Ġvol unt -Ġschol ar -ĠEm peror -Ġ4 3 -v able -ĠD us -ĠG U -Ġtarget ing -ww w -Ġamend ment -ìĨ Įë -Ġt ing -Ġn asty -Ġg auge -ĠÑĢ од -ĠH ans -Y our -α ν -Ġpro jet -ĠHawai i -Ġsusp icious -Ġsch w -Ġremo val -Ġint rig -ĠM U -Ġp onto -ठ¾ -Ġоб ÑĢаз -Ġguess ing -p ace -Ġm others -Ġmill imeter -л ение -没 æľī -Ġavail ability -ic z -æŃ ¤ -Ġfr act -Ġbas es -k m -ĠB TS -ĠF ield -Ġd zie -Ġseg undo -ĠëĤĺ ëĬĶ -Ġlegit imate -im as -Ġв н -Ġcor ruption -Ġsm ash -ĠVal ent -Ġalign ed -ĠPennsy lvania -Ġg ab -ĠE un -ent h -ĠMor ning -Ġcand le -Ġback pack -ĠIslam ic -a ções -Ġenc ry -Ġmushroom s -íĮ Į -d it -Ġtrans it -ĠW isconsin -Ġparticip ated -ĠIl s -Ġunf old -¶ Ģë -Ġprof its -Ġwar ming -ĠG ang -Ġnetwork ing -Ġme ga -Ġthorough ly -le ments -ĠH m -Ġdec iding -Ġemotion ally -Ġexha usted -ĠÐŁ оÑĤ -c ido -ĠHT ML -Ġcopy right -Ġmel ody -y im -Ġand ers -osh op -Ġë³ ¼ -Ġathlet e -ĠG E -Ġfrequ ent -Ġdes ires -Ġneed ing -ĠY un -Ġrif le -Ġlo ver -' T -Ġd ense -Ġt ão -Ġnot ified -Ġid i -ìĹ Ń -í Ĩ -Ġinteract ing -Ġrapp ort -еÑĢ и -s ki -Ġb esser -Ġmanufact urer -ĠK yle -Ġaccount able -ĠS ak -ĠP il -ĠD omin -Ġpres um -ĠÐĴÑģ е -Ġvine gar -Ġguarante ed -çľĭ åĪ° -Ġhand led -éŁ ³ -c at -Ġcivil ization -Ġaccom p -ĠV M -é mon -Ġde ze -Ġgrad es -Ġsoll te -Ġst aring -×IJ× ª -ar nt -Ġhoriz on -Ġtrav ail -h our -第 ä¸Ģ -ĠE D -ĠD ak -Ġn y -Ġcon ve -ĠCh am -Ġfir ms -ĠL iu -ĠÑģÑĤ ÑĢан -Ġli bert -Ġlens es -Ġint ake -ĠвÑĭ б -Ġmens en -h el -Ġpract ition -Ġ3 50 -ãĤ ³ -F O -Ġbed s -Ġancest ors -ĠìĹĦ ì²Ń -Ġdistur b -ĠLast ly -ĠSupp ort -ี à¹ī -ĠCor ona -Ġenthus i -Ġвоз м -ĠìĤ¬ëŀ Įë -Ġ5 2 -b ird -Ġredu ces -ĠìŀĪ ìĿĦ -ĠG ene -êµ IJ -ÄĻ p -ĠÃľ ber -Ġconcer ning -us er -Ġconcent rate -ĠWH AT -ish op -onym ous -no ld -Ġsuggest ing -© ° -ĠF ish -.... .... -Ġvess el -Ġtrabaj o -ãģ µ -ĠO cean -å§ IJ -y g -Ġtown s -d el -Ġterr ifying -Ġçal Ä±ÅŁ -Ġs ino -Ġe ats -Ġge z -Ġg eme -ĠìĻ Ħ -Ġcomp art -Ġimplement ing -ĠPot ter -ĠGerm ans -Ġg ÅĤ -Ġt ennis -Ġcar pet -au er -ĠSaud i -ye ong -Ġcur ry -ĠFore st -Ñĭ л -Ġfif teen -Ġbol ts -Ġ{ \ -¬ ´ -Ġsett lement -Ġl ange -Ġb am -G et -íķ Ļ -Ġsw ap -ĠK han -Ġcomm ence -Ġquar antine -Ġsc ored -ç ĸ -Ġ19 50 -Ġthick er -Ġsû r -åı £ -ĠLar ry -Ġall ez -ìĭľ ëĬĶ -Ġg ü -Ġspect acular -/ / -b oth -Ġst ats -å¦ ³ -ĠN ancy -Ġbun u -Ġcr ust -Ġactiv ated -Ġê·¸ë ŀ -out he -Ġport s -Ġne ural -Ġj aw -Ġobserv ations -Ġvo it -ab an -ả i -¦¬ë ¥¼ -om es -௠ĭ -qu i -Ġkind ness -Ð ij -Ġ4 1 -Ġmoder ate -Ġang els -ĠT amb -è t -Ġch lor -ĠBill y -ì² ĺë -ac on -Ġselect ing -ĠDel ta -Ġn ull -den ly -Ġci ud -Ġtend ency -Ġbreak down -Ġm int -ÑĦ оÑĢм -or ph -Ġda wn -s pr -ĠW ILL -äch lich -Ġpu ppy -7 00 -Ġà® ¤ -Ġfail s -ĠCon c -Ġrel atives -Ġinv iting -Ġaut onom -Ġcomp osed -Ġun ity -Ġdec is -Ġaccess ories -ĠC ass -Ġb ist -ĠT ip -ì§ ¸ -Ġp unt -Ġr áp -éĢ ² -AN K -ãģ ļ -ex ist -Ġcompat ible -Ġn er -Ġе мÑĥ -Ġa plic -Ġb apt -Ġfail ing -ĠTam am -Ġos cill -Ġletz ten -Ġrepeated ly -Ġjung le -ĠP ush -h ai -ĠÎ · -Ġdead ly -Ñı ж -wi Äħ -ĠComm on -ĠÎ ķ -Ġsk ate -T C -ĠMin i -Ġhob by -ầ n -Ġrout es -Ġam igos -Ġcon jun -Ġpartners hips -Ġno vo -Ġa ver -Ġpou vez -br idge -Ġpre oc -h im -Ġtur b -Ġso b -ĠSn ap -Ġì° ¸ -min ute -Ġtra ject -uj ÄĻ -Ġe ager -Ġregul atory -Ġbank ing -b ling -ÑĪ ÑĮ -a ż -Ġbiz arre -it ated -d ire -Ġthreat ened -Ġsh ining -Ġn esse -Ġcor ps -ĠÑģ Ñĥ -Ġt eles -Ġtem p -t em -Ġк ан -Ġfe ver -N ew -Ġheav ier -ĠS ah -b ud -Ġout ros -Ġì° ¾ -Ġëª ħ -arr ing -Ġê´ľ ì°® -ĠN ap -Ġse min -ĠTh an -if s -Ġdes en -ĠÑĤак ое -Ġlos es -ĠB alt -k on -Ġнап ÑĢ -Ġvo is -ĠMosc ow -Ġch airs -h is -Ġrefuge es -k g -Ġk ole -į ¨ -аÑģ ибо -¦ ½ -ĠUn iverse -ĠDire ct -Ġche ating -ĠC in -Ġpat ri -Ġadv ise -ĠN ether -Ġprime iro -Ġmention ing -n ut -5 6 -ar ı -Ġpet ite -b led -Ġpens ar -ic io -IN D -Ġveter an -Ġlad der -Ġconsequ ence -ож ал -ĠB urn -Ġr ug -ĠM ade -Ġg it -" ... -Ġcompet itors -Ġprz ed -Ġapp arent -ĠArgent ina -ĠWork ing -Ġcollabor ate -w oman -Ġret ain -Ġle urs -Ġdash board -×Ļ× ĵ -ĠEar ly -B M -Ġе Ñij -ол ог -Ġsatisf ying -Ġoft entimes -Ġma pping -ünk ü -ar th -f old -Ġlaunch ing -Ġa ura -Ġprec ision -work s -G od -Ġstra p -ĠIm per -Ġr ivers -Ġ | -Ġcu er -reg on -Ġarri val -ка Ñħ -ĠM iami -ан Ñĭ -Ġsurviv ors -ĠSen ior -Dav id -Ġest ado -Ġse ctors -Ġpop ping -Ġch im -ay ı -Ġkun nen -Ġgall ery -Ġsun light -ese hen -Ġye lling -ĠMe in -ĠPho enix -Ġman o -Ġhistor ia -Ġoccur ring -æ¬ ¸ -ì ¸ -ад и -å¾ ħ -Ġinstitution al -ĠT ut -ç ² -Ġsl aves -ãģ© ãģĨ -Ġforg iveness -Ġtw in -ĠHy un -н ÑĮ -ĠK omm -and ra -sh ot -ss ä -ĠÑĨ е -at ta -Ġexp ense -ĠG PU -ĠP ast -rib ly -ĠëŃIJ ìķ¼ -Ġгод а -Ġresp ir -æĿ ± -ĠQue ens -h ops -Ġs érie -Ġpre f -Ġcom ed -Ġpl ut -ĠOver all -Ġãģ Ŀ -Ġc ush -Ġring ing -Ġincor rect -ĠÑģÑĤ ÑĢ -Ġgeomet ry -Ġadvert is -ĠÐ ¨ -Ġreview ed -ãģĤ ãģĤ -Ġdo zens -Ġdeterm ination -ĠPh ill -Ġcontrib uted -ĠC it -Ġpass engers -Ġcôt é -Ġre ver -Ġtechn ological -Ġall en -Ġr aining -av i -Ġsal ty -Ġtyp ing -ĠÑĤ е -Ġt ilt -Ġì¹ ĺ -Ġо ÑĢ -ĠпÑĢ Ñıм -Ġr ou -Ġare na -ar at -åĪ « -HH HH -Ġmanufact urers -ĠEd ward -Ġt uck -Ġbl ows -ing o -ĠMar c -ìķĦ ìĦľ -M ich -ĠCle an -è ´ -est o -ĠP ack -Ġsha ft -BRUN O -Ġa ven -u ur -Ñģк олÑĮко -ê´ Ģ -Ġautom ated -Ġvent ure -Ġsurve illance -ĠG row -ĠE mer -Ġд оÑĢ -Ġinvest or -ĠY ok -Ġl atter -ĠN I -Ġfunction ing -ĠHam ilton -Ġ5 1 -Ġmurder ed -Ġanch or -Ġc uc -ĠSC P -ĠMad am -Ġconstra ints -Ġb arn -ank en -Ġë§İ ìĿĢ -ĠMot or -ĠDo ing -Ġam en -et ts -Ġinst ructor -eg t -ak o -Ġpost ure -iv ia -ĠPol ish -Ġдв а -Ġcolor ful -Ġel bow -Ġpar le -Ġpass er -Ġcond em -ort al -Ġfert il -ا د -ĠCol omb -Ġalign ment -Ġastron aut -ĠM ut -Ġsal mon -Ġstructure d -ŀ ר -Ġclick s -Ġm iej -æĶ ¿ -ãģĦ ãĤĦ -ĠR ound -Ġrain bow -ĠV A -ãģĶ ãģĸ -ì§ Ī -ot z -, -Ġch ords -ĠSand ers -Ġë¶ Ħë -B en -Ġdar über -ili ans -Ġorder ing -ĠMan h -Ġkil ogram -Ġkar ÅŁ -Ġgr asp -Ġghost s -al en -ĠJ edi -Ġб ли -Ġdownload ed -Ġconduct ing -ĠH ak -Ġresearch er -il an -go od -ĠH annah -ĠdÃ¼ÅŁ ün -ĠMess iah -u ity -ion a -Ġprob able -ĠY E -Ġindepend ently -Ġbuff er -b urn -our d -ĠMc K -Ġl ingu -uj emy -еÑĢ ÑĤ -Ġintuit ive -Ġcrack s -app ropri -nt y -Ġge en -Ġl end -Ġcert ification -ID S -un ter -pe es -Ġtr ump -Ġbank rupt -Ġfe as -è Ĺ -Ġdu ż -æ¸ ħ -Ġvirus es -Ġ5 8 -g od -Ġж ел -Ġst alk -I nd -ach i -ĠC F -ĠC ond -Ġsan ct -Ġcont en -Ġfre ed -ĠR T -Ġment ors -ì¡ ± -Ġport able -ĠPaul o -r ane -HA HA -ĠS ection -ç Ĩ -hy un -ĠÎŃ Ïĩ -ĠP ub -ĠInd epend -Ġcomp ounds -ĠÑģ Ñĭ -Ġmess aging -Ġded ication -Ġnot icing -Ġdevot ed -ÑİÑĤ ÑģÑı -Ġsn akes -Ġbattle field -p ers -Ġdel a -9 2 -Ġha i -ill ä -ér er -e very -Ġrespons ive -×Ļ ×ķ -op f -é ī -Ĭ ¸ -Be cause -Ġtour ism -Ġê·¸ ê²Į -×ķ× ¦ -Ġcan s -st üt -Ġdon ne -ĠD ios -ĠU ber -act ory -Ġorient ed -ĠH erm -Ġpat ron -ur f -be i -Ġprogram a -ĠOh h -gen er -Ġf ist -ĠW endy -Ġand a -Ġguess ed -Ġfre ak -ä¸Ń åľĭ -ĠK ings -ch ool -Ġoff line -ĠIndian a -ĠAll iance -Ġ5 3 -Ġpartic ul -ĠF ocus -Ġinhab it -Ġê°ĻìĿĢ ëį° -ĠMc G -ows ki -ĠìĿ´ ê±´ -Ġpa ÅĦst -он и -itt a -Ġconfirm ation -ĠBrook lyn -Ġnood le -f und -it ud -Ġgrand parents -Ġbar becue -ει ÏĤ -Ġ á -Ġball ot -ĠV eter -Ġpip es -ig ious -ĠG raph -est ed -Ġë¸ Įë -ĠK E -ãģ¡ãĤĩ ãģ£ãģ¨ -Ġe ins -Ġhat red -ãģij ãģ© -Ġd ang -ee ee -Ġarch ae -ĠJes se -Ġdetect ed -Ġsen i -burg h -Ġdispl acement -Ġdo p -Ġcondition ing -Ġне ÑģколÑĮко -Ġdistur bing -P H -Ġthin ner -Ġwound ed -ĠCu ando -Ġcush ion -Ġwh ites -Ġprefer ences -Ġì¤Ģë ¹Ħ -Ġka ż -ĠG ate -ĠP ath -d les -à¸Ħ ร -im ore -Ġë³´ìĹ ¬ -Ġdiscipl ines -á» ı -Ġmes ma -Ġìĥ Īë -Ġìĭ ¬ -Ġg ing -Ġumbre lla -IGH T -Ġp ension -Ġcomb ining -S S -Ġrect angle -á»ĩ t -Ġpro xim -ĠC ow -¸ Į -Ġintention al -æķ Ļ -Ġdec id -ĠÑģк аж -ĠU ma -ias m -b uz -Ġdebr is -Ġc ass -ĠP rop -is ka -ë ł¥ -ester ol -uss ian -ìĿ´ë ŀij -Ġun limited -Ġadm ire -Ġtight ly -Ġgen ome -ĠJun ior -ven ir -g us -Ġc Äĥ -ĠV lad -Ġí Ĥ -Ġrel ativ -in ci -Ġaun que -ĠBo ys -ÑĨи он -ĠSw iss -Ġphys icians -Ġíı ī -ĠP ET -Ġw ounds -ab out -Ãł i -on z -ur ities -ĠÑĥв ид -å· ¦ -Ġment ality -Ġvari ance -Ġseg unda -Ġvol cano -al ie -ॠĩ -Ġt iles -ĠT erry -ĠاÙĦÙĦ Ùĩ -Ġcan on -Ġsc attered -pt on -Ġdefin itions -Ġal gebra -ot en -ab lo -ij uana -Ġwra pping -Ġses ame -ĠнаÑĩ ина -ĠAl f -ĠÐł оÑģÑģ -or no -Ġan kle -Ġspecial ty -Ġattempt ing -ili ation -Ġ19 20 -Ġphen omena -ĠPro duct -ĠB uck -ĠA ww -se en -Ġvo id -ĠFrank lin -Ġadvoc acy -ĠS ep -Ġcool est -ĠÑģ ÑĢазÑĥ -ĠQu and -Ġ9 00 -ĠTr ad -d ies -Ġhas h -æĪij å°± -ä¹Ł æĺ¯ -Ġpot s -Ġsad ly -Ġvi able -ĠT iger -ĠON E -Ġneur ons -ow anie -Ä Ĺ -ĠSh ar -ĠLand es -Ġconfer ences -è© ² -Ġcred ential -Ġl ime -ine e -x it -p ay -Ġinc ons -Ġ>> : -èª į -Ġí ŀĺë -Ġless er -Ġsp ill -Ġprem ise -Ġ36 5 -ĠH ost -Ġtom ar -×IJ× ľ -ë ²Ī -ĠWhat s -Ġlight weight -ĠM ap -f ia -ells chaft -Ġvend ors -uest o -ĠM ister -ĠÐŁ ÑĢи -åı ³ -h ma -Ġintention ally -ĠT ang -éĹ ® -Ġident ification -Ġetc etera -ĠN ee -ĠÑĤ ÑĢи -ê· ¸ -Ġcrypt ocur -Ġin hale -Ġadd ict -åIJĦ ä½į -Ġma u -ĠÑĤак аÑı -Ġë² Ħ -Ġcomp rar -ied zieÄĩ -ĠоÑĤ но -Ġbegin ner -Ġм Ñĥж -Ġobs c -Ġlim iting -asc ular -Ġins pection -ac i -Ġre jo -M us -Ġz aten -Ġsz cz -ĠMad rid -Ġvar ieties -Ġest Ãł -ĠSh akes -Ġk its -Ġad minister -Ġla va -Ġg Ã¥ -è© ¦ -ת ×Ļ -ĠWay ne -Ġinst agram -Ġr ated -p aper -Ġb ild -Ġpret ending -Ġobser ving -ĠÑģам ом -Ġtr or -Ġorgan isms -Ġfal ta -Ġh ometown -ç ± -Ġí ĭ -Ġche g -Ġì ¡ -Ġcomm a -is é -Ġlike lihood -av ored -Ġgel di -ни ков -Ġmed io -Ġjak ie -ĠJ up -Ġgreen house -Ġsp it -ко е -Ġк аж -ĠG ram -ĠCon ference -Ġdef icit -s ın -in se -u ÄŁ -Ġr icht -Ġcoinc idence -åı į -Ġeu rop -Ġbutter fly -p read -Ġìĸ ¼ -èĢ ¶ -Ġwa vel -ĠIn fin -ĠPlan et -Ġself ie -ient ras -Ġar rog -os er -id al -ł×Š׳×ķ -üt ün -Ġfresh man -ĠMach ine -Ïĥ ÏĦ -ĠD ia -ìĿ´ ëĭ¤ -ãģĵ ãģĨ -ne a -Ġlist ing -Ġconfig ure -ut or -U p -ts chaft -ri ère -Ġup wards -ĠÑħоÑĩ Ñĥ -Ġswe ep -B r -Ġexpress ing -Ġun happy -Ġmand atory -g ender -ĠA ÃŃ -Ġindic ators -Ġoil s -n ote -Ġseg ur -ож еÑĤ -yn asty -Ġdist ances -Ġmer ge -BER T -Ġsur render -Ġbu at -ĠA wards -Ġseñ or -od ox -Ġfl avour -Ġab dom -Ġconfig ur -8 6 -ĠDI Y -Ġrig id -° ĺ -Ġcorpor ation -Ġg room -j aw -ĠNe ar -ил о -Ġoper a -ĠIn nov -и ÑĢа -ĵ ± -Ġspec ified -Ġcos m -ĠFre edom -Ġcl own -ĠN em -Ġв ол -Ñij н -Ġchar ger -à¹ģ ล -Ġinflu ential -äs ident -é ¤ -ĠìĦ łë -Ġvol umes -æ IJ -Ġout ras -ĠTw itch -Ġfound ing -Ġa while -Ġco il -ê° Ļ -Ġc ả -ĠTh row -ĠH ence -omm t -ĠBen jamin -глÑı д -T ime -ob ic -Ġm our -Ġd read -ĠL Ãł -ĠCh ile -Ġpre val -Ġv ain -Ġart ık -Ġpres erved -ĠоÑĤ д -Ġware house -Ġbest e -ĠSever al -ĠS ituation -Ġcard board -T od -er na -Ġgar ant -Ġgest ure -Ġh en -Ġspe lling -ose xual -Ġan ne -Ġm ice -ĠMe ine -c ard -Ġre bell -Ġcert o -Ġìľ łë -Ġvers chied -ĠB os -Ġinv ention -Ġtr ze -Ġman ière -ĠCh ad -Ġsp re -Ġorganis ations -Ġpoor ly -Ġan terior -Ġst air -к ÑĢ -Ġatom ic -Ġsymp ath -Ġcontin ually -Ġkle ine -è te -и Ñī -ο ÏĤ -pe ut -Ġrep osit -Ġent ra -E m -Ġfinan cing -Ġмн ог -Ġthe sis -ĠCom puter -e au -ĠT ree -Ġbr ide -ons ieur -sh ire -w ic -D E -ĠìĪ ĺë -Ġac om -ĠP O -ers ch -Ġпом оÑī -ĠAr men -Ġì£ ½ -Ġz or -Ġprint s -ĠD ass -æ¸ ¯ -Ġdur able -ĠTrans port -ìŀIJ ê°Ģ -Ġл ег -Ġdé t -ô le -am ous -Y N -Ġcl iff -Ġgramm ar -ĠÐŁÐ¾ ÑįÑĤомÑĥ -ĠlÃł m -es ch -Ġmiser able -Ġvol ts -ĠC ad -uk an -ÑĤ ив -r ust -Ġìĺ¬ë Ŀ¼ -Ġver k -Ġchick ens -ĠY oo -Ġout fits -c ode -Ġhier archy -net es -Ġcounter part -Ġt ôi -Ġt ed -ĠB art -Ġë Ŀ¼ -ĠGen au -Ġinc oming -ĠA BC -ri que -ĠоÑĤ п -qu al -Ġincent ive -Ġih ren -׳ ×Ļ -lo e -Ġ19 30 -Ġbar g -Ġd iction -Ġön ce -IN S -Ġre h -isia j -m outh -Ġsc oring -l ık -ĠìķĦ 주 -OR IA -ĠEst ados -Ġcompan ion -Ġasse mble -Ġpun ished -Ġit al -Ġprev ents -ist es -ĠKent ucky -Ġloc ate -Ġfast ing -ãģ¨ æĢĿ -ĥ Ģ -ĠSe b -ĠCr own -op ia -Ġwh ip -us z -к ами -Ġdatab ases -åŃ Ĺ -Ġprose c -Ġ199 7 -ĠìĤ´ì §Ŀ -ĠSol ar -ĠP ues -ĠZ en -oll o -ĠG uru -Ġsque ez -ĠÐĹ Ð° -ĠÄ į -cept ions -c ca -iz able -m and -Ġbreak through -Ġtables poon -ĠS EC -ik h -ĠS ão -Ġп ло -am en -Ġpr ac -Ġdar ling -Ġtall er -Ġrend ering -Ġìļ°ë¦¬ ê°Ģ -ĠÏĦη ÏĤ -Ġm ã -Ġes os -uer do -ĠÑģ ÑĩиÑĤ -all er -ìĹĪ ìĸ´ìļĶ -Ġmill ones -ler in -Ġpe gar -on ne -Ġenroll ment -Ġli egt -Ġbo a -w iÄĻ -bs p -Ġcy cling -ĠBern ie -Ġ198 9 -Ġд алÑĮ -ĠDak ota -ĠÑģв Ñıз -ĠC P -Ġst are -íĤ ¤ -Ġprosper ity -Ġarrange ments -Ġarri ving -m ä -Ġkay ak -ip t -Ġp ardon -Ġrel at -Ġver ste -ĠF ig -Ġfo il -ĠTalk ing -pe are -Ġno i -ĠпÑĢи ÑĪ -Ġhoc key -Ġad o -ĠO UT -6 7 -Ġhorm ones -ĠAven ue -ĠSuper man -Ġpres cription -uber netes -C L -ot ive -N IS -ien en -Ġsad ness -ĠV it -T y -Ġstar ter -Ġbed e -Ġfound ations -Ġso re -åº Ĺ -Ñīе ÑģÑĤв -ìļ °ë -ĠÑĩ Ñĥв -l ink -Ġmane u -work ing -Ãł n -ĠAtt ack -ĠC art -ve is -ĠRes p -ens ing -Ġì¢ĭ ìķĦìļĶ -Ġesc uch -ĠR NA -Ĥ ´ -Ġad op -Ġb ending -ع د -Ġman ages -us p -Ġt art -Ġrout er -B o -Ġestab lishing -Ġbal ancing -Ġathlet ic -ĠS lo -Ġf ills -Ġн аб -Ġд ал -Ġpos so -ĠV ielen -Ġcrit ics -Ġlaws uit -ĠIsa ac -ĠÑĦилÑĮ м -Ġtr as -Ġpra w -ĠCra zy -Ġne u -Ġk ull -Ġtum or -ĠAP P -g ate -ĠA RE -9 8 -ĠSte am -Ġfuck ed -l age -ĠâĻ ¬ -ĠM D -f y -Ġshell s -ĠSe ems -iz ers -Ġr anges -ĠAnton io -AT ION -ĠB aba -Ġìĥ ī -k un -Ġpray ed -ÑĢ Ñı -ĠпÑĢоÑĤ ив -Ġse as -b ury -Ġ×Ķ× © -Ġtra it -ĠDep ending -Ġd re -Ġkön nt -ÑĨ Ñĥ -Ġlip stick -ee z -ĠпÑĢ имеÑĢ -Ġassign ments -B ob -Ġmet als -Ġspe cially -å°į ä¸įå°į -Ġìĺ Īë -ĠÅ ¡ -Ġv ista -ĠÎ ¬ -Ġtw ins -Ġnot able -ĠS au -Ġdé velop -Ġç ek -Ġpoly nom -av am -Ġtamb é -он ом -Ġpl asma -Ġe fect -Ġlä ng -Ġcas i -Ñģ а -ım ı -ãģĻ ãĤĭ -ĵ¤ ìĿĢ -Ġlab our -oss en -ĠP un -r if -Ġd oses -Ġoper ates -ил ли -Ġja ar -st aw -ĠìĤ¬ëŀ ij -Ġat m -Ġprotect s -Ġimp ed -H O -Ġc ima -Ġto ch -ab is -Ġsend o -la us -Ġcur l -ĠN um -Ġspons ors -Ġdé but -ĠAlex a -ĠB ür -ĠA mer -Ġc ope -Ġиз в -j al -Ġ199 5 -ap at -res se -ĠPri ze -ĠCla ire -ĠBrand on -Ġwszyst ko -Ġval ued -à¸Ļ ะ -Ġse ct -Ġsecret ly -Ġdiam onds -ĠEv an -ĠRP G -ãģ« ãģª -Īë ıĦ -ĠUnivers al -Ġdoub ts -ĠP in -wiÄħ z -ļ © -Ġal bo -Ġbra ucht -AU L -ĠM obile -gr ades -Ġsch em -wh y -ĠN icht -p i -g le -Ġchor us -Ġg ly -Ġrein force -Ġm uff -ĠSh en -ĠH ola -Ñĥ г -vid emment -v ial -ac ious -laim ed -ĠR ico -Ġve gg -Ġillust ration -ĠBut ter -ow ad -Ġeu x -Ġenf ants -ĠLe ader -ĠVill age -et ically -ÙĨ ÙĬ -Ġst ew -Ġsurpr ises -Ġc ue -ĠGrand ma -ĠC elsius -ĠR icht -en c -Ġpet ition -Ġher b -Ġw icked -Ġsch le -oc aly -Ġtrans f -Ġtok ens -ĠGr ay -ĠB BC -I K -Ġ15 00 -z n -ĠNe v -Ġk oy -Ġz ar -Ġbull shit -ĠColomb ia -ul ative -Ġwides pread -y ect -k it -Ġempres a -Ġn our -Ġburn s -at in -a ired -Ġrevolution ary -Ġгод Ñĥ -ĠLog an -Ġ199 6 -ĠGra ham -re b -ĠN HS -æľ Ľ -Ġcost umes -Ġnaw et -Ġlo vers -ĠLuc y -ĠInd igenous -íķĺ 기 -Ġimmun ity -¥ ´ë -uit o -Ġexcess ive -Ġdon ations -Ġ×Ķ ר -Ġì² « -éī Ħ -Ġdry ing -mel on -Ġsurve ys -Ġ무ì Ĭ¨ -é¢ ¨ -aa a -Ġpro be -an cial -Ġlou der -Ġhot els -ü ÄŁ -ag ner -Ġorig ins -Ġë§Ī ì§Ģë§ī -Ġ* * -Ġstr angers -ĠHa us -com ed -Ġan throp -Ġus o -ĠìķĦ ì§ģ -ĠY uan -ĠíķĦ ìļĶ -pl er -ress ive -Ġsp raw -ĠSt ew -Ġ199 4 -Ġeld ers -Ġme inen -Ġj unt -Ġac oust -ĠW ohn -Ġban anas -Ġproject ion -ĠSt ick -leg t -spe ed -ĠcÅ ©ng -ĠW ort -ĠBalt imore -ĠÑĨ ел -Ġdun no -å¼ · -? , -ãĥī ãĥ³ -ĠLoc al -ost o -Ð Ń -од а -ĠPort uguese -Ġtheir s -Ġdé m -åı ¦ -Ġdra uf -ĠBuddh ist -ert a -G e -Ġcar rot -ĠWonder ful -Ġso ak -Ġchair man -gg i -IC A -f ried -Ġfl ick -ĠThrough out -Ġìļ °ë -Ġc ough -Ġfl uffy -sch ool -Ġr ipped ----- ---- -ĠZuk unft -Ġн еб -Ġst o -ĠB O -p ent -ĠLaw rence -Ïī ÏĤ -st icks -ĠE ins -ĠÑĢ Ñĭ -ĠStr ong -Ġcar amel -Ġsp ite -az ar -éĥ½ æĺ¯ -Ġcrit ically -Ġob ra -ow itz -ĠZ one -ĠÑĢ ек -Ġsu g -ard ed -Ġg ì -ff entlich -an che -Ø Ł -ast ically -ìĿ ¼ë -л ав -Ġsimpl est -ĠF riend -Ġque llo -Ġamb ition -Ġabb iamo -åº ķ -ĠÑĦ оÑĢм -ĠEs sa -Ġeduc ators -Ġstatist ical -éĢĻ éĤĬ -Ġchang er -Ġat au -éta is -ĠShakes peare -ë IJĺ -Ġtr iggers -Ġreal iz -Ġcel ui -whe el -Ġloyal ty -Ġscream s -ke hr -ĠM ega -e ast -Ġtop s -ĠTot ally -ount ain -l ord -Ġviol ation -ĠG A -Ġnic er -ĠF resh -ĠMel issa -fun ction -Ġra pe -Ġexcept ions -Ġsil icon -Ġliber ty -Ġhousehold s -ãģį ãģ¾ãģĻ -ĠC A -ĠÐŀ б -Ġli b -ŀ Į -c ific -Ġtrop ical -Ġinvestig ating -H D -Ġad apter -ĠP itt -an cia -ĠShe ll -friend ly -Ġconclus ions -Ġtur tle -Ġdec omp -Ġanim ations -ĠÑģ ек -ins i -Ġret ention -k ie -Ġinject ion -ĠMad ison -ì° ° -Ġv ient -Ġvar ied -Ġviol in -ĠB il -Ġluck ily -Ġh tt -l ä -Ġr anch -çľĭ çľĭ -Ġsó lo -ìķ ħ -ĠD erek -ĠScript ure -оÑĢ а -Ġclassroom s -av il -form ed -Ġbefore hand -ĠG em -pre ch -Ġl in -Ġgre ens -ÑĨ ев -ĠMer cedes -Ġdr ought -gas ps -Ġab ortion -Ġter ribly -Ġspos ób -Ġsec ured -Ġat rás -Ġwavel ength -Ġgra ins -ect ive -Ġspace craft -Ġtour s -Ġprof es -Ġsur geon -ĠP ie -Ġide ally -arn er -U P -op ard -s ce -Ġimm ense -ĠOr t -roll er -ĠD allas -ĠNich olas -Ġs ulf -ĠToy ota -Ġquant ities -ce ans -Ġcu i -an ça -ĠC AN -itzer land -åĦ ¿ -Ġz ou -ĠCy ber -le gen -ĠIn it -ed u -Ġa pert -Ġad jac -ou v -èĢĮ ä¸Ķ -r s -Ġcab bage -Ġwheel chair -iny l -ĠD ynam -ĠìķĦëĭĪë Ŀ¼ -Ġl ing -h l -Ġмог Ñĥ -Ġcris p -Ġm ij -Ġd ug -n in -Ġbl oss -Ġbelong ing -Ġloud ly -Ġminer als -Ġconclud ed -Ġsearch ed -9 6 -ĠMe et -ĠS EO -ĠС к -ĠH ob -ot ta -Ġpropag anda -Ġcin namon -Ġhun ter -Ġgeme ins -Ġsculpt ure -uls ion -Ġv äl -Ġmagaz ines -Ġcontrovers y -ä¸Ģ 樣 -Ġsequ ences -ãģĦ ãĤĭ -Ġíļ Į -Ġdel eted -ä½ ¿ -IJë ıĦ -Ġvary ing -ãĥ Ĩ -Ġmount ing -Ġaff air -Ġpath ways -æ ¦ -Ġdig o -äº ® -Ġд ок -A lex -Ġtob acco -ĠC V -Ġbother ed -Ġamb ient -ink y -ĠS L -Ġh ates -Ġje żeli -Ġcon greg -Ġel as -Ġde uts -ĠStud ios -ch ÄĻ -Ġdocument ed -ĠCru z -ĠL en -ĠDoug las -ĠPort ugal -ent i -Ġsp ouse -Ġanal ys -av ia -Ġed ited -Ġl ại -bu ilt -Ġv ille -ad ora -Ġbrac elet -Ġs ushi -Ġp m -Ġtra ils -Ġl ug -Ġö ver -Ġs orrow -Ġcol ony -ado x -Ġser ie -any ak -ĠØ · -ĠG ulf -æĺ¯ ä¸įæĺ¯ -ĠP V -ĠSam uel -ĠK it -ĠR al -ont in -ex pl -Ġent ries -Ġactiv ists -P s -Ġs ant -ĠÑĤо Ñĩ -ĠBr uno -ke ley -Ġtut to -é Ķ -Ġv intage -Ġterr ified -Ġпо Ñħ -us ive -ow ers -ай ÑĤ -ë ıĻ -Ġtwist ed -ĠTh ought -Ġt ah -Ġshr ink -Ġshe er -l it -Ġdal am -Ġd ib -Ġv ard -ow ane -Ġdo br -ĠR ena -ĠÑģво Ñİ -ĠpaÃŃs es -ĠE ra -ãģ® ãģ§ -ĠB UT -s ighs -Ġê·¸ ê±° -Ġgro ÃŁen -Ġë¹ ¨ë¦¬ -Ġn erves -Ġconst it -Ġpreoc up -ĠG ay -ĠX u -keep er -he ure -.. ) -ĠCal m -ĠUn idos -ĠìĿ´ ê²ĥ -ĠAqu i -Ġìłľ ìĿ¼ -d ır -ì¦ ĺ -y our -ĠÑįÑĤ им -20 20 -Ġr und -ĠH O -ĠC atherine -iel i -Ġf usion -Ġide ology -Ġfor am -sh aped -ĠíĽ Ħë -Ġw t -Ġret r -Ġpr éc -Ġê° ij -Ġopen ly -v ity -구 ìļĶ -Ġobst acle -Ġbo o -Ġse iner -ic orn -Ġeigen lijk -Ġhead er -are mos -Ġso fter -ĠÐŁ од -Ġpre jud -Ġdefin es -ier te -Ġbl ending -Ġbelie vers -ĠWo chen -Ġник ак -ĠÐļ огда -ĠTyp ically -Ġíģ ¬ -ç® ¡ -ci os -Ġmiss iles -Ġsp onge -ĠK itchen -Ġt ren -ning en -Ġsc rap -Ġser ait -´ì ł -ç ¹ -Ġë° ĺë -Ġrest ored -Ġprzy kÅĤad -ĠK ubernetes -Ġsa it -Ġu w -Ġen abling -Ġtra vers -amp s -åı Ĺ -ĠOM G -ens or -Ġz osta -Ġpronoun ced -A ng -norm al -Ġeconom ies -t in -ĠChamp ion -iz en -Ġar beiten -ĠG ospel -ĠZ u -ng a -Ġliter acy -ĠM ans -Ġcircul ation -Ġad ap -ĠTot al -Ġmere ka -Ġol acak -ÑģÑĤ аÑĤи -J ack -Ġm und -Ġth ief -b ies -Ġê² ģ -a que -ĠÚ© ÛĮ -ĠSc ar -å ² -Ġab ol -Ġdev ote -Ġ0 1 -Ġs itten -ĠVis ual -we ek -s ome -ing t -Ġjournal ism -ĠH ir -ĠB achelor -in ery -Ãľ ND -ãĥ Ł -ç» Ļ -Ġcolor ing -ĠCr ist -Ġcelebr ities -ĠÑĩ иÑģ -ĠC rit -Ġdifferent iate -ĠÐľ не -el im -Ġse afood -Ġalgum as -otherap y -æĪ ° -Ġgla ub -Ġarbitr ary -g ens -ĠбÑĥд ем -Ġt av -Ġcream y -ĠCount ry -a ñ -м еÑĤ -Ġh inter -Ġm ism -Ġillust rate -ÃľND NIS -Ġdecre asing -Ġwen iger -AK I -ix on -Ġн ей -Ġfat to -Ġn erd -ç ł -Ġb itte -P er -Ġt ane -Ġgö z -Ġfor te -ĠE y -Ġнав еÑĢ -è¢ « -ĠWord Press -ĠM is -Å ¯ -z äh -Ġinté ress -osa urs -ĠFall s -Ġn essa -9 7 -Ġmuseum s -Ġcorrespond s -Ġs ings -f our -Ġed er -ĠCommun ist -o a -ne k -ĠWH O -Ġcor po -Ġmess ing -ÏĦ αι -Ġbrush es -Ġb isc -ĠAr beits -ĠT ax -Ġse le -Ġflag s -ou pe -Ġanticip ated -ãĥ ij -ĠN ad -Ġpou red -Ġm l -Ġll ama -Ġvisual ize -Ġlisten ers -ÙĦ Ùĥ -al ten -Mich ael -Ġcos ì -Õ¡ Õ -op us -Ġíķ´ì £¼ -Ġh ike -ĠAtt orney -ĠHill ary -ud ed -Ġíķĺ ì§Ģë§Į -Ġdo ve -Ġstorm s -ак Ñģ -Ġdoct rine -Ġhe x -ik s -no ÅĽÄĩ -Ġscript s -Ġδ εν -ĠÑįÑĤи Ñħ -ĠÐ Ĩ -ab er -ĠV as -Ġcent imeters -×ŀ ×Ķ -ни б -Ġrid ers -ĠT rib -åĮ ħ -Ġtak że -Ġn oun -Ġic ons -Ġsole ly -mind ed -Ġdisp on -ĠSw itzerland -Ġcl usters -Ġqu eda -ail ing -Ġman ga -Ġ6 8 -Ħ Ī -Ġt et -g ins -ha us -ç© º -å· ¥ -ĠO P -ot ed -Ġnouve au -AL LY -ÙĪ د -ò n -Ġmort ality -ĠGit Hub -d rop -Ġdis gu -Ġrec om -Ġloc als -Ġhome made -amb a -Ġpron unciation -Ġal phabet -ан ÑĮ -ow any -ir as -id ency -OM E -ĠÑĢаÑģ Ñģ -ar ak -v iamente -Ġnon profit -ĠYouT uber -Ġp arenth -ĠB oo -v at -ĠSt ir -Ġpre cip -Ġan ts -Ġall y -ĠMa ori -ĠëĮĢ íķľ -åı¯ æĺ¯ -og ene -ĠLab our -aret te -Ġrecy cling -ens a -Ġpurs uit -Ġs ak -ĠÐĹд еÑģÑĮ -Ġtoler ance -Ġsa at -Ġclick ed -âĻ ¥ -Ġface book -ĠInt o -Ġincent ives -기 ëĬĶ -ĠD ennis -ĠW ik -ges ch -à¹ĢภĽ -ĠÏĢ α -ĠWh oo -Ġround ed -Ġdo pe -Ġcapt uring -ĠWar ri -Ġcivil ian -Ġchar ming -Ġes as -Ġsust ained -Ġle aning -Ġabund ance -ÃŃ lia -алÑĮ нÑĭй -Ġph ải -ac ja -Ġê°Ļ ìķĦ -act iv -า ย -Ġ9 7 -Ġм ой -c ro -ĠJack ie -itt ees -br acht -ul ent -Ġìł ľë -Ġplug in -v antage -part y -Ġsu as -Ġan te -Ñĥ л -ÐĿ ÐIJ -æĤ ¨ -ĠÏĥ Ïħ -Ġmet h -Ġenthus iasm -ÑıÑĤ ÑģÑı -íĻ Ķë -Ġsynth etic -Ġseason ing -ĠL ost -on omy -ĠSp ark -Ġb ure -Ġass ured -Ġimag in -Ġcar ro -S ha -Äħ t -нÑĥ ÑĤÑĮ -át ica -T Y -Ġk ern -ĠBrazil ian -à ° -Ġsusp ended -ĠCar ib -Ġbiz im -ĠOl iver -ãģ ¶ -T om -Ġпл ан -Ġn ope -omet hing -Ġbe iden -ÑĨ ен -Ġflu ct -Ġμ οÏħ -Ġf athers -ĠBl ake -Ġup ward -ĠD ash -ĠL il -ĠìĪ ĺëıĦ -Ġrevel ation -Ġelev ated -ĠJi ang -LE D -ĠThom pson -Ġмог ÑĥÑĤ -ÑģÑĤ ÑĢÑĥ -if iers -Ġcome back -Ġbuy ers -ê² ° -ĠS ales -иÑĩ е -c iones -Ġwh istle -Ġd ull -LE X -Ġíķĺ ê²łìĬµëĭĪëĭ¤ -Ġcrimin als -Ġdes cent -ipp le -mas ı -Ġfool ish -ĠдÑĥм аÑİ -t ar -Ġman go -Ġchore ography -M att -Ġterr itor -Ġac aba -ĠEin stein -ĠI BM -ĠMet al -ĠCry stal -Ġr ah -Ġf oul -ĠIsland s -Ġint act -ĠR ail -. : -Ġac á -ĠпÑĢ оп -еÑĢ е -ĠWr ite -he he -ĠF O -ĠÏĥ ÏĦη -Ġdo in -h eld -Ġappropri ately -Ġdeliber ately -Ġarch ive -Ġgive away -ãģĵ ãģĵ -Ġfin ale -л аÑģ -ен о -Æ¡ n -æ£ Ĵ -og o -çī © -ĠAud ience -ãħ ł -Ġsub ur -Ġhead ache -ан нÑı -ĠW itch -ĠSwed ish -ĠB I -Ġer ase -Ġk hi -Ġcomment ary -ĠS ultan -íĥ Ŀ -ĠLe ban -Ġë³´ì ĭ -ĠP am -pe kt -mon th -Ġground ed -ê ¾ -ĠÅŁek ilde -2 50 -ĠS CH -ios o -Ġin aug -he imer -Ġreflect ing -ĠR uth -ĠO il -Ġtrou ver -u ep -.. ] -Ġìŀ Īë -Ġol ha -Ġreason ably -Ġgl itch -U B -ĠGr an -Ġad alah -Ġl ent -ر ا -Ġtr action -Ġadjust ing -´ ¤ -ниб ÑĥдÑĮ -Ġд оп -Ġstretch ed -Ġor t -Ġcos ine -vi ol -Ġì ħ -c ir -Ġbast ard -ä¸ ĩ -ĠÑħ од -Ġqu ier -Ġpress ures -ĠAn h -å¹ ¾ -Ġell es -Ġд ÑĢÑĥз -ĠможеÑĤ е -Ġch á» -ĠM é -ö k -ầ u -ìł Ī -z in -Ġca ution -ib an -Ġjud ging -ÑĥÑİ ÑĤ -Ġb aj -ĠС ейÑĩаÑģ -ĠPo or -ĠNaz i -Ġup beat -y ang -Ġweek ends -ĠEss entially -Ġol uyor -Ġspat ial -ack er -Ġsell er -Ġ×IJ ×ķת -ij ׾ -Ġv ivid -ĠB ond -ê ¶Į -is kt -ãĤ µ -Ġgo at -dri ver -Ġm ug -ict ional -Ġall t -ĠIn iti -ĠR and -Ġfinish es -Ġê° Ī -Ġvit am -Ġteen agers -ĠMor ris -ì¤ Ħ -ĠO ri -i ya -Ġmy ös -St ep -ĠK re -è¾ ¦ -Ġdin osaur -Ġëª ĩ -aff e -ĠëIJ ©ëĭĪëĭ¤ -Ġz eg -åĪ ĩ -ĠManh attan -Ġsu jet -ue lle -st off -Ġd ür -Ġsub mar -es es -Ġa quele -Ġn ou -ĠFa ith -t z -ĠÑĤ омÑĥ -ace ut -li ers -Ġband width -Æ°á» Ŀ -Ġrespect ive -ĠA ve -Ġspread she -ĠS ent -ic amente -Ġinf ra -Ġlearn ers -Ġà® ī -ai ah -ren al -Ġmust ard -Ġhab t -ç ĥ -ĠQu é -Ġanaly zing -æ¯ ı -Ġso lic -Ġ×Ķ ×ķ×IJ -Ġcaus a -Ġwel comed -ĠS uccess -Ġfac ile -ĠÐŁÐ¾ÑĤ омÑĥ -sche in -Ġf etch -Ġstr at -ĠÑģÑĤо иÑĤ -ìĹIJìĦľ ëĬĶ -ĠÑģп оÑģоб -m am -Ġser ÃŃa -nam ents -wr iter -Ġconsult ing -íĺ Ģ -ĠBer keley -e u -as ive -U U -ĠAnal yt -Ġsubm ission -Ġmagnific ent -en za -Ġe con -Ġprof iles -Ġinc ar -A b -ĠN un -Ġh ic -scream ing -Ġresil ient -åĪ © -gr und -Ġconc ur -Ġbere its -L D -Ġnur t -ì ī -Ġfe ast -Ġenc uent -ĠMich el -Ġsup rem -" ] -Ġfeed s -ĠKoll egen -iss er -ĠF eng -ĠW en -m un -Ġten ÃŃa -ĠW rest -Ġìĺ¤ëĬĺ ìĿĢ -Ġst ead -Ġrest oration -Ġdon ated -Ġdel s -Ġc ensus -Ġdesper ately -worth y -H E -ĠSp a -ĠBry an -Ġh j -ĠR aw -ìķĦ ë -ĠCam era -Ġz ien -Ġst yl -ĠT W -ĠChe ese -bor ne -Ġob l -ĠAl ready -Ġunst able -Ġfl ames -p ost -H a -rom agn -ĠìĹ Ħë§Ī -d est -Ġkole j -Ġtempor arily -Ġdeterm ining -ĠGl ass -ÑĢ он -ol an -Ġdom inated -åĮ ĸ -__ __ -ĠÙĩ ذا -ĠD ana -Ġdin heiro -a qu -ë ¯¼ -ĠÃł s -ĠJo ey -ĠGr iff -Ġatt ain -Ġtrans itions -ĠLiter ally -ен д -ĠHa ven -Ġgrab bing -Ġcryst als -ĠFour th -Ġcand les -ĠÑģлÑĥÑĩ а -ric o -Ġ5 000 -et to -Ġund o -Ġk to -Ġdi vert -Ġch ir -Ġper sec -Ġh iking -Ġannounce ments -çĶ ± -з Ñĭ -Ġa uc -Ġsystem ic -ĠR M -Ïĥ α -ĠÐĶ ж -Ġy ar -ĠW ard -Ġpiss ed -Ġcar n -Ġautonom ous -ãħİ ãħİ -so ver -æ²Ĵ éĮ¯ -å¾Ī 好 -Ġref lex -Ġgard ens -Ġd ated -ì ± -ami ÄĻ -Ġcontinu ity -Ġcitizens hip -Ġsch wer -Ġz ak -t able -ĠÑģ Ñĩ -è§ ģ -ĠÏĥ ε -Ġgener ates -구ë Ĥĺ -ö h -ó m -al am -ĠJUD Y -ĠB ug -Ġãģ ¦ -Ġdr ones -Ġá gua -ac aks -æ ļ -ĠÐļ он -× ĸ×Ķ -Ġstri ve -ĠAl tern -Ġne arest -Ġpro yect -ter a -ĠASH LEY -Ġwor m -Ġre play -Ġt ara -ĠInd ians -ãĤ ° -ica id -ĠìĪ ľ -Ġappe aling -ĠW es -Ġment ions -Ġдел е -Ġk w -Ġfrag ile -is z -k ów -h ang -col or -Ġpresident e -8 7 -е ÑĦ -çĪ ¸ -Ġдоб ав -ĠN elson -á fic -ĠMIC HAEL -Ġmechan ic -Ġmet res -Ġo czywiÅĽcie -ĠC ind -Ġog sÃ¥ -Ġlands ca -AC E -Ġhead lines -Ġcat alyst -ĠC atch -ink les -Ġp ills -ord o -Ġimmig rant -Ġexam ination -Ġacc idents -zÄħ d -Ġqui ere -Ġne lla -Ġ6 7 -Ġpass a -Ġsuper fic -ist or -Ġno v -ëĭ µ -Ġmand ate -is ons -ĠVirt ual -Ġsel ber -Ġcounsel ing -ĠN BA -Ġse pt -Ġbelie ver -Ġmar vel -ĠInte gr -Ġм Ñĸ -Ġor ph -Ġback ward -ĠGen eration -ĠP ict -ĠÑĤо ÑĤ -Ġtap i -pro chen -Ġhall way -ht e -ĠÛģ ÛĴ -ĠZ um -èĢģ 師 -ach ment -iqu er -fol g -ĠEd die -ĠK il -Ġwell ness -st ock -è¼ ĥ -Ġka ç -Ġterror ism -Ġpo inter -O f -her ic -ĠUlt imately -Ġmes es -ĠTr ade -Ġp int -Ġtu ition -Ġdisag re -Ġê²Į ìŀĦ -Ġmanus cript -Ġro omm -Ġoutput s -е ÑĨи -Ġr ies -Ġsal ud -otz dem -Ġmass es -Ġby ÅĤa -Ġclear ing -Ġdisc ourse -ats on -Ġfold ed -ĠJ ar -ÙĦ Ùī -9 00 -ĠÑĥ Ñģп -Ġprophe cy -Ġinterf ere -иÑħ од -๠Į -Ġth ri -Ġ×ŀ× © -Ġlaz ım -Ġ199 2 -Ġfut uro -Ġlock ing -Ġembar go -ĠNe ither -iv amente -ĠmÃ¥ ste -Ġm ik -Ġcollect or -еко ÑĤоÑĢ -ĠG and -Ġsent ir -ĠM ight -å¡ Ķ -Ġgan zen -U C -Ġrel ating -S D -Ġmos quito -G R -Ġho llow -âĺ ħ -ĠWalk er -Ġaffili ate -Ġduplic ate -н ем -Ġgra pe -ĠOrgan ization -Ġsy nt -J oe -Ġg eg -Ġreve aling -ĠEth an -out er -Ġy ay -é« Ķ -л аÑĢ -Ġreported ly -Ġihr er -Ġrecogn ise -Ġbum per -ĠR andy -ĠVen us -t les -Ġappet ite -Ġgluc ose -Ġch odzi -ĠFurther more -t ir -Ġcont a -Ġint uition -Ġalt itude -Ġch unks -ĠJosh ua -ıģ ım -ry lic -le ans -ĠíĶ ¼ë -L L -Q ue -Ġg or -Ġзна ÑĩиÑĤ -Ġpo ems -Ġexc el -Ġexpl ored -Ġpop ul -Ġinclus o -st ä -ĠG avin -all ing -ĠÏĦο ν -é © -ar beit -ĠG as -Ġgl orious -rie ben -Ġsp am -Ġindo or -Ġthr ust -ĠA ld -ĠPri or -Ġon board -ãģł ãģķãģĦ -o ca -AS H -£ ł -ĠChrist ine -Ġdra wer -Ġno on -Ġìŀ ĺë -Ġperman ently -æ· ± -ĠнапÑĢ имеÑĢ -Ġpodcast s -era peut -pr it -Ġstain less -ĠÚ© ÛĴ -Ġfamil ia -ĠÑĢаз ÑĢ -un to -ĠÑģÑĤ ол -Ġh ä -ĠH ai -ĠP B -iz on -Ġkon nte -Ġbüy ük -Ġutil izar -Ú Ĩ -Ġaqu esta -Ġmix er -ud ent -лек Ñģ -ÅĤ u -ĠÑģиÑģÑĤ ем -Ġн оÑĢм -Ġfat al -Ġconsider ations -Ġvalid ation -Ġo li -Ġk ardeÅŁ -ĠGL ORIA -Ġp all -еÑģÑĤ е -Ġrect ang -Ġmed ieval -allah i -ast i -ĠSy rian -Ġshe ar -Ġdeb ug -ĠM ai -Ġknock ing -ĠLe x -ard an -ro v -Ġmem orial -æ° £ -ook y -Ġstuff ed -Ġpass é -Ġw ig -Ĥ ł -Ġpróxim a -Ġ199 1 -Ġм еждÑĥ -Ġnuest ros -ĠBe ast -Ġsm o -atch ed -olog ia -Ġм од -Ġge e -Ġconcept ual -Ġà ´ -Ġdecre ases -Ġquer ies -олÑĮ ÑĪ -ĠA part -Ġex empl -å± ± -Ġfl ed -ĠO FF -gg ak -Ġbe ad -h ir -l ies -ĠClear ly -ı lar -Ġch ess -Ġwhich ever -Ġ9 6 -Ạ± -Ġrespect s -Ġм оÑĢ -Ġorgan ism -Ġgrand pa -ĠV ie -è·Ł ä½ł -Ġflo oding -Ġupgrad ed -Ñij ÑĢ -Ġcheek s -Ġcon quer -Ġstub born -Ġpuzz les -Ġau ction -Ġre lying -ĠPRO F -ĠEs per -ĠÐľ У -Ġhy pe -Ġposs ibil -Ġimp rison -ĠEr n -ìĹĪ ìĬµëĭĪëĭ¤ -Ġenv ie -Ġresur rection -ä¸į è¡Į -Ġs per -ĠVenez uela -s om -Ġìŀł ê¹ -Ġnouve lle -Ġclos es -Ġ19 40 -Ġqu a -ĠJ ared -ĠP ir -Ġind e -Ġscr ub -uk u -Ġrequ iring -Ġв ами -Ġconsider able -åIJ Ľ -il ia -Ġin ne -Ġmein em -Ġhard ship -Ġtra ps -ro c -ĠìĦ ¤ë -Ġresearch ing -ĠMarg aret -Ġpen ny -Ġbı rak -Ñij л -Ġw ool -Ġr het -Ġflat ten -ç ĩ -à¹Ģภ£ -Ġp ied -ĠCh ap -Ġunder m -Ġf ret -Ġcrash ed -ĠFra uen -Ø° Ùĩ -iv an -Ġliter ary -late go -Ġsp äter -Ġsimilar ities -â Ĩ -ĠCor on -ĠC reek -Ġboss es -Ġaccompan ied -Ġdeb ates -Ġassemb led -Ġà ģ -ĠV ai -Ġtr act -Ġsimple ment -ĠAr in -Ġvulner ability -Ġhorm one -I EL -OO K -Ġrel ay -ĠAnd rea -r il -Ġnecess ity -aceut ical -Ñİ Ñī -ous ing -nah men -Ġfoot print -m ap -ĠT ier -ann ya -int end -åĸ ® -å ¢ -Ġdecor ate -Ġzomb ies -ĠHy d -ĠSu z -Ġcampus es -ĠE mb -Ġthr ottle -Ġad min -Ġop ortun -Ġmir rors -Ġident ities -ĠCl in -Ġë¹ Ħë -á¹ £ -ĠO tt -Ġbl ues -Ġimpress ions -- , -Ġv ague -a fe -Ġinfer ior -eral d -Ġmedic ines -Ġpre gunta -os ely -Ġt élé -ĠMon th -ĠLe aders -ĠEgypt ian -Ġr ation -k ers -he its -Ġre cht -P lay -Ġe g -Ġpoll s -ĠWOO DR -Ġsl ots -j am -B oth -ĠR at -ÑĢ аж -ĠBr ight -ä¸Ģ å®ļ -á»ij i -ur ious -Ġsing ers -Ġlo gin -Ġt êm -l ation -ĠM um -Æ°á»Ŀ ng -ĠEd itor -åIJ ij -Ġinnov ations -h ave -ĠS ek -Ġwe aker -ĠG ob -A fter -´ì §Ģ -Ġ문 ìłľ -ãĥ¼ ãĥ¼ -Ġdisad vantage -ç¢ º -Ġg aze -ĠM ack -Ïģ ί -ĠK iss -ĠH olo -ĠBir th -iz i -b ab -ä¿ Ŀ -ìĭľ ê³ł -д еÑĢж -Ġsqu at -кÑĥ Ñģ -un i -ĠComm e -ĠWOODR UFF -ĠChampions hip -Ġwel che -ĠY outh -z em -Ġod pow -Ġpersist ent -r ut -ìĶ © -íĸ ¥ -la ir -ik u -Ġvend or -Ġch úng -Ġfinan ci -Ġover ly -â u -Ġgl uten -Ġ18 00 -Ġdiv isions -Ġciud ad -Ġob ed -Ġwar um -Ġe her -Ġel im -ĠÐĴ о -Ġpeu vent -ĠW anna -Ġattend ance -Ġassess ments -ĠB og -Ġimag ery -Ġcollect ively -Ġinform al -ĠSch we -Ġde utlich -ĠCh el -ĠP E -ow ed -Ġb anner -Ġshel ves -ĠRet urn -æĭ ¿ -LAUGH S -Ġcongrat ulate -ĠNor way -Ġd well -ĠCarib bean -Ġnorm s -ĠAn imal -ĠValent ine -Ġext ending -ĠV ou -or r -ĠCh eng - ¡ -ĠдоÑĢ ог -Ġve g -Ġh Ã¥ -ĠX in -Ġì¹ ´ë -em et -Ġhyp oth -Ġinteress ante -ric es -I Z -ĠUS D -Ġrun ner -ĠB ag -Ġê ½ -Ġcomeç ar -Ġpig s -Ġweakness es -P h -ĠVi ol -ä¸į çĶ¨ -Ġdra gging -ĠAqu ÃŃ -ĠCS S -Ġmill imeters -Ġest ás -Ġac ute -Ġde jar -i ÄŁ -ob ra -L ove -Ġsil k -** ** -Ġjo ins -Ġpro l -Ġê°IJìĤ¬ íķ©ëĭĪëĭ¤ -æĶ ¯ -ØŃ Ø¯ -agh etti -än ner -Ġstr ang -Ġdoub led -Ġdescri ptions -Ġst ellen -Ġpart i -ç« ĭ -² Ħë -Ġö ÄŁ -ig hing -Ġang ular -Ġnat uur -ĠSh el -Æ° Æ¡ -Ġr ays -Ġse per -st art -v ised -Ġrush ed -Ġinternation ally -Ġnive l -Ġbox ing -fall en -á»ij c -Ġse inen -plic ity -Ġcarb oh -ĠTra vis -us o -ĠPh ase -Ġactiv ation -Ġop io -· ¨ -Ġdecre ased -C ar -Ġbund le -Ġexp end -orm al -Ġadjac ent -Ġme e -ĠоÑĢ г -Ġtrans cript -ĠLang uage -G S -è§ ī -Ġse ul -Ãł nh -Ġn ya -ning s -Ġìĭ ľë -ĠëĶ°ë Ŀ¼ -ĠA gr -ÃŃ d -çķ Ļ -Ġab y -ĠNe o -ıyor uz -ĠThink ing -a ime -Ġv ite -Ġtrav és -Ġ×ij× ¢ -Ġм ед -O ur -ho ot -Ġl iner -ĠP izza -Ġhy g -fl ies -ĠContin ue -Ġdent al -ĠT ib -Ġreg ulate -lie ÃŁ -AL K -ĠTa e -ê¸ ¸ -ĠBre xit -ĠG ut -Ġoccup ation -Ġz robi -â m -Ġwh isk -ä¸ĸ çķĮ -Ġkans ke -om on -ro be -Ġwar fare -Ġth á»ĥ -Ġjak i -Ġstro kes -Ġpe as -ĠDam it -H AN -Ġinter ference -Ġмин ÑĥÑĤ -N ER -out ing -Ġtext ures -Ł ī -ow i -Ġíķ Ļ -Ġd ens -Ġprotagon ist -än n -Ġgod dess -Ġwoll te -ij o -ĠWo che -ĠV PN -st ory -Ġkind erg -Ġfun nel -Ġdist ress -ноÑģÑĤÑĮ Ñİ -Ġno isy -ĠпÑĢод олж -Ġdar an -Ġenzy me -л ож -Ġm ute -Ġd war -Ġا س -Ġkom pl -Ġmer it -Ġf osse -ĠDr ink -Ġfor a -Ġw ohl -Ġbree ze -Ġsan it -Ġdr in -ĠìĿ´ê±° ëĬĶ -Ġ6 2 -Ġì° ¨ë -aby tes -Ġde eds -ĠÐ ¹ -i ème -igg ling -Ġ" ' -ĠÑĩа ÑģÑĤÑĮ -ĠAns wer -Ġev angel -Ġ10 80 -ĠVis it -ic ient -Ġreli ability -Ñİ ÑģÑĮ -ĠEar lier -Ġf id -çŃī ä¸Ģä¸ĭ -Ġslee ves -iy orsun -Ġb ib -ĠAcc ount -Ñı ли -cipl inary -z as -Ġб еÑĢ -Ġneck lace -Ġbl ender -ĠPhill ips -et i -ĠJup iter -Ġprov oc -ĠYe ars -ent re -ac io -Ġk ü -Ġanten na -Ġnovel s -Ġf art -ĠS ugar -ĠJud y -Ġcollaps ed -ç ° -rit is -Ġìĥģ íĻ© -ÐĹ Ð« -ĠVer f -rane an -ere um -ĠTar get -Ġ8 8 -ĠÐĺ з -ide o -Ġreg ression -ì¶ ľ -Ġmów i -Ġstud ios -i ens -ip h -Ġfr ying -Ġfasc inated -ĠW ah -b ucks -m aya -ĠSat urn -ĠM ommy -Ġrating s -Ġaut umn -Æ°Æ¡ ng -Ġlos er -Ġcent ro -érie ur -ĠF old -Ġsuper visor -ĠNo bel -Ġunder est -ob ia -Ġв ÑģÑı -Ġver w -Ġfu els -Ġartif acts -Ġë¶ Ļ -ĠAut om -çļĦ æĺ¯ -Û Ķ -×ķ× ¡ -Ġih nen -Ġ5 9 -ound ing -еÑĢ Ñĭ -in ars -ch ant -Ġadd icted -Ġexplos ive -Ġdisp ers -â ĸĪ -ax is -AR Y -Ġl um -ĠÑĥ Ñģл -ĠØ Į -Ġru pees -ĠPe arl -c amp -t v -oy a -Ġconclud es -Ġcoll ision -Ġbuy er -Ġplay ground -Ġspr ings -Ġfemin ine -ĠR as -Ġincar cer -íĹ ĺ -Ġdial ect -Ġclos ure -Ġchat ting -Ġb abe -Ġspot light -Ġnot ation -è· ¯ -St ar -i ão -Ġt ête -Ġt ide -Ġjun to -Ġsen ator -Ð ¥ -Ġexcus es -Ġbl ink -Ġadm ission -ĠL ily -Ñĭ ми -Ġam igo -Ġl ust -ëĭ ¬ -Ġam ino -äºĭ æĥħ -Ġconsult ant -ĠElect ric -Ġëħ¸ë ŀĺ -uj ah -Ġshoot er -icht en -ĠUkrain ian -Ġaim s -ĠEnter tain -Ġmir acles -èŃ ° -Ġze igen -Ġl am -Ġres s -ĠJ ill -yl an -Ġro ok -Ġh aya -Ġpass port -ad ata -Ġju icy -con f -л ей -ĠS z -Ġinter cept -ãģĤãĤĬãģĮãģ¨ãģĨ ãģĶãģĸ -ĠTe ams -Ġmak en -ir rel -ĠLI KE -áºŃ y -êµ ° -Ġshort age -Ġparad igm -Ġpap el -Ġast ero -ãģ¾ ãģŁ -Ġsoll en -ĠMic key -ĠOr leans -Ġchol esterol -Ġgo ose -ÑĨи Ñİ -ãģĤ ãĤĭ -ĠF L -Ġгол ов -Ġtrib ute -ĠG am -Ġé videmment -Ñı Ñħ -å® ŀ -çĶ ° -Ġin appropri -uh an -Ġorganiz ational -ail ed -Ġend ure -Ġ7 6 -Ġshot gun -Ġliv re -Ġsu ited -Ġwarm th -ĠS IM -Ġenv ision -Ġde grad -î ne -La ughing -ĠWho ever -ĠBuddh ism -Ġspr inkle -ceÄŁ iz -Ġru ins -Ġst arch -ĠHer z -Ġinjust ice -Ġhum idity -ожал Ñĥй -ĠOb ject -ĠI gn -ĠEx am -ig ers -Ġth ou -ĠSo y -iv as -Ġpol es -m ath -Ġв ним -ING ING -ed ral -Ġexpl or -Ġroast ed -Ġcraw l -Ġco ff -Ġan om -Ġw ij -Ġimpro ves -Ġtreat y -Ġdiscover ing -Ġstat ute -Ġmerc ado -ĠÑģ ил -Ġint el -ĠChance llor -ĠMed icaid -ug i -Ġver bal -Ġd ön -Ġscript ure -Ġit eration -ek s -ĠOx ford -Ġw äh -ĠV ad -ĠA K -ĠìķĦ ìĿ´ë -Ġi ets -Ġneed les -Ùĥ Ùħ -Ġpas ado -Ġalbum s -Ġye a -et zen -Ħë ıĦ -Ġdeterm ines -Ġthe e -ĠPlay ing -är t -Ġ× ¦ -c led -Ġdown ward -al one -Ġsol u -Ġpart ition -Ġw z -d d -Ġpesso al -å ª½ -Ġfact ories -Ġble ibt -ม า -als a -ĠNF L -Ġfu era -Ġres erved -ĠE arn -Ġhel t -Ġshort cut -Ġconvin cing -sp ace -Ġen force -Ġc ores -Ġe fter -Ġrecess ion -x ico -Ġprop osition -ar ians -rop ol -Ġëª °ë -ĠÎ ľ -ĠìļĶ ì¦ĺ -Ġactiv ist -Ġconv iction -Ġz ab -Ġcancel ed -ÑĤо Ñĩно -ĠÎ ® -éĢĻ樣 åŃIJ -n ite -Ġfund ra -buz zer -ел о -ic ations -Ġz ona -Ġte ens -Ġmethod ology -Ġì¤ij ìļĶ -th an -ĠU l -ĠG rey -Ġh og -IN K -ĠS ung -ĠC laud -ĠCN N -Ġdel ivers -al in -ĠAd obe -ot he -ĠDes wegen -ภ³ -Ġwer de -Ġgre ase -Ġup grades -ĠFin land -ac cept -Ġinter rog -be e -Ġãģ « -Ġpre de -ĠN ep -ĠCam bridge -Ġgraph s -Ġha unted -Ñģ ем -æ § -åħ ĭ -S ome -ĠM all -Ġrehears al -ĠUr ban -ĠL ag -Ġn im -ê° ķ -Ġposition ed -Ġavo ided -EM A -Ġlleg ar -Ġráp ido -Ġgou vern -Ġh ing -Ġdeal er -Ġreform s -Ġfat ty -к ол -ĠA ce -Ġne p -Ġì² Ń -Ġcomput ation -ĠSt ream -bour ne -t ur -P or -Ġsleep y -Ġbang et -ãģĤ ãģ® -Ġwe ighs -Ġble iben -ĠG ren -Ġun ions -Ġêµ IJ -Ġap render -uit ar -ĠJ est -um ing -ĠPlay er -ĠExt rem -Ġinteg er -аÑĩ е -Ġconcert s -×ķ× Ľ -Ġtro chÄĻ -ĠRe pe -éĩį è¦ģ -๠Ĥ -ż en -Ġsound ing -Ġan onymous -Ġex ca -ĠIran ian -Ġener getic -Ġw ives -ĠÑĨ веÑĤ -Ġa is -ãģĭ ãģª -Ġsud ah -Ġunder wear -Ġcrunch y -ĠP ain -Ġger çek -red ict -Ġm isma -Ñĸ ÑĤ -Ġsurv iving -ÎŃ ÏĤ -Ġparticip ant -ĠH essen -ári as -Ġsub way -ist ä -Ġcor al -Ġmar ijuana -ĠMem orial -ÑĪ ий -ri z -Ġsatell ites -Ġle ase -ĠCam eron -um ph -Ġclass mates -äh än -ÑģÑĤв е -Ġh ue -ĵ¤ ìĿĦ -Ġproport ional -Ġn oss -Ġl aps -r Ã¥ -Ġbit coin -ÐĹЫ ÐļÐIJ -Ġì¶ © -ĠÙĦ ÙĦ -ĠM ort -ĠEs p -arn os -ĠÑģказ ал -Ġä nd -åħ Ħ -×Ļ ×Ļ×Ŀ -ĠGe b -ge hen -I naudible -bor ough -ÑĦ ÑĦ -Ġfellow ship -ĠP aper -Ġcur ved -ĠGE OR -Ġcalcul ator -ĠCat al -ĠvÃł o -Ġby pass -л еÑĤ -à ³ -tr ans -ren cies -ì ¡Į -ig ent -Ġtast ed -Ġo ceans -u ft -erv ice -ĠÐľÐ£ ÐĹЫÐļÐIJ -ĠClass ic -Ġrespect ively -~ ) -î tre -ĠN ash -Ġz it -ĠìĽ ĥ -ĠëĨ Ĵ -qu ote -ĠUn s -Ġt ac -Ġpro ves -ĠPort land -b ly -Ġ ere -ì¶ Ķ -Ġépo ca -ĠÑĤÑĭ ÑģÑıÑĩ -7 6 -Ġhad e -ĠF ro -ĠpolÃŃt ica -t ag -Ġíķ Ń -Ġsch ö -are tt -Ġprov isions -Ġmot ors -Ġimag ing -Ġdo k -ul ously -Ġme ille -çİ° åľ¨ -ë IJ -ĠIS O -ĠST EM -ĠBow l -Ġto wers -ĠE e -ĠPerform ance -Ġlo in -cuss ion -Ġcoast al -ial e -com pass -Ġspell s -Ġdisappoint ing -Ġë²Ī 째 -E ER -Ġvers atile -as ury -Ġen fin -Ġdown side -Ġgu iding -ĠاÙĦ ÙĤ -Ġnin ety -char ged -ĠF ans -Ġphilosoph ical -Ġg arn -ĠmÃ¥ nga -Ġwilling ness -Ġport ions -ab en -Ġ ï - ¿ -ra ul -Ġspr int -if en -ıy la -Ġк Ñĥп -ãģı ãģłãģķãģĦ -Ġens uite -ĠCap itol -Ġ6 3 -ĠговоÑĢ иÑĤ -Ġappoint ments -æī ¾ -omi ast -Ġcare g -Ġpubl isher -Ġher aus -Ġε ί -ĠV S -ãģĿ ãģĹãģ¦ -ä¸Ń åħ± -Ġsacrific es -th ird -Ġhuman itarian -ĠëĤ ´ì -im on -Ġine qu -Ġz ob -Ġcomfort ably -ĠD inge -Ġcancell ed -ĠPS AKI -ĠRob inson -Ġfin s -) ? -ĠHist or -ĠÑĩеловек а -Ġt bsp -te xt -k im -Ġupd ating -Ġgel d -f eld -ı ¼ -Ġm ä -Ġcaf é -Ö Ģ -ĠS ri -ĠReg ion -ĠH ahaha -Ġfin ances -ĠاÙĦØ ´ -Ġb unk -ru k -ha ft -Ġlater al -Ġext ensions -ĠìķĦ ìĿ´ -Ġdefin ite -ĠZ hao -ĠLu is -st y -Ġcas os -ĠK lim -Ġ199 3 -Ġreal ization -Ġhistor ian -Ġcrack ed -ëĤ ´ -Ġsyst ème -ĠC IA -ĠÑĤ во -osp heric -Ġfle e -Ġr ất -ĠRegard less -Ġrel uct -Ġtim ely -ĠJul ian -G M -é Ĵ -ad ura -é£ Ł -Ġdress es -çģ £ -ĠëĶ Ķ -Ġnom inated -Ġadvoc ates -ym ph -Ġrecord ings -Ġdev iation -Ġpriorit ize -Ġspir al -ĠYOU R -Ġtransp ose -amp oo -ĠìĽIJë ŀĺ -ĠV ision -Ġpol ite -Ġha mb -ĠPat ient -æ¯Ķ è¼ĥ -íģ ¬ë -Ġs ia -Ġê³ ³ -Ġž e -è§ Ģ -Ġsuper market -ë ¹ -ĠS ierra -Ġgr illed -ĠUp on -Ġabs ent -Ġme c -ĠAp ollo -Ġp unk -ĠPa ÅĦst -ĠÑģв ой -Ġê±° 기 -G irl -Ġskin ny -ĠPrem ier -Ġterrit ories -Ġli ability -Ġj erk -r atic -Ġdan cers -ĠÑĥ ÑĢов -Ġê´ Ģë -on ly -ĠSt u -Ġske leton -ĠëŃ IJë -Ġзак он -ı kt -ĠMI KE -Ġl ö -m ie -Ġre iter -ãģĵãĤĮ ãģ¯ -ĠKoll eg -ĠAd ams -lich er -Ġçoc uk -Ñı г -Ġbl ush -Ġsun shine -Ġe z -ĠDev il -Ġê¸ ¸ -Ġãģ Ĭ -ad d -Ġlic ensed -Ġv inyl -ĠC zech -im ag -Ġcrack ing -Ġì º -Ġud ah -Ġs ommes -Ġìĸ¼ êµ -wa Äĩ -Ġf res -åij ½ -ĠWal mart -ĠТ епеÑĢÑĮ -at isf -C I -l ang -Ġdiff usion -çĶ · -Ġsom os -ĠM akes -æĪij æĥ³ -ĠRick y -Ġmuch a -íķ ¨ -Ġhorse power -as ia -Ġfib ers -Ġ erm -Ñģ кие -Ġjest e -Ġfire fight -Ġcu isine -Ġbesond ers -d ig -Ġì¢ ħ -ĠÑĥ ж -Ġtr acing -Ġcertain s -ĠApp ly -Ñĭв аÑĤÑĮ -ç Į -Ġbr u -ĠY ES -ĠB ai -ĠD it -ĠB is -Ġun le -ÑģÑĤа ÑĤоÑĩно -ĠAw ak -.. " -Ġ12 5 -Ġroot ed -Ġcaut ious -con st -Ġorchest ra -çľ ¼ -Ġвн ÑĥÑĤ -Ġquel qu -ĠоÑĤ веÑĤ -ĠMet hod -ì¹ ľ -Ġμ αÏĤ -l ü -ĠìķĦ ê¹Į -Ġn aming -C har -ĠS icher -Ġprivile ged -ĠF ly -Ġãģ ĭ -áºŃ t -Ġadv ances -ĠZel da -Ġand ra -Ġgr inding -ĠEd ition -p f -Ġwarri ors -Ġh edge -Ġuns eren -ĠÑģÑİ Ð´Ð° -el iness -Ġpersonal ities -Ġf ö -' M -ĠÑĤо Ñĩно -Ġsh ipped -Ġmete or -Ġsurround ings -ĠF ill -u esta -ĠPerson al -ĠAll e -OR T -ä¹ ħ -ĠS che -V I -Ġcompar able -dam n -Ġd itch -Y AN -ism us -Ġpick up -Ġd ak -ĠE P -b est -ĠS ue -äll t -Ġpop corn -Ġfold ing -h ome -ив аеÑĤ -å·² ç¶ĵ -Ġan not -ch uck -Ġfier ce -Ġdam aging -Ġfl op -Ġpas ar -Ġre ef -ĠÑģво ей -Ġz oo -o vers -j ets -Ġpr ès -ĠSil icon -te ok -ĠS eth -at amente -Ġtransm itted -Ġrepl icate -Ġsl im -ĠC ream -æĦŁ ãģĺ -Ġside walk -ìĪ ĺë -Ġжиз нÑĮ -ĠMon ica -ä¾Ĩ äºĨ -Ġcop ied -ĠTer ra -ist ent -ç³ » -Ġо но -Ġwh ale -ĠW ITH -л ÑĥÑĪ -å½± çīĩ -ĠE en -ĠÑģво и -Ġord in -Ġpl ural -Ġsp okes -Ġdisp ute -Ġsens ible -Ġpre aching -Ġktó rzy -pt ed -av ier -Ġpist ol -ĠTap i -Ġ ÅĤ -ff ff -Ġac rylic -Ġignor ance -ĠZ iel -r ans -Ġweld ing -m id -æĪij ä¸į -Ġзан им -Ġlan es -Ġmin es -Ġmom s -×ķ× Ĺ -ĠCham ber -t ier -Ġmod est -ĠìĹ¬ê¸° ìĦľ -Ġun as -Ġw rench -hand ed -Ġsatur ated -ĠF ang -ĠCommission er -ठ° -Ġ× ĸ -ĠLouis iana -ĠM ask -Ġcub es -ìĶ ¨ -Ġvidé os -ĠnÃ¥ gon -Ġr ider -Ġì¶ ľ -Ġs ón -ĠLat ino -b ank -íķ´ì £¼ -ĠB rend -Ġsexual ity -... , -Ġforget ting -Ġ ÛĮ -ĠAven gers -ĠBon jour -cess or -кÑĢа ÑĹ -c ence -Ġge ograph -cul o -о ÑģÑĤÑĮ -Ġswe ating -íĥ Ģ -Ġsymm etry -ts Ã¥ -Ġj an -ĠFer r -é¦ ĸ -Ġamb assador -ziÄĻ k -Ġmus un -ĠÑĥ ÑĤ -ĠL G -iss ent -comm un -Ġcour s -Ġdevelop s -Ġbron ze -Ġsubst ances -dri ven -주 ìĦ¸ìļĶ -Ġa os -åĦ Ħ -ĠPROF ESS -h alf -Ġsort ed -ĠB omb -л аг -ĠMalays ia -ĠChrist ina -Ġteam mate -èģ ŀ -F T -Ġk ı -heart ed -+ + -ogen ic -Ġbell s -ĠOu ais -Ġspecial ists -б Ñĭ -dep th -lass es -g ies -ĠCo ffee -Ġmark ing -Ġfo ll -ul i -Ġad hesive -ĠB ot -ĠP unkt -e ye -ĠB ub -el ong -åĪ ¶ -ĠпÑĢ ик -Ġdon or -8 4 -Ġen for -Ġcatch es -Ġbr icks -Ġkn itting -ĠKnow ing -ok s -H Y -r ide -ĠFant asy -im an -Ġp se -Ġìĺ ¨ -Ġв д -Ġrest ra -Ġevalu ated -ÑĢ ев -Ġfortun ately -Ġche gar -ر ب -Ġdom ains -ib i -ar ry -Ġshut ter -Ġfic ou -M ike -Ġinc lu -Ġdon ors -Ġa pl -ĠL ower -Ġimport ed -Ġacad emy -Ġfin als -Ġdisappe ars -ÙĬ ا -Ġadministr ator -j s -Ġcut ter -Ġr anging -ör per -Ġconstra int -ĠT able -ĠSh an -v ic -ĠF ix -ĠSw ift -oun ces -ĠWar um -Ġlett uce -app elle -Ġsh ave -Ġb ás -Ġ7 7 -ĠO oo -a o -ĠMc M -ĠD rew -Ġl ump -Ġl ashes -schein lich -R ep -in is -ĠC ette -Ġcompos ite -emet ery -Ġsort e -ĠFin ancial -он е -ron es -ĠV oy -Ġt éc -ł ¹ -ĠNin ja -ĠCor in -ен нÑı -ìĿ´ìĹ Ī -Ġn ich -Ġdetect ive -âĢ¦ " -Ïĥ ε -Ŀ¼ë ıĦ -Ġë³ Ģ -Ġë¸ Ķë -Ġpro pe -ĠW right -Ġ×Ķ× ª -ĠSh i -Ġãģ Ł -Ġinvestig ations -éĤĦ æĺ¯ -ĠPower Point -ĠCh u -Ġìĺ ¤í -ĠìĻĦ ìłĦ -ĠFra gen -un ning -Ġpour rait -Ġtext book -м Ñĭ -Ġf ahren -Ġ ÑĤоÑĢ -Ġl akes -ünd e -I nt -ĠMet ro -Ġmans ion -Ġа б -ĠZh ou -Ġcorrid or -Ġesc ol -Ġindic ating -ia ÅĤa -Ġm ommy -Ġarch ives -Ġfound ers -eng ine -ĠDie u -Ġsick ness -Ġë³´ ëĭĪê¹Į -Ġar b -Ġn ed -ĠCh op -Ġco vid -Ġsl am -Ġpublic ations -D C -Ġsp ends -æ ¾ -Ġrefuge e -Ġd ile -Ġ×IJ× ĸ -ific ar -ĠS ach -G u -Ġre load -?? ?? -Ġje ÅĽli -ĠÑģ оÑģÑĤо -Ġsim plicity -Ġbull ying -Ġм ол -Ġreal idad -Ġuncle ar -app a -le vant -ĠIS IS -ĠW atson -Ġde in -ĠMic ro -íķ ľë -ü g -Ġdev am -Ġtwe eted -å° İ -Ġunderstand able -at an -Ġvers a -Ġpre ca -Ġv á»ģ -ĠCop y -ĠOr acle -Ġmindful ness -Ġdisc ret -ern en -ĠP le -H ave -Ġisol ate -Ġde u -Ġsevent y -ĠH ills -Ġarc ade -ĠÑģп еÑĨи -Ġsigu iente -ĠB ÃľNDNIS -lig a -ĠвÑģÑĤÑĢ еÑĩ -ô m -Ġtwe ets -Ġsch auen -Ġcrit ique -ĠðŁİ µ -Ġst att -ĠÑģам ое -ân cia -Ġsuper natural -Ġplug ged -F l -yn ı -ĠTamb ién -Ġencourage ment -ĠSer ver -ëĤ ľ -up a -Ġast on -Ġhe ars -ÑĢа Ñħ -Ġsch e -Ġr ats -Ġrec uper -Ġun ten -ĠFight ing -Ġacadem ics -ç¤ º -ĠS ü -Ñģ киÑħ -Ġpa ired -Ģ ìĿĦ -Ġá rea -Ġsweet ness -åı Ĭ -Ġde fer -Ġmuit as -ĠAud io -Ġlock er -ÙĬ د -ĠÑģÑĤ ав -Ġbu ena -AN S -Ġdetect or -av o -be k -Ġα ν -íİ ¸ -Ġdra gged -Ġдолж ен -à ĸ -ر Ø© -ìĿ´ì §Ģ -Ġcell e -ck ing -ĠاÙĦØ ¬ -ĠCan vas -Ġespa ñ -Ġgl imp -Ġspread s -ong o -ĠM ason -ĠIn g -Ġê°Ģ ëĬ¥ -ÏĦ ικ -Ġsec ular -Ġb ater -Ġinqu iry -Ġenerg ies -Ġmanufact ured -Ġveget arian -Ġpine apple -ÑıÑĤ а -Ġpractition ers -2 000 -Ġíķ´ì ļĶ -ĠìĹ¬ëŁ¬ë ¶Ħëĵ¤ -Ġë¶ Īë -ĠJeff erson -ĠJo an -Ġtr am -å® ¹ -ch mal -ĠH ait -á¹ ĩ -Ġun real -Ġsymbol ic -Ġste alth -Ġspl ash -ĠEntertain ment -Ġmetall ic -?" . -è¶ Ĭ -ar ound -Ġdesp air -ĠNev ada -ĠFin ance -Ġk rie -ĠL ux -ĠSm ash -ke eping -Ġз аг -Ġnarc iss -Ġdz isiaj -Ġtoler ate -o ard -Ġlink ing -ĠEconom ic -Ġì ¼ -Ġmor ph -ĠN ak -ĠB aker -at on -r ings -ĠP eng -ĠAir port -ãģĭ ãģ£ãģŁ -íķĺ ëĭ¤ -§ ģ -pr ints -Ġhad i -Ġemp ir -ĠL ives -ann ers -Ġн им -ĠPROFESS OR -Ġpositive ly -ant om -Ġbad ge -ke lt -Ġinter fer -Ġfulf illing -Ġvisual ization -éĹľ ä¿Ĥ -ĠPr ice -� � -Ġscen ery -Ġpr one -Ġw izard -Ġb anyak -ver b -s ky -Ġwish ed -Ġrail way -Ġü zer -Ġalgu ien -ĠA W -Ġкол иÑĩе -Ġreact ing -ĠB uch -ภ¶ -Ġan th -Ġsi h -Ġh ust -ĠSc reen -il ant -ah o -Ġfragr ance -Ġelev ation -ĠMed iter -Ġë ¿ -Ġé qu -Ġwra ps -Ġin ert -Ġrecre ate -л аÑĤ -Ġbo leh -Ġharass ment -unk y -Ġglimp se -reg ierung -Ġfut ur -Ġreposit ory -Ġeng ra -Ġtraff icking -ass is -ĠTre k -Ġë² Į -Ġë§ Īë -ĠK ab -ani u -g ive -Ġdin osaurs -Ġfe ather -Ġatt itudes -Ġpl um -ĠR S -ĠAn fang -ill ery -ĠìĬ ¤ -M Y -Ġtrze ba -Ġsk ies -ĠA j -ur able -C U -ĠSh ane -Ġdepart ure -ĠT ON -iet en -r ats -æ° Ĺ -is u -Ġb ord -Ġinteresting ly -çĻ » -oug hing -Ġr ushing -Ġvol atility -Ġp yt -Ġform ats -Ġз аÑĤ -Ġê¼ Ń -Ġwhat not -Ġcomp ort -s w -ore an -ĠRel ax -Ġcl an -ĠA H -Ġpe w -Ġdiction ary -T ake -sh irts -ĠH ugh -ĠعÙĦ ÙĬ -ĠP ic -Ġenroll ed -Ġjed nak -Ġoffer ings -Ġcor az -L ife -Ġ !!! -Ġcl er -ĠVide os -ĠRod rig -ĠId ent -ĠP os -ĠSt age -ĠR ace -Ġen act -ãģĦ ãģ¾ãģĹãģŁ -ĠG y -ĠHis pan -Ġdef ence -ĠCamp bell -m atic -Ġrele v -Ġpe ach -Ħ¸ ìļĶ -Ġparad ise -Ġcere mon -Ġannoy ed -æĮ ĩ -la x -Ġexplo it -Ġcla use -ek er -ĠBlo om -n ant -ate urs -Ġhe ights -E ven -Ñģ он -Ġoutra ge -ĠVietnam ese -ãģ¯ ãģ¯ -T R -Ġe er -Ġcann on -ĠCom b -IJë §Į -è» Ĭ -Ġê²ĥ ëıĦ -Ġaccomplish ments -ĠAnalyt ics -Ġshap ing -re iben -Ġb achelor -Ġfing ert -ack ed -Ġpyram id -ĠStew art -á st -Ġsurviv or -Ġdu ct -Ġdeal ers -æ´ » -ع Ùħ -ли н -Ġed e -×ķ× ¢ -ĠÙĥ اÙĨ -ĠÏĦ ι -Ġcho oses -ĠO wn -го ÑĤов -h ire -алÑĮ нÑĭе -ĠÐĽ Ñİ -Ġо ÑģÑĤав -te ch -Ġdro it -Ġsubject ive -en es -Ġdiv is -ave z -Ġmaneu ver -à¹Ħ à¸Ķ -ade ce -ĠEn s -ac ial -ĠProt ection -ĸ ´ -Ġform ally -Ġwy d -ingu ém -Ġz iem -Ġrecru iting -×Ļ× ļ -n em -Ġforb idden -ĠB apt -×IJ× ł×Ļ -Ġsubs et -ĠMag az -n ement -Ġaqu ela -rag on -Ġcomm ittees -Ġéta ient -ud i -ĠDa wn -Ġb ore -Ġcompos er -ĠwiÄĻ cej -ang a -Ġdis like -ĠD ays -åŁ º -Ġpar al -Ġm ientras -Ġheaven s -ãģ Ĵ -he id -Ġtrad ers -on ce -Ġmasc ara -ĠÏĢ Ïģο -Ġwhis per -ĠMus k -éĽ Ĩ -ĠFamil ie -All ah -ĠOl ivia -ĠPr os -Ġol ika -il im -Ġrép ond -ĠP eters -Ġ å¾Ī -Ġbit es -Ġv ic -ĠN Y -em ption -Ġ4 50 -Ġvisual s -Ġlie u -ück en -ĠSte el -ĠG P -w ait -Ġnotice able -uch a -Ġreh abil -Ġreject ion -ĠÑģлед ÑĥÑİÑī -Ġsl ider -Ġregard ed -Ġgrav it -ĠRes erve -c ount -Ġbre eding -Ġlon ge -ale b -Ġkn ight -Ġв ой -Ġprés ent -Ĥĺ ìļĶ -ĠSpec ifically -Ġpos es -Ġve ure -ok ay -em as -Ġ ãģ§ãģĻ -Ġma jÄħ -Ġweb inars -Ġcann abis -Ġdam als -ĠNorth west -Ġp ada -Ġcrowd s -Ġfut ures -Ġä n -Ġciv ilians -ĠS achen -æ į -Ġtr aces -Ġ먹 ê³ł -Q U -é¡ĺ ãģĦ -ĠI F -an ın -ìĤ ´ -Ġb iblical -ĠV ed -Ġst oring -ÑĢав лÑı -æĩī 該 -Ġn ast -Ġd ö -ÑĢ оп -el ia -Ġside ways -ĠUnder stand -ĠQ ur -Ġper pend -ĠMill ionen -Ġwater melon -ĠDiv ine -ult ur -ab ord -Ġsuccess es -Ġhom bre -Ġcar p -Ġsus cept -ung kin -Ġk ij -ul us -Ø§Ø ¬ -Ġnot ch -Ġpolynom ial -å¹ ² -å © -Ġún ico -Ġteles cope -Ġpolit ique -k iem -ĠÎŃ Î½Î± -Ġaggreg ate -ĠGe off -Ġtr il -ĠG RA -Ġsubscri ber -im et -Ġдол лаÑĢ -op ing -Ġth erapeut -ĠCan cer -Ġpar ade -Ġir rig -âĻª âĻª -Ġclear er -Ġb og -ĠM aur -า à¸ĩ -ĠShang hai -acht e -ĠK ol -el ujah -Ġha v -ĠCr ime -se k -Ġë ¡ľ -ien na -ĠG or -è Ľ -ĠпоÑĤ ÑĢ -Ġкаж еÑĤÑģÑı -ĠL ift -ĠS ort -ĠP sal -Ġp ing -ĵ Ŀ -ph is -ĠF UCK -ĠS yn -Ġbam boo -¬ ìĺģ -c uts -Ġm mm -Ġfunktion iert -Ġ _ -ÃŃ cio -St op -Ġimag inary -Ġnot amment -ĠIniti ative -ãĥ ¥ -ĠK urt -Ġlo osen -Ġbus car -çģ « -Ġz elf -Ġpro ps -åĽ ī -Ġmoet en -Ġmill i -Ġhall s -ĠM atch -Ġbrack ets -ĠC ou -æ¦ Ĥ -ĠÐľ аÑĢ -IS A -Ġcig arette -Ġcompet itions -ĠM IN -Ġbeh ö -vo or -Ġ ust -ĠZ i -ĠO cc -ul ates -Ġball oons -Ġpr onto -ĠM iy -ĠF ile -Ġкл аÑģÑģ -нÑĥ л -Ġcere al -Ġincre ment -Ġref ined -åı¦ å¤ĸ -pr ising -ĠR F -Ġrespect ful -Ġlo ot -ask et -Ġdeix a -ing le -Ġfuncion a -ĠRe vel -Ġso ber -Ġperform s -ĠG entle -ãĤ ¨ -Ġrecip ient -ĠHa use -Ġë ĥ -F rom -Ġmin isters -Ġpar adox -å°±æĺ¯ èªª -Ġtast ing -Ġ×Ķ× Ĺ -Ġre use -ĠL ane -ĠÑģов еÑĢÑĪ -Ġremem bers -Ġfemin ist -Ġcommit ments -Ġproject ed -Ġg az -iyor uz -Ġoblig ations -R o -z ar -Ġch w -ĠJ AM -ĠbÄĻd Äħ -asp berry -Ġм еÑģÑĤо -ë² ķ -Ġreg ulated -Ġw icht -ĠTre vor -Ġsecond ly -ĠIh re -els h -Ġrep orters -ÑĤоÑĢ а -oy o -G I -Ġinter connect -é IJĺ -OS H -æŃ ² -Ġbr ass -Ġign oring -ä»Ĭ æĹ¥ -in fect -Ġpro jekt -ore t -ÏĦα ν -ĠÑĤ ип -Ġmut ta -Ġunbox ing -Ħ ° -å¡ Ĭ -Ġadv ised -ĠDen ver -Ġsevere ly -ĠM hm -Ġfl ipped -Ġp ien -Ġkomm un -ĠF RE -Ġà®ĩ à®° -aint ed -Ġkn ives -Ġhab l -Ġgew orden -arett es -C S -Ġмал енÑĮ -Ġgal ax -Ġnin ete -ê±°ë Ĥĺ -Ġs is -Ġadvis ory -Ġdr illing -ĠWould n -ün f -gest ellt -ĠHel en -Ġ×ŀ× IJ -ap olis -Ġrze czy -Ġter ra -Ġhe p -Ġalg ún -ik k -Ġastron om -ĠStar bucks -k Äħ -Ġpat rol -Ġì½ Ķ -Ġg on -Ġ ãĢIJ -Ġson st -Ġencoun ters -Ġret rou -Ġshark s -Ġd or -ĠR ever -Ġev apor -Ġreserv oir -Ġalleg ed -ul er -Ġver m -Ġcommer ce -Ġf itted -ge m -Ġtact ical -Ġl ith -éīĦ å¡Ķ -h ad -è® Ĭ -Ġcarboh yd -Ġlength s -ι ο -Ġdem ographic -R ob -ĠS kin -cc oli -Ġsimpl ified -Ġread ily -ĠC um -ades h -ĠD Ã¥ -us st -ig ne -et on -Ġmen or -q i -OO M -à¸Ń à¸Ļ -Ġpsych iat -Ġeight y -Ġм илли -ĠT ob -ed o -ç¶ ² -ĠÄij ến -Ġcirc uits -ĠLAU GH -ic ism -em or -Ġreg ener -eg ree -Ġbure auc -ĠAl ber -ä¹ĭ å¾Į -ĠW or -å¤ « -Ġres in -Ġby ÅĤy -ĠI G -à¯į , -Ġ7 8 -Ġwe eds -ĠMy th -9 3 -æ ¿ -ĠëĤĺ ìĻĶ -é v -á ½ -ö ren -ç ar -ĠP AUL -Ġdisad vant -Ġposition ing -Ġcock tail -Ġagre es -n n -ĠS ally -M s -Ġinher ent -Ġmonet ary -Ġnat ur -ĠN h -ĠImp ort -Ġle ben -Ġw i -uss y -Ġob es -Ġwand ering -Ġìĭ łë -Äħ da -etch up -Ġdispos al -ĠJ A -ĠC er -z illa -Ġvir gin -ĠSl ide -and el -Ġrighteous ness -ĠÎ £ -Ġide ia -ä½ł 好 -иÑĢов аÑĤÑĮ -ר ×IJ -Com ment -Ġpre lim -ĠV ale -Ġì§Ģë Ĥľ -ĠV anc -OM AN -Ġп Ñĸд -Ġy um -st re -ce m -Ġpo cz -Ġfrag ment -ĠÑģлÑĥÑĩа е -Ġunder go -ĠH ank -ce ks -ĠF PS -Ġoc ur -Ġdeter ior -æ³ ¨ -Ġempres as -Pa ul -Ġ) )) -ĠвÑĢем ени -Ġsc old -×Ļ× ¢ -Ġsuspect ed -Ġaccess ing -Ġsubst it -Ġhistor ians -ä» » -Ġдел о -Ġsoci ed -r one -Ġre den -Ġext ends -epher d -Ġbal con -ä¸į èµ· -ĠSol o -Ġpolit ician -олÑĮ но -Ġirgend w -Ġtraum atic -Ġrapp er -ĠRO BERT -Re ally -æģ ¯ -Ġline up -AS E -Ġcontract or -ĠCorpor ation -g or -ĠTod o -ÑģÑĤÑĢ ой -F BE -Ġnews letter -Ġko ÅĦ -alt ies -ĠпÑĢ иÑĩ -ĠHe avy -Ġsw ords -Ġmanip ulation -Ġfun k -Ġv Ã¥r -ĠTal iban -Ġë° ¥ -Ġac ne -ür ü -Ġdes wegen -ĠD ust -Ġsil ic -Ġhook s -Ġbl ij -Ġpet its -Ġfil me -ĠBere ich -ĠSa id -Ġimp osed -Ġdi ary -Ġго ÑĢ -ĠG ates -Ġal ta -å¸ Į -Ġch cia -ple asant -Ġë° Ŀ -Ġmoż emy -ĠAust ria -Ġbro ker -Ġsuck ed -èĢ ĥ -Ġcomp artment -Ġcl one -Ġ×Ķ× ¢ -ĠDan ke -Ġnoch mal -ез д -Ġad renal -Ġkle inen -ãģ¾ ãģĹãĤĩãģĨ -Ġsubsequ ently -Ġdecent ral -Ġgen etics -Ġê´ ij -Ġmon itors -ĠApp lic -ĠRep orter -w ert -Ġwie m -ĠMove ment -Ġinterview ing -Ġhair s -Ġpu ò -ĠChel sea -Ġco her -Ġc ot -Ġz as -Ġpatch es -Ġl ah -Ñĥн к -ĠRe agan -ĠMar co -c ity -Ġdef ender -Ġdecor ation -ij i -Ġl itter -Ð ¨ -Ġj ego -RE W -ĠP ik -ĠHe e -ĠI v -Ġи де -ĠThe ater -ĠÑĩаÑģ ÑĤо -Ġswe ater -Ġhighlight ing -Ġa insi -Ġdipl omatic -ĠNever theless -å ³ -AS ON -Ġpúblic o -Ġf erm -reat ed -c od -Ġë¬ ¼ë -Ġm ister -ĠVanc ouver -Ġrecogn izes -ec d -Ġcomplic ations -en cial -ãģĹ ãģı -Ġê°Ģ ì§Ģ -ĠUlt imate -Ġva ig -ĠM erry -×ķ× Ĵ -ĠMar cus -ç¸ ½ -ow ego -Ġm ente -S m -Ġa ja -ĠTa o -Ġjud icial -Ġentrepreneurs hip -Ġнем ного -Ġp is -Ġer g -Ġch rist -ĠC urt -ĠÑĢаÑģ п -λ ε -ens ch -ÃŃ re -Ġfo cal -ĠDiam ond -av ÃŃa -Ġh anno -ĠSqu ad -Ġassoci ations -ĠCreat ive -Ġmess enger -Ġbe gging -Ġdec imal -Ġd Ä±ÅŁ -Ġmet adata -sel s -ĠÄ° ÅŁ -ữ a -Ġdiffic ile -d ı -Ġs laughter -ĠVer g -Ġ×Ĵ ×Ŀ -ç° ¡ -æĮ ī -ĠTe a -ass es -O k -Ġsynth es -ot iation -Ġpain ter -Ġel bows -Ġarchitect ural -ĠÑĢ ад -Ġgl or -im age -amp a -cul iar -ł ¨ -Ġte ve -ĠSt elle -ĠB am -Ġì´ Ī -as is -ip edia -ĠG I -ĠAct ive -çĦ¶ åIJİ -az i -ãĤĮ ãģ¦ -ĠL ucky -íķ © -ĠпÑĢ иÑħод -Ġrun way -Ġauthent ication -Ġpos ible -Ġsupp lements -Ġsurg ical -G en -Ġfeas ible -D O -Ġout look -Ġinter vals -Ġan ecd -Ãł ng -Ġstra ps -ĠSh u -ud d -iss enschaft -Ġport e -Ġcomm itting -Ġall ey -Ġco venant -ĠPed ro -less ness -ĠSol id -ĠM olly -Ġн екоÑĤоÑĢ -Ġcooper ate -åĮ Ĺ -oll en -Ġtun a -Ġkinderg arten -ĠS iz -Ġduż o -ĠM BA -ĠGEOR GE -ĠF isher -å¿ ĺ -ĠCa esar -ĠкÑĢаÑģ ив -ĠDel hi -zy m -Ġexpl icar -ê°Ģ ì§Ģ -un s -gr ow -ĠпÑĢ иÑģ -Ġ8 6 -Ġst ating -Ġmass a -ch ter -Ġì»¬ë Ł¬ -Ġdep uty -S M -n oc -Ġge ography -ĠEnter prise -ĠC ant -ö z -Ġun pack -ĠíĻ Ķë -Ġsearch es -Ġpres idency -Ġtri vial -Ġp ige -ou bt -ãĤ ļ -ì¼ ĢìĿ´ -Ġbudget s -Ġu b -Ġp ne -ĠY ale -ĠÅŁ öyle -reg ular -Ġimper fect -AR A -Ġfam ÃŃlia -ur m -ĠAdvent ure -ãĥ Ĭ -c is -em ark -Ġne go -Ġinappropri ate -ĠпÑĢи з -ĠÑĢ ол -Ġdream ed -B ry -Ġshut tle -Ġpill ars -Ġb ik -in um -ĠÑĥ Ñģ -ĠNe br -Ġperpend icular -Ġbook ed -ber y -Ġv ikt -be ar -es us -Ġвозм ожно -¨ ¹ -Ġpresum ably -ĠMem phis -Ġambul ance -×ķ× ŀר -Ġthumbna il -Ġmod ification -éĩ ı -Ġinterpret ed -Ġprom o -Ġκ ά -Ġε ÏĢ -Ġacoust ic -ĠD B -åĵ İ -Ġnon etheless -ou le -Ġpe qu -Ġkn ob -ãĤ £ -ĠëıĮ ìķĦ -Ġpurch ases -ĠÃĩ ünkü -Ġdivid ing -per form -ract ion -health y -ĠTit le -Ġu k -Ġcer ca -Ġargu ably -Ġf ale -ë³ µ -Ġgam ers -Ġutil izing -Ġoff ended -Ġt ava -al ı -Ġmed ian -Ġinfect ious -ĠAn nie -Ġsmart phones -Ġpar ole -åĸ Ŀ -ĠEp ic -z za -Ġun ified -Ġê·¸ë ķĮ -Ġcur tain -ĠÄ ĥ -Ġsex ually -Ġuns erem -ĠCon vention -Ġalleg edly -Y a -ĠH oo -en ment -æĢ ª -íĽ Ħ -Ġgig antic -Ġnot ing -Ġre bo -ĠJ ama -ĠAl z -Ġborrow ed -ì¹ ¨ -Ġper ipher -оÑĤ а -ĠG B -ĠGe ar -Ġeconom ically -Ġtele fon -Ġqu eremos -ĠдалÑĮ ÑĪе -Ġr as -ĠTe ach -ic ios -at os -Ġpl edge -b au -ĠHim self -L ink -Ġesper o -Ġchrom os -ĠP ER -Ġer le -Ġpod ium -ç os -Ġnie u -Ġf en -ĠGO D -ĠCh ocolate -wer k -Ġt ừ -Ġsupp ress -λ η -Ġ24 0 -Ġsit ä -Ġhonest y -ĠB io -ĠB ard -ĠобÑī ем -Ġм Ñĥз -Ġmar ble -ĠÑĨ енÑĤ -Ġproc ure -Ġrot or -ber n -Ġtu h -Ġhead set -at em -Ġwarrant y -à® ´ -Ġfil ing -ι ά -Ġcomp rendre -Ġimp ulse -Ġsal v -wr itten -Ġinstit ute -K im -ĠLGBT Q -fic iente -H is -ĠαÏħÏĦ ÏĮ -Ġteen age -or us -ĠÑĢаз б -S ee -ĠCons erv -á»ģ n -ful ness -Ġstraw berries -ĠAb u -и он -Ġo lla -NO ISE -ĠEm ploy -Ġwip ed -ur ger -Ġmod ifications -Ġíķĺ ì§Ģ -Ġfoot steps -Ġhon ors -Ġad ul -Ġfl ipping -ĠH U -Z Y -Ġintegr ating -ب ر -ull a -Ġnatuur lijk -ĠíĹ Ī -ĠEth ereum -ÙĬ ÙĦ -w ed -Ġpe aks -ĠK es -Ġblo om -Ġcr ashing -Ġ9 11 -ĠоÑĤ лиÑĩ -Ġcontro llers -ĠD od -Ġвм еÑģÑĤе -Ġsort ir -å¥ ĩ -ĠStra ight -ĠGrac ias -Ġgro ove -Ġto gg -Ġìĭ¶ ìĿĢ -é ro -Ġout ward -ĠW A -ĠRock y -Ġsc am -Ġhay at -ig nty -â Ħ -pl ings -Ġantibiot ics -Ġ ä¸Ģ -Ġnever theless -j ang -com merce -Ġspo iler -Ġglo ve -Ġch atter -ĠB Y -~ ? -Ġíĺ ¸ -Ġdem ol -we chsel -im ir -Ġra id -еÑĢ Ñħ -ìŀIJ 기 -en f -Ġcomment ed -Ġoptim ized -Ġconv icted -Ġb ats -ĠS B -ĠA ur -ĠT ong -Ġimplic it -ĠJan et -Ġre ag -ãģ ² -ĠAdv anced -Ġimp ose -ש ×Ķ -Ġschem es -oug her -ab olic -Ġê±° ì£ł -Ġslow ing -Ġwt edy -Ġdest ructive -Ġоп ÑĢед -Ġland mark -Ġëı Ī -ĠWalk ing -Ạ¹ -Ġt ijd -ĠK N -ĠQu ant -ìĺ ¤ë -Ġк ÑĢÑĥ -Ġper der -Ġno ve -änd e -Ġãģ Ĺ -b ia -Ġcust ody -Ġb iod -æĿ± 西 -Ġdirect ing -... âĢĭ -Ġre loc -Ġdemand e -ãĤĵ ãģł -Ġo ÄŁlum -Ġод на -ĠMil k -åı · -ĠK ra -ĠH onda -Ġp ue -Ġele kt -Ġbegin ners -Ġspe ar -ÃŃ nh -ĠLu ft -Ġn ig -ĠSchool s -Ġfor ums -ĠQ in -pp o -Ġz ag -ĠÐ ® -Ġtooth p -ĠSt yle -ì´ Ī -Ġpun ct -Ġrep s -ĠA ly -Ġamend ments -Ġö z -Ġdig its -ur ai -Ġcha otic -ĠMas ters -e on -ĠC ash -ĠC uz -Ġbede utet -Ġscan ning -Ġж д -н еÑĤ -Ġcertain ty -j ek -Ġdi jo -ĠCl imate -Ġr inse -Ġk rij -vel and -Ġsound track -ĠSa fe -ĠNo va -9 4 -Ġa the -ĠVer b -ol er -ìĿ´ì £ł -Ġv in -Ġrespir atory -ĠStud y -ĠC AM -Ġav ocado -ĠZ hen -Ġlat ency -Ġfe athers -Ġcont ar -Ġв еÑī -Ġf ark -Ġbl ended -Ġexpl oded -ĠX X -ĠBen im -Ġalgu ém -isto ire -Ġconfident ial -Ġm ast -Ġì ¿ -ge h -Ġdis respect -ĠSystem s -Æ° a -E d -Ġw ys -Ġex otic -Ġgl owing -ù ng -oun ge -è Ħ -ани з -Ġpal av -ĠSw ord -Ġg im -ĠC row -Ġpot ent -b ish -Ġab used -ĠJ ed -Ġg ambling -ĠS pect -Ġinvestig ators -æĻ ļ -Ġr att -Ġdo b -ĠD ES -h og -ĠоÑĤк ÑĢÑĭ -íĮ ħ -ĠденÑĮ ги -Ġíĺ ¹ -Ġë¨ ¸ë¦¬ -Ġsat uration -Ġinher ited -ĠInnov ation -ìĹ Īëįĺ -Ġtang ible -Ġdep ri -h ed -Ġпом ог -Ġslic ed -ॠį -Ġth ế -Å ¥ -6 8 -Ġcor ona -Ġgift ed -Ġso ir -Ġhum ility -ĠìĿ´ 걸 -Ġflaw s -ĠпÑĢ акÑĤи -Ġk ald -wa ż -y w -ãĤĵ ãģ§ãģĻ -ir teen -Ġcroch ets -¦¬ ê°Ģ -ĠìłĦ ìĹIJ -Ġdes e -æ¥ Ń -Ġм аг -Ġdz iaÅĤ -Ġl ég -ch anging -Ġlle v -ÅĦ sk -çĶ » -Ġ198 4 -orn s -ĠW elsh -Ġpharm aceutical -Ġpump ing -ĠSh aw -p unk -Ġva ult -Ġkin etic -Ġhur ricane -ĠInc luding -ứ c -ĠGrand pa -ans hip -é¦Ļ 港 -ĠвÑĭ Ñħод -н ож -ľ ł -ut ta -Ġê²ģ ëĭĪëĭ¤ -Ġb az -Ġпо ÑĪ -Ġpe culiar -zy Äĩ -ĠEll ie -Ġlearn s -ĠKr ishna -Ġconse cut -Ġemp ath -ĠD in -Ġtrad ed -ĠBor is -ugg age -oll a -Ġназ в -Ġetern ity -Ġв п -è mes -Ġgra pp -b é -ĠпÑĢед ÑģÑĤав -ĠF C -į ëĭĪëĭ¤ -e ven -ĠNebr aska -ortun e -Ġk arena -ĠAg ent -Ġst ing -ĠP I -Ġmunicip al -power ed -Ġconse gue -ĠMan chester -Ġrain y -Ġbl i -Ġk ost -Ġhal ten -ĠAh hh -ins ula -er ting -ĠاÙĦ Ùģ -Ġrel acion -Ġk omen -Ġd ome -Ġpri ests -ĠInt rodu -rop he -sh ore -vel t -clip se -ĠÑĢ ÑĥÑģ -×Ļ× ¡ -Ġsab emos -ĠHoll and -og i -ank i -ĠM ats -Ġsm oked -ull ie -Ġeuro pe -ĠдейÑģÑĤв иÑĤелÑĮно -Ġbard ziej -Ġtransform ing -ĠE z -op ath -Ġìĸ¸ ëĭĪ -ĠÑģÑĤ ан -ằ ng -ั à¹ī -ĠO uch -Ġclear ance -ust ain -Ġsolid arity -Ġpro ving -ĠÐĺ н -ĠÑģ ÑĬ -Ġpro long -ад но -Ġs os -ĠDe al -Ġ17 0 -m ons -Ġз ем -Ġlo gged -Ġlif elong -Ġsens ory -Ġbe hold -ĠF AR -èt ement -ĠFed eration -Ġdod ge -ĠSh ir -Ġdrag ons -ĠAr ctic -Äħ ż -Å į - º -Ġden ke -Ġpodr ÃŃa -co le -ÑĥлÑĮÑĤ аÑĤ -Ġsystem atic -ам а -ch os -Ġclin ics -ĠB S -Ġtal es -us ions -Ġí Ī¬ -Ġpres ervation -Ġl ore -ĠProt est -á» Ľ -å¸ Ĥ -Ġacknowled ged -ĠIs aiah -ĠëķĮ ëĬĶ -Ġ× ĺ -Ġcompet itor -Ġadv ancing -z ip -Ġtent h -ĠLa ure -Ġh ints -Ġexerc ising -ŀ ľë -ĠIntell igence -u ated -OU T -op ed -Ġaut onomy -Ġbrand ing -ĠMediter ranean -Ñĸ к -Ġscrew driver -Ġsu pre -Ġst ap -Ġjurisd iction -ĠSetting s -Ġfore front -ĠF emale -com fort -Ġmultiplic ation -ĠMur ray -Ġbo b -ĠT as -Ġt ahu -Ġon un -et ter -Ġproph ets -l ag -Ġreven ues -Ġpr á -Ġupload ing -Ġmach inery -asc al -ĠEst á -ĠG oth -ĠB ald -ĠS aw -Ġstri pes -ìł ij -Ġpow in -æĹ¥ æľ¬ -Ġhost ile -Ġdar um -Ġprevent ed -ожалÑĥй ÑģÑĤа -Ġalgun as -Ġhop eless -Ġz naj -Ġread ings -Ġcra ving -t at -ĠP ig -Ġli ar -çĪ ± -Ġmulti player -Ġd ale -ĠCour se -íģ ¼ -ĠK ita -Ġcustom s -Ġrespond s -end ra -è¦ ĸ -Ġmet ro -Ñģ ол -Ġmitig ate -Ġopp ression -Ġ æĪijåĢij -qu inho -Ġam mo -Ġen fer -Ġp ony -Ġ ounces -° Ķ -ĠìĪĺ ê°Ģ -Ġdich o -ĠDe b -Ġwond ers -ĠRo ose -Ġpri zes -ĠA LEX -Ġthank fully -Ġtiss ues -ĠÑĢав но -ĠL una -intell igible -ĠìĻ ¸ -ê° ij -ĠHe at -ĠÑģ ид -ĠQu i -Ġ ions -Ġaccommod ation -ä¾ ¿ -ĠK art -ien st -Ġt arde -Ġso aked -ĠCase y -Ġì´ Ŀ -ĠÑĢ Ñĥб -Ġdifferent i -Ġleft over -Ġexch anges -sec ond -Ġfirst ly -Ġbuild er -ri en -Ġd w -Ġboun cing -? < -olog ÃŃa -we alth -Ġmed itate -ĵ¤ ìĿĺ -ĠC raft -è§ī å¾Ĺ -æĻ ® -ri v -ĠAgain st -Ġcer amic -esp ère -Ġcompet ent -ĠHop kins -Ġkil os -Ġgra vel -Ġpist on -Ġfriends hips -Ġesc re -Ġvo z -ĠGes ellschaft -Ġunter stüt -Ġmu j -Ġwarning s -p os -ĠProfess ional -w szy -od le -b ands -Ġteam work -stell ung -Ġd x -åį Ĭ -Ġatt orneys -Ġweit ere -ãħĭãħĭ ãħĭ -ĠOrig inal -×Ļ× Ĺ -Ġbroadcast ing -ĠпеÑĢв Ñĭй -uch i -Ġhe ure -Ġgra bs -ĠW OR -ĠPla id -M in -Ġp az -ĠP uis -um u -it ates -Ġco ats -Ġbu en -Ġhe ir -Ġpne um -ש ר -ens er -ĠJUD GE -Ġbl onde -á¹ Ľ -Ġg ak -Ġs ık -Ġquot ed -Ġequip o -Ġw ishing -ÃŃ cia -Ġver bs -çµ Ħ -ĠCanad ians -Ġgover ning -ĠEv ans -E uro -Ġgen res -Ġunters chied -ĠBeck y -³¼ ê²ĮìļĶ -Ġe inge -ĠRa ise -ol and -ĠStr ateg -Ġer es -ĠVeter ans -Ġbreak out -Ġsant é -Ġad el -Ġinvestig ated -Ġpe ur -Ġag ile -Ġrail road -ans ka -Ġе й -Ġexp os -ator ies -ĠCont ent -Ġtruth s -ĠTra il -Ġgu a -Ġp ores -Ġwrit ings -ĠU hr -ĠThat s -Ġic ing -O C -ĠProdu ction -Ġcar ne -IS S -Ġn inguém -n on -Ġv icious -×ķ× Ķ -Ġrecon nect -Ġcent res -ĠK em -Ġcre ase -ĠìĿ´ë ¯¸ -айÑĤ еÑģÑĮ -Ġб оÑĢ -ĠHay ır -ĠÑģ Ñĥд -Ġún ica -owa ÅĤ -Ġad her -h ua -Z Z -Ġprecis o -Ġcurrent s -Ġseason ed -ĠIo T -ĠB ishop -è¨ Ī -st ed -ĠBern ard -ì¤ ĺ -æ² » -ĠGl enn -Ġktóry m -ื à¹Ī -Ġast rolog -ĠK ot -å¤ ľ -Ġparf ois -Ġfor wards -ĠW iÄĻ -ĠÎ ĺ -Ġn ano -è» į -s ub -ĠBr ill -Ġgr it -Ġc ited -g ado -Ġmel ts -Ġfor cé -âĸĪ âĸĪ -Ġb ajo -Ġdiscret ion -° ° -at ivity -Ġsitu ated -ãĥ« ãĤ¯ -Ñīе е -åľ° æĸ¹ -ĠпÑĢин ÑĨип -am az -Ġaqu arium -Ġdissol ve -ĠGod s -S uper -Ġam id -z k -Ġ ãģĦ -éł IJ -amp f -Ġhel a -' ! -Ġdevelopment al -ĠD ise -ĠÑĢабоÑĤ аеÑĤ -Ġsnaps hot -好 好 -Õ ¸ -ĠY ue -ĠH ulk -ĠDo om -ĠFel ix -Ġré f -M ale -ç· Ĭ -ph ants -EN S -ĠMe chan -ĠG olf -åĨį è¦ĭ -Ġgener osity -ät ze -Ġunlock ed -Ġ ãĤĴ -íĥ ģ -ocaly pse -Al right -Ġê° ľë -Ġ×IJ× ij׾ -ĠKeep ing -Ġcollabor ating -ch ief -ĠFern ando -Ġchef s -ĠíĶ¼ë ¶Ģ -Ġsk ipped -Ġperson n -Ġax e -che z -Ġextract ion -ĠA V -ĠGib bs -Ġí ľ -Ġs ı -I AM -V iew -ĠGR ANT -Ġëª ¸ -Ġver ification -Ġdep icted -ĠMo z -ou x -Ġt ul -Ġsc anner -Ġcomed ian -ĠVol ks -ĠJE FF -è¨Ĥ éĸ± -§ Ħ -Ġdistract ion -r á -ĠIN TER -Ġsin cer -Ġ×ŀ× ª -Ġש ׳ -Ġconstruct ive -ar f -ĠëĪ Ħë -Ġe co -r amos -Ġrenew ed -in ement -ĠU b -ĠPe pper -ì§Ģ ê°Ģ -ĠDar win -Ġmerch and -Ġv árias -è ce -N G -ĠìľĦ íķ´ìĦľ -Ġак ÑĤив -ĠUn ters -ع ÙĦ -Ġint ric -omm a -ie ving -ĠCarol ine -åĵ ģ -ĠPR ES -Ġperform er -Ġaut our -ãģ¾ãģĽ ãĤĵ -Ġutter ly -Ġsynth esis -Ġles bian -Ġretrie ve -Ġmane ira -Ġimp air -Ġment oring -ĠSoul s -ĠGo Pro -ÑĢ аÑĤÑĮ -Ġc ose -ĠSS D -I RE -Ġup front -ĠA un -Ġgam er -Ġl itt -Ġag gression -ĠLike wise -ĠBet ty -ĠD art -ĠD LC -ish ment -ìŀ¥ ìĿĦ -Ġ 对 -ç» ı -c ream -ĠBaby lon -Ġn ug -br ar -Ġa ynı -am ily -b ike -ahah aha -lo yd -Ġmir a -Ġper me -ĠG aming -Ġfirm ware -M a -Ġassist ed -at ics -Ġìķŀ ìľ¼ë¡ľ -ĠM ental -niej s -ĠI z -ow Äħ -Ġt ougher -Ġde ed -èĭ ¦ -Ġsty lish -ĠTool s -ĠH amp -Ġsun screen -Ġartic ulate -i ye -и ÑĦ -ĠSp read -ĠHA VE -Ġsw irl -Ġspons oring -ä» ĭ -iov ascular -mes i -Ġrelax ation -ĠÑģво иÑħ -Ġmar gins -Ġsa ÄŁ -ĠPr ide -ĠÏĦοÏħ ÏĤ -и ÑĨи -en ci -Do es -Ġcor pse -Ġend urance -Ġí ŀĺ -ì¹ ´ -Ġhair cut -Ġinterrupt ed -Ġwind y -ĠC aleb -Ïģ Ïĩ -ĠPour quoi -Ġhol istic -uc lear -ĠWho le -å£ « -A ct -Ġgall on -c ade -ĠReg ional -ro ads -ĠSch ne -á ng -Ġиз мен -ãĤĪ ãģŃ -Ġmen us -Ġspl itting -Ġpr iced -ĠÎ ĵ -Ġus ername -ĠÐŀ Ñĩ -Ġcomp ressed -y in -Ġguard ian -Ġgo of -Ġcheck list -Ġinter change -Ġexped ition -Ġex tern -Ġinfra red -eng o -Ġden ying -Ġpack ets -on ent -B B -ĠInc re -Ġsin i -ÃŁ er -è g -ma al -gen eration -Ġminor ities -Ġlle var -Ġnom ination -Ġcons id -Ġ×ľ× ¢ -m uÅŁ -ĠEs c -Ġnumer ator -Ġka ik -Ġktóry ch -ies en -Ġv ê -ĠUS S -ĠPri vate -Ġод но -Ġal ém -ÃŃt ulo -Ġlim b -Ġforg iven -Ġdiscl osure -ÏĦ ί -Ġning ún -Ġtherapeut ic -Ġnegoti ating -ĠN ike -ense ful -Ġin cap -Ġflag ship -t own -â Ī -ĠÏĢ ολ -Ġwol ves -Ġviol ations -ĠAr nold -Ġinterven e -Ġhe ater -Ġrecurs os -Ġma id -ê² ¼ -Ġдав айÑĤе -ĠCe lebr -Ġca pe -ĠSt y -ain en -s ite -b ij -Ġп олÑĮз -Ġfr amed -Ġpublish ers -ĠÑĩ ÑĥÑĤÑĮ -Ġtempt ation -Ġcert eza -Ġex empt -ìĬ ¹ -se lling -ĠT ask -ho on -ĠC oc -ĠPark s -Ġrepet ition -ĠÑĤ Ñĥда -Ġens l -ĠdeÄŁ iÅŁ -ĠOr lando -ĠMain ten -æŃ ¢ -oc ument -ĠH C -Ġscoot er -Ġнап иÑģ -Ġtight er -Ġte ase -Ġremo ves -Ġkij ken -ĠÑģÑĥ ÑīеÑģÑĤв -Ġth é -ĠвÑĭ глÑıд -Ġrel ieve -Ġmit ä -Ġstation ary -ö ff -p able -Ġar ter -Ġdé f -r ative -Ġcon ect -Ġsad dle -ĠD iane -Ġcomm emor -fend im -S ÃŃ -Ġíģ ´ë -Ġman ge -at te -Ġarrog ant -Ġrobot ic -Ġgi Ãł -æĺ¯ çļĦ -Ġneighbour hood -iss on -Ġдв иж -ĠR I -ĠNorm an -b rand -am ation -Ġraz or -Ġmur ders -ĠÑĤ Ñĥ -Ġwszystk im -Ġut ilities -Ġmicros cop -ê ¿ -Ġda qui -oll ar -ĠÐĶав айÑĤе -Ġann ée -Ġkilomet res -Ġhom osexual -Ġarchitect s -ãģ¡ ãģ¯ -Ġni ye -L ER -Ġmicro phones -ĠSt unden -Ġconsecut ive -iend a -v änd -D ER -Ġlif ts -ĠMe at -Ġsave z -íĸ Īëįĺ -M en -Ġdism ant -ê±°ë ¥¼ -Ġins ulation -Ġsc all -Ġsp ooky -Ġpar c -Ġball et -ĠWhats App -Ġfr anc -Ġdeliber ate -Ġíħ Į -Ġm ars -ĠZ ur -P r -dis ciplinary -Ġobs ession -м е -Ġmarch ing -ĠEmer gency -ig uous -Ġs zy -ĠL ands -Ġboard ing -ĠпоÑĩ ÑĤи -Ġenv y -Ġcompassion ate -Ġmer ci -Ġdes irable -d ale -Ġcan ım -ĠAnt ar -tem ps -Ġconfig ured -ĠComp ared -ne h -ic ating -Ġnic kel -ÙĪ ÙĤ -Ùĥ ÙĪÙĨ -op es -Ġform ulas -ĠÐķ ÑģÑĤÑĮ -Ġpo bl -ĠP J -ĠL ud -ä»Ĭ åĽŀ -ĠBr id -ĠH og -ĠBr is -J en -Ġshad ing -ĠY as -Ġdistur bed -Ġrecomm ending -Ġc é -ĠH OW -ìĹĪ ìĸ´ -Ġrevers ed -ĠInteresting ly -iox id -åħ Ń -Ġìĺ¤ ì¼ĢìĿ´ -ế u -x x -Ġou ais -ĠYouT ubers -ĠR osa -ĠH aupt -j adi -Ġvlog s -Ġcult ura -ĠLeaders hip -ĠH ep -Ġill um -´ë ıĻ -Ġcustom ized -Ġmar ca -Ġqu atro -Ġн аг -ĠSpace X -ĠE igen -ast ing -ĠolduÄŁ u -Ġfor ts -ãģ ī -r iment -ien cia -Ġten ir -ro ffen -Ġ197 9 -Ġc ie -ĠëIJĺ ê³ł -Ġes cri -ÏĮ ÏĤ -íı ¬ -uz zy -C ong -ìĿ¸ ìĿ´ -G reat -s il -é ch -ãģ¨ ãģĭ -Ġmult ic -ĠDis k -² ķ -Ġfaz la -Ġle vant -Ġab ajo -ur ry -st ru -Ġ먹 ëĬĶ -Ġaccess ory -Ġдв иг -ĠR id -20 19 -Ġdown stream -æķ ¸ -Ġk az -ut an -Ġchar coal -Ġa fect -w u -Ġcontext s -Ġfe ared -ĠìĦ ¤ -Ġhist ories -Ġf as -ens ible -Ġcoco a -ill ar -ge ons -Ġspiritual ity -ĠP ew -Ġpharm acy -Ġpass ions -Ġb os -Ġall á -Ġthri ving -ĠRe act -Ġoccup y -Ġwithdraw al -Ġallow ance -ĠFra ktion -Ġbud dies -Ġid le -Ġdissol ved -Ġpreval ent -Ġmil itar -Ġsens ing -Ġpo jaw -Ġanc ora -Ġabund ant -Ġha irst -ãģĤ ãĤĮ -Ġtw ee -Ġnäch ste -ĠMöglich keit -Ġho o -uff icient -Ġfant ast -Ġed ible -Ġëĸ¨ ìĸ´ì -ìĽ ĥ -Ġve in -uc ci -Ġdevot ion -Ġconce aler -in come -Ġrecy cled -ĠìĬ¤í ĥĢ -Ġpont os -Ġdess us -Ġvé rit -Ġreflect ions -ĠA A -Ġtake away -b are -ĠCont act -e il -ĠHe ar -Ġmir ac -ĠGer ilim -ĠÑģам Ñĭй -Ġv ivo -Ġkilogram s -ĠCr im -û t -7 8 -Ġsincere ly -ra z -Ġë³ µ -Ġarri v -Ġconcept ion -ĠPers ian -Ġsj äl -Ġst arring -ĠìķĦë ¬´ -ĠFore ver -е ÑģÑĤÑĮ -Ġve il -Ġsubt it -od ka -ĠоÑĤно ÑĪ -Ġcook s -ен Ñı -K ay -Ġni ños -ĠPh one -Ġstitch ing -Ġfinger print -é¢ ĺ -λ ά -Ġded icate -ĠL ob -Ġblack s -ĠB le -b out -ĠÄij ang -Ġe ks -Ġsqu ash -ĠK ü -od i -Ġn Æ°á»Ľc -Ġvoy age -Ġplay ful -ĠØ¥ ÙĦÙī -an ic -Ġcondem n -ĠB öyle -ĠPol ize -ãĤ¿ ãĥ¼ -Ġay uda -Ġp am -à¹Ħ à¸Ľ -ĠK athy -ед ин -нов а -Ġbr ig -eg er -Ġe agle -Ġvis ions -ĠíķŃ ìĥģ -Ġsh itty -Ġh ott -ĠBr itt -ut ors -ENT E -æĽ ² -Ġph on -ĠB ing -Ġпод деÑĢж -spr ing -æĸ ¯ -et ten -Ġpil gr -Ġed iyor -енÑĤ Ñĭ -ag gio -Ġj ul -Ġcomp rend -te il -ĠØ ² -Ġperform ers -Ġinf amous -ĠM K -ç ª -æ³ ģ -ot le -e ff -ĠH ash -Ġcow ard -ĠB RA -ĠD D -Ġcom ida -Ġpl ata -Ġfl ap -ĠMe hr -rib ution -ĠY emen -Ġmyster ies -ĠÄ° yi -Ġst ell -Ġeyel iner -Ġdel es -Ġnail ed -Ġillness es -Ġst acks -Ġtrabaj ar -fl ower -ci u -Ġcr ude -Ġsubstant ially -Ġhome m -Ġnep hew -Ġstamp s -Ġcar bs -ÑĮ ÑĤе -mo oth -Ġtun nels -ac ie -æ³ ¢ -ĠSe ñ -ĠH era -ĠìķĦëĭĪ ìĹIJìļĶ -ĠWy oming -ĠHD MI -ĠL is -u ción -Ġste er -о Ñİ -иÑĤ а -N T -Ġìĸ¼êµ ´ -Ġpal ms -Ġne on -ов аниÑı -Ġfilter ing -Ġjou er -ĠH ö -Ġне Ñģ -ê²ł ìĸ´ìļĶ -Ġ8 1 -Ġstory line -Ġprz ep -Ġthank ing -ĠBo eing -Ġsoft ly -j em -алÑĮ нÑĭÑħ -Ġflash light -Ġп Ñĥ -ĠW OMAN -ắ c -ÃŃ ch -Ġlux urious -Ġw ün -Ġimpact ful -Ġcons on -re u -ir ring -if ter -Ġconstitu ents -èIJ ½ -Ġ9 4 -ĠT ou -g om -ĠìĥĿê°ģ ìĿĦ -Ġstere otypes -Ġmoż li -åĪĨ 享 -Ĥ ¨ -Ġpencil s -ĠÑģл ож -Ġih rem -ĠBes ch -ĠK oh -ĠEnt scheid -Ġle k -Ġför s -Ġtotal mente -Ġlive ly -Ġent ropy -Ġdisc ern -ĠÐĹ Ð½Ð° -Ġdo v -Ġmyth ology -è¨ĺ å¾Ĺ -apan ese -Ġapprox imate -аÑĤ ив -if iable -ĠSe o -åĢ Ĵ -´ìĭ¬ íŀĪ -Ġìĺ · -Ġtempor al -Ġi T -Ġest at -к им -Ġspr ink -Ġgr und -Ġinfant ry -Ġsch affen -ç´ Ħ -Ġan k -ri ages -ĠYe on -ĠMor oc -Ġinv asive -ģ Ķ -Ġparent ing -ĠR is -ib ile -Ġmod s -å½ ¢ -ĠпÑĢов еÑĢ -ĠTh ing -ĠWhere ver -Ġacknowled ging -Ġpa wn -um mer -or b -6 9 -Ġretr ouve -Ġrel ies -ĠHigh way -Ġa we -ãģ§ãģĻ ãģĭ -ita ire -Ġapplic ant -Ġais le -w orm -Ġpay load -Ġcar re -ĠB ach -æł ¼ -Ġì¹ľ 구ë -ни е -Ġit ÃŃs -onna ise -s ol -èı ¯ -alg ia -Ġrock ing -Ġbest en -rit es -^ ^ -ин ой -Ġba ixo -Ġ기 ìĸµ -оÑĤ ÑĢи -s im -Ġinc arn -ëĭ¤ ìĿĮ -Ġl ick -s ided -Ġ7 1 -f order -Ġreson ance -Ġte gen -Ġmet aph -ows er -Ġ×IJ× ł×Ĺ׳×ķ -? ãĢį -Ġsp ielen -Ġvoll ey -ĶìĿ´íģ¬ ìĹħ -lo oked -Ġsent enced -Ġmultip lying -Ġide als -Ġwahr scheinlich -Ġdepos its -bil ir -Ġeff et -ill on -Īë §Į -Ġtestim on -Ġz awsze -ĠпÑĢоÑĨ еÑģÑģ -ĠL av -ä¸į éĮ¯ -Ġtrava iller -Ġla isse -ĠMount ains -ĠÑĢ об -Ġexam ined -it us -W as -л Ñĭ -Ġattrib uted -ĠìĬ ¹ -ĠBar on -Ġg ep -Ġatt ent -ĠColl ection -Ġthe at -ĠC ai -Ġwell s -Ġhuman o -çĹ ħ -ĠH ast -ĠÑħоÑĤ Ñı -cz as -Ġperm its -Ġle gg -Ġe po -ĠF en -Ġth i -ĠF oi -Ġé lect -Ġ8 3 -Ġover th -Ġ è¬Ŀè¬Ŀ -Ġten ant -è² · -N ext -Ġpra ised -sec urity -ĠImp act -为 ä»Ģä¹Ī -Ġv ouch -Ġneg ó -Ġun ve -Ġcritic ize -ĠKen ya -Ġtact ic -Ġlo gr -Ġpo is -Ġpap a -spe aks -ðŁ ij -isp ers -Ġsur plus -Ġcold er -åį Ĺ -åIJ ¬ -pl ets -ĠV ienna -ĠLe ad -Ġaer ial -ĠT ah -енÑĤ ов -ĠGree ks -C am -Ġmá xim -Ġk uin -ch io -Ġdemonst rates -an os -ĠC ert -ĠÑį н -Ġblog s -ĠìĦľ ìļ¸ -Ġbe ams -ик ов -Ġprompt ed -Ġfright ening -ĠPors che -ãģĪ ãģ¦ -lar ını -Ġch illing -is phere -Ġfl ashing -ĠK ard -b read -Ġex h -Ġty cker -Ġec ological -ĠMa e -Ġ×ŀ×IJ ×ķ×ĵ -ĠëĤ ĺëıĦ -л он -ys s -Ġper gunt -Ġpri x -izz ard -Ġcan cers -Ġ9 1 -s usp -ĠIt em -ÅŁ a -Ġp est -Ġtak Äħ -Ġl ymph -ĠPat ri -f ill -Ġrec onna -Ġoptim ism -Ġmim ic -Ġì² ľ -ĠMad ame -oc y -l ining -åijĬ 訴 -erm e -Ġfold ers -Ġcz ÅĤ -uch ar -Ġcur so -Ġbre ach -ни ÑĤÑĮ -Ġp amiÄĻ -Ġel ig -Ġaut op -F low -Ġprogram med -ĠPro cess -Ġfig ur -ĠS F -ĠE les -Ġprogram mes -Ġdiz zy -ìĭľ ê°Ħ -Ġли бо -Ġsn iff -ĠSeb astian -ĠH ye -Ġ4 000 -Ġperm ite -æ¢ Ŀ -Ġза Ñī -Ġgu it -ĠD ais -Ġaccord ance -Ġmod ular -ogene ous -æĭ į -Ġpou quinho -Ġart illery -Ġlub ric -Ġvol can -ĠN H -ðŁ ¤ -Ġde an -R h -Ġminist re -åĿ IJ -ĠIn v -ĠBul gar -ĠD aten -è İ -I m -Ġorigin ated -ĠN ixon -inte gr -Ġlack s -ĠN acht -ìĸ´ë Ĥĺ -cam era -Ġrad ish -ki ye -Ġang es -Ġpré f -j uk -ĠBe e -ĠB U -ĠвоÑģ п -ĠB T -ê mes -ĠSt ück -ĠIn k -æĪĸ èĢħ -ĠSerge ant -ĠMult ip -Ġhiç bir -ĠС ам -ĠD é -ol ph -ìĸ ¸ -Ġimp at -ĠìķĬ ê³ł -ĠÑĤак ого -ĠнавеÑĢ ное -Ġunpredict able -Ġm end -ĠìĹĨ ìĸ´ìļĶ -Ġjakie ÅĽ -Ġann i -Ġdon né -ĠK irsty -Ġrectang ular -Ġempez ar -ĠEx change -ê° Ķ -Ġé conom -ãģĵ ãĤĵ -el in -re ibt -Ġ×Ķ× ¤ -Ġc emetery -Ġespañ ol -ol in -лÑİ Ð´ -Ġgr âce -all en -ĠPh ilos -ĠEr st -Ġìĥ Ī -ĠV id -G ive -O H -μ ο -ĠP are -Ġmetabol ism -Ġma ple -Ġax le -ĠD y -Ġkomm e -Ïİ Î½ -Ġgreat ness -Ġver ified -Ġsp é -ĠFahren heit -ĠB ren -ĠConf eder -Ġhist oire -Ġelimin ating -ĠAd ding -ĠAb i -æĿ İ -Ġhospital ity -t im -Ġbon ito -Ġpart es -ĠдÑĢÑĥг иÑħ -ĠSh ay -ĠS ed -Ġreg rets -Ñı ми -Ġten ants -éĢ Ł -ĠP TS -Ġdev i -ĠL ate -ue z -Ġsö yl -ãĤ » -Ġìŀ¬ë °Į -Ġtogg le -Ġmas king -алÑĮ ного -Ġpers ön -Ġamer ican -f ik -ĠR GB -ens on -ĠK A -ww ww -ĠÑĢ ег -met ics -Ġeduc ator -ãĤ· ãĥ«ãĤ¯ -p ark -елÑĮ зÑı -ar us -ÑĢ еÑĤ -Ġfe ito -Ġcho ir -Ġlar go -Ġe ens -Ġwat ts -ĠSing le -Ġsuscept ible -ic er -Ġв клÑİÑĩ -Ġp us -íĻ ĺ -E ng -Ġfant as -Ġspecific ation -Ġconfront ed -ĠColumb us -ив еÑĤ -ar ım -Ġcaffe ine -mun ition -Ġmig rants -l ide -it ations -ĠG eme -Ạ« -Ġpl anner -Ġstim ulate -Ġapro xim -ce u -ĠN om -Ġv og -ĠÑĢ аÑģÑĤ -Ġense ñ -Ġsell ers -Ġgut en -z d -C al -Ġdescri pt -Ġrecon ciliation -z inho -á¹ĩ a -ãģĺãĤĥ ãģĤ -acy j -ĠCO L -s aw -ĠíĻķ ìĿ¸ -Ġvar it -Ġpartner ing -Ġdet ention -Ġbomb ing -c lapping -ien cies -ond u -AM E -Ġê°Ļ ìĬµëĭĪëĭ¤ -c ÃŃa -ĠпоÑģ ÑĤо -ĠAS MR -Ġhome page -Ġsi è -an tha -ĠP oll -Ġ igen -cy ch -Ġê°ij ìŀIJ기 -Ġconsider ably -ä»ĸ çļĦ -ĠAr ist -Ġwith stand -Ġqual itative -ĠK raft -ĠÑį лекÑĤ -ĠBe ad -екÑĤ ив -Ġcr ushing -ì³ IJ -Ġnav y -ÙĪ Úº -s ho -Ġo ak -ipp ers -Ġso ils -Ġpig ment -Ġev itar -ãĥ ĩ -Ġf use -ĠD ale -: " -Ġcompl ètement -Ġke l -๠Ĩ -Ġqu atre -ĠU M -Ġë§ IJë -æł ¹ -ÃŃ r -Ġle isure -ĠH ousing -Ġfold s -est ion -AR S -Ġm ash -urp ose -Ġaccum ulated -ĠSt uff -èª ŀ -Ġtap es -ĠÑģ илÑĮно -ĠLO VE -Ġ198 2 -Ġsc ars -Ġcapital ist -ĠN ed -Ġsoft en -Ġnot ably -Ġforcé ment -ĠRa um -Ġнеоб Ñħод -Ġtrad emark -Ġfert ig -Ġ? ! -æĹ ł -Ġreinfor ced -Ġre charge -ĠPut ting -Ġvill ains -Ġhand ic -Ġadvertis ement -ت ÙĬ -ĠÑģ Ñĥм -ĠR iley -×ķ× ij× -äº ¬ -O s -Ø§Ø ² -B oy -Ġsqu ish -ock et -Ġtest ify -æ¼ Ķ -Ġ×ľ× ŀ× -Ġм аÑģÑģ -man uel -ĠArk ansas -if fe -Ġanalyst s -ĠDe af -Ġj ó -Ġgrocer ies -ĠWhe el -ĠÑĢ иÑģ -Ġc òn -ĠC ob -Ġpris ons -è ve -ĠCab inet -Ġpos ed -Ġguer re -ĠL loyd -Ġcl erk -Ġcr ises -ĠSh o -ĠO re -ĠFoot ball -ĠAd vis -ĠZh eng -è į -ĠAM Y -Ġun for -Ġmon aster -Ġcomp ile -Ġimm ortal -at able -Ġpar ano -Ġt iver -ĠStep h -ĠFu ÃŁ -Ġdisc ontin -Ġr ipe -Ġhack ing -Ġs iendo -Ġsegu ro -alt res -Ġand eres -Ġë ¦¬ë -Ġexp orts -æŃ ¥ -Ġtab ii -Ġ기 ëĭ¤ë -Ġbother ing -Ġpick le -ĠBRI AN -Ġalt ar -ĠпÑĢи б -Ġtransfer ring -ĠV ors -ĠÙĩ ÙĪ -ĠZ a -ĠFr ances -Ġbrow se -em it -Ġche wing -ĠFred dy -Ġedit ors -ä lle -Ġí ĮĢ -ĠS que -ĠC ultural -aw k -ĠS ache -ĠCar bon -ắ t -F L -ĠN GO -pe ÅĤ -ĠS ou -Ġh vor -un intelligible -Ġë² ķ -Ġ ° -i in -Ġ×¢ ×Ŀ -Ġder rière -Ġczy m -ĠAp ost -Ġregard er -Ġag rade -ĠC andy -Ġma re -Ġintrodu ces -bird s -Ġuniqu ely -Ġm uk -Ġcook er -Ġcrew s -Ġje ito -ER T -¶ Ħë -n isse -Ġe f -Ġcart e -ĠY ak -ĠP AT -и но -bok ki -Ġm ates -Ġdist int -Ġì½Ķë¡ľ ëĤĺ -Ġy ıl -Ġκ άν -Ġconfigur ations -eng a -re cht -H appy -ãĤĦ ãģ£ãģ¦ -in vest -Ġreconst ruct -ĠÑįÑĤ омÑĥ -Ġmos que -ra um -Ġvoy ez -ĠN BC -ĠìŀIJ ìĭł -Ġstur dy -Ġк ап -Ġans ch -al id -Ġmas ih -ĠR EP -Ġì½ Ķë -Ġded uct -Ġsal ir -w urf -il ot -ĠM utter -old s -ĠF EMA -ĠB ib -Ġneighb oring -Ġbl iss -Ġíĺ ¼ -ли ÑģÑĮ -ĠÑĤÑĢ еб -Ġ å°±æĺ¯ -Ġgren ade -Ġe gal -Ġfin ely -Ġpet als -Ġke er -Ġch yba -Ġsk ipping -Ġth irteen -Ġgrav y -ĠS AT -6 1 -Ġн ог -Ġmin s -IT E -Ġso zial -íķĺë ©´ìĦľ -rukt ur -Ġвозм ож -Ġоп ÑıÑĤÑĮ -Ġar th -ĠCub an -Ġtre asures -Ġfertil izer -Ġawak ening -Ġë°± ìĭł -Ġr all -Ġdep ict -ĠP ablo -Ġninete en -Ġw att -Ġentire ty -K S -ĠWood s -S ch -ĠÚ© ÙĪ -ĠD ry -ãģ ŀ -u ve -Ġreconst ruction -Ġanat omy -Īë ¥¼ -Ġb aba -Ġlisten er -Ġshar pen -ĠPer u -ĠвÑĭ з -Ġrecre ation -Ġiniti ate -Ġcal or -ĠN aj -ge e -ĠFe els -ĠSnap chat -ĠT et -ĠN est -ĠD af -ĠFin ish -ĠÑĤак им -ú c -iz ens -Ġsp ins -Ġemb ry -Ġpass ages -Ġc ient -Ġjust ification -ä»ĸ 說 -Ġolm az -Ġflood ed -Ġemo ji -Ġembr acing -Ġdisc ard -ĠBas ic -ag og -ĠìľĦ íķ´ -Ġas ylum -er in -Ġf im -Ġnin ja -Ġautom ate -Ġaller gic -ÿÿ ÿÿ -am am -Ġм аÑĢ -ĠO i -ä us -Ġin duct -ĠB EN -Ġz ÅĤ -Ġkaż dy -ĠAM P -n ÄĽ -S ure -Ġqu il -Ġespe c -ro k -BS CRI -Ġlie be -p us -ach sen -Ġcr icket -ëĬ IJ -ĠFr ame -ekk ür -ar b -Ġp ÅĻ -иÑģ Ñģ -Ġzeg gen -Ġdou bles -ĠD re -t est -ins p -bo ys -Ġm ão -ĠVer se -Ġmus cular -ĠMA LE -Ġd ulu -Ġoccas ional -L o -conom ic -Ġv ak -Ġrem edy -å¤ ł -ĠâĻªâĻª âĻª -ve m -Ġön em -ĠkarÅŁ ı -ĠSh arp -h ur -Ġë°© ë²ķ -Ġgrand son -Ġakt iv -ĠTh rones -ĠìķĪ ìĹIJ -Ġto ts -Ġsub d -ĠPa ula -Ġgra ves -ĠB rent -Ġник ÑĤо -Ġsö z -Ġcre c -ĠVlad imir -çĸ « -Ġп ой -Ġ" - -Ġp sy -at ri -id an -Ġa ún -Ġstandard ized -ì¹ ĺë -Ġк ÑĢов -ĠZh u -s omething -Ġ7 50 -Ġmuj eres -Ġa it -éĹ ´ -ag u -Ġcorrect ed -ik ka -el ed -ĠCare er -ow ym -Ġroomm ate -Ġdescend ants -ĠNapole on -ĠÐĶ о -íĸĪ ìĸ´ìļĶ -Ġbun un -ĠMich a -ç· ļ -Ġdesc ob -P I -Ġpalab ra -Ġtrack ed -Ġdepend ence -ĠBar ack -åģ ĩ -Ġfert ility -ĠSouth west -Ġincom plete -Ġcomun ic -Ġcomp ris -ĠRest aur -Ġac ron -κ α -Ġapprent ices -Ġmus st -ĠA br -Ġpent ru -ĠCons ort -ĠAve c -Ġdum plings -L R -Ġwszystk ie -Ġsw amp -н ев -ugg le -Ġwater color -Ġprot on -ĠEspa ña -ock ing -ов ал -Ġtak im -V ery -Ġdement ia -ĠÅŁey i -J ac -ĠMac Book -ĠL iv -ffic ients -ĠH unt -Ġover lay -æĦŁ 覺 -ĠSky pe -p unkt -Ġconf ined -ĠAd rian -ر Ùĥ -ĠJe ep -Ġenqu anto -Ġan est -оÑĤ веÑĤ -Ġм енÑĮ -Ġirrig ation -á»ij n -Ġeight een -ĠP on -Ġresc ued -Ġ198 3 -r ü -ja e -ĠJe ong -Ġamazing ly -ĠF DP -Ġback stage -c ue -ĠÏĥÏĦη ν -ĠاÙĦØ µ -Ġlivest ock -ĠW arner -Ġmaj ors -ãĥģ ãĥ£ -Ġcooper ative -ĠBr ady -ra ined -rie b -Ġ×ij× ŀ× -Ġдов олÑĮно -ĠF E -Ġle aked -ĠMerc ury -Ġpersu ade -Ġtransform er -ĠNor weg -ĠìĹ¬ë Ł¬ -Ġzrobi Äĩ -Ġcard iovascular -ĠCr ash -Ġg ossip -а ÑģÑĤÑĮ -Ġì ª½ -Ġsw ept -ĠH orn -ĠAt é -Ġbu kan -ĠK aw -K Y -ĠSt ories -G ary -Ġgard ening -ĠQuick ly -ĠFal con -Ġov at -c ı -ĠCom plet -ĠD ate -ĠпÑĢ им -Ġlä uft -ĠAud rey -ĠW ent -Ġpel ÃŃcul -Ġcar riage -Ġun acceptable -ny mi -ĠÑģл ÑĭÑĪ -Ġter re -uell ement -EE EE -Ġpharm ac -h ões -Ġz ich -Ġmig rate -ĠF ry -ñ ana -ĠM uito -EO VER -Ġfort ress -ĠCom pan -ĠJ SON -ord nung -Ġw arto -Ġun gef -ìħĶ ìĦľ -ĠÑĢ ок -Ġpad dle -J ared -Ġsubm itting -Ġl atch -Ġf ug -Ġк оÑģ -ĠE f -Ġlaunch es -Ġf t -ote chn -Ġtrave lled -ا Ùģ -éģ ķ -Ġpro ch -Ġded im -8 3 -Ġreb ound -ĠL U -p ath -ĠÑģп ÑĢав -Ġö l -ĠíĤ ¤ -Ġpriv at -Ġtr actor -ĠAtt ention -S er -Ġcos es -á ria -p al -ĠìĿ Ģ -Ġsuccess or -Ġconnect ors -ĠÑĥÑģÑĤ анов -Ġgen ocide -Ġsufficient ly -ĠA ixò -Ġstabil ize -Ġcon gest -Ġcar ving -Ġz ost -ĠбÑĭ ÑģÑĤÑĢо -Ġshort est -Ġli vel -Ġ8 9 -éģ Ĭ -Ġer k -Ġport raits -ॠĢ -è ĺ -bo at -ll ah -AN C -Ġempir ical -ĠE cho -ĠNeder land -è¿Ļ ä¹Ī -N et -Ġcuid ado -ĠR oma -Ġc alf -Ġgi ants -ĠExpl orer -ĠColl ect -al ition -ĠDest iny -Ġaus ge -ĠE du -ĠC lo -Ġear rings -ĠTr ack -ĠR OS -ĠBe lle -çĻ ¾ -Ġpu eda -Ġday time -Ġsupp lier -ĠS V -ĠEx hale -Ġgal era -c ourse -Ġcent imeter -ĠB ast -m ud -Ġsang at -ĠPhys ical -Ġpriv ately -Ġtr ata -lyn n -ill i -Ġë© ĶìĿ´íģ¬ìĹħ -Ġcryst all -Ġpod s -ả n -in ator -ĠRec ords -å® ĺ -ÄŁim iz -isse ment -h are -h adow -ĠD K -ĠìķĮ ê³ł -Ġw yn -Ġrequest ing -ĠD onna -ĠìĹ ´ìĭ¬íŀĪ -ine a -Ġex ert -ĠDun can -Ġв еÑĩ -ĠH ah -ठĤ -ĠL if -ĠF inding -ĠNo v -Ġзн ак -Ġо ÑĦ -ĠQu è -Ġquarter back -ĠÑĦ ак -Ġbipart isan -ÄŁ in -Ġné cess -Ġrefer endum -Ġcomp iler -Ġprob abil -ед и -Ġtrad er -æĺ ĵ -ĠR um -ge me -Ġd io -ĠbÄĻdzie my -ĠÏĢ ά -ê¾ ¸ -×ķ× ĺ -Ġठķ -Ġбл аг -Ġscal p -ĠPa use -Ġcapt ion -Ġend anger -Ġen lar -Ġrot ten -ãĥĥ ãĥĪ -Ġw ah -èĤ ī -Ġd zi -ĠInst all -A y -Ġcre ar -енÑĤ а -Ġwe ighing -Ġbutter flies -ĠG ast -äº ķ -h orn -war z -IC EOVER -Ġнай ÑĤи -Ġcoe fficients -ç°¡ åĸ® -ĠSp encer -ĠH igher -Ġcow ork -å¨ ĺ -ĠкоÑĤоÑĢ ое -Ġmon it -Ġdys function -ĠÑģÑĤ анов -Ġtour naments -Ġoy ster -B N -Ġtr ud -sl ow -ĠPen ny -ĠOd ys -æ r -Ġf ou -Ġenjoy ment -аÑĤ Ñĭ -Ġwygl Äħda -алÑĮ наÑı -ĠProt ect -Ġmo y -Ġcl aw -Ġsusp icion -Ġsacrific ed -Ġgost o -B ig -Ġaggress ively -Ġvor ne -ãĥ ł -Ġbl amed -ĠSe hr -פ ר -c ito -Ġse als -Ġmu jer -ĠWe ird -Ġfore ns -Ġcontrib utes -est ra -Ġp og -L OL -Ġhacer lo -о ÑĤÑĮ -f iction -7 9 -λ ο -大 æ¦Ĥ -å£ ° -ĠÑĤ об -ĠG S -ĠCl ara -ite z -Ġadvoc ating -ĠíĶ Ħë -s ung -Ġvert ices -Ġnavig ating -Ġeurop é -çļ Ĩ -Ġslow ed -Ġfore ground -ĠIndust rial -Ġad ore -ìĭ Ń -Ġcré er -æŀ Ĺ -chn itt -Ġun aware -Ġcur ly -ent ar -Ġl er -Ġprohib ited -ĠHero es -ĠRe ed -u ca -Ġsm ok -Ġkun na -zeit ig -im men -ĠL un -Ġаб ÑģолÑİÑĤ -Ġdeg li -Ġvill agers -Ġpres et -z ept -ud s -Ġem it -ä½ł è¦ģ -Ġë ī -ëĬĶ ì§Ģ -нак о -Ġos ób -Ġ196 9 -ĠÐIJ ÑĢ -Ġman chmal -ĠBro ck -Ġmant ra -ĠW IL -b ach -in ä -el as -kel n -Ġdisci ple -Ġqual c -Ġde hyd -ìĿ´ë Ŀ¼ëĬĶ -A f -ìĦ± ìĿ´ -R yan -Ġpupp et -ĠдÑĢÑĥг ие -Ġr ud -Ġp ending -P lus -ĠìķĬ ìĿĦ -Ġb á»ĭ -ĠSe ga -ç e -Ġprogram mer -b li -Ġun l -Ġensl aved -Ġsoci été -Äģ h -Ġinherit ance -ĠBang l -erm aid -Ġpractition er -ĠSt alin -ĠUs er -ci ble -Ġcard iac -ĠKore ans -Ġdump ed -Ġ×Ķ ×Ļ×Ķ -á is -Ġhydraul ic -oubt edly -ĠP it -Ġpic nic -Ġbehö ver -ĠÑģм ог -Ġbra king -é» ij -ut ar -ĠìĦ ¸ë -ub l -Ġü z -Ġmaj esty -Ġb ers -ut able -Ġhot ter -çħ § -ÛĮ ÙĨ -Ġbi ases -Ġsubject ed -Ġnaught y -Ġcir cus -ãģĹ ãģĭ -ĠIm medi -ĠSte fan -ĠTri ple -en k -Ġw it -Ġrecy cle -em ie -d ated -Ġun load -Ġpop ula -ch in -Ġyield s -Ġeng lish -ĠBon nie -Ġsp iders -à ģ -Ġer osion -éĥ¨ åĪĨ -ĠN ICK -иÑı Ñħ -Ġimp art -Ġк ни -Ġres olutions -Ġlith ium -Ġconver gence -ĠT ara -Ġдв е -th s -ĠCind y -æĪij è¦ģ -å¹ « -ĠD IE -Ġass urance -Ġоп иÑģ -Ġbu ckets -Ġc ues -ĠQu iet -Ġsimilar ity -Ġfound ational -ĠMin ist -æ» ¿ -Ġp ian -Ġcent r -Ġnum b -Ġmon ks -uj ourd -en zie -Ġskate board -Ġd latego -ĠÑģ оÑĤ -ĠA E -Ġmaster piece -ĠSol omon -ĠRed dit -Ġr iot -ab l -ĠJ azz -Ġelectromagn etic -Ġinsec ure -ĠComp et -ger ies -об од -ł ×ķ -ðŁ Ĵ -Ġsen ators -ĠBris bane -ĠAl b -utter ing -ĠAll ow -z ero -Ġp ai -ĠÐIJ лекÑģ -ĠDis play -ĠBl ade -ĠApp s -Ġp ä -Ġд еÑģÑı -Ġque lla -ĠGa o -ен нÑĭÑħ -Ġspoil ers -Ġgall ons -ĠÙĦ ÙĬ -ĠZ ion -æľī ä¸Ģ -on ie -rag t -ĠCh and -Ġë³ ij -Ġbl unt -Ġus u -ĠK ad -ra kt -Ġcin ematic -Ġam munition -re ne -Ġfour teen -ĠC arn -c rit -Ġten ure -v u -Ġprincipal mente -Ġalle en -éĢĻ ä¸Ģ -Ġkompl ett -Ġdü ny -J ames -Ġrecept or -Ġones elf -g uru -Ġmerch ant -l iness -Ġover looked -Ġharmon ic -éķ ¿ -ies o -×ķ× ŀ -col m -ĠпÑĢо екÑĤ -ĠAd a -ا س -T im -Ġrecur ring -Ġproceed s -ĠPart icularly -ĠDown load -et rical -Ġmat rices -Ġproyect o -anc ies -ĠUh m -Ġc aves -Ġìĸ´ë ł¤ -ĠLe af -Ġоб ÑĭÑĩ -ĠìĿ´ì ľł -Euro pe -Ġt Äħ -Ġpul s -Ġtak iego -ÐĿ е -G U -Ġfor s -Ïģ γ -Ġfot os -Ġ) ) -Ġë© ¤ë -Ġaqu ilo -ĠK urd -ï¸ ı -pt ic -ĠD ort -Ġmis ery -aus o -åĬ Ł -chuck ling -ĠR idge -ĠíĸĪ ìĬµëĭĪëĭ¤ -Ġ* ** -å® ¢ -ĠHmm m -Ġge ographic -Ġany s -Ġtal vez -Ġske let -Ġsign atures -Ġlit ers -IJë ©´ -ĠÑģво его -Ġski ing -ĠÐľ оÑģ -Ġadop ting -Ġha ft -Ġsymm etric -ĠL iqu -Ġthy roid -Ġmis in -lud e -Ġh ull -ĠX D -ĠG ust -ze ich -Ġvibr ations -Ġes emp -ĠвÑģ Ñİ -ĠQu em -Ġü brig -ĠS ke -ĠLyn ch -room s -art et -f est -Ġfr üher -Ġl ure -ä¸į好 æĦıæĢĿ -ĠìķĮ ìķĦ -ĠW IN -ĠR YAN -ĠкоÑĤоÑĢ ÑĥÑİ -ĠK ash -Ġ×Ķ× ŀ -Ġsaf eg -ĠHall elujah -Ġдв ÑĥÑħ -Ġstap le -Ġsed iment -ĠAct s -Ġbl aming -Ġmain land -Ġsport ing -Ġdecor ations -Ġexecut ing -Ġpar an -ĠDoll ar -Ġproject ions -Ġcommission ed -Ġb our -ö m -Ġste amed -ĠëŃ ĺ -Ġpet rol -Ġcel ular -å¸ ¶ -ĠHung ary -Ġrent ed -Ġв аÑĢи -bb ie -Ġsé cur -ü ll -Ġsw ings -bet ween -Ġи ÑĤ -est ro -Ġnie mand -ĠìĤ ¼ -ĠP ardon -ess es -ĠM ID -Ġcentral ized -ĠAl ien -cul os -Ġcr ise -裡 éĿ¢ -Ġcl asse -beit et -i ÄŁi -Ġwh ales -Ġper imeter -Ġty ing -Ġstr ony -Ġlike wise -ĠP unch -D a -ĠBapt ist -Ġsort ing -Ġ iv -Ġíķ © -Ġre hab -Ġet a -ri ver -Ġsa i -ãģĦãģŁ ãģł -od us -ãģĬé¡ĺãģĦ ãģĹãģ¾ãģĻ -Ġess ayer -Ġtur tles -ĠHaz rat -Ġfab rics -Ġcav ity -Ġpon ieważ -Ġschle cht -Ġs alsa -ÅŁ ekkür -Ġse ating -Ġeconom ists -Ġman g -Ġsegu inte -Ġr ang -Ġrat ios -Ġconst ell -Ġlong temps -u ating -Ġspo iled -Ġrecip ients -Ġsn iper -ä¹ĭ åīį -ìĬµ ëĭĪê¹Į -Ġw p -ĠLIN KE -Ġfl are -ĠAd ri -ñ as -Ġback l -mä ÃŁ -ĠB end -Ġworkload s -ĠÑģ Ñĥп -Ġ197 5 -им ÑģÑı -ан е -Ġм он -Ġaspir ations -ĠA er -ĠговоÑĢ иÑĤÑĮ -ĠQ ian -å¦ Ī -Ġcomprom ised -Ġyol k -ла ÑģÑĤ -Ġhe men -ro ve -d ens -Ġком менÑĤ -Ġ- -- -Ġflu ores -но Ñģ -ĠLiver pool -ĠÑģоб ой -ĠZ we -Ġl umin -ĠO G -á ¸ -hol m -pro fits -S N -Ġproport ions -Ġm ica -ĠB oh -ĠAt las -Ġuns ure -Ġtour ing -Ġn ied -Ġt ÄĻ -Ġimper ative -Ġdem ek -ĠSher iff -r ance -Ġhom eland -ĠH ail -ĠG anz -y mm -M on -åĨ · -v ida -Ġdesar roll -æĬ Ģ -Ġintrig uing -ĠH ugo -Ġ ãĤĤ -é ¬ -а ÑĨ -ĠWiÄĻ c -att ed -ĠìķĦëĭĪ ê³ł -ĠV ari -á d -Ġsur real -Ġdispar ities -Ġm ó -ull en -ĠìŀĪ ëĭ¤ê³ł -Ġп ожалÑĥйÑģÑĤа -Ġma ins -Ġe ject -Ġmeth ane -Ġmarginal ized -Ġchill i -r ès -Ġy em -ä½ł æĺ¯ -ĠCh un -Ġdeb ts -Ġdownload ing -ĠAth ens -is ierung -ry n -Ġte kn -ĠQu indi -éľ Ģ -Ġtara f -Ġh é -Ġconscious ly -Ġfix es -uck le -may ın -Ġfre i -Ġsp a -Ġì§Ħ íĸī -ĠاÙĦØ ° -ĠÑĥ к -let t -Ġolm uÅŁ -Ġche esy -า à¸ģ -na ire -Ġw iden -Ġli en -Ġesca ping -igg s -ĠBl ick -c Äħ -ĠìĦ ľë -Ġ×Ķ× ¡ -Ġв пеÑĢ -oph one -ie ll -ĠSU BSCRI -Ġl ions -Ġê·¸ ê²ĥ -Ġinsp ires -Ġguarante es -Ġcome ça -ĠGrow ing -Ġneg lig -ĠFrank f -Ġge geben -ĠÄij ầu -Ġend lich -Ġì į¨ -ĠT T -ĠL ith -ÏĢ α -aster n -ĠA zer -Ġlun ar -h ic -Ġна ÑĢод -Ġnen hum -è· ij -ĠSalv ador -ĠPro gress -Ġprivile ges -ĠëıĻ ìķĪ -Ġant agon -ĠImp f -Ġdesc ub -ĠLe i -ĠìĥĪë ¡ľ -Ñĩ е -Ġdó lares -ĠMeg han -ĠW ire -to o -ay ing -us c -Ġt ud -Ġappe als -ed uc -Ġp ane -Ġj i -Ġde cks -ĠAl ter -Ġ å°± -ìĦ ¤ -åĪĨ éIJĺ -Ġproduct ions -ĠWILL IAM -Ġimpl ied -Ġfulfill ment -ĠA ah -Ġsa ja -x us -ĠÎļ αι -Ãł s -uc ch -ок о -ĠDisc ord -ĠS Y -j sk -ĠWall ace -un ction -Dan iel -Ġk öt -ij ah -Ġmarch e -Ġdis gr -Ġm ungkin -Ġal ma -³ µ -Ġextensive ly -ĠFl oren -ĠAll ison -ãĤ ± -ÙĬ Ùħ -Ġju ven -ĠRena issance -Ġfundra ising -ĠCha os -Ġpar aly -Ġnarr ator -Ġecosystem s -A sh -Ġmitig ation -ĠA ujourd -ĠIde e -! , -Ġ ½ -Ġland lord -Ġdefect s -Ġac re -uls ive -Ġalg ae -pe k -Ġem ba -ĠR oc -éĽ ¢ -ks om -ä che -Ġle uk -Ġlever aging -Ġê·¸ëłĩ ì§Ģ -ĠPal m -Ġä ven -Ġl is -ĠIn sp -ĠR ita -ĠAb b -ith m -Ġsuper vision -Ġrevis it -Ġpi ÄĻ -Ġeu h -Ġf ades -Ġmot to -åį ¡ -ез ж -ĠSh im -Ġrelev ance -Ġo o -Ġo stat -n ica -Ġcho ix -ĠFac ulty -Ġì¤ij ìĹIJ -ĠAb ove -Ġнеб олÑĮÑĪ -Ġsequ encing -Ġnutri ent -Ġconqu ered -Ġdigest ive -Ġback drop -ĠL ori -ail able -G ame -Ġneglect ed -om orph -ill ah -Ġkn e -Ġsi itä -Ġworks pace -ĠVen ice -ĠK ne -Ñī о -ħ Ģ -ĠH ass -Ġv ita -Ŀ¼ë ©´ -Ġlay s -ên cias -é rica -ĠL l -æ± Ĥ -ĠCo ca -ĠWH Y -èĪ ŀ -Ġrout ing -Ġperm issions -Ġd ings -pre nd -pro gram -Ġcro cod -br al -AAAA AAAA -ag it -ĠN ä -Ġgek ommen -at ten -Ġrefer enced -Ġpair ing -ĠPart ner -ĠCoron avirus -Ñĸ Ñģ -è½ ī -Ġ×Ķ× ĵ -Ġespec ÃŃfic -ars i -qu elle -Ġspont aneous -çĨ ± -Ġê²ĥ ìĿĦ -ĠÐŁÐ¾Ñģ ле -ĠاÙĦ د -ĠSh out -Ġн ал -Ġdisgu ise -ĠJ ord -Ġwe e -Ġmiej sc -Ġser um -Ġplais ir -Ġcred ible -Ġb Ã¥ -ĠA J -ma res -Ġrod s -Ġer an -ãģ¾ ãģĤ -Ġp ää -ĠU A -ĠUn known -ĠÙĦ Ùħ -ĠRab bi -Ġla at -Ġhairst yle -ĠØ º -éģ ĭ -Ġc ach -ĠWr iting -оÑĩ ки -ab ad -Ġstraight en --- " -w ife -Ġhott est -Ġpun ya -ĠF ashion -gr iff -ĠQ R -ot ch -ĠÐľ ожеÑĤ -Cl oud -ĠStri ke -ĠHe in -Ġ 羣çļĦ -Ġle i -ĠFl ow -weg s -Ġha br -åīĽ åīĽ -nah me -Ì ģ -Ġple asing -op ping -Ġ구ë ıħ -Ġdr an -Ġbang s -Ġ7 9 -Ġsk et -Ġcav al -ĠMac ron -Ġweight ed -Ġm uted -Ġnuest ras -EE P -Ġmath ematic -ĠM RI -ag us -Ġtherap ies -θ ε -Ġun pl -Ġcomm encer -f ull -Ġtow els -Ġpr ue -Ġlic enses -׼ ×ķ׾ -ĠÐŁ оÑĩемÑĥ -Ġpoint less -B ye -Ġelig ibility -Ġscra pe -Ġab usive -ĠM ant -Ġje unes -t al -ĠPrin cip -ĠOrth odox -Ġmel od -ĠмаÑĤ еÑĢи -Ġprosecut or -Ġopio id -ĠÑĥ веÑĢ -ĠBe en -Ġìłij ì¢ħ -Ġd ynasty -Ġajud a -Ġent reg -Ġweigh ed -Ġe ure -ĠB em -Ġab normal -8 2 -ĠJ R -ĠA kt -ĠB ri -ú t -Ġst agn -! * -Ġwe gen -Ġle aking -ĠW ords -ĠM au -Ġv ue -ĠL iam -ани ем -Ġclin icians -ĠP ump -Ġför st -? ... -Ġautom otive -ĠOw en -zus agen -ĠH undred -Ġdecentral ized -Ġbul bs -Ġ×ľ× Ľ -Ġprovin ces -ĠMil an -8 1 -k as -Ġëĵ £ -Ġfor ça -Ġright ly -å³ ¶ -r Äħ -Ġven ues -Ġw ai -Ġpred icting -ĠWi Fi -Ġê¶ģ ê¸Ī -ر ÙĪ -Ġ×Ķ× ĸ -cent ury -Ġgrad ual -ĠProblem e -ĠìĹ ħ -Ġcop ing -ĠBr us -Ġpean uts -irts chaft -Ġз ал -ĠT roy -Ġsper m -ĠM itar -ĠTür kiye -g rand -¦ Ń -Ġ×ŀ× ¡ -Ġp ans -ĠKnow ledge -ber ly -ĠÐķ го -Ġdan ced -ĠFr ost -ĠB urg -Ġbit ing -ìłķ ìĿĦ -me al -Ġhero ic -Ġmother board -ĠL icht -ãģ£ ãģ -ll an -ай н -ĠÑĢ Ñıд -Ġ à¹Ģภ-on en -ir ie -Ar t -r ang -ν η -Ġnew born -Ġam is -Ġا ÙĪر -Ġsoph om -ĠCare ful -Ġprospect s -ens en -Ġthr ill -ĠVi á»ĩt -A dam -r ition -ent ric -ud en -Ġcertific ates -Ġas hes -èª ¿ -play ing -Ġs adece -Ġo st -Ġairpl anes -ÑĢ ок -on er -Ġmagnes ium -Ġgod damn -Ġ197 2 -ĠSch ule -Ġtem at -Ġpart out -௠Ĥ -Ġin ve -ĠScient ists -ĠHud son -win ning -ceks in -Ġcongress ional -or u -Ġro pes -в ед -Ġmad re -Ġf erry -ĠCoh en -ĠP red -Ġvag y -Ġб еÑģп -Ġmult im -Ġdrain age -Ġsim ulator -g iggles -ĠSt adium -об Ñī -Ġnot ices -Ġcraw ling -Ġgr oupe -åı ¸ -Ġkto ÅĽ -ĠY oga -Ġmed ida -ĠÑħ ваÑĤ -ĠL ite -Ġr av -or ama -Ġdisc ord -ĠDI RE -Ġte h -ĠN urs -ç² ī -Ġpitch ed -Ġbark ing -ĠC oke -wi ad -Ġpop ulated -éĻ ¤ -pe lled -Ġб ог -Ġpe wno -ĠC ube -Ġrecru ited -éĢĻ 種 -ĠC ara -ıģ ını -im ated -ĠÑĪ кол -ic ional -ĠпÑĢо ÑĦ -Ġcontam ination -Ġúlt imos -Ġfear ful -Ġele phants -us i -ĠiT unes -ĠSw ami -ê ¼ -ĠìĦ¤ë ªħ -ĠRich ards -Ġmagn ets -ĠRicht ung -ĠLeg ion -èı ľ -Ġk itty -Ġkiss ed -Ġwater ing -Ġcon o -ĠPalest ine -id ir -Ġma ze -Ġflu ids -ĠProdu cer -ĠKr sna -好 åķ¦ -la f -Ġ×IJ ×ķ -Ġm iesz -ĠX ing -oint ed -se in -ĠF uk -ĠDep ression -ĠD uty -ĠPan ther -Ġsu nd -Ġref ere -Ġexc lusion -Ġnav al -ĠWin ston -Ġsl ogan -Ġhypoth etical -Ġelev ate -ë ł¹ -Ġcabe ça -ĠGes und -m eter -ĠìķĦëĭĪë ©´ -Ġcloud y -âĢ¦ ? -ĠSch ritt -ĠJ S -ì į -ĠSpr ings -ĠB atter -· ° -Ġtail or -ĠPTS D -ĠG ent -Ġba ÄŁ -Ġspat ula -Ġcr ay -ĠLeg isl -Ġs ú -Ġle ve -า ม -Ġer ad -Ġdon g -Ġd erm -ĠBank s -ich o -åħĪ çĶŁ -ĠFr anz -ra vel -éģ Ķ -ол о -Ġfl ute -ĠE k -Ġjoy ful -Ġch ased -ĠLar ge -O ver -Ġentrepreneur ial -Ġcons iders -Ñĥ ем -op a -Ġdorm ir -ĠElement ary -Ġprzy pad -ÑĥÑģ ка -ĠоÑĩ еÑĢ -ug ene -Ġten ido -Ġlug ares -ë ¥ -ĠÑĩ аÑģÑĤ -Ġsa o -Ġbra id -ĠV ere -ĠRe ich -ĠP oss -Ġin an -w and -re f -Ġmont rer -Ġ198 1 -çķ ª -as ında -Ġch rome -ĠTr inity -Ġexplo itation -ĠS ense -ĠC MS -ĠNo ble -ĠìĦł íĥĿ -Ġswe lling -elect ronic -] ? -Ġbr ushing -Ġliquid ity -ĠH ook -ĠCon nor -ĠAl um -Ġgu cken -su ite -Ġwie le -Ġbarrel s -ĠReg el -ĠM ent -ĠT rip -ĠBr ush -ĠE rik -ur ate -ÉĻ r -ĠC yr -ou ble -ĠBe cca -Ġpass words -Å ± -bor g -Ġv endo -ĠCla us -ĠF az -ind est -Ġdece ased -Ġcompar isons -ĠL CD -ĠP ork -Ġevent ual -Ġpat reon -Ġin ability -Ġext inction -Ġì¢ĭìķĦ íķĺëĬĶ -ĠÑģ оÑģ -aj u -Ġ×ij× IJ× -Ġso fort -Ġdest ined -ĠR in -Ġmouth s -ĠNat ürlich -Ġpres erving -Ġlim p -é» ¨ -oc used -ин г -Ġexp osing -ĠÎ ¾ -ë į -la ugh -Ġhis s -ãģł ãģĭãĤī -Ġind ie -Ġdet al -ÑĢав ÑģÑĤв -Ġtr ên -æķ ° -Ġog ni -Ġsimple mente -Ġ197 8 -Ġgo o -Ġ196 7 -Ġgen ug -h ö -Ġhist ó -å® Ł -Ġlob ster -c endo -Ġte il -Ġalle vi -00 00 -OL D -Ġpes os -Ġbon uses -Ġam i -Ġrev ival -ĠHor se -Ġs ack -T alk -Ġmul her -ĠпоÑģÑĤо Ñıн -ĠH ood -H uh -Ġë¶ ģ -Ġhy ung -ĠMe eting -Ġimport a -Ġì°¾ ìķĦ -ĠV ern -Ġstri pped -Ġref uses -Ġqual ifications -op l -Ģë ıĦ -ix ÃŃ -Ġdi ab -it ime -fl ows -Ġin ac -ĠG ong -Ġmeaning less -Ġcourage ous -Ġmicro bi -az y -h ist -Ġvolunte ering -V IE -Ġviol ated -Ġsymp athy -ĠEd it -好 åĥı -elect ric -produ ct -Ġpand emia -Ġgeomet ric -ĠCon vers -g re -Ġgl ut -ist ed -ĠاÙĦ Ùĥ -ĠCh ain -ĠPres ent -ĠY in -ĠÑģ ог -ĠV log -Ġìĸ´ë ¨¸ -Ġdon n -Ġh itch -uck ing -ãģĬ ãģĦ -w ald -ris k -Ġhar i -ĠK ens -ĠId ol -Ġвним ание -Ġtod d -Ġsm ashed -Ġinv ari -Ġкон ÑĤÑĢ -Ġaut istic -ìŀ¥ ëĭĺ -R es -д Ñĭ -ch au -Ġsel v -Ġhät ten -ठ¿ -Ġexpect s -Ïģ η -Ġaç ık -ĠHT TP -le ÅŁ -Ġswe eping -ĠBet a -Ġcounterpart s -ab ile -ĠSim s -C s -Ġrep ar -s qu -Ġprovin cial -Ġshare holders -Ġrun ter -Ġged acht -ĠTe en -Ġgrand s -çĶ ¢ -ag les -Ġrock y -ven s -Ġr ivals -un al -Ġreact s -ë © -Ġmerc ury -ĠLu igi -Ġо г -ĠJ UST -Ġl od -Ġcort ex -w ig -Ġl akh -ì¤ij ìĹIJ -ĠV ic -ĠM und -Ġma pped -ĠD ell -ĠD ruck -Ġlif es -алÑĮ ное -ivid ual -ad ım -Ġat rav -ĠFl ug -ĠKle in -ê±° ìķ¼ -ห à¸Ļ -Ġapp li -ா ? -ü yorum -ĠинÑĤеÑĢеÑģ но -Ġdis infect -> - -Ġchamp agne -Ġk la -op ers -Tr ans -ĠDes ert -Ġcultiv ate -ĠFuck ing -idel ity -ĠÑĤ ан -Ġinc ub -Ġtem u -Ġlearn er -found er -ĠSy l -ãĤ Ģ -Ġf ato -z ier -ĠìĹĨ ìĿ´ -ĠìĪ ¨ -Ġpsych o -ĠÑĤел еÑĦ -Ġregard e -Ġrepresent ations -Ġlit igation -Ġsp ann -ult s -b ior -è¦ĭ ãģ¦ -ä¸į å¤ļ -ĠSur vey -ĠLED s -Ġtr ä -Ġl ên -Ġant ioxid -еÑĢ ом -Ġindu ction -Ġfool ed -ät zlich -ĠговоÑĢ ÑıÑĤ -ĠF act -umb ai -Ġw iggle -NO UN -Ġdévelop p -ĠCl aro -Ġì ¸ -ë ¬ -ãģªãĤĵ ãģł -Ġaccum ulate -Ġmaint ains -ë Ħ -ĠFight er -íĨ ł -Ġmat in -Ġcoup on -Ġst unt -Ġdeb uted -å¾ħ ãģ£ãģ¦ -Ġpra g -ив аем -7 3 -Ġexp res -Ġìĺ¤ë ¹ł -ĠпеÑĢ Ñģон -Ġcalcul us -Ġab rupt -ĠInspect or -our t -æĸ Ļ -ź niej -int ense -B a -Ġl ounge -Ġast hma -ĠHi ç -ª » -Ġeditor ial -Ġse ize -Ġk ır -Ġm ouve -Ġtier ra -Ġtestoster one -Ġr h -ĠKing ston -EL LE -ĠRepresent ative -Ġ197 4 -Ġi ba -T s -Ġsort a -Ġ( ?) -Ġت ÙĪ -ĠëĤ´ë ł¤ -Ġbek ommt -Ġspirit ually -Ġdist orted -M ad -Ġre im -á nh -ĠOtt oman -ĠRel ig -ĠEl s -Ġret ained -ĠLa ughs -æĢ » -ĠS AS -ĠколиÑĩе ÑģÑĤво -×ķת ר -Ġinnov ate -Ġk ork -ĠÑĢаÑģÑģк азÑĭв -ond ere -iv i -ay e -ount y -ĠполÑĥÑĩ аеÑĤÑģÑı -Ġbun s -åħ « -Ġyüz den -Ġsur geries -Ø£ ÙĨ -Ġbankrupt cy -w elt -Ġsi amo -Ġdark est -ĠH ann -gg a -Ġform as -ĠD j -n amed -Ġshield s -ue ller -ĠF ew -Ġl ace -Ġfur ious -ĠY U -Ġsociet al -Ġjudge ment -ĠD os -Ġj ab -law s -Ġrein vent -ĠK atherine -ĠCh oi -ad ows -Ġr ans -od en -ĠMid west -n ın -Ġdep ort -ĠD ip -ç´ ħ -Ġaten ción -ĠCourt ney -ivid ad -ĠÚ© Ûģ -Ġeffic acy -ĠBrook s -Ġrefer ral -Ġкон ÑĨ -Ġmal icious -Ġk ir -ĠGod dess -Ġfun ky -Ġinter im -ĠK örper -Ġìĸ¼ë § -k ur -Ġк ли -Ġtruc s -ges etz -Ġz ug -ĠGl ück -ĠMin ute -Ġprest igious -Ġnie z -Ġconcent rations -ла ÑģÑĤи -ĠS is -ĠVit amin -ko v -ĠP BS -Ġне е -Ġretail ers -Ġcon ventions -ĠSam antha -Ġproud ly -J ordan -ĠJ ASON -at k -Ġtr iste -Ġst är -Ġreiter ate -Ġpos terior -Ġ197 3 -ĠP ine -ĠJul iet -Ġped ir -k il -Ġover lapping -Ġexclud e -Ġecon óm -Ġaccept s -ĠS ter -æ± º -Ġìļ ´ëıĻ -est ab -Ġt ug -ar g -Ġliv ro -Ø§Ø µ -Ġse ams -Ġbur aya -Ġe llo -ĠT M -ĠP aw -ĠInd ex -Ex c -Ġinspir ational -Ġd unk -è° ģ -ak ter -Ġcondition er -ĠSal ut -ÅĤ ec -Ġìī ½ -ĠÑĥз на -ĠRome o -f ruit -ĠY O -Ġchá» ī -б Ñĥ -b ons -Ġreprodu ctive -Ġor ada -Ġíļ ¨ -Ġtent ar -Ġma ñana -ãĤ ¬ -Ġsol vent -Jess ica -ĠLeg al -Ġtu a -Ġs ic -ĠE Q -au kee -ìĭľ ëĭ¤ -ĠÅŀ u -Ġad here -ĠT ul -Ġà® Ĩ -Ġtext books -ĠFif th -Ġexper i -Ġch ic -Ġhe ap -in ely -at ra -T wo -Ġhele maal -Ġf ren -æİ ¨ -Ġbis her -Ø§Ø ´ -ĠìĦł ìĥĿ -ĠT ages -Ġs á»± -Ġbull ied -Ø ¤ -Ġbenef ited -ĠPre viously -ĠÑį ÑĦÑĦ -Ù į -Ġsen ate -ĠM orm -ij ke -ĠF lu -Ġincorpor ating -j ack -Ġп иÑĤ -Ġimp ly -Ġha cks -ĠR ICH -Ġк ваÑĢ -ĠпÑĢек ÑĢаÑģ -Ġdepend ency -Ġìļ © -Ġì± ħ -Ġwäh rend -Ġsu lla -ĠPitts burgh -Ġesemp io -¼ë ¡ľ -pr ot -ĠR osen -ĠIndepend ence -Ġpars ley -ie gen -Ġha w -Ġaqu ell -ĠC AP -ĠÑĢабоÑĤ аÑĤÑĮ -ĠCl iff -ion ar -Ġsec uring -æĪijåĢij çļĦ -ν ε -Ġutil is -Ġcou le -ĠP ing -Ġtre k -Ġf ak -Ġenorm e -Ġìĭ « -è® © -Ġdoub ling -ĠнÑĢав иÑĤÑģÑı -Ġh ed -ho ven -ĠStand ing -Ġm ÃŃn -ĠJ imin -Ġmon arch -Ġco ke -Ġm r -Ġcl ic -à į -Ġimpe achment -Ġdur ability -Ġvar ios -Ġcommercial s -Ġgreet ings -ĠR i -ĠApp reci -ìŀĪ ëĬĶ -Ġrés ult -ér t -Ġsal ute -Ġpoder ia -Ġsun rise -ve ck -Ġreluct ant -Ġcommission er -å¿ µ -â te -ĠKen ny -ĠSir i -ãĥĥ ãĥĹ -ĠëĬ ĺ -ĠE E -Ġun ch -к он -ĠاÙĦØ ¥ -Ġbel ts -Ġhas s -Ġмо Ñı -Ġdispl aced -Ġab ra -ÎŃ Î» -Ġscratch es -Ġcom et -Ġauthor ization -ĠL LC -Ġprodu k -Ġrehabil itation -å ŀ -Ñĸ Ñĩ -ud ing -ol it -Ġ10 5 -Ġexp ands -Ġalt ri -ĠKom ment -Ġan f -P l -ĠM ana -f ed -Ġb ri -Ġor a -G s -ĠG ur -uck land -Ġjun ction -Ġiron ic -ĠFe ed -Ġpra kt -ĠHam mer -Įë ıĦ -ĠTr acy -çµ ± -ĠAs ide -н его -ĠиÑģполÑĮз оваÑĤÑĮ -Ġz aj -Ġequ itable -Ġcur b -Ġãģĵ ãĤĮ -Ġderiv atives -Ġpupp ies -ĠKenn eth -ĠCom pl -ig ram -ĠGar cia -) " -ĠHar bor -est ial -Ġ ä¾Ĩ -Ġ ers -æ ¹ -Ġunw anted -Ġbel ang -аР³Ð¾ -em b -d os -ĠìĻ ľë -ĠBud get -Ġbatt ling -ØŃ Øª -k ok -наÑĩ ала -Ġpl ag -Ġcant idad -Ġgrup os -Ġplug ins -ler ini -Ġиме еÑĤ -Ġso zusagen -ol ics -Ġpue blo -Ġrem inis -r än -ĠMor rison -Ġl inha -Ġbreath s -ĠT aste -Ġenf rent -ĠDo cker -Ġд ен -Ġethnic ity -Ġw ob -Ġsuff ers -Ġtransition ing -ĠR ange -ÄĻd zy -Ġк аÑĤ -Ġsy ner -Ġdon ut -Ġprob abilities -ĠO mar -Wh ich -u ish -is in -Ġdem os -ĠìłĢ 기 -Ġëĺij ê°Ļ -Ġед ин -Ġc erve -Ġj oka -I AN -Ġkilomet er -Ġhorizont ally -ĠBh ag -Ġ- > -ĠMon itor -Ġknowledge able -Ġf av -Ġpin ned -Ġe Bay -ick er -Ġìŀłê¹ IJë§Į -ĠXia omi -Ġcap it -Ġn p -Ġ196 5 -ho e -Ġn ok -ĠS age -Ġн елÑĮзÑı -ĠT ow -g am -Ġdic en -ĠSUBSCRI BE -Ġrebo ot -Ġp aj -Ġë³´ìĹ ¬ë -Ġth icken -ĠRe ality -id än -N a -Ġê²ĥ ìĿĢ -!! ) -Ġrout ines -Ġод ного -Ġex ting -Ġì¦ Ŀ -Ġsulf ur -Ġcar ve -Ġastero id -ĠWarri or -Ġphotograph ers -Ġpe ll -Ġcros sover -æĪij çŁ¥éģĵ -Ġhace mos -ĠNe j -Ġsett ling -Ġir m -ĠBook s -ient ôt -Ġesp acio -ĠSchol ars -Ġdo omed -ĠIR S -w ohl -Ġseg ue -ĠëĪĦ ê°Ģ -Ġpr atic -B T -ĠConsider ing -ĠBuff alo -Ġtrain ings -Ġge bru -ĠG leich -Ġpir ates -Ġen velop -Ġre open -im at -Ġte e -Ġsu ed -fe h -Ġ×Ķ× § -Ġdi ets -Ġjunt os -ast o -Ġmisunder stood -Ġru im -Ġclass ify -ĠпÑĢод Ñĥк -Ġin se -Ġillust rated -Ġcorros ion -Ġacc red -ĠAunt ie -ĠпÑĢив еÑĤ -ĠLI VE -Ġre k -Ġrece ipt -åĪ° åºķ -ĠBar bie -ĠSn ake -t urn -Je ff -ãģĬ ãģĬ -ķ Ħ -VO ICEOVER -co ll -Ġrun ners -ìł ľë -os os -mo on -Ġkey note -ĠInst it -S PEAK -Ġplug s -Ġcur v -ĠY uri -ĠTh eres -ĠP s -Ġμ ÏĢο -Ġconver ter -Ġref ine -Ġbad ass -Ġο ι -Ġreg en -az zi -ÙĬ Ùģ -Ġse ized -Ġiç er -ile e -Ġup stream -Ġbud s -Ġp im -Ġíķĺë £¨ -Ġall uded -Ġthem ed -Ġconsist ing -Ġb ons -un uz -ĠпÑĢов од -ĠLove ly -ॠĭ -Ġpar ach -ĠSta ats -éļ Ĭ -Ġselect ive -Ġf ase -ĠGeor get -Ġcoc aine -Ġreprodu ction -ĠL ara -ĠL D -Ġg h -J on -Ġl Ã¥ -Ġëij IJë -Ġtyp ed -ĠB ana -ë ĵľë -Ġsav ory -ĠZ omb -stand en -Ġpedest rian -Ġdifférent s -Ġìĭ ¸ -èī ¯ -Ġcompl ained -ç¦ ı -ĠÐļ ÑĤо -Ġ×ľ× ¤ -ali ÅĽmy -Ġmort ar -Ġverd ict -Ġsu ficiente -ĠMill ion -mitt el -in als -ĠاÙĦØ ® -аÑİ ÑģÑĮ -Ġmi ÄĻdzy -ĠO le -Ġin vert -czy Äĩ -озм ожно -star ter -Ġaud itor -ĠSc out -ch ien -ĠSver ige -uff led -Ġze hn -ĠA uckland -Ġarg ent -Ġ197 6 -ĠHo e -Ġboth ers -Ġsocial ist -Ġpl iers -Ġemer gen -ĠX P -еÑĢ ов -M ore -ĠLe vi -ĠAnd ers -ibil idad -ĠP arents -Ġindu ced -ìĸ´ì ¤ -Ġbal ances -ĠвÑĭ ÑĪ -Ġsubmar ine -St art -Ġdri es -Ġvol ver -Ġtick ing -c ott -Ġf aj -pr és -ĠS abb -Ġза Ñĩ -Ġпок Ñĥп -Ġbapt ized -ĠBrill iant -ĠÐij ог -Ġm ots -b its -Ġlatt ice -æĪij è·Łä½ł -Ġcor iander -Ġresid ency -yn c -Ġpier wszy -ĠKn ock -ĠZ ap -ĠÐķ в -ê² ¬ -å°ı å¿ĥ -Ġune ven -ĠJ as -od or -ç¿ Ĵ -7 4 -ĠS ite -Ġacontece u -ym pt -Ġtril ogy -Ġlan tern -ĠZ ucker -v ari -we lling -ĠPot ato -gom ery -Ġreact ed -ĠChr on -Ġj ede -be eld -Ġtw ent -Ġl act -æ¨ Ĥ -Ġré se -Ġrel ent -Ġfurn ace -Ġwid get -Ġearthqu akes -ĠAd just -il it -ĠØ£ ÙĪ -Ġhear ings -Ġdefend ant -irs iniz -Ġbas k -c ja -ľ ¨ -Ġrif les -Ġinst al -ĠFor give -p ical -ĠÐŀÑĩ енÑĮ -Ġpet ites -Ġh p -Ġren owned -ĠIn n -Ġ주 ìĦ¸ìļĶ -Ġemphas ized -éĹ® é¢ĺ -ĠìŀĪ ì£ł -Ġê²ĥ ìľ¼ë¡ľ -ãĤ Ĩ -Å ĵ -g ili -D ave -Ġexha usting -ÅĤ ug -Ġsch ema -μ ά -cy cl -Ġaut ant -Ġpar cel -Ġmater ia -ĠB erry -ĠÑģ ами -Ġextract ed -ĠSay ing -ism atic -Ġпоп ÑĢоб -Ġneur on -g raph -ľë ©´ -Ġencl osure -ĠJoh ann -Ġafter math -ÑĤ об -Ġu ży -Ġs amp -3 60 -ĠMe i -Ġt aco -Ġrecept ors -Ġpunch es -ĠHo je -ĠÙĩ ÙĨا -=" # -ĠAng ular -Ġmus ique -Ġro l -Ġà ± -ster reich -Ġcl am -ĠTre asury -chem ical -Ġap ar -Ġapp end -Ġforb id -ĠHamb urg -ак ов -Ġê¸ Ī -ild a -Ġprepar ations -Ġmog Äħ -Ġcam ino -E ric -ĠBl ind -èĪ ĩ -å¹´ çļĦ -ĠDis covery -ì¸ ł -çĪ ¶ -Ġinterpre ter -Ġb red -ĠPsal m -Ġdef ended -ìī ¬ -ĠEr fahr -ĠPe ach -Ġmo ons -ĠO st -Ġspé cial -Ġarri ver -ĠW is -u ci -Ġrobot ics -I VE -Ġsie ge -ar la -Ġsepar ates -ĠT C -íı ° -quis ite -Ġparenth eses -ик е -ç« Ļ -Ġtr ous -å» º -ĠÑģ илÑĮ -Ġbe ers -Ġпл аÑĤ -ãģĻãģĶ ãģĦ -Ġso la -Ġd ès -ming ham -ik te -Ġo ops -Ġtw itch -å° ĩ -Ï Ī -ĠShould n -uv re -Ġle er -cript ions -Ġeyes hadow -ĠGu o -ĠPow ell -Ġsup uesto -Ġan a -r als -ĠMont real -Ġsurf ing -ĠÐŁÐµÑĢ в -×ŀ ×ķ -Ġmillise conds -Ġsubur bs -Ġplanet a -ÑĥÑĪ ка -hr lich -ĠH Y -Ġس ÛĴ -ĠM M -ĠE ff -åı¯ æĦĽ -ĠH S -ans on -Ġì§ģ ìłij -Ġsu o -Ġdeploy ing -Ġk unt -ter ing -Ġere ct -ìŀ¥ ìĿ´ -ĠìĿĮ ìĭĿ -Ġspec imen -! ... -æĪij 說 -Ġlig ne -Ġk onst -ade qu -Ġìĥģ íĥľ -Ġaccess ed -ĠP ole -k ill -Ġë² Ħë -Ġauthentic ity -Ġapp elle -ull e -Ġrev ision -Ġgo ats -г ли -Ġp au -ĠR anger -ĠIm ag -aut hor -Ġe ve -ĠMess enger -Ġn ay -Ġwh oles -ät te -Ġon wards -ĠDep ois -Ġíijľ íĺĦ -ĠSAR S -Ġwszystk ich -Ġdest ru -umb ing -Ġcompat ibility -Ġmis information -od ore -ĠF avor -ek o -ı Į -w aukee -ĠTe aching -ĠK O -Ġbet ting -Ġquest s -Ġviv re -ĠмÑĥз Ñĭ -Ġs aga -Ġswe ll -Ġge he -æĢİ麼 樣 -ĠоÑĢг аниз -Ġg ide -ĠG ross -Ġdale j -Ġcl aws -á»Ļ c -Ġprejud ice -Ġins ign -i hood -Ġpl ed -Ġdó nde -ĠPolit ical -Ġprem ises -und ert -ع ت -on nen -Ġespa ço -Ġf é -ĠHarr ison -ĠC ensus -Ġcard io -Ġdi y -Ġmil ieu -Ġjourn ée -ĠRe lease -N IE -ĠM uk -id ée -á»į i -Ġiç inde -ŀ Ļ -Ġreson ate -Ġm oles -ĠF lying -ĠGl oria -ĠPast or -ĠAre na -好 ä¸į好 -N ON -ол ов -Ġall ÃŃ -om at -ìĸ´ë ıĦ -Ġcaracter ÃŃst -Ġdecl ining -Ñĸ Ñı -an co -ĠIn form -Ġbarg ain -Ġbus hes -ĠNat urally -Ġre chts -ĠT ensor -ĠPat ricia -Ġprincip io -ĠM umbai -Ġwom b -Ġnost ra -Ġdile mma -Ġirgendw ann -Ġ196 4 -Ġenerg ÃŃa -Ġна ÑĢ -Ġseg regation -ĠA thlet -Ġ» , -Ġy eni -ĠSe it -Ġven om -Ġdak ika -Ġëı Įë -ĠÃī l -Ġf us -ĠM og -¦½ ëĭĪëĭ¤ -Ġrem ar -ĠTed dy -Ġbreast s -ic ans -æĶ¶ çľĭ -k ap -Ġh Æ¡n -ĠJ P -ãĥ³ ãĤ¿ -Ġresur rect -ĠìĿ ¸ë -her ical -Ġfot ograf -ĠJos é -Ġlivel ihood -Ġbib li -ter i -Ġvor stellen -ĠA AA -Ġassess ing -Y A -Ġspl end -Ġexca v -Ġbapt ism -y ll -w ow -M ac -Ġpl astics -teok bokki -Ġintéress ant -Ġcommand ed -Ġfamous ly -ĠÐĺ ли -ĠMan uel -Ġsouth west -Ġde formation -ÃŃcul o -ĠнаÑħод иÑĤÑģÑı -ĠP atter -d egree -ĠczÄĻ sto -" - -Ġìħ ĭ -Ġman ger -ĠTrust ee -Ģë ¦¬ -Ġpunt os -iv able -Ġvol atile -ĠëĬ IJ -Ġinst ability -Ġc iel -ci Äħ -Ġpur ity -но ÑģÑĤ -S il -ed ar -åĻ ¨ -NOUN CER -Ġspe lled -G ER -Ġsanct uary -Ġacceler ating -Ġsc out -ĠпÑĢ ев -f ahren -ãģĵ ãģ¡ãĤī -ĠëĤĺìĺ ¨ -Ġpocz Äħt -ĠMe u -ka ar -³´ ê³ł -ak ra -D own -ĠÃĦ r -ĠEl ite -Ġall ons -Ġmay onnaise -ĠS ustain -prising ly -Ġsuper vis -Ġê·¸ëłĩ ì£ł -Ġunemploy ed -Ġfresh ly -Ġ×ŀ× ¢ -ĠD h -Ġtack ling -Ġo gr -Ġì´ Īë -ãĤĪ ãĤį -Ġlo ft -ar ah -ĠA irl -ĠD ir -ĠÐľ ожно -Ġbook ing -ĠC RA -Ġhtt ps -Ġcho ke -Ġg own -Ġno ite -Ġz ac -ist ol -Ġsec re -Ġresemb les -Ġcu ad -ìĤ¬ ê°Ģ -sh ow -Ġbl anc -Ġag u -ĠPr int -ast ed -ĠWe ather -i pl -Ġobsc ure -Ġcont e -ough s -) ; -ĠD ame -ä¸Ģ 缴 -Ġclar ification -Ġintim acy -Ġup hold -ĠMir ror -Ġw agon -x ide -Ġcl og -app er -ĠImmedi ately -ú de -Ġtouch down -Ġro oft -аÑĪ а -Ġç ıkt -Ġla isser -ĠUn real -ens itive -Ġ12 3 -Ġpl aster -Ġduck s -Ġet me -Ġb ishop -bre vi -Ġb ic -ä¸ĭ åİ» -Ġrun time -Ġamb itions -м аÑĤ -ĠWe in -ĠMar i -ĠíĬ ¸ë -Ġresol ver -Ġng Ãły -ĠR ise -ãĤĪãģĨ ãģ« -ĠCr us -Ġmerchand ise -Ġel i -Ġstate wide -Ġow l -éģ ł -æĶ ¹ -Ġtwist ing -Ġcontam inated -ĠCom merce -hy thm -Ġà Ī -Ġìĭ ¤ë -Ġmus ste -u ir -Ġsum s -ĠSome where -ãĥ İ -Ġk ami -Ġa ired -ĠAND REW -Ġê º -Ġv iendo -Ġantib ody -Ġabsol ument -Ġprotest ers -ĠQué bec -st adt -Sha un -Ġcham bers -ĠWe ar -ĠEffect s -Ġhaz ards -Ġne i -Ġcoraz ón -Ġá ¼ -ĠS G -Ķ © -ĠìĹŃ ìĭľ -Ġcom fy -ĠC ody -Ġpens ando -Ġg anska -ĠAc ross -öll ig -aby te -Ġwed ge -Ġkal ian -Ġsig ue -end es -ĠGro ÃŁ -Ġutil iser -Ġfl own -ани Ñİ -Ġle var -rest rial -Ġillust rations -Ġas lında -BLE EP -Ġдо ÑģÑĤ -Ġtur ret -Ġsuit case -ziÄĻ ki -Ġsket ches -Ġac red -ĠRe i -Ġt sun -ĠS ag -Ġthird s -ĠKIR BY -ra i -Ġhuman os -Ġrecomm ends -Ġextraordin arily -Ġcommence ment -K N -ope z -Ġ×ij× © -Ġlet hal -ĠEst amos -Ġinspect or -ĠSe ok -e un -Ġoff shore -Ġget tin -ye ars -ĠSil ence -ĠNat ur -up un -Ġtr zy -Ġno get -Ġhamb urger -ĠPra ise -é nd -Ġ197 1 -yl ie -k rit -ĠìĥĿê°ģ ìĿ´ -çļ ® -Ġmoment os -Ġest é -Ġdisse min -Ġgig s -Ġdes af -Ġav is -ĠZ oo -ĠìķĬ ìĿĢ -h äng -åı ¥ -h ake -ĠB ism -Ġre think -ĠMal colm -Ġident ifies -l ower -ix el -Ġtv Ã¥ -k ed -ier z -Ġö ffentlich -Ġproc laim -so on -l ol -Ġlo i -Ġb itten -ro llo -Ġser mon -Ġes qu -Ġjack ets -Ġgr áfic -Ġпок азÑĭв -Ġcabe za -ch odzi -Ġpel vis -Ġnost algia -Ġbre w -Ġshort cuts -ĠAd emás -Ġsuperfic ial -åħ© åĢĭ -Ġbo ca -ĠæĪij æĺ¯ -iment os -åĽł 为 -Ġspr outs -é£ Ľ -ĠJon as -ĠFloren ce -st atic -da ughter -* ) -ÅĤ by -f ashion -ĠG inger -Ġë§ ¤ë -Ġhust le -ut os -ĠÑĤ Ñıж -ĠL ös -ש ×Ļ×Ŀ -any ch -tu ber -Ġtid y -Ġfront al -Ġwhis key -Ġhum id -ĠÎ Ł -Ġr idge -Ġmar in -Ġb ientôt -ĠCarr ie -ch w -Ġtah un -ĠEr geb -F R -Ġìłķ ë¶Ģ -ĠSold ier -Ġenlight enment -Ġexam ining -ĠNot re -Ġer am -ĠSun ny -Ġlay ered -ĠD azu -r ades -好 åIJĥ -ĠнаÑĪ ей -Ġtim ber -Ġman ners -ĠBir mingham -Ġmini ature -omet ers -Ġfill er -ĠR ip -ĠK omb -own er -ì ¿ -id ian -Ġdem ás -ĠÙĪ ت -Ġpreca utions -Ġgovern o -z elf -ĠCom plete -å¸ ĥ -ĠPh antom -ãģ¾ ãģļ -Ġн ез -ĠкаÑĢ ÑĤ -ĠAnt wort -ĠPf izer -ĠFran co -Ġw ÅĤ -Ġfr ig -es per -Ġk ale -Ġfilm maker -Ġk urt -Ġinv alid -å± Ģ -are lla -Äĥ ng -ram ento -Ġnutr itional -Ġdict ators -Ġaf in -Ġf uzzy -ĠG ina -ó t -ĠExtrem adura -Ġdemonst rations -ĠMont gomery -íķ´ì Ħ¤ -ĠGand hi -ãĥ Ŀ -ç½ ® -Ġreun ion -Ġjaki ÅĽ -ĠZ ug -OU GH -l ifting -Ġ ಠ-á¹Ľ á¹£ -e b -ĠW OW -ĠSh iva -omet ry -Ġwild ly -Ġt ended -Ġmeg ap -ì² ĺ -Ġna use -Ġg erek -ãĥ ĭ -ĠMar cel -Ġn este -Ø® ر -Ġfe h -åĨ ħ -susp enseful -ĠWrest le -ĠPalestin ians -ĠG ORD -iy et -ĠÑĢ ади -Ġvers uchen -Ġtrans istor -ĠÐŁÑĢ оÑģÑĤо -Ġпон ÑĢав -Ġrhy me -ĠVerm ont -pl atz -è® ° -ĠÄ°ÅŁ te -ĠH ag -ĠÐĺ м -ĠÑĢаÑģÑģк аз -Ġmet ros -ĠInfin ity -w olf -ib al -ft ig -Ġ ÚĨ -Ġíĺ¹ ìĭľ -Ġo ggi -Ġdisp osit -ĠпÑĢ ил -ĠвÑĭ пол -Ġth ôi -ĠK ENN -Ġhand ing -act us -Ġtac os -Ġformer ly -ĠCorinth ians -ãģ« ãģ¯ -ÑĨÑĸ ÑĹ -Ġpad re -Ġcongreg ation -æ ij -fer t -Ġsub ir -ais er -qu a -ara oh -ĠCur ry -ĠìķĬ ëĬĶ -ел Ñİ -Ġf uss -Ġbo oty -Ġl ows -Ġh ommes -ĠM H -ĠDisney land -w ent -Ġresid ue -Ġbe eping -è¼ ķ -ät ta -Ġm ould -ĠPro jekt -st alk -Ġartif act -ĠAnt rag -ĠAM D -ĠCry pt -Ġë© Ķ -ĠFel ipe -ĠCO B -el u -Ġself ies -ĠS anti -ch utz -ĠУ кÑĢаÑĹ -ges amt -Ġflo ck -j az -pl ain -Ġwr inkles -Ġre ais -Ġpal jon -Ġempower ment -Ġattend ees -pp a -Ġn eden -он Ñĭ -Ġtime frame -ĠCher ry -Ġid ée -Ġg ag -Ġdon key -Ġô ng -ĠH are -éļ Ľ -ĠK ara -Ġacom pan -pl aces -im ientos -ĠH amm -б и -ub en -ili yor -Ġth irst -Ġk ry -ĠGeorget own -׳ ×Ķ -Ġor ch -Ġheart beat -Ġtransform ations -est ones -ĠK H -Ġcart oons -Ġan ci -Ġworth less -Ġtail ored -p u -Americ ans -Ġp iles -ĠMon key -Ġbas in -ĠTem per -ĠP aint -Ġpunch ing -Ġba ik -ĠOak land -v re -ÅŁ allah -yd d -Ġcas ually -od u -Ġc oded -ĠNorweg ian -ĠV ince -Ġprem ature -ĠProm ise -ек ÑģÑĤ -Ġdevast ated -ĠPrem ium -ĠPar am -ĠÃĸ yle -um uz -P O -r ators -Ġlamp s -Ġterritor ial -Ġback bone -list ed -D Y -ĠاÙĦ ر -Ġpurs ued -ĠComm ons -Ġê³ ¡ -lo cks -ed or -Ġconce ived -g ere -Ġdisappe aring -ĠS ull -ĠìĹ °ë -Ġho ffe -Ġdet ox -íĶ Į -Ġret ir -ĠëģĿ ëĤ -Ġper gunta -ĠB OY -ç² ¾ -Ġp enn -æĿ¥ äºĨ -h és -h on -Ġcatastroph ic -Ġa ust -Ġtor so -Ġìĸ´ ëĬIJ -ĠìĤ¬ëŀĮë ĵ¤ìĿ´ -Ġmarvel ous -ĠHar ley -ach ine -Ġti ế -itt o -ĠI ÃŃm -yl on -Ġshut down -.' ' -Ġap ologies -ĠCommun ication -ĠговоÑĢ Ñİ -ãģĤ ãĥ¼ -âĦ ¢ -ÃŃ veis -ac un -Ġret aining -Ġcontrad iction -ĠAD AM -C OM -Bry an -ĠM onsieur -Ġadap ting -Ш ÐIJ -ĠSc r -änd ert -Ġpl aus -ä»Ĭ天 çļĦ -Ġon set -Ġassist ants -Ġval ves -Ġsc atter -ĠR ust -aw ia -Ġread iness -Ġp ais -Ġb ible -Ġamb iente -Ġа меÑĢик -Ġunc ond -Ġk alk -åĬ ¨ -Ġmo c -un n -Ġact u -Ġhum ming -iss imo -ĠPat rol -g ow -ãĥ ¤ -ĠTHE Y -ĠBod en -ĠB ie -Ġre el -ĠÑĥÑģл ов -Ġende avor -ĠPer iod -ustom ed -m als -al on -B ox -ĠÏĥ αÏĤ -Ġom dat -Ġal tre -ĠHe h -k ad -Ġprotect or -Ġdomin ance -odynam ic -Ġcommunic ated -k ö -Ġprede cessor -ĠL uk -ĠFl ower -Ġãģ © -po que -ÑĤи ÑĢов -Ġret rospect -Ġdecis ive -Ġexem pel -{ \ -ĠR ück -r ite -ĠZe us -Ġcal orie -Ġattract ions -ĠH inter -Ġuh m -ĠíĮ IJ -Ġrul ers -Ġdiscour aged -Ġaconte cer -Ġacc ents -ĠOpt im -ĠAl g -k ids -20 21 -ĠLind say -Ġfilm makers -pr owad -Ġter ug -ëĭ ´ -ĠSom mer -20 18 -Ġborrow ing -ĠTrans fer -н оп -ari as -Ġhead phone -ì¼ ľ -Ġtransl ating -Ġauf ge -ப à®Ł -we is -av ant -pa id -b aby -Ġtough est -Ġrepe ats -ĠTer esa -L ord -Ġacab ar -ĠR ide -d ir -Ġl eng -Ġd wa -Ġhead aches -Ġn ữa -ĠнаÑģ ÑĤоÑıÑī -Ġbo ils -Ġlong ing -ri as -ó rio -ĠParad ise -ĠSeñ or -erd em -Ġrein st -Ġsal aries -Ġinsec urity -ÅĤo ÅĽci -ĠабÑģолÑİÑĤ но -ink en -ĠEd dy -ud os -Ġd ummy -Ðļ ак -s ix -Ġin box -Ạ© -Pe ople -á»ĵ ng -Ġorganiz ers -f ind -Ġü l -ĠCO M -ż a -we ile -Comment ary -íĬ¸ë ¥¼ -ĠMitt el -k us -èĽ ĭ -ठ¨ -ir al -Ġgar ment -ικ ά -Ġst ool -pay ers -Ġsh immer -ĠO llie -ĠJe żeli -è¿ĺ æľī -Ġ197 7 -Ġje ux -Ġext inct -ĠTransport ation -ĠM aker -Ġj ohn -Ġrich est -Ġtraum at -Ġli egen -´ë ¥¼ -è¿Ļ éĩĮ -Ġun rest -ĠSt raw -æĭľ æĭľ -Ġcom a -ĠKr isten -ĠÐļон еÑĩно -ĠBry ce -ĠÑıк Ñĸ -Ġpearl s -Ġпоним аÑİ -Ġadd itions -Ġas ympt -ĠменÑĮ ÑĪе -Ġsc ans -Ch ild -ĠH ide -к ÑĥÑİ -et as -Ġd ank -Ġple as -Ġess ays -Ġj ets -åħ Ĵ -Ġв ед -Ġposit ives -ho f -- ) -zz o -Ġstar ters -Ġsm iled -Ġ194 4 -qu iera -Ġro k -Ġpu esto -N ico -Ġsim ulations -Ġ ච-Ġintrig ued -ĠOver watch -åĸ Ĥ -s igh -b ai -Ġë§IJ ê³ł -id é -Ġcra bs -áºŃ p -ĠIraq i -ìĿ´ë ¥¼ -ÑĤ Ñı -ĠSoph ia -ĠDN S -Ġönem li -ĠLu o -Ŀ ¤ -ĠCoun sel -l igen -анÑĮ ÑĪе -Ġtrump et -Ġd apat -ĠJ M -ĠEVER Y -Ġå°į ä¸įå°į -å¤ ¢ -ĠL ayer -Ġc ô -н ал -ĠJ oo -ĠH ack -Ġs unt -ĠLeon ard -ĠFire base -äng er -Ġexpl oding -v oy -Ġì¦ IJ -ĠÑģ еÑĢÑĮ -Ġsever ity -Ġbest imm -çµIJ æŀľ -Ġt iring -Ġprocure ment -Ġdiplom acy -Ġdecor ative -ĠÙĬ ا -Ġpenet ration -Õ « -Ġout right -EN E -ĠUn i -od les -Ġz eros -Ġdelight ful -j m -Ġdo po -没 äºĭ -Ġposit ivity -ĠVIS TA -ĠRes ource -íĥ Ģë -ÑĪ ие -C arl -Ġpip ing -Ġchop ping -ĠGan ze -ü ss -ĠA o -Ġsh attered -ĠDet ective -Ġund oubtedly -Ġhall uc -Ġen ch -Ñĭ Ñĩно -ÑĥлÑı ÑĢ -is esti -Ġped als -Ġdur um -¤í Ķ -la imer -Ġprop re -C u -Ġtransl ator -Ġca ÅĤ -Ġê·¸ 걸 -Ġca ÅĤy -U A -Ġrev ised -Ġпод об -ĠArt icle -ĠHait i -Ġà ĵ -ĠC trl -Ġroz m -la it -Ġletz te -is pering -dis play -Ġalumin ium -Ġpalab ras -Ġconoc er -Ġz itten -Ġdir ig -åıª æľī -Ġbrain storm -Ġw ifi -ĠPart icip -Ġview point -ĠQu an -Ġhier arch -W elcome -å¯ ¾ -Ġoff en -ĠRe covery -gan o -W ould -Ġrep ro -Ġper ceptions -Ġdem asi -ĠBangl adesh -ĠIncred ible -Ġlet zt -Ġbehav ing -Ġaston ishing -Ġâ Ĩ -ĠëĤ¨ ìŀIJ -èµ° äºĨ -ãĥ Ķ -ĠGORD ON -C AR -? !" -ĠP rest -Ġë§ŀ ìķĦìļĶ -Ġt and -Ġl ash -ç Ĭ -ific ant -Ġint oler -Ġг еÑĢо -Ġte u -as o -ĠÑģов еÑĤ -Ġtravel ers -ĠSy nd -ĠвеÑĢ Ñģ -F onda -ad ı -Ġtrans cription -Ġtit anium -Ġtw ists -Ġgear box -ens ation -f at -C oll -ĠCommon wealth -z on -ĠPolize i -ĠAPP LAUSE -f ry -ĠJud a -este em -Ġso ck -ĠJug end -Ġк ÑģÑĤаÑĤи -ĠD ro -Ġproch aine -ãĥ¼ ãĥ« -Ġli ksom -ĠEner gie -ĠMar ina -Ġ2 30 -Ġê°Ģ ìĦľ -ump ing -Ġl one -ç´ ļ -Ġfont s -Ġbusiness man -Ġp ly -Ġdo e -gr id -ĠMil waukee -ĠE den -! ". -ĠÛĮ Ûģ -og ens -Ġteas er -Ġqui én -Ġincent iv -go vern -Ġchild care -Ġsneak ers -Ġimprison ed - ® -иÑĤ еÑģÑĮ -an bul -Ġreg ain -Ġtranqu il -Red ner -éĽ ¨ -IF A -Ġide ological -Ġmayor ÃŃa -Ġb ureau -et erm -ĠD ID -ìĬ · -Ġw aving -Ġbe b -Ġá r -Ġк в -Ġenv oy -an ut -ик Ñĥ -ĠEnviron ment -ĠAss ass -ãĤĵ ãģ§ -ĠB read -ĠТ ÑĥÑĤ -Ġstair case -ĠDise ase -Ġauc un -Ġëĭ Ī -Ġconfront ation -Ġ194 1 -Ġiron y -Ġwor sh -ãĤĮ ãĤĭ -Ġf ick -ĠNa omi -Ġback side -ie ux -K ap -Ġved ere -Ġlength y -Ġbreak er -ĠRoll e -Ġpred ator -Ġnoss os -Ġadvert ise -è³ ĩ -ÑĢод е -Redner wechsel -re ten -Ġcollect ors -ıģ ımız -Ġtr ig -Ġax es -in ters -Ġpen alties -ĠOs man -ĠJen na -Ġfl akes -Ġtrain ers -Ġstun ned -ĠSc roll -ĠP ip -Ġна ÑģÑĤ -Ġnh Ãł -ĠSm ack -ẫ n -rat os -ĠÑĢабоÑĤ Ñĭ -Ġu cz -ĠLem on -ĠS ind -Ġpsych ic -ĠAb g -Ġmamm als -Ġimmers ive -Ġb ots -Ġverschied ene -Ġg eral -Ġfoll ower -Ġ ä»ĸ -Ġsegur idad -Ġimmers ed -fe ito -c ross -Ġö ld -íĥ Ħ -Ġãģĵ ãģ® -Ġ×Ķ ×Ļ×IJ -ĠJ ian -Ġbili yor -are a -Ġk af -Ġgod t -缸 ä¿¡ -Ġë°© ìĨ¡ -Ġdet riment -æ¥ ļ -Ñĸ л -ĠÄij âu -Ġchlor ide -ø re -le i -Ġmont e -Ġdifférent es -à¯ģ . -Ġcareg ivers -Ġin adequ -Ġfare well -ĠÑĤип а -ont ec -ĠE ph -HH H -ĠTod os -ĠС ШÐIJ -Ġtro v -Ġl ige -Ġc ông -ĠC iv -Ġcap az -ĠV allahi -Ġquest e -Ġrepl ica -س ب -z na -ĠÑģл Ñĥж -ĠP T -w ave -ien i -Ġrel ied -de velop -Ġdem e -ĠA man -Ġ[ ...] -Ġcompl iments -u ais -ĠíĮ ¨ -Ġsmell ing -Ġdad urch -ÙĪ ت -Ġor anges -Ġл ай -Ġstabil ization -åĢ į -ãĤĮ ãģŁ -æ¥ ½ -Ġappl iances -Ġh m -ĥ IJë©´ -odynam ics -Ġc iÄĻ -ĠC ott -M ON -ĠM ang -æĶ¯ æĮģ -Ġall erdings -ικ ή -sh ots -Ġt s -ĠG ör -ĠCH AR -Ġ: ( -Ġwr ath -Ġf ique -Ġfüh ren -Ġtest ament -Ġ^ ^ -á¹Ľá¹£ á¹ĩa -AL D -Ġtext o -ĠDog s -Ġs ib -Ġpath etic -ock s -Ġrad ically -ĠM ORE -ĠJAM ES -Ġing l -ĠTechn ical -Ġpor ch -ĠU T -ĠобÑıз аÑĤелÑĮно -Ġrenew al -Ġaesthet ics -ik um -Ġbe verage -der n -Ġpredict ive -Ġch uy -ĠRegard ing -ĠFor ward -ĠÙĪ ÙĦ -Ġcontext ual -Ġdwar f -Ġpre he -Ġgovern ed -ħ Ħ -Ġtrabal har -Ġnegó cio -ĠболÑĮÑĪ ой -еÑĩ аÑĤ -Ġд ÑĥÑħ -Ġflood s -Ġbow ling -ĠO B -ĠH är -Ġgrad ing -주 ëĬĶ -Ġg ars -d ling -Ġr ak -ë Ī -c reat -ĠÑī е -Ġneighb ours -f ood -Qu ery -Ġhero in -ice ps -ĠK inda -N ET -Ġmar i -Ġim itate -Ġach ter -Ġsettle ments -ra re -cc iones -Ġë ĵľ -Ġf ik -it ung -Ġм акÑģим -Ġel f -Ġd alla -ĠPol sce -ĠP ul -Ч ÑĤо -ĠMor gen -ØŃ Ùħ -Ġsuprem acy -Ġk ys -ĠHur ricane -ĠG TA -ĠFe h -Ġfinal mente -m und -ĠK rie -é poque -ĠT ucker -IT T -Ġl ur -Ġdi pping -ä v -Ġeer ste -ĠFl int -bild ung -ู à¹ī -Ġto im -Ġpr acy -Ġtransform s -Ġspeed ing -Ġpresent er -Ġfellow s -f illed -ie za -Ġadv ising -ĠInter view -и гÑĢ -we hr -ĠD ante -pt ure -Īë¬ ¸ -¯ ¸ë -IJ IJ -ĠCoun ter -Ġcr ist -Ġì§ ľ -Ġje une -ĠÑģÑĤ ÑĢаÑĪ -Ġmie Äĩ -Ġtut or -Ġmas ala -Ġpowder ed -Ġn au -ĠFreder ick -Ġbill ing -ĠE isen -Ġд обÑĢ -Ġm est -æ ½ -Ġsn ipp -Ġmon o -ĠA lo -ĠMer cy -éri ence -Ġcasual ties -ĠAN NOUNCER -ä» İ -Ġto car -Ġbacter ial -H o -Ġstre ak -ĠJ ENN -Ġpl ast -Ñģ лед -Ġre app -Ġpay check -Ġmin ers -hab t -ĠJ ap -н ÑĥÑĤ -Ġred emption -Ġqu ir -hn lich -Ġaccum ulation -Ġsh ove -Ġadrenal ine -M ake -ĠH ern -oss ing -ĠV il -ub by -her tz -bre aks -Ġsp ur -ĠD aha -US TIN -Ġcontinu er -ĠSa ul -ãģ® ãģ¯ -Ġíı Ń -ĠëIJĺë ©´ -Ġë§IJìĶ Ģ -Ġо ж -Ġsuspect s -Ġla quelle -ĠMuch as -Ġv öllig -ul en -Ġimp res -Ġlo bb -ene e -Ġн аж -T a -Ġréal ité -ĠRe x -Ġharvest ing -Ġest r -æ ¶ -osp ace -OS S -Ġdisturb ance -ass ic -ĠIs ab -Ġdéc ouv -ĠHamp shire -Ġor nament -Ġlu ôn -ĠU W -Ġj Äħ -éĤ£ ä¹Ī -Ġrespect o -Ġcomun idad -Ġcom igo -ag na -Ġintrins ic -ĠAlum ni -Ġses leri -Ġestim ation -âĢĶ âĢĶ -Ġprodu it -ãĢĤ ãĢį -Ġв ÑĢ -Ġwh irl -Ġac ces -ç u -Ġvari ability -Ġv odka -its u -Ġinternship s -Ġalloc ate -R R -íĽ Ī -Ġinstruction al -t ant -Ġà®ħ த -Ġinv ites -Ġha k -Ġsca res -Ġe clipse -п ов -к олÑĮ -ativ as -Ġstab bed -ĠD OM -ä¸į åĪ° -ro ots -ĠPict ure -íĺ ¼ -ĠC HA -ie c -ı ı -han ol -Ġmisunder stand -R ay -Ġroad map -ocument ed -iz ione -ĠOl ive -r ift -Ġ×Ķ× ł -æ¯ į -l est -; ; -ĠE A -éľĢ è¦ģ -од Ñĥ -Ġhob bies -Ġbur ial -ãģ« ãģ¡ãģ¯ -Ð ¤ -le ge -ĠH J -Ġobject ion -Ġãģ Ń -ct ory -Ġincre mental -Ġgym n -Ġepid emi -Ñģ Ñĭл -à ij -Ġadvance ment -Ġpar ch -New s -Ġa yr -л ам -Ġ×ľ× © -Ġdipl oma -ãģ¡ãĤĥ ãĤĵ -Ġrob bed -On ly -Ġinc ur -Ġch anting -Ġíķ´ë ıĦ -Ġrich es -ĠCar men -Ġnost ro -λ ÎŃ -ĠPow der -à¹Ģภ« -ĠìŀĪ ìľ¼ë©´ -Ġgerçek ten -ĠPik achu -ем он -OL L -Ġplanet ary -Ġsl ows -Ġclock wise -al ion -Ġì Į -Ġver n -Ġh omme -Ġend point -Ġinnoc ence -Ġelement os -Ġsophom ore -Ġnot ions -ĠCould n -p ur -Ġz at -Ġobs ess -Ġmotiv o -ĠK ub -ĠDr ug -A nt -ĠPlay ers -ĠHum ans -Ġme lee -ĠWild life -ĠV P -Ġvolcan ic -Ġcom in -ĠGu ang -ĠÏĦι ÏĤ -ĠоÑģоб енно -ĠS ize -L isten -ĠA aa -app ro -Ġbar bar -ĠPark inson -нÑı ÑĤÑĮ -å į° -Ġunderest imate -Ġsubst itution -Ġcosm etic -ä¸ĭ 次 -Ġwill en -Ġbe ide -ann i -Ġcondition ed -ĠDe bbie -Ġis to -ĠEd wards -ìĽĮ ìļĶ -ĠÑĤ ов -Ġab brevi -ĠM ün -ĠPr inc -ĠLi ang -Ġst ink -Ġradio active -ãģĨ ãĤı -Ġac ontec -Ġun con -ĠTur bo -ãģ IJ -Ġkiss es -æĺ¯ ä»Ģ麼 -еÑĤ ÑĢов -Ġfront ier -ĠSp y -ĠBel arus -ĠC BS -á» Ĺ -am oto -íķľë į° -ĠÑģÑĤ ÑĢо -ĠEn fin -Ġbread th -éĺ ² -ĠCa fe -ĠDaf ür -ĠB our -ar as -Ġbl ueprint -an ı -Ġconst ants -Ġattack er -ĠForm ula -za Äĩ -Ġs owie -Ġeyebr ow -ob ook -Ġset zen -第 ä¸ī -ons ider -aw ning -Ġsöyle ye -Ġinv aded -Ġpronoun s -Ġdob ry -S i -ĠÐ¥ оÑĤ -Ġvolley ball -Ġl ament -is ches -ar me -ap i -ĠW iki -ли ÑĪ -Ġkas ih -Ġp ess -ĠÑĦ оÑĤ -ĠS ul -å¾ · -Ġpse udo -Ġmem o -ĠìĹ° ìĬµ -ĠдоллаÑĢ ов -ĠпеÑĢ ем -ĠRe ach -mir al -alt ed -Ġstat ut -read ing -Ġsöy led -ĠLind sey -ĠAh mad -ë ¶Ģë -ĠС егоднÑı -Ġprzy got -Ġhy ster -U RE -ĠNe igh -Rep orter -ĠB unu -ĠTreat y -ĠR ank -ĠF ame -in ished -Ġge ared -Ġcomp ose -od ia -ĠL on -Ġjeste ÅĽmy -ĠDIRE CTOR -Ġel kaar -ĠV iel -×IJ× © -ynth ia -ä¸ ¦ -Ġm ère -ĠTom ato -Ġex atamente -ni ÄĻ -ĠFre i -ĠD if -Ġopen ings -Ġgraph ical -ĠÑĥд об -ĠвÑģ п -ĠWeek ly -ев а -Ġhang s -Ġuns afe -Ġem blem -ĠKolleg innen -al ay -Ġk si -Ġh ides -Ġol may -Ġent ste -Ġarth ritis -ÃŁ erdem -Ġbin nen -Ġlist ens -ĠH ess -åĨį ä¾Ĩ -ĠLou ise -ld en -ен Ñģ -ĠVers ion -ĠAgric ulture -ìĬ¤ë ¥¼ -м ан -ë Ħ¤ìļĶ -Ġw ines -ĠIN F -r ul -ĠJ K -ıyor lar -sh ield -reat h -Ġter us -ĠL um -Ġanticip ation -Ġacc ustomed -ĠM ina -Ġw ield -io è -mer a -Ġcount down -Ġcl ing -Ġcomm end -Ġfakt iskt -Ġdef enses -Ġcock pit -Ġком анд -Ġdish was -ĠThan os -Ġkid neys -Ġse he -Ġmicro bes -Ġc uff -ĠвÑĭÑģ ок -ĠSp icy -çŃī çŃī -வ à®° -cul us -or c -ç¾ ħ -ix es -ĠC redit -Ġr aj -Ġbring t -ĠN iss -Ġgr im -ĠS OL -Ġten im -ĠSud an -ĠSp art -Ġpromot es -ĠN ossa -ĠÑģоÑģÑĤо Ñıни -Ġì° © -Ġunc ont -ĠLiber al -ĠТ олÑĮко -ĠV iele -Ġktóre j -Ġ* *** -M ax -ĠЧ ÑĤобÑĭ -3 50 -Ġíĺ¼ ìŀIJ -Ġë¶Ħë ĵ¤ìĿ´ -Ġwar p -Ġteng a -Ġsympath etic -Ġbiz i -ĠZ ack -ied o -Ġëī ´ì -p iel -ĠÑĤ ол -Ġsc aled -ĠPET ER -ĠCO MM -ĠC ame -Ġcatast rophe -Ġsweat y -ig ration -Ġstuff ing -ĠÏĢολ Ïį -ĠDri ver -zy st -T ech -Ġassess ed -ĠSur face -ır ım -s ur -ler weile -Ġд ог -Ġshut ting -Ġfr actions -ĠÑģ ол -every one -Ġer n -ĠÐĿ ов -Ġdefend ers -Ġvers ucht -ãĥ³ãĥ Ģ -Ġpol ity -ĠÐŁ он -ver ständ -Ġbrows ers -Ġtransform ative -Ġdict ate -ĠLE GO -Ġning una -ê´ ij -Ġp izz -ĠHar old -ĠL opez -Ú¾ ÛĮ -an ız -atch et -ÙĬ ت -Ġl ernen -Ġê·Ģ ìŬ -Ġhous ed -Ġclean se -ĠW AT -lar ation -Ġby tes -Ġtuck ed -Ġfault s -д о -F X -Ġìĸ¼ë§ ĪëĤĺ -Ġde form -Ġcontract ing -ĠTIM E -ir se -Ġne ben -Ġc erc -ĠArm strong -Ġtest er -Ġparf ait -Ġjealous y -Ġtox ins -Ġdis bel -ÑĥÑĢ Ñĭ -imp ression -Ġprost ate -Ġfire wall -Ġclass ics -еÑĩ ÑĮ -Ġsocial ism -Ġgrac ious -ĠÑģ нова -Ġд нÑı -Ġburn er -ĠMin or -Ġìļ°ë ¦¬ë -Ġjed es -Ġcontinu um -Ġh ots -Ġoccur rence -Ġadminister ed -Ġзам еÑĤ -Ġhes itation -Ġdr ills -er ca -ĠвÑĤоÑĢ ой -Ġstead ily -Ġinsan lar -Ġi han -í ij -Ġhel per -ĠSen in -åģ ľ -ов ание -ĠER IC -b la -ĠAcad emic -Ġhuman ities -bl ack -ump y -ort ex -Ġìł Īë -ĠØ¥ ÙĨ -Ġdiscl ose -ĠEl ijah -Ġλ ÎŃ -ĠQu er -ب ÙĦ -ãĤ ¡ -T ell -ar le -Ñĸ ÑĢ -Ġaug mented -Ġë¹Ħ ìĬ· -Ġand roid -ठ¤ -ar ma -Ġs zer -ge ord -Ġge ek -Ġye ux -Ġp ong -ĠãģĿ ãģĨ -Ġtort ured -ĠB ath -z ig -ason able -Ġn ets -Ġbar u -ĠFl at -ĠV ater -ĠTer ror -ĠA vo -Ġceremon ies -ro e -Ùģ س -O ps -Ġhy vin -Ġap resent -ol or -ĠигÑĢ Ñĭ -ort on -Ġê·¸ëŀ ¬ -Ġlook in -ĠT Y -ĠM int -Ad d -Ġm ite -ĠSm oke -Ġnot a -Ġm oss -ĠAb end -Ġì» ¨ -Ġexagger ated -f ires -Ġred ist -ff iti -Ġopen ness -ê°IJ ìĿ´ -ende u -ен ной -W atch -Ġav atar -ĠP ey -ur un -Ġsen za -Ġì§Ģ ìĹŃ -ĠNat omiast -Ġemer gence -ray s -Ġcraft ed -g ary -ãģł ãģij -ü ng -- " -Ġhack ed -Ġstr ay -en cie -em o -Ġcom en -ĠK ız -ĠJ asmine -ĠH indi -man as -Ġinfin itely -em on -ìĿ¸ëį° ìļĶ -j ak -Ġro aring -éri que -s weise -ĠRo lex -åł± å°İ -ĠStu art -bn b -Ġdiagn ose -Ġcoher ent -ĠM J -æºĸ åĤĻ -Ġp ike -l av -Ġorchest ral -а ÑģÑĤи -Ġterm inar -Ġgather ings -Ġcompl iant -Ġupgrad ing -Ġregul ator -Ġlan ç -éĢ £ -Ġmerch ants -ta wa -Ġmonit ored -Ġrend re -ä¸ ¤ -Ġunter wegs -ang uard -g ard -ĠBel ow -du ino -ĠЦ е -Ġimped ance -ìľ ¡ -ä» ½ -Ġakt uell -ĠV atic -åŃ © -Ġste wards -Ġbright est -Ġk enn -Ġk au -ĠMat rix -ĠB ark -ĠðŁ ij -Ġt aper -Ġcas ino -ר ×Ķ -ys ical -Ġbuild ers -ĠczÅĤ owie -ĠNep al -Ġ! " -Ġterm e -Ġin nych -Ġmath s -Ġdraft ed -ĠB alk -Ġhesit ant -Ġvolt ar -Ġrev ive -ĠÑĦилÑĮ ма -Ġassass in -ĠS olutions -Ġdu el -Ġbear ings -à¸Ħ ะ -Ġrook ie -ik at -Ġbisc uits -Ġc ords -Ñĥв аÑĤи -AR IN -Ġprogress ing -ĠG ir -Ġpenet rate -ĠSt orage -e ight -ĠÑĤ ÑĢÑĥ -Ġdon ÃŃt -Ġsiz in -Ġout dated -ĠнаÑĪ и -Ġaff ir -Ġspo ons -Ġon i -Ġfl ank -ĠG ol -h ã -Ġp éri -Ġhonor able -ĠBreat he -sc enes -Ġob viamente -ик Ñģ -Ġש ×ŀ× -Ġsmooth ie -ŀ Īë -Ġd ime -ĠíĸĪ ìĸ´ìļĶ -Ġapp el -ĠCath olics -Ġsing les -Ġlat en -Ġç ünkü -ĠV ader -æı Ľ -Ġvard ı -ĠIst anbul -gr é -ĠEl sa -ë l -Ġinve ce -Ġcr ane -Ġo be -ĠSh ark -Ġsm ack -Ġrest oring -. \ -Ġë¹ łë -Ġf aded -um bers -S inging -Ġdep ressing -th est -ĠW ahr -Ġmult itude -ÑĢавÑģÑĤв ÑĥйÑĤе -rij k -ek a -Ġcomplet es -ĠWell s -Ġro y -ĠPr ay -ĠKal au -iz in -iaÅĤ em -Ġlo com -ĠNash ville -ĠPent agon -ë ¯¸ -ĠNE W -Äħ Äĩ -ÃŃ ss -Ġmarry ing -Ġfe ud -íĻ ķ -æĢ ¥ -) ! -ĠOper ations -Ñĥ ÑĶ -Ġmo je -Ġinstruct ed -ĠëĪĦ 구 -Ġ×Ķ× Ĵ -ĠпомоÑī ÑĮÑİ -Ġsab ia -ìķĺ ìĸ´ìļĶ -pl ane -p ri -Ġпол ноÑģÑĤÑĮÑİ -ĠK itty -Ġpróp rio -ed ere -Ġinteres ante -Ġд е -Ġcond ensed -Ġav ent -T OR -Ġgre asy -AR K -ort a -A J -Ġdis reg -Ġcorrect ions -Ġst ero -Ġinfluen za -Ġdess es -Ġball ots -Ġme get -Ġma fia -Ġb öl -n ost -ĠÑģÑĤ аÑĤÑĮ -Ġrespond er -Ġhint en -g rav -à¸Ń ะ -yn chron -Ġvi ens -Ġsam o -Ġd t -pan nt -ĠÅĽwi at -Ġзап иÑģ -Ġmer ged -Ġke p -Ġmis leading -Ġdig amos -Ġam mon -è¾ Ľ -ch et -Ġê°Ģ ìł¸ -Ġun i -ĠëIJĺ ëĬĶëį° -Ġнап ÑĢав -ĠкоÑĤоÑĢ ого -Ġanim ate -×ķ× IJ× -еÑĢ в -Ġmin ced -Ġka um -ãģĤ ãģģ -ÏĢ ε -л ег -exist ing -Ġplata form -ĠK RIS -ìĽ ł -ĠFamil ien -ĠLib ya -Ġbiod iversity -Ġidi ots -ird i -Ġszy b -ĠRoll ing -ü cht -ĠÑĥд ив -Ñģ Ñĥд -Ġreal izar -Ġcan ned -ĠÑĢ ан -Ġmet abolic -ĠBe ef -Ġkil ka -лÑİ Ñģ -Ġreg istry -моÑĤÑĢ иÑĤе -Ġviel ä -Ġod c -Ġcondem ned -æ© ĭ -f al -ĠD il -wo ÅĽci -A w -Ġstatist ically -Ġso gen -ĠB ETH -Ġsh aving -å¹ ¸ -oc al -ĠFun ny -Ġpeace fully -Ġaddict ive -ĠIns ert -la uf -Ġexperien cia -é¦ĸ åħĪ -иÑĤ елÑı -ÃŃ gen -ág ina -Ġabdom en -íķľ ëĭ¤ -ic us -im ana -ì į¨ -arch ing -Ġkonk ret -ìķ ĺë -ек а -ou fl -ive l -Ġn ude -èt res -Ġm onsieur -Ġcl ash -Ġtherap ists -Ġcub ed -Ġretrou ver -Ġwave form -Ġpot em -ĠForm er -is ión -åº ľ -Ġ×IJ× Ŀ -und os -ĠMein ung -ص ÙĦ -ĠJ ude -Ġn Ã¥r -ĠLeon ardo -ĠCr isto -ĠG OT -ÑģÑĤÑĢÑĥ к -L AN -Ġg Ã¥ng -Ġdé b -ĠFrankf urt -Ġcra ppy -Ġli l -ann ée -ĠмеÑģÑĤ е -RE T -ĠN er -ĠCO STA -Ġjed em -Ġcurt ains -Ġiter ations -Ġun av -Ġpla que -or um -ĠÎ ¶ -Ġnúmer os -Ġdes ap -² ½ -Ġcomp iled -Ġref le -Ġrank ings -Ġrep aired -ĠÐĿап ÑĢ -Ġdownload s -Ġarm our -Ġ×Ļ ×ķתר -Ġlonge vity -ĠTON ER -ĠкомменÑĤ аÑĢ -Ġcz ego -Ġnot ify -Ġairport s -Ġend uring -let te -Ġapp arat -Ġhab il -á»ĩ c -n ad -IC O -ĠBra h -Ġseg ún -Ġgovern ors -k aha -ĠSchl uss -Ġodpow ied -ir ting -Ġrem pl -ĠAb original -ident ally -Ġenhan cing -lic ting -ĠHawai ian -Ġstri ving -ĠN iet -Ġzn aczy -Ġobed ience -ĠnÃ¥ got -Ġexp ired -Ġ19 18 -pres ented -Ġpr owad -ĠTer r -ĠPrinc eton -Ġmor gen -Ġattract ing -ĠS igma -ign er -ĠRe chts -ĠP eki -Ġmet hy -Ġha mm -Ġdire ito -Ġdeleg ation -ив аÑİÑĤ -Ġg in -You ng -Ġdepend encies -ĠBrad ley -bud s -Ġf is -Ġpyt anie -Ġinterconnect ed -Ġemba ixo -ĠS as -Ġr uh -ĠS icht -S ur -Ġsuper b -ĠSabb ath -ĠD anger -k ol -Ġh ou -s upp -ĠN acional -Ġsuccess ion -Ġv á -ĠMaÃŁ nahmen -ĠJess ie -ĠId aho -fore st -ħ ĺ -Ġ×ŀ× ĵ -ĠØ£ ÙĬ -Ġsweet heart -Ġneat ly -ĠEv angel -ê³ ¡ -ĠSu ite -úblic a -ĠÑĥ ли -ĠAnn ouncer -l igh -Ġsens ations -Ġshel ters -Ġh art -Ġsqueez ing -ĠR ivers -ĠCook ing -ì± ħ -person al -Ġman os -ÑijÑĤ ÑģÑı -w ij -Ġgo gg -ĠMill i -ĠF P -ün st -ĠL S -Ġspray ing -Ġf aux -Ġaut ograph -olog ic -Ġtor ment -Ġencry pted -á» ħ -Ġest re -ç¹ ¼ -à ± -Ġst umbled -Ġa ider -Ġsab en -x ter -ĠC ities -ĠTür k -ëĭ ¥ -ch ine -Ġto pping -Ġpoison ed -ĠRoman ia -×ĵ ×Ļ -Ģë ¡ľ -ĠпоÑĢ Ñıд -Ġchir ping -ĠìĻ Ħë -×ij× ¢ -Ġcu anto -Ġdon ating -ĠReg ent -ĠBer uf -Ġdistract ing -Ġstam ina -ĠDar ren -Ġì¶ ķ -l ists -d al -ch uss -Ġeconom ist -ãģĪ ãĥ¼ -org t -Ġist iyorum -è¿ Ľ -ĠSur prise -ĠHa o -Ġìµľ ê³ł -ĠG W -ĠIn ner -Ġqu ieren -Ġmind ed -Ġsupercom puter -Ġdiagram s -íĬ ľë -ê²ł ìĸ´ -ĠобÑĬ ÑıÑģ -Ġestab an -Ġdestro ys -ĠBre aking -Ġkar Ä±ÅŁ -Ġrebuild ing -ľë ĮĢ -ли во -ĠSau ce -ĠF usion -×ķ× ŀ× -ĠQu inn -Ġga uche -ĠÙĪ Ø£ -Ġ È -ç ĵľ -Ġtechn o -Ġdisp atch -ĠaÅŁ k -Ġein zel -ĠG mail -ç ŀ -Ġê°ľ ìĿ¸ -ĠÑģем ÑĮ -Ġjour neys -Ġi ht -Ġfib re -Ġdram as -ouch ed -Ġren ame -Ġоп еÑĢ -Ġpo o -ĠD ru -ĠиÑĤ ог -Ġz ast -Ġco z -Ġz ucch -Ġobt aining -Ġcomm ute -Ġsub mer -ĠV ish -ĠR abb -og g -Ġh ut -íĸĪ ìĸ´ -æ¯Ķ å¦Ĥ -ere mi -Ġμ α -Ġdisk ut -Ġб Ñĥк -Ġimp aired -d epend -ĠÙĪ ا -ĠÑĢ Ñĥк -Ġб аÑĢ -Ġoxid ation -Ġsitu ação -ÉĻ n -u ção -Ġsag te -ĠS ER -ĠC ake -Ġtur meric -ĠK ak -b ung -ĠK á¹Ľá¹£á¹ĩa -Ġpoison ing -Ġsl ipping -ĠS ays -å°± åı¯ä»¥ -ò ng -çŁ ³ - « -ĠClaud ia -ĠChar acter -ни ÑĨ -co at -Ġprogress ed -ĠFer gus -Ġìĺ¤ ëĬ -Ġo at -ord able -ĠLe y -ĠHera us -Ġresult ados -ĠKay la -Ġr iff -Ġcheg ou -Ġx i -Ġsp acious -Ġrecogn ised -Ġe ch -ĠT ie -Ġlaunch er -J im -Ġsupp ression -ĠImp ossible -Ġguit ars -ĠFour ier -иÑĩеÑģ кий -ĠTh erap -ĠK af -cent ered -ĠÑģо оÑĤвеÑĤ -Ġk lim -Ġcarbohyd rates -ign ant -ĠAst ron -Ġem ple -Ġdr astic -ĠмиÑĢ е -в ин -u w -Ġpret tier -Ġdon uts -ĠAth ena -Ġdiss ert -Ġpl ante -Ġur anium -ìĿ Įë -ar é -Ġrze cz -Ġdisplay ing -æĪ ² -Ġsar c -r ão -Ġtamp oco -Ġphilosoph ers -ĠRe cht -æĵ ļ -Ġcoment arios -y se -Ġìľ ¤ -Ġm ise -ĠG in -Ġн ом -ĠFR OM -l iner -at if -Ġspo ÅĤec -x a -ĠÑĤ ÑĢÑĥд -Ġw ag -기 ìĹIJ -ĠM G -Ġoff spring -ĠUnder standing -åıª æĺ¯ -OR A -Ġwh irring -Ġsur rend -Ġpok er -Ġmon uments -ĠâĻ © -Ġorgan ised -ĠSo zial -ĠF actory -Ñħ а -Ġrese mble -з д -Ġexplos ions -Ġpay roll -Ġom n -ĠJ orge -ι Ïĥ -Ġfract ure -Ġpersec ution -Ġdem ais -E CH -, ) -Ġcri ar -ĠJ OSH -Ġdem ographics -Ġ16 00 -Ġcur rencies -ĠT ips -Ġ éĢĻåĢĭ -ĠRe fer -ĠDan cing -Ġincons istent -Ġde h -Ġimm ens -Ġme ist -Ġimpat ient -Ġbehav es -æĿ ¾ -ĠëĤ´ì ļ© -Ġback story -Ġagree ing -ĠÅ ģ -ih in -Ġtemper atura -ĠBack ground -Ġnut zen -Ġëħ ¹ -ĠM änner -Ġcollabor ations -ĠK os -éģİ åİ» -Ġnight mares -ë ĵ± -ĠQueens land -Ġassoci ates -ĠK ok -Ġfact orial -ĠHy ung -Ġê·¸ ëĭ¤ìĿĮ -Ġfil ho -Ġel ét -Ġíĸī ë³µ -° ± -Ġgef unden -Ġsemic ondu -Ġcounsel ors -ĠU pper -ĠA ub -ick ers -V er -Ġnorth west -ĠMainten ant -ĠL akes -аÑı в -int é -ì° ½ -Ġг аз -Ġgi orn -Ġdigit ally -ĠCirc uit -ì¼ Ģ -ãĤĬ ãģ¾ãģĹãģŁ -Ġcheer ful -ĠPet erson -ĠDan ish -ativ os -Ġli ken -Ġhar bor -али ÑģÑĤ -x e -Ġcur ls -ĠR hod -E nd -ĠE T -Ġacqu aint -ĠKel vin -Ġtr if -ĠA way -ìŀIJ ëĬĶ -v s -Ġp ágina -Ġin let -ĠSant os -Ġìļ° ìĻĢ -Ġyap ıyorsun -th eme -Ġsou ff -Ġinject ed -Ġpó źniej -iver so -amp ed -Ġda her -Ġd agger -ĠлÑİб им -Ġt ummy -Ġenlight ened -c ents -ĠD ah -Ġcu est -ä¾Ĩ 說 -IL Y -Ġ×ij ר -Ġbang ing -ĠEm il -ĠC ler -ĠB order -иж Ñĥ -Ġpresent ers -ĠST UD -co ins -ĠíĻ į -Ġper ks -Ġpar ap -Ġcertain es -ĠL ore -ö st -ĠMAR TIN -Ġb ios -Ġwhere by -ver ts -ĠMir anda -Ġst ip -æ¾ ¤ -and ez -׼ ׾ -uj in -Ġê ¾ -Ġaller gies -pl ate -Ġyap ıl -Ġundert ake -ĠëĤĺ ê°Ģ -P art -Ġkız ım -h guru -ãģĤ ãģ¨ -ĠJohn s -Ġeyel ashes -Ġdra ined -Ġst Ã¥r -ãģĤãĤĬ ãģ¾ãģĻ -ĠJ ade -Ġcal end -fil m -Ġmes a -Ġlud zie -Ġattract s -Ġju ices -Ġк ил -Ġnieu we -Ġmen cion -Ġign ition -Ġbl adder -anda ag -ĠExt ension -íĤ ¨ -fe ed -ĠÙĪ Ùĩ -Ġsp un -Ġt ät -оÑĢ оÑĤ -ty ard -ron ics -ĠH uge -Ñĥж д -st ring -Ġun just -Ġpra wn -Ġfrost ing -Ġdisappear ance -ios a -Ġcard i -ĠPri est -Ġcient ÃŃfic -åĵª 裡 -ĠÐĴ аÑģ -Ġë¶Ģ íĥģ -Ġth ieves -Ġphys ique -ĠE ugene -Ġбли з -Ġmon opoly -Ġbi ography -Ġho ÅŁ -Ġt ö -m ac -Ġshock s -ìĦ ¸ë -h it -Ġsn ug -Ġinc l -Ġded ic -Ġult ras -Ġизв еÑģÑĤ -Ġutil ization -ĠÑģовеÑĢÑĪ енно -Ġserv i -st ag -1 80 -Ġse wer -ĠCh oice -Ġdis charged -ĠJ D -ол еÑĤ -ĠкваÑĢ ÑĤи -Ġteles cop -ĠJe ÅĽli -ĠN ana -c ale -ĠÑĤ он -mm m -äºĨ åIJ§ -Ġge habt -ëĤ ł -æĬ ķ -à¸Ļ à¸Ļ -Ġet her -Ġz en -Ġresearch ed -ĠCzy li -å®Į åħ¨ -work ers -Ġê²½ ì°° -Ġsher iff -all o -Ġtip os -Ġprosec ution -Ġfrog s -Ġf alt -j d -ĠíĮ Ķ -Ġfilter ed -ĠO ft -Ġì į -Ġdis fr -ĠMust ang -Ġwo ah -ĠRE ALLY -Ġмог ли -Ġentr ada -Ġиг ÑĢа -Ġmix es -ĠавÑĤом об -Ð Ļ -Ġsh in -Ġparan ormal -Ġsome place -Ġdish on -eta an -Ġfu erte -Ù ¹ -Ġdo om -ìĪ ľ -Ġexist ential -Ġbu ld -ĠSD K -ĠпÑĢав да -Ġturn over -ĠìĹ¬ê¸° ìĹIJ -Ġठ¹ -Ġmodel ed -Ġbug ün -Ġexperiment ation -Ġmorning s -Ġmed o -Ste vie -Ġplay able -Ġairl ines -g ments -Ġê¸°ë ¶Ħ -ĠT omb -ĠMV P -AUDI ENCE -Ġcheck out -Ġpas st -Ġbe ispiel -ĠLink s -he avy -Ġquestion able -Ġìĵ °ë -Ġs ill -Ġmanip ulated -ĠL oren -Ġìľ ¼ -Ġver ge -á k -I ES -Ġsab ot -ĠCustom er -ale ży -Ġnom inee -ĠG ad -Ġnouve lles -ĠS PE -ist ling -Ġo val -обÑĢ аж -if ty -éĩ İ -Ġbez el -y et -Ġfre ight -ĠHan ım -r ÃŃa -Ġz oning -Ġind em -ĠB ü -Ġfemin ism -Ġvo ix -Ġof icial -Ġdi yorum -» IJ -Ġar ose -Ġpar ar -ìĿ¸ ì§Ģ -ĠMart ine -ĠL ect -Ġrest er -Ġdrown ing -u ya -c ida -ĠAri el -Ġ0 2 -Ġ×Ķ ×Ķ -ç´ ł -ĠW ert -Т Ñĭ -Ġwid ow -Ġparch ment -Ġcott age -ĠX L -ĠSl ack -ĠN ES -Ġro be -Ġg imm -Ġcam inho -ĠHar per -Ġcit rus -Ġfirefight ers -Ġdop amine -el ets -Ġdemocr at -ìł ľë¡ľ -Ġplay back -o j -ĠпÑĢ ок -ĠSull ivan -se mble -ĠW orth -ĠMust afa -า ร -Ġmet s -éĸ Ģ -л оÑģÑĮ -Ġinert ia -Ġuniform s -è¶ ³ -é rio -×ķר ×Ķ -é nt -Ġà® Ĵ -ĠÑģам ÑĭÑħ -Ġvou lais -ĠZ immer -ê² łë -Ġн оÑģ -en cias -Ġrel ación -Ġê± ¸ë -Ġfact ion -Ġg osp -пол ож -n ap -h ak -Ġproceed ings -ĠìĨ Ķ -ìķĦ ëĭĪ -ĠìŀIJ 기 -Ġwer d -Ġso f -Ġsch lim -Ġfl avored -Ġquad ratic -ĠBo ot -Ġpublic ity -ĠCar o -Ġ ?" -ни ÑĨа -man ia -ĠS UR -ĠB UR -l ance -ét ica -Ġzob aczy -Ġtri o -s ama -Ġta ÅŁ -Ġas ymm -ress er -Ġت ع -Ġп еÑģ -Ġbeginning s -lad ım -ĠбÑĭ ÑģÑĤÑĢ -Ġmo o -ĠGene va -Ġ åľ¨ -er us -bor ah -Ġref using -b ull -ĠWait ing -ĠInd ividual -Ġan onym -im ens -Ġmed idas -Ġfragr ant -Ġdirect ement -ĠìķĦ ë§Ī -ur ia -Ġsp herical -Ġab ge -ĠVictor ian -Ġspect acle -ĠRodrig uez -Ġoc up -ĠN är -mark s -ng ulo -ĠLu ci -Ġshout ed -Ġregul ators -ÄŁ ini -Ġdis ent -ĠÑĢÑĭ н -ëĤ ¨ -ĠìĤ ´ë -Ġprobl èmes -ĠF inger -asse mble -Ġpe ar -Ġdro ite -ĠEvery where -t am -оÑĤ ив -в ой -ordin ate -ĠL ak -Ġm Ỽi -ĠTele vision -Ġexpon entially -av as -Ġble v -ĠM T -ä¿ º -Con nell -ĠêµŃ 민 -ĠÑģво им -Ġach a -ĠD ynasty -J in -Ġto re -Ġfl or -Ġмног ие -æ²Ĵ äºĭ -ow an -b ah -Ġì£ Ħ -ĠC ela -Ġìµľ ê·¼ -Ġpermett re -Ġab ras -Ġverste hen -Ġesc ort -ĠThe m -är ke -por ter -Ġkah kaha -Ġhe ct -Ġda u -w ah -ol ve -ĠAg es -s chaft -ĠSt ell -ne lle -ĠEn suite -ĠÐĴÑģ ем -Ġcr éd -ĠP P -l ords -gr unting -Ġcontract ion -G ot -Ġacqu iring -Ġso pr -Ġpoison ous -R NA -Ġan ar -ĠH of -' ) -Ġremark ably -Ġintern acional -ü cke -in qu -Ġdu y -Ġbeast s -ĠL AN -Ġpreced ent -ĠRP M -åij ¨ -Ġsel on -Ġmort e -Ġcomeç ou -Ñı ла -Ġinterpre ting -ĠBur ke -ÑĤ ÑĢа -ĠìĿ´ë Ł¬ -Ġpess im -ĠN ok -íĮ Ŀ -F emale -Ġìĭ ¤í -Ļ Ģ -Ġstim ulation -Ġsl ick -Ġê°Ģ ëĬĶ -Ġк аз -ĠH BO -Ġpap ier -Ġkön nten -Ñĥб ли -ĠConst ant -SPEAK ING -Ġktó rÄħ -Ġcos metics -ĠT rend -Ġrob bery -Ġt itt -Ġgj ort -Ġdiet ary -ł Į -ĠKir by -ĠпÑĢимеÑĢ но -Ġqual ification -Ġìķ ī -Ġcabin ets -Ġhtt p -ĠEric a -ç¾ © -Ġdisadvant ages -Ġch attering -y z -fe it -Ġgu ild -ĠE TF -ĠDrag ons -ĠH ERE -vent h -ÙĦ اÙħ -Ġmarch é -D am -Ġphot on -Ġest able -M ag -Ġol har -Ġcou pling -ĠHil fe -ĠW izard -Ġм ало -hel p -ĠlÃŃ nea -Ġì « -Ġstand alone -Ġmor ale -Ġzwe ite -ãĤĪãĤį ãģĹãģı -ähr t -Ġd otted -Ġdri pping -ĠFl ag -éĿ Ĵ -ro cket -rate gy -ir im -Ġíķĺë ©´ìĦľ -Ġsogen an -ĠUn o -ĠSch utz -Ġest ilo -ĠS ubs -ĠDais y -ÐĿ еÑĤ -' ... -Ġplat inum -Ġb irl -ĠSo vi -Ġviol ate -Ñĥ еÑĤÑģÑı -r ill -Ġtra z -Ġsn ip -Ġcum pl -à¸Ń à¸ģ -Ġc uk -éħ Ĵ -ĠParl ament -Ġhyper t -Ġpul p -Ġtong ues -at to -Ġbus ca -ih n -ER O -ĠÙĬ ع -Ġvari as -ĠMar ian -Ġbound ed -Ġpitch ing -Ġdefic iency -ĠBless ed -ĠEx erc -uch s -ĠnhÆ° ng -æľ¬ å½ĵ -Ġrap ed -h ales -Ġmal a -p ic -Ġ40 1 -ÅĽ niej -ar ina -ëĵ¤ ìĿĦ -ott i -Ġдол го -Ġtrack er -ĠShel by -Ġvan ished -Ġbak ery -Kap ı -J esus -ĠK R -J O -ħ ¸ -Ġdisc s -ìĦ ¯ -ì§Ģ ë -×Ļ× ¦ -em ary -K endra -Ġy ük -ück t -Ġv az -Ġk up -akt u -ĠÑģп аÑģибо -Ġa ik -Ġnurs ery -Ġendanger ed -êm ement -emat ics -Ġrespond ers -ĠRepresent atives -Ġsculpt ures -ig keiten -Ġde pl -Ġinterpret ations -Ġdead lines -Ġ194 2 -Ã Ĺ -Ġsug ars -em u -l ively -Ġrecre ational -Ġdist ort -Ġunders core -Ġun quote -Ġsaf est -Ġsw ollen -Ġanalys es -Ġcommen cé -å¦ ¹ -and in -ĠÐ¥ оÑĢоÑĪо -Ġdi arr -ãģ¾ ãģģ -zi est -Ġtooth brush -éł» éģĵ -u ations -Ġc ade -Ġbackl ash -h ind -Ġris que -z ess -ĠìĿ´ìķ¼ 기 -Ġesper ar -Ġtransl ations -ion ed -gro ans -Ġп ÑĥÑĤ -Ġgen etically -éĢ ł -Ġhapp iest -Ġwer k -ato on -Ġmus i -Ġfun ção -Ġìŀħ ëĭĪëĭ¤ -ĠÑĢ ай -Ġbe vor -BL ANK -Ġrepent ance -P ut -Ġpotrze b -Ġsal a -Ġcamp a -W ER -Ġdec ÃŃa -Ġsécur ité -ĠAppreci ate -Ñĩ и -ĠR andom -ë³ Ħ -k ah -Ġmö j -Ġsä ger -Ġ×Ļ ׼×ķ׾ -Ġ19 0 -xt ures -E u -Ġg ä -Ġ×ij× ª -ĠC roat -ap o -P LE -Ġpersist ence -åĬ © -Ġbl ends -Ġtre ffen -ĠSanti ago -yd ia -al do -ĠTensor Flow -ĠD ual -ãĥ ľ -Ġch iff -ìĹ ´ -Ġcontract ed -Ġseg reg -ĠFair y -Ġwis ely -Ġvulner abilities -Ġhand held -Ġgad gets -Ġbo ÅŁ -ĠPop ular -Ġcurv ature -ë ¬¸ -ĠMAR Y -ìĿ´ì Ĭ -Ġform ulation -Ġcel ery -Ġblur ry -ĠT S -ale z -Ġw s -Ġprogram m -ĠSt ack -ĠJ IM -ов али -ı ll -Ġp ère -ĠKan ye -ĠDel aware -Ġãģ ł -Ġda unting -Ġб еÑģ -ĠSt upid -b ig -ffic ial -Ġprecip itation -Ġpl ung -ụ c -bur se -Ġdar le -Ġcri pp -Ġpione er -Ġdis put -Ġse an -ãģĵ ãĤĵãģª -Ġresist or -Ġalle in -ipp les -are l -Ġend ors -z ust -ĠÑĢеб ÑıÑĤа -ed ed -Ġì¹´ë ©Ķë -Ġlle va -Ġken nt -Ġб ал -ĠDoc ument -ĠKn ights -Ġbuck le -Ġìī ¬ -Ġal k -ĠEvery day -atter s -Ġtoil ets -Ġj ugar -ĠìŀĪ ì§Ģ -Ġgen auso -ĠLandes regierung -ãģ£ãģ ± -ij e -Ġtrail ers -ĠT igers -Ġg itti -Ġforg iving -Ġconcur rent -ĠV u -ĠíĬ¹ íŀĪ -ĠBR OWN -ound ed -" ; -Ġtre mb -Ġt iet -ĠÑĢеж им -Ġnuts hell -ел иÑĩ -Ġlos ers -ric ting -Ġrede em -def ined -N ice -Ġbroad band -K O -Ġte asing -Ġpart isan -ı ma -Ġìŀ¬ë ¯¸ -ĠJour ney -Ġslop es -un ing -gr unts -Ġt äll -Ġuncover ed -Ġmy ÅĽlÄĻ -ĠEst her -äº İ -ĠHealth y -Ġë° ij -r ée -Ġpolar ization -Ġfl av -Ġcambi ar -Ġy r -ĠR anch -Ġspl its -Ġtrou vé -åľĭ 家 -Ġrecord er -Ġdé part -ÙĪ ب -ĠK ry -Ġinteress ant -Ġeder im -ÅĽ wiad -il ateral -w right -Ġpour ra -ê ter -Ġcam el -á ŀ -Ġrapid ement -Ġme j -Ġstiff ness -AD AS -Ġdiff ers -Ġal ot -ĠS ig -ÑıÑĤ елÑĮ -Ġabstract ion -åľ ĺ -Ġke iner -gr upp -ĠSher lock -íĺ Ķ -Ġc ite -Ġover flow -Ġt ại -ú car -b ula -Ġconjun to -ĠC I -Ġmoder ator -Ġindirect ly -Ġalle ine -â Ĥ -ÑĪ иб -Ġб аб -Ġdan ach -Ġ19 39 -Ġpr omet -Ġdest inations -ĠIll ust -ικ ÏĮ -Ġsab es -Ġhe h -ĠGesetz ent -ĠM iz -ен ко -ĠM ys -Ð ¬ -ĠJuda ism -Ġmust ache -Ġst immt -ĠG aza -Ġvol te -Ġnu o -Ġm ón -ĠCom put -ู à¹Ī -ĠR adi -Ġexception ally -Ġassum es -éĸĭ å¿ĥ -ãģĪ ãģ° -in form -Ġshr ine -æĵ Ĭ -Ġimplic ation -ĠF itz -æ²Ĵ éĹľä¿Ĥ -! . -Ġl t -Ġall oy -Ġeth ic -Ġmonaster y -ìĭľ ì£ł -ica ção -Ġcoordin ating -ĠM oto -Ġover look -Ġcho is -Ġantibiot ic -ĠMin ne -ĠB J -ĠA pa -or ian -Ġsp illed -J am -Ġhus bands -Ġcre ations -Ġa ñ -üs sel -ĠìĿ´ì ļ© -Ġanaly se -r ose -Ġpunch ed -Ġpres que -Ġastron omy -Ġschwier ig -ĠEb ola -Ġc is -Ġac et -ĠF X -end re -ĠìĿĮ ìķħ -Ġweb page -Ġfre aked -Ġlat te -Ġì¿ ł -Ġë¨ ¸ë -N ever -G ra -íĻĶë ¥¼ -ey ed -Ġë°ľë Ŀ¼ -Ġesper a -Ġapare ce -ra ção -Ġdisrupt ive -ĠJo int -ur ous -re as -Ġquer ÃŃa -Ġdistrib utions -Ġexpon ent -ì¹ ĺ를 -Ġd l -z hou -ĠHe aring -å·® ä¸įå¤ļ -ĠC raw -Ġflo ats -oun ced -L ab -W orld -Ġbur dens -Ġauthor itarian -ĠB olt -Ġод нÑĥ -Ġpige on -Ġdistract ions -ĠHeraus forder -Ġz est -es c -Ġsh akes -at as -ĠÙħ Ø´ -hol es -Ġthink ers -al ta -Ġar che -ĠS uk -an ha -Ġtempt ing -Ġyou tuber -Ġv ì -Ġdz iaÅĤa -ĠVatic an -P ark -Ġsup ers -ĠNik ki -ëĬ IJë -or ang -ram ient -é ¬¼ -Ġê°ĸ ê³ł -Ġdessert s -Ġav ere -ĠGreg ory -Ġëĵ¤ìĸ´ì ĺ -Ġcost ing -ĠClin ic -Ġreb els -ĠM ob -Ġbun lar -ĠYour s -ert ime -Ġret ali -m ara -at us -all es -Ġд ÑĢ -Ġд иÑģ -Ġdiscount s -ĠGU Y -Ġкак ое -ĠExper iment -re ment -ĠXi ang -Ġb ate -W E -Ġspecial ize -Ġde ity -ĠL oki -m ag -ĠN it -W est -Ġmater nal -Ġqu is -åŁº æľ¬ -bro ken -Ġlas ers -Ġha kk -ĠAng els -Ġmaster y -ant is -T iffany -ee e -ç ij -ore m -Ġin acc -Ġjurisd ictions -ĠKard ash -æľ º -I l -ĠS inn -åĭķ çĶ» -Ġathlet ics -c ÄĻ -Ġlo osely -Ġdiet a -A g -Ġ? ? -ĠëĮĢ íijľ -Ġsuper v -Ġnut rit -Ġdr ifting -ĠìĦłìĥĿ ëĭĺ -Ġпон Ñıл -ĠVict ory -ÙĦ Ø© -×ķ׳ ×Ķ -Ġп иÑĪ -Ġsh aved -Ġmes ure -ond en -Ùĥ ر -Ġex ile -ĠDes de -ĠP interest -Ġattach ments -Ġh ombres -Ġfin es -ĠìĦ¸ ìĥģ -Ġsleep s -ĠT aco -ĠI RA -ri os -Ġo ll -et es -Ġun ut -fashion ed -Ġtre ball -ĠNear ly -ĠÑĢе алÑĮно -Ġch il -éĢ ± -ÄŁ a -ĠM EL -ros cop -ĠC G -Ġv enge -Ġdishwas her -al gic -Ġmod ifier -Ġemb assy -t imer -em ics -Ġintric ate -Ġev et -ĠëĮĢë °ķ -Ġis ot -Ġна ÑĥÑĩ -ĠQu iz -res o -δ Ïİ -Ġye lled -Ġfed er -ELL ER -Ġexceed ed -on as -ic ano -Ġжив оÑĤ -ĠMa o -ĠKaz uto -Ġ ãħĭãħĭãħĭãħĭ -Ġfront line -ĠHung arian -Ġüber all -aw at -Ġgri ps -i ções -arn ya -ĠÍ ¡ -Ġse id -Ġan ak -Ġacab ou -íķ ij -Ġnot orious -ĠGod zilla -Ġover coming -ĠP end -Ġol abilir -ül me -Ġer halten -ãĤī ãģĦ -ê· ¹ -ĠM eter -Ġsta an -O l -Ġch ats -ĠBu enos -ÃŃ ve -alu able -Ġstrateg ically -Ġcompr ised -ĠпеÑĢÑģон аж -Ġw ann -ĠC en -н иÑĤе -Ł ģ -ĠÑĤоб ой -i ad -ĠkardeÅŁ im -ĠCongress man -ream ing -h omme -Ġcommun aut -Ġalcohol ic -Ġpick led -Ġac ord -p osition -eg ól -Ġtrou bling -ĠMarch eg -Ġzum indest -Ġseam lessly -Ġol un -ĠTV s -ĠпÑĢакÑĤи ÑĩеÑģки -Ġback end -ãģĵãĤĵ ãģ«ãģ¡ãģ¯ -id able -Ġgad get -Ġfa ço -ĠMarcheg iani -Ġë° ¤ -Ġaccident al -ĠL P -Ġeld est -ĠAd miral -Ġn Äĥm -le ver -Ġpast el -Ġfond o -Con nie -Ġter cer -Ġp act -ĠMont e -Ġme ats -ĠS MS -ĠAustral ians -ç ¼ -Rh ett -Ġexact ement -Ġë¹ ¼ -ĠM OD -ç ¡ -ĠR apt -ĠNo ch -Ġab ort -ĠNav al -ĠFu ji -IN TER -Ġнов Ñĭй -Ġmiej sce -ĠIC U -ĠGrad uate -ĠGl en -ard i -ĠÈ ĺ -Ġsold er -Ġprofess ions -Ġorth og -om n -int rodu -ĠDen ise -ìŀIJë ¥¼ -Ġcorrespond ence -AM A -Ġinf lict -Ġf and -ĠG ü -ĠÑĩ еÑĤ -Ġtr aced -Ġpat ents -Ġamb ush -Ġlot ta -ff er -ĠW agner -Ġimp erson -Ġextr êmement -ÙĤ ت -cond uct -A tt -ĠM ueller -ĠAl icia -Ġcy c -Ġha cker -Ġt ys -Ġha il -Ġз аÑıв -Ġpas so -Ġì¶ Ķê°Ģ -ĠÎ Ī -Ġpack aged -ĠC ynthia -he et -ä¸Ń åĽ½ -ĠNiss an -ĠQuest o -é ¨ -d id -Ġμ ια -ĠEll is -ĠAnal ysis -ce mos -Ġas eg -ĠMy ster -ĠCa o -Ġtu v -ĠIndust ry -주 ê³ł -ot al -Ġpeque ño -br as -Ġcompreh end -ĠSim pson -ÑģÑĤв ие -ocr acy -иÑĩеÑģ ки -ĠM ush -ĠLaur ie -Ġtriang ular -ĠPres ents -ĠK unden -ç´ ¹ -æŃ ¦ -ĠIs s -ĠDe ck -á»ĥ n -ĠDark ness -Ġinflamm atory -eremi ah -Ġwar med -vey ard -ĠMem ory -et ty -Ġtax payers -ภĵ -Ø ¡ -Ġpract ise -ëĭ ¬ë -Ġdr illed -m Ã¼ÅŁ -log o -ĠF ach -¤ë ¡ľ -Ġübrig ens -Ġkon nten -Ġnormal mente -Ġarg ues -iling ual -°ë ¥¼ -eg al -Ġtrava ill -ov y -а ÑĤо -Ġr uth -ĠL ights -Ġconsist ed -×ijר ×Ļ×Ŀ -Ġstere otype -Ġpay er -ĠRe e -ĠAir bnb -Ġdr owned -ĠZ oe -Ġcan opy -Ġbar r -Ġн оÑĩ -Ġpag an -Ġj ars -Ġr ê -er ver -æĪ ¿ -ie ben -Ġes pect -ĠF i -Ġunw illing -Ġtechn ician -ặ t -m ember -ĠCan al -س Ùħ -Ġlie ber -Ġin ference -Ġhon oring -åij µ -ĠCamp aign -Ġline age -ĠSt ress -Ġvict ories -Ġde ja -× £ -ê tes -bl ick -Ġмен ее -oth s -ĠCou ple -J ason -ĠNic olas -ек Ñģ -l ib -Ġher ramient -Ġ×IJ ×ķ×ŀר -Ġвид им -mill imeter -Ġsil houette -Ġdrive way -Ġcher ish -ãħł ãħł -Ġrans om -Ġinter disciplinary -ĠPort al -Ġtra g -th ood -Ġted ious -Ġgloss y -Ġpré par -ĠC ay -ĠT ook -ĠBott om -Ġz ig -å « -åį ± -re presented -à¹Ģล ย -Ġdesar rollo -ìĦ ľë -Ġvis cos -Ġmill igram -ĠG und -Ġfer ment -d rum -Ġdraw ers -La ugh -Ġpel os -Ġpave ment -Ġmem oir -av ait -Ġ20 50 -¤ë ¥¼ -Ġraz ón -Ġflour ish -Ġst ern -ä¸ Ī -ĠCh ung -Ġser pent -ĠGentle men -羣çļĦ å¾Ī -k ook -Ġl ut -import e -p arent -Ġw sz -Ġsc ree -ĠMitar beiter -å· ´ -m ut -Ġìĸĺ 기를 -Ġsem ble -ĠO W -Ġinvestig ator -ĠCher yl -ĠG erald -Ġpr ere -Ġcomp ares -ny t -Ġdiferen ça -? - -Ġqu á -ר ×Ļ -S en -Ġhe ps -Ġgrat uit -Ġcons ort -ĠST OP -ĠProtest ant -Ġelectro de -â Ĺ -Ġsecure ly -иÑĩеÑģ кой -Ġt ää -Ġreg isters -ĠHeaven ly -og ly -iss ä -ĠPhys ics -ĠMer kel -Ġré v -éĻ ¢ -Ġer ased -ĠSac ramento -Ġcoff in -Ġex acer -Ġl anz -Ġpo ets -ul if -Ġì¹ ĺë -ĠN erd -ĠN CT -ĠH our -neh mer -ŀ ĺëıĦ -ĠPrin ci -S w -m ies -ar med -ĠBeat les -Ġpropag ation -Ġexch anged -Ġcum ulative -Ġì§ij ìĹIJ -Ġdefe ating -æĬ ± -b els -Ġw es -ĠOdys sey -ä½ł æĥ³ -av ior -ĠìľĦ ìĹIJ -Ġbr it -Ġhij o -D AY -ĠاÙĦت ÙĬ -ĠС еÑĢг -Ñĥ ка -eds iÄĻ -Ġimp os -Ġell as -Ġfire arms -ĠN R -Ġ×ij× IJ -ĠÐŁ ока -aw i -ĠìĦ± ê³µ -Ġpup ils -ĠT ack -Ġfr ase -ĠSh ip -Ġst ad -ä¸ ľ -ĠGreat er -un un -imm ung -gr own -ĠN XT -ĠAmeric as -f ox -Ġmant en -éłIJ åĤĻ -ĠÑģ ок -Ġr ikt -lect ric -de ep -Ġзна еÑĪÑĮ -Ġben ut -ĠInf rast -ĠEm ir -ĠоÑĤп ÑĢав -ĠKim chi -ĠFinn ish -´ìł ģ -ina ire -Ġo ike -æ¸ħ æ¥ļ -Ġhost age -ĠBut ton -ÙĤ ÙĬ -ek ing -ĠKaz akh -Ġcomfort ing -Ġso g -Ġgreet ed -g uitar -p ayer -Ġrel ational -Ġconstru ir -çī¹ åĪ¥ -op ian -ĠVol ume -iet h -ÑģÑĤв ом -ur rection -li ÅĽmy -Ġhem isphere -ĠBe an -IG N -Ġköt ü -ĠFall out -Ġbr ace -ç¹¼ çºĮ -ÏĢ ά -ĠH AS -Ġg é -Ġcharacter ize -ặ c -ĠMil ky -Ġtum ors -Ġn uit -ĠG az -ĠìŀĪ ëĭ¤ëĬĶ -Ġг аÑĢ -ess ment -ĠA be -Ġë½ ij -ĠEins atz -J IN -j ä -C ry -ĠProm ised -ĠÑģеÑĢ д -ok us -Ġscal able -ĠпоÑģмоÑĤÑĢ еÑĤÑĮ -ück lich -Ġreal ism -Ġmay o -Ġjuven ile -Ġhead lights -Ġgör Ã¼ÅŁ -ĠRe form -Ġhal ves -cz ne -Ġbreak up -że j -Ġr ätt -D ay -ĠìĿ¼ë ³¸ -Ġmu erte -Ġtun es -ĠSm ile -rec ord -Ġrecher che -atisf ied -Ġpo zi -Ġcelebr ations -ise xual -ĠRO B -third s -ĠF ortune -ĠÑĤ ой -Ġbrand ed -lo o -Ġd ud -Ġrandom ized -Ġcomb in -ä¸Ģ äºĽ -ier an -c zenia -į ãĥ« -Ġcur ator -Ġar tery -ĠÑĥ ÑĪ -ĠÑĩ иÑĤ -Ġsubsid ies -Ġbloss om -ĠTw ilight -Ġhy vä -ĠPom pe -ĠC isco -ĠÐŁÑĢ о -Ġbir i -Ġg ern -Ġre built -Ġw cze -Ġbenefic i -Ġdrum mer -Ġsol ids -Ġdi yorsun -ãģĤãĤĬãģĮãģ¨ãģĨãģĶãģĸ ãģĦãģ¾ãģĹãģŁ -l ated -Ġmud dy -Ġh olog -Ġcl aps -ĠR ings -ĠO key -ĠBra ve -Ġvalu ation -Ġmig rant -Ġinter mitt -Ġeig ene -ili ary -ãĥ¼ ãĥĪ -mark t -k r -ĠR ib -á»Ļ i -Ġaccus ations -Ġa rab -w ash -ĠBard zo -Ġu gh -est ers -oph ren -Ġaliment os -ĠU z -Ö Ĥ -Ġ6 50 -ĠпÑĢи еÑħ -F I -Ġsamp ai -Ġparl é -hes ion -Ġs ır -Ġapparat us -Ġcor related -ĠPrincip al -Ġcor r -ĠOffic ial -иÑĩеÑģ кие -Ġtermin als -Sh ould -Ġvac un -Ġst ellt -Ġmo oi -etz ung -Ġк ÑĢа -Ġda i -Ġп ож -Te am -ĠP PE -ĠÐŀ Ñģ -ĠLe ah -ĠI vy -y st -Ġuh hh -Ġnight time -Ġtrend y -Ġsec urities -Ġcontin ents -Ġfirst hand -ĠVer on -ĠëĤ ® -Ġbrows ing -ĠC ada -t ro -Ġtr amp -re ib -Ġerst mal -irl er -Ġps ic -Ġget ir -ĠN P -Ġdzie ci -об ÑĢаз -Ġmagic ian -Ġscrut iny -Ġsl ab -ĠO T -ist y -ir ies -ore st -Ġtask ed -Ġmor ally -ìķ¼ ì§Ģ -ust ered -Ġfool s -Ġir respons -Ġein f -Ġvi á»ĩc -Ġsc or -Ġpill ows -ĠG egen -Ġtut te -Ġquarter ly -Ġdid nt -ĠG ym -ĠE ther -ĠØ « -лиÑĪ ком -Ġsign aling -ĠN ode -ĠDonc s -Ġy ah -ĠKan al -Ġf ading -et in -Ġinfluen cers -Ġmed als -Ġengine ered -Ġfer mented -ê²ł ì§Ģë§Į -ĠBeet hoven -×ŀ× © -inent al -ĠìķĮë ł¤ -üt fen -al nya -Ġo vere -Ġden kt -ак ÑĤеÑĢ -Ġâ ĺ -Ġneces it -Ġgener ators -gr ass -Ġпод Ñĥм -lie ÃŁen -B ar -ľë ıĻ -ĠдеÑĤ ей -Ġsuck ing -Ġsten cil -Ġprim o -ĠBreat h -st rom -Ġimmens ely -Ġapp reh -ìłķ ìĿ´ -P op -Ġj ong -ĠGi ul -ĠAD HD -Ġhö ren -Ġe lo -iv ent -Ġr us -Ġoutrage ous -Ġmaster ed -Ġì» ¤ -ÙĪ Ùģ -ip es -ĠRud y -Jac ob -Ġbull ish -Ġt apped -Ġfa ud -iz ophren -ĠÑģо Ñħ -ĠDar ling -Ġ196 3 -ĠPre vention -² Ķ -Ġabdom inal -st ones -Ġav aient -á»ķ i -m ake -Ġs are -ĠInst ant -к ам -Ġkeep er -Ġblank ets -ãģ§ ãģĹãĤĩãģĨ -Ġswe ats -ĠMinne apolis -åħ¨ éĥ¨ -Ġgen ommen -Ġfast en -ĠBrus sels -åij ¼ -Ġcaf eter -Ġabsor bing -Ġha go -ĠEl mo -Ġgust o -ĠY ap -M úsica -Ġt ert -Ġband a -Ġm ily -Ġthere after -ĠStock holm -ĠC arson -Ġcalib ration -ava ÅŁ -ans a -ik ke -Ġfore see -Ġqual che -Ġdest e -æ ¤ -ün üz -Ġfor ge -D is -est en -Ġδ ια -Ġenca ps -ĠGes pr -Ġcher cher -ick ets -ÑĤоÑĢ Ñĭ -C r -ĠТак же -Ġrabb its -ĠD ot -he iten -Ġcaus al -ĠF oster -ajÄħ c -Ġbere it -Ġayud ar -é« Ļ -ãģ ³ -s ong -com b -Ġfr inge -Ġcyber security -Ġëľ ¨ -Ġk ier -Ġbesch äft -Ġкон ÑĨе -Ġfacil it -ĠNam en -Ġbil ateral -t x -ĠW issenschaft -Ġnu ances -Ġr ipping -Ġf y -ĠSicher heit -ĠGh ana -ol on -Ġto pped -ĠMoroc co -Ġrad ial -ĠL EE -ĠAndre as -ed d -ĠìĹ ´ë -ĠAirl ines -ãģĵ ãĤį -Ġval ores -ê· ľ -H y -Ġзад аÑĩ -ĠKend all -ĠÑħ аÑĢ -ĠV amp -Ġpy thon -Ġmanage able -ĠG ente -o ise -ici ary -Ġimp oss -ĠBun ny -iest a -And rew -Ġser t -ĠC ec -zz arella -Ġautom obile -ĠT iere -all ows -åĨ Ĩ -Ġë° Ģ -ĠSc orp -ĠJ elly -ag ara -ĠSt retch -Ġrede f -Ġexacer b -ĠS HA -é f -ors a -Ġflaw ed -ĠNo el -?! ? -Ġpro cent -Ġmen stru -ĠпÑĢо Ñĩ -Ġinf ants -ðŁİ µ -pa use -ĠR acing -Ġ194 8 -Ġsuper intendent -id ores -id y -bra him -Ġunl ucky -Ġper k -an ci -Ġë§Įë Ĥĺ -ĠÐľÐ¾Ñģ кв -Ġfin ans -Ġdiferen cia -łĪ ìĿ´ -éħ į -OR Y -ĠT ac -ÛĮ ا -Ġdes em -Ġваж но -ĠJ U -ĠìŀĪ ìŀĸìķĦìļĶ -ĠÎ Ŀ -Ġinform ations -ĠH EL -h st -Ġпог овоÑĢ -Ġvo iture -Ġre us -änd ig -ĠпоÑħ ож -j ing -Ġd ru -alt ra -Ġprodu its -Ġk ite -Ġeye ball -ĠB elt -ĠRestaur ant -Ġg amb -Ġpor ridge -it ters -Ġconver ts -Ġyard ım -Ġmáxim o -w irtschaft -Ġíķĺë Ĥĺë -Ġì¤ Ģ -Ġice berg -Ġvor bei -Ġ25 6 -ocr atic -Ġreck less -on ner -Ġm ús -Ġlog ically -ĠPr ison -ĠNet z -Ġvac ant -Ġn immt -ĠH ARR -Ġз ов -ĠDe e -ring e -ni est -ĠR ules -ìĬ¤ë Ł½ -cuss ions -Ġfl oral -Ġconstra ined -Ġdifferent iation -ĠQue bec -ĠÛģ ÛĮÚº -Ġpúblic a -it el -Ġaccommod ations -ĠGr ü -í ľ -Ġpick les -иÑĩеÑģ киÑħ -Ġcomm issions -ĠBa ek -Ġçoc uÄŁ -ĠMed ium -Ġperiod ically -Ġwonder fully -Ġstaff ing -ìĽ IJë -ri re -f le -ĠMc L -ĠÑĤ еп -ĠпеÑĢ ек -н олог -Ġíģ¬ ê²Į -çĻ¼ çı¾ -Ġprosper ous -ĠSpirit ual -ĠCh ick -DI A -ĠÐŁÑĢ ивеÑĤ -Ġper ÃŃ -ÑĮ ÑİÑĤ -Ġconsult ants -ĠEar l -ä»Ĭ å¹´ -Ġru ining -оÑĢ е -Ġpens er -Ġtak iej -Ġstrength ened -ĠLiqu id -он еÑĨ -ав аÑĤÑĮ -Ġcam er -Ġdisagre ement -Ġbat hing -ĠY osh -a al -pre chen -RIS ADAS -Ġsuper star -æģ Ń -лÑı ÑĤÑĮ -Ġn ib -ĠTh erm -ĠDAN IEL -Ġp aw -Ġliqu ids -Ġcapac it -ark en -Ġvag ina -Ġm ashed -Ġemer ges -ys cy -Ġun related -ĠGu ild -Ġin verted -it ives -T ra -Ġbe gr -Ġal te -ì§ ķ -ãĤģ ãģ¦ -ĠÑĢазÑĢ абоÑĤ -f inder -Ġдал ее -Ġблаг одаÑĢ -walk er -Ġcr ater -ass adors -ren ces -ins ki -ĠK IM -ĠEll iot -20 17 -ĠS r -ink a -ano v -Ġìŀĺë ª» -Ġpropriet ary -display style -ĠÑģ им -Ġиз б -ĠPan el -Ġinstinct s -ĠCommun ications -éº » -mid t -Ġë§Įëĵ¤ ìĸ´ -ĠÑģл ова -ĠGil bert -缮 åīį -Т ак -voor beeld -е ÑİÑģÑĮ -ary n -que z -Ġd art -Ñĸ ÑĪ -ĠH ut -S al -Ġs outheast -Ġpestic ides -Ġhelicop ters -Ġend ured -i ada -Ġbre wing -ìĹ ¬ë -ĠÑģв обод -ĠS aints -ĠFr ançais -ĠEconom ics -Ġdis loc -oph obia -C amer -Ġnegoti ated -ĠÑģÑĤ али -ìĬ¤í ģ -og ie -Ġtsun ami -Ġpeel ed -Ġmotiv ations -è¨ Ń -ost at -fl an -ĠD AC -Ġk av -' RE -ĠPe arson -b be -c zenie -Ġaten ção -íĨµ ëł¹ -ãģ£ ãģ¡ -ĠÑĥд аÑĢ -Ġintrodu ctory -ĠI ci -ë ĮĢë -ak at -Ġt rench -Ġproceed ed -ĠCo in -Ġdere cho -ĠRed e -æ¯ Ľ -ан нÑĭй -Ġincarcer ated -ĠRich mond -R ock -ĠP av -ĠKar ma -ug es -Ġconte ú -ë ¹Ħ -Ġê·¸ë §Į -ĠG one -Ġwsp óÅĤ -ĠRah men -un ken -Ġì¤ijìļĶ íķľ -Ġi b -Ġatt aching -H ay -Ġsu ka -ìį ¹ -Ġpivot al -ĠRes pect -ÃŃ da -I B -ĠVer antwort -w iet -Ġforens ic -ÑĢи ÑģÑĤ -ĠпÑĢинÑĨип е -Ġmark ings -Ġk ettle -ĠOper a -ĠDo ctors -Ġshred ded -Ġrec uer -Ġvig il -ĠF ail -Ġentre v -Ġд ÑĥÑĪ -Ġout breaks -èµ° åIJ§ -ĠÏĢ ο -Ġro gue -ang led -Ġyear ly -ĠCre ed -Ġw am -Ġlot us -ê³ ¼ë -ãĢģ ãĢģ -ĠSp it -ĠIt u -Ġstra ins -Ġstamp ed -Ġpl aint -Ġpot ion -Ġconsolid ation -è© ķ -оÑĩ кÑĥ -Ġvlog ging -Ġsl ate -ĠAu ft -ĠInc or -ừ ng -§ IJ -en h -Ġhe iÃŁ -Ġdom est -ĠSt rom -åį ³ -ak is -Ġfra gen -Ġfin er -ĠS ug -Ġup hill -Ġé én -âĢ¦ ) -ĠÑģ оп -ĠCore y -Ġsie bie -Ġm use -Ġclo ves -Ġp ous -ĠFin anz -ĠR oute -am at -Ġmut ually -ĠвнÑĥÑĤ ÑĢи -ĠSel ena -ë Ķ -ĠGa ussian -ë ¶ĢíĦ° -Ġ×ij× Ľ -Ġej erc -å¾ ® -ke a -ĠG erry -ĠS ic -大 çļĦ -Ġ196 6 -ies e -Ġfoss ils -Ġest ad -ĠK ane -ci Äĩ -Ġìľł íĬľë -Ġп ам -ĠCru ise -int érieur -Ġbe kannt -ĠP ode -Ġdem ander -R em -Ġinv ade -Ġdecor ating -rop ic -Ġcow boy -ĠPh oto -opol it -Ġì»¬ë Ł¬ë -Ġre ap -Ġhand writing -à¹Ħ ร -Ġë ļ -Ġب عد -ĠM t -Ù Ģ -Ġspaces hip -Ġnational ism -Ġcouncil s -ĠGriff in -ĠAh med -Ġcl ich -ĠO L -w l -ĠPil ot -å® ® -Ġacron ym -Ġg els -Ġelectro ly -è ĵ -Ġм ной -Ġepis od -ĠDies es -ĠAT P -Ġed iyorum -Ġexpress es -Ġexhib its -C omm -Ġк ÑĢÑĥп -Ġmat ar -Ġ20 25 -ĠArt em -vas ive -r Ãł -Ġbe ÅŁ -é» ĥ -Ġliz ard -Ġfill e -Ġì§ Ī문 -Ġмо Ñī -Ġt ür -Ġcul prit -Ġwo ven -ĠAN Y -n im -Ġt ay -Ġprom in -Ġacom pa -Ġid é -Ġbo iler -ĠThe men -Ġaven ue -ĠM ud -Ġнов Ñĭе -Ġwitness ing -Ġl ance -ĠCH AN -ĠBe ver -ت Ùħ -Ġchem otherapy -K ing -ĠbÄĻd ÄĻ -Ġat ual -Ġt ive -Ġtalk in -Ġqued ar -ie ÃŁ -ed el -Ġìĸ´ì łľ -Ġjog ar -Ġö r -Ġundert aking -ĠStre ngth -Ġmil hões -ĠW ine -ĠM olt -è® ² -ãģij ãĤĮ -Ġunderm ine -ĠArch ives -v ana -mer cial -M C -Ġcast e -п ÑĢ -Ġlegisl ators -ul ators -ên io -Ġëį °ë -ĠÑħоÑĤ иÑĤе -Ġн ек -Ġs urn -Ġcons ci -ĠP OW -Ġcul inary -ĠK AT -ĠFol ks -Ñĭв аем -Ġв ок -ãģij ãĤĭ -s ervice -pt s -Ġпоб ед -æĺ¯ åķĬ -Ġt ents -Ġn ord -ST E -Ġrepublic an -Ġwy k -Ġmin ions -èĻ ķ -Ġmem ang -j est -Ġcompar ative -Ġty le -car bon -bed ingt -ks en -Ġneg ativity -Ġsjäl v -Ġd ú -æīĢ æľī -Ġrec alled -c ra -ĠT ada -ĠÑĢÑĥ ки -ĠопÑĢед ел -Ġproc rast -Ġjog os -ĠO o -ĠHe arts -Ġé ch -Ġksi Äħż -Ġco arse -ĠT ube -ĠG reens -Ġé n -Ġdumb bell -ĠÑĤ и -Ġquer er -ا ØŃ -Ïĥ ει -ĠпÑĢав илÑĮно -Ġп ап -Ġcomp ra -Ġt ér -ĠAnt es -Ġoptim um -Ġbisc uit -κ ι -acz ego -Ġìĭľê°Ħ ìĿ´ -ĠMar ines -ver o -Ġvacc inations -Ġpet ty -rit ers -Ġа л -count ry -Ġcoun ters -Ġattend ant -ĠH ui -ãģ¨ãģĦãģĨãģĵãģ¨ ãģ§ -ck a -ÑģÑĤвен нÑĭй -gu y -Ġtrick ed -ĠR ED -Ġthr illing -ÏĢο ι -Ġpig gy -Ġan unci -OR TER -ĠVal ue -Ġr ond -ĠA DA -Ġpos er -h ores -ĠR oland -ĵ ¯ -Ġno ir -Ġש ×IJ× -ë° ľ -iem and -ĠпоÑĤ еÑĢ -ê³ ³ -Ġê± ± -Ġformat ting -ĠL ed -è§Ģ çľ¾ -Ġkill ers -ĠÄij ấy -Ġha ar -ag ain -! > [ -min ster -Ġв ли -Ġident ifier -ĠLamb da -Ġtr os -Ġflaw less -Ġdetriment al -Ġbun ları -W ar -Ġreg ião -羣çļĦ æĺ¯ -ĠB ike -cess ors -Ġc ùng -ĠR N -Ġê½ ĥ -Ġküç ük -ĠBegin ning -íĺ ¸ë -Ġge we -Ġden ote -ĠAlber to -Ġprob iot -Ġo de -Ġmol ar -Ġburst ing -ass umed -Ġfoot prints -ved a -Ġstero ids -Ġfl aming -ĠE ller -Ġerk ennen -ät zen -Ġlife cycle -ĠD OU -ĠK arena -ĠGuer ra -è¿ĺ æĺ¯ -Ġsin ister -Ġpod éis -Ġpar ab -Ġok o -Ġmat éri -Ġcar ic -son aro -Ġpratic amente -ÑĥÑģ а -Ġcomun que -Ġvig ilant -Ġreg imes -ĠShoot ing -Ġra ids -ĠN ora -ĠW ieder -m ens -ĠÑģ од -Ġê²½ìļ° ìĹIJëĬĶ -Ġв Ñħод -Ġaut obi -ĠS chn -ĠRob bie -ĠF itness -Ġкон ÑĦ -Ġpeng uin -моÑĤÑĢ Ñı -Ġми ним -play s -Ġdeleg ates -M er -Ġsist em -ĠMicha els -m ale -ا ع -Ġcá ch -ĠH ä -Ġ×Ļ ×ķ×ĵ×¢ -Ġsuper power -Ġstr on -Ġro ver -Ġdé pend -éĻ ³ -Ġret iring -Ġvamp ires -Ġmer de -ĠCh anging -Ġt ame -Ġspokes person -Ġc ay -Ġfl irting -ĠGr ö -Ġw är -Ġwy b -Ġcoe ur -ạ nh -ĠìĻĢ ìĦľ -Ġconna is -ĠHundred s -ĠBe a -Ġα ÏĢ -pr uch -Ġsocied ade -ĠWh ilst -ĠK ait -esp ace -Ġch ia -ĠEr m -Ġë°Ķ ê¿ -Ġf ences -ĠM ortal -ê² ģ -Ġг ÑĢаÑĦ -ĠHom eland -ĠJ UN -is st -Ġpar lar -Ġsport y -é o -Ġdeep en -ĠBeh avior -éĢ ı -åĵĪåĵĪ åĵĪ -Ġer rand -Ġrot ary -ĠWell ington -W ind -Ġmes ela -ả ng -iend e -Ġex cell -ĠGen ius -ĠEdu ardo -æľī 人 -ĠÅŁ unu -ĠÄ° stanbul -Ġprod uto -Ġ ãħİãħİ -O FF -Ġwoll t -çĪ Ĩ -Ġëī´ì Ĭ¤ -Ġl ass -Ġher tz -Ġar omatic -Ġзв он -Ġaut oc -ĠL ust -Ġ11 2 -ĠÎ Ĺ -Ġreview ers -Ġrecept ive -å°į äºĨ -â nd -og lo -ĠìķĦëĭ Ļ -Ġn go -Ñĸ ÑĤи -Ã¥ t -con o -Ġtek rar -Ġ주 ê³ł -Ġgel miÅŁ -Ġbed time -ĠAr gh -AD A -ĠгоÑĢод а -ĠÄ ĩ -Ġall iances -g iggling -Ġyer de -Ġsp ies -Ġg utes -ç i -Ġallt id -ĠL ah -ŀ IJë -Ġdo kÅĤad -ÙĪ ÙĬ -Ġtoxic ity -Ġcancell ation -Ġ195 8 -d ro -Ġìŀij ìĿĢ -ĠMotor ola -Ġmult in -Ġenthusi asts -ĠM ighty -ĠCoc onut -: ãĢĮ -ĠPict ures -Ġsang re -Ġbl inking -ol esome -ĠìĬ¤íĥĢ ìĿ¼ -F P -Ġboom ing -ĠдеÑģÑı ÑĤ -Ġr atchet -Ġtim elines -len ess -Ġc ages -ĠGood night -omet imes -Ġc unning -ĠR isk -ul ed -d ade -Ġpr ata -Ġgust arÃŃa -am us -ĠJin ping -Ġest rut -Ġdescob rir -ĠM Äģ -ĠAll an -Ġ åĪĨ -Ġ×ľ× § -Ġpres erv -ĠStraw berry -Ä ı -L u -Ġk ro -ĠRep orts -ìħĶ ìķ¼ -Ġval t -Ġpouv ait -Ġapp ar -ĠB one -Ġprefer ably -ĠRep ública -å°± åĪ° -Ġher zlich -Ġchim ney -Ġç ev -Ġvis as -Ġver r -Ġcultiv ation -ĠArmen ia -Ġвд ÑĢÑĥг -Ġcock ro -retch ed -art z -ĠлÑİд Ñıм -ĠpolÃŃt icas -ĠP anz -ĠA KA -ĠëĪ Į룬 -Ġer ro -Ġcam per -Ġ10 2 -ठ¸ -d one -Ġho ard -ĠÐŁÐ¾ÑĤ ом -je ong -Ġdest a -p ak -Ġin im -Ġgrow ers -ĠMess age -Ġele ctor -eng age -ĠFor bes -ĠCincinn ati -Ġdiffé rence -d f -Ġsp ar -Ġawait s -ĠUSS R -ĠR ising -ĠHo ÅŁ -Ġfoot ing -Ġcond iciones -ÑĤоÑĢ ов -Ġclin ician -ĠDisk uss -å£ ĵ -ר ×Ĵ -× ¥ -ite it -g ren -Ġchar isma -Ġle uke -Ġirrit ating -Ġcir ca -ĠRhod es -Ġp ior -Ġhandic ap -roy able -Ġv ull -O G -Ġin ÃŃcio -ier i -Ġspl ashing -Ġdem ise -Ġassist ir -Ñĩ ÑĤо -Ġcover t -ĠG ud -ภī -kl är -ĠìŀIJ 꾸 -Ġver ändert -ĠR EM -ĠCon ven -at ge -Ġpierws ze -Ġcler gy -ling ton -l iv -V PN -ĠÑģ ожал -ĠH ate -ãģ¨ ãģĵãĤį -ÏĨ ο -ĠResp ons -оз д -Ġet mek -Ġchem in -Ùħ Ø© -Ġê°Ģ 족 -T re -Ġum as -ĠBur ton -Ġpatri arch -ĠSmithson ian -¥ ĺ -M oon -A ir -Ġmed ios -Ġer aser -Ġwoll ten -Ġpare il -ĠBill ie -æĬ ½ -еÑĢÑĤ в -Ġparl ament -Ġag ony -ĠQU E -sequ ently -An other -ĠWh ew -ĠAnn ual -Ġse ben -ìĥģ ìĿĦ -val ues -ŀľë §Į -Ġsin on -ere al -ĠEn light -ĠChem istry -ĠCatal unya -Ġdoct r -ant on -Ġst uk -ĠPl ate -ĠKardash ian -Ġfil os -ĠW et -Ġпоп ÑĭÑĤ -Ġunknown s -ĠSch on -ĠBald win -Ġtelescop es -ĠG ucci -ox ide -ĠConserv ative -ìĦ± ìĿĦ -Ġhina us -P ower -Ġê±´ ê°ķ -Ġprev ail -orm an -m achine -Ġ194 6 -Ġun bel -Ġsch aut -Ġp iel -e enth -Ġobject ively -Ġch akra -aud io -Ġch icos -ĠV ault -å° Ī -Ġmedic inal -ĠT ail -Wh ile -Ġas phalt -Ġfro ze -ĠE K -unch ing -n osis -20 15 -ĠG ri -Ġodd ly -ĠM är -ĠA eg -c olo -P ar -Ġëĵ¤ ìĸ´ë -Ġv inden -ĠO VER -Ġ iced -Ġsc orp -Ġha c -qual ified -ĠÑĥвид еÑĤÑĮ -erm o -H EN -Ġso i -Ġmulti ples -Ġlay outs -Ġblind ness -ĠB owser -Ġпод ÑĤ -ĠÃ İ -vention al -Ġm ata -mad ı -Ġge ez -Ġcad ence -Ġważ ne -ĠChrist ie -ven ge -C all -Ġturn around -Ġblo b -ĠЯ к -ĠVoice over -Ġper il -ĠJa ime -ĠH OY -l ane -Ġse bel -ĠDu o -ĠHistor ical -Ġd ni -Ġg ema -y k -Ġsab em -ắ ng -Ġv ars -ĠRon nie -ĠRon aldo -ĠPer què -ns inn -h air -Ġrelent less -Ġl yn -Ġtravel er -æĢİ麼 äºĨ -n ine -Ġant im -Ġì¼ Ģ -Ġsnow ball -ĠÑħаÑĢ акÑĤеÑĢ -Ġintern s -Ġconstitu ency -ĠÐĿ ам -׾ ׾ -V EL -Ġvikt igt -Ġap oyo -ÙĦ ب -Ġj ard -Ġheight ened -ÑĢо ÑģÑĤ -ĠSM ITH -Ġдел а -Ġrepair ing -Ġr igt -ĠShe ikh -ĠBrit ney -Ġevery time -Ġadvent urous -oc key -er nt -Ġat aque -ĠAltern atively -e ffect -Ġpalav ras -ĠElli ott -Ġréuss i -Ġhypert ension -ĠMan ual -Ġproph etic -Ġhand c -ÑĮ е -Ġref rain -ĠSqu id -ìŀ ¡ -Ġком ан -äll en -Ġlleg ó -Ġbas h -ion y -ĠÑģк лад -Ġк аб -Ġcare less -ĠP ool -Ġtr ás -Ġfil s -ĠSch r -Ġsp rawd -ĠMon aten -Ġunfor gettable -ĠCott on -Ġinconven ient -ĠR X -or is -Ġhum bled -ת ×Ĺ -ĠØ¢ Ù¾ -Ġincre ÃŃ -ĠKomment are -èĪ Ĵ -r ación -Ġv antage -ĠSe al -ĠìĿ´ 거를 -Ġjou e -ãģĿãģĨ ãģ§ãģĻãģŃ -Ġìĺ¤ë ŀĺ -ĠиÑģп ÑĭÑĤ -ob en -Ġgr ate -Ġcontro le -ĠPer cy -ÅĤ ada -Ġsimult aneous -Ġprot oty -ĠgroÃŁ er -Ġbew usst -iniz i -Ġpass ieren -ĠHapp iness -åī ĩ -sh i -ge ht -Ġstation ed -ĠErgeb nis -Ġdirect amente -Ġsurv ives -Ġperson es -BER G -Ġvom iting -Ġconhe cer -Ġad jour -ĠCiv ic -pe i -bur st -Ġëĭ¤ ëĭĪ -é ı -Ġsl ed -Ġplataform a -ĠS ect -ĠDe fin -çĻ» éĮ² -én om -chn et -Ġprofit ability -Ġerre icht -á»ı i -c ation -Ġì§Ģ ê¸ -Ġperd re -Ġfel ony -Ġ195 7 -æĪij å¾Ī -Ġunsuccess ful -Ġnag yon -Ġelastic ity -Ġfac ade -Ġearth ly -ĠамеÑĢик ан -Ġcon n -c la -D u -Ġpolit iques -Ġhal o -iant es -Ġмо ей -ãĥ³ ãĥī -ton es -el ier -è® ļ -ht aking -Ġwicht ige -Ġan no -ĠL ok -ill ions -Ġv iver -Ġsol chen -Ġsu f -ĠSal z -ĠN vidia -z uge -ĠSp ike -V ideo -Ġtw or -ĠA la -èij ī -Ġh anya -ĠAd m -ìĿ µ -ĠPatient en -ĠOn ion -ĠKo be -ĠSc ene -ĠR ash -æ¨ Ļ -ÑĢа ÑģÑĤ -ist ani -Gen eral -le ye -imb ap -Ġconce aled -ĠFr idays -ĠW ool -Ġнов ÑĭÑħ -Ø´ ر -Ġê²° ê³¼ -Ġjed och -´ìĭ ľ -ĵ¤ ëıĦ -Ġìŀ¥ ëĤľ -uk t -L ou -Ġ먹 ìĸ´ -ĠEx pect -Ġдом ой -Ġirrespons ible -Ġac erca -ĠZ ust -ר ×ĺ -U I -Ġyout ubers -ĠPos itive -Ġsoci oe -Ġsn atch -èĥ Į -Ġrefresh ed -Ġnom inations -ĠP att -Ġobsol ete -Ġdem iÅŁ -åı ¤ -orm uÅŁ -ĠìĨĶì§ģ íŀĪ -Ġf la -Ġcra ziest -ĠZ ie -ĠT ú -z ep -ic em -Ġë©ĭ ìŀĪ -Ġcyn ical -ãģĿ ãĤĵãģª -Ġt resp -Ġcra z -Õ¥ Õ -Ġne lle -Ġm ph -ĠN ered -ĠK ob -ĠE ck -¨¸ ëĭĪ -J an -ĠТ огда -Ġde ci -ĠV og -Ġbubb ling -éĢ Ģ -ú a -Ġproduct os -iber al -Ġrepl icated -ĠImp rove -ill ary -C ha -Ġré du -ĥIJ íķĺë©´ -Ġcon not -ĠK rit -ĠдÑĥÑħ ов -Ġtread mill -ĠP W -Ġзов ÑĥÑĤ -Ġcl ams -Ġdra fting -Ġ195 6 -un ta -Ġexpend itures -ĠHoo ver -W OO -ÑĪе е -Ġded uction -mon ary -Ġreci b -Ġpo vo -Ġëį Ķë -ĠP AL -ĠBl ow -Ġwy p -Ġdest ac -de al -Gra eme -Ġnécess aire -Ġdamn ed -Ġ19 38 -Ġìĭ¤ ìłľë¡ľ -Ġtro op -Ġinsight ful -ĠT J -ĠоÑģ в -Ġf idelity -ĠSk ip -ĠMay o -ë§ Ŀ -app e -Ġbl as -ĠW Y -ĠG N -ct ar -S u -Ġcu ent -he ws -Ġcorps es -A bs -Ġwaste water -Ġc iek -ĠOn u -Ġexplos ives -Ġar ma -ĠSTEP HAN -polit ik -ĠOs aka -ta ÅĤ -Ġyap ıyor -Ġiz quier -Ġbele za -ĠWy att -åIJ ¸ -Ġsu k -Ġspec jal -Ġdan ke -wh istle -ĠfÃŃs ica -ĠHar riet -ĠìķĦ íĮĮ -Ġwill kommen -ip ing -ĠÑģмоÑĤÑĢ иÑĤе -Ġмож еÑĪÑĮ -Ġinacc urate -Ġarrog ance -ĠRem o -γ ά -ass ed -Ġdeliver ies -Ġst inky -ĠпеÑĢ еж -j ay -Ġtrans itional -Ġr ere -ĠNGO s -ĠAT M -Ø® ت -i ology -Ġв лад -Ġsch me -ĠSh ine -ìķ ¡ -p ants -Ġser ge -Ġsen hor -Ġab duct -ĠBry ant -V ES -Ġawak ened -ĠL az -rop olis -ĠLa o -è¾Ľ èĭ¦ -Ġvill a -Ġsumm ers -Ġent hal -Ġ194 9 -V ia -Ġìĸ´ì ¨ -Ġtend on -Ġviol et -Ġintellect ually -Ġboun ced -ara us -Ġ19 19 -Ġvra ag -Ġsp el -ĠSch war -Sc ott -ĠInd o -Ġë§ Ŀ -Ġcanon ical -ĠI KE -Ġthat ÃŃs -Ġme llan -æ¯ Ĵ -ig mat -C ould -... ?) -Ġfo arte -ĠKum ar -rend o -Ġél é -à ´ -val uation -c ases -Ġintuit ively -h ong -ett ed -Ġsou ven -Ġmor b -Ġc ors -ĠN V -ĠHas an -æĥħ åĨµ -ie ved -Ġì§Ģê¸Ī ìĿĢ -Ġdum pling -Ġcontr ôle -Ġambigu ity -æ©Ł æľĥ -Ġco g -ĠScript ures -Ġc ai -Ġbe ver -大家 éĥ½ -Ġhu is -Ġa ime -Ġerkl ären -ĠL M -ĠF ey -éļ ¾ -à®± த -Ġsuper vised -Ġje we -s pl -ĠÑĨенÑĤ ÑĢ -Ġcoll isions -ÙĦ Ùģ -ĠHog warts -ĠDur ham -×ķ× £ -Ġphosph ate -Ġoverse e -Ġinspect ions -Ġbr inc -ĠZ ak -Ġpay off -Ġch aud -ĠHung er -ã os -v ir -Ġf iance -Ġb oug -l ived -c ry -åĽŀ ä¾Ĩ -Ġjoint ly -Ġgirl friends -ĠNe xus -¦¬ ê²łìĬµëĭĪëĭ¤ -ĠK wang -åĵĪ åĽī -å§ ij -ÅĤ ÄĻ -ĠN eden -ie ce -Ġins erting -æŁ ĵ -ĠM ummy -ĠGlo be -Ġle e -Ġg erman -Ġcre ams -ach o -Ġch Æ°a -ĠGal ile -Ġfür s -Ġest iver -c idos -Christ ian -Ġlors qu -Ġcut est -v ale -ĠкÑĢ еп -Ġw ary -Ġslic ing -Ġesper ando -ĠV ander -ĠDe ixa -Ġ195 4 -Ġmów iÄħ -Ñĸ ÑĶ -Ġtool ing -Ġrest or -Ġpos ición -Ġintent ar -ĠAp ache -OU L -ĠÙĪ ب -Ġmat ière -ãĥ¼ ãĤĵ -Ġl inen -Ġestrat ég -ĠMut ta -é¡ ¯ -è¡Į äºĨ -Ġpart ing -Ġminim izing -Ġapp rendre -æľ Ŀ -Ġан глий -ĠDo o -ĠFire fox -c ómo -Ġge opolit -Ġmak an -Ġmog elijk -ĠÏĢε Ïģι -Ġcá» © -Ġinstall er -Ġdib uj -ĠHe ath -lo op -ĠBro ken -HY UN -sh elf -Ġf izer -Ġenh ances -ä¾ĭ ãģĪãģ° -Ġдо ÑģÑĤи -ĠP UB -ĠKolleg in -Ġatt ained -Ä ¾ -Ġmist ress -ĠOft entimes -×ŀ ×Ļ×Ŀ -Ġbe we -ĠS ora -ra uen -ba um -Ġroll ers -Ġm ering -ĠP AC -Ġн Ñĸ -ĠRép ublique -ĠÑĤ ÑĢав -ĠV anguard -uc iones -Ġ무ë ĮĢ -Ġg our -¯ ¤ -ĠÏ ī -Ġsa una -Ġpe ine -ĠVal erie -ĠS ikh -fend imiz -ber o -ĠÑĩ и -Ġdo ÅĽwiad -ĠE uros -Ġcomment aires -Ġtwe aks -ĠF aster -ĠÑĢаÑģ к -Ġprogress ively -ĠE uch -bor o -ĠIng red -C ap -Ġun check -Ġìĺ¤ë ¥¸ -Ġw re -ĠF T -ör ung -Ġmemor ized -ĠD inner -ĠP hew -ou bl -Ġput a -Ġadm its -ез де -op od -Ġpand a -Ġhing es -ci pe -Ġtrans act -Ġpod ia -Ġp ics -Ġcriter ion -ĠOrchest ra -ĠBl og -Ġsolem n -ĠPix ar -Th ree -Ġв низ -ĠVol unte -ĠSav age -ĠPV C -ĠC af -Ġwy kon -Ġgrad ers -Ġcr ouch -Ġcl iche -Ġsoy beans -ĠM UR -ĠGonz alez -ĠM imi -ĠBol sonaro -Ġdi aphrag -Ġbil ang -ëIJĺ ëĬĶ -éĤ£ æĪijåĢij -Ġregul ating -M c -J udge -Ġн ож -Ġjak Äħ -ites se -ĠW ij -Ġl ata -gro aning -POS ING -Ġ×IJ×ķת ×ķ -Ġha ga -Ġground ing -Ġviol ently -Ġt ills -Ġeng ag -ĠHo llow -Ġпоп ÑĥлÑıÑĢ -Ġw prowad -Ġrepl aces -Ġfluores cent -urg ical -igg ly -ĠTrad itional -t te -ĠÙĦ Ùĩ -Ġphosph orus -Ġapr on -ĠWat ers -ĠK ultur -ав ай -Ġol ives -Ġ×Ķ×IJ× ľ -Ġteil weise -Ġsen cill -Ġprend s -Ġnarr ower -Ġj ätte -ĠInformation en -ìĥģ ìĿ´ -Ġstar ve -Ġfr ick -ĠBe weg -ठ² -Ġdolph in -ĠLAUGH TER -ĠINTER VIE -åĶ ī -Ġyan lÄ±ÅŁ -Ġtor pedo -Ġshort ages -ìĿ´ë ĵľ -ıld ı -Ġp aws -Ġo zone -Ġcultiv ated -ĠF ot -Ġnot or -н оз -Ġко ÑĪ -Ġtouch screen -ĠAll y -æľĢ è¿ij -Ġ맼ìŀĪ ìĸ´ìļĶ -ĠС еÑĢ -Ġв полне -Ġpap rika -ĠDust in -Ġefect o -Ġop ini -Ġmu ut -Ġhá»į c -Ġinter ject -ÄĻ t -Ġbut ts -ure z -ĠP ike -ĠH ok -ĠGu inea -ĠCath edral -Ġ14 00 -C ra -+ , -ë§ Ľ -³´ë ıĦë¡Ŀ -aby rin -Ġvide og -Ġо ÑĢÑĥж -Ġu ž -Ġbus cando -ĠAss istance -éĻ ½ -Ġmel hores -ì¡ ´ -Ġëģ ¼ -ĠR J -Ġت Ùħ -Ġo min -Ġmotor cycles -ĠS app -Ġsupply ing -ĠAl gun -Ġaer ospace -×¢ ׾ -oc cup -le ist -Ġê±° ëĬĶ -Ġcomplet a -b res -! ( -ĠÐŁÑĢ ед -Ġdisadvant aged -ĠAtt end -ĠJud ah -á»ĭ ch -yl ene -act ly -Ġset ups -Ġammon ia -ĠSchwe iz -ĠSh ame -Ġband e -ĠF uel -Ġtroubles ome -Ġnum ero -ĠM OM -ĠпÑĢед лаг -ment ioned -ĠболÑĮÑĪ ое -ĠVikt or -ĠSty les -Ġcruc ified -ructure d -en viron -Ġmor als -Ġmed itating -Ġax ial -is ance -ĠAb st -G reen -Ġê± ´ì -Ġquad rant -Ġper gi -Ġcamer aman -ĠSe qu -Ġpa used -ĠLa ughing -ê· Ģ -? .. -ĠÅ» e -Ġpermit ir -Ġdetect ors -ĠH UD -av al -ĠìĹ¬ê¸° ê¹Įì§Ģ -Ġh ubs -Ġbest immt -ĠбÑĥдеÑĤ е -INTER POSING -Ġten gan -Ġcra ve -ĠBundes regierung -ĠBlo ody -Ġus ability -ĠE as -ĠÄijá»Ļ ng -Ġ195 5 -Ġkrie gen -Ġhabit ual -Ġessential s -rim inal -Ġroomm ates -éĤ£ å°± -ĠпеÑĢе Ñħод -Ġng hi -Ġmen ing -ĠSym phony -ĠH ug -ag gi -Ġw ied -Ġmit ad -ãģ£ãģ¦ ãģĦãģĨ -te enth -ida Äĩ -S ave -Ġrob iÄĩ -Ġboun ces -° ĸìĹIJ -st ars -Ġprag matic -Ġcogn ition -Ġwra pper -Ġw arten -ad h -Ġpens a -ĠHert z -Ġn ÄĽ -ĠRe id -ĠPC s -ĠMo le -Ġ.. ... -Ġpre cio -ĠChampions hips -ê°Ģë Ŀ½ -Ġv ér -Ġcorrid ors -ĠElect ronic -S l -Ġа ле -Ġoverth row -Ġk abul -ĠR ES -ĠCyber punk -ог од -ĠÐĿ ав -Ġw an -Ġmanifest ations -Ġcual es -ĠW ise -ĠLös ung -Ġex fol -Ġearn s -ÑĥÑģÑĤ иÑĤÑĮ -Ġsa pp -ĠBra un -ĠBRAND ON -ì¹ Ļ -Ġs ano -ĠF EL -Ñĭв айÑĤеÑģÑĮ -ожд ениÑı -Ġse wn -F un -Ġrecipro cal -Ġexpans ive -ĠTra ffic -Ġktóre go -ĠÙĪ س -æĺ ¥ -Ġë¹ ¨ -pro ve -ig are -Ġlo h -Ø§Ø ¶ -H ope -Ġdevote es -ĠG om -Ġste als -ĠU ms -ĠTw ice -ãĤ ² -iy im -Ġrhythm ic -ĠV orte -Ġpref ix -om ination -Ġdat o -Ġcust ard -ĠVO ICE -å· ŀ -Ġmen y -ist ors -Ġíĺ ij -ĠìĤ´ì ķĦ -Ġíĥ Ħ -Ġk ort -Ġab a -ĠV era -ep y -Ġì¹´ë©Ķë Ŀ¼ -Ġsubmer ged -ĠC lock -Ġthumbna ils -Ġbo ast -ĠF are -!! ] -ĠÅĽ m -Ġkaik ki -ĠTechn ologies -ìĻ ¸ -ãĥ Ĵ -иÑĤ ай -å°ı æĻĤ -Ġа ÑĤ -Ġkn obs -Ġre icht -ượ ng -gl io -Ġ맼 ìĿ´ -ê°IJ ìĿĦ -Ġjot ka -ĠHand y -ĠHab en -n ous -Ġin land -Ġam azon -ho oting -S L -Ġle isten -~ " -Ġprov oke -ĠTw ist -Ġ×ij× Ĺ -Ġdepart ed -ê° ľë¥¼ -Ġk onse -ĠCar wyn -íķĺ ìĭł -ident al -ES CO -Ġt teokbokki -Ġdiz endo -ç· ´ -ınd aki -imas u -af ar -Ġland fill -Ġcorrect ing -Ġcle ars -ĠNum mer -H AM -Ġcart ridges -ĠDies el -p aced -Ġobl iv -Ġmoy ens -ĠSin ne -ĠPre is -il iz -ĠÑģм ож -Ġbroad en -ä»ĸ æĺ¯ -x es -Ġcarbohyd rate -íĺ ¹ -se ok -Ġecho es -Ġc ess -ë° Ķ -Ġб изнеÑģ -Ġllam ado -Ġess ent -ĠìĿ¼ë °ĺ -ĠA ires -ph en -Ġze bra -Ġsymbol ism -On ce -Ġr acks -ĠKaf ka -ĠÑģеÑĢÑĮ ез -Ġsin n -p icious -ka a -Ġmotherf ucker -Ġapprentices hip -Ġr pm -Ġtax ation -Ġfur ry -ĠSac red -ĠÑĢаз м -por a -eng es -ĠíĹ Īë -ĠÑģ ин -Ġsanit izer -Ġcr inge -ĠS ca -оÑĩ но -Ġof ere -Ġmel odies -ĠVel vet -ĠIhr er -ĠHy brid -ĠG iov -Ġirgend was -Ġdep ende -ĠUs ers -Ġh ump -dri ving -Ġs f -Ġruth less -à¹ĢภĦ -Ġlem ons -Ġfö ret -ĠO j -Ġм ама -Ġinter personal -Ġge v -Ġab norm -иÑģ л -Ġин д -Ġkont roll -Ġreg res -Ġled ge -Ġerzäh lt -ĠT act -Ġarri vé -Ġsubstant ive -Ġspoon ful -zw ischen -oooo o -Ġconten ido -Ġbes l -á»ĥ m -k ten -Jam ie -Ġsand y -ä¸į åIJĮ -â ĭ -Ġp ase -Ġdet te -ĠBelg ian -ê° ľë -ula res -r ud -ig or -ĠíĮ ¬ë -Ġremed ies -Ġblast ing -ĠS ich -Ġож ид -Ġmon str -Ġmanif old -Ġglaub en -ĠE ST -Ġstream line -Ġlobb ying -ĠGoth ic -to ire -.. ' -Ġdém ocr -Ġнаб лÑİд -Ġwsp ól -ĠczÄĻ ÅĽÄĩ -ä¸ĭ éĿ¢ -is és -g angen -Ġbez pie -rem lin -ê° Ŀ -St ill -Ġres ides -Ġgele cek -Ġtélé phone -Ġpe wn -Ġle opard -Ġcompliment ary -Ġc rib -ĠAnim als -Ġge il -ess el -Ġgard er -Ġcatch y -æ¨ ¹ -ĠE ts -ĠCom mercial -ĠD ENNIS -ĠCoordin ator -ĠAb igail -ffff ff -ấ p -Ġpeque ña -Ġinject ions -ce kt -Ġphilanthrop y -Ġp uck -Ġcelebr ates -ĠD unk -ĠD latego -ãģ¾ ãģł -δ ή -grad uate -ĠM obil -t ill -ac am -Ġyol ks -Ġtang led -Ġman iac -Ġoblig ed -ĠLa ink -Ġver der -ĠDam on -Ġmut ant -Ġhop ping -Ġre ins -Ġinver ter -Ġcont empt -׳ ס -le arning -M iss -ĠÐĵ оÑģ -ĠMe yer -ê»ĺ ìĦľ -é£ İ -×ķ׳ ×Ļ×Ŀ -ask ing -Ġtrim ming -Ġtre asury -Ġs ente -A ust -ĠUnterstüt zung -ĠCom edy -ĠAn akin -é ¹ -ÑĢÑĥ ÑĤ -ĠH ari -ograph ers -Ġoat meal -ĠB ots -ä¸į äºĨ -Ġп алÑĮ -Ġacknowledge ment -x ic -Ġê´Ģ ìĭ¬ -gas ping -Ġãģ ķ -Ġterr ace -Ġor naments -ĠM ER -comm ittee -ĠìĹĨ ìĬµëĭĪëĭ¤ -Ġr ij -é ³ -צ ×Ŀ -le me -Ġlibert ies -Ġfell as -ĠCop per -ben ch -ĠIde a -á»į n -ÑĪ а -Ġvers ión -ÏĦο Ïį -ĠÐľ и -ĠпÑĢил ож -Ġbox er -ĠT anner -ĠM oy -ì¹ĺ ëĬĶ -T hr -Ġtin ham -Ġpol ishing -Ġconsequ ently -Ġamen ities -ĠK I -ĠGRE EN -ĠFrank ie -н иÑĤ -itt el -Ñģ кое -urs ed -Ġup bringing -Ġth ứ -ĠìĭĿ ìľ¼ë¡ľ -Ġwh im -Ġchin ese -conf idence -ĠJ eder -ãģª ãģ®ãģ§ -aj cie -ĠT ous -ĠPow ers -ừ a -other mal -ĠвÑĭ ÑĪе -r ale -Ø§Ø ® -Ġì§Ģ ìĽIJ -Ġép isode -Ġsul ph -Ġenc ara -k raft -alar ı -ĠCom es -Ġdiv ul -ĠRud olph -ĠM use -Ġut ens -ĠìŀIJ 주 -Ġp ana -ĠVeget a -ĠPH P -ĠN SA -ent in -ĠCarne gie -ا ÙĬ -iÄĻ cy -H arry -Ġf ır -С п -Ġglad ly -Ġaver aging -íķĺ ê²łìĬµëĭĪëĭ¤ -лÑı ÑİÑĤÑģÑı -ĠÐľ енÑı -Ġquot ation -ri res -itch ens -ay ed -Ġun att -ĠP erez -ĠоÑĤ меÑĤ -Ġtact ile -ĠEu h -is ini -b uh -Ġhat ır -ĠìŀĪ ìľ¼ -Ġpolicy makers -³´ì Ħ¸ìļĶ -ac ı -Ġκ ι -Ġregister ing -re to -ĠSpr inkle -ĠGram my -ax ter -Ġб и -Ġsit ter -Ġpred ic -Ġthin ly -Ġstr um -Ġag grav -Ġa ha -ر ج -m ellow -Ġconst ante -ĠL aut -ist on -Ġtransition ed -ĠCamb odia -ãģĦ ãģįãģ¾ãģĻ -è·Ł 大家 -art ed -Ġmis f -ĠPunk te -Įë ĵł -Ġtremb ling -Ġges pannt -ĠعÙĦÙĬ Ùĩ -Ġникак иÑħ -Ġë¶Ģë ĵľë -ĠÑĢазв иÑĤ -Ġit chy -Ġc iento -Ġpl ains -Ġk ittens -Ġback log -ĠPres iding -pt a -Ġha voc -ĠDarr in -ĠÐĽÑİ Ð± -Ġsegreg ated -Ġg hetto -Ġerle bt -Ġdrug iej -ĠSi xt -åı ĥ -ร ะ -uen cia -Ġíķĺ 기 -ĠëĨ į -Ġrob i -Ġpione ers -Ġmilli ards -ĠWitch er -Ġ무ìĹ ĩ -or ro -m ass -Ġdiver gence -ĠRiver a -ĠNo odles -Ġend roit -ĠK osten -ĠдÑĢÑĥг а -ĠmÃŃn imo -ĠKazakh stan -ت Ùĩ -Ġвоз дÑĥ -Ġgesch rieben -ĠN il -Ñģ ки -ĠFr üh -Ġbever ages -æº IJ -ĠG on -æĺ ¨ -Ar in -ĠInt ro -ocaly ptic -Ġexhaust ion -ĠStat us -ĠBatter y -és z -£ ¼ë -air y -Ġë³´ìŬë ĵľë -Ġdispar ity -Ù Į -ĠTuc son -Ġbright ly -pro blem -Ġbiom ass -éĻ į -§ ī -Ġhur dle -Ġwavelength s -Ġ< < -Ġteam ed -FF FF -ĠS lim -om ial -Ġunve iled -ĠVere in -ÙĤ Ø· -est ry -Ġcl ás -Ġch eddar -Ġaccus ing -ĠScient ific -ĠбÑĥд е -ĠCyr us -ε ÏĦε -Ĩĵ ê³ł -Ġë³ Ħ -Ġcur d -Ġrefer rals -sh ift -åį ķ -nik ów -Ġm ier -Ġconf ronting -ê²ĥ ëıĦ -aw l -Ġtry in -Ġê·¸ëŀĺ ìļĶ -Ġch iar -Ġìĺ¤ëĬ ĺëıĦ -æĶ¿ æ²» -es que -Ġmism os -ĠSh ak -Ġsoci aux -Ġpi ÅŁ -ĠkiÅŁ i -Ġcy an -h ay -be w -b od -ĠÎ ¹ -ĠMain ly -Ñİ ÑĤÑĮ -hab itude -ĠÑģп окой -è·Ł æĪij -Ġpre con -ĠM andy -ðŁ¤ £ -ill os -Ġgr upp -Ġcr umble -Ġconstru ctor -erv ices -Ġlight house -ĠCon cept -ан ÑĤи -alt ro -h ope -ĠAll eg -ìĸ´ë ¥¼ -pie ces -oun ter -Ġíķĺ ëĭĪê¹Į -ĠìĿ¸ íĦ°ë -Ġvérit able -Ġthread ed -bl ind -Ĥĺë Ŀ¼ -Ġtr ays -ĠEd ison -ĠÃĸ z -ĠSte vie -Ġl ender -Ġbrig ade -Ġdeuts che -m uffled -b art -Ġinsan ity -Ġsav vy -Ġsens ational -Ġdere chos -ĠM X -ĠпÑĢ еп -Ġthreat ens -Ġrealt Ãł -Ġindic ative -Ġch ops -Ġbenef iting -ĠVern on -ĠSt rand -n un -qu ently -10 1 -Ġe el -ìĪ Ļ -r ints -ĠÙħ س -Ġب د -Ġпо ÑģÑĤÑĢо -Ġyap mÄ±ÅŁ -Ġol ması -Ġi edereen -ol é -ke f -Ġë°ľ ìĥĿ -Ġr ained -Ġalm ighty -ĠвÑĭ д -ĠC PR -F re -Ġinhab ited -Ġarb ets -Ġa kin -а ÑģÑĤв -v ania -Ġhäuf ig -ĠMat te -s orry -Jen ny -ĠгÑĢ ад -Ġwh it -Ġbro kers -å¯ Ł -Ġh ine -ast en -Ġг ÑĢÑĥ -M B -ĠP RI -S ab -Ġwrest ler -Ġfacil itating -Ġeh kä -ĠC red -Ġ12 7 -Ġnot hin -Ġmand ated -å¯ Į -ÑĥÑĤ ÑģÑĤв -F rank -Ġwor s -Ġdzie ÅĦ -ĠUnder ground -Ġznaj du -ĠB ä -ĠPrin zip -аÑĤ елей -Ġveter inar -Ġsplend id -Ġroz p -Ġpsych opath -ig on -Ġh ops -Ġc ần -ĠX ian -Ġtro isième -Ġproduct o -ĠdeÄŁ er -ĠContin uing -ив ал -c ık -Ġmoistur izer -Wh ite -Ġsi is -ĠEver est -ien ced -Ġcả m -ĠJ apon -´ìł Ħ -Ġten ÃŃan -Ġenc anta -M m -Ġdrop down -ĠI ya -³´ë ©´ -Ġword ing -ĠSque eze -ĠMap le -Ġclar ified -ĠMun icip -ĠRou ge -ĠNick i -ĠGo o -v olt -t ek -fect ure -f red -ar rive -ãĥ¼ ãģĦ -te z -E p -Ġob ras -ĠV ID -ĠR iv -ĠMod i -i be -Ġacontec endo -Ġim itation -Ġcamoufl age -Ġspan ning -ĠSEC RET -ĠOre o -ìĨĮë ¦¬ -Ġh unch -Ġca ÅĤe -Ġspont aneously -ĠPer d -Ġet ap -ĠHo le -ĠDis ability -Ġafter life -æģ © -Ġtest ified -Ġpres up -Ġpet roleum -Ġcontr ario -ĠAss essment -ÄŁ lu -Ġp ests -Ġdil ig -ĠвÑģÑĤÑĢ еÑĤ -Ġcons équ -Ġcann ons -Ġcan oe -ĠM ile -Ġcit oy -Ġbe gged -ĠMin nie -ÅĤy ch -Ġprinci pe -ÏĢÏĮ ν -m niej -Ġw ert -Ġëĭ¤ë ĵ¤ -an se -Ġunc les -Ġprovoc ative -Ġinter sections -Ġdemocr ats -ĠJul ius -ин ки -yg usal -Ġ׾ ×ķ -Ġgj orde -Ġg asket -ĠB ock -ĠÄ° n -b reat -ĠEqu ity -ard ı -Ġкан але -Ġд ней -Ġt Ỽi -Ġfi xture -Ġab uses -Ġv aya -Ġou vert -Ġmultic ultural -Ġcontext o -ĠSes ame -Ġdé pl -Ġcons omm -ĠPart e -Ġp em -ĠCon an -Ġб ÑĸлÑĮ -Ġpersu aded -Ġdra ins -M oo -F ORE -Ġб аÑĤ -Ġf od -ĠProduct s -ì§Ħ ì§ľ -Ġ" [ -ĠW ick -ĠNar uto -н али -ry w -Ġl odge -Ġin h -Ġvont ade -Ġdi j -ĠJes ús -Look ing -Ġfore arm -ĠIntegr ation -ĠHARR IS -Ġtool bar -le ader -Ġsel dom -Ġб ÑĢоÑģ -ĠK ook -он д -Ġmon opol -Ġmill et -Ġl ira -ĠAs ians -Ġ18 90 -ci ÄŁim -Ġed en -ĠIKE A -ĠNeigh bor -ĠKazu ya -ü d -Ġpsych edel -Ġenvision ed -åĿ Ĺ -Ġï· » -Ġw under -ĠBulgar ia -B rid -Ġmar row -Ġdep iction -ĠT in -ĠPhar ise -Ġeinz ige -Ġblind ly -ãģĽ ãģ¦ -Ġdef ens -D ire -Ġvibr ating -Ġtroll s -Ġdisrespect ful -Ġw od -Ġstimul i -Ġcreep ing -Ġcla irement -Ġsc ariest -Ġdécouv rir -Ġ10 4 -ĠвеÑĢ Ñħ -ĠÅĤ at -Ġróż ne -Ġbar ley -ĠRe pl -ĠT we -k ke -ĠãģĿ ãĤĮ -ĠRed mi -ĠMet roid -Ġή ÏĦαν -Che ck -ĠS EN -Ġ ido -ÑĤоÑĢ ии -ó p -UN KNOWN -Ġänd ern -ĠJu ice -ĠGes icht -å°± æľĥ -ĠнаÑģÑĤ олÑĮко -íĥ ķ -Â Ń -ex hales -Ġì´ ī -Ġj sem -ÏĢ ÏīÏĤ -Ġit t -ëªħ ìĿ´ -Ġrem ix -Ġbloss oms -ĠR enee -is ations -ìĬ¤í Ħ° -Ġë³´ ìĿ´ëĬĶ -uest as -op edia -ĠA im -ìĿ´ì¦ Ī -sc ene -Ġleak age -uck t -S ad -A sk -Ġsusp ense -Ġimp ost -ĠStrateg ic -ĠIt ÃŃs -âĢ Į -Ġkey boards -Ġam using -og r -id erman -ŀ ĸ -Ġв ижÑĥ -Ġd ips -Ġapolog ized -ĠST AR -Ġesc uela -ĠC hing -н ениÑı -Ġë¶Ģë¶Ħ ìĿ´ -ĠFle et -Ġs amb -Ġentsprech end -Ġelectrod es -ĠFrei heit -æĪij ä¸įçŁ¥éģĵ -ĠSh rim -iÃŁ e -Ġselect ions -Ġfor di -Ġd oss -Ñı Ñĩ -Ġdiscrimin ate -ĠAu ÃŁerdem -Ġdesenvol v -ĠIntern al -ĠBened ict -å¯ Ĩ -ĠSh iv -M issy -Ġоб наÑĢÑĥж -Ġна ÑģÑĤÑĢо -Ġcontrol ar -ĠL ia -Ġopio ids -ant u -Ġcup board -æģ IJ -г е -acht s -Ġcur ated -Ġx em -Ġwe ary -Ġbre thren -Ġbudget ing -Ġpour tant -éļ » -ais ia -ĠоÑĤв еÑĩ -ĠG IS -μ αι -Ġש×Ķ ×ķ×IJ -Ġsa ud -Ġl Ỽ -Ðķ Т -ub ine -ĠнÑĥж ен -Ġkidna pping -Ġbr at -ĠTer re -ĠMon et -Ġë§Ī ìĬ¤íģ -Ġflash y -ĠIS BN -Ġfreel ance -i age -Ġjun ge -ì¶ © -cer al -ĠÑĤоÑĩ ки -Ġform ulate -ĠF ER -ĠDart mouth -ìľ¼ë ©´ìĦľ -å¢ ĥ -ow iÄħ -ĠëĶĶ ìŀIJ -Ġreg iment -Ġmetabol ismo -ĠP arr -Ġ충 ë¶Ħ -Ġsan ity -ĠL al -ĠG ö -ĠG la -Ġprot o -Ġmicroscop ic -Ġk ang -ĠSc alia -Ġp ug -ĠSc ore -ĠSav annah -Ġgard e -ĠN OR -å°į åIJ§ -Ġsche int -Ġp óÅĤ -Ġcor ri -Ġbr ute -Ġ ÅĤad -ä»ĸ 们 -Ġsucceed ing -Ġbicy cles -N on -Ġseek ers -Ġuncond itional -Ġrhy mes -ĠGar age -Ġinv oice -Ġcan vi -ne ck -Ġcustom izable -irit ual -Que en -íķĺ ìĭľëĬĶ -Ġpower less -Ġcs ak -ä¸į ä¼ļ -is oft -Ġìłķ íĻķ -Ġnh ân -ĠM AND -ĠH af -Ġrevol ves -ä¹Ł åı¯ä»¥ -ov an -ar oo -ĠGr ind -éĽ ª -Ġindispens able -Ġconsult ed -ĠClin ical -A cc -Ġol hos -Ġmon ter -ĠH ana -et ah -Ġva an -Ġt igers -Ġcau cus -ðŁĺ Ĥ -³´ì ŀIJ -pow ers -ium s -ĠíĨ łë -Ġtrad icional -Ġreson ated -Ġìĭł 기 -th em -Ro bert -Ġelement o -Ġant id -Ġоб Ñģ -Ġnat ives -Ġlo ca -ow ment -ĠT ight -Ġ æĢĿ -Ġmel an -ĠN ue -am is -Ġsor gen -as ına -H ome -ĠPUB G -Ġaw fully -ĠSh ore -ĠPer ché -ĠL au -ĠCind erella -ĠCh est -Ġsem antic -Ġdesert ed -ĠMom o -ĠHern andez -gen es -ĠAd ult -иÑĩеÑģ кого -osh ima -ĠcaracterÃŃst icas -ĠK L -´ìŀ ¥ -oc ar -Ġfeh lt -Ġd ruk -ĠPop py -EN GLISH -ĠVerg leich -B rien -Ġrec omp -ĠÑģ д -Ġmer ger -Ġmarket ers -Ġhoney moon -Ġpen so -Ġbell i -еÑĤ Ñĥ -Ġbank er -Cam era -ĠSt all -ĠSt amp -ĠB ite -еж де -Ġs ür -Ġgü ç -ĠPas sover -ĠBug ün -ĠÑģожал ениÑİ -Ġн из -Ġman ure -Ġglac ier -è« ĩ -RA Y -ter ror -Ġsal ads -Ġhur ricanes -ĠDesign er -ator io -Ġfact ual -ĠTam my -Ġзв ÑĥÑĩ -Ġintrodu ctions -Ġhouse keeping -Ġh anger -ëĭ ĺë -ak te -ĠCol a -' ] -ĠG ender -оÑĢ он -ip se -ic ias -Ġsuccess ive -Ġpolit ic -Ġhö her -ĠQ iao -ĠG imme -Ġл ож -Ġse b -ĠWe iter -ĠSak ura -ĠB oulder -ĠAm érica -peÅĤ nie -Ġtecn ologÃŃa -ish ops -f ur -Ġmoon light -Ġdispers ed -Ġre z -ен ное -алÑĮ нÑĥÑİ -ĠTw elve -ĠH OR -ìĭ¤í ŀĪ -il age -Ġshad ed -Ġres umes -ĠPe anut -ĠM ILL -ap ons -ĠU FC -ĠSo le -Ġjoy stick -ĠOliv ier -war ming -Ġsyll abus -Ġоб Ñīе -Ġhi á»ĩn -Ġfest a -Ġcr adle -ĠZ ac -Ġremem brance -Ġê°Ļ ìķĦìĦľ -ĠpiÄĻ k -Ġco exist -ĠV II -Ġá reas -Ġu waż -Ġobser vers -Ġmännisk or -co on -ĠD AM -Ġnas zym -Ġall igator -ĠFree ze -ĠEst ate -ĠÑĤÑĢ ади -Ġunder cover -Ġn ies -ĠFeh ler -pl in -ĠK abul -il ate -Ġê³ł ìĸij -Ġm op -ìĦ ¼ -Ġand erer -ĠK ELL -ок и -Ġж еÑģÑĤ -Ġgra zing -Ġda ÃŃ -Ġcapital ize -Ġa pex -Ġnurt uring -Ġcort ar -Ġcontr ac -ımız ı -Ġtand em -éĥ½ æľī -ge ment -ĠÑģиÑģÑĤем а -Ġman que -ia jÄħ -W OR -Ġا ب -Ġcart s -AN O -Ġë°Ľ ê³ł -ĠC ena -ĠBi ology -id ar -Ġa ż -er ne -an u -Ġthank ed -Ġsubmar ines -Ġman ic -Ġм оз -ä¼ Ĭ -inst ant -ess ential -Ġsam urai -Ġpast i -Ġal an -Ġbro ch -Ġb aker -ĠGu ill -¨ ¼ -Ġwithd rawn -ëĭ Ŀ -Per fect -qu ency -Ġstream lined -Ġ13 00 -´ë ıĦ -Ġëĸ łë -Ġãģ¯ ãģĦ -Ġh vad -ä¸Ģå®ļ è¦ģ -Ġverb ally -ĠK ons -Ġì¡° ìĭ¬ -Ġdie z -æİ° æİ° -Ġchuck ling -ĠM ih -Ġrall ies -Ġman ter -Ġearn est -s uper -Ġge ce -ĠR end -ĠGer ade -jen igen -ĠV all -Ġìŀ ĪëĤĺ -ĠÑģказ ала -Ġtrabal h -ĠнаÑĪ ем -Ġм еÑħ -ik it -Ġnoun s -Ġneurolog ical -Ġmotiv ational -ĠMcM ahon -ĠFin ished -Ġë³´ ìĿ´ -ĠField s -Ġadoles cents -ĠT isch -ĠNe ben -ĠFl owers -ĠEner g -Ġdire t -ĠTh i -ĠP icas -æĥ ľ -æĢİä¹Ī æł· -Ġav ete -ĠF ors -ĠChap el -N ão -E t -ĠÑģод еÑĢж -ren o -Ġs ven -Ġdost ÄĻp -ne e -ĠSnap dragon -ĠID s -ìķĺ ëĬĶëį° -ר ×ļ -Ġsun flower -Ġperpet ual -ç³ ĸ -Ġkn ights -Ġg ird -ĠTo ld -Ġvolcano es -Ġadvers ary -ĠEconom y -Ġextra pol -Ġbl uetooth -Ġzoom ing -Ġsk ys -Ġgen ial -ÃŃcul os -amb re -Ġм еÑĢ -Ġteen y -Ġstress ing -ìķ Į -ON Y -Ġtransluc ent -Ġround ing -Ġgr ues -×Ļ׳ ×Ķ -ap rès -Ġprue ba -Ġpoly gon -Ġblue berry -ĠProgram m -Ġtren ches -Ġse bagai -Ġpal ate -Ġla ude -Ġbehav ed -Ġlongitud inal -ĠMod ule -Ġadm ir -λ ι -G reg -Ġwy st -Ġpropag ate -Ġmold s -ĠT ub -ĠL oud -ust o -Ġun stoppable -Ġreinfor cing -éĿŀ常 çļĦ -ĠпÑĢоблем а -Ġpot encial -Ġhe mp -ìŀ Ķ -ठ¯ -Ġopt ic -Ġerfolg reich -Ñģ Ñĭ -олÑĮ ÑĪе -ur st -ĠPo is -Ġrespond ents -Ġneh me -ĠEx ternal -ol ate -H yun -Ġquart z -Ġmathematic ian -Ġbás icamente -Ġa il -ìł ľë¥¼ -att utto -Ġno oit -Ġaff lict -ĠOl ga -èŃ · -Ġна ÑĤ -Ġd ites -Ġreal idade -Ġk än -Ġuniqu eness -Ġpad res -Ġsubs idi -Ġpige ons -β α -st ad -Ġder en -ĠС лед -d oo -ĠопиÑģ ании -Ġam ber -Ġgoose bumps -ĠfrÃ¥ gor -ĠV ital -ĠIsrael ites -w asser -Is n -Ġcomm its -ĠSTE VEN -ĠBev ölker -uit ive -Ġleg en -Ġbr uk -иÑĢов ан -yn en -hel m -Ġgener ational -ĠL ändern -οι ÏĢÏĮν -uz u -Ġcall er -он ÑĮ -üm ü -Ġbes ar -Ġpl ats -Ġmig rated -Ġj ap -ĠW AR -Ġdis sect -ĠZus ch -ĠZe iten -ĠL ions -ĠD F -â Ķ -ки в -Ġpedest rians -ĠMar ilyn -d ock -Ġy ht -Ġre incarn -ĠSon o -ĠGrow th -ÑĥÑģ ов -Ġdun geons -Ġbag us -k ich -ĠÑĥ кÑĢаÑĹ -éĨ « -ĠK eller -chem istry -J apanese -Ġwill st -Ġdecomp osition -ĠÑģÑĤ ен -Ġrev ived -íķĻ êµIJ -ĠÅ ĵ -ä½ IJ -ìĭ ¸ -ipp y -Ġhour ly -j än -ĠWork shop -Ŀ¼ ìĦľ -Ġcu arto -Ġpat rim -ĠB urch -ĠìŀĪ 기 -Ġhe pat -Ġh Ãłng -ĠëĮĢ íķ´ -ĠваÑĪ и -Ġre work -Ġpar se -Ġçıkt ı -ĠS ax -ĠMong o -ĠAa ah -ram ble -D J -Ġstabil ized -ĠSpe ech -Book s -Ġhur dles -ĠW O -ĠLamb org -Ġ19 33 -Ġvor bere -Ġclin ically -Ġbreat htaking -ĠGate way -пеÑĢв ÑĭÑħ -ut ers -Ġë¹ µ -Ġyet er -Ġpull ey -Ġmuff in -ĠPre fer -ĠP ence -Ġinform ação -ìĬ¤í Ĭ¸ë -ãĤ¸ ãĥ£ -ĠTur tle -ĠReg ina -ĠLo ad -do es -pan ze -¸ Ķ -Ġmin a -ĠLatin os -amm ers -ĠT ort -ĠBey once -имо ÑģÑĤи -ĠвопÑĢоÑģ Ñĭ -Ġbul un -èĢĮ å·² -ine k -bere ich -Ġpast ure -ĠO A -ĠM elt -ĠEt t -ĠD Y -Ġob wohl -Ġle agues -ÑĤ еÑģÑĮ -Ġк ÑĥÑģ -Ġv ors -Ġto pp -ograph ical -as st -Ġl indo -Ġë°Ŀ íĺĶ -Ġré fl -Ġclim bs -Ġv arsa -Ġmethy l -ĠKar ere -Æ°á» Ł -R ad -Ġprepared ness -он Ñĩ -ĠO D -ĠC GI -Ġठ® -Ġspeech less -Ġlas ci -Ġbol ag -ĠÑħоÑĩ еÑĤÑģÑı -Ġgr ieving -ĠJohann es -ĠCar roll -ad aki -Ī ¬ë -ĠsÅĤ u -Ġinner halb -Ġgymn astics -п ÑĢи -if iques -Ġkar ate -Ġdom u -ãģĿãĤĮ ãģ§ -OTH ER -Ġdemand é -Ġbook let -ĠKy oto -Ġw oh -ĠMar ÃŃa -viol ent -J E -Ġl óg -Ġbrut ally -c ot -ĠÙħ ÛĮ -ĠWars z -å® Ī -w ol -Ġmik ä -ĠPron ounce -ĠBrend an -Ġr oup -Ġital iano -å¦Ĥ æѤ -Ġкомп ÑĮÑİÑĤ -Ġur ging -ed es -Ġcarbon o -ĠRichards on -ĠÐĿ аÑĩ -ĠTra iner -ĠCrime a -Ġdi apers -Ġco vet -ĠMah ar -ĠH utch -ĠAus w -ber ty -Ġind ifferent -кÑĢ еÑĤ -uld ade -Ġhar ms -¢ ÙĨ -les ia -Ġg io -ĠMist ress -ĠK nox -ĠFRE E -Ġë £¨ë -ĠнаÑĪ а -Ġinvinci ble -Ġma iden -ĠJ eez -Ġbre ve -po le -Ġcritic isms -ĠRus ia -ठ® -ph in -ĠComp are -ĠB ON -Ġsne aking -ĠR ails -ĠG eral -Ġ195 3 -H ola -Ġоп ÑĭÑĤ -Ġrain forest -Ġbel um -ĠOb i -ĠIS S -ãĤĮ ãģªãģĦ -ĠС в -Ġbl ond -Ġwz gl -Ġpowiedz iaÅĤ -Ġch oking -ĠSong s -ĠBir az -Ġyell s -Ġstyl ist -ÏĮ ÏĦε -Ġsch reiben -ĠJ aw -ĠEle ven -ĠR if -/ . -Ġìĺ¤ë ŀľë§Į -Ġtreat ies -uff ed -ĠâĪ Ĵ -Ġroof s -à¹Ģภª -Ġë » -Ġspark le -ĠK iev -ĠAr gu -ere cht -ĠÐĿад о -ĠF IL -Ġmol ta -ĠDe vi -Ġcam pe -Ġbene vol -ĠT ough -Ġmo im -Ġevac uate -Ġer rado -å© Ĩ -ÑĢÑĥ го -Ġíİ ĺ -ĠÎĵ ια -Ġweak en -Ġillum inated -Ġsig lo -ĠV acc -и ей -al is -ĠÑĥ ÑģÑĤÑĢой -Ġdon a -ÅĤ os -ü man -Ġprodu cción -Ġcl ot -ĠM ango -Ġune asy -Ġsh uts -ĠExam ples -ve ll -e be -Ġprompt ly -ĠT eles -ĠпÑĢоÑĪ л -Ġpu erta -Ġüber zeug -Ġco ch -so cial -ĠB enson -ĠM eth -ĠEx ped -Ġsupplement al -Ġconce ive -Ġ×ĺ ×ķ×ij -Ġcapt ivity -ıĻ ìķĪ -ĠÑħ Ñĥд -form ing -Ġupload s -Ġturbul ence -j oint -Ġsatisf actory -ĠAn ime -Ġwash es -Ġliber als -ĠSun shine -ĠRE AL -ub lik -b inary -T ony -Ġpolar ized -Ġenrich ed -t aking -ĠëģĿ ëĤĺ -Ġple asures -Ġex termin -in ese -at l -v är -аÑĢ Ñĭ -Ġmy ÅĽ -n arrator -Ġод ном -Ġnaj wiÄĻ -Ġmobil ize -Ġmill or -Ġat a -æ· · -ĠpolÃŃt ico -Ġple ad -Ġpain ters -ĠS ow -о ÑĦ -ĠìĺĽ ëĤł -ĠÑĩ ÑĤоб -Ġs abor -ĠUnd ert -ĠJER RY -Å¡ ÃŃ -Ġë° ĸìĹIJ -Ġpréc éd -Ġannot ation -ĠI naudible -Ġtext ured -Ġfisher man -v ordan -icher ung -Ġìłģ ìĿ´ -Ġge zeigt -Ġmand ates -Ġbe ak -ĠTW O -ĠAk bar -il ian -Ġtiế p -Ġsuperior ity -ink u -Ġl ys -ĠF CC -ĠC PA -ust ering -nic os -an ja -Ġch ills -ĠC age -Ġse aling -Ġsa ç -Ġded ans -ĠAl ger -Ġspe zie -Ġcol oss -ıy ı -clock wise -Ġexact amente -Ġ iemand -am ı -Ġmand ar -ra j -f aced -ag ua -Ġê¹ Ķë -Ġins besondere -Ġdri zzle -Ġdimin ish -ĠY oda -A I -Ġbil miyorum -ĠM MA -ateg ory -ĠпеÑĢ еп -Ġparticip ar -Ġnormal ized -Ġcomplex ities -æ´ ² -æİ § -аÑĢ ов -m ist -ich a -Gr oup -Ġresil iency -Ġnog le -ĠCN C -pr ü -Ġphysic ists -н ок -L I -Ġstuff s -Ġsist emas -Ġinterfer ing -ĠMar vin -ér cito -ĠìĹĨ ê³ł -Ġson ic -Ġequ iv -Ġab ord -ĠRam en -Ġ0 9 -med im -at iques -Ġдел аÑİÑĤ -Ġunanim ously -Ġsk irts -ĠíĬ¹ ë³Ħ -ĠP rix -k ami -Ġfr uition -Ġbirthday s -ик ом -Ġinaug ural -Ġcorrel ate -ĠT ory -ĠëĤĺ ìģ -Ġde w -ĠPre cis -ih i -Ġë¬¸ìłľ ê°Ģ -Ġc iting -ĠL ana -ĠK ag -Ġplay through -ĠProt ocol -fr ist -hov ah -Ġmerc iful -Ġb ilingual -ĠG uitar -r h -Ġglam orous -ĠVik ings -ĠOoo oh -íķĺ ëĬĶëį° -ĠUg anda -Ġcollaps es -ent ry -Ġantioxid ants -ëĤ ĺë -ÑĪ аÑı -Ġtri via -Ġgä ller -Ġfun gi -Ġmil ks -Ġd icht -μ η -po ke -ĠвÑĭп ÑĥÑģк -Ġfeed er -ĠAl cohol -h ower -Ġdes erving -ĠRe bel -ios is -Ġ10 3 -Ġhand out -Ġen m -Ġland lords -Ġge ology -r ils -Ġco bra -ĠV old -ĠP anch -ĠGRE G -Ġpr oss -Ġbrac elets -ĠV ega -Ġroz um -æ¬ ¾ -аз д -ĠLy nd -ĠHon ors -Ġsurrend ered -Ġlibr arians -12 5 -ĠÑģ иг -Ġuniform ly -ĠE agles -ìķ Ļ -иÑĤ ан -and id -ĠìłĪë ĮĢ -ĠØ ¶ -Ġarrest s -ĠCS V -ĠAzerbai jan -ort ic -ĠD X -ĠAdvent ures -Ġab us -ĠF au -Ġschlim m -Ġratt ling -Ġconsum es -ĠTol kien -Ġresurrect ed -ĠX Y -íĬ¸ ê°Ģ -ĠвÑĭ ÑģÑĤÑĥп -ĠAng ie -żen ia -M ic -ĠShe ila -acht et -Ġover st -Ġl â -Ġine ffective -æĿ ¡ -æĢİä¹Ī äºĨ -å¿ Ļ -Ġwicht iger -Ġv ino -Ġp um -Ġang led -ĠP ione -ĠM ỹ -ãģĿãĤĮ ãģ¯ -wo ÅĽÄĩ -d raw -ั à¹Ī -mark ets -Ġcaf es -ĠC em -â Ŀ¤ -ĠS uit -M K -Ġemphas izes -Ġtort illa -Ġmejor ar -ĠSur viv -cast ing -Ġeduc ación -ĠG um -u ely -ĠìĹ¬ê¸° ëĬĶ -Ġstretch y -en ça -Ġwith hold -Ġex iting -Ġenthal py -ĠTrans it -ıl mÄ±ÅŁ -al ies -Ġsal var -Ġlean ed -ĠgroÃŁ es -Ġf itt -ак и -S arah -Ġhost el -Ġfinger na -Ġnadzie jÄĻ -w ives -R ec -Ġsp ool -аÑĤ ов -ĠEn emy -Ġf ury -Ġdet ta -ĠF ay -éļ ¨ -Ñı ÑİÑĤ -Ġaproxim adamente -Ġsil os -Ġmag ist -Ġc ree -ĠKr ank -ĠD OWN -Ġstart led -Ġre born -ĠUm welt -ĠSuz anne -ни ÑĨÑĭ -out ez -ĠJ AC -y ards -rad as -ra u -ip ts -h ail -Ġparagraph s -Ġme glio -Ġisol ating -Ġace ite -ĠH arsh -Ġcy st -ĠBlock chain -ĠÑħоÑĢоÑĪ ий -Ġvirt uous -Ġinvestig ación -Ġdev oir -Ġmast urb -ĠS ale -ÙĬر Ø© -ĠÎ § -ĠStra ÃŁen -Ġdi kk -Ġa fore -ĠJung kook -Ġcho ciaż -ĠDebat te -Ġweird ly -Ġvia je -reg ist -H elp -Ġkind eren -Ġform ulated -Ġenf im -ĠTow ards -ко ÑĹ -iver ing -ĠдеÑĤ и -char ger -Ġpur l -Ġacadem ically -ĠNur se -Ġdel eting -ay o -Ġref usal -Ġdepict s -ĠDr acula -Ġtoast ed -ĠZomb ie -ĠSuper ior -ĠB old -Ġquizz es -Ġg le -4 50 -Ġcome ço -yn n -Ġver st -ĠO laf -Ġpom oc -ĠS ask -ë ĺ -ĠT CP -ĠProper ty -íķĺ ì£ł -à¸ľ ม -bo om -ar os -ĠÑĢоÑģÑģ ий -ĠбÑĭв аеÑĤ -åĩº åİ» -ĠìĿ´ìķ¼ 기를 -Ġcomb ien -v acc -Ġeben falls -par a -Ġз м -Ġdesper ation -ord re -Ġש׾ ×Ļ -Ġgener ously -ĠÐŀ к -Ġorb iting -> ", "eos_token": "<|endoftext|>", "unk_token": "<|endoftext|>"} \ No newline at end of file diff --git a/funasr_detach/models/whisper/utils/assets/multilingual/tokenizer_config.json b/funasr_detach/models/whisper/utils/assets/multilingual/tokenizer_config.json deleted file mode 100644 index 0235ccff90efbc022fbe30360220375c5bad68da..0000000000000000000000000000000000000000 --- a/funasr_detach/models/whisper/utils/assets/multilingual/tokenizer_config.json +++ /dev/null @@ -1 +0,0 @@ -{"unk_token": {"content": "<|endoftext|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "__type": "AddedToken"}, "bos_token": {"content": "<|endoftext|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "__type": "AddedToken"}, "eos_token": {"content": "<|endoftext|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "__type": "AddedToken"}, "add_prefix_space": false, "model_max_length": 1024, "special_tokens_map_file": null, "name_or_path": "multilingual", "errors": "replace", "tokenizer_class": "GPT2Tokenizer"} \ No newline at end of file diff --git a/funasr_detach/models/whisper/utils/assets/multilingual/vocab.json b/funasr_detach/models/whisper/utils/assets/multilingual/vocab.json deleted file mode 100644 index 406c3f2600089389bd8bf3920c2fa3770d11789a..0000000000000000000000000000000000000000 --- a/funasr_detach/models/whisper/utils/assets/multilingual/vocab.json +++ /dev/null @@ -1 +0,0 @@ -{"!": 0, "\"": 1, "#": 2, "$": 3, "%": 4, "&": 5, "'": 6, "(": 7, ")": 8, "*": 9, "+": 10, ",": 11, "-": 12, ".": 13, "/": 14, "0": 15, "1": 16, "2": 17, "3": 18, "4": 19, "5": 20, "6": 21, "7": 22, "8": 23, "9": 24, ":": 25, ";": 26, "<": 27, "=": 28, ">": 29, "?": 30, "@": 31, "A": 32, "B": 33, "C": 34, "D": 35, "E": 36, "F": 37, "G": 38, "H": 39, "I": 40, "J": 41, "K": 42, "L": 43, "M": 44, "N": 45, "O": 46, "P": 47, "Q": 48, "R": 49, "S": 50, "T": 51, "U": 52, "V": 53, "W": 54, "X": 55, "Y": 56, "Z": 57, "[": 58, "\\": 59, "]": 60, "^": 61, "_": 62, "`": 63, "a": 64, "b": 65, "c": 66, "d": 67, "e": 68, "f": 69, "g": 70, "h": 71, "i": 72, "j": 73, "k": 74, "l": 75, "m": 76, "n": 77, "o": 78, "p": 79, "q": 80, "r": 81, "s": 82, "t": 83, "u": 84, "v": 85, "w": 86, "x": 87, "y": 88, "z": 89, "{": 90, "|": 91, "}": 92, "~": 93, "¡": 94, "¢": 95, "£": 96, "¤": 97, "¥": 98, "¦": 99, "§": 100, "¨": 101, "©": 102, "ª": 103, "«": 104, "¬": 105, "®": 106, "¯": 107, "°": 108, "±": 109, "²": 110, "³": 111, "´": 112, "µ": 113, "¶": 114, "·": 115, "¸": 116, "¹": 117, "º": 118, "»": 119, "¼": 120, "½": 121, "¾": 122, "¿": 123, "À": 124, "Á": 125, "Â": 126, "Ã": 127, "Ä": 128, "Å": 129, "Æ": 130, "Ç": 131, "È": 132, "É": 133, "Ê": 134, "Ë": 135, "Ì": 136, "Í": 137, "Î": 138, "Ï": 139, "Ð": 140, "Ñ": 141, "Ò": 142, "Ó": 143, "Ô": 144, "Õ": 145, "Ö": 146, "×": 147, "Ø": 148, "Ù": 149, "Ú": 150, "Û": 151, "Ü": 152, "Ý": 153, "Þ": 154, "ß": 155, "à": 156, "á": 157, "â": 158, "ã": 159, "ä": 160, "å": 161, "æ": 162, "ç": 163, "è": 164, "é": 165, "ê": 166, "ë": 167, "ì": 168, "í": 169, "î": 170, "ï": 171, "ð": 172, "ñ": 173, "ò": 174, "ó": 175, "ô": 176, "õ": 177, "ö": 178, "÷": 179, "ø": 180, "ù": 181, "ú": 182, "û": 183, "ü": 184, "ý": 185, "þ": 186, "ÿ": 187, "Ā": 188, "ā": 189, "Ă": 190, "ă": 191, "Ą": 192, "ą": 193, "Ć": 194, "ć": 195, "Ĉ": 196, "ĉ": 197, "Ċ": 198, "ċ": 199, "Č": 200, "č": 201, "Ď": 202, "ď": 203, "Đ": 204, "đ": 205, "Ē": 206, "ē": 207, "Ĕ": 208, "ĕ": 209, "Ė": 210, "ė": 211, "Ę": 212, "ę": 213, "Ě": 214, "ě": 215, "Ĝ": 216, "ĝ": 217, "Ğ": 218, "ğ": 219, "Ġ": 220, "ġ": 221, "Ģ": 222, "ģ": 223, "Ĥ": 224, "ĥ": 225, "Ħ": 226, "ħ": 227, "Ĩ": 228, "ĩ": 229, "Ī": 230, "ī": 231, "Ĭ": 232, "ĭ": 233, "Į": 234, "į": 235, "İ": 236, "ı": 237, "IJ": 238, "ij": 239, "Ĵ": 240, "ĵ": 241, "Ķ": 242, "ķ": 243, "ĸ": 244, "Ĺ": 245, "ĺ": 246, "Ļ": 247, "ļ": 248, "Ľ": 249, "ľ": 250, "Ŀ": 251, "ŀ": 252, "Ł": 253, "ł": 254, "Ń": 255, "Ġt": 256, "Ġa": 257, "Ġth": 258, "in": 259, "er": 260, "Ġw": 261, "Ġs": 262, "ou": 263, "Ġthe": 264, "re": 265, "on": 266, "at": 267, "en": 268, "Ġc": 269, "it": 270, "is": 271, "Ġb": 272, "nd": 273, "Ġd": 274, "Ġm": 275, "Ġh": 276, "Ġo": 277, "ing": 278, "es": 279, "Ġp": 280, "Ġto": 281, "an": 282, "Ġf": 283, "or": 284, "ll": 285, "ĠI": 286, "Ġl": 287, "Ġy": 288, "ar": 289, "Ġg": 290, "Ġyou": 291, "ed": 292, "Ġand": 293, "Ġin": 294, "Ġof": 295, "as": 296, "Ġn": 297, "om": 298, "ic": 299, "Ġthat": 300, "us": 301, "et": 302, "ve": 303, "al": 304, "ow": 305, "le": 306, "Ġis": 307, "Ġe": 308, "Ġit": 309, "ot": 310, "'s": 311, "Ġbe": 312, "ion": 313, "ĠT": 314, "Ġwh": 315, "ĠA": 316, "ent": 317, "ĠS": 318, "Ġre": 319, "ay": 320, "Ġwe": 321, "Ġon": 322, "ere": 323, "Ġha": 324, "ut": 325, "ac": 326, "id": 327, "ig": 328, "os": 329, "ke": 330, "ver": 331, "im": 332, "ĠÐ": 333, "ĠTh": 334, "am": 335, "all": 336, "Ġfor": 337, "el": 338, "ch": 339, "ro": 340, "Ġthis": 341, "Ġst": 342, "ĠW": 343, "Ġu": 344, "ad": 345, "out": 346, "ir": 347, "ld": 348, "ct": 349, "Ġk": 350, "if": 351, "Ġgo": 352, "..": 353, "о": 354, "ith": 355, "ly": 356, "ht": 357, "qu": 358, "Ġ-": 359, "Ġdo": 360, "Ġj": 361, "Ġhave": 362, "ĠB": 363, "Ġan": 364, "Ġwith": 365, "Ġare": 366, "Ġr": 367, "Ġde": 368, "Ġse": 369, "Ġso": 370, "Ġv": 371, "st": 372, "ill": 373, "ur": 374, "Ġli": 375, "ĠM": 376, "est": 377, "od": 378, "ally": 379, "'t": 380, "ust": 381, "Ġas": 382, "ĠC": 383, "ce": 384, "Ġme": 385, "а": 386, "е": 387, "il": 388, "ĠH": 389, "Ġwas": 390, "ter": 391, "th": 392, "Ġcan": 393, "ant": 394, "Ġcom": 395, "our": 396, "ight": 397, "ĠY": 398, "ation": 399, "ĠAnd": 400, "ol": 401, "Ġsh": 402, "ÑĤ": 403, "op": 404, "se": 405, "Ġnot": 406, "ĠSo": 407, "Ġne": 408, "un": 409, "Ġab": 410, "Ġlike": 411, "Ġat": 412, "ĠD": 413, "ie": 414, "Ġhe": 415, "Ġcon": 416, "Ġch": 417, "ore": 418, "Ġal": 419, "Ġor": 420, "Ġqu": 421, "ĠO": 422, "ome": 423, "ra": 424, "ul": 425, "ĠN": 426, "pp": 427, "Ġyour": 428, "ould": 429, "ĠP": 430, "Ġfr": 431, "ge": 432, "ers": 433, "'re": 434, "и": 435, "Ġthey": 436, "Ġwhat": 437, "use": 438, "Ġall": 439, "ĠThe": 440, "ĠL": 441, "ess": 442, "em": 443, "Ġkn": 444, "Ġjust": 445, "art": 446, "Ġpro": 447, "very": 448, "um": 449, "Ġlo": 450, "Ġì": 451, "Ġmy": 452, "ok": 453, "Ġex": 454, "ab": 455, "Ġthere": 456, "Ġbut": 457, "Ġknow": 458, "Ġsu": 459, "ĠG": 460, "Ñģ": 461, "ĠE": 462, "Ġma": 463, "оÐ": 464, "Ġen": 465, "Ġabout": 466, "ĠIt": 467, "ist": 468, "Ġwor": 469, "ri": 470, "ind": 471, "Ġone": 472, "ate": 473, "and": 474, "ink": 475, "Ġle": 476, "ort": 477, "'m": 478, "ĠF": 479, "ich": 480, "ÑĢ": 481, "ide": 482, "Ġget": 483, "Ġout": 484, "...": 485, "Ġwill": 486, "ãģ": 487, "ive": 488, "н": 489, "Ġfrom": 490, "ain": 491, "ĠWe": 492, "Ġup": 493, "pe": 494, "res": 495, "ca": 496, "ĠR": 497, "Ġif": 498, "Ġpl": 499, "Ġdon": 500, "ack": 501, "Ġ1": 502, "Ġ\"": 503, "Ġtr": 504, "Ġus": 505, "ĠWh": 506, "ity": 507, "ĠJ": 508, "ĠYou": 509, "Ġhere": 510, "her": 511, "Ġsome": 512, "oug": 513, "ak": 514, "ard": 515, "Ġgoing": 516, "Ġun": 517, "ment": 518, "Ġthink": 519, "Ġpe": 520, "end": 521, "Ġ(": 522, "cause": 523, "Ġtim": 524, "ast": 525, "é": 526, "Ġour": 527, "Ġwant": 528, "ame": 529, "ies": 530, "Ġë": 531, "ud": 532, "ine": 533, "Ġreally": 534, "Ġte": 535, "Ġsee": 536, "ci": 537, "Ġby": 538, "so": 539, "ure": 540, "ose": 541, "Ġ[": 542, "are": 543, "Ġmore": 544, "ah": 545, "one": 546, "ck": 547, "ople": 548, "аÐ": 549, "Ġthen": 550, "Ġthing": 551, "Ġthem": 552, "ven": 553, "ound": 554, "ost": 555, "ong": 556, "ect": 557, "Ġright": 558, "ag": 559, "Ġint": 560, "Ġpeople": 561, "Ġwhen": 562, "ous": 563, "pl": 564, "Ġtime": 565, "Ġim": 566, "Ġwho": 567, "Ġ2": 568, "ap": 569, "Ġbecause": 570, "hing": 571, "Ġno": 572, "ice": 573, "Ġlook": 574, "Ġhas": 575, "Ġwould": 576, "Ġhow": 577, "act": 578, "Ġfe": 579, "nt": 580, "ough": 581, "Ġpr": 582, "ĠBut": 583, "Ġsay": 584, "Ñĥ": 585, "Ġnow": 586, "Ġman": 587, "Ġvery": 588, "Ġwork": 589, "iz": 590, "ĠK": 591, "iv": 592, "itt": 593, "Ġar": 594, "ep": 595, "Ġcl": 596, "Ġwhich": 597, "Ġco": 598, "ans": 599, "'ve": 600, "Ġsa": 601, "ff": 602, "'ll": 603, "Ġany": 604, "Ġact": 605, "Ġye": 606, "ber": 607, "ach": 608, "age": 609, "per": 610, "Ġalso": 611, "fer": 612, "Ġthese": 613, "Ġad": 614, "еÐ": 615, "ther": 616, "ace": 617, "ick": 618, "ake": 619, "reat": 620, "ire": 621, "ue": 622, "Ġag": 623, "ĠU": 624, "uch": 625, "ions": 626, "ry": 627, "00": 628, "na": 629, "Ġdid": 630, "Ġque": 631, "Ġhad": 632, "Ġevery": 633, "ĠHe": 634, "Ġla": 635, "Ġway": 636, "Ġsp": 637, "ble": 638, "ĠThis": 639, "ass": 640, "Ġtheir": 641, "ite": 642, "Ġneed": 643, "Ġpart": 644, "Ġwere": 645, "Ġback": 646, "ip": 647, "own": 648, "omet": 649, "be": 650, "ase": 651, "Ġmake": 652, "irst": 653, "ia": 654, "ence": 655, "ang": 656, "ank": 657, "Ġgot": 658, "Ġpre": 659, "Ġcont": 660, "Ġother": 661, "pt": 662, "ĠThat": 663, "og": 664, "Ġgood": 665, "Ġinto": 666, "alk": 667, "Ġbeen": 668, "Ġam": 669, "Ġover": 670, "ually": 671, "Ġâ": 672, "ìĿ": 673, "Ġund": 674, "he": 675, "way": 676, "Ġgr": 677, "ÑĮ": 678, "Ġdif": 679, "Ġper": 680, "Ñı": 681, "ĠIn": 682, "Ġtw": 683, "ond": 684, "ars": 685, "int": 686, "orm": 687, "Ġlot": 688, "Ġwhere": 689, "ĠÃ": 690, "ĠV": 691, "Ġsomet": 692, "л": 693, "ens": 694, "Ġgu": 695, "Ġac": 696, "ug": 697, "Ñĭ": 698, "ı": 699, "Ġfirst": 700, "ree": 701, "Ġhis": 702, "ittle": 703, "Ġimp": 704, "Ġmo": 705, "av": 706, "Ġlittle": 707, "ĠWhat": 708, "Ġmuch": 709, "Ġz": 710, "Ġê": 711, "able": 712, "Ġп": 713, "Ġpo": 714, "Ġcomp": 715, "ne": 716, "Ġdis": 717, "Ġlet": 718, "ance": 719, "Ġher": 720, "Ġthings": 721, "Ġstart": 722, "ult": 723, "Ġapp": 724, "Ġres": 725, "Ġfo": 726, "Ġcould": 727, "Ġinter": 728, "Ġthose": 729, "Ġdes": 730, "Ġwell": 731, "Ġtwo": 732, "Ġkind": 733, "xt": 734, "ress": 735, "ely": 736, "ä": 737, "Ġbr": 738, "Ġthr": 739, "Ġв": 740, "Ġi": 741, "ish": 742, "Ġdiffer": 743, "Ġro": 744, "ĠSt": 745, "Ġsomething": 746, "Ġtake": 747, "Ġbo": 748, "ys": 749, "Ġshe": 750, "Ġtalk": 751, "lo": 752, "Ñĩ": 753, "Ġeven": 754, "к": 755, "ãĢ": 756, "Ġн": 757, "Ġbu": 758, "ĠIf": 759, "Ġdown": 760, "ĠCh": 761, "ade": 762, "ations": 763, "Ġuse": 764, "ord": 765, "Ġoff": 766, "Ġactually": 767, "Ġspe": 768, "du": 769, "ated": 770, "ater": 771, "oss": 772, "ning": 773, "ü": 774, "Ġdoes": 775, "ĠÑģ": 776, "Ġnew": 777, "Ġbet": 778, "vel": 779, "cess": 780, "ple": 781, "Ġhapp": 782, "ting": 783, "onna": 784, "Ġes": 785, "Ġday": 786, "Ġonly": 787, "ign": 788, "kay": 789, "sel": 790, "ents": 791, "ount": 792, "ild": 793, "ile": 794, "Ġsc": 795, "Ġhim": 796, "Ġagain": 797, "ving": 798, "Ġgonna": 799, "Ġcomm": 800, "Ġhel": 801, "other": 802, "Ġke": 803, "ical": 804, "Ġ3": 805, "Ġel": 806, "Ġthrough": 807, "Ġcome": 808, "ark": 809, "day": 810, "ier": 811, "ó": 812, "Ġthan": 813, "ĠThey": 814, "Ġmay": 815, "Ġser": 816, "íķ": 817, "Ġcall": 818, "Ġdifferent": 819, "Ġshould": 820, "ĠThere": 821, "ary": 822, "ĠNow": 823, "ãĤ": 824, "thing": 825, "we": 826, "ory": 827, "fter": 828, "Ġput": 829, "ors": 830, "ial": 831, "ëĭ": 832, "Ġunder": 833, "Ġinc": 834, "ĠYe": 835, "ub": 836, "form": 837, "Ġvide": 838, "à¸": 839, "vers": 840, "Ġfeel": 841, "á": 842, "ody": 843, "ft": 844, "fore": 845, "Ġem": 846, "get": 847, "Ġsaid": 848, "ition": 849, "Ġrec": 850, "ious": 851, "atch": 852, "Ġtry": 853, "Ġhelp": 854, "Ġshow": 855, "д": 856, "Ġbit": 857, "ull": 858, "в": 859, "ÑĤо": 860, "gr": 861, "Ġplay": 862, "ife": 863, "ail": 864, "ĠYeah": 865, "Ġquest": 866, "Ġmany": 867, "Ġpers": 868, "Ġgreat": 869, "ÃŃ": 870, "Ġest": 871, "ng": 872, "ĠâĻ": 873, "ty": 874, "la": 875, "ĠOh": 876, "Ġ×": 877, "à®": 878, "ĠBe": 879, "ady": 880, "Ġmost": 881, "ction": 882, "ĠNo": 883, "Ġdoing": 884, "Ġbeing": 885, "Ġtoo": 886, "ces": 887, "Ġbl": 888, ".\"": 889, "Ġrem": 890, "iss": 891, "ons": 892, ">>": 893, "ru": 894, "wn": 895, "ont": 896, "ib": 897, "ell": 898, "Ġsm": 899, "oth": 900, "ual": 901, "Ġ>>": 902, "Ġph": 903, "les": 904, "oc": 905, "ful": 906, "Ġsec": 907, "ise": 908, "Ġadd": 909, "igh": 910, "ert": 911, "Ġsame": 912, "âĢ": 913, "Ġmean": 914, "Ġfind": 915, "ek": 916, "Ġend": 917, "--": 918, "м": 919, "Ġstill": 920, "az": 921, "Ġ'": 922, "Ġmin": 923, "Ġyears": 924, "urn": 925, "Ġaround": 926, "self": 927, "Ġwr": 928, "bs": 929, "ought": 930, "ĠâĻª": 931, "Ġfl": 932, "ange": 933, "Ġafter": 934, "Ġpoint": 935, "mer": 936, "ved": 937, "Ġlong": 938, "oy": 939, "ä¸": 940, "Ġcr": 941, "ways": 942, "Ġsy": 943, "Ġtra": 944, "Ġ20": 945, "ave": 946, "Ġche": 947, "Ġent": 948, "Ġbefore": 949, "ph": 950, "Ġatt": 951, "ian": 952, "ily": 953, "Ġperson": 954, "Ġbig": 955, "Ġsch": 956, "Ġreal": 957, "Ġnext": 958, "Ġlove": 959, "Ġvideo": 960, "ĠLet": 961, "Ġfin": 962, "Ġmak": 963, "ible": 964, "Ġtoday": 965, "erm": 966, "ĠAl": 967, "ower": 968, "ann": 969, "ix": 970, "Ġpar": 971, "Ġstud": 972, "ö": 973, "Ġimport": 974, "te": 975, "Ġgive": 976, "ves": 977, "Ġdie": 978, "Ġdec": 979, "Ġtell": 980, "Ġк": 981, "ÑģÑĤ": 982, "Ġwhy": 983, "ically": 984, "ict": 985, "red": 986, "Ġbas": 987, "Ġsure": 988, "Ġbel": 989, "ating": 990, "Ġtak": 991, "Ġset": 992, "Ġlife": 993, "Ġdidn": 994, "ا": 995, "ob": 996, "und": 997, "ath": 998, "Ġop": 999, "Ġо": 1000, "ait": 1001, "Ġworld": 1002, "Ġsupp": 1003, "io": 1004, "Ġcour": 1005, "Ġи": 1006, "ward": 1007, "ен": 1008, "Ġalways": 1009, "up": 1010, "Ġhand": 1011, "ĠHow": 1012, "cial": 1013, "Ġcons": 1014, "ĠÑ": 1015, "Ġind": 1016, "Ġ4": 1017, "ĠAs": 1018, "Ġfun": 1019, "ject": 1020, "Ġimportant": 1021, "Ġsur": 1022, "ew": 1023, "ates": 1024, "Ġ5": 1025, "Ġdi": 1026, "Ġmade": 1027, "Ġins": 1028, "Ġask": 1029, "Ġet": 1030, "Ġnum": 1031, "Ġcar": 1032, "ĠOkay": 1033, "Ġsim": 1034, "ik": 1035, "Ġlast": 1036, "ĠGo": 1037, "Ġmus": 1038, "Ġrel": 1039, "ular": 1040, "´ì": 1041, "ĠWell": 1042, "pect": 1043, "ĠThank": 1044, "Ġthree": 1045, "ã": 1046, "ãĥ": 1047, "Ġinv": 1048, "Ġgen": 1049, "lic": 1050, "Ġhappen": 1051, "ëĬ": 1052, "ien": 1053, "ever": 1054, "ов": 1055, "Ġstr": 1056, "ĠAll": 1057, "Ġinst": 1058, "ĠâĢ": 1059, "Ġdef": 1060, "Ġsl": 1061, "Ġmight": 1062, "ung": 1063, "Ġyear": 1064, "Ġown": 1065, "Ġkeep": 1066, "body": 1067, "der": 1068, "ĠÑĤ": 1069, "Ġд": 1070, "Ġanother": 1071, "Ġmod": 1072, "Ġev": 1073, "Ġguys": 1074, "Ġable": 1075, "ão": 1076, "que": 1077, "ident": 1078, "ĠYes": 1079, "Ġits": 1080, "Ġplace": 1081, "Ġprodu": 1082, "arn": 1083, "Ġм": 1084, "Ġrep": 1085, "Ġexper": 1086, "Ġfam": 1087, "ities": 1088, "ific": 1089, "Ġhigh": 1090, "ied": 1091, "ool": 1092, "iew": 1093, "еÑĤ": 1094, "ren": 1095, "Ġdone": 1096, "Ġ...": 1097, "ëĬĶ": 1098, "stem": 1099, "ĠSe": 1100, "Ġbetter": 1101, "come": 1102, "Ġdel": 1103, "Ġty": 1104, "Ġum": 1105, "Ġho": 1106, "ĠAn": 1107, "Ġmon": 1108, "ings": 1109, "Ġsk": 1110, "Ġob": 1111, "com": 1112, "blem": 1113, "ope": 1114, "stand": 1115, "'d": 1116, "ments": 1117, "Ġele": 1118, "ĠIs": 1119, "Ġda": 1120, "Ġreg": 1121, "lease": 1122, "ike": 1123, "als": 1124, "ize": 1125, "ê°": 1126, "Ġcare": 1127, "Ġnever": 1128, "ìĿ´": 1129, "ese": 1130, "Ġmet": 1131, "olog": 1132, "ĠWhen": 1133, "uck": 1134, "еÑĢ": 1135, "Ġé": 1136, "Ġdat": 1137, "ç": 1138, "Ġexam": 1139, "ility": 1140, "Ġdet": 1141, "cri": 1142, "Ġused": 1143, "ĠDo": 1144, "Ġtrans": 1145, "eg": 1146, "ten": 1147, "Ñİ": 1148, "cus": 1149, "Ġsecond": 1150, "Ġbest": 1151, "Ġhard": 1152, "Ġide": 1153, "Ġproblem": 1154, "ê³": 1155, "ĠUn": 1156, "Ñħ": 1157, "ĠÎ": 1158, "Ġwatch": 1159, "ĠSh": 1160, "atter": 1161, "Ġpret": 1162, "Ġder": 1163, "Ġcourse": 1164, "ÅŁ": 1165, "ative": 1166, "ics": 1167, "Ġquestion": 1168, "ute": 1169, "ìĹ": 1170, "ĠFor": 1171, "ather": 1172, "Ġcol": 1173, "iend": 1174, "Ġí": 1175, "ĠZ": 1176, "Ġdoesn": 1177, "arch": 1178, "Ġinterest": 1179, "Ġpol": 1180, "Ġcor": 1181, "ience": 1182, "Ġpres": 1183, "Ġeach": 1184, "Ġsystem": 1185, "Ġfact": 1186, "iel": 1187, "ably": 1188, "Ġer": 1189, "Ġrun": 1190, "ĠìĿ": 1191, "Ġtop": 1192, "ner": 1193, "Ġthought": 1194, "Ġeas": 1195, "ient": 1196, "Ġcre": 1197, "ÑĪ": 1198, "Ġcommun": 1199, "ye": 1200, "ready": 1201, "llow": 1202, "Ġeverything": 1203, "omm": 1204, "Ġmed": 1205, "ļĶ": 1206, "Ġcount": 1207, "its": 1208, "Ġcompl": 1209, "hip": 1210, "ÙĦ": 1211, "ook": 1212, "Ġtoget": 1213, "Ġtogether": 1214, "amp": 1215, "Ġgame": 1216, "Ġalready": 1217, "ал": 1218, "Ġcalled": 1219, "ale": 1220, "ÅĤ": 1221, "ĠMy": 1222, "Ġunderstand": 1223, "Ġdr": 1224, "Ġmom": 1225, "ited": 1226, "ол": 1227, "Ġusing": 1228, "zy": 1229, "Ġnumber": 1230, "ãĢģ": 1231, "ced": 1232, "Ġcle": 1233, "но": 1234, "ëĭ¤": 1235, "ince": 1236, "Ġlooking": 1237, "Ġpretty": 1238, "Ġprob": 1239, "ĠShe": 1240, "Ġve": 1241, "Ġgetting": 1242, "Ġweek": 1243, "Ġeff": 1244, "uff": 1245, "air": 1246, "ues": 1247, "ern": 1248, "ĠQ": 1249, "oup": 1250, "ention": 1251, "Ġside": 1252, "ом": 1253, "Ġform": 1254, "Ġbus": 1255, "Ġass": 1256, "Ġed": 1257, "ason": 1258, "ween": 1259, "âĢ¦": 1260, "Ġturn": 1261, "Ġcur": 1262, "Ġcoll": 1263, "Ġdire": 1264, "ĠGod": 1265, "Ġ10": 1266, "Ġequ": 1267, "Ġб": 1268, "Ġopen": 1269, "Ġsuch": 1270, "ird": 1271, "ак": 1272, "Ġear": 1273, "ÄĻ": 1274, "gan": 1275, "Ġpartic": 1276, "Ġfriend": 1277, "Ġexp": 1278, "Ġext": 1279, "Ġhome": 1280, "Ġwater": 1281, "ĠOn": 1282, "ÑĤÑĮ": 1283, "ork": 1284, "ĠпÑĢ": 1285, "Ġmove": 1286, "ness": 1287, "ense": 1288, "ho": 1289, "Ġchar": 1290, "co": 1291, "ins": 1292, "Ġboth": 1293, "Ġ19": 1294, "Ġgra": 1295, "Ġbetween": 1296, "á»": 1297, "Ġìķ": 1298, "ash": 1299, "ĠRe": 1300, "ai": 1301, "alth": 1302, "ures": 1303, "ember": 1304, "Ġav": 1305, "Ġver": 1306, "ê": 1307, "oney": 1308, "Ġthank": 1309, "Ġmaybe": 1310, "uc": 1311, "ime": 1312, "ê³ł": 1313, "Ġaway": 1314, "Ġname": 1315, "ouse": 1316, "Ġacc": 1317, "Ġmusic": 1318, "Ġchange": 1319, "Ġpass": 1320, "ger": 1321, "Ġbuild": 1322, "Ġval": 1323, "iness": 1324, "any": 1325, "Ġfew": 1326, "´ë": 1327, "ta": 1328, "Ġlist": 1329, "Ã¥": 1330, "Ġold": 1331, "Ġìŀ": 1332, "Ġsort": 1333, "Ġmem": 1334, "Ġca": 1335, "cept": 1336, "Ġgener": 1337, "Ġyeah": 1338, "Ġwhile": 1339, "Ġanything": 1340, "ric": 1341, "gram": 1342, "Ġein": 1343, "cy": 1344, "uring": 1345, "ĠDe": 1346, "Ġpower": 1347, "Ġcoming": 1348, "Ġword": 1349, "Ġ--": 1350, "Ġbelie": 1351, "Ġfound": 1352, "to": 1353, "п": 1354, "Ġmeans": 1355, "Ġinform": 1356, "ĠØ": 1357, "ĠÑĩ": 1358, "Ġsmall": 1359, "000": 1360, "Ġcame": 1361, "Ġíķ": 1362, "wh": 1363, "Ġworking": 1364, "Ġexample": 1365, "Ġpos": 1366, "Ġdep": 1367, "ê²": 1368, "äº": 1369, "ote": 1370, "Ġdem": 1371, "ì§": 1372, "ts": 1373, "Ġvar": 1374, "aut": 1375, "Ġtri": 1376, "chn": 1377, "Ġhead": 1378, "Ġwhole": 1379, "×Ļ": 1380, "ze": 1381, "Ġtrying": 1382, "Ġtem": 1383, "Ġcou": 1384, "ets": 1385, "Ġ6": 1386, "Ġfil": 1387, "velop": 1388, "Ġcase": 1389, "à¯": 1390, "Ġprobably": 1391, "Ġokay": 1392, "Ġplan": 1393, "Ġsit": 1394, "Ġschool": 1395, "ĠThen": 1396, "¸ë": 1397, "me": 1398, "Ġprocess": 1399, "Ġfar": 1400, "Ġread": 1401, "Ġposs": 1402, "Ġbre": 1403, "Ġsol": 1404, "icht": 1405, "Ġsupport": 1406, "ĠTo": 1407, "ertain": 1408, "Ġstarted": 1409, "Ġcap": 1410, "Ġleft": 1411, "Ġdata": 1412, "Ġtimes": 1413, "ел": 1414, "Ġwanted": 1415, "ан": 1416, "Ġtalking": 1417, "Ġist": 1418, "Ġhaving": 1419, "ump": 1420, "Ġcontin": 1421, "Ġsub": 1422, "Ġз": 1423, "pr": 1424, "ëĭĪ": 1425, "ina": 1426, "ż": 1427, "Ġcreat": 1428, "ode": 1429, "×ķ": 1430, "æĺ": 1431, "!!": 1432, "Ġterm": 1433, "ism": 1434, "од": 1435, "ĠBecause": 1436, "Ġwent": 1437, "ider": 1438, "Ġprov": 1439, "Ġchild": 1440, "Ġden": 1441, "Ġlight": 1442, "br": 1443, "³Ð¾": 1444, "oh": 1445, "Ġbook": 1446, "ĠÙ": 1447, "ution": 1448, "ĠJust": 1449, "ene": 1450, "Ġfour": 1451, "Ġvis": 1452, "ê°Ģ": 1453, "Ġhope": 1454, "Ġmaking": 1455, "ĠLe": 1456, "ìķ": 1457, "Ġopp": 1458, "au": 1459, "Ġmoney": 1460, "Ġprogram": 1461, "è": 1462, "Ġstand": 1463, "IN": 1464, "Ġsign": 1465, "Ġlearn": 1466, "Ãł": 1467, "ĠDon": 1468, "Ġteam": 1469, "Ġна": 1470, "lud": 1471, "Ġrest": 1472, "ices": 1473, "æľ": 1474, "ĠÑĢ": 1475, "Ġaut": 1476, "Ġlead": 1477, "ational": 1478, "de": 1479, "gy": 1480, "Ġnice": 1481, "Ġdas": 1482, "Ġdist": 1483, "Ġhum": 1484, "ĠOne": 1485, "æĪ": 1486, "Ġcomes": 1487, "Ġjo": 1488, "Ġcent": 1489, "Ġexpl": 1490, "Ġmark": 1491, "reen": 1492, "led": 1493, "gin": 1494, "ìļĶ": 1495, "Ġlevel": 1496, "Ġconf": 1497, "ush": 1498, "Ġdevelop": 1499, "Ġtest": 1500, "eng": 1501, "vious": 1502, "ature": 1503, "ем": 1504, "ret": 1505, "Ġje": 1506, "Ġstuff": 1507, "Ġclass": 1508, "ows": 1509, "Ġê·": 1510, "Ġsi": 1511, "Ġles": 1512, "rop": 1513, "çļ": 1514, "Ġpor": 1515, "Ġwar": 1516, "ìĹIJ": 1517, "Ġeveryone": 1518, "Ġge": 1519, "Ġcheck": 1520, "ott": 1521, "Ġsing": 1522, "Ġart": 1523, "Ġfollow": 1524, "Ġ201": 1525, "ĠFr": 1526, "ais": 1527, "ìĸ": 1528, "α": 1529, "å°": 1530, "ĠÃł": 1531, "imes": 1532, "Ġret": 1533, "Ġchang": 1534, "Ġpub": 1535, "Ġinf": 1536, "Ġtechn": 1537, "ada": 1538, "ives": 1539, "Ġbeh": 1540, "æĺ¯": 1541, "Ġlooks": 1542, "ãĢĤ": 1543, "з": 1544, "ĠWhy": 1545, "çļĦ": 1546, "Ġenough": 1547, "Ġbra": 1548, "itch": 1549, "ä»": 1550, "Ġadv": 1551, "б": 1552, "Ġwithout": 1553, "wer": 1554, "meric": 1555, "den": 1556, "Ġcomplet": 1557, "Ġidea": 1558, "ters": 1559, "ock": 1560, "Ġdefin": 1561, "Ġever": 1562, "Ġgl": 1563, "Ġonce": 1564, "Ġbring": 1565, "Ġsaying": 1566, "Ġans": 1567, "Ġhear": 1568, "nect": 1569, "Ġless": 1570, "go": 1571, "ream": 1572, "ado": 1573, "ìŀ": 1574, "Ġmind": 1575, "ente": 1576, "Ġfull": 1577, "Ġbad": 1578, "Ġwom": 1579, "Ġsomeone": 1580, "Ġdu": 1581, "Ġwon": 1582, "Ġcontro": 1583, "ortun": 1584, "Ġhealth": 1585, "Ġcho": 1586, "ĠAr": 1587, "Ġconc": 1588, "Ġinformation": 1589, "Ġstop": 1590, "att": 1591, "ately": 1592, "ä½": 1593, "Ġgroup": 1594, "ĠÑĥ": 1595, "Ġquite": 1596, "Ġresp": 1597, "ER": 1598, "ught": 1599, "ê¸": 1600, "man": 1601, "ized": 1602, "ĠBr": 1603, "Ġremember": 1604, "Ġfamily": 1605, "Ġbusiness": 1606, "aw": 1607, "Ġspec": 1608, "Ġau": 1609, "ĠOr": 1610, "Äħ": 1611, "Ġseen": 1612, "Ġlar": 1613, "Ġ7": 1614, "gg": 1615, "bers": 1616, "Ġdra": 1617, "Ġmonth": 1618, "Ġsays": 1619, "Ġiss": 1620, "Ġlive": 1621, "Ġline": 1622, "Ġmoment": 1623, "Ġexc": 1624, "els": 1625, "Ġsound": 1626, "Ġcool": 1627, "Ġloc": 1628, "Ġcertain": 1629, "Ġdri": 1630, "оÑĤ": 1631, "ames": 1632, "Ġmust": 1633, "ny": 1634, "иÑĤ": 1635, "Ġkid": 1636, "Ġinclud": 1637, "ìĿĦ": 1638, "ator": 1639, "ÄŁ": 1640, "ha": 1641, "ared": 1642, "Ġseem": 1643, "й": 1644, "ìĦ": 1645, "Ġelse": 1646, "Ġìł": 1647, "irl": 1648, "Ġ8": 1649, "Ġvo": 1650, "Ġquestions": 1651, "ines": 1652, "ee": 1653, "æĪij": 1654, "ür": 1655, "ĠAmeric": 1656, "Ġstory": 1657, "Ġserv": 1658, "vern": 1659, "ages": 1660, "land": 1661, "ĠâĢĵ": 1662, "era": 1663, "ĠCan": 1664, "Ġpop": 1665, "ether": 1666, "Ġna": 1667, "Ġorder": 1668, "Ġmakes": 1669, "Ġsince": 1670, "con": 1671, "ctor": 1672, "Ġthough": 1673, "Ġproduct": 1674, "ли": 1675, "Ġleg": 1676, "Ġmeet": 1677, "alf": 1678, "ÑģÑı": 1679, "unch": 1680, "iter": 1681, "ove": 1682, "×ķ×": 1683, "iet": 1684, "ам": 1685, "ital": 1686, "Ġsuper": 1687, "ling": 1688, "Ġpay": 1689, "Ġpara": 1690, "Ġjob": 1691, "ĠHere": 1692, "Ġsw": 1693, "ks": 1694, "ption": 1695, "ma": 1696, "Ġbelieve": 1697, "¬ë": 1698, "Ġwait": 1699, "ой": 1700, "Ġunt": 1701, "Ġquick": 1702, "hr": 1703, "ĠÑį": 1704, "ĠPro": 1705, "Ġmen": 1706, "à¹": 1707, "Ġdays": 1708, "Ġgoes": 1709, "Ġspeak": 1710, "ĠAt": 1711, "ement": 1712, "Ġmiss": 1713, "Ġaw": 1714, "Ġdesign": 1715, "Ġproject": 1716, "оÑĢ": 1717, "ij": 1718, "ants": 1719, "ats": 1720, "ĠChr": 1721, "Ġ9": 1722, "Ġcut": 1723, "Ġrequ": 1724, "Ġне": 1725, "ĠNot": 1726, "aster": 1727, "Ġmill": 1728, "Ġparticular": 1729, "Ġpie": 1730, "Ġstudents": 1731, "Ġfive": 1732, "oun": 1733, "ĠNe": 1734, "Ġgi": 1735, "Ġpas": 1736, "Ġfree": 1737, "ĠSp": 1738, "lich": 1739, "Ġprof": 1740, "Ġeng": 1741, "Ġprot": 1742, "ĠLike": 1743, "osed": 1744, "Ġconnect": 1745, "app": 1746, "Ġë§": 1747, "iting": 1748, "Ġblo": 1749, "Ġlos": 1750, "ists": 1751, "Ġexperience": 1752, "rent": 1753, "Ġstay": 1754, "Ġfood": 1755, "ton": 1756, "ruct": 1757, "Ġhist": 1758, "view": 1759, "ining": 1760, "most": 1761, "ivers": 1762, "bo": 1763, "ãģĦ": 1764, "ĠTr": 1765, "gen": 1766, "Ġplease": 1767, "Ġcommunity": 1768, "Ġce": 1769, "AN": 1770, "no": 1771, "Ġbody": 1772, "Ġhour": 1773, "Ġvers": 1774, "áº": 1775, "cer": 1776, "Ġê°": 1777, "Ġreason": 1778, "ĠRight": 1779, "Ġlater": 1780, "ÏĦ": 1781, "Ġhouse": 1782, "ĠX": 1783, "он": 1784, "Ġstate": 1785, "fic": 1786, "å¤": 1787, "ÅĽ": 1788, "ield": 1789, "Ġpri": 1790, "Ġpast": 1791, "Ġwalk": 1792, "ology": 1793, "ering": 1794, "anna": 1795, "Ġter": 1796, "Ġhold": 1797, "Ġorgan": 1798, "ben": 1799, "ο": 1800, "ón": 1801, "Ġeffect": 1802, "Ġyourself": 1803, "Ġplus": 1804, "aj": 1805, "ando": 1806, "ural": 1807, "Ġroom": 1808, "lect": 1809, "ê²Į": 1810, "?\"": 1811, "side": 1812, "Ġbecome": 1813, "ÑĨ": 1814, "ĠÂ": 1815, "ood": 1816, "Ġconst": 1817, "Ġnight": 1818, "utes": 1819, "ж": 1820, "Ġbreak": 1821, "Ġpain": 1822, "Ġstep": 1823, "ired": 1824, "Ġnothing": 1825, "Ġuntil": 1826, "Ñĸ": 1827, "ав": 1828, "ÙĬ": 1829, "Ġduring": 1830, "ì§Ģ": 1831, "less": 1832, "oll": 1833, "нÑĭ": 1834, "ι": 1835, "fect": 1836, "iver": 1837, "ıĦ": 1838, "ither": 1839, "ying": 1840, "Ġbegin": 1841, "×Ļ×": 1842, "ivid": 1843, "Ġç": 1844, "Ġsal": 1845, "Ġta": 1846, "Ġpot": 1847, "Ġ$": 1848, "Ġmar": 1849, "Ġclear": 1850, "Ġface": 1851, "Ġgrow": 1852, "Ġ*": 1853, "Ġinside": 1854, "Ġfriends": 1855, "Ġleave": 1856, "enn": 1857, "Ġeasy": 1858, "Ġarea": 1859, "ality": 1860, "oud": 1861, "Ġeat": 1862, "ÙĨ": 1863, "Ġpur": 1864, "orn": 1865, "Ġsaw": 1866, "Ġanswer": 1867, "Ġfront": 1868, "Ġbeaut": 1869, "¼ë": 1870, "Ġmatter": 1871, "Ġson": 1872, "ĠNew": 1873, "Ġresult": 1874, "ides": 1875, "che": 1876, "Ġfut": 1877, "ps": 1878, "Ġfocus": 1879, "Ġinteresting": 1880, "å¥": 1881, "Ġap": 1882, "\".": 1883, "Ġcreate": 1884, "оÑģ": 1885, "Ġpress": 1886, "ross": 1887, "Ġpick": 1888, "line": 1889, "Ġtook": 1890, "ĠMay": 1891, "row": 1892, "Ġich": 1893, "ĺë": 1894, "Ġref": 1895, "Ġmor": 1896, "ract": 1897, "arent": 1898, "AR": 1899, "Ġexact": 1900, "Ġspace": 1901, "work": 1902, "ни": 1903, "Ġbir": 1904, "Ġdev": 1905, "г": 1906, "Ġtold": 1907, "Ġpublic": 1908, "cially": 1909, "Ġview": 1910, "ĠHey": 1911, "med": 1912, "llo": 1913, "cc": 1914, "Ġfac": 1915, "Ġcouple": 1916, "Ġheart": 1917, "ler": 1918, "Ġready": 1919, "Ġalmost": 1920, "aring": 1921, "Ġhalf": 1922, "ĠMe": 1923, "avor": 1924, "ique": 1925, "Ġcharac": 1926, "Ġpract": 1927, "ON": 1928, "ane": 1929, "Ġil": 1930, "на": 1931, "Ġvi": 1932, "lish": 1933, "head": 1934, "Ġleast": 1935, "Ġbasically": 1936, "ased": 1937, "right": 1938, "Ġyet": 1939, "Ġtaking": 1940, "Ġcountry": 1941, "Ġwin": 1942, "Ġisn": 1943, "Ġpossible": 1944, "Ġcam": 1945, "Ġincre": 1946, "Ġpat": 1947, "Ġwanna": 1948, "Ġconsider": 1949, "Ġabs": 1950, "Ġwithin": 1951, "Ġhuman": 1952, "Ġthinking": 1953, "Ġoh": 1954, "¡ľ": 1955, "Ġqui": 1956, "ases": 1957, "Ġ0": 1958, "itely": 1959, "ä¸į": 1960, "Ġkill": 1961, "Ġmil": 1962, "Ġinvest": 1963, "ister": 1964, "Ġsuc": 1965, "ional": 1966, "elf": 1967, "Ġwhether": 1968, "Ġcontrol": 1969, "Ġagainst": 1970, "ots": 1971, "ëĭĪëĭ¤": 1972, "ior": 1973, "Ġpresent": 1974, "Ġا": 1975, "Ġwatching": 1976, "ube": 1977, "erv": 1978, "Ġnicht": 1979, "Ġgovern": 1980, "ĠThese": 1981, "Ġ:": 1982, "uit": 1983, "ugh": 1984, "Ġworks": 1985, "oo": 1986, "Ġwir": 1987, "Ġair": 1988, "ĠTe": 1989, "аз": 1990, "ision": 1991, "where": 1992, "Ġtot": 1993, "joy": 1994, "ìĭ": 1995, "Ġvol": 1996, "Ġе": 1997, "Ġclose": 1998, "ĠAd": 1999, "Ñī": 2000, "ined": 2001, "Ġuna": 2002, "Ġê·¸ë": 2003, "°ë": 2004, "orry": 2005, "Ġbro": 2006, "Ġfilm": 2007, "ift": 2008, "20": 2009, "Ġtype": 2010, "Ġhappened": 2011, "ĠAm": 2012, "Ġgirl": 2013, "ĠAre": 2014, "wards": 2015, "Ġpour": 2016, "Ġcolor": 2017, "elt": 2018, "аÑģ": 2019, "Ġsense": 2020, "lex": 2021, "ĠWith": 2022, "uss": 2023, "rib": 2024, "Ġrese": 2025, "Ġnorm": 2026, "Ġfuture": 2027, "Ġdeal": 2028, "ending": 2029, "ey": 2030, "Ġx": 2031, "ero": 2032, "ĠCl": 2033, "uk": 2034, "Ġwhatever": 2035, "selves": 2036, "Ġyoung": 2037, "ìĬ": 2038, "ĠMar": 2039, "ĠChrist": 2040, "Ġguess": 2041, "Ġperform": 2042, "Ġener": 2043, "ron": 2044, "Ġhit": 2045, "Ġwond": 2046, "Ġdirect": 2047, "ĠEvery": 2048, "Ġoften": 2049, "Ġfa": 2050, "Ġalong": 2051, "Ġclick": 2052, "ĠLook": 2053, "Ġsitu": 2054, "Ġhappy": 2055, "ead": 2056, "Ġago": 2057, "Ġenc": 2058, "Ġmyself": 2059, "Ġcover": 2060, "об": 2061, "Ġmid": 2062, "Ġcost": 2063, "Ġten": 2064, "ĠSch": 2065, "Ġexpect": 2066, "Ġwasn": 2067, "Ġstrong": 2068, "iful": 2069, "Ġopportun": 2070, "inal": 2071, "yle": 2072, "Ġshare": 2073, "Ġtrue": 2074, "Ġappro": 2075, "Ġchall": 2076, "Ġminutes": 2077, "Ġchann": 2078, "ĠëĤ": 2079, "ε": 2080, "li": 2081, "Ġmess": 2082, "ories": 2083, "pecially": 2084, "Ġwrong": 2085, "Ġyes": 2086, "ĠìĹ": 2087, "iron": 2088, "Ġallow": 2089, "Ġsubs": 2090, "Ġfore": 2091, "Ġfight": 2092, "Ġsocial": 2093, "Ġcra": 2094, "ana": 2095, "Ġaff": 2096, "Ġess": 2097, "Ġways": 2098, "Ġshort": 2099, "Ġfall": 2100, "Ġlaw": 2101, "ĠWho": 2102, "Ġenjoy": 2103, "Ġcal": 2104, "Ġaccess": 2105, "fe": 2106, "Ġnon": 2107, "Ġacross": 2108, "ery": 2109, "viously": 2110, "ĠEx": 2111, "ided": 2112, "Ġlink": 2113, "ĠPr": 2114, "Ġterms": 2115, "aces": 2116, "Ġland": 2117, "azing": 2118, "Ġ15": 2119, "Ġmult": 2120, "Ġspecial": 2121, "åĢ": 2122, "iving": 2123, "ìĿĢ": 2124, "Ġtyp": 2125, "Ġste": 2126, "ĠÄ": 2127, "Ġforward": 2128, "åı": 2129, "Ġfre": 2130, "好": 2131, "Ġresearch": 2132, "à¯į": 2133, "аÑĤ": 2134, "Ġmain": 2135, "Ġrecord": 2136, "Ġhu": 2137, "Ġdefinitely": 2138, "Ġeither": 2139, "Ġlisten": 2140, "Ġkey": 2141, "Ġmarket": 2142, "ĠÑĩÑĤо": 2143, "ization": 2144, "Ġvideos": 2145, "Ġguy": 2146, "Ġfig": 2147, "Ġstra": 2148, "ĠPl": 2149, "ully": 2150, "amos": 2151, "Ġmention": 2152, "Ġsong": 2153, "Ġintern": 2154, "ral": 2155, "urs": 2156, "Ġhon": 2157, "Ġvalue": 2158, "Ġbar": 2159, "cle": 2160, "ож": 2161, "Äĩ": 2162, "ľë": 2163, "Ġzu": 2164, "им": 2165, "ä½ł": 2166, "Ġsingle": 2167, "Ġauch": 2168, "cuss": 2169, "Ġgets": 2170, "Ġsometimes": 2171, "å¾": 2172, "amb": 2173, "mm": 2174, "cing": 2175, "Ġperfect": 2176, "ĠBl": 2177, "outh": 2178, "ìł": 2179, "Ġsci": 2180, "par": 2181, "Ġred": 2182, "Ġpost": 2183, "Ġmot": 2184, "Ġelect": 2185, "ĠEu": 2186, "itive": 2187, "ĠSome": 2188, "Ġdescri": 2189, "Ġcurrent": 2190, "és": 2191, "Ġtre": 2192, "ĠEn": 2193, "Ġmit": 2194, "EN": 2195, "Īë": 2196, "ium": 2197, "Ġheard": 2198, "Ġsimple": 2199, "lar": 2200, "Ġeverybody": 2201, "ilar": 2202, "Ġneeds": 2203, "Ġdiffic": 2204, "ĠGood": 2205, "ument": 2206, "cent": 2207, "Ġoper": 2208, "аÑĤÑĮ": 2209, "ety": 2210, "Ġblack": 2211, "Ġgiven": 2212, "ones": 2213, "Ġwel": 2214, "éĢ": 2215, "ĠìķĦ": 2216, "Ġ30": 2217, "AT": 2218, "Ġstat": 2219, "ouch": 2220, "ĠMr": 2221, "аÑĢ": 2222, "Ġsho": 2223, "Ġcond": 2224, "×Ķ": 2225, "my": 2226, "Ġchildren": 2227, "Ġeu": 2228, "ед": 2229, "ìķĦ": 2230, "tern": 2231, "Ġuh": 2232, "Ġhar": 2233, "Ġprom": 2234, "Ġpull": 2235, "rew": 2236, "Ġcompany": 2237, "Ġbeautiful": 2238, "ustom": 2239, "íķĺ": 2240, "ки": 2241, "Ġstre": 2242, "Ġamazing": 2243, "ries": 2244, "Ġsuccess": 2245, "Ġmach": 2246, "not": 2247, "Ġdiscuss": 2248, "Ġnat": 2249, "¦¬": 2250, "Ġune": 2251, "Ġdifficult": 2252, "Ġris": 2253, "ν": 2254, "Ġcamp": 2255, "Ġbuy": 2256, "ä¸Ģ": 2257, "Ġmag": 2258, "po": 2259, "ĠYour": 2260, "Ġbehind": 2261, "ica": 2262, "ın": 2263, "ĠOK": 2264, "Ġlang": 2265, "Ġwomen": 2266, "Ġenv": 2267, "Ġrece": 2268, "Ġchannel": 2269, "ially": 2270, "ule": 2271, "Ġ12": 2272, "thers": 2273, "Ġbott": 2274, "Ġreport": 2275, "ently": 2276, "fully": 2277, "The": 2278, "Ġsent": 2279, "Ġevent": 2280, "Ġenergy": 2281, "lt": 2282, "Ġwords": 2283, "arr": 2284, "dle": 2285, "Ġahead": 2286, "ards": 2287, "ر": 2288, "äºĨ": 2289, "Ġtool": 2290, "conom": 2291, "еÑģ": 2292, "Ġexactly": 2293, "Ġfavor": 2294, "Ġlow": 2295, "Ġproper": 2296, "ĠìŀĪ": 2297, "Ġ!": 2298, "Ġrelations": 2299, "Ġmas": 2300, "Ġkids": 2301, "Ġentire": 2302, "ude": 2303, "Ùħ": 2304, "ĠWhere": 2305, "Ġones": 2306, "Ġcity": 2307, "olut": 2308, "Ġsix": 2309, "ability": 2310, "ör": 2311, "ili": 2312, "ĠEs": 2313, "Ġhappens": 2314, "ains": 2315, "Ġmodel": 2316, "Ġpict": 2317, "Ġespecially": 2318, "Ġ100": 2319, "kt": 2320, "Ġsoon": 2321, "by": 2322, "rodu": 2323, "Ġann": 2324, "Ġsubscri": 2325, "ĠQu": 2326, "Ġavail": 2327, "iment": 2328, "Ġvoc": 2329, "ka": 2330, "Ġ200": 2331, "aper": 2332, "ĠInd": 2333, "Ġì§": 2334, "hor": 2335, "į°": 2336, "jor": 2337, "ил": 2338, "Ġsqu": 2339, "AU": 2340, "arning": 2341, "Ġг": 2342, "IS": 2343, "Ġл": 2344, "ей": 2345, "yes": 2346, "åħ": 2347, "ĠÐĴ": 2348, "Ġorig": 2349, "ого": 2350, "Ġasked": 2351, "ilt": 2352, "ог": 2353, "Ġcontinue": 2354, "Ġìĺ": 2355, "ram": 2356, "Ġothers": 2357, "ES": 2358, "ohn": 2359, "Ġlay": 2360, "Ġbased": 2361, "Ġpu": 2362, "Ġappe": 2363, "Ġlim": 2364, "Ġprop": 2365, "Ģë": 2366, "min": 2367, "Ġhot": 2368, "ĠLa": 2369, "Ġfast": 2370, "Ġprotect": 2371, "Ġamount": 2372, "Ġaqu": 2373, "Ġfund": 2374, "Ġcustom": 2375, "Ġcult": 2376, "Ġhands": 2377, "Ġhaven": 2378, "Ġaud": 2379, "Ġoutside": 2380, "ĠAfter": 2381, "aps": 2382, "Ġanim": 2383, "ploy": 2384, "Ġhat": 2385, "ĠFirst": 2386, "Ġtreat": 2387, "Ġep": 2388, "Ġmater": 2389, "Ġbuilding": 2390, "Ġë°": 2391, "åIJ": 2392, "ìĦľ": 2393, "za": 2394, "ughter": 2395, "ĠPe": 2396, "ney": 2397, "eter": 2398, "atic": 2399, "Ġeduc": 2400, "기": 2401, "Ġmov": 2402, "ĵ¤": 2403, "ama": 2404, "ration": 2405, "Ġsn": 2406, "ÙĪ": 2407, "Ġsum": 2408, "Ġphot": 2409, "ĠÐĿ": 2410, "Ġ.": 2411, "æľī": 2412, "Ġfinish": 2413, "itting": 2414, "å®": 2415, "Ġlarge": 2416, "Ġìĸ": 2417, "Ġwhite": 2418, "ara": 2419, "Ġmais": 2420, "ĠHi": 2421, "Ġdam": 2422, "ĠاÙĦ": 2423, "Ġbox": 2424, "ĠHello": 2425, "Ġsle": 2426, "Ġopt": 2427, "ried": 2428, "¥¼": 2429, "Ġactiv": 2430, "Ġnão": 2431, "ĠCom": 2432, "Ġplaying": 2433, "Th": 2434, "Ġavailable": 2435, "Ġport": 2436, "åĪ": 2437, "ĠAh": 2438, "Ġlas": 2439, "Ġearly": 2440, "Ġwonder": 2441, "±°": 2442, "Ġ18": 2443, "cul": 2444, "Ġfunction": 2445, "Ġmorning": 2446, "lle": 2447, "ients": 2448, "ux": 2449, "Ġcir": 2450, "itions": 2451, "Ġdeep": 2452, "Ġpolit": 2453, "yor": 2454, "mp": 2455, "aking": 2456, "Įë": 2457, "ĠMan": 2458, "Ġmillion": 2459, "Ġ/": 2460, "Ġindivid": 2461, "Ġpan": 2462, "Ġgovernment": 2463, "Ġwrite": 2464, "ĠTod": 2465, "ament": 2466, "ĠÏ": 2467, "Ġwind": 2468, "ĠEng": 2469, "chen": 2470, "Wh": 2471, "ìľ": 2472, "Ġident": 2473, "ãģ§": 2474, "vent": 2475, "urch": 2476, "Ġhy": 2477, "Ġya": 2478, "Ġtrad": 2479, "Ġrelationship": 2480, "ú": 2481, "Ġdou": 2482, "OR": 2483, "Ġswe": 2484, "Ġneg": 2485, "ination": 2486, "Ġtext": 2487, "ipp": 2488, "Ġfine": 2489, "ás": 2490, "ĠDr": 2491, "ĠCome": 2492, "Ġmonths": 2493, ",\"": 2494, "ени": 2495, "Ġhours": 2496, "Ġpod": 2497, "irt": 2498, "Ġinvol": 2499, "Ġcollect": 2500, "Ġauf": 2501, "Ġpa": 2502, "Ġhistory": 2503, "mb": 2504, "ify": 2505, "Ġ?": 2506, "Ġbelow": 2507, "asure": 2508, "aby": 2509, "Ġlangu": 2510, "Ġant": 2511, "Ġcomb": 2512, "ato": 2513, "Ġexist": 2514, "Ġëĭ": 2515, "Ġtakes": 2516, "Ġcharacter": 2517, "aff": 2518, "Ġfield": 2519, "Ġeconom": 2520, "ief": 2521, "Ġpiece": 2522, "åľ": 2523, "Ġreach": 2524, "Ġê²": 2525, "ony": 2526, "Ġmaterial": 2527, "Ġdig": 2528, "Ġphys": 2529, "Ġimpro": 2530, "Ġsimilar": 2531, "IC": 2532, "Ġnet": 2533, "yn": 2534, "Ġposition": 2535, "ÃŁ": 2536, "Ġbene": 2537, "read": 2538, "Ġlearning": 2539, "ume": 2540, "Ġclean": 2541, "ÑĤоÑĢ": 2542, "Ġcook": 2543, "Ġseems": 2544, "Ġol": 2545, "ĠUS": 2546, "ĠJes": 2547, "Ġà®": 2548, "ential": 2549, "iversity": 2550, "acy": 2551, "ĠÑı": 2552, "olutely": 2553, "rect": 2554, "ĠPlease": 2555, "Ġrepres": 2556, "Ġtouch": 2557, "men": 2558, "Ġа": 2559, "ión": 2560, "ĠThanks": 2561, "Ġang": 2562, "Ġmajor": 2563, "Ġitself": 2564, "ills": 2565, "\",": 2566, "ians": 2567, "Ġscreen": 2568, "Ġhor": 2569, "Ġknown": 2570, "Ġenviron": 2571, "Ġfinal": 2572, "Ġfigure": 2573, "ĠTw": 2574, "Ġeyes": 2575, "Ġimag": 2576, "Ġseeing": 2577, "Ġhair": 2578, "rem": 2579, "Ġapplic": 2580, "ends": 2581, "put": 2582, "Ġnews": 2583, "Ġcompletely": 2584, "ughs": 2585, "Ġknew": 2586, "ified": 2587, "ĠJe": 2588, "ĠDid": 2589, "Ġsituation": 2590, "Ġflo": 2591, "ms": 2592, "Ġphone": 2593, "Ġball": 2594, "do": 2595, "Ġparent": 2596, "Ġsorry": 2597, "ury": 2598, "ин": 2599, "ips": 2600, "ад": 2601, "Ġinstead": 2602, "Ġhuge": 2603, "Ġtu": 2604, "Ġãģ": 2605, "ĠGr": 2606, "Ġdetail": 2607, "ĠÐŁ": 2608, "Ġindividual": 2609, "Ġfire": 2610, "Ġclos": 2611, "Ġwer": 2612, "une": 2613, "Ġrunning": 2614, "Ġconvers": 2615, "Ġrecomm": 2616, "Ġcomo": 2617, "Ġsomebody": 2618, "ĠJohn": 2619, "ĠìĿ´": 2620, "ĠOur": 2621, "ples": 2622, "ĠPh": 2623, "Ġanal": 2624, "Ġ50": 2625, "Ġoffer": 2626, "Ġ<": 2627, "itional": 2628, "gest": 2629, "Ġvous": 2630, "let": 2631, "icy": 2632, "Ġfeeling": 2633, "LE": 2634, "ros": 2635, "Ġthird": 2636, "ок": 2637, "Ġseries": 2638, "ĠAny": 2639, "ised": 2640, "old": 2641, "Ġdraw": 2642, "Ġservice": 2643, "Ġcannot": 2644, "bal": 2645, "ãģĨ": 2646, "Ġliving": 2647, "ım": 2648, "Ġdifference": 2649, "Ġopportunity": 2650, "Ġnear": 2651, "orth": 2652, "ken": 2653, "Ġlocal": 2654, "ت": 2655, "ĠCon": 2656, "Ġobject": 2657, "Ġdass": 2658, "ãģĻ": 2659, "IJ×": 2660, "Ġquickly": 2661, "raph": 2662, "Ġissues": 2663, "éĢĻ": 2664, "ĠAmerican": 2665, "Ġprep": 2666, "ences": 2667, "Ġprofess": 2668, "lling": 2669, "of": 2670, "Ġfoot": 2671, "bre": 2672, "Ġusually": 2673, "Ġgeneral": 2674, "da": 2675, "ances": 2676, "Ġdest": 2677, "Ġocc": 2678, "Ġmembers": 2679, "Ġdans": 2680, "Ġequal": 2681, "zt": 2682, "Ġbecom": 2683, "Ġmoving": 2684, "Ġspecific": 2685, "ÃŃa": 2686, "Ġfur": 2687, "Ġnecess": 2688, "Ġcommon": 2689, "Ġattack": 2690, "ĠÑįÑĤо": 2691, "ĠToday": 2692, "Ġuns": 2693, "ĠGu": 2694, "iod": 2695, "Ġaccount": 2696, "Ġgrand": 2697, "Ġself": 2698, "ĠEl": 2699, "Ġtast": 2700, "Ġcontent": 2701, "Ġcu": 2702, "Ħë": 2703, "ĠMaybe": 2704, "ĠJesus": 2705, "ores": 2706, "port": 2707, "©´": 2708, "Ġgives": 2709, "Ġnormal": 2710, "ÑĢÑĥ": 2711, "Ġimpact": 2712, "är": 2713, "Ġdies": 2714, "Ġlab": 2715, "sh": 2716, "ios": 2717, "ĠPres": 2718, "ĠUnd": 2719, "ĠOf": 2720, "Ġfinally": 2721, "Ġdoll": 2722, "Ġvocê": 2723, "ply": 2724, "ĠAg": 2725, "Ġtaken": 2726, "Ġground": 2727, "fort": 2728, "Ġgave": 2729, "ĠInst": 2730, "Ġlost": 2731, "Ġworked": 2732, "Ġliter": 2733, "Ġissue": 2734, "Ġindust": 2735, "Ġreturn": 2736, "Ġhappening": 2737, "Ġwants": 2738, "ив": 2739, "Ġproblems": 2740, "ĠCar": 2741, "Ŀ¼": 2742, "ĠAlso": 2743, "Ġsize": 2744, "Ġobviously": 2745, "ĠSu": 2746, "ĠSc": 2747, "Ġrecommend": 2748, "ources": 2749, "astic": 2750, "....": 2751, "Ġmi": 2752, "lier": 2753, "ĠEven": 2754, "cia": 2755, "Ġhur": 2756, "va": 2757, "Ġmass": 2758, "Ġwouldn": 2759, "unt": 2760, "cks": 2761, "Ġfelt": 2762, "osp": 2763, "light": 2764, "олÑĮ": 2765, "nie": 2766, "Ġbottom": 2767, "ĠбÑĭ": 2768, "ored": 2769, "ison": 2770, "Ġgrad": 2771, "Ġuma": 2772, "Ġva": 2773, "ĠìĤ": 2774, "ression": 2775, "ulation": 2776, "ID": 2777, "idence": 2778, "Ġbur": 2779, "Ġgone": 2780, "lu": 2781, "ìĸ´ì": 2782, "Ġredu": 2783, "Ġja": 2784, "ìĿĺ": 2785, "ita": 2786, "Ġsoft": 2787, "Ġça": 2788, "ico": 2789, "eral": 2790, "ñ": 2791, "af": 2792, "Ġpoints": 2793, "gu": 2794, "Ġdé": 2795, "apt": 2796, "ax": 2797, "ĠAlright": 2798, "Ġcamera": 2799, "Ġach": 2800, "Ġпо": 2801, "Ġsever": 2802, "50": 2803, "Ġsie": 2804, "Ïģ": 2805, "Ġmal": 2806, "Ġcomput": 2807, "Ġmiddle": 2808, "Ġcouldn": 2809, "ming": 2810, "Ġìĭ": 2811, "ĠHis": 2812, "Ġgames": 2813, "Ġintrodu": 2814, "Ġcell": 2815, "por": 2816, "Ġsleep": 2817, "Ġë³": 2818, "iding": 2819, "Ġou": 2820, "Ġdeg": 2821, "Ġdrink": 2822, "Ġenvironment": 2823, "ĠUnited": 2824, "Ġtalked": 2825, "Ġchoose": 2826, "Ġjour": 2827, "ege": 2828, "ĠMin": 2829, "Ġinte": 2830, "Ġrather": 2831, "Ġoffic": 2832, "ка": 2833, "aching": 2834, "Ġmentioned": 2835, "Ġfill": 2836, "Ġtrack": 2837, "Ġnie": 2838, "Ġut": 2839, "ĠвÑĭ": 2840, "ibility": 2841, "Ġvac": 2842, "Ġrad": 2843, "Ġpack": 2844, "Ġsend": 2845, "ĠDas": 2846, "ĠAb": 2847, "Ġengine": 2848, "ãģĹ": 2849, "Ġcompet": 2850, "ô": 2851, "ĠвÑģ": 2852, "Ġdoor": 2853, "Ġlonger": 2854, "å°į": 2855, "Ġlanguage": 2856, "Ġextra": 2857, "play": 2858, "Ġwebs": 2859, "umb": 2860, "room": 2861, "çľ": 2862, "Ġbeginning": 2863, "Ġrefer": 2864, "AM": 2865, "nen": 2866, "igher": 2867, "face": 2868, "erc": 2869, "Ġforget": 2870, "Ġcomment": 2871, "ек": 2872, "лÑı": 2873, "ror": 2874, "że": 2875, "ĠGe": 2876, "Ġdark": 2877, "Ġanyone": 2878, "ante": 2879, "ges": 2880, "ìĬµ": 2881, "Ñij": 2882, "bed": 2883, "je": 2884, "ructure": 2885, "Ġprim": 2886, "ida": 2887, "è¦": 2888, "ãģ¾": 2889, "Ġmix": 2890, "Ġstarting": 2891, "ĠìĿ´ë": 2892, "Ġprovide": 2893, "action": 2894, "Ġmother": 2895, "Ġperiod": 2896, "Ġstick": 2897, "ĠYouT": 2898, "Ġtechnology": 2899, "ê¹": 2900, "Ġbed": 2901, "Ġgiving": 2902, "Ġexplain": 2903, "zen": 2904, "imate": 2905, "Ġrepresent": 2906, "load": 2907, "ĠHowever": 2908, "Ġlives": 2909, "uth": 2910, "irit": 2911, "ogn": 2912, "Ġlik": 2913, "Ġrespons": 2914, "Ġpriv": 2915, "Ġtom": 2916, "ção": 2917, "iam": 2918, "Ġexcited": 2919, "Ġcard": 2920, "ground": 2921, "Ġ×Ķ": 2922, "Ġsens": 2923, "Ġteach": 2924, "ido": 2925, "hod": 2926, "Ġepis": 2927, "Ġwelcome": 2928, "Ġwall": 2929, "ä¹": 2930, "Ġchance": 2931, "hen": 2932, "ĠС": 2933, "ĠÄij": 2934, "Ġsimply": 2935, "ĠÑĤак": 2936, "ring": 2937, "ja": 2938, "book": 2939, "Ġseveral": 2940, "ste": 2941, "Ġcreated": 2942, "ĠоÑĤ": 2943, "Ġpush": 2944, "==": 2945, "Ġhigher": 2946, "uf": 2947, "ource": 2948, "oke": 2949, "Ġonline": 2950, "Ġrele": 2951, "Ġton": 2952, "ensive": 2953, "Ġfavorite": 2954, "Ñĥд": 2955, "Ġlooked": 2956, "Ġvon": 2957, "âĢĶ": 2958, "Ġfür": 2959, "Ġbutton": 2960, "Ġbill": 2961, "Ġchanges": 2962, "!\"": 2963, "Ġslow": 2964, "ables": 2965, "Ġdeath": 2966, "ands": 2967, "ateg": 2968, "Ġthemselves": 2969, "ãģ£": 2970, "Ġcop": 2971, "ãģ®": 2972, "Ġpersonal": 2973, "ughing": 2974, "Ġ11": 2975, "gar": 2976, "ades": 2977, "Ġneeded": 2978, "Ġstudy": 2979, "aged": 2980, "ÑģÑĤв": 2981, "ino": 2982, "Ġdisc": 2983, "ki": 2984, "Ġaddress": 2985, "ר": 2986, "itten": 2987, "esome": 2988, "Ġж": 2989, "¤ë": 2990, "ura": 2991, "Ġmu": 2992, "Ġcontinu": 2993, "for": 2994, "Ġmatch": 2995, "ãģ¦": 2996, "Ġstraight": 2997, "IJë": 2998, "ners": 2999, "Ġdog": 3000, "Ġdeb": 3001, "ĠCO": 3002, "Ġos": 3003, "ged": 3004, "came": 3005, "Ġcorrect": 3006, "ette": 3007, "ĠSee": 3008, "Ġincluding": 3009, "ĠEuro": 3010, "ester": 3011, "Ġjump": 3012, "ĠWhich": 3013, "Ġкак": 3014, "son": 3015, "ya": 3016, "ING": 3017, "Ġeine": 3018, "osh": 3019, "ency": 3020, "Ġmedia": 3021, "Ġsubscribe": 3022, "éĤ": 3023, "Ġprin": 3024, "Ġhab": 3025, "ĠPer": 3026, "ĠWas": 3027, "Ġpage": 3028, "itor": 3029, "Ġtowards": 3030, "Ġtried": 3031, "enge": 3032, "artment": 3033, "Ġvari": 3034, "Ġpaper": 3035, "Ġpicture": 3036, "Ġversion": 3037, "Ġbrought": 3038, "ware": 3039, "ĠStates": 3040, "Ġsich": 3041, "ledge": 3042, "Ġpercent": 3043, "Ġgod": 3044, "ec": 3045, "ĠComm": 3046, "Ġdecided": 3047, "Ġselect": 3048, "íķľ": 3049, ").": 3050, "urity": 3051, "Ġfurther": 3052, "Ġcomments": 3053, "lement": 3054, "Ġdream": 3055, "Ġcenter": 3056, "mi": 3057, "Ġcas": 3058, "Ġwoman": 3059, "Ġroad": 3060, "Ġfail": 3061, "Ġbecame": 3062, "lus": 3063, "ilities": 3064, "ãģ¯": 3065, "ĠCo": 3066, "Ġmanage": 3067, "Ġrecogn": 3068, "Ġaction": 3069, "Ġbenef": 3070, "Ġearlier": 3071, "׾": 3072, "Ġspeed": 3073, "Ġment": 3074, "Ġsoci": 3075, "Ġshoot": 3076, "ui": 3077, "Ġä": 3078, "Ġapply": 3079, "vo": 3080, "xim": 3081, "Ġcause": 3082, "Ġsurpr": 3083, "Ġhaben": 3084, "DI": 3085, "Ġfather": 3086, "ĠNext": 3087, "ĠYouTube": 3088, "Ġcode": 3089, "Ġrole": 3090, "gress": 3091, "Ġgreen": 3092, "ett": 3093, "Ġbuilt": 3094, "Ġflow": 3095, "Ġbase": 3096, "Ġtraining": 3097, "Ġround": 3098, "ĠWill": 3099, "Ġpath": 3100, "ĠRo": 3101, "Ġinterested": 3102, "ìĸ´": 3103, "Ġrespect": 3104, "Ġchanged": 3105, "ission": 3106, "Ġstudent": 3107, "ograph": 3108, "Ġapproach": 3109, "Ġshows": 3110, "å°±": 3111, "Ġtar": 3112, "Ġcrit": 3113, "Ġglo": 3114, "ìĬµëĭĪëĭ¤": 3115, "Ġdead": 3116, "ĠPresident": 3117, "Ġthous": 3118, "Ġbal": 3119, "ster": 3120, "ex": 3121, "Ġabsolutely": 3122, "Ġmic": 3123, "Ġpractice": 3124, "Ġquality": 3125, "Ġlower": 3126, "ogle": 3127, "Ġsepar": 3128, "ball": 3129, "medi": 3130, "Ġreview": 3131, "ĠApp": 3132, "Ġok": 3133, "âĢĭ": 3134, "Ġexperien": 3135, "Ġconcern": 3136, "entially": 3137, "more": 3138, "ĠJo": 3139, "apan": 3140, "ĠIch": 3141, "istic": 3142, "Ġfair": 3143, "Ġwebsite": 3144, "ires": 3145, "ĠBy": 3146, "Ġtravel": 3147, "Ġrisk": 3148, "Ġmir": 3149, "Ġboard": 3150, "Ġsen": 3151, "Ġparents": 3152, "ĠWow": 3153, "Ġfeed": 3154, "Ġsave": 3155, "Ġserious": 3156, "Ġinit": 3157, "EL": 3158, "undred": 3159, "AS": 3160, "Ġvan": 3161, "orrow": 3162, "Ġworth": 3163, "Ġsearch": 3164, "Ġ16": 3165, "Ġparts": 3166, "ÑģÑĤÑĮ": 3167, "Ġcompan": 3168, "Ġmovie": 3169, "Ġmethod": 3170, "Ġill": 3171, "Ġwish": 3172, "dy": 3173, "Ġitem": 3174, "Ġminus": 3175, "anger": 3176, "Ġvoice": 3177, "Ġskin": 3178, "Ġareas": 3179, "Ġeight": 3180, "Ġobs": 3181, "Ġ,": 3182, "ай": 3183, "Ġoil": 3184, "Ġcy": 3185, "Ġbaby": 3186, "sy": 3187, "Ġemploy": 3188, "ĠKe": 3189, "Ġplaces": 3190, "Ġfix": 3191, "Ġestá": 3192, "ãģ¨": 3193, "ived": 3194, "Ġlots": 3195, "Ġseason": 3196, "unk": 3197, "alt": 3198, "Ġtable": 3199, "ĠТ": 3200, "â": 3201, "Ġattention": 3202, "ãģª": 3203, "ĠHer": 3204, "Ġage": 3205, "Ġpra": 3206, "back": 3207, "cil": 3208, "Ġnetwork": 3209, "rit": 3210, "Ġdoc": 3211, "Ġaren": 3212, "igen": 3213, "ĠëĦ": 3214, "د": 3215, "ender": 3216, "Ġtotal": 3217, "Ġprice": 3218, "Ġcrazy": 3219, "ìļ": 3220, "iqu": 3221, "though": 3222, "You": 3223, "Ùĩ": 3224, "ãĤĵ": 3225, "Ïħ": 3226, "Ġsat": 3227, "Ġbi": 3228, "ĠDie": 3229, "Ġsha": 3230, "Ġthanks": 3231, "uh": 3232, "Ġstage": 3233, "аж": 3234, "ĠFl": 3235, "Ġleav": 3236, "Ġboy": 3237, "Ġaf": 3238, "ön": 3239, "ĠGet": 3240, "Ġaccept": 3241, "Ġenter": 3242, "Ġtur": 3243, "ĠsiÄĻ": 3244, "Ġhonest": 3245, "ãĢĮ": 3246, "Ġsam": 3247, "Ġrepl": 3248, "ging": 3249, "Ġdevelopment": 3250, "ĠAct": 3251, "ora": 3252, "ãĢį": 3253, "ä¾": 3254, "Ġknows": 3255, "Ġimage": 3256, "ĠLord": 3257, "иÑĤÑĮ": 3258, "Ġweeks": 3259, "Ġsex": 3260, "Ķë": 3261, "Ġhundred": 3262, "Ġsounds": 3263, "Ġlearned": 3264, "Ġbud": 3265, "ĠÑģÑĤ": 3266, "Ġincred": 3267, "âĻ": 3268, "Ġnos": 3269, "Ġdrop": 3270, "Ġben": 3271, "ĠÐĺ": 3272, "Ġsafe": 3273, "ata": 3274, "Ġfuck": 3275, "soci": 3276, "Ġdan": 3277, "Ġcross": 3278, "10": 3279, "mo": 3280, "vert": 3281, "Ġ17": 3282, "zie": 3283, "åķ": 3284, "Ġdom": 3285, "ĠBo": 3286, "Ġsetting": 3287, "Ġinvolved": 3288, "arily": 3289, "Ġsind": 3290, "Ġsus": 3291, "Ġworry": 3292, "eth": 3293, "ê¹Į": 3294, "Ġsun": 3295, "Ġhier": 3296, "Ġcertainly": 3297, "oul": 3298, "orts": 3299, "ĠEr": 3300, "ĠUm": 3301, "Ġcaus": 3302, "Ġnatural": 3303, "Ġü": 3304, "Ġcry": 3305, "ĠSec": 3306, "Ġsom": 3307, "æ²": 3308, "Ġeducation": 3309, "аеÑĤ": 3310, "Ġmultip": 3311, "Ġalone": 3312, "Ġeye": 3313, "Ġrate": 3314, "ĠEurope": 3315, "è¿": 3316, "mon": 3317, "Ġfit": 3318, "izing": 3319, "pped": 3320, "Ġpressure": 3321, "the": 3322, "иÑģ": 3323, "ites": 3324, "ĠAf": 3325, "reci": 3326, "attle": 3327, "Ġservices": 3328, "ĠGoogle": 3329, "éģ": 3330, "Ġcases": 3331, "Ġdrive": 3332, "Ġchalleng": 3333, "uz": 3334, "ĠMo": 3335, "ìľ¼ë": 3336, "val": 3337, "åĢĭ": 3338, "Ġfol": 3339, "Ġì¢": 3340, "ffic": 3341, "Ġra": 3342, "Ġsin": 3343, "Ġblue": 3344, "Ġaffect": 3345, "Ġmis": 3346, "Ġshot": 3347, "Ġоб": 3348, "asing": 3349, "Ġsignific": 3350, "ĠChe": 3351, "Ġê³": 3352, "Ġpositive": 3353, "ì£": 3354, "Ġwie": 3355, "Ġ40": 3356, "ording": 3357, "ĠFrom": 3358, "êµ": 3359, "Ġbrand": 3360, "Ġtrust": 3361, "Ġple": 3362, "Ġcommunic": 3363, "Ġweight": 3364, "Ġasking": 3365, "Ġtax": 3366, "ĠJapan": 3367, "ãģŁ": 3368, "Ġíķĺ": 3369, "ops": 3370, "ÏĤ": 3371, "Ġputting": 3372, "Ġroll": 3373, "ĠAmerica": 3374, "reg": 3375, "ŀ×": 3376, "atures": 3377, "ension": 3378, "ĠSomet": 3379, "Ġoriginal": 3380, "ping": 3381, "ĠÅŁ": 3382, "Ġproducts": 3383, "ãĥ¼": 3384, "Ġcontact": 3385, "olution": 3386, "Ġgoal": 3387, "Ġpow": 3388, "Ġperformance": 3389, "Ġblood": 3390, "ators": 3391, "ĠMich": 3392, "Ġtemper": 3393, "ĠDan": 3394, "Ġsugg": 3395, "ÑĤи": 3396, "Ġimm": 3397, "Ġoffice": 3398, "Ġarri": 3399, "Ġcomfort": 3400, "ĠÐĶ": 3401, "Ġsuggest": 3402, "Ġplat": 3403, "Ĥĺ": 3404, "19": 3405, "Ġom": 3406, "Ġseven": 3407, "ĠCent": 3408, "ille": 3409, "Ġconcept": 3410, "Ġbag": 3411, "ün": 3412, "ively": 3413, "Ġdiv": 3414, "mos": 3415, "æī": 3416, "Ġfeels": 3417, "Ġir": 3418, "akes": 3419, "ley": 3420, "Ġparticip": 3421, "ĠÐļ": 3422, "fl": 3423, "just": 3424, "Ġsil": 3425, "ĠPa": 3426, "AL": 3427, "Ġgotta": 3428, "Ġfan": 3429, "Ġchallenge": 3430, "Ġcompanies": 3431, "ĠPeople": 3432, "": 12331, "Ġheroes": 12332, "ĠBoston": 12333, "Ġdependent": 12334, "Ġmotivation": 12335, "flix": 12336, "Ġseam": 12337, "кие": 12338, "Ġdrain": 12339, "oded": 12340, "Ġguilty": 12341, "ĠJenn": 12342, "ingen": 12343, "Ġgranted": 12344, "ĠKelly": 12345, "ĠSav": 12346, "ĠUncle": 12347, "ĠHonestly": 12348, "ELI": 12349, "Ġnavigate": 12350, "Ġblessed": 12351, "core": 12352, "Ġearning": 12353, "Ġsignals": 12354, "Ġdisk": 12355, "ials": 12356, "Ġages": 12357, "æħ": 12358, "Ġparticle": 12359, "ĠÑĩеÑĢ": 12360, "Ġcann": 12361, "Ġtier": 12362, "Ġstatements": 12363, "ê³łìļĶ": 12364, "ĠëķĮ문ìĹIJ": 12365, "ĠCho": 12366, "Ġpolar": 12367, "anç": 12368, "ĠKenn": 12369, "ĠNi": 12370, "ĠFight": 12371, "organ": 12372, "éķ": 12373, "ĠCha": 12374, "ĠSÃŃ": 12375, "ãĥª": 12376, "Ġslic": 12377, "Ġcertific": 12378, "Ġtemplate": 12379, "ĠFederal": 12380, "Ġconsideration": 12381, "Ġexplo": 12382, "ĠMain": 12383, "ĠNE": 12384, "Ġalongside": 12385, "Ġdressed": 12386, "ĠPoint": 12387, "Ġenvironments": 12388, "Ġpróxim": 12389, "Ġdaar": 12390, "Ġprompt": 12391, "Ġpursue": 12392, "Ġentertainment": 12393, "Ġthroat": 12394, "Ġproblema": 12395, "Ġmart": 12396, "ì¼": 12397, "Ġprovider": 12398, "ØĮ": 12399, "Ġ×Ĺ": 12400, "inte": 12401, "making": 12402, "Ġstroke": 12403, "Ġtissue": 12404, "Un": 12405, "Ġprecious": 12406, "ĠArts": 12407, "inking": 12408, "ĠÐŀн": 12409, "ĠиÑģ": 12410, "nah": 12411, "ĠÐķÑģли": 12412, "Ġcorners": 12413, "Ġtricky": 12414, "inch": 12415, "lijk": 12416, "Ġpressing": 12417, "level": 12418, "ANG": 12419, "Ġradiation": 12420, "ìĦł": 12421, "Ġconfront": 12422, "Ġvet": 12423, "Ġrepresentative": 12424, "Ġpropag": 12425, "Ġcrap": 12426, "ĠDec": 12427, "Ġramp": 12428, "епеÑĢÑĮ": 12429, "ués": 12430, "essen": 12431, "cription": 12432, "Ġbills": 12433, "ĠMatthew": 12434, "Ġanime": 12435, "ất": 12436, "Ġlowest": 12437, "has": 12438, "screen": 12439, "ograp": 12440, "ало": 12441, "inton": 12442, "ĠJah": 12443, "èĢħ": 12444, "itÃł": 12445, "Ġkay": 12446, "Ġrotation": 12447, "ĠWere": 12448, "abei": 12449, "Ġtrials": 12450, "Ġlever": 12451, "ighty": 12452, "Ġspoon": 12453, "Ġhunt": 12454, "cling": 12455, "Ġdism": 12456, "ĠболÑĮÑĪ": 12457, "Ġassault": 12458, "Ġíĺķ": 12459, "Ġweekly": 12460, "Ġmismo": 12461, "Ġgenetic": 12462, "ulpt": 12463, "ĠStudent": 12464, "Ġrealistic": 12465, "Ġauthentic": 12466, "æīĵ": 12467, "asta": 12468, "Ġarrested": 12469, "Ġguidelines": 12470, "Ġ׾×IJ": 12471, "Ġдав": 12472, "ĠComing": 12473, "für": 12474, "Ġrequests": 12475, "ĥIJ": 12476, "Ġanalyze": 12477, "Ġinteress": 12478, "Ġhalt": 12479, "ĠOper": 12480, "onom": 12481, "Ġduck": 12482, "Ġwithd": 12483, "ser": 12484, "ĠÏĮ": 12485, "ĠHistory": 12486, "Ġyoutube": 12487, "ãĤį": 12488, "Ġsaber": 12489, "walk": 12490, "font": 12491, "Ġoverview": 12492, "39": 12493, "üy": 12494, "etti": 12495, "Ġfrozen": 12496, "Ġflesh": 12497, "ÄŁi": 12498, "ĠPM": 12499, "ĠìĻĢ": 12500, "é¢": 12501, "ÑĨии": 12502, "Ġ기ë": 12503, "íģ¬": 12504, "Ġprose": 12505, "oooo": 12506, "rates": 12507, "WS": 12508, "Ġautomatic": 12509, "Ġcollecting": 12510, "Åij": 12511, "Ġneighbors": 12512, "».": 12513, "ĠExpl": 12514, "Ġcircul": 12515, "cover": 12516, "weg": 12517, "Ġsticks": 12518, "Ġeller": 12519, "Ġwww": 12520, "Ġdorm": 12521, "ĠExper": 12522, "Ġstatistics": 12523, "Ġemails": 12524, "Ġgrave": 12525, "imiz": 12526, "HS": 12527, "Ġuit": 12528, ",'": 12529, "Ġlaser": 12530, "èī": 12531, "ĠÑĤем": 12532, "ÑĭÑĪ": 12533, "ÑīÑij": 12534, "Ġgenau": 12535, "Ġtienen": 12536, "Ġmeditation": 12537, "ĠOrgan": 12538, "Ġestimate": 12539, "Ġ무ì": 12540, "lets": 12541, "ĠnÃły": 12542, "Ġmindset": 12543, "Ġreson": 12544, "Ġmés": 12545, "Ġnumerous": 12546, "Ġvielleicht": 12547, "ĠThird": 12548, "uous": 12549, "ĠDead": 12550, "анд": 12551, "HN": 12552, "Ġracing": 12553, "Ġagents": 12554, "ĠUt": 12555, "Ġtear": 12556, "ĠHP": 12557, "Ġchemistry": 12558, "Ġsurvival": 12559, "æĸ°": 12560, "Ġconvinced": 12561, "Ġ;": 12562, "Ġregulations": 12563, "ĠES": 12564, "åĴĮ": 12565, "300": 12566, "Ġense": 12567, "Ġìµ": 12568, "Ġdict": 12569, "GA": 12570, "ĠahÃŃ": 12571, "åĭķ": 12572, "Ġtej": 12573, "ĠоÑģÑĤ": 12574, "ĠElect": 12575, "Ġintellectual": 12576, "Ġbias": 12577, "Ġburden": 12578, "çĤ¹": 12579, "Ġìĸ´ëĸ»": 12580, "Ġcheer": 12581, "Ġsoph": 12582, "Ġportfolio": 12583, "uba": 12584, "Ġestos": 12585, "TV": 12586, "For": 12587, "Ġash": 12588, "Ġkommer": 12589, "Ġcollective": 12590, "Ġwrest": 12591, "ĠJetzt": 12592, "ĠWat": 12593, "reich": 12594, "Ġprimer": 12595, "active": 12596, "Ġmie": 12597, "icked": 12598, "Ġhunting": 12599, "Ġtestim": 12600, "Ġcompassion": 12601, "Ġر": 12602, "Ġbrut": 12603, "Ġsalad": 12604, "обÑīе": 12605, "Ġsolving": 12606, "Ġfloating": 12607, "ç·": 12608, "Ġattractive": 12609, "ÙĪÙĦ": 12610, "Ġperd": 12611, "iffer": 12612, "Ġsculpt": 12613, "hhh": 12614, "ĠWeek": 12615, "Ġenthus": 12616, "Ġnad": 12617, "Ġmerch": 12618, "ĠíĻķ": 12619, "Ġmile": 12620, "好äºĨ": 12621, "Ġθ": 12622, "ĠëĤĺë": 12623, "éĩį": 12624, "38": 12625, "Ġchains": 12626, "ĠAlmost": 12627, "Ġtickets": 12628, "rin": 12629, "ĠCC": 12630, "Ġdistributed": 12631, "abetes": 12632, "Ġtemperatures": 12633, "Ġgained": 12634, "Ġflexibility": 12635, "Ġscreaming": 12636, "Ġabroad": 12637, "uno": 12638, "Ġentrepreneurs": 12639, "ĠNetwork": 12640, "ĠCanadian": 12641, "Ġprev": 12642, "Ġsö": 12643, "ĠÑĤебÑı": 12644, "ĠPoke": 12645, "ĠPod": 12646, "ĠTurkey": 12647, "çı¾åľ¨": 12648, "Ġabstract": 12649, "Ġsnake": 12650, "ĠAmy": 12651, "ĠëĬIJëĤĮ": 12652, "Ġbrave": 12653, "ĠìŀĪìĸ´ìļĶ": 12654, "ĠKal": 12655, "Ġ2007": 12656, "ário": 12657, "Ġmarked": 12658, "gines": 12659, "Ġalloc": 12660, "ONG": 12661, "Ġscientist": 12662, "Ġesca": 12663, "Ġracism": 12664, "×ij×": 12665, "ĠSams": 12666, "ĠPenn": 12667, "Ġloads": 12668, "Ġந": 12669, "über": 12670, "Me": 12671, "ixò": 12672, "Ġperò": 12673, "anne": 12674, "Ġexpressed": 12675, "меÑĢ": 12676, "Ġmoet": 12677, "Ġreturning": 12678, "nia": 12679, "Ġexpon": 12680, "Pro": 12681, "Ġloyal": 12682, "ML": 12683, "Ġlamp": 12684, "Ġshy": 12685, "Ġcomposition": 12686, "ĠLy": 12687, "Ġmagnetic": 12688, "Ġpremier": 12689, "Ġmeasured": 12690, "Ġsummary": 12691, "Ġattacked": 12692, "Ġfinishing": 12693, "ÐĹ": 12694, "ç¥": 12695, "Ġsits": 12696, "Ġhydrogen": 12697, "Ġmai": 12698, "ĠDeutsch": 12699, "ası": 12700, "Ġobtain": 12701, "vie": 12702, "Ġsoit": 12703, "Ġë°Ķ": 12704, "Ġlane": 12705, "Ġconsegu": 12706, "во": 12707, "Ġease": 12708, "akin": 12709, "ĠFa": 12710, "Ġuntuk": 12711, "Ġburst": 12712, "Ġcum": 12713, "alım": 12714, "úblic": 12715, "idi": 12716, "ĠRoyal": 12717, "ĠKon": 12718, "Ġcommonly": 12719, "Ġremoving": 12720, "Ġjur": 12721, "ilib": 12722, "Ġanch": 12723, "íĸī": 12724, "ượ": 12725, "ĠÐľÑĭ": 12726, "ĠAnth": 12727, "ĠSÃ¥": 12728, "Ġinterrupt": 12729, "Ġstere": 12730, "ĠOS": 12731, "onym": 12732, "tery": 12733, "ĠMaria": 12734, "ê²ĥ": 12735, "Ġexploring": 12736, "Ġtransparent": 12737, "Ġfate": 12738, "ĠJung": 12739, "Ġgrup": 12740, "Ġdarker": 12741, "ĠDoug": 12742, "Ġmane": 12743, "æĶ¾": 12744, "ại": 12745, "dri": 12746, "look": 12747, "ĠDesign": 12748, "Ġtutaj": 12749, "Ġhorizontal": 12750, "reon": 12751, "orte": 12752, "ĠCorrect": 12753, "ĠSteven": 12754, "Ġvine": 12755, "02": 12756, "iÄĩ": 12757, "Ġsiempre": 12758, "ĠKey": 12759, "åĥı": 12760, "ĠGames": 12761, "Ġnaar": 12762, "Ġshocked": 12763, "elve": 12764, "ĠRose": 12765, "ìĭ¬": 12766, "Ġstopping": 12767, "ohl": 12768, "ĠMix": 12769, "Ġsuffered": 12770, "Ġsigma": 12771, "Ġweakness": 12772, "ĠOw": 12773, "ีà¹Ī": 12774, "IF": 12775, "Ġà®ħ": 12776, "aded": 12777, "ĠNetflix": 12778, "anes": 12779, "Ġremained": 12780, "iry": 12781, "Ġrip": 12782, "ellt": 12783, "Ġsilent": 12784, "Ġproven": 12785, "Ġtoxic": 12786, "Ġalumin": 12787, "Ġmultipl": 12788, "aland": 12789, "Ġ34": 12790, "06": 12791, "ĠBru": 12792, "Ġìłķë§IJ": 12793, "Just": 12794, "boy": 12795, "Ġshoe": 12796, "Ġcreature": 12797, "Ġheaded": 12798, "ĠоÑĤк": 12799, "æ±": 12800, "Ġessence": 12801, "Ġremarkable": 12802, "Ġnúmer": 12803, "Ġdrew": 12804, "Ġpuzzle": 12805, "ĠLibrary": 12806, "ĠFu": 12807, "ashes": 12808, "kk": 12809, "ĠIst": 12810, "¦°": 12811, "ĠBry": 12812, "Ġceremony": 12813, "Ġà®İ": 12814, "Ġcri": 12815, "equ": 12816, "ãĤ¢": 12817, "Ġprize": 12818, "Ġdimensions": 12819, "ogram": 12820, "Ġleather": 12821, "Ġpopulations": 12822, "uum": 12823, "Ġvegan": 12824, "Ñıд": 12825, "Ġcómo": 12826, "åĦ": 12827, "Ġstrip": 12828, "å£": 12829, "Ġvacation": 12830, "ħķ": 12831, "Ġmeals": 12832, "ilipp": 12833, "Ġents": 12834, "aram": 12835, "richt": 12836, "Ġgrain": 12837, "ĠSpain": 12838, "Ġcheek": 12839, "ĠAff": 12840, "ION": 12841, "ĠBring": 12842, "Ġ38": 12843, "ielen": 12844, "ulu": 12845, "ĠболÑĮÑĪе": 12846, "Ġannouncement": 12847, "ĠÑĤÑĥÑĤ": 12848, "ĠProphet": 12849, "ardo": 12850, "37": 12851, "Ġwoke": 12852, "Ġtranslation": 12853, "ĠNOT": 12854, "ĠCL": 12855, "ĠdÃ¼ÅŁ": 12856, "ÑĨÑĸ": 12857, "acer": 12858, "ĠLoc": 12859, "Ġperception": 12860, "NO": 12861, "Ġdiesen": 12862, "Look": 12863, "heart": 12864, "aved": 12865, "Ġboundary": 12866, "Ġflows": 12867, "Ñijм": 12868, "Ġarguments": 12869, "Ġelections": 12870, "ıs": 12871, "Ġheck": 12872, "Ġsuitable": 12873, "Ġfiber": 12874, "ĠStra": 12875, "xy": 12876, "ĠHum": 12877, "Ġmonthly": 12878, "uper": 12879, "Ġgolf": 12880, "Ġlately": 12881, "ĠGard": 12882, "ĠRen": 12883, "ĠAst": 12884, "ĠFant": 12885, "аÑģÑģ": 12886, "Ġobser": 12887, "ë¡ľ": 12888, "Ġeasiest": 12889, "įĶë": 12890, "Ġwebsites": 12891, "pol": 12892, "Ġcocon": 12893, "Ġà®ĩ": 12894, "ĠVeg": 12895, "Ġwalks": 12896, "Ġintro": 12897, "Ġdirected": 12898, "ĠAnna": 12899, "Ġëĵ¤ìĸ´": 12900, "ĠEastern": 12901, "ĠSaint": 12902, "ĠBow": 12903, "Ġroast": 12904, "ĠURL": 12905, "Ġjeden": 12906, "uras": 12907, "aja": 12908, "Ġsemi": 12909, "Ġrapidly": 12910, "Ġtargets": 12911, "ĠControl": 12912, "Ġbah": 12913, "Ġreflection": 12914, "Ġcreativity": 12915, "holders": 12916, "Ġìĺ¬ë": 12917, "Ġamongst": 12918, "Ġfeeding": 12919, "ÑįÑĤомÑĥ": 12920, "Ġвиде": 12921, "Ġë§Įëĵ¤": 12922, "ĠSmart": 12923, "Ġreliable": 12924, "Ġvezes": 12925, "Ġר": 12926, "chuckles": 12927, "azione": 12928, "ĠWilliams": 12929, "Ġaç": 12930, "Ġslee": 12931, "еÑī": 12932, "Ġtimeline": 12933, "Ġthorough": 12934, "á»į": 12935, "ĠOt": 12936, "ạn": 12937, "Ġimagination": 12938, "Ġmechanics": 12939, "rist": 12940, "Ġclaimed": 12941, "ÏĦη": 12942, "ête": 12943, "ĠHurry": 12944, "ĠiPad": 12945, "Ġconstru": 12946, "ĠCla": 12947, "ĠAls": 12948, "ä¼ļ": 12949, "utz": 12950, "Ġcultures": 12951, "Ġìĸ´ëĸ»ê²Į": 12952, "Ġbelongs": 12953, "Ġyer": 12954, "ĠDoesn": 12955, "Ġgeomet": 12956, "Ġbid": 12957, "Ġfoam": 12958, "Ġhob": 12959, "ĠBritain": 12960, "Ġsubstance": 12961, "Ġanniversary": 12962, "ĠëĦĪ": 12963, "Ġnoted": 12964, "Ġgovernor": 12965, "Ġstocks": 12966, "31": 12967, "Ġdiye": 12968, "ìĬ¤ë": 12969, "Ġreb": 12970, "zel": 12971, "Ġmultiply": 12972, "Ġoperator": 12973, "Ħ¤ìļĶ": 12974, "Ġwaters": 12975, "Ġdär": 12976, "Ġunser": 12977, "ĠElizabeth": 12978, "é«ĺ": 12979, "Ġincreasingly": 12980, "ĠGro": 12981, "Ġengines": 12982, "irs": 12983, "Ø«": 12984, "Ġtreasure": 12985, "PC": 12986, "inction": 12987, "iri": 12988, "Ġaccum": 12989, "Ġvariation": 12990, "Ġpom": 12991, "Ġtitles": 12992, "ĠFest": 12993, "ós": 12994, "Ġelder": 12995, "nym": 12996, "run": 12997, "Ñıв": 12998, "Ġinnovative": 12999, "Ġnombre": 13000, "Ġcoinc": 13001, "Ġfranch": 13002, "Ġentonces": 13003, "Ġnichts": 13004, "Ġexclusive": 13005, "ĠCheers": 13006, "ĠBi": 13007, "uje": 13008, "æŃ¡": 13009, "Ġpok": 13010, "ĠPrem": 13011, "Ġrocket": 13012, "ELIPE": 13013, "Ġhospitals": 13014, "rium": 13015, "Ġjuste": 13016, "Ġhammer": 13017, "Ġquantum": 13018, "Ġresponses": 13019, "lly": 13020, "endi": 13021, "Ġactively": 13022, "Ġfridge": 13023, "iate": 13024, "long": 13025, "Ġquem": 13026, "Ġdeaths": 13027, "Ġsuperior": 13028, "cken": 13029, "ìĿ´ìĹIJ": 13030, "ktop": 13031, "Ġgathered": 13032, "£¨": 13033, "Ġdazu": 13034, "Ġrecipes": 13035, "Ġbuzz": 13036, "cen": 13037, "Ġanytime": 13038, "onsense": 13039, "Ġcircles": 13040, "Ġsolved": 13041, "Ġìĭł": 13042, "Ġcoronavirus": 13043, "ĠLuke": 13044, "Ġbubb": 13045, "Ġcontempor": 13046, "rzy": 13047, "ĠJane": 13048, "Ġдом": 13049, "Ġscrews": 13050, "Ġhybrid": 13051, "Ġcasual": 13052, "Ġselbst": 13053, "being": 13054, "ĠÄIJ": 13055, "ĠColumb": 13056, "ĠÑħоÑĩ": 13057, "Ġbucket": 13058, "Ġevaluate": 13059, "Ġidol": 13060, "Ġreputation": 13061, "ĠìĨĮë": 13062, "ÙĪر": 13063, "Ġhecho": 13064, "Ġpoem": 13065, "Ġsubjects": 13066, "plant": 13067, "ĠBeh": 13068, "ĠSpeaking": 13069, "Ġbatteries": 13070, "Ġfollowers": 13071, "öl": 13072, "Ġgently": 13073, "Ġsixt": 13074, "Ġparameter": 13075, "Ġikke": 13076, "ĠTour": 13077, "ĠDJ": 13078, "otte": 13079, "ĠJahren": 13080, "Ġpreparation": 13081, "ĠдÑĥм": 13082, "Ġ800": 13083, "cop": 13084, "iking": 13085, "Ġ문": 13086, "ĠнÑĥ": 13087, "ĠлеÑĤ": 13088, "åIJĮ": 13089, "ĠIde": 13090, "Ġì¡°ê¸Ī": 13091, "Ġlaughter": 13092, "Ġmolecules": 13093, "ĠRest": 13094, "Ġobserved": 13095, "dzie": 13096, "Ġadvertising": 13097, "erto": 13098, "Ġmoins": 13099, "ĠMIT": 13100, "Ġexcit": 13101, "Ġtum": 13102, "Ġtyl": 13103, "Ġinvested": 13104, "Ġpharm": 13105, "Ġunexpected": 13106, "Ġphi": 13107, "otype": 13108, "weise": 13109, "Ġgeç": 13110, "jourd": 13111, "Ġhorses": 13112, "nÄħ": 13113, "=\"": 13114, "ĠSM": 13115, "Ġfib": 13116, "Ġclips": 13117, "çķ¶": 13118, "å¦Ĥæŀľ": 13119, "Ġregime": 13120, "Ġrotate": 13121, "rou": 13122, "nik": 13123, "Ġarmor": 13124, "ðŁĺ": 13125, "еÑĢа": 13126, "度": 13127, "ĠOch": 13128, "Ġrichtig": 13129, "üzel": 13130, "aneously": 13131, "mek": 13132, "éĮ¯": 13133, "ĠXiao": 13134, "Ġexisted": 13135, "worth": 13136, "ãģ£ãģ¨": 13137, "Ġnaught": 13138, "ĠheiÃŁt": 13139, "ĠBal": 13140, "Ġresid": 13141, "ivot": 13142, "omatic": 13143, "Ġhired": 13144, "Ġgradually": 13145, "Ġonions": 13146, "Ġcompat": 13147, "Ġintim": 13148, "Ġjew": 13149, "Ġcontribution": 13150, "ĠIre": 13151, "acji": 13152, "Ġslice": 13153, "Ġimmun": 13154, "ĠRus": 13155, "Ġgrows": 13156, "ĠSimilarly": 13157, "Ġhardest": 13158, "Ġstruck": 13159, "Ġmeasurement": 13160, "...]": 13161, "they": 13162, "ĠìłĢë": 13163, "Ġsneak": 13164, "Ġapplies": 13165, "Ġнем": 13166, "æĵ": 13167, "×ijר": 13168, "ĠЧÑĤо": 13169, "Ġoutro": 13170, "Ġinnocent": 13171, "Ġmog": 13172, "ĠSamsung": 13173, "Ġmercy": 13174, "Ġhandling": 13175, "Ġintervention": 13176, "idays": 13177, "got": 13178, "Ġcurric": 13179, "Ġboundaries": 13180, "Ġconfusing": 13181, "Ŀ¼ëĬĶ": 13182, "æĩ": 13183, "Ġstitches": 13184, "ÃŃvel": 13185, "Ġtunnel": 13186, "itä": 13187, "Ġgost": 13188, "imy": 13189, "Ġczas": 13190, "Ġmé": 13191, "Ġcatal": 13192, "ĠSimon": 13193, "ĠLIAM": 13194, "mic": 13195, "ĠФ": 13196, "Ġeyel": 13197, "isas": 13198, "ĠCPU": 13199, "ĠDou": 13200, "Ġnäch": 13201, "Ġinfinity": 13202, "Ġrif": 13203, "ĠPeace": 13204, "ĠCu": 13205, "Ġminimal": 13206, "Ġlistened": 13207, "Ġpole": 13208, "halb": 13209, "Ġloaded": 13210, "Ġsteady": 13211, "ĠBesides": 13212, "êm": 13213, "Ġlap": 13214, "Ġcoop": 13215, "Ġfriendship": 13216, "world": 13217, "Ġgeh": 13218, "Ġtylko": 13219, "ĠLaura": 13220, "Ġsurrounded": 13221, "ĠEvent": 13222, "Ġchap": 13223, "ĠWonder": 13224, "break": 13225, "Ġdrove": 13226, "Ġbroader": 13227, "Ġchi": 13228, "Fi": 13229, "Ġgehen": 13230, "Ġwestern": 13231, "Ġintelligent": 13232, "Ġpersist": 13233, "Ġfounded": 13234, "ãģĵãģ¨": 13235, "Ġhistoric": 13236, "ĠfrÃ¥": 13237, "cksÃ¥": 13238, "Ġhandy": 13239, "Ġsymp": 13240, "Ġrows": 13241, "Ġnutri": 13242, "bur": 13243, "ĠLeon": 13244, "Ġsistema": 13245, "Ġextensive": 13246, "ĠÑĥв": 13247, "íı": 13248, "Ġnights": 13249, "Ġcác": 13250, "Ġcounting": 13251, "ĠMust": 13252, "allow": 13253, "еÑģÑģ": 13254, "Mom": 13255, "Ġнадо": 13256, "Ġbarrel": 13257, "ãĥŀ": 13258, "ARD": 13259, "Ġinstallation": 13260, "Ġinsect": 13261, "Ġëħ¸ë": 13262, "ujÄħ": 13263, "ĠÄiji": 13264, "Ġpacked": 13265, "Ġfiction": 13266, "Now": 13267, "ĠYay": 13268, "Ġpert": 13269, "rons": 13270, "unde": 13271, "aches": 13272, "Ġstyles": 13273, "Ġaprès": 13274, "oku": 13275, "ĠVice": 13276, "ınız": 13277, "comm": 13278, "Ġassigned": 13279, "Ġinteractions": 13280, "Ġacab": 13281, "FELIPE": 13282, "Ġrescue": 13283, "Ġindustries": 13284, "ĠAndy": 13285, "Ġpraise": 13286, "Ġflame": 13287, "Ġsnack": 13288, "íĤ": 13289, "çģ": 13290, "Ġswo": 13291, "render": 13292, "Ġboards": 13293, "ĠÑĤом": 13294, "enne": 13295, "Ġpasta": 13296, "Ġdevil": 13297, "ĠFel": 13298, "Ġhatte": 13299, "Ġcolleg": 13300, "eh": 13301, "ì»": 13302, "ãģĵãģ®": 13303, "Ġproductive": 13304, "forward": 13305, "ип": 13306, "Ġsmartphone": 13307, "Ġinvis": 13308, "Ġbum": 13309, "Ġwhoa": 13310, "ìŀĦ": 13311, "ĠocksÃ¥": 13312, "ĠLang": 13313, "ĠSyria": 13314, "Ġsesi": 13315, "ία": 13316, "Ġapproval": 13317, "48": 13318, "Ġодин": 13319, "Ġëĸ": 13320, "ĠHarr": 13321, "ĠAdminist": 13322, "Ġפ": 13323, "ĠDean": 13324, "fi": 13325, "Ġcitizen": 13326, "Ġshark": 13327, "05": 13328, "Ġboil": 13329, "Ġindicate": 13330, "å¡": 13331, "Are": 13332, "Ġlayout": 13333, "Ġrefr": 13334, "ĠPacific": 13335, "AAAA": 13336, "ĠAustralian": 13337, "gression": 13338, "Voice": 13339, "алÑģÑı": 13340, "Ġshelter": 13341, "To": 13342, "aupt": 13343, "Ġevaluation": 13344, "apor": 13345, "Ġcurrency": 13346, "Ġмного": 13347, "igos": 13348, "ãģ°": 13349, "Ġoct": 13350, "Ġroyal": 13351, "è³": 13352, "asil": 13353, "ĠChildren": 13354, "Ġrien": 13355, "Ġëĵľë": 13356, "Ġbarrier": 13357, "Ġejemplo": 13358, "Ġek": 13359, "ND": 13360, "esp": 13361, "ена": 13362, "Ġpic": 13363, "Ġkiller": 13364, "Ġintegrate": 13365, "Ġfewer": 13366, "Ġdisabilities": 13367, "Ġ....": 13368, "Ġtriangle": 13369, "Ġfees": 13370, "Ġwidely": 13371, "emi": 13372, "Ġoverwhelming": 13373, "Ġzomb": 13374, "Ġbere": 13375, "Ġhood": 13376, "ĠAye": 13377, "ĠHarvard": 13378, "ev": 13379, "ĠÏĦοÏħ": 13380, "Ġcups": 13381, "ĠAuch": 13382, "zona": 13383, "Ġ1990": 13384, "ĠweiÃŁ": 13385, "Ġcrunch": 13386, "æ¥": 13387, "Ġзав": 13388, "Ġmeasuring": 13389, "Ġstations": 13390, "ĠStephen": 13391, "Ġshortly": 13392, "Ġsigning": 13393, "Ġcomedy": 13394, "omo": 13395, "Ġsuggestions": 13396, "Ġsignature": 13397, "ĠпÑĢив": 13398, "Ġdisorder": 13399, "aska": 13400, "Ġworlds": 13401, "Ġprecisely": 13402, "norm": 13403, "rav": 13404, "ĠCivil": 13405, "Inter": 13406, "ĠCertain": 13407, "Ġinjured": 13408, "Ġsuggests": 13409, "ĠGolden": 13410, "Ġcyber": 13411, "ĠØ´": 13412, "Ġtemporary": 13413, "Ġcooper": 13414, "Ġvoted": 13415, "Ġought": 13416, "ấy": 13417, "xual": 13418, "Ġpanels": 13419, "Ġ95": 13420, "Ġhandsome": 13421, "ĠпÑĢов": 13422, "Ġpermit": 13423, "Ġkein": 13424, "Ġbadly": 13425, "Ġnotifications": 13426, "iza": 13427, "ĠNotice": 13428, "Ġinclusive": 13429, "Ġanswering": 13430, "ĠíĹ": 13431, "uld": 13432, "íħĮ": 13433, "Ġnowadays": 13434, "Ġ37": 13435, "Ġbolt": 13436, "Ġstatic": 13437, "ĠHop": 13438, "Ġavant": 13439, "ajo": 13440, "Ġ맼ìŀĪ": 13441, "Ġfifty": 13442, "ĠFinal": 13443, "Ġscores": 13444, "ĠTap": 13445, "Ġcyl": 13446, "Ġconvince": 13447, "Ġanyways": 13448, "oda": 13449, "Ġìķ¼": 13450, "Ġserves": 13451, "ĠÑĤакой": 13452, "ĠZoom": 13453, "Ġsavings": 13454, "ulo": 13455, "Ġsouthern": 13456, "viewer": 13457, "Ġhoje": 13458, "Ġseja": 13459, "Ġrepresenting": 13460, "Īëįĺ": 13461, "lik": 13462, "ĠSomebody": 13463, "Ġbeast": 13464, "Ġsticking": 13465, "Ġinsist": 13466, "Ġtalented": 13467, "Ġexplaining": 13468, "Ġattorney": 13469, "éĥ¨": 13470, "Ġstairs": 13471, "ĠDog": 13472, "íĭ": 13473, "Ġcig": 13474, "Ġshaped": 13475, "Ġsons": 13476, "Ïģι": 13477, "utt": 13478, "ĠìĶ": 13479, "Ġparad": 13480, "ìĿ¸ëį°": 13481, "Ġhorn": 13482, "ĠJour": 13483, "anno": 13484, "Ġworldwide": 13485, "åĬĽ": 13486, "Ġparticipation": 13487, "¦Ħ": 13488, "Ġmów": 13489, "Ġburned": 13490, "Ġwriters": 13491, "allah": 13492, "ĠFund": 13493, "Ġclever": 13494, "ĠLeute": 13495, "bin": 13496, "Ġbeating": 13497, "foot": 13498, "ĠìĽIJ": 13499, "ĠStudio": 13500, "Ġvag": 13501, "bey": 13502, "rze": 13503, "Ġopposition": 13504, "Ġжиз": 13505, "who": 13506, "Ġê±´": 13507, "Ġtrace": 13508, "ĠденÑĮ": 13509, "Ġepid": 13510, "Ġgesch": 13511, "ĠNar": 13512, "ĠBE": 13513, "Ñĥй": 13514, "ĠSign": 13515, "edly": 13516, "Ġclay": 13517, "Ġinstantly": 13518, "Ġgathering": 13519, "ĠGalaxy": 13520, "Ġbored": 13521, "ĠBuddh": 13522, "cé": 13523, "Ġmam": 13524, "Ġslope": 13525, "Ġëĭ¤ìĿĮ": 13526, "Ġschön": 13527, "Ġpir": 13528, "gef": 13529, "amer": 13530, "Ġhö": 13531, "Ġcolleague": 13532, "Ġpresents": 13533, "adium": 13534, "Ġவ": 13535, "Ġfalar": 13536, "beep": 13537, "Ġdried": 13538, "isms": 13539, "Ġrope": 13540, "Ġworkshop": 13541, "Ġestud": 13542, "Ġbands": 13543, "Ġthemes": 13544, "åħ¬": 13545, "ÙĬر": 13546, "åIJİ": 13547, "Ġreminder": 13548, "ÑĤÑĥ": 13549, "ĠBh": 13550, "Ġcoconut": 13551, "ĠÑģÑĤо": 13552, "ĠChannel": 13553, "Ġimmigration": 13554, "äs": 13555, ".....": 13556, "主": 13557, "çĻ½": 13558, "stop": 13559, "ĠкаÑĢ": 13560, "Ġcoins": 13561, "ĠÑĩаÑģ": 13562, "Ġdestruction": 13563, "lined": 13564, "Ġbarriers": 13565, "antine": 13566, "Ġprinted": 13567, "Ġcongratulations": 13568, "ĠHeart": 13569, "Ġinqu": 13570, "tha": 13571, "Ġhardly": 13572, "ĠAven": 13573, "Ġtinha": 13574, "ĠSony": 13575, "ĠNF": 13576, "Ġgraduates": 13577, "Ġsqueeze": 13578, "eremy": 13579, "ÏĦι": 13580, "Ġepic": 13581, "ĠJu": 13582, "Ġolm": 13583, "ĠLaughter": 13584, "Ġbeliefs": 13585, "ĠCru": 13586, "ĠTrue": 13587, "ĠSoul": 13588, "oween": 13589, "Ġromantic": 13590, "Ġзв": 13591, "Ġanos": 13592, "ĠYup": 13593, "éĺ¿": 13594, "dim": 13595, "Ġinfer": 13596, "Ġзам": 13597, "Ġsoc": 13598, "uka": 13599, "Ġprecise": 13600, "Ġdropping": 13601, "Ġclue": 13602, "Ġerrors": 13603, "charge": 13604, "ĠPu": 13605, "ometer": 13606, "Ġlambda": 13607, "acional": 13608, "ĠDong": 13609, "Ġchamber": 13610, "Ġthankful": 13611, "ĠNu": 13612, "ĠHawai": 13613, "Ġinfo": 13614, "Ġactivate": 13615, "ĠQual": 13616, "Ġqued": 13617, "ÑĥлÑĮ": 13618, "Ġcloth": 13619, "åĸľ": 13620, "Ġwichtig": 13621, "55": 13622, "Ġotra": 13623, "ographer": 13624, "Ġcurios": 13625, "Ġ1980": 13626, "Ġempres": 13627, "dess": 13628, "eur": 13629, "Ġcluster": 13630, "arter": 13631, "obile": 13632, "ĠYan": 13633, "ĠAdv": 13634, "Ġdiscipline": 13635, "ĠìłķëıĦ": 13636, "ĠPlace": 13637, "ĠSelect": 13638, "TE": 13639, "ĠбÑĭла": 13640, "Ġwhis": 13641, "Ġbay": 13642, "ĠDor": 13643, "encing": 13644, "Ġrepet": 13645, "Ġficar": 13646, "pad": 13647, "Ġfog": 13648, "uyor": 13649, "Ġsnap": 13650, "ibt": 13651, "Ġsobie": 13652, "Ġappointment": 13653, "ĠRy": 13654, "Ġceiling": 13655, "ourse": 13656, "Ġwrites": 13657, "ĠAfghanistan": 13658, "Ġmos": 13659, "aze": 13660, "Ġpenal": 13661, "Ġcrystal": 13662, "ICE": 13663, "ê°IJ": 13664, "éŁ": 13665, "ĠTesla": 13666, "Ġtheories": 13667, "Ġappeal": 13668, "Ġnewspaper": 13669, "Ġcookies": 13670, "æ©": 13671, "ĠاÙĦÙĦ": 13672, "Ġmaj": 13673, "ĠGetting": 13674, "kommen": 13675, "ĠHeaven": 13676, "ells": 13677, "Ġdivine": 13678, "Ä«": 13679, "Ġakt": 13680, "Ġhopes": 13681, "ĠChen": 13682, "wegen": 13683, "***": 13684, "ĠFrage": 13685, "Ġни": 13686, "ู": 13687, "minister": 13688, "nesota": 13689, "which": 13690, "Ġexplicit": 13691, "Ġverdad": 13692, "Ġgraduated": 13693, "ĠPhilipp": 13694, "QL": 13695, "ĠMI": 13696, "Ġdevot": 13697, "Ġcure": 13698, "Ġclosest": 13699, "ĠÃĦ": 13700, "Ġsexy": 13701, "ãģĽ": 13702, "ĠDeath": 13703, "oko": 13704, "ugu": 13705, "ĠAnne": 13706, "itarian": 13707, "esa": 13708, "егод": 13709, "ĠDur": 13710, "Ġ000": 13711, "zeit": 13712, "Ġtournament": 13713, "Ġmelhor": 13714, "ส": 13715, "Ġindu": 13716, "Ġflaw": 13717, "Ġwars": 13718, "ĠMind": 13719, "ĠIron": 13720, "ÑĤак": 13721, "ĠVR": 13722, "Ġsiz": 13723, "ĠSouthern": 13724, "Ġê·¸ëŁ¬ë": 13725, "Ġawak": 13726, "Ġìķŀ": 13727, "Ġcube": 13728, "believable": 13729, "ifall": 13730, "dis": 13731, "Ġabandoned": 13732, "mind": 13733, "Ġparl": 13734, "Ġclassical": 13735, "èĭ": 13736, "á»Ļt": 13737, "ĠAuto": 13738, "ĠBor": 13739, "ç©": 13740, "400": 13741, "ĠSociety": 13742, "Ġsubtle": 13743, "Ġmissions": 13744, "Ġremembered": 13745, "ĠEither": 13746, "Ġdafür": 13747, "ORD": 13748, "Ġintensity": 13749, "ESIN": 13750, "ĠCup": 13751, "Ġrarely": 13752, "Ġtoys": 13753, "ĠCharlie": 13754, "ợ": 13755, "Ġglaube": 13756, "Ġrounds": 13757, "TIN": 13758, "Ġcapability": 13759, "Ġderivative": 13760, "Ġreferring": 13761, "ĠdÃ¥": 13762, "ĠTALI": 13763, "Ġcotton": 13764, "Ġconfer": 13765, "Ġcolumns": 13766, "Ġliberal": 13767, "Ġnunca": 13768, "Ġμε": 13769, "Ġindo": 13770, "iben": 13771, "ĠBeispiel": 13772, "Ġê·¸ëłĩ": 13773, "ĠÑĥÑĩ": 13774, "Ġhoy": 13775, "Ġfry": 13776, "ĠScottish": 13777, "èĬ": 13778, "Ġciv": 13779, "Ġconservative": 13780, "Ġairpl": 13781, "Ġsar": 13782, "rus": 13783, "Ġinvestments": 13784, "Ġinfinite": 13785, "Ġà®ķ": 13786, "ĠTALIESIN": 13787, "ĠGary": 13788, "uell": 13789, "Ġак": 13790, "ĠCir": 13791, "Ġritual": 13792, "Ġ>>>": 13793, "Ġtempt": 13794, "ĠTech": 13795, "ĠPokemon": 13796, "Ġimprovements": 13797, "Ġspare": 13798, "Ġtranslate": 13799, "Ġsonra": 13800, "ĠFilm": 13801, "wort": 13802, "Ġми": 13803, "Ġperiods": 13804, "Ġjealous": 13805, "ãģĦãģĦ": 13806, "Ġtir": 13807, "MI": 13808, "Ġconducted": 13809, "ĠìķĪëħķ": 13810, "09": 13811, "ĠPolit": 13812, "ĠWhereas": 13813, "Ġmoisture": 13814, "Ġsins": 13815, "Ġkap": 13816, "ĠÑįк": 13817, "Ġbenim": 13818, "Ġeliminate": 13819, "Ġathletes": 13820, "ĠManager": 13821, "Ġfeatured": 13822, "apore": 13823, "äºĽ": 13824, "Ġë°ľ": 13825, "Ġperf": 13826, "ĠThus": 13827, "Ġdebut": 13828, "обÑĢ": 13829, "Ġseñ": 13830, "Ġmysterious": 13831, "words": 13832, "Ķê°Ģ": 13833, "Ġchecks": 13834, "Ġvolunteer": 13835, "Ġwashing": 13836, "ĠMarvel": 13837, "ĠAB": 13838, "issors": 13839, "!'": 13840, "ĠFull": 13841, "yeon": 13842, "Ġweigh": 13843, "ĠJOHN": 13844, "Ġvos": 13845, "Ġprocedures": 13846, "Ġaddressed": 13847, "ĠBerlin": 13848, "puter": 13849, "ĠBan": 13850, "Ġmedication": 13851, "Ġdrone": 13852, "ĠÑĥб": 13853, "ĠJean": 13854, "Ġcaps": 13855, "Ġdisappointed": 13856, "Ġwore": 13857, "ĠêµŃ": 13858, "Ġorganize": 13859, "ĠHalloween": 13860, "Ġfantasy": 13861, "yard": 13862, "Ġnosotros": 13863, "Ġjumped": 13864, "Ġphotography": 13865, "ĠName": 13866, "rec": 13867, "AB": 13868, "Ġblessing": 13869, "ĠShut": 13870, "Ġbitter": 13871, "pop": 13872, "ãģĿãĤĮ": 13873, "Ġdei": 13874, "Ġfulfill": 13875, "çIJĨ": 13876, "Ġdengan": 13877, "Ġbelo": 13878, "ĠMeanwhile": 13879, "Ġdepois": 13880, "Ġdiabetes": 13881, "Ġbund": 13882, "ĠZealand": 13883, "Ġdigest": 13884, "Ġtires": 13885, "Ġdod": 13886, "agne": 13887, "ết": 13888, "Ġpeel": 13889, "Ġзаб": 13890, "Ġnodes": 13891, "Ġtrends": 13892, "ĠSwitch": 13893, "ĠAward": 13894, "ĠOrig": 13895, "ĠHal": 13896, "Ġestas": 13897, "Ġ360": 13898, "Ġsimult": 13899, "Ġcomic": 13900, "ĠmÃł": 13901, "Ġbalanced": 13902, "ĠPrincess": 13903, "Ġkilometers": 13904, "ứ": 13905, "Ġpartir": 13906, "ì¤ij": 13907, "soft": 13908, "ĠView": 13909, "Ġbiological": 13910, "inst": 13911, "44": 13912, "Ġmanera": 13913, "Ġcomprehensive": 13914, "ĠSab": 13915, "Ġcrimes": 13916, "yers": 13917, "ĠCompany": 13918, "ĠPhot": 13919, "Ġpouco": 13920, "iac": 13921, "Ġbeim": 13922, "inate": 13923, "Ġsubsequ": 13924, "ĠMayor": 13925, "Ġcenturies": 13926, "ères": 13927, "ìŀĸìķĦìļĶ": 13928, "Ġê·¸ëŁ¼": 13929, "ĠFrau": 13930, "ĠOH": 13931, "ĠëģĿ": 13932, "ĠNah": 13933, "ĠSeries": 13934, "Ġovernight": 13935, "íĴĪ": 13936, "ĠâĢ¢": 13937, "Ġtrave": 13938, "attered": 13939, "Ġwarri": 13940, "ĠGrund": 13941, "ĠIndones": 13942, "Ġscra": 13943, "oby": 13944, "ĠBrook": 13945, "Ġcurs": 13946, "Ġë¸": 13947, "Ġexplains": 13948, "ramatic": 13949, "Ġparticipating": 13950, "Ġminut": 13951, "Ġcontracts": 13952, "Ġgegen": 13953, "Ġdisappeared": 13954, "ĠSN": 13955, "Ġrobust": 13956, "aph": 13957, "Ġshrim": 13958, "Ġdevast": 13959, "cope": 13960, "Ġmeets": 13961, "Ġpeaceful": 13962, "mate": 13963, "Ġweld": 13964, "Ġת": 13965, "don": 13966, "ÑĥÑĤÑĮ": 13967, "Ġregistered": 13968, "ĠNik": 13969, "jin": 13970, "Ġcav": 13971, "Ġecht": 13972, "iox": 13973, "Ġflowing": 13974, "ноÑģÑĤи": 13975, "Ġtoe": 13976, "Ġentity": 13977, "ова": 13978, "fits": 13979, "ĠPatrick": 13980, "ÑĤÑĢ": 13981, "Ġleverage": 13982, "Ġcorrel": 13983, "iah": 13984, "Ġstrings": 13985, "istinct": 13986, "Ġgue": 13987, "archy": 13988, "Ġtengo": 13989, "ımız": 13990, "Ġorbit": 13991, "为": 13992, "ĠеÑīÑij": 13993, "cake": 13994, "Ġ׾×Ķ": 13995, "ĠMinnesota": 13996, "Ġbrake": 13997, "owie": 13998, "Ġcraw": 13999, "기를": 14000, "Ġprogramme": 14001, "ĠÑģлÑĥÑĩ": 14002, "åıª": 14003, "iences": 14004, "ĠOui": 14005, "ĠPers": 14006, "imiento": 14007, "ĠInvest": 14008, "Ġslower": 14009, "æĻĤåĢĻ": 14010, "ĠBeth": 14011, "Ġnurse": 14012, "ĠSpring": 14013, "Sp": 14014, "Ġunemploy": 14015, "ди": 14016, "Ġgenius": 14017, "ĠAaron": 14018, "Ġê·¸ëŁ¬": 14019, "Ġei": 14020, "ãģĹãĤĩ": 14021, "Ġtanks": 14022, "Ġaujourd": 14023, "Ġcomplexity": 14024, "ĠÑĢеÑĪ": 14025, "Ġoldest": 14026, "Ġletz": 14027, "åħ¥": 14028, "Ġphenomenon": 14029, "print": 14030, "ĠBundes": 14031, "itat": 14032, "ê»ĺ": 14033, "Ġ42": 14034, "ĠWi": 14035, "Ġincom": 14036, "Ġgek": 14037, "Ġembrace": 14038, "Ġties": 14039, "oute": 14040, "Ġdose": 14041, "ĠFriends": 14042, "ÑĭÑĤ": 14043, "егоднÑı": 14044, "Ġorg": 14045, "Ħë¡ľ": 14046, "óg": 14047, "Ġexceed": 14048, "Ġgods": 14049, "Ġê±°ìĺĪìļĶ": 14050, "Ġsociet": 14051, "ĠUnivers": 14052, "ität": 14053, "Ġworden": 14054, "Ġsmoking": 14055, "Ġintens": 14056, "abul": 14057, "emia": 14058, "èij": 14059, "47": 14060, "fly": 14061, "Ġ2006": 14062, "ĠSeriously": 14063, "Ġprzez": 14064, "æ¼": 14065, "cre": 14066, "Ġnan": 14067, "Ġmodes": 14068, "оваÑĤÑĮ": 14069, "ĠHang": 14070, "emen": 14071, "Ġbeneficial": 14072, "Ġvoters": 14073, "ĠBroad": 14074, "Ġbent": 14075, "Wow": 14076, "Ġmul": 14077, "åĵ¥": 14078, "ĠUC": 14079, "Ġdamaged": 14080, "ĠUkraine": 14081, "Ġwipe": 14082, "Ġstones": 14083, "Ġmanagers": 14084, "Ġrab": 14085, "ÑģÑĤÑĢо": 14086, "lat": 14087, "Ġdece": 14088, "Ġgraphic": 14089, "Ġfoss": 14090, "Ġdisagree": 14091, "ĠAmen": 14092, "Ġsecrets": 14093, "hole": 14094, "inkle": 14095, "Ġfortunate": 14096, "Ġì±": 14097, "ìľĦ": 14098, "èIJ¬": 14099, "Ġhabits": 14100, "Ġburied": 14101, "Ġhin": 14102, "Ġvirtually": 14103, "olas": 14104, "ĠRP": 14105, "ĠTab": 14106, "low": 14107, "Ġsacrific": 14108, "Ġestimated": 14109, "oln": 14110, "Ùĭ": 14111, "cur": 14112, "ĠFeel": 14113, "Ġcastle": 14114, "Ġuseless": 14115, "Ġdisg": 14116, "ĠJacob": 14117, "Ġgaan": 14118, "Ġupside": 14119, "Ġparece": 14120, "ãĥ³ãĥ": 14121, "Ġshipping": 14122, "ĠCR": 14123, "Ġdisrupt": 14124, "acter": 14125, "UND": 14126, "fu": 14127, "å®Į": 14128, "ĠPick": 14129, "ĠCharl": 14130, "ĠBull": 14131, "Ġenterprise": 14132, "Ġpunishment": 14133, "acking": 14134, "Ġfraction": 14135, "Ġtablet": 14136, "Ġchord": 14137, "Ġsimilarly": 14138, "åħ¶å¯¦": 14139, "ĠToronto": 14140, "Ġcourts": 14141, "ÄŁl": 14142, "eszcze": 14143, "Ġpronoun": 14144, "ĠSister": 14145, "ĠMP": 14146, "Ġgreatly": 14147, "ĠDank": 14148, "icop": 14149, "Ġgarbage": 14150, "Ġresolve": 14151, "ĠSaf": 14152, "ĠGun": 14153, "Ġcompound": 14154, "Ġë°°": 14155, "ĠMusik": 14156, "âĻ«": 14157, "Ġchaos": 14158, "ĠWhenever": 14159, "Ġeuros": 14160, "Ġorchest": 14161, "Ġrefriger": 14162, "alan": 14163, "ื": 14164, "ĠAmazing": 14165, "Ġpud": 14166, "agan": 14167, "Ġjeszcze": 14168, "isy": 14169, "Ġaccuracy": 14170, "ĠAma": 14171, "isode": 14172, "ëĮĢ": 14173, "Ġinterpretation": 14174, "ĠLiber": 14175, "æ·": 14176, "cam": 14177, "Ġevolved": 14178, "ĠKay": 14179, "ÑĨÑĭ": 14180, "Ġcreator": 14181, "itas": 14182, "Ġalarm": 14183, "Ġcelebration": 14184, "zent": 14185, "Ġfuncion": 14186, "Ġov": 14187, "umbling": 14188, "Ġ%": 14189, "à¸Ī": 14190, "Ġrestrictions": 14191, "Ġнав": 14192, "ĠKinder": 14193, "Ġbanana": 14194, "ÑĮÑı": 14195, "Ġdiameter": 14196, "Ġnorthern": 14197, "urers": 14198, "ĠPas": 14199, "æĪijçļĦ": 14200, "Ġworkforce": 14201, "Ġjung": 14202, "Ġguarante": 14203, "Ġequilib": 14204, "Ġsuite": 14205, "Ġeuro": 14206, "Ġdeliber": 14207, "Ste": 14208, "Ġdowntown": 14209, "Ġchin": 14210, "Ġcodes": 14211, "edia": 14212, "Ġsheep": 14213, "reshold": 14214, "wnie": 14215, "ób": 14216, "Ġunderlying": 14217, "lia": 14218, "jer": 14219, "ÏĢÏĮ": 14220, "çĿ": 14221, "throp": 14222, "Ġzap": 14223, "Ġvacuum": 14224, "ĠHab": 14225, "Ġwrapped": 14226, "ì¢": 14227, "Ġinventory": 14228, "ма": 14229, "Ġcoord": 14230, "Ġplates": 14231, "Ġsymm": 14232, "Te": 14233, "ĠwÅĤaÅĽnie": 14234, "Ġreaches": 14235, "Ġlonely": 14236, "Script": 14237, "lee": 14238, "esser": 14239, "Ġ걸": 14240, "ĠGesch": 14241, "ĠMoving": 14242, "Ġrép": 14243, "ĠVill": 14244, "åIJĪ": 14245, "ĠRachel": 14246, "Ġtemos": 14247, "ONE": 14248, "Ġstrain": 14249, "Ġangel": 14250, "ĠfÃ¥": 14251, "Tr": 14252, "Ġacho": 14253, "Ġhighlights": 14254, "ĠWer": 14255, "ĠCarl": 14256, "Ġblur": 14257, "Ġregards": 14258, "·": 14259, "илÑģÑı": 14260, "Ġrecre": 14261, "ĠYani": 14262, "UCK": 14263, "ł¸": 14264, "Ġelectrons": 14265, "ĠSpiel": 14266, "Ġved": 14267, "Ú¾": 14268, "Ġbeam": 14269, "Ġidiot": 14270, "ëĵ¤": 14271, "наÑĩ": 14272, "idd": 14273, "Ġski": 14274, "itative": 14275, "Ġhypothes": 14276, "ãģ§ãģĻãģŃ": 14277, "enter": 14278, "ĠìķĦëĭĪë": 14279, "Ġihre": 14280, "Ġpreview": 14281, "angel": 14282, "Ġdemon": 14283, "Ġdus": 14284, "Ġdic": 14285, "ĠKom": 14286, "LEY": 14287, "...!": 14288, "Ġsieht": 14289, "ĠSonic": 14290, "Ġtenho": 14291, "anas": 14292, "Ġdigit": 14293, "ĠMaar": 14294, "Ġundergrad": 14295, "ouncer": 14296, "uffy": 14297, "Ġconversion": 14298, "Ġdisconnect": 14299, "Ġecho": 14300, "omer": 14301, "Ġcurriculum": 14302, "Ġperché": 14303, "Ġwand": 14304, "..?": 14305, "Ġrolled": 14306, "Ġentrepreneur": 14307, "Ġtheoret": 14308, "ĠÑīо": 14309, "Ġinsights": 14310, "Ġzusammen": 14311, "oin": 14312, "rett": 14313, "produ": 14314, "Ġvisitors": 14315, "eous": 14316, "Ġgrandmother": 14317, "Ġhumor": 14318, "ĠниÑħ": 14319, "zenia": 14320, "inson": 14321, "Ġreset": 14322, "Ġbaseball": 14323, "Ġmatching": 14324, "ëĭ¤ê°Ģ": 14325, "Ġpunto": 14326, "ì¡": 14327, "Ġrede": 14328, "Ġaddressing": 14329, "Ġforecast": 14330, "ĠBol": 14331, "Ġcolored": 14332, "Ġdocumentation": 14333, "Ġexpectation": 14334, "ĠNorthern": 14335, "Ġcreo": 14336, "Ġà®ļ": 14337, "fon": 14338, "Ġunsere": 14339, "UM": 14340, "Ġcopies": 14341, "Ġexpanded": 14342, "Ġveterans": 14343, "ĠAlm": 14344, "ĠвообÑīе": 14345, "Ġpsychological": 14346, "Ġnosso": 14347, "Ġpayments": 14348, "imeters": 14349, "Ġ-->": 14350, "ĠJennifer": 14351, "Ġvolunteers": 14352, "osse": 14353, "orious": 14354, "ĠбÑĭли": 14355, "èĤ": 14356, "ĠEss": 14357, "ws": 14358, "ĠBC": 14359, "ĠIC": 14360, "Woman": 14361, "Ġvont": 14362, "Ġethnic": 14363, "ENN": 14364, "имо": 14365, "Ġlob": 14366, "Ġoui": 14367, "cs": 14368, "Ġrehe": 14369, "Ġìłģ": 14370, "Ġchick": 14371, "úsica": 14372, "Ġkont": 14373, "ĠDistrict": 14374, "Ġpile": 14375, "Ġав": 14376, "ейÑģÑĤв": 14377, "Ġ£": 14378, "Ġissued": 14379, "Ġкомп": 14380, "Ġprosper": 14381, "Ġprofound": 14382, "ĠDear": 14383, "Ġãģĵ": 14384, "Ġfunded": 14385, "Ġbisa": 14386, "ŀĺë": 14387, "ף": 14388, "ĠìĿĺ": 14389, "Ġtwelve": 14390, "ĠChampions": 14391, "éĿŀ常": 14392, "Ñģл": 14393, "Ġ2005": 14394, "pm": 14395, "Ġonde": 14396, "Ġdiffé": 14397, "ĠChall": 14398, "Ġdifficulties": 14399, "Ġgarage": 14400, "Ġdá": 14401, "ünk": 14402, "Ġ물": 14403, "Ġtran": 14404, "Ġsubmitted": 14405, "zw": 14406, "ÙĪا": 14407, "Ġark": 14408, "ĠìĦ±": 14409, "Ġgrocery": 14410, "она": 14411, "iere": 14412, "Ġaest": 14413, "Ġexhibition": 14414, "Ġrés": 14415, "Ġconsistency": 14416, "Ġcookie": 14417, "ней": 14418, "Ġreplacement": 14419, "æ²¹": 14420, "ĠSem": 14421, "ĠìĤ¬ìļ©": 14422, "800": 14423, "Ġgenes": 14424, "Ġtransaction": 14425, "ĠEL": 14426, "Ġdurante": 14427, "ibles": 14428, "ĠEat": 14429, "tail": 14430, "issance": 14431, "Ġtoss": 14432, "Ġsurvived": 14433, "Ġoffices": 14434, "Ġsupportive": 14435, "Where": 14436, "Ġtoutes": 14437, "Ġë§ī": 14438, "Ġjokes": 14439, "ieron": 14440, "apers": 14441, "Ġmature": 14442, "ĠMarsh": 14443, "Ġsido": 14444, "kind": 14445, "Ġrealmente": 14446, "ĠChef": 14447, "Ġquelque": 14448, "Ġjudges": 14449, "eft": 14450, "ERS": 14451, "Ġjet": 14452, "Ġpersons": 14453, "è»": 14454, "izations": 14455, "rik": 14456, "Ġshops": 14457, "ĠWy": 14458, "Ġeleg": 14459, "què": 14460, "quoi": 14461, "Ġjuga": 14462, "Ġíķľë²Ī": 14463, "ĠQuestion": 14464, "ĠGlobal": 14465, "Ġìķ½ê°Ħ": 14466, "ĠStation": 14467, "æİ¥": 14468, "ĠOhio": 14469, "Ġsticky": 14470, "Ġstressed": 14471, "Ġgün": 14472, "ĠíĿ": 14473, "ÑģÑĤÑĥп": 14474, "é¡Į": 14475, "ĠPhD": 14476, "immer": 14477, "Ġmentor": 14478, "Ġinvented": 14479, "Ġreun": 14480, "Ġinevit": 14481, "ĠpolÃŃt": 14482, "Ġexecute": 14483, "ĠStory": 14484, "Ġoutstanding": 14485, "Ġguer": 14486, "ĠRain": 14487, "Ġchoses": 14488, "ĠTit": 14489, "ĠÑģеÑĢ": 14490, "ĠSingapore": 14491, "ĠNone": 14492, "Ġchronic": 14493, "°ëį°": 14494, "Ġego": 14495, "æł·": 14496, "EST": 14497, "ãģĤãĤĬ": 14498, "ĠWang": 14499, "ĠNAT": 14500, "Ġaug": 14501, "Ġdesktop": 14502, "Ġeternal": 14503, "ĠìĤ¬ìĭ¤": 14504, "ĠConstitution": 14505, "ìĤ¬ë": 14506, "×Ļ׾": 14507, "pres": 14508, "ĠТÑĭ": 14509, "Ġinterf": 14510, "Ġlists": 14511, "Ġfights": 14512, "ften": 14513, "ĠIowa": 14514, "Ġmotivated": 14515, "ĠHosp": 14516, "Ġelsewhere": 14517, "Ġpaths": 14518, "Ġinstances": 14519, "Bl": 14520, "range": 14521, "á»±": 14522, "ĠSit": 14523, "mana": 14524, "Ġìĭľìŀij": 14525, "Ġmình": 14526, "ansas": 14527, "Ġsna": 14528, "Ġphilosoph": 14529, "Ġpasse": 14530, "Æ°á»Ŀi": 14531, "akh": 14532, "ental": 14533, "Ġihn": 14534, "ructor": 14535, "ĠваÑĪ": 14536, "Ġgenerous": 14537, "Ġpivot": 14538, "пол": 14539, "Ġjamais": 14540, "Ġcoment": 14541, "ĠLew": 14542, "odzi": 14543, "ĠXbox": 14544, "Ġвод": 14545, "Ġconsent": 14546, "īìŀ¥": 14547, "Ġdispar": 14548, "lass": 14549, "ĠGovernor": 14550, "Beifall": 14551, "Ġê°ľ": 14552, "Ġbeloved": 14553, "׳×ķ": 14554, "sell": 14555, "Ġhonored": 14556, "leh": 14557, "Ġwäre": 14558, "unting": 14559, "Ġfraud": 14560, "ĠRAM": 14561, "걸": 14562, "Ġkills": 14563, "Ġeconomics": 14564, "04": 14565, "пеÑĢ": 14566, "Ġcoisas": 14567, "ĠигÑĢ": 14568, "ÃŃm": 14569, "Ġmöchte": 14570, "Ġìµľ": 14571, "Ġstimul": 14572, "Ġfastest": 14573, "lv": 14574, "Ġgén": 14575, "ĠSounds": 14576, "Ġ1970": 14577, "Ġhomework": 14578, "speaking": 14579, "Ġencouraging": 14580, "Ġquery": 14581, "Ġrevers": 14582, "profit": 14583, "Ġdy": 14584, "Ġìŀij": 14585, "ëĬĶëį°ìļĶ": 14586, "Ġsoap": 14587, "ĠGall": 14588, "ĠCN": 14589, "ĠAns": 14590, "Ġfic": 14591, "anks": 14592, "Ġdessert": 14593, "ĠìłĢíĿ¬": 14594, "ĠMaking": 14595, "Ġcomeç": 14596, "ê³Ħ": 14597, "Ġassociation": 14598, "Dad": 14599, "hee": 14600, "Ġhogy": 14601, "Ġapro": 14602, "Ġinvisible": 14603, "American": 14604, "íİ": 14605, "Ġvibe": 14606, "Ġemissions": 14607, "Ġadvocate": 14608, "Ġkicked": 14609, "Ġvel": 14610, "Ġsummar": 14611, "Ġfreaking": 14612, "chron": 14613, "Ġpinch": 14614, "Ġwszystk": 14615, "iscal": 14616, "Ġproved": 14617, "Ġmindful": 14618, "Ġtä": 14619, "Ġnoises": 14620, "Ġisolated": 14621, "Ġcrossed": 14622, "Ġê°ķ": 14623, "ĠvoilÃł": 14624, "Ġchore": 14625, "ĠRA": 14626, "Com": 14627, "Ġrelaxed": 14628, "atro": 14629, "Ġprevention": 14630, "Voiceover": 14631, "OD": 14632, "ĠCovid": 14633, "Ġseparation": 14634, "Ġ-[": 14635, "иÑĩего": 14636, "çĻ¼": 14637, "ĠSD": 14638, "bleep": 14639, "Ġindependence": 14640, "Ġpartial": 14641, "Ġalgorithms": 14642, "ĠAnyone": 14643, "Ġassociate": 14644, "hum": 14645, "icular": 14646, "Ġbạn": 14647, "Ġbattles": 14648, "Good": 14649, "Applause": 14650, "Ġbastante": 14651, "Ġadvant": 14652, "ĠSweet": 14653, "Ġrefused": 14654, "ãĤ¸": 14655, "ĠÑĤебе": 14656, "plet": 14657, "Ġencouraged": 14658, "åĵ¦": 14659, "Ġmiracle": 14660, "ĠBun": 14661, "ĠVar": 14662, "rimination": 14663, "elect": 14664, "ĠMult": 14665, "Ġdelivering": 14666, "eing": 14667, "Ġcm": 14668, "nehmen": 14669, "ĠLine": 14670, "Ġë§Į": 14671, "enced": 14672, "ĠSound": 14673, "ĠContin": 14674, "ijd": 14675, "UNG": 14676, "kle": 14677, "Ġthreshold": 14678, "Ġcompact": 14679, "adt": 14680, "Ġtoes": 14681, "ĠPur": 14682, "owned": 14683, "mented": 14684, "Ġdesigning": 14685, "Ġvaccinated": 14686, "Ġexhaust": 14687, "Ġbasics": 14688, "Ġconsists": 14689, "ĠGuy": 14690, "aczy": 14691, "ĠmÃŃ": 14692, "won": 14693, "害": 14694, "Ġ85": 14695, "æĤ": 14696, "Ġmum": 14697, "Ġignor": 14698, "Ġprinting": 14699, "acular": 14700, "pow": 14701, "Ġexpanding": 14702, "Ġgir": 14703, "ĠCab": 14704, "íĺ¸": 14705, "ÑĤÑĮÑģÑı": 14706, "ĠìŬ룬ë¶Ħ": 14707, "Ġangles": 14708, "Ġterminal": 14709, "ĠWon": 14710, "ĠInteresting": 14711, "Ġcrossing": 14712, "Ġbonds": 14713, "Ġpueden": 14714, "Ġorb": 14715, "ların": 14716, "Ġcreepy": 14717, "Ġnutrition": 14718, "Ġallies": 14719, "Ġwireless": 14720, "Ġdesired": 14721, "Ġcompute": 14722, "ĠArizona": 14723, "ĠBeautiful": 14724, "Ġproduces": 14725, "Ġnuestro": 14726, "ted": 14727, "Ġeligible": 14728, "ĠÑģоз": 14729, "icial": 14730, "ĠHero": 14731, "Ġconsume": 14732, "Ġrobots": 14733, "Ġpurchased": 14734, "cción": 14735, "Ġiz": 14736, "ược": 14737, "ίναι": 14738, "ĠØ£ÙĨ": 14739, "Ġshadows": 14740, "ĠMedia": 14741, "Ġprincess": 14742, "Ġklar": 14743, "Ġwooden": 14744, "Ġusar": 14745, "Ġgüzel": 14746, "Ġslot": 14747, "rade": 14748, "ĠëĴ": 14749, "Ġharmon": 14750, "Ġingredient": 14751, "orship": 14752, "eki": 14753, "Ġgrandfather": 14754, "Ġexcitement": 14755, "Ġpoliticians": 14756, "..!": 14757, "Ġouts": 14758, "Ġseparately": 14759, "ĠÑıк": 14760, "ĠWelt": 14761, "ĠPow": 14762, "jan": 14763, "Ġorientation": 14764, "åıĭ": 14765, "LC": 14766, "agem": 14767, "ÛĮÚº": 14768, "åIJĹ": 14769, "Ġbranches": 14770, "aden": 14771, "rente": 14772, "ĠIhr": 14773, "asm": 14774, "Ġestão": 14775, "ĠNic": 14776, "Ġslave": 14777, "Ġcompress": 14778, "crowd": 14779, "Ġclimbing": 14780, "ĠManagement": 14781, "ĠBah": 14782, "Ġpanic": 14783, "Ġkor": 14784, "Ġcooling": 14785, "Ġbind": 14786, "Ġзад": 14787, "Ġrack": 14788, "Ġentit": 14789, "Ġsends": 14790, "Ġyourselves": 14791, "des": 14792, "ĠMuslims": 14793, "Ġíļ": 14794, "isma": 14795, "cycle": 14796, "unkt": 14797, "ĠCore": 14798, "Ġinjuries": 14799, "Ġidentical": 14800, "каÑı": 14801, "ĠDeutschland": 14802, "Ġее": 14803, "isan": 14804, "Ġtruc": 14805, "leton": 14806, "Ġbackup": 14807, "Ġultra": 14808, "Ġabund": 14809, "illeurs": 14810, "ĠbyÅĤo": 14811, "åħĥ": 14812, "orted": 14813, "Ġearthqu": 14814, "Ġкл": 14815, "Ġobservation": 14816, "Ġmaintenant": 14817, "elen": 14818, "Ġsettled": 14819, "Ġpela": 14820, "ĠEconom": 14821, "ĠÕ": 14822, "Ġsteering": 14823, "ĠALL": 14824, "ĠCher": 14825, "Ġpatience": 14826, "ĠSnow": 14827, "Ġbor": 14828, "Ġworthy": 14829, "Ġcái": 14830, "Ġק": 14831, "Ġκα": 14832, "dog": 14833, "ĠKaren": 14834, "illes": 14835, "β": 14836, "Ġagriculture": 14837, "×ķף": 14838, "ĠSean": 14839, "Ġsensors": 14840, "íķ´ë": 14841, "agh": 14842, "Ġpublicly": 14843, "Ġpeux": 14844, "ĠAlexander": 14845, "Ġpriorit": 14846, "Ġlazy": 14847, "ardon": 14848, "attering": 14849, "Ġcostume": 14850, "ست": 14851, "è¿ĺ": 14852, "Ġunw": 14853, "ÐĽ": 14854, "Ġthickness": 14855, "quito": 14856, "gunt": 14857, "istas": 14858, "neys": 14859, "ĠëIJĺê²Į": 14860, "ĠBrasil": 14861, "Ġtoken": 14862, "Ġaffili": 14863, "lon": 14864, "ĠfÃ¥r": 14865, "ĠBeach": 14866, "Ġwitch": 14867, "ĠSeven": 14868, "Ġpant": 14869, "λλ": 14870, "Ġcaptain": 14871, "åĿ": 14872, "Ġveut": 14873, "Ġpouvoir": 14874, "acz": 14875, "ĠBarb": 14876, "Ġutility": 14877, "Ġcontemporary": 14878, "Ġobtained": 14879, "Ġpaintings": 14880, "ear": 14881, "Ġpean": 14882, "ĠOg": 14883, "Ġcust": 14884, "лем": 14885, "Ĥĺë": 14886, "ĠIsso": 14887, "Ġaconte": 14888, "ĠTele": 14889, "ĠAssistant": 14890, "Ãī": 14891, "íĸĪìĬµëĭĪëĭ¤": 14892, "Ġcounts": 14893, "Ġbuck": 14894, "ĠDeep": 14895, "Ġtackle": 14896, "Ġharsh": 14897, "Ġdecides": 14898, "éĹľ": 14899, ".âĢĭ": 14900, "éĤĬ": 14901, "ĠAngel": 14902, "Ġlaying": 14903, "Ġcalories": 14904, "Ġcontrolling": 14905, "Ġadvantages": 14906, "ĠÑįÑĤой": 14907, "Ġapproaching": 14908, "Ġthreats": 14909, "akan": 14910, "ematic": 14911, "mann": 14912, "ê³µ": 14913, "mumbles": 14914, "ació": 14915, "Ġmaintaining": 14916, "Ġfounder": 14917, "lah": 14918, "fight": 14919, "Ġadmitted": 14920, "âĢ¦.": 14921, "ķĮ": 14922, "abol": 14923, "Ġusage": 14924, "Ġnonsense": 14925, "ĠPalest": 14926, "Ġcontre": 14927, "ĠDemocratic": 14928, "ĠER": 14929, "jekt": 14930, "Ġarbit": 14931, "Ġгол": 14932, "ĠMichelle": 14933, "icher": 14934, "esh": 14935, "ĠPho": 14936, "ком": 14937, "49": 14938, "ĠEnergy": 14939, "οÏį": 14940, "Ġcents": 14941, "Ġrefers": 14942, "Ġgospel": 14943, "ĠSha": 14944, "ĠShare": 14945, "×Ļ׳": 14946, "Ġclinic": 14947, "ĠëĦ£": 14948, "Ġequality": 14949, "ugs": 14950, "Ġshed": 14951, "Ġplanes": 14952, "Ġtoute": 14953, "reck": 14954, "Ġstrand": 14955, "Ġbiology": 14956, "Ġleague": 14957, "ĠPok": 14958, "Ġnúmero": 14959, "ĠCoast": 14960, "Ġconsistently": 14961, "Ġnucle": 14962, "OOOO": 14963, "Ġobjet": 14964, "Ġchor": 14965, "Ġginger": 14966, "Ġdabei": 14967, "Ġcooperation": 14968, "à¯į.": 14969, "nten": 14970, "ç¤": 14971, "lÃł": 14972, "ìĸij": 14973, "rado": 14974, "Ġpassive": 14975, "Ġgloves": 14976, "Ġunderground": 14977, "Ġlogical": 14978, "Ġket": 14979, "Ġfunctionality": 14980, "¸ë¦¬": 14981, "Ġportal": 14982, "eller": 14983, "×Ļר": 14984, "ĠTed": 14985, "ĠGre": 14986, "IJľ": 14987, "Ġpersonnel": 14988, "Ġemerging": 14989, "ĠFür": 14990, "Ġmeantime": 14991, "usalem": 14992, "ĠClear": 14993, "Ġtrapped": 14994, "Ġìļ°": 14995, "Ġdispl": 14996, "Ġmettre": 14997, "Ġmunicip": 14998, "Ġwithdraw": 14999, "Ġspat": 15000, "unes": 15001, "Ġaccessibility": 15002, "æĪij们": 15003, "Ġapare": 15004, "Ġprospect": 15005, "Ġназ": 15006, "Ġcopper": 15007, "ĠPRO": 15008, "ÏħÏĦ": 15009, "Ġattacking": 15010, "ĠVin": 15011, "ĠStone": 15012, "Ġinvestigate": 15013, "style": 15014, "Ġλ": 15015, "ë¡Ŀ": 15016, "ë§Ī": 15017, "Ġinspect": 15018, "Ġliver": 15019, "алиÑģÑĮ": 15020, "Ġsera": 15021, "halten": 15022, "eman": 15023, "Ġministry": 15024, "''": 15025, "Ġdots": 15026, "ãħĭãħĭãħĭãħĭ": 15027, "ÑĥÑģÑĤ": 15028, "ĠJak": 15029, "AKE": 15030, "Ġgaps": 15031, "ucker": 15032, "ĠинÑĤеÑĢеÑģ": 15033, "ĠEmily": 15034, "Ġinterval": 15035, "Ġtender": 15036, "ĠTechnology": 15037, "game": 15038, "Ġtrib": 15039, "ÙĦا": 15040, "ĠDevelopment": 15041, "Ùħا": 15042, "Ġwrist": 15043, "Ġfires": 15044, "Ġtargeted": 15045, "ìłIJ": 15046, "Ġsod": 15047, "íļĮ": 15048, "ĠolduÄŁ": 15049, "Ġseasons": 15050, "ventions": 15051, "Ġнего": 15052, "Ġsometime": 15053, "лив": 15054, "né": 15055, "Ġtú": 15056, "ĠDeus": 15057, "Ġexecution": 15058, "áp": 15059, "ĠChange": 15060, "ĠIndeed": 15061, "Ġregulation": 15062, "ĠHung": 15063, "éis": 15064, "Ġwishes": 15065, "Ġjazz": 15066, "Ġstructural": 15067, "Ġblowing": 15068, "ĠbyÄĩ": 15069, "Ġthermal": 15070, "phant": 15071, "ÑĢÑĥз": 15072, "анÑĤ": 15073, "ĠPull": 15074, "Ġconfusion": 15075, "нÑĭми": 15076, "Ġscenarios": 15077, "ìłģìľ¼ë¡ľ": 15078, "ĠдеÑĤ": 15079, "Ġtattoo": 15080, "Ġautre": 15081, "Ġheating": 15082, "Ġtreating": 15083, "Ġпоним": 15084, "Ġexclus": 15085, "ĠLOL": 15086, "wear": 15087, "agle": 15088, "Ġzurück": 15089, "Ġrational": 15090, "su": 15091, "Ġdeter": 15092, "ĠNative": 15093, "à®ķள": 15094, "ached": 15095, "Ġãĥ": 15096, "ĠEntonces": 15097, "Ġhora": 15098, "ìĿ´ìĹIJìļĶ": 15099, "Ġlite": 15100, "ë": 15101, "Ġsixth": 15102, "Ġболее": 15103, "actor": 15104, "Ġpsychology": 15105, "缸": 15106, "Ġdemands": 15107, "Ġpeer": 15108, "Ġnewly": 15109, "ĠWWE": 15110, "Donald": 15111, "ĠBox": 15112, "Ġpine": 15113, "Ġloading": 15114, "ĠNico": 15115, "ĠsÅĤ": 15116, "omme": 15117, "ART": 15118, "Ġrecruit": 15119, "Ġbugs": 15120, "arents": 15121, "ĠпÑĢоб": 15122, "ĠInside": 15123, "ipper": 15124, "dramatic": 15125, "Ġplanets": 15126, "orde": 15127, "Ġyoga": 15128, "child": 15129, "ĠMarie": 15130, "ĠãģĤ": 15131, "ĠBL": 15132, "Ġfilmed": 15133, "Ġrefresh": 15134, "Ġtomatoes": 15135, "Ġfet": 15136, "Qué": 15137, "Ġ!!": 15138, "ĠëĤ´ë": 15139, "rine": 15140, "Ġinteractive": 15141, "sal": 15142, "annah": 15143, "pez": 15144, "ç¶ĵ": 15145, "Ġunderstands": 15146, "ĠTokyo": 15147, "Ġlibraries": 15148, "Ġreader": 15149, "ijIJ": 15150, "oz": 15151, "ĠEnde": 15152, "ĠFlo": 15153, "Ġmild": 15154, "Ġpoetry": 15155, "Ġжив": 15156, "æĦĽ": 15157, "Ġbehave": 15158, "Ġdoen": 15159, "ĠSusan": 15160, "page": 15161, "raham": 15162, "Ġcommunications": 15163, "Ġtuning": 15164, "Ġpac": 15165, "Ġanxious": 15166, "IO": 15167, "Mark": 15168, "Ġhiç": 15169, "books": 15170, "Ġpiss": 15171, "Ġenabled": 15172, "achelor": 15173, "ĠFOR": 15174, "Ġéc": 15175, "ĠTR": 15176, "ilst": 15177, "hat": 15178, "ĠìĿĮ": 15179, "Ġtych": 15180, "Ġjar": 15181, "Ġbuilds": 15182, "ĠArgent": 15183, "Ġintermedi": 15184, "Ġlou": 15185, "Ġara": 15186, "Ġassignment": 15187, "Ġcabinet": 15188, "Ġretirement": 15189, "ãģ»": 15190, "Ġdisabled": 15191, "rica": 15192, "Ġawards": 15193, "Ġboots": 15194, "Ġacknowled": 15195, "Ġthy": 15196, "Ġ구": 15197, "Ġsynd": 15198, "ний": 15199, "ilton": 15200, "Ġprobl": 15201, "ĠFal": 15202, "Ġverdade": 15203, "Ġ700": 15204, "ĠLearning": 15205, "ocus": 15206, "Ġpalace": 15207, "Not": 15208, "tain": 15209, "cm": 15210, "Ġmagnet": 15211, "incoln": 15212, "Ġfiguring": 15213, "ĠLyn": 15214, "ĠBoss": 15215, "ĠVO": 15216, "Ġdiagnosis": 15217, "Ġequipped": 15218, "watch": 15219, "inos": 15220, "aders": 15221, "Ġshelf": 15222, "Ġorganis": 15223, "Ġnod": 15224, "Ġkız": 15225, "ppers": 15226, "Ġrestore": 15227, "Ġartic": 15228, "ĠVoice": 15229, "ıyorum": 15230, "격": 15231, "Ġspreading": 15232, "Ġhips": 15233, "Ġward": 15234, "ureau": 15235, "Ġintersection": 15236, "66": 15237, "Ġ39": 15238, "ç³": 15239, "Ġwaited": 15240, "ì´": 15241, "hhhh": 15242, "Ġdys": 15243, "ĠEN": 15244, "Ġbatch": 15245, "Ġcaf": 15246, "Ġmarker": 15247, "大家好": 15248, "orable": 15249, "ória": 15250, "Ġstepped": 15251, "Ġcelebrating": 15252, "ана": 15253, "Ġworn": 15254, "ĠFol": 15255, "Ġpla": 15256, "Ġattempts": 15257, "Ġtweet": 15258, "Ġrust": 15259, "gence": 15260, "íĨµ": 15261, "Ġrevel": 15262, "Ġrecept": 15263, "eness": 15264, "Ġ((": 15265, "ãĥ¼ãĥ": 15266, "!âĢĭ": 15267, "ĠìĨIJ": 15268, "Ġinfluenced": 15269, "иж": 15270, "ĠконеÑĩно": 15271, "Ġcolleges": 15272, "ioni": 15273, "Ġsag": 15274, "Ann": 15275, "olar": 15276, "Ġexpressions": 15277, "Ġsuits": 15278, "Ġownership": 15279, "eland": 15280, "piece": 15281, "æĢİä¹Ī": 15282, "Ġdespués": 15283, "Ġtel": 15284, "Ġinsult": 15285, "Ġêµīìŀ¥": 15286, "ĠSmall": 15287, "ĠFR": 15288, "oka": 15289, "berries": 15290, "ĠAnton": 15291, "елÑı": 15292, "ÑıÑģ": 15293, "Ġvalve": 15294, "acts": 15295, "Ġwoods": 15296, "ண": 15297, "Ġcultiv": 15298, "Ġfá": 15299, "ãģ¨ãģĦãģĨ": 15300, "Ġcheers": 15301, "Ġassumption": 15302, "Ġfitness": 15303, "ÃŃcul": 15304, "Ġpodr": 15305, "Ġweit": 15306, "ĠHind": 15307, "Ġdign": 15308, "Ġзн": 15309, "Ġsquad": 15310, "Ġdestro": 15311, "cere": 15312, "shirt": 15313, "immt": 15314, "engers": 15315, "Ġsä": 15316, "kÅĤad": 15317, "ĠÈĻ": 15318, "Ġoccas": 15319, "Ġì¤Ħ": 15320, "Ġprocessor": 15321, "ĠDM": 15322, "ĠDaddy": 15323, "Ġsooner": 15324, "Ġstraightforward": 15325, "Ġdepartments": 15326, "ĠChrome": 15327, "Ġworkplace": 15328, "ĠPython": 15329, "Ġmeng": 15330, "ĠDAN": 15331, "ĠIce": 15332, "ĠëĪĪ": 15333, "ĠGi": 15334, "Ġhiring": 15335, "Ġlanded": 15336, "Ġdemocratic": 15337, "iedz": 15338, "ãģĺãĤĥ": 15339, "Ġsev": 15340, "icia": 15341, "Ġespecial": 15342, "ĠNous": 15343, "Ġhät": 15344, "Ġbou": 15345, "pert": 15346, "iesz": 15347, "åijĢ": 15348, "Ġvil": 15349, "ÅĽli": 15350, "Ġîn": 15351, "Ġlosses": 15352, "éķ·": 15353, "Ġtoast": 15354, "Ġrealm": 15355, "ĠAustin": 15356, "ĠInformation": 15357, "Ġresume": 15358, "Ġchase": 15359, "Ġsalary": 15360, "Ġë¶Ħ": 15361, "лиÑĩ": 15362, "ĠÑģлед": 15363, "ĠFurther": 15364, "Ġcaring": 15365, "Ġvig": 15366, "Ġvalor": 15367, "è¿Ļ个": 15368, "ĠÑĩа": 15369, "Ġanalytics": 15370, "Ġglobe": 15371, "ĠMAN": 15372, "Ġnel": 15373, "ìĿ´ìķ¼": 15374, "Ł¼": 15375, "Ġoy": 15376, "íķĺìĦ¸ìļĶ": 15377, "jen": 15378, "Ġtroubles": 15379, "ahaha": 15380, "Ġchurches": 15381, "uet": 15382, "Ġmeasurements": 15383, "bil": 15384, "ì½": 15385, "ifully": 15386, "инÑĥ": 15387, "ĠWilson": 15388, "¦´": 15389, "ĠíĮĮ": 15390, "Ġì°¨": 15391, "Ġpúblic": 15392, "ĠJerusalem": 15393, "Ġnails": 15394, "Ġspine": 15395, "Ġhemos": 15396, "Ġzn": 15397, "quis": 15398, "ĠLeben": 15399, "Ġreferences": 15400, "ITH": 15401, "iper": 15402, "ĠÑģебÑı": 15403, "ìģ": 15404, "ĠWa": 15405, "state": 15406, "§Ŀ": 15407, "åħ±": 15408, "ĠGener": 15409, "Ġactress": 15410, "ĠEnjoy": 15411, "à¹ĥ": 15412, "Ġ×Ĵ": 15413, "Ġinfected": 15414, "Ġshaking": 15415, "Ġnick": 15416, "ุ": 15417, "Ġfot": 15418, "Ġaccomplished": 15419, "uke": 15420, "Ġsheets": 15421, "Ġfence": 15422, "Ġnursing": 15423, "Ġintroducing": 15424, "Ġfeat": 15425, "One": 15426, "TO": 15427, "Ġclubs": 15428, "ĠBruce": 15429, "onge": 15430, "change": 15431, "ĠBatman": 15432, "åı°": 15433, "ĠOfficer": 15434, "Ġhydro": 15435, "Ġsupplement": 15436, "Ġcela": 15437, "Ġlongest": 15438, "Ġcompeting": 15439, "Ġconhe": 15440, "giving": 15441, "Ġbrains": 15442, "Ġloans": 15443, "Ġwage": 15444, "ĠClinton": 15445, "ĠsÄĥ": 15446, "aneous": 15447, "Ġlord": 15448, "ÑĢÑĥж": 15449, "Ġquiz": 15450, "Ġstiff": 15451, "ĠLGB": 15452, "sz": 15453, "ME": 15454, "mare": 15455, "there": 15456, "Ġnär": 15457, "ĠMand": 15458, "last": 15459, "Ġdag": 15460, "Ġhalfway": 15461, "ĠBand": 15462, "Ġëĭ¤ìĭľ": 15463, "ĠAren": 15464, "Ġile": 15465, "PN": 15466, "ento": 15467, "Ġalgum": 15468, "Ġsoccer": 15469, "Ġblocked": 15470, "ĠJonathan": 15471, "Ġsew": 15472, "ĠTestament": 15473, "Ġvale": 15474, "Ġbehavi": 15475, "å§ĭ": 15476, "Ġconna": 15477, "ICH": 15478, "Ġaudiences": 15479, "ml": 15480, "ammad": 15481, "ĠìĤ´ì": 15482, "IGH": 15483, "Ġraces": 15484, "emed": 15485, "Ġmá»Ļt": 15486, "ï": 15487, "Ġovers": 15488, "Ġdeclared": 15489, "Ġsana": 15490, "ĠUna": 15491, "ĠÑĢе": 15492, "ucks": 15493, "Ġpairs": 15494, "Ġange": 15495, "Ne": 15496, "Ġups": 15497, "avy": 15498, "ør": 15499, "reek": 15500, "Ġbehaviors": 15501, "Ġreflected": 15502, "Ġpriorities": 15503, "Ġcondu": 15504, "Ġretreat": 15505, "Ġexpenses": 15506, "Ġë´IJ": 15507, "Ġtriple": 15508, "Ġêµīìŀ¥íŀĪ": 15509, "ält": 15510, "Ġindigenous": 15511, "Ġmining": 15512, "Ġacceptable": 15513, "Ġruin": 15514, "CA": 15515, "uine": 15516, "Ġpipeline": 15517, "ctic": 15518, "êt": 15519, "ĠвÑģего": 15520, "Ġboun": 15521, "ĠDigital": 15522, "ĠBoom": 15523, "ÑĨе": 15524, "ĠлÑĥÑĩ": 15525, "Ġasc": 15526, "ĮĢë¡ľ": 15527, "ĠGoodbye": 15528, "Ġrender": 15529, "enez": 15530, "arre": 15531, "ĠTHAT": 15532, "bour": 15533, "ición": 15534, "ãĤŃ": 15535, "Every": 15536, "Ġwires": 15537, "ĠParliament": 15538, "nung": 15539, "ateur": 15540, "ĠSave": 15541, "ĠPhys": 15542, "Ġamor": 15543, "ĠEve": 15544, "Ġfright": 15545, "Ġgamma": 15546, "Ġmicros": 15547, "mitt": 15548, "ĠCode": 15549, "ĠBey": 15550, "pled": 15551, "ĠиÑģполÑĮз": 15552, "çĹ": 15553, "ìĥī": 15554, "她": 15555, "Ġmonet": 15556, "ĠJahre": 15557, "Ġluxury": 15558, "Ġdeaf": 15559, "Ġbetray": 15560, "Ġê²°": 15561, "ики": 15562, "Ġdefeated": 15563, "Ġundert": 15564, "Ġweg": 15565, "Ġcooler": 15566, "ãģķãĤĵ": 15567, "iami": 15568, "éĤĦæľī": 15569, "ĠJessica": 15570, "ĠJoy": 15571, "Ġsophistic": 15572, "ении": 15573, "ðĿĺ": 15574, "Ġchili": 15575, "ĠType": 15576, "Ġproteins": 15577, "Ġpresenting": 15578, "alia": 15579, "ìļ¸": 15580, "ĠMajor": 15581, "Ġmolecule": 15582, "umer": 15583, "Ġcollapse": 15584, "ĠAnyways": 15585, "ĠMountain": 15586, "anted": 15587, "ãĢIJ": 15588, "Ġвидео": 15589, "æ°´": 15590, "Aud": 15591, "Ġconqu": 15592, "Ġvoll": 15593, "Ġknit": 15594, "Ġmembr": 15595, "ĠMarket": 15596, "Ġdari": 15597, "Ġcalculated": 15598, "ги": 15599, "Ġshrimp": 15600, "ĠMu": 15601, "ĠпÑĢоÑĤ": 15602, "Ġìĺģìĥģ": 15603, "Ġproductivity": 15604, "Ġcognitive": 15605, "ĠHeb": 15606, "ictions": 15607, "ê²½": 15608, "Ġcré": 15609, "för": 15610, "Ġpraying": 15611, "ashi": 15612, "ĠTik": 15613, "ór": 15614, "wen": 15615, "ÑĮÑİ": 15616, "ixo": 15617, "Ġ(\"": 15618, "ĠÑĤел": 15619, "Ġìĸ´ëĸ¤": 15620, "ĠпеÑĢед": 15621, "ĠDrive": 15622, "ãĢij": 15623, "ĠEqu": 15624, "Ġequilibrium": 15625, "Ġdescribes": 15626, "нее": 15627, "42": 15628, "ĠCurrent": 15629, "yy": 15630, "Ġabsorb": 15631, "Ġsoldier": 15632, "ders": 15633, "Ġtestimony": 15634, "Ġdecline": 15635, "ľë¡ľ": 15636, "gage": 15637, "Ġinspire": 15638, "lapping": 15639, "Ġspinning": 15640, "Ġslavery": 15641, "Ġfacial": 15642, "Ġtraditions": 15643, "ários": 15644, "ĠHospital": 15645, "Ġnest": 15646, "ĠëĪĦ": 15647, "Ġtoi": 15648, "Ġfears": 15649, "ìħ¨": 15650, "ĠMuh": 15651, "Ġgraduation": 15652, "Ġimpacted": 15653, "Ġaunt": 15654, "ĠLets": 15655, "Ġaluminum": 15656, "Ġdominant": 15657, "ĠDavis": 15658, "ĠNavy": 15659, "Ġcompt": 15660, "oples": 15661, "Ġestava": 15662, "è¥": 15663, "Ġscal": 15664, "Ġpreserve": 15665, "ĠOpp": 15666, "Ġpractically": 15667, "Ġmagnitude": 15668, "Ġfitting": 15669, "Ġcoordinate": 15670, "Ġfurniture": 15671, "ĠFamil": 15672, "Ġexplosion": 15673, "Ġdocumentary": 15674, "ĠScript": 15675, "Ġportray": 15676, "mat": 15677, "Ġscheduled": 15678, "Ġdynamics": 15679, "phy": 15680, "aky": 15681, "ĠUI": 15682, "Che": 15683, "Ġcontinuously": 15684, "ĠProv": 15685, "å°ij": 15686, "Ñĥз": 15687, "rah": 15688, "Ġgerne": 15689, "proof": 15690, "Ġsecretary": 15691, "ĠPatreon": 15692, "scream": 15693, "ĠKids": 15694, "á»ĵi": 15695, "Ġkg": 15696, "Ġuncertainty": 15697, "Ġкажд": 15698, "Ġmitig": 15699, "Ġreads": 15700, "å·²": 15701, "ĠRu": 15702, "Ġpriest": 15703, "Ġнед": 15704, "Ġlimitations": 15705, "Ġfloat": 15706, "600": 15707, "ĠToy": 15708, "ĠJimmy": 15709, "Ġoffensive": 15710, "eni": 15711, "ĠXi": 15712, "Ġeyebr": 15713, "ĠTurk": 15714, "Ġaccidentally": 15715, "Ġohne": 15716, "ĠSaud": 15717, "95": 15718, "ĠDutch": 15719, "анÑģ": 15720, "ĠSeattle": 15721, "Ġëĵ±": 15722, "check": 15723, "kÄĻ": 15724, "Ġcontributions": 15725, "Ġbeside": 15726, "Ġquindi": 15727, "Ġflew": 15728, "æŶ": 15729, "ذا": 15730, "ĠLO": 15731, "Ġwaist": 15732, "ĠEV": 15733, "Ġholidays": 15734, "jon": 15735, "Ġmisunder": 15736, "Ñıн": 15737, "Ġbout": 15738, "Ġdimin": 15739, "ẽ": 15740, "ól": 15741, "ĠGrace": 15742, "Ġinputs": 15743, "Ġdeny": 15744, "Ġforming": 15745, "ĠBild": 15746, "Ġadequ": 15747, "Ġfolk": 15748, "Ġrejected": 15749, "semb": 15750, "Ġfrustrated": 15751, "open": 15752, "ĠBetter": 15753, "ilon": 15754, "Ġtowel": 15755, "Ġdifferential": 15756, "Ġsacred": 15757, "Ġsail": 15758, "éĩĮ": 15759, "entimes": 15760, "Ġgentleman": 15761, "Ġiconic": 15762, "Ġcomparing": 15763, "Ġsagt": 15764, "Ġtexts": 15765, "Ġgrandma": 15766, "Ġrolls": 15767, "Ġcontents": 15768, "ä¸į好": 15769, "оÑģÑģ": 15770, "Ġsuspension": 15771, "roit": 15772, "¦¼": 15773, "Ġassez": 15774, "Ġdort": 15775, "ĠMath": 15776, "ĠVictor": 15777, "ĠJavaScript": 15778, "ä¸įå°į": 15779, "Ġenhan": 15780, "ÅĻ": 15781, "ĠBush": 15782, "Ġpromotion": 15783, "Ġkin": 15784, "Ġmonsters": 15785, "ĠColorado": 15786, "Ġβ": 15787, "íķ´ìļĶ": 15788, "æŃ£": 15789, "ifferent": 15790, "Ġnaked": 15791, "Ġprod": 15792, "etics": 15793, "ĠWoman": 15794, "Ġtreatments": 15795, "Ġestoy": 15796, "vé": 15797, "Ġlifting": 15798, "Ġyapt": 15799, "ĠRober": 15800, "Ġì¹ľ": 15801, "Ġsubstitute": 15802, "aku": 15803, "ridge": 15804, "Ġê±°ë": 15805, "Ġresponded": 15806, "Ġbé": 15807, "ĠEngineer": 15808, "Ġtransferred": 15809, "ë²": 15810, "Ġhaber": 15811, "oop": 15812, "ĠWE": 15813, "Ġvest": 15814, "Ġforty": 15815, "ĠDS": 15816, "Ġ2004": 15817, "Ġcoaching": 15818, "nom": 15819, "ĠBab": 15820, "Ġnossa": 15821, "ĠJake": 15822, "Ġgy": 15823, "Ġdeleg": 15824, "Ġìŀł": 15825, "ĠкÑĢаÑģ": 15826, "Ġstandpoint": 15827, "Ġdisad": 15828, "Ġartwork": 15829, "Ad": 15830, "illo": 15831, "ĠÄijược": 15832, "ĠProm": 15833, "ĠLib": 15834, "Ġcriticism": 15835, "Ġcontacts": 15836, "ÑĢам": 15837, "Ġachievement": 15838, "ÐĶа": 15839, "Ġdissol": 15840, "ĠVegas": 15841, "Ġstreams": 15842, "ĠKent": 15843, "ĠعÙĦÙī": 15844, "Ġradius": 15845, "Ġsucks": 15846, "ĠAch": 15847, "Ġfi": 15848, "oust": 15849, "ĠлÑİди": 15850, "Ġpalette": 15851, "ĠHaz": 15852, "ĠAnthony": 15853, "Ġtema": 15854, "ĠCos": 15855, "Ġsafer": 15856, "αÏĤ": 15857, "Ġcontrad": 15858, "Ġmaior": 15859, "Ġinflation": 15860, "ĠSilver": 15861, "Ġattending": 15862, "íķľíħĮ": 15863, "arto": 15864, "Ġapplauding": 15865, "Ġcomputing": 15866, "ĠHat": 15867, "æ»": 15868, "know": 15869, "makers": 15870, "Ġconoc": 15871, "Ġeducated": 15872, "Ġmodified": 15873, "Ġinclusion": 15874, "mental": 15875, "ŀIJ": 15876, "isia": 15877, "ĠÏĢοÏħ": 15878, "Ġaun": 15879, "ĠIreland": 15880, "Ġkö": 15881, "Ġcompliance": 15882, "Ġinspiring": 15883, "иÑĤелÑĮно": 15884, "Ġdispos": 15885, "ì°¨": 15886, "Ġwip": 15887, "rical": 15888, "rawd": 15889, "Ġtres": 15890, "Ġmobil": 15891, "olutions": 15892, "BO": 15893, "Ġbounce": 15894, "Ġassumed": 15895, "ĠMedical": 15896, "Ġfiscal": 15897, "ĠngÆ°á»Ŀi": 15898, "itionally": 15899, "Ġstolen": 15900, "ĠBM": 15901, "Ġmechanisms": 15902, "εί": 15903, "Ġqualified": 15904, "ĠìŀIJë": 15905, "ughters": 15906, "ĠHIV": 15907, "ĠLots": 15908, "Ġservers": 15909, "Ġcarr": 15910, "ĠTogether": 15911, "Ġattracted": 15912, "Ġkr": 15913, "æĪijæĺ¯": 15914, "thur": 15915, "inin": 15916, "ĠHalf": 15917, "ÈĽ": 15918, "ĠPap": 15919, "Ġreminded": 15920, "ALL": 15921, "Ġhelmet": 15922, "Ġbottles": 15923, "Ġprofessors": 15924, "Ġseine": 15925, "ÅĤÄħ": 15926, "ãĥı": 15927, "Ġê±°ìķ¼": 15928, "Ġ×¢×ľ": 15929, "fun": 15930, "ĠBird": 15931, "Ġfighter": 15932, "ĠëĶ°ë": 15933, "ĠTool": 15934, "Ġtin": 15935, "inois": 15936, "ë¶Ħ": 15937, "×Ļף": 15938, "ĠCAR": 15939, "åIJį": 15940, "irsty": 15941, "Ġoutdoor": 15942, "ĠNS": 15943, "ãħİ": 15944, "ffen": 15945, "Ġlud": 15946, "Hello": 15947, "Ġroller": 15948, "iele": 15949, "ĠPoland": 15950, "Ġapa": 15951, "exp": 15952, "Ġcertificate": 15953, "ĠTown": 15954, "аÑİÑĤÑģÑı": 15955, "ilde": 15956, "Ġdetermin": 15957, "PR": 15958, "Ġfreeze": 15959, "Ġmainstream": 15960, "Ġobjectives": 15961, "blo": 15962, "Ġtakie": 15963, "åĵĪåĵĪ": 15964, "Ġë°Ķë¡ľ": 15965, "elet": 15966, "ĠIV": 15967, "ĠFast": 15968, "Ġdere": 15969, "emp": 15970, "ĠDra": 15971, "ĠìŀĪìĹĪ": 15972, "Ġdiscrimination": 15973, "Ġείναι": 15974, "necess": 15975, "æ®": 15976, "ıģı": 15977, "Ġposting": 15978, "wiÅĽcie": 15979, "Ġlub": 15980, "Ġolive": 15981, "Ġrim": 15982, "Ġmodeling": 15983, "Ġaño": 15984, "ĠPakistan": 15985, "Ġoverl": 15986, "Ġinflam": 15987, "NE": 15988, "ìĹIJê²Į": 15989, "Ġattended": 15990, "Ġdealt": 15991, "ĠAlt": 15992, "ĠLincoln": 15993, "Ġawake": 15994, "Ġfilters": 15995, "ĠWithin": 15996, "czywiÅĽcie": 15997, "Ġsû": 15998, "ĠJohnny": 15999, "Ġintegrity": 16000, "Ġisolation": 16001, "ĠEasy": 16002, "ĠпÑĢин": 16003, "ĠAlice": 16004, "Ġsmiling": 16005, "enix": 16006, ",...": 16007, "ζ": 16008, "Ġbegun": 16009, "Ġjewel": 16010, "Ġconventional": 16011, "Ġstatist": 16012, "Ġhanded": 16013, "Ġirre": 16014, "Ġprohib": 16015, "Ġsatellite": 16016, "é¦Ļ": 16017, "ĠIndust": 16018, "Ġtraged": 16019, "Ġtrava": 16020, "Ġihm": 16021, "Ġcruel": 16022, "ĠAgora": 16023, "ĠDoc": 16024, "Ġzones": 16025, "Ġmall": 16026, "Ġtray": 16027, "×ķ׳": 16028, "Ġirrit": 16029, "Ġkans": 16030, "ĠBeat": 16031, "udge": 16032, "ielle": 16033, "Ġtrusted": 16034, "Ġbikes": 16035, "ĠÑĥп": 16036, "ĠMember": 16037, "wick": 16038, "Ġcreators": 16039, "Ġheritage": 16040, "indistinct": 16041, "Ġresur": 16042, "ennen": 16043, "Come": 16044, "Ġfiring": 16045, "ĠBueno": 16046, "ĠТо": 16047, "ikan": 16048, "ettes": 16049, "Ġkes": 16050, "Ġtrips": 16051, "Ġdivorce": 16052, "ĠKl": 16053, "Ġconsol": 16054, "keep": 16055, "기ê°Ģ": 16056, "ĠReport": 16057, "Ġhosting": 16058, "Ġdiamond": 16059, "Ġcomplic": 16060, "Ġhelicop": 16061, "Ġdepuis": 16062, "ds": 16063, "ĠChan": 16064, "Ñıл": 16065, "Ġscissors": 16066, "ilation": 16067, "Ġproportion": 16068, "ERE": 16069, "ĠÙĪاÙĦ": 16070, "inta": 16071, "Ġmuchas": 16072, "uation": 16073, "itis": 16074, "æĬĬ": 16075, "ÑıÑī": 16076, "Ġniin": 16077, "Ġemphasize": 16078, "uela": 16079, "Ġproducers": 16080, "Ġrze": 16081, "änder": 16082, "ETH": 16083, "æº": 16084, "Ġconstitu": 16085, "åĽ½": 16086, "Ġperformances": 16087, "istle": 16088, "gov": 16089, "ĠLiter": 16090, "Ġincorporate": 16091, "Ġeducate": 16092, "ĠNin": 16093, "쪽": 16094, "ÙĩÙħ": 16095, "eleration": 16096, "×ķ×ij": 16097, "ĠyaÅŁ": 16098, "orous": 16099, "ĠCas": 16100, "Ġgrants": 16101, "ëĬ¥": 16102, "amel": 16103, "Ġê·¸ëłĩê²Į": 16104, "ĠEste": 16105, "ÑħодиÑĤ": 16106, "ĠпоÑģле": 16107, "Ġgent": 16108, "Ġfocuses": 16109, "alities": 16110, "ĠRh": 16111, "ë³´": 16112, "æ°ij": 16113, "ĠDance": 16114, "rr": 16115, "Ġamer": 16116, "Ġutilize": 16117, "ĠlÃŃ": 16118, "ĠAmong": 16119, "Ġpregnancy": 16120, "Ġloops": 16121, "алоÑģÑĮ": 16122, "ĠMoh": 16123, "Ġcatching": 16124, "Ġglob": 16125, "Ġajud": 16126, "Ġ[?": 16127, "ĠAnal": 16128, "looking": 16129, "Ġsurfaces": 16130, "Ġprogressive": 16131, "Ġviral": 16132, "08": 16133, "ξ": 16134, "KA": 16135, "Ġży": 16136, "Ġpicks": 16137, "annon": 16138, "Ġbulk": 16139, "ĠRoss": 16140, "Ġdescribing": 16141, "ĠGel": 16142, "Ġlocally": 16143, "Ġendless": 16144, "Ġmassage": 16145, "Ġcleaned": 16146, "Ġtraveled": 16147, "енÑĭ": 16148, "Ġsentiment": 16149, "igma": 16150, "ĠNas": 16151, "Ġchemicals": 16152, "Ġrighteous": 16153, "ĠMagic": 16154, "Ġrelates": 16155, "Ġtrucks": 16156, "Ġ1960": 16157, "åĪ¥": 16158, "Ġappet": 16159, "Ġsnacks": 16160, "ĠSummer": 16161, "Ġyüz": 16162, "Ġpris": 16163, "ĠMexican": 16164, "Ġtransparen": 16165, "Ġminority": 16166, "Ġverte": 16167, "Ġlassen": 16168, "46": 16169, "лек": 16170, "ép": 16171, "ĠÑĦилÑĮ": 16172, "Ġiyi": 16173, "Ġspan": 16174, "íķĺì§Ģ": 16175, "Ġindicated": 16176, "quar": 16177, "Ġscholarship": 16178, "ĠLGBT": 16179, "Ġhistorically": 16180, "óÅĤ": 16181, "Ġminist": 16182, "Ġpenet": 16183, "ĠRap": 16184, "Ġconservation": 16185, "缴": 16186, "ĠHoney": 16187, "ĠBei": 16188, "idel": 16189, "Ġresponsibilities": 16190, "Ġmessy": 16191, "ĠExcept": 16192, "ORE": 16193, "Ġinitiatives": 16194, "Ġjunior": 16195, "Ġdesigners": 16196, "Ġexploration": 16197, "Ġsponsor": 16198, "Ġmobility": 16199, "Ġinteg": 16200, "lando": 16201, "Ġbark": 16202, "Ġindicates": 16203, "à¶": 16204, "Ġemployer": 16205, "å®ī": 16206, "Ġcousin": 16207, "Ġboiling": 16208, "Ġchrom": 16209, "Ġçal": 16210, "Ġperpet": 16211, "Ġcontained": 16212, "Ġparks": 16213, "Ы": 16214, "ĠEngineering": 16215, "Please": 16216, "ĠStarting": 16217, "hero": 16218, "Ġlawyers": 16219, "西": 16220, "Ġzd": 16221, "Ġfranchise": 16222, "rage": 16223, "Ġintuit": 16224, "ĠGL": 16225, "reach": 16226, "ĠElle": 16227, "ĠnhÆ°": 16228, "ĠNord": 16229, "Ġbean": 16230, "07": 16231, "Ġpleasant": 16232, "å½ĵ": 16233, "viron": 16234, "Ġgradient": 16235, "zus": 16236, "ĠEM": 16237, "Ġessay": 16238, "ìĹIJìļĶ": 16239, "ến": 16240, "nu": 16241, "ừ": 16242, "ĠÃīs": 16243, "Ġdenomin": 16244, "ĠGirls": 16245, "Ġpersonnes": 16246, "ĠاÙĦØ£": 16247, "bild": 16248, "ĠStat": 16249, "Ġcompliment": 16250, "ĠKate": 16251, "Ġoptimal": 16252, "Ġhid": 16253, "دÙĬ": 16254, "Ġquicker": 16255, "wall": 16256, "En": 16257, "INE": 16258, "???": 16259, "ì²´": 16260, "ĠAction": 16261, "åŁ": 16262, "Ġpenalty": 16263, "ĠKaz": 16264, "'?": 16265, "Ġcried": 16266, "Ġcanvas": 16267, "fte": 16268, "Ġexclud": 16269, "¸ë¡ľ": 16270, "Ġemphasis": 16271, "Ġenzy": 16272, "ĠHou": 16273, "Ġoverseas": 16274, "ÃŃamos": 16275, "師": 16276, "öglich": 16277, "Ġheadphones": 16278, "cn": 16279, "ĠAge": 16280, "Ġakan": 16281, "Ġcharacteristic": 16282, "íķĺë©´": 16283, "gets": 16284, "Ġë¶Ī": 16285, "Ġrival": 16286, "Ġborders": 16287, "emente": 16288, "emás": 16289, "Ġyol": 16290, "Ġcompe": 16291, "enders": 16292, "ından": 16293, "Ġmöglich": 16294, "Ġbubbles": 16295, "natural": 16296, "Ġarmed": 16297, "Ġelabor": 16298, "ĠìĿ´ë²Ī": 16299, "Ġwashed": 16300, "οÏħμε": 16301, "è«ĭ": 16302, "Ġflavors": 16303, "Ġexiste": 16304, "Ġprest": 16305, "ĠThema": 16306, "опÑĢоÑģ": 16307, "eron": 16308, "UE": 16309, "eri": 16310, "Ġconcer": 16311, "Ġaixò": 16312, "åħ©": 16313, "Ġprotective": 16314, "ĠзнаÑİ": 16315, "ĠëĤł": 16316, "ĠIII": 16317, "Ġmeer": 16318, "ĠShop": 16319, "lli": 16320, "ĠOrder": 16321, "ĠMY": 16322, "ĠGhost": 16323, "ãĤĤãģĨ": 16324, "adel": 16325, "Ġstole": 16326, "Ġreleasing": 16327, "ĠComment": 16328, "Ġtrains": 16329, "ëªħ": 16330, "Ġwissen": 16331, "ensed": 16332, "Ġdescend": 16333, "Ġfier": 16334, "Ġradi": 16335, "Ġpersu": 16336, "ç¢": 16337, "Ġмн": 16338, "ĠDest": 16339, "Ġworries": 16340, "itet": 16341, "bas": 16342, "Ġstab": 16343, "name": 16344, "oric": 16345, "ĠClose": 16346, "Ġalumni": 16347, "ĠSelf": 16348, "ffe": 16349, "itating": 16350, "atherine": 16351, "ĠRights": 16352, "Ġellos": 16353, "Ġwarrant": 16354, "Ġnerve": 16355, "Ġvegetable": 16356, "ĠTeil": 16357, "Ġê°ĻìĿ´": 16358, "RY": 16359, "Ġsustainability": 16360, "Ġsteht": 16361, "Ġbrid": 16362, "adaÅŁ": 16363, "Ġtv": 16364, "Ġduration": 16365, "Ġpessoa": 16366, "Ġmetrics": 16367, "Ġadam": 16368, "cas": 16369, "аÑĢи": 16370, "Ġevident": 16371, "Ġdisplayed": 16372, "ائ": 16373, "Ġreck": 16374, "ĠBuddha": 16375, "Ġdele": 16376, "ĠDiego": 16377, "osph": 16378, "Ġbla": 16379, "ĠMik": 16380, "ulator": 16381, "Ġ2001": 16382, "Ġpromoting": 16383, "ych": 16384, "ĠEX": 16385, "Ġlastly": 16386, "Ġoutline": 16387, "Ġspirits": 16388, "Ġveux": 16389, "Ġsubtract": 16390, "ĠÅŁimdi": 16391, "Ġpins": 16392, "Ġburger": 16393, "Ġmolto": 16394, "ĠhabÃŃa": 16395, "Ġë°ĺ": 16396, "igu": 16397, "erst": 16398, "Ġnen": 16399, "Ġbacon": 16400, "itious": 16401, "Ġcarries": 16402, "Ġpromises": 16403, "nde": 16404, "ĠLeft": 16405, "ĠLim": 16406, "æ£": 16407, "Ġ44": 16408, "Ġcareers": 16409, "Ġ주ë": 16410, "Ġspeeds": 16411, "qué": 16412, "mad": 16413, "market": 16414, "isme": 16415, "Ġ2003": 16416, "Ġrecess": 16417, "ĠJUD": 16418, "Ġracist": 16419, "ĠSchl": 16420, "Ġparler": 16421, "Ġotros": 16422, "ishes": 16423, "Ġconverted": 16424, "aaaa": 16425, "ании": 16426, "ĠArk": 16427, "ĠChance": 16428, "Ġelementary": 16429, "εν": 16430, "inks": 16431, "Interviewer": 16432, "Ġfreely": 16433, "alah": 16434, "Ġëĭ¤ë¥¸": 16435, "Ġrequested": 16436, "Ġtorque": 16437, "noÅĽci": 16438, "oured": 16439, "ĠStaff": 16440, "Ġstain": 16441, "ĠAlan": 16442, "Ġvere": 16443, "ĠWinter": 16444, "Ġdefect": 16445, "iedy": 16446, "Ġbeats": 16447, "Ġhá": 16448, "umn": 16449, "oons": 16450, "itudes": 16451, "Ġseit": 16452, "oly": 16453, "Ġreserv": 16454, "Ġextr": 16455, "Ġphysician": 16456, "visor": 16457, "Ġhandful": 16458, "ĠNations": 16459, "Ġì¢ĭìĿĢ": 16460, "uccess": 16461, "Ġupstairs": 16462, "ĠSquare": 16463, "Ġhein": 16464, "ĠSeason": 16465, "olis": 16466, "Ġprince": 16467, "Ġdefensive": 16468, "ç½": 16469, "ĠмеÑģÑĤ": 16470, "Ñĸй": 16471, "ĠاÙĨ": 16472, "umble": 16473, "ê¹ĮìļĶ": 16474, "Ġassass": 16475, "Ġcircular": 16476, "Ġqualities": 16477, "Ġhmm": 16478, "Ġblown": 16479, "ĠLiz": 16480, "ĠKur": 16481, "ĠSA": 16482, "Ġfindings": 16483, "Ġcolours": 16484, "Ġdelle": 16485, "ĠIR": 16486, "ĠAth": 16487, "ĠDub": 16488, "ĠOx": 16489, "ĠØ®": 16490, "Ġpockets": 16491, "Ġgrill": 16492, "Ġswitching": 16493, "Ġpreferred": 16494, "ĠWales": 16495, "Ġexemplo": 16496, "Ġchopped": 16497, "Ġvaccination": 16498, "Ġneuro": 16499, "Ġspecify": 16500, "ivos": 16501, "Ġserá": 16502, "Ġzie": 16503, "Ġà®®": 16504, "Ġresulting": 16505, "ĠUgh": 16506, "Ġmessed": 16507, "CD": 16508, "Ġpaar": 16509, "Ġcomer": 16510, "Ġcouch": 16511, "ĠFestival": 16512, "Ġ49": 16513, "vous": 16514, "zens": 16515, "種": 16516, "ĠKennedy": 16517, "ĠTs": 16518, "Ġë³´ìĹ": 16519, "Ġdemonstration": 16520, "Ġunto": 16521, "Ġfrustrating": 16522, "Ġlaboratory": 16523, "Ġegy": 16524, "Ġbeautifully": 16525, "Ġìŀ¬ë": 16526, "Ġalgu": 16527, "Ġöyle": 16528, "ä½łçľĭ": 16529, "ĠPH": 16530, "Ġfortune": 16531, "Ġcleaner": 16532, "ĠRobin": 16533, "Ġsaus": 16534, "ĠGeld": 16535, "Ġkat": 16536, "obs": 16537, "Ġolur": 16538, "Ġmatt": 16539, "Ġquesta": 16540, "Ġsuggestion": 16541, "encer": 16542, "оÑģÑĤ": 16543, "Ġradar": 16544, "Ġìŀ¡": 16545, "isha": 16546, "ந": 16547, "ãĤĵãģª": 16548, "jes": 16549, "Ġveel": 16550, "ìĤ°": 16551, "Ġauthors": 16552, "ãĢİ": 16553, "plan": 16554, "Ġcollaborative": 16555, "Ġinstinct": 16556, "Ġfarming": 16557, "auge": 16558, "Edu": 16559, "Ġmembership": 16560, "Ġsimultaneously": 16561, "Ġbake": 16562, "Ġkä": 16563, "Ġlectures": 16564, "ÑĩеÑģ": 16565, "Ġprendre": 16566, "Ġcollaps": 16567, "ĠSaya": 16568, "ĠFut": 16569, "Ġyog": 16570, "ĠRather": 16571, "رÙĬ": 16572, "Ġcamps": 16573, "олод": 16574, "Ġsimulation": 16575, "ĠMak": 16576, "Laughs": 16577, "Ġgrey": 16578, "Ġsentences": 16579, "yen": 16580, "ĠUnless": 16581, "Je": 16582, "ĠSatan": 16583, "ĠÑĤакже": 16584, "ĠNA": 16585, "Ġbron": 16586, "Ġ?]": 16587, "Ġsouls": 16588, "Ġlightning": 16589, "Ġimagined": 16590, "Ġczyli": 16591, "psilon": 16592, "etta": 16593, "Ġbelieving": 16594, "Ġstrongest": 16595, "ĠCON": 16596, "Ġquelques": 16597, "Ġimmigrants": 16598, "Ġwallet": 16599, "éĢĻæĺ¯": 16600, "ĠJersey": 16601, "Ġimplications": 16602, "Ġforb": 16603, "ãĢı": 16604, "Ġunbelievable": 16605, "اء": 16606, "Ġoperational": 16607, "üs": 16608, "ĠGM": 16609, "Ġê·¸ëŁ°ëį°": 16610, "Ġgracias": 16611, "Ġentend": 16612, "ĠRegard": 16613, "rob": 16614, "ĠÑĤеÑħ": 16615, "èı": 16616, "ĠRevolution": 16617, "Ġwaar": 16618, "ĠBiz": 16619, "theless": 16620, "Ġsponsored": 16621, "quier": 16622, "ĠìĿ¼ë": 16623, "Ġtek": 16624, "ĠëIJł": 16625, "igkeit": 16626, "ĠLuck": 16627, "ĠCertainly": 16628, "Ġtoll": 16629, "ĠниÑĩего": 16630, "ĠMoney": 16631, "ĠÑģÑĤоÑĢ": 16632, "ĠDouble": 16633, "ĠWolf": 16634, "Ġchunk": 16635, "άν": 16636, "ités": 16637, "oning": 16638, "Mar": 16639, "Ġgrandes": 16640, "Ġcollections": 16641, "ĠEuropa": 16642, "ĠаÑĢ": 16643, "ĠâĢĭâĢĭâĢĭ": 16644, "Ġê·¸ëŁ¬ë©´": 16645, "ĠобÑĬ": 16646, "Ġãģª": 16647, "Ġìĭľê°Ħ": 16648, "ĠCustom": 16649, "Ġì²ĺ": 16650, "ÑĸлÑĮ": 16651, "Ġindividually": 16652, "íĹ": 16653, "Ġdozen": 16654, "Ġowe": 16655, "ĠVictoria": 16656, "åı¯èĥ½": 16657, "Ġbeet": 16658, "urb": 16659, "Ġanalog": 16660, "ição": 16661, "Ĥľ": 16662, "soever": 16663, "Ġmodo": 16664, "Ġsubscribed": 16665, "ìŀ¬": 16666, "Ġentities": 16667, "çīĩ": 16668, "Ġcloset": 16669, "Ġresponding": 16670, "Ġprinter": 16671, "ĠStephan": 16672, "ĠbyÅĤ": 16673, "ĠDom": 16674, "ĠFern": 16675, "ĠPier": 16676, "ĠwiÄĻc": 16677, "Ġhence": 16678, "Ġmodules": 16679, "ãĥ¬": 16680, "ĠëĶ±": 16681, "ĠDanny": 16682, "ĠÑģебе": 16683, "Ġvad": 16684, "ĠìĹĦ": 16685, "Ġsous": 16686, "Ġsphere": 16687, "BY": 16688, "ĠPed": 16689, "igned": 16690, "Ġwheat": 16691, "Ġunders": 16692, "Ġevolve": 16693, "Ġdeclar": 16694, "Ġlightly": 16695, "Ġidentifying": 16696, "æĦıæĢĿ": 16697, "Ġlegendary": 16698, "Ġgenuine": 16699, "Ġgrind": 16700, "ĠUne": 16701, "geben": 16702, "Ġbicy": 16703, "Ġjumps": 16704, "Ġprovince": 16705, "ziÄĻ": 16706, "Ġ×IJ׳×Ļ": 16707, "Ġhoc": 16708, "Ġбл": 16709, "ĠGrad": 16710, "Ġrevenge": 16711, "ĠاÙĦت": 16712, "ooh": 16713, "æĭľ": 16714, "аÑĨии": 16715, "å¹³": 16716, "Ġelectro": 16717, "ĠëIJIJ": 16718, "ãģ§ãģ¯": 16719, "Ġfals": 16720, "riel": 16721, "oker": 16722, "ĠExcellent": 16723, "ĠMorgan": 16724, "Ġbrick": 16725, "Ġsubstantial": 16726, "Ġpollution": 16727, "ĠTür": 16728, "ĠEvet": 16729, "Ġlung": 16730, "ãģĸ": 16731, "×Ļש": 16732, "ommes": 16733, "Ġrealizing": 16734, "Ġhumble": 16735, "ĠLock": 16736, "Ġbod": 16737, "Ġìĸ¸": 16738, "Ġpeers": 16739, "uzz": 16740, "Ġembedded": 16741, "Ġclaro": 16742, "Ġaggreg": 16743, "Ġemployers": 16744, "ĠRaj": 16745, "Ġãģ¨": 16746, "ĠYi": 16747, "Ġjeu": 16748, "aters": 16749, "Ġstrikes": 16750, "nos": 16751, "autres": 16752, "dr": 16753, "opher": 16754, "ĠApparently": 16755, "íĺĦ": 16756, "Ġinfant": 16757, "اب": 16758, "ÑĤÑĭ": 16759, "íĽ": 16760, "Ú¯": 16761, "Ġredes": 16762, "acaģım": 16763, "ĠDAVID": 16764, "ĠChicken": 16765, "Ġperspectives": 16766, "Ġviewer": 16767, "Ġshar": 16768, "ĠпÑĢоиз": 16769, "ligt": 16770, "eros": 16771, "itable": 16772, "илоÑģÑĮ": 16773, "ĠdifÃŃ": 16774, "´ëį°": 16775, "Ġretired": 16776, "Ġthats": 16777, "zenie": 16778, "beiten": 16779, "Ġmycket": 16780, "ĠRab": 16781, "Ġinflamm": 16782, "ì°®": 16783, "Ġdum": 16784, "Ġdaddy": 16785, "æľŁ": 16786, "Ġimmers": 16787, "Ġplaylist": 16788, "à¯Ĩ": 16789, "Ġtraum": 16790, "Ġrefuse": 16791, "step": 16792, "à®ļ": 16793, "cup": 16794, "Ġpops": 16795, "rimin": 16796, "ayım": 16797, "Ġald": 16798, "Ġunnecess": 16799, "Ġdah": 16800, "ĠIrish": 16801, "Ġcompr": 16802, "laÅŁ": 16803, "TP": 16804, "Ġtranslated": 16805, "Sc": 16806, "ceÄŁim": 16807, "´IJ": 16808, "Ġdrei": 16809, "ĠлÑİдей": 16810, "Ġquiero": 16811, "Ġhele": 16812, "zlich": 16813, "Ġapples": 16814, "Ġdistricts": 16815, "Ġcredits": 16816, "Ġasp": 16817, "Ġëĭ¨": 16818, "oral": 16819, "å½±": 16820, "Ġstepping": 16821, "ĠVa": 16822, "Ġgains": 16823, "65": 16824, "Ġnuestra": 16825, "eday": 16826, "assador": 16827, "ĠLind": 16828, "Ġcrops": 16829, "ciendo": 16830, "igue": 16831, "Ġbana": 16832, "Am": 16833, "Ġpent": 16834, "Ġaddiction": 16835, "Ġpackaging": 16836, "äd": 16837, "ª¨": 16838, "Ġperquè": 16839, "Ġcampaigns": 16840, "Ġsteep": 16841, "Ġneue": 16842, "Ġembarrassed": 16843, "Ġdistinction": 16844, "itzer": 16845, "åijĬ": 16846, "Ġregistration": 16847, "Ġllam": 16848, "ĠAlmighty": 16849, "liest": 16850, "Ġuz": 16851, "nak": 16852, "çº": 16853, "Ġteraz": 16854, "iamente": 16855, "Ġtransactions": 16856, "Ġcôt": 16857, "Ġswitched": 16858, "Ġcombo": 16859, "Ġprayers": 16860, "Ġinternship": 16861, "Ġaddresses": 16862, "Ġcharity": 16863, "ĠWOO": 16864, "Ġbait": 16865, "è¿ĩ": 16866, "Ġ�": 16867, "Ġfica": 16868, "ĠTyler": 16869, "aru": 16870, "Ġatoms": 16871, "ĠLevel": 16872, "ĠпоÑĤом": 16873, "Ġfame": 16874, "ulk": 16875, "Ġteaches": 16876, "Ġrebuild": 16877, "едÑĮ": 16878, "ĠIndonesia": 16879, "ushi": 16880, "ĠShort": 16881, "Ġensuring": 16882, "fs": 16883, "ele": 16884, "Ġmarginal": 16885, "Ġconclude": 16886, "amt": 16887, "Ġverify": 16888, "ĠMcDonald": 16889, "Ġskal": 16890, "Ġreconst": 16891, "ĠMann": 16892, "Ġbasement": 16893, "Ġtransformed": 16894, "Ġoccasionally": 16895, "zone": 16896, "ĠDans": 16897, "Ġкакой": 16898, "Ġdiagnosed": 16899, "ĠÏĦα": 16900, "Ġcommands": 16901, "Ġpresidential": 16902, "Ġabb": 16903, "Ġbracket": 16904, "ĠLem": 16905, "Ã¥ng": 16906, "Ġfavorites": 16907, "Ġrevol": 16908, "ĠíĬ¹": 16909, "Ġharass": 16910, "éħ": 16911, "Ġcleans": 16912, "ständ": 16913, "Ġknocked": 16914, "Ġpeoples": 16915, "Ġmusicians": 16916, "Ġmutual": 16917, "ĠCold": 16918, "88": 16919, "zej": 16920, "atie": 16921, "ĠHonor": 16922, "Ġobsessed": 16923, "ĠMUSIC": 16924, "ĠBreak": 16925, "úng": 16926, "Ġmodify": 16927, "Ġsöyle": 16928, "Ġ×ŀ×Ķ": 16929, "ĠOnline": 16930, "fo": 16931, "ĠMiller": 16932, "Ġliking": 16933, "Ġinhab": 16934, "Ġgratitude": 16935, "ĠJournal": 16936, "arness": 16937, "John": 16938, "ĠGit": 16939, "åīĽ": 16940, "Ġsincere": 16941, "ĠSci": 16942, "ĠEli": 16943, "Ġsymbols": 16944, "Ġmanually": 16945, "εÏĤ": 16946, "ĠвÑĸд": 16947, "ĠFat": 16948, "Ġlabels": 16949, "Ġsophisticated": 16950, "umps": 16951, "Ġreleases": 16952, "Ġ47": 16953, "ĠOM": 16954, "ê°Ģë": 16955, "ĠBien": 16956, "ĠRef": 16957, "è¨ĺ": 16958, "ĠSta": 16959, "ĠEgg": 16960, "Ġindicator": 16961, "pson": 16962, "Ġnasıl": 16963, "Right": 16964, "Ġconvey": 16965, "Ġknot": 16966, "Ġconnects": 16967, "ulas": 16968, "Ġpreced": 16969, "Ġinequality": 16970, "amiento": 16971, "Ġreply": 16972, "OY": 16973, "Ġdismiss": 16974, "ĠëIJľ": 16975, "çĦ¡": 16976, "ĠÑħоÑĢоÑĪо": 16977, "Ġméd": 16978, "Ġrandomly": 16979, "ĠOnt": 16980, "uard": 16981, "Ġpulls": 16982, "ĠÑĤепеÑĢÑĮ": 16983, "ĠNeed": 16984, "ĠSoft": 16985, "Ġstrengths": 16986, "Ġgoed": 16987, "umen": 16988, "æŃ»": 16989, "Ġíݸ": 16990, "Ġдоб": 16991, "Ġclarity": 16992, "ĠAi": 16993, "Ġballoon": 16994, "ĠPand": 16995, "ĠìķĦëĭ": 16996, "Ġshiny": 16997, "Ġsmallest": 16998, "onia": 16999, "hill": 17000, "oting": 17001, "Ġeing": 17002, "Ġmerely": 17003, "Ġseus": 17004, "Ġнеп": 17005, "ĠíĨµ": 17006, "Ġguides": 17007, "Ġspecialist": 17008, "Ġsteak": 17009, "ãĤĪãģĨ": 17010, "Ġmigration": 17011, "quele": 17012, "Ġruined": 17013, "Ġpupp": 17014, "女": 17015, "Ġkend": 17016, "angan": 17017, "Ġpalm": 17018, "Ġunfair": 17019, "Ġzm": 17020, "ĠDV": 17021, "chester": 17022, "иÑİ": 17023, "Ġooh": 17024, "erg": 17025, "ATH": 17026, "°©": 17027, "åĵª": 17028, "rison": 17029, "Ġinvolving": 17030, "Ġpartly": 17031, "ançais": 17032, "Ġvow": 17033, "Ġprominent": 17034, "Ġcryst": 17035, "iba": 17036, "Ġdeserves": 17037, "Ġovert": 17038, "Ġsensit": 17039, "ĠWhe": 17040, "Ġtighten": 17041, "Ġintimid": 17042, "Ġaliment": 17043, "will": 17044, "Ġstrengthen": 17045, "ĠTan": 17046, "åıĪ": 17047, "ãģĹãģ¾ãģĻ": 17048, "oni": 17049, "ĠMun": 17050, "Ġproph": 17051, "Ġrehears": 17052, "ĠKle": 17053, "Ġveces": 17054, "Ġwondered": 17055, "oki": 17056, "Ġsenses": 17057, "´ìĭ": 17058, "Æ°á»Ľ": 17059, "ĠÈĻi": 17060, "Ġmuchos": 17061, "Ġwatches": 17062, "ortunate": 17063, "ĠJuan": 17064, "ìŀĸìķĦ": 17065, "ÑĢе": 17066, "ei": 17067, "ionen": 17068, "Ġexperimental": 17069, "Ġdaughters": 17070, "à¸Ľ": 17071, "Ġmentally": 17072, "becca": 17073, "aware": 17074, "ìĦĿ": 17075, "Ġwhatsoever": 17076, "Ġenables": 17077, "ĠLow": 17078, "oid": 17079, "à¸Ĭ": 17080, "ód": 17081, "غ": 17082, "Ġconstructed": 17083, "ĠLadies": 17084, "Ġaccused": 17085, "Ġан": 17086, "Dan": 17087, "Ġspawn": 17088, "Ġcontainers": 17089, "Ġartistic": 17090, "ıp": 17091, "Ġdiscl": 17092, "Ġautres": 17093, "inas": 17094, "ĠNation": 17095, "Ġnag": 17096, "bean": 17097, "whe": 17098, "ľëıĦ": 17099, "ĠSeoul": 17100, "Ġíı¬": 17101, "ĠNich": 17102, "Ġcomplement": 17103, "Ġinterven": 17104, "ĠModel": 17105, "ĠOrange": 17106, "namon": 17107, "Ġcalculation": 17108, "see": 17109, "Ġustedes": 17110, "Ġleb": 17111, "Ġdoct": 17112, "Ñĸн": 17113, "Ġfoster": 17114, "Ġelastic": 17115, "ĠAhh": 17116, "Ġace": 17117, "ĠPink": 17118, "ĠJeg": 17119, "Ġdeer": 17120, "ãģĹãģĦ": 17121, "sis": 17122, "Ġjako": 17123, "ĠEmma": 17124, "ÑģÑĤвенно": 17125, "Ġportrait": 17126, "Ġmaker": 17127, "Ġaument": 17128, "ÑĢоб": 17129, "Ġairplane": 17130, "Ġtransparency": 17131, "Ġadjustment": 17132, "ĠCDC": 17133, "çon": 17134, "Ġuploaded": 17135, "ĠдейÑģÑĤв": 17136, "ĠгоÑĤов": 17137, "Ġiter": 17138, "Ġcurse": 17139, "ôn": 17140, "merce": 17141, "aran": 17142, "Ġleak": 17143, "çµIJ": 17144, "Ġabsence": 17145, "Ñģкий": 17146, "Ġreaders": 17147, "aler": 17148, "Ġbeneath": 17149, "ango": 17150, "hetic": 17151, "Ġfinns": 17152, "Ġpoop": 17153, "Ġduplic": 17154, "Hi": 17155, "igs": 17156, "ologically": 17157, "opp": 17158, "Ġdizer": 17159, "ĠAllen": 17160, "Ġgli": 17161, "Ġacceleration": 17162, "Ġvitamin": 17163, "ãĥŃ": 17164, "vä": 17165, "ĠAccess": 17166, "à®Ļ": 17167, "rás": 17168, "Ġappreciated": 17169, "Ġnah": 17170, "Ġposter": 17171, "Ġtale": 17172, "Ġhighlighted": 17173, "æĸĩ": 17174, "żeli": 17175, "Ġblockchain": 17176, "Ġmicrow": 17177, "Ġcinema": 17178, "ĠChang": 17179, "ĠSearch": 17180, "usters": 17181, "ĠZero": 17182, "ĠDivision": 17183, "ÑĢаÑģ": 17184, "Ġscare": 17185, "Ġjelly": 17186, "ĠAdministration": 17187, "SO": 17188, "Ġlined": 17189, "Ġê°Ħ": 17190, "Ġgeben": 17191, "Ġsoda": 17192, "Ġwinners": 17193, "³¼": 17194, "ÙĴ": 17195, "ĠAmb": 17196, "åķıé¡Į": 17197, "åĶ": 17198, "Ġpeg": 17199, "å·±": 17200, "43": 17201, "Ġraus": 17202, "Ġrewards": 17203, "Ġinclus": 17204, "Ġhighway": 17205, "Ġhah": 17206, "Ġmultiplied": 17207, "Ġsẽ": 17208, "Ġdisciples": 17209, "Ġning": 17210, "Ġdressing": 17211, "Ġattributes": 17212, "ĠMosc": 17213, "ĠGreece": 17214, "Ġsek": 17215, "ĠLearn": 17216, "Ġjus": 17217, "rendre": 17218, "Ġpersonne": 17219, "plete": 17220, "Ġplacing": 17221, "Ġluego": 17222, "illance": 17223, "ĠобÑī": 17224, "Ġprovision": 17225, "Ġlion": 17226, "tra": 17227, "boards": 17228, "Ġbehaviour": 17229, "hey": 17230, "Ġsubscription": 17231, "Ġprotagon": 17232, "ãĥ£": 17233, "Ġvara": 17234, "ĠÅŁu": 17235, "Ġhaha": 17236, "Ġteaspoon": 17237, "æŁ": 17238, "avoir": 17239, "Ġcrypto": 17240, "ĠÑģÑĤаÑĢ": 17241, "ĠStore": 17242, "abs": 17243, "ĠStudents": 17244, "Ġlaund": 17245, "into": 17246, "Ġapproached": 17247, "°ľ": 17248, "ÑĥÑİÑī": 17249, "ĠLabor": 17250, "otes": 17251, "iatric": 17252, "ĠgroÃŁ": 17253, "utive": 17254, "Ġид": 17255, "ĠGib": 17256, "Ġplacement": 17257, "ĠdifÃŃcil": 17258, "Ġfrog": 17259, "ĠвÑģеÑħ": 17260, "ĠJr": 17261, "azed": 17262, "ÑĥÑī": 17263, "Ġê¼": 17264, "frame": 17265, "аеÑĪÑĮ": 17266, "Ġlockdown": 17267, "åij³": 17268, "Ġmedi": 17269, "Ġ×Ķ×ŀ×": 17270, "ений": 17271, "emale": 17272, "ì¢ħ": 17273, "ateral": 17274, "Ġdistant": 17275, "Ġbears": 17276, "Ġjournalist": 17277, "解": 17278, "ĠMarshall": 17279, "ĠIhnen": 17280, "uetooth": 17281, "bag": 17282, "ĠÄijã": 17283, "ĠHighness": 17284, "Ġì°į": 17285, "ика": 17286, "ĠWu": 17287, "ĠFran": 17288, "Ġpeng": 17289, "Ġfon": 17290, "Ġhypothesis": 17291, "ĠÑĢÑĥ": 17292, "Ġly": 17293, "×ļ": 17294, "ìĽĶ": 17295, "ĠRadio": 17296, "à¸ŀ": 17297, "Dav": 17298, "Ġembarrassing": 17299, "ĠìŀĪìĸ´": 17300, "Ġcasting": 17301, "Ġcage": 17302, "ĠPsych": 17303, "ĠìĿ¼ëĭ¨": 17304, "Ġž": 17305, "imb": 17306, "Ġdirectors": 17307, "SH": 17308, "ĠÏĦην": 17309, "á»ģu": 17310, "ĠkonuÅŁ": 17311, "Ġoptional": 17312, "quarters": 17313, "iker": 17314, "ĠSant": 17315, "Ġverses": 17316, "ë¶Ģ": 17317, "Ġolar": 17318, "ĠÏĩ": 17319, "ãĥķ": 17320, "Ġγια": 17321, "ĠImm": 17322, "Ġcontroversial": 17323, "Ġersten": 17324, "Ġrecip": 17325, "ĠChristianity": 17326, "Ġê´ľ": 17327, "ordon": 17328, "×ķש": 17329, "Ġslash": 17330, "ĠPf": 17331, "ÑĥдÑĮ": 17332, "×ķ×Ŀ": 17333, "ĠPerry": 17334, "Ġmamy": 17335, "Ġbackgrounds": 17336, "Ġà®İன": 17337, "Ġpendant": 17338, "ĠColumbia": 17339, "Ġinverse": 17340, "ĠÑĩеÑĢез": 17341, "Ġsv": 17342, "Ġdigging": 17343, "41": 17344, "chem": 17345, "Ġnavigation": 17346, "ĠShin": 17347, "ĠFront": 17348, "PD": 17349, "Ġbearing": 17350, "ĠWasser": 17351, "Ġwax": 17352, "ĠCHRIS": 17353, "ching": 17354, "Ġpressed": 17355, "El": 17356, "ĠDal": 17357, "onsin": 17358, "Ġbinding": 17359, "Ñģкой": 17360, "poons": 17361, "Ġmock": 17362, "arest": 17363, "кÑĢа": 17364, "MM": 17365, "Ġcorrupt": 17366, "storm": 17367, "Ġrefres": 17368, "ĠCoach": 17369, "llä": 17370, "ĠTHIS": 17371, "Ġparag": 17372, "Ġìĵ°": 17373, "pool": 17374, "Ġbillions": 17375, "Ġê¹Ģ": 17376, "group": 17377, "Ġwelcoming": 17378, "cellence": 17379, "ĠDuke": 17380, "긴": 17381, "Ġprimera": 17382, "ìł¸": 17383, "Ġpond": 17384, "Ġstatue": 17385, "Ġ구ë": 17386, "Ġhatch": 17387, "Ġinstrumental": 17388, "Ġresidential": 17389, "커": 17390, "Ġaccepting": 17391, "oshi": 17392, "date": 17393, "ĠìĶ¨": 17394, "Ġplanted": 17395, "Ġjoking": 17396, "ĠìĦľ": 17397, "Ġhated": 17398, "ĠÑĢаÑģÑģк": 17399, "Ġslept": 17400, "Ġpackages": 17401, "Ġislands": 17402, "esen": 17403, "ģı": 17404, "Ġdiagon": 17405, "ĠOsc": 17406, "Ġmesh": 17407, "Ġscales": 17408, "arity": 17409, "ĠDefense": 17410, "ãģ¡ãĤĩ": 17411, "ĠLewis": 17412, "ĠÑģегоднÑı": 17413, "Ġflies": 17414, "uinely": 17415, "ĠConsider": 17416, "Ġstark": 17417, "hew": 17418, "ĠAsÃŃ": 17419, "³´ë": 17420, "Ġpropose": 17421, "Ġíķĺë©´": 17422, "odo": 17423, "ĠNormally": 17424, "Ġheeft": 17425, "ĠHarris": 17426, "gro": 17427, "ĠBlood": 17428, "base": 17429, "ĠiOS": 17430, "Ġtouches": 17431, "Ġinspir": 17432, "Ġ×ĵ": 17433, "Ġbinary": 17434, "Ġì¶Ķ": 17435, "Ġserial": 17436, "Ġion": 17437, "Ġunemployment": 17438, "Ġodds": 17439, "ĠFab": 17440, "ĠFBI": 17441, "BRUN": 17442, "Ġweights": 17443, "νο": 17444, "atile": 17445, "Ġnurses": 17446, "Ġinvolvement": 17447, "ĠíĶ¼": 17448, "Ġgovernance": 17449, "ĠâĤ¬": 17450, "ÑĢÑĥп": 17451, "ierra": 17452, "íĺķ": 17453, "ĠJerry": 17454, "Ġbeard": 17455, "Ġsalvation": 17456, "ĠAlong": 17457, "gentle": 17458, "ĠKi": 17459, "bol": 17460, "ĠPlat": 17461, "Ġhasht": 17462, "è¿ij": 17463, "Ġware": 17464, "Ġpartie": 17465, "ycz": 17466, "Ġintr": 17467, "Fih": 17468, "nent": 17469, "Ġcheat": 17470, "ilen": 17471, "Ġë¯": 17472, "orie": 17473, "Ġfácil": 17474, "etric": 17475, "Ġaffecting": 17476, "unciation": 17477, "Ġaffairs": 17478, "Ġbee": 17479, "Ġviewing": 17480, "Ġorang": 17481, "ĠLan": 17482, "ĠСÑĤ": 17483, "ä¸ĸ": 17484, "ĠMes": 17485, "ĥģ": 17486, "erie": 17487, "Ġespa": 17488, "Ġinterpre": 17489, "Ġpossess": 17490, "Ġpurely": 17491, "rito": 17492, "found": 17493, "asma": 17494, "ìłģìĿ¸": 17495, "Ġexamine": 17496, "ĠÑĥм": 17497, "Ġbesch": 17498, "ĠTomorrow": 17499, "ĠBlock": 17500, "Ġvariant": 17501, "Ġpreference": 17502, "Ġcoaches": 17503, "Ġmedications": 17504, "ĠíĺĦ": 17505, "Ġempire": 17506, "ëĦ¤": 17507, "ĠIllinois": 17508, "Ġcrispy": 17509, "Ġthì": 17510, "Ġbees": 17511, "77": 17512, "Ġglow": 17513, "èº": 17514, "ĠStudies": 17515, "åIJĦ": 17516, "ĠChallenge": 17517, "Ġunlikely": 17518, "Ч": 17519, "ıyorsun": 17520, "DIE": 17521, "Ġminimize": 17522, "izard": 17523, "Ġún": 17524, "Ġencontrar": 17525, "ĠKill": 17526, "å»": 17527, "Ġvanilla": 17528, "ĠGrant": 17529, "ĠGT": 17530, "sea": 17531, "Ġsought": 17532, "вод": 17533, "Ġnäm": 17534, "ĠAunt": 17535, "OWN": 17536, "Ġpumpkin": 17537, "stellen": 17538, "Ġrag": 17539, "егда": 17540, "Ġstoryt": 17541, "Ġforum": 17542, "æ©Ł": 17543, "Ġestaba": 17544, "uche": 17545, "Ġcongress": 17546, "ĠRey": 17547, "Ġdramatically": 17548, "ĠSport": 17549, "ĠYellow": 17550, "Ġê³ĦìĨį": 17551, "Ġdisgusting": 17552, "ĠRecent": 17553, "Ġacquired": 17554, "Ġcables": 17555, "çĶļ": 17556, "din": 17557, "Ġvisto": 17558, "Ġcommunicating": 17559, "ÑģÑĤавлÑı": 17560, "еÑģÑĤо": 17561, "ãĥ»ãĥ»ãĥ»": 17562, "Ġrég": 17563, "Ġsocks": 17564, "Ġproces": 17565, "because": 17566, "Ġutter": 17567, "Ġcolocar": 17568, "Ġnewest": 17569, "Ġgramm": 17570, "表": 17571, "ä¸įçŁ¥éģĵ": 17572, "Ġshifting": 17573, "Ġcarrier": 17574, "ĠÑģкоÑĢ": 17575, "ĠSchw": 17576, "Ġexecuted": 17577, "Ġmaintained": 17578, "ĠÏĨ": 17579, "ĠMoses": 17580, "Ġdisse": 17581, "Ġhorr": 17582, "ãĢľ": 17583, "Ġrally": 17584, "Ġallem": 17585, "ĠEventually": 17586, "Ġdiyor": 17587, "lvania": 17588, "Ġschnell": 17589, "Ġê³¼": 17590, "Ġ매": 17591, "Ġstruggles": 17592, "late": 17593, "Ġclarify": 17594, "ément": 17595, "Ġmultiplic": 17596, "ибо": 17597, "Ġjourn": 17598, "Ġfragr": 17599, "Ġsurprisingly": 17600, "Ġdesperate": 17601, "52": 17602, "Ġsul": 17603, "ĠRead": 17604, "ĠFried": 17605, "Ġmond": 17606, "woo": 17607, "Ġorganizing": 17608, "ãģĹãĤĩãģĨ": 17609, "ĠSoon": 17610, "ĠвопÑĢоÑģ": 17611, "ĠNur": 17612, "ĠÐĹд": 17613, "Ġspider": 17614, "еÑģÑı": 17615, "Ġtutorials": 17616, "Ġnutrients": 17617, "orer": 17618, "Ġcoefficient": 17619, "Ġarrangement": 17620, "Ġpricing": 17621, "nan": 17622, "yu": 17623, "BL": 17624, "Ġtribe": 17625, "ĠHoward": 17626, "unks": 17627, "Ġnewer": 17628, "Ġprovin": 17629, "Ġprediction": 17630, "hos": 17631, "Ġolsun": 17632, "ĠAround": 17633, "Ġvier": 17634, "ĠÑģÑĤоÑĢон": 17635, "Ġvalley": 17636, "ĠEla": 17637, "ifi": 17638, "Ġgalaxy": 17639, "Ġtranqu": 17640, "Ġadvers": 17641, "ĠTemple": 17642, "iffs": 17643, "igence": 17644, "èĩªå·±": 17645, "Ġkönnte": 17646, "ĠÄijó": 17647, "Did": 17648, "Ġphotographs": 17649, "ĠAWS": 17650, "ÑĨиÑı": 17651, "Ġguards": 17652, "Ġappointed": 17653, "ĠGil": 17654, "Ġмом": 17655, "Ġcod": 17656, "ĠUnlike": 17657, "Ġevenly": 17658, "isconsin": 17659, "Ġestou": 17660, "Ġmnie": 17661, "ĠExec": 17662, "ĠMV": 17663, "ĠEine": 17664, "ä¿¡": 17665, "ĠRoger": 17666, "ĠFac": 17667, "ĠList": 17668, "Ġfuer": 17669, "аеÑĤе": 17670, "omed": 17671, "Ġattraction": 17672, "èī²": 17673, "Ġterrain": 17674, "ĠDrop": 17675, "Ġcorporations": 17676, "Ġsciences": 17677, "Ġthrone": 17678, "ãģĦãģŁ": 17679, "Ġaj": 17680, "ĠRot": 17681, "çī¹": 17682, "Ġsupporters": 17683, "ĠBere": 17684, "Here": 17685, "Ġdiferentes": 17686, "Ġsignificance": 17687, "Ïĥη": 17688, "æĪij覺å¾Ĺ": 17689, "Ġclamp": 17690, "ĠëĮĢë": 17691, "Ġfabulous": 17692, "rez": 17693, "æĮģ": 17694, "Ġassumptions": 17695, "uther": 17696, "wid": 17697, "pot": 17698, "è¿İ": 17699, "Ġyan": 17700, "ulin": 17701, "ÑĢÑĭв": 17702, "ĠSlow": 17703, "ĠPennsy": 17704, "Ġíķ´ìĦľ": 17705, "Ġmeio": 17706, "Ġwealthy": 17707, "ĠEight": 17708, "Ġpulse": 17709, "Ġfriction": 17710, "idity": 17711, "ĠHoll": 17712, "iyorum": 17713, "Ġsounded": 17714, "ĠCarr": 17715, "Ġfork": 17716, "âĺ": 17717, "ĠPA": 17718, "Ġconspir": 17719, "Ġcoding": 17720, "rt": 17721, "ĠTyp": 17722, "Ġìĸij": 17723, "Ġпог": 17724, "Ġmiser": 17725, "ĠÑģмоÑĤÑĢ": 17726, "ĠSweden": 17727, "Ġolarak": 17728, "ĠZhang": 17729, "ĠChi": 17730, "ĠTitan": 17731, "Ġscreening": 17732, "ĠSpider": 17733, "ĠÅŀimdi": 17734, "Ġobstacles": 17735, "lara": 17736, "Ġchallenged": 17737, "pse": 17738, "TON": 17739, "ụ": 17740, "ĠPi": 17741, "Ġlagi": 17742, "ieurs": 17743, "Ġhurting": 17744, "Ġneglect": 17745, "Ġgenerating": 17746, "Ġyoungest": 17747, "Ġaudit": 17748, "ĠÑĢез": 17749, "Ïģά": 17750, "Ġdonate": 17751, "ĠPDF": 17752, "Ġvisits": 17753, "Ġcruise": 17754, "PP": 17755, "aser": 17756, "Ġwsp": 17757, "backs": 17758, "ivals": 17759, "ãģĨãĤĵ": 17760, "Ġdeve": 17761, "Ġproport": 17762, "Ġcath": 17763, "ĠEffect": 17764, "Ġwinds": 17765, "ĠìĻĶ": 17766, "Ġcharts": 17767, "Ġsama": 17768, "Ġautomation": 17769, "Ġпока": 17770, "Ġolan": 17771, "Ġboats": 17772, "Ġcafe": 17773, "Ġdenied": 17774, "ĠMama": 17775, "Ġblocking": 17776, "ĠThor": 17777, "Ġphenomenal": 17778, "Ġstakeholders": 17779, "Ġunos": 17780, "ÑĥеÑĤ": 17781, "ĠAbraham": 17782, "ãģ§ãĤĤ": 17783, "Ġdetection": 17784, "Ġjuris": 17785, "Ġpowered": 17786, "zial": 17787, "Ġwelfare": 17788, "Ġupgrad": 17789, "Ġmożna": 17790, "ĠCase": 17791, "cular": 17792, "ĶìĿ´": 17793, "ãĥģ": 17794, "ĠGuess": 17795, "Ġcycles": 17796, "ä¾ĭ": 17797, "給": 17798, "rock": 17799, "umi": 17800, "Ġelite": 17801, "Ġquè": 17802, "åł±": 17803, "ÑĤом": 17804, "Ġshore": 17805, "gunta": 17806, "Ġku": 17807, "Ġfaithful": 17808, "ĠJeremy": 17809, "aid": 17810, "à·": 17811, "ugal": 17812, "å°įåķĬ": 17813, "ĠVel": 17814, "Ġvrai": 17815, "stell": 17816, "¨¸": 17817, "Ġkol": 17818, "è½": 17819, "Ġquanto": 17820, "ĠзаÑĢ": 17821, "Ġ2002": 17822, "esy": 17823, "Ġreserve": 17824, "ĠмоменÑĤ": 17825, "Ġdeployed": 17826, "Ġdefining": 17827, "Ġsau": 17828, "Ġgaat": 17829, "\")": 17830, "Ġtransmit": 17831, "Ġpublishing": 17832, "Ġranking": 17833, "Ġoffense": 17834, "Ġ46": 17835, "pin": 17836, "ĠTaking": 17837, "Ġentitled": 17838, "Ġgenuinely": 17839, "Ġvariations": 17840, "Ġfinde": 17841, "Ġtau": 17842, "Ġunfortunate": 17843, "ĠRah": 17844, "ports": 17845, "ĠcÅ": 17846, "Ġmonkey": 17847, "Ġbrac": 17848, "wei": 17849, "lung": 17850, "Ġartif": 17851, "Ġsyrup": 17852, "ĠÐĶав": 17853, "Ġlifted": 17854, "Ġchez": 17855, "ĠAdvent": 17856, "ĠStock": 17857, "Ġdol": 17858, "мен": 17859, "иÑĪÑĮ": 17860, "Ġyn": 17861, "gio": 17862, "det": 17863, "Ġdesse": 17864, "Ġgri": 17865, "ĠChairman": 17866, "çħ": 17867, "Ġcuenta": 17868, "anim": 17869, "Ġcrab": 17870, "Ġescal": 17871, "Ġpremière": 17872, "ĠGef": 17873, "Ġdining": 17874, "Ġseventh": 17875, "Ġchasing": 17876, "ĠTower": 17877, "Ġbrutal": 17878, "Ġfundamentally": 17879, "ãģ¨ãģĨ": 17880, "лениÑı": 17881, "stage": 17882, "Ġacquis": 17883, "Ġcylinder": 17884, "Ġcommander": 17885, "mem": 17886, "ĠUV": 17887, "happy": 17888, "Ġepsilon": 17889, "Ġinvitation": 17890, "Ġfarmer": 17891, "chair": 17892, "Ġdestiny": 17893, "Ġsovere": 17894, "ĠHebrew": 17895, "Ġservant": 17896, "Ġbew": 17897, "Ġgast": 17898, "uties": 17899, "Ġadministrative": 17900, "ĠCommand": 17901, "éta": 17902, "Ġnitrogen": 17903, "ê·¼": 17904, "Ġabi": 17905, "Ġvillain": 17906, "Ġblanket": 17907, "ĠSend": 17908, "Ġbeaten": 17909, "²Ħ": 17910, "Ġvolunt": 17911, "Ġscholar": 17912, "ĠEmperor": 17913, "Ġ43": 17914, "vable": 17915, "ĠDus": 17916, "ĠGU": 17917, "Ġtargeting": 17918, "www": 17919, "Ġamendment": 17920, "ìĨĮë": 17921, "Ġting": 17922, "Ġnasty": 17923, "Ġgauge": 17924, "ĠÑĢод": 17925, "ĠHans": 17926, "Your": 17927, "αν": 17928, "Ġprojet": 17929, "ĠHawaii": 17930, "Ġsuspicious": 17931, "Ġschw": 17932, "Ġremoval": 17933, "Ġintrig": 17934, "ĠMU": 17935, "Ġponto": 17936, "ा": 17937, "ĠобÑĢаз": 17938, "Ġguessing": 17939, "pace": 17940, "Ġmothers": 17941, "Ġmillimeter": 17942, "ление": 17943, "没æľī": 17944, "Ġavailability": 17945, "icz": 17946, "æѤ": 17947, "Ġfract": 17948, "Ġbases": 17949, "km": 17950, "ĠBTS": 17951, "ĠField": 17952, "Ġdzie": 17953, "Ġsegundo": 17954, "ĠëĤĺëĬĶ": 17955, "Ġlegitimate": 17956, "imas": 17957, "Ġвн": 17958, "Ġcorruption": 17959, "Ġsmash": 17960, "ĠValent": 17961, "Ġaligned": 17962, "ĠPennsylvania": 17963, "Ġgab": 17964, "ĠEun": 17965, "enth": 17966, "ĠMorning": 17967, "Ġcandle": 17968, "Ġbackpack": 17969, "ĠIslamic": 17970, "ações": 17971, "Ġencry": 17972, "Ġmushrooms": 17973, "íĮĮ": 17974, "dit": 17975, "Ġtransit": 17976, "ĠWisconsin": 17977, "Ġparticipated": 17978, "ĠIls": 17979, "Ġunfold": 17980, "¶Ģë": 17981, "Ġprofits": 17982, "Ġwarming": 17983, "ĠGang": 17984, "Ġnetworking": 17985, "Ġmega": 17986, "Ġthoroughly": 17987, "lements": 17988, "ĠHm": 17989, "Ġdeciding": 17990, "Ġemotionally": 17991, "Ġexhausted": 17992, "ĠÐŁÐ¾ÑĤ": 17993, "cido": 17994, "ĠHTML": 17995, "Ġcopyright": 17996, "Ġmelody": 17997, "yim": 17998, "Ġanders": 17999, "oshop": 18000, "Ġë³¼": 18001, "Ġathlete": 18002, "ĠGE": 18003, "Ġfrequent": 18004, "Ġdesires": 18005, "Ġneeding": 18006, "ĠYun": 18007, "Ġrifle": 18008, "Ġlover": 18009, "'T": 18010, "Ġdense": 18011, "Ġtão": 18012, "Ġnotified": 18013, "Ġidi": 18014, "ìĹŃ": 18015, "íĨ": 18016, "Ġinteracting": 18017, "Ġrapport": 18018, "еÑĢи": 18019, "ski": 18020, "Ġbesser": 18021, "Ġmanufacturer": 18022, "ĠKyle": 18023, "Ġaccountable": 18024, "ĠSak": 18025, "ĠPil": 18026, "ĠDomin": 18027, "Ġpresum": 18028, "ĠÐĴÑģе": 18029, "Ġvinegar": 18030, "Ġguaranteed": 18031, "çľĭåĪ°": 18032, "Ġhandled": 18033, "éŁ³": 18034, "cat": 18035, "Ġcivilization": 18036, "Ġaccomp": 18037, "ĠVM": 18038, "émon": 18039, "Ġdeze": 18040, "Ġgrades": 18041, "Ġsollte": 18042, "Ġstaring": 18043, "×IJת": 18044, "arnt": 18045, "Ġhorizon": 18046, "Ġtravail": 18047, "hour": 18048, "第ä¸Ģ": 18049, "ĠED": 18050, "ĠDak": 18051, "Ġny": 18052, "Ġconve": 18053, "ĠCham": 18054, "Ġfirms": 18055, "ĠLiu": 18056, "ĠÑģÑĤÑĢан": 18057, "Ġlibert": 18058, "Ġlenses": 18059, "Ġintake": 18060, "ĠвÑĭб": 18061, "Ġmensen": 18062, "hel": 18063, "Ġpractition": 18064, "Ġ350": 18065, "ãĤ³": 18066, "FO": 18067, "Ġbeds": 18068, "Ġancestors": 18069, "ĠìĹĦì²Ń": 18070, "Ġdisturb": 18071, "ĠLastly": 18072, "ĠSupport": 18073, "ีà¹ī": 18074, "ĠCorona": 18075, "Ġenthusi": 18076, "Ġвозм": 18077, "ĠìĤ¬ëŀĮë": 18078, "Ġ52": 18079, "bird": 18080, "Ġreduces": 18081, "ĠìŀĪìĿĦ": 18082, "ĠGene": 18083, "êµIJ": 18084, "ÄĻp": 18085, "ĠÃľber": 18086, "Ġconcerning": 18087, "user": 18088, "Ġconcentrate": 18089, "ĠWHAT": 18090, "ishop": 18091, "onymous": 18092, "nold": 18093, "Ġsuggesting": 18094, "©°": 18095, "ĠFish": 18096, "........": 18097, "Ġvessel": 18098, "Ġtrabajo": 18099, "ãģµ": 18100, "ĠOcean": 18101, "å§IJ": 18102, "yg": 18103, "Ġtowns": 18104, "del": 18105, "Ġterrifying": 18106, "ĠçalÄ±ÅŁ": 18107, "Ġsino": 18108, "Ġeats": 18109, "Ġgez": 18110, "Ġgeme": 18111, "ĠìĻĦ": 18112, "Ġcompart": 18113, "Ġimplementing": 18114, "ĠPotter": 18115, "ĠGermans": 18116, "ĠgÅĤ": 18117, "Ġtennis": 18118, "Ġcarpet": 18119, "auer": 18120, "ĠSaudi": 18121, "yeong": 18122, "Ġcurry": 18123, "ĠForest": 18124, "Ñĭл": 18125, "Ġfifteen": 18126, "Ġbolts": 18127, "Ġ{\\": 18128, "¬´": 18129, "Ġsettlement": 18130, "Ġlange": 18131, "Ġbam": 18132, "Get": 18133, "íķĻ": 18134, "Ġswap": 18135, "ĠKhan": 18136, "Ġcommence": 18137, "Ġquarantine": 18138, "Ġscored": 18139, "çĸ": 18140, "Ġ1950": 18141, "Ġthicker": 18142, "Ġsûr": 18143, "åı£": 18144, "ĠLarry": 18145, "Ġallez": 18146, "ìĭľëĬĶ": 18147, "Ġgü": 18148, "Ġspectacular": 18149, "//": 18150, "both": 18151, "Ġstats": 18152, "妳": 18153, "ĠNancy": 18154, "Ġbunu": 18155, "Ġcrust": 18156, "Ġactivated": 18157, "Ġê·¸ëŀ": 18158, "outhe": 18159, "Ġports": 18160, "Ġneural": 18161, "Ġjaw": 18162, "Ġobservations": 18163, "Ġvoit": 18164, "aban": 18165, "ải": 18166, "¦¬ë¥¼": 18167, "omes": 18168, "à¯ĭ": 18169, "qui": 18170, "Ġkindness": 18171, "Ðij": 18172, "Ġ41": 18173, "Ġmoderate": 18174, "Ġangels": 18175, "ĠTamb": 18176, "èt": 18177, "Ġchlor": 18178, "ĠBilly": 18179, "ì²ĺë": 18180, "acon": 18181, "Ġselecting": 18182, "ĠDelta": 18183, "Ġnull": 18184, "denly": 18185, "Ġciud": 18186, "Ġtendency": 18187, "Ġbreakdown": 18188, "Ġmint": 18189, "ÑĦоÑĢм": 18190, "orph": 18191, "Ġdawn": 18192, "spr": 18193, "ĠWILL": 18194, "ächlich": 18195, "Ġpuppy": 18196, "700": 18197, "Ġத": 18198, "Ġfails": 18199, "ĠConc": 18200, "Ġrelatives": 18201, "Ġinviting": 18202, "Ġautonom": 18203, "Ġcomposed": 18204, "Ġunity": 18205, "Ġdecis": 18206, "Ġaccessories": 18207, "ĠCass": 18208, "Ġbist": 18209, "ĠTip": 18210, "째": 18211, "Ġpunt": 18212, "Ġráp": 18213, "éĢ²": 18214, "ANK": 18215, "ãģļ": 18216, "exist": 18217, "Ġcompatible": 18218, "Ġner": 18219, "ĠемÑĥ": 18220, "Ġaplic": 18221, "Ġbapt": 18222, "Ġfailing": 18223, "ĠTamam": 18224, "Ġoscill": 18225, "Ġletzten": 18226, "Ġrepeatedly": 18227, "Ġjungle": 18228, "ĠPush": 18229, "hai": 18230, "Ġη": 18231, "Ġdeadly": 18232, "Ñıж": 18233, "wiÄħ": 18234, "ĠCommon": 18235, "ĠÎķ": 18236, "Ġskate": 18237, "TC": 18238, "ĠMini": 18239, "Ġhobby": 18240, "ần": 18241, "Ġroutes": 18242, "Ġamigos": 18243, "Ġconjun": 18244, "Ġpartnerships": 18245, "Ġnovo": 18246, "Ġaver": 18247, "Ġpouvez": 18248, "bridge": 18249, "Ġpreoc": 18250, "him": 18251, "Ġturb": 18252, "Ġsob": 18253, "ĠSnap": 18254, "Ġì°¸": 18255, "minute": 18256, "Ġtraject": 18257, "ujÄĻ": 18258, "Ġeager": 18259, "Ġregulatory": 18260, "Ġbanking": 18261, "bling": 18262, "ÑĪÑĮ": 18263, "aż": 18264, "Ġbizarre": 18265, "itated": 18266, "dire": 18267, "Ġthreatened": 18268, "Ġshining": 18269, "Ġnesse": 18270, "Ġcorps": 18271, "ĠÑģÑĥ": 18272, "Ġteles": 18273, "Ġtemp": 18274, "tem": 18275, "Ġкан": 18276, "Ġfever": 18277, "New": 18278, "Ġheavier": 18279, "ĠSah": 18280, "bud": 18281, "Ġoutros": 18282, "Ġì°¾": 18283, "Ġëªħ": 18284, "arring": 18285, "Ġê´ľì°®": 18286, "ĠNap": 18287, "Ġsemin": 18288, "ĠThan": 18289, "ifs": 18290, "Ġdesen": 18291, "ĠÑĤакое": 18292, "Ġloses": 18293, "ĠBalt": 18294, "kon": 18295, "ĠнапÑĢ": 18296, "Ġvois": 18297, "ĠMoscow": 18298, "Ġchairs": 18299, "his": 18300, "Ġrefugees": 18301, "kg": 18302, "Ġkole": 18303, "į¨": 18304, "аÑģибо": 18305, "¦½": 18306, "ĠUniverse": 18307, "ĠDirect": 18308, "Ġcheating": 18309, "ĠCin": 18310, "Ġpatri": 18311, "Ġadvise": 18312, "ĠNether": 18313, "Ġprimeiro": 18314, "Ġmentioning": 18315, "nut": 18316, "56": 18317, "arı": 18318, "Ġpetite": 18319, "bled": 18320, "Ġpensar": 18321, "icio": 18322, "IND": 18323, "Ġveteran": 18324, "Ġladder": 18325, "Ġconsequence": 18326, "ожал": 18327, "ĠBurn": 18328, "Ġrug": 18329, "ĠMade": 18330, "Ġgit": 18331, "\"...": 18332, "Ġcompetitors": 18333, "Ġprzed": 18334, "Ġapparent": 18335, "ĠArgentina": 18336, "ĠWorking": 18337, "Ġcollaborate": 18338, "woman": 18339, "Ġretain": 18340, "Ġleurs": 18341, "Ġdashboard": 18342, "×Ļ×ĵ": 18343, "ĠEarly": 18344, "BM": 18345, "ĠеÑij": 18346, "олог": 18347, "Ġsatisfying": 18348, "Ġoftentimes": 18349, "Ġmapping": 18350, "ünkü": 18351, "arth": 18352, "fold": 18353, "Ġlaunching": 18354, "Ġaura": 18355, "Ġprecision": 18356, "works": 18357, "God": 18358, "Ġstrap": 18359, "ĠImper": 18360, "Ġrivers": 18361, "Ġ|": 18362, "Ġcuer": 18363, "regon": 18364, "Ġarrival": 18365, "каÑħ": 18366, "ĠMiami": 18367, "анÑĭ": 18368, "Ġsurvivors": 18369, "ĠSenior": 18370, "David": 18371, "Ġestado": 18372, "Ġsectors": 18373, "Ġpopping": 18374, "Ġchim": 18375, "ayı": 18376, "Ġkunnen": 18377, "Ġgallery": 18378, "Ġsunlight": 18379, "esehen": 18380, "Ġyelling": 18381, "ĠMein": 18382, "ĠPhoenix": 18383, "Ġmano": 18384, "Ġhistoria": 18385, "Ġoccurring": 18386, "欸": 18387, "ì¸": 18388, "ади": 18389, "å¾ħ": 18390, "Ġinstitutional": 18391, "ĠTut": 18392, "ç²": 18393, "Ġslaves": 18394, "ãģ©ãģĨ": 18395, "Ġforgiveness": 18396, "Ġtwin": 18397, "ĠHyun": 18398, "нÑĮ": 18399, "ĠKomm": 18400, "andra": 18401, "shot": 18402, "ssä": 18403, "ĠÑĨе": 18404, "atta": 18405, "Ġexpense": 18406, "ĠGPU": 18407, "ĠPast": 18408, "ribly": 18409, "ĠëŃIJìķ¼": 18410, "Ġгода": 18411, "Ġrespir": 18412, "æĿ±": 18413, "ĠQueens": 18414, "hops": 18415, "Ġsérie": 18416, "Ġpref": 18417, "Ġcomed": 18418, "Ġplut": 18419, "ĠOverall": 18420, "ĠãģĿ": 18421, "Ġcush": 18422, "Ġringing": 18423, "Ġincorrect": 18424, "ĠÑģÑĤÑĢ": 18425, "Ġgeometry": 18426, "Ġadvertis": 18427, "ĠШ": 18428, "Ġreviewed": 18429, "ãģĤãģĤ": 18430, "Ġdozens": 18431, "Ġdetermination": 18432, "ĠPhill": 18433, "Ġcontributed": 18434, "ĠCit": 18435, "Ġpassengers": 18436, "Ġcôté": 18437, "Ġrever": 18438, "Ġtechnological": 18439, "Ġallen": 18440, "Ġraining": 18441, "avi": 18442, "Ġsalty": 18443, "Ġtyping": 18444, "ĠÑĤе": 18445, "Ġtilt": 18446, "Ġì¹ĺ": 18447, "ĠоÑĢ": 18448, "ĠпÑĢÑıм": 18449, "Ġrou": 18450, "Ġarena": 18451, "arat": 18452, "åĪ«": 18453, "HHHH": 18454, "Ġmanufacturers": 18455, "ĠEdward": 18456, "Ġtuck": 18457, "Ġblows": 18458, "ingo": 18459, "ĠMarc": 18460, "ìķĦìĦľ": 18461, "Mich": 18462, "ĠClean": 18463, "è´": 18464, "esto": 18465, "ĠPack": 18466, "Ġshaft": 18467, "BRUNO": 18468, "Ġaven": 18469, "uur": 18470, "ÑģколÑĮко": 18471, "ê´Ģ": 18472, "Ġautomated": 18473, "Ġventure": 18474, "Ġsurveillance": 18475, "ĠGrow": 18476, "ĠEmer": 18477, "ĠдоÑĢ": 18478, "Ġinvestor": 18479, "ĠYok": 18480, "Ġlatter": 18481, "ĠNI": 18482, "Ġfunctioning": 18483, "ĠHamilton": 18484, "Ġ51": 18485, "Ġmurdered": 18486, "Ġanchor": 18487, "Ġcuc": 18488, "ĠSCP": 18489, "ĠMadam": 18490, "Ġconstraints": 18491, "Ġbarn": 18492, "anken": 18493, "Ġë§İìĿĢ": 18494, "ĠMotor": 18495, "ĠDoing": 18496, "Ġamen": 18497, "etts": 18498, "Ġinstructor": 18499, "egt": 18500, "ako": 18501, "Ġposture": 18502, "ivia": 18503, "ĠPolish": 18504, "Ġдва": 18505, "Ġcolorful": 18506, "Ġelbow": 18507, "Ġparle": 18508, "Ġpasser": 18509, "Ġcondem": 18510, "ortal": 18511, "Ġfertil": 18512, "اد": 18513, "ĠColomb": 18514, "Ġalignment": 18515, "Ġastronaut": 18516, "ĠMut": 18517, "Ġsalmon": 18518, "Ġstructured": 18519, "ŀר": 18520, "Ġclicks": 18521, "Ġmiej": 18522, "æĶ¿": 18523, "ãģĦãĤĦ": 18524, "ĠRound": 18525, "Ġrainbow": 18526, "ĠVA": 18527, "ãģĶãģĸ": 18528, "ì§Ī": 18529, "otz": 18530, ",": 21732, "Ġchords": 21733, "ĠSanders": 21734, "Ġë¶Ħë": 21735, "Ben": 21736, "Ġdarüber": 21737, "ilians": 21738, "Ġordering": 21739, "ĠManh": 21740, "Ġkilogram": 21741, "ĠkarÅŁ": 21742, "Ġgrasp": 21743, "Ġghosts": 21744, "alen": 21745, "ĠJedi": 21746, "Ġбли": 21747, "Ġdownloaded": 21748, "Ġconducting": 21749, "ĠHak": 21750, "Ġresearcher": 21751, "ilan": 21752, "good": 21753, "ĠHannah": 21754, "ĠdÃ¼ÅŁÃ¼n": 21755, "ĠMessiah": 21756, "uity": 21757, "iona": 21758, "Ġprobable": 21759, "ĠYE": 21760, "Ġindependently": 21761, "Ġbuffer": 21762, "burn": 21763, "ourd": 21764, "ĠMcK": 21765, "Ġlingu": 21766, "ujemy": 21767, "еÑĢÑĤ": 21768, "Ġintuitive": 21769, "Ġcracks": 21770, "appropri": 21771, "nty": 21772, "Ġgeen": 21773, "Ġlend": 21774, "Ġcertification": 21775, "IDS": 21776, "unter": 21777, "pees": 21778, "Ġtrump": 21779, "Ġbankrupt": 21780, "Ġfeas": 21781, "èĹ": 21782, "Ġduż": 21783, "æ¸ħ": 21784, "Ġviruses": 21785, "Ġ58": 21786, "god": 21787, "Ġжел": 21788, "Ġstalk": 21789, "Ind": 21790, "achi": 21791, "ĠCF": 21792, "ĠCond": 21793, "Ġsanct": 21794, "Ġconten": 21795, "Ġfreed": 21796, "ĠRT": 21797, "Ġmentors": 21798, "족": 21799, "Ġportable": 21800, "ĠPaulo": 21801, "rane": 21802, "HAHA": 21803, "ĠSection": 21804, "çĨ": 21805, "hyun": 21806, "ĠÎŃÏĩ": 21807, "ĠPub": 21808, "ĠIndepend": 21809, "Ġcompounds": 21810, "ĠÑģÑĭ": 21811, "Ġmessaging": 21812, "Ġdedication": 21813, "Ġnoticing": 21814, "Ġdevoted": 21815, "ÑİÑĤÑģÑı": 21816, "Ġsnakes": 21817, "Ġbattlefield": 21818, "pers": 21819, "Ġdela": 21820, "92": 21821, "Ġhai": 21822, "illä": 21823, "érer": 21824, "every": 21825, "Ġresponsive": 21826, "×Ļ×ķ": 21827, "opf": 21828, "éī": 21829, "Ĭ¸": 21830, "Because": 21831, "Ġtourism": 21832, "Ġê·¸ê²Į": 21833, "×ķצ": 21834, "Ġcans": 21835, "stüt": 21836, "Ġdonne": 21837, "ĠDios": 21838, "ĠUber": 21839, "actory": 21840, "Ġoriented": 21841, "ĠHerm": 21842, "Ġpatron": 21843, "urf": 21844, "bei": 21845, "Ġprograma": 21846, "ĠOhh": 21847, "gener": 21848, "Ġfist": 21849, "ĠWendy": 21850, "Ġanda": 21851, "Ġguessed": 21852, "Ġfreak": 21853, "ä¸Ńåľĭ": 21854, "ĠKings": 21855, "chool": 21856, "Ġoffline": 21857, "ĠIndiana": 21858, "ĠAlliance": 21859, "Ġ53": 21860, "Ġparticul": 21861, "ĠFocus": 21862, "Ġinhabit": 21863, "Ġê°ĻìĿĢëį°": 21864, "ĠMcG": 21865, "owski": 21866, "ĠìĿ´ê±´": 21867, "ĠpaÅĦst": 21868, "они": 21869, "itta": 21870, "Ġconfirmation": 21871, "ĠBrooklyn": 21872, "Ġnoodle": 21873, "fund": 21874, "itud": 21875, "Ġgrandparents": 21876, "Ġbarbecue": 21877, "ειÏĤ": 21878, "Ġá": 21879, "Ġballot": 21880, "ĠVeter": 21881, "Ġpipes": 21882, "igious": 21883, "ĠGraph": 21884, "ested": 21885, "Ġë¸Įë": 21886, "ĠKE": 21887, "ãģ¡ãĤĩãģ£ãģ¨": 21888, "Ġeins": 21889, "Ġhatred": 21890, "ãģijãģ©": 21891, "Ġdang": 21892, "eeee": 21893, "Ġarchae": 21894, "ĠJesse": 21895, "Ġdetected": 21896, "Ġseni": 21897, "burgh": 21898, "Ġdisplacement": 21899, "Ġdop": 21900, "Ġconditioning": 21901, "ĠнеÑģколÑĮко": 21902, "Ġdisturbing": 21903, "PH": 21904, "Ġthinner": 21905, "Ġwounded": 21906, "ĠCuando": 21907, "Ġcushion": 21908, "Ġwhites": 21909, "Ġpreferences": 21910, "Ġì¤Ģë¹Ħ": 21911, "Ġkaż": 21912, "ĠGate": 21913, "ĠPath": 21914, "dles": 21915, "à¸Ħร": 21916, "imore": 21917, "Ġë³´ìŬ": 21918, "Ġdisciplines": 21919, "á»ı": 21920, "Ġmesma": 21921, "ĠìĥĪë": 21922, "Ġìĭ¬": 21923, "Ġging": 21924, "Ġumbrella": 21925, "IGHT": 21926, "Ġpension": 21927, "Ġcombining": 21928, "SS": 21929, "Ġrectangle": 21930, "á»ĩt": 21931, "Ġproxim": 21932, "ĠCow": 21933, "¸Į": 21934, "Ġintentional": 21935, "æķĻ": 21936, "Ġdecid": 21937, "ĠÑģкаж": 21938, "ĠUma": 21939, "iasm": 21940, "buz": 21941, "Ġdebris": 21942, "Ġcass": 21943, "ĠProp": 21944, "iska": 21945, "ëł¥": 21946, "esterol": 21947, "ussian": 21948, "ìĿ´ëŀij": 21949, "Ġunlimited": 21950, "Ġadmire": 21951, "Ġtightly": 21952, "Ġgenome": 21953, "ĠJunior": 21954, "venir": 21955, "gus": 21956, "ĠcÄĥ": 21957, "ĠVlad": 21958, "ĠíĤ": 21959, "Ġrelativ": 21960, "inci": 21961, "Ġaunque": 21962, "ĠBoys": 21963, "ÑĨион": 21964, "ĠSwiss": 21965, "Ġphysicians": 21966, "Ġíıī": 21967, "ĠPET": 21968, "Ġwounds": 21969, "about": 21970, "Ãłi": 21971, "onz": 21972, "urities": 21973, "ĠÑĥвид": 21974, "å·¦": 21975, "Ġmentality": 21976, "Ġvariance": 21977, "Ġsegunda": 21978, "Ġvolcano": 21979, "alie": 21980, "à¥ĩ": 21981, "Ġtiles": 21982, "ĠTerry": 21983, "ĠاÙĦÙĦÙĩ": 21984, "Ġcanon": 21985, "Ġscattered": 21986, "pton": 21987, "Ġdefinitions": 21988, "Ġalgebra": 21989, "oten": 21990, "ablo": 21991, "ijuana": 21992, "Ġwrapping": 21993, "Ġsesame": 21994, "ĠнаÑĩина": 21995, "ĠAlf": 21996, "ĠÐłÐ¾ÑģÑģ": 21997, "orno": 21998, "Ġankle": 21999, "Ġspecialty": 22000, "Ġattempting": 22001, "iliation": 22002, "Ġ1920": 22003, "Ġphenomena": 22004, "ĠProduct": 22005, "ĠBuck": 22006, "ĠAww": 22007, "seen": 22008, "Ġvoid": 22009, "ĠFranklin": 22010, "Ġadvocacy": 22011, "ĠSep": 22012, "Ġcoolest": 22013, "ĠÑģÑĢазÑĥ": 22014, "ĠQuand": 22015, "Ġ900": 22016, "ĠTrad": 22017, "dies": 22018, "Ġhash": 22019, "æĪijå°±": 22020, "ä¹Łæĺ¯": 22021, "Ġpots": 22022, "Ġsadly": 22023, "Ġviable": 22024, "ĠTiger": 22025, "ĠONE": 22026, "Ġneurons": 22027, "owanie": 22028, "ÄĹ": 22029, "ĠShar": 22030, "ĠLandes": 22031, "Ġconferences": 22032, "該": 22033, "Ġcredential": 22034, "Ġlime": 22035, "inee": 22036, "xit": 22037, "pay": 22038, "Ġincons": 22039, "Ġ>>:": 22040, "èªį": 22041, "Ġíŀĺë": 22042, "Ġlesser": 22043, "Ġspill": 22044, "Ġpremise": 22045, "Ġ365": 22046, "ĠHost": 22047, "Ġtomar": 22048, "×IJ׾": 22049, "ë²Ī": 22050, "ĠWhats": 22051, "Ġlightweight": 22052, "ĠMap": 22053, "fia": 22054, "ellschaft": 22055, "Ġvendors": 22056, "uesto": 22057, "ĠMister": 22058, "ĠÐŁÑĢи": 22059, "åı³": 22060, "hma": 22061, "Ġintentionally": 22062, "ĠTang": 22063, "éĹ®": 22064, "Ġidentification": 22065, "Ġetcetera": 22066, "ĠNee": 22067, "ĠÑĤÑĢи": 22068, "ê·¸": 22069, "Ġcryptocur": 22070, "Ġinhale": 22071, "Ġaddict": 22072, "åIJĦä½į": 22073, "Ġmau": 22074, "ĠÑĤакаÑı": 22075, "Ġë²Ħ": 22076, "Ġcomprar": 22077, "iedzieÄĩ": 22078, "ĠоÑĤно": 22079, "Ġbeginner": 22080, "ĠмÑĥж": 22081, "Ġobsc": 22082, "Ġlimiting": 22083, "ascular": 22084, "Ġinspection": 22085, "aci": 22086, "Ġrejo": 22087, "Mus": 22088, "Ġzaten": 22089, "Ġszcz": 22090, "ĠMadrid": 22091, "Ġvarieties": 22092, "ĠestÃł": 22093, "ĠShakes": 22094, "Ġkits": 22095, "Ġadminister": 22096, "Ġlava": 22097, "ĠgÃ¥": 22098, "試": 22099, "ת×Ļ": 22100, "ĠWayne": 22101, "Ġinstagram": 22102, "Ġrated": 22103, "paper": 22104, "Ġbild": 22105, "Ġpretending": 22106, "Ġobserving": 22107, "ĠÑģамом": 22108, "Ġtror": 22109, "Ġorganisms": 22110, "Ġfalta": 22111, "Ġhometown": 22112, "ç±": 22113, "Ġíĭ": 22114, "Ġcheg": 22115, "Ġì¡": 22116, "Ġcomma": 22117, "isé": 22118, "Ġlikelihood": 22119, "avored": 22120, "Ġgeldi": 22121, "ников": 22122, "Ġmedio": 22123, "Ġjakie": 22124, "ĠJup": 22125, "Ġgreenhouse": 22126, "Ġspit": 22127, "кое": 22128, "Ġкаж": 22129, "ĠGram": 22130, "ĠConference": 22131, "Ġdeficit": 22132, "sın": 22133, "inse": 22134, "uÄŁ": 22135, "Ġricht": 22136, "Ġcoincidence": 22137, "åıį": 22138, "Ġeurop": 22139, "Ġbutterfly": 22140, "pread": 22141, "Ġìĸ¼": 22142, "èĢ¶": 22143, "Ġwavel": 22144, "ĠInfin": 22145, "ĠPlanet": 22146, "Ġselfie": 22147, "ientras": 22148, "Ġarrog": 22149, "oser": 22150, "idal": 22151, "ł×Ĺ׳×ķ": 22152, "ütün": 22153, "Ġfreshman": 22154, "ĠMachine": 22155, "ÏĥÏĦ": 22156, "ĠDia": 22157, "ìĿ´ëĭ¤": 22158, "ãģĵãģĨ": 22159, "nea": 22160, "Ġlisting": 22161, "Ġconfigure": 22162, "utor": 22163, "Up": 22164, "tschaft": 22165, "rière": 22166, "Ġupwards": 22167, "ĠÑħоÑĩÑĥ": 22168, "Ġsweep": 22169, "Br": 22170, "Ġexpressing": 22171, "Ġunhappy": 22172, "Ġmandatory": 22173, "gender": 22174, "ĠAÃŃ": 22175, "Ġindicators": 22176, "Ġoils": 22177, "note": 22178, "Ġsegur": 22179, "ожеÑĤ": 22180, "ynasty": 22181, "Ġdistances": 22182, "Ġmerge": 22183, "BERT": 22184, "Ġsurrender": 22185, "Ġbuat": 22186, "ĠAwards": 22187, "Ġseñor": 22188, "odox": 22189, "Ġflavour": 22190, "Ġabdom": 22191, "Ġconfigur": 22192, "86": 22193, "ĠDIY": 22194, "Ġrigid": 22195, "°ĺ": 22196, "Ġcorporation": 22197, "Ġgroom": 22198, "jaw": 22199, "ĠNear": 22200, "ило": 22201, "Ġopera": 22202, "ĠInnov": 22203, "иÑĢа": 22204, "ĵ±": 22205, "Ġspecified": 22206, "Ġcosm": 22207, "ĠFreedom": 22208, "Ġclown": 22209, "ĠNem": 22210, "Ġвол": 22211, "Ñijн": 22212, "Ġcharger": 22213, "à¹ģล": 22214, "Ġinfluential": 22215, "äsident": 22216, "é¤": 22217, "ĠìĦłë": 22218, "Ġvolumes": 22219, "æIJ": 22220, "Ġoutras": 22221, "ĠTwitch": 22222, "Ġfounding": 22223, "Ġawhile": 22224, "Ġcoil": 22225, "ê°Ļ": 22226, "Ġcả": 22227, "ĠThrow": 22228, "ĠHence": 22229, "ommt": 22230, "ĠBenjamin": 22231, "глÑıд": 22232, "Time": 22233, "obic": 22234, "Ġmour": 22235, "Ġdread": 22236, "ĠLÃł": 22237, "ĠChile": 22238, "Ġpreval": 22239, "Ġvain": 22240, "Ġartık": 22241, "Ġpreserved": 22242, "ĠоÑĤд": 22243, "Ġwarehouse": 22244, "Ġbeste": 22245, "ĠSeveral": 22246, "ĠSituation": 22247, "Ġcardboard": 22248, "Tod": 22249, "erna": 22250, "Ġgarant": 22251, "Ġgesture": 22252, "Ġhen": 22253, "Ġspelling": 22254, "osexual": 22255, "Ġanne": 22256, "Ġmice": 22257, "ĠMeine": 22258, "card": 22259, "Ġrebell": 22260, "Ġcerto": 22261, "Ġìľłë": 22262, "Ġverschied": 22263, "ĠBos": 22264, "Ġinvention": 22265, "Ġtrze": 22266, "Ġmanière": 22267, "ĠChad": 22268, "Ġspre": 22269, "Ġorganisations": 22270, "Ġpoorly": 22271, "Ġanterior": 22272, "Ġstair": 22273, "кÑĢ": 22274, "Ġatomic": 22275, "Ġsympath": 22276, "Ġcontinually": 22277, "Ġkleine": 22278, "ète": 22279, "иÑī": 22280, "οÏĤ": 22281, "peut": 22282, "Ġreposit": 22283, "Ġentra": 22284, "Em": 22285, "Ġfinancing": 22286, "Ġмног": 22287, "Ġthesis": 22288, "ĠComputer": 22289, "eau": 22290, "ĠTree": 22291, "Ġbride": 22292, "onsieur": 22293, "shire": 22294, "wic": 22295, "DE": 22296, "ĠìĪĺë": 22297, "Ġacom": 22298, "ĠPO": 22299, "ersch": 22300, "ĠпомоÑī": 22301, "ĠArmen": 22302, "Ġ죽": 22303, "Ġzor": 22304, "Ġprints": 22305, "ĠDass": 22306, "港": 22307, "Ġdurable": 22308, "ĠTransport": 22309, "ìŀIJê°Ģ": 22310, "Ġлег": 22311, "Ġdét": 22312, "ôle": 22313, "amous": 22314, "YN": 22315, "Ġcliff": 22316, "Ġgrammar": 22317, "ĠÐŁÐ¾ÑįÑĤомÑĥ": 22318, "ĠlÃłm": 22319, "esch": 22320, "Ġmiserable": 22321, "Ġvolts": 22322, "ĠCad": 22323, "ukan": 22324, "ÑĤив": 22325, "rust": 22326, "Ġìĺ¬ëĿ¼": 22327, "Ġverk": 22328, "Ġchickens": 22329, "ĠYoo": 22330, "Ġoutfits": 22331, "code": 22332, "Ġhierarchy": 22333, "netes": 22334, "Ġcounterpart": 22335, "Ġtôi": 22336, "Ġted": 22337, "ĠBart": 22338, "ĠëĿ¼": 22339, "ĠGenau": 22340, "Ġincoming": 22341, "ĠABC": 22342, "rique": 22343, "ĠоÑĤп": 22344, "qual": 22345, "Ġincentive": 22346, "Ġihren": 22347, "׳×Ļ": 22348, "loe": 22349, "Ġ1930": 22350, "Ġbarg": 22351, "Ġdiction": 22352, "Ġönce": 22353, "INS": 22354, "Ġreh": 22355, "isiaj": 22356, "mouth": 22357, "Ġscoring": 22358, "lık": 22359, "ĠìķĦ주": 22360, "ORIA": 22361, "ĠEstados": 22362, "Ġcompanion": 22363, "Ġassemble": 22364, "Ġpunished": 22365, "Ġital": 22366, "Ġprevents": 22367, "istes": 22368, "ĠKentucky": 22369, "Ġlocate": 22370, "Ġfasting": 22371, "ãģ¨æĢĿ": 22372, "ĥĢ": 22373, "ĠSeb": 22374, "ĠCrown": 22375, "opia": 22376, "Ġwhip": 22377, "usz": 22378, "ками": 22379, "Ġdatabases": 22380, "åŃĹ": 22381, "Ġprosec": 22382, "Ġ1997": 22383, "ĠìĤ´ì§Ŀ": 22384, "ĠSolar": 22385, "ĠPues": 22386, "ĠZen": 22387, "ollo": 22388, "ĠGuru": 22389, "Ġsqueez": 22390, "ĠÐĹа": 22391, "ĠÄį": 22392, "ceptions": 22393, "cca": 22394, "izable": 22395, "mand": 22396, "Ġbreakthrough": 22397, "Ġtablespoon": 22398, "ĠSEC": 22399, "ikh": 22400, "ĠSão": 22401, "Ġпло": 22402, "amen": 22403, "Ġprac": 22404, "Ġdarling": 22405, "Ġtaller": 22406, "Ġrendering": 22407, "Ġìļ°ë¦¬ê°Ģ": 22408, "ĠÏĦηÏĤ": 22409, "Ġmã": 22410, "Ġesos": 22411, "uerdo": 22412, "ĠÑģÑĩиÑĤ": 22413, "aller": 22414, "ìĹĪìĸ´ìļĶ": 22415, "Ġmillones": 22416, "lerin": 22417, "Ġpegar": 22418, "onne": 22419, "Ġenrollment": 22420, "Ġliegt": 22421, "Ġboa": 22422, "wiÄĻ": 22423, "bsp": 22424, "Ġcycling": 22425, "ĠBernie": 22426, "Ġ1989": 22427, "ĠдалÑĮ": 22428, "ĠDakota": 22429, "ĠÑģвÑıз": 22430, "ĠCP": 22431, "Ġstare": 22432, "íĤ¤": 22433, "Ġprosperity": 22434, "Ġarrangements": 22435, "Ġarriving": 22436, "mä": 22437, "Ġkayak": 22438, "ipt": 22439, "Ġpardon": 22440, "Ġrelat": 22441, "Ġverste": 22442, "ĠFig": 22443, "Ġfoil": 22444, "ĠTalking": 22445, "peare": 22446, "Ġnoi": 22447, "ĠпÑĢиÑĪ": 22448, "Ġhockey": 22449, "Ġado": 22450, "ĠOUT": 22451, "67": 22452, "Ġhormones": 22453, "ĠAvenue": 22454, "ĠSuperman": 22455, "Ġprescription": 22456, "ubernetes": 22457, "CL": 22458, "otive": 22459, "NIS": 22460, "ienen": 22461, "Ġsadness": 22462, "ĠVit": 22463, "Ty": 22464, "Ġstarter": 22465, "Ġbede": 22466, "Ġfoundations": 22467, "Ġsore": 22468, "åºĹ": 22469, "ÑīеÑģÑĤв": 22470, "ìļ°ë": 22471, "ĠÑĩÑĥв": 22472, "link": 22473, "Ġmaneu": 22474, "working": 22475, "Ãłn": 22476, "ĠAttack": 22477, "ĠCart": 22478, "veis": 22479, "ĠResp": 22480, "ensing": 22481, "Ġì¢ĭìķĦìļĶ": 22482, "Ġescuch": 22483, "ĠRNA": 22484, "Ĥ´": 22485, "Ġadop": 22486, "Ġbending": 22487, "عد": 22488, "Ġmanages": 22489, "usp": 22490, "Ġtart": 22491, "Ġrouter": 22492, "Bo": 22493, "Ġestablishing": 22494, "Ġbalancing": 22495, "Ġathletic": 22496, "ĠSlo": 22497, "Ġfills": 22498, "Ġнаб": 22499, "Ġдал": 22500, "Ġposso": 22501, "ĠVielen": 22502, "Ġcritics": 22503, "Ġlawsuit": 22504, "ĠIsaac": 22505, "ĠÑĦилÑĮм": 22506, "Ġtras": 22507, "Ġpraw": 22508, "ĠCrazy": 22509, "Ġneu": 22510, "Ġkull": 22511, "Ġtumor": 22512, "ĠAPP": 22513, "gate": 22514, "ĠARE": 22515, "98": 22516, "ĠSteam": 22517, "Ġfucked": 22518, "lage": 22519, "ĠâĻ¬": 22520, "ĠMD": 22521, "fy": 22522, "Ġshells": 22523, "ĠSeems": 22524, "izers": 22525, "Ġranges": 22526, "ĠAntonio": 22527, "ATION": 22528, "ĠBaba": 22529, "Ġìĥī": 22530, "kun": 22531, "Ġprayed": 22532, "ÑĢÑı": 22533, "ĠпÑĢоÑĤив": 22534, "Ġseas": 22535, "bury": 22536, "Ġ×Ķש": 22537, "Ġtrait": 22538, "ĠDepending": 22539, "Ġdre": 22540, "Ġkönnt": 22541, "ÑĨÑĥ": 22542, "Ġlipstick": 22543, "eez": 22544, "ĠпÑĢимеÑĢ": 22545, "Ġassignments": 22546, "Bob": 22547, "Ġmetals": 22548, "Ġspecially": 22549, "å°įä¸įå°į": 22550, "ĠìĺĪë": 22551, "ĠÅ¡": 22552, "Ġvista": 22553, "Ġά": 22554, "Ġtwins": 22555, "Ġnotable": 22556, "ĠSau": 22557, "Ġdévelop": 22558, "Ġçek": 22559, "Ġpolynom": 22560, "avam": 22561, "Ġtambé": 22562, "оном": 22563, "Ġplasma": 22564, "Ġefect": 22565, "Ġläng": 22566, "Ġcasi": 22567, "Ñģа": 22568, "ımı": 22569, "ãģĻãĤĭ": 22570, "ĵ¤ìĿĢ": 22571, "Ġlabour": 22572, "ossen": 22573, "ĠPun": 22574, "rif": 22575, "Ġdoses": 22576, "Ġoperates": 22577, "илли": 22578, "Ġjaar": 22579, "staw": 22580, "ĠìĤ¬ëŀij": 22581, "Ġatm": 22582, "Ġprotects": 22583, "Ġimped": 22584, "HO": 22585, "Ġcima": 22586, "Ġtoch": 22587, "abis": 22588, "Ġsendo": 22589, "laus": 22590, "Ġcurl": 22591, "ĠNum": 22592, "Ġsponsors": 22593, "Ġdébut": 22594, "ĠAlexa": 22595, "ĠBür": 22596, "ĠAmer": 22597, "Ġcope": 22598, "Ġизв": 22599, "jal": 22600, "Ġ1995": 22601, "apat": 22602, "resse": 22603, "ĠPrize": 22604, "ĠClaire": 22605, "ĠBrandon": 22606, "Ġwszystko": 22607, "Ġvalued": 22608, "à¸Ļะ": 22609, "Ġsect": 22610, "Ġsecretly": 22611, "Ġdiamonds": 22612, "ĠEvan": 22613, "ĠRPG": 22614, "ãģ«ãģª": 22615, "ĪëıĦ": 22616, "ĠUniversal": 22617, "Ġdoubts": 22618, "ĠPin": 22619, "wiÄħz": 22620, "ļ©": 22621, "Ġalbo": 22622, "Ġbraucht": 22623, "AUL": 22624, "ĠMobile": 22625, "grades": 22626, "Ġschem": 22627, "why": 22628, "ĠNicht": 22629, "pi": 22630, "gle": 22631, "Ġchorus": 22632, "Ġgly": 22633, "Ġreinforce": 22634, "Ġmuff": 22635, "ĠShen": 22636, "ĠHola": 22637, "Ñĥг": 22638, "videmment": 22639, "vial": 22640, "acious": 22641, "laimed": 22642, "ĠRico": 22643, "Ġvegg": 22644, "Ġillustration": 22645, "ĠButter": 22646, "owad": 22647, "Ġeux": 22648, "Ġenfants": 22649, "ĠLeader": 22650, "ĠVillage": 22651, "etically": 22652, "ÙĨÙĬ": 22653, "Ġstew": 22654, "Ġsurprises": 22655, "Ġcue": 22656, "ĠGrandma": 22657, "ĠCelsius": 22658, "ĠRicht": 22659, "enc": 22660, "Ġpetition": 22661, "Ġherb": 22662, "Ġwicked": 22663, "Ġschle": 22664, "ocaly": 22665, "Ġtransf": 22666, "Ġtokens": 22667, "ĠGray": 22668, "ĠBBC": 22669, "IK": 22670, "Ġ1500": 22671, "zn": 22672, "ĠNev": 22673, "Ġkoy": 22674, "Ġzar": 22675, "Ġbullshit": 22676, "ĠColombia": 22677, "ulative": 22678, "Ġwidespread": 22679, "yect": 22680, "kit": 22681, "Ġempresa": 22682, "Ġnour": 22683, "Ġburns": 22684, "atin": 22685, "aired": 22686, "Ġrevolutionary": 22687, "ĠгодÑĥ": 22688, "ĠLogan": 22689, "Ġ1996": 22690, "ĠGraham": 22691, "reb": 22692, "ĠNHS": 22693, "æľĽ": 22694, "Ġcostumes": 22695, "Ġnawet": 22696, "Ġlovers": 22697, "ĠLucy": 22698, "ĠIndigenous": 22699, "íķĺ기": 22700, "Ġimmunity": 22701, "¥´ë": 22702, "uito": 22703, "Ġexcessive": 22704, "Ġdonations": 22705, "Ġ×Ķר": 22706, "Ġ첫": 22707, "éīĦ": 22708, "Ġdrying": 22709, "melon": 22710, "Ġsurveys": 22711, "Ġ무ìĬ¨": 22712, "風": 22713, "aaa": 22714, "Ġprobe": 22715, "ancial": 22716, "Ġlouder": 22717, "Ġhotels": 22718, "Ã¼ÄŁ": 22719, "agner": 22720, "Ġorigins": 22721, "Ġë§Īì§Ģë§ī": 22722, "Ġ**": 22723, "Ġstrangers": 22724, "ĠHaus": 22725, "comed": 22726, "Ġanthrop": 22727, "Ġuso": 22728, "ĠìķĦì§ģ": 22729, "ĠYuan": 22730, "ĠíķĦìļĶ": 22731, "pler": 22732, "ressive": 22733, "Ġspraw": 22734, "ĠStew": 22735, "Ġ1994": 22736, "Ġelders": 22737, "Ġmeinen": 22738, "Ġjunt": 22739, "Ġacoust": 22740, "ĠWohn": 22741, "Ġbananas": 22742, "Ġprojection": 22743, "ĠStick": 22744, "legt": 22745, "speed": 22746, "ĠcÅ©ng": 22747, "ĠWort": 22748, "ĠBaltimore": 22749, "ĠÑĨел": 22750, "Ġdunno": 22751, "å¼·": 22752, "?,": 22753, "ãĥīãĥ³": 22754, "ĠLocal": 22755, "osto": 22756, "ÐŃ": 22757, "ода": 22758, "ĠPortuguese": 22759, "Ġtheirs": 22760, "Ġdém": 22761, "åı¦": 22762, "Ġdrauf": 22763, "ĠBuddhist": 22764, "erta": 22765, "Ge": 22766, "Ġcarrot": 22767, "ĠWonderful": 22768, "Ġsoak": 22769, "Ġchairman": 22770, "ggi": 22771, "ICA": 22772, "fried": 22773, "Ġflick": 22774, "ĠThroughout": 22775, "Ġìļ°ë": 22776, "Ġcough": 22777, "Ġfluffy": 22778, "school": 22779, "Ġripped": 22780, "--------": 22781, "ĠZukunft": 22782, "Ġнеб": 22783, "Ġsto": 22784, "ĠBO": 22785, "pent": 22786, "ĠLawrence": 22787, "ÏīÏĤ": 22788, "sticks": 22789, "ĠEins": 22790, "ĠÑĢÑĭ": 22791, "ĠStrong": 22792, "Ġcaramel": 22793, "Ġspite": 22794, "azar": 22795, "éĥ½æĺ¯": 22796, "Ġcritically": 22797, "Ġobra": 22798, "owitz": 22799, "ĠZone": 22800, "ĠÑĢек": 22801, "Ġsug": 22802, "arded": 22803, "Ġgì": 22804, "ffentlich": 22805, "anche": 22806, "ØŁ": 22807, "astically": 22808, "ìĿ¼ë": 22809, "лав": 22810, "Ġsimplest": 22811, "ĠFriend": 22812, "Ġquello": 22813, "Ġambition": 22814, "Ġabbiamo": 22815, "åºķ": 22816, "ĠÑĦоÑĢм": 22817, "ĠEssa": 22818, "Ġeducators": 22819, "Ġstatistical": 22820, "éĢĻéĤĬ": 22821, "Ġchanger": 22822, "Ġatau": 22823, "étais": 22824, "ĠShakespeare": 22825, "ëIJĺ": 22826, "Ġtriggers": 22827, "Ġrealiz": 22828, "Ġcelui": 22829, "wheel": 22830, "Ġloyalty": 22831, "Ġscreams": 22832, "kehr": 22833, "ĠMega": 22834, "east": 22835, "Ġtops": 22836, "ĠTotally": 22837, "ountain": 22838, "lord": 22839, "Ġviolation": 22840, "ĠGA": 22841, "Ġnicer": 22842, "ĠFresh": 22843, "ĠMelissa": 22844, "function": 22845, "Ġrape": 22846, "Ġexceptions": 22847, "Ġsilicon": 22848, "Ġliberty": 22849, "Ġhouseholds": 22850, "ãģįãģ¾ãģĻ": 22851, "ĠCA": 22852, "ĠÐŀб": 22853, "Ġlib": 22854, "ŀĮ": 22855, "cific": 22856, "Ġtropical": 22857, "Ġinvestigating": 22858, "HD": 22859, "Ġadapter": 22860, "ĠPitt": 22861, "ancia": 22862, "ĠShell": 22863, "friendly": 22864, "Ġconclusions": 22865, "Ġturtle": 22866, "Ġdecomp": 22867, "Ġanimations": 22868, "ĠÑģек": 22869, "insi": 22870, "Ġretention": 22871, "kie": 22872, "Ġinjection": 22873, "ĠMadison": 22874, "ì°°": 22875, "Ġvient": 22876, "Ġvaried": 22877, "Ġviolin": 22878, "ĠBil": 22879, "Ġluckily": 22880, "Ġhtt": 22881, "lä": 22882, "Ġranch": 22883, "çľĭçľĭ": 22884, "Ġsólo": 22885, "ìķħ": 22886, "ĠDerek": 22887, "ĠScripture": 22888, "оÑĢа": 22889, "Ġclassrooms": 22890, "avil": 22891, "formed": 22892, "Ġbeforehand": 22893, "ĠGem": 22894, "prech": 22895, "Ġlin": 22896, "Ġgreens": 22897, "ÑĨев": 22898, "ĠMercedes": 22899, "Ġdrought": 22900, "gasps": 22901, "Ġabortion": 22902, "Ġterribly": 22903, "Ġsposób": 22904, "Ġsecured": 22905, "Ġatrás": 22906, "Ġwavelength": 22907, "Ġgrains": 22908, "ective": 22909, "Ġspacecraft": 22910, "Ġtours": 22911, "Ġprofes": 22912, "Ġsurgeon": 22913, "ĠPie": 22914, "Ġideally": 22915, "arner": 22916, "UP": 22917, "opard": 22918, "sce": 22919, "Ġimmense": 22920, "ĠOrt": 22921, "roller": 22922, "ĠDallas": 22923, "ĠNicholas": 22924, "Ġsulf": 22925, "ĠToyota": 22926, "Ġquantities": 22927, "ceans": 22928, "Ġcui": 22929, "ança": 22930, "ĠCAN": 22931, "itzerland": 22932, "åĦ¿": 22933, "Ġzou": 22934, "ĠCyber": 22935, "legen": 22936, "ĠInit": 22937, "edu": 22938, "Ġapert": 22939, "Ġadjac": 22940, "ouv": 22941, "èĢĮä¸Ķ": 22942, "rs": 22943, "Ġcabbage": 22944, "Ġwheelchair": 22945, "inyl": 22946, "ĠDynam": 22947, "ĠìķĦëĭĪëĿ¼": 22948, "Ġling": 22949, "hl": 22950, "ĠмогÑĥ": 22951, "Ġcrisp": 22952, "Ġmij": 22953, "Ġdug": 22954, "nin": 22955, "Ġbloss": 22956, "Ġbelonging": 22957, "Ġloudly": 22958, "Ġminerals": 22959, "Ġconcluded": 22960, "Ġsearched": 22961, "96": 22962, "ĠMeet": 22963, "ĠSEO": 22964, "ĠСк": 22965, "ĠHob": 22966, "otta": 22967, "Ġpropaganda": 22968, "Ġcinnamon": 22969, "Ġhunter": 22970, "Ġgemeins": 22971, "Ġsculpture": 22972, "ulsion": 22973, "Ġväl": 22974, "Ġmagazines": 22975, "Ġcontroversy": 22976, "ä¸Ģ樣": 22977, "Ġsequences": 22978, "ãģĦãĤĭ": 22979, "ĠíļĮ": 22980, "Ġdeleted": 22981, "使": 22982, "IJëıĦ": 22983, "Ġvarying": 22984, "ãĥĨ": 22985, "Ġmounting": 22986, "Ġaffair": 22987, "Ġpathways": 22988, "æ¦": 22989, "Ġdigo": 22990, "亮": 22991, "Ġдок": 22992, "Alex": 22993, "Ġtobacco": 22994, "ĠCV": 22995, "Ġbothered": 22996, "Ġambient": 22997, "inky": 22998, "ĠSL": 22999, "Ġhates": 23000, "Ġjeżeli": 23001, "Ġcongreg": 23002, "Ġelas": 23003, "Ġdeuts": 23004, "ĠStudios": 23005, "chÄĻ": 23006, "Ġdocumented": 23007, "ĠCruz": 23008, "ĠLen": 23009, "ĠDouglas": 23010, "ĠPortugal": 23011, "enti": 23012, "Ġspouse": 23013, "Ġanalys": 23014, "avia": 23015, "Ġedited": 23016, "Ġlại": 23017, "built": 23018, "Ġville": 23019, "adora": 23020, "Ġbracelet": 23021, "Ġsushi": 23022, "Ġpm": 23023, "Ġtrails": 23024, "Ġlug": 23025, "Ġöver": 23026, "Ġsorrow": 23027, "Ġcolony": 23028, "adox": 23029, "Ġserie": 23030, "anyak": 23031, "ĠØ·": 23032, "ĠGulf": 23033, "æĺ¯ä¸įæĺ¯": 23034, "ĠPV": 23035, "ĠSamuel": 23036, "ĠKit": 23037, "ĠRal": 23038, "ontin": 23039, "expl": 23040, "Ġentries": 23041, "Ġactivists": 23042, "Ps": 23043, "Ġsant": 23044, "ĠÑĤоÑĩ": 23045, "ĠBruno": 23046, "keley": 23047, "Ġtutto": 23048, "éĶ": 23049, "Ġvintage": 23050, "Ġterrified": 23051, "ĠпоÑħ": 23052, "usive": 23053, "owers": 23054, "айÑĤ": 23055, "ëıĻ": 23056, "Ġtwisted": 23057, "ĠThought": 23058, "Ġtah": 23059, "Ġshrink": 23060, "Ġsheer": 23061, "lit": 23062, "Ġdalam": 23063, "Ġdib": 23064, "Ġvard": 23065, "owane": 23066, "Ġdobr": 23067, "ĠRena": 23068, "ĠÑģвоÑİ": 23069, "ĠpaÃŃses": 23070, "ĠEra": 23071, "ãģ®ãģ§": 23072, "ĠBUT": 23073, "sighs": 23074, "Ġ그거": 23075, "ĠgroÃŁen": 23076, "Ġ빨리": 23077, "Ġnerves": 23078, "Ġconstit": 23079, "Ġpreocup": 23080, "ĠGay": 23081, "ĠXu": 23082, "keeper": 23083, "heure": 23084, "..)": 23085, "ĠCalm": 23086, "ĠUnidos": 23087, "ĠìĿ´ê²ĥ": 23088, "ĠAqui": 23089, "ĠìłľìĿ¼": 23090, "dır": 23091, "ì¦ĺ": 23092, "your": 23093, "ĠÑįÑĤим": 23094, "2020": 23095, "Ġrund": 23096, "ĠHO": 23097, "ĠCatherine": 23098, "ieli": 23099, "Ġfusion": 23100, "Ġideology": 23101, "Ġforam": 23102, "shaped": 23103, "ĠíĽĦë": 23104, "Ġwt": 23105, "Ġretr": 23106, "Ġpréc": 23107, "Ġê°ij": 23108, "Ġopenly": 23109, "vity": 23110, "구ìļĶ": 23111, "Ġobstacle": 23112, "Ġboo": 23113, "Ġseiner": 23114, "icorn": 23115, "Ġeigenlijk": 23116, "Ġheader": 23117, "aremos": 23118, "Ġsofter": 23119, "ĠÐŁÐ¾Ð´": 23120, "Ġprejud": 23121, "Ġdefines": 23122, "ierte": 23123, "Ġblending": 23124, "Ġbelievers": 23125, "ĠWochen": 23126, "Ġникак": 23127, "ĠÐļогда": 23128, "ĠTypically": 23129, "Ġíģ¬": 23130, "管": 23131, "cios": 23132, "Ġmissiles": 23133, "Ġsponge": 23134, "ĠKitchen": 23135, "Ġtren": 23136, "ningen": 23137, "Ġscrap": 23138, "Ġserait": 23139, "´ìł": 23140, "ç¹": 23141, "Ġë°ĺë": 23142, "Ġrestored": 23143, "ĠprzykÅĤad": 23144, "ĠKubernetes": 23145, "Ġsait": 23146, "Ġuw": 23147, "Ġenabling": 23148, "Ġtravers": 23149, "amps": 23150, "åıĹ": 23151, "ĠOMG": 23152, "ensor": 23153, "Ġzosta": 23154, "Ġpronounced": 23155, "Ang": 23156, "normal": 23157, "Ġeconomies": 23158, "tin": 23159, "ĠChampion": 23160, "izen": 23161, "Ġarbeiten": 23162, "ĠGospel": 23163, "ĠZu": 23164, "nga": 23165, "Ġliteracy": 23166, "ĠMans": 23167, "Ġcirculation": 23168, "Ġadap": 23169, "ĠTotal": 23170, "Ġmereka": 23171, "Ġolacak": 23172, "ÑģÑĤаÑĤи": 23173, "Jack": 23174, "Ġmund": 23175, "Ġthief": 23176, "bies": 23177, "Ġê²ģ": 23178, "aque": 23179, "ĠÚ©ÛĮ": 23180, "ĠScar": 23181, "å²": 23182, "Ġabol": 23183, "Ġdevote": 23184, "Ġ01": 23185, "Ġsitten": 23186, "ĠVisual": 23187, "week": 23188, "some": 23189, "ingt": 23190, "Ġjournalism": 23191, "ĠHir": 23192, "ĠBachelor": 23193, "inery": 23194, "ÃľND": 23195, "ãĥŁ": 23196, "ç»Ļ": 23197, "Ġcoloring": 23198, "ĠCrist": 23199, "Ġcelebrities": 23200, "ĠÑĩиÑģ": 23201, "ĠCrit": 23202, "Ġdifferentiate": 23203, "ĠÐľÐ½Ðµ": 23204, "elim": 23205, "Ġseafood": 23206, "Ġalgumas": 23207, "otherapy": 23208, "æĪ°": 23209, "Ġglaub": 23210, "Ġarbitrary": 23211, "gens": 23212, "ĠбÑĥдем": 23213, "Ġtav": 23214, "Ġcreamy": 23215, "ĠCountry": 23216, "añ": 23217, "меÑĤ": 23218, "Ġhinter": 23219, "Ġmism": 23220, "Ġillustrate": 23221, "ÃľNDNIS": 23222, "Ġdecreasing": 23223, "Ġweniger": 23224, "AKI": 23225, "ixon": 23226, "Ġней": 23227, "Ġfatto": 23228, "Ġnerd": 23229, "çł": 23230, "Ġbitte": 23231, "Per": 23232, "Ġtane": 23233, "Ġgöz": 23234, "Ġforte": 23235, "ĠEy": 23236, "ĠнавеÑĢ": 23237, "被": 23238, "ĠWordPress": 23239, "ĠMis": 23240, "ů": 23241, "zäh": 23242, "Ġintéress": 23243, "osaurs": 23244, "ĠFalls": 23245, "Ġnessa": 23246, "97": 23247, "Ġmuseums": 23248, "Ġcorresponds": 23249, "Ġsings": 23250, "four": 23251, "Ġeder": 23252, "ĠCommunist": 23253, "oa": 23254, "nek": 23255, "ĠWHO": 23256, "Ġcorpo": 23257, "Ġmessing": 23258, "ÏĦαι": 23259, "Ġbrushes": 23260, "Ġbisc": 23261, "ĠArbeits": 23262, "ĠTax": 23263, "Ġsele": 23264, "Ġflags": 23265, "oupe": 23266, "Ġanticipated": 23267, "ãĥij": 23268, "ĠNad": 23269, "Ġpoured": 23270, "Ġml": 23271, "Ġllama": 23272, "Ġvisualize": 23273, "Ġlisteners": 23274, "ÙĦÙĥ": 23275, "alten": 23276, "Michael": 23277, "Ġcosì": 23278, "Õ¡Õ": 23279, "opus": 23280, "Ġíķ´ì£¼": 23281, "Ġhike": 23282, "ĠAttorney": 23283, "ĠHillary": 23284, "uded": 23285, "Ġíķĺì§Ģë§Į": 23286, "Ġdove": 23287, "Ġstorms": 23288, "акÑģ": 23289, "Ġdoctrine": 23290, "Ġhex": 23291, "iks": 23292, "noÅĽÄĩ": 23293, "Ġscripts": 23294, "Ġδεν": 23295, "ĠÑįÑĤиÑħ": 23296, "ĠÐĨ": 23297, "aber": 23298, "ĠVas": 23299, "Ġcentimeters": 23300, "×ŀ×Ķ": 23301, "ниб": 23302, "Ġriders": 23303, "ĠTrib": 23304, "åĮħ": 23305, "Ġtakże": 23306, "Ġnoun": 23307, "Ġicons": 23308, "Ġsolely": 23309, "minded": 23310, "Ġdispon": 23311, "ĠSwitzerland": 23312, "Ġclusters": 23313, "Ġqueda": 23314, "ailing": 23315, "Ġmanga": 23316, "Ġ68": 23317, "ĦĪ": 23318, "Ġtet": 23319, "gins": 23320, "haus": 23321, "空": 23322, "å·¥": 23323, "ĠOP": 23324, "oted": 23325, "Ġnouveau": 23326, "ALLY": 23327, "ÙĪد": 23328, "òn": 23329, "Ġmortality": 23330, "ĠGitHub": 23331, "drop": 23332, "Ġdisgu": 23333, "Ġrecom": 23334, "Ġlocals": 23335, "Ġhomemade": 23336, "amba": 23337, "Ġpronunciation": 23338, "Ġalphabet": 23339, "анÑĮ": 23340, "owany": 23341, "iras": 23342, "idency": 23343, "OME": 23344, "ĠÑĢаÑģÑģ": 23345, "arak": 23346, "viamente": 23347, "Ġnonprofit": 23348, "ĠYouTuber": 23349, "Ġparenth": 23350, "ĠBoo": 23351, "vat": 23352, "ĠStir": 23353, "Ġprecip": 23354, "Ġants": 23355, "Ġally": 23356, "ĠMaori": 23357, "ĠëĮĢíķľ": 23358, "åı¯æĺ¯": 23359, "ogene": 23360, "ĠLabour": 23361, "arette": 23362, "Ġrecycling": 23363, "ensa": 23364, "Ġpursuit": 23365, "Ġsak": 23366, "ĠÐĹдеÑģÑĮ": 23367, "Ġtolerance": 23368, "Ġsaat": 23369, "Ġclicked": 23370, "âĻ¥": 23371, "Ġfacebook": 23372, "ĠInto": 23373, "Ġincentives": 23374, "기ëĬĶ": 23375, "ĠDennis": 23376, "ĠWik": 23377, "gesch": 23378, "à¹Ģà¸Ľ": 23379, "ĠÏĢα": 23380, "ĠWhoo": 23381, "Ġrounded": 23382, "Ġdope": 23383, "Ġcapturing": 23384, "ĠWarri": 23385, "Ġcivilian": 23386, "Ġcharming": 23387, "Ġesas": 23388, "Ġsustained": 23389, "Ġleaning": 23390, "Ġabundance": 23391, "ÃŃlia": 23392, "алÑĮнÑĭй": 23393, "Ġphải": 23394, "acja": 23395, "Ġê°ĻìķĦ": 23396, "activ": 23397, "าย": 23398, "Ġ97": 23399, "Ġмой": 23400, "cro": 23401, "ĠJackie": 23402, "ittees": 23403, "bracht": 23404, "ulent": 23405, "Ġìłľë": 23406, "Ġplugin": 23407, "vantage": 23408, "party": 23409, "Ġsuas": 23410, "Ġante": 23411, "Ñĥл": 23412, "ÐĿÐIJ": 23413, "æĤ¨": 23414, "ĠÏĥÏħ": 23415, "Ġmeth": 23416, "Ġenthusiasm": 23417, "ÑıÑĤÑģÑı": 23418, "íĻĶë": 23419, "Ġsynthetic": 23420, "Ġseasoning": 23421, "ĠLost": 23422, "onomy": 23423, "ĠSpark": 23424, "Ġbure": 23425, "Ġassured": 23426, "Ġimagin": 23427, "Ġcarro": 23428, "Sha": 23429, "Äħt": 23430, "нÑĥÑĤÑĮ": 23431, "ática": 23432, "TY": 23433, "Ġkern": 23434, "ĠBrazilian": 23435, "ð": 23436, "Ġsuspended": 23437, "ĠCarib": 23438, "Ġbizim": 23439, "ĠOliver": 23440, "ãģ¶": 23441, "Tom": 23442, "Ġплан": 23443, "Ġnope": 23444, "omething": 23445, "Ġbeiden": 23446, "ÑĨен": 23447, "Ġfluct": 23448, "ĠμοÏħ": 23449, "Ġfathers": 23450, "ĠBlake": 23451, "Ġupward": 23452, "ĠDash": 23453, "ĠLil": 23454, "ĠìĪĺëıĦ": 23455, "Ġrevelation": 23456, "Ġelevated": 23457, "ĠJiang": 23458, "LED": 23459, "ĠThompson": 23460, "ĠмогÑĥÑĤ": 23461, "ÑģÑĤÑĢÑĥ": 23462, "ifiers": 23463, "Ġcomeback": 23464, "Ġbuyers": 23465, "ê²°": 23466, "ĠSales": 23467, "иÑĩе": 23468, "ciones": 23469, "Ġwhistle": 23470, "Ġdull": 23471, "LEX": 23472, "Ġíķĺê²łìĬµëĭĪëĭ¤": 23473, "Ġcriminals": 23474, "Ġdescent": 23475, "ipple": 23476, "ması": 23477, "Ġfoolish": 23478, "ĠдÑĥмаÑİ": 23479, "tar": 23480, "Ġmango": 23481, "Ġchoreography": 23482, "Matt": 23483, "Ġterritor": 23484, "Ġacaba": 23485, "ĠEinstein": 23486, "ĠIBM": 23487, "ĠMetal": 23488, "ĠCrystal": 23489, "Ġrah": 23490, "Ġfoul": 23491, "ĠIslands": 23492, "Ġintact": 23493, "ĠRail": 23494, ".:": 23495, "Ġacá": 23496, "ĠпÑĢоп": 23497, "еÑĢе": 23498, "ĠWrite": 23499, "hehe": 23500, "ĠFO": 23501, "ĠÏĥÏĦη": 23502, "Ġdoin": 23503, "held": 23504, "Ġappropriately": 23505, "Ġdeliberately": 23506, "Ġarchive": 23507, "Ġgiveaway": 23508, "ãģĵãģĵ": 23509, "Ġfinale": 23510, "лаÑģ": 23511, "ено": 23512, "Æ¡n": 23513, "æ£Ĵ": 23514, "ogo": 23515, "çī©": 23516, "ĠAudience": 23517, "ãħł": 23518, "Ġsubur": 23519, "Ġheadache": 23520, "аннÑı": 23521, "ĠWitch": 23522, "ĠSwedish": 23523, "ĠBI": 23524, "Ġerase": 23525, "Ġkhi": 23526, "Ġcommentary": 23527, "ĠSultan": 23528, "íĥĿ": 23529, "ĠLeban": 23530, "Ġë³´ìĭ": 23531, "ĠPam": 23532, "pekt": 23533, "month": 23534, "Ġgrounded": 23535, "ê¾": 23536, "ĠÅŁekilde": 23537, "250": 23538, "ĠSCH": 23539, "ioso": 23540, "Ġinaug": 23541, "heimer": 23542, "Ġreflecting": 23543, "ĠRuth": 23544, "ĠOil": 23545, "Ġtrouver": 23546, "uep": 23547, "..]": 23548, "ĠìŀĪë": 23549, "Ġolha": 23550, "Ġreasonably": 23551, "Ġglitch": 23552, "UB": 23553, "ĠGran": 23554, "Ġadalah": 23555, "Ġlent": 23556, "را": 23557, "Ġtraction": 23558, "Ġadjusting": 23559, "´¤": 23560, "нибÑĥдÑĮ": 23561, "Ġдоп": 23562, "Ġstretched": 23563, "Ġort": 23564, "Ġcosine": 23565, "viol": 23566, "Ġìħ": 23567, "cir": 23568, "Ġbastard": 23569, "ä¸ĩ": 23570, "ĠÑħод": 23571, "Ġquier": 23572, "Ġpressures": 23573, "ĠAnh": 23574, "å¹¾": 23575, "Ġelles": 23576, "ĠдÑĢÑĥз": 23577, "ĠможеÑĤе": 23578, "Ġchá»": 23579, "ĠMé": 23580, "ök": 23581, "ầu": 23582, "ìłĪ": 23583, "zin": 23584, "Ġcaution": 23585, "iban": 23586, "Ġjudging": 23587, "ÑĥÑİÑĤ": 23588, "Ġbaj": 23589, "ĠСейÑĩаÑģ": 23590, "ĠPoor": 23591, "ĠNazi": 23592, "Ġupbeat": 23593, "yang": 23594, "Ġweekends": 23595, "ĠEssentially": 23596, "Ġoluyor": 23597, "Ġspatial": 23598, "acker": 23599, "Ġseller": 23600, "Ġ×IJ×ķת": 23601, "ij׾": 23602, "Ġvivid": 23603, "ĠBond": 23604, "ê¶Į": 23605, "iskt": 23606, "ãĤµ": 23607, "Ġgoat": 23608, "driver": 23609, "Ġmug": 23610, "ictional": 23611, "Ġallt": 23612, "ĠIniti": 23613, "ĠRand": 23614, "Ġfinishes": 23615, "Ġê°Ī": 23616, "Ġvitam": 23617, "Ġteenagers": 23618, "ĠMorris": 23619, "ì¤Ħ": 23620, "ĠOri": 23621, "iya": 23622, "Ġmyös": 23623, "Step": 23624, "ĠKre": 23625, "辦": 23626, "Ġdinosaur": 23627, "Ġëªĩ": 23628, "affe": 23629, "ĠëIJ©ëĭĪëĭ¤": 23630, "Ġzeg": 23631, "åĪĩ": 23632, "ĠManhattan": 23633, "Ġsujet": 23634, "uelle": 23635, "stoff": 23636, "Ġdür": 23637, "Ġsubmar": 23638, "eses": 23639, "Ġaquele": 23640, "Ġnou": 23641, "ĠFaith": 23642, "tz": 23643, "ĠÑĤомÑĥ": 23644, "aceut": 23645, "liers": 23646, "Ġbandwidth": 23647, "Æ°á»Ŀ": 23648, "Ġrespective": 23649, "ĠAve": 23650, "Ġspreadshe": 23651, "ĠSent": 23652, "icamente": 23653, "Ġinfra": 23654, "Ġlearners": 23655, "Ġà®ī": 23656, "aiah": 23657, "renal": 23658, "Ġmustard": 23659, "Ġhabt": 23660, "çĥ": 23661, "ĠQué": 23662, "Ġanalyzing": 23663, "æ¯ı": 23664, "Ġsolic": 23665, "Ġ×Ķ×ķ×IJ": 23666, "Ġcausa": 23667, "Ġwelcomed": 23668, "ĠSuccess": 23669, "Ġfacile": 23670, "ĠÐŁÐ¾ÑĤомÑĥ": 23671, "schein": 23672, "Ġfetch": 23673, "Ġstrat": 23674, "ĠÑģÑĤоиÑĤ": 23675, "ìĹIJìĦľëĬĶ": 23676, "ĠÑģпоÑģоб": 23677, "mam": 23678, "ĠserÃŃa": 23679, "naments": 23680, "writer": 23681, "Ġconsulting": 23682, "íĺĢ": 23683, "ĠBerkeley": 23684, "eu": 23685, "asive": 23686, "UU": 23687, "ĠAnalyt": 23688, "Ġsubmission": 23689, "Ġmagnificent": 23690, "enza": 23691, "Ġecon": 23692, "Ġprofiles": 23693, "Ġincar": 23694, "Ab": 23695, "ĠNun": 23696, "Ġhic": 23697, "screaming": 23698, "Ġresilient": 23699, "åĪ©": 23700, "grund": 23701, "Ġconcur": 23702, "Ġbereits": 23703, "LD": 23704, "Ġnurt": 23705, "ìī": 23706, "Ġfeast": 23707, "Ġencuent": 23708, "ĠMichel": 23709, "Ġsuprem": 23710, "\"]": 23711, "Ġfeeds": 23712, "ĠKollegen": 23713, "isser": 23714, "ĠFeng": 23715, "ĠWen": 23716, "mun": 23717, "ĠtenÃŃa": 23718, "ĠWrest": 23719, "Ġìĺ¤ëĬĺìĿĢ": 23720, "Ġstead": 23721, "Ġrestoration": 23722, "Ġdonated": 23723, "Ġdels": 23724, "Ġcensus": 23725, "Ġdesperately": 23726, "worthy": 23727, "HE": 23728, "ĠSpa": 23729, "ĠBryan": 23730, "Ġhj": 23731, "ĠRaw": 23732, "ìķĦë": 23733, "ĠCamera": 23734, "Ġzien": 23735, "Ġstyl": 23736, "ĠTW": 23737, "ĠCheese": 23738, "borne": 23739, "Ġobl": 23740, "ĠAlready": 23741, "Ġunstable": 23742, "Ġflames": 23743, "post": 23744, "Ha": 23745, "romagn": 23746, "ĠìĹĦë§Ī": 23747, "dest": 23748, "Ġkolej": 23749, "Ġtemporarily": 23750, "Ġdetermining": 23751, "ĠGlass": 23752, "ÑĢон": 23753, "olan": 23754, "Ġdominated": 23755, "åĮĸ": 23756, "____": 23757, "ĠÙĩذا": 23758, "ĠDana": 23759, "Ġdinheiro": 23760, "aqu": 23761, "민": 23762, "ĠÃłs": 23763, "ĠJoey": 23764, "ĠGriff": 23765, "Ġattain": 23766, "Ġtransitions": 23767, "ĠLiterally": 23768, "енд": 23769, "ĠHaven": 23770, "Ġgrabbing": 23771, "Ġcrystals": 23772, "ĠFourth": 23773, "Ġcandles": 23774, "ĠÑģлÑĥÑĩа": 23775, "rico": 23776, "Ġ5000": 23777, "etto": 23778, "Ġundo": 23779, "Ġkto": 23780, "Ġdivert": 23781, "Ġchir": 23782, "Ġpersec": 23783, "Ġhiking": 23784, "Ġannouncements": 23785, "çĶ±": 23786, "зÑĭ": 23787, "Ġauc": 23788, "Ġsystemic": 23789, "ĠRM": 23790, "Ïĥα": 23791, "ĠÐĶж": 23792, "Ġyar": 23793, "ĠWard": 23794, "Ġpissed": 23795, "Ġcarn": 23796, "Ġautonomous": 23797, "ãħİãħİ": 23798, "sover": 23799, "æ²ĴéĮ¯": 23800, "å¾Ī好": 23801, "Ġreflex": 23802, "Ġgardens": 23803, "Ġdated": 23804, "ì±": 23805, "amiÄĻ": 23806, "Ġcontinuity": 23807, "Ġcitizenship": 23808, "Ġschwer": 23809, "Ġzak": 23810, "table": 23811, "ĠÑģÑĩ": 23812, "è§ģ": 23813, "ĠÏĥε": 23814, "Ġgenerates": 23815, "구ëĤĺ": 23816, "öh": 23817, "óm": 23818, "alam": 23819, "ĠJUDY": 23820, "ĠBug": 23821, "Ġãģ¦": 23822, "Ġdrones": 23823, "Ġágua": 23824, "acaks": 23825, "æļ": 23826, "ĠÐļон": 23827, "×ĸ×Ķ": 23828, "Ġstrive": 23829, "ĠAltern": 23830, "Ġnearest": 23831, "Ġproyect": 23832, "tera": 23833, "ĠASHLEY": 23834, "Ġworm": 23835, "Ġreplay": 23836, "Ġtara": 23837, "ĠIndians": 23838, "ãĤ°": 23839, "icaid": 23840, "ĠìĪľ": 23841, "Ġappealing": 23842, "ĠWes": 23843, "Ġmentions": 23844, "Ġделе": 23845, "Ġkw": 23846, "Ġfragile": 23847, "isz": 23848, "ków": 23849, "hang": 23850, "color": 23851, "Ġpresidente": 23852, "87": 23853, "еÑĦ": 23854, "çĪ¸": 23855, "Ġдобав": 23856, "ĠNelson": 23857, "áfic": 23858, "ĠMICHAEL": 23859, "Ġmechanic": 23860, "Ġmetres": 23861, "ĠoczywiÅĽcie": 23862, "ĠCind": 23863, "ĠogsÃ¥": 23864, "Ġlandsca": 23865, "ACE": 23866, "Ġheadlines": 23867, "Ġcatalyst": 23868, "ĠCatch": 23869, "inkles": 23870, "Ġpills": 23871, "ordo": 23872, "Ġimmigrant": 23873, "Ġexamination": 23874, "Ġaccidents": 23875, "zÄħd": 23876, "Ġquiere": 23877, "Ġnella": 23878, "Ġ67": 23879, "Ġpassa": 23880, "Ġsuperfic": 23881, "istor": 23882, "Ġnov": 23883, "ëĭµ": 23884, "Ġmandate": 23885, "isons": 23886, "ĠVirtual": 23887, "Ġselber": 23888, "Ġcounseling": 23889, "ĠNBA": 23890, "Ġsept": 23891, "Ġbeliever": 23892, "Ġmarvel": 23893, "ĠIntegr": 23894, "ĠмÑĸ": 23895, "Ġorph": 23896, "Ġbackward": 23897, "ĠGeneration": 23898, "ĠPict": 23899, "ĠÑĤоÑĤ": 23900, "Ġtapi": 23901, "prochen": 23902, "Ġhallway": 23903, "hte": 23904, "ĠÛģÛĴ": 23905, "ĠZum": 23906, "èĢģ師": 23907, "achment": 23908, "iquer": 23909, "folg": 23910, "ĠEddie": 23911, "ĠKil": 23912, "Ġwellness": 23913, "stock": 23914, "è¼ĥ": 23915, "Ġkaç": 23916, "Ġterrorism": 23917, "Ġpointer": 23918, "Of": 23919, "heric": 23920, "ĠUltimately": 23921, "Ġmeses": 23922, "ĠTrade": 23923, "Ġpint": 23924, "Ġtuition": 23925, "Ġdisagre": 23926, "Ġê²ĮìŀĦ": 23927, "Ġmanuscript": 23928, "Ġroomm": 23929, "Ġoutputs": 23930, "еÑĨи": 23931, "Ġries": 23932, "Ġsalud": 23933, "otzdem": 23934, "Ġmasses": 23935, "ĠbyÅĤa": 23936, "Ġclearing": 23937, "Ġdiscourse": 23938, "atson": 23939, "Ġfolded": 23940, "ĠJar": 23941, "ÙĦÙī": 23942, "900": 23943, "ĠÑĥÑģп": 23944, "Ġprophecy": 23945, "Ġinterfere": 23946, "иÑħод": 23947, "à¹Į": 23948, "Ġthri": 23949, "Ġ×ŀש": 23950, "Ġlazım": 23951, "Ġ1992": 23952, "Ġfuturo": 23953, "Ġlocking": 23954, "Ġembargo": 23955, "ĠNeither": 23956, "ivamente": 23957, "ĠmÃ¥ste": 23958, "Ġmik": 23959, "Ġcollector": 23960, "екоÑĤоÑĢ": 23961, "ĠGand": 23962, "Ġsentir": 23963, "ĠMight": 23964, "å¡Ķ": 23965, "Ġganzen": 23966, "UC": 23967, "Ġrelating": 23968, "SD": 23969, "Ġmosquito": 23970, "GR": 23971, "Ġhollow": 23972, "âĺħ": 23973, "ĠWalker": 23974, "Ġaffiliate": 23975, "Ġduplicate": 23976, "нем": 23977, "Ġgrape": 23978, "ĠOrganization": 23979, "Ġsynt": 23980, "Joe": 23981, "Ġgeg": 23982, "Ġrevealing": 23983, "ĠEthan": 23984, "outer": 23985, "Ġyay": 23986, "é«Ķ": 23987, "лаÑĢ": 23988, "Ġreportedly": 23989, "Ġihrer": 23990, "Ġrecognise": 23991, "Ġbumper": 23992, "ĠRandy": 23993, "ĠVenus": 23994, "tles": 23995, "Ġappetite": 23996, "Ġglucose": 23997, "Ġchodzi": 23998, "ĠFurthermore": 23999, "tir": 24000, "Ġconta": 24001, "Ġintuition": 24002, "Ġaltitude": 24003, "Ġchunks": 24004, "ĠJoshua": 24005, "ıģım": 24006, "rylic": 24007, "leans": 24008, "ĠíĶ¼ë": 24009, "LL": 24010, "Que": 24011, "Ġgor": 24012, "ĠзнаÑĩиÑĤ": 24013, "Ġpoems": 24014, "Ġexcel": 24015, "Ġexplored": 24016, "Ġpopul": 24017, "Ġincluso": 24018, "stä": 24019, "ĠGavin": 24020, "alling": 24021, "ĠÏĦον": 24022, "é©": 24023, "arbeit": 24024, "ĠGas": 24025, "Ġglorious": 24026, "rieben": 24027, "Ġspam": 24028, "Ġindoor": 24029, "Ġthrust": 24030, "ĠAld": 24031, "ĠPrior": 24032, "Ġonboard": 24033, "ãģłãģķãģĦ": 24034, "oca": 24035, "ASH": 24036, "£ł": 24037, "ĠChristine": 24038, "Ġdrawer": 24039, "Ġnoon": 24040, "Ġìŀĺë": 24041, "Ġpermanently": 24042, "æ·±": 24043, "ĠнапÑĢимеÑĢ": 24044, "Ġpodcasts": 24045, "erapeut": 24046, "prit": 24047, "Ġstainless": 24048, "ĠÚ©ÛĴ": 24049, "Ġfamilia": 24050, "ĠÑĢазÑĢ": 24051, "unto": 24052, "ĠÑģÑĤол": 24053, "Ġhä": 24054, "ĠHai": 24055, "ĠPB": 24056, "izon": 24057, "Ġkonnte": 24058, "Ġbüyük": 24059, "Ġutilizar": 24060, "ÚĨ": 24061, "Ġaquesta": 24062, "Ġmixer": 24063, "udent": 24064, "лекÑģ": 24065, "ÅĤu": 24066, "ĠÑģиÑģÑĤем": 24067, "ĠноÑĢм": 24068, "Ġfatal": 24069, "Ġconsiderations": 24070, "Ġvalidation": 24071, "Ġoli": 24072, "ĠkardeÅŁ": 24073, "ĠGLORIA": 24074, "Ġpall": 24075, "еÑģÑĤе": 24076, "Ġrectang": 24077, "Ġmedieval": 24078, "allahi": 24079, "asti": 24080, "ĠSyrian": 24081, "Ġshear": 24082, "Ġdebug": 24083, "ĠMai": 24084, "Ġknocking": 24085, "ĠLex": 24086, "ardan": 24087, "rov": 24088, "Ġmemorial": 24089, "æ°£": 24090, "ooky": 24091, "Ġstuffed": 24092, "Ġpassé": 24093, "Ġwig": 24094, "Ĥł": 24095, "Ġpróxima": 24096, "Ġ1991": 24097, "ĠмеждÑĥ": 24098, "Ġnuestros": 24099, "ĠBeast": 24100, "Ġsmo": 24101, "atched": 24102, "ologia": 24103, "Ġмод": 24104, "Ġgee": 24105, "Ġconceptual": 24106, "Ġô": 24107, "Ġdecreases": 24108, "Ġqueries": 24109, "олÑĮÑĪ": 24110, "ĠApart": 24111, "Ġexempl": 24112, "å±±": 24113, "Ġfled": 24114, "ĠOFF": 24115, "ggak": 24116, "Ġbead": 24117, "hir": 24118, "lies": 24119, "ĠClearly": 24120, "ılar": 24121, "Ġchess": 24122, "Ġwhichever": 24123, "Ġ96": 24124, "ằ": 24125, "Ġrespects": 24126, "ĠмоÑĢ": 24127, "Ġorganism": 24128, "Ġgrandpa": 24129, "ĠVie": 24130, "è·Łä½ł": 24131, "Ġflooding": 24132, "Ġupgraded": 24133, "ÑijÑĢ": 24134, "Ġcheeks": 24135, "Ġconquer": 24136, "Ġstubborn": 24137, "Ġpuzzles": 24138, "Ġauction": 24139, "Ġrelying": 24140, "ĠPROF": 24141, "ĠEsper": 24142, "ĠÐľÐ£": 24143, "Ġhype": 24144, "Ġpossibil": 24145, "Ġimprison": 24146, "ĠErn": 24147, "ìĹĪìĬµëĭĪëĭ¤": 24148, "Ġenvie": 24149, "Ġresurrection": 24150, "ä¸įè¡Į": 24151, "Ġsper": 24152, "ĠVenezuela": 24153, "som": 24154, "Ġìŀłê¹": 24155, "Ġnouvelle": 24156, "Ġcloses": 24157, "Ġ1940": 24158, "Ġqua": 24159, "ĠJared": 24160, "ĠPir": 24161, "Ġinde": 24162, "Ġscrub": 24163, "uku": 24164, "Ġrequiring": 24165, "Ġвами": 24166, "Ġconsiderable": 24167, "åIJĽ": 24168, "ilia": 24169, "Ġinne": 24170, "Ġmeinem": 24171, "Ġhardship": 24172, "Ġtraps": 24173, "roc": 24174, "ĠìĦ¤ë": 24175, "Ġresearching": 24176, "ĠMargaret": 24177, "Ġpenny": 24178, "Ġbırak": 24179, "Ñijл": 24180, "Ġwool": 24181, "Ġrhet": 24182, "Ġflatten": 24183, "çĩ": 24184, "à¹Ģร": 24185, "Ġpied": 24186, "ĠChap": 24187, "Ġunderm": 24188, "Ġfret": 24189, "Ġcrashed": 24190, "ĠFrauen": 24191, "Ø°Ùĩ": 24192, "ivan": 24193, "Ġliterary": 24194, "latego": 24195, "Ġspäter": 24196, "Ġsimilarities": 24197, "âĨ": 24198, "ĠCoron": 24199, "ĠCreek": 24200, "Ġbosses": 24201, "Ġaccompanied": 24202, "Ġdebates": 24203, "Ġassembled": 24204, "ĠÃģ": 24205, "ĠVai": 24206, "Ġtract": 24207, "Ġsimplement": 24208, "ĠArin": 24209, "Ġvulnerability": 24210, "Ġhormone": 24211, "IEL": 24212, "OOK": 24213, "Ġrelay": 24214, "ĠAndrea": 24215, "ril": 24216, "Ġnecessity": 24217, "aceutical": 24218, "ÑİÑī": 24219, "ousing": 24220, "nahmen": 24221, "Ġfootprint": 24222, "map": 24223, "ĠTier": 24224, "annya": 24225, "intend": 24226, "åĸ®": 24227, "å¢": 24228, "Ġdecorate": 24229, "Ġzombies": 24230, "ĠHyd": 24231, "ĠSuz": 24232, "Ġcampuses": 24233, "ĠEmb": 24234, "Ġthrottle": 24235, "Ġadmin": 24236, "Ġoportun": 24237, "Ġmirrors": 24238, "Ġidentities": 24239, "ĠClin": 24240, "Ġë¹Ħë": 24241, "á¹£": 24242, "ĠOtt": 24243, "Ġblues": 24244, "Ġimpressions": 24245, "-,": 24246, "Ġvague": 24247, "afe": 24248, "Ġinferior": 24249, "erald": 24250, "Ġmedicines": 24251, "Ġpregunta": 24252, "osely": 24253, "Ġtélé": 24254, "ĠMonth": 24255, "ĠLeaders": 24256, "ĠEgyptian": 24257, "Ġration": 24258, "kers": 24259, "heits": 24260, "Ġrecht": 24261, "Play": 24262, "Ġeg": 24263, "Ġpolls": 24264, "ĠWOODR": 24265, "Ġslots": 24266, "jam": 24267, "Both": 24268, "ĠRat": 24269, "ÑĢаж": 24270, "ĠBright": 24271, "ä¸Ģå®ļ": 24272, "á»iji": 24273, "urious": 24274, "Ġsingers": 24275, "Ġlogin": 24276, "Ġtêm": 24277, "lation": 24278, "ĠMum": 24279, "Æ°á»Ŀng": 24280, "ĠEditor": 24281, "åIJij": 24282, "Ġinnovations": 24283, "have": 24284, "ĠSek": 24285, "Ġweaker": 24286, "ĠGob": 24287, "After": 24288, "´ì§Ģ": 24289, "Ġë¬¸ìłľ": 24290, "ãĥ¼ãĥ¼": 24291, "Ġdisadvantage": 24292, "確": 24293, "Ġgaze": 24294, "ĠMack": 24295, "Ïģί": 24296, "ĠKiss": 24297, "ĠHolo": 24298, "ĠBirth": 24299, "izi": 24300, "bab": 24301, "ä¿Ŀ": 24302, "ìĭľê³ł": 24303, "деÑĢж": 24304, "Ġsquat": 24305, "кÑĥÑģ": 24306, "uni": 24307, "ĠComme": 24308, "ĠWOODRUFF": 24309, "ĠChampionship": 24310, "Ġwelche": 24311, "ĠYouth": 24312, "zem": 24313, "Ġodpow": 24314, "Ġpersistent": 24315, "rut": 24316, "ìĶ©": 24317, "íĸ¥": 24318, "lair": 24319, "iku": 24320, "Ġvendor": 24321, "Ġchúng": 24322, "Ġfinanci": 24323, "Ġoverly": 24324, "âu": 24325, "Ġgluten": 24326, "Ġ1800": 24327, "Ġdivisions": 24328, "Ġciudad": 24329, "Ġobed": 24330, "Ġwarum": 24331, "Ġeher": 24332, "Ġelim": 24333, "ĠÐĴо": 24334, "Ġpeuvent": 24335, "ĠWanna": 24336, "Ġattendance": 24337, "Ġassessments": 24338, "ĠBog": 24339, "Ġimagery": 24340, "Ġcollectively": 24341, "Ġinformal": 24342, "ĠSchwe": 24343, "Ġdeutlich": 24344, "ĠChel": 24345, "ĠPE": 24346, "owed": 24347, "Ġbanner": 24348, "Ġshelves": 24349, "ĠReturn": 24350, "æĭ¿": 24351, "LAUGHS": 24352, "Ġcongratulate": 24353, "ĠNorway": 24354, "Ġdwell": 24355, "ĠCaribbean": 24356, "Ġnorms": 24357, "ĠAnimal": 24358, "ĠValentine": 24359, "Ġextending": 24360, "ĠVou": 24361, "orr": 24362, "ĠCheng": 24363, "¡": 24364, "ĠдоÑĢог": 24365, "Ġveg": 24366, "ĠhÃ¥": 24367, "ĠXin": 24368, "Ġì¹´ë": 24369, "emet": 24370, "Ġhypoth": 24371, "Ġinteressante": 24372, "rices": 24373, "IZ": 24374, "ĠUSD": 24375, "Ġrunner": 24376, "ĠBag": 24377, "Ġê½": 24378, "Ġcomeçar": 24379, "Ġpigs": 24380, "Ġweaknesses": 24381, "Ph": 24382, "ĠViol": 24383, "ä¸įçĶ¨": 24384, "Ġdragging": 24385, "ĠAquÃŃ": 24386, "ĠCSS": 24387, "Ġmillimeters": 24388, "Ġestás": 24389, "Ġacute": 24390, "Ġdejar": 24391, "iÄŁ": 24392, "obra": 24393, "Love": 24394, "Ġsilk": 24395, "****": 24396, "Ġjoins": 24397, "Ġprol": 24398, "Ġê°IJìĤ¬íķ©ëĭĪëĭ¤": 24399, "æĶ¯": 24400, "ØŃد": 24401, "aghetti": 24402, "änner": 24403, "Ġstrang": 24404, "Ġdoubled": 24405, "Ġdescriptions": 24406, "Ġstellen": 24407, "Ġparti": 24408, "ç«ĭ": 24409, "²Ħë": 24410, "ĠÃ¶ÄŁ": 24411, "ighing": 24412, "Ġangular": 24413, "Ġnatuur": 24414, "ĠShel": 24415, "Æ°Æ¡": 24416, "Ġrays": 24417, "Ġseper": 24418, "start": 24419, "vised": 24420, "Ġrushed": 24421, "Ġinternationally": 24422, "Ġnivel": 24423, "Ġboxing": 24424, "fallen": 24425, "á»ijc": 24426, "Ġseinen": 24427, "plicity": 24428, "Ġcarboh": 24429, "ĠTravis": 24430, "uso": 24431, "ĠPhase": 24432, "Ġactivation": 24433, "Ġopio": 24434, "·¨": 24435, "Ġdecreased": 24436, "Car": 24437, "Ġbundle": 24438, "Ġexpend": 24439, "ormal": 24440, "Ġadjacent": 24441, "Ġmee": 24442, "ĠоÑĢг": 24443, "Ġtranscript": 24444, "ĠLanguage": 24445, "GS": 24446, "è§ī": 24447, "Ġseul": 24448, "Ãłnh": 24449, "Ġnya": 24450, "nings": 24451, "Ġìĭľë": 24452, "ĠëĶ°ëĿ¼": 24453, "ĠAgr": 24454, "ÃŃd": 24455, "çķĻ": 24456, "Ġaby": 24457, "ĠNeo": 24458, "ıyoruz": 24459, "ĠThinking": 24460, "aime": 24461, "Ġvite": 24462, "Ġtravés": 24463, "Ġ×ij×¢": 24464, "Ġмед": 24465, "Our": 24466, "hoot": 24467, "Ġliner": 24468, "ĠPizza": 24469, "Ġhyg": 24470, "flies": 24471, "ĠContinue": 24472, "Ġdental": 24473, "ĠTib": 24474, "Ġregulate": 24475, "lieÃŁ": 24476, "ALK": 24477, "ĠTae": 24478, "길": 24479, "ĠBrexit": 24480, "ĠGut": 24481, "Ġoccupation": 24482, "Ġzrobi": 24483, "âm": 24484, "Ġwhisk": 24485, "ä¸ĸçķĮ": 24486, "Ġkanske": 24487, "omon": 24488, "robe": 24489, "Ġwarfare": 24490, "Ġthá»ĥ": 24491, "Ġjaki": 24492, "Ġstrokes": 24493, "Ġpeas": 24494, "ĠDamit": 24495, "HAN": 24496, "Ġinterference": 24497, "ĠминÑĥÑĤ": 24498, "NER": 24499, "outing": 24500, "Ġtextures": 24501, "Łī": 24502, "owi": 24503, "ĠíķĻ": 24504, "Ġdens": 24505, "Ġprotagonist": 24506, "änn": 24507, "Ġgoddess": 24508, "Ġwollte": 24509, "ijo": 24510, "ĠWoche": 24511, "ĠVPN": 24512, "story": 24513, "Ġkinderg": 24514, "Ġfunnel": 24515, "Ġdistress": 24516, "ноÑģÑĤÑĮÑİ": 24517, "Ġnoisy": 24518, "ĠпÑĢодолж": 24519, "Ġdaran": 24520, "Ġenzyme": 24521, "лож": 24522, "Ġmute": 24523, "Ġdwar": 24524, "Ġاس": 24525, "Ġkompl": 24526, "Ġmerit": 24527, "Ġfosse": 24528, "ĠDrink": 24529, "Ġfora": 24530, "Ġwohl": 24531, "Ġbreeze": 24532, "Ġsanit": 24533, "Ġdrin": 24534, "ĠìĿ´ê±°ëĬĶ": 24535, "Ġ62": 24536, "Ġì°¨ë": 24537, "abytes": 24538, "Ġdeeds": 24539, "Ġй": 24540, "ième": 24541, "iggling": 24542, "Ġ\"'": 24543, "ĠÑĩаÑģÑĤÑĮ": 24544, "ĠAnswer": 24545, "Ġevangel": 24546, "Ġ1080": 24547, "ĠVisit": 24548, "icient": 24549, "Ġreliability": 24550, "ÑİÑģÑĮ": 24551, "ĠEarlier": 24552, "Ġfid": 24553, "çŃīä¸Ģä¸ĭ": 24554, "Ġsleeves": 24555, "iyorsun": 24556, "Ġbib": 24557, "ĠAccount": 24558, "Ñıли": 24559, "ciplinary": 24560, "zas": 24561, "ĠбеÑĢ": 24562, "Ġnecklace": 24563, "Ġblender": 24564, "ĠPhillips": 24565, "eti": 24566, "ĠJupiter": 24567, "Ġprovoc": 24568, "ĠYears": 24569, "entre": 24570, "acio": 24571, "Ġkü": 24572, "Ġantenna": 24573, "Ġnovels": 24574, "Ġfart": 24575, "ĠSugar": 24576, "ĠJudy": 24577, "Ġcollapsed": 24578, "ç°": 24579, "ritis": 24580, "ĠìĥģíĻ©": 24581, "ÐĹЫ": 24582, "ĠVerf": 24583, "ranean": 24584, "ereum": 24585, "ĠTarget": 24586, "Ġ88": 24587, "ĠÐĺз": 24588, "ideo": 24589, "Ġregression": 24590, "ì¶ľ": 24591, "Ġmówi": 24592, "Ġstudios": 24593, "iens": 24594, "iph": 24595, "Ġfrying": 24596, "Ġfascinated": 24597, "ĠWah": 24598, "bucks": 24599, "maya": 24600, "ĠSaturn": 24601, "ĠMommy": 24602, "Ġratings": 24603, "Ġautumn": 24604, "Æ°Æ¡ng": 24605, "Ġloser": 24606, "Ġcentro": 24607, "érieur": 24608, "ĠFold": 24609, "Ġsupervisor": 24610, "ĠNobel": 24611, "Ġunderest": 24612, "obia": 24613, "ĠвÑģÑı": 24614, "Ġverw": 24615, "Ġfuels": 24616, "Ġartifacts": 24617, "Ġë¶Ļ": 24618, "ĠAutom": 24619, "çļĦæĺ¯": 24620, "ÛĶ": 24621, "×ķס": 24622, "Ġihnen": 24623, "Ġ59": 24624, "ounding": 24625, "еÑĢÑĭ": 24626, "inars": 24627, "chant": 24628, "Ġaddicted": 24629, "Ġexplosive": 24630, "Ġdispers": 24631, "âĸĪ": 24632, "axis": 24633, "ARY": 24634, "Ġlum": 24635, "ĠÑĥÑģл": 24636, "ĠØĮ": 24637, "Ġrupees": 24638, "ĠPearl": 24639, "camp": 24640, "tv": 24641, "oya": 24642, "Ġconcludes": 24643, "Ġcollision": 24644, "Ġbuyer": 24645, "Ġplayground": 24646, "Ġsprings": 24647, "Ġfeminine": 24648, "ĠRas": 24649, "Ġincarcer": 24650, "íĹĺ": 24651, "Ġdialect": 24652, "Ġclosure": 24653, "Ġchatting": 24654, "Ġbabe": 24655, "Ġspotlight": 24656, "Ġnotation": 24657, "è·¯": 24658, "Star": 24659, "ião": 24660, "Ġtête": 24661, "Ġtide": 24662, "Ġjunto": 24663, "Ġsenator": 24664, "Ð¥": 24665, "Ġexcuses": 24666, "Ġblink": 24667, "Ġadmission": 24668, "ĠLily": 24669, "Ñĭми": 24670, "Ġamigo": 24671, "Ġlust": 24672, "ëĭ¬": 24673, "Ġamino": 24674, "äºĭæĥħ": 24675, "Ġconsultant": 24676, "ĠElectric": 24677, "Ġëħ¸ëŀĺ": 24678, "ujah": 24679, "Ġshooter": 24680, "ichten": 24681, "ĠUkrainian": 24682, "Ġaims": 24683, "ĠEntertain": 24684, "Ġmiracles": 24685, "èŃ°": 24686, "Ġzeigen": 24687, "Ġlam": 24688, "Ġress": 24689, "ĠJill": 24690, "ylan": 24691, "Ġrook": 24692, "Ġhaya": 24693, "Ġpassport": 24694, "adata": 24695, "Ġjuicy": 24696, "conf": 24697, "лей": 24698, "ĠSz": 24699, "Ġintercept": 24700, "ãģĤãĤĬãģĮãģ¨ãģĨãģĶãģĸ": 24701, "ĠTeams": 24702, "Ġmaken": 24703, "irrel": 24704, "ĠLIKE": 24705, "áºŃy": 24706, "êµ°": 24707, "Ġshortage": 24708, "Ġparadigm": 24709, "Ġpapel": 24710, "Ġastero": 24711, "ãģ¾ãģŁ": 24712, "Ġsollen": 24713, "ĠMickey": 24714, "ĠOrleans": 24715, "Ġcholesterol": 24716, "Ġgoose": 24717, "ÑĨиÑİ": 24718, "ãģĤãĤĭ": 24719, "ĠFL": 24720, "Ġголов": 24721, "Ġtribute": 24722, "ĠGam": 24723, "Ġévidemment": 24724, "ÑıÑħ": 24725, "å®ŀ": 24726, "çĶ°": 24727, "Ġinappropri": 24728, "uhan": 24729, "Ġorganizational": 24730, "ailed": 24731, "Ġendure": 24732, "Ġ76": 24733, "Ġshotgun": 24734, "Ġlivre": 24735, "Ġsuited": 24736, "Ġwarmth": 24737, "ĠSIM": 24738, "Ġenvision": 24739, "Ġdegrad": 24740, "îne": 24741, "Laughing": 24742, "ĠWhoever": 24743, "ĠBuddhism": 24744, "Ġsprinkle": 24745, "ceÄŁiz": 24746, "Ġruins": 24747, "Ġstarch": 24748, "ĠHerz": 24749, "Ġinjustice": 24750, "Ġhumidity": 24751, "ожалÑĥй": 24752, "ĠObject": 24753, "ĠIgn": 24754, "ĠExam": 24755, "igers": 24756, "Ġthou": 24757, "ĠSoy": 24758, "ivas": 24759, "Ġpoles": 24760, "math": 24761, "Ġвним": 24762, "INGING": 24763, "edral": 24764, "Ġexplor": 24765, "Ġroasted": 24766, "Ġcrawl": 24767, "Ġcoff": 24768, "Ġanom": 24769, "Ġwij": 24770, "Ġimproves": 24771, "Ġtreaty": 24772, "Ġdiscovering": 24773, "Ġstatute": 24774, "Ġmercado": 24775, "ĠÑģил": 24776, "Ġintel": 24777, "ĠChancellor": 24778, "ĠMedicaid": 24779, "ugi": 24780, "Ġverbal": 24781, "Ġdön": 24782, "Ġscripture": 24783, "Ġiteration": 24784, "eks": 24785, "ĠOxford": 24786, "Ġwäh": 24787, "ĠVad": 24788, "ĠAK": 24789, "ĠìķĦìĿ´ë": 24790, "Ġiets": 24791, "Ġneedles": 24792, "ÙĥÙħ": 24793, "Ġpasado": 24794, "Ġalbums": 24795, "Ġyea": 24796, "etzen": 24797, "ĦëıĦ": 24798, "Ġdetermines": 24799, "Ġthee": 24800, "ĠPlaying": 24801, "ärt": 24802, "Ġצ": 24803, "cled": 24804, "Ġdownward": 24805, "alone": 24806, "Ġsolu": 24807, "Ġpartition": 24808, "Ġwz": 24809, "dd": 24810, "Ġpessoal": 24811, "媽": 24812, "Ġfactories": 24813, "Ġbleibt": 24814, "มา": 24815, "alsa": 24816, "ĠNFL": 24817, "Ġfuera": 24818, "Ġreserved": 24819, "ĠEarn": 24820, "Ġhelt": 24821, "Ġshortcut": 24822, "Ġconvincing": 24823, "space": 24824, "Ġenforce": 24825, "Ġcores": 24826, "Ġefter": 24827, "Ġrecession": 24828, "xico": 24829, "Ġproposition": 24830, "arians": 24831, "ropol": 24832, "Ġ몰ë": 24833, "ĠÎľ": 24834, "ĠìļĶì¦ĺ": 24835, "Ġactivist": 24836, "Ġconviction": 24837, "Ġzab": 24838, "Ġcanceled": 24839, "ÑĤоÑĩно": 24840, "Ġή": 24841, "éĢĻ樣åŃIJ": 24842, "nite": 24843, "Ġfundra": 24844, "buzzer": 24845, "ело": 24846, "ications": 24847, "Ġzona": 24848, "Ġteens": 24849, "Ġmethodology": 24850, "Ġì¤ijìļĶ": 24851, "than": 24852, "ĠUl": 24853, "ĠGrey": 24854, "Ġhog": 24855, "INK": 24856, "ĠSung": 24857, "ĠClaud": 24858, "ĠCNN": 24859, "Ġdelivers": 24860, "alin": 24861, "ĠAdobe": 24862, "othe": 24863, "ĠDeswegen": 24864, "ำ": 24865, "Ġwerde": 24866, "Ġgrease": 24867, "Ġupgrades": 24868, "ĠFinland": 24869, "accept": 24870, "Ġinterrog": 24871, "bee": 24872, "Ġãģ«": 24873, "Ġprede": 24874, "ĠNep": 24875, "ĠCambridge": 24876, "Ġgraphs": 24877, "Ġhaunted": 24878, "Ñģем": 24879, "æ§": 24880, "åħĭ": 24881, "Some": 24882, "ĠMall": 24883, "Ġrehearsal": 24884, "ĠUrban": 24885, "ĠLag": 24886, "Ġnim": 24887, "ê°ķ": 24888, "Ġpositioned": 24889, "Ġavoided": 24890, "EMA": 24891, "Ġllegar": 24892, "Ġrápido": 24893, "Ġgouvern": 24894, "Ġhing": 24895, "Ġdealer": 24896, "Ġreforms": 24897, "Ġfatty": 24898, "кол": 24899, "ĠAce": 24900, "Ġnep": 24901, "Ġì²Ń": 24902, "Ġcomputation": 24903, "ĠStream": 24904, "bourne": 24905, "tur": 24906, "Por": 24907, "Ġsleepy": 24908, "Ġbanget": 24909, "ãģĤãģ®": 24910, "Ġweighs": 24911, "Ġbleiben": 24912, "ĠGren": 24913, "Ġunions": 24914, "ĠêµIJ": 24915, "Ġaprender": 24916, "uitar": 24917, "ĠJest": 24918, "uming": 24919, "ĠPlayer": 24920, "ĠExtrem": 24921, "Ġinteger": 24922, "аÑĩе": 24923, "Ġconcerts": 24924, "×ķ׼": 24925, "ĠtrochÄĻ": 24926, "ĠRepe": 24927, "éĩįè¦ģ": 24928, "à¹Ĥ": 24929, "żen": 24930, "Ġsounding": 24931, "Ġanonymous": 24932, "Ġexca": 24933, "ĠIranian": 24934, "Ġenergetic": 24935, "Ġwives": 24936, "ĠÑĨвеÑĤ": 24937, "Ġais": 24938, "ãģĭãģª": 24939, "Ġsudah": 24940, "Ġunderwear": 24941, "Ġcrunchy": 24942, "ĠPain": 24943, "Ġgerçek": 24944, "redict": 24945, "Ġmisma": 24946, "ÑĸÑĤ": 24947, "Ġsurviving": 24948, "ÎŃÏĤ": 24949, "Ġparticipant": 24950, "ĠHessen": 24951, "árias": 24952, "Ġsubway": 24953, "istä": 24954, "Ġcoral": 24955, "Ġmarijuana": 24956, "ĠMemorial": 24957, "ÑĪий": 24958, "riz": 24959, "Ġsatellites": 24960, "Ġlease": 24961, "ĠCameron": 24962, "umph": 24963, "Ġclassmates": 24964, "ähän": 24965, "ÑģÑĤве": 24966, "Ġhue": 24967, "ĵ¤ìĿĦ": 24968, "Ġproportional": 24969, "Ġnoss": 24970, "Ġlaps": 24971, "rÃ¥": 24972, "Ġbitcoin": 24973, "ÐĹЫÐļÐIJ": 24974, "Ġ충": 24975, "ĠÙĦÙĦ": 24976, "ĠMort": 24977, "ĠEsp": 24978, "arnos": 24979, "ĠÑģказал": 24980, "Ġänd": 24981, "åħĦ": 24982, "×Ļ×Ļ×Ŀ": 24983, "ĠGeb": 24984, "gehen": 24985, "Inaudible": 24986, "borough": 24987, "ÑĦÑĦ": 24988, "Ġfellowship": 24989, "ĠPaper": 24990, "Ġcurved": 24991, "ĠGEOR": 24992, "Ġcalculator": 24993, "ĠCatal": 24994, "ĠvÃło": 24995, "Ġbypass": 24996, "леÑĤ": 24997, "à³": 24998, "trans": 24999, "rencies": 25000, "ì¡Į": 25001, "igent": 25002, "Ġtasted": 25003, "Ġoceans": 25004, "uft": 25005, "ervice": 25006, "ĠÐľÐ£ÐĹЫÐļÐIJ": 25007, "ĠClassic": 25008, "Ġrespectively": 25009, "~)": 25010, "ître": 25011, "ĠNash": 25012, "Ġzit": 25013, "ĠìĽĥ": 25014, "ĠëĨĴ": 25015, "quote": 25016, "ĠUns": 25017, "Ġtac": 25018, "Ġproves": 25019, "ĠPortland": 25020, "bly": 25021, "Ġere": 25022, "ì¶Ķ": 25023, "Ġépoca": 25024, "ĠÑĤÑĭÑģÑıÑĩ": 25025, "76": 25026, "Ġhade": 25027, "ĠFro": 25028, "ĠpolÃŃtica": 25029, "tag": 25030, "ĠíķŃ": 25031, "Ġschö": 25032, "arett": 25033, "Ġprovisions": 25034, "Ġmotors": 25035, "Ġimaging": 25036, "Ġdok": 25037, "ulously": 25038, "Ġmeille": 25039, "çİ°åľ¨": 25040, "ëIJ": 25041, "ĠISO": 25042, "ĠSTEM": 25043, "ĠBowl": 25044, "Ġtowers": 25045, "ĠEe": 25046, "ĠPerformance": 25047, "Ġloin": 25048, "cussion": 25049, "Ġcoastal": 25050, "iale": 25051, "compass": 25052, "Ġspells": 25053, "Ġdisappointing": 25054, "Ġë²Ī째": 25055, "EER": 25056, "Ġversatile": 25057, "asury": 25058, "Ġenfin": 25059, "Ġdownside": 25060, "Ġguiding": 25061, "ĠاÙĦÙĤ": 25062, "Ġninety": 25063, "charged": 25064, "ĠFans": 25065, "Ġphilosophical": 25066, "Ġgarn": 25067, "ĠmÃ¥nga": 25068, "Ġwillingness": 25069, "Ġportions": 25070, "aben": 25071, "Ġï": 25072, "¿": 25073, "raul": 25074, "Ġsprint": 25075, "ifen": 25076, "ıyla": 25077, "ĠкÑĥп": 25078, "ãģıãģłãģķãģĦ": 25079, "Ġensuite": 25080, "ĠCapitol": 25081, "Ġ63": 25082, "ĠговоÑĢиÑĤ": 25083, "Ġappointments": 25084, "æī¾": 25085, "omiast": 25086, "Ġcareg": 25087, "Ġpublisher": 25088, "Ġheraus": 25089, "Ġεί": 25090, "ĠVS": 25091, "ãģĿãģĹãģ¦": 25092, "ä¸Ńåħ±": 25093, "Ġsacrifices": 25094, "third": 25095, "Ġhumanitarian": 25096, "ĠëĤ´ì": 25097, "imon": 25098, "Ġinequ": 25099, "Ġzob": 25100, "Ġcomfortably": 25101, "ĠDinge": 25102, "Ġcancelled": 25103, "ĠPSAKI": 25104, "ĠRobinson": 25105, "Ġfins": 25106, ")?": 25107, "ĠHistor": 25108, "ĠÑĩеловека": 25109, "Ġtbsp": 25110, "text": 25111, "kim": 25112, "Ġupdating": 25113, "Ġgeld": 25114, "feld": 25115, "ı¼": 25116, "Ġmä": 25117, "Ġcafé": 25118, "ÖĢ": 25119, "ĠSri": 25120, "ĠRegion": 25121, "ĠHahaha": 25122, "Ġfinances": 25123, "ĠاÙĦØ´": 25124, "Ġbunk": 25125, "ruk": 25126, "haft": 25127, "Ġlateral": 25128, "Ġextensions": 25129, "ĠìķĦìĿ´": 25130, "Ġdefinite": 25131, "ĠZhao": 25132, "ĠLuis": 25133, "sty": 25134, "Ġcasos": 25135, "ĠKlim": 25136, "Ġ1993": 25137, "Ġrealization": 25138, "Ġhistorian": 25139, "Ġcracked": 25140, "ëĤ´": 25141, "Ġsystème": 25142, "ĠCIA": 25143, "ĠÑĤво": 25144, "ospheric": 25145, "Ġflee": 25146, "Ġrất": 25147, "ĠRegardless": 25148, "Ġreluct": 25149, "Ġtimely": 25150, "ĠJulian": 25151, "GM": 25152, "éĴ": 25153, "adura": 25154, "é£Ł": 25155, "Ġdresses": 25156, "çģ£": 25157, "ĠëĶĶ": 25158, "Ġnominated": 25159, "Ġadvocates": 25160, "ymph": 25161, "Ġrecordings": 25162, "Ġdeviation": 25163, "Ġprioritize": 25164, "Ġspiral": 25165, "ĠYOUR": 25166, "Ġtranspose": 25167, "ampoo": 25168, "ĠìĽIJëŀĺ": 25169, "ĠVision": 25170, "Ġpolite": 25171, "Ġhamb": 25172, "ĠPatient": 25173, "æ¯Ķè¼ĥ": 25174, "íģ¬ë": 25175, "Ġsia": 25176, "Ġê³³": 25177, "Ġže": 25178, "è§Ģ": 25179, "Ġsupermarket": 25180, "ë¹": 25181, "ĠSierra": 25182, "Ġgrilled": 25183, "ĠUpon": 25184, "Ġabsent": 25185, "Ġmec": 25186, "ĠApollo": 25187, "Ġpunk": 25188, "ĠPaÅĦst": 25189, "ĠÑģвой": 25190, "Ġ거기": 25191, "Girl": 25192, "Ġskinny": 25193, "ĠPremier": 25194, "Ġterritories": 25195, "Ġliability": 25196, "Ġjerk": 25197, "ratic": 25198, "Ġdancers": 25199, "ĠÑĥÑĢов": 25200, "Ġê´Ģë": 25201, "only": 25202, "ĠStu": 25203, "Ġskeleton": 25204, "ĠëŃIJë": 25205, "Ġзакон": 25206, "ıkt": 25207, "ĠMIKE": 25208, "Ġlö": 25209, "mie": 25210, "Ġreiter": 25211, "ãģĵãĤĮãģ¯": 25212, "ĠKolleg": 25213, "ĠAdams": 25214, "licher": 25215, "Ġçocuk": 25216, "Ñıг": 25217, "Ġblush": 25218, "Ġsunshine": 25219, "Ġez": 25220, "ĠDevil": 25221, "Ġ길": 25222, "ĠãģĬ": 25223, "add": 25224, "Ġlicensed": 25225, "Ġvinyl": 25226, "ĠCzech": 25227, "imag": 25228, "Ġcracking": 25229, "Ġìº": 25230, "Ġudah": 25231, "Ġsommes": 25232, "Ġìĸ¼êµ": 25233, "waÄĩ": 25234, "Ġfres": 25235, "åij½": 25236, "ĠWalmart": 25237, "ĠТепеÑĢÑĮ": 25238, "atisf": 25239, "CI": 25240, "lang": 25241, "Ġdiffusion": 25242, "çĶ·": 25243, "Ġsomos": 25244, "ĠMakes": 25245, "æĪijæĥ³": 25246, "ĠRicky": 25247, "Ġmucha": 25248, "íķ¨": 25249, "Ġhorsepower": 25250, "asia": 25251, "Ġfibers": 25252, "Ġerm": 25253, "Ñģкие": 25254, "Ġjeste": 25255, "Ġfirefight": 25256, "Ġcuisine": 25257, "Ġbesonders": 25258, "dig": 25259, "Ġì¢ħ": 25260, "ĠÑĥж": 25261, "Ġtracing": 25262, "Ġcertains": 25263, "ĠApply": 25264, "ÑĭваÑĤÑĮ": 25265, "çĮ": 25266, "Ġbru": 25267, "ĠYES": 25268, "ĠBai": 25269, "ĠDit": 25270, "ĠBis": 25271, "Ġunle": 25272, "ÑģÑĤаÑĤоÑĩно": 25273, "ĠAwak": 25274, "..\"": 25275, "Ġ125": 25276, "Ġrooted": 25277, "Ġcautious": 25278, "const": 25279, "Ġorchestra": 25280, "çľ¼": 25281, "ĠвнÑĥÑĤ": 25282, "Ġquelqu": 25283, "ĠоÑĤвеÑĤ": 25284, "ĠMethod": 25285, "ì¹ľ": 25286, "ĠμαÏĤ": 25287, "lü": 25288, "ĠìķĦê¹Į": 25289, "Ġnaming": 25290, "Char": 25291, "ĠSicher": 25292, "Ġprivileged": 25293, "ĠFly": 25294, "Ġãģĭ": 25295, "áºŃt": 25296, "Ġadvances": 25297, "ĠZelda": 25298, "Ġandra": 25299, "Ġgrinding": 25300, "ĠEdition": 25301, "pf": 25302, "Ġwarriors": 25303, "Ġhedge": 25304, "Ġunseren": 25305, "ĠÑģÑİда": 25306, "eliness": 25307, "Ġpersonalities": 25308, "Ġfö": 25309, "'M": 25310, "ĠÑĤоÑĩно": 25311, "Ġshipped": 25312, "Ġmeteor": 25313, "Ġsurroundings": 25314, "ĠFill": 25315, "uesta": 25316, "ĠPersonal": 25317, "ĠAlle": 25318, "ORT": 25319, "ä¹ħ": 25320, "ĠSche": 25321, "VI": 25322, "Ġcomparable": 25323, "damn": 25324, "Ġditch": 25325, "YAN": 25326, "ismus": 25327, "Ġpickup": 25328, "Ġdak": 25329, "ĠEP": 25330, "best": 25331, "ĠSue": 25332, "ällt": 25333, "Ġpopcorn": 25334, "Ġfolding": 25335, "home": 25336, "иваеÑĤ": 25337, "å·²ç¶ĵ": 25338, "Ġannot": 25339, "chuck": 25340, "Ġfierce": 25341, "Ġdamaging": 25342, "Ġflop": 25343, "Ġpasar": 25344, "Ġreef": 25345, "ĠÑģвоей": 25346, "Ġzoo": 25347, "overs": 25348, "jets": 25349, "Ġprès": 25350, "ĠSilicon": 25351, "teok": 25352, "ĠSeth": 25353, "atamente": 25354, "Ġtransmitted": 25355, "Ġreplicate": 25356, "Ġslim": 25357, "ĠCream": 25358, "æĦŁãģĺ": 25359, "Ġsidewalk": 25360, "ìĪĺë": 25361, "ĠжизнÑĮ": 25362, "ĠMonica": 25363, "ä¾ĨäºĨ": 25364, "Ġcopied": 25365, "ĠTerra": 25366, "istent": 25367, "ç³»": 25368, "Ġоно": 25369, "Ġwhale": 25370, "ĠWITH": 25371, "лÑĥÑĪ": 25372, "å½±çīĩ": 25373, "ĠEen": 25374, "ĠÑģвои": 25375, "Ġordin": 25376, "Ġplural": 25377, "Ġspokes": 25378, "Ġdispute": 25379, "Ġsensible": 25380, "Ġpreaching": 25381, "Ġktórzy": 25382, "pted": 25383, "avier": 25384, "Ġpistol": 25385, "ĠTapi": 25386, "ĠÅĤ": 25387, "ffff": 25388, "Ġacrylic": 25389, "Ġignorance": 25390, "ĠZiel": 25391, "rans": 25392, "Ġwelding": 25393, "mid": 25394, "æĪijä¸į": 25395, "Ġзаним": 25396, "Ġlanes": 25397, "Ġmines": 25398, "Ġmoms": 25399, "×ķ×Ĺ": 25400, "ĠChamber": 25401, "tier": 25402, "Ġmodest": 25403, "ĠìĹ¬ê¸°ìĦľ": 25404, "Ġunas": 25405, "Ġwrench": 25406, "handed": 25407, "Ġsaturated": 25408, "ĠFang": 25409, "ĠCommissioner": 25410, "र": 25411, "Ġ×ĸ": 25412, "ĠLouisiana": 25413, "ĠMask": 25414, "Ġcubes": 25415, "ìĶ¨": 25416, "Ġvidéos": 25417, "ĠnÃ¥gon": 25418, "Ġrider": 25419, "Ġì¶ľ": 25420, "Ġsón": 25421, "ĠLatino": 25422, "bank": 25423, "íķ´ì£¼": 25424, "ĠBrend": 25425, "Ġsexuality": 25426, "...,": 25427, "Ġforgetting": 25428, "ĠÛĮ": 25429, "ĠAvengers": 25430, "ĠBonjour": 25431, "cessor": 25432, "кÑĢаÑĹ": 25433, "cence": 25434, "Ġgeograph": 25435, "culo": 25436, "оÑģÑĤÑĮ": 25437, "Ġsweating": 25438, "íĥĢ": 25439, "Ġsymmetry": 25440, "tsÃ¥": 25441, "Ġjan": 25442, "ĠFerr": 25443, "é¦ĸ": 25444, "Ġambassador": 25445, "ziÄĻk": 25446, "Ġmusun": 25447, "ĠÑĥÑĤ": 25448, "ĠLG": 25449, "issent": 25450, "commun": 25451, "Ġcours": 25452, "Ġdevelops": 25453, "Ġbronze": 25454, "Ġsubstances": 25455, "driven": 25456, "주ìĦ¸ìļĶ": 25457, "Ġaos": 25458, "åĦĦ": 25459, "ĠPROFESS": 25460, "half": 25461, "Ġsorted": 25462, "ĠBomb": 25463, "лаг": 25464, "ĠMalaysia": 25465, "ĠChristina": 25466, "Ġteammate": 25467, "èģŀ": 25468, "FT": 25469, "Ġkı": 25470, "hearted": 25471, "++": 25472, "ogenic": 25473, "Ġbells": 25474, "ĠOuais": 25475, "Ġspecialists": 25476, "бÑĭ": 25477, "depth": 25478, "lasses": 25479, "gies": 25480, "ĠCoffee": 25481, "Ġmarking": 25482, "Ġfoll": 25483, "uli": 25484, "Ġadhesive": 25485, "ĠBot": 25486, "ĠPunkt": 25487, "eye": 25488, "ĠBub": 25489, "elong": 25490, "åĪ¶": 25491, "ĠпÑĢик": 25492, "Ġdonor": 25493, "84": 25494, "Ġenfor": 25495, "Ġcatches": 25496, "Ġbricks": 25497, "Ġknitting": 25498, "ĠKnowing": 25499, "oks": 25500, "HY": 25501, "ride": 25502, "ĠFantasy": 25503, "iman": 25504, "Ġpse": 25505, "Ġìĺ¨": 25506, "Ġвд": 25507, "Ġrestra": 25508, "Ġevaluated": 25509, "ÑĢев": 25510, "Ġfortunately": 25511, "Ġchegar": 25512, "رب": 25513, "Ġdomains": 25514, "ibi": 25515, "arry": 25516, "Ġshutter": 25517, "Ġficou": 25518, "Mike": 25519, "Ġinclu": 25520, "Ġdonors": 25521, "Ġapl": 25522, "ĠLower": 25523, "Ġimported": 25524, "Ġacademy": 25525, "Ġfinals": 25526, "Ġdisappears": 25527, "ÙĬا": 25528, "Ġadministrator": 25529, "js": 25530, "Ġcutter": 25531, "Ġranging": 25532, "örper": 25533, "Ġconstraint": 25534, "ĠTable": 25535, "ĠShan": 25536, "vic": 25537, "ĠFix": 25538, "ĠSwift": 25539, "ounces": 25540, "ĠWarum": 25541, "Ġlettuce": 25542, "appelle": 25543, "Ġshave": 25544, "Ġbás": 25545, "Ġ77": 25546, "ĠOoo": 25547, "ao": 25548, "ĠMcM": 25549, "ĠDrew": 25550, "Ġlump": 25551, "Ġlashes": 25552, "scheinlich": 25553, "Rep": 25554, "inis": 25555, "ĠCette": 25556, "Ġcomposite": 25557, "emetery": 25558, "Ġsorte": 25559, "ĠFinancial": 25560, "оне": 25561, "rones": 25562, "ĠVoy": 25563, "Ġtéc": 25564, "ł¹": 25565, "ĠNinja": 25566, "ĠCorin": 25567, "еннÑı": 25568, "ìĿ´ìĹĪ": 25569, "Ġnich": 25570, "Ġdetective": 25571, "âĢ¦\"": 25572, "Ïĥε": 25573, "Ŀ¼ëıĦ": 25574, "Ġë³Ģ": 25575, "Ġë¸Ķë": 25576, "Ġprope": 25577, "ĠWright": 25578, "Ġ×Ķת": 25579, "ĠShi": 25580, "ĠãģŁ": 25581, "Ġinvestigations": 25582, "éĤĦæĺ¯": 25583, "ĠPowerPoint": 25584, "ĠChu": 25585, "Ġìĺ¤í": 25586, "ĠìĻĦìłĦ": 25587, "ĠFragen": 25588, "unning": 25589, "Ġpourrait": 25590, "Ġtextbook": 25591, "мÑĭ": 25592, "Ġfahren": 25593, "ĠÑĤоÑĢ": 25594, "Ġlakes": 25595, "ünde": 25596, "Int": 25597, "ĠMetro": 25598, "Ġmansion": 25599, "Ġаб": 25600, "ĠZhou": 25601, "Ġcorridor": 25602, "Ġescol": 25603, "Ġindicating": 25604, "iaÅĤa": 25605, "Ġmommy": 25606, "Ġarchives": 25607, "Ġfounders": 25608, "engine": 25609, "ĠDieu": 25610, "Ġsickness": 25611, "Ġë³´ëĭĪê¹Į": 25612, "Ġarb": 25613, "Ġned": 25614, "ĠChop": 25615, "Ġcovid": 25616, "Ġslam": 25617, "Ġpublications": 25618, "DC": 25619, "Ġspends": 25620, "æ¾": 25621, "Ġrefugee": 25622, "Ġdile": 25623, "Ġ×IJ×ĸ": 25624, "ificar": 25625, "ĠSach": 25626, "Gu": 25627, "Ġreload": 25628, "????": 25629, "ĠjeÅĽli": 25630, "ĠÑģоÑģÑĤо": 25631, "Ġsimplicity": 25632, "Ġbullying": 25633, "Ġмол": 25634, "Ġrealidad": 25635, "Ġunclear": 25636, "appa": 25637, "levant": 25638, "ĠISIS": 25639, "ĠWatson": 25640, "Ġdein": 25641, "ĠMicro": 25642, "íķľë": 25643, "üg": 25644, "Ġdevam": 25645, "Ġtweeted": 25646, "å°İ": 25647, "Ġunderstandable": 25648, "atan": 25649, "Ġversa": 25650, "Ġpreca": 25651, "Ġvá»ģ": 25652, "ĠCopy": 25653, "ĠOracle": 25654, "Ġmindfulness": 25655, "Ġdiscret": 25656, "ernen": 25657, "ĠPle": 25658, "Have": 25659, "Ġisolate": 25660, "Ġdeu": 25661, "Ġseventy": 25662, "ĠHills": 25663, "Ġarcade": 25664, "ĠÑģпеÑĨи": 25665, "Ġsiguiente": 25666, "ĠBÃľNDNIS": 25667, "liga": 25668, "ĠвÑģÑĤÑĢеÑĩ": 25669, "ôm": 25670, "Ġtweets": 25671, "Ġschauen": 25672, "Ġcritique": 25673, "ĠðŁİµ": 25674, "Ġstatt": 25675, "ĠÑģамое": 25676, "ância": 25677, "Ġsupernatural": 25678, "Ġplugged": 25679, "Fl": 25680, "ynı": 25681, "ĠTambién": 25682, "Ġencouragement": 25683, "ĠServer": 25684, "ëĤľ": 25685, "upa": 25686, "Ġaston": 25687, "Ġhears": 25688, "ÑĢаÑħ": 25689, "Ġsche": 25690, "Ġrats": 25691, "Ġrecuper": 25692, "Ġunten": 25693, "ĠFighting": 25694, "Ġacademics": 25695, "示": 25696, "ĠSü": 25697, "ÑģкиÑħ": 25698, "Ġpaired": 25699, "ĢìĿĦ": 25700, "Ġárea": 25701, "Ġsweetness": 25702, "åıĬ": 25703, "Ġdefer": 25704, "Ġmuitas": 25705, "ĠAudio": 25706, "Ġlocker": 25707, "ÙĬد": 25708, "ĠÑģÑĤав": 25709, "Ġbuena": 25710, "ANS": 25711, "Ġdetector": 25712, "avo": 25713, "bek": 25714, "Ġαν": 25715, "íݸ": 25716, "Ġdragged": 25717, "Ġдолжен": 25718, "Ãĸ": 25719, "رة": 25720, "ìĿ´ì§Ģ": 25721, "Ġcelle": 25722, "cking": 25723, "ĠاÙĦج": 25724, "ĠCanvas": 25725, "Ġespañ": 25726, "Ġglimp": 25727, "Ġspreads": 25728, "ongo": 25729, "ĠMason": 25730, "ĠIng": 25731, "Ġê°ĢëĬ¥": 25732, "ÏĦικ": 25733, "Ġsecular": 25734, "Ġbater": 25735, "Ġinquiry": 25736, "Ġenergies": 25737, "Ġmanufactured": 25738, "Ġvegetarian": 25739, "Ġpineapple": 25740, "ÑıÑĤа": 25741, "Ġpractitioners": 25742, "2000": 25743, "Ġíķ´ìļĶ": 25744, "ĠìŬ룬ë¶Ħëĵ¤": 25745, "Ġë¶Īë": 25746, "ĠJefferson": 25747, "ĠJoan": 25748, "Ġtram": 25749, "容": 25750, "chmal": 25751, "ĠHait": 25752, "á¹ĩ": 25753, "Ġunreal": 25754, "Ġsymbolic": 25755, "Ġstealth": 25756, "Ġsplash": 25757, "ĠEntertainment": 25758, "Ġmetallic": 25759, "?\".": 25760, "è¶Ĭ": 25761, "around": 25762, "Ġdespair": 25763, "ĠNevada": 25764, "ĠFinance": 25765, "Ġkrie": 25766, "ĠLux": 25767, "ĠSmash": 25768, "keeping": 25769, "Ġзаг": 25770, "Ġnarciss": 25771, "Ġdzisiaj": 25772, "Ġtolerate": 25773, "oard": 25774, "Ġlinking": 25775, "ĠEconomic": 25776, "Ġì¼": 25777, "Ġmorph": 25778, "ĠNak": 25779, "ĠBaker": 25780, "aton": 25781, "rings": 25782, "ĠPeng": 25783, "ĠAirport": 25784, "ãģĭãģ£ãģŁ": 25785, "íķĺëĭ¤": 25786, "§ģ": 25787, "prints": 25788, "Ġhadi": 25789, "Ġempir": 25790, "ĠLives": 25791, "anners": 25792, "Ġним": 25793, "ĠPROFESSOR": 25794, "Ġpositively": 25795, "antom": 25796, "Ġbadge": 25797, "kelt": 25798, "Ġinterfer": 25799, "Ġfulfilling": 25800, "Ġvisualization": 25801, "éĹľä¿Ĥ": 25802, "ĠPrice": 25803, "��": 25804, "Ġscenery": 25805, "Ġprone": 25806, "Ġwizard": 25807, "Ġbanyak": 25808, "verb": 25809, "sky": 25810, "Ġwished": 25811, "Ġrailway": 25812, "Ġüzer": 25813, "Ġalguien": 25814, "ĠAW": 25815, "ĠколиÑĩе": 25816, "Ġreacting": 25817, "ĠBuch": 25818, "ึ": 25819, "Ġanth": 25820, "Ġsih": 25821, "Ġhust": 25822, "ĠScreen": 25823, "ilant": 25824, "aho": 25825, "Ġfragrance": 25826, "Ġelevation": 25827, "ĠMediter": 25828, "Ġë¿": 25829, "Ġéqu": 25830, "Ġwraps": 25831, "Ġinert": 25832, "Ġrecreate": 25833, "лаÑĤ": 25834, "Ġboleh": 25835, "Ġharassment": 25836, "unky": 25837, "Ġglimpse": 25838, "regierung": 25839, "Ġfutur": 25840, "Ġrepository": 25841, "Ġengra": 25842, "Ġtrafficking": 25843, "assis": 25844, "ĠTrek": 25845, "Ġë²Į": 25846, "Ġë§Īë": 25847, "ĠKab": 25848, "aniu": 25849, "give": 25850, "Ġdinosaurs": 25851, "Ġfeather": 25852, "Ġattitudes": 25853, "Ġplum": 25854, "ĠRS": 25855, "ĠAnfang": 25856, "illery": 25857, "ĠìĬ¤": 25858, "MY": 25859, "Ġtrzeba": 25860, "Ġskies": 25861, "ĠAj": 25862, "urable": 25863, "CU": 25864, "ĠShane": 25865, "Ġdeparture": 25866, "ĠTON": 25867, "ieten": 25868, "rats": 25869, "æ°Ĺ": 25870, "isu": 25871, "Ġbord": 25872, "Ġinterestingly": 25873, "çĻ»": 25874, "oughing": 25875, "Ġrushing": 25876, "Ġvolatility": 25877, "Ġpyt": 25878, "Ġformats": 25879, "ĠзаÑĤ": 25880, "Ġê¼Ń": 25881, "Ġwhatnot": 25882, "Ġcomport": 25883, "sw": 25884, "orean": 25885, "ĠRelax": 25886, "Ġclan": 25887, "ĠAH": 25888, "Ġpew": 25889, "Ġdictionary": 25890, "Take": 25891, "shirts": 25892, "ĠHugh": 25893, "ĠعÙĦÙĬ": 25894, "ĠPic": 25895, "Ġenrolled": 25896, "Ġjednak": 25897, "Ġofferings": 25898, "Ġcoraz": 25899, "Life": 25900, "Ġ!!!": 25901, "Ġcler": 25902, "ĠVideos": 25903, "ĠRodrig": 25904, "ĠIdent": 25905, "ĠPos": 25906, "ĠStage": 25907, "ĠRace": 25908, "Ġenact": 25909, "ãģĦãģ¾ãģĹãģŁ": 25910, "ĠGy": 25911, "ĠHispan": 25912, "Ġdefence": 25913, "ĠCampbell": 25914, "matic": 25915, "Ġrelev": 25916, "Ġpeach": 25917, "Ħ¸ìļĶ": 25918, "Ġparadise": 25919, "Ġceremon": 25920, "Ġannoyed": 25921, "æĮĩ": 25922, "lax": 25923, "Ġexploit": 25924, "Ġclause": 25925, "eker": 25926, "ĠBloom": 25927, "nant": 25928, "ateurs": 25929, "Ġheights": 25930, "Even": 25931, "Ñģон": 25932, "Ġoutrage": 25933, "ĠVietnamese": 25934, "ãģ¯ãģ¯": 25935, "TR": 25936, "Ġeer": 25937, "Ġcannon": 25938, "ĠComb": 25939, "IJë§Į": 25940, "è»Ĭ": 25941, "Ġê²ĥëıĦ": 25942, "Ġaccomplishments": 25943, "ĠAnalytics": 25944, "Ġshaping": 25945, "reiben": 25946, "Ġbachelor": 25947, "Ġfingert": 25948, "acked": 25949, "Ġpyramid": 25950, "ĠStewart": 25951, "ást": 25952, "Ġsurvivor": 25953, "Ġduct": 25954, "Ġdealers": 25955, "æ´»": 25956, "عÙħ": 25957, "лин": 25958, "Ġede": 25959, "×ķ×¢": 25960, "ĠÙĥاÙĨ": 25961, "ĠÏĦι": 25962, "Ġchooses": 25963, "ĠOwn": 25964, "гоÑĤов": 25965, "hire": 25966, "алÑĮнÑĭе": 25967, "ĠÐĽÑİ": 25968, "ĠоÑģÑĤав": 25969, "tech": 25970, "Ġdroit": 25971, "Ġsubjective": 25972, "enes": 25973, "Ġdivis": 25974, "avez": 25975, "Ġmaneuver": 25976, "à¹Ħà¸Ķ": 25977, "adece": 25978, "ĠEns": 25979, "acial": 25980, "ĠProtection": 25981, "ĸ´": 25982, "Ġformally": 25983, "Ġwyd": 25984, "inguém": 25985, "Ġziem": 25986, "Ġrecruiting": 25987, "×Ļ×ļ": 25988, "nem": 25989, "Ġforbidden": 25990, "ĠBapt": 25991, "×IJ׳×Ļ": 25992, "Ġsubset": 25993, "ĠMagaz": 25994, "nement": 25995, "Ġaquela": 25996, "ragon": 25997, "Ġcommittees": 25998, "Ġétaient": 25999, "udi": 26000, "ĠDawn": 26001, "Ġbore": 26002, "Ġcomposer": 26003, "ĠwiÄĻcej": 26004, "anga": 26005, "Ġdislike": 26006, "ĠDays": 26007, "åŁº": 26008, "Ġparal": 26009, "Ġmientras": 26010, "Ġheavens": 26011, "ãģĴ": 26012, "heid": 26013, "Ġtraders": 26014, "once": 26015, "Ġmascara": 26016, "ĠÏĢÏģο": 26017, "Ġwhisper": 26018, "ĠMusk": 26019, "éĽĨ": 26020, "ĠFamilie": 26021, "Allah": 26022, "ĠOlivia": 26023, "ĠPros": 26024, "Ġolika": 26025, "ilim": 26026, "Ġrépond": 26027, "ĠPeters": 26028, "Ġå¾Ī": 26029, "Ġbites": 26030, "Ġvic": 26031, "ĠNY": 26032, "emption": 26033, "Ġ450": 26034, "Ġvisuals": 26035, "Ġlieu": 26036, "ücken": 26037, "ĠSteel": 26038, "ĠGP": 26039, "wait": 26040, "Ġnoticeable": 26041, "ucha": 26042, "Ġrehabil": 26043, "Ġrejection": 26044, "ĠÑģледÑĥÑİÑī": 26045, "Ġslider": 26046, "Ġregarded": 26047, "Ġgravit": 26048, "ĠReserve": 26049, "count": 26050, "Ġbreeding": 26051, "Ġlonge": 26052, "aleb": 26053, "Ġknight": 26054, "Ġвой": 26055, "Ġprésent": 26056, "ĤĺìļĶ": 26057, "ĠSpecifically": 26058, "Ġposes": 26059, "Ġveure": 26060, "okay": 26061, "emas": 26062, "Ġãģ§ãģĻ": 26063, "ĠmajÄħ": 26064, "Ġwebinars": 26065, "Ġcannabis": 26066, "Ġdamals": 26067, "ĠNorthwest": 26068, "Ġpada": 26069, "Ġcrowds": 26070, "Ġfutures": 26071, "Ġän": 26072, "Ġcivilians": 26073, "ĠSachen": 26074, "æį": 26075, "Ġtraces": 26076, "Ġë¨¹ê³ł": 26077, "QU": 26078, "é¡ĺãģĦ": 26079, "ĠIF": 26080, "anın": 26081, "ìĤ´": 26082, "Ġbiblical": 26083, "ĠVed": 26084, "Ġstoring": 26085, "ÑĢавлÑı": 26086, "æĩī該": 26087, "Ġnast": 26088, "Ġdö": 26089, "ÑĢоп": 26090, "elia": 26091, "Ġsideways": 26092, "ĠUnderstand": 26093, "ĠQur": 26094, "Ġperpend": 26095, "ĠMillionen": 26096, "Ġwatermelon": 26097, "ĠDivine": 26098, "ultur": 26099, "abord": 26100, "Ġsuccesses": 26101, "Ġhombre": 26102, "Ġcarp": 26103, "Ġsuscept": 26104, "ungkin": 26105, "Ġkij": 26106, "ulus": 26107, "اج": 26108, "Ġnotch": 26109, "Ġpolynomial": 26110, "å¹²": 26111, "å©": 26112, "Ġúnico": 26113, "Ġtelescope": 26114, "Ġpolitique": 26115, "kiem": 26116, "ĠÎŃνα": 26117, "Ġaggregate": 26118, "ĠGeoff": 26119, "Ġtril": 26120, "ĠGRA": 26121, "Ġsubscriber": 26122, "imet": 26123, "ĠдоллаÑĢ": 26124, "oping": 26125, "Ġtherapeut": 26126, "ĠCancer": 26127, "Ġparade": 26128, "Ġirrig": 26129, "âĻªâĻª": 26130, "Ġclearer": 26131, "Ġbog": 26132, "ĠMaur": 26133, "าà¸ĩ": 26134, "ĠShanghai": 26135, "achte": 26136, "ĠKol": 26137, "elujah": 26138, "Ġhav": 26139, "ĠCrime": 26140, "sek": 26141, "Ġë¡ľ": 26142, "ienna": 26143, "ĠGor": 26144, "èĽ": 26145, "ĠпоÑĤÑĢ": 26146, "ĠкажеÑĤÑģÑı": 26147, "ĠLift": 26148, "ĠSort": 26149, "ĠPsal": 26150, "Ġping": 26151, "ĵĿ": 26152, "phis": 26153, "ĠFUCK": 26154, "ĠSyn": 26155, "Ġbamboo": 26156, "¬ìĺģ": 26157, "cuts": 26158, "Ġmmm": 26159, "Ġfunktioniert": 26160, "Ġ_": 26161, "ÃŃcio": 26162, "Stop": 26163, "Ġimaginary": 26164, "Ġnotamment": 26165, "ĠInitiative": 26166, "ãĥ¥": 26167, "ĠKurt": 26168, "Ġloosen": 26169, "Ġbuscar": 26170, "çģ«": 26171, "Ġzelf": 26172, "Ġprops": 26173, "åĽī": 26174, "Ġmoeten": 26175, "Ġmilli": 26176, "Ġhalls": 26177, "ĠMatch": 26178, "Ġbrackets": 26179, "ĠCou": 26180, "æ¦Ĥ": 26181, "ĠÐľÐ°ÑĢ": 26182, "ISA": 26183, "Ġcigarette": 26184, "Ġcompetitions": 26185, "ĠMIN": 26186, "Ġbehö": 26187, "voor": 26188, "Ġust": 26189, "ĠZi": 26190, "ĠOcc": 26191, "ulates": 26192, "Ġballoons": 26193, "Ġpronto": 26194, "ĠMiy": 26195, "ĠFile": 26196, "ĠклаÑģÑģ": 26197, "нÑĥл": 26198, "Ġcereal": 26199, "Ġincrement": 26200, "Ġrefined": 26201, "åı¦å¤ĸ": 26202, "prising": 26203, "ĠRF": 26204, "Ġrespectful": 26205, "Ġloot": 26206, "asket": 26207, "Ġdeixa": 26208, "ingle": 26209, "Ġfunciona": 26210, "ĠRevel": 26211, "Ġsober": 26212, "Ġperforms": 26213, "ĠGentle": 26214, "ãĤ¨": 26215, "Ġrecipient": 26216, "ĠHause": 26217, "Ġëĥ": 26218, "From": 26219, "Ġministers": 26220, "Ġparadox": 26221, "å°±æĺ¯èªª": 26222, "Ġtasting": 26223, "Ġ×Ķ×Ĺ": 26224, "Ġreuse": 26225, "ĠLane": 26226, "ĠÑģовеÑĢÑĪ": 26227, "Ġremembers": 26228, "Ġfeminist": 26229, "Ġcommitments": 26230, "Ġprojected": 26231, "Ġgaz": 26232, "iyoruz": 26233, "Ġobligations": 26234, "Ro": 26235, "zar": 26236, "Ġchw": 26237, "ĠJAM": 26238, "ĠbÄĻdÄħ": 26239, "aspberry": 26240, "ĠмеÑģÑĤо": 26241, "ë²ķ": 26242, "Ġregulated": 26243, "Ġwicht": 26244, "ĠTrevor": 26245, "Ġsecondly": 26246, "ĠIhre": 26247, "elsh": 26248, "Ġreporters": 26249, "ÑĤоÑĢа": 26250, "oyo": 26251, "GI": 26252, "Ġinterconnect": 26253, "éIJĺ": 26254, "OSH": 26255, "æŃ²": 26256, "Ġbrass": 26257, "Ġignoring": 26258, "ä»ĬæĹ¥": 26259, "infect": 26260, "Ġprojekt": 26261, "oret": 26262, "ÏĦαν": 26263, "ĠÑĤип": 26264, "Ġmutta": 26265, "Ġunboxing": 26266, "Ħ°": 26267, "å¡Ĭ": 26268, "Ġadvised": 26269, "ĠDenver": 26270, "Ġseverely": 26271, "ĠMhm": 26272, "Ġflipped": 26273, "Ġpien": 26274, "Ġkommun": 26275, "ĠFRE": 26276, "Ġà®ĩà®°": 26277, "ainted": 26278, "Ġknives": 26279, "Ġhabl": 26280, "Ġgeworden": 26281, "arettes": 26282, "CS": 26283, "ĠмаленÑĮ": 26284, "Ġgalax": 26285, "Ġninete": 26286, "ê±°ëĤĺ": 26287, "Ġsis": 26288, "Ġadvisory": 26289, "Ġdrilling": 26290, "ĠWouldn": 26291, "ünf": 26292, "gestellt": 26293, "ĠHelen": 26294, "Ġ×ŀ×IJ": 26295, "apolis": 26296, "Ġrzeczy": 26297, "Ġterra": 26298, "Ġhep": 26299, "Ġalgún": 26300, "ikk": 26301, "Ġastronom": 26302, "ĠStarbucks": 26303, "kÄħ": 26304, "Ġpatrol": 26305, "Ġì½Ķ": 26306, "Ġgon": 26307, "ĠãĢIJ": 26308, "Ġsonst": 26309, "Ġencounters": 26310, "Ġretrou": 26311, "Ġsharks": 26312, "Ġdor": 26313, "ĠRever": 26314, "Ġevapor": 26315, "Ġreservoir": 26316, "Ġalleged": 26317, "uler": 26318, "Ġverm": 26319, "Ġcommerce": 26320, "Ġfitted": 26321, "gem": 26322, "Ġtactical": 26323, "Ġlith": 26324, "éīĦå¡Ķ": 26325, "had": 26326, "è®Ĭ": 26327, "Ġcarbohyd": 26328, "Ġlengths": 26329, "ιο": 26330, "Ġdemographic": 26331, "Rob": 26332, "ĠSkin": 26333, "ccoli": 26334, "Ġsimplified": 26335, "Ġreadily": 26336, "ĠCum": 26337, "adesh": 26338, "ĠDÃ¥": 26339, "usst": 26340, "igne": 26341, "eton": 26342, "Ġmenor": 26343, "qi": 26344, "OOM": 26345, "à¸Ńà¸Ļ": 26346, "Ġpsychiat": 26347, "Ġeighty": 26348, "Ġмилли": 26349, "ĠTob": 26350, "edo": 26351, "網": 26352, "ĠÄijến": 26353, "Ġcircuits": 26354, "ĠLAUGH": 26355, "icism": 26356, "emor": 26357, "Ġregener": 26358, "egree": 26359, "Ġbureauc": 26360, "ĠAlber": 26361, "ä¹ĭå¾Į": 26362, "ĠWor": 26363, "夫": 26364, "Ġresin": 26365, "ĠbyÅĤy": 26366, "ĠIG": 26367, "à¯į,": 26368, "Ġ78": 26369, "Ġweeds": 26370, "ĠMyth": 26371, "93": 26372, "æ¿": 26373, "ĠëĤĺìĻĶ": 26374, "év": 26375, "á½": 26376, "ören": 26377, "çar": 26378, "ĠPAUL": 26379, "Ġdisadvant": 26380, "Ġpositioning": 26381, "Ġcocktail": 26382, "Ġagrees": 26383, "nn": 26384, "ĠSally": 26385, "Ms": 26386, "Ġinherent": 26387, "Ġmonetary": 26388, "Ġnatur": 26389, "ĠNh": 26390, "ĠImport": 26391, "Ġleben": 26392, "Ġwi": 26393, "ussy": 26394, "Ġobes": 26395, "Ġwandering": 26396, "Ġìĭłë": 26397, "Äħda": 26398, "etchup": 26399, "Ġdisposal": 26400, "ĠJA": 26401, "ĠCer": 26402, "zilla": 26403, "Ġvirgin": 26404, "ĠSlide": 26405, "andel": 26406, "Ġrighteousness": 26407, "ĠΣ": 26408, "Ġideia": 26409, "ä½łå¥½": 26410, "иÑĢоваÑĤÑĮ": 26411, "ר×IJ": 26412, "Comment": 26413, "Ġprelim": 26414, "ĠVale": 26415, "Ġì§ĢëĤľ": 26416, "ĠVanc": 26417, "OMAN": 26418, "ĠпÑĸд": 26419, "Ġyum": 26420, "stre": 26421, "cem": 26422, "Ġpocz": 26423, "Ġfragment": 26424, "ĠÑģлÑĥÑĩае": 26425, "Ġundergo": 26426, "ĠHank": 26427, "ceks": 26428, "ĠFPS": 26429, "Ġocur": 26430, "Ġdeterior": 26431, "注": 26432, "Ġempresas": 26433, "Paul": 26434, "Ġ)))": 26435, "ĠвÑĢемени": 26436, "Ġscold": 26437, "×Ļ×¢": 26438, "Ġsuspected": 26439, "Ġaccessing": 26440, "Ġsubstit": 26441, "Ġhistorians": 26442, "ä»»": 26443, "Ġдело": 26444, "Ġsocied": 26445, "rone": 26446, "Ġreden": 26447, "Ġextends": 26448, "epherd": 26449, "Ġbalcon": 26450, "ä¸įèµ·": 26451, "ĠSolo": 26452, "Ġpolitician": 26453, "олÑĮно": 26454, "Ġirgendw": 26455, "Ġtraumatic": 26456, "Ġrapper": 26457, "ĠROBERT": 26458, "Really": 26459, "æģ¯": 26460, "Ġlineup": 26461, "ASE": 26462, "Ġcontractor": 26463, "ĠCorporation": 26464, "gor": 26465, "ĠTodo": 26466, "ÑģÑĤÑĢой": 26467, "FBE": 26468, "Ġnewsletter": 26469, "ĠkoÅĦ": 26470, "alties": 26471, "ĠпÑĢиÑĩ": 26472, "ĠHeavy": 26473, "Ġswords": 26474, "Ġmanipulation": 26475, "Ġfunk": 26476, "ĠvÃ¥r": 26477, "ĠTaliban": 26478, "Ġë°¥": 26479, "Ġacne": 26480, "ürü": 26481, "Ġdeswegen": 26482, "ĠDust": 26483, "Ġsilic": 26484, "Ġhooks": 26485, "Ġblij": 26486, "Ġpetits": 26487, "Ġfilme": 26488, "ĠBereich": 26489, "ĠSaid": 26490, "Ġimposed": 26491, "Ġdiary": 26492, "ĠгоÑĢ": 26493, "ĠGates": 26494, "Ġalta": 26495, "å¸Į": 26496, "Ġchcia": 26497, "pleasant": 26498, "Ġë°Ŀ": 26499, "Ġmożemy": 26500, "ĠAustria": 26501, "Ġbroker": 26502, "Ġsucked": 26503, "èĢĥ": 26504, "Ġcompartment": 26505, "Ġclone": 26506, "Ġ×Ķ×¢": 26507, "ĠDanke": 26508, "Ġnochmal": 26509, "езд": 26510, "Ġadrenal": 26511, "Ġkleinen": 26512, "ãģ¾ãģĹãĤĩãģĨ": 26513, "Ġsubsequently": 26514, "Ġdecentral": 26515, "Ġgenetics": 26516, "Ġê´ij": 26517, "Ġmonitors": 26518, "ĠApplic": 26519, "ĠReporter": 26520, "wert": 26521, "Ġwiem": 26522, "ĠMovement": 26523, "Ġinterviewing": 26524, "Ġhairs": 26525, "Ġpuò": 26526, "ĠChelsea": 26527, "Ġcoher": 26528, "Ġcot": 26529, "Ġzas": 26530, "Ġpatches": 26531, "Ġlah": 26532, "Ñĥнк": 26533, "ĠReagan": 26534, "ĠMarco": 26535, "city": 26536, "Ġdefender": 26537, "Ġdecoration": 26538, "iji": 26539, "Ġlitter": 26540, "Ш": 26541, "Ġjego": 26542, "REW": 26543, "ĠPik": 26544, "ĠHee": 26545, "ĠIv": 26546, "Ġиде": 26547, "ĠTheater": 26548, "ĠÑĩаÑģÑĤо": 26549, "Ġsweater": 26550, "Ġhighlighting": 26551, "Ġainsi": 26552, "Ġdiplomatic": 26553, "ĠNevertheless": 26554, "å³": 26555, "ASON": 26556, "Ġpúblico": 26557, "Ġferm": 26558, "reated": 26559, "cod": 26560, "Ġ물ë": 26561, "Ġmister": 26562, "ĠVancouver": 26563, "Ġrecognizes": 26564, "ecd": 26565, "Ġcomplications": 26566, "encial": 26567, "ãģĹãģı": 26568, "Ġê°Ģì§Ģ": 26569, "ĠUltimate": 26570, "Ġvaig": 26571, "ĠMerry": 26572, "×ķ×Ĵ": 26573, "ĠMarcus": 26574, "總": 26575, "owego": 26576, "Ġmente": 26577, "Sm": 26578, "Ġaja": 26579, "ĠTao": 26580, "Ġjudicial": 26581, "Ġentrepreneurship": 26582, "Ġнемного": 26583, "Ġpis": 26584, "Ġerg": 26585, "Ġchrist": 26586, "ĠCurt": 26587, "ĠÑĢаÑģп": 26588, "λε": 26589, "ensch": 26590, "ÃŃre": 26591, "Ġfocal": 26592, "ĠDiamond": 26593, "avÃŃa": 26594, "Ġhanno": 26595, "ĠSquad": 26596, "Ġassociations": 26597, "ĠCreative": 26598, "Ġmessenger": 26599, "Ġbegging": 26600, "Ġdecimal": 26601, "ĠdÄ±ÅŁ": 26602, "Ġmetadata": 26603, "sels": 26604, "ĠÄ°ÅŁ": 26605, "ữa": 26606, "Ġdifficile": 26607, "dı": 26608, "Ġslaughter": 26609, "ĠVerg": 26610, "Ġ×Ĵ×Ŀ": 26611, "ç°¡": 26612, "æĮī": 26613, "ĠTea": 26614, "asses": 26615, "Ok": 26616, "Ġsynthes": 26617, "otiation": 26618, "Ġpainter": 26619, "Ġelbows": 26620, "Ġarchitectural": 26621, "ĠÑĢад": 26622, "Ġglor": 26623, "image": 26624, "ampa": 26625, "culiar": 26626, "ł¨": 26627, "Ġteve": 26628, "ĠStelle": 26629, "ĠBam": 26630, "Ġì´Ī": 26631, "asis": 26632, "ipedia": 26633, "ĠGI": 26634, "ĠActive": 26635, "çĦ¶åIJİ": 26636, "azi": 26637, "ãĤĮãģ¦": 26638, "ĠLucky": 26639, "íķ©": 26640, "ĠпÑĢиÑħод": 26641, "Ġrunway": 26642, "Ġauthentication": 26643, "Ġposible": 26644, "Ġsupplements": 26645, "Ġsurgical": 26646, "Gen": 26647, "Ġfeasible": 26648, "DO": 26649, "Ġoutlook": 26650, "Ġintervals": 26651, "Ġanecd": 26652, "Ãłng": 26653, "Ġstraps": 26654, "ĠShu": 26655, "udd": 26656, "issenschaft": 26657, "Ġporte": 26658, "Ġcommitting": 26659, "Ġalley": 26660, "Ġcovenant": 26661, "ĠPedro": 26662, "lessness": 26663, "ĠSolid": 26664, "ĠMolly": 26665, "ĠнекоÑĤоÑĢ": 26666, "Ġcooperate": 26667, "åĮĹ": 26668, "ollen": 26669, "Ġtuna": 26670, "Ġkindergarten": 26671, "ĠSiz": 26672, "Ġdużo": 26673, "ĠMBA": 26674, "ĠGEORGE": 26675, "ĠFisher": 26676, "å¿ĺ": 26677, "ĠCaesar": 26678, "ĠкÑĢаÑģив": 26679, "ĠDelhi": 26680, "zym": 26681, "Ġexplicar": 26682, "ê°Ģì§Ģ": 26683, "uns": 26684, "grow": 26685, "ĠпÑĢиÑģ": 26686, "Ġ86": 26687, "Ġstating": 26688, "Ġmassa": 26689, "chter": 26690, "Ġì»¬ëŁ¬": 26691, "Ġdeputy": 26692, "SM": 26693, "noc": 26694, "Ġgeography": 26695, "ĠEnterprise": 26696, "ĠCant": 26697, "öz": 26698, "Ġunpack": 26699, "ĠíĻĶë": 26700, "Ġsearches": 26701, "Ġpresidency": 26702, "Ġtrivial": 26703, "Ġpige": 26704, "oubt": 26705, "ãĤļ": 26706, "ì¼ĢìĿ´": 26707, "Ġbudgets": 26708, "Ġub": 26709, "Ġpne": 26710, "ĠYale": 26711, "ĠÅŁÃ¶yle": 26712, "regular": 26713, "Ġimperfect": 26714, "ARA": 26715, "ĠfamÃŃlia": 26716, "urm": 26717, "ĠAdventure": 26718, "ãĥĬ": 26719, "cis": 26720, "emark": 26721, "Ġnego": 26722, "Ġinappropriate": 26723, "ĠпÑĢиз": 26724, "ĠÑĢол": 26725, "Ġdreamed": 26726, "Bry": 26727, "Ġshuttle": 26728, "Ġpillars": 26729, "Ġbik": 26730, "inum": 26731, "ĠÑĥÑģ": 26732, "ĠNebr": 26733, "Ġperpendicular": 26734, "Ġbooked": 26735, "bery": 26736, "Ġvikt": 26737, "bear": 26738, "esus": 26739, "Ġвозможно": 26740, "¨¹": 26741, "Ġpresumably": 26742, "ĠMemphis": 26743, "Ġambulance": 26744, "×ķ×ŀר": 26745, "Ġthumbnail": 26746, "Ġmodification": 26747, "éĩı": 26748, "Ġinterpreted": 26749, "Ġpromo": 26750, "Ġκά": 26751, "ĠεÏĢ": 26752, "Ġacoustic": 26753, "ĠDB": 26754, "åĵİ": 26755, "Ġnonetheless": 26756, "oule": 26757, "Ġpequ": 26758, "Ġknob": 26759, "ãĤ£": 26760, "ĠëıĮìķĦ": 26761, "Ġpurchases": 26762, "ĠÃĩünkü": 26763, "Ġdividing": 26764, "perform": 26765, "raction": 26766, "healthy": 26767, "ĠTitle": 26768, "Ġuk": 26769, "Ġcerca": 26770, "Ġarguably": 26771, "Ġfale": 26772, "ë³µ": 26773, "Ġgamers": 26774, "Ġutilizing": 26775, "Ġoffended": 26776, "Ġtava": 26777, "alı": 26778, "Ġmedian": 26779, "Ġinfectious": 26780, "ĠAnnie": 26781, "Ġsmartphones": 26782, "Ġparole": 26783, "åĸĿ": 26784, "ĠEpic": 26785, "zza": 26786, "Ġunified": 26787, "Ġê·¸ëķĮ": 26788, "Ġcurtain": 26789, "ĠÄĥ": 26790, "Ġsexually": 26791, "Ġunserem": 26792, "ĠConvention": 26793, "Ġallegedly": 26794, "Ya": 26795, "ĠHoo": 26796, "enment": 26797, "æĢª": 26798, "íĽĦ": 26799, "Ġgigantic": 26800, "Ġnoting": 26801, "Ġrebo": 26802, "ĠJama": 26803, "ĠAlz": 26804, "Ġborrowed": 26805, "침": 26806, "Ġperipher": 26807, "оÑĤа": 26808, "ĠGB": 26809, "ĠGear": 26810, "Ġeconomically": 26811, "Ġtelefon": 26812, "Ġqueremos": 26813, "ĠдалÑĮÑĪе": 26814, "Ġras": 26815, "ĠTeach": 26816, "icios": 26817, "atos": 26818, "Ġpledge": 26819, "bau": 26820, "ĠHimself": 26821, "Link": 26822, "Ġespero": 26823, "Ġchromos": 26824, "ĠPER": 26825, "Ġerle": 26826, "Ġpodium": 26827, "ços": 26828, "Ġnieu": 26829, "Ġfen": 26830, "ĠGOD": 26831, "ĠChocolate": 26832, "werk": 26833, "Ġtừ": 26834, "Ġsuppress": 26835, "λη": 26836, "Ġ240": 26837, "Ġsitä": 26838, "Ġhonesty": 26839, "ĠBio": 26840, "ĠBard": 26841, "ĠобÑīем": 26842, "ĠмÑĥз": 26843, "Ġmarble": 26844, "ĠÑĨенÑĤ": 26845, "Ġprocure": 26846, "Ġrotor": 26847, "bern": 26848, "Ġtuh": 26849, "Ġheadset": 26850, "atem": 26851, "Ġwarranty": 26852, "à®´": 26853, "Ġfiling": 26854, "ιά": 26855, "Ġcomprendre": 26856, "Ġimpulse": 26857, "Ġsalv": 26858, "written": 26859, "Ġinstitute": 26860, "Kim": 26861, "ĠLGBTQ": 26862, "ficiente": 26863, "His": 26864, "ĠαÏħÏĦÏĮ": 26865, "Ġteenage": 26866, "orus": 26867, "ĠÑĢазб": 26868, "See": 26869, "ĠConserv": 26870, "á»ģn": 26871, "fulness": 26872, "Ġstrawberries": 26873, "ĠAbu": 26874, "ион": 26875, "Ġolla": 26876, "NOISE": 26877, "ĠEmploy": 26878, "Ġwiped": 26879, "urger": 26880, "Ġmodifications": 26881, "Ġíķĺì§Ģ": 26882, "Ġfootsteps": 26883, "Ġhonors": 26884, "Ġadul": 26885, "Ġflipping": 26886, "ĠHU": 26887, "ZY": 26888, "Ġintegrating": 26889, "بر": 26890, "ulla": 26891, "Ġnatuurlijk": 26892, "ĠíĹĪ": 26893, "ĠEthereum": 26894, "ÙĬÙĦ": 26895, "wed": 26896, "Ġpeaks": 26897, "ĠKes": 26898, "Ġbloom": 26899, "Ġcrashing": 26900, "Ġ911": 26901, "ĠоÑĤлиÑĩ": 26902, "Ġcontrollers": 26903, "ĠDod": 26904, "ĠвмеÑģÑĤе": 26905, "Ġsortir": 26906, "å¥ĩ": 26907, "ĠStraight": 26908, "ĠGracias": 26909, "Ġgroove": 26910, "Ġtogg": 26911, "Ġìĭ¶ìĿĢ": 26912, "éro": 26913, "Ġoutward": 26914, "ĠWA": 26915, "ĠRocky": 26916, "Ġscam": 26917, "Ġhayat": 26918, "ignty": 26919, "âĦ": 26920, "plings": 26921, "Ġantibiotics": 26922, "Ġä¸Ģ": 26923, "Ġnevertheless": 26924, "jang": 26925, "commerce": 26926, "Ġspoiler": 26927, "Ġglove": 26928, "Ġchatter": 26929, "ĠBY": 26930, "~?": 26931, "Ġíĺ¸": 26932, "Ġdemol": 26933, "wechsel": 26934, "imir": 26935, "Ġraid": 26936, "еÑĢÑħ": 26937, "ìŀIJ기": 26938, "enf": 26939, "Ġcommented": 26940, "Ġoptimized": 26941, "Ġconvicted": 26942, "Ġbats": 26943, "ĠSB": 26944, "ĠAur": 26945, "ĠTong": 26946, "Ġimplicit": 26947, "ĠJanet": 26948, "Ġreag": 26949, "ãģ²": 26950, "ĠAdvanced": 26951, "Ġimpose": 26952, "ש×Ķ": 26953, "Ġschemes": 26954, "ougher": 26955, "abolic": 26956, "Ġê±°ì£ł": 26957, "Ġslowing": 26958, "Ġwtedy": 26959, "Ġdestructive": 26960, "ĠопÑĢед": 26961, "Ġlandmark": 26962, "ĠëıĪ": 26963, "ĠWalking": 26964, "ẹ": 26965, "Ġtijd": 26966, "ĠKN": 26967, "ĠQuant": 26968, "ìĺ¤ë": 26969, "ĠкÑĢÑĥ": 26970, "Ġperder": 26971, "Ġnove": 26972, "ände": 26973, "ĠãģĹ": 26974, "bia": 26975, "Ġcustody": 26976, "Ġbiod": 26977, "æĿ±è¥¿": 26978, "Ġdirecting": 26979, "...âĢĭ": 26980, "Ġreloc": 26981, "Ġdemande": 26982, "ãĤĵãģł": 26983, "ĠoÄŁlum": 26984, "Ġодна": 26985, "ĠMilk": 26986, "åı·": 26987, "ĠKra": 26988, "ĠHonda": 26989, "Ġpue": 26990, "Ġelekt": 26991, "Ġbeginners": 26992, "Ġspear": 26993, "ÃŃnh": 26994, "ĠLuft": 26995, "Ġnig": 26996, "ĠSchools": 26997, "Ġforums": 26998, "ĠQin": 26999, "ppo": 27000, "Ġzag": 27001, "ĠЮ": 27002, "Ġtoothp": 27003, "ĠStyle": 27004, "ì´Ī": 27005, "Ġpunct": 27006, "Ġreps": 27007, "ĠAly": 27008, "Ġamendments": 27009, "Ġöz": 27010, "Ġdigits": 27011, "urai": 27012, "Ġchaotic": 27013, "ĠMasters": 27014, "eon": 27015, "ĠCash": 27016, "ĠCuz": 27017, "Ġbedeutet": 27018, "Ġscanning": 27019, "Ġжд": 27020, "неÑĤ": 27021, "Ġcertainty": 27022, "jek": 27023, "Ġdijo": 27024, "ĠClimate": 27025, "Ġrinse": 27026, "Ġkrij": 27027, "veland": 27028, "Ġsoundtrack": 27029, "ĠSafe": 27030, "ĠNova": 27031, "94": 27032, "Ġathe": 27033, "ĠVerb": 27034, "oler": 27035, "ìĿ´ì£ł": 27036, "Ġvin": 27037, "Ġrespiratory": 27038, "ĠStudy": 27039, "ĠCAM": 27040, "Ġavocado": 27041, "ĠZhen": 27042, "Ġlatency": 27043, "Ġfeathers": 27044, "Ġcontar": 27045, "ĠвеÑī": 27046, "Ġfark": 27047, "Ġblended": 27048, "Ġexploded": 27049, "ĠXX": 27050, "ĠBenim": 27051, "Ġalguém": 27052, "istoire": 27053, "Ġconfidential": 27054, "Ġmast": 27055, "Ġì¿": 27056, "geh": 27057, "Ġdisrespect": 27058, "ĠSystems": 27059, "Æ°a": 27060, "Ed": 27061, "Ġwys": 27062, "Ġexotic": 27063, "Ġglowing": 27064, "ùng": 27065, "ounge": 27066, "èĦ": 27067, "аниз": 27068, "Ġpalav": 27069, "ĠSword": 27070, "Ġgim": 27071, "ĠCrow": 27072, "Ġpotent": 27073, "bish": 27074, "Ġabused": 27075, "ĠJed": 27076, "Ġgambling": 27077, "ĠSpect": 27078, "Ġinvestigators": 27079, "æĻļ": 27080, "Ġratt": 27081, "Ġdob": 27082, "ĠDES": 27083, "hog": 27084, "ĠоÑĤкÑĢÑĭ": 27085, "íĮħ": 27086, "ĠденÑĮги": 27087, "Ġíĺ¹": 27088, "Ġ머리": 27089, "Ġsaturation": 27090, "Ġinherited": 27091, "ĠInnovation": 27092, "ìĹĪëįĺ": 27093, "Ġtangible": 27094, "Ġdepri": 27095, "hed": 27096, "Ġпомог": 27097, "Ġsliced": 27098, "à¥į": 27099, "Ġthế": 27100, "Å¥": 27101, "68": 27102, "Ġcorona": 27103, "Ġgifted": 27104, "Ġsoir": 27105, "Ġhumility": 27106, "ĠìĿ´ê±¸": 27107, "Ġflaws": 27108, "ĠпÑĢакÑĤи": 27109, "Ġkald": 27110, "waż": 27111, "yw": 27112, "ãĤĵãģ§ãģĻ": 27113, "irteen": 27114, "Ġcrochets": 27115, "¦¬ê°Ģ": 27116, "ĠìłĦìĹIJ": 27117, "Ġdese": 27118, "æ¥Ń": 27119, "Ġмаг": 27120, "ĠdziaÅĤ": 27121, "Ġlég": 27122, "changing": 27123, "Ġllev": 27124, "ÅĦsk": 27125, "çĶ»": 27126, "Ġ1984": 27127, "orns": 27128, "ĠWelsh": 27129, "Ġpharmaceutical": 27130, "Ġpumping": 27131, "ĠShaw": 27132, "punk": 27133, "Ġvault": 27134, "Ġkinetic": 27135, "Ġhurricane": 27136, "ĠIncluding": 27137, "ức": 27138, "ĠGrandpa": 27139, "anship": 27140, "é¦Ļ港": 27141, "ĠвÑĭÑħод": 27142, "нож": 27143, "ľł": 27144, "utta": 27145, "Ġê²ģëĭĪëĭ¤": 27146, "Ġbaz": 27147, "ĠпоÑĪ": 27148, "Ġpeculiar": 27149, "zyÄĩ": 27150, "ĠEllie": 27151, "Ġlearns": 27152, "ĠKrishna": 27153, "Ġconsecut": 27154, "Ġempath": 27155, "ĠDin": 27156, "Ġtraded": 27157, "ĠBoris": 27158, "uggage": 27159, "olla": 27160, "Ġназв": 27161, "Ġeternity": 27162, "Ġвп": 27163, "èmes": 27164, "Ġgrapp": 27165, "bé": 27166, "ĠпÑĢедÑģÑĤав": 27167, "ĠFC": 27168, "įëĭĪëĭ¤": 27169, "even": 27170, "ĠNebraska": 27171, "ortune": 27172, "Ġkarena": 27173, "ĠAgent": 27174, "Ġsting": 27175, "ĠPI": 27176, "Ġmunicipal": 27177, "powered": 27178, "Ġconsegue": 27179, "ĠManchester": 27180, "Ġrainy": 27181, "Ġbli": 27182, "Ġkost": 27183, "Ġhalten": 27184, "ĠAhhh": 27185, "insula": 27186, "erting": 27187, "ĠاÙĦÙģ": 27188, "Ġrelacion": 27189, "Ġkomen": 27190, "Ġdome": 27191, "Ġpriests": 27192, "ĠIntrodu": 27193, "rophe": 27194, "shore": 27195, "velt": 27196, "clipse": 27197, "ĠÑĢÑĥÑģ": 27198, "×Ļס": 27199, "Ġsabemos": 27200, "ĠHolland": 27201, "ogi": 27202, "anki": 27203, "ĠMats": 27204, "Ġsmoked": 27205, "ullie": 27206, "Ġeurope": 27207, "ĠдейÑģÑĤвиÑĤелÑĮно": 27208, "Ġbardziej": 27209, "Ġtransforming": 27210, "ĠEz": 27211, "opath": 27212, "Ġìĸ¸ëĭĪ": 27213, "ĠÑģÑĤан": 27214, "ằng": 27215, "ัà¹ī": 27216, "ĠOuch": 27217, "Ġclearance": 27218, "ustain": 27219, "Ġsolidarity": 27220, "Ġproving": 27221, "ĠÐĺн": 27222, "ĠÑģÑĬ": 27223, "Ġprolong": 27224, "адно": 27225, "Ġsos": 27226, "ĠDeal": 27227, "Ġ170": 27228, "mons": 27229, "Ġзем": 27230, "Ġlogged": 27231, "Ġlifelong": 27232, "Ġsensory": 27233, "Ġbehold": 27234, "ĠFAR": 27235, "ètement": 27236, "ĠFederation": 27237, "Ġdodge": 27238, "ĠShir": 27239, "Ġdragons": 27240, "ĠArctic": 27241, "Äħż": 27242, "Åį": 27243, "º": 27244, "Ġdenke": 27245, "ĠpodrÃŃa": 27246, "cole": 27247, "ÑĥлÑĮÑĤаÑĤ": 27248, "Ġsystematic": 27249, "ама": 27250, "chos": 27251, "Ġclinics": 27252, "ĠBS": 27253, "Ġtales": 27254, "usions": 27255, "ĠíĪ¬": 27256, "Ġpreservation": 27257, "Ġlore": 27258, "ĠProtest": 27259, "Ỽ": 27260, "å¸Ĥ": 27261, "Ġacknowledged": 27262, "ĠIsaiah": 27263, "ĠëķĮëĬĶ": 27264, "Ġ×ĺ": 27265, "Ġcompetitor": 27266, "Ġadvancing": 27267, "zip": 27268, "Ġtenth": 27269, "ĠLaure": 27270, "Ġhints": 27271, "Ġexercising": 27272, "ŀľë": 27273, "ĠIntelligence": 27274, "uated": 27275, "OUT": 27276, "oped": 27277, "Ġautonomy": 27278, "Ġbranding": 27279, "ĠMediterranean": 27280, "Ñĸк": 27281, "Ġscrewdriver": 27282, "Ġsupre": 27283, "Ġstap": 27284, "Ġjurisdiction": 27285, "ĠSettings": 27286, "Ġforefront": 27287, "ĠFemale": 27288, "comfort": 27289, "Ġmultiplication": 27290, "ĠMurray": 27291, "Ġbob": 27292, "ĠTas": 27293, "Ġtahu": 27294, "Ġonun": 27295, "etter": 27296, "Ġprophets": 27297, "lag": 27298, "Ġrevenues": 27299, "Ġprá": 27300, "Ġuploading": 27301, "Ġmachinery": 27302, "ascal": 27303, "ĠEstá": 27304, "ĠGoth": 27305, "ĠBald": 27306, "ĠSaw": 27307, "Ġstripes": 27308, "ìłij": 27309, "Ġpowin": 27310, "æĹ¥æľ¬": 27311, "Ġhostile": 27312, "Ġdarum": 27313, "Ġprevented": 27314, "ожалÑĥйÑģÑĤа": 27315, "Ġalgunas": 27316, "Ġhopeless": 27317, "Ġznaj": 27318, "Ġreadings": 27319, "Ġcraving": 27320, "tat": 27321, "ĠPig": 27322, "Ġliar": 27323, "çĪ±": 27324, "Ġmultiplayer": 27325, "Ġdale": 27326, "ĠCourse": 27327, "íģ¼": 27328, "ĠKita": 27329, "Ġcustoms": 27330, "Ġresponds": 27331, "endra": 27332, "è¦ĸ": 27333, "Ġmetro": 27334, "Ñģол": 27335, "Ġmitigate": 27336, "Ġoppression": 27337, "ĠæĪijåĢij": 27338, "quinho": 27339, "Ġammo": 27340, "Ġenfer": 27341, "Ġpony": 27342, "Ġounces": 27343, "°Ķ": 27344, "ĠìĪĺê°Ģ": 27345, "Ġdicho": 27346, "ĠDeb": 27347, "Ġwonders": 27348, "ĠRoose": 27349, "Ġprizes": 27350, "ĠALEX": 27351, "Ġthankfully": 27352, "Ġtissues": 27353, "ĠÑĢавно": 27354, "ĠLuna": 27355, "intelligible": 27356, "ĠìĻ¸": 27357, "ê°ij": 27358, "ĠHeat": 27359, "ĠÑģид": 27360, "ĠQui": 27361, "Ġions": 27362, "Ġaccommodation": 27363, "便": 27364, "ĠKart": 27365, "ienst": 27366, "Ġtarde": 27367, "Ġsoaked": 27368, "ĠCasey": 27369, "Ġì´Ŀ": 27370, "ĠÑĢÑĥб": 27371, "Ġdifferenti": 27372, "Ġleftover": 27373, "Ġexchanges": 27374, "second": 27375, "Ġfirstly": 27376, "Ġbuilder": 27377, "rien": 27378, "Ġdw": 27379, "Ġbouncing": 27380, "?<": 29986, "ologÃŃa": 29987, "wealth": 29988, "Ġmeditate": 29989, "ĵ¤ìĿĺ": 29990, "ĠCraft": 29991, "è§īå¾Ĺ": 29992, "æĻ®": 29993, "riv": 29994, "ĠAgainst": 29995, "Ġceramic": 29996, "espère": 29997, "Ġcompetent": 29998, "ĠHopkins": 29999, "Ġkilos": 30000, "Ġgravel": 30001, "Ġpiston": 30002, "Ġfriendships": 30003, "Ġescre": 30004, "Ġvoz": 30005, "ĠGesellschaft": 30006, "Ġunterstüt": 30007, "Ġmuj": 30008, "Ġwarnings": 30009, "pos": 30010, "ĠProfessional": 30011, "wszy": 30012, "odle": 30013, "bands": 30014, "Ġteamwork": 30015, "stellung": 30016, "Ġdx": 30017, "åįĬ": 30018, "Ġattorneys": 30019, "Ġweitere": 30020, "ãħĭãħĭãħĭ": 30021, "ĠOriginal": 30022, "×Ļ×Ĺ": 30023, "Ġbroadcasting": 30024, "ĠпеÑĢвÑĭй": 30025, "uchi": 30026, "Ġheure": 30027, "Ġgrabs": 30028, "ĠWOR": 30029, "ĠPlaid": 30030, "Min": 30031, "Ġpaz": 30032, "ĠPuis": 30033, "umu": 30034, "itates": 30035, "Ġcoats": 30036, "Ġbuen": 30037, "Ġheir": 30038, "Ġpneum": 30039, "שר": 30040, "enser": 30041, "ĠJUDGE": 30042, "Ġblonde": 30043, "á¹Ľ": 30044, "Ġgak": 30045, "Ġsık": 30046, "Ġquoted": 30047, "Ġequipo": 30048, "Ġwishing": 30049, "ÃŃcia": 30050, "Ġverbs": 30051, "çµĦ": 30052, "ĠCanadians": 30053, "Ġgoverning": 30054, "ĠEvans": 30055, "Euro": 30056, "Ġgenres": 30057, "Ġunterschied": 30058, "ĠBecky": 30059, "³¼ê²ĮìļĶ": 30060, "Ġeinge": 30061, "ĠRaise": 30062, "oland": 30063, "ĠStrateg": 30064, "Ġeres": 30065, "ĠVeterans": 30066, "Ġbreakout": 30067, "Ġsanté": 30068, "Ġadel": 30069, "Ġinvestigated": 30070, "Ġpeur": 30071, "Ġagile": 30072, "Ġrailroad": 30073, "anska": 30074, "Ġей": 30075, "Ġexpos": 30076, "atories": 30077, "ĠContent": 30078, "Ġtruths": 30079, "ĠTrail": 30080, "Ġgua": 30081, "Ġpores": 30082, "Ġwritings": 30083, "ĠUhr": 30084, "ĠThats": 30085, "Ġicing": 30086, "OC": 30087, "ĠProduction": 30088, "Ġcarne": 30089, "ISS": 30090, "Ġninguém": 30091, "non": 30092, "Ġvicious": 30093, "×ķ×Ķ": 30094, "Ġreconnect": 30095, "Ġcentres": 30096, "ĠKem": 30097, "Ġcrease": 30098, "ĠìĿ´ë¯¸": 30099, "айÑĤеÑģÑĮ": 30100, "ĠбоÑĢ": 30101, "ĠHayır": 30102, "ĠÑģÑĥд": 30103, "Ġúnica": 30104, "owaÅĤ": 30105, "Ġadher": 30106, "hua": 30107, "ZZ": 30108, "Ġpreciso": 30109, "Ġcurrents": 30110, "Ġseasoned": 30111, "ĠIoT": 30112, "ĠBishop": 30113, "è¨Ī": 30114, "sted": 30115, "ĠBernard": 30116, "ì¤ĺ": 30117, "æ²»": 30118, "ĠGlenn": 30119, "Ġktórym": 30120, "ืà¹Ī": 30121, "Ġastrolog": 30122, "ĠKot": 30123, "å¤ľ": 30124, "Ġparfois": 30125, "Ġforwards": 30126, "ĠWiÄĻ": 30127, "ĠÎĺ": 30128, "Ġnano": 30129, "è»į": 30130, "sub": 30131, "ĠBrill": 30132, "Ġgrit": 30133, "Ġcited": 30134, "gado": 30135, "Ġmelts": 30136, "Ġforcé": 30137, "âĸĪâĸĪ": 30138, "Ġbajo": 30139, "Ġdiscretion": 30140, "°°": 30141, "ativity": 30142, "Ġsituated": 30143, "ãĥ«ãĤ¯": 30144, "Ñīее": 30145, "åľ°æĸ¹": 30146, "ĠпÑĢинÑĨип": 30147, "amaz": 30148, "Ġaquarium": 30149, "Ġdissolve": 30150, "ĠGods": 30151, "Super": 30152, "Ġamid": 30153, "zk": 30154, "ĠãģĦ": 30155, "éłIJ": 30156, "ampf": 30157, "Ġhela": 30158, "'!": 30159, "Ġdevelopmental": 30160, "ĠDise": 30161, "ĠÑĢабоÑĤаеÑĤ": 30162, "Ġsnapshot": 30163, "好好": 30164, "Õ¸": 30165, "ĠYue": 30166, "ĠHulk": 30167, "ĠDoom": 30168, "ĠFelix": 30169, "Ġréf": 30170, "Male": 30171, "ç·Ĭ": 30172, "phants": 30173, "ENS": 30174, "ĠMechan": 30175, "ĠGolf": 30176, "åĨįè¦ĭ": 30177, "Ġgenerosity": 30178, "ätze": 30179, "Ġunlocked": 30180, "ĠãĤĴ": 30181, "íĥģ": 30182, "ocalypse": 30183, "Alright": 30184, "Ġê°ľë": 30185, "Ġ×IJ×ij׾": 30186, "ĠKeeping": 30187, "Ġcollaborating": 30188, "chief": 30189, "ĠFernando": 30190, "Ġchefs": 30191, "ĠíĶ¼ë¶Ģ": 30192, "Ġskipped": 30193, "Ġpersonn": 30194, "Ġaxe": 30195, "chez": 30196, "Ġextraction": 30197, "ĠAV": 30198, "ĠGibbs": 30199, "Ġíľ": 30200, "Ġsı": 30201, "IAM": 30202, "View": 30203, "ĠGRANT": 30204, "Ġ몸": 30205, "Ġverification": 30206, "Ġdepicted": 30207, "ĠMoz": 30208, "oux": 30209, "Ġtul": 30210, "Ġscanner": 30211, "Ġcomedian": 30212, "ĠVolks": 30213, "ĠJEFF": 30214, "è¨Ĥéĸ±": 30215, "§Ħ": 30216, "Ġdistraction": 30217, "rá": 30218, "ĠINTER": 30219, "Ġsincer": 30220, "Ġ×ŀת": 30221, "Ġש׳": 30222, "Ġconstructive": 30223, "arf": 30224, "ĠëĪĦë": 30225, "Ġeco": 30226, "ramos": 30227, "Ġrenewed": 30228, "inement": 30229, "ĠUb": 30230, "ĠPepper": 30231, "ì§Ģê°Ģ": 30232, "ĠDarwin": 30233, "Ġmerchand": 30234, "Ġvárias": 30235, "èce": 30236, "NG": 30237, "ĠìľĦíķ´ìĦľ": 30238, "ĠакÑĤив": 30239, "ĠUnters": 30240, "عÙĦ": 30241, "Ġintric": 30242, "omma": 30243, "ieving": 30244, "ĠCaroline": 30245, "åĵģ": 30246, "ĠPRES": 30247, "Ġperformer": 30248, "Ġautour": 30249, "ãģ¾ãģĽãĤĵ": 30250, "Ġutterly": 30251, "Ġsynthesis": 30252, "Ġlesbian": 30253, "Ġretrieve": 30254, "Ġmaneira": 30255, "Ġimpair": 30256, "Ġmentoring": 30257, "ĠSouls": 30258, "ĠGoPro": 30259, "ÑĢаÑĤÑĮ": 30260, "Ġcose": 30261, "ĠSSD": 30262, "IRE": 30263, "Ġupfront": 30264, "ĠAun": 30265, "Ġgamer": 30266, "Ġlitt": 30267, "Ġaggression": 30268, "ĠLikewise": 30269, "ĠBetty": 30270, "ĠDart": 30271, "ĠDLC": 30272, "ishment": 30273, "ìŀ¥ìĿĦ": 30274, "Ġ对": 30275, "ç»ı": 30276, "cream": 30277, "ĠBabylon": 30278, "Ġnug": 30279, "brar": 30280, "Ġaynı": 30281, "amily": 30282, "bike": 30283, "ahahaha": 30284, "loyd": 30285, "Ġmira": 30286, "Ġperme": 30287, "ĠGaming": 30288, "Ġfirmware": 30289, "Ma": 30290, "Ġassisted": 30291, "atics": 30292, "Ġìķŀìľ¼ë¡ľ": 30293, "ĠMental": 30294, "niejs": 30295, "ĠIz": 30296, "owÄħ": 30297, "Ġtougher": 30298, "Ġdeed": 30299, "èĭ¦": 30300, "Ġstylish": 30301, "ĠTools": 30302, "ĠHamp": 30303, "Ġsunscreen": 30304, "Ġarticulate": 30305, "iye": 30306, "иÑĦ": 30307, "ĠSpread": 30308, "ĠHAVE": 30309, "Ġswirl": 30310, "Ġsponsoring": 30311, "ä»ĭ": 30312, "iovascular": 30313, "mesi": 30314, "Ġrelaxation": 30315, "ĠÑģвоиÑħ": 30316, "Ġmargins": 30317, "ĠsaÄŁ": 30318, "ĠPride": 30319, "ĠÏĦοÏħÏĤ": 30320, "иÑĨи": 30321, "enci": 30322, "Does": 30323, "Ġcorpse": 30324, "Ġendurance": 30325, "Ġíŀĺ": 30326, "ì¹´": 30327, "Ġhaircut": 30328, "Ġinterrupted": 30329, "Ġwindy": 30330, "ĠCaleb": 30331, "ÏģÏĩ": 30332, "ĠPourquoi": 30333, "Ġholistic": 30334, "uclear": 30335, "ĠWhole": 30336, "士": 30337, "Act": 30338, "Ġgallon": 30339, "cade": 30340, "ĠRegional": 30341, "roads": 30342, "ĠSchne": 30343, "áng": 30344, "Ġизмен": 30345, "ãĤĪãģŃ": 30346, "Ġmenus": 30347, "Ġsplitting": 30348, "Ġpriced": 30349, "ĠÎĵ": 30350, "Ġusername": 30351, "ĠÐŀÑĩ": 30352, "Ġcompressed": 30353, "yin": 30354, "Ġguardian": 30355, "Ġgoof": 30356, "Ġchecklist": 30357, "Ġinterchange": 30358, "Ġexpedition": 30359, "Ġextern": 30360, "Ġinfrared": 30361, "engo": 30362, "Ġdenying": 30363, "Ġpackets": 30364, "onent": 30365, "BB": 30366, "ĠIncre": 30367, "Ġsini": 30368, "ÃŁer": 30369, "èg": 30370, "maal": 30371, "generation": 30372, "Ġminorities": 30373, "Ġllevar": 30374, "Ġnomination": 30375, "Ġconsid": 30376, "Ġ×ľ×¢": 30377, "muÅŁ": 30378, "ĠEsc": 30379, "Ġnumerator": 30380, "Ġkaik": 30381, "Ġktórych": 30382, "iesen": 30383, "Ġvê": 30384, "ĠUSS": 30385, "ĠPrivate": 30386, "Ġодно": 30387, "Ġalém": 30388, "ÃŃtulo": 30389, "Ġlimb": 30390, "Ġforgiven": 30391, "Ġdisclosure": 30392, "ÏĦί": 30393, "Ġningún": 30394, "Ġtherapeutic": 30395, "Ġnegotiating": 30396, "ĠNike": 30397, "enseful": 30398, "Ġincap": 30399, "Ġflagship": 30400, "town": 30401, "âĪ": 30402, "ĠÏĢολ": 30403, "Ġwolves": 30404, "Ġviolations": 30405, "ĠArnold": 30406, "Ġintervene": 30407, "Ġheater": 30408, "Ġrecursos": 30409, "Ġmaid": 30410, "ê²¼": 30411, "ĠдавайÑĤе": 30412, "ĠCelebr": 30413, "Ġcape": 30414, "ĠSty": 30415, "ainen": 30416, "site": 30417, "bij": 30418, "ĠполÑĮз": 30419, "Ġframed": 30420, "Ġpublishers": 30421, "ĠÑĩÑĥÑĤÑĮ": 30422, "Ġtemptation": 30423, "Ġcerteza": 30424, "Ġexempt": 30425, "ìĬ¹": 30426, "selling": 30427, "ĠTask": 30428, "hoon": 30429, "ĠCoc": 30430, "ĠParks": 30431, "Ġrepetition": 30432, "ĠÑĤÑĥда": 30433, "Ġensl": 30434, "ĠdeÄŁiÅŁ": 30435, "ĠOrlando": 30436, "ĠMainten": 30437, "æŃ¢": 30438, "ocument": 30439, "ĠHC": 30440, "Ġscooter": 30441, "ĠнапиÑģ": 30442, "Ġtighter": 30443, "Ġtease": 30444, "Ġremoves": 30445, "Ġkijken": 30446, "ĠÑģÑĥÑīеÑģÑĤв": 30447, "Ġthé": 30448, "ĠвÑĭглÑıд": 30449, "Ġrelieve": 30450, "Ġmitä": 30451, "Ġstationary": 30452, "öff": 30453, "pable": 30454, "Ġarter": 30455, "Ġdéf": 30456, "rative": 30457, "Ġconect": 30458, "Ġsaddle": 30459, "ĠDiane": 30460, "Ġcommemor": 30461, "fendim": 30462, "SÃŃ": 30463, "Ġíģ´ë": 30464, "Ġmange": 30465, "atte": 30466, "Ġarrogant": 30467, "Ġrobotic": 30468, "ĠgiÃł": 30469, "æĺ¯çļĦ": 30470, "Ġneighbourhood": 30471, "isson": 30472, "Ġдвиж": 30473, "ĠRI": 30474, "ĠNorman": 30475, "brand": 30476, "amation": 30477, "Ġrazor": 30478, "Ġmurders": 30479, "ĠÑĤÑĥ": 30480, "Ġwszystkim": 30481, "Ġutilities": 30482, "Ġmicroscop": 30483, "ê¿": 30484, "Ġdaqui": 30485, "ollar": 30486, "ĠÐĶавайÑĤе": 30487, "Ġannée": 30488, "Ġkilometres": 30489, "Ġhomosexual": 30490, "Ġarchitects": 30491, "ãģ¡ãģ¯": 30492, "Ġniye": 30493, "LER": 30494, "Ġmicrophones": 30495, "ĠStunden": 30496, "Ġconsecutive": 30497, "ienda": 30498, "vänd": 30499, "DER": 30500, "Ġlifts": 30501, "ĠMeat": 30502, "Ġsavez": 30503, "íĸĪëįĺ": 30504, "Men": 30505, "Ġdismant": 30506, "거를": 30507, "Ġinsulation": 30508, "Ġscall": 30509, "Ġspooky": 30510, "Ġparc": 30511, "Ġballet": 30512, "ĠWhatsApp": 30513, "Ġfranc": 30514, "Ġdeliberate": 30515, "ĠíħĮ": 30516, "Ġmars": 30517, "ĠZur": 30518, "Pr": 30519, "disciplinary": 30520, "Ġobsession": 30521, "ме": 30522, "Ġmarching": 30523, "ĠEmergency": 30524, "iguous": 30525, "Ġszy": 30526, "ĠLands": 30527, "Ġboarding": 30528, "ĠпоÑĩÑĤи": 30529, "Ġenvy": 30530, "Ġcompassionate": 30531, "Ġmerci": 30532, "Ġdesirable": 30533, "dale": 30534, "Ġcanım": 30535, "ĠAntar": 30536, "temps": 30537, "Ġconfigured": 30538, "ĠCompared": 30539, "neh": 30540, "icating": 30541, "Ġnickel": 30542, "ÙĪÙĤ": 30543, "ÙĥÙĪÙĨ": 30544, "opes": 30545, "Ġformulas": 30546, "ĠÐķÑģÑĤÑĮ": 30547, "Ġpobl": 30548, "ĠPJ": 30549, "ĠLud": 30550, "ä»ĬåĽŀ": 30551, "ĠBrid": 30552, "ĠHog": 30553, "ĠBris": 30554, "Jen": 30555, "Ġshading": 30556, "ĠYas": 30557, "Ġdisturbed": 30558, "Ġrecommending": 30559, "Ġcé": 30560, "ĠHOW": 30561, "ìĹĪìĸ´": 30562, "Ġreversed": 30563, "ĠInterestingly": 30564, "ioxid": 30565, "åħŃ": 30566, "Ġìĺ¤ì¼ĢìĿ´": 30567, "ếu": 30568, "xx": 30569, "Ġouais": 30570, "ĠYouTubers": 30571, "ĠRosa": 30572, "ĠHaupt": 30573, "jadi": 30574, "Ġvlogs": 30575, "Ġcultura": 30576, "ĠLeadership": 30577, "ĠHep": 30578, "Ġillum": 30579, "´ëıĻ": 30580, "Ġcustomized": 30581, "Ġmarca": 30582, "Ġquatro": 30583, "Ġнаг": 30584, "ĠSpaceX": 30585, "ĠEigen": 30586, "asting": 30587, "ĠolduÄŁu": 30588, "Ġforts": 30589, "ãģī": 30590, "riment": 30591, "iencia": 30592, "Ġtenir": 30593, "roffen": 30594, "Ġ1979": 30595, "Ġcie": 30596, "ĠëIJĺê³ł": 30597, "Ġescri": 30598, "ÏĮÏĤ": 30599, "íı¬": 30600, "uzzy": 30601, "Cong": 30602, "ìĿ¸ìĿ´": 30603, "Great": 30604, "sil": 30605, "éch": 30606, "ãģ¨ãģĭ": 30607, "Ġmultic": 30608, "ĠDisk": 30609, "²ķ": 30610, "Ġfazla": 30611, "Ġlevant": 30612, "Ġabajo": 30613, "urry": 30614, "stru": 30615, "Ġ먹ëĬĶ": 30616, "Ġaccessory": 30617, "Ġдвиг": 30618, "ĠRid": 30619, "2019": 30620, "Ġdownstream": 30621, "æķ¸": 30622, "Ġkaz": 30623, "utan": 30624, "Ġcharcoal": 30625, "Ġafect": 30626, "wu": 30627, "Ġcontexts": 30628, "Ġfeared": 30629, "ĠìĦ¤": 30630, "Ġhistories": 30631, "Ġfas": 30632, "ensible": 30633, "Ġcocoa": 30634, "illar": 30635, "geons": 30636, "Ġspirituality": 30637, "ĠPew": 30638, "Ġpharmacy": 30639, "Ġpassions": 30640, "Ġbos": 30641, "Ġallá": 30642, "Ġthriving": 30643, "ĠReact": 30644, "Ġoccupy": 30645, "Ġwithdrawal": 30646, "Ġallowance": 30647, "ĠFraktion": 30648, "Ġbuddies": 30649, "Ġidle": 30650, "Ġdissolved": 30651, "Ġprevalent": 30652, "Ġmilitar": 30653, "Ġsensing": 30654, "Ġpojaw": 30655, "Ġancora": 30656, "Ġabundant": 30657, "Ġhairst": 30658, "ãģĤãĤĮ": 30659, "Ġtwee": 30660, "Ġnächste": 30661, "ĠMöglichkeit": 30662, "Ġhoo": 30663, "ufficient": 30664, "Ġfantast": 30665, "Ġedible": 30666, "Ġëĸ¨ìĸ´ì": 30667, "ìĽĥ": 30668, "Ġvein": 30669, "ucci": 30670, "Ġdevotion": 30671, "Ġconcealer": 30672, "income": 30673, "Ġrecycled": 30674, "ĠìĬ¤íĥĢ": 30675, "Ġpontos": 30676, "Ġdessus": 30677, "Ġvérit": 30678, "Ġreflections": 30679, "ĠAA": 30680, "Ġtakeaway": 30681, "bare": 30682, "ĠContact": 30683, "eil": 30684, "ĠHear": 30685, "Ġmirac": 30686, "ĠGerilim": 30687, "ĠÑģамÑĭй": 30688, "Ġvivo": 30689, "Ġkilograms": 30690, "ĠCrim": 30691, "ût": 30692, "78": 30693, "Ġsincerely": 30694, "raz": 30695, "Ġë³µ": 30696, "Ġarriv": 30697, "Ġconception": 30698, "ĠPersian": 30699, "Ġsjäl": 30700, "Ġstarring": 30701, "ĠìķĦ무": 30702, "ĠForever": 30703, "еÑģÑĤÑĮ": 30704, "Ġveil": 30705, "Ġsubtit": 30706, "odka": 30707, "ĠоÑĤноÑĪ": 30708, "Ġcooks": 30709, "енÑı": 30710, "Kay": 30711, "Ġniños": 30712, "ĠPhone": 30713, "Ġstitching": 30714, "Ġfingerprint": 30715, "é¢ĺ": 30716, "λά": 30717, "Ġdedicate": 30718, "ĠLob": 30719, "Ġblacks": 30720, "ĠBle": 30721, "bout": 30722, "ĠÄijang": 30723, "Ġeks": 30724, "Ġsquash": 30725, "ĠKü": 30726, "odi": 30727, "ĠnÆ°á»Ľc": 30728, "Ġvoyage": 30729, "Ġplayful": 30730, "ĠØ¥ÙĦÙī": 30731, "anic": 30732, "Ġcondemn": 30733, "ĠBöyle": 30734, "ĠPolize": 30735, "ãĤ¿ãĥ¼": 30736, "Ġayuda": 30737, "Ġpam": 30738, "à¹Ħà¸Ľ": 30739, "ĠKathy": 30740, "един": 30741, "нова": 30742, "Ġbrig": 30743, "eger": 30744, "Ġeagle": 30745, "Ġvisions": 30746, "ĠíķŃìĥģ": 30747, "Ġshitty": 30748, "Ġhott": 30749, "ĠBritt": 30750, "utors": 30751, "ENTE": 30752, "æĽ²": 30753, "Ġphon": 30754, "ĠBing": 30755, "ĠподдеÑĢж": 30756, "spring": 30757, "æĸ¯": 30758, "etten": 30759, "Ġpilgr": 30760, "Ġediyor": 30761, "енÑĤÑĭ": 30762, "aggio": 30763, "Ġjul": 30764, "Ġcomprend": 30765, "teil": 30766, "Ġز": 30767, "Ġperformers": 30768, "Ġinfamous": 30769, "ĠMK": 30770, "çª": 30771, "æ³ģ": 30772, "otle": 30773, "eff": 30774, "ĠHash": 30775, "Ġcoward": 30776, "ĠBRA": 30777, "ĠDD": 30778, "Ġcomida": 30779, "Ġplata": 30780, "Ġflap": 30781, "ĠMehr": 30782, "ribution": 30783, "ĠYemen": 30784, "Ġmysteries": 30785, "ĠÄ°yi": 30786, "Ġstell": 30787, "Ġeyeliner": 30788, "Ġdeles": 30789, "Ġnailed": 30790, "Ġillnesses": 30791, "Ġstacks": 30792, "Ġtrabajar": 30793, "flower": 30794, "ciu": 30795, "Ġcrude": 30796, "Ġsubstantially": 30797, "Ġhomem": 30798, "Ġnephew": 30799, "Ġstamps": 30800, "Ġcarbs": 30801, "ÑĮÑĤе": 30802, "mooth": 30803, "Ġtunnels": 30804, "acie": 30805, "æ³¢": 30806, "ĠSeñ": 30807, "ĠHera": 30808, "ĠìķĦëĭĪìĹIJìļĶ": 30809, "ĠWyoming": 30810, "ĠHDMI": 30811, "ĠLis": 30812, "ución": 30813, "Ġsteer": 30814, "оÑİ": 30815, "иÑĤа": 30816, "NT": 30817, "Ġìĸ¼êµ´": 30818, "Ġpalms": 30819, "Ġneon": 30820, "ованиÑı": 30821, "Ġfiltering": 30822, "Ġjouer": 30823, "ĠHö": 30824, "ĠнеÑģ": 30825, "ê²łìĸ´ìļĶ": 30826, "Ġ81": 30827, "Ġstoryline": 30828, "Ġprzep": 30829, "Ġthanking": 30830, "ĠBoeing": 30831, "Ġsoftly": 30832, "jem": 30833, "алÑĮнÑĭÑħ": 30834, "Ġflashlight": 30835, "ĠпÑĥ": 30836, "ĠWOMAN": 30837, "ắc": 30838, "ÃŃch": 30839, "Ġluxurious": 30840, "Ġwün": 30841, "Ġimpactful": 30842, "Ġconson": 30843, "reu": 30844, "irring": 30845, "ifter": 30846, "Ġconstituents": 30847, "èIJ½": 30848, "Ġ94": 30849, "ĠTou": 30850, "gom": 30851, "ĠìĥĿê°ģìĿĦ": 30852, "Ġstereotypes": 30853, "Ġmożli": 30854, "åĪĨ享": 30855, "Ĥ¨": 30856, "Ġpencils": 30857, "ĠÑģлож": 30858, "Ġihrem": 30859, "ĠBesch": 30860, "ĠKoh": 30861, "ĠEntscheid": 30862, "Ġlek": 30863, "Ġförs": 30864, "Ġtotalmente": 30865, "Ġlively": 30866, "Ġentropy": 30867, "Ġdiscern": 30868, "ĠÐĹна": 30869, "Ġdov": 30870, "Ġmythology": 30871, "è¨ĺå¾Ĺ": 30872, "apanese": 30873, "Ġapproximate": 30874, "аÑĤив": 30875, "ifiable": 30876, "ĠSeo": 30877, "åĢĴ": 30878, "´ìĭ¬íŀĪ": 30879, "Ġìĺ·": 30880, "Ġtemporal": 30881, "ĠiT": 30882, "Ġestat": 30883, "ким": 30884, "Ġsprink": 30885, "Ġgrund": 30886, "Ġinfantry": 30887, "Ġschaffen": 30888, "ç´Ħ": 30889, "Ġank": 30890, "riages": 30891, "ĠYeon": 30892, "ĠMoroc": 30893, "Ġinvasive": 30894, "ģĶ": 30895, "Ġparenting": 30896, "ĠRis": 30897, "ibile": 30898, "Ġmods": 30899, "å½¢": 30900, "ĠпÑĢовеÑĢ": 30901, "ĠThing": 30902, "ĠWherever": 30903, "Ġacknowledging": 30904, "Ġpawn": 30905, "ummer": 30906, "orb": 30907, "69": 30908, "Ġretrouve": 30909, "Ġrelies": 30910, "ĠHighway": 30911, "Ġawe": 30912, "ãģ§ãģĻãģĭ": 30913, "itaire": 30914, "Ġapplicant": 30915, "Ġaisle": 30916, "worm": 30917, "Ġpayload": 30918, "Ġcarre": 30919, "ĠBach": 30920, "æł¼": 30921, "Ġì¹ľêµ¬ë": 30922, "ние": 30923, "ĠitÃŃs": 30924, "onnaise": 30925, "sol": 30926, "èı¯": 30927, "algia": 30928, "Ġrocking": 30929, "Ġbesten": 30930, "rites": 30931, "^^": 30932, "иной": 30933, "Ġbaixo": 30934, "Ġ기ìĸµ": 30935, "оÑĤÑĢи": 30936, "sim": 30937, "Ġincarn": 30938, "ëĭ¤ìĿĮ": 30939, "Ġlick": 30940, "sided": 30941, "Ġ71": 30942, "forder": 30943, "Ġresonance": 30944, "Ġtegen": 30945, "Ġmetaph": 30946, "owser": 30947, "Ġ×IJ׳×Ĺ׳×ķ": 30948, "?ãĢį": 30949, "Ġspielen": 30950, "Ġvolley": 30951, "ĶìĿ´íģ¬ìĹħ": 30952, "looked": 30953, "Ġsentenced": 30954, "Ġmultiplying": 30955, "Ġideals": 30956, "Ġwahrscheinlich": 30957, "Ġdeposits": 30958, "bilir": 30959, "Ġeffet": 30960, "illon": 30961, "Īë§Į": 30962, "Ġtestimon": 30963, "Ġzawsze": 30964, "ĠпÑĢоÑĨеÑģÑģ": 30965, "ĠLav": 30966, "ä¸įéĮ¯": 30967, "Ġtravailler": 30968, "Ġlaisse": 30969, "ĠMountains": 30970, "ĠÑĢоб": 30971, "Ġexamined": 30972, "itus": 30973, "Was": 30974, "лÑĭ": 30975, "Ġattributed": 30976, "ĠìĬ¹": 30977, "ĠBaron": 30978, "Ġgep": 30979, "Ġattent": 30980, "ĠCollection": 30981, "Ġtheat": 30982, "ĠCai": 30983, "Ġwells": 30984, "Ġhumano": 30985, "çĹħ": 30986, "ĠHast": 30987, "ĠÑħоÑĤÑı": 30988, "czas": 30989, "Ġpermits": 30990, "Ġlegg": 30991, "Ġepo": 30992, "ĠFen": 30993, "Ġthi": 30994, "ĠFoi": 30995, "Ġélect": 30996, "Ġ83": 30997, "Ġoverth": 30998, "Ġè¬Ŀè¬Ŀ": 30999, "Ġtenant": 31000, "è²·": 31001, "Next": 31002, "Ġpraised": 31003, "security": 31004, "ĠImpact": 31005, "为ä»Ģä¹Ī": 31006, "Ġvouch": 31007, "Ġnegó": 31008, "Ġunve": 31009, "Ġcriticize": 31010, "ĠKenya": 31011, "Ġtactic": 31012, "Ġlogr": 31013, "Ġpois": 31014, "Ġpapa": 31015, "speaks": 31016, "ðŁij": 31017, "ispers": 31018, "Ġsurplus": 31019, "Ġcolder": 31020, "åįĹ": 31021, "åIJ¬": 31022, "plets": 31023, "ĠVienna": 31024, "ĠLead": 31025, "Ġaerial": 31026, "ĠTah": 31027, "енÑĤов": 31028, "ĠGreeks": 31029, "Cam": 31030, "Ġmáxim": 31031, "Ġkuin": 31032, "chio": 31033, "Ġdemonstrates": 31034, "anos": 31035, "ĠCert": 31036, "ĠÑįн": 31037, "Ġblogs": 31038, "ĠìĦľìļ¸": 31039, "Ġbeams": 31040, "иков": 31041, "Ġprompted": 31042, "Ġfrightening": 31043, "ĠPorsche": 31044, "ãģĪãģ¦": 31045, "larını": 31046, "Ġchilling": 31047, "isphere": 31048, "Ġflashing": 31049, "ĠKard": 31050, "bread": 31051, "Ġexh": 31052, "Ġtycker": 31053, "Ġecological": 31054, "ĠMae": 31055, "Ġ×ŀ×IJ×ķ×ĵ": 31056, "ĠëĤĺëıĦ": 31057, "лон": 31058, "yss": 31059, "Ġpergunt": 31060, "Ġprix": 31061, "izzard": 31062, "Ġcancers": 31063, "Ġ91": 31064, "susp": 31065, "ĠItem": 31066, "ÅŁa": 31067, "Ġpest": 31068, "ĠtakÄħ": 31069, "Ġlymph": 31070, "ĠPatri": 31071, "fill": 31072, "Ġreconna": 31073, "Ġoptimism": 31074, "Ġmimic": 31075, "Ġì²ľ": 31076, "ĠMadame": 31077, "ocy": 31078, "lining": 31079, "åijĬ訴": 31080, "erme": 31081, "Ġfolders": 31082, "ĠczÅĤ": 31083, "uchar": 31084, "Ġcurso": 31085, "Ġbreach": 31086, "ниÑĤÑĮ": 31087, "ĠpamiÄĻ": 31088, "Ġelig": 31089, "Ġautop": 31090, "Flow": 31091, "Ġprogrammed": 31092, "ĠProcess": 31093, "Ġfigur": 31094, "ĠSF": 31095, "ĠEles": 31096, "Ġprogrammes": 31097, "Ġdizzy": 31098, "ìĭľê°Ħ": 31099, "Ġлибо": 31100, "Ġsniff": 31101, "ĠSebastian": 31102, "ĠHye": 31103, "Ġ4000": 31104, "Ġpermite": 31105, "æ¢Ŀ": 31106, "ĠзаÑī": 31107, "Ġguit": 31108, "ĠDais": 31109, "Ġaccordance": 31110, "Ġmodular": 31111, "ogeneous": 31112, "æĭį": 31113, "Ġpouquinho": 31114, "Ġartillery": 31115, "Ġlubric": 31116, "Ġvolcan": 31117, "ĠNH": 31118, "ðŁ¤": 31119, "Ġdean": 31120, "Rh": 31121, "Ġministre": 31122, "åĿIJ": 31123, "ĠInv": 31124, "ĠBulgar": 31125, "ĠDaten": 31126, "èİ": 31127, "Im": 31128, "Ġoriginated": 31129, "ĠNixon": 31130, "integr": 31131, "Ġlacks": 31132, "ĠNacht": 31133, "ìĸ´ëĤĺ": 31134, "camera": 31135, "Ġradish": 31136, "kiye": 31137, "Ġanges": 31138, "Ġpréf": 31139, "juk": 31140, "ĠBee": 31141, "ĠBU": 31142, "ĠвоÑģп": 31143, "ĠBT": 31144, "êmes": 31145, "ĠStück": 31146, "ĠInk": 31147, "æĪĸèĢħ": 31148, "ĠSergeant": 31149, "ĠMultip": 31150, "Ġhiçbir": 31151, "ĠСам": 31152, "ĠDé": 31153, "olph": 31154, "ìĸ¸": 31155, "Ġimpat": 31156, "ĠìķĬê³ł": 31157, "ĠÑĤакого": 31158, "ĠнавеÑĢное": 31159, "Ġunpredictable": 31160, "Ġmend": 31161, "ĠìĹĨìĸ´ìļĶ": 31162, "ĠjakieÅĽ": 31163, "Ġanni": 31164, "Ġdonné": 31165, "ĠKirsty": 31166, "Ġrectangular": 31167, "Ġempezar": 31168, "ĠExchange": 31169, "ê°Ķ": 31170, "Ġéconom": 31171, "ãģĵãĤĵ": 31172, "elin": 31173, "reibt": 31174, "Ġ×Ķפ": 31175, "Ġcemetery": 31176, "Ġespañol": 31177, "olin": 31178, "лÑİд": 31179, "Ġgrâce": 31180, "allen": 31181, "ĠPhilos": 31182, "ĠErst": 31183, "ĠìĥĪ": 31184, "ĠVid": 31185, "Give": 31186, "OH": 31187, "μο": 31188, "ĠPare": 31189, "Ġmetabolism": 31190, "Ġmaple": 31191, "Ġaxle": 31192, "ĠDy": 31193, "Ġkomme": 31194, "Ïİν": 31195, "Ġgreatness": 31196, "Ġverified": 31197, "Ġspé": 31198, "ĠFahrenheit": 31199, "ĠBren": 31200, "ĠConfeder": 31201, "Ġhistoire": 31202, "Ġeliminating": 31203, "ĠAdding": 31204, "ĠAbi": 31205, "æĿİ": 31206, "Ġhospitality": 31207, "tim": 31208, "Ġbonito": 31209, "Ġpartes": 31210, "ĠдÑĢÑĥгиÑħ": 31211, "ĠShay": 31212, "ĠSed": 31213, "Ġregrets": 31214, "Ñıми": 31215, "Ġtenants": 31216, "éĢŁ": 31217, "ĠPTS": 31218, "Ġdevi": 31219, "ĠLate": 31220, "uez": 31221, "Ġsöyl": 31222, "ãĤ»": 31223, "Ġìŀ¬ë°Į": 31224, "Ġtoggle": 31225, "Ġmasking": 31226, "алÑĮного": 31227, "Ġpersön": 31228, "Ġamerican": 31229, "fik": 31230, "ĠRGB": 31231, "enson": 31232, "ĠKA": 31233, "wwww": 31234, "ĠÑĢег": 31235, "metics": 31236, "Ġeducator": 31237, "ãĤ·ãĥ«ãĤ¯": 31238, "park": 31239, "елÑĮзÑı": 31240, "arus": 31241, "ÑĢеÑĤ": 31242, "Ġfeito": 31243, "Ġchoir": 31244, "Ġlargo": 31245, "Ġeens": 31246, "Ġwatts": 31247, "ĠSingle": 31248, "Ġsusceptible": 31249, "icer": 31250, "ĠвклÑİÑĩ": 31251, "Ġpus": 31252, "íĻĺ": 31253, "Eng": 31254, "Ġfantas": 31255, "Ġspecification": 31256, "Ġconfronted": 31257, "ĠColumbus": 31258, "ивеÑĤ": 31259, "arım": 31260, "Ġcaffeine": 31261, "munition": 31262, "Ġmigrants": 31263, "lide": 31264, "itations": 31265, "ĠGeme": 31266, "ẫ": 31267, "Ġplanner": 31268, "Ġstimulate": 31269, "Ġaproxim": 31270, "ceu": 31271, "ĠNom": 31272, "Ġvog": 31273, "ĠÑĢаÑģÑĤ": 31274, "Ġenseñ": 31275, "Ġsellers": 31276, "Ġguten": 31277, "zd": 31278, "Cal": 31279, "Ġdescript": 31280, "Ġreconciliation": 31281, "zinho": 31282, "á¹ĩa": 31283, "ãģĺãĤĥãģĤ": 31284, "acyj": 31285, "ĠCOL": 31286, "saw": 31287, "ĠíĻķìĿ¸": 31288, "Ġvarit": 31289, "Ġpartnering": 31290, "Ġdetention": 31291, "Ġbombing": 31292, "clapping": 31293, "iencies": 31294, "ondu": 31295, "AME": 31296, "Ġê°ĻìĬµëĭĪëĭ¤": 31297, "cÃŃa": 31298, "ĠпоÑģÑĤо": 31299, "ĠASMR": 31300, "Ġhomepage": 31301, "Ġsiè": 31302, "antha": 31303, "ĠPoll": 31304, "Ġigen": 31305, "cych": 31306, "Ġê°ijìŀIJ기": 31307, "Ġconsiderably": 31308, "ä»ĸçļĦ": 31309, "ĠArist": 31310, "Ġwithstand": 31311, "Ġqualitative": 31312, "ĠKraft": 31313, "ĠÑįлекÑĤ": 31314, "ĠBead": 31315, "екÑĤив": 31316, "Ġcrushing": 31317, "ì³IJ": 31318, "Ġnavy": 31319, "ÙĪÚº": 31320, "sho": 31321, "Ġoak": 31322, "ippers": 31323, "Ġsoils": 31324, "Ġpigment": 31325, "Ġevitar": 31326, "ãĥĩ": 31327, "Ġfuse": 31328, "ĠDale": 31329, ":\"": 31330, "Ġcomplètement": 31331, "Ġkel": 31332, "à¹Ĩ": 31333, "Ġquatre": 31334, "ĠUM": 31335, "Ġë§IJë": 31336, "æł¹": 31337, "ÃŃr": 31338, "Ġleisure": 31339, "ĠHousing": 31340, "Ġfolds": 31341, "estion": 31342, "ARS": 31343, "Ġmash": 31344, "urpose": 31345, "Ġaccumulated": 31346, "ĠStuff": 31347, "èªŀ": 31348, "Ġtapes": 31349, "ĠÑģилÑĮно": 31350, "ĠLOVE": 31351, "Ġ1982": 31352, "Ġscars": 31353, "Ġcapitalist": 31354, "ĠNed": 31355, "Ġsoften": 31356, "Ġnotably": 31357, "Ġforcément": 31358, "ĠRaum": 31359, "ĠнеобÑħод": 31360, "Ġtrademark": 31361, "Ġfertig": 31362, "Ġ?!": 31363, "æĹł": 31364, "Ġreinforced": 31365, "Ġrecharge": 31366, "ĠPutting": 31367, "Ġvillains": 31368, "Ġhandic": 31369, "Ġadvertisement": 31370, "تÙĬ": 31371, "ĠÑģÑĥм": 31372, "ĠRiley": 31373, "×ķ×ij×": 31374, "京": 31375, "Os": 31376, "از": 31377, "Boy": 31378, "Ġsquish": 31379, "ocket": 31380, "Ġtestify": 31381, "æ¼Ķ": 31382, "Ġ׾×ŀ×": 31383, "ĠмаÑģÑģ": 31384, "manuel": 31385, "ĠArkansas": 31386, "iffe": 31387, "Ġanalysts": 31388, "ĠDeaf": 31389, "Ġjó": 31390, "Ġgroceries": 31391, "ĠWheel": 31392, "ĠÑĢиÑģ": 31393, "Ġcòn": 31394, "ĠCob": 31395, "Ġprisons": 31396, "ève": 31397, "ĠCabinet": 31398, "Ġposed": 31399, "Ġguerre": 31400, "ĠLloyd": 31401, "Ġclerk": 31402, "Ġcrises": 31403, "ĠSho": 31404, "ĠOre": 31405, "ĠFootball": 31406, "ĠAdvis": 31407, "ĠZheng": 31408, "èį": 31409, "ĠAMY": 31410, "Ġunfor": 31411, "Ġmonaster": 31412, "Ġcompile": 31413, "Ġimmortal": 31414, "atable": 31415, "Ġparano": 31416, "Ġtiver": 31417, "ĠSteph": 31418, "ĠFuÃŁ": 31419, "Ġdiscontin": 31420, "Ġripe": 31421, "Ġhacking": 31422, "Ġsiendo": 31423, "Ġseguro": 31424, "altres": 31425, "Ġanderes": 31426, "Ġ리ë": 31427, "Ġexports": 31428, "æŃ¥": 31429, "Ġtabii": 31430, "Ġ기ëĭ¤ë": 31431, "Ġbothering": 31432, "Ġpickle": 31433, "ĠBRIAN": 31434, "Ġaltar": 31435, "ĠпÑĢиб": 31436, "Ġtransferring": 31437, "ĠVors": 31438, "ĠÙĩÙĪ": 31439, "ĠZa": 31440, "ĠFrances": 31441, "Ġbrowse": 31442, "emit": 31443, "Ġchewing": 31444, "ĠFreddy": 31445, "Ġeditors": 31446, "älle": 31447, "ĠíĮĢ": 31448, "ĠSque": 31449, "ĠCultural": 31450, "awk": 31451, "ĠSache": 31452, "ĠCarbon": 31453, "ắt": 31454, "FL": 31455, "ĠNGO": 31456, "peÅĤ": 31457, "ĠSou": 31458, "Ġhvor": 31459, "unintelligible": 31460, "Ġë²ķ": 31461, "Ġ°": 31462, "iin": 31463, "Ġ×¢×Ŀ": 31464, "Ġderrière": 31465, "Ġczym": 31466, "ĠApost": 31467, "Ġregarder": 31468, "Ġagrade": 31469, "ĠCandy": 31470, "Ġmare": 31471, "Ġintroduces": 31472, "birds": 31473, "Ġuniquely": 31474, "Ġmuk": 31475, "Ġcooker": 31476, "Ġcrews": 31477, "Ġjeito": 31478, "ERT": 31479, "¶Ħë": 31480, "nisse": 31481, "Ġef": 31482, "Ġcarte": 31483, "ĠYak": 31484, "ĠPAT": 31485, "ино": 31486, "bokki": 31487, "Ġmates": 31488, "Ġdistint": 31489, "Ġì½Ķë¡ľëĤĺ": 31490, "Ġyıl": 31491, "Ġκάν": 31492, "Ġconfigurations": 31493, "enga": 31494, "recht": 31495, "Happy": 31496, "ãĤĦãģ£ãģ¦": 31497, "invest": 31498, "Ġreconstruct": 31499, "ĠÑįÑĤомÑĥ": 31500, "Ġmosque": 31501, "raum": 31502, "Ġvoyez": 31503, "ĠNBC": 31504, "ĠìŀIJìĭł": 31505, "Ġsturdy": 31506, "Ġкап": 31507, "Ġansch": 31508, "alid": 31509, "Ġmasih": 31510, "ĠREP": 31511, "Ġì½Ķë": 31512, "Ġdeduct": 31513, "Ġsalir": 31514, "wurf": 31515, "ilot": 31516, "ĠMutter": 31517, "olds": 31518, "ĠFEMA": 31519, "ĠBib": 31520, "Ġneighboring": 31521, "Ġbliss": 31522, "Ġíĺ¼": 31523, "лиÑģÑĮ": 31524, "ĠÑĤÑĢеб": 31525, "Ġå°±æĺ¯": 31526, "Ġgrenade": 31527, "Ġegal": 31528, "Ġfinely": 31529, "Ġpetals": 31530, "Ġkeer": 31531, "Ġchyba": 31532, "Ġskipping": 31533, "Ġthirteen": 31534, "Ġgravy": 31535, "ĠSAT": 31536, "61": 31537, "Ġног": 31538, "Ġmins": 31539, "ITE": 31540, "Ġsozial": 31541, "íķĺë©´ìĦľ": 31542, "ruktur": 31543, "Ġвозмож": 31544, "ĠопÑıÑĤÑĮ": 31545, "Ġarth": 31546, "ĠCuban": 31547, "Ġtreasures": 31548, "Ġfertilizer": 31549, "Ġawakening": 31550, "Ġë°±ìĭł": 31551, "Ġrall": 31552, "Ġdepict": 31553, "ĠPablo": 31554, "Ġnineteen": 31555, "Ġwatt": 31556, "Ġentirety": 31557, "KS": 31558, "ĠWoods": 31559, "Sch": 31560, "ĠÚ©ÙĪ": 31561, "ĠDry": 31562, "ãģŀ": 31563, "uve": 31564, "Ġreconstruction": 31565, "Ġanatomy": 31566, "Ī를": 31567, "Ġbaba": 31568, "Ġlistener": 31569, "Ġsharpen": 31570, "ĠPeru": 31571, "ĠвÑĭз": 31572, "Ġrecreation": 31573, "Ġinitiate": 31574, "Ġcalor": 31575, "ĠNaj": 31576, "gee": 31577, "ĠFeels": 31578, "ĠSnapchat": 31579, "ĠTet": 31580, "ĠNest": 31581, "ĠDaf": 31582, "ĠFinish": 31583, "ĠÑĤаким": 31584, "úc": 31585, "izens": 31586, "Ġspins": 31587, "Ġembry": 31588, "Ġpassages": 31589, "Ġcient": 31590, "Ġjustification": 31591, "ä»ĸ說": 31592, "Ġolmaz": 31593, "Ġflooded": 31594, "Ġemoji": 31595, "Ġembracing": 31596, "Ġdiscard": 31597, "ĠBasic": 31598, "agog": 31599, "ĠìľĦíķ´": 31600, "Ġasylum": 31601, "erin": 31602, "Ġfim": 31603, "Ġninja": 31604, "Ġautomate": 31605, "Ġallergic": 31606, "ÿÿÿÿ": 31607, "amam": 31608, "ĠмаÑĢ": 31609, "ĠOi": 31610, "äus": 31611, "Ġinduct": 31612, "ĠBEN": 31613, "ĠzÅĤ": 31614, "Ġkażdy": 31615, "ĠAMP": 31616, "nÄĽ": 31617, "Sure": 31618, "Ġquil": 31619, "Ġespec": 31620, "rok": 31621, "BSCRI": 31622, "Ġliebe": 31623, "pus": 31624, "achsen": 31625, "Ġcricket": 31626, "ëĬIJ": 31627, "ĠFrame": 31628, "ekkür": 31629, "arb": 31630, "ĠpÅĻ": 31631, "иÑģÑģ": 31632, "Ġzeggen": 31633, "Ġdoubles": 31634, "ĠDre": 31635, "test": 31636, "insp": 31637, "boys": 31638, "Ġmão": 31639, "ĠVerse": 31640, "Ġmuscular": 31641, "ĠMALE": 31642, "Ġdulu": 31643, "Ġoccasional": 31644, "Lo": 31645, "conomic": 31646, "Ġvak": 31647, "Ġremedy": 31648, "å¤ł": 31649, "ĠâĻªâĻªâĻª": 31650, "vem": 31651, "Ġönem": 31652, "ĠkarÅŁÄ±": 31653, "ĠSharp": 31654, "hur": 31655, "Ġë°©ë²ķ": 31656, "Ġgrandson": 31657, "Ġaktiv": 31658, "ĠThrones": 31659, "ĠìķĪìĹIJ": 31660, "Ġtots": 31661, "Ġsubd": 31662, "ĠPaula": 31663, "Ġgraves": 31664, "ĠBrent": 31665, "ĠникÑĤо": 31666, "Ġsöz": 31667, "Ġcrec": 31668, "ĠVladimir": 31669, "çĸ«": 31670, "Ġпой": 31671, "Ġ\"-": 31672, "Ġpsy": 31673, "atri": 31674, "idan": 31675, "Ġaún": 31676, "Ġstandardized": 31677, "ì¹ĺë": 31678, "ĠкÑĢов": 31679, "ĠZhu": 31680, "something": 31681, "Ġ750": 31682, "Ġmujeres": 31683, "Ġait": 31684, "éĹ´": 31685, "agu": 31686, "Ġcorrected": 31687, "ikka": 31688, "eled": 31689, "ĠCareer": 31690, "owym": 31691, "Ġroommate": 31692, "Ġdescendants": 31693, "ĠNapoleon": 31694, "ĠÐĶо": 31695, "íĸĪìĸ´ìļĶ": 31696, "Ġbunun": 31697, "ĠMicha": 31698, "ç·ļ": 31699, "Ġdescob": 31700, "PI": 31701, "Ġpalabra": 31702, "Ġtracked": 31703, "Ġdependence": 31704, "ĠBarack": 31705, "åģĩ": 31706, "Ġfertility": 31707, "ĠSouthwest": 31708, "Ġincomplete": 31709, "Ġcomunic": 31710, "Ġcompris": 31711, "ĠRestaur": 31712, "Ġacron": 31713, "κα": 31714, "Ġapprentices": 31715, "Ġmusst": 31716, "ĠAbr": 31717, "Ġpentru": 31718, "ĠConsort": 31719, "ĠAvec": 31720, "Ġdumplings": 31721, "LR": 31722, "Ġwszystkie": 31723, "Ġswamp": 31724, "нев": 31725, "uggle": 31726, "Ġwatercolor": 31727, "Ġproton": 31728, "ĠEspaña": 31729, "ocking": 31730, "овал": 31731, "Ġtakim": 31732, "Very": 31733, "Ġdementia": 31734, "ĠÅŁeyi": 31735, "Jac": 31736, "ĠMacBook": 31737, "ĠLiv": 31738, "fficients": 31739, "ĠHunt": 31740, "Ġoverlay": 31741, "æĦŁè¦º": 31742, "ĠSkype": 31743, "punkt": 31744, "Ġconfined": 31745, "ĠAdrian": 31746, "رÙĥ": 31747, "ĠJeep": 31748, "Ġenquanto": 31749, "Ġanest": 31750, "оÑĤвеÑĤ": 31751, "ĠменÑĮ": 31752, "Ġirrigation": 31753, "á»ijn": 31754, "Ġeighteen": 31755, "ĠPon": 31756, "Ġrescued": 31757, "Ġ1983": 31758, "rü": 31759, "jae": 31760, "ĠJeong": 31761, "Ġamazingly": 31762, "ĠFDP": 31763, "Ġbackstage": 31764, "cue": 31765, "ĠÏĥÏĦην": 31766, "ĠاÙĦص": 31767, "Ġlivestock": 31768, "ĠWarner": 31769, "Ġmajors": 31770, "ãĥģãĥ£": 31771, "Ġcooperative": 31772, "ĠBrady": 31773, "rained": 31774, "rieb": 31775, "Ġ×ij×ŀ×": 31776, "ĠдоволÑĮно": 31777, "ĠFE": 31778, "Ġleaked": 31779, "ĠMercury": 31780, "Ġpersuade": 31781, "Ġtransformer": 31782, "ĠNorweg": 31783, "ĠìŬ룬": 31784, "ĠzrobiÄĩ": 31785, "Ġcardiovascular": 31786, "ĠCrash": 31787, "Ġgossip": 31788, "аÑģÑĤÑĮ": 31789, "Ġ쪽": 31790, "Ġswept": 31791, "ĠHorn": 31792, "ĠAté": 31793, "Ġbukan": 31794, "ĠKaw": 31795, "KY": 31796, "ĠStories": 31797, "Gary": 31798, "Ġgardening": 31799, "ĠQuickly": 31800, "ĠFalcon": 31801, "Ġovat": 31802, "cı": 31803, "ĠComplet": 31804, "ĠDate": 31805, "ĠпÑĢим": 31806, "Ġläuft": 31807, "ĠAudrey": 31808, "ĠWent": 31809, "ĠpelÃŃcul": 31810, "Ġcarriage": 31811, "Ġunacceptable": 31812, "nymi": 31813, "ĠÑģлÑĭÑĪ": 31814, "Ġterre": 31815, "uellement": 31816, "EEEE": 31817, "Ġpharmac": 31818, "hões": 31819, "Ġzich": 31820, "Ġmigrate": 31821, "ĠFry": 31822, "ñana": 31823, "ĠMuito": 31824, "EOVER": 31825, "Ġfortress": 31826, "ĠCompan": 31827, "ĠJSON": 31828, "ordnung": 31829, "Ġwarto": 31830, "Ġungef": 31831, "ìħĶìĦľ": 31832, "ĠÑĢок": 31833, "Ġpaddle": 31834, "Jared": 31835, "Ġsubmitting": 31836, "Ġlatch": 31837, "Ġfug": 31838, "ĠкоÑģ": 31839, "ĠEf": 31840, "Ġlaunches": 31841, "Ġft": 31842, "otechn": 31843, "Ġtravelled": 31844, "اÙģ": 31845, "éģķ": 31846, "Ġproch": 31847, "Ġdedim": 31848, "83": 31849, "Ġrebound": 31850, "ĠLU": 31851, "path": 31852, "ĠÑģпÑĢав": 31853, "Ġöl": 31854, "ĠíĤ¤": 31855, "Ġprivat": 31856, "Ġtractor": 31857, "ĠAttention": 31858, "Ser": 31859, "Ġcoses": 31860, "ária": 31861, "pal": 31862, "ĠìĿĢ": 31863, "Ġsuccessor": 31864, "Ġconnectors": 31865, "ĠÑĥÑģÑĤанов": 31866, "Ġgenocide": 31867, "Ġsufficiently": 31868, "ĠAixò": 31869, "Ġstabilize": 31870, "Ġcongest": 31871, "Ġcarving": 31872, "Ġzost": 31873, "ĠбÑĭÑģÑĤÑĢо": 31874, "Ġshortest": 31875, "Ġlivel": 31876, "Ġ89": 31877, "éģĬ": 31878, "Ġerk": 31879, "Ġportraits": 31880, "à¥Ģ": 31881, "èĺ": 31882, "boat": 31883, "llah": 31884, "ANC": 31885, "Ġempirical": 31886, "ĠEcho": 31887, "ĠNederland": 31888, "è¿Ļä¹Ī": 31889, "Net": 31890, "Ġcuidado": 31891, "ĠRoma": 31892, "Ġcalf": 31893, "Ġgiants": 31894, "ĠExplorer": 31895, "ĠCollect": 31896, "alition": 31897, "ĠDestiny": 31898, "Ġausge": 31899, "ĠEdu": 31900, "ĠClo": 31901, "Ġearrings": 31902, "ĠTrack": 31903, "ĠROS": 31904, "ĠBelle": 31905, "çĻ¾": 31906, "Ġpueda": 31907, "Ġdaytime": 31908, "Ġsupplier": 31909, "ĠSV": 31910, "ĠExhale": 31911, "Ġgalera": 31912, "course": 31913, "Ġcentimeter": 31914, "ĠBast": 31915, "mud": 31916, "Ġsangat": 31917, "ĠPhysical": 31918, "Ġprivately": 31919, "Ġtrata": 31920, "lynn": 31921, "illi": 31922, "Ġë©ĶìĿ´íģ¬ìĹħ": 31923, "Ġcrystall": 31924, "Ġpods": 31925, "ản": 31926, "inator": 31927, "ĠRecords": 31928, "å®ĺ": 31929, "ÄŁimiz": 31930, "issement": 31931, "hare": 31932, "hadow": 31933, "ĠDK": 31934, "ĠìķĮê³ł": 31935, "Ġwyn": 31936, "Ġrequesting": 31937, "ĠDonna": 31938, "ĠìĹ´ìĭ¬íŀĪ": 31939, "inea": 31940, "Ġexert": 31941, "ĠDuncan": 31942, "ĠвеÑĩ": 31943, "ĠHah": 31944, "à¤Ĥ": 31945, "ĠLif": 31946, "ĠFinding": 31947, "ĠNov": 31948, "Ġзнак": 31949, "ĠоÑĦ": 31950, "ĠQuè": 31951, "Ġquarterback": 31952, "ĠÑĦак": 31953, "Ġbipartisan": 31954, "ÄŁin": 31955, "Ġnécess": 31956, "Ġreferendum": 31957, "Ġcompiler": 31958, "Ġprobabil": 31959, "еди": 31960, "Ġtrader": 31961, "æĺĵ": 31962, "ĠRum": 31963, "geme": 31964, "Ġdio": 31965, "ĠbÄĻdziemy": 31966, "ĠÏĢά": 31967, "꾸": 31968, "×ķ×ĺ": 31969, "Ġà¤ķ": 31970, "Ġблаг": 31971, "Ġscalp": 31972, "ĠPause": 31973, "Ġcaption": 31974, "Ġendanger": 31975, "Ġenlar": 31976, "Ġrotten": 31977, "ãĥĥãĥĪ": 31978, "Ġwah": 31979, "èĤī": 31980, "Ġdzi": 31981, "ĠInstall": 31982, "Ay": 31983, "Ġcrear": 31984, "енÑĤа": 31985, "Ġweighing": 31986, "Ġbutterflies": 31987, "ĠGast": 31988, "äºķ": 31989, "horn": 31990, "warz": 31991, "ICEOVER": 31992, "ĠнайÑĤи": 31993, "Ġcoefficients": 31994, "ç°¡åĸ®": 31995, "ĠSpencer": 31996, "ĠHigher": 31997, "Ġcowork": 31998, "å¨ĺ": 31999, "ĠкоÑĤоÑĢое": 32000, "Ġmonit": 32001, "Ġdysfunction": 32002, "ĠÑģÑĤанов": 32003, "Ġtournaments": 32004, "Ġoyster": 32005, "BN": 32006, "Ġtrud": 32007, "slow": 32008, "ĠPenny": 32009, "ĠOdys": 32010, "ær": 32011, "Ġfou": 32012, "Ġenjoyment": 32013, "аÑĤÑĭ": 32014, "ĠwyglÄħda": 32015, "алÑĮнаÑı": 32016, "ĠProtect": 32017, "Ġmoy": 32018, "Ġclaw": 32019, "Ġsuspicion": 32020, "Ġsacrificed": 32021, "Ġgosto": 32022, "Big": 32023, "Ġaggressively": 32024, "Ġvorne": 32025, "ãĥł": 32026, "Ġblamed": 32027, "ĠSehr": 32028, "פר": 32029, "cito": 32030, "Ġseals": 32031, "Ġmujer": 32032, "ĠWeird": 32033, "Ġforens": 32034, "Ġcontributes": 32035, "estra": 32036, "Ġpog": 32037, "LOL": 32038, "Ġhacerlo": 32039, "оÑĤÑĮ": 32040, "fiction": 32041, "79": 32042, "λο": 32043, "大æ¦Ĥ": 32044, "声": 32045, "ĠÑĤоб": 32046, "ĠGS": 32047, "ĠClara": 32048, "itez": 32049, "Ġadvocating": 32050, "ĠíĶĦë": 32051, "sung": 32052, "Ġvertices": 32053, "Ġnavigating": 32054, "Ġeuropé": 32055, "çļĨ": 32056, "Ġslowed": 32057, "Ġforeground": 32058, "ĠIndustrial": 32059, "Ġadore": 32060, "ìĭŃ": 32061, "Ġcréer": 32062, "æŀĹ": 32063, "chnitt": 32064, "Ġunaware": 32065, "Ġcurly": 32066, "entar": 32067, "Ġler": 32068, "Ġprohibited": 32069, "ĠHeroes": 32070, "ĠReed": 32071, "uca": 32072, "Ġsmok": 32073, "Ġkunna": 32074, "zeitig": 32075, "immen": 32076, "ĠLun": 32077, "ĠабÑģолÑİÑĤ": 32078, "Ġdegli": 32079, "Ġvillagers": 32080, "Ġpreset": 32081, "zept": 32082, "uds": 32083, "Ġemit": 32084, "ä½łè¦ģ": 32085, "Ġëī": 32086, "ëĬĶì§Ģ": 32087, "нако": 32088, "Ġosób": 32089, "Ġ1969": 32090, "ĠÐIJÑĢ": 32091, "Ġmanchmal": 32092, "ĠBrock": 32093, "Ġmantra": 32094, "ĠWIL": 32095, "bach": 32096, "inä": 32097, "elas": 32098, "keln": 32099, "Ġdisciple": 32100, "Ġqualc": 32101, "Ġdehyd": 32102, "ìĿ´ëĿ¼ëĬĶ": 32103, "Af": 32104, "ìĦ±ìĿ´": 32105, "Ryan": 32106, "Ġpuppet": 32107, "ĠдÑĢÑĥгие": 32108, "Ġrud": 32109, "Ġpending": 32110, "Plus": 32111, "ĠìķĬìĿĦ": 32112, "Ġbá»ĭ": 32113, "ĠSega": 32114, "çe": 32115, "Ġprogrammer": 32116, "bli": 32117, "Ġunl": 32118, "Ġenslaved": 32119, "Ġsociété": 32120, "Äģh": 32121, "Ġinheritance": 32122, "ĠBangl": 32123, "ermaid": 32124, "Ġpractitioner": 32125, "ĠStalin": 32126, "ĠUser": 32127, "cible": 32128, "Ġcardiac": 32129, "ĠKoreans": 32130, "Ġdumped": 32131, "Ġ×Ķ×Ļ×Ķ": 32132, "áis": 32133, "Ġhydraulic": 32134, "oubtedly": 32135, "ĠPit": 32136, "Ġpicnic": 32137, "Ġbehöver": 32138, "ĠÑģмог": 32139, "Ġbraking": 32140, "é»ij": 32141, "utar": 32142, "ĠìĦ¸ë": 32143, "ubl": 32144, "Ġüz": 32145, "Ġmajesty": 32146, "Ġbers": 32147, "utable": 32148, "Ġhotter": 32149, "çħ§": 32150, "ÛĮÙĨ": 32151, "Ġbiases": 32152, "Ġsubjected": 32153, "Ġnaughty": 32154, "Ġcircus": 32155, "ãģĹãģĭ": 32156, "ĠImmedi": 32157, "ĠStefan": 32158, "ĠTriple": 32159, "enk": 32160, "Ġwit": 32161, "Ġrecycle": 32162, "emie": 32163, "dated": 32164, "Ġunload": 32165, "Ġpopula": 32166, "chin": 32167, "Ġyields": 32168, "Ġenglish": 32169, "ĠBonnie": 32170, "Ġspiders": 32171, "Ãģ": 32172, "Ġerosion": 32173, "éĥ¨åĪĨ": 32174, "ĠNICK": 32175, "иÑıÑħ": 32176, "Ġimpart": 32177, "Ġкни": 32178, "Ġresolutions": 32179, "Ġlithium": 32180, "Ġconvergence": 32181, "ĠTara": 32182, "Ġдве": 32183, "ths": 32184, "ĠCindy": 32185, "æĪijè¦ģ": 32186, "幫": 32187, "ĠDIE": 32188, "Ġassurance": 32189, "ĠопиÑģ": 32190, "Ġbuckets": 32191, "Ġcues": 32192, "ĠQuiet": 32193, "Ġsimilarity": 32194, "Ġfoundational": 32195, "ĠMinist": 32196, "滿": 32197, "Ġpian": 32198, "Ġcentr": 32199, "Ġnumb": 32200, "Ġmonks": 32201, "ujourd": 32202, "enzie": 32203, "Ġskateboard": 32204, "Ġdlatego": 32205, "ĠÑģоÑĤ": 32206, "ĠAE": 32207, "Ġmasterpiece": 32208, "ĠSolomon": 32209, "ĠReddit": 32210, "Ġriot": 32211, "abl": 32212, "ĠJazz": 32213, "Ġelectromagnetic": 32214, "Ġinsecure": 32215, "ĠCompet": 32216, "geries": 32217, "обод": 32218, "ł×ķ": 32219, "ðŁĴ": 32220, "Ġsenators": 32221, "ĠBrisbane": 32222, "ĠAlb": 32223, "uttering": 32224, "ĠAllow": 32225, "zero": 32226, "Ġpai": 32227, "ĠÐIJлекÑģ": 32228, "ĠDisplay": 32229, "ĠBlade": 32230, "ĠApps": 32231, "Ġpä": 32232, "ĠдеÑģÑı": 32233, "Ġquella": 32234, "ĠGao": 32235, "еннÑĭÑħ": 32236, "Ġspoilers": 32237, "Ġgallons": 32238, "ĠÙĦÙĬ": 32239, "ĠZion": 32240, "æľīä¸Ģ": 32241, "onie": 32242, "ragt": 32243, "ĠChand": 32244, "Ġë³ij": 32245, "Ġblunt": 32246, "Ġusu": 32247, "ĠKad": 32248, "rakt": 32249, "Ġcinematic": 32250, "Ġammunition": 32251, "rene": 32252, "Ġfourteen": 32253, "ĠCarn": 32254, "crit": 32255, "Ġtenure": 32256, "vu": 32257, "Ġprincipalmente": 32258, "Ġalleen": 32259, "éĢĻä¸Ģ": 32260, "Ġkomplett": 32261, "Ġdüny": 32262, "James": 32263, "Ġreceptor": 32264, "Ġoneself": 32265, "guru": 32266, "Ġmerchant": 32267, "liness": 32268, "Ġoverlooked": 32269, "Ġharmonic": 32270, "éķ¿": 32271, "ieso": 32272, "×ķ×ŀ": 32273, "colm": 32274, "ĠпÑĢоекÑĤ": 32275, "ĠAda": 32276, "اس": 32277, "Tim": 32278, "Ġrecurring": 32279, "Ġproceeds": 32280, "ĠParticularly": 32281, "ĠDownload": 32282, "etrical": 32283, "Ġmatrices": 32284, "Ġproyecto": 32285, "ancies": 32286, "ĠUhm": 32287, "Ġcaves": 32288, "Ġìĸ´ëł¤": 32289, "ĠLeaf": 32290, "ĠобÑĭÑĩ": 32291, "ĠìĿ´ìľł": 32292, "Europe": 32293, "ĠtÄħ": 32294, "Ġpuls": 32295, "Ġtakiego": 32296, "ÐĿе": 32297, "GU": 32298, "Ġfors": 32299, "Ïģγ": 32300, "Ġfotos": 32301, "Ġ))": 32302, "Ġ멤ë": 32303, "Ġaquilo": 32304, "ĠKurd": 32305, "ï¸ı": 32306, "ptic": 32307, "ĠDort": 32308, "Ġmisery": 32309, "auso": 32310, "åĬŁ": 32311, "chuckling": 32312, "ĠRidge": 32313, "ĠíĸĪìĬµëĭĪëĭ¤": 32314, "Ġ***": 32315, "客": 32316, "ĠHmmm": 32317, "Ġgeographic": 32318, "Ġanys": 32319, "Ġtalvez": 32320, "Ġskelet": 32321, "Ġsignatures": 32322, "Ġliters": 32323, "IJë©´": 32324, "ĠÑģвоего": 32325, "Ġskiing": 32326, "ĠÐľÐ¾Ñģ": 32327, "Ġadopting": 32328, "Ġhaft": 32329, "Ġsymmetric": 32330, "ĠLiqu": 32331, "Ġthyroid": 32332, "Ġmisin": 32333, "lude": 32334, "Ġhull": 32335, "ĠXD": 32336, "ĠGust": 32337, "zeich": 32338, "Ġvibrations": 32339, "Ġesemp": 32340, "ĠвÑģÑİ": 32341, "ĠQuem": 32342, "Ġübrig": 32343, "ĠSke": 32344, "ĠLynch": 32345, "rooms": 32346, "artet": 32347, "fest": 32348, "Ġfrüher": 32349, "Ġlure": 32350, "ä¸į好æĦıæĢĿ": 32351, "ĠìķĮìķĦ": 32352, "ĠWIN": 32353, "ĠRYAN": 32354, "ĠкоÑĤоÑĢÑĥÑİ": 32355, "ĠKash": 32356, "Ġ×Ķ×ŀ": 32357, "Ġsafeg": 32358, "ĠHallelujah": 32359, "ĠдвÑĥÑħ": 32360, "Ġstaple": 32361, "Ġsediment": 32362, "ĠActs": 32363, "Ġblaming": 32364, "Ġmainland": 32365, "Ġsporting": 32366, "Ġdecorations": 32367, "Ġexecuting": 32368, "Ġparan": 32369, "ĠDollar": 32370, "Ġprojections": 32371, "Ġcommissioned": 32372, "Ġbour": 32373, "öm": 32374, "Ġsteamed": 32375, "ĠëŃĺ": 32376, "Ġpetrol": 32377, "Ġcelular": 32378, "帶": 32379, "ĠHungary": 32380, "Ġrented": 32381, "ĠваÑĢи": 32382, "bbie": 32383, "Ġsécur": 32384, "üll": 32385, "Ġswings": 32386, "between": 32387, "ĠиÑĤ": 32388, "estro": 32389, "Ġniemand": 32390, "ĠìĤ¼": 32391, "ĠPardon": 32392, "esses": 32393, "ĠMID": 32394, "Ġcentralized": 32395, "ĠAlien": 32396, "culos": 32397, "Ġcrise": 32398, "裡éĿ¢": 32399, "Ġclasse": 32400, "beitet": 32401, "iÄŁi": 32402, "Ġwhales": 32403, "Ġperimeter": 32404, "Ġtying": 32405, "Ġstrony": 32406, "Ġlikewise": 32407, "ĠPunch": 32408, "Da": 32409, "ĠBaptist": 32410, "Ġsorting": 32411, "Ġiv": 32412, "Ġíķ©": 32413, "Ġrehab": 32414, "Ġeta": 32415, "river": 32416, "Ġsai": 32417, "ãģĦãģŁãģł": 32418, "odus": 32419, "ãģĬé¡ĺãģĦãģĹãģ¾ãģĻ": 32420, "Ġessayer": 32421, "Ġturtles": 32422, "ĠHazrat": 32423, "Ġfabrics": 32424, "Ġcavity": 32425, "Ġponieważ": 32426, "Ġschlecht": 32427, "Ġsalsa": 32428, "ÅŁekkür": 32429, "Ġseating": 32430, "Ġeconomists": 32431, "Ġmang": 32432, "Ġseguinte": 32433, "Ġrang": 32434, "Ġratios": 32435, "Ġconstell": 32436, "Ġlongtemps": 32437, "uating": 32438, "Ġspoiled": 32439, "Ġrecipients": 32440, "Ġsniper": 32441, "ä¹ĭåīį": 32442, "ìĬµëĭĪê¹Į": 32443, "Ġwp": 32444, "ĠLINKE": 32445, "Ġflare": 32446, "ĠAdri": 32447, "ñas": 32448, "Ġbackl": 32449, "mÃ¤ÃŁ": 32450, "ĠBend": 32451, "Ġworkloads": 32452, "ĠÑģÑĥп": 32453, "Ġ1975": 32454, "имÑģÑı": 32455, "ане": 32456, "Ġмон": 32457, "Ġaspirations": 32458, "ĠAer": 32459, "ĠговоÑĢиÑĤÑĮ": 32460, "ĠQian": 32461, "å¦Ī": 32462, "Ġcompromised": 32463, "Ġyolk": 32464, "лаÑģÑĤ": 32465, "Ġhemen": 32466, "rove": 32467, "dens": 32468, "ĠкомменÑĤ": 32469, "Ġ---": 32470, "Ġfluores": 32471, "ноÑģ": 32472, "ĠLiverpool": 32473, "ĠÑģобой": 32474, "ĠZwe": 32475, "Ġlumin": 32476, "ĠOG": 32477, "á¸": 32478, "holm": 32479, "profits": 32480, "SN": 32481, "Ġproportions": 32482, "Ġmica": 32483, "ĠBoh": 32484, "ĠAtlas": 32485, "Ġunsure": 32486, "Ġtouring": 32487, "Ġnied": 32488, "ĠtÄĻ": 32489, "Ġimperative": 32490, "Ġdemek": 32491, "ĠSheriff": 32492, "rance": 32493, "Ġhomeland": 32494, "ĠHail": 32495, "ĠGanz": 32496, "ymm": 32497, "Mon": 32498, "åĨ·": 32499, "vida": 32500, "Ġdesarroll": 32501, "æĬĢ": 32502, "Ġintriguing": 32503, "ĠHugo": 32504, "ĠãĤĤ": 32505, "é¬": 32506, "аÑĨ": 32507, "ĠWiÄĻc": 32508, "atted": 32509, "ĠìķĦëĭĪê³ł": 32510, "ĠVari": 32511, "ád": 32512, "Ġsurreal": 32513, "Ġdisparities": 32514, "Ġmó": 32515, "ullen": 32516, "ĠìŀĪëĭ¤ê³ł": 32517, "ĠпожалÑĥйÑģÑĤа": 32518, "Ġmains": 32519, "Ġeject": 32520, "Ġmethane": 32521, "Ġmarginalized": 32522, "Ġchilli": 32523, "rès": 32524, "Ġyem": 32525, "ä½łæĺ¯": 32526, "ĠChun": 32527, "Ġdebts": 32528, "Ġdownloading": 32529, "ĠAthens": 32530, "isierung": 32531, "ryn": 32532, "Ġtekn": 32533, "ĠQuindi": 32534, "éľĢ": 32535, "Ġtaraf": 32536, "Ġhé": 32537, "Ġconsciously": 32538, "Ġfixes": 32539, "uckle": 32540, "mayın": 32541, "Ġfrei": 32542, "Ġspa": 32543, "Ġì§Ħíĸī": 32544, "ĠاÙĦØ°": 32545, "ĠÑĥк": 32546, "lett": 32547, "ĠolmuÅŁ": 32548, "Ġcheesy": 32549, "าà¸ģ": 32550, "naire": 32551, "Ġwiden": 32552, "Ġlien": 32553, "Ġescaping": 32554, "iggs": 32555, "ĠBlick": 32556, "cÄħ": 32557, "ĠìĦľë": 32558, "Ġ×Ķס": 32559, "ĠвпеÑĢ": 32560, "ophone": 32561, "iell": 32562, "ĠSUBSCRI": 32563, "Ġlions": 32564, "Ġê·¸ê²ĥ": 32565, "Ġinspires": 32566, "Ġguarantees": 32567, "Ġcomeça": 32568, "ĠGrowing": 32569, "Ġneglig": 32570, "ĠFrankf": 32571, "Ġgegeben": 32572, "ĠÄijầu": 32573, "Ġendlich": 32574, "Ġìį¨": 32575, "ĠTT": 32576, "ĠLith": 32577, "ÏĢα": 32578, "astern": 32579, "ĠAzer": 32580, "Ġlunar": 32581, "hic": 32582, "ĠнаÑĢод": 32583, "Ġnenhum": 32584, "è·ij": 32585, "ĠSalvador": 32586, "ĠProgress": 32587, "Ġprivileges": 32588, "ĠëıĻìķĪ": 32589, "Ġantagon": 32590, "ĠImpf": 32591, "Ġdescub": 32592, "ĠLei": 32593, "ĠìĥĪë¡ľ": 32594, "Ñĩе": 32595, "Ġdólares": 32596, "ĠMeghan": 32597, "ĠWire": 32598, "too": 32599, "aying": 32600, "usc": 32601, "Ġtud": 32602, "Ġappeals": 32603, "educ": 32604, "Ġpane": 32605, "Ġji": 32606, "Ġdecks": 32607, "ĠAlter": 32608, "Ġå°±": 32609, "ìĦ¤": 32610, "åĪĨéIJĺ": 32611, "Ġproductions": 32612, "ĠWILLIAM": 32613, "Ġimplied": 32614, "Ġfulfillment": 32615, "ĠAah": 32616, "Ġsaja": 32617, "xus": 32618, "ĠÎļαι": 32619, "Ãłs": 32620, "ucch": 32621, "око": 32622, "ĠDiscord": 32623, "ĠSY": 32624, "jsk": 32625, "ĠWallace": 32626, "unction": 32627, "Daniel": 32628, "Ġköt": 32629, "ijah": 32630, "Ġmarche": 32631, "Ġdisgr": 32632, "Ġmungkin": 32633, "Ġalma": 32634, "³µ": 32635, "Ġextensively": 32636, "ĠFloren": 32637, "ĠAllison": 32638, "ãĤ±": 32639, "ÙĬÙħ": 32640, "Ġjuven": 32641, "ĠRenaissance": 32642, "Ġfundraising": 32643, "ĠChaos": 32644, "Ġparaly": 32645, "Ġnarrator": 32646, "Ġecosystems": 32647, "Ash": 32648, "Ġmitigation": 32649, "ĠAujourd": 32650, "ĠIdee": 32651, "!,": 32652, "Ġ½": 32653, "Ġlandlord": 32654, "Ġdefects": 32655, "Ġacre": 32656, "ulsive": 32657, "Ġalgae": 32658, "pek": 32659, "Ġemba": 32660, "ĠRoc": 32661, "éĽ¢": 32662, "ksom": 32663, "äche": 32664, "Ġleuk": 32665, "Ġleveraging": 32666, "Ġê·¸ëłĩì§Ģ": 32667, "ĠPalm": 32668, "Ġäven": 32669, "Ġlis": 32670, "ĠInsp": 32671, "ĠRita": 32672, "ĠAbb": 32673, "ithm": 32674, "Ġsupervision": 32675, "Ġrevisit": 32676, "ĠpiÄĻ": 32677, "Ġeuh": 32678, "Ġfades": 32679, "Ġmotto": 32680, "åį¡": 32681, "езж": 32682, "ĠShim": 32683, "Ġrelevance": 32684, "Ġoo": 32685, "Ġostat": 32686, "nica": 32687, "Ġchoix": 32688, "ĠFaculty": 32689, "Ġì¤ijìĹIJ": 32690, "ĠAbove": 32691, "ĠнеболÑĮÑĪ": 32692, "Ġsequencing": 32693, "Ġnutrient": 32694, "Ġconquered": 32695, "Ġdigestive": 32696, "Ġbackdrop": 32697, "ĠLori": 32698, "ailable": 32699, "Game": 32700, "Ġneglected": 32701, "omorph": 32702, "illah": 32703, "Ġkne": 32704, "Ġsiitä": 32705, "Ġworkspace": 32706, "ĠVenice": 32707, "ĠKne": 32708, "Ñīо": 32709, "ħĢ": 32710, "ĠHass": 32711, "Ġvita": 32712, "Ŀ¼ë©´": 32713, "Ġlays": 32714, "ências": 32715, "érica": 32716, "ĠLl": 32717, "æ±Ĥ": 32718, "ĠCoca": 32719, "ĠWHY": 32720, "èĪŀ": 32721, "Ġrouting": 32722, "Ġpermissions": 32723, "Ġdings": 32724, "prend": 32725, "program": 32726, "Ġcrocod": 32727, "bral": 32728, "AAAAAAAA": 32729, "agit": 32730, "ĠNä": 32731, "Ġgekommen": 32732, "atten": 32733, "Ġreferenced": 32734, "Ġpairing": 32735, "ĠPartner": 32736, "ĠCoronavirus": 32737, "ÑĸÑģ": 32738, "è½ī": 32739, "Ġ×Ķ×ĵ": 32740, "ĠespecÃŃfic": 32741, "arsi": 32742, "quelle": 32743, "Ġspontaneous": 32744, "çĨ±": 32745, "Ġê²ĥìĿĦ": 32746, "ĠÐŁÐ¾Ñģле": 32747, "ĠاÙĦد": 32748, "ĠShout": 32749, "Ġнал": 32750, "Ġdisguise": 32751, "ĠJord": 32752, "Ġwee": 32753, "Ġmiejsc": 32754, "Ġserum": 32755, "Ġplaisir": 32756, "Ġcredible": 32757, "ĠbÃ¥": 32758, "ĠAJ": 32759, "mares": 32760, "Ġrods": 32761, "Ġeran": 32762, "ãģ¾ãģĤ": 32763, "Ġpää": 32764, "ĠUA": 32765, "ĠUnknown": 32766, "ĠÙĦÙħ": 32767, "ĠRabbi": 32768, "Ġlaat": 32769, "Ġhairstyle": 32770, "Ġغ": 32771, "éģĭ": 32772, "Ġcach": 32773, "ĠWriting": 32774, "оÑĩки": 32775, "abad": 32776, "Ġstraighten": 32777, "--\"": 32778, "wife": 32779, "Ġhottest": 32780, "Ġpunya": 32781, "ĠFashion": 32782, "griff": 32783, "ĠQR": 32784, "otch": 32785, "ĠÐľÐ¾Ð¶ÐµÑĤ": 32786, "Cloud": 32787, "ĠStrike": 32788, "ĠHein": 32789, "Ġ羣çļĦ": 32790, "Ġlei": 32791, "ĠFlow": 32792, "wegs": 32793, "Ġhabr": 32794, "åīĽåīĽ": 32795, "nahme": 32796, "Ìģ": 32797, "Ġpleasing": 32798, "opping": 32799, "Ġ구ëıħ": 32800, "Ġdran": 32801, "Ġbangs": 32802, "Ġ79": 32803, "Ġsket": 32804, "Ġcaval": 32805, "ĠMacron": 32806, "Ġweighted": 32807, "Ġmuted": 32808, "Ġnuestras": 32809, "EEP": 32810, "Ġmathematic": 32811, "ĠMRI": 32812, "agus": 32813, "Ġtherapies": 32814, "θε": 32815, "Ġunpl": 32816, "Ġcommencer": 32817, "full": 32818, "Ġtowels": 32819, "Ġprue": 32820, "Ġlicenses": 32821, "׼×ķ׾": 32822, "ĠÐŁÐ¾ÑĩемÑĥ": 32823, "Ġpointless": 32824, "Bye": 32825, "Ġeligibility": 32826, "Ġscrape": 32827, "Ġabusive": 32828, "ĠMant": 32829, "Ġjeunes": 32830, "tal": 32831, "ĠPrincip": 32832, "ĠOrthodox": 32833, "Ġmelod": 32834, "ĠмаÑĤеÑĢи": 32835, "Ġprosecutor": 32836, "Ġopioid": 32837, "ĠÑĥвеÑĢ": 32838, "ĠBeen": 32839, "Ġìłijì¢ħ": 32840, "Ġdynasty": 32841, "Ġajuda": 32842, "Ġentreg": 32843, "Ġweighed": 32844, "Ġeure": 32845, "ĠBem": 32846, "Ġabnormal": 32847, "82": 32848, "ĠJR": 32849, "ĠAkt": 32850, "ĠBri": 32851, "út": 32852, "Ġstagn": 32853, "!*": 32854, "Ġwegen": 32855, "Ġleaking": 32856, "ĠWords": 32857, "ĠMau": 32858, "Ġvue": 32859, "ĠLiam": 32860, "анием": 32861, "Ġclinicians": 32862, "ĠPump": 32863, "Ġförst": 32864, "?...": 32865, "Ġautomotive": 32866, "ĠOwen": 32867, "zusagen": 32868, "ĠHundred": 32869, "Ġdecentralized": 32870, "Ġbulbs": 32871, "Ġ׾׼": 32872, "Ġprovinces": 32873, "ĠMilan": 32874, "81": 32875, "kas": 32876, "Ġëĵ£": 32877, "Ġforça": 32878, "Ġrightly": 32879, "島": 32880, "rÄħ": 32881, "Ġvenues": 32882, "Ġwai": 32883, "Ġpredicting": 32884, "ĠWiFi": 32885, "Ġê¶ģê¸Ī": 32886, "رÙĪ": 32887, "Ġ×Ķ×ĸ": 32888, "century": 32889, "Ġgradual": 32890, "ĠProbleme": 32891, "ĠìĹħ": 32892, "Ġcoping": 32893, "ĠBrus": 32894, "Ġpeanuts": 32895, "irtschaft": 32896, "Ġзал": 32897, "ĠTroy": 32898, "Ġsperm": 32899, "ĠMitar": 32900, "ĠTürkiye": 32901, "grand": 32902, "¦Ń": 32903, "Ġ×ŀס": 32904, "Ġpans": 32905, "ĠKnowledge": 32906, "berly": 32907, "ĠÐķго": 32908, "Ġdanced": 32909, "ĠFrost": 32910, "ĠBurg": 32911, "Ġbiting": 32912, "ìłķìĿĦ": 32913, "meal": 32914, "Ġheroic": 32915, "Ġmotherboard": 32916, "ĠLicht": 32917, "ãģ£ãģ": 32918, "llan": 32919, "айн": 32920, "ĠÑĢÑıд": 32921, "Ġà¹Ģà¸": 32922, "onen": 32923, "irie": 32924, "Art": 32925, "rang": 32926, "νη": 32927, "Ġnewborn": 32928, "Ġamis": 32929, "ĠاÙĪر": 32930, "Ġsophom": 32931, "ĠCareful": 32932, "Ġprospects": 32933, "ensen": 32934, "Ġthrill": 32935, "ĠViá»ĩt": 32936, "Adam": 32937, "rition": 32938, "entric": 32939, "uden": 32940, "Ġcertificates": 32941, "Ġashes": 32942, "調": 32943, "playing": 32944, "Ġsadece": 32945, "Ġost": 32946, "Ġairplanes": 32947, "ÑĢок": 32948, "oner": 32949, "Ġmagnesium": 32950, "Ġgoddamn": 32951, "Ġ1972": 32952, "ĠSchule": 32953, "Ġtemat": 32954, "Ġpartout": 32955, "à¯Ĥ": 32956, "Ġinve": 32957, "ĠScientists": 32958, "ĠHudson": 32959, "winning": 32960, "ceksin": 32961, "Ġcongressional": 32962, "oru": 32963, "Ġropes": 32964, "вед": 32965, "Ġmadre": 32966, "Ġferry": 32967, "ĠCohen": 32968, "ĠPred": 32969, "Ġvagy": 32970, "ĠбеÑģп": 32971, "Ġmultim": 32972, "Ġdrainage": 32973, "Ġsimulator": 32974, "giggles": 32975, "ĠStadium": 32976, "обÑī": 32977, "Ġnotices": 32978, "Ġcrawling": 32979, "Ġgroupe": 32980, "åı¸": 32981, "ĠktoÅĽ": 32982, "ĠYoga": 32983, "Ġmedida": 32984, "ĠÑħваÑĤ": 32985, "ĠLite": 32986, "Ġrav": 32987, "orama": 32988, "Ġdiscord": 32989, "ĠDIRE": 32990, "Ġteh": 32991, "ĠNurs": 32992, "ç²ī": 32993, "Ġpitched": 32994, "Ġbarking": 32995, "ĠCoke": 32996, "wiad": 32997, "Ġpopulated": 32998, "éĻ¤": 32999, "pelled": 33000, "Ġбог": 33001, "Ġpewno": 33002, "ĠCube": 33003, "Ġrecruited": 33004, "éĢĻ種": 33005, "ĠCara": 33006, "ıģını": 33007, "imated": 33008, "ĠÑĪкол": 33009, "icional": 33010, "ĠпÑĢоÑĦ": 33011, "Ġcontamination": 33012, "Ġúltimos": 33013, "Ġfearful": 33014, "Ġelephants": 33015, "usi": 33016, "ĠiTunes": 33017, "ĠSwami": 33018, "ê¼": 33019, "ĠìĦ¤ëªħ": 33020, "ĠRichards": 33021, "Ġmagnets": 33022, "ĠRichtung": 33023, "ĠLegion": 33024, "èıľ": 33025, "Ġkitty": 33026, "Ġkissed": 33027, "Ġwatering": 33028, "Ġcono": 33029, "ĠPalestine": 33030, "idir": 33031, "Ġmaze": 33032, "Ġfluids": 33033, "ĠProducer": 33034, "ĠKrsna": 33035, "好åķ¦": 33036, "laf": 33037, "Ġ×IJ×ķ": 33038, "Ġmiesz": 33039, "ĠXing": 33040, "ointed": 33041, "sein": 33042, "ĠFuk": 33043, "ĠDepression": 33044, "ĠDuty": 33045, "ĠPanther": 33046, "Ġsund": 33047, "Ġrefere": 33048, "Ġexclusion": 33049, "Ġnaval": 33050, "ĠWinston": 33051, "Ġslogan": 33052, "Ġhypothetical": 33053, "Ġelevate": 33054, "ëł¹": 33055, "Ġcabeça": 33056, "ĠGesund": 33057, "meter": 33058, "ĠìķĦëĭĪë©´": 33059, "Ġcloudy": 33060, "âĢ¦?": 33061, "ĠSchritt": 33062, "ĠJS": 33063, "ìį": 33064, "ĠSprings": 33065, "ĠBatter": 33066, "·°": 33067, "Ġtailor": 33068, "ĠPTSD": 33069, "ĠGent": 33070, "ĠbaÄŁ": 33071, "Ġspatula": 33072, "Ġcray": 33073, "ĠLegisl": 33074, "Ġsú": 33075, "Ġleve": 33076, "าม": 33077, "Ġerad": 33078, "Ġdong": 33079, "Ġderm": 33080, "ĠBanks": 33081, "icho": 33082, "åħĪçĶŁ": 33083, "ĠFranz": 33084, "ravel": 33085, "éģĶ": 33086, "оло": 33087, "Ġflute": 33088, "ĠEk": 33089, "Ġjoyful": 33090, "Ġchased": 33091, "ĠLarge": 33092, "Over": 33093, "Ġentrepreneurial": 33094, "Ġconsiders": 33095, "Ñĥем": 33096, "opa": 33097, "Ġdormir": 33098, "ĠElementary": 33099, "Ġprzypad": 33100, "ÑĥÑģка": 33101, "ĠоÑĩеÑĢ": 33102, "ugene": 33103, "Ġtenido": 33104, "Ġlugares": 33105, "ë¥": 33106, "ĠÑĩаÑģÑĤ": 33107, "Ġsao": 33108, "Ġbraid": 33109, "ĠVere": 33110, "ĠReich": 33111, "ĠPoss": 33112, "Ġinan": 33113, "wand": 33114, "ref": 33115, "Ġmontrer": 33116, "Ġ1981": 33117, "çķª": 33118, "asında": 33119, "Ġchrome": 33120, "ĠTrinity": 33121, "Ġexploitation": 33122, "ĠSense": 33123, "ĠCMS": 33124, "ĠNoble": 33125, "ĠìĦłíĥĿ": 33126, "Ġswelling": 33127, "electronic": 33128, "]?": 33129, "Ġbrushing": 33130, "Ġliquidity": 33131, "ĠHook": 33132, "ĠConnor": 33133, "ĠAlum": 33134, "Ġgucken": 33135, "suite": 33136, "Ġwiele": 33137, "Ġbarrels": 33138, "ĠRegel": 33139, "ĠMent": 33140, "ĠTrip": 33141, "ĠBrush": 33142, "ĠErik": 33143, "urate": 33144, "ÉĻr": 33145, "ĠCyr": 33146, "ouble": 33147, "ĠBecca": 33148, "Ġpasswords": 33149, "ű": 33150, "borg": 33151, "Ġvendo": 33152, "ĠClaus": 33153, "ĠFaz": 33154, "indest": 33155, "Ġdeceased": 33156, "Ġcomparisons": 33157, "ĠLCD": 33158, "ĠPork": 33159, "Ġeventual": 33160, "Ġpatreon": 33161, "Ġinability": 33162, "Ġextinction": 33163, "Ġì¢ĭìķĦíķĺëĬĶ": 33164, "ĠÑģоÑģ": 33165, "aju": 33166, "Ġ×ij×IJ×": 33167, "Ġsofort": 33168, "Ġdestined": 33169, "ĠRin": 33170, "Ġmouths": 33171, "ĠNatürlich": 33172, "Ġpreserving": 33173, "Ġlimp": 33174, "黨": 33175, "ocused": 33176, "инг": 33177, "Ġexposing": 33178, "Ġξ": 33179, "ëį": 33180, "laugh": 33181, "Ġhiss": 33182, "ãģłãģĭãĤī": 33183, "Ġindie": 33184, "Ġdetal": 33185, "ÑĢавÑģÑĤв": 33186, "Ġtrên": 33187, "æķ°": 33188, "Ġogni": 33189, "Ġsimplemente": 33190, "Ġ1978": 33191, "Ġgoo": 33192, "Ġ1967": 33193, "Ġgenug": 33194, "hö": 33195, "Ġhistó": 33196, "å®Ł": 33197, "Ġlobster": 33198, "cendo": 33199, "Ġteil": 33200, "Ġallevi": 33201, "0000": 33202, "OLD": 33203, "Ġpesos": 33204, "Ġbonuses": 33205, "Ġami": 33206, "Ġrevival": 33207, "ĠHorse": 33208, "Ġsack": 33209, "Talk": 33210, "Ġmulher": 33211, "ĠпоÑģÑĤоÑıн": 33212, "ĠHood": 33213, "Huh": 33214, "Ġë¶ģ": 33215, "Ġhyung": 33216, "ĠMeeting": 33217, "Ġimporta": 33218, "Ġì°¾ìķĦ": 33219, "ĠVern": 33220, "Ġstripped": 33221, "Ġrefuses": 33222, "Ġqualifications": 33223, "opl": 33224, "ĢëıĦ": 33225, "ixÃŃ": 33226, "Ġdiab": 33227, "itime": 33228, "flows": 33229, "Ġinac": 33230, "ĠGong": 33231, "Ġmeaningless": 33232, "Ġcourageous": 33233, "Ġmicrobi": 33234, "azy": 33235, "hist": 33236, "Ġvolunteering": 33237, "VIE": 33238, "Ġviolated": 33239, "Ġsympathy": 33240, "ĠEdit": 33241, "好åĥı": 33242, "electric": 33243, "product": 33244, "Ġpandemia": 33245, "Ġgeometric": 33246, "ĠConvers": 33247, "gre": 33248, "Ġglut": 33249, "isted": 33250, "ĠاÙĦÙĥ": 33251, "ĠChain": 33252, "ĠPresent": 33253, "ĠYin": 33254, "ĠÑģог": 33255, "ĠVlog": 33256, "Ġìĸ´ë¨¸": 33257, "Ġdonn": 33258, "Ġhitch": 33259, "ucking": 33260, "ãģĬãģĦ": 33261, "wald": 33262, "risk": 33263, "Ġhari": 33264, "ĠKens": 33265, "ĠIdol": 33266, "Ġвнимание": 33267, "Ġtodd": 33268, "Ġsmashed": 33269, "Ġinvari": 33270, "ĠконÑĤÑĢ": 33271, "Ġautistic": 33272, "ìŀ¥ëĭĺ": 33273, "Res": 33274, "дÑĭ": 33275, "chau": 33276, "Ġselv": 33277, "Ġhätten": 33278, "ि": 33279, "Ġexpects": 33280, "Ïģη": 33281, "Ġaçık": 33282, "ĠHTTP": 33283, "leÅŁ": 33284, "Ġsweeping": 33285, "ĠBeta": 33286, "Ġcounterparts": 33287, "abile": 33288, "ĠSims": 33289, "Cs": 33290, "Ġrepar": 33291, "squ": 33292, "Ġprovincial": 33293, "Ġshareholders": 33294, "Ġrunter": 33295, "Ġgedacht": 33296, "ĠTeen": 33297, "Ġgrands": 33298, "çĶ¢": 33299, "agles": 33300, "Ġrocky": 33301, "vens": 33302, "Ġrivals": 33303, "unal": 33304, "Ġreacts": 33305, "ë©": 33306, "Ġmercury": 33307, "ĠLuigi": 33308, "Ġог": 33309, "ĠJUST": 33310, "Ġlod": 33311, "Ġcortex": 33312, "wig": 33313, "Ġlakh": 33314, "ì¤ijìĹIJ": 33315, "ĠVic": 33316, "ĠMund": 33317, "Ġmapped": 33318, "ĠDell": 33319, "ĠDruck": 33320, "Ġlifes": 33321, "алÑĮное": 33322, "ividual": 33323, "adım": 33324, "Ġatrav": 33325, "ĠFlug": 33326, "ĠKlein": 33327, "ê±°ìķ¼": 33328, "หà¸Ļ": 33329, "Ġappli": 33330, "ா?": 33331, "üyorum": 33332, "ĠинÑĤеÑĢеÑģно": 33333, "Ġdisinfect": 33334, ">-": 33335, "Ġchampagne": 33336, "Ġkla": 33337, "opers": 33338, "Trans": 33339, "ĠDesert": 33340, "Ġcultivate": 33341, "ĠFucking": 33342, "idelity": 33343, "ĠÑĤан": 33344, "Ġincub": 33345, "Ġtemu": 33346, "Ġlearner": 33347, "founder": 33348, "ĠSyl": 33349, "ãĤĢ": 33350, "Ġfato": 33351, "zier": 33352, "ĠìĹĨìĿ´": 33353, "ĠìĪ¨": 33354, "Ġpsycho": 33355, "ĠÑĤелеÑĦ": 33356, "Ġregarde": 33357, "Ġrepresentations": 33358, "Ġlitigation": 33359, "Ġspann": 33360, "ults": 33361, "bior": 33362, "è¦ĭãģ¦": 33363, "ä¸įå¤ļ": 33364, "ĠSurvey": 33365, "ĠLEDs": 33366, "Ġträ": 33367, "Ġlên": 33368, "Ġantioxid": 33369, "еÑĢом": 33370, "Ġinduction": 33371, "Ġfooled": 33372, "ätzlich": 33373, "ĠговоÑĢÑıÑĤ": 33374, "ĠFact": 33375, "umbai": 33376, "Ġwiggle": 33377, "NOUN": 33378, "Ġdévelopp": 33379, "ĠClaro": 33380, "Ġì¸": 33381, "ë¬": 33382, "ãģªãĤĵãģł": 33383, "Ġaccumulate": 33384, "Ġmaintains": 33385, "ëĦ": 33386, "ĠFighter": 33387, "íĨł": 33388, "Ġmatin": 33389, "Ġcoupon": 33390, "Ġstunt": 33391, "Ġdebuted": 33392, "å¾ħãģ£ãģ¦": 33393, "Ġprag": 33394, "иваем": 33395, "73": 33396, "Ġexpres": 33397, "Ġìĺ¤ë¹ł": 33398, "ĠпеÑĢÑģон": 33399, "Ġcalculus": 33400, "Ġabrupt": 33401, "ĠInspector": 33402, "ourt": 33403, "æĸĻ": 33404, "źniej": 33405, "intense": 33406, "Ba": 33407, "Ġlounge": 33408, "Ġasthma": 33409, "ĠHiç": 33410, "ª»": 33411, "Ġeditorial": 33412, "Ġseize": 33413, "Ġkır": 33414, "Ġmouve": 33415, "Ġtierra": 33416, "Ġtestosterone": 33417, "Ġrh": 33418, "ĠKingston": 33419, "ELLE": 33420, "ĠRepresentative": 33421, "Ġ1974": 33422, "Ġiba": 33423, "Ts": 33424, "Ġsorta": 33425, "Ġ(?)": 33426, "ĠتÙĪ": 33427, "ĠëĤ´ëł¤": 33428, "Ġbekommt": 33429, "Ġspiritually": 33430, "Ġdistorted": 33431, "Mad": 33432, "Ġreim": 33433, "ánh": 33434, "ĠOttoman": 33435, "ĠRelig": 33436, "ĠEls": 33437, "Ġretained": 33438, "ĠLaughs": 33439, "æĢ»": 33440, "ĠSAS": 33441, "ĠколиÑĩеÑģÑĤво": 33442, "×ķתר": 33443, "Ġinnovate": 33444, "Ġkork": 33445, "ĠÑĢаÑģÑģказÑĭв": 33446, "ondere": 33447, "ivi": 33448, "aye": 33449, "ounty": 33450, "ĠполÑĥÑĩаеÑĤÑģÑı": 33451, "Ġbuns": 33452, "åħ«": 33453, "Ġyüzden": 33454, "Ġsurgeries": 33455, "Ø£ÙĨ": 33456, "Ġbankruptcy": 33457, "welt": 33458, "Ġsiamo": 33459, "Ġdarkest": 33460, "ĠHann": 33461, "gga": 33462, "Ġformas": 33463, "ĠDj": 33464, "named": 33465, "Ġshields": 33466, "ueller": 33467, "ĠFew": 33468, "Ġlace": 33469, "Ġfurious": 33470, "ĠYU": 33471, "Ġsocietal": 33472, "Ġjudgement": 33473, "ĠDos": 33474, "Ġjab": 33475, "laws": 33476, "Ġreinvent": 33477, "ĠKatherine": 33478, "ĠChoi": 33479, "adows": 33480, "Ġrans": 33481, "oden": 33482, "ĠMidwest": 33483, "nın": 33484, "Ġdeport": 33485, "ĠDip": 33486, "ç´ħ": 33487, "Ġatención": 33488, "ĠCourtney": 33489, "ividad": 33490, "ĠÚ©Ûģ": 33491, "Ġefficacy": 33492, "ĠBrooks": 33493, "Ġreferral": 33494, "ĠконÑĨ": 33495, "Ġmalicious": 33496, "Ġkir": 33497, "ĠGoddess": 33498, "Ġfunky": 33499, "Ġinterim": 33500, "ĠKörper": 33501, "Ġìĸ¼ë§": 33502, "kur": 33503, "Ġкли": 33504, "Ġtrucs": 33505, "gesetz": 33506, "Ġzug": 33507, "ĠGlück": 33508, "ĠMinute": 33509, "Ġprestigious": 33510, "Ġniez": 33511, "Ġconcentrations": 33512, "лаÑģÑĤи": 33513, "ĠSis": 33514, "ĠVitamin": 33515, "kov": 33516, "ĠPBS": 33517, "Ġнее": 33518, "Ġretailers": 33519, "Ġconventions": 33520, "ĠSamantha": 33521, "Ġproudly": 33522, "Jordan": 33523, "ĠJASON": 33524, "atk": 33525, "Ġtriste": 33526, "Ġstär": 33527, "Ġreiterate": 33528, "Ġposterior": 33529, "Ġ1973": 33530, "ĠPine": 33531, "ĠJuliet": 33532, "Ġpedir": 33533, "kil": 33534, "Ġoverlapping": 33535, "Ġexclude": 33536, "Ġeconóm": 33537, "Ġaccepts": 33538, "ĠSter": 33539, "決": 33540, "Ġìļ´ëıĻ": 33541, "estab": 33542, "Ġtug": 33543, "arg": 33544, "Ġlivro": 33545, "اص": 33546, "Ġseams": 33547, "Ġburaya": 33548, "Ġello": 33549, "ĠTM": 33550, "ĠPaw": 33551, "ĠIndex": 33552, "Exc": 33553, "Ġinspirational": 33554, "Ġdunk": 33555, "è°ģ": 33556, "akter": 33557, "Ġconditioner": 33558, "ĠSalut": 33559, "ÅĤec": 33560, "Ġìī½": 33561, "ĠÑĥзна": 33562, "ĠRomeo": 33563, "fruit": 33564, "ĠYO": 33565, "Ġchá»ī": 33566, "бÑĥ": 33567, "bons": 33568, "Ġreproductive": 33569, "Ġorada": 33570, "Ġíļ¨": 33571, "Ġtentar": 33572, "Ġmañana": 33573, "ãĤ¬": 33574, "Ġsolvent": 33575, "Jessica": 33576, "ĠLegal": 33577, "Ġtua": 33578, "Ġsic": 33579, "ĠEQ": 33580, "aukee": 33581, "ìĭľëĭ¤": 33582, "ĠÅŀu": 33583, "Ġadhere": 33584, "ĠTul": 33585, "Ġà®Ĩ": 33586, "Ġtextbooks": 33587, "ĠFifth": 33588, "Ġexperi": 33589, "Ġchic": 33590, "Ġheap": 33591, "inely": 33592, "atra": 33593, "Two": 33594, "Ġhelemaal": 33595, "Ġfren": 33596, "æݨ": 33597, "Ġbisher": 33598, "اش": 33599, "ĠìĦłìĥĿ": 33600, "ĠTages": 33601, "Ġsá»±": 33602, "Ġbullied": 33603, "ؤ": 33604, "Ġbenefited": 33605, "ĠPreviously": 33606, "ĠÑįÑĦÑĦ": 33607, "Ùį": 33608, "Ġsenate": 33609, "ĠMorm": 33610, "ijke": 33611, "ĠFlu": 33612, "Ġincorporating": 33613, "jack": 33614, "ĠпиÑĤ": 33615, "Ġimply": 33616, "Ġhacks": 33617, "ĠRICH": 33618, "ĠкваÑĢ": 33619, "ĠпÑĢекÑĢаÑģ": 33620, "Ġdependency": 33621, "Ġìļ©": 33622, "Ġì±ħ": 33623, "Ġwährend": 33624, "Ġsulla": 33625, "ĠPittsburgh": 33626, "Ġesempio": 33627, "¼ë¡ľ": 33628, "prot": 33629, "ĠRosen": 33630, "ĠIndependence": 33631, "Ġparsley": 33632, "iegen": 33633, "Ġhaw": 33634, "Ġaquell": 33635, "ĠCAP": 33636, "ĠÑĢабоÑĤаÑĤÑĮ": 33637, "ĠCliff": 33638, "ionar": 33639, "Ġsecuring": 33640, "æĪijåĢijçļĦ": 33641, "νε": 33642, "Ġutilis": 33643, "Ġcoule": 33644, "ĠPing": 33645, "Ġtrek": 33646, "Ġfak": 33647, "Ġenorme": 33648, "Ġìĭ«": 33649, "让": 33650, "Ġdoubling": 33651, "ĠнÑĢавиÑĤÑģÑı": 33652, "Ġhed": 33653, "hoven": 33654, "ĠStanding": 33655, "ĠmÃŃn": 33656, "ĠJimin": 33657, "Ġmonarch": 33658, "Ġcoke": 33659, "Ġmr": 33660, "Ġclic": 33661, "Ãį": 33662, "Ġimpeachment": 33663, "Ġdurability": 33664, "Ġvarios": 33665, "Ġcommercials": 33666, "Ġgreetings": 33667, "ĠRi": 33668, "ĠAppreci": 33669, "ìŀĪëĬĶ": 33670, "Ġrésult": 33671, "ért": 33672, "Ġsalute": 33673, "Ġpoderia": 33674, "Ġsunrise": 33675, "veck": 33676, "Ġreluctant": 33677, "Ġcommissioner": 33678, "念": 33679, "âte": 33680, "ĠKenny": 33681, "ĠSiri": 33682, "ãĥĥãĥĹ": 33683, "ĠëĬĺ": 33684, "ĠEE": 33685, "Ġunch": 33686, "кон": 33687, "ĠاÙĦØ¥": 33688, "Ġbelts": 33689, "Ġhass": 33690, "ĠмоÑı": 33691, "Ġdisplaced": 33692, "Ġabra": 33693, "ÎŃλ": 33694, "Ġscratches": 33695, "Ġcomet": 33696, "Ġauthorization": 33697, "ĠLLC": 33698, "Ġproduk": 33699, "Ġrehabilitation": 33700, "åŀ": 33701, "ÑĸÑĩ": 33702, "uding": 33703, "olit": 33704, "Ġ105": 33705, "Ġexpands": 33706, "Ġaltri": 33707, "ĠKomment": 33708, "Ġanf": 33709, "Pl": 33710, "ĠMana": 33711, "fed": 33712, "Ġbri": 33713, "Ġora": 33714, "Gs": 33715, "ĠGur": 33716, "uckland": 33717, "Ġjunction": 33718, "Ġironic": 33719, "ĠFeed": 33720, "Ġprakt": 33721, "ĠHammer": 33722, "ĮëıĦ": 33723, "ĠTracy": 33724, "çµ±": 33725, "ĠAside": 33726, "него": 33727, "ĠиÑģполÑĮзоваÑĤÑĮ": 33728, "Ġzaj": 33729, "Ġequitable": 33730, "Ġcurb": 33731, "ĠãģĵãĤĮ": 33732, "Ġderivatives": 33733, "Ġpuppies": 33734, "ĠKenneth": 33735, "ĠCompl": 33736, "igram": 33737, "ĠGarcia": 33738, ")\"": 33739, "ĠHarbor": 33740, "estial": 33741, "Ġä¾Ĩ": 33742, "Ġers": 33743, "æ¹": 33744, "Ġunwanted": 33745, "Ġbelang": 33746, "аго": 33747, "emb": 33748, "dos": 33749, "ĠìĻľë": 33750, "ĠBudget": 33751, "Ġbattling": 33752, "ØŃت": 33753, "kok": 33754, "наÑĩала": 33755, "Ġplag": 33756, "Ġcantidad": 33757, "Ġgrupos": 33758, "Ġplugins": 33759, "lerini": 33760, "ĠимееÑĤ": 33761, "Ġsozusagen": 33762, "olics": 33763, "Ġpueblo": 33764, "Ġreminis": 33765, "rän": 33766, "ĠMorrison": 33767, "Ġlinha": 33768, "Ġbreaths": 33769, "ĠTaste": 33770, "Ġenfrent": 33771, "ĠDocker": 33772, "Ġден": 33773, "Ġethnicity": 33774, "Ġwob": 33775, "Ġsuffers": 33776, "Ġtransitioning": 33777, "ĠRange": 33778, "ÄĻdzy": 33779, "ĠкаÑĤ": 33780, "Ġsyner": 33781, "Ġdonut": 33782, "Ġprobabilities": 33783, "ĠOmar": 33784, "Which": 33785, "uish": 33786, "isin": 33787, "Ġdemos": 33788, "ĠìłĢ기": 33789, "Ġëĺijê°Ļ": 33790, "Ġедин": 33791, "Ġcerve": 33792, "Ġjoka": 33793, "IAN": 33794, "Ġkilometer": 33795, "Ġhorizontally": 33796, "ĠBhag": 33797, "Ġ->": 33798, "ĠMonitor": 33799, "Ġknowledgeable": 33800, "Ġfav": 33801, "Ġpinned": 33802, "ĠeBay": 33803, "icker": 33804, "Ġìŀłê¹IJë§Į": 33805, "ĠXiaomi": 33806, "Ġcapit": 33807, "Ġnp": 33808, "Ġ1965": 33809, "hoe": 33810, "Ġnok": 33811, "ĠSage": 33812, "ĠнелÑĮзÑı": 33813, "ĠTow": 33814, "gam": 33815, "Ġdicen": 33816, "ĠSUBSCRIBE": 33817, "Ġreboot": 33818, "Ġpaj": 33819, "Ġë³´ìŬë": 33820, "Ġthicken": 33821, "ĠReality": 33822, "idän": 33823, "Na": 33824, "Ġê²ĥìĿĢ": 33825, "!!)": 33826, "Ġroutines": 33827, "Ġодного": 33828, "Ġexting": 33829, "Ġì¦Ŀ": 33830, "Ġsulfur": 33831, "Ġcarve": 33832, "Ġasteroid": 33833, "ĠWarrior": 33834, "Ġphotographers": 33835, "Ġpell": 33836, "Ġcrossover": 33837, "æĪijçŁ¥éģĵ": 33838, "Ġhacemos": 33839, "ĠNej": 33840, "Ġsettling": 33841, "Ġirm": 33842, "ĠBooks": 33843, "ientôt": 33844, "Ġespacio": 33845, "ĠScholars": 33846, "Ġdoomed": 33847, "ĠIRS": 33848, "wohl": 33849, "Ġsegue": 33850, "ĠëĪĦê°Ģ": 33851, "Ġpratic": 33852, "BT": 33853, "ĠConsidering": 33854, "ĠBuffalo": 33855, "Ġtrainings": 33856, "Ġgebru": 33857, "ĠGleich": 33858, "Ġpirates": 33859, "Ġenvelop": 33860, "Ġreopen": 33861, "imat": 33862, "Ġtee": 33863, "Ġsued": 33864, "feh": 33865, "Ġ×Ķק": 33866, "Ġdiets": 33867, "Ġjuntos": 33868, "asto": 33869, "Ġmisunderstood": 33870, "Ġruim": 33871, "Ġclassify": 33872, "ĠпÑĢодÑĥк": 33873, "Ġinse": 33874, "Ġillustrated": 33875, "Ġcorrosion": 33876, "Ġaccred": 33877, "ĠAuntie": 33878, "ĠпÑĢивеÑĤ": 33879, "ĠLIVE": 33880, "Ġrek": 33881, "Ġreceipt": 33882, "åĪ°åºķ": 33883, "ĠBarbie": 33884, "ĠSnake": 33885, "turn": 33886, "Jeff": 33887, "ãģĬãģĬ": 33888, "ķĦ": 33889, "VOICEOVER": 33890, "coll": 33891, "Ġrunners": 33892, "ìłľë": 33893, "osos": 33894, "moon": 33895, "Ġkeynote": 33896, "ĠInstit": 33897, "SPEAK": 33898, "Ġplugs": 33899, "Ġcurv": 33900, "ĠYuri": 33901, "ĠTheres": 33902, "ĠPs": 33903, "ĠμÏĢο": 33904, "Ġconverter": 33905, "Ġrefine": 33906, "Ġbadass": 33907, "Ġοι": 33908, "Ġregen": 33909, "azzi": 33910, "ÙĬÙģ": 33911, "Ġseized": 33912, "Ġiçer": 33913, "ilee": 33914, "Ġupstream": 33915, "Ġbuds": 33916, "Ġpim": 33917, "Ġíķĺ루": 33918, "Ġalluded": 33919, "Ġthemed": 33920, "Ġconsisting": 33921, "Ġbons": 33922, "unuz": 33923, "ĠпÑĢовод": 33924, "ĠLovely": 33925, "à¥ĭ": 33926, "Ġparach": 33927, "ĠStaats": 33928, "éļĬ": 33929, "Ġselective": 33930, "Ġfase": 33931, "ĠGeorget": 33932, "Ġcocaine": 33933, "Ġreproduction": 33934, "ĠLara": 33935, "ĠLD": 33936, "Ġgh": 33937, "Jon": 33938, "ĠlÃ¥": 33939, "ĠëijIJë": 33940, "Ġtyped": 33941, "ĠBana": 33942, "ëĵľë": 33943, "Ġsavory": 33944, "ĠZomb": 33945, "standen": 33946, "Ġpedestrian": 33947, "Ġdifférents": 33948, "Ġìĭ¸": 33949, "èī¯": 33950, "Ġcomplained": 33951, "ç¦ı": 33952, "ĠÐļÑĤо": 33953, "Ġ׾פ": 33954, "aliÅĽmy": 33955, "Ġmortar": 33956, "Ġverdict": 33957, "Ġsuficiente": 33958, "ĠMillion": 33959, "mittel": 33960, "inals": 33961, "ĠاÙĦØ®": 33962, "аÑİÑģÑĮ": 33963, "ĠmiÄĻdzy": 33964, "ĠOle": 33965, "Ġinvert": 33966, "czyÄĩ": 33967, "озможно": 33968, "starter": 33969, "Ġauditor": 33970, "ĠScout": 33971, "chien": 33972, "ĠSverige": 33973, "uffled": 33974, "Ġzehn": 33975, "ĠAuckland": 33976, "Ġargent": 33977, "Ġ1976": 33978, "ĠHoe": 33979, "Ġbothers": 33980, "Ġsocialist": 33981, "Ġpliers": 33982, "Ġemergen": 33983, "ĠXP": 33984, "еÑĢов": 33985, "More": 33986, "ĠLevi": 33987, "ĠAnders": 33988, "ibilidad": 33989, "ĠParents": 33990, "Ġinduced": 33991, "ìĸ´ì¤": 33992, "Ġbalances": 33993, "ĠвÑĭÑĪ": 33994, "Ġsubmarine": 33995, "Start": 33996, "Ġdries": 33997, "Ġvolver": 33998, "Ġticking": 33999, "cott": 34000, "Ġfaj": 34001, "prés": 34002, "ĠSabb": 34003, "ĠзаÑĩ": 34004, "ĠпокÑĥп": 34005, "Ġbaptized": 34006, "ĠBrilliant": 34007, "ĠÐijог": 34008, "Ġmots": 34009, "bits": 34010, "Ġlattice": 34011, "æĪijè·Łä½ł": 34012, "Ġcoriander": 34013, "Ġresidency": 34014, "ync": 34015, "Ġpierwszy": 34016, "ĠKnock": 34017, "ĠZap": 34018, "ĠÐķв": 34019, "견": 34020, "å°ıå¿ĥ": 34021, "Ġuneven": 34022, "ĠJas": 34023, "odor": 34024, "ç¿Ĵ": 34025, "74": 34026, "ĠSite": 34027, "Ġaconteceu": 34028, "ympt": 34029, "Ġtrilogy": 34030, "Ġlantern": 34031, "ĠZucker": 34032, "vari": 34033, "welling": 34034, "ĠPotato": 34035, "gomery": 34036, "Ġreacted": 34037, "ĠChron": 34038, "Ġjede": 34039, "beeld": 34040, "Ġtwent": 34041, "Ġlact": 34042, "æ¨Ĥ": 34043, "Ġrése": 34044, "Ġrelent": 34045, "Ġfurnace": 34046, "Ġwidget": 34047, "Ġearthquakes": 34048, "ĠAdjust": 34049, "ilit": 34050, "ĠØ£ÙĪ": 34051, "Ġhearings": 34052, "Ġdefendant": 34053, "irsiniz": 34054, "Ġbask": 34055, "cja": 34056, "ľ¨": 34057, "Ġrifles": 34058, "Ġinstal": 34059, "ĠForgive": 34060, "pical": 34061, "ĠÐŀÑĩенÑĮ": 34062, "Ġpetites": 34063, "Ġhp": 34064, "Ġrenowned": 34065, "ĠInn": 34066, "Ġ주ìĦ¸ìļĶ": 34067, "Ġemphasized": 34068, "éĹ®é¢ĺ": 34069, "ĠìŀĪì£ł": 34070, "Ġê²ĥìľ¼ë¡ľ": 34071, "ãĤĨ": 34072, "Åĵ": 34073, "gili": 34074, "Dave": 34075, "Ġexhausting": 34076, "ÅĤug": 34077, "Ġschema": 34078, "μά": 34079, "cycl": 34080, "Ġautant": 34081, "Ġparcel": 34082, "Ġmateria": 34083, "ĠBerry": 34084, "ĠÑģами": 34085, "Ġextracted": 34086, "ĠSaying": 34087, "ismatic": 34088, "ĠпопÑĢоб": 34089, "Ġneuron": 34090, "graph": 34091, "ľë©´": 34092, "Ġenclosure": 34093, "ĠJohann": 34094, "Ġaftermath": 34095, "ÑĤоб": 34096, "Ġuży": 34097, "Ġsamp": 34098, "360": 34099, "ĠMei": 34100, "Ġtaco": 34101, "Ġreceptors": 34102, "Ġpunches": 34103, "ĠHoje": 34104, "ĠÙĩÙĨا": 34105, "=\"#": 34106, "ĠAngular": 34107, "Ġmusique": 34108, "Ġrol": 34109, "Ġñ": 34110, "sterreich": 34111, "Ġclam": 34112, "ĠTreasury": 34113, "chemical": 34114, "Ġapar": 34115, "Ġappend": 34116, "Ġforbid": 34117, "ĠHamburg": 34118, "аков": 34119, "Ġê¸Ī": 34120, "ilda": 34121, "Ġpreparations": 34122, "ĠmogÄħ": 34123, "Ġcamino": 34124, "Eric": 34125, "ĠBlind": 34126, "èĪĩ": 34127, "å¹´çļĦ": 34128, "ĠDiscovery": 34129, "ì¸ł": 34130, "çĪ¶": 34131, "Ġinterpreter": 34132, "Ġbred": 34133, "ĠPsalm": 34134, "Ġdefended": 34135, "ìī¬": 34136, "ĠErfahr": 34137, "ĠPeach": 34138, "Ġmoons": 34139, "ĠOst": 34140, "Ġspécial": 34141, "Ġarriver": 34142, "ĠWis": 34143, "uci": 34144, "Ġrobotics": 34145, "IVE": 34146, "Ġsiege": 34147, "arla": 34148, "Ġseparates": 34149, "ĠTC": 34150, "íı°": 34151, "quisite": 34152, "Ġparentheses": 34153, "ике": 34154, "ç«Ļ": 34155, "Ġtrous": 34156, "建": 34157, "ĠÑģилÑĮ": 34158, "Ġbeers": 34159, "ĠплаÑĤ": 34160, "ãģĻãģĶãģĦ": 34161, "Ġsola": 34162, "Ġdès": 34163, "mingham": 34164, "ikte": 34165, "Ġoops": 34166, "Ġtwitch": 34167, "å°ĩ": 34168, "ÏĪ": 34169, "ĠShouldn": 34170, "uvre": 34171, "Ġleer": 34172, "criptions": 34173, "Ġeyeshadow": 34174, "ĠGuo": 34175, "ĠPowell": 34176, "Ġsupuesto": 34177, "Ġana": 34178, "rals": 34179, "ĠMontreal": 34180, "Ġsurfing": 34181, "ĠÐŁÐµÑĢв": 34182, "×ŀ×ķ": 34183, "Ġmilliseconds": 34184, "Ġsuburbs": 34185, "Ġplaneta": 34186, "ÑĥÑĪка": 34187, "hrlich": 34188, "ĠHY": 34189, "ĠسÛĴ": 34190, "ĠMM": 34191, "ĠEff": 34192, "åı¯æĦĽ": 34193, "ĠHS": 34194, "anson": 34195, "Ġì§ģìłij": 34196, "Ġsuo": 34197, "Ġdeploying": 34198, "Ġkunt": 34199, "tering": 34200, "Ġerect": 34201, "ìŀ¥ìĿ´": 34202, "ĠìĿĮìĭĿ": 34203, "Ġspecimen": 34204, "!...": 34205, "æĪij說": 34206, "Ġligne": 34207, "Ġkonst": 34208, "adequ": 34209, "Ġìĥģíĥľ": 34210, "Ġaccessed": 34211, "ĠPole": 34212, "kill": 34213, "Ġë²Ħë": 34214, "Ġauthenticity": 34215, "Ġappelle": 34216, "ulle": 34217, "Ġrevision": 34218, "Ġgoats": 34219, "гли": 34220, "Ġpau": 34221, "ĠRanger": 34222, "ĠImag": 34223, "author": 34224, "Ġeve": 34225, "ĠMessenger": 34226, "Ġnay": 34227, "Ġwholes": 34228, "ätte": 34229, "Ġonwards": 34230, "ĠDepois": 34231, "ĠíijľíĺĦ": 34232, "ĠSARS": 34233, "Ġwszystkich": 34234, "Ġdestru": 34235, "umbing": 34236, "Ġcompatibility": 34237, "Ġmisinformation": 34238, "odore": 34239, "ĠFavor": 34240, "eko": 34241, "ıĮ": 34242, "waukee": 34243, "ĠTeaching": 34244, "ĠKO": 34245, "Ġbetting": 34246, "Ġquests": 34247, "Ġvivre": 34248, "ĠмÑĥзÑĭ": 34249, "Ġsaga": 34250, "Ġswell": 34251, "Ġgehe": 34252, "æĢİ麼樣": 34253, "ĠоÑĢганиз": 34254, "Ġgide": 34255, "ĠGross": 34256, "Ġdalej": 34257, "Ġclaws": 34258, "á»Ļc": 34259, "Ġprejudice": 34260, "Ġinsign": 34261, "ihood": 34262, "Ġpled": 34263, "Ġdónde": 34264, "ĠPolitical": 34265, "Ġpremises": 34266, "undert": 34267, "عت": 34268, "onnen": 34269, "Ġespaço": 34270, "Ġfé": 34271, "ĠHarrison": 34272, "ĠCensus": 34273, "Ġcardio": 34274, "Ġdiy": 34275, "Ġmilieu": 34276, "Ġjournée": 34277, "ĠRelease": 34278, "NIE": 34279, "ĠMuk": 34280, "idée": 34281, "á»įi": 34282, "Ġiçinde": 34283, "ŀĻ": 34284, "Ġresonate": 34285, "Ġmoles": 34286, "ĠFlying": 34287, "ĠGloria": 34288, "ĠPastor": 34289, "ĠArena": 34290, "好ä¸į好": 34291, "NON": 34292, "олов": 34293, "ĠallÃŃ": 34294, "omat": 34295, "ìĸ´ëıĦ": 34296, "ĠcaracterÃŃst": 34297, "Ġdeclining": 34298, "ÑĸÑı": 34299, "anco": 34300, "ĠInform": 34301, "Ġbargain": 34302, "Ġbushes": 34303, "ĠNaturally": 34304, "Ġrechts": 34305, "ĠTensor": 34306, "ĠPatricia": 34307, "Ġprincipio": 34308, "ĠMumbai": 34309, "Ġwomb": 34310, "Ġnostra": 34311, "Ġdilemma": 34312, "Ġirgendwann": 34313, "Ġ1964": 34314, "ĠenergÃŃa": 34315, "ĠнаÑĢ": 34316, "Ġsegregation": 34317, "ĠAthlet": 34318, "Ġ»,": 34319, "Ġyeni": 34320, "ĠSeit": 34321, "Ġvenom": 34322, "Ġdakika": 34323, "ĠëıĮë": 34324, "ĠÃīl": 34325, "Ġfus": 34326, "ĠMog": 34327, "¦½ëĭĪëĭ¤": 34328, "Ġremar": 34329, "ĠTeddy": 34330, "Ġbreasts": 34331, "icans": 34332, "æĶ¶çľĭ": 34333, "kap": 34334, "ĠhÆ¡n": 34335, "ĠJP": 34336, "ãĥ³ãĤ¿": 34337, "Ġresurrect": 34338, "ĠìĿ¸ë": 34339, "herical": 34340, "Ġfotograf": 34341, "ĠJosé": 34342, "Ġlivelihood": 34343, "Ġbibli": 34344, "teri": 34345, "Ġvorstellen": 34346, "ĠAAA": 34347, "Ġassessing": 34348, "YA": 34349, "Ġsplend": 34350, "Ġexcav": 34351, "Ġbaptism": 34352, "yll": 34353, "wow": 34354, "Mac": 34355, "Ġplastics": 34356, "teokbokki": 34357, "Ġintéressant": 34358, "Ġcommanded": 34359, "Ġfamously": 34360, "ĠÐĺли": 34361, "ĠManuel": 34362, "Ġsouthwest": 34363, "Ġdeformation": 34364, "ÃŃculo": 34365, "ĠнаÑħодиÑĤÑģÑı": 34366, "ĠPatter": 34367, "degree": 34368, "ĠczÄĻsto": 34369, "\"-": 34370, "Ġìħĭ": 34371, "Ġmanger": 34372, "ĠTrustee": 34373, "Ģ리": 34374, "Ġpuntos": 34375, "ivable": 34376, "Ġvolatile": 34377, "ĠëĬIJ": 34378, "Ġinstability": 34379, "Ġciel": 34380, "ciÄħ": 34381, "Ġpurity": 34382, "ноÑģÑĤ": 34383, "Sil": 34384, "edar": 34385, "åĻ¨": 34386, "NOUNCER": 34387, "Ġspelled": 34388, "GER": 34389, "Ġsanctuary": 34390, "Ġaccelerating": 34391, "Ġscout": 34392, "ĠпÑĢев": 34393, "fahren": 34394, "ãģĵãģ¡ãĤī": 34395, "ĠëĤĺìĺ¨": 34396, "ĠpoczÄħt": 34397, "ĠMeu": 34398, "kaar": 34399, "³´ê³ł": 34400, "akra": 34401, "Down": 34402, "ĠÃĦr": 34403, "ĠElite": 34404, "Ġallons": 34405, "Ġmayonnaise": 34406, "ĠSustain": 34407, "prisingly": 34408, "Ġsupervis": 34409, "Ġê·¸ëłĩì£ł": 34410, "Ġunemployed": 34411, "Ġfreshly": 34412, "Ġ×ŀ×¢": 34413, "ĠDh": 34414, "Ġtackling": 34415, "Ġogr": 34416, "Ġì´Īë": 34417, "ãĤĪãĤį": 34418, "Ġloft": 34419, "arah": 34420, "ĠAirl": 34421, "ĠDir": 34422, "ĠÐľÐ¾Ð¶Ð½Ð¾": 34423, "Ġbooking": 34424, "ĠCRA": 34425, "Ġhttps": 34426, "Ġchoke": 34427, "Ġgown": 34428, "Ġnoite": 34429, "Ġzac": 34430, "istol": 34431, "Ġsecre": 34432, "Ġresembles": 34433, "Ġcuad": 34434, "ìĤ¬ê°Ģ": 34435, "show": 34436, "Ġblanc": 34437, "Ġagu": 34438, "ĠPrint": 34439, "asted": 34440, "ĠWeather": 34441, "ipl": 34442, "Ġobscure": 34443, "Ġconte": 34444, "oughs": 34445, ");": 34446, "ĠDame": 34447, "ä¸Ģ缴": 34448, "Ġclarification": 34449, "Ġintimacy": 34450, "Ġuphold": 34451, "ĠMirror": 34452, "Ġwagon": 34453, "xide": 34454, "Ġclog": 34455, "apper": 34456, "ĠImmediately": 34457, "úde": 34458, "Ġtouchdown": 34459, "Ġrooft": 34460, "аÑĪа": 34461, "Ġçıkt": 34462, "Ġlaisser": 34463, "ĠUnreal": 34464, "ensitive": 34465, "Ġ123": 34466, "Ġplaster": 34467, "Ġducks": 34468, "Ġetme": 34469, "Ġbishop": 34470, "brevi": 34471, "Ġbic": 34472, "ä¸ĭåİ»": 34473, "Ġruntime": 34474, "Ġambitions": 34475, "маÑĤ": 34476, "ĠWein": 34477, "ĠMari": 34478, "ĠíĬ¸ë": 34479, "Ġresolver": 34480, "ĠngÃły": 34481, "ĠRise": 34482, "ãĤĪãģĨãģ«": 34483, "ĠCrus": 34484, "Ġmerchandise": 34485, "Ġeli": 34486, "Ġstatewide": 34487, "Ġowl": 34488, "éģł": 34489, "æĶ¹": 34490, "Ġtwisting": 34491, "Ġcontaminated": 34492, "ĠCommerce": 34493, "hythm": 34494, "ĠÃĪ": 34495, "Ġìĭ¤ë": 34496, "Ġmusste": 34497, "uir": 34498, "Ġsums": 34499, "ĠSomewhere": 34500, "ãĥİ": 34501, "Ġkami": 34502, "Ġaired": 34503, "ĠANDREW": 34504, "Ġêº": 34505, "Ġviendo": 34506, "Ġantibody": 34507, "Ġabsolument": 34508, "Ġprotesters": 34509, "ĠQuébec": 34510, "stadt": 34511, "Shaun": 34512, "Ġchambers": 34513, "ĠWear": 34514, "ĠEffects": 34515, "Ġhazards": 34516, "Ġnei": 34517, "Ġcorazón": 34518, "Ġá¼": 34519, "ĠSG": 34520, "Ķ©": 34521, "ĠìĹŃìĭľ": 34522, "Ġcomfy": 34523, "ĠCody": 34524, "Ġpensando": 34525, "Ġganska": 34526, "ĠAcross": 34527, "öllig": 34528, "abyte": 34529, "Ġwedge": 34530, "Ġkalian": 34531, "Ġsigue": 34532, "endes": 34533, "ĠGroÃŁ": 34534, "Ġutiliser": 34535, "Ġflown": 34536, "аниÑİ": 34537, "Ġlevar": 34538, "restrial": 34539, "Ġillustrations": 34540, "Ġaslında": 34541, "BLEEP": 34542, "ĠдоÑģÑĤ": 34543, "Ġturret": 34544, "Ġsuitcase": 34545, "ziÄĻki": 34546, "Ġsketches": 34547, "Ġacred": 34548, "ĠRei": 34549, "Ġtsun": 34550, "ĠSag": 34551, "Ġthirds": 34552, "ĠKIRBY": 34553, "rai": 34554, "Ġhumanos": 34555, "Ġrecommends": 34556, "Ġextraordinarily": 34557, "Ġcommencement": 34558, "KN": 34559, "opez": 34560, "Ġ×ijש": 34561, "Ġlethal": 34562, "ĠEstamos": 34563, "Ġinspector": 34564, "ĠSeok": 34565, "eun": 34566, "Ġoffshore": 34567, "Ġgettin": 34568, "years": 34569, "ĠSilence": 34570, "ĠNatur": 34571, "upun": 34572, "Ġtrzy": 34573, "Ġnoget": 34574, "Ġhamburger": 34575, "ĠPraise": 34576, "énd": 34577, "Ġ1971": 34578, "ylie": 34579, "krit": 34580, "ĠìĥĿê°ģìĿ´": 34581, "çļ®": 34582, "Ġmomentos": 34583, "Ġesté": 34584, "Ġdissemin": 34585, "Ġgigs": 34586, "Ġdesaf": 34587, "Ġavis": 34588, "ĠZoo": 34589, "ĠìķĬìĿĢ": 34590, "häng": 34591, "åı¥": 34592, "hake": 34593, "ĠBism": 34594, "Ġrethink": 34595, "ĠMalcolm": 34596, "Ġidentifies": 34597, "lower": 34598, "ixel": 34599, "ĠtvÃ¥": 34600, "ked": 34601, "ierz": 34602, "Ġöffentlich": 34603, "Ġproclaim": 34604, "soon": 34605, "lol": 34606, "Ġloi": 34607, "Ġbitten": 34608, "rollo": 34609, "Ġsermon": 34610, "Ġesqu": 34611, "Ġjackets": 34612, "Ġgráfic": 34613, "ĠпоказÑĭв": 34614, "Ġcabeza": 34615, "chodzi": 34616, "Ġpelvis": 34617, "Ġnostalgia": 34618, "Ġbrew": 34619, "Ġshortcuts": 34620, "ĠAdemás": 34621, "Ġsuperficial": 34622, "åħ©åĢĭ": 34623, "Ġboca": 34624, "ĠæĪijæĺ¯": 34625, "imentos": 34626, "åĽłä¸º": 34627, "Ġsprouts": 34628, "é£Ľ": 34629, "ĠJonas": 34630, "ĠFlorence": 34631, "static": 34632, "daughter": 34633, "*)": 34634, "ÅĤby": 34635, "fashion": 34636, "ĠGinger": 34637, "Ġ매ë": 34638, "Ġhustle": 34639, "utos": 34640, "ĠÑĤÑıж": 34641, "ĠLös": 34642, "ש×Ļ×Ŀ": 34643, "anych": 34644, "tuber": 34645, "Ġtidy": 34646, "Ġfrontal": 34647, "Ġwhiskey": 34648, "Ġhumid": 34649, "ĠÎŁ": 34650, "Ġridge": 34651, "Ġmarin": 34652, "Ġbientôt": 34653, "ĠCarrie": 34654, "chw": 34655, "Ġtahun": 34656, "ĠErgeb": 34657, "FR": 34658, "Ġìłķë¶Ģ": 34659, "ĠSoldier": 34660, "Ġenlightenment": 34661, "Ġexamining": 34662, "ĠNotre": 34663, "Ġeram": 34664, "ĠSunny": 34665, "Ġlayered": 34666, "ĠDazu": 34667, "rades": 34668, "好åIJĥ": 34669, "ĠнаÑĪей": 34670, "Ġtimber": 34671, "Ġmanners": 34672, "ĠBirmingham": 34673, "Ġminiature": 34674, "ometers": 34675, "Ġfiller": 34676, "ĠRip": 34677, "ĠKomb": 34678, "owner": 34679, "ì¿": 34680, "idian": 34681, "Ġdemás": 34682, "ĠÙĪت": 34683, "Ġprecautions": 34684, "Ġgoverno": 34685, "zelf": 34686, "ĠComplete": 34687, "å¸ĥ": 34688, "ĠPhantom": 34689, "ãģ¾ãģļ": 34690, "Ġнез": 34691, "ĠкаÑĢÑĤ": 34692, "ĠAntwort": 34693, "ĠPfizer": 34694, "ĠFranco": 34695, "ĠwÅĤ": 34696, "Ġfrig": 34697, "esper": 34698, "Ġkale": 34699, "Ġfilmmaker": 34700, "Ġkurt": 34701, "Ġinvalid": 34702, "å±Ģ": 34703, "arella": 34704, "Äĥng": 34705, "ramento": 34706, "Ġnutritional": 34707, "Ġdictators": 34708, "Ġafin": 34709, "Ġfuzzy": 34710, "ĠGina": 34711, "ót": 34712, "ĠExtremadura": 34713, "Ġdemonstrations": 34714, "ĠMontgomery": 34715, "íķ´ìĦ¤": 34716, "ĠGandhi": 34717, "ãĥĿ": 34718, "ç½®": 34719, "Ġreunion": 34720, "ĠjakiÅĽ": 34721, "ĠZug": 34722, "OUGH": 34723, "lifting": 34724, "Ġà²": 34725, "á¹Ľá¹£": 34726, "eb": 34727, "ĠWOW": 34728, "ĠShiva": 34729, "ometry": 34730, "Ġwildly": 34731, "Ġtended": 34732, "Ġmegap": 34733, "ì²ĺ": 34734, "Ġnause": 34735, "Ġgerek": 34736, "ãĥĭ": 34737, "ĠMarcel": 34738, "Ġneste": 34739, "خر": 34740, "Ġfeh": 34741, "åĨħ": 34742, "suspenseful": 34743, "ĠWrestle": 34744, "ĠPalestinians": 34745, "ĠGORD": 34746, "iyet": 34747, "ĠÑĢади": 34748, "Ġversuchen": 34749, "Ġtransistor": 34750, "ĠÐŁÑĢоÑģÑĤо": 34751, "ĠпонÑĢав": 34752, "Ġrhyme": 34753, "ĠVermont": 34754, "platz": 34755, "è®°": 34756, "ĠÄ°ÅŁte": 34757, "ĠHag": 34758, "ĠÐĺм": 34759, "ĠÑĢаÑģÑģказ": 34760, "Ġmetros": 34761, "ĠInfinity": 34762, "wolf": 34763, "ibal": 34764, "ftig": 34765, "ĠÚĨ": 34766, "Ġíĺ¹ìĭľ": 34767, "Ġoggi": 34768, "Ġdisposit": 34769, "ĠпÑĢил": 34770, "ĠвÑĭпол": 34771, "Ġthôi": 34772, "ĠKENN": 34773, "Ġhanding": 34774, "actus": 34775, "Ġtacos": 34776, "Ġformerly": 34777, "ĠCorinthians": 34778, "ãģ«ãģ¯": 34779, "ÑĨÑĸÑĹ": 34780, "Ġpadre": 34781, "Ġcongregation": 34782, "æij": 34783, "fert": 34784, "Ġsubir": 34785, "aiser": 34786, "qua": 34787, "araoh": 34788, "ĠCurry": 34789, "ĠìķĬëĬĶ": 34790, "елÑİ": 34791, "Ġfuss": 34792, "Ġbooty": 34793, "Ġlows": 34794, "Ġhommes": 34795, "ĠMH": 34796, "ĠDisneyland": 34797, "went": 34798, "Ġresidue": 34799, "Ġbeeping": 34800, "è¼ķ": 34801, "ätta": 34802, "Ġmould": 34803, "ĠProjekt": 34804, "stalk": 34805, "Ġartifact": 34806, "ĠAntrag": 34807, "ĠAMD": 34808, "ĠCrypt": 34809, "Ġë©Ķ": 34810, "ĠFelipe": 34811, "ĠCOB": 34812, "elu": 34813, "Ġselfies": 34814, "ĠSanti": 34815, "chutz": 34816, "ĠУкÑĢаÑĹ": 34817, "gesamt": 34818, "Ġflock": 34819, "jaz": 34820, "plain": 34821, "Ġwrinkles": 34822, "Ġreais": 34823, "Ġpaljon": 34824, "Ġempowerment": 34825, "Ġattendees": 34826, "ppa": 34827, "Ġneden": 34828, "онÑĭ": 34829, "Ġtimeframe": 34830, "ĠCherry": 34831, "Ġidée": 34832, "Ġgag": 34833, "Ġdonkey": 34834, "Ġông": 34835, "ĠHare": 34836, "éļĽ": 34837, "ĠKara": 34838, "Ġacompan": 34839, "places": 34840, "imientos": 34841, "ĠHamm": 34842, "би": 34843, "uben": 34844, "iliyor": 34845, "Ġthirst": 34846, "Ġkry": 34847, "ĠGeorgetown": 34848, "׳×Ķ": 34849, "Ġorch": 34850, "Ġheartbeat": 34851, "Ġtransformations": 34852, "estones": 34853, "ĠKH": 34854, "Ġcartoons": 34855, "Ġanci": 34856, "Ġworthless": 34857, "Ġtailored": 34858, "pu": 34859, "Americans": 34860, "Ġpiles": 34861, "ĠMonkey": 34862, "Ġbasin": 34863, "ĠTemper": 34864, "ĠPaint": 34865, "Ġpunching": 34866, "Ġbaik": 34867, "ĠOakland": 34868, "vre": 34869, "ÅŁallah": 34870, "ydd": 34871, "Ġcasually": 34872, "odu": 34873, "Ġcoded": 34874, "ĠNorwegian": 34875, "ĠVince": 34876, "Ġpremature": 34877, "ĠPromise": 34878, "екÑģÑĤ": 34879, "Ġdevastated": 34880, "ĠPremium": 34881, "ĠParam": 34882, "ĠÃĸyle": 34883, "umuz": 34884, "PO": 34885, "rators": 34886, "Ġlamps": 34887, "Ġterritorial": 34888, "Ġbackbone": 34889, "listed": 34890, "DY": 34891, "ĠاÙĦر": 34892, "Ġpursued": 34893, "ĠCommons": 34894, "Ġ곡": 34895, "locks": 34896, "edor": 34897, "Ġconceived": 34898, "gere": 34899, "Ġdisappearing": 34900, "ĠSull": 34901, "ĠìĹ°ë": 34902, "Ġhoffe": 34903, "Ġdetox": 34904, "íĶĮ": 34905, "Ġretir": 34906, "ĠëģĿëĤ": 34907, "Ġpergunta": 34908, "ĠBOY": 34909, "ç²¾": 34910, "Ġpenn": 34911, "æĿ¥äºĨ": 34912, "hés": 34913, "hon": 34914, "Ġcatastrophic": 34915, "Ġaust": 34916, "Ġtorso": 34917, "Ġìĸ´ëĬIJ": 34918, "ĠìĤ¬ëŀĮëĵ¤ìĿ´": 34919, "Ġmarvelous": 34920, "ĠHarley": 34921, "achine": 34922, "Ġtiế": 34923, "itto": 34924, "ĠIÃŃm": 34925, "ylon": 34926, "Ġshutdown": 34927, ".''": 34928, "Ġapologies": 34929, "ĠCommunication": 34930, "ĠговоÑĢÑİ": 34931, "ãģĤãĥ¼": 34932, "âĦ¢": 34933, "ÃŃveis": 34934, "acun": 34935, "Ġretaining": 34936, "Ġcontradiction": 34937, "ĠADAM": 34938, "COM": 34939, "Bryan": 34940, "ĠMonsieur": 34941, "Ġadapting": 34942, "ШÐIJ": 34943, "ĠScr": 34944, "ändert": 34945, "Ġplaus": 34946, "ä»Ĭ天çļĦ": 34947, "Ġonset": 34948, "Ġassistants": 34949, "Ġvalves": 34950, "Ġscatter": 34951, "ĠRust": 34952, "awia": 34953, "Ġreadiness": 34954, "Ġpais": 34955, "Ġbible": 34956, "Ġambiente": 34957, "ĠамеÑĢик": 34958, "Ġuncond": 34959, "Ġkalk": 34960, "åĬ¨": 34961, "Ġmoc": 34962, "unn": 34963, "Ġactu": 34964, "Ġhumming": 34965, "issimo": 34966, "ĠPatrol": 34967, "gow": 34968, "ãĥ¤": 34969, "ĠTHEY": 34970, "ĠBoden": 34971, "ĠBie": 34972, "Ġreel": 34973, "ĠÑĥÑģлов": 34974, "Ġendeavor": 34975, "ĠPeriod": 34976, "ustomed": 34977, "mals": 34978, "alon": 34979, "Box": 34980, "ĠÏĥαÏĤ": 34981, "Ġomdat": 34982, "Ġaltre": 34983, "ĠHeh": 34984, "kad": 34985, "Ġprotector": 34986, "Ġdominance": 34987, "odynamic": 34988, "Ġcommunicated": 34989, "kö": 34990, "Ġpredecessor": 34991, "ĠLuk": 34992, "ĠFlower": 34993, "Ġãģ©": 34994, "poque": 34995, "ÑĤиÑĢов": 34996, "Ġretrospect": 34997, "Ġdecisive": 34998, "Ġexempel": 34999, "{\\": 35000, "ĠRück": 35001, "rite": 35002, "ĠZeus": 35003, "Ġcalorie": 35004, "Ġattractions": 35005, "ĠHinter": 35006, "Ġuhm": 35007, "ĠíĮIJ": 35008, "Ġrulers": 35009, "Ġdiscouraged": 35010, "Ġacontecer": 35011, "Ġaccents": 35012, "ĠOptim": 35013, "ĠAlg": 35014, "kids": 35015, "2021": 35016, "ĠLindsay": 35017, "Ġfilmmakers": 35018, "prowad": 35019, "Ġterug": 35020, "ëĭ´": 35021, "ĠSommer": 35022, "2018": 35023, "Ġborrowing": 35024, "ĠTransfer": 35025, "ноп": 35026, "arias": 35027, "Ġheadphone": 35028, "ì¼ľ": 35029, "Ġtranslating": 35030, "Ġaufge": 35031, "à®ªà®Ł": 35032, "weis": 35033, "avant": 35034, "paid": 35035, "baby": 35036, "Ġtoughest": 35037, "Ġrepeats": 35038, "ĠTeresa": 35039, "Lord": 35040, "Ġacabar": 35041, "ĠRide": 35042, "dir": 35043, "Ġleng": 35044, "Ġdwa": 35045, "Ġheadaches": 35046, "Ġnữa": 35047, "ĠнаÑģÑĤоÑıÑī": 35048, "Ġboils": 35049, "Ġlonging": 35050, "rias": 35051, "ório": 35052, "ĠParadise": 35053, "ĠSeñor": 35054, "erdem": 35055, "Ġreinst": 35056, "Ġsalaries": 35057, "Ġinsecurity": 35058, "ÅĤoÅĽci": 35059, "ĠабÑģолÑİÑĤно": 35060, "inken": 35061, "ĠEddy": 35062, "udos": 35063, "Ġdummy": 35064, "Ðļак": 35065, "six": 35066, "Ġinbox": 35067, "ẩ": 35068, "People": 35069, "á»ĵng": 35070, "Ġorganizers": 35071, "find": 35072, "Ġül": 35073, "ĠCOM": 35074, "ża": 35075, "weile": 35076, "Commentary": 35077, "íĬ¸ë¥¼": 35078, "ĠMittel": 35079, "kus": 35080, "èĽĭ": 35081, "न": 35082, "iral": 35083, "Ġgarment": 35084, "ικά": 35085, "Ġstool": 35086, "payers": 35087, "Ġshimmer": 35088, "ĠOllie": 35089, "ĠJeżeli": 35090, "è¿ĺæľī": 35091, "Ġ1977": 35092, "Ġjeux": 35093, "Ġextinct": 35094, "ĠTransportation": 35095, "ĠMaker": 35096, "Ġjohn": 35097, "Ġrichest": 35098, "Ġtraumat": 35099, "Ġliegen": 35100, "´ë¥¼": 35101, "è¿ĻéĩĮ": 35102, "Ġunrest": 35103, "ĠStraw": 35104, "æĭľæĭľ": 35105, "Ġcoma": 35106, "ĠKristen": 35107, "ĠÐļонеÑĩно": 35108, "ĠBryce": 35109, "ĠÑıкÑĸ": 35110, "Ġpearls": 35111, "ĠпонимаÑİ": 35112, "Ġadditions": 35113, "Ġasympt": 35114, "ĠменÑĮÑĪе": 35115, "Ġscans": 35116, "Child": 35117, "ĠHide": 35118, "кÑĥÑİ": 35119, "etas": 35120, "Ġdank": 35121, "Ġpleas": 35122, "Ġessays": 35123, "Ġjets": 35124, "åħĴ": 35125, "Ġвед": 35126, "Ġpositives": 35127, "hof": 35128, "-)": 35129, "zzo": 35130, "Ġstarters": 35131, "Ġsmiled": 35132, "Ġ1944": 35133, "quiera": 35134, "Ġrok": 35135, "Ġpuesto": 35136, "Nico": 35137, "Ġsimulations": 35138, "Ġà¶": 35139, "Ġintrigued": 35140, "ĠOverwatch": 35141, "åĸĤ": 35142, "sigh": 35143, "bai": 35144, "Ġë§IJê³ł": 35145, "idé": 35146, "Ġcrabs": 35147, "áºŃp": 35148, "ĠIraqi": 35149, "ìĿ´ë¥¼": 35150, "ÑĤÑı": 35151, "ĠSophia": 35152, "ĠDNS": 35153, "Ġönemli": 35154, "ĠLuo": 35155, "Ŀ¤": 35156, "ĠCounsel": 35157, "ligen": 35158, "анÑĮÑĪе": 35159, "Ġtrumpet": 35160, "Ġdapat": 35161, "ĠJM": 35162, "ĠEVERY": 35163, "Ġå°įä¸įå°į": 35164, "夢": 35165, "ĠLayer": 35166, "Ġcô": 35167, "нал": 35168, "ĠJoo": 35169, "ĠHack": 35170, "Ġsunt": 35171, "ĠLeonard": 35172, "ĠFirebase": 35173, "änger": 35174, "Ġexploding": 35175, "voy": 35176, "Ġì¦IJ": 35177, "ĠÑģеÑĢÑĮ": 35178, "Ġseverity": 35179, "Ġbestimm": 35180, "çµIJæŀľ": 35181, "Ġtiring": 35182, "Ġprocurement": 35183, "Ġdiplomacy": 35184, "Ġdecorative": 35185, "ĠÙĬا": 35186, "Ġpenetration": 35187, "Õ«": 35188, "Ġoutright": 35189, "ENE": 35190, "ĠUni": 35191, "odles": 35192, "Ġzeros": 35193, "Ġdelightful": 35194, "jm": 35195, "Ġdopo": 35196, "没äºĭ": 35197, "Ġpositivity": 35198, "ĠVISTA": 35199, "ĠResource": 35200, "íĥĢë": 35201, "ÑĪие": 35202, "Carl": 35203, "Ġpiping": 35204, "Ġchopping": 35205, "ĠGanze": 35206, "üss": 35207, "ĠAo": 35208, "Ġshattered": 35209, "ĠDetective": 35210, "Ġundoubtedly": 35211, "Ġhalluc": 35212, "Ġench": 35213, "ÑĭÑĩно": 35214, "ÑĥлÑıÑĢ": 35215, "isesti": 35216, "Ġpedals": 35217, "Ġdurum": 35218, "¤íĶ": 35219, "laimer": 35220, "Ġpropre": 35221, "Cu": 35222, "Ġtranslator": 35223, "ĠcaÅĤ": 35224, "Ġ그걸": 35225, "ĠcaÅĤy": 35226, "UA": 35227, "Ġrevised": 35228, "Ġподоб": 35229, "ĠArticle": 35230, "ĠHaiti": 35231, "ĠÃĵ": 35232, "ĠCtrl": 35233, "Ġrozm": 35234, "lait": 35235, "Ġletzte": 35236, "ispering": 35237, "display": 35238, "Ġaluminium": 35239, "Ġpalabras": 35240, "Ġconocer": 35241, "Ġzitten": 35242, "Ġdirig": 35243, "åıªæľī": 35244, "Ġbrainstorm": 35245, "Ġwifi": 35246, "ĠParticip": 35247, "Ġviewpoint": 35248, "ĠQuan": 35249, "Ġhierarch": 35250, "Welcome": 35251, "対": 35252, "Ġoffen": 35253, "ĠRecovery": 35254, "gano": 35255, "Would": 35256, "Ġrepro": 35257, "Ġperceptions": 35258, "Ġdemasi": 35259, "ĠBangladesh": 35260, "ĠIncredible": 35261, "Ġletzt": 35262, "Ġbehaving": 35263, "Ġastonishing": 35264, "ĠâĨ": 35265, "ĠëĤ¨ìŀIJ": 35266, "èµ°äºĨ": 35267, "ãĥĶ": 35268, "ĠGORDON": 35269, "CAR": 35270, "?!\"": 35271, "ĠPrest": 35272, "Ġë§ŀìķĦìļĶ": 35273, "Ġtand": 35274, "Ġlash": 35275, "çĬ": 35276, "ificant": 35277, "Ġintoler": 35278, "ĠгеÑĢо": 35279, "Ġteu": 35280, "aso": 35281, "ĠÑģовеÑĤ": 35282, "Ġtravelers": 35283, "ĠSynd": 35284, "ĠвеÑĢÑģ": 35285, "Fonda": 35286, "adı": 35287, "Ġtranscription": 35288, "Ġtitanium": 35289, "Ġtwists": 35290, "Ġgearbox": 35291, "ensation": 35292, "fat": 35293, "Coll": 35294, "ĠCommonwealth": 35295, "zon": 35296, "ĠPolizei": 35297, "ĠAPPLAUSE": 35298, "fry": 35299, "ĠJuda": 35300, "esteem": 35301, "Ġsock": 35302, "ĠJugend": 35303, "ĠкÑģÑĤаÑĤи": 35304, "ĠDro": 35305, "Ġprochaine": 35306, "ãĥ¼ãĥ«": 35307, "Ġliksom": 35308, "ĠEnergie": 35309, "ĠMarina": 35310, "Ġ230": 35311, "Ġê°ĢìĦľ": 35312, "umping": 35313, "Ġlone": 35314, "ç´ļ": 35315, "Ġfonts": 35316, "Ġbusinessman": 35317, "Ġply": 35318, "Ġdoe": 35319, "grid": 35320, "ĠMilwaukee": 35321, "ĠEden": 35322, "!\".": 35323, "ĠÛĮÛģ": 35324, "ogens": 35325, "Ġteaser": 35326, "Ġquién": 35327, "Ġincentiv": 35328, "govern": 35329, "Ġchildcare": 35330, "Ġsneakers": 35331, "Ġimprisoned": 35332, "®": 35333, "иÑĤеÑģÑĮ": 35334, "anbul": 35335, "Ġregain": 35336, "Ġtranquil": 35337, "Redner": 35338, "鼨": 35339, "IFA": 35340, "Ġideological": 35341, "ĠmayorÃŃa": 35342, "Ġbureau": 35343, "eterm": 35344, "ĠDID": 35345, "ìĬ·": 35346, "Ġwaving": 35347, "Ġbeb": 35348, "Ġár": 35349, "Ġкв": 35350, "Ġenvoy": 35351, "anut": 35352, "икÑĥ": 35353, "ĠEnvironment": 35354, "ĠAssass": 35355, "ãĤĵãģ§": 35356, "ĠBread": 35357, "ĠТÑĥÑĤ": 35358, "Ġstaircase": 35359, "ĠDisease": 35360, "Ġaucun": 35361, "ĠëĭĪ": 35362, "Ġconfrontation": 35363, "Ġ1941": 35364, "Ġirony": 35365, "Ġworsh": 35366, "ãĤĮãĤĭ": 35367, "Ġfick": 35368, "ĠNaomi": 35369, "Ġbackside": 35370, "ieux": 35371, "Kap": 35372, "Ġvedere": 35373, "Ġlengthy": 35374, "Ġbreaker": 35375, "ĠRolle": 35376, "Ġpredator": 35377, "Ġnossos": 35378, "Ġadvertise": 35379, "è³ĩ": 35380, "ÑĢоде": 35381, "Rednerwechsel": 35382, "reten": 35383, "Ġcollectors": 35384, "ıģımız": 35385, "Ġtrig": 35386, "Ġaxes": 35387, "inters": 35388, "Ġpenalties": 35389, "ĠOsman": 35390, "ĠJenna": 35391, "Ġflakes": 35392, "Ġtrainers": 35393, "Ġstunned": 35394, "ĠScroll": 35395, "ĠPip": 35396, "ĠнаÑģÑĤ": 35397, "ĠnhÃł": 35398, "ĠSmack": 35399, "ẫn": 35400, "ratos": 35401, "ĠÑĢабоÑĤÑĭ": 35402, "Ġucz": 35403, "ĠLemon": 35404, "ĠSind": 35405, "Ġpsychic": 35406, "ĠAbg": 35407, "Ġmammals": 35408, "Ġimmersive": 35409, "Ġbots": 35410, "Ġverschiedene": 35411, "Ġgeral": 35412, "Ġfollower": 35413, "Ġä»ĸ": 35414, "Ġseguridad": 35415, "Ġimmersed": 35416, "feito": 35417, "cross": 35418, "Ġöld": 35419, "íĥĦ": 35420, "Ġãģĵãģ®": 35421, "Ġ×Ķ×Ļ×IJ": 35422, "ĠJian": 35423, "Ġbiliyor": 35424, "area": 35425, "Ġkaf": 35426, "Ġgodt": 35427, "çĽ¸ä¿¡": 35428, "Ġë°©ìĨ¡": 35429, "Ġdetriment": 35430, "æ¥ļ": 35431, "Ñĸл": 35432, "ĠÄijâu": 35433, "Ġchloride": 35434, "øre": 35435, "lei": 35436, "Ġmonte": 35437, "Ġdifférentes": 35438, "à¯ģ.": 35439, "Ġcaregivers": 35440, "Ġinadequ": 35441, "Ġfarewell": 35442, "ĠÑĤипа": 35443, "ontec": 35444, "ĠEph": 35445, "HHH": 35446, "ĠTodos": 35447, "ĠСШÐIJ": 35448, "Ġtrov": 35449, "Ġlige": 35450, "Ġcông": 35451, "ĠCiv": 35452, "Ġcapaz": 35453, "ĠVallahi": 35454, "Ġqueste": 35455, "Ġreplica": 35456, "سب": 35457, "zna": 35458, "ĠÑģлÑĥж": 35459, "ĠPT": 35460, "wave": 35461, "ieni": 35462, "Ġrelied": 35463, "develop": 35464, "Ġdeme": 35465, "ĠAman": 35466, "Ġ[...]": 35467, "Ġcompliments": 35468, "uais": 35469, "ĠíĮ¨": 35470, "Ġsmelling": 35471, "Ġdadurch": 35472, "ÙĪت": 35473, "Ġoranges": 35474, "Ġлай": 35475, "Ġstabilization": 35476, "åĢį": 35477, "ãĤĮãģŁ": 35478, "楽": 35479, "Ġappliances": 35480, "Ġhm": 35481, "ĥIJë©´": 35482, "odynamics": 35483, "ĠciÄĻ": 35484, "ĠCott": 35485, "MON": 35486, "ĠMang": 35487, "æĶ¯æĮģ": 35488, "Ġallerdings": 35489, "ική": 35490, "shots": 35491, "Ġts": 35492, "ĠGör": 35493, "ĠCHAR": 35494, "Ġ:(": 35495, "Ġwrath": 35496, "Ġfique": 35497, "Ġführen": 35498, "Ġtestament": 35499, "Ġ^^": 35500, "á¹Ľá¹£á¹ĩa": 35501, "ALD": 35502, "Ġtexto": 35503, "ĠDogs": 35504, "Ġsib": 35505, "Ġpathetic": 35506, "ocks": 35507, "Ġradically": 35508, "ĠMORE": 35509, "ĠJAMES": 35510, "Ġingl": 35511, "ĠTechnical": 35512, "Ġporch": 35513, "ĠUT": 35514, "ĠобÑıзаÑĤелÑĮно": 35515, "Ġrenewal": 35516, "Ġaesthetics": 35517, "ikum": 35518, "Ġbeverage": 35519, "dern": 35520, "Ġpredictive": 35521, "Ġchuy": 35522, "ĠRegarding": 35523, "ĠForward": 35524, "ĠÙĪÙĦ": 35525, "Ġcontextual": 35526, "Ġdwarf": 35527, "Ġprehe": 35528, "Ġgoverned": 35529, "ħĦ": 35530, "Ġtrabalhar": 35531, "Ġnegócio": 35532, "ĠболÑĮÑĪой": 35533, "еÑĩаÑĤ": 35534, "ĠдÑĥÑħ": 35535, "Ġfloods": 35536, "Ġbowling": 35537, "ĠOB": 35538, "ĠHär": 35539, "Ġgrading": 35540, "주ëĬĶ": 35541, "Ġgars": 35542, "dling": 35543, "Ġrak": 35544, "ëĪ": 35545, "creat": 35546, "ĠÑīе": 35547, "Ġneighbours": 35548, "food": 35549, "Query": 35550, "Ġheroin": 35551, "iceps": 35552, "ĠKinda": 35553, "NET": 35554, "Ġmari": 35555, "Ġimitate": 35556, "Ġachter": 35557, "Ġsettlements": 35558, "rare": 35559, "cciones": 35560, "Ġëĵľ": 35561, "Ġfik": 35562, "itung": 35563, "ĠмакÑģим": 35564, "Ġelf": 35565, "Ġdalla": 35566, "ĠPolsce": 35567, "ĠPul": 35568, "ЧÑĤо": 35569, "ĠMorgen": 35570, "ØŃÙħ": 35571, "Ġsupremacy": 35572, "Ġkys": 35573, "ĠHurricane": 35574, "ĠGTA": 35575, "ĠFeh": 35576, "Ġfinalmente": 35577, "mund": 35578, "ĠKrie": 35579, "époque": 35580, "ĠTucker": 35581, "ITT": 35582, "Ġlur": 35583, "Ġdipping": 35584, "äv": 35585, "Ġeerste": 35586, "ĠFlint": 35587, "bildung": 35588, "ูà¹ī": 35589, "Ġtoim": 35590, "Ġpracy": 35591, "Ġtransforms": 35592, "Ġspeeding": 35593, "Ġpresenter": 35594, "Ġfellows": 35595, "filled": 35596, "ieza": 35597, "Ġadvising": 35598, "ĠInterview": 35599, "игÑĢ": 35600, "wehr": 35601, "ĠDante": 35602, "pture": 35603, "Ī문": 35604, "¯¸ë": 35605, "IJIJ": 35606, "ĠCounter": 35607, "Ġcrist": 35608, "Ġì§ľ": 35609, "Ġjeune": 35610, "ĠÑģÑĤÑĢаÑĪ": 35611, "ĠmieÄĩ": 35612, "Ġtutor": 35613, "Ġmasala": 35614, "Ġpowdered": 35615, "Ġnau": 35616, "ĠFrederick": 35617, "Ġbilling": 35618, "ĠEisen": 35619, "ĠдобÑĢ": 35620, "Ġmest": 35621, "æ½": 35622, "Ġsnipp": 35623, "Ġmono": 35624, "ĠAlo": 35625, "ĠMercy": 35626, "érience": 35627, "Ġcasualties": 35628, "ĠANNOUNCER": 35629, "ä»İ": 35630, "Ġtocar": 35631, "Ġbacterial": 35632, "Ho": 35633, "Ġstreak": 35634, "ĠJENN": 35635, "Ġplast": 35636, "Ñģлед": 35637, "Ġreapp": 35638, "Ġpaycheck": 35639, "Ġminers": 35640, "habt": 35641, "ĠJap": 35642, "нÑĥÑĤ": 35643, "Ġredemption": 35644, "Ġquir": 35645, "hnlich": 35646, "Ġaccumulation": 35647, "Ġshove": 35648, "Ġadrenaline": 35649, "Make": 35650, "ĠHern": 35651, "ossing": 35652, "ĠVil": 35653, "ubby": 35654, "hertz": 35655, "breaks": 35656, "Ġspur": 35657, "ĠDaha": 35658, "USTIN": 35659, "Ġcontinuer": 35660, "ĠSaul": 35661, "ãģ®ãģ¯": 35662, "ĠíıŃ": 35663, "ĠëIJĺë©´": 35664, "Ġë§IJìĶĢ": 35665, "Ġож": 35666, "Ġsuspects": 35667, "Ġlaquelle": 35668, "ĠMuchas": 35669, "Ġvöllig": 35670, "ulen": 35671, "Ġimpres": 35672, "Ġlobb": 35673, "enee": 35674, "Ġнаж": 35675, "Ta": 35676, "Ġréalité": 35677, "ĠRex": 35678, "Ġharvesting": 35679, "Ġestr": 35680, "æ¶": 35681, "ospace": 35682, "OSS": 35683, "Ġdisturbance": 35684, "assic": 35685, "ĠIsab": 35686, "Ġdécouv": 35687, "ĠHampshire": 35688, "Ġornament": 35689, "Ġluôn": 35690, "ĠUW": 35691, "ĠjÄħ": 35692, "éĤ£ä¹Ī": 35693, "Ġrespecto": 35694, "Ġcomunidad": 35695, "Ġcomigo": 35696, "agna": 35697, "Ġintrinsic": 35698, "ĠAlumni": 35699, "Ġsesleri": 35700, "Ġestimation": 35701, "âĢĶâĢĶ": 35702, "Ġproduit": 35703, "ãĢĤãĢį": 35704, "ĠвÑĢ": 35705, "Ġwhirl": 35706, "Ġacces": 35707, "çu": 35708, "Ġvariability": 35709, "Ġvodka": 35710, "itsu": 35711, "Ġinternships": 35712, "Ġallocate": 35713, "RR": 35714, "íĽĪ": 35715, "Ġinstructional": 35716, "tant": 35717, "Ġà®ħத": 35718, "Ġinvites": 35719, "Ġhak": 35720, "Ġscares": 35721, "Ġeclipse": 35722, "пов": 35723, "колÑĮ": 35724, "ativas": 35725, "Ġstabbed": 35726, "ĠDOM": 35727, "ä¸įåĪ°": 35728, "roots": 35729, "ĠPicture": 35730, "íĺ¼": 35731, "ĠCHA": 35732, "iec": 35733, "ıı": 35734, "hanol": 35735, "Ġmisunderstand": 35736, "Ray": 35737, "Ġroadmap": 35738, "ocumented": 35739, "izione": 35740, "ĠOlive": 35741, "rift": 35742, "Ġ×Ķ׳": 35743, "æ¯į": 35744, "lest": 35745, ";;": 35746, "ĠEA": 35747, "éľĢè¦ģ": 35748, "одÑĥ": 35749, "Ġhobbies": 35750, "Ġburial": 35751, "ãģ«ãģ¡ãģ¯": 35752, "Ф": 35753, "lege": 35754, "ĠHJ": 35755, "Ġobjection": 35756, "ĠãģŃ": 35757, "ctory": 35758, "Ġincremental": 35759, "Ġgymn": 35760, "Ġepidemi": 35761, "ÑģÑĭл": 35762, "Ãij": 35763, "Ġadvancement": 35764, "Ġparch": 35765, "News": 35766, "Ġayr": 35767, "лам": 35768, "Ġ׾ש": 35769, "Ġdiploma": 35770, "ãģ¡ãĤĥãĤĵ": 35771, "Ġrobbed": 35772, "Only": 35773, "Ġincur": 35774, "Ġchanting": 35775, "Ġíķ´ëıĦ": 35776, "Ġriches": 35777, "ĠCarmen": 35778, "Ġnostro": 35779, "λÎŃ": 35780, "ĠPowder": 35781, "à¹Ģห": 35782, "ĠìŀĪìľ¼ë©´": 35783, "Ġgerçekten": 35784, "ĠPikachu": 35785, "емон": 35786, "OLL": 35787, "Ġplanetary": 35788, "Ġslows": 35789, "Ġclockwise": 35790, "alion": 35791, "ĠìĮ": 35792, "Ġvern": 35793, "Ġhomme": 35794, "Ġendpoint": 35795, "Ġinnocence": 35796, "Ġelementos": 35797, "Ġsophomore": 35798, "Ġnotions": 35799, "ĠCouldn": 35800, "pur": 35801, "Ġzat": 35802, "Ġobsess": 35803, "Ġmotivo": 35804, "ĠKub": 35805, "ĠDrug": 35806, "Ant": 35807, "ĠPlayers": 35808, "ĠHumans": 35809, "Ġmelee": 35810, "ĠWildlife": 35811, "ĠVP": 35812, "Ġvolcanic": 35813, "Ġcomin": 35814, "ĠGuang": 35815, "ĠÏĦιÏĤ": 35816, "ĠоÑģобенно": 35817, "ĠSize": 35818, "Listen": 35819, "ĠAaa": 35820, "appro": 35821, "Ġbarbar": 35822, "ĠParkinson": 35823, "нÑıÑĤÑĮ": 35824, "åį°": 35825, "Ġunderestimate": 35826, "Ġsubstitution": 35827, "Ġcosmetic": 35828, "ä¸ĭ次": 35829, "Ġwillen": 35830, "Ġbeide": 35831, "anni": 35832, "Ġconditioned": 35833, "ĠDebbie": 35834, "Ġisto": 35835, "ĠEdwards": 35836, "ìĽĮìļĶ": 35837, "ĠÑĤов": 35838, "Ġabbrevi": 35839, "ĠMün": 35840, "ĠPrinc": 35841, "ĠLiang": 35842, "Ġstink": 35843, "Ġradioactive": 35844, "ãģĨãĤı": 35845, "Ġacontec": 35846, "Ġuncon": 35847, "ĠTurbo": 35848, "ãģIJ": 35849, "Ġkisses": 35850, "æĺ¯ä»Ģ麼": 35851, "еÑĤÑĢов": 35852, "Ġfrontier": 35853, "ĠSpy": 35854, "ĠBelarus": 35855, "ĠCBS": 35856, "á»Ĺ": 35857, "amoto": 35858, "íķľëį°": 35859, "ĠÑģÑĤÑĢо": 35860, "ĠEnfin": 35861, "Ġbreadth": 35862, "éĺ²": 35863, "ĠCafe": 35864, "ĠDafür": 35865, "ĠBour": 35866, "aras": 35867, "Ġblueprint": 35868, "anı": 35869, "Ġconstants": 35870, "Ġattacker": 35871, "ĠFormula": 35872, "zaÄĩ": 35873, "Ġsowie": 35874, "Ġeyebrow": 35875, "obook": 35876, "Ġsetzen": 35877, "第ä¸ī": 35878, "onsider": 35879, "awning": 35880, "Ġsöyleye": 35881, "Ġinvaded": 35882, "Ġpronouns": 35883, "Ġdobry": 35884, "Si": 35885, "ĠХоÑĤ": 35886, "Ġvolleyball": 35887, "Ġlament": 35888, "isches": 35889, "arme": 35890, "api": 35891, "ĠWiki": 35892, "лиÑĪ": 35893, "Ġkasih": 35894, "Ġpess": 35895, "ĠÑĦоÑĤ": 35896, "ĠSul": 35897, "å¾·": 35898, "Ġpseudo": 35899, "Ġmemo": 35900, "ĠìĹ°ìĬµ": 35901, "ĠдоллаÑĢов": 35902, "ĠпеÑĢем": 35903, "ĠReach": 35904, "miral": 35905, "alted": 35906, "Ġstatut": 35907, "reading": 35908, "Ġsöyled": 35909, "ĠLindsey": 35910, "ĠAhmad": 35911, "ë¶Ģë": 35912, "ĠСегоднÑı": 35913, "Ġprzygot": 35914, "Ġhyster": 35915, "URE": 35916, "ĠNeigh": 35917, "Reporter": 35918, "ĠBunu": 35919, "ĠTreaty": 35920, "ĠRank": 35921, "ĠFame": 35922, "inished": 35923, "Ġgeared": 35924, "Ġcompose": 35925, "odia": 35926, "ĠLon": 35927, "ĠjesteÅĽmy": 35928, "ĠDIRECTOR": 35929, "Ġelkaar": 35930, "ĠViel": 35931, "×IJש": 35932, "ynthia": 35933, "並": 35934, "Ġmère": 35935, "ĠTomato": 35936, "Ġexatamente": 35937, "niÄĻ": 35938, "ĠFrei": 35939, "ĠDif": 35940, "Ġopenings": 35941, "Ġgraphical": 35942, "ĠÑĥдоб": 35943, "ĠвÑģп": 35944, "ĠWeekly": 35945, "ева": 35946, "Ġhangs": 35947, "Ġunsafe": 35948, "Ġemblem": 35949, "ĠKolleginnen": 35950, "alay": 35951, "Ġksi": 35952, "Ġhides": 35953, "Ġolmay": 35954, "Ġentste": 35955, "Ġarthritis": 35956, "ÃŁerdem": 35957, "Ġbinnen": 35958, "Ġlistens": 35959, "ĠHess": 35960, "åĨįä¾Ĩ": 35961, "ĠLouise": 35962, "lden": 35963, "енÑģ": 35964, "ĠVersion": 35965, "ĠAgriculture": 35966, "ìĬ¤ë¥¼": 35967, "ман": 35968, "ëĦ¤ìļĶ": 35969, "Ġwines": 35970, "ĠINF": 35971, "rul": 35972, "ĠJK": 35973, "ıyorlar": 35974, "shield": 35975, "reath": 35976, "Ġterus": 35977, "ĠLum": 35978, "Ġanticipation": 35979, "Ġaccustomed": 35980, "ĠMina": 35981, "Ġwield": 35982, "ioè": 35983, "mera": 35984, "Ġcountdown": 35985, "Ġcling": 35986, "Ġcommend": 35987, "Ġfaktiskt": 35988, "Ġdefenses": 35989, "Ġcockpit": 35990, "Ġкоманд": 35991, "Ġdishwas": 35992, "ĠThanos": 35993, "Ġkidneys": 35994, "Ġsehe": 35995, "Ġmicrobes": 35996, "Ġcuff": 35997, "ĠвÑĭÑģок": 35998, "ĠSpicy": 35999, "çŃīçŃī": 36000, "வர": 36001, "culus": 36002, "orc": 36003, "ç¾ħ": 36004, "ixes": 36005, "ĠCredit": 36006, "Ġraj": 36007, "Ġbringt": 36008, "ĠNiss": 36009, "Ġgrim": 36010, "ĠSOL": 36011, "Ġtenim": 36012, "ĠSudan": 36013, "ĠSpart": 36014, "Ġpromotes": 36015, "ĠNossa": 36016, "ĠÑģоÑģÑĤоÑıни": 36017, "Ġì°©": 36018, "Ġuncont": 36019, "ĠLiberal": 36020, "ĠТолÑĮко": 36021, "ĠViele": 36022, "Ġktórej": 36023, "Ġ****": 36024, "Max": 36025, "ĠЧÑĤобÑĭ": 36026, "350": 36027, "Ġíĺ¼ìŀIJ": 36028, "Ġë¶Ħëĵ¤ìĿ´": 36029, "Ġwarp": 36030, "Ġtenga": 36031, "Ġsympathetic": 36032, "Ġbizi": 36033, "ĠZack": 36034, "iedo": 36035, "Ġëī´ì": 36036, "piel": 36037, "ĠÑĤол": 36038, "Ġscaled": 36039, "ĠPETER": 36040, "ĠCOMM": 36041, "ĠCame": 36042, "Ġcatastrophe": 36043, "Ġsweaty": 36044, "igration": 36045, "Ġstuffing": 36046, "ĠÏĢολÏį": 36047, "ĠDriver": 36048, "zyst": 36049, "Tech": 36050, "Ġassessed": 36051, "ĠSurface": 36052, "ırım": 36053, "sur": 36054, "lerweile": 36055, "Ġдог": 36056, "Ġshutting": 36057, "Ġfractions": 36058, "ĠÑģол": 36059, "everyone": 36060, "Ġern": 36061, "ĠÐĿов": 36062, "Ġdefenders": 36063, "Ġversucht": 36064, "ãĥ³ãĥĢ": 36065, "Ġpolity": 36066, "ĠÐŁÐ¾Ð½": 36067, "verständ": 36068, "Ġbrowsers": 36069, "Ġtransformative": 36070, "Ġdictate": 36071, "ĠLEGO": 36072, "Ġninguna": 36073, "ê´ij": 36074, "Ġpizz": 36075, "ĠHarold": 36076, "ĠLopez": 36077, "Ú¾ÛĮ": 36078, "anız": 36079, "atchet": 36080, "ÙĬت": 36081, "Ġlernen": 36082, "Ġê·ĢìŬ": 36083, "Ġhoused": 36084, "Ġcleanse": 36085, "ĠWAT": 36086, "laration": 36087, "Ġbytes": 36088, "Ġtucked": 36089, "Ġfaults": 36090, "до": 36091, "FX": 36092, "Ġìĸ¼ë§ĪëĤĺ": 36093, "Ġdeform": 36094, "Ġcontracting": 36095, "ĠTIME": 36096, "irse": 36097, "Ġneben": 36098, "Ġcerc": 36099, "ĠArmstrong": 36100, "Ġtester": 36101, "Ġparfait": 36102, "Ġjealousy": 36103, "Ġtoxins": 36104, "Ġdisbel": 36105, "ÑĥÑĢÑĭ": 36106, "impression": 36107, "Ġprostate": 36108, "Ġfirewall": 36109, "Ġclassics": 36110, "еÑĩÑĮ": 36111, "Ġsocialism": 36112, "Ġgracious": 36113, "ĠÑģнова": 36114, "ĠднÑı": 36115, "Ġburner": 36116, "ĠMinor": 36117, "Ġìļ°ë¦¬ë": 36118, "Ġjedes": 36119, "Ġcontinuum": 36120, "Ġhots": 36121, "Ġoccurrence": 36122, "Ġadministered": 36123, "ĠзамеÑĤ": 36124, "Ġhesitation": 36125, "Ġdrills": 36126, "erca": 36127, "ĠвÑĤоÑĢой": 36128, "Ġsteadily": 36129, "Ġinsanlar": 36130, "Ġihan": 36131, "íij": 36132, "Ġhelper": 36133, "ĠSenin": 36134, "åģľ": 36135, "ование": 36136, "ĠERIC": 36137, "bla": 36138, "ĠAcademic": 36139, "Ġhumanities": 36140, "black": 36141, "umpy": 36142, "ortex": 36143, "ĠìłĪë": 36144, "ĠØ¥ÙĨ": 36145, "Ġdisclose": 36146, "ĠElijah": 36147, "ĠλÎŃ": 36148, "ĠQuer": 36149, "بÙĦ": 36150, "ãĤ¡": 36151, "Tell": 36152, "arle": 36153, "ÑĸÑĢ": 36154, "Ġaugmented": 36155, "Ġë¹ĦìĬ·": 36156, "Ġandroid": 36157, "त": 36158, "arma": 36159, "Ġszer": 36160, "geord": 36161, "Ġgeek": 36162, "Ġyeux": 36163, "Ġpong": 36164, "ĠãģĿãģĨ": 36165, "Ġtortured": 36166, "ĠBath": 36167, "zig": 36168, "asonable": 36169, "Ġnets": 36170, "Ġbaru": 36171, "ĠFlat": 36172, "ĠVater": 36173, "ĠTerror": 36174, "ĠAvo": 36175, "Ġceremonies": 36176, "roe": 36177, "Ùģس": 36178, "Ops": 36179, "Ġhyvin": 36180, "Ġapresent": 36181, "olor": 36182, "ĠигÑĢÑĭ": 36183, "orton": 36184, "Ġê·¸ëŀ¬": 36185, "Ġlookin": 36186, "ĠTY": 36187, "ĠMint": 36188, "Add": 36189, "Ġmite": 36190, "ĠSmoke": 36191, "Ġnota": 36192, "Ġmoss": 36193, "ĠAbend": 36194, "Ġ컨": 36195, "Ġexaggerated": 36196, "fires": 36197, "Ġredist": 36198, "ffiti": 36199, "Ġopenness": 36200, "ê°IJìĿ´": 36201, "endeu": 36202, "енной": 36203, "Watch": 36204, "Ġavatar": 36205, "ĠPey": 36206, "urun": 36207, "Ġsenza": 36208, "Ġì§ĢìĹŃ": 36209, "ĠNatomiast": 36210, "Ġemergence": 36211, "rays": 36212, "Ġcrafted": 36213, "gary": 36214, "ãģłãģij": 36215, "üng": 36216, "-\"": 36217, "Ġhacked": 36218, "Ġstray": 36219, "encie": 36220, "emo": 36221, "Ġcomen": 36222, "ĠKız": 36223, "ĠJasmine": 36224, "ĠHindi": 36225, "manas": 36226, "Ġinfinitely": 36227, "emon": 36228, "ìĿ¸ëį°ìļĶ": 36229, "jak": 36230, "Ġroaring": 36231, "érique": 36232, "sweise": 36233, "ĠRolex": 36234, "åł±å°İ": 36235, "ĠStuart": 36236, "bnb": 36237, "Ġdiagnose": 36238, "Ġcoherent": 36239, "ĠMJ": 36240, "æºĸåĤĻ": 36241, "Ġpike": 36242, "lav": 36243, "Ġorchestral": 36244, "аÑģÑĤи": 36245, "Ġterminar": 36246, "Ġgatherings": 36247, "Ġcompliant": 36248, "Ġupgrading": 36249, "Ġregulator": 36250, "Ġlanç": 36251, "éĢ£": 36252, "Ġmerchants": 36253, "tawa": 36254, "Ġmonitored": 36255, "Ġrendre": 36256, "两": 36257, "Ġunterwegs": 36258, "anguard": 36259, "gard": 36260, "ĠBelow": 36261, "duino": 36262, "ĠЦе": 36263, "Ġimpedance": 36264, "ìľ¡": 36265, "份": 36266, "Ġaktuell": 36267, "ĠVatic": 36268, "åŃ©": 36269, "Ġstewards": 36270, "Ġbrightest": 36271, "Ġkenn": 36272, "Ġkau": 36273, "ĠMatrix": 36274, "ĠBark": 36275, "ĠðŁij": 36276, "Ġtaper": 36277, "Ġcasino": 36278, "ר×Ķ": 36279, "ysical": 36280, "Ġbuilders": 36281, "ĠczÅĤowie": 36282, "ĠNepal": 36283, "Ġ!\"": 36284, "Ġterme": 36285, "Ġinnych": 36286, "Ġmaths": 36287, "Ġdrafted": 36288, "ĠBalk": 36289, "Ġhesitant": 36290, "Ġvoltar": 36291, "Ġrevive": 36292, "ĠÑĦилÑĮма": 36293, "Ġassassin": 36294, "ĠSolutions": 36295, "Ġduel": 36296, "Ġbearings": 36297, "à¸Ħะ": 36298, "Ġrookie": 36299, "ikat": 36300, "Ġbiscuits": 36301, "Ġcords": 36302, "ÑĥваÑĤи": 36303, "ARIN": 36304, "Ġprogressing": 36305, "ĠGir": 36306, "Ġpenetrate": 36307, "ĠStorage": 36308, "eight": 36309, "ĠÑĤÑĢÑĥ": 36310, "ĠdonÃŃt": 36311, "Ġsizin": 36312, "Ġoutdated": 36313, "ĠнаÑĪи": 36314, "Ġaffir": 36315, "Ġspoons": 36316, "Ġoni": 36317, "Ġflank": 36318, "ĠGol": 36319, "hã": 36320, "Ġpéri": 36321, "Ġhonorable": 36322, "ĠBreathe": 36323, "scenes": 36324, "Ġobviamente": 36325, "икÑģ": 36326, "Ġש×ŀ×": 36327, "Ġsmoothie": 36328, "ŀĪë": 36329, "Ġdime": 36330, "ĠíĸĪìĸ´ìļĶ": 36331, "Ġappel": 36332, "ĠCatholics": 36333, "Ġsingles": 36334, "Ġlaten": 36335, "Ġçünkü": 36336, "ĠVader": 36337, "æıĽ": 36338, "Ġvardı": 36339, "ĠIstanbul": 36340, "gré": 36341, "ĠElsa": 36342, "ël": 36343, "Ġinvece": 36344, "Ġcrane": 36345, "Ġobe": 36346, "ĠShark": 36347, "Ġsmack": 36348, "Ġrestoring": 36349, ".\\": 36350, "Ġë¹łë": 36351, "Ġfaded": 36352, "umbers": 36353, "Singing": 36354, "Ġdepressing": 36355, "thest": 36356, "ĠWahr": 36357, "Ġmultitude": 36358, "ÑĢавÑģÑĤвÑĥйÑĤе": 36359, "rijk": 36360, "eka": 36361, "Ġcompletes": 36362, "ĠWells": 36363, "Ġroy": 36364, "ĠPray": 36365, "ĠKalau": 36366, "izin": 36367, "iaÅĤem": 36368, "Ġlocom": 36369, "ĠNashville": 36370, "ĠPentagon": 36371, "미": 36372, "ĠNEW": 36373, "ÄħÄĩ": 36374, "ÃŃss": 36375, "Ġmarrying": 36376, "Ġfeud": 36377, "íĻķ": 36378, "æĢ¥": 36379, ")!": 36380, "ĠOperations": 36381, "ÑĥÑĶ": 36382, "Ġmoje": 36383, "Ġinstructed": 36384, "ĠëĪĦ구": 36385, "Ġ×Ķ×Ĵ": 36386, "ĠпомоÑīÑĮÑİ": 36387, "Ġsabia": 36388, "ìķĺìĸ´ìļĶ": 36389, "plane": 36390, "pri": 36391, "ĠполноÑģÑĤÑĮÑİ": 36392, "ĠKitty": 36393, "Ġpróprio": 36394, "edere": 36395, "Ġinteresante": 36396, "Ġде": 36397, "Ġcondensed": 36398, "Ġavent": 36399, "TOR": 36400, "Ġgreasy": 36401, "ARK": 36402, "orta": 36403, "AJ": 36404, "Ġdisreg": 36405, "Ġcorrections": 36406, "Ġstero": 36407, "Ġinfluenza": 36408, "Ġdesses": 36409, "Ġballots": 36410, "Ġmeget": 36411, "Ġmafia": 36412, "Ġböl": 36413, "nost": 36414, "ĠÑģÑĤаÑĤÑĮ": 36415, "Ġresponder": 36416, "Ġhinten": 36417, "grav": 36418, "à¸Ńะ": 36419, "ynchron": 36420, "Ġviens": 36421, "Ġsamo": 36422, "Ġdt": 36423, "pannt": 36424, "ĠÅĽwiat": 36425, "ĠзапиÑģ": 36426, "Ġmerged": 36427, "Ġkep": 36428, "Ġmisleading": 36429, "Ġdigamos": 36430, "Ġammon": 36431, "è¾Ľ": 36432, "chet": 36433, "Ġê°Ģìł¸": 36434, "Ġuni": 36435, "ĠëIJĺëĬĶëį°": 36436, "ĠнапÑĢав": 36437, "ĠкоÑĤоÑĢого": 36438, "Ġanimate": 36439, "×ķ×IJ×": 36440, "еÑĢв": 36441, "Ġminced": 36442, "Ġkaum": 36443, "ãģĤãģģ": 36444, "ÏĢε": 36445, "лег": 36446, "existing": 36447, "Ġplataform": 36448, "ĠKRIS": 36449, "ìĽł": 36450, "ĠFamilien": 36451, "ĠLibya": 36452, "Ġbiodiversity": 36453, "Ġidiots": 36454, "irdi": 36455, "Ġszyb": 36456, "ĠRolling": 36457, "ücht": 36458, "ĠÑĥдив": 36459, "ÑģÑĥд": 36460, "Ġrealizar": 36461, "Ġcanned": 36462, "ĠÑĢан": 36463, "Ġmetabolic": 36464, "ĠBeef": 36465, "Ġkilka": 36466, "лÑİÑģ": 36467, "Ġregistry": 36468, "моÑĤÑĢиÑĤе": 36469, "Ġvielä": 36470, "Ġodc": 36471, "Ġcondemned": 36472, "æ©ĭ": 36473, "fal": 36474, "ĠDil": 36475, "woÅĽci": 36476, "Aw": 36477, "Ġstatistically": 36478, "Ġsogen": 36479, "ĠBETH": 36480, "Ġshaving": 36481, "幸": 36482, "ocal": 36483, "ĠFunny": 36484, "Ġpeacefully": 36485, "Ġaddictive": 36486, "ĠInsert": 36487, "lauf": 36488, "Ġexperiencia": 36489, "é¦ĸåħĪ": 36490, "иÑĤелÑı": 36491, "ÃŃgen": 36492, "ágina": 36493, "Ġabdomen": 36494, "íķľëĭ¤": 36495, "icus": 36496, "imana": 36497, "ìį¨": 36498, "arching": 36499, "Ġkonkret": 36500, "ìķĺë": 36501, "ека": 36502, "oufl": 36503, "ivel": 36504, "Ġnude": 36505, "ètres": 36506, "Ġmonsieur": 36507, "Ġclash": 36508, "Ġtherapists": 36509, "Ġcubed": 36510, "Ġretrouver": 36511, "Ġwaveform": 36512, "Ġpotem": 36513, "ĠFormer": 36514, "isión": 36515, "åºľ": 36516, "Ġ×IJ×Ŀ": 36517, "undos": 36518, "ĠMeinung": 36519, "صÙĦ": 36520, "ĠJude": 36521, "ĠnÃ¥r": 36522, "ĠLeonardo": 36523, "ĠCristo": 36524, "ĠGOT": 36525, "ÑģÑĤÑĢÑĥк": 36526, "LAN": 36527, "ĠgÃ¥ng": 36528, "Ġdéb": 36529, "ĠFrankfurt": 36530, "Ġcrappy": 36531, "Ġlil": 36532, "année": 36533, "ĠмеÑģÑĤе": 36534, "RET": 36535, "ĠNer": 36536, "ĠCOSTA": 36537, "Ġjedem": 36538, "Ġcurtains": 36539, "Ġiterations": 36540, "Ġunav": 36541, "Ġplaque": 36542, "orum": 36543, "Ġζ": 36544, "Ġnúmeros": 36545, "Ġdesap": 36546, "²½": 36547, "Ġcompiled": 36548, "Ġrefle": 36549, "Ġrankings": 36550, "Ġrepaired": 36551, "ĠÐĿапÑĢ": 36552, "Ġdownloads": 36553, "Ġarmour": 36554, "Ġ×Ļ×ķתר": 36555, "Ġlongevity": 36556, "ĠTONER": 36557, "ĠкомменÑĤаÑĢ": 36558, "Ġczego": 36559, "Ġnotify": 36560, "Ġairports": 36561, "Ġenduring": 36562, "lette": 36563, "Ġapparat": 36564, "Ġhabil": 36565, "á»ĩc": 36566, "nad": 36567, "ICO": 36568, "ĠBrah": 36569, "Ġsegún": 36570, "Ġgovernors": 36571, "kaha": 36572, "ĠSchluss": 36573, "Ġodpowied": 36574, "irting": 36575, "Ġrempl": 36576, "ĠAboriginal": 36577, "identally": 36578, "Ġenhancing": 36579, "licting": 36580, "ĠHawaiian": 36581, "Ġstriving": 36582, "ĠNiet": 36583, "Ġznaczy": 36584, "Ġobedience": 36585, "ĠnÃ¥got": 36586, "Ġexpired": 36587, "Ġ1918": 36588, "presented": 36589, "Ġprowad": 36590, "ĠTerr": 36591, "ĠPrinceton": 36592, "Ġmorgen": 36593, "Ġattracting": 36594, "ĠSigma": 36595, "igner": 36596, "ĠRechts": 36597, "ĠPeki": 36598, "Ġmethy": 36599, "Ġhamm": 36600, "Ġdireito": 36601, "Ġdelegation": 36602, "иваÑİÑĤ": 36603, "Ġgin": 36604, "Young": 36605, "Ġdependencies": 36606, "ĠBradley": 36607, "buds": 36608, "Ġfis": 36609, "Ġpytanie": 36610, "Ġinterconnected": 36611, "Ġembaixo": 36612, "ĠSas": 36613, "Ġruh": 36614, "ĠSicht": 36615, "Sur": 36616, "Ġsuperb": 36617, "ĠSabbath": 36618, "ĠDanger": 36619, "kol": 36620, "Ġhou": 36621, "supp": 36622, "ĠNacional": 36623, "Ġsuccession": 36624, "Ġvá": 36625, "ĠMaÃŁnahmen": 36626, "ĠJessie": 36627, "ĠIdaho": 36628, "forest": 36629, "ħĺ": 36630, "Ġ×ŀ×ĵ": 36631, "ĠØ£ÙĬ": 36632, "Ġsweetheart": 36633, "Ġneatly": 36634, "ĠEvangel": 36635, "곡": 36636, "ĠSuite": 36637, "ública": 36638, "ĠÑĥли": 36639, "ĠAnnouncer": 36640, "ligh": 36641, "Ġsensations": 36642, "Ġshelters": 36643, "Ġhart": 36644, "Ġsqueezing": 36645, "ĠRivers": 36646, "ĠCooking": 36647, "ì±ħ": 36648, "personal": 36649, "Ġmanos": 36650, "ÑijÑĤÑģÑı": 36651, "wij": 36652, "Ġgogg": 36653, "ĠMilli": 36654, "ĠFP": 36655, "ünst": 36656, "ĠLS": 36657, "Ġspraying": 36658, "Ġfaux": 36659, "Ġautograph": 36660, "ologic": 36661, "Ġtorment": 36662, "Ġencrypted": 36663, "á»ħ": 36664, "Ġestre": 36665, "ç¹¼": 36666, "à±": 36667, "Ġstumbled": 36668, "Ġaider": 36669, "Ġsaben": 36670, "xter": 36671, "ĠCities": 36672, "ĠTürk": 36673, "ëĭ¥": 36674, "chine": 36675, "Ġtopping": 36676, "Ġpoisoned": 36677, "ĠRomania": 36678, "×ĵ×Ļ": 36679, "Ģë¡ľ": 36680, "ĠпоÑĢÑıд": 36681, "Ġchirping": 36682, "ĠìĻĦë": 36683, "×ij×¢": 36684, "Ġcuanto": 36685, "Ġdonating": 36686, "ĠRegent": 36687, "ĠBeruf": 36688, "Ġdistracting": 36689, "Ġstamina": 36690, "ĠDarren": 36691, "Ġì¶ķ": 36692, "lists": 36693, "dal": 36694, "chuss": 36695, "Ġeconomist": 36696, "ãģĪãĥ¼": 36697, "orgt": 36698, "Ġistiyorum": 36699, "è¿Ľ": 36700, "ĠSurprise": 36701, "ĠHao": 36702, "Ġìµľê³ł": 36703, "ĠGW": 36704, "ĠInner": 36705, "Ġquieren": 36706, "Ġminded": 36707, "Ġsupercomputer": 36708, "Ġdiagrams": 36709, "íĬľë": 36710, "ê²łìĸ´": 36711, "ĠобÑĬÑıÑģ": 36712, "Ġestaban": 36713, "Ġdestroys": 36714, "ĠBreaking": 36715, "ĠkarÄ±ÅŁ": 36716, "Ġrebuilding": 36717, "ľëĮĢ": 36718, "ливо": 36719, "ĠSauce": 36720, "ĠFusion": 36721, "×ķ×ŀ×": 36722, "ĠQuinn": 36723, "Ġgauche": 36724, "ĠÙĪØ£": 36725, "ĠÈ": 36726, "çĵľ": 36727, "Ġtechno": 36728, "Ġdispatch": 36729, "ĠaÅŁk": 36730, "Ġeinzel": 36731, "ĠGmail": 36732, "çŀ": 36733, "Ġê°ľìĿ¸": 36734, "ĠÑģемÑĮ": 36735, "Ġjourneys": 36736, "Ġiht": 36737, "Ġfibre": 36738, "Ġdramas": 36739, "ouched": 36740, "Ġrename": 36741, "ĠопеÑĢ": 36742, "Ġpoo": 36743, "ĠDru": 36744, "ĠиÑĤог": 36745, "Ġzast": 36746, "Ġcoz": 36747, "Ġzucch": 36748, "Ġobtaining": 36749, "Ġcommute": 36750, "Ġsubmer": 36751, "ĠVish": 36752, "ĠRabb": 36753, "ogg": 36754, "Ġhut": 36755, "íĸĪìĸ´": 36756, "æ¯Ķå¦Ĥ": 36757, "eremi": 36758, "Ġμα": 36759, "Ġdiskut": 36760, "ĠбÑĥк": 36761, "Ġimpaired": 36762, "depend": 36763, "ĠÙĪا": 36764, "ĠÑĢÑĥк": 36765, "ĠбаÑĢ": 36766, "Ġoxidation": 36767, "Ġsituação": 36768, "ÉĻn": 36769, "ução": 36770, "Ġsagte": 36771, "ĠSER": 36772, "ĠCake": 36773, "Ġturmeric": 36774, "ĠKak": 36775, "bung": 36776, "ĠKá¹Ľá¹£á¹ĩa": 36777, "Ġpoisoning": 36778, "Ġslipping": 36779, "ĠSays": 36780, "å°±åı¯ä»¥": 36781, "òng": 36782, "çŁ³": 36783, "«": 36784, "ĠClaudia": 36785, "ĠCharacter": 36786, "ниÑĨ": 36787, "coat": 36788, "Ġprogressed": 36789, "ĠFergus": 36790, "Ġìĺ¤ëĬ": 36791, "Ġoat": 36792, "ordable": 36793, "ĠLey": 36794, "ĠHeraus": 36795, "Ġresultados": 36796, "ĠKayla": 36797, "Ġriff": 36798, "Ġchegou": 36799, "Ġxi": 36800, "Ġspacious": 36801, "Ġrecognised": 36802, "Ġech": 36803, "ĠTie": 36804, "Ġlauncher": 36805, "Jim": 36806, "Ġsuppression": 36807, "ĠImpossible": 36808, "Ġguitars": 36809, "ĠFourier": 36810, "иÑĩеÑģкий": 36811, "ĠTherap": 36812, "ĠKaf": 36813, "centered": 36814, "ĠÑģооÑĤвеÑĤ": 36815, "Ġklim": 36816, "Ġcarbohydrates": 36817, "ignant": 36818, "ĠAstron": 36819, "Ġemple": 36820, "Ġdrastic": 36821, "ĠмиÑĢе": 36822, "вин": 36823, "uw": 36824, "Ġprettier": 36825, "Ġdonuts": 36826, "ĠAthena": 36827, "Ġdissert": 36828, "Ġplante": 36829, "Ġuranium": 36830, "ìĿĮë": 36831, "aré": 36832, "Ġrzecz": 36833, "Ġdisplaying": 36834, "æĪ²": 36835, "Ġsarc": 36836, "rão": 36837, "Ġtampoco": 36838, "Ġphilosophers": 36839, "ĠRecht": 36840, "æĵļ": 36841, "Ġcomentarios": 36842, "yse": 36843, "Ġìľ¤": 36844, "Ġmise": 36845, "ĠGin": 36846, "Ġном": 36847, "ĠFROM": 36848, "liner": 36849, "atif": 36850, "ĠspoÅĤec": 36851, "xa": 36852, "ĠÑĤÑĢÑĥд": 36853, "Ġwag": 36854, "기ìĹIJ": 36855, "ĠMG": 36856, "Ġoffspring": 36857, "ĠUnderstanding": 36858, "åıªæĺ¯": 36859, "ORA": 36860, "Ġwhirring": 36861, "Ġsurrend": 36862, "Ġpoker": 36863, "Ġmonuments": 36864, "ĠâĻ©": 36865, "Ġorganised": 36866, "ĠSozial": 36867, "ĠFactory": 36868, "Ñħа": 36869, "Ġresemble": 36870, "зд": 36871, "Ġexplosions": 36872, "Ġpayroll": 36873, "Ġomn": 36874, "ĠJorge": 36875, "ιÏĥ": 36876, "Ġfracture": 36877, "Ġpersecution": 36878, "Ġdemais": 36879, "ECH": 36880, ",)": 36881, "Ġcriar": 36882, "ĠJOSH": 36883, "Ġdemographics": 36884, "Ġ1600": 36885, "Ġcurrencies": 36886, "ĠTips": 36887, "ĠéĢĻåĢĭ": 36888, "ĠRefer": 36889, "ĠDancing": 36890, "Ġinconsistent": 36891, "Ġdeh": 36892, "Ġimmens": 36893, "Ġmeist": 36894, "Ġimpatient": 36895, "Ġbehaves": 36896, "æĿ¾": 36897, "ĠëĤ´ìļ©": 36898, "Ġbackstory": 36899, "Ġagreeing": 36900, "ĠÅģ": 36901, "ihin": 36902, "Ġtemperatura": 36903, "ĠBackground": 36904, "Ġnutzen": 36905, "Ġëħ¹": 36906, "ĠMänner": 36907, "Ġcollaborations": 36908, "ĠKos": 36909, "éģİåİ»": 36910, "Ġnightmares": 36911, "ëĵ±": 36912, "ĠQueensland": 36913, "Ġassociates": 36914, "ĠKok": 36915, "Ġfactorial": 36916, "ĠHyung": 36917, "Ġê·¸ëĭ¤ìĿĮ": 36918, "Ġfilho": 36919, "Ġelét": 36920, "Ġíĸīë³µ": 36921, "°±": 36922, "Ġgefunden": 36923, "Ġsemicondu": 36924, "Ġcounselors": 36925, "ĠUpper": 36926, "ĠAub": 36927, "ickers": 36928, "Ver": 36929, "Ġnorthwest": 36930, "ĠMaintenant": 36931, "ĠLakes": 36932, "аÑıв": 36933, "inté": 36934, "ì°½": 36935, "Ġгаз": 36936, "Ġgiorn": 36937, "Ġdigitally": 36938, "ĠCircuit": 36939, "ì¼Ģ": 36940, "ãĤĬãģ¾ãģĹãģŁ": 36941, "Ġcheerful": 36942, "ĠPeterson": 36943, "ĠDanish": 36944, "ativos": 36945, "Ġliken": 36946, "Ġharbor": 36947, "алиÑģÑĤ": 36948, "xe": 36949, "Ġcurls": 36950, "ĠRhod": 36951, "End": 36952, "ĠET": 36953, "Ġacquaint": 36954, "ĠKelvin": 36955, "Ġtrif": 36956, "ĠAway": 36957, "ìŀIJëĬĶ": 36958, "vs": 36959, "Ġpágina": 36960, "Ġinlet": 36961, "ĠSantos": 36962, "Ġìļ°ìĻĢ": 36963, "Ġyapıyorsun": 36964, "theme": 36965, "Ġsouff": 36966, "Ġinjected": 36967, "Ġpóźniej": 36968, "iverso": 36969, "amped": 36970, "Ġdaher": 36971, "Ġdagger": 36972, "ĠлÑİбим": 36973, "Ġtummy": 36974, "Ġenlightened": 36975, "cents": 36976, "ĠDah": 36977, "Ġcuest": 36978, "ä¾Ĩ說": 36979, "ILY": 36980, "Ġ×ijר": 36981, "Ġbanging": 36982, "ĠEmil": 36983, "ĠCler": 36984, "ĠBorder": 36985, "ижÑĥ": 36986, "Ġpresenters": 36987, "ĠSTUD": 36988, "coins": 36989, "ĠíĻį": 36990, "Ġperks": 36991, "Ġparap": 36992, "Ġcertaines": 36993, "ĠLore": 36994, "öst": 36995, "ĠMARTIN": 36996, "Ġbios": 36997, "Ġwhereby": 36998, "verts": 36999, "ĠMiranda": 37000, "Ġstip": 37001, "澤": 37002, "andez": 37003, "׼׾": 37004, "ujin": 37005, "Ġê¾": 37006, "Ġallergies": 37007, "plate": 37008, "Ġyapıl": 37009, "Ġundertake": 37010, "ĠëĤĺê°Ģ": 37011, "Part": 37012, "Ġkızım": 37013, "hguru": 37014, "ãģĤãģ¨": 37015, "ĠJohns": 37016, "Ġeyelashes": 37017, "Ġdrained": 37018, "ĠstÃ¥r": 37019, "ãģĤãĤĬãģ¾ãģĻ": 37020, "ĠJade": 37021, "Ġcalend": 37022, "film": 37023, "Ġmesa": 37024, "Ġludzie": 37025, "Ġattracts": 37026, "Ġjuices": 37027, "Ġкил": 37028, "Ġnieuwe": 37029, "Ġmencion": 37030, "Ġignition": 37031, "Ġbladder": 37032, "andaag": 37033, "ĠExtension": 37034, "íĤ¨": 37035, "feed": 37036, "ĠÙĪÙĩ": 37037, "Ġspun": 37038, "Ġtät": 37039, "оÑĢоÑĤ": 37040, "tyard": 37041, "ronics": 37042, "ĠHuge": 37043, "Ñĥжд": 37044, "string": 37045, "Ġunjust": 37046, "Ġprawn": 37047, "Ġfrosting": 37048, "Ġdisappearance": 37049, "iosa": 37050, "Ġcardi": 37051, "ĠPriest": 37052, "ĠcientÃŃfic": 37053, "åĵªè£¡": 37054, "ĠÐĴаÑģ": 37055, "Ġë¶Ģíĥģ": 37056, "Ġthieves": 37057, "Ġphysique": 37058, "ĠEugene": 37059, "Ġблиз": 37060, "Ġmonopoly": 37061, "Ġbiography": 37062, "ĠhoÅŁ": 37063, "Ġtö": 37064, "mac": 37065, "Ġshocks": 37066, "ìĦ¸ë": 37067, "hit": 37068, "Ġsnug": 37069, "Ġincl": 37070, "Ġdedic": 37071, "Ġultras": 37072, "ĠизвеÑģÑĤ": 37073, "Ġutilization": 37074, "ĠÑģовеÑĢÑĪенно": 37075, "Ġservi": 37076, "stag": 37077, "180": 37078, "Ġsewer": 37079, "ĠChoice": 37080, "Ġdischarged": 37081, "ĠJD": 37082, "олеÑĤ": 37083, "ĠкваÑĢÑĤи": 37084, "Ġtelescop": 37085, "ĠJeÅĽli": 37086, "ĠNana": 37087, "cale": 37088, "ĠÑĤон": 37089, "mmm": 37090, "äºĨåIJ§": 37091, "Ġgehabt": 37092, "ëĤł": 37093, "æĬķ": 37094, "à¸Ļà¸Ļ": 37095, "Ġether": 37096, "Ġzen": 37097, "Ġresearched": 37098, "ĠCzyli": 37099, "å®Įåħ¨": 37100, "workers": 37101, "Ġ경찰": 37102, "Ġsheriff": 37103, "allo": 37104, "Ġtipos": 37105, "Ġprosecution": 37106, "Ġfrogs": 37107, "Ġfalt": 37108, "jd": 37109, "ĠíĮĶ": 37110, "Ġfiltered": 37111, "ĠOft": 37112, "Ġìį": 37113, "Ġdisfr": 37114, "ĠMustang": 37115, "Ġwoah": 37116, "ĠREALLY": 37117, "Ġмогли": 37118, "Ġentrada": 37119, "ĠигÑĢа": 37120, "Ġmixes": 37121, "ĠавÑĤомоб": 37122, "ÐĻ": 37123, "Ġshin": 37124, "Ġparanormal": 37125, "Ġsomeplace": 37126, "Ġdishon": 37127, "etaan": 37128, "Ġfuerte": 37129, "Ù¹": 37130, "Ġdoom": 37131, "ìĪľ": 37132, "Ġexistential": 37133, "Ġbuld": 37134, "ĠSDK": 37135, "ĠпÑĢавда": 37136, "Ġturnover": 37137, "ĠìĹ¬ê¸°ìĹIJ": 37138, "Ġह": 37139, "Ġmodeled": 37140, "Ġbugün": 37141, "Ġexperimentation": 37142, "Ġmornings": 37143, "Ġmedo": 37144, "Stevie": 37145, "Ġplayable": 37146, "Ġairlines": 37147, "gments": 37148, "Ġ기ë¶Ħ": 37149, "ĠTomb": 37150, "ĠMVP": 37151, "AUDIENCE": 37152, "Ġcheckout": 37153, "Ġpasst": 37154, "Ġbeispiel": 37155, "ĠLinks": 37156, "heavy": 37157, "Ġquestionable": 37158, "Ġìĵ°ë": 37159, "Ġsill": 37160, "Ġmanipulated": 37161, "ĠLoren": 37162, "Ġìľ¼": 37163, "Ġverge": 37164, "ák": 37165, "IES": 37166, "Ġsabot": 37167, "ĠCustomer": 37168, "ależy": 37169, "Ġnominee": 37170, "ĠGad": 37171, "Ġnouvelles": 37172, "ĠSPE": 37173, "istling": 37174, "Ġoval": 37175, "обÑĢаж": 37176, "ifty": 37177, "éĩİ": 37178, "Ġbezel": 37179, "yet": 37180, "Ġfreight": 37181, "ĠHanım": 37182, "rÃŃa": 37183, "Ġzoning": 37184, "Ġindem": 37185, "ĠBü": 37186, "Ġfeminism": 37187, "Ġvoix": 37188, "Ġoficial": 37189, "Ġdiyorum": 37190, "»IJ": 37191, "Ġarose": 37192, "Ġparar": 37193, "ìĿ¸ì§Ģ": 37194, "ĠMartine": 37195, "ĠLect": 37196, "Ġrester": 37197, "Ġdrowning": 37198, "uya": 37199, "cida": 37200, "ĠAriel": 37201, "Ġ02": 37202, "Ġ×Ķ×Ķ": 37203, "ç´ł": 37204, "ĠWert": 37205, "ТÑĭ": 37206, "Ġwidow": 37207, "Ġparchment": 37208, "Ġcottage": 37209, "ĠXL": 37210, "ĠSlack": 37211, "ĠNES": 37212, "Ġrobe": 37213, "Ġgimm": 37214, "Ġcaminho": 37215, "ĠHarper": 37216, "Ġcitrus": 37217, "Ġfirefighters": 37218, "Ġdopamine": 37219, "elets": 37220, "Ġdemocrat": 37221, "ìłľë¡ľ": 37222, "Ġplayback": 37223, "oj": 37224, "ĠпÑĢок": 37225, "ĠSullivan": 37226, "semble": 37227, "ĠWorth": 37228, "ĠMustafa": 37229, "าร": 37230, "Ġmets": 37231, "éĸĢ": 37232, "лоÑģÑĮ": 37233, "Ġinertia": 37234, "Ġuniforms": 37235, "足": 37236, "ério": 37237, "×ķר×Ķ": 37238, "ént": 37239, "Ġà®Ĵ": 37240, "ĠÑģамÑĭÑħ": 37241, "Ġvoulais": 37242, "ĠZimmer": 37243, "ê²łë": 37244, "ĠноÑģ": 37245, "encias": 37246, "Ġrelación": 37247, "Ġ걸ë": 37248, "Ġfaction": 37249, "Ġgosp": 37250, "полож": 37251, "nap": 37252, "hak": 37253, "Ġproceedings": 37254, "ĠìĨĶ": 37255, "ìķĦëĭĪ": 37256, "ĠìŀIJ기": 37257, "Ġwerd": 37258, "Ġsof": 37259, "Ġschlim": 37260, "Ġflavored": 37261, "Ġquadratic": 37262, "ĠBoot": 37263, "Ġpublicity": 37264, "ĠCaro": 37265, "Ġ?\"": 37266, "ниÑĨа": 37267, "mania": 37268, "ĠSUR": 37269, "ĠBUR": 37270, "lance": 37271, "ética": 37272, "Ġzobaczy": 37273, "Ġtrio": 37274, "sama": 37275, "ĠtaÅŁ": 37276, "Ġasymm": 37277, "resser": 37278, "Ġتع": 37279, "ĠпеÑģ": 37280, "Ġbeginnings": 37281, "ladım": 37282, "ĠбÑĭÑģÑĤÑĢ": 37283, "Ġmoo": 37284, "ĠGeneva": 37285, "Ġåľ¨": 37286, "erus": 37287, "borah": 37288, "Ġrefusing": 37289, "bull": 37290, "ĠWaiting": 37291, "ĠIndividual": 37292, "Ġanonym": 37293, "imens": 37294, "Ġmedidas": 37295, "Ġfragrant": 37296, "Ġdirectement": 37297, "ĠìķĦë§Ī": 37298, "uria": 37299, "Ġspherical": 37300, "Ġabge": 37301, "ĠVictorian": 37302, "Ġspectacle": 37303, "ĠRodriguez": 37304, "Ġocup": 37305, "ĠNär": 37306, "marks": 37307, "ngulo": 37308, "ĠLuci": 37309, "Ġshouted": 37310, "Ġregulators": 37311, "ÄŁini": 37312, "Ġdisent": 37313, "ĠÑĢÑĭн": 37314, "ëĤ¨": 37315, "ĠìĤ´ë": 37316, "Ġproblèmes": 37317, "ĠFinger": 37318, "assemble": 37319, "Ġpear": 37320, "Ġdroite": 37321, "ĠEverywhere": 37322, "tam": 37323, "оÑĤив": 37324, "вой": 37325, "ordinate": 37326, "ĠLak": 37327, "ĠmỼi": 37328, "ĠTelevision": 37329, "Ġexponentially": 37330, "avas": 37331, "Ġblev": 37332, "ĠMT": 37333, "俺": 37334, "Connell": 37335, "ĠêµŃ민": 37336, "ĠÑģвоим": 37337, "Ġacha": 37338, "ĠDynasty": 37339, "Jin": 37340, "Ġtore": 37341, "Ġflor": 37342, "Ġмногие": 37343, "æ²Ĵäºĭ": 37344, "owan": 37345, "bah": 37346, "Ġì£Ħ": 37347, "ĠCela": 37348, "Ġìµľê·¼": 37349, "Ġpermettre": 37350, "Ġabras": 37351, "Ġverstehen": 37352, "Ġescort": 37353, "ĠThem": 37354, "ärke": 37355, "porter": 37356, "Ġkahkaha": 37357, "Ġhect": 37358, "Ġdau": 37359, "wah": 37360, "olve": 37361, "ĠAges": 37362, "schaft": 37363, "ĠStell": 37364, "nelle": 37365, "ĠEnsuite": 37366, "ĠÐĴÑģем": 37367, "Ġcréd": 37368, "ĠPP": 37369, "lords": 37370, "grunting": 37371, "Ġcontraction": 37372, "Got": 37373, "Ġacquiring": 37374, "Ġsopr": 37375, "Ġpoisonous": 37376, "RNA": 37377, "Ġanar": 37378, "ĠHof": 37379, "')": 37380, "Ġremarkably": 37381, "Ġinternacional": 37382, "ücke": 37383, "inqu": 37384, "Ġduy": 37385, "Ġbeasts": 37386, "ĠLAN": 37387, "Ġprecedent": 37388, "ĠRPM": 37389, "åij¨": 37390, "Ġselon": 37391, "Ġmorte": 37392, "Ġcomeçou": 37393, "Ñıла": 37394, "Ġinterpreting": 37395, "ĠBurke": 37396, "ÑĤÑĢа": 37397, "ĠìĿ´ëŁ¬": 37398, "Ġpessim": 37399, "ĠNok": 37400, "íĮĿ": 37401, "Female": 37402, "Ġìĭ¤í": 37403, "ĻĢ": 37404, "Ġstimulation": 37405, "Ġslick": 37406, "Ġê°ĢëĬĶ": 37407, "Ġказ": 37408, "ĠHBO": 37409, "Ġpapier": 37410, "Ġkönnten": 37411, "Ñĥбли": 37412, "ĠConstant": 37413, "SPEAKING": 37414, "ĠktórÄħ": 37415, "Ġcosmetics": 37416, "ĠTrend": 37417, "Ġrobbery": 37418, "Ġtitt": 37419, "Ġgjort": 37420, "Ġdietary": 37421, "łĮ": 37422, "ĠKirby": 37423, "ĠпÑĢимеÑĢно": 37424, "Ġqualification": 37425, "Ġìķī": 37426, "Ġcabinets": 37427, "Ġhttp": 37428, "ĠErica": 37429, "義": 37430, "Ġdisadvantages": 37431, "Ġchattering": 37432, "yz": 37433, "feit": 37434, "Ġguild": 37435, "ĠETF": 37436, "ĠDragons": 37437, "ĠHERE": 37438, "venth": 37439, "ÙĦاÙħ": 37440, "Ġmarché": 37441, "Dam": 37442, "Ġphoton": 37443, "Ġestable": 37444, "Mag": 37445, "Ġolhar": 37446, "Ġcoupling": 37447, "ĠHilfe": 37448, "ĠWizard": 37449, "Ġмало": 37450, "help": 37451, "ĠlÃŃnea": 37452, "Ġì«": 37453, "Ġstandalone": 37454, "Ġmorale": 37455, "Ġzweite": 37456, "ãĤĪãĤįãģĹãģı": 37457, "ährt": 37458, "Ġdotted": 37459, "Ġdripping": 37460, "ĠFlag": 37461, "éĿĴ": 37462, "rocket": 37463, "rategy": 37464, "irim": 37465, "Ġíķĺë©´ìĦľ": 37466, "Ġsogenan": 37467, "ĠUno": 37468, "ĠSchutz": 37469, "Ġestilo": 37470, "ĠSubs": 37471, "ĠDaisy": 37472, "ÐĿеÑĤ": 37473, "'...": 37474, "Ġplatinum": 37475, "Ġbirl": 37476, "ĠSovi": 37477, "Ġviolate": 37478, "ÑĥеÑĤÑģÑı": 37479, "rill": 37480, "Ġtraz": 37481, "Ġsnip": 37482, "Ġcumpl": 37483, "à¸Ńà¸ģ": 37484, "Ġcuk": 37485, "éħĴ": 37486, "ĠParlament": 37487, "Ġhypert": 37488, "Ġpulp": 37489, "Ġtongues": 37490, "atto": 37491, "Ġbusca": 37492, "ihn": 37493, "ERO": 37494, "ĠÙĬع": 37495, "Ġvarias": 37496, "ĠMarian": 37497, "Ġbounded": 37498, "Ġpitching": 37499, "Ġdeficiency": 37500, "ĠBlessed": 37501, "ĠExerc": 37502, "uchs": 37503, "ĠnhÆ°ng": 37504, "æľ¬å½ĵ": 37505, "Ġraped": 37506, "hales": 37507, "Ġmala": 37508, "pic": 37509, "Ġ401": 37510, "ÅĽniej": 37511, "arina": 37512, "ëĵ¤ìĿĦ": 37513, "otti": 37514, "Ġдолго": 37515, "Ġtracker": 37516, "ĠShelby": 37517, "Ġvanished": 37518, "Ġbakery": 37519, "Kapı": 37520, "Jesus": 37521, "ĠKR": 37522, "JO": 37523, "ħ¸": 37524, "Ġdiscs": 37525, "ìĦ¯": 37526, "ì§Ģë": 37527, "×Ļצ": 37528, "emary": 37529, "Kendra": 37530, "Ġyük": 37531, "ückt": 37532, "Ġvaz": 37533, "Ġkup": 37534, "aktu": 37535, "ĠÑģпаÑģибо": 37536, "Ġaik": 37537, "Ġnursery": 37538, "Ġendangered": 37539, "êmement": 37540, "ematics": 37541, "Ġresponders": 37542, "ĠRepresentatives": 37543, "Ġsculptures": 37544, "igkeiten": 37545, "Ġdepl": 37546, "Ġinterpretations": 37547, "Ġdeadlines": 37548, "Ġ1942": 37549, "ÃĹ": 37550, "Ġsugars": 37551, "emu": 37552, "lively": 37553, "Ġrecreational": 37554, "Ġdistort": 37555, "Ġunderscore": 37556, "Ġunquote": 37557, "Ġsafest": 37558, "Ġswollen": 37559, "Ġanalyses": 37560, "Ġcommencé": 37561, "妹": 37562, "andin": 37563, "ĠХоÑĢоÑĪо": 37564, "Ġdiarr": 37565, "ãģ¾ãģģ": 37566, "ziest": 37567, "Ġtoothbrush": 37568, "éł»éģĵ": 37569, "uations": 37570, "Ġcade": 37571, "Ġbacklash": 37572, "hind": 37573, "Ġrisque": 37574, "zess": 37575, "ĠìĿ´ìķ¼ê¸°": 37576, "Ġesperar": 37577, "Ġtranslations": 37578, "ioned": 37579, "groans": 37580, "ĠпÑĥÑĤ": 37581, "Ġgenetically": 37582, "éĢł": 37583, "Ġhappiest": 37584, "Ġwerk": 37585, "atoon": 37586, "Ġmusi": 37587, "Ġfunção": 37588, "ĠìŀħëĭĪëĭ¤": 37589, "ĠÑĢай": 37590, "Ġbevor": 37591, "BLANK": 37592, "Ġrepentance": 37593, "Put": 37594, "Ġpotrzeb": 37595, "Ġsala": 37596, "Ġcampa": 37597, "WER": 37598, "ĠdecÃŃa": 37599, "Ġsécurité": 37600, "ĠAppreciate": 37601, "Ñĩи": 37602, "ĠRandom": 37603, "ë³Ħ": 37604, "kah": 37605, "Ġmöj": 37606, "Ġsäger": 37607, "Ġ×Ļ׼×ķ׾": 37608, "Ġ190": 37609, "xtures": 37610, "Eu": 37611, "Ġgä": 37612, "Ġ×ijת": 37613, "ĠCroat": 37614, "apo": 37615, "PLE": 37616, "Ġpersistence": 37617, "åĬ©": 37618, "Ġblends": 37619, "Ġtreffen": 37620, "ĠSantiago": 37621, "ydia": 37622, "aldo": 37623, "ĠTensorFlow": 37624, "ĠDual": 37625, "ãĥľ": 37626, "Ġchiff": 37627, "ìĹ´": 37628, "Ġcontracted": 37629, "Ġsegreg": 37630, "ĠFairy": 37631, "Ġwisely": 37632, "Ġvulnerabilities": 37633, "Ġhandheld": 37634, "Ġgadgets": 37635, "ĠboÅŁ": 37636, "ĠPopular": 37637, "Ġcurvature": 37638, "문": 37639, "ĠMARY": 37640, "ìĿ´ìĬ": 37641, "Ġformulation": 37642, "Ġcelery": 37643, "Ġblurry": 37644, "ĠTS": 37645, "alez": 37646, "Ġws": 37647, "Ġprogramm": 37648, "ĠStack": 37649, "ĠJIM": 37650, "овали": 37651, "ıll": 37652, "Ġpère": 37653, "ĠKanye": 37654, "ĠDelaware": 37655, "Ġãģł": 37656, "Ġdaunting": 37657, "ĠбеÑģ": 37658, "ĠStupid": 37659, "big": 37660, "fficial": 37661, "Ġprecipitation": 37662, "Ġplung": 37663, "ục": 37664, "burse": 37665, "Ġdarle": 37666, "Ġcripp": 37667, "Ġpioneer": 37668, "Ġdisput": 37669, "Ġsean": 37670, "ãģĵãĤĵãģª": 37671, "Ġresistor": 37672, "Ġallein": 37673, "ipples": 37674, "arel": 37675, "Ġendors": 37676, "zust": 37677, "ĠÑĢебÑıÑĤа": 37678, "eded": 37679, "Ġì¹´ë©Ķë": 37680, "Ġlleva": 37681, "Ġkennt": 37682, "Ġбал": 37683, "ĠDocument": 37684, "ĠKnights": 37685, "Ġbuckle": 37686, "Ġìī¬": 37687, "Ġalk": 37688, "ĠEveryday": 37689, "atters": 37690, "Ġtoilets": 37691, "Ġjugar": 37692, "ĠìŀĪì§Ģ": 37693, "Ġgenauso": 37694, "ĠLandesregierung": 37695, "ãģ£ãģ±": 37696, "ije": 37697, "Ġtrailers": 37698, "ĠTigers": 37699, "Ġgitti": 37700, "Ġforgiving": 37701, "Ġconcurrent": 37702, "ĠVu": 37703, "ĠíĬ¹íŀĪ": 37704, "ĠBROWN": 37705, "ounded": 37706, "\";": 37707, "Ġtremb": 37708, "Ġtiet": 37709, "ĠÑĢежим": 37710, "Ġnutshell": 37711, "елиÑĩ": 37712, "Ġlosers": 37713, "ricting": 37714, "Ġredeem": 37715, "defined": 37716, "Nice": 37717, "Ġbroadband": 37718, "KO": 37719, "Ġteasing": 37720, "Ġpartisan": 37721, "ıma": 37722, "Ġìŀ¬ë¯¸": 37723, "ĠJourney": 37724, "Ġslopes": 37725, "uning": 37726, "grunts": 37727, "Ġtäll": 37728, "Ġuncovered": 37729, "ĠmyÅĽlÄĻ": 37730, "ĠEsther": 37731, "äºİ": 37732, "ĠHealthy": 37733, "Ġë°ij": 37734, "rée": 37735, "Ġpolarization": 37736, "Ġflav": 37737, "Ġcambiar": 37738, "Ġyr": 37739, "ĠRanch": 37740, "Ġsplits": 37741, "Ġtrouvé": 37742, "åľĭ家": 37743, "Ġrecorder": 37744, "Ġdépart": 37745, "ÙĪب": 37746, "ĠKry": 37747, "Ġinteressant": 37748, "Ġederim": 37749, "ÅĽwiad": 37750, "ilateral": 37751, "wright": 37752, "Ġpourra": 37753, "êter": 37754, "Ġcamel": 37755, "áŀ": 37756, "Ġrapidement": 37757, "Ġmej": 37758, "Ġstiffness": 37759, "ADAS": 37760, "Ġdiffers": 37761, "Ġalot": 37762, "ĠSig": 37763, "ÑıÑĤелÑĮ": 37764, "Ġabstraction": 37765, "åľĺ": 37766, "Ġkeiner": 37767, "grupp": 37768, "ĠSherlock": 37769, "íĺĶ": 37770, "Ġcite": 37771, "Ġoverflow": 37772, "Ġtại": 37773, "úcar": 37774, "bula": 37775, "Ġconjunto": 37776, "ĠCI": 37777, "Ġmoderator": 37778, "Ġindirectly": 37779, "Ġalleine": 37780, "âĤ": 37781, "ÑĪиб": 37782, "Ġбаб": 37783, "Ġdanach": 37784, "Ġ1939": 37785, "Ġpromet": 37786, "Ġdestinations": 37787, "ĠIllust": 37788, "ικÏĮ": 37789, "Ġsabes": 37790, "Ġheh": 37791, "ĠGesetzent": 37792, "ĠMiz": 37793, "енко": 37794, "ĠMys": 37795, "Ь": 37796, "ĠJudaism": 37797, "Ġmustache": 37798, "Ġstimmt": 37799, "ĠGaza": 37800, "Ġvolte": 37801, "Ġnuo": 37802, "Ġmón": 37803, "ĠComput": 37804, "ูà¹Ī": 37805, "ĠRadi": 37806, "Ġexceptionally": 37807, "Ġassumes": 37808, "éĸĭå¿ĥ": 37809, "ãģĪãģ°": 37810, "inform": 37811, "Ġshrine": 37812, "æĵĬ": 37813, "Ġimplication": 37814, "ĠFitz": 37815, "æ²ĴéĹľä¿Ĥ": 37816, "!.": 37817, "Ġlt": 37818, "Ġalloy": 37819, "Ġethic": 37820, "Ġmonastery": 37821, "ìĭľì£ł": 37822, "icação": 37823, "Ġcoordinating": 37824, "ĠMoto": 37825, "Ġoverlook": 37826, "Ġchois": 37827, "Ġantibiotic": 37828, "ĠMinne": 37829, "ĠBJ": 37830, "ĠApa": 37831, "orian": 37832, "Ġspilled": 37833, "Jam": 37834, "Ġhusbands": 37835, "Ġcreations": 37836, "Ġañ": 37837, "üssel": 37838, "ĠìĿ´ìļ©": 37839, "Ġanalyse": 37840, "rose": 37841, "Ġpunched": 37842, "Ġpresque": 37843, "Ġastronomy": 37844, "Ġschwierig": 37845, "ĠEbola": 37846, "Ġcis": 37847, "Ġacet": 37848, "ĠFX": 37849, "endre": 37850, "ĠìĿĮìķħ": 37851, "Ġwebpage": 37852, "Ġfreaked": 37853, "Ġlatte": 37854, "Ġì¿ł": 37855, "Ġ머ë": 37856, "Never": 37857, "Gra": 37858, "íĻĶ를": 37859, "eyed": 37860, "Ġë°ľëĿ¼": 37861, "Ġespera": 37862, "Ġaparece": 37863, "ração": 37864, "Ġdisruptive": 37865, "ĠJoint": 37866, "urous": 37867, "reas": 37868, "ĠquerÃŃa": 37869, "Ġdistributions": 37870, "Ġexponent": 37871, "ì¹ĺ를": 37872, "Ġdl": 37873, "zhou": 37874, "ĠHearing": 37875, "å·®ä¸įå¤ļ": 37876, "ĠCraw": 37877, "Ġfloats": 37878, "ounced": 37879, "Lab": 37880, "World": 37881, "Ġburdens": 37882, "Ġauthoritarian": 37883, "ĠBolt": 37884, "ĠоднÑĥ": 37885, "Ġpigeon": 37886, "Ġdistractions": 37887, "ĠHerausforder": 37888, "Ġzest": 37889, "esc": 37890, "Ġshakes": 37891, "atas": 37892, "ĠÙħØ´": 37893, "holes": 37894, "Ġthinkers": 37895, "alta": 37896, "Ġarche": 37897, "ĠSuk": 37898, "anha": 37899, "Ġtempting": 37900, "Ġyoutuber": 37901, "Ġvì": 37902, "ĠdziaÅĤa": 37903, "ĠVatican": 37904, "Park": 37905, "Ġsupers": 37906, "ĠNikki": 37907, "ëĬIJë": 37908, "orang": 37909, "ramient": 37910, "鬼": 37911, "Ġê°ĸê³ł": 37912, "Ġdesserts": 37913, "Ġavere": 37914, "ĠGregory": 37915, "Ġëĵ¤ìĸ´ìĺ": 37916, "Ġcosting": 37917, "ĠClinic": 37918, "Ġrebels": 37919, "ĠMob": 37920, "Ġbunlar": 37921, "ĠYours": 37922, "ertime": 37923, "Ġretali": 37924, "mara": 37925, "atus": 37926, "alles": 37927, "ĠдÑĢ": 37928, "ĠдиÑģ": 37929, "Ġdiscounts": 37930, "ĠGUY": 37931, "Ġкакое": 37932, "ĠExperiment": 37933, "rement": 37934, "ĠXiang": 37935, "Ġbate": 37936, "WE": 37937, "Ġspecialize": 37938, "Ġdeity": 37939, "ĠLoki": 37940, "mag": 37941, "ĠNit": 37942, "West": 37943, "Ġmaternal": 37944, "Ġquis": 37945, "åŁºæľ¬": 37946, "broken": 37947, "Ġlasers": 37948, "Ġhakk": 37949, "ĠAngels": 37950, "Ġmastery": 37951, "antis": 37952, "Tiffany": 37953, "eee": 37954, "çij": 37955, "orem": 37956, "Ġinacc": 37957, "Ġjurisdictions": 37958, "ĠKardash": 37959, "æľº": 37960, "Il": 37961, "ĠSinn": 37962, "åĭķçĶ»": 37963, "Ġathletics": 37964, "cÄĻ": 37965, "Ġloosely": 37966, "Ġdieta": 37967, "Ag": 37968, "Ġ??": 37969, "ĠëĮĢíijľ": 37970, "Ġsuperv": 37971, "Ġnutrit": 37972, "Ġdrifting": 37973, "ĠìĦłìĥĿëĭĺ": 37974, "ĠпонÑıл": 37975, "ĠVictory": 37976, "ÙĦØ©": 37977, "×ķ׳×Ķ": 37978, "ĠпиÑĪ": 37979, "Ġshaved": 37980, "Ġmesure": 37981, "onden": 37982, "Ùĥر": 37983, "Ġexile": 37984, "ĠDesde": 37985, "ĠPinterest": 37986, "Ġattachments": 37987, "Ġhombres": 37988, "Ġfines": 37989, "ĠìĦ¸ìĥģ": 37990, "Ġsleeps": 37991, "ĠTaco": 37992, "ĠIRA": 37993, "rios": 37994, "Ġoll": 37995, "etes": 37996, "Ġunut": 37997, "fashioned": 37998, "Ġtreball": 37999, "ĠNearly": 38000, "ĠÑĢеалÑĮно": 38001, "Ġchil": 38002, "éĢ±": 38003, "ÄŁa": 38004, "ĠMEL": 38005, "roscop": 38006, "ĠCG": 38007, "Ġvenge": 38008, "Ġdishwasher": 38009, "algic": 38010, "Ġmodifier": 38011, "Ġembassy": 38012, "timer": 38013, "emics": 38014, "Ġintricate": 38015, "Ġevet": 38016, "ĠëĮĢë°ķ": 38017, "Ġisot": 38018, "ĠнаÑĥÑĩ": 38019, "ĠQuiz": 38020, "reso": 38021, "δÏİ": 38022, "Ġyelled": 38023, "Ġfeder": 38024, "ELLER": 38025, "Ġexceeded": 38026, "onas": 38027, "icano": 38028, "ĠживоÑĤ": 38029, "ĠMao": 38030, "ĠKazuto": 38031, "Ġãħĭãħĭãħĭãħĭ": 38032, "Ġfrontline": 38033, "ĠHungarian": 38034, "Ġüberall": 38035, "awat": 38036, "Ġgrips": 38037, "ições": 38038, "arnya": 38039, "ĠÍ¡": 38040, "Ġseid": 38041, "Ġanak": 38042, "Ġacabou": 38043, "íķij": 38044, "Ġnotorious": 38045, "ĠGodzilla": 38046, "Ġovercoming": 38047, "ĠPend": 38048, "Ġolabilir": 38049, "ülme": 38050, "Ġerhalten": 38051, "ãĤīãģĦ": 38052, "ê·¹": 38053, "ĠMeter": 38054, "Ġstaan": 38055, "Ol": 38056, "Ġchats": 38057, "ĠBuenos": 38058, "ÃŃve": 38059, "aluable": 38060, "Ġstrategically": 38061, "Ġcomprised": 38062, "ĠпеÑĢÑģонаж": 38063, "Ġwann": 38064, "ĠCen": 38065, "ниÑĤе": 38066, "Łģ": 38067, "ĠÑĤобой": 38068, "iad": 38069, "ĠkardeÅŁim": 38070, "ĠCongressman": 38071, "reaming": 38072, "homme": 38073, "Ġcommunaut": 38074, "Ġalcoholic": 38075, "Ġpickled": 38076, "Ġacord": 38077, "position": 38078, "egól": 38079, "Ġtroubling": 38080, "ĠMarcheg": 38081, "Ġzumindest": 38082, "Ġseamlessly": 38083, "Ġolun": 38084, "ĠTVs": 38085, "ĠпÑĢакÑĤиÑĩеÑģки": 38086, "Ġbackend": 38087, "ãģĵãĤĵãģ«ãģ¡ãģ¯": 38088, "idable": 38089, "Ġgadget": 38090, "Ġfaço": 38091, "ĠMarchegiani": 38092, "Ġë°¤": 38093, "Ġaccidental": 38094, "ĠLP": 38095, "Ġeldest": 38096, "ĠAdmiral": 38097, "ĠnÄĥm": 38098, "lever": 38099, "Ġpastel": 38100, "Ġfondo": 38101, "Connie": 38102, "Ġtercer": 38103, "Ġpact": 38104, "ĠMonte": 38105, "Ġmeats": 38106, "ĠSMS": 38107, "ĠAustralians": 38108, "ç¼": 38109, "Rhett": 38110, "Ġexactement": 38111, "Ġë¹¼": 38112, "ĠMOD": 38113, "ç¡": 38114, "ĠRapt": 38115, "ĠNoch": 38116, "Ġabort": 38117, "ĠNaval": 38118, "ĠFuji": 38119, "INTER": 38120, "ĠновÑĭй": 38121, "Ġmiejsce": 38122, "ĠICU": 38123, "ĠGraduate": 38124, "ĠGlen": 38125, "ardi": 38126, "ĠÈĺ": 38127, "Ġsolder": 38128, "Ġprofessions": 38129, "Ġorthog": 38130, "omn": 38131, "introdu": 38132, "ĠDenise": 38133, "ìŀIJ를": 38134, "Ġcorrespondence": 38135, "AMA": 38136, "Ġinflict": 38137, "Ġfand": 38138, "ĠGü": 38139, "ĠÑĩеÑĤ": 38140, "Ġtraced": 38141, "Ġpatents": 38142, "Ġambush": 38143, "Ġlotta": 38144, "ffer": 38145, "ĠWagner": 38146, "Ġimperson": 38147, "Ġextrêmement": 38148, "ÙĤت": 38149, "conduct": 38150, "Att": 38151, "ĠMueller": 38152, "ĠAlicia": 38153, "Ġcyc": 38154, "Ġhacker": 38155, "Ġtys": 38156, "Ġhail": 38157, "ĠзаÑıв": 38158, "Ġpasso": 38159, "Ġì¶Ķê°Ģ": 38160, "ĠÎĪ": 38161, "Ġpackaged": 38162, "ĠCynthia": 38163, "heet": 38164, "ä¸ŃåĽ½": 38165, "ĠNissan": 38166, "ĠQuesto": 38167, "é¨": 38168, "did": 38169, "Ġμια": 38170, "ĠEllis": 38171, "ĠAnalysis": 38172, "cemos": 38173, "Ġaseg": 38174, "ĠMyster": 38175, "ĠCao": 38176, "Ġtuv": 38177, "ĠIndustry": 38178, "ì£¼ê³ł": 38179, "otal": 38180, "Ġpequeño": 38181, "bras": 38182, "Ġcomprehend": 38183, "ĠSimpson": 38184, "ÑģÑĤвие": 38185, "ocracy": 38186, "иÑĩеÑģки": 38187, "ĠMush": 38188, "ĠLaurie": 38189, "Ġtriangular": 38190, "ĠPresents": 38191, "ĠKunden": 38192, "ç´¹": 38193, "æѦ": 38194, "ĠIss": 38195, "ĠDeck": 38196, "á»ĥn": 38197, "ĠDarkness": 38198, "Ġinflammatory": 38199, "eremiah": 38200, "Ġwarmed": 38201, "veyard": 38202, "ĠMemory": 38203, "etty": 38204, "Ġtaxpayers": 38205, "à¸ĵ": 38206, "Ø¡": 38207, "Ġpractise": 38208, "ëĭ¬ë": 38209, "Ġdrilled": 38210, "mÃ¼ÅŁ": 38211, "logo": 38212, "ĠFach": 38213, "¤ë¡ľ": 38214, "Ġübrigens": 38215, "Ġkonnten": 38216, "Ġnormalmente": 38217, "Ġargues": 38218, "ilingual": 38219, "°ë¥¼": 38220, "egal": 38221, "Ġtravaill": 38222, "ovy": 38223, "аÑĤо": 38224, "Ġruth": 38225, "ĠLights": 38226, "Ġconsisted": 38227, "×ijר×Ļ×Ŀ": 38228, "Ġstereotype": 38229, "Ġpayer": 38230, "ĠRee": 38231, "ĠAirbnb": 38232, "Ġdrowned": 38233, "ĠZoe": 38234, "Ġcanopy": 38235, "Ġbarr": 38236, "ĠноÑĩ": 38237, "Ġpagan": 38238, "Ġjars": 38239, "Ġrê": 38240, "erver": 38241, "æĪ¿": 38242, "ieben": 38243, "Ġespect": 38244, "ĠFi": 38245, "Ġunwilling": 38246, "Ġtechnician": 38247, "ặt": 38248, "member": 38249, "ĠCanal": 38250, "سÙħ": 38251, "Ġlieber": 38252, "Ġinference": 38253, "Ġhonoring": 38254, "åijµ": 38255, "ĠCampaign": 38256, "Ġlineage": 38257, "ĠStress": 38258, "Ġvictories": 38259, "Ġdeja": 38260, "×£": 38261, "êtes": 38262, "blick": 38263, "Ġменее": 38264, "oths": 38265, "ĠCouple": 38266, "Jason": 38267, "ĠNicolas": 38268, "екÑģ": 38269, "lib": 38270, "Ġherramient": 38271, "Ġ×IJ×ķ×ŀר": 38272, "Ġвидим": 38273, "millimeter": 38274, "Ġsilhouette": 38275, "Ġdriveway": 38276, "Ġcherish": 38277, "ãħłãħł": 38278, "Ġransom": 38279, "Ġinterdisciplinary": 38280, "ĠPortal": 38281, "Ġtrag": 38282, "thood": 38283, "Ġtedious": 38284, "Ġglossy": 38285, "Ġprépar": 38286, "ĠCay": 38287, "ĠTook": 38288, "ĠBottom": 38289, "Ġzig": 38290, "å«": 38291, "åį±": 38292, "represented": 38293, "à¹Ģลย": 38294, "Ġdesarrollo": 38295, "ìĦľë": 38296, "Ġviscos": 38297, "Ġmilligram": 38298, "ĠGund": 38299, "Ġferment": 38300, "drum": 38301, "Ġdrawers": 38302, "Laugh": 38303, "Ġpelos": 38304, "Ġpavement": 38305, "Ġmemoir": 38306, "avait": 38307, "Ġ2050": 38308, "¤ë¥¼": 38309, "Ġrazón": 38310, "Ġflourish": 38311, "Ġstern": 38312, "ä¸Ī": 38313, "ĠChung": 38314, "Ġserpent": 38315, "ĠGentlemen": 38316, "羣çļĦå¾Ī": 38317, "kook": 38318, "Ġlut": 38319, "importe": 38320, "parent": 38321, "Ġwsz": 38322, "Ġscree": 38323, "ĠMitarbeiter": 38324, "å·´": 38325, "mut": 38326, "Ġìĸĺ기를": 38327, "Ġsemble": 38328, "ĠOW": 38329, "Ġinvestigator": 38330, "ĠCheryl": 38331, "ĠGerald": 38332, "Ġprere": 38333, "Ġcompares": 38334, "nyt": 38335, "Ġdiferença": 38336, "?-": 38337, "Ġquá": 38338, "ר×Ļ": 38339, "Sen": 38340, "Ġheps": 38341, "Ġgratuit": 38342, "Ġconsort": 38343, "ĠSTOP": 38344, "ĠProtestant": 38345, "Ġelectrode": 38346, "âĹ": 38347, "Ġsecurely": 38348, "иÑĩеÑģкой": 38349, "Ġtää": 38350, "Ġregisters": 38351, "ĠHeavenly": 38352, "ogly": 38353, "issä": 38354, "ĠPhysics": 38355, "ĠMerkel": 38356, "Ġrév": 38357, "éĻ¢": 38358, "Ġerased": 38359, "ĠSacramento": 38360, "Ġcoffin": 38361, "Ġexacer": 38362, "Ġlanz": 38363, "Ġpoets": 38364, "ulif": 38365, "Ġì¹ĺë": 38366, "ĠNerd": 38367, "ĠNCT": 38368, "ĠHour": 38369, "nehmer": 38370, "ŀĺëıĦ": 38371, "ĠPrinci": 38372, "Sw": 38373, "mies": 38374, "armed": 38375, "ĠBeatles": 38376, "Ġpropagation": 38377, "Ġexchanged": 38378, "Ġcumulative": 38379, "Ġì§ijìĹIJ": 38380, "Ġdefeating": 38381, "æĬ±": 38382, "bels": 38383, "Ġwes": 38384, "ĠOdyssey": 38385, "ä½łæĥ³": 38386, "avior": 38387, "ĠìľĦìĹIJ": 38388, "Ġbrit": 38389, "Ġhijo": 38390, "DAY": 38391, "ĠاÙĦتÙĬ": 38392, "ĠСеÑĢг": 38393, "Ñĥка": 38394, "edsiÄĻ": 38395, "Ġimpos": 38396, "Ġellas": 38397, "Ġfirearms": 38398, "ĠNR": 38399, "Ġ×ij×IJ": 38400, "ĠÐŁÐ¾ÐºÐ°": 38401, "awi": 38402, "ĠìĦ±ê³µ": 38403, "Ġpupils": 38404, "ĠTack": 38405, "Ġfrase": 38406, "ĠShip": 38407, "Ġstad": 38408, "举": 38409, "ĠGreater": 38410, "unun": 38411, "immung": 38412, "grown": 38413, "ĠNXT": 38414, "ĠAmericas": 38415, "fox": 38416, "Ġmanten": 38417, "éłIJåĤĻ": 38418, "ĠÑģок": 38419, "Ġrikt": 38420, "lectric": 38421, "deep": 38422, "ĠзнаеÑĪÑĮ": 38423, "Ġbenut": 38424, "ĠInfrast": 38425, "ĠEmir": 38426, "ĠоÑĤпÑĢав": 38427, "ĠKimchi": 38428, "ĠFinnish": 38429, "´ìłģ": 38430, "inaire": 38431, "Ġoike": 38432, "æ¸ħæ¥ļ": 38433, "Ġhostage": 38434, "ĠButton": 38435, "ÙĤÙĬ": 38436, "eking": 38437, "ĠKazakh": 38438, "Ġcomforting": 38439, "Ġsog": 38440, "Ġgreeted": 38441, "guitar": 38442, "payer": 38443, "Ġrelational": 38444, "Ġconstruir": 38445, "çī¹åĪ¥": 38446, "opian": 38447, "ĠVolume": 38448, "ieth": 38449, "ÑģÑĤвом": 38450, "urrection": 38451, "liÅĽmy": 38452, "Ġhemisphere": 38453, "ĠBean": 38454, "IGN": 38455, "Ġkötü": 38456, "ĠFallout": 38457, "Ġbrace": 38458, "ç¹¼çºĮ": 38459, "ÏĢά": 38460, "ĠHAS": 38461, "Ġgé": 38462, "Ġcharacterize": 38463, "ặc": 38464, "ĠMilky": 38465, "Ġtumors": 38466, "Ġnuit": 38467, "ĠGaz": 38468, "ĠìŀĪëĭ¤ëĬĶ": 38469, "ĠгаÑĢ": 38470, "essment": 38471, "ĠAbe": 38472, "Ġë½ij": 38473, "ĠEinsatz": 38474, "JIN": 38475, "jä": 38476, "Cry": 38477, "ĠPromised": 38478, "ĠÑģеÑĢд": 38479, "okus": 38480, "Ġscalable": 38481, "ĠпоÑģмоÑĤÑĢеÑĤÑĮ": 38482, "ücklich": 38483, "Ġrealism": 38484, "Ġmayo": 38485, "Ġjuvenile": 38486, "Ġheadlights": 38487, "ĠgörÃ¼ÅŁ": 38488, "ĠReform": 38489, "Ġhalves": 38490, "czne": 38491, "Ġbreakup": 38492, "żej": 38493, "Ġrätt": 38494, "Day": 38495, "ĠìĿ¼ë³¸": 38496, "Ġmuerte": 38497, "Ġtunes": 38498, "ĠSmile": 38499, "record": 38500, "Ġrecherche": 38501, "atisfied": 38502, "Ġpozi": 38503, "Ġcelebrations": 38504, "isexual": 38505, "ĠROB": 38506, "thirds": 38507, "ĠFortune": 38508, "ĠÑĤой": 38509, "Ġbranded": 38510, "loo": 38511, "Ġdud": 38512, "Ġrandomized": 38513, "Ġcombin": 38514, "ä¸ĢäºĽ": 38515, "ieran": 38516, "czenia": 38517, "įãĥ«": 38518, "Ġcurator": 38519, "Ġartery": 38520, "ĠÑĥÑĪ": 38521, "ĠÑĩиÑĤ": 38522, "Ġsubsidies": 38523, "Ġblossom": 38524, "ĠTwilight": 38525, "Ġhyvä": 38526, "ĠPompe": 38527, "ĠCisco": 38528, "ĠÐŁÑĢо": 38529, "Ġbiri": 38530, "Ġgern": 38531, "Ġrebuilt": 38532, "Ġwcze": 38533, "Ġbenefici": 38534, "Ġdrummer": 38535, "Ġsolids": 38536, "Ġdiyorsun": 38537, "ãģĤãĤĬãģĮãģ¨ãģĨãģĶãģĸãģĦãģ¾ãģĹãģŁ": 38538, "lated": 38539, "Ġmuddy": 38540, "Ġholog": 38541, "Ġclaps": 38542, "ĠRings": 38543, "ĠOkey": 38544, "ĠBrave": 38545, "Ġvaluation": 38546, "Ġmigrant": 38547, "Ġintermitt": 38548, "Ġeigene": 38549, "iliary": 38550, "ãĥ¼ãĥĪ": 38551, "markt": 38552, "kr": 38553, "ĠRib": 38554, "á»Ļi": 38555, "Ġaccusations": 38556, "Ġarab": 38557, "wash": 38558, "ĠBardzo": 38559, "Ġugh": 38560, "esters": 38561, "ophren": 38562, "Ġalimentos": 38563, "ĠUz": 38564, "ÖĤ": 38565, "Ġ650": 38566, "ĠпÑĢиеÑħ": 38567, "FI": 38568, "Ġsampai": 38569, "Ġparlé": 38570, "hesion": 38571, "Ġsır": 38572, "Ġapparatus": 38573, "Ġcorrelated": 38574, "ĠPrincipal": 38575, "Ġcorr": 38576, "ĠOfficial": 38577, "иÑĩеÑģкие": 38578, "Ġterminals": 38579, "Should": 38580, "Ġvacun": 38581, "Ġstellt": 38582, "Ġmooi": 38583, "etzung": 38584, "ĠкÑĢа": 38585, "Ġdai": 38586, "Ġпож": 38587, "Team": 38588, "ĠPPE": 38589, "ĠÐŀÑģ": 38590, "ĠLeah": 38591, "ĠIvy": 38592, "yst": 38593, "Ġuhhh": 38594, "Ġnighttime": 38595, "Ġtrendy": 38596, "Ġsecurities": 38597, "Ġcontinents": 38598, "Ġfirsthand": 38599, "ĠVeron": 38600, "ĠëĤ®": 38601, "Ġbrowsing": 38602, "ĠCada": 38603, "tro": 38604, "Ġtramp": 38605, "reib": 38606, "Ġerstmal": 38607, "irler": 38608, "Ġpsic": 38609, "Ġgetir": 38610, "ĠNP": 38611, "Ġdzieci": 38612, "обÑĢаз": 38613, "Ġmagician": 38614, "Ġscrutiny": 38615, "Ġslab": 38616, "ĠOT": 38617, "isty": 38618, "iries": 38619, "orest": 38620, "Ġtasked": 38621, "Ġmorally": 38622, "ìķ¼ì§Ģ": 38623, "ustered": 38624, "Ġfools": 38625, "Ġirrespons": 38626, "Ġeinf": 38627, "Ġviá»ĩc": 38628, "Ġscor": 38629, "Ġpillows": 38630, "ĠGegen": 38631, "Ġtutte": 38632, "Ġquarterly": 38633, "Ġdidnt": 38634, "ĠGym": 38635, "ĠEther": 38636, "ĠØ«": 38637, "лиÑĪком": 38638, "Ġsignaling": 38639, "ĠNode": 38640, "ĠDoncs": 38641, "Ġyah": 38642, "ĠKanal": 38643, "Ġfading": 38644, "etin": 38645, "Ġinfluencers": 38646, "Ġmedals": 38647, "Ġengineered": 38648, "Ġfermented": 38649, "ê²łì§Ģë§Į": 38650, "ĠBeethoven": 38651, "×ŀש": 38652, "inental": 38653, "ĠìķĮ볤": 38654, "ütfen": 38655, "alnya": 38656, "Ġovere": 38657, "Ġdenkt": 38658, "акÑĤеÑĢ": 38659, "Ġâĺ": 38660, "Ġnecesit": 38661, "Ġgenerators": 38662, "grass": 38663, "ĠподÑĥм": 38664, "lieÃŁen": 38665, "Bar": 38666, "ľëıĻ": 38667, "ĠдеÑĤей": 38668, "Ġsucking": 38669, "Ġstencil": 38670, "Ġprimo": 38671, "ĠBreath": 38672, "strom": 38673, "Ġimmensely": 38674, "Ġappreh": 38675, "ìłķìĿ´": 38676, "Pop": 38677, "Ġjong": 38678, "ĠGiul": 38679, "ĠADHD": 38680, "Ġhören": 38681, "Ġelo": 38682, "ivent": 38683, "Ġrus": 38684, "Ġoutrageous": 38685, "Ġmastered": 38686, "Ġ커": 38687, "ÙĪÙģ": 38688, "ipes": 38689, "ĠRudy": 38690, "Jacob": 38691, "Ġbullish": 38692, "Ġtapped": 38693, "Ġfaud": 38694, "izophren": 38695, "ĠÑģоÑħ": 38696, "ĠDarling": 38697, "Ġ1963": 38698, "ĠPrevention": 38699, "²Ķ": 38700, "Ġabdominal": 38701, "stones": 38702, "Ġavaient": 38703, "á»ķi": 38704, "make": 38705, "Ġsare": 38706, "ĠInstant": 38707, "кам": 38708, "Ġkeeper": 38709, "Ġblankets": 38710, "ãģ§ãģĹãĤĩãģĨ": 38711, "Ġsweats": 38712, "ĠMinneapolis": 38713, "åħ¨éĥ¨": 38714, "Ġgenommen": 38715, "Ġfasten": 38716, "ĠBrussels": 38717, "åij¼": 38718, "Ġcafeter": 38719, "Ġabsorbing": 38720, "Ġhago": 38721, "ĠElmo": 38722, "Ġgusto": 38723, "ĠYap": 38724, "Música": 38725, "Ġtert": 38726, "Ġbanda": 38727, "Ġmily": 38728, "Ġthereafter": 38729, "ĠStockholm": 38730, "ĠCarson": 38731, "Ġcalibration": 38732, "avaÅŁ": 38733, "ansa": 38734, "ikke": 38735, "Ġforesee": 38736, "Ġqualche": 38737, "Ġdeste": 38738, "æ¤": 38739, "ünüz": 38740, "Ġforge": 38741, "Dis": 38742, "esten": 38743, "Ġδια": 38744, "Ġencaps": 38745, "ĠGespr": 38746, "Ġchercher": 38747, "ickets": 38748, "ÑĤоÑĢÑĭ": 38749, "Cr": 38750, "ĠТакже": 38751, "Ġrabbits": 38752, "ĠDot": 38753, "heiten": 38754, "Ġcausal": 38755, "ĠFoster": 38756, "ajÄħc": 38757, "Ġbereit": 38758, "Ġayudar": 38759, "é«Ļ": 38760, "ãģ³": 38761, "song": 38762, "comb": 38763, "Ġfringe": 38764, "Ġcybersecurity": 38765, "Ġ뾨": 38766, "Ġkier": 38767, "Ġbeschäft": 38768, "ĠконÑĨе": 38769, "Ġfacilit": 38770, "ĠNamen": 38771, "Ġbilateral": 38772, "tx": 38773, "ĠWissenschaft": 38774, "Ġnuances": 38775, "Ġripping": 38776, "Ġfy": 38777, "ĠSicherheit": 38778, "ĠGhana": 38779, "olon": 38780, "Ġtopped": 38781, "ĠMorocco": 38782, "Ġradial": 38783, "ĠLEE": 38784, "ĠAndreas": 38785, "edd": 38786, "ĠìĹ´ë": 38787, "ĠAirlines": 38788, "ãģĵãĤį": 38789, "Ġvalores": 38790, "ê·ľ": 38791, "Hy": 38792, "ĠзадаÑĩ": 38793, "ĠKendall": 38794, "ĠÑħаÑĢ": 38795, "ĠVamp": 38796, "Ġpython": 38797, "Ġmanageable": 38798, "ĠGente": 38799, "oise": 38800, "iciary": 38801, "Ġimposs": 38802, "ĠBunny": 38803, "iesta": 38804, "Andrew": 38805, "Ġsert": 38806, "ĠCec": 38807, "zzarella": 38808, "Ġautomobile": 38809, "ĠTiere": 38810, "allows": 38811, "åĨĨ": 38812, "Ġë°Ģ": 38813, "ĠScorp": 38814, "ĠJelly": 38815, "agara": 38816, "ĠStretch": 38817, "Ġredef": 38818, "Ġexacerb": 38819, "ĠSHA": 38820, "éf": 38821, "orsa": 38822, "Ġflawed": 38823, "ĠNoel": 38824, "?!?": 38825, "Ġprocent": 38826, "Ġmenstru": 38827, "ĠпÑĢоÑĩ": 38828, "Ġinfants": 38829, "ðŁİµ": 38830, "pause": 38831, "ĠRacing": 38832, "Ġ1948": 38833, "Ġsuperintendent": 38834, "idores": 38835, "idy": 38836, "brahim": 38837, "Ġunlucky": 38838, "Ġperk": 38839, "anci": 38840, "Ġë§ĮëĤĺ": 38841, "ĠÐľÐ¾Ñģкв": 38842, "Ġfinans": 38843, "Ġdiferencia": 38844, "łĪìĿ´": 38845, "éħį": 38846, "ORY": 38847, "ĠTac": 38848, "ÛĮا": 38849, "Ġdesem": 38850, "Ġважно": 38851, "ĠJU": 38852, "ĠìŀĪìŀĸìķĦìļĶ": 38853, "ĠÎĿ": 38854, "Ġinformations": 38855, "ĠHEL": 38856, "hst": 38857, "ĠпоговоÑĢ": 38858, "Ġvoiture": 38859, "Ġreus": 38860, "ändig": 38861, "ĠпоÑħож": 38862, "jing": 38863, "Ġdru": 38864, "altra": 38865, "Ġproduits": 38866, "Ġkite": 38867, "Ġeyeball": 38868, "ĠBelt": 38869, "ĠRestaurant": 38870, "Ġgamb": 38871, "Ġporridge": 38872, "itters": 38873, "Ġconverts": 38874, "Ġyardım": 38875, "Ġmáximo": 38876, "wirtschaft": 38877, "ĠíķĺëĤĺë": 38878, "Ġì¤Ģ": 38879, "Ġiceberg": 38880, "Ġvorbei": 38881, "Ġ256": 38882, "ocratic": 38883, "Ġreckless": 38884, "onner": 38885, "Ġmús": 38886, "Ġlogically": 38887, "ĠPrison": 38888, "ĠNetz": 38889, "Ġvacant": 38890, "Ġnimmt": 38891, "ĠHARR": 38892, "Ġзов": 38893, "ĠDee": 38894, "ringe": 38895, "niest": 38896, "ĠRules": 38897, "ìĬ¤ëŁ½": 38898, "cussions": 38899, "Ġfloral": 38900, "Ġconstrained": 38901, "Ġdifferentiation": 38902, "ĠQuebec": 38903, "ĠÛģÛĮÚº": 38904, "Ġpública": 38905, "itel": 38906, "Ġaccommodations": 38907, "ĠGrü": 38908, "íľ": 38909, "Ġpickles": 38910, "иÑĩеÑģкиÑħ": 38911, "Ġcommissions": 38912, "ĠBaek": 38913, "ĠçocuÄŁ": 38914, "ĠMedium": 38915, "Ġperiodically": 38916, "Ġwonderfully": 38917, "Ġstaffing": 38918, "ìĽIJë": 38919, "rire": 38920, "fle": 38921, "ĠMcL": 38922, "ĠÑĤеп": 38923, "ĠпеÑĢек": 38924, "нолог": 38925, "Ġíģ¬ê²Į": 38926, "çĻ¼çı¾": 38927, "Ġprosperous": 38928, "ĠSpiritual": 38929, "ĠChick": 38930, "DIA": 38931, "ĠÐŁÑĢивеÑĤ": 38932, "ĠperÃŃ": 38933, "ÑĮÑİÑĤ": 38934, "Ġconsultants": 38935, "ĠEarl": 38936, "ä»Ĭå¹´": 38937, "Ġruining": 38938, "оÑĢе": 38939, "Ġpenser": 38940, "Ġtakiej": 38941, "Ġstrengthened": 38942, "ĠLiquid": 38943, "онеÑĨ": 38944, "аваÑĤÑĮ": 38945, "Ġcamer": 38946, "Ġdisagreement": 38947, "Ġbathing": 38948, "ĠYosh": 38949, "aal": 38950, "prechen": 38951, "RISADAS": 38952, "Ġsuperstar": 38953, "æģŃ": 38954, "лÑıÑĤÑĮ": 38955, "Ġnib": 38956, "ĠTherm": 38957, "ĠDANIEL": 38958, "Ġpaw": 38959, "Ġliquids": 38960, "Ġcapacit": 38961, "arken": 38962, "Ġvagina": 38963, "Ġmashed": 38964, "Ġemerges": 38965, "yscy": 38966, "Ġunrelated": 38967, "ĠGuild": 38968, "Ġinverted": 38969, "itives": 38970, "Tra": 38971, "Ġbegr": 38972, "Ġalte": 38973, "ì§ķ": 38974, "ãĤģãģ¦": 38975, "ĠÑĢазÑĢабоÑĤ": 38976, "finder": 38977, "Ġдалее": 38978, "ĠблагодаÑĢ": 38979, "walker": 38980, "Ġcrater": 38981, "assadors": 38982, "rences": 38983, "inski": 38984, "ĠKIM": 38985, "ĠElliot": 38986, "2017": 38987, "ĠSr": 38988, "inka": 38989, "anov": 38990, "Ġìŀĺ못": 38991, "Ġproprietary": 38992, "displaystyle": 38993, "ĠÑģим": 38994, "Ġизб": 38995, "ĠPanel": 38996, "Ġinstincts": 38997, "ĠCommunications": 38998, "麻": 38999, "midt": 39000, "Ġë§Įëĵ¤ìĸ´": 39001, "ĠÑģлова": 39002, "ĠGilbert": 39003, "缮åīį": 39004, "Так": 39005, "voorbeeld": 39006, "еÑİÑģÑĮ": 39007, "aryn": 39008, "quez": 39009, "Ġdart": 39010, "ÑĸÑĪ": 39011, "ĠHut": 39012, "Sal": 39013, "Ġsoutheast": 39014, "Ġpesticides": 39015, "Ġhelicopters": 39016, "Ġendured": 39017, "iada": 39018, "Ġbrewing": 39019, "ìŬë": 39020, "ĠÑģвобод": 39021, "ĠSaints": 39022, "ĠFrançais": 39023, "ĠEconomics": 39024, "Ġdisloc": 39025, "ophobia": 39026, "Camer": 39027, "Ġnegotiated": 39028, "ĠÑģÑĤали": 39029, "ìĬ¤íģ": 39030, "ogie": 39031, "Ġtsunami": 39032, "Ġpeeled": 39033, "Ġmotivations": 39034, "è¨Ń": 39035, "ostat": 39036, "flan": 39037, "ĠDAC": 39038, "Ġkav": 39039, "'RE": 39040, "ĠPearson": 39041, "bbe": 39042, "czenie": 39043, "Ġatenção": 39044, "íĨµëł¹": 39045, "ãģ£ãģ¡": 39046, "ĠÑĥдаÑĢ": 39047, "Ġintroductory": 39048, "ĠIci": 39049, "ëĮĢë": 39050, "akat": 39051, "Ġtrench": 39052, "Ġproceeded": 39053, "ĠCoin": 39054, "Ġderecho": 39055, "ĠRede": 39056, "æ¯Ľ": 39057, "аннÑĭй": 39058, "Ġincarcerated": 39059, "ĠRichmond": 39060, "Rock": 39061, "ĠPav": 39062, "ĠKarma": 39063, "uges": 39064, "Ġconteú": 39065, "ë¹Ħ": 39066, "Ġê·¸ë§Į": 39067, "ĠGone": 39068, "ĠwspóÅĤ": 39069, "ĠRahmen": 39070, "unken": 39071, "Ġì¤ijìļĶíķľ": 39072, "Ġib": 39073, "Ġattaching": 39074, "Hay": 39075, "Ġsuka": 39076, "ìį¹": 39077, "Ġpivotal": 39078, "ĠRespect": 39079, "ÃŃda": 39080, "IB": 39081, "ĠVerantwort": 39082, "wiet": 39083, "Ġforensic": 39084, "ÑĢиÑģÑĤ": 39085, "ĠпÑĢинÑĨипе": 39086, "Ġmarkings": 39087, "Ġkettle": 39088, "ĠOpera": 39089, "ĠDoctors": 39090, "Ġshredded": 39091, "Ġrecuer": 39092, "Ġvigil": 39093, "ĠFail": 39094, "Ġentrev": 39095, "ĠдÑĥÑĪ": 39096, "Ġoutbreaks": 39097, "èµ°åIJ§": 39098, "ĠÏĢο": 39099, "Ġrogue": 39100, "angled": 39101, "Ġyearly": 39102, "ĠCreed": 39103, "Ġwam": 39104, "Ġlotus": 39105, "ê³¼ë": 39106, "ãĢģãĢģ": 39107, "ĠSpit": 39108, "ĠItu": 39109, "Ġstrains": 39110, "Ġstamped": 39111, "Ġplaint": 39112, "Ġpotion": 39113, "Ġconsolidation": 39114, "è©ķ": 39115, "оÑĩкÑĥ": 39116, "Ġvlogging": 39117, "Ġslate": 39118, "ĠAuft": 39119, "ĠIncor": 39120, "ừng": 39121, "§IJ": 39122, "enh": 39123, "ĠheiÃŁ": 39124, "Ġdomest": 39125, "ĠStrom": 39126, "åį³": 39127, "akis": 39128, "Ġfragen": 39129, "Ġfiner": 39130, "ĠSug": 39131, "Ġuphill": 39132, "Ġéén": 39133, "âĢ¦)": 39134, "ĠÑģоп": 39135, "ĠCorey": 39136, "Ġsiebie": 39137, "Ġmuse": 39138, "Ġcloves": 39139, "Ġpous": 39140, "ĠFinanz": 39141, "ĠRoute": 39142, "amat": 39143, "Ġmutually": 39144, "ĠвнÑĥÑĤÑĢи": 39145, "ĠSelena": 39146, "ëĶ": 39147, "ĠGaussian": 39148, "ë¶ĢíĦ°": 39149, "Ġ×ij׼": 39150, "Ġejerc": 39151, "å¾®": 39152, "kea": 39153, "ĠGerry": 39154, "ĠSic": 39155, "大çļĦ": 39156, "Ġ1966": 39157, "iese": 39158, "Ġfossils": 39159, "Ġestad": 39160, "ĠKane": 39161, "ciÄĩ": 39162, "ĠìľłíĬľë": 39163, "Ġпам": 39164, "ĠCruise": 39165, "intérieur": 39166, "Ġbekannt": 39167, "ĠPode": 39168, "Ġdemander": 39169, "Rem": 39170, "Ġinvade": 39171, "Ġdecorating": 39172, "ropic": 39173, "Ġcowboy": 39174, "ĠPhoto": 39175, "opolit": 39176, "Ġì»¬ëŁ¬ë": 39177, "Ġreap": 39178, "Ġhandwriting": 39179, "à¹Ħร": 39180, "Ġëļ": 39181, "Ġبعد": 39182, "ĠMt": 39183, "ÙĢ": 39184, "Ġspaceship": 39185, "Ġnationalism": 39186, "Ġcouncils": 39187, "ĠGriffin": 39188, "ĠAhmed": 39189, "Ġclich": 39190, "ĠOL": 39191, "wl": 39192, "ĠPilot": 39193, "å®®": 39194, "Ġacronym": 39195, "Ġgels": 39196, "Ġelectroly": 39197, "èĵ": 39198, "Ġмной": 39199, "Ġepisod": 39200, "ĠDieses": 39201, "ĠATP": 39202, "Ġediyorum": 39203, "Ġexpresses": 39204, "Ġexhibits": 39205, "Comm": 39206, "ĠкÑĢÑĥп": 39207, "Ġmatar": 39208, "Ġ2025": 39209, "ĠArtem": 39210, "vasive": 39211, "rÃł": 39212, "ĠbeÅŁ": 39213, "é»ĥ": 39214, "Ġlizard": 39215, "Ġfille": 39216, "Ġì§Ī문": 39217, "ĠмоÑī": 39218, "Ġtür": 39219, "Ġculprit": 39220, "Ġwoven": 39221, "ĠANY": 39222, "nim": 39223, "Ġtay": 39224, "Ġpromin": 39225, "Ġacompa": 39226, "Ġidé": 39227, "Ġboiler": 39228, "ĠThemen": 39229, "Ġavenue": 39230, "ĠMud": 39231, "ĠновÑĭе": 39232, "Ġwitnessing": 39233, "Ġlance": 39234, "ĠCHAN": 39235, "ĠBever": 39236, "تÙħ": 39237, "Ġchemotherapy": 39238, "King": 39239, "ĠbÄĻdÄĻ": 39240, "Ġatual": 39241, "Ġtive": 39242, "Ġtalkin": 39243, "Ġquedar": 39244, "ieÃŁ": 39245, "edel": 39246, "Ġìĸ´ìłľ": 39247, "Ġjogar": 39248, "Ġör": 39249, "Ġundertaking": 39250, "ĠStrength": 39251, "Ġmilhões": 39252, "ĠWine": 39253, "ĠMolt": 39254, "讲": 39255, "ãģijãĤĮ": 39256, "Ġundermine": 39257, "ĠArchives": 39258, "vana": 39259, "mercial": 39260, "MC": 39261, "Ġcaste": 39262, "пÑĢ": 39263, "Ġlegislators": 39264, "ulators": 39265, "ênio": 39266, "Ġëį°ë": 39267, "ĠÑħоÑĤиÑĤе": 39268, "Ġнек": 39269, "Ġsurn": 39270, "Ġconsci": 39271, "ĠPOW": 39272, "Ġculinary": 39273, "ĠKAT": 39274, "ĠFolks": 39275, "Ñĭваем": 39276, "Ġвок": 39277, "ãģijãĤĭ": 39278, "service": 39279, "pts": 39280, "Ġпобед": 39281, "æĺ¯åķĬ": 39282, "Ġtents": 39283, "Ġnord": 39284, "STE": 39285, "Ġrepublican": 39286, "Ġwyk": 39287, "Ġminions": 39288, "èĻķ": 39289, "Ġmemang": 39290, "jest": 39291, "Ġcomparative": 39292, "Ġtyle": 39293, "carbon": 39294, "bedingt": 39295, "ksen": 39296, "Ġnegativity": 39297, "Ġsjälv": 39298, "Ġdú": 39299, "æīĢæľī": 39300, "Ġrecalled": 39301, "cra": 39302, "ĠTada": 39303, "ĠÑĢÑĥки": 39304, "ĠопÑĢедел": 39305, "Ġprocrast": 39306, "Ġjogos": 39307, "ĠOo": 39308, "ĠHearts": 39309, "Ġéch": 39310, "ĠksiÄħż": 39311, "Ġcoarse": 39312, "ĠTube": 39313, "ĠGreens": 39314, "Ġén": 39315, "Ġdumbbell": 39316, "ĠÑĤи": 39317, "Ġquerer": 39318, "اØŃ": 39319, "Ïĥει": 39320, "ĠпÑĢавилÑĮно": 39321, "Ġпап": 39322, "Ġcompra": 39323, "Ġtér": 39324, "ĠAntes": 39325, "Ġoptimum": 39326, "Ġbiscuit": 39327, "κι": 39328, "aczego": 39329, "Ġìĭľê°ĦìĿ´": 39330, "ĠMarines": 39331, "vero": 39332, "Ġvaccinations": 39333, "Ġpetty": 39334, "riters": 39335, "Ġал": 39336, "country": 39337, "Ġcounters": 39338, "Ġattendant": 39339, "ĠHui": 39340, "ãģ¨ãģĦãģĨãģĵãģ¨ãģ§": 39341, "cka": 39342, "ÑģÑĤвеннÑĭй": 39343, "guy": 39344, "Ġtricked": 39345, "ĠRED": 39346, "Ġthrilling": 39347, "ÏĢοι": 39348, "Ġpiggy": 39349, "Ġanunci": 39350, "ORTER": 39351, "ĠValue": 39352, "Ġrond": 39353, "ĠADA": 39354, "Ġposer": 39355, "hores": 39356, "ĠRoland": 39357, "ĵ¯": 39358, "Ġnoir": 39359, "Ġש×IJ×": 39360, "ë°ľ": 39361, "iemand": 39362, "ĠпоÑĤеÑĢ": 39363, "ê³³": 39364, "Ġê±±": 39365, "Ġformatting": 39366, "ĠLed": 39367, "è§Ģçľ¾": 39368, "Ġkillers": 39369, "ĠÄijấy": 39370, "Ġhaar": 39371, "again": 39372, "!>[": 45687, "minster": 45688, "Ġвли": 45689, "Ġidentifier": 45690, "ĠLambda": 45691, "Ġtros": 45692, "Ġflawless": 45693, "Ġdetrimental": 45694, "Ġbunları": 45695, "War": 45696, "Ġregião": 45697, "羣çļĦæĺ¯": 45698, "ĠBike": 45699, "cessors": 45700, "Ġcùng": 45701, "ĠRN": 45702, "Ġê½ĥ": 45703, "Ġküçük": 45704, "ĠBeginning": 45705, "íĺ¸ë": 45706, "Ġgewe": 45707, "Ġdenote": 45708, "ĠAlberto": 45709, "Ġprobiot": 45710, "Ġode": 45711, "Ġmolar": 45712, "Ġbursting": 45713, "assumed": 45714, "Ġfootprints": 45715, "veda": 45716, "Ġsteroids": 45717, "Ġflaming": 45718, "ĠEller": 45719, "Ġerkennen": 45720, "ätzen": 45721, "Ġlifecycle": 45722, "ĠDOU": 45723, "ĠKarena": 45724, "ĠGuerra": 45725, "è¿ĺæĺ¯": 45726, "Ġsinister": 45727, "Ġpodéis": 45728, "Ġparab": 45729, "Ġoko": 45730, "Ġmatéri": 45731, "Ġcaric": 45732, "sonaro": 45733, "Ġpraticamente": 45734, "ÑĥÑģа": 45735, "Ġcomunque": 45736, "Ġvigilant": 45737, "Ġregimes": 45738, "ĠShooting": 45739, "Ġraids": 45740, "ĠNora": 45741, "ĠWieder": 45742, "mens": 45743, "ĠÑģод": 45744, "Ġê²½ìļ°ìĹIJëĬĶ": 45745, "ĠвÑħод": 45746, "Ġautobi": 45747, "ĠSchn": 45748, "ĠRobbie": 45749, "ĠFitness": 45750, "ĠконÑĦ": 45751, "Ġpenguin": 45752, "моÑĤÑĢÑı": 45753, "Ġминим": 45754, "plays": 45755, "Ġdelegates": 45756, "Mer": 45757, "Ġsistem": 45758, "ĠMichaels": 45759, "male": 45760, "اع": 45761, "Ġcách": 45762, "ĠHä": 45763, "Ġ×Ļ×ķ×ĵ×¢": 45764, "Ġsuperpower": 45765, "Ġstron": 45766, "Ġrover": 45767, "Ġdépend": 45768, "éĻ³": 45769, "Ġretiring": 45770, "Ġvampires": 45771, "Ġmerde": 45772, "ĠChanging": 45773, "Ġtame": 45774, "Ġspokesperson": 45775, "Ġcay": 45776, "Ġflirting": 45777, "ĠGrö": 45778, "Ġwär": 45779, "Ġwyb": 45780, "Ġcoeur": 45781, "ạnh": 45782, "ĠìĻĢìĦľ": 45783, "Ġconnais": 45784, "ĠHundreds": 45785, "ĠBea": 45786, "ĠαÏĢ": 45787, "pruch": 45788, "Ġsociedade": 45789, "ĠWhilst": 45790, "ĠKait": 45791, "espace": 45792, "Ġchia": 45793, "ĠErm": 45794, "Ġë°Ķê¿": 45795, "Ġfences": 45796, "ĠMortal": 45797, "ê²ģ": 45798, "ĠгÑĢаÑĦ": 45799, "ĠHomeland": 45800, "ĠJUN": 45801, "isst": 45802, "Ġparlar": 45803, "Ġsporty": 45804, "éo": 45805, "Ġdeepen": 45806, "ĠBehavior": 45807, "éĢı": 45808, "åĵĪåĵĪåĵĪ": 45809, "Ġerrand": 45810, "Ġrotary": 45811, "ĠWellington": 45812, "Wind": 45813, "Ġmesela": 45814, "ảng": 45815, "iende": 45816, "Ġexcell": 45817, "ĠGenius": 45818, "ĠEduardo": 45819, "æľī人": 45820, "ĠÅŁunu": 45821, "ĠÄ°stanbul": 45822, "Ġproduto": 45823, "Ġãħİãħİ": 45824, "OFF": 45825, "Ġwollt": 45826, "çĪĨ": 45827, "Ġëī´ìĬ¤": 45828, "Ġlass": 45829, "Ġhertz": 45830, "Ġaromatic": 45831, "Ġзвон": 45832, "Ġautoc": 45833, "ĠLust": 45834, "Ġ112": 45835, "ĠÎĹ": 45836, "Ġreviewers": 45837, "Ġreceptive": 45838, "å°įäºĨ": 45839, "ând": 45840, "oglo": 45841, "ĠìķĦëĭĻ": 45842, "Ġngo": 45843, "ÑĸÑĤи": 45844, "Ã¥t": 45845, "cono": 45846, "Ġtekrar": 45847, "Ġì£¼ê³ł": 45848, "ĠgelmiÅŁ": 45849, "Ġbedtime": 45850, "ĠArgh": 45851, "ADA": 45852, "ĠгоÑĢода": 45853, "ĠÄĩ": 45854, "Ġalliances": 45855, "giggling": 45856, "Ġyerde": 45857, "Ġspies": 45858, "Ġgutes": 45859, "çi": 45860, "Ġalltid": 45861, "ĠLah": 45862, "ŀIJë": 45863, "ĠdokÅĤad": 45864, "ÙĪÙĬ": 45865, "Ġtoxicity": 45866, "Ġcancellation": 45867, "Ġ1958": 45868, "dro": 45869, "ĠìŀijìĿĢ": 45870, "ĠMotorola": 45871, "Ġmultin": 45872, "Ġenthusiasts": 45873, "ĠMighty": 45874, "ĠCoconut": 45875, ":ãĢĮ": 45876, "ĠPictures": 45877, "Ġsangre": 45878, "Ġblinking": 45879, "olesome": 45880, "ĠìĬ¤íĥĢìĿ¼": 45881, "FP": 45882, "Ġbooming": 45883, "ĠдеÑģÑıÑĤ": 45884, "Ġratchet": 45885, "Ġtimelines": 45886, "leness": 45887, "Ġcages": 45888, "ĠGoodnight": 45889, "ometimes": 45890, "Ġcunning": 45891, "ĠRisk": 45892, "uled": 45893, "dade": 45894, "Ġprata": 45895, "ĠgustarÃŃa": 45896, "amus": 45897, "ĠJinping": 45898, "Ġestrut": 45899, "Ġdescobrir": 45900, "ĠMÄģ": 45901, "ĠAllan": 45902, "ĠåĪĨ": 45903, "Ġ׾ק": 45904, "Ġpreserv": 45905, "ĠStrawberry": 45906, "Äı": 45907, "Lu": 45908, "Ġkro": 45909, "ĠReports": 45910, "ìħĶìķ¼": 45911, "Ġvalt": 45912, "Ġpouvait": 45913, "Ġappar": 45914, "ĠBone": 45915, "Ġpreferably": 45916, "ĠRepública": 45917, "å°±åĪ°": 45918, "Ġherzlich": 45919, "Ġchimney": 45920, "Ġçev": 45921, "Ġvisas": 45922, "Ġverr": 45923, "Ġcultivation": 45924, "ĠArmenia": 45925, "ĠвдÑĢÑĥг": 45926, "Ġcockro": 45927, "retched": 45928, "artz": 45929, "ĠлÑİдÑıм": 45930, "ĠpolÃŃticas": 45931, "ĠPanz": 45932, "ĠAKA": 45933, "ĠëĪĮ룬": 45934, "Ġerro": 45935, "Ġcamper": 45936, "Ġ102": 45937, "स": 45938, "done": 45939, "Ġhoard": 45940, "ĠÐŁÐ¾ÑĤом": 45941, "jeong": 45942, "Ġdesta": 45943, "pak": 45944, "Ġinim": 45945, "Ġgrowers": 45946, "ĠMessage": 45947, "Ġelector": 45948, "engage": 45949, "ĠForbes": 45950, "ĠCincinnati": 45951, "Ġdifférence": 45952, "df": 45953, "Ġspar": 45954, "Ġawaits": 45955, "ĠUSSR": 45956, "ĠRising": 45957, "ĠHoÅŁ": 45958, "Ġfooting": 45959, "Ġcondiciones": 45960, "ÑĤоÑĢов": 45961, "Ġclinician": 45962, "ĠDiskuss": 45963, "å£ĵ": 45964, "ר×Ĵ": 45965, "×¥": 45966, "iteit": 45967, "gren": 45968, "Ġcharisma": 45969, "Ġleuke": 45970, "Ġirritating": 45971, "Ġcirca": 45972, "ĠRhodes": 45973, "Ġpior": 45974, "Ġhandicap": 45975, "royable": 45976, "Ġvull": 45977, "OG": 45978, "ĠinÃŃcio": 45979, "ieri": 45980, "Ġsplashing": 45981, "Ġdemise": 45982, "Ġassistir": 45983, "ÑĩÑĤо": 45984, "Ġcovert": 45985, "ĠGud": 45986, "à¸ī": 45987, "klär": 45988, "ĠìŀIJ꾸": 45989, "Ġverändert": 45990, "ĠREM": 45991, "ĠConven": 45992, "atge": 45993, "Ġpierwsze": 45994, "Ġclergy": 45995, "lington": 45996, "liv": 45997, "VPN": 45998, "ĠÑģожал": 45999, "ĠHate": 46000, "ãģ¨ãģĵãĤį": 46001, "ÏĨο": 46002, "ĠRespons": 46003, "озд": 46004, "Ġetmek": 46005, "Ġchemin": 46006, "ÙħØ©": 46007, "Ġê°Ģ족": 46008, "Tre": 46009, "Ġumas": 46010, "ĠBurton": 46011, "Ġpatriarch": 46012, "ĠSmithsonian": 46013, "¥ĺ": 46014, "Moon": 46015, "Air": 46016, "Ġmedios": 46017, "Ġeraser": 46018, "Ġwollten": 46019, "Ġpareil": 46020, "ĠBillie": 46021, "æĬ½": 46022, "еÑĢÑĤв": 46023, "Ġparlament": 46024, "Ġagony": 46025, "ĠQUE": 46026, "sequently": 46027, "Another": 46028, "ĠWhew": 46029, "ĠAnnual": 46030, "Ġseben": 46031, "ìĥģìĿĦ": 46032, "values": 46033, "ŀľë§Į": 46034, "Ġsinon": 46035, "ereal": 46036, "ĠEnlight": 46037, "ĠChemistry": 46038, "ĠCatalunya": 46039, "Ġdoctr": 46040, "anton": 46041, "Ġstuk": 46042, "ĠPlate": 46043, "ĠKardashian": 46044, "Ġfilos": 46045, "ĠWet": 46046, "ĠпопÑĭÑĤ": 46047, "Ġunknowns": 46048, "ĠSchon": 46049, "ĠBaldwin": 46050, "Ġtelescopes": 46051, "ĠGucci": 46052, "oxide": 46053, "ĠConservative": 46054, "ìĦ±ìĿĦ": 46055, "Ġhinaus": 46056, "Power": 46057, "Ġê±´ê°ķ": 46058, "Ġprevail": 46059, "orman": 46060, "machine": 46061, "Ġ1946": 46062, "Ġunbel": 46063, "Ġschaut": 46064, "Ġpiel": 46065, "eenth": 46066, "Ġobjectively": 46067, "Ġchakra": 46068, "audio": 46069, "Ġchicos": 46070, "ĠVault": 46071, "å°Ī": 46072, "Ġmedicinal": 46073, "ĠTail": 46074, "While": 46075, "Ġasphalt": 46076, "Ġfroze": 46077, "ĠEK": 46078, "unching": 46079, "nosis": 46080, "2015": 46081, "ĠGri": 46082, "Ġoddly": 46083, "ĠMär": 46084, "ĠAeg": 46085, "colo": 46086, "Par": 46087, "Ġëĵ¤ìĸ´ë": 46088, "Ġvinden": 46089, "ĠOVER": 46090, "Ġiced": 46091, "Ġscorp": 46092, "Ġhac": 46093, "qualified": 46094, "ĠÑĥвидеÑĤÑĮ": 46095, "ermo": 46096, "HEN": 46097, "Ġsoi": 46098, "Ġmultiples": 46099, "Ġlayouts": 46100, "Ġblindness": 46101, "ĠBowser": 46102, "ĠподÑĤ": 46103, "ĠÃİ": 46104, "ventional": 46105, "Ġmata": 46106, "madı": 46107, "Ġgeez": 46108, "Ġcadence": 46109, "Ġważne": 46110, "ĠChristie": 46111, "venge": 46112, "Call": 46113, "Ġturnaround": 46114, "Ġblob": 46115, "ĠЯк": 46116, "ĠVoiceover": 46117, "Ġperil": 46118, "ĠJaime": 46119, "ĠHOY": 46120, "lane": 46121, "Ġsebel": 46122, "ĠDuo": 46123, "ĠHistorical": 46124, "Ġdni": 46125, "Ġgema": 46126, "yk": 46127, "Ġsabem": 46128, "ắng": 46129, "Ġvars": 46130, "ĠRonnie": 46131, "ĠRonaldo": 46132, "ĠPerquè": 46133, "nsinn": 46134, "hair": 46135, "Ġrelentless": 46136, "Ġlyn": 46137, "Ġtraveler": 46138, "æĢİ麼äºĨ": 46139, "nine": 46140, "Ġantim": 46141, "Ġì¼Ģ": 46142, "Ġsnowball": 46143, "ĠÑħаÑĢакÑĤеÑĢ": 46144, "Ġinterns": 46145, "Ġconstituency": 46146, "ĠÐĿам": 46147, "׾׾": 46148, "VEL": 46149, "Ġviktigt": 46150, "Ġapoyo": 46151, "ÙĦب": 46152, "Ġjard": 46153, "Ġheightened": 46154, "ÑĢоÑģÑĤ": 46155, "ĠSMITH": 46156, "Ġдела": 46157, "Ġrepairing": 46158, "Ġrigt": 46159, "ĠSheikh": 46160, "ĠBritney": 46161, "Ġeverytime": 46162, "Ġadventurous": 46163, "ockey": 46164, "ernt": 46165, "Ġataque": 46166, "ĠAlternatively": 46167, "effect": 46168, "Ġpalavras": 46169, "ĠElliott": 46170, "Ġréussi": 46171, "Ġhypertension": 46172, "ĠManual": 46173, "Ġprophetic": 46174, "Ġhandc": 46175, "ÑĮе": 46176, "Ġrefrain": 46177, "ĠSquid": 46178, "ìŀ¡": 46179, "Ġкоман": 46180, "ällen": 46181, "Ġllegó": 46182, "Ġbash": 46183, "iony": 46184, "ĠÑģклад": 46185, "Ġкаб": 46186, "Ġcareless": 46187, "ĠPool": 46188, "Ġtrás": 46189, "Ġfils": 46190, "ĠSchr": 46191, "Ġsprawd": 46192, "ĠMonaten": 46193, "Ġunforgettable": 46194, "ĠCotton": 46195, "Ġinconvenient": 46196, "ĠRX": 46197, "oris": 46198, "Ġhumbled": 46199, "ת×Ĺ": 46200, "Ġآپ": 46201, "ĠincreÃŃ": 46202, "ĠKommentare": 46203, "èĪĴ": 46204, "ración": 46205, "Ġvantage": 46206, "ĠSeal": 46207, "ĠìĿ´ê±°ë¥¼": 46208, "Ġjoue": 46209, "ãģĿãģĨãģ§ãģĻãģŃ": 46210, "Ġìĺ¤ëŀĺ": 46211, "ĠиÑģпÑĭÑĤ": 46212, "oben": 46213, "Ġgrate": 46214, "Ġcontrole": 46215, "ĠPercy": 46216, "ÅĤada": 46217, "Ġsimultaneous": 46218, "Ġprototy": 46219, "ĠgroÃŁer": 46220, "Ġbewusst": 46221, "inizi": 46222, "Ġpassieren": 46223, "ĠHappiness": 46224, "åīĩ": 46225, "shi": 46226, "geht": 46227, "Ġstationed": 46228, "ĠErgebnis": 46229, "Ġdirectamente": 46230, "Ġsurvives": 46231, "Ġpersones": 46232, "BERG": 46233, "Ġvomiting": 46234, "Ġconhecer": 46235, "Ġadjour": 46236, "ĠCivic": 46237, "pei": 46238, "burst": 46239, "Ġëĭ¤ëĭĪ": 46240, "éı": 46241, "Ġsled": 46242, "Ġplataforma": 46243, "ĠSect": 46244, "ĠDefin": 46245, "çĻ»éĮ²": 46246, "énom": 46247, "chnet": 46248, "Ġprofitability": 46249, "Ġerreicht": 46250, "á»ıi": 46251, "cation": 46252, "Ġì§Ģê¸": 46253, "Ġperdre": 46254, "Ġfelony": 46255, "Ġ1957": 46256, "æĪijå¾Ī": 46257, "Ġunsuccessful": 46258, "Ġnagyon": 46259, "Ġelasticity": 46260, "Ġfacade": 46261, "Ġearthly": 46262, "ĠамеÑĢикан": 46263, "Ġconn": 46264, "cla": 46265, "Du": 46266, "Ġpolitiques": 46267, "Ġhalo": 46268, "iantes": 46269, "Ġмоей": 46270, "ãĥ³ãĥī": 46271, "tones": 46272, "elier": 46273, "è®ļ": 46274, "htaking": 46275, "Ġwichtige": 46276, "Ġanno": 46277, "ĠLok": 46278, "illions": 46279, "Ġviver": 46280, "Ġsolchen": 46281, "Ġsuf": 46282, "ĠSalz": 46283, "ĠNvidia": 46284, "zuge": 46285, "ĠSpike": 46286, "Video": 46287, "Ġtwor": 46288, "ĠAla": 46289, "èijī": 46290, "Ġhanya": 46291, "ĠAdm": 46292, "ìĿµ": 46293, "ĠPatienten": 46294, "ĠOnion": 46295, "ĠKobe": 46296, "ĠScene": 46297, "ĠRash": 46298, "æ¨Ļ": 46299, "ÑĢаÑģÑĤ": 46300, "istani": 46301, "General": 46302, "leye": 46303, "imbap": 46304, "Ġconcealed": 46305, "ĠFridays": 46306, "ĠWool": 46307, "ĠновÑĭÑħ": 46308, "شر": 46309, "Ġê²°ê³¼": 46310, "Ġjedoch": 46311, "´ìĭľ": 46312, "ĵ¤ëıĦ": 46313, "Ġìŀ¥ëĤľ": 46314, "ukt": 46315, "Lou": 46316, "Ġ먹ìĸ´": 46317, "ĠExpect": 46318, "Ġдомой": 46319, "Ġirresponsible": 46320, "Ġacerca": 46321, "ĠZust": 46322, "ר×ĺ": 46323, "UI": 46324, "Ġyoutubers": 46325, "ĠPositive": 46326, "Ġsocioe": 46327, "Ġsnatch": 46328, "èĥĮ": 46329, "Ġrefreshed": 46330, "Ġnominations": 46331, "ĠPatt": 46332, "Ġobsolete": 46333, "ĠdemiÅŁ": 46334, "åı¤": 46335, "ormuÅŁ": 46336, "ĠìĨĶì§ģíŀĪ": 46337, "Ġfla": 46338, "Ġcraziest": 46339, "ĠZie": 46340, "ĠTú": 46341, "zep": 46342, "icem": 46343, "Ġë©ĭìŀĪ": 46344, "Ġcynical": 46345, "ãģĿãĤĵãģª": 46346, "Ġtresp": 46347, "Ġcraz": 46348, "Õ¥Õ": 46349, "Ġnelle": 46350, "Ġmph": 46351, "ĠNered": 46352, "ĠKob": 46353, "ĠEck": 46354, "¨¸ëĭĪ": 46355, "Jan": 46356, "ĠТогда": 46357, "Ġdeci": 46358, "ĠVog": 46359, "Ġbubbling": 46360, "éĢĢ": 46361, "úa": 46362, "Ġproductos": 46363, "iberal": 46364, "Ġreplicated": 46365, "ĠImprove": 46366, "illary": 46367, "Cha": 46368, "Ġrédu": 46369, "ĥIJíķĺë©´": 46370, "Ġconnot": 46371, "ĠKrit": 46372, "ĠдÑĥÑħов": 46373, "Ġtreadmill": 46374, "ĠPW": 46375, "ĠзовÑĥÑĤ": 46376, "Ġclams": 46377, "Ġdrafting": 46378, "Ġ1956": 46379, "unta": 46380, "Ġexpenditures": 46381, "ĠHoover": 46382, "WOO": 46383, "ÑĪее": 46384, "Ġdeduction": 46385, "monary": 46386, "Ġrecib": 46387, "Ġpovo": 46388, "ĠëįĶë": 46389, "ĠPAL": 46390, "ĠBlow": 46391, "Ġwyp": 46392, "Ġdestac": 46393, "deal": 46394, "Graeme": 46395, "Ġnécessaire": 46396, "Ġdamned": 46397, "Ġ1938": 46398, "Ġìĭ¤ìłľë¡ľ": 46399, "Ġtroop": 46400, "Ġinsightful": 46401, "ĠTJ": 46402, "ĠоÑģв": 46403, "Ġfidelity": 46404, "ĠSkip": 46405, "ĠMayo": 46406, "ë§Ŀ": 46407, "appe": 46408, "Ġblas": 46409, "ĠWY": 46410, "ĠGN": 46411, "ctar": 46412, "Su": 46413, "Ġcuent": 46414, "hews": 46415, "Ġcorpses": 46416, "Abs": 46417, "Ġwastewater": 46418, "Ġciek": 46419, "ĠOnu": 46420, "Ġexplosives": 46421, "Ġarma": 46422, "ĠSTEPHAN": 46423, "politik": 46424, "ĠOsaka": 46425, "taÅĤ": 46426, "Ġyapıyor": 46427, "Ġizquier": 46428, "Ġbeleza": 46429, "ĠWyatt": 46430, "åIJ¸": 46431, "Ġsuk": 46432, "Ġspecjal": 46433, "Ġdanke": 46434, "whistle": 46435, "ĠfÃŃsica": 46436, "ĠHarriet": 46437, "ĠìķĦíĮĮ": 46438, "Ġwillkommen": 46439, "iping": 46440, "ĠÑģмоÑĤÑĢиÑĤе": 46441, "ĠможеÑĪÑĮ": 46442, "Ġinaccurate": 46443, "Ġarrogance": 46444, "ĠRemo": 46445, "γά": 46446, "assed": 46447, "Ġdeliveries": 46448, "Ġstinky": 46449, "ĠпеÑĢеж": 46450, "jay": 46451, "Ġtransitional": 46452, "Ġrere": 46453, "ĠNGOs": 46454, "ĠATM": 46455, "خت": 46456, "iology": 46457, "Ġвлад": 46458, "Ġschme": 46459, "ĠShine": 46460, "ìķ¡": 46461, "pants": 46462, "Ġserge": 46463, "Ġsenhor": 46464, "Ġabduct": 46465, "ĠBryant": 46466, "VES": 46467, "Ġawakened": 46468, "ĠLaz": 46469, "ropolis": 46470, "ĠLao": 46471, "è¾Ľèĭ¦": 46472, "Ġvilla": 46473, "Ġsummers": 46474, "Ġenthal": 46475, "Ġ1949": 46476, "Via": 46477, "Ġìĸ´ì¨": 46478, "Ġtendon": 46479, "Ġviolet": 46480, "Ġintellectually": 46481, "Ġbounced": 46482, "araus": 46483, "Ġ1919": 46484, "Ġvraag": 46485, "Ġspel": 46486, "ĠSchwar": 46487, "Scott": 46488, "ĠIndo": 46489, "Ġë§Ŀ": 46490, "Ġcanonical": 46491, "ĠIKE": 46492, "ĠthatÃŃs": 46493, "Ġmellan": 46494, "æ¯Ĵ": 46495, "igmat": 46496, "Could": 46497, "...?)": 46498, "Ġfoarte": 46499, "ĠKumar": 46500, "rendo": 46501, "Ġélé": 46502, "à´": 46503, "valuation": 46504, "cases": 46505, "Ġintuitively": 46506, "hong": 46507, "etted": 46508, "Ġsouven": 46509, "Ġmorb": 46510, "Ġcors": 46511, "ĠNV": 46512, "ĠHasan": 46513, "æĥħåĨµ": 46514, "ieved": 46515, "Ġì§Ģê¸ĪìĿĢ": 46516, "Ġdumpling": 46517, "Ġcontrôle": 46518, "Ġambiguity": 46519, "æ©Łæľĥ": 46520, "Ġcog": 46521, "ĠScriptures": 46522, "Ġcai": 46523, "Ġbever": 46524, "大家éĥ½": 46525, "Ġhuis": 46526, "Ġaime": 46527, "Ġerklären": 46528, "ĠLM": 46529, "ĠFey": 46530, "éļ¾": 46531, "றத": 46532, "Ġsupervised": 46533, "Ġjewe": 46534, "spl": 46535, "ĠÑĨенÑĤÑĢ": 46536, "Ġcollisions": 46537, "ÙĦÙģ": 46538, "ĠHogwarts": 46539, "ĠDurham": 46540, "×ķ×£": 46541, "Ġphosphate": 46542, "Ġoversee": 46543, "Ġinspections": 46544, "Ġbrinc": 46545, "ĠZak": 46546, "Ġpayoff": 46547, "Ġchaud": 46548, "ĠHunger": 46549, "ãos": 46550, "vir": 46551, "Ġfiance": 46552, "Ġboug": 46553, "lived": 46554, "cry": 46555, "åĽŀä¾Ĩ": 46556, "Ġjointly": 46557, "Ġgirlfriends": 46558, "ĠNexus": 46559, "¦¬ê²łìĬµëĭĪëĭ¤": 46560, "ĠKwang": 46561, "åĵĪåĽī": 46562, "å§ij": 46563, "ÅĤÄĻ": 46564, "ĠNeden": 46565, "iece": 46566, "Ġinserting": 46567, "æŁĵ": 46568, "ĠMummy": 46569, "ĠGlobe": 46570, "Ġlee": 46571, "Ġgerman": 46572, "Ġcreams": 46573, "acho": 46574, "ĠchÆ°a": 46575, "ĠGalile": 46576, "Ġfürs": 46577, "Ġestiver": 46578, "cidos": 46579, "Christian": 46580, "Ġlorsqu": 46581, "Ġcutest": 46582, "vale": 46583, "ĠкÑĢеп": 46584, "Ġwary": 46585, "Ġslicing": 46586, "Ġesperando": 46587, "ĠVander": 46588, "ĠDeixa": 46589, "Ġ1954": 46590, "ĠmówiÄħ": 46591, "ÑĸÑĶ": 46592, "Ġtooling": 46593, "Ġrestor": 46594, "Ġposición": 46595, "Ġintentar": 46596, "ĠApache": 46597, "OUL": 46598, "ĠÙĪب": 46599, "Ġmatière": 46600, "ãĥ¼ãĤĵ": 46601, "Ġlinen": 46602, "Ġestratég": 46603, "ĠMutta": 46604, "顯": 46605, "è¡ĮäºĨ": 46606, "Ġparting": 46607, "Ġminimizing": 46608, "Ġapprendre": 46609, "æľĿ": 46610, "Ġанглий": 46611, "ĠDoo": 46612, "ĠFirefox": 46613, "cómo": 46614, "Ġgeopolit": 46615, "Ġmakan": 46616, "Ġmogelijk": 46617, "ĠÏĢεÏģι": 46618, "Ġcứ": 46619, "Ġinstaller": 46620, "Ġdibuj": 46621, "ĠHeath": 46622, "loop": 46623, "ĠBroken": 46624, "HYUN": 46625, "shelf": 46626, "Ġfizer": 46627, "Ġenhances": 46628, "ä¾ĭãģĪãģ°": 46629, "ĠдоÑģÑĤи": 46630, "ĠPUB": 46631, "ĠKollegin": 46632, "Ġattained": 46633, "ľ": 46634, "Ġmistress": 46635, "ĠOftentimes": 46636, "×ŀ×Ļ×Ŀ": 46637, "Ġbewe": 46638, "ĠSora": 46639, "rauen": 46640, "baum": 46641, "Ġrollers": 46642, "Ġmering": 46643, "ĠPAC": 46644, "ĠнÑĸ": 46645, "ĠRépublique": 46646, "ĠÑĤÑĢав": 46647, "ĠVanguard": 46648, "uciones": 46649, "Ġ무ëĮĢ": 46650, "Ġgour": 46651, "¯¤": 46652, "ĠÏī": 46653, "Ġsauna": 46654, "Ġpeine": 46655, "ĠValerie": 46656, "ĠSikh": 46657, "fendimiz": 46658, "bero": 46659, "ĠÑĩи": 46660, "ĠdoÅĽwiad": 46661, "ĠEuros": 46662, "Ġcommentaires": 46663, "Ġtweaks": 46664, "ĠFaster": 46665, "ĠÑĢаÑģк": 46666, "Ġprogressively": 46667, "ĠEuch": 46668, "boro": 46669, "ĠIngred": 46670, "Cap": 46671, "Ġuncheck": 46672, "Ġìĺ¤ë¥¸": 46673, "Ġwre": 46674, "ĠFT": 46675, "örung": 46676, "Ġmemorized": 46677, "ĠDinner": 46678, "ĠPhew": 46679, "oubl": 46680, "Ġputa": 46681, "Ġadmits": 46682, "езде": 46683, "opod": 46684, "Ġpanda": 46685, "Ġhinges": 46686, "cipe": 46687, "Ġtransact": 46688, "Ġpodia": 46689, "Ġpics": 46690, "Ġcriterion": 46691, "ĠOrchestra": 46692, "ĠBlog": 46693, "Ġsolemn": 46694, "ĠPixar": 46695, "Three": 46696, "Ġвниз": 46697, "ĠVolunte": 46698, "ĠSavage": 46699, "ĠPVC": 46700, "ĠCaf": 46701, "Ġwykon": 46702, "Ġgraders": 46703, "Ġcrouch": 46704, "Ġcliche": 46705, "Ġsoybeans": 46706, "ĠMUR": 46707, "ĠGonzalez": 46708, "ĠMimi": 46709, "ĠBolsonaro": 46710, "Ġdiaphrag": 46711, "Ġbilang": 46712, "ëIJĺëĬĶ": 46713, "éĤ£æĪijåĢij": 46714, "Ġregulating": 46715, "Mc": 46716, "Judge": 46717, "Ġнож": 46718, "ĠjakÄħ": 46719, "itesse": 46720, "ĠWij": 46721, "Ġlata": 46722, "groaning": 46723, "POSING": 46724, "Ġ×IJ×ķת×ķ": 46725, "Ġhaga": 46726, "Ġgrounding": 46727, "Ġviolently": 46728, "Ġtills": 46729, "Ġengag": 46730, "ĠHollow": 46731, "ĠпопÑĥлÑıÑĢ": 46732, "Ġwprowad": 46733, "Ġreplaces": 46734, "Ġfluorescent": 46735, "urgical": 46736, "iggly": 46737, "ĠTraditional": 46738, "tte": 46739, "ĠÙĦÙĩ": 46740, "Ġphosphorus": 46741, "Ġapron": 46742, "ĠWaters": 46743, "ĠKultur": 46744, "авай": 46745, "Ġolives": 46746, "Ġ×Ķ×IJ׾": 46747, "Ġteilweise": 46748, "Ġsencill": 46749, "Ġprends": 46750, "Ġnarrower": 46751, "Ġjätte": 46752, "ĠInformationen": 46753, "ìĥģìĿ´": 46754, "Ġstarve": 46755, "Ġfrick": 46756, "ĠBeweg": 46757, "ल": 46758, "Ġdolphin": 46759, "ĠLAUGHTER": 46760, "ĠINTERVIE": 46761, "åĶī": 46762, "ĠyanlÄ±ÅŁ": 46763, "Ġtorpedo": 46764, "Ġshortages": 46765, "ìĿ´ëĵľ": 46766, "ıldı": 46767, "Ġpaws": 46768, "Ġozone": 46769, "Ġcultivated": 46770, "ĠFot": 46771, "Ġnotor": 46772, "ноз": 46773, "ĠкоÑĪ": 46774, "Ġtouchscreen": 46775, "ĠAlly": 46776, "æľĢè¿ij": 46777, "Ġ맼ìŀĪìĸ´ìļĶ": 46778, "ĠСеÑĢ": 46779, "Ġвполне": 46780, "Ġpaprika": 46781, "ĠDustin": 46782, "Ġefecto": 46783, "Ġopini": 46784, "Ġmuut": 46785, "Ġhá»įc": 46786, "Ġinterject": 46787, "ÄĻt": 46788, "Ġbutts": 46789, "urez": 46790, "ĠPike": 46791, "ĠHok": 46792, "ĠGuinea": 46793, "ĠCathedral": 46794, "Ġ1400": 46795, "Cra": 46796, "+,": 46797, "맼": 46798, "³´ëıĦë¡Ŀ": 46799, "abyrin": 46800, "Ġvideog": 46801, "ĠоÑĢÑĥж": 46802, "Ġuž": 46803, "Ġbuscando": 46804, "ĠAssistance": 46805, "éĻ½": 46806, "Ġmelhores": 46807, "ì¡´": 46808, "Ġëģ¼": 46809, "ĠRJ": 46810, "ĠتÙħ": 46811, "Ġomin": 46812, "Ġmotorcycles": 46813, "ĠSapp": 46814, "Ġsupplying": 46815, "ĠAlgun": 46816, "Ġaerospace": 46817, "×¢×ľ": 46818, "occup": 46819, "leist": 46820, "Ġê±°ëĬĶ": 46821, "Ġcompleta": 46822, "bres": 46823, "!(": 46824, "ĠÐŁÑĢед": 46825, "Ġdisadvantaged": 46826, "ĠAttend": 46827, "ĠJudah": 46828, "á»ĭch": 46829, "ylene": 46830, "actly": 46831, "Ġsetups": 46832, "Ġammonia": 46833, "ĠSchweiz": 46834, "ĠShame": 46835, "Ġbande": 46836, "ĠFuel": 46837, "Ġtroublesome": 46838, "Ġnumero": 46839, "ĠMOM": 46840, "ĠпÑĢедлаг": 46841, "mentioned": 46842, "ĠболÑĮÑĪое": 46843, "ĠViktor": 46844, "ĠStyles": 46845, "Ġcrucified": 46846, "ructured": 46847, "environ": 46848, "Ġmorals": 46849, "Ġmeditating": 46850, "Ġaxial": 46851, "isance": 46852, "ĠAbst": 46853, "Green": 46854, "Ġê±´ì": 46855, "Ġquadrant": 46856, "Ġpergi": 46857, "Ġcameraman": 46858, "ĠSequ": 46859, "Ġpaused": 46860, "ĠLaughing": 46861, "ê·Ģ": 46862, "?..": 46863, "ĠÅ»e": 46864, "Ġpermitir": 46865, "Ġdetectors": 46866, "ĠHUD": 46867, "aval": 46868, "ĠìĹ¬ê¸°ê¹Įì§Ģ": 46869, "Ġhubs": 46870, "Ġbestimmt": 46871, "ĠбÑĥдеÑĤе": 46872, "INTERPOSING": 46873, "Ġtengan": 46874, "Ġcrave": 46875, "ĠBundesregierung": 46876, "ĠBloody": 46877, "Ġusability": 46878, "ĠEas": 46879, "ĠÄijá»Ļng": 46880, "Ġ1955": 46881, "Ġkriegen": 46882, "Ġhabitual": 46883, "Ġessentials": 46884, "riminal": 46885, "Ġroommates": 46886, "éĤ£å°±": 46887, "ĠпеÑĢеÑħод": 46888, "Ġnghi": 46889, "Ġmening": 46890, "ĠSymphony": 46891, "ĠHug": 46892, "aggi": 46893, "Ġwied": 46894, "Ġmitad": 46895, "ãģ£ãģ¦ãģĦãģĨ": 46896, "teenth": 46897, "idaÄĩ": 46898, "Save": 46899, "ĠrobiÄĩ": 46900, "Ġbounces": 46901, "°ĸìĹIJ": 46902, "stars": 46903, "Ġpragmatic": 46904, "Ġcognition": 46905, "Ġwrapper": 46906, "Ġwarten": 46907, "adh": 46908, "Ġpensa": 46909, "ĠHertz": 46910, "ĠnÄĽ": 46911, "ĠReid": 46912, "ĠPCs": 46913, "ĠMole": 46914, "Ġ.....": 46915, "Ġprecio": 46916, "ĠChampionships": 46917, "ê°ĢëĿ½": 46918, "Ġvér": 46919, "Ġcorridors": 46920, "ĠElectronic": 46921, "Sl": 46922, "Ġале": 46923, "Ġoverthrow": 46924, "Ġkabul": 46925, "ĠRES": 46926, "ĠCyberpunk": 46927, "огод": 46928, "ĠÐĿав": 46929, "Ġwan": 46930, "Ġmanifestations": 46931, "Ġcuales": 46932, "ĠWise": 46933, "ĠLösung": 46934, "Ġexfol": 46935, "Ġearns": 46936, "ÑĥÑģÑĤиÑĤÑĮ": 46937, "Ġsapp": 46938, "ĠBraun": 46939, "ĠBRANDON": 46940, "ì¹Ļ": 46941, "Ġsano": 46942, "ĠFEL": 46943, "ÑĭвайÑĤеÑģÑĮ": 46944, "ождениÑı": 46945, "Ġsewn": 46946, "Fun": 46947, "Ġreciprocal": 46948, "Ġexpansive": 46949, "ĠTraffic": 46950, "Ġktórego": 46951, "ĠÙĪس": 46952, "æĺ¥": 46953, "Ġ빨": 46954, "prove": 46955, "igare": 46956, "Ġloh": 46957, "اض": 46958, "Hope": 46959, "Ġdevotees": 46960, "ĠGom": 46961, "Ġsteals": 46962, "ĠUms": 46963, "ĠTwice": 46964, "ãĤ²": 46965, "iyim": 46966, "Ġrhythmic": 46967, "ĠVorte": 46968, "Ġprefix": 46969, "omination": 46970, "Ġdato": 46971, "Ġcustard": 46972, "ĠVOICE": 46973, "å·ŀ": 46974, "Ġmeny": 46975, "istors": 46976, "Ġíĺij": 46977, "ĠìĤ´ìķĦ": 46978, "ĠíĥĦ": 46979, "Ġkort": 46980, "Ġaba": 46981, "ĠVera": 46982, "epy": 46983, "Ġì¹´ë©ĶëĿ¼": 46984, "Ġsubmerged": 46985, "ĠClock": 46986, "Ġthumbnails": 46987, "Ġboast": 46988, "ĠFare": 46989, "!!]": 46990, "ĠÅĽm": 46991, "Ġkaikki": 46992, "ĠTechnologies": 46993, "ìĻ¸": 46994, "ãĥĴ": 46995, "иÑĤай": 46996, "å°ıæĻĤ": 46997, "ĠаÑĤ": 46998, "Ġknobs": 46999, "Ġreicht": 47000, "ượng": 47001, "glio": 47002, "Ġ맼ìĿ´": 47003, "ê°IJìĿĦ": 47004, "Ġjotka": 47005, "ĠHandy": 47006, "ĠHaben": 47007, "nous": 47008, "Ġinland": 47009, "Ġamazon": 47010, "hooting": 47011, "SL": 47012, "Ġleisten": 47013, "~\"": 47014, "Ġprovoke": 47015, "ĠTwist": 47016, "Ġ×ij×Ĺ": 47017, "Ġdeparted": 47018, "ê°ľë¥¼": 47019, "Ġkonse": 47020, "ĠCarwyn": 47021, "íķĺìĭł": 47022, "idental": 47023, "ESCO": 47024, "Ġtteokbokki": 47025, "Ġdizendo": 47026, "ç·´": 47027, "ındaki": 47028, "imasu": 47029, "afar": 47030, "Ġlandfill": 47031, "Ġcorrecting": 47032, "Ġclears": 47033, "ĠNummer": 47034, "HAM": 47035, "Ġcartridges": 47036, "ĠDiesel": 47037, "paced": 47038, "Ġobliv": 47039, "Ġmoyens": 47040, "ĠSinne": 47041, "ĠPreis": 47042, "iliz": 47043, "ĠÑģмож": 47044, "Ġbroaden": 47045, "ä»ĸæĺ¯": 47046, "xes": 47047, "Ġcarbohydrate": 47048, "íĺ¹": 47049, "seok": 47050, "Ġechoes": 47051, "Ġcess": 47052, "ë°Ķ": 47053, "ĠбизнеÑģ": 47054, "Ġllamado": 47055, "Ġessent": 47056, "ĠìĿ¼ë°ĺ": 47057, "ĠAires": 47058, "phen": 47059, "Ġzebra": 47060, "Ġsymbolism": 47061, "Once": 47062, "Ġracks": 47063, "ĠKafka": 47064, "ĠÑģеÑĢÑĮез": 47065, "Ġsinn": 47066, "picious": 47067, "kaa": 47068, "Ġmotherfucker": 47069, "Ġapprenticeship": 47070, "Ġrpm": 47071, "Ġtaxation": 47072, "Ġfurry": 47073, "ĠSacred": 47074, "ĠÑĢазм": 47075, "pora": 47076, "enges": 47077, "ĠíĹĪë": 47078, "ĠÑģин": 47079, "Ġsanitizer": 47080, "Ġcringe": 47081, "ĠSca": 47082, "оÑĩно": 47083, "Ġofere": 47084, "Ġmelodies": 47085, "ĠVelvet": 47086, "ĠIhrer": 47087, "ĠHybrid": 47088, "ĠGiov": 47089, "Ġirgendwas": 47090, "Ġdepende": 47091, "ĠUsers": 47092, "Ġhump": 47093, "driving": 47094, "Ġsf": 47095, "Ġruthless": 47096, "à¹Ģà¸Ħ": 47097, "Ġlemons": 47098, "Ġföret": 47099, "ĠOj": 47100, "Ġмама": 47101, "Ġinterpersonal": 47102, "Ġgev": 47103, "Ġabnorm": 47104, "иÑģл": 47105, "Ġинд": 47106, "Ġkontroll": 47107, "Ġregres": 47108, "Ġledge": 47109, "Ġerzählt": 47110, "ĠTact": 47111, "Ġarrivé": 47112, "Ġsubstantive": 47113, "Ġspoonful": 47114, "zwischen": 47115, "ooooo": 47116, "Ġcontenido": 47117, "Ġbesl": 47118, "á»ĥm": 47119, "kten": 47120, "Jamie": 47121, "Ġsandy": 47122, "ä¸įåIJĮ": 47123, "âĭ": 47124, "Ġpase": 47125, "Ġdette": 47126, "ĠBelgian": 47127, "ê°ľë": 47128, "ulares": 47129, "rud": 47130, "igor": 47131, "ĠíĮ¬ë": 47132, "Ġremedies": 47133, "Ġblasting": 47134, "ĠSich": 47135, "Ġожид": 47136, "Ġmonstr": 47137, "Ġmanifold": 47138, "Ġglauben": 47139, "ĠEST": 47140, "Ġstreamline": 47141, "Ġlobbying": 47142, "ĠGothic": 47143, "toire": 47144, "..'": 47145, "Ġdémocr": 47146, "ĠнаблÑİд": 47147, "Ġwspól": 47148, "ĠczÄĻÅĽÄĩ": 47149, "ä¸ĭéĿ¢": 47150, "isés": 47151, "gangen": 47152, "Ġbezpie": 47153, "remlin": 47154, "ê°Ŀ": 47155, "Still": 47156, "Ġresides": 47157, "Ġgelecek": 47158, "Ġtéléphone": 47159, "Ġpewn": 47160, "Ġleopard": 47161, "Ġcomplimentary": 47162, "Ġcrib": 47163, "ĠAnimals": 47164, "Ġgeil": 47165, "essel": 47166, "Ġgarder": 47167, "Ġcatchy": 47168, "樹": 47169, "ĠEts": 47170, "ĠCommercial": 47171, "ĠDENNIS": 47172, "ĠCoordinator": 47173, "ĠAbigail": 47174, "ffffff": 47175, "ấp": 47176, "Ġpequeña": 47177, "Ġinjections": 47178, "cekt": 47179, "Ġphilanthropy": 47180, "Ġpuck": 47181, "Ġcelebrates": 47182, "ĠDunk": 47183, "ĠDlatego": 47184, "ãģ¾ãģł": 47185, "δή": 47186, "graduate": 47187, "ĠMobil": 47188, "till": 47189, "acam": 47190, "Ġyolks": 47191, "Ġtangled": 47192, "Ġmaniac": 47193, "Ġobliged": 47194, "ĠLaink": 47195, "Ġverder": 47196, "ĠDamon": 47197, "Ġmutant": 47198, "Ġhopping": 47199, "Ġreins": 47200, "Ġinverter": 47201, "Ġcontempt": 47202, "×ł×¡": 47203, "learning": 47204, "Miss": 47205, "ĠÐĵоÑģ": 47206, "ĠMeyer": 47207, "ê»ĺìĦľ": 47208, "é£İ": 47209, "×ķ׳×Ļ×Ŀ": 47210, "asking": 47211, "Ġtrimming": 47212, "Ġtreasury": 47213, "Ġsente": 47214, "Aust": 47215, "ĠUnterstützung": 47216, "ĠComedy": 47217, "ĠAnakin": 47218, "é¹": 47219, "ÑĢÑĥÑĤ": 47220, "ĠHari": 47221, "ographers": 47222, "Ġoatmeal": 47223, "ĠBots": 47224, "ä¸įäºĨ": 47225, "ĠпалÑĮ": 47226, "Ġacknowledgement": 47227, "xic": 47228, "Ġê´Ģìĭ¬": 47229, "gasping": 47230, "Ġãģķ": 47231, "Ġterrace": 47232, "Ġornaments": 47233, "ĠMER": 47234, "committee": 47235, "ĠìĹĨìĬµëĭĪëĭ¤": 47236, "Ġrij": 47237, "é³": 47238, "צ×Ŀ": 47239, "leme": 47240, "Ġliberties": 47241, "Ġfellas": 47242, "ĠCopper": 47243, "bench": 47244, "ĠIdea": 47245, "á»įn": 47246, "ÑĪа": 47247, "Ġversión": 47248, "ÏĦοÏį": 47249, "ĠÐľÐ¸": 47250, "ĠпÑĢилож": 47251, "Ġboxer": 47252, "ĠTanner": 47253, "ĠMoy": 47254, "ì¹ĺëĬĶ": 47255, "Thr": 47256, "Ġtinham": 47257, "Ġpolishing": 47258, "Ġconsequently": 47259, "Ġamenities": 47260, "ĠKI": 47261, "ĠGREEN": 47262, "ĠFrankie": 47263, "ниÑĤ": 47264, "ittel": 47265, "Ñģкое": 47266, "ursed": 47267, "Ġupbringing": 47268, "Ġthứ": 47269, "ĠìĭĿìľ¼ë¡ľ": 47270, "Ġwhim": 47271, "Ġchinese": 47272, "confidence": 47273, "ĠJeder": 47274, "ãģªãģ®ãģ§": 47275, "ajcie": 47276, "ĠTous": 47277, "ĠPowers": 47278, "ừa": 47279, "othermal": 47280, "ĠвÑĭÑĪе": 47281, "rale": 47282, "اخ": 47283, "Ġì§ĢìĽIJ": 47284, "Ġépisode": 47285, "Ġsulph": 47286, "Ġencara": 47287, "kraft": 47288, "aları": 47289, "ĠComes": 47290, "Ġdivul": 47291, "ĠRudolph": 47292, "ĠMuse": 47293, "Ġutens": 47294, "ĠìŀIJ주": 47295, "Ġpana": 47296, "ĠVegeta": 47297, "ĠPHP": 47298, "ĠNSA": 47299, "entin": 47300, "ĠCarnegie": 47301, "اÙĬ": 47302, "iÄĻcy": 47303, "Harry": 47304, "Ġfır": 47305, "Сп": 47306, "Ġgladly": 47307, "Ġaveraging": 47308, "íķĺê²łìĬµëĭĪëĭ¤": 47309, "лÑıÑİÑĤÑģÑı": 47310, "ĠÐľÐµÐ½Ñı": 47311, "Ġquotation": 47312, "rires": 47313, "itchens": 47314, "ayed": 47315, "Ġunatt": 47316, "ĠPerez": 47317, "ĠоÑĤмеÑĤ": 47318, "Ġtactile": 47319, "ĠEuh": 47320, "isini": 47321, "buh": 47322, "Ġhatır": 47323, "ĠìŀĪìľ¼": 47324, "Ġpolicymakers": 47325, "³´ìĦ¸ìļĶ": 47326, "acı": 47327, "Ġκι": 47328, "Ġregistering": 47329, "reto": 47330, "ĠSprinkle": 47331, "ĠGrammy": 47332, "axter": 47333, "Ġби": 47334, "Ġsitter": 47335, "Ġpredic": 47336, "Ġthinly": 47337, "Ġstrum": 47338, "Ġaggrav": 47339, "Ġaha": 47340, "رج": 47341, "mellow": 47342, "Ġconstante": 47343, "ĠLaut": 47344, "iston": 47345, "Ġtransitioned": 47346, "ĠCambodia": 47347, "ãģĦãģįãģ¾ãģĻ": 47348, "è·Łå¤§å®¶": 47349, "arted": 47350, "Ġmisf": 47351, "ĠPunkte": 47352, "Įëĵł": 47353, "Ġtrembling": 47354, "Ġgespannt": 47355, "ĠعÙĦÙĬÙĩ": 47356, "ĠникакиÑħ": 47357, "Ġë¶Ģëĵľë": 47358, "ĠÑĢазвиÑĤ": 47359, "Ġitchy": 47360, "Ġciento": 47361, "Ġplains": 47362, "Ġkittens": 47363, "Ġbacklog": 47364, "ĠPresiding": 47365, "pta": 47366, "Ġhavoc": 47367, "ĠDarrin": 47368, "ĠÐĽÑİб": 47369, "Ġsegregated": 47370, "Ġghetto": 47371, "Ġerlebt": 47372, "Ġdrugiej": 47373, "ĠSixt": 47374, "åıĥ": 47375, "ระ": 47376, "uencia": 47377, "Ġíķĺ기": 47378, "ĠëĨį": 47379, "Ġrobi": 47380, "Ġpioneers": 47381, "Ġmilliards": 47382, "ĠWitcher": 47383, "Ġ무ìĹĩ": 47384, "orro": 47385, "mass": 47386, "Ġdivergence": 47387, "ĠRivera": 47388, "ĠNoodles": 47389, "Ġendroit": 47390, "ĠKosten": 47391, "ĠдÑĢÑĥга": 47392, "ĠmÃŃnimo": 47393, "ĠKazakhstan": 47394, "تÙĩ": 47395, "ĠвоздÑĥ": 47396, "Ġgeschrieben": 47397, "ĠNil": 47398, "Ñģки": 47399, "ĠFrüh": 47400, "Ġbeverages": 47401, "æºIJ": 47402, "ĠGon": 47403, "æĺ¨": 47404, "Arin": 47405, "ĠIntro": 47406, "ocalyptic": 47407, "Ġexhaustion": 47408, "ĠStatus": 47409, "ĠBattery": 47410, "ész": 47411, "£¼ë": 47412, "airy": 47413, "Ġë³´ìŬëĵľë": 47414, "Ġdisparity": 47415, "ÙĮ": 47416, "ĠTucson": 47417, "Ġbrightly": 47418, "problem": 47419, "Ġbiomass": 47420, "éĻį": 47421, "§ī": 47422, "Ġhurdle": 47423, "Ġwavelengths": 47424, "Ġ<<": 47425, "Ġteamed": 47426, "FFFF": 47427, "ĠSlim": 47428, "omial": 47429, "Ġunveiled": 47430, "ĠVerein": 47431, "ÙĤØ·": 47432, "estry": 47433, "Ġclás": 47434, "Ġcheddar": 47435, "Ġaccusing": 47436, "ĠScientific": 47437, "ĠбÑĥде": 47438, "ĠCyrus": 47439, "εÏĦε": 47440, "Ĩĵê³ł": 47441, "Ġë³Ħ": 47442, "Ġcurd": 47443, "Ġreferrals": 47444, "shift": 47445, "åįķ": 47446, "ników": 47447, "Ġmier": 47448, "Ġconfronting": 47449, "ê²ĥëıĦ": 47450, "awl": 47451, "Ġtryin": 47452, "Ġê·¸ëŀĺìļĶ": 47453, "Ġchiar": 47454, "Ġìĺ¤ëĬĺëıĦ": 47455, "æĶ¿æ²»": 47456, "esque": 47457, "Ġmismos": 47458, "ĠShak": 47459, "Ġsociaux": 47460, "ĠpiÅŁ": 47461, "ĠkiÅŁi": 47462, "Ġcyan": 47463, "hay": 47464, "bew": 47465, "bod": 47466, "Ġι": 47467, "ĠMainly": 47468, "ÑİÑĤÑĮ": 47469, "habitude": 47470, "ĠÑģпокой": 47471, "è·ŁæĪij": 47472, "Ġprecon": 47473, "ĠMandy": 47474, "ðŁ¤£": 47475, "illos": 47476, "Ġgrupp": 47477, "Ġcrumble": 47478, "Ġconstructor": 47479, "ervices": 47480, "Ġlighthouse": 47481, "ĠConcept": 47482, "анÑĤи": 47483, "altro": 47484, "hope": 47485, "ĠAlleg": 47486, "ìĸ´ë¥¼": 47487, "pieces": 47488, "ounter": 47489, "ĠíķĺëĭĪê¹Į": 47490, "ĠìĿ¸íĦ°ë": 47491, "Ġvéritable": 47492, "Ġthreaded": 47493, "blind": 47494, "ĤĺëĿ¼": 47495, "Ġtrays": 47496, "ĠEdison": 47497, "ĠÃĸz": 47498, "ĠStevie": 47499, "Ġlender": 47500, "Ġbrigade": 47501, "Ġdeutsche": 47502, "muffled": 47503, "bart": 47504, "Ġinsanity": 47505, "Ġsavvy": 47506, "Ġsensational": 47507, "Ġderechos": 47508, "ĠMX": 47509, "ĠпÑĢеп": 47510, "Ġthreatens": 47511, "ĠrealtÃł": 47512, "Ġindicative": 47513, "Ġchops": 47514, "Ġbenefiting": 47515, "ĠVernon": 47516, "ĠStrand": 47517, "nun": 47518, "quently": 47519, "101": 47520, "Ġeel": 47521, "ìĪĻ": 47522, "rints": 47523, "ĠÙħس": 47524, "Ġبد": 47525, "ĠпоÑģÑĤÑĢо": 47526, "ĠyapmÄ±ÅŁ": 47527, "Ġolması": 47528, "Ġiedereen": 47529, "olé": 47530, "kef": 47531, "Ġë°ľìĥĿ": 47532, "Ġrained": 47533, "Ġalmighty": 47534, "ĠвÑĭд": 47535, "ĠCPR": 47536, "Fre": 47537, "Ġinhabited": 47538, "Ġarbets": 47539, "Ġakin": 47540, "аÑģÑĤв": 47541, "vania": 47542, "Ġhäufig": 47543, "ĠMatte": 47544, "sorry": 47545, "Jenny": 47546, "ĠгÑĢад": 47547, "Ġwhit": 47548, "Ġbrokers": 47549, "å¯Ł": 47550, "Ġhine": 47551, "asten": 47552, "ĠгÑĢÑĥ": 47553, "MB": 47554, "ĠPRI": 47555, "Sab": 47556, "Ġwrestler": 47557, "Ġfacilitating": 47558, "Ġehkä": 47559, "ĠCred": 47560, "Ġ127": 47561, "Ġnothin": 47562, "Ġmandated": 47563, "å¯Į": 47564, "ÑĥÑĤÑģÑĤв": 47565, "Frank": 47566, "Ġwors": 47567, "ĠdzieÅĦ": 47568, "ĠUnderground": 47569, "Ġznajdu": 47570, "ĠBä": 47571, "ĠPrinzip": 47572, "аÑĤелей": 47573, "Ġveterinar": 47574, "Ġsplendid": 47575, "Ġrozp": 47576, "Ġpsychopath": 47577, "igon": 47578, "Ġhops": 47579, "Ġcần": 47580, "ĠXian": 47581, "Ġtroisième": 47582, "Ġproducto": 47583, "ĠdeÄŁer": 47584, "ĠContinuing": 47585, "ивал": 47586, "cık": 47587, "Ġmoisturizer": 47588, "White": 47589, "Ġsiis": 47590, "ĠEverest": 47591, "ienced": 47592, "Ġcảm": 47593, "ĠJapon": 47594, "´ìłĦ": 47595, "ĠtenÃŃan": 47596, "Ġencanta": 47597, "Mm": 47598, "Ġdropdown": 47599, "ĠIya": 47600, "³´ë©´": 47601, "Ġwording": 47602, "ĠSqueeze": 47603, "ĠMaple": 47604, "Ġclarified": 47605, "ĠMunicip": 47606, "ĠRouge": 47607, "ĠNicki": 47608, "ĠGoo": 47609, "volt": 47610, "tek": 47611, "fecture": 47612, "fred": 47613, "arrive": 47614, "ãĥ¼ãģĦ": 47615, "tez": 47616, "Ep": 47617, "Ġobras": 47618, "ĠVID": 47619, "ĠRiv": 47620, "ĠModi": 47621, "ibe": 47622, "Ġacontecendo": 47623, "Ġimitation": 47624, "Ġcamouflage": 47625, "Ġspanning": 47626, "ĠSECRET": 47627, "ĠOreo": 47628, "ìĨĮ리": 47629, "Ġhunch": 47630, "ĠcaÅĤe": 47631, "Ġspontaneously": 47632, "ĠPerd": 47633, "Ġetap": 47634, "ĠHole": 47635, "ĠDisability": 47636, "Ġafterlife": 47637, "æģ©": 47638, "Ġtestified": 47639, "Ġpresup": 47640, "Ġpetroleum": 47641, "Ġcontrario": 47642, "ĠAssessment": 47643, "ÄŁlu": 47644, "Ġpests": 47645, "Ġdilig": 47646, "ĠвÑģÑĤÑĢеÑĤ": 47647, "Ġconséqu": 47648, "Ġcannons": 47649, "Ġcanoe": 47650, "ĠMile": 47651, "Ġcitoy": 47652, "Ġbegged": 47653, "ĠMinnie": 47654, "ÅĤych": 47655, "Ġprincipe": 47656, "ÏĢÏĮν": 47657, "mniej": 47658, "Ġwert": 47659, "Ġëĭ¤ëĵ¤": 47660, "anse": 47661, "Ġuncles": 47662, "Ġprovocative": 47663, "Ġintersections": 47664, "Ġdemocrats": 47665, "ĠJulius": 47666, "инки": 47667, "ygusal": 47668, "Ġ׾×ķ": 47669, "Ġgjorde": 47670, "Ġgasket": 47671, "ĠBock": 47672, "ĠÄ°n": 47673, "breat": 47674, "ĠEquity": 47675, "ardı": 47676, "Ġканале": 47677, "Ġдней": 47678, "ĠtỼi": 47679, "Ġfixture": 47680, "Ġabuses": 47681, "Ġvaya": 47682, "Ġouvert": 47683, "Ġmulticultural": 47684, "Ġcontexto": 47685, "ĠSesame": 47686, "Ġdépl": 47687, "Ġconsomm": 47688, "ĠParte": 47689, "Ġpem": 47690, "ĠConan": 47691, "ĠбÑĸлÑĮ": 47692, "Ġpersuaded": 47693, "Ġdrains": 47694, "Moo": 47695, "FORE": 47696, "ĠбаÑĤ": 47697, "Ġfod": 47698, "ĠProducts": 47699, "ì§Ħì§ľ": 47700, "Ġ\"[": 47701, "ĠWick": 47702, "ĠNaruto": 47703, "нали": 47704, "ryw": 47705, "Ġlodge": 47706, "Ġinh": 47707, "Ġvontade": 47708, "Ġdij": 47709, "ĠJesús": 47710, "Looking": 47711, "Ġforearm": 47712, "ĠIntegration": 47713, "ĠHARRIS": 47714, "Ġtoolbar": 47715, "leader": 47716, "Ġseldom": 47717, "ĠбÑĢоÑģ": 47718, "ĠKook": 47719, "онд": 47720, "Ġmonopol": 47721, "Ġmillet": 47722, "Ġlira": 47723, "ĠAsians": 47724, "Ġ1890": 47725, "ciÄŁim": 47726, "Ġeden": 47727, "ĠIKEA": 47728, "ĠNeighbor": 47729, "ĠKazuya": 47730, "üd": 47731, "Ġpsychedel": 47732, "Ġenvisioned": 47733, "åĿĹ": 47734, "Ġï·»": 47735, "Ġwunder": 47736, "ĠBulgaria": 47737, "Brid": 47738, "Ġmarrow": 47739, "Ġdepiction": 47740, "ĠTin": 47741, "ĠPharise": 47742, "Ġeinzige": 47743, "Ġblindly": 47744, "ãģĽãģ¦": 47745, "Ġdefens": 47746, "Dire": 47747, "Ġvibrating": 47748, "Ġtrolls": 47749, "Ġdisrespectful": 47750, "Ġwod": 47751, "Ġstimuli": 47752, "Ġcreeping": 47753, "Ġclairement": 47754, "Ġscariest": 47755, "Ġdécouvrir": 47756, "Ġ104": 47757, "ĠвеÑĢÑħ": 47758, "ĠÅĤat": 47759, "Ġróżne": 47760, "Ġbarley": 47761, "ĠRepl": 47762, "ĠTwe": 47763, "kke": 47764, "ĠãģĿãĤĮ": 47765, "ĠRedmi": 47766, "ĠMetroid": 47767, "ĠήÏĦαν": 47768, "Check": 47769, "ĠSEN": 47770, "Ġido": 47771, "ÑĤоÑĢии": 47772, "óp": 47773, "UNKNOWN": 47774, "Ġändern": 47775, "ĠJuice": 47776, "ĠGesicht": 47777, "å°±æľĥ": 47778, "ĠнаÑģÑĤолÑĮко": 47779, "íĥķ": 47780, "ÂŃ": 47781, "exhales": 47782, "Ġì´ī": 47783, "Ġjsem": 47784, "ÏĢÏīÏĤ": 47785, "Ġitt": 47786, "ëªħìĿ´": 47787, "Ġremix": 47788, "Ġblossoms": 47789, "ĠRenee": 47790, "isations": 47791, "ìĬ¤íĦ°": 47792, "Ġë³´ìĿ´ëĬĶ": 47793, "uestas": 47794, "opedia": 47795, "ĠAim": 47796, "ìĿ´ì¦Ī": 47797, "scene": 47798, "Ġleakage": 47799, "uckt": 47800, "Sad": 47801, "Ask": 47802, "Ġsuspense": 47803, "Ġimpost": 47804, "ĠStrategic": 47805, "ĠItÃŃs": 47806, "âĢĮ": 47807, "Ġkeyboards": 47808, "Ġamusing": 47809, "ogr": 47810, "iderman": 47811, "ŀĸ": 47812, "ĠвижÑĥ": 47813, "Ġdips": 47814, "Ġapologized": 47815, "ĠSTAR": 47816, "Ġescuela": 47817, "ĠChing": 47818, "нениÑı": 47819, "Ġë¶Ģë¶ĦìĿ´": 47820, "ĠFleet": 47821, "Ġsamb": 47822, "Ġentsprechend": 47823, "Ġelectrodes": 47824, "ĠFreiheit": 47825, "æĪijä¸įçŁ¥éģĵ": 47826, "ĠShrim": 47827, "iÃŁe": 47828, "Ġselections": 47829, "Ġfordi": 47830, "Ġdoss": 47831, "ÑıÑĩ": 47832, "Ġdiscriminate": 47833, "ĠAuÃŁerdem": 47834, "Ġdesenvolv": 47835, "ĠInternal": 47836, "ĠBenedict": 47837, "å¯Ĩ": 47838, "ĠShiv": 47839, "Missy": 47840, "ĠобнаÑĢÑĥж": 47841, "ĠнаÑģÑĤÑĢо": 47842, "Ġcontrolar": 47843, "ĠLia": 47844, "Ġopioids": 47845, "antu": 47846, "Ġcupboard": 47847, "æģIJ": 47848, "ге": 47849, "achts": 47850, "Ġcurated": 47851, "Ġxem": 47852, "Ġweary": 47853, "Ġbrethren": 47854, "Ġbudgeting": 47855, "Ġpourtant": 47856, "éļ»": 47857, "aisia": 47858, "ĠоÑĤвеÑĩ": 47859, "ĠGIS": 47860, "μαι": 47861, "Ġש×Ķ×ķ×IJ": 47862, "Ġsaud": 47863, "ĠlỼ": 47864, "ÐķТ": 47865, "ubine": 47866, "ĠнÑĥжен": 47867, "Ġkidnapping": 47868, "Ġbrat": 47869, "ĠTerre": 47870, "ĠMonet": 47871, "Ġë§ĪìĬ¤íģ": 47872, "Ġflashy": 47873, "ĠISBN": 47874, "Ġfreelance": 47875, "iage": 47876, "Ġjunge": 47877, "충": 47878, "ceral": 47879, "ĠÑĤоÑĩки": 47880, "Ġformulate": 47881, "ĠFER": 47882, "ĠDartmouth": 47883, "ìľ¼ë©´ìĦľ": 47884, "å¢ĥ": 47885, "owiÄħ": 47886, "ĠëĶĶìŀIJ": 47887, "Ġregiment": 47888, "Ġmetabolismo": 47889, "ĠParr": 47890, "Ġ충ë¶Ħ": 47891, "Ġsanity": 47892, "ĠLal": 47893, "ĠGö": 47894, "ĠGla": 47895, "Ġproto": 47896, "Ġmicroscopic": 47897, "Ġkang": 47898, "ĠScalia": 47899, "Ġpug": 47900, "ĠScore": 47901, "ĠSavannah": 47902, "Ġgarde": 47903, "ĠNOR": 47904, "å°įåIJ§": 47905, "Ġscheint": 47906, "ĠpóÅĤ": 47907, "Ġcorri": 47908, "Ġbrute": 47909, "ĠÅĤad": 47910, "ä»ĸ们": 47911, "Ġsucceeding": 47912, "Ġbicycles": 47913, "Non": 47914, "Ġseekers": 47915, "Ġunconditional": 47916, "Ġrhymes": 47917, "ĠGarage": 47918, "Ġinvoice": 47919, "Ġcanvi": 47920, "neck": 47921, "Ġcustomizable": 47922, "iritual": 47923, "Queen": 47924, "íķĺìĭľëĬĶ": 47925, "Ġpowerless": 47926, "Ġcsak": 47927, "ä¸įä¼ļ": 47928, "isoft": 47929, "ĠìłķíĻķ": 47930, "Ġnhân": 47931, "ĠMAND": 47932, "ĠHaf": 47933, "Ġrevolves": 47934, "ä¹Łåı¯ä»¥": 47935, "ovan": 47936, "aroo": 47937, "ĠGrind": 47938, "éĽª": 47939, "Ġindispensable": 47940, "Ġconsulted": 47941, "ĠClinical": 47942, "Acc": 47943, "Ġolhos": 47944, "Ġmonter": 47945, "ĠHana": 47946, "etah": 47947, "Ġvaan": 47948, "Ġtigers": 47949, "Ġcaucus": 47950, "ðŁĺĤ": 47951, "³´ìŀIJ": 47952, "powers": 47953, "iums": 47954, "ĠíĨłë": 47955, "Ġtradicional": 47956, "Ġresonated": 47957, "Ġìĭłê¸°": 47958, "them": 47959, "Robert": 47960, "Ġelemento": 47961, "Ġantid": 47962, "ĠобÑģ": 47963, "Ġnatives": 47964, "Ġloca": 47965, "owment": 47966, "ĠTight": 47967, "ĠæĢĿ": 47968, "Ġmelan": 47969, "ĠNue": 47970, "amis": 47971, "Ġsorgen": 47972, "asına": 47973, "Home": 47974, "ĠPUBG": 47975, "Ġawfully": 47976, "ĠShore": 47977, "ĠPerché": 47978, "ĠLau": 47979, "ĠCinderella": 47980, "ĠChest": 47981, "Ġsemantic": 47982, "Ġdeserted": 47983, "ĠMomo": 47984, "ĠHernandez": 47985, "genes": 47986, "ĠAdult": 47987, "иÑĩеÑģкого": 47988, "oshima": 47989, "ĠcaracterÃŃsticas": 47990, "ĠKL": 47991, "´ìŀ¥": 47992, "ocar": 47993, "Ġfehlt": 47994, "Ġdruk": 47995, "ĠPoppy": 47996, "ENGLISH": 47997, "ĠVergleich": 47998, "Brien": 47999, "Ġrecomp": 48000, "ĠÑģд": 48001, "Ġmerger": 48002, "Ġmarketers": 48003, "Ġhoneymoon": 48004, "Ġpenso": 48005, "Ġbelli": 48006, "еÑĤÑĥ": 48007, "Ġbanker": 48008, "Camera": 48009, "ĠStall": 48010, "ĠStamp": 48011, "ĠBite": 48012, "ежде": 48013, "Ġsür": 48014, "Ġgüç": 48015, "ĠPassover": 48016, "ĠBugün": 48017, "ĠÑģожалениÑİ": 48018, "Ġниз": 48019, "Ġmanure": 48020, "Ġglacier": 48021, "è«ĩ": 48022, "RAY": 48023, "terror": 48024, "Ġsalads": 48025, "Ġhurricanes": 48026, "ĠDesigner": 48027, "atorio": 48028, "Ġfactual": 48029, "ĠTammy": 48030, "ĠзвÑĥÑĩ": 48031, "Ġintroductions": 48032, "Ġhousekeeping": 48033, "Ġhanger": 48034, "ëĭĺë": 48035, "akte": 48036, "ĠCola": 48037, "']": 48038, "ĠGender": 48039, "оÑĢон": 48040, "ipse": 48041, "icias": 48042, "Ġsuccessive": 48043, "Ġpolitic": 48044, "Ġhöher": 48045, "ĠQiao": 48046, "ĠGimme": 48047, "Ġлож": 48048, "Ġseb": 48049, "ĠWeiter": 48050, "ĠSakura": 48051, "ĠBoulder": 48052, "ĠAmérica": 48053, "peÅĤnie": 48054, "ĠtecnologÃŃa": 48055, "ishops": 48056, "fur": 48057, "Ġmoonlight": 48058, "Ġdispersed": 48059, "Ġrez": 48060, "енное": 48061, "алÑĮнÑĥÑİ": 48062, "ĠTwelve": 48063, "ĠHOR": 48064, "ìĭ¤íŀĪ": 48065, "ilage": 48066, "Ġshaded": 48067, "Ġresumes": 48068, "ĠPeanut": 48069, "ĠMILL": 48070, "apons": 48071, "ĠUFC": 48072, "ĠSole": 48073, "Ġjoystick": 48074, "ĠOlivier": 48075, "warming": 48076, "Ġsyllabus": 48077, "ĠобÑīе": 48078, "Ġhiá»ĩn": 48079, "Ġfesta": 48080, "Ġcradle": 48081, "ĠZac": 48082, "Ġremembrance": 48083, "Ġê°ĻìķĦìĦľ": 48084, "ĠpiÄĻk": 48085, "Ġcoexist": 48086, "ĠVII": 48087, "Ġáreas": 48088, "Ġuważ": 48089, "Ġobservers": 48090, "Ġmänniskor": 48091, "coon": 48092, "ĠDAM": 48093, "Ġnaszym": 48094, "Ġalligator": 48095, "ĠFreeze": 48096, "ĠEstate": 48097, "ĠÑĤÑĢади": 48098, "Ġundercover": 48099, "Ġnies": 48100, "ĠFehler": 48101, "plin": 48102, "ĠKabul": 48103, "ilate": 48104, "Ġê³łìĸij": 48105, "Ġmop": 48106, "ìĦ¼": 48107, "Ġanderer": 48108, "ĠKELL": 48109, "оки": 48110, "ĠжеÑģÑĤ": 48111, "Ġgrazing": 48112, "ĠdaÃŃ": 48113, "Ġcapitalize": 48114, "Ġapex": 48115, "Ġnurturing": 48116, "Ġcortar": 48117, "Ġcontrac": 48118, "ımızı": 48119, "Ġtandem": 48120, "éĥ½æľī": 48121, "gement": 48122, "ĠÑģиÑģÑĤема": 48123, "Ġmanque": 48124, "iajÄħ": 48125, "WOR": 48126, "Ġاب": 48127, "Ġcarts": 48128, "ANO": 48129, "Ġë°Ľê³ł": 48130, "ĠCena": 48131, "ĠBiology": 48132, "idar": 48133, "Ġaż": 48134, "erne": 48135, "anu": 48136, "Ġthanked": 48137, "Ġsubmarines": 48138, "Ġmanic": 48139, "Ġмоз": 48140, "ä¼Ĭ": 48141, "instant": 48142, "essential": 48143, "Ġsamurai": 48144, "Ġpasti": 48145, "Ġalan": 48146, "Ġbroch": 48147, "Ġbaker": 48148, "ĠGuill": 48149, "¨¼": 48150, "Ġwithdrawn": 48151, "ëĭĿ": 48152, "Perfect": 48153, "quency": 48154, "Ġstreamlined": 48155, "Ġ1300": 48156, "´ëıĦ": 48157, "Ġëĸłë": 48158, "Ġãģ¯ãģĦ": 48159, "Ġhvad": 48160, "ä¸Ģå®ļè¦ģ": 48161, "Ġverbally": 48162, "ĠKons": 48163, "Ġì¡°ìĭ¬": 48164, "Ġdiez": 48165, "æİ°æİ°": 48166, "Ġchuckling": 48167, "ĠMih": 48168, "Ġrallies": 48169, "Ġmanter": 48170, "Ġearnest": 48171, "super": 48172, "Ġgece": 48173, "ĠRend": 48174, "ĠGerade": 48175, "jenigen": 48176, "ĠVall": 48177, "ĠìŀĪëĤĺ": 48178, "ĠÑģказала": 48179, "Ġtrabalh": 48180, "ĠнаÑĪем": 48181, "ĠмеÑħ": 48182, "ikit": 48183, "Ġnouns": 48184, "Ġneurological": 48185, "Ġmotivational": 48186, "ĠMcMahon": 48187, "ĠFinished": 48188, "Ġë³´ìĿ´": 48189, "ĠFields": 48190, "Ġadolescents": 48191, "ĠTisch": 48192, "ĠNeben": 48193, "ĠFlowers": 48194, "ĠEnerg": 48195, "Ġdiret": 48196, "ĠThi": 48197, "ĠPicas": 48198, "æĥľ": 48199, "æĢİä¹Īæł·": 48200, "Ġavete": 48201, "ĠFors": 48202, "ĠChapel": 48203, "Não": 48204, "Et": 48205, "ĠÑģодеÑĢж": 48206, "reno": 48207, "Ġsven": 48208, "ĠdostÄĻp": 48209, "nee": 48210, "ĠSnapdragon": 48211, "ĠIDs": 48212, "ìķĺëĬĶëį°": 48213, "ר×ļ": 48214, "Ġsunflower": 48215, "Ġperpetual": 48216, "ç³ĸ": 48217, "Ġknights": 48218, "Ġgird": 48219, "ĠTold": 48220, "Ġvolcanoes": 48221, "Ġadversary": 48222, "ĠEconomy": 48223, "Ġextrapol": 48224, "Ġbluetooth": 48225, "Ġzooming": 48226, "Ġskys": 48227, "Ġgenial": 48228, "ÃŃculos": 48229, "ambre": 48230, "ĠмеÑĢ": 48231, "Ġteeny": 48232, "Ġstressing": 48233, "ìķĮ": 48234, "ONY": 48235, "Ġtranslucent": 48236, "Ġrounding": 48237, "Ġgrues": 48238, "×Ļ׳×Ķ": 48239, "après": 48240, "Ġprueba": 48241, "Ġpolygon": 48242, "Ġblueberry": 48243, "ĠProgramm": 48244, "Ġtrenches": 48245, "Ġsebagai": 48246, "Ġpalate": 48247, "Ġlaude": 48248, "Ġbehaved": 48249, "Ġlongitudinal": 48250, "ĠModule": 48251, "Ġadmir": 48252, "λι": 48253, "Greg": 48254, "Ġwyst": 48255, "Ġpropagate": 48256, "Ġmolds": 48257, "ĠTub": 48258, "ĠLoud": 48259, "usto": 48260, "Ġunstoppable": 48261, "Ġreinforcing": 48262, "éĿŀ常çļĦ": 48263, "ĠпÑĢоблема": 48264, "Ġpotencial": 48265, "Ġhemp": 48266, "ìŀĶ": 48267, "य": 48268, "Ġoptic": 48269, "Ġerfolgreich": 48270, "ÑģÑĭ": 48271, "олÑĮÑĪе": 48272, "urst": 48273, "ĠPois": 48274, "Ġrespondents": 48275, "Ġnehme": 48276, "ĠExternal": 48277, "olate": 48278, "Hyun": 48279, "Ġquartz": 48280, "Ġmathematician": 48281, "Ġbásicamente": 48282, "Ġail": 48283, "ìłľë¥¼": 48284, "attutto": 48285, "Ġnooit": 48286, "Ġafflict": 48287, "ĠOlga": 48288, "èŃ·": 48289, "ĠнаÑĤ": 48290, "Ġdites": 48291, "Ġrealidade": 48292, "Ġkän": 48293, "Ġuniqueness": 48294, "Ġpadres": 48295, "Ġsubsidi": 48296, "Ġpigeons": 48297, "βα": 48298, "stad": 48299, "Ġderen": 48300, "ĠСлед": 48301, "doo": 48302, "ĠопиÑģании": 48303, "Ġamber": 48304, "Ġgoosebumps": 48305, "ĠfrÃ¥gor": 48306, "ĠVital": 48307, "ĠIsraelites": 48308, "wasser": 48309, "Isn": 48310, "Ġcommits": 48311, "ĠSTEVEN": 48312, "ĠBevölker": 48313, "uitive": 48314, "Ġlegen": 48315, "Ġbruk": 48316, "иÑĢован": 48317, "ynen": 48318, "helm": 48319, "Ġgenerational": 48320, "ĠLändern": 48321, "οιÏĢÏĮν": 48322, "uzu": 48323, "Ġcaller": 48324, "онÑĮ": 48325, "ümü": 48326, "Ġbesar": 48327, "Ġplats": 48328, "Ġmigrated": 48329, "Ġjap": 48330, "ĠWAR": 48331, "Ġdissect": 48332, "ĠZusch": 48333, "ĠZeiten": 48334, "ĠLions": 48335, "ĠDF": 48336, "âĶ": 48337, "кив": 48338, "Ġpedestrians": 48339, "ĠMarilyn": 48340, "dock": 48341, "Ġyht": 48342, "Ġreincarn": 48343, "ĠSono": 48344, "ĠGrowth": 48345, "ÑĥÑģов": 48346, "Ġdungeons": 48347, "Ġbagus": 48348, "kich": 48349, "ĠÑĥкÑĢаÑĹ": 48350, "éĨ«": 48351, "ĠKeller": 48352, "chemistry": 48353, "Japanese": 48354, "Ġwillst": 48355, "Ġdecomposition": 48356, "ĠÑģÑĤен": 48357, "Ġrevived": 48358, "íķĻêµIJ": 48359, "ĠÅĵ": 48360, "ä½IJ": 48361, "ìĭ¸": 48362, "ippy": 48363, "Ġhourly": 48364, "jän": 48365, "ĠWorkshop": 48366, "Ŀ¼ìĦľ": 48367, "Ġcuarto": 48368, "Ġpatrim": 48369, "ĠBurch": 48370, "ĠìŀĪ기": 48371, "Ġhepat": 48372, "ĠhÃłng": 48373, "ĠëĮĢíķ´": 48374, "ĠваÑĪи": 48375, "Ġrework": 48376, "Ġparse": 48377, "Ġçıktı": 48378, "ĠSax": 48379, "ĠMongo": 48380, "ĠAaah": 48381, "ramble": 48382, "DJ": 48383, "Ġstabilized": 48384, "ĠSpeech": 48385, "Books": 48386, "Ġhurdles": 48387, "ĠWO": 48388, "ĠLamborg": 48389, "Ġ1933": 48390, "Ġvorbere": 48391, "Ġclinically": 48392, "Ġbreathtaking": 48393, "ĠGateway": 48394, "пеÑĢвÑĭÑħ": 48395, "uters": 48396, "Ġë¹µ": 48397, "Ġyeter": 48398, "Ġpulley": 48399, "Ġmuffin": 48400, "ĠPrefer": 48401, "ĠPence": 48402, "Ġinformação": 48403, "ìĬ¤íĬ¸ë": 48404, "ãĤ¸ãĥ£": 48405, "ĠTurtle": 48406, "ĠRegina": 48407, "ĠLoad": 48408, "does": 48409, "panze": 48410, "¸Ķ": 48411, "Ġmina": 48412, "ĠLatinos": 48413, "ammers": 48414, "ĠTort": 48415, "ĠBeyonce": 48416, "имоÑģÑĤи": 48417, "ĠвопÑĢоÑģÑĭ": 48418, "Ġbulun": 48419, "èĢĮå·²": 48420, "inek": 48421, "bereich": 48422, "Ġpasture": 48423, "ĠOA": 48424, "ĠMelt": 48425, "ĠEtt": 48426, "ĠDY": 48427, "Ġobwohl": 48428, "Ġleagues": 48429, "ÑĤеÑģÑĮ": 48430, "ĠкÑĥÑģ": 48431, "Ġvors": 48432, "Ġtopp": 48433, "ographical": 48434, "asst": 48435, "Ġlindo": 48436, "Ġë°ĿíĺĶ": 48437, "Ġréfl": 48438, "Ġclimbs": 48439, "Ġvarsa": 48440, "Ġmethyl": 48441, "ĠKarere": 48442, "Æ°á»Ł": 48443, "Rad": 48444, "Ġpreparedness": 48445, "онÑĩ": 48446, "ĠOD": 48447, "ĠCGI": 48448, "Ġम": 48449, "Ġspeechless": 48450, "Ġlasci": 48451, "Ġbolag": 48452, "ĠÑħоÑĩеÑĤÑģÑı": 48453, "Ġgrieving": 48454, "ĠJohannes": 48455, "ĠCarroll": 48456, "adaki": 48457, "Ī¬ë": 48458, "ĠsÅĤu": 48459, "Ġinnerhalb": 48460, "Ġgymnastics": 48461, "пÑĢи": 48462, "ifiques": 48463, "Ġkarate": 48464, "Ġdomu": 48465, "ãģĿãĤĮãģ§": 48466, "OTHER": 48467, "Ġdemandé": 48468, "Ġbooklet": 48469, "ĠKyoto": 48470, "Ġwoh": 48471, "ĠMarÃŃa": 48472, "violent": 48473, "JE": 48474, "Ġlóg": 48475, "Ġbrutally": 48476, "cot": 48477, "ĠÙħÛĮ": 48478, "ĠWarsz": 48479, "å®Ī": 48480, "wol": 48481, "Ġmikä": 48482, "ĠPronounce": 48483, "ĠBrendan": 48484, "Ġroup": 48485, "Ġitaliano": 48486, "å¦ĤæѤ": 48487, "ĠкомпÑĮÑİÑĤ": 48488, "Ġurging": 48489, "edes": 48490, "Ġcarbono": 48491, "ĠRichardson": 48492, "ĠÐĿаÑĩ": 48493, "ĠTrainer": 48494, "ĠCrimea": 48495, "Ġdiapers": 48496, "Ġcovet": 48497, "ĠMahar": 48498, "ĠHutch": 48499, "ĠAusw": 48500, "berty": 48501, "Ġindifferent": 48502, "кÑĢеÑĤ": 48503, "uldade": 48504, "Ġharms": 48505, "¢ÙĨ": 48506, "lesia": 48507, "Ġgio": 48508, "ĠMistress": 48509, "ĠKnox": 48510, "ĠFREE": 48511, "Ġ루ë": 48512, "ĠнаÑĪа": 48513, "Ġinvincible": 48514, "Ġmaiden": 48515, "ĠJeez": 48516, "Ġbreve": 48517, "pole": 48518, "Ġcriticisms": 48519, "ĠRusia": 48520, "म": 48521, "phin": 48522, "ĠCompare": 48523, "ĠBON": 48524, "Ġsneaking": 48525, "ĠRails": 48526, "ĠGeral": 48527, "Ġ1953": 48528, "Hola": 48529, "ĠопÑĭÑĤ": 48530, "Ġrainforest": 48531, "Ġbelum": 48532, "ĠObi": 48533, "ĠISS": 48534, "ãĤĮãģªãģĦ": 48535, "ĠСв": 48536, "Ġblond": 48537, "Ġwzgl": 48538, "ĠpowiedziaÅĤ": 48539, "Ġchoking": 48540, "ĠSongs": 48541, "ĠBiraz": 48542, "Ġyells": 48543, "Ġstylist": 48544, "ÏĮÏĦε": 48545, "Ġschreiben": 48546, "ĠJaw": 48547, "ĠEleven": 48548, "ĠRif": 48549, "/.": 48550, "Ġìĺ¤ëŀľë§Į": 48551, "Ġtreaties": 48552, "uffed": 48553, "ĠâĪĴ": 48554, "Ġroofs": 48555, "à¹Ģส": 48556, "Ġë»": 48557, "Ġsparkle": 48558, "ĠKiev": 48559, "ĠArgu": 48560, "erecht": 48561, "ĠÐĿадо": 48562, "ĠFIL": 48563, "Ġmolta": 48564, "ĠDevi": 48565, "Ġcampe": 48566, "Ġbenevol": 48567, "ĠTough": 48568, "Ġmoim": 48569, "Ġevacuate": 48570, "Ġerrado": 48571, "å©Ĩ": 48572, "ÑĢÑĥго": 48573, "Ġíİĺ": 48574, "ĠÎĵια": 48575, "Ġweaken": 48576, "Ġilluminated": 48577, "Ġsiglo": 48578, "ĠVacc": 48579, "ией": 48580, "alis": 48581, "ĠÑĥÑģÑĤÑĢой": 48582, "Ġdona": 48583, "ÅĤos": 48584, "üman": 48585, "Ġproducción": 48586, "Ġclot": 48587, "ĠMango": 48588, "Ġuneasy": 48589, "Ġshuts": 48590, "ĠExamples": 48591, "vell": 48592, "ebe": 48593, "Ġpromptly": 48594, "ĠTeles": 48595, "ĠпÑĢоÑĪл": 48596, "Ġpuerta": 48597, "Ġüberzeug": 48598, "Ġcoch": 48599, "social": 48600, "ĠBenson": 48601, "ĠMeth": 48602, "ĠExped": 48603, "Ġsupplemental": 48604, "Ġconceive": 48605, "Ġ×ĺ×ķ×ij": 48606, "Ġcaptivity": 48607, "ıĻìķĪ": 48608, "ĠÑħÑĥд": 48609, "forming": 48610, "Ġuploads": 48611, "Ġturbulence": 48612, "joint": 48613, "Ġsatisfactory": 48614, "ĠAnime": 48615, "Ġwashes": 48616, "Ġliberals": 48617, "ĠSunshine": 48618, "ĠREAL": 48619, "ublik": 48620, "binary": 48621, "Tony": 48622, "Ġpolarized": 48623, "Ġenriched": 48624, "taking": 48625, "ĠëģĿëĤĺ": 48626, "Ġpleasures": 48627, "Ġextermin": 48628, "inese": 48629, "atl": 48630, "vär": 48631, "аÑĢÑĭ": 48632, "ĠmyÅĽ": 48633, "narrator": 48634, "Ġодном": 48635, "ĠnajwiÄĻ": 48636, "Ġmobilize": 48637, "Ġmillor": 48638, "Ġata": 48639, "æ··": 48640, "ĠpolÃŃtico": 48641, "Ġplead": 48642, "Ġpainters": 48643, "ĠSow": 48644, "оÑĦ": 48645, "ĠìĺĽëĤł": 48646, "ĠÑĩÑĤоб": 48647, "Ġsabor": 48648, "ĠUndert": 48649, "ĠJERRY": 48650, "Å¡ÃŃ": 48651, "Ġë°ĸìĹIJ": 48652, "Ġprécéd": 48653, "Ġannotation": 48654, "ĠInaudible": 48655, "Ġtextured": 48656, "Ġfisherman": 48657, "vordan": 48658, "icherung": 48659, "ĠìłģìĿ´": 48660, "Ġgezeigt": 48661, "Ġmandates": 48662, "Ġbeak": 48663, "ĠTWO": 48664, "ĠAkbar": 48665, "ilian": 48666, "Ġtiếp": 48667, "Ġsuperiority": 48668, "inku": 48669, "Ġlys": 48670, "ĠFCC": 48671, "ĠCPA": 48672, "ustering": 48673, "nicos": 48674, "anja": 48675, "Ġchills": 48676, "ĠCage": 48677, "Ġsealing": 48678, "Ġsaç": 48679, "Ġdedans": 48680, "ĠAlger": 48681, "Ġspezie": 48682, "Ġcoloss": 48683, "ıyı": 48684, "clockwise": 48685, "Ġexactamente": 48686, "Ġiemand": 48687, "amı": 48688, "Ġmandar": 48689, "raj": 48690, "faced": 48691, "agua": 48692, "Ġê¹Ķë": 48693, "Ġinsbesondere": 48694, "Ġdrizzle": 48695, "Ġdiminish": 48696, "ĠYoda": 48697, "AI": 48698, "Ġbilmiyorum": 48699, "ĠMMA": 48700, "ategory": 48701, "ĠпеÑĢеп": 48702, "Ġparticipar": 48703, "Ġnormalized": 48704, "Ġcomplexities": 48705, "æ´²": 48706, "æݧ": 48707, "аÑĢов": 48708, "mist": 48709, "icha": 48710, "Group": 48711, "Ġresiliency": 48712, "Ġnogle": 48713, "ĠCNC": 48714, "prü": 48715, "Ġphysicists": 48716, "нок": 48717, "LI": 48718, "Ġstuffs": 48719, "Ġsistemas": 48720, "Ġinterfering": 48721, "ĠMarvin": 48722, "ército": 48723, "ĠìĹĨê³ł": 48724, "Ġsonic": 48725, "Ġequiv": 48726, "Ġabord": 48727, "ĠRamen": 48728, "Ġ09": 48729, "medim": 48730, "atiques": 48731, "ĠделаÑİÑĤ": 48732, "Ġunanimously": 48733, "Ġskirts": 48734, "ĠíĬ¹ë³Ħ": 48735, "ĠPrix": 48736, "kami": 48737, "Ġfruition": 48738, "Ġbirthdays": 48739, "иком": 48740, "Ġinaugural": 48741, "Ġcorrelate": 48742, "ĠTory": 48743, "ĠëĤĺìģ": 48744, "Ġdew": 48745, "ĠPrecis": 48746, "ihi": 48747, "Ġë¬¸ìłľê°Ģ": 48748, "Ġciting": 48749, "ĠLana": 48750, "ĠKag": 48751, "Ġplaythrough": 48752, "ĠProtocol": 48753, "frist": 48754, "hovah": 48755, "Ġmerciful": 48756, "Ġbilingual": 48757, "ĠGuitar": 48758, "rh": 48759, "Ġglamorous": 48760, "ĠVikings": 48761, "ĠOoooh": 48762, "íķĺëĬĶëį°": 48763, "ĠUganda": 48764, "Ġcollapses": 48765, "entry": 48766, "Ġantioxidants": 48767, "ëĤĺë": 48768, "ÑĪаÑı": 48769, "Ġtrivia": 48770, "Ġgäller": 48771, "Ġfungi": 48772, "Ġmilks": 48773, "Ġdicht": 48774, "μη": 48775, "poke": 48776, "ĠвÑĭпÑĥÑģк": 48777, "Ġfeeder": 48778, "ĠAlcohol": 48779, "hower": 48780, "Ġdeserving": 48781, "ĠRebel": 48782, "iosis": 48783, "Ġ103": 48784, "Ġhandout": 48785, "Ġenm": 48786, "Ġlandlords": 48787, "Ġgeology": 48788, "rils": 48789, "Ġcobra": 48790, "ĠVold": 48791, "ĠPanch": 48792, "ĠGREG": 48793, "Ġpross": 48794, "Ġbracelets": 48795, "ĠVega": 48796, "Ġrozum": 48797, "款": 48798, "азд": 48799, "ĠLynd": 48800, "ĠHonors": 48801, "Ġsurrendered": 48802, "Ġlibrarians": 48803, "125": 48804, "ĠÑģиг": 48805, "Ġuniformly": 48806, "ĠEagles": 48807, "ìķĻ": 48808, "иÑĤан": 48809, "andid": 48810, "ĠìłĪëĮĢ": 48811, "Ġض": 48812, "Ġarrests": 48813, "ĠCSV": 48814, "ĠAzerbaijan": 48815, "ortic": 48816, "ĠDX": 48817, "ĠAdventures": 48818, "Ġabus": 48819, "ĠFau": 48820, "Ġschlimm": 48821, "Ġrattling": 48822, "Ġconsumes": 48823, "ĠTolkien": 48824, "Ġresurrected": 48825, "ĠXY": 48826, "íĬ¸ê°Ģ": 48827, "ĠвÑĭÑģÑĤÑĥп": 48828, "ĠAngie": 48829, "żenia": 48830, "Mic": 48831, "ĠSheila": 48832, "achtet": 48833, "Ġoverst": 48834, "Ġlâ": 48835, "Ġineffective": 48836, "æĿ¡": 48837, "æĢİä¹ĪäºĨ": 48838, "å¿Ļ": 48839, "Ġwichtiger": 48840, "Ġvino": 48841, "Ġpum": 48842, "Ġangled": 48843, "ĠPione": 48844, "ĠMỹ": 48845, "ãģĿãĤĮãģ¯": 48846, "woÅĽÄĩ": 48847, "draw": 48848, "ัà¹Ī": 48849, "markets": 48850, "Ġcafes": 48851, "ĠCem": 48852, "âĿ¤": 48853, "ĠSuit": 48854, "MK": 48855, "Ġemphasizes": 48856, "Ġtortilla": 48857, "Ġmejorar": 48858, "ĠSurviv": 48859, "casting": 48860, "Ġeducación": 48861, "ĠGum": 48862, "uely": 48863, "ĠìĹ¬ê¸°ëĬĶ": 48864, "Ġstretchy": 48865, "ença": 48866, "Ġwithhold": 48867, "Ġexiting": 48868, "Ġenthalpy": 48869, "ĠTransit": 48870, "ılmÄ±ÅŁ": 48871, "alies": 48872, "Ġsalvar": 48873, "Ġleaned": 48874, "ĠgroÃŁes": 48875, "Ġfitt": 48876, "аки": 48877, "Sarah": 48878, "Ġhostel": 48879, "Ġfingerna": 48880, "ĠnadziejÄĻ": 48881, "wives": 48882, "Rec": 48883, "Ġspool": 48884, "аÑĤов": 48885, "ĠEnemy": 48886, "Ġfury": 48887, "Ġdetta": 48888, "ĠFay": 48889, "éļ¨": 48890, "ÑıÑİÑĤ": 48891, "Ġaproximadamente": 48892, "Ġsilos": 48893, "Ġmagist": 48894, "Ġcree": 48895, "ĠKrank": 48896, "ĠDOWN": 48897, "Ġstartled": 48898, "Ġreborn": 48899, "ĠUmwelt": 48900, "ĠSuzanne": 48901, "ниÑĨÑĭ": 48902, "outez": 48903, "ĠJAC": 48904, "yards": 48905, "radas": 48906, "rau": 48907, "ipts": 48908, "hail": 48909, "Ġparagraphs": 48910, "Ġmeglio": 48911, "Ġisolating": 48912, "Ġaceite": 48913, "ĠHarsh": 48914, "Ġcyst": 48915, "ĠBlockchain": 48916, "ĠÑħоÑĢоÑĪий": 48917, "Ġvirtuous": 48918, "Ġinvestigación": 48919, "Ġdevoir": 48920, "Ġmasturb": 48921, "ĠSale": 48922, "ÙĬرة": 48923, "ĠΧ": 48924, "ĠStraÃŁen": 48925, "Ġdikk": 48926, "Ġafore": 48927, "ĠJungkook": 48928, "Ġchociaż": 48929, "ĠDebatte": 48930, "Ġweirdly": 48931, "Ġviaje": 48932, "regist": 48933, "Help": 48934, "Ġkinderen": 48935, "Ġformulated": 48936, "Ġenfim": 48937, "ĠTowards": 48938, "коÑĹ": 48939, "ivering": 48940, "ĠдеÑĤи": 48941, "charger": 48942, "Ġpurl": 48943, "Ġacademically": 48944, "ĠNurse": 48945, "Ġdeleting": 48946, "ayo": 48947, "Ġrefusal": 48948, "Ġdepicts": 48949, "ĠDracula": 48950, "Ġtoasted": 48951, "ĠZombie": 48952, "ĠSuperior": 48953, "ĠBold": 48954, "Ġquizzes": 48955, "Ġgle": 48956, "450": 48957, "Ġcomeço": 48958, "ynn": 48959, "Ġverst": 48960, "ĠOlaf": 48961, "Ġpomoc": 48962, "ĠSask": 48963, "ëĺ": 48964, "ĠTCP": 48965, "ĠProperty": 48966, "íķĺì£ł": 48967, "à¸ľà¸¡": 48968, "boom": 48969, "aros": 48970, "ĠÑĢоÑģÑģий": 48971, "ĠбÑĭваеÑĤ": 48972, "åĩºåİ»": 48973, "ĠìĿ´ìķ¼ê¸°ë¥¼": 48974, "Ġcombien": 48975, "vacc": 48976, "Ġebenfalls": 48977, "para": 48978, "Ġзм": 48979, "Ġdesperation": 48980, "ordre": 48981, "Ġש׾×Ļ": 48982, "Ġgenerously": 48983, "ĠÐŀк": 48984, "Ġorbiting": 48985, "> length: - array = array.index_select( - dim=axis, index=torch.arange(length, device=array.device) - ) - - if array.shape[axis] < length: - pad_widths = [(0, 0)] * array.ndim - pad_widths[axis] = (0, length - array.shape[axis]) - array = F.pad(array, [pad for sizes in pad_widths[::-1] for pad in sizes]) - else: - if array.shape[axis] > length: - array = array.take(indices=range(length), axis=axis) - - if array.shape[axis] < length: - pad_widths = [(0, 0)] * array.ndim - pad_widths[axis] = (0, length - array.shape[axis]) - array = np.pad(array, pad_widths) - - return array - - -@lru_cache(maxsize=None) -def mel_filters(device, n_mels: int = N_MELS) -> torch.Tensor: - """ - load the mel filterbank matrix for projecting STFT into a Mel spectrogram. - Allows decoupling librosa dependency; saved using: - - np.savez_compressed( - "mel_filters.npz", - mel_80=librosa.filters.mel(sr=16000, n_fft=400, n_mels=80), - ) - """ - assert n_mels == 80, f"Unsupported n_mels: {n_mels}" - with np.load( - os.path.join(os.path.dirname(__file__), "assets", "mel_filters.npz") - ) as f: - return torch.from_numpy(f[f"mel_{n_mels}"]).to(device) - - -def log_mel_spectrogram( - audio: Union[str, np.ndarray, torch.Tensor], n_mels: int = N_MELS -): - """ - Compute the log-Mel spectrogram of - - Parameters - ---------- - audio: Union[str, np.ndarray, torch.Tensor], shape = (*) - The path to audio or either a NumPy array or Tensor containing the audio waveform in 16 kHz - - n_mels: int - The number of Mel-frequency filters, only 80 is supported - - Returns - ------- - torch.Tensor, shape = (80, n_frames) - A Tensor that contains the Mel spectrogram - """ - if not torch.is_tensor(audio): - if isinstance(audio, str): - audio = load_audio(audio) - audio = torch.from_numpy(audio) - - window = torch.hann_window(N_FFT).to(audio.device) - stft = torch.stft(audio, N_FFT, HOP_LENGTH, window=window, return_complex=True) - magnitudes = stft[..., :-1].abs() ** 2 - - filters = mel_filters(audio.device, n_mels) - mel_spec = filters @ magnitudes - - log_spec = torch.clamp(mel_spec, min=1e-10).log10() - log_spec = torch.maximum(log_spec, log_spec.max() - 8.0) - log_spec = (log_spec + 4.0) / 4.0 - return log_spec diff --git a/funasr_detach/models/whisper/utils/decoding.py b/funasr_detach/models/whisper/utils/decoding.py deleted file mode 100644 index 9c5bf3c6d6c252499677416bac58fa56b2ab27e3..0000000000000000000000000000000000000000 --- a/funasr_detach/models/whisper/utils/decoding.py +++ /dev/null @@ -1,811 +0,0 @@ -from dataclasses import dataclass, field -from typing import Dict, List, Tuple, Iterable, Optional, Sequence, Union, TYPE_CHECKING - -import numpy as np -import torch -import torch.nn.functional as F -from torch import Tensor -from torch.distributions import Categorical - -from funasr_detach.models.whisper.utils.audio import CHUNK_LENGTH -from funasr_detach.models.whisper.utils.tokenizer import Tokenizer, get_tokenizer -from funasr_detach.models.whisper.utils.utils import compression_ratio - -if TYPE_CHECKING: - from funasr_detach.models.whisper_models.model import Whisper - - -@torch.no_grad() -def detect_language( - model: "Whisper", mel: Tensor, tokenizer: Tokenizer = None -) -> Tuple[Tensor, List[dict]]: - """ - Detect the spoken language in the audio, and return them as list of strings, along with the ids - of the most probable language tokens and the probability distribution over all language tokens. - This is performed outside the main decode loop in order to not interfere with kv-caching. - - Returns - ------- - language_tokens : Tensor, shape = (n_audio,) - ids of the most probable language tokens, which appears after the startoftranscript token. - language_probs : List[Dict[str, float]], length = n_audio - list of dictionaries containing the probability distribution over all languages. - """ - if tokenizer is None: - tokenizer = get_tokenizer(model.is_multilingual) - if ( - tokenizer.language is None - or tokenizer.language_token not in tokenizer.sot_sequence - ): - raise ValueError( - f"This model doesn't have language tokens so it can't perform lang id" - ) - - single = mel.ndim == 2 - if single: - mel = mel.unsqueeze(0) - - # skip encoder forward pass if already-encoded audio features were given - if mel.shape[-2:] != (model.dims.n_audio_ctx, model.dims.n_audio_state): - mel = model.encoder(mel) - - # forward pass using a single token, startoftranscript - n_audio = mel.shape[0] - x = torch.tensor([[tokenizer.sot]] * n_audio).to(mel.device) # [n_audio, 1] - logits = model.logits(x, mel)[:, 0] - - # collect detected languages; suppress all non-language tokens - mask = torch.ones(logits.shape[-1], dtype=torch.bool) - mask[list(tokenizer.all_language_tokens)] = False - logits[:, mask] = -np.inf - language_tokens = logits.argmax(dim=-1) - language_token_probs = logits.softmax(dim=-1).cpu() - language_probs = [ - { - c: language_token_probs[i, j].item() - for j, c in zip(tokenizer.all_language_tokens, tokenizer.all_language_codes) - } - for i in range(n_audio) - ] - - if single: - language_tokens = language_tokens[0] - language_probs = language_probs[0] - - return language_tokens, language_probs - - -@dataclass(frozen=True) -class DecodingOptions: - task: str = ( - "transcribe" # whether to perform X->X "transcribe" or X->English "translate" - ) - language: Optional[str] = ( - None # language that the audio is in; uses detected language if None - ) - - # sampling-related options - temperature: float = 0.0 - sample_len: Optional[int] = None # maximum number of tokens to sample - best_of: Optional[int] = ( - None # number of independent samples to collect, when t > 0 - ) - beam_size: Optional[int] = None # number of beams in beam search, when t == 0 - patience: Optional[float] = ( - None # patience in beam search (https://arxiv.org/abs/2204.05424) - ) - - # options for ranking generations (either beams or best-of-N samples) - length_penalty: Optional[float] = ( - None # "alpha" in Google NMT, None defaults to length norm - ) - - # prompt, prefix, and token suppression - prompt: Optional[Union[str, List[int]]] = ( - None # text or tokens for the previous context - ) - prefix: Optional[Union[str, List[int]]] = ( - None # text or tokens to prefix the current context - ) - suppress_blank: bool = True # this will suppress blank outputs - - # list of tokens ids (or comma-separated token ids) to suppress - # "-1" will suppress a set of symbols as defined in `tokenizer.non_speech_tokens()` - suppress_tokens: Optional[Union[str, Iterable[int]]] = "-1" - - # timestamp sampling options - without_timestamps: bool = False # use <|notimestamps|> to sample text tokens only - max_initial_timestamp: Optional[float] = ( - 1.0 # the initial timestamp cannot be later than this - ) - - # implementation details - fp16: bool = True # use fp16 for most of the calculation - - -@dataclass(frozen=True) -class DecodingResult: - audio_features: Tensor - language: str - language_probs: Optional[Dict[str, float]] = None - tokens: List[int] = field(default_factory=list) - text: str = "" - avg_logprob: float = np.nan - no_speech_prob: float = np.nan - temperature: float = np.nan - compression_ratio: float = np.nan - - -class Inference: - def logits(self, tokens: Tensor, audio_features: Tensor) -> Tensor: - """Perform a forward pass on the decoder and return per-token logits""" - raise NotImplementedError - - def rearrange_kv_cache(self, source_indices) -> None: - """Update the key-value cache according to the updated beams""" - raise NotImplementedError - - def cleanup_caching(self) -> None: - """Clean up any resources or hooks after decoding is finished""" - pass - - -class PyTorchInference(Inference): - def __init__(self, model: "Whisper", initial_token_length: int): - self.model: "Whisper" = model - self.initial_token_length = initial_token_length - self.kv_cache = {} - self.hooks = [] - - def logits(self, tokens: Tensor, audio_features: Tensor) -> Tensor: - if not self.kv_cache: - self.kv_cache, self.hooks = self.model.install_kv_cache_hooks() - - if tokens.shape[-1] > self.initial_token_length: - # only need to use the last token except in the first forward pass - tokens = tokens[:, -1:] - - return self.model.decoder(tokens, audio_features, kv_cache=self.kv_cache) - - def cleanup_caching(self): - for hook in self.hooks: - hook.remove() - - self.kv_cache = {} - self.hooks = [] - - def rearrange_kv_cache(self, source_indices): - for module, tensor in self.kv_cache.items(): - # update the key/value cache to contain the selected sequences - self.kv_cache[module] = tensor[source_indices].detach() - - -class SequenceRanker: - def rank( - self, tokens: List[List[Tensor]], sum_logprobs: List[List[float]] - ) -> List[int]: - """ - Given a list of groups of samples and their cumulative log probabilities, - return the indices of the samples in each group to select as the final result - """ - raise NotImplementedError - - -class MaximumLikelihoodRanker(SequenceRanker): - """ - Select the sample with the highest log probabilities, penalized using either - a simple length normalization or Google NMT paper's length penalty - """ - - def __init__(self, length_penalty: Optional[float]): - self.length_penalty = length_penalty - - def rank(self, tokens: List[List[Tensor]], sum_logprobs: List[List[float]]): - def scores(logprobs, lengths): - result = [] - for logprob, length in zip(logprobs, lengths): - if self.length_penalty is None: - penalty = length - else: - # from the Google NMT paper - penalty = ((5 + length) / 6) ** self.length_penalty - result.append(logprob / penalty) - return result - - # get the sequence with the highest score - lengths = [[len(t) for t in s] for s in tokens] - return [np.argmax(scores(p, l)) for p, l in zip(sum_logprobs, lengths)] - - -class TokenDecoder: - def reset(self): - """Initialize any stateful variables for decoding a new sequence""" - - def update( - self, tokens: Tensor, logits: Tensor, sum_logprobs: Tensor - ) -> Tuple[Tensor, bool]: - """Specify how to select the next token, based on the current trace and logits - - Parameters - ---------- - tokens : Tensor, shape = (n_batch, current_sequence_length) - all tokens in the context so far, including the prefix and sot_sequence tokens - - logits : Tensor, shape = (n_batch, vocab_size) - per-token logits of the probability distribution at the current step - - sum_logprobs : Tensor, shape = (n_batch) - cumulative log probabilities for each sequence - - Returns - ------- - tokens : Tensor, shape = (n_batch, current_sequence_length + 1) - the tokens, appended with the selected next token - - completed : bool - True if all sequences has reached the end of text - - """ - raise NotImplementedError - - def finalize( - self, tokens: Tensor, sum_logprobs: Tensor - ) -> Tuple[Sequence[Sequence[Tensor]], List[List[float]]]: - """Finalize search and return the final candidate sequences - - Parameters - ---------- - tokens : Tensor, shape = (n_audio, n_group, current_sequence_length) - all tokens in the context so far, including the prefix and sot_sequence - - sum_logprobs : Tensor, shape = (n_audio, n_group) - cumulative log probabilities for each sequence - - Returns - ------- - tokens : Sequence[Sequence[Tensor]], length = n_audio - sequence of Tensors containing candidate token sequences, for each audio input - - sum_logprobs : List[List[float]], length = n_audio - sequence of cumulative log probabilities corresponding to the above - - """ - raise NotImplementedError - - -class GreedyDecoder(TokenDecoder): - def __init__(self, temperature: float, eot: int): - self.temperature = temperature - self.eot = eot - - def update( - self, tokens: Tensor, logits: Tensor, sum_logprobs: Tensor - ) -> Tuple[Tensor, bool]: - temperature = self.temperature - if temperature == 0: - next_tokens = logits.argmax(dim=-1) - else: - next_tokens = Categorical(logits=logits / temperature).sample() - - logprobs = F.log_softmax(logits.float(), dim=-1) - current_logprobs = logprobs[torch.arange(logprobs.shape[0]), next_tokens] - sum_logprobs += current_logprobs * (tokens[:, -1] != self.eot) - - next_tokens[tokens[:, -1] == self.eot] = self.eot - tokens = torch.cat([tokens, next_tokens[:, None]], dim=-1) - - completed = (tokens[:, -1] == self.eot).all() - return tokens, completed - - def finalize(self, tokens: Tensor, sum_logprobs: Tensor): - # make sure each sequence has at least one EOT token at the end - tokens = F.pad(tokens, (0, 1), value=self.eot) - return tokens, sum_logprobs.tolist() - - -class BeamSearchDecoder(TokenDecoder): - def __init__( - self, - beam_size: int, - eot: int, - inference: Inference, - patience: Optional[float] = None, - ): - self.beam_size = beam_size - self.eot = eot - self.inference = inference - self.patience = patience or 1.0 - self.max_candidates: int = round(beam_size * self.patience) - self.finished_sequences = None - - assert ( - self.max_candidates > 0 - ), f"Invalid beam size ({beam_size}) or patience ({patience})" - - def reset(self): - self.finished_sequences = None - - def update( - self, tokens: Tensor, logits: Tensor, sum_logprobs: Tensor - ) -> Tuple[Tensor, bool]: - if tokens.shape[0] % self.beam_size != 0: - raise ValueError(f"{tokens.shape}[0] % {self.beam_size} != 0") - - n_audio = tokens.shape[0] // self.beam_size - if self.finished_sequences is None: # for the first update - self.finished_sequences = [{} for _ in range(n_audio)] - - logprobs = F.log_softmax(logits.float(), dim=-1) - next_tokens, source_indices, finished_sequences = [], [], [] - for i in range(n_audio): - scores, sources, finished = {}, {}, {} - - # STEP 1: calculate the cumulative log probabilities for possible candidates - for j in range(self.beam_size): - idx = i * self.beam_size + j - prefix = tokens[idx].tolist() - for logprob, token in zip(*logprobs[idx].topk(self.beam_size + 1)): - new_logprob = (sum_logprobs[idx] + logprob).item() - sequence = tuple(prefix + [token.item()]) - scores[sequence] = new_logprob - sources[sequence] = idx - - # STEP 2: rank the candidates and keep the top beam_size sequences for each audio - saved = 0 - for sequence in sorted(scores, key=scores.get, reverse=True): - if sequence[-1] == self.eot: - finished[sequence] = scores[sequence] - else: - sum_logprobs[len(next_tokens)] = scores[sequence] - next_tokens.append(sequence) - source_indices.append(sources[sequence]) - - saved += 1 - if saved == self.beam_size: - break - - finished_sequences.append(finished) - - tokens = torch.tensor(next_tokens, device=tokens.device) - self.inference.rearrange_kv_cache(source_indices) - - # add newly finished sequences to self.finished_sequences - assert len(self.finished_sequences) == len(finished_sequences) - for previously_finished, newly_finished in zip( - self.finished_sequences, finished_sequences - ): - for seq in sorted(newly_finished, key=newly_finished.get, reverse=True): - if len(previously_finished) >= self.max_candidates: - break # the candidate list is full - previously_finished[seq] = newly_finished[seq] - - # mark as completed if all audio has enough number of samples - completed = all( - len(sequences) >= self.max_candidates - for sequences in self.finished_sequences - ) - return tokens, completed - - def finalize(self, preceding_tokens: Tensor, sum_logprobs: Tensor): - # collect all finished sequences, including patience, and add unfinished ones if not enough - sum_logprobs = sum_logprobs.cpu() - for i, sequences in enumerate(self.finished_sequences): - if ( - len(sequences) < self.beam_size - ): # when not enough sequences are finished - for j in list(np.argsort(sum_logprobs[i]))[::-1]: - sequence = preceding_tokens[i, j].tolist() + [self.eot] - sequences[tuple(sequence)] = sum_logprobs[i][j].item() - if len(sequences) >= self.beam_size: - break - - tokens: List[List[Tensor]] = [ - [torch.tensor(seq) for seq in sequences.keys()] - for sequences in self.finished_sequences - ] - sum_logprobs: List[List[float]] = [ - list(sequences.values()) for sequences in self.finished_sequences - ] - return tokens, sum_logprobs - - -class LogitFilter: - def apply(self, logits: Tensor, tokens: Tensor) -> None: - """Apply any filtering or masking to logits in-place - - Parameters - ---------- - logits : Tensor, shape = (n_batch, vocab_size) - per-token logits of the probability distribution at the current step - - tokens : Tensor, shape = (n_batch, current_sequence_length) - all tokens in the context so far, including the prefix and sot_sequence tokens - - """ - raise NotImplementedError - - -class SuppressBlank(LogitFilter): - def __init__(self, tokenizer: Tokenizer, sample_begin: int): - self.tokenizer = tokenizer - self.sample_begin = sample_begin - - def apply(self, logits: Tensor, tokens: Tensor): - if tokens.shape[1] == self.sample_begin: - logits[:, self.tokenizer.encode(" ") + [self.tokenizer.eot]] = -np.inf - - -class SuppressTokens(LogitFilter): - def __init__(self, suppress_tokens: Sequence[int]): - self.suppress_tokens = list(suppress_tokens) - - def apply(self, logits: Tensor, tokens: Tensor): - logits[:, self.suppress_tokens] = -np.inf - - -class ApplyTimestampRules(LogitFilter): - def __init__( - self, - tokenizer: Tokenizer, - sample_begin: int, - max_initial_timestamp_index: Optional[int], - ): - self.tokenizer = tokenizer - self.sample_begin = sample_begin - self.max_initial_timestamp_index = max_initial_timestamp_index - - def apply(self, logits: Tensor, tokens: Tensor): - # suppress <|notimestamps|> which is handled by without_timestamps - if self.tokenizer.no_timestamps is not None: - logits[:, self.tokenizer.no_timestamps] = -np.inf - - # timestamps have to appear in pairs, except directly before EOT; mask logits accordingly - for k in range(tokens.shape[0]): - seq = [t for t in tokens[k, self.sample_begin :].tolist()] - last_was_timestamp = ( - len(seq) >= 1 and seq[-1] >= self.tokenizer.timestamp_begin - ) - penultimate_was_timestamp = ( - len(seq) < 2 or seq[-2] >= self.tokenizer.timestamp_begin - ) - - if last_was_timestamp: - if penultimate_was_timestamp: # has to be non-timestamp - logits[k, self.tokenizer.timestamp_begin :] = -np.inf - else: # cannot be normal text tokens - logits[k, : self.tokenizer.eot] = -np.inf - - if tokens.shape[1] == self.sample_begin: - # suppress generating non-timestamp tokens at the beginning - logits[:, : self.tokenizer.timestamp_begin] = -np.inf - - # apply the `max_initial_timestamp` option - if self.max_initial_timestamp_index is not None: - last_allowed = ( - self.tokenizer.timestamp_begin + self.max_initial_timestamp_index - ) - logits[:, last_allowed + 1 :] = -np.inf - - # if sum of probability over timestamps is above any other token, sample timestamp - logprobs = F.log_softmax(logits.float(), dim=-1) - for k in range(tokens.shape[0]): - timestamp_logprob = logprobs[k, self.tokenizer.timestamp_begin :].logsumexp( - dim=-1 - ) - max_text_token_logprob = logprobs[k, : self.tokenizer.timestamp_begin].max() - if timestamp_logprob > max_text_token_logprob: - logits[k, : self.tokenizer.timestamp_begin] = -np.inf - - -class DecodingTask: - inference: Inference - sequence_ranker: SequenceRanker - decoder: TokenDecoder - logit_filters: List[LogitFilter] - - def __init__(self, model: "Whisper", options: DecodingOptions): - self.model = model - - language = options.language or "en" - tokenizer = get_tokenizer( - model.is_multilingual, language=language, task=options.task - ) - self.tokenizer: Tokenizer = tokenizer - self.options: DecodingOptions = self._verify_options(options) - - self.n_group: int = options.beam_size or options.best_of or 1 - self.n_ctx: int = model.dims.n_text_ctx - self.sample_len: int = options.sample_len or model.dims.n_text_ctx // 2 - - self.sot_sequence: Tuple[int] = tokenizer.sot_sequence - if self.options.without_timestamps: - self.sot_sequence = tokenizer.sot_sequence_including_notimestamps - - self.initial_tokens: Tuple[int] = self._get_initial_tokens() - self.sample_begin: int = len(self.initial_tokens) - self.sot_index: int = self.initial_tokens.index(tokenizer.sot) - - # inference: implements the forward pass through the decoder, including kv caching - self.inference = PyTorchInference(model, len(self.initial_tokens)) - - # sequence ranker: implements how to rank a group of sampled sequences - self.sequence_ranker = MaximumLikelihoodRanker(options.length_penalty) - - # decoder: implements how to select the next tokens, given the autoregressive distribution - if options.beam_size is not None: - self.decoder = BeamSearchDecoder( - options.beam_size, tokenizer.eot, self.inference, options.patience - ) - else: - self.decoder = GreedyDecoder(options.temperature, tokenizer.eot) - - # logit filters: applies various rules to suppress or penalize certain tokens - self.logit_filters = [] - if self.options.suppress_blank: - self.logit_filters.append(SuppressBlank(self.tokenizer, self.sample_begin)) - if self.options.suppress_tokens: - self.logit_filters.append(SuppressTokens(self._get_suppress_tokens())) - if not options.without_timestamps: - precision = CHUNK_LENGTH / model.dims.n_audio_ctx # usually 0.02 seconds - max_initial_timestamp_index = None - if options.max_initial_timestamp: - max_initial_timestamp_index = round( - self.options.max_initial_timestamp / precision - ) - self.logit_filters.append( - ApplyTimestampRules( - tokenizer, self.sample_begin, max_initial_timestamp_index - ) - ) - - def _verify_options(self, options: DecodingOptions) -> DecodingOptions: - if options.beam_size is not None and options.best_of is not None: - raise ValueError("beam_size and best_of can't be given together") - if options.temperature == 0: - if options.best_of is not None: - raise ValueError("best_of with greedy sampling (T=0) is not compatible") - if options.patience is not None and options.beam_size is None: - raise ValueError("patience requires beam_size to be given") - if options.length_penalty is not None and not ( - 0 <= options.length_penalty <= 1 - ): - raise ValueError("length_penalty (alpha) should be a value between 0 and 1") - - return options - - def _get_initial_tokens(self) -> Tuple[int]: - tokens = list(self.sot_sequence) - prefix = self.options.prefix - prompt = self.options.prompt - - if prefix: - prefix_tokens = ( - self.tokenizer.encode(" " + prefix.strip()) - if isinstance(prefix, str) - else prefix - ) - if self.sample_len is not None: - max_prefix_len = self.n_ctx // 2 - self.sample_len - prefix_tokens = prefix_tokens[-max_prefix_len:] - tokens = tokens + prefix_tokens - - if prompt: - prompt_tokens = ( - self.tokenizer.encode(" " + prompt.strip()) - if isinstance(prompt, str) - else prompt - ) - tokens = ( - [self.tokenizer.sot_prev] - + prompt_tokens[-(self.n_ctx // 2 - 1) :] - + tokens - ) - - return tuple(tokens) - - def _get_suppress_tokens(self) -> Tuple[int]: - suppress_tokens = self.options.suppress_tokens - - if isinstance(suppress_tokens, str): - suppress_tokens = [int(t) for t in suppress_tokens.split(",")] - - if -1 in suppress_tokens: - suppress_tokens = [t for t in suppress_tokens if t >= 0] - suppress_tokens.extend(self.tokenizer.non_speech_tokens) - elif suppress_tokens is None or len(suppress_tokens) == 0: - suppress_tokens = [] # interpret empty string as an empty list - else: - assert isinstance(suppress_tokens, list), "suppress_tokens must be a list" - - suppress_tokens.extend( - [self.tokenizer.sot, self.tokenizer.sot_prev, self.tokenizer.sot_lm] - ) - if self.tokenizer.no_speech is not None: - # no-speech probability is collected separately - suppress_tokens.append(self.tokenizer.no_speech) - - return tuple(sorted(set(suppress_tokens))) - - def _get_audio_features(self, mel: Tensor): - if self.options.fp16: - mel = mel.half() - - if mel.shape[-2:] == ( - self.model.dims.n_audio_ctx, - self.model.dims.n_audio_state, - ): - # encoded audio features are given; skip audio encoding - audio_features = mel - else: - audio_features = self.model.encoder(mel) - - if audio_features.dtype != ( - torch.float16 if self.options.fp16 else torch.float32 - ): - return TypeError( - f"audio_features has an incorrect dtype: {audio_features.dtype}" - ) - - return audio_features - - def _detect_language(self, audio_features: Tensor, tokens: Tensor): - languages = [self.options.language] * audio_features.shape[0] - lang_probs = None - - if self.options.language is None or self.options.task == "lang_id": - lang_tokens, lang_probs = self.model.detect_language( - audio_features, self.tokenizer - ) - languages = [max(probs, key=probs.get) for probs in lang_probs] - if self.options.language is None: - tokens[:, self.sot_index + 1] = lang_tokens # write language tokens - - return languages, lang_probs - - def _main_loop(self, audio_features: Tensor, tokens: Tensor): - assert audio_features.shape[0] == tokens.shape[0] - n_batch = tokens.shape[0] - sum_logprobs: Tensor = torch.zeros(n_batch, device=audio_features.device) - no_speech_probs = [np.nan] * n_batch - - try: - for i in range(self.sample_len): - logits = self.inference.logits(tokens, audio_features) - - if ( - i == 0 and self.tokenizer.no_speech is not None - ): # save no_speech_probs - probs_at_sot = logits[:, self.sot_index].float().softmax(dim=-1) - no_speech_probs = probs_at_sot[:, self.tokenizer.no_speech].tolist() - - # now we need to consider the logits at the last token only - logits = logits[:, -1] - - # apply the logit filters, e.g. for suppressing or applying penalty to - for logit_filter in self.logit_filters: - logit_filter.apply(logits, tokens) - - # expand the tokens tensor with the selected next tokens - tokens, completed = self.decoder.update(tokens, logits, sum_logprobs) - - if completed or tokens.shape[-1] > self.n_ctx: - break - finally: - self.inference.cleanup_caching() - - return tokens, sum_logprobs, no_speech_probs - - @torch.no_grad() - def run(self, mel: Tensor) -> List[DecodingResult]: - self.decoder.reset() - tokenizer: Tokenizer = self.tokenizer - n_audio: int = mel.shape[0] - - audio_features: Tensor = self._get_audio_features(mel) # encoder forward pass - tokens: Tensor = torch.tensor([self.initial_tokens]).repeat(n_audio, 1) - - # detect language if requested, overwriting the language token - languages, language_probs = self._detect_language(audio_features, tokens) - if self.options.task == "lang_id": - return [ - DecodingResult( - audio_features=features, language=language, language_probs=probs - ) - for features, language, probs in zip( - audio_features, languages, language_probs - ) - ] - - # repeat the audio & text tensors by the group size, for beam search or best-of-n sampling - audio_features = audio_features.repeat_interleave(self.n_group, dim=0) - tokens = tokens.repeat_interleave(self.n_group, dim=0).to(audio_features.device) - - # call the main sampling loop - tokens, sum_logprobs, no_speech_probs = self._main_loop(audio_features, tokens) - - # reshape the tensors to have (n_audio, n_group) as the first two dimensions - audio_features = audio_features[:: self.n_group] - no_speech_probs = no_speech_probs[:: self.n_group] - assert audio_features.shape[0] == len(no_speech_probs) == n_audio - - tokens = tokens.reshape(n_audio, self.n_group, -1) - sum_logprobs = sum_logprobs.reshape(n_audio, self.n_group) - - # get the final candidates for each group, and slice between the first sampled token and EOT - tokens, sum_logprobs = self.decoder.finalize(tokens, sum_logprobs) - tokens: List[List[Tensor]] = [ - [t[self.sample_begin : (t == tokenizer.eot).nonzero()[0, 0]] for t in s] - for s in tokens - ] - - # select the top-ranked sample in each group - selected = self.sequence_ranker.rank(tokens, sum_logprobs) - tokens: List[List[int]] = [t[i].tolist() for i, t in zip(selected, tokens)] - texts: List[str] = [tokenizer.decode(t).strip() for t in tokens] - - sum_logprobs: List[float] = [lp[i] for i, lp in zip(selected, sum_logprobs)] - avg_logprobs: List[float] = [ - lp / (len(t) + 1) for t, lp in zip(tokens, sum_logprobs) - ] - - fields = ( - texts, - languages, - tokens, - audio_features, - avg_logprobs, - no_speech_probs, - ) - if len(set(map(len, fields))) != 1: - raise RuntimeError(f"inconsistent result lengths: {list(map(len, fields))}") - - return [ - DecodingResult( - audio_features=features, - language=language, - tokens=tokens, - text=text, - avg_logprob=avg_logprob, - no_speech_prob=no_speech_prob, - temperature=self.options.temperature, - compression_ratio=compression_ratio(text), - ) - for text, language, tokens, features, avg_logprob, no_speech_prob in zip( - *fields - ) - ] - - -@torch.no_grad() -def decode( - model: "Whisper", mel: Tensor, options: DecodingOptions = DecodingOptions() -) -> Union[DecodingResult, List[DecodingResult]]: - """ - Performs decoding of 30-second audio segment(s), provided as Mel spectrogram(s). - - Parameters - ---------- - model: Whisper - the Whisper model instance - - mel: torch.Tensor, shape = (80, 3000) or (*, 80, 3000) - A tensor containing the Mel spectrogram(s) - - options: DecodingOptions - A dataclass that contains all necessary options for decoding 30-second segments - - Returns - ------- - result: Union[DecodingResult, List[DecodingResult]] - The result(s) of decoding contained in `DecodingResult` dataclass instance(s) - """ - single = mel.ndim == 2 - if single: - mel = mel.unsqueeze(0) - - result = DecodingTask(model, options).run(mel) - - if single: - result = result[0] - - return result diff --git a/funasr_detach/models/whisper/utils/tokenizer.py b/funasr_detach/models/whisper/utils/tokenizer.py deleted file mode 100644 index 925598f845be0dea17bc7a9bc7b169348cfd3f70..0000000000000000000000000000000000000000 --- a/funasr_detach/models/whisper/utils/tokenizer.py +++ /dev/null @@ -1,351 +0,0 @@ -import os -from dataclasses import dataclass -from functools import lru_cache -from typing import List, Optional, Tuple, Union - -import numpy as np -import torch - -try: - from transformers import GPT2TokenizerFast -except ImportError: - raise ImportError( - "transformers was not installed. Please install transformers first." - ) - -LANGUAGES = { - "en": "english", - "zh": "chinese", - "de": "german", - "es": "spanish", - "ru": "russian", - "ko": "korean", - "fr": "french", - "ja": "japanese", - "pt": "portuguese", - "tr": "turkish", - "pl": "polish", - "ca": "catalan", - "nl": "dutch", - "ar": "arabic", - "sv": "swedish", - "it": "italian", - "id": "indonesian", - "hi": "hindi", - "fi": "finnish", - "vi": "vietnamese", - "he": "hebrew", - "uk": "ukrainian", - "el": "greek", - "ms": "malay", - "cs": "czech", - "ro": "romanian", - "da": "danish", - "hu": "hungarian", - "ta": "tamil", - "no": "norwegian", - "th": "thai", - "ur": "urdu", - "hr": "croatian", - "bg": "bulgarian", - "lt": "lithuanian", - "la": "latin", - "mi": "maori", - "ml": "malayalam", - "cy": "welsh", - "sk": "slovak", - "te": "telugu", - "fa": "persian", - "lv": "latvian", - "bn": "bengali", - "sr": "serbian", - "az": "azerbaijani", - "sl": "slovenian", - "kn": "kannada", - "et": "estonian", - "mk": "macedonian", - "br": "breton", - "eu": "basque", - "is": "icelandic", - "hy": "armenian", - "ne": "nepali", - "mn": "mongolian", - "bs": "bosnian", - "kk": "kazakh", - "sq": "albanian", - "sw": "swahili", - "gl": "galician", - "mr": "marathi", - "pa": "punjabi", - "si": "sinhala", - "km": "khmer", - "sn": "shona", - "yo": "yoruba", - "so": "somali", - "af": "afrikaans", - "oc": "occitan", - "ka": "georgian", - "be": "belarusian", - "tg": "tajik", - "sd": "sindhi", - "gu": "gujarati", - "am": "amharic", - "yi": "yiddish", - "lo": "lao", - "uz": "uzbek", - "fo": "faroese", - "ht": "haitian creole", - "ps": "pashto", - "tk": "turkmen", - "nn": "nynorsk", - "mt": "maltese", - "sa": "sanskrit", - "lb": "luxembourgish", - "my": "myanmar", - "bo": "tibetan", - "tl": "tagalog", - "mg": "malagasy", - "as": "assamese", - "tt": "tatar", - "haw": "hawaiian", - "ln": "lingala", - "ha": "hausa", - "ba": "bashkir", - "jw": "javanese", - "su": "sundanese", -} - -# language code lookup by name, with a few language aliases -TO_LANGUAGE_CODE = { - **{language: code for code, language in LANGUAGES.items()}, - "burmese": "my", - "valencian": "ca", - "flemish": "nl", - "haitian": "ht", - "letzeburgesch": "lb", - "pushto": "ps", - "panjabi": "pa", - "moldavian": "ro", - "moldovan": "ro", - "sinhalese": "si", - "castilian": "es", -} - - -@dataclass(frozen=True) -class Tokenizer: - """A thin wrapper around `GPT2TokenizerFast` providing quick access to special tokens""" - - tokenizer: "GPT2TokenizerFast" - language: Optional[str] - sot_sequence: Tuple[int] - - def encode(self, text, **kwargs): - return self.tokenizer.encode(text, **kwargs) - - def decode( - self, token_ids: Union[int, List[int], np.ndarray, torch.Tensor], **kwargs - ): - return self.tokenizer.decode(token_ids, **kwargs) - - def decode_with_timestamps(self, tokens) -> str: - """ - Timestamp tokens are above the special tokens' id range and are ignored by `decode()`. - This method decodes given tokens with timestamps tokens annotated, e.g. "<|1.08|>". - """ - outputs = [[]] - for token in tokens: - if token >= self.timestamp_begin: - timestamp = f"<|{(token - self.timestamp_begin) * 0.02:.2f}|>" - outputs.append(timestamp) - outputs.append([]) - else: - outputs[-1].append(token) - outputs = [ - s if isinstance(s, str) else self.tokenizer.decode(s) for s in outputs - ] - return "".join(outputs) - - @property - @lru_cache() - def eot(self) -> int: - return self.tokenizer.eos_token_id - - @property - @lru_cache() - def sot(self) -> int: - return self._get_single_token_id("<|startoftranscript|>") - - @property - @lru_cache() - def sot_lm(self) -> int: - return self._get_single_token_id("<|startoflm|>") - - @property - @lru_cache() - def sot_prev(self) -> int: - return self._get_single_token_id("<|startofprev|>") - - @property - @lru_cache() - def no_speech(self) -> int: - return self._get_single_token_id("<|nospeech|>") - - @property - @lru_cache() - def no_timestamps(self) -> int: - return self._get_single_token_id("<|notimestamps|>") - - @property - @lru_cache() - def timestamp_begin(self) -> int: - return self.tokenizer.all_special_ids[-1] + 1 - - @property - @lru_cache() - def language_token(self) -> int: - """Returns the token id corresponding to the value of the `language` field""" - if self.language is None: - raise ValueError(f"This tokenizer does not have language token configured") - - additional_tokens = dict( - zip( - self.tokenizer.additional_special_tokens, - self.tokenizer.additional_special_tokens_ids, - ) - ) - candidate = f"<|{self.language}|>" - if candidate in additional_tokens: - return additional_tokens[candidate] - - raise KeyError(f"Language {self.language} not found in tokenizer.") - - @property - @lru_cache() - def all_language_tokens(self) -> Tuple[int]: - result = [] - for token, token_id in zip( - self.tokenizer.additional_special_tokens, - self.tokenizer.additional_special_tokens_ids, - ): - if token.strip("<|>") in LANGUAGES: - result.append(token_id) - return tuple(result) - - @property - @lru_cache() - def all_language_codes(self) -> Tuple[str]: - return tuple(self.decode([l]).strip("<|>") for l in self.all_language_tokens) - - @property - @lru_cache() - def sot_sequence_including_notimestamps(self) -> Tuple[int]: - return tuple(list(self.sot_sequence) + [self.no_timestamps]) - - @property - @lru_cache() - def non_speech_tokens(self) -> Tuple[int]: - """ - Returns the list of tokens to suppress in order to avoid any speaker tags or non-speech - annotations, to prevent sampling texts that are not actually spoken in the audio, e.g. - - - ♪♪♪ - - ( SPEAKING FOREIGN LANGUAGE ) - - [DAVID] Hey there, - - keeping basic punctuations like commas, periods, question marks, exclamation points, etc. - """ - symbols = list('"#()*+/:;<=>@[\\]^_`{|}~「」『』') - symbols += ( - "<< >> <<< >>> -- --- -( -[ (' (\" (( )) ((( ))) [[ ]] {{ }} ♪♪ ♪♪♪".split() - ) - - # symbols that may be a single token or multiple tokens depending on the tokenizer. - # In case they're multiple tokens, suppress the first token, which is safe because: - # These are between U+2640 and U+267F miscellaneous symbols that are okay to suppress - # in generations, and in the 3-byte UTF-8 representation they share the first two bytes. - miscellaneous = set("♩♪♫♬♭♮♯") - assert all(0x2640 <= ord(c) <= 0x267F for c in miscellaneous) - - # allow hyphens "-" and single quotes "'" between words, but not at the beginning of a word - result = {self.tokenizer.encode(" -")[0], self.tokenizer.encode(" '")[0]} - for symbol in symbols + list(miscellaneous): - for tokens in [ - self.tokenizer.encode(symbol), - self.tokenizer.encode(" " + symbol), - ]: - if len(tokens) == 1 or symbol in miscellaneous: - result.add(tokens[0]) - - return tuple(sorted(result)) - - def _get_single_token_id(self, text) -> int: - tokens = self.tokenizer.encode(text) - assert len(tokens) == 1, f"{text} is not encoded as a single token" - return tokens[0] - - -@lru_cache(maxsize=None) -def build_tokenizer(name: str = "gpt2", resource_path: str = None): - os.environ["TOKENIZERS_PARALLELISM"] = "false" - if resource_path is not None: - path = os.path.join(resource_path, name) - else: - path = os.path.join(os.path.dirname(__file__), "assets", name) - tokenizer = GPT2TokenizerFast.from_pretrained(path) - - specials = [ - "<|startoftranscript|>", - *[f"<|{lang}|>" for lang in LANGUAGES.keys()], - "<|translate|>", - "<|transcribe|>", - "<|startoflm|>", - "<|startofprev|>", - "<|nospeech|>", - "<|notimestamps|>", - ] - - tokenizer.add_special_tokens(dict(additional_special_tokens=specials)) - return tokenizer - - -@lru_cache(maxsize=None) -def get_tokenizer( - multilingual: bool, - *, - task: Optional[str] = None, # Literal["transcribe", "translate", None] - language: Optional[str] = None, -) -> Tokenizer: - if language is not None: - language = language.lower() - if language not in LANGUAGES: - if language in TO_LANGUAGE_CODE: - language = TO_LANGUAGE_CODE[language] - else: - raise ValueError(f"Unsupported language: {language}") - - if multilingual: - tokenizer_name = "multilingual" - task = task or "transcribe" - language = language or "en" - else: - tokenizer_name = "gpt2" - task = None - language = None - - tokenizer = build_tokenizer(name=tokenizer_name) - all_special_ids: List[int] = tokenizer.all_special_ids - sot: int = all_special_ids[1] - translate: int = all_special_ids[-6] - transcribe: int = all_special_ids[-5] - - langs = tuple(LANGUAGES.keys()) - sot_sequence = [sot] - if language is not None: - sot_sequence.append(sot + 1 + langs.index(language)) - if task is not None: - sot_sequence.append(transcribe if task == "transcribe" else translate) - - return Tokenizer( - tokenizer=tokenizer, language=language, sot_sequence=tuple(sot_sequence) - ) diff --git a/funasr_detach/models/whisper/utils/transcribe.py b/funasr_detach/models/whisper/utils/transcribe.py deleted file mode 100644 index c993b4ad5bbc42c919406c6c8b82e30287c7fe41..0000000000000000000000000000000000000000 --- a/funasr_detach/models/whisper/utils/transcribe.py +++ /dev/null @@ -1,494 +0,0 @@ -import argparse -import os -import warnings -from typing import Optional, Tuple, Union, TYPE_CHECKING - -import numpy as np -import torch -import tqdm - -from funasr_detach.models.whisper.utils.audio import ( - SAMPLE_RATE, - N_FRAMES, - HOP_LENGTH, - pad_or_trim, - log_mel_spectrogram, -) -from funasr_detach.models.whisper.utils.decoding import DecodingOptions, DecodingResult -from funasr_detach.models.whisper.utils.tokenizer import ( - LANGUAGES, - TO_LANGUAGE_CODE, - get_tokenizer, -) -from funasr_detach.models.whisper.utils.utils import ( - exact_div, - format_timestamp, - make_safe, - optional_int, - optional_float, - str2bool, - get_writer, -) - -if TYPE_CHECKING: - from .model import Whisper - - -def transcribe( - model: "Whisper", - audio: Union[str, np.ndarray, torch.Tensor], - *, - verbose: Optional[bool] = None, - temperature: Union[float, Tuple[float, ...]] = (0.0, 0.2, 0.4, 0.6, 0.8, 1.0), - compression_ratio_threshold: Optional[float] = 2.4, - logprob_threshold: Optional[float] = -1.0, - no_speech_threshold: Optional[float] = 0.6, - condition_on_previous_text: bool = True, - **decode_options, -): - """ - Transcribe an audio file using Whisper - - Parameters - ---------- - model: Whisper - The Whisper model instance - - audio: Union[str, np.ndarray, torch.Tensor] - The path to the audio file to open, or the audio waveform - - verbose: bool - Whether to display the text being decoded to the console. If True, displays all the details, - If False, displays minimal details. If None, does not display anything - - temperature: Union[float, Tuple[float, ...]] - Temperature for sampling. It can be a tuple of temperatures, which will be successively used - upon failures according to either `compression_ratio_threshold` or `logprob_threshold`. - - compression_ratio_threshold: float - If the gzip compression ratio is above this value, treat as failed - - logprob_threshold: float - If the average log probability over sampled tokens is below this value, treat as failed - - no_speech_threshold: float - If the no_speech probability is higher than this value AND the average log probability - over sampled tokens is below `logprob_threshold`, consider the segment as silent - - condition_on_previous_text: bool - if True, the previous output of the model is provided as a prompt for the next window; - disabling may make the text inconsistent across windows, but the model becomes less prone to - getting stuck in a failure loop, such as repetition looping or timestamps going out of sync. - - decode_options: dict - Keyword arguments to construct `DecodingOptions` instances - - Returns - ------- - A dictionary containing the resulting text ("text") and segment-level details ("segments"), and - the spoken language ("language"), which is detected when `decode_options["language"]` is None. - """ - dtype = torch.float16 if decode_options.get("fp16", True) else torch.float32 - if model.device == torch.device("cpu"): - if torch.cuda.is_available(): - warnings.warn("Performing inference on CPU when CUDA is available") - if dtype == torch.float16: - warnings.warn("FP16 is not supported on CPU; using FP32 instead") - dtype = torch.float32 - - if dtype == torch.float32: - decode_options["fp16"] = False - - mel = log_mel_spectrogram(audio) - - if decode_options.get("language", None) is None: - if not model.is_multilingual: - decode_options["language"] = "en" - else: - if verbose: - print( - "Detecting language using up to the first 30 seconds. Use `--language` to specify the language" - ) - segment = pad_or_trim(mel, N_FRAMES).to(model.device).to(dtype) - _, probs = model.detect_language(segment) - decode_options["language"] = max(probs, key=probs.get) - if verbose is not None: - print( - f"Detected language: {LANGUAGES[decode_options['language']].title()}" - ) - - language = decode_options["language"] - task = decode_options.get("task", "transcribe") - tokenizer = get_tokenizer(model.is_multilingual, language=language, task=task) - - def decode_with_fallback(segment: torch.Tensor) -> DecodingResult: - temperatures = ( - [temperature] if isinstance(temperature, (int, float)) else temperature - ) - decode_result = None - - for t in temperatures: - kwargs = {**decode_options} - if t > 0: - # disable beam_size and patience when t > 0 - kwargs.pop("beam_size", None) - kwargs.pop("patience", None) - else: - # disable best_of when t == 0 - kwargs.pop("best_of", None) - - options = DecodingOptions(**kwargs, temperature=t) - decode_result = model.decode(segment, options) - - needs_fallback = False - if ( - compression_ratio_threshold is not None - and decode_result.compression_ratio > compression_ratio_threshold - ): - needs_fallback = True # too repetitive - if ( - logprob_threshold is not None - and decode_result.avg_logprob < logprob_threshold - ): - needs_fallback = True # average log probability is too low - - if not needs_fallback: - break - - return decode_result - - seek = 0 - input_stride = exact_div( - N_FRAMES, model.dims.n_audio_ctx - ) # mel frames per output token: 2 - time_precision = ( - input_stride * HOP_LENGTH / SAMPLE_RATE - ) # time per output token: 0.02 (seconds) - all_tokens = [] - all_segments = [] - prompt_reset_since = 0 - - initial_prompt = decode_options.pop("initial_prompt", None) or [] - if initial_prompt: - initial_prompt = tokenizer.encode(" " + initial_prompt.strip()) - all_tokens.extend(initial_prompt) - - def add_segment( - *, start: float, end: float, text_tokens: torch.Tensor, result: DecodingResult - ): - text = tokenizer.decode( - [token for token in text_tokens if token < tokenizer.eot] - ) - if len(text.strip()) == 0: # skip empty text output - return - - all_segments.append( - { - "id": len(all_segments), - "seek": seek, - "start": start, - "end": end, - "text": text, - "tokens": text_tokens.tolist(), - "temperature": result.temperature, - "avg_logprob": result.avg_logprob, - "compression_ratio": result.compression_ratio, - "no_speech_prob": result.no_speech_prob, - } - ) - if verbose: - print( - make_safe( - f"[{format_timestamp(start)} --> {format_timestamp(end)}] {text}" - ) - ) - - # show the progress bar when verbose is False (otherwise the transcribed text will be printed) - num_frames = mel.shape[-1] - previous_seek_value = seek - - with tqdm.tqdm( - total=num_frames, unit="frames", disable=verbose is not False - ) as pbar: - while seek < num_frames: - timestamp_offset = float(seek * HOP_LENGTH / SAMPLE_RATE) - segment = pad_or_trim(mel[:, seek:], N_FRAMES).to(model.device).to(dtype) - segment_duration = segment.shape[-1] * HOP_LENGTH / SAMPLE_RATE - - decode_options["prompt"] = all_tokens[prompt_reset_since:] - result: DecodingResult = decode_with_fallback(segment) - tokens = torch.tensor(result.tokens) - - if no_speech_threshold is not None: - # no voice activity check - should_skip = result.no_speech_prob > no_speech_threshold - if ( - logprob_threshold is not None - and result.avg_logprob > logprob_threshold - ): - # don't skip if the logprob is high enough, despite the no_speech_prob - should_skip = False - - if should_skip: - seek += segment.shape[ - -1 - ] # fast-forward to the next segment boundary - continue - - timestamp_tokens: torch.Tensor = tokens.ge(tokenizer.timestamp_begin) - consecutive = torch.where(timestamp_tokens[:-1] & timestamp_tokens[1:])[ - 0 - ].add_(1) - if ( - len(consecutive) > 0 - ): # if the output contains two consecutive timestamp tokens - last_slice = 0 - for current_slice in consecutive: - sliced_tokens = tokens[last_slice:current_slice] - start_timestamp_position = ( - sliced_tokens[0].item() - tokenizer.timestamp_begin - ) - end_timestamp_position = ( - sliced_tokens[-1].item() - tokenizer.timestamp_begin - ) - add_segment( - start=timestamp_offset - + start_timestamp_position * time_precision, - end=timestamp_offset + end_timestamp_position * time_precision, - text_tokens=sliced_tokens[1:-1], - result=result, - ) - last_slice = current_slice - last_timestamp_position = ( - tokens[last_slice - 1].item() - tokenizer.timestamp_begin - ) - seek += last_timestamp_position * input_stride - all_tokens.extend(tokens[: last_slice + 1].tolist()) - else: - duration = segment_duration - timestamps = tokens[timestamp_tokens.nonzero().flatten()] - if ( - len(timestamps) > 0 - and timestamps[-1].item() != tokenizer.timestamp_begin - ): - # no consecutive timestamps but it has a timestamp; use the last one. - # single timestamp at the end means no speech after the last timestamp. - last_timestamp_position = ( - timestamps[-1].item() - tokenizer.timestamp_begin - ) - duration = last_timestamp_position * time_precision - - add_segment( - start=timestamp_offset, - end=timestamp_offset + duration, - text_tokens=tokens, - result=result, - ) - - seek += segment.shape[-1] - all_tokens.extend(tokens.tolist()) - - if not condition_on_previous_text or result.temperature > 0.5: - # do not feed the prompt tokens if a high temperature was used - prompt_reset_since = len(all_tokens) - - # update progress bar - pbar.update(min(num_frames, seek) - previous_seek_value) - previous_seek_value = seek - - return dict( - text=tokenizer.decode(all_tokens[len(initial_prompt) :]), - segments=all_segments, - language=language, - ) - - -def cli(): - from . import available_models - - parser = argparse.ArgumentParser( - formatter_class=argparse.ArgumentDefaultsHelpFormatter - ) - parser.add_argument( - "audio", nargs="+", type=str, help="audio file(s) to transcribe" - ) - parser.add_argument( - "--model", - default="small", - choices=available_models(), - help="name of the Whisper model to use", - ) - parser.add_argument( - "--model_dir", - type=str, - default=None, - help="the path to save model files; uses ~/.cache/whisper by default", - ) - parser.add_argument( - "--device", - default="cuda" if torch.cuda.is_available() else "cpu", - help="device to use for PyTorch inference", - ) - parser.add_argument( - "--output_dir", - "-o", - type=str, - default=".", - help="directory to save the outputs", - ) - parser.add_argument( - "--output_format", - "-f", - type=str, - default="all", - choices=["txt", "vtt", "srt", "tsv", "json", "all"], - help="format of the output file; if not specified, all available formats will be produced", - ) - parser.add_argument( - "--verbose", - type=str2bool, - default=True, - help="whether to print out the progress and debug messages", - ) - - parser.add_argument( - "--task", - type=str, - default="transcribe", - choices=["transcribe", "translate"], - help="whether to perform X->X speech recognition ('transcribe') or X->English translation ('translate')", - ) - parser.add_argument( - "--language", - type=str, - default=None, - choices=sorted(LANGUAGES.keys()) - + sorted([k.title() for k in TO_LANGUAGE_CODE.keys()]), - help="language spoken in the audio, specify None to perform language detection", - ) - - parser.add_argument( - "--temperature", type=float, default=0, help="temperature to use for sampling" - ) - parser.add_argument( - "--best_of", - type=optional_int, - default=5, - help="number of candidates when sampling with non-zero temperature", - ) - parser.add_argument( - "--beam_size", - type=optional_int, - default=5, - help="number of beams in beam search, only applicable when temperature is zero", - ) - parser.add_argument( - "--patience", - type=float, - default=None, - help="optional patience value to use in beam decoding, as in https://arxiv.org/abs/2204.05424, the default (1.0) is equivalent to conventional beam search", - ) - parser.add_argument( - "--length_penalty", - type=float, - default=None, - help="optional token length penalty coefficient (alpha) as in https://arxiv.org/abs/1609.08144, uses simple length normalization by default", - ) - - parser.add_argument( - "--suppress_tokens", - type=str, - default="-1", - help="comma-separated list of token ids to suppress during sampling; '-1' will suppress most special characters except common punctuations", - ) - parser.add_argument( - "--initial_prompt", - type=str, - default=None, - help="optional text to provide as a prompt for the first window.", - ) - parser.add_argument( - "--condition_on_previous_text", - type=str2bool, - default=True, - help="if True, provide the previous output of the model as a prompt for the next window; disabling may make the text inconsistent across windows, but the model becomes less prone to getting stuck in a failure loop", - ) - parser.add_argument( - "--fp16", - type=str2bool, - default=True, - help="whether to perform inference in fp16; True by default", - ) - - parser.add_argument( - "--temperature_increment_on_fallback", - type=optional_float, - default=0.2, - help="temperature to increase when falling back when the decoding fails to meet either of the thresholds below", - ) - parser.add_argument( - "--compression_ratio_threshold", - type=optional_float, - default=2.4, - help="if the gzip compression ratio is higher than this value, treat the decoding as failed", - ) - parser.add_argument( - "--logprob_threshold", - type=optional_float, - default=-1.0, - help="if the average log probability is lower than this value, treat the decoding as failed", - ) - parser.add_argument( - "--no_speech_threshold", - type=optional_float, - default=0.6, - help="if the probability of the <|nospeech|> token is higher than this value AND the decoding has failed due to `logprob_threshold`, consider the segment as silence", - ) - parser.add_argument( - "--threads", - type=optional_int, - default=0, - help="number of threads used by torch for CPU inference; supercedes MKL_NUM_THREADS/OMP_NUM_THREADS", - ) - - args = parser.parse_args().__dict__ - model_name: str = args.pop("model") - model_dir: str = args.pop("model_dir") - output_dir: str = args.pop("output_dir") - output_format: str = args.pop("output_format") - device: str = args.pop("device") - os.makedirs(output_dir, exist_ok=True) - - if model_name.endswith(".en") and args["language"] not in {"en", "English"}: - if args["language"] is not None: - warnings.warn( - f"{model_name} is an English-only model but receipted '{args['language']}'; using English instead." - ) - args["language"] = "en" - - temperature = args.pop("temperature") - temperature_increment_on_fallback = args.pop("temperature_increment_on_fallback") - if temperature_increment_on_fallback is not None: - temperature = tuple( - np.arange(temperature, 1.0 + 1e-6, temperature_increment_on_fallback) - ) - else: - temperature = [temperature] - - threads = args.pop("threads") - if threads > 0: - torch.set_num_threads(threads) - - from . import load_model - - model = load_model(model_name, device=device, download_root=model_dir) - - writer = get_writer(output_format, output_dir) - - for audio_path in args.pop("audio"): - result = transcribe(model, audio_path, temperature=temperature, **args) - writer(result, audio_path) - - -if __name__ == "__main__": - cli() diff --git a/funasr_detach/models/whisper/utils/utils.py b/funasr_detach/models/whisper/utils/utils.py deleted file mode 100644 index e9c2c712217a44a1a343aa02710b64478e0cd9cb..0000000000000000000000000000000000000000 --- a/funasr_detach/models/whisper/utils/utils.py +++ /dev/null @@ -1,172 +0,0 @@ -import json -import os -import sys -import zlib -from typing import Callable, TextIO - -system_encoding = sys.getdefaultencoding() - -if system_encoding != "utf-8": - - def make_safe(string): - # replaces any character not representable using the system default encoding with an '?', - # avoiding UnicodeEncodeError (https://github.com/openai/whisper/discussions/729). - return string.encode(system_encoding, errors="replace").decode(system_encoding) - -else: - - def make_safe(string): - # utf-8 can encode any Unicode code point, so no need to do the round-trip encoding - return string - - -def exact_div(x, y): - assert x % y == 0 - return x // y - - -def str2bool(string): - str2val = {"True": True, "False": False} - if string in str2val: - return str2val[string] - else: - raise ValueError(f"Expected one of {set(str2val.keys())}, got {string}") - - -def optional_int(string): - return None if string == "None" else int(string) - - -def optional_float(string): - return None if string == "None" else float(string) - - -def compression_ratio(text) -> float: - text_bytes = text.encode("utf-8") - return len(text_bytes) / len(zlib.compress(text_bytes)) - - -def format_timestamp( - seconds: float, always_include_hours: bool = False, decimal_marker: str = "." -): - assert seconds >= 0, "non-negative timestamp expected" - milliseconds = round(seconds * 1000.0) - - hours = milliseconds // 3_600_000 - milliseconds -= hours * 3_600_000 - - minutes = milliseconds // 60_000 - milliseconds -= minutes * 60_000 - - seconds = milliseconds // 1_000 - milliseconds -= seconds * 1_000 - - hours_marker = f"{hours:02d}:" if always_include_hours or hours > 0 else "" - return ( - f"{hours_marker}{minutes:02d}:{seconds:02d}{decimal_marker}{milliseconds:03d}" - ) - - -class ResultWriter: - extension: str - - def __init__(self, output_dir: str): - self.output_dir = output_dir - - def __call__(self, result: dict, audio_path: str): - audio_basename = os.path.basename(audio_path) - output_path = os.path.join( - self.output_dir, audio_basename + "." + self.extension - ) - - with open(output_path, "w", encoding="utf-8") as f: - self.write_result(result, file=f) - - def write_result(self, result: dict, file: TextIO): - raise NotImplementedError - - -class WriteTXT(ResultWriter): - extension: str = "txt" - - def write_result(self, result: dict, file: TextIO): - for segment in result["segments"]: - print(segment["text"].strip(), file=file, flush=True) - - -class WriteVTT(ResultWriter): - extension: str = "vtt" - - def write_result(self, result: dict, file: TextIO): - print("WEBVTT\n", file=file) - for segment in result["segments"]: - print( - f"{format_timestamp(segment['start'])} --> {format_timestamp(segment['end'])}\n" - f"{segment['text'].strip().replace('-->', '->')}\n", - file=file, - flush=True, - ) - - -class WriteSRT(ResultWriter): - extension: str = "srt" - - def write_result(self, result: dict, file: TextIO): - for i, segment in enumerate(result["segments"], start=1): - # write srt lines - print( - f"{i}\n" - f"{format_timestamp(segment['start'], always_include_hours=True, decimal_marker=',')} --> " - f"{format_timestamp(segment['end'], always_include_hours=True, decimal_marker=',')}\n" - f"{segment['text'].strip().replace('-->', '->')}\n", - file=file, - flush=True, - ) - - -class WriteTSV(ResultWriter): - """ - Write a transcript to a file in TSV (tab-separated values) format containing lines like: - \t\t - - Using integer milliseconds as start and end times means there's no chance of interference from - an environment setting a language encoding that causes the decimal in a floating point number - to appear as a comma; also is faster and more efficient to parse & store, e.g., in C++. - """ - - extension: str = "tsv" - - def write_result(self, result: dict, file: TextIO): - print("start", "end", "text", sep="\t", file=file) - for segment in result["segments"]: - print(round(1000 * segment["start"]), file=file, end="\t") - print(round(1000 * segment["end"]), file=file, end="\t") - print(segment["text"].strip().replace("\t", " "), file=file, flush=True) - - -class WriteJSON(ResultWriter): - extension: str = "json" - - def write_result(self, result: dict, file: TextIO): - json.dump(result, file) - - -def get_writer(output_format: str, output_dir: str) -> Callable[[dict, TextIO], None]: - writers = { - "txt": WriteTXT, - "vtt": WriteVTT, - "srt": WriteSRT, - "tsv": WriteTSV, - "json": WriteJSON, - } - - if output_format == "all": - all_writers = [writer(output_dir) for writer in writers.values()] - - def write_all(result: dict, file: TextIO): - for writer in all_writers: - writer(result, file) - - return write_all - - return writers[output_format](output_dir) diff --git a/funasr_detach/models/xvector/__init__.py b/funasr_detach/models/xvector/__init__.py deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/funasr_detach/models/xvector/e2e_sv.py b/funasr_detach/models/xvector/e2e_sv.py deleted file mode 100644 index a24f10c93b86122d9b1fe940315ebb75c6634086..0000000000000000000000000000000000000000 --- a/funasr_detach/models/xvector/e2e_sv.py +++ /dev/null @@ -1,270 +0,0 @@ -""" -Author: Speech Lab, Alibaba Group, China -""" - -import logging -from contextlib import contextmanager -from distutils.version import LooseVersion -from typing import Dict -from typing import List -from typing import Optional -from typing import Tuple -from typing import Union - -import torch - -from funasr_detach.layers.abs_normalize import AbsNormalize -from funasr_detach.losses.label_smoothing_loss import ( - LabelSmoothingLoss, # noqa: H301 -) -from funasr_detach.models.ctc import CTC -from funasr_detach.models.decoder.abs_decoder import AbsDecoder -from funasr_detach.models.encoder.abs_encoder import AbsEncoder -from funasr_detach.frontends.abs_frontend import AbsFrontend -from funasr_detach.models.postencoder.abs_postencoder import AbsPostEncoder -from funasr_detach.models.preencoder.abs_preencoder import AbsPreEncoder -from funasr_detach.models.specaug.abs_specaug import AbsSpecAug -from funasr_detach.models.transformer.utils.add_sos_eos import add_sos_eos -from funasr_detach.metrics import ErrorCalculator -from funasr_detach.metrics.compute_acc import th_accuracy -from funasr_detach.train_utils.device_funcs import force_gatherable -from funasr_detach.models.base_model import FunASRModel - -if LooseVersion(torch.__version__) >= LooseVersion("1.6.0"): - from torch.cuda.amp import autocast -else: - # Nothing to do if torch<1.6.0 - @contextmanager - def autocast(enabled=True): - yield - - -class ESPnetSVModel(FunASRModel): - """CTC-attention hybrid Encoder-Decoder model""" - - def __init__( - self, - vocab_size: int, - token_list: Union[Tuple[str, ...], List[str]], - frontend: Optional[AbsFrontend], - specaug: Optional[AbsSpecAug], - normalize: Optional[AbsNormalize], - preencoder: Optional[AbsPreEncoder], - encoder: AbsEncoder, - postencoder: Optional[AbsPostEncoder], - pooling_layer: torch.nn.Module, - decoder: AbsDecoder, - ): - - super().__init__() - # note that eos is the same as sos (equivalent ID) - self.vocab_size = vocab_size - self.token_list = token_list.copy() - - self.frontend = frontend - self.specaug = specaug - self.normalize = normalize - self.preencoder = preencoder - self.postencoder = postencoder - self.encoder = encoder - self.pooling_layer = pooling_layer - self.decoder = decoder - - def forward( - self, - speech: torch.Tensor, - speech_lengths: torch.Tensor, - text: torch.Tensor, - text_lengths: torch.Tensor, - ) -> Tuple[torch.Tensor, Dict[str, torch.Tensor], torch.Tensor]: - """Frontend + Encoder + Decoder + Calc loss - Args: - speech: (Batch, Length, ...) - speech_lengths: (Batch, ) - text: (Batch, Length) - text_lengths: (Batch,) - """ - assert text_lengths.dim() == 1, text_lengths.shape - # Check that batch_size is unified - assert ( - speech.shape[0] - == speech_lengths.shape[0] - == text.shape[0] - == text_lengths.shape[0] - ), (speech.shape, speech_lengths.shape, text.shape, text_lengths.shape) - batch_size = speech.shape[0] - - # for data-parallel - text = text[:, : text_lengths.max()] - - # 1. Encoder - encoder_out, encoder_out_lens = self.encode(speech, speech_lengths) - intermediate_outs = None - if isinstance(encoder_out, tuple): - intermediate_outs = encoder_out[1] - encoder_out = encoder_out[0] - - loss_att, acc_att, cer_att, wer_att = None, None, None, None - loss_ctc, cer_ctc = None, None - loss_transducer, cer_transducer, wer_transducer = None, None, None - stats = dict() - - # 1. CTC branch - if self.ctc_weight != 0.0: - loss_ctc, cer_ctc = self._calc_ctc_loss( - encoder_out, encoder_out_lens, text, text_lengths - ) - - # Collect CTC branch stats - stats["loss_ctc"] = loss_ctc.detach() if loss_ctc is not None else None - stats["cer_ctc"] = cer_ctc - - # Intermediate CTC (optional) - loss_interctc = 0.0 - if self.interctc_weight != 0.0 and intermediate_outs is not None: - for layer_idx, intermediate_out in intermediate_outs: - # we assume intermediate_out has the same length & padding - # as those of encoder_out - loss_ic, cer_ic = self._calc_ctc_loss( - intermediate_out, encoder_out_lens, text, text_lengths - ) - loss_interctc = loss_interctc + loss_ic - - # Collect Intermedaite CTC stats - stats["loss_interctc_layer{}".format(layer_idx)] = ( - loss_ic.detach() if loss_ic is not None else None - ) - stats["cer_interctc_layer{}".format(layer_idx)] = cer_ic - - loss_interctc = loss_interctc / len(intermediate_outs) - - # calculate whole encoder loss - loss_ctc = ( - 1 - self.interctc_weight - ) * loss_ctc + self.interctc_weight * loss_interctc - - if self.use_transducer_decoder: - # 2a. Transducer decoder branch - ( - loss_transducer, - cer_transducer, - wer_transducer, - ) = self._calc_transducer_loss( - encoder_out, - encoder_out_lens, - text, - ) - - if loss_ctc is not None: - loss = loss_transducer + (self.ctc_weight * loss_ctc) - else: - loss = loss_transducer - - # Collect Transducer branch stats - stats["loss_transducer"] = ( - loss_transducer.detach() if loss_transducer is not None else None - ) - stats["cer_transducer"] = cer_transducer - stats["wer_transducer"] = wer_transducer - - else: - # 2b. Attention decoder branch - if self.ctc_weight != 1.0: - loss_att, acc_att, cer_att, wer_att = self._calc_att_loss( - encoder_out, encoder_out_lens, text, text_lengths - ) - - # 3. CTC-Att loss definition - if self.ctc_weight == 0.0: - loss = loss_att - elif self.ctc_weight == 1.0: - loss = loss_ctc - else: - loss = self.ctc_weight * loss_ctc + (1 - self.ctc_weight) * loss_att - - # Collect Attn branch stats - stats["loss_att"] = loss_att.detach() if loss_att is not None else None - stats["acc"] = acc_att - stats["cer"] = cer_att - stats["wer"] = wer_att - - # Collect total loss stats - stats["loss"] = torch.clone(loss.detach()) - - # force_gatherable: to-device and to-tensor if scalar for DataParallel - loss, stats, weight = force_gatherable((loss, stats, batch_size), loss.device) - return loss, stats, weight - - def collect_feats( - self, - speech: torch.Tensor, - speech_lengths: torch.Tensor, - text: torch.Tensor, - text_lengths: torch.Tensor, - ) -> Dict[str, torch.Tensor]: - if self.extract_feats_in_collect_stats: - feats, feats_lengths = self._extract_feats(speech, speech_lengths) - else: - # Generate dummy stats if extract_feats_in_collect_stats is False - logging.warning( - "Generating dummy stats for feats and feats_lengths, " - "because encoder_conf.extract_feats_in_collect_stats is " - f"{self.extract_feats_in_collect_stats}" - ) - feats, feats_lengths = speech, speech_lengths - return {"feats": feats, "feats_lengths": feats_lengths} - - def encode( - self, speech: torch.Tensor, speech_lengths: torch.Tensor - ) -> Tuple[torch.Tensor, torch.Tensor]: - """Frontend + Encoder. Note that this method is used by asr_inference.py - Args: - speech: (Batch, Length, ...) - speech_lengths: (Batch, ) - """ - with autocast(False): - # 1. Extract feats - feats, feats_lengths = self._extract_feats(speech, speech_lengths) - - # 2. Data augmentation - if self.specaug is not None and self.training: - feats, feats_lengths = self.specaug(feats, feats_lengths) - - # 3. Normalization for feature: e.g. Global-CMVN, Utterance-CMVN - if self.normalize is not None: - feats, feats_lengths = self.normalize(feats, feats_lengths) - - # Pre-encoder, e.g. used for raw input data - if self.preencoder is not None: - feats, feats_lengths = self.preencoder(feats, feats_lengths) - - # 4. Forward encoder - # feats: (Batch, Length, Dim) -> (Batch, Channel, Length2, Dim2) - encoder_out, encoder_out_lens = self.encoder(feats, feats_lengths) - - # Post-encoder, e.g. NLU - if self.postencoder is not None: - encoder_out, encoder_out_lens = self.postencoder( - encoder_out, encoder_out_lens - ) - - return encoder_out, encoder_out_lens - - def _extract_feats( - self, speech: torch.Tensor, speech_lengths: torch.Tensor - ) -> Tuple[torch.Tensor, torch.Tensor]: - assert speech_lengths.dim() == 1, speech_lengths.shape - - # for data-parallel - speech = speech[:, : speech_lengths.max()] - - if self.frontend is not None: - # Frontend - # e.g. STFT and Feature extract - # data_loader may send time-domain signal in this case - # speech (Batch, NSamples) -> feats: (Batch, NFrames, Dim) - feats, feats_lengths = self.frontend(speech, speech_lengths) - else: - # No frontend and no feature extract - feats, feats_lengths = speech, speech_lengths - return feats, feats_lengths diff --git a/funasr_detach/optimizers/__init__.py b/funasr_detach/optimizers/__init__.py deleted file mode 100644 index 2d6bd7e0b17a5eebc6945cdde8b86652d39a3156..0000000000000000000000000000000000000000 --- a/funasr_detach/optimizers/__init__.py +++ /dev/null @@ -1,17 +0,0 @@ -import torch -from funasr_detach.optimizers.fairseq_adam import FairseqAdam -from funasr_detach.optimizers.sgd import SGD - -optim_classes = dict( - adam=torch.optim.Adam, - fairseq_adam=FairseqAdam, - adamw=torch.optim.AdamW, - sgd=SGD, - adadelta=torch.optim.Adadelta, - adagrad=torch.optim.Adagrad, - adamax=torch.optim.Adamax, - asgd=torch.optim.ASGD, - lbfgs=torch.optim.LBFGS, - rmsprop=torch.optim.RMSprop, - rprop=torch.optim.Rprop, -) diff --git a/funasr_detach/optimizers/fairseq_adam.py b/funasr_detach/optimizers/fairseq_adam.py deleted file mode 100644 index 1d686aaf5d2706ed73f4e0baacdf98b52e949e06..0000000000000000000000000000000000000000 --- a/funasr_detach/optimizers/fairseq_adam.py +++ /dev/null @@ -1,152 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - - -import math - -import torch -import torch.optim - - -class FairseqAdam(torch.optim.Optimizer): - r"""Implements Adam algorithm. - - This implementation is modified from torch.optim.Adam based on: - `Fixed Weight Decay Regularization in Adam` - (see https://arxiv.org/abs/1711.05101) - - It has been proposed in `Adam: A Method for Stochastic Optimization`_. - - Args: - params (iterable): iterable of parameters to optimize or dicts defining - parameter groups - lr (float, optional): learning rate (default: 1e-3) - betas (Tuple[float, float], optional): coefficients used for computing - running averages of gradient and its square (default: (0.9, 0.999)) - eps (float, optional): term added to the denominator to improve - numerical stability (default: 1e-8) - weight_decay (float, optional): weight decay (L2 penalty) (default: 0) - amsgrad (boolean, optional): whether to use the AMSGrad variant of this - algorithm from the paper `On the Convergence of Adam and Beyond`_ - - .. _Adam\: A Method for Stochastic Optimization: - https://arxiv.org/abs/1412.6980 - .. _On the Convergence of Adam and Beyond: - https://openreview.net/forum?id=ryQu7f-RZ - """ - - def __init__( - self, - params, - lr=1e-3, - adam_betas=(0.9, 0.999), - adam_eps=1e-8, - weight_decay=0, - amsgrad=False, - ): - defaults = dict( - lr=lr, - betas=adam_betas, - eps=adam_eps, - weight_decay=weight_decay, - amsgrad=amsgrad, - ) - super(FairseqAdam, self).__init__(params, defaults) - self.optimizer_lr = lr - - @property - def supports_memory_efficient_fp16(self): - return True - - @property - def supports_flat_params(self): - return True - - def step(self, closure=None): - """Performs a single optimization step. - - Args: - closure (callable, optional): A closure that reevaluates the model - and returns the loss. - """ - loss = None - if closure is not None: - loss = closure() - - for group in self.param_groups: - for p in group["params"]: - if p.grad is None: - continue - grad = p.grad.data - if grad.dtype in {torch.float16, torch.bfloat16}: - grad = grad.float() - if grad.is_sparse: - raise RuntimeError( - "Adam does not support sparse gradients, please consider SparseAdam instead" - ) - amsgrad = group.get("amsgrad", False) - - p_data_fp32 = p.data - if p.data.dtype in {torch.float16, torch.bfloat16}: - p_data_fp32 = p_data_fp32.float() - - state = self.state[p] - - # State initialization - if len(state) == 0: - state["step"] = 0 - # Exponential moving average of gradient values - state["exp_avg"] = torch.zeros_like(p_data_fp32) - # Exponential moving average of squared gradient values - state["exp_avg_sq"] = torch.zeros_like(p_data_fp32) - if amsgrad: - # Maintains max of all exp. moving avg. of sq. grad. values - state["max_exp_avg_sq"] = torch.zeros_like(p_data_fp32) - else: - state["exp_avg"] = state["exp_avg"].to(p_data_fp32) - state["exp_avg_sq"] = state["exp_avg_sq"].to(p_data_fp32) - if amsgrad: - state["max_exp_avg_sq"] = state["max_exp_avg_sq"].to( - p_data_fp32 - ) - - exp_avg, exp_avg_sq = state["exp_avg"], state["exp_avg_sq"] - if amsgrad: - max_exp_avg_sq = state["max_exp_avg_sq"] - beta1, beta2 = group["betas"] - - state["step"] += 1 - - # Decay the first and second moment running average coefficient - exp_avg.mul_(beta1).add_(grad, alpha=1 - beta1) - exp_avg_sq.mul_(beta2).addcmul_(grad, grad, value=1 - beta2) - if amsgrad: - # Maintains the maximum of all 2nd moment running avg. till now - torch.max(max_exp_avg_sq, exp_avg_sq, out=max_exp_avg_sq) - # Use the max. for normalizing running avg. of gradient - denom = max_exp_avg_sq.sqrt().add_(group["eps"]) - else: - denom = exp_avg_sq.sqrt().add_(group["eps"]) - - bias_correction1 = 1 - beta1 ** state["step"] - bias_correction2 = 1 - beta2 ** state["step"] - step_size = group["lr"] * math.sqrt(bias_correction2) / bias_correction1 - - if group["weight_decay"] != 0: - p_data_fp32.add_( - p_data_fp32, alpha=-group["weight_decay"] * group["lr"] - ) - - p_data_fp32.addcdiv_(exp_avg, denom, value=-step_size) - - if p.data.dtype in {torch.float16, torch.bfloat16}: - p.data.copy_(p_data_fp32) - - return loss - - def set_lr(self, lr): - """Set the learning rate.""" - for param_group in self.param_groups: - param_group["lr"] = lr diff --git a/funasr_detach/optimizers/sgd.py b/funasr_detach/optimizers/sgd.py deleted file mode 100644 index fb7a3df01ed428038361626ab1a8e40405bedb14..0000000000000000000000000000000000000000 --- a/funasr_detach/optimizers/sgd.py +++ /dev/null @@ -1,30 +0,0 @@ -import torch - - -class SGD(torch.optim.SGD): - """Thin inheritance of torch.optim.SGD to bind the required arguments, 'lr' - - Note that - the arguments of the optimizer invoked by AbsTask.main() - must have default value except for 'param'. - - I can't understand why only SGD.lr doesn't have the default value. - """ - - def __init__( - self, - params, - lr: float = 0.1, - momentum: float = 0.0, - dampening: float = 0.0, - weight_decay: float = 0.0, - nesterov: bool = False, - ): - super().__init__( - params, - lr=lr, - momentum=momentum, - dampening=dampening, - weight_decay=weight_decay, - nesterov=nesterov, - ) diff --git a/funasr_detach/register.py b/funasr_detach/register.py deleted file mode 100644 index 7280d2d2c63328111ef2203410544ba282103e37..0000000000000000000000000000000000000000 --- a/funasr_detach/register.py +++ /dev/null @@ -1,93 +0,0 @@ -import logging -import inspect -from dataclasses import dataclass -import re - - -@dataclass -class RegisterTables: - model_classes = {} - frontend_classes = {} - specaug_classes = {} - normalize_classes = {} - encoder_classes = {} - decoder_classes = {} - joint_network_classes = {} - predictor_classes = {} - stride_conv_classes = {} - tokenizer_classes = {} - batch_sampler_classes = {} - dataset_classes = {} - index_ds_classes = {} - - def print(self, key=None): - print("\ntables: \n") - fields = vars(self) - for classes_key, classes_dict in fields.items(): - - flag = True - if key is not None: - flag = key in classes_key - if classes_key.endswith("_meta") and flag: - print( - f"----------- ** {classes_key.replace('_meta', '')} ** --------------" - ) - headers = ["class name", "class location"] - metas = [] - for register_key, meta in classes_dict.items(): - metas.append(meta) - metas.sort(key=lambda x: x[0]) - data = [headers] + metas - col_widths = [max(len(str(item)) for item in col) for col in zip(*data)] - - for row in data: - print( - "| " - + " | ".join( - str(item).ljust(width) - for item, width in zip(row, col_widths) - ) - + " |" - ) - print("\n") - - def register(self, register_tables_key: str, key=None): - def decorator(target_class): - - if not hasattr(self, register_tables_key): - setattr(self, register_tables_key, {}) - logging.info( - "new registry table has been added: {}".format(register_tables_key) - ) - - registry = getattr(self, register_tables_key) - registry_key = key if key is not None else target_class.__name__ - - # assert not registry_key in registry, "(key: {} / class: {}) has been registered already，in {}".format( - # registry_key, target_class, register_tables_key) - - registry[registry_key] = target_class - - # meta， headers = ["class name", "register name", "class location"] - register_tables_key_meta = register_tables_key + "_meta" - if not hasattr(self, register_tables_key_meta): - setattr(self, register_tables_key_meta, {}) - registry_meta = getattr(self, register_tables_key_meta) - # doc = target_class.__doc__ - class_file = inspect.getfile(target_class) - class_line = inspect.getsourcelines(target_class)[1] - pattern = r"^.+/funasr/" - class_file = re.sub(pattern, "funasr/", class_file) - meata_data = [f"{target_class.__name__}", f"{class_file}:{class_line}"] - # meata_data = [f"{target_class.__name__}", f"{registry_key}", f"{class_file}:{class_line}"] - registry_meta[registry_key] = meata_data - # print(f"Registering class: {class_file}:{class_line} - {target_class.__name__} as {registry_key}") - return target_class - - return decorator - - -tables = RegisterTables() - -# from funasr_detach.models.paraformer_streaming.model import ParaformerStreaming -import funasr diff --git a/funasr_detach/schedulers/__init__.py b/funasr_detach/schedulers/__init__.py deleted file mode 100644 index 25c2292ad062632f8df6f9aaf38c9f2b88d98cce..0000000000000000000000000000000000000000 --- a/funasr_detach/schedulers/__init__.py +++ /dev/null @@ -1,23 +0,0 @@ -import torch -import torch.multiprocessing -import torch.nn -import torch.optim - -from funasr_detach.schedulers.noam_lr import NoamLR -from funasr_detach.schedulers.tri_stage_scheduler import TriStageLR -from funasr_detach.schedulers.warmup_lr import WarmupLR - -scheduler_classes = dict( - ReduceLROnPlateau=torch.optim.lr_scheduler.ReduceLROnPlateau, - lambdalr=torch.optim.lr_scheduler.LambdaLR, - steplr=torch.optim.lr_scheduler.StepLR, - multisteplr=torch.optim.lr_scheduler.MultiStepLR, - exponentiallr=torch.optim.lr_scheduler.ExponentialLR, - CosineAnnealingLR=torch.optim.lr_scheduler.CosineAnnealingLR, - noamlr=NoamLR, - warmuplr=WarmupLR, - tri_stage=TriStageLR, - cycliclr=torch.optim.lr_scheduler.CyclicLR, - onecyclelr=torch.optim.lr_scheduler.OneCycleLR, - CosineAnnealingWarmRestarts=torch.optim.lr_scheduler.CosineAnnealingWarmRestarts, -) diff --git a/funasr_detach/schedulers/abs_scheduler.py b/funasr_detach/schedulers/abs_scheduler.py deleted file mode 100644 index 7395f259c3e563f560ea99080e00dac61967b421..0000000000000000000000000000000000000000 --- a/funasr_detach/schedulers/abs_scheduler.py +++ /dev/null @@ -1,84 +0,0 @@ -from abc import ABC -from abc import abstractmethod - -import torch.optim.lr_scheduler as L - - -class AbsScheduler(ABC): - @abstractmethod - def step(self, epoch: int = None): - pass - - @abstractmethod - def state_dict(self): - pass - - @abstractmethod - def load_state_dict(self, state): - pass - - -# If you need to define custom scheduler, please inherit these classes -class AbsBatchStepScheduler(AbsScheduler): - @abstractmethod - def step(self, epoch: int = None): - pass - - @abstractmethod - def state_dict(self): - pass - - @abstractmethod - def load_state_dict(self, state): - pass - - -class AbsEpochStepScheduler(AbsScheduler): - @abstractmethod - def step(self, epoch: int = None): - pass - - @abstractmethod - def state_dict(self): - pass - - @abstractmethod - def load_state_dict(self, state): - pass - - -class AbsValEpochStepScheduler(AbsEpochStepScheduler): - @abstractmethod - def step(self, val, epoch: int = None): - pass - - @abstractmethod - def state_dict(self): - pass - - @abstractmethod - def load_state_dict(self, state): - pass - - -# Create alias type to check the type -# Note(kamo): Currently PyTorch doesn't provide the base class -# to judge these classes. -AbsValEpochStepScheduler.register(L.ReduceLROnPlateau) -for s in [ - L.ReduceLROnPlateau, - L.LambdaLR, - L.StepLR, - L.MultiStepLR, - L.MultiStepLR, - L.ExponentialLR, - L.CosineAnnealingLR, -]: - AbsEpochStepScheduler.register(s) - -AbsBatchStepScheduler.register(L.CyclicLR) -for s in [ - L.OneCycleLR, - L.CosineAnnealingWarmRestarts, -]: - AbsBatchStepScheduler.register(s) diff --git a/funasr_detach/schedulers/noam_lr.py b/funasr_detach/schedulers/noam_lr.py deleted file mode 100644 index 975feb61ac2bb3d96cc999c6553a5ae5f870fc36..0000000000000000000000000000000000000000 --- a/funasr_detach/schedulers/noam_lr.py +++ /dev/null @@ -1,64 +0,0 @@ -"""Noam learning rate scheduler module.""" - -from typing import Union -import warnings - -import torch -from torch.optim.lr_scheduler import _LRScheduler - -from funasr_detach.schedulers.abs_scheduler import AbsBatchStepScheduler - - -class NoamLR(_LRScheduler, AbsBatchStepScheduler): - """The LR scheduler proposed by Noam - - Ref: - "Attention Is All You Need", https://arxiv.org/pdf/1706.03762.pdf - - FIXME(kamo): PyTorch doesn't provide _LRScheduler as public class, - thus the behaviour isn't guaranteed at forward PyTorch version. - - NOTE(kamo): The "model_size" in original implementation is derived from - the model, but in this implementation, this parameter is a constant value. - You need to change it if the model is changed. - - """ - - def __init__( - self, - optimizer: torch.optim.Optimizer, - model_size: Union[int, float] = 320, - warmup_steps: Union[int, float] = 25000, - last_epoch: int = -1, - ): - self.model_size = model_size - self.warmup_steps = warmup_steps - - lr = list(optimizer.param_groups)[0]["lr"] - new_lr = self.lr_for_WarmupLR(lr) - warnings.warn( - f"NoamLR is deprecated. " - f"Use WarmupLR(warmup_steps={warmup_steps}) with Optimizer(lr={new_lr})", - ) - - # __init__() must be invoked before setting field - # because step() is also invoked in __init__() - super().__init__(optimizer, last_epoch) - - def lr_for_WarmupLR(self, lr: float) -> float: - return lr / self.model_size**0.5 / self.warmup_steps**0.5 - - def __repr__(self): - return ( - f"{self.__class__.__name__}(model_size={self.model_size}, " - f"warmup_steps={self.warmup_steps})" - ) - - def get_lr(self): - step_num = self.last_epoch + 1 - return [ - lr - * self.model_size**-0.5 - * min(step_num**-0.5, step_num * self.warmup_steps**-1.5) - for lr in self.base_lrs - ] diff --git a/funasr_detach/schedulers/tri_stage_scheduler.py b/funasr_detach/schedulers/tri_stage_scheduler.py deleted file mode 100644 index 86c8a6a82e9d42b5c36f5f296ce2e060f4d6bf20..0000000000000000000000000000000000000000 --- a/funasr_detach/schedulers/tri_stage_scheduler.py +++ /dev/null @@ -1,106 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import math -from typing import Optional, List - -import torch -from torch.optim.lr_scheduler import _LRScheduler - -from funasr_detach.schedulers.abs_scheduler import AbsBatchStepScheduler - - -class TriStageLR(_LRScheduler, AbsBatchStepScheduler): - def __init__( - self, - optimizer: torch.optim.Optimizer, - last_epoch: int = -1, - phase_ratio: Optional[List[float]] = None, - init_lr_scale: float = 0.01, - final_lr_scale: float = 0.01, - ): - self.optimizer = optimizer - self.last_epoch = last_epoch - self.phase_ratio = phase_ratio - self.init_lr_scale = init_lr_scale - self.final_lr_scale = final_lr_scale - self.optimizer_lr = self.optimizer.defaults["lr"] - - def init_tri_stage_scheudler(self, max_update): - self.max_update = max_update - self.peak_lr = self.optimizer_lr - self.init_lr = self.init_lr_scale * self.optimizer_lr - self.final_lr = self.final_lr_scale * self.optimizer_lr - - assert self.max_update > 0 - assert sum(self.phase_ratio) == 1, "phase ratios must add up to 1" - assert len(self.phase_ratio) == 3 - self.warmup_steps = int(self.max_update * self.phase_ratio[0]) - self.hold_steps = int(self.max_update * self.phase_ratio[1]) - self.decay_steps = int(self.max_update * self.phase_ratio[2]) - - self.warmup_rate = ( - (self.peak_lr - self.init_lr) / self.warmup_steps - if self.warmup_steps != 0 - else 0 - ) - self.decay_factor = -math.log(self.final_lr_scale) / self.decay_steps - - # initial learning rate - self.lr = self.init_lr - - # __init__() must be invoked before setting field - # because step() is also invoked in __init__() - self.set_optimizer_lr(self.lr) - super().__init__(self.optimizer, self.last_epoch) - - def _decide_stage(self, update_step): - """ - return stage, and the corresponding steps within the current stage - """ - if update_step < self.warmup_steps: - # warmup state - return 0, update_step - - offset = self.warmup_steps - - if update_step < offset + self.hold_steps: - # hold stage - return 1, update_step - offset - - offset += self.hold_steps - - if update_step <= offset + self.decay_steps: - # decay stage - return 2, update_step - offset - - offset += self.decay_steps - - # still here ? constant lr stage - return 3, update_step - offset - - def step_update(self, num_updates): - """Update the learning rate after each update.""" - stage, steps_in_stage = self._decide_stage(num_updates) - if stage == 0: - self.lr = self.init_lr + self.warmup_rate * steps_in_stage - elif stage == 1: - self.lr = self.peak_lr - elif stage == 2: - self.lr = self.peak_lr * math.exp(-self.decay_factor * steps_in_stage) - elif stage == 3: - self.lr = self.final_lr - else: - raise ValueError("Undefined stage") - self.set_optimizer_lr(self.lr) - - def set_optimizer_lr(self, lr): - for param_group in self.optimizer.param_groups: - param_group["lr"] = lr - - def get_lr(self): - step_num = self.last_epoch + 1 - self.step_update(step_num) - return [self.lr] diff --git a/funasr_detach/schedulers/warmup_lr.py b/funasr_detach/schedulers/warmup_lr.py deleted file mode 100644 index 361a5be12e9d3586117fe9da6a9ea456963e3b88..0000000000000000000000000000000000000000 --- a/funasr_detach/schedulers/warmup_lr.py +++ /dev/null @@ -1,49 +0,0 @@ -"""Warm up learning rate scheduler module.""" - -from typing import Union - -import torch -from torch.optim.lr_scheduler import _LRScheduler - -from funasr_detach.schedulers.abs_scheduler import AbsBatchStepScheduler - - -class WarmupLR(_LRScheduler, AbsBatchStepScheduler): - """The WarmupLR scheduler - - This scheduler is almost same as NoamLR Scheduler except for following difference: - - NoamLR: - lr = optimizer.lr * model_size ** -0.5 - * min(step ** -0.5, step * warmup_step ** -1.5) - WarmupLR: - lr = optimizer.lr * warmup_step ** 0.5 - * min(step ** -0.5, step * warmup_step ** -1.5) - - Note that the maximum lr equals to optimizer.lr in this scheduler. - - """ - - def __init__( - self, - optimizer: torch.optim.Optimizer, - warmup_steps: Union[int, float] = 25000, - last_epoch: int = -1, - ): - self.warmup_steps = warmup_steps - - # __init__() must be invoked before setting field - # because step() is also invoked in __init__() - super().__init__(optimizer, last_epoch) - - def __repr__(self): - return f"{self.__class__.__name__}(warmup_steps={self.warmup_steps})" - - def get_lr(self): - step_num = self.last_epoch + 1 - return [ - lr - * self.warmup_steps**0.5 - * min(step_num**-0.5, step_num * self.warmup_steps**-1.5) - for lr in self.base_lrs - ] diff --git a/funasr_detach/tokenizer/__init__.py b/funasr_detach/tokenizer/__init__.py deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/funasr_detach/tokenizer/abs_tokenizer.py b/funasr_detach/tokenizer/abs_tokenizer.py deleted file mode 100644 index 69cce662807443d8bcbacbafb505b828f6012029..0000000000000000000000000000000000000000 --- a/funasr_detach/tokenizer/abs_tokenizer.py +++ /dev/null @@ -1,95 +0,0 @@ -import json -import numpy as np -from abc import ABC -from pathlib import Path -from abc import abstractmethod -from typing import Union, Iterable, List, Dict - - -class AbsTokenizer(ABC): - @abstractmethod - def text2tokens(self, line: str) -> List[str]: - raise NotImplementedError - - @abstractmethod - def tokens2text(self, tokens: Iterable[str]) -> str: - raise NotImplementedError - - -class BaseTokenizer(ABC): - def __init__( - self, - token_list: Union[Path, str, Iterable[str]] = None, - unk_symbol: str = "", - **kwargs, - ): - - if token_list is not None: - if isinstance(token_list, (Path, str)) and token_list.endswith(".txt"): - token_list = Path(token_list) - self.token_list_repr = str(token_list) - self.token_list: List[str] = [] - - with token_list.open("r", encoding="utf-8") as f: - for idx, line in enumerate(f): - line = line.rstrip() - self.token_list.append(line) - elif isinstance(token_list, (Path, str)) and token_list.endswith(".json"): - token_list = Path(token_list) - self.token_list_repr = str(token_list) - self.token_list: List[str] = [] - - with open(token_list, "r", encoding="utf-8") as f: - self.token_list = json.load(f) - - else: - self.token_list: List[str] = list(token_list) - self.token_list_repr = "" - for i, t in enumerate(self.token_list): - if i == 3: - break - self.token_list_repr += f"{t}, " - self.token_list_repr += f"... (NVocab={(len(self.token_list))})" - - self.token2id: Dict[str, int] = {} - for i, t in enumerate(self.token_list): - if t in self.token2id: - raise RuntimeError(f'Symbol "{t}" is duplicated') - self.token2id[t] = i - - self.unk_symbol = unk_symbol - if self.unk_symbol not in self.token2id: - raise RuntimeError( - f"Unknown symbol '{unk_symbol}' doesn't exist in the token_list" - ) - self.unk_id = self.token2id[self.unk_symbol] - - def encode(self, text): - tokens = self.text2tokens(text) - text_ints = self.tokens2ids(tokens) - - return text_ints - - def decode(self, text_ints): - token = self.ids2tokens(text_ints) - text = self.tokens2text(token) - return text - - def get_num_vocabulary_size(self) -> int: - return len(self.token_list) - - def ids2tokens(self, integers: Union[np.ndarray, Iterable[int]]) -> List[str]: - if isinstance(integers, np.ndarray) and integers.ndim != 1: - raise ValueError(f"Must be 1 dim ndarray, but got {integers.ndim}") - return [self.token_list[i] for i in integers] - - def tokens2ids(self, tokens: Iterable[str]) -> List[int]: - return [self.token2id.get(i, self.unk_id) for i in tokens] - - @abstractmethod - def text2tokens(self, line: str) -> List[str]: - raise NotImplementedError - - @abstractmethod - def tokens2text(self, tokens: Iterable[str]) -> str: - raise NotImplementedError diff --git a/funasr_detach/tokenizer/build_tokenizer.py b/funasr_detach/tokenizer/build_tokenizer.py deleted file mode 100644 index a4e717f62f79516fe714f7ab97a1f15c9f640edf..0000000000000000000000000000000000000000 --- a/funasr_detach/tokenizer/build_tokenizer.py +++ /dev/null @@ -1,61 +0,0 @@ -from pathlib import Path -from typing import Iterable -from typing import Union - - -from funasr_detach.tokenizer.abs_tokenizer import AbsTokenizer -from funasr_detach.tokenizer.char_tokenizer import CharTokenizer -from funasr_detach.tokenizer.phoneme_tokenizer import PhonemeTokenizer -from funasr_detach.tokenizer.sentencepiece_tokenizer import SentencepiecesTokenizer -from funasr_detach.tokenizer.word_tokenizer import WordTokenizer - - -def build_tokenizer( - token_type: str, - bpemodel: Union[Path, str, Iterable[str]] = None, - non_linguistic_symbols: Union[Path, str, Iterable[str]] = None, - remove_non_linguistic_symbols: bool = False, - space_symbol: str = "", - delimiter: str = None, - g2p_type: str = None, -) -> AbsTokenizer: - """A helper function to instantiate Tokenizer""" - if token_type == "bpe": - if bpemodel is None: - raise ValueError('bpemodel is required if token_type = "bpe"') - - if remove_non_linguistic_symbols: - raise RuntimeError( - "remove_non_linguistic_symbols is not implemented for token_type=bpe" - ) - return SentencepiecesTokenizer(bpemodel) - - elif token_type == "word": - if remove_non_linguistic_symbols and non_linguistic_symbols is not None: - return WordTokenizer( - delimiter=delimiter, - non_linguistic_symbols=non_linguistic_symbols, - remove_non_linguistic_symbols=True, - ) - else: - return WordTokenizer(delimiter=delimiter) - - elif token_type == "char": - return CharTokenizer( - non_linguistic_symbols=non_linguistic_symbols, - space_symbol=space_symbol, - remove_non_linguistic_symbols=remove_non_linguistic_symbols, - ) - - elif token_type == "phn": - return PhonemeTokenizer( - g2p_type=g2p_type, - non_linguistic_symbols=non_linguistic_symbols, - space_symbol=space_symbol, - remove_non_linguistic_symbols=remove_non_linguistic_symbols, - ) - - else: - raise ValueError( - f"token_mode must be one of bpe, word, char or phn: " f"{token_type}" - ) diff --git a/funasr_detach/tokenizer/char_tokenizer.py b/funasr_detach/tokenizer/char_tokenizer.py deleted file mode 100644 index 916429080fa2964ad93160793c2d4427092a9ad8..0000000000000000000000000000000000000000 --- a/funasr_detach/tokenizer/char_tokenizer.py +++ /dev/null @@ -1,111 +0,0 @@ -from pathlib import Path -from typing import Iterable -from typing import List -from typing import Union -import warnings -import re - -from funasr_detach.tokenizer.abs_tokenizer import BaseTokenizer -from funasr_detach.register import tables - - -@tables.register("tokenizer_classes", "CharTokenizer") -class CharTokenizer(BaseTokenizer): - def __init__( - self, - non_linguistic_symbols: Union[Path, str, Iterable[str]] = None, - space_symbol: str = "", - remove_non_linguistic_symbols: bool = False, - split_with_space: bool = False, - seg_dict: str = None, - **kwargs, - ): - super().__init__(**kwargs) - self.space_symbol = space_symbol - if non_linguistic_symbols is None: - self.non_linguistic_symbols = set() - elif isinstance(non_linguistic_symbols, (Path, str)): - non_linguistic_symbols = Path(non_linguistic_symbols) - try: - with non_linguistic_symbols.open("r", encoding="utf-8") as f: - self.non_linguistic_symbols = set(line.rstrip() for line in f) - except FileNotFoundError: - warnings.warn(f"{non_linguistic_symbols} doesn't exist.") - self.non_linguistic_symbols = set() - else: - self.non_linguistic_symbols = set(non_linguistic_symbols) - self.remove_non_linguistic_symbols = remove_non_linguistic_symbols - self.split_with_space = split_with_space - self.seg_dict = None - if seg_dict is not None: - self.seg_dict = load_seg_dict(seg_dict) - - def __repr__(self): - return ( - f"{self.__class__.__name__}(" - f'space_symbol="{self.space_symbol}"' - f'non_linguistic_symbols="{self.non_linguistic_symbols}"' - f")" - ) - - def text2tokens(self, line: Union[str, list]) -> List[str]: - - # if self.split_with_space: - - if self.seg_dict is not None: - tokens = line.strip().split(" ") - tokens = seg_tokenize(tokens, self.seg_dict) - else: - tokens = [] - while len(line) != 0: - for w in self.non_linguistic_symbols: - if line.startswith(w): - if not self.remove_non_linguistic_symbols: - tokens.append(line[: len(w)]) - line = line[len(w) :] - break - else: - t = line[0] - if t == " ": - # t = "" - line = line[1:] - continue - tokens.append(t) - line = line[1:] - return tokens - - def tokens2text(self, tokens: Iterable[str]) -> str: - tokens = [t if t != self.space_symbol else " " for t in tokens] - return "".join(tokens) - - -def load_seg_dict(seg_dict_file): - seg_dict = {} - assert isinstance(seg_dict_file, str) - with open(seg_dict_file, "r", encoding="utf8") as f: - lines = f.readlines() - for line in lines: - s = line.strip().split() - key = s[0] - value = s[1:] - seg_dict[key] = " ".join(value) - return seg_dict - - -def seg_tokenize(txt, seg_dict): - pattern = re.compile(r"^[\u4E00-\u9FA50-9]+$") - out_txt = "" - for word in txt: - word = word.lower() - if word in seg_dict: - out_txt += seg_dict[word] + " " - else: - if pattern.match(word): - for char in word: - if char in seg_dict: - out_txt += seg_dict[char] + " " - else: - out_txt += "" + " " - else: - out_txt += "" + " " - return out_txt.strip().split() diff --git a/funasr_detach/tokenizer/cleaner.py b/funasr_detach/tokenizer/cleaner.py deleted file mode 100644 index 882db5c970c40350547a75d706f547ff12799430..0000000000000000000000000000000000000000 --- a/funasr_detach/tokenizer/cleaner.py +++ /dev/null @@ -1,48 +0,0 @@ -from typing import Collection - -from jaconv import jaconv - -# import tacotron_cleaner.cleaners - -try: - from vietnamese_cleaner import vietnamese_cleaners -except ImportError: - vietnamese_cleaners = None - - -class TextCleaner: - """Text cleaner. - - Examples: - >>> cleaner = TextCleaner("tacotron") - >>> cleaner("(Hello-World); & jr. & dr.") - 'HELLO WORLD, AND JUNIOR AND DOCTOR' - - """ - - def __init__(self, cleaner_types: Collection[str] = None): - - if cleaner_types is None: - self.cleaner_types = [] - elif isinstance(cleaner_types, str): - self.cleaner_types = [cleaner_types] - else: - self.cleaner_types = list(cleaner_types) - - def __call__(self, text: str) -> str: - for t in self.cleaner_types: - if t == "tacotron": - # text = tacotron_cleaner.cleaners.custom_english_cleaners(text) - pass - elif t == "jaconv": - text = jaconv.normalize(text) - elif t == "vietnamese": - if vietnamese_cleaners is None: - raise RuntimeError("Please install underthesea") - text = vietnamese_cleaners.vietnamese_cleaner(text) - elif t == "korean_cleaner": - text = KoreanCleaner.normalize_text(text) - else: - raise RuntimeError(f"Not supported: type={t}") - - return text diff --git a/funasr_detach/tokenizer/korean_cleaner.py b/funasr_detach/tokenizer/korean_cleaner.py deleted file mode 100644 index e98d0b81921894712cf02959f2df344e7bda047a..0000000000000000000000000000000000000000 --- a/funasr_detach/tokenizer/korean_cleaner.py +++ /dev/null @@ -1,79 +0,0 @@ -# Referenced from https://github.com/hccho2/Tacotron-Wavenet-Vocoder-Korean - -import re - - -class KoreanCleaner: - @classmethod - def _normalize_numbers(cls, text): - number_to_kor = { - "0": "영", - "1": "일", - "2": "이", - "3": "삼", - "4": "사", - "5": "오", - "6": "육", - "7": "칠", - "8": "팔", - "9": "구", - } - new_text = "".join( - number_to_kor[char] if char in number_to_kor.keys() else char - for char in text - ) - return new_text - - @classmethod - def _normalize_english_text(cls, text): - upper_alphabet_to_kor = { - "A": "에이", - "B": "비", - "C": "씨", - "D": "디", - "E": "이", - "F": "에프", - "G": "지", - "H": "에이치", - "I": "아이", - "J": "제이", - "K": "케이", - "L": "엘", - "M": "엠", - "N": "엔", - "O": "오", - "P": "피", - "Q": "큐", - "R": "알", - "S": "에스", - "T": "티", - "U": "유", - "V": "브이", - "W": "더블유", - "X": "엑스", - "Y": "와이", - "Z": "지", - } - new_text = re.sub("[a-z]+", lambda x: str.upper(x.group()), text) - new_text = "".join( - ( - upper_alphabet_to_kor[char] - if char in upper_alphabet_to_kor.keys() - else char - ) - for char in new_text - ) - - return new_text - - @classmethod - def normalize_text(cls, text): - # stage 0 : text strip - text = text.strip() - - # stage 1 : normalize numbers - text = cls._normalize_numbers(text) - - # stage 2 : normalize english text - text = cls._normalize_english_text(text) - return text diff --git a/funasr_detach/tokenizer/phoneme_tokenizer.py b/funasr_detach/tokenizer/phoneme_tokenizer.py deleted file mode 100644 index 2ce6eaedb14d0c1bc467cde3c644c97ced3dd0b3..0000000000000000000000000000000000000000 --- a/funasr_detach/tokenizer/phoneme_tokenizer.py +++ /dev/null @@ -1,526 +0,0 @@ -import logging -from pathlib import Path -import re -from typing import Iterable -from typing import List -from typing import Optional -from typing import Union -import warnings - -# import g2p_en -import jamo - -from funasr_detach.tokenizer.abs_tokenizer import AbsTokenizer - - -g2p_classes = [ - None, - "g2p_en", - "g2p_en_no_space", - "pyopenjtalk", - "pyopenjtalk_kana", - "pyopenjtalk_accent", - "pyopenjtalk_accent_with_pause", - "pyopenjtalk_prosody", - "pypinyin_g2p", - "pypinyin_g2p_phone", - "espeak_ng_arabic", - "espeak_ng_german", - "espeak_ng_french", - "espeak_ng_spanish", - "espeak_ng_russian", - "espeak_ng_greek", - "espeak_ng_finnish", - "espeak_ng_hungarian", - "espeak_ng_dutch", - "espeak_ng_english_us_vits", - "espeak_ng_hindi", - "g2pk", - "g2pk_no_space", - "korean_jaso", - "korean_jaso_no_space", -] - - -def split_by_space(text) -> List[str]: - if " " in text: - text = text.replace(" ", " ") - return [c.replace("", " ") for c in text.split(" ")] - else: - return text.split(" ") - - -def pyopenjtalk_g2p(text) -> List[str]: - import pyopenjtalk - - # phones is a str object separated by space - phones = pyopenjtalk.g2p(text, kana=False) - phones = phones.split(" ") - return phones - - -def pyopenjtalk_g2p_accent(text) -> List[str]: - import pyopenjtalk - import re - - phones = [] - for labels in pyopenjtalk.run_frontend(text)[1]: - p = re.findall(r"\-(.*?)\+.*?\/A:([0-9\-]+).*?\/F:.*?_([0-9]+)", labels) - if len(p) == 1: - phones += [p[0][0], p[0][2], p[0][1]] - return phones - - -def pyopenjtalk_g2p_accent_with_pause(text) -> List[str]: - import pyopenjtalk - import re - - phones = [] - for labels in pyopenjtalk.run_frontend(text)[1]: - if labels.split("-")[1].split("+")[0] == "pau": - phones += ["pau"] - continue - p = re.findall(r"\-(.*?)\+.*?\/A:([0-9\-]+).*?\/F:.*?_([0-9]+)", labels) - if len(p) == 1: - phones += [p[0][0], p[0][2], p[0][1]] - return phones - - -def pyopenjtalk_g2p_kana(text) -> List[str]: - import pyopenjtalk - - kanas = pyopenjtalk.g2p(text, kana=True) - return list(kanas) - - -def pyopenjtalk_g2p_prosody(text: str, drop_unvoiced_vowels: bool = True) -> List[str]: - """Extract phoneme + prosoody symbol sequence from input full-context labels. - - The algorithm is based on `Prosodic features control by symbols as input of - sequence-to-sequence acoustic modeling for neural TTS`_ with some r9y9's tweaks. - - Args: - text (str): Input text. - drop_unvoiced_vowels (bool): whether to drop unvoiced vowels. - - Returns: - List[str]: List of phoneme + prosody symbols. - - Examples: - >>> from funasr_detach.tokenizer.phoneme_tokenizer import pyopenjtalk_g2p_prosody - >>> pyopenjtalk_g2p_prosody("こんにちは。") - ['^', 'k', 'o', '[', 'N', 'n', 'i', 'ch', 'i', 'w', 'a', '$'] - - .. _`Prosodic features control by symbols as input of sequence-to-sequence acoustic - modeling for neural TTS`: https://doi.org/10.1587/transinf.2020EDP7104 - - """ - import pyopenjtalk - - labels = pyopenjtalk.run_frontend(text)[1] - N = len(labels) - - phones = [] - for n in range(N): - lab_curr = labels[n] - - # current phoneme - p3 = re.search(r"\-(.*?)\+", lab_curr).group(1) - - # deal unvoiced vowels as normal vowels - if drop_unvoiced_vowels and p3 in "AEIOU": - p3 = p3.lower() - - # deal with sil at the beginning and the end of text - if p3 == "sil": - assert n == 0 or n == N - 1 - if n == 0: - phones.append("^") - elif n == N - 1: - # check question form or not - e3 = _numeric_feature_by_regex(r"!(\d+)_", lab_curr) - if e3 == 0: - phones.append("$") - elif e3 == 1: - phones.append("?") - continue - elif p3 == "pau": - phones.append("_") - continue - else: - phones.append(p3) - - # accent type and position info (forward or backward) - a1 = _numeric_feature_by_regex(r"/A:([0-9\-]+)\+", lab_curr) - a2 = _numeric_feature_by_regex(r"\+(\d+)\+", lab_curr) - a3 = _numeric_feature_by_regex(r"\+(\d+)/", lab_curr) - - # number of mora in accent phrase - f1 = _numeric_feature_by_regex(r"/F:(\d+)_", lab_curr) - - a2_next = _numeric_feature_by_regex(r"\+(\d+)\+", labels[n + 1]) - # accent phrase border - if a3 == 1 and a2_next == 1 and p3 in "aeiouAEIOUNcl": - phones.append("#") - # pitch falling - elif a1 == 0 and a2_next == a2 + 1 and a2 != f1: - phones.append("]") - # pitch rising - elif a2 == 1 and a2_next == 2: - phones.append("[") - - return phones - - -def _numeric_feature_by_regex(regex, s): - match = re.search(regex, s) - if match is None: - return -50 - return int(match.group(1)) - - -def pypinyin_g2p(text) -> List[str]: - from pypinyin import pinyin - from pypinyin import Style - - phones = [phone[0] for phone in pinyin(text, style=Style.TONE3)] - return phones - - -def pypinyin_g2p_phone(text) -> List[str]: - from pypinyin import pinyin - from pypinyin import Style - from pypinyin.style._utils import get_finals - from pypinyin.style._utils import get_initials - - phones = [ - p - for phone in pinyin(text, style=Style.TONE3) - for p in [ - get_initials(phone[0], strict=True), - get_finals(phone[0], strict=True), - ] - if len(p) != 0 - ] - return phones - - -class G2p_en: - """On behalf of g2p_en.G2p. - - g2p_en.G2p isn't pickalable and it can't be copied to the other processes - via multiprocessing module. - As a workaround, g2p_en.G2p is instantiated upon calling this class. - - """ - - def __init__(self, no_space: bool = False): - self.no_space = no_space - self.g2p = None - - def __call__(self, text) -> List[str]: - if self.g2p is None: - self.g2p = g2p_en.G2p() - - phones = self.g2p(text) - if self.no_space: - # remove space which represents word serapater - phones = list(filter(lambda s: s != " ", phones)) - return phones - - -class G2pk: - """On behalf of g2pk.G2p. - - g2pk.G2p isn't pickalable and it can't be copied to the other processes - via multiprocessing module. - As a workaround, g2pk.G2p is instantiated upon calling this class. - - """ - - def __init__( - self, descritive=False, group_vowels=False, to_syl=False, no_space=False - ): - self.descritive = descritive - self.group_vowels = group_vowels - self.to_syl = to_syl - self.no_space = no_space - self.g2p = None - - def __call__(self, text) -> List[str]: - if self.g2p is None: - import g2pk - - self.g2p = g2pk.G2p() - - phones = list( - self.g2p( - text, - descriptive=self.descritive, - group_vowels=self.group_vowels, - to_syl=self.to_syl, - ) - ) - if self.no_space: - # remove space which represents word serapater - phones = list(filter(lambda s: s != " ", phones)) - return phones - - -class Jaso: - PUNC = "!'(),-.:;?" - SPACE = " " - - JAMO_LEADS = "".join([chr(_) for _ in range(0x1100, 0x1113)]) - JAMO_VOWELS = "".join([chr(_) for _ in range(0x1161, 0x1176)]) - JAMO_TAILS = "".join([chr(_) for _ in range(0x11A8, 0x11C3)]) - - VALID_CHARS = JAMO_LEADS + JAMO_VOWELS + JAMO_TAILS + PUNC + SPACE - - def __init__(self, space_symbol=" ", no_space=False): - self.space_symbol = space_symbol - self.no_space = no_space - - def _text_to_jaso(self, line: str) -> List[str]: - jasos = list(jamo.hangul_to_jamo(line)) - return jasos - - def _remove_non_korean_characters(self, tokens): - new_tokens = [token for token in tokens if token in self.VALID_CHARS] - return new_tokens - - def __call__(self, text) -> List[str]: - graphemes = [x for x in self._text_to_jaso(text)] - graphemes = self._remove_non_korean_characters(graphemes) - - if self.no_space: - graphemes = list(filter(lambda s: s != " ", graphemes)) - else: - graphemes = [x if x != " " else self.space_symbol for x in graphemes] - return graphemes - - -class Phonemizer: - """Phonemizer module for various languages. - - This is wrapper module of https://github.com/bootphon/phonemizer. - You can define various g2p modules by specifying options for phonemizer. - - See available options: - https://github.com/bootphon/phonemizer/blob/master/phonemizer/phonemize.py#L32 - - """ - - def __init__( - self, - backend, - word_separator: Optional[str] = None, - syllable_separator: Optional[str] = None, - phone_separator: Optional[str] = " ", - strip=False, - split_by_single_token: bool = False, - **phonemizer_kwargs, - ): - # delayed import - from phonemizer.backend import BACKENDS - from phonemizer.separator import Separator - - self.separator = Separator( - word=word_separator, - syllable=syllable_separator, - phone=phone_separator, - ) - - # define logger to suppress the warning in phonemizer - logger = logging.getLogger("phonemizer") - logger.setLevel(logging.ERROR) - self.phonemizer = BACKENDS[backend]( - **phonemizer_kwargs, - logger=logger, - ) - self.strip = strip - self.split_by_single_token = split_by_single_token - - def __call__(self, text) -> List[str]: - tokens = self.phonemizer.phonemize( - [text], - separator=self.separator, - strip=self.strip, - njobs=1, - )[0] - if not self.split_by_single_token: - return tokens.split() - else: - # "a: ab" -> ["a", ":", "", "a", "b"] - # TODO(kan-bayashi): space replacement should be dealt in PhonemeTokenizer - return [c.replace(" ", "") for c in tokens] - - -class PhonemeTokenizer(AbsTokenizer): - def __init__( - self, - g2p_type: Union[None, str], - non_linguistic_symbols: Union[Path, str, Iterable[str]] = None, - space_symbol: str = "", - remove_non_linguistic_symbols: bool = False, - ): - if g2p_type is None: - self.g2p = split_by_space - elif g2p_type == "g2p_en": - self.g2p = G2p_en(no_space=False) - elif g2p_type == "g2p_en_no_space": - self.g2p = G2p_en(no_space=True) - elif g2p_type == "pyopenjtalk": - self.g2p = pyopenjtalk_g2p - elif g2p_type == "pyopenjtalk_kana": - self.g2p = pyopenjtalk_g2p_kana - elif g2p_type == "pyopenjtalk_accent": - self.g2p = pyopenjtalk_g2p_accent - elif g2p_type == "pyopenjtalk_accent_with_pause": - self.g2p = pyopenjtalk_g2p_accent_with_pause - elif g2p_type == "pyopenjtalk_prosody": - self.g2p = pyopenjtalk_g2p_prosody - elif g2p_type == "pypinyin_g2p": - self.g2p = pypinyin_g2p - elif g2p_type == "pypinyin_g2p_phone": - self.g2p = pypinyin_g2p_phone - elif g2p_type == "espeak_ng_arabic": - self.g2p = Phonemizer( - language="ar", - backend="espeak", - with_stress=True, - preserve_punctuation=True, - ) - elif g2p_type == "espeak_ng_german": - self.g2p = Phonemizer( - language="de", - backend="espeak", - with_stress=True, - preserve_punctuation=True, - ) - elif g2p_type == "espeak_ng_french": - self.g2p = Phonemizer( - language="fr-fr", - backend="espeak", - with_stress=True, - preserve_punctuation=True, - ) - elif g2p_type == "espeak_ng_spanish": - self.g2p = Phonemizer( - language="es", - backend="espeak", - with_stress=True, - preserve_punctuation=True, - ) - elif g2p_type == "espeak_ng_russian": - self.g2p = Phonemizer( - language="ru", - backend="espeak", - with_stress=True, - preserve_punctuation=True, - ) - elif g2p_type == "espeak_ng_greek": - self.g2p = Phonemizer( - language="el", - backend="espeak", - with_stress=True, - preserve_punctuation=True, - ) - elif g2p_type == "espeak_ng_finnish": - self.g2p = Phonemizer( - language="fi", - backend="espeak", - with_stress=True, - preserve_punctuation=True, - ) - elif g2p_type == "espeak_ng_hungarian": - self.g2p = Phonemizer( - language="hu", - backend="espeak", - with_stress=True, - preserve_punctuation=True, - ) - elif g2p_type == "espeak_ng_dutch": - self.g2p = Phonemizer( - language="nl", - backend="espeak", - with_stress=True, - preserve_punctuation=True, - ) - elif g2p_type == "espeak_ng_hindi": - self.g2p = Phonemizer( - language="hi", - backend="espeak", - with_stress=True, - preserve_punctuation=True, - ) - elif g2p_type == "g2pk": - self.g2p = G2pk(no_space=False) - elif g2p_type == "g2pk_no_space": - self.g2p = G2pk(no_space=True) - elif g2p_type == "espeak_ng_english_us_vits": - # VITS official implementation-like processing - # Reference: https://github.com/jaywalnut310/vits - self.g2p = Phonemizer( - language="en-us", - backend="espeak", - with_stress=True, - preserve_punctuation=True, - strip=True, - word_separator=" ", - phone_separator="", - split_by_single_token=True, - ) - elif g2p_type == "korean_jaso": - self.g2p = Jaso(space_symbol=space_symbol, no_space=False) - elif g2p_type == "korean_jaso_no_space": - self.g2p = Jaso(no_space=True) - else: - raise NotImplementedError(f"Not supported: g2p_type={g2p_type}") - - self.g2p_type = g2p_type - self.space_symbol = space_symbol - if non_linguistic_symbols is None: - self.non_linguistic_symbols = set() - elif isinstance(non_linguistic_symbols, (Path, str)): - non_linguistic_symbols = Path(non_linguistic_symbols) - try: - with non_linguistic_symbols.open("r", encoding="utf-8") as f: - self.non_linguistic_symbols = set(line.rstrip() for line in f) - except FileNotFoundError: - warnings.warn(f"{non_linguistic_symbols} doesn't exist.") - self.non_linguistic_symbols = set() - else: - self.non_linguistic_symbols = set(non_linguistic_symbols) - self.remove_non_linguistic_symbols = remove_non_linguistic_symbols - - def __repr__(self): - return ( - f"{self.__class__.__name__}(" - f'g2p_type="{self.g2p_type}", ' - f'space_symbol="{self.space_symbol}", ' - f'non_linguistic_symbols="{self.non_linguistic_symbols}"' - ")" - ) - - def text2tokens(self, line: str) -> List[str]: - tokens = [] - while len(line) != 0: - for w in self.non_linguistic_symbols: - if line.startswith(w): - if not self.remove_non_linguistic_symbols: - tokens.append(line[: len(w)]) - line = line[len(w) :] - break - else: - t = line[0] - tokens.append(t) - line = line[1:] - - line = "".join(tokens) - tokens = self.g2p(line) - return tokens - - def tokens2text(self, tokens: Iterable[str]) -> str: - # phoneme type is not invertible - return "".join(tokens) diff --git a/funasr_detach/tokenizer/sentencepiece_tokenizer.py b/funasr_detach/tokenizer/sentencepiece_tokenizer.py deleted file mode 100644 index 573947c35ac9b46af02afcbc5d0a6aa573860939..0000000000000000000000000000000000000000 --- a/funasr_detach/tokenizer/sentencepiece_tokenizer.py +++ /dev/null @@ -1,47 +0,0 @@ -from pathlib import Path -from typing import Iterable -from typing import List -from typing import Union - -import sentencepiece as spm - -from funasr_detach.tokenizer.abs_tokenizer import BaseTokenizer -from funasr_detach.register import tables - - -@tables.register("tokenizer_classes", "SentencepiecesTokenizer") -class SentencepiecesTokenizer(BaseTokenizer): - def __init__(self, bpemodel: Union[Path, str], **kwargs): - super().__init__(**kwargs) - self.bpemodel = str(bpemodel) - # NOTE(kamo): - # Don't build SentencePieceProcessor in __init__() - # because it's not picklable and it may cause following error, - # "TypeError: can't pickle SwigPyObject objects", - # when giving it as argument of "multiprocessing.Process()". - self.sp = None - - def __repr__(self): - return f'{self.__class__.__name__}(model="{self.bpemodel}")' - - def _build_sentence_piece_processor(self): - # Build SentencePieceProcessor lazily. - if self.sp is None: - self.sp = spm.SentencePieceProcessor() - self.sp.load(self.bpemodel) - - def text2tokens(self, line: str) -> List[str]: - self._build_sentence_piece_processor() - return self.sp.EncodeAsPieces(line) - - def tokens2text(self, tokens: Iterable[str]) -> str: - self._build_sentence_piece_processor() - return self.sp.DecodePieces(list(tokens)) - - def encode(self, line: str) -> List[int]: - self._build_sentence_piece_processor() - return self.sp.EncodeAsIds(line) - - def decode(self, line: List[int]): - self._build_sentence_piece_processor() - return self.sp.DecodeIds(line) diff --git a/funasr_detach/tokenizer/token_id_converter.py b/funasr_detach/tokenizer/token_id_converter.py deleted file mode 100644 index 1888d750837b008e128b9ae89b8c109fc17a167e..0000000000000000000000000000000000000000 --- a/funasr_detach/tokenizer/token_id_converter.py +++ /dev/null @@ -1,58 +0,0 @@ -from pathlib import Path -from typing import Dict -from typing import Iterable -from typing import List -from typing import Union - -import numpy as np - - -class TokenIDConverter: - def __init__( - self, - token_list: Union[Path, str, Iterable[str]], - unk_symbol: str = "", - ): - - if isinstance(token_list, (Path, str)): - token_list = Path(token_list) - self.token_list_repr = str(token_list) - self.token_list: List[str] = [] - - with token_list.open("r", encoding="utf-8") as f: - for idx, line in enumerate(f): - line = line.rstrip() - self.token_list.append(line) - - else: - self.token_list: List[str] = list(token_list) - self.token_list_repr = "" - for i, t in enumerate(self.token_list): - if i == 3: - break - self.token_list_repr += f"{t}, " - self.token_list_repr += f"... (NVocab={(len(self.token_list))})" - - self.token2id: Dict[str, int] = {} - for i, t in enumerate(self.token_list): - if t in self.token2id: - raise RuntimeError(f'Symbol "{t}" is duplicated') - self.token2id[t] = i - - self.unk_symbol = unk_symbol - if self.unk_symbol not in self.token2id: - raise RuntimeError( - f"Unknown symbol '{unk_symbol}' doesn't exist in the token_list" - ) - self.unk_id = self.token2id[self.unk_symbol] - - def get_num_vocabulary_size(self) -> int: - return len(self.token_list) - - def ids2tokens(self, integers: Union[np.ndarray, Iterable[int]]) -> List[str]: - if isinstance(integers, np.ndarray) and integers.ndim != 1: - raise ValueError(f"Must be 1 dim ndarray, but got {integers.ndim}") - return [self.token_list[i] for i in integers] - - def tokens2ids(self, tokens: Iterable[str]) -> List[int]: - return [self.token2id.get(i, self.unk_id) for i in tokens] diff --git a/funasr_detach/tokenizer/word_tokenizer.py b/funasr_detach/tokenizer/word_tokenizer.py deleted file mode 100644 index 8a9ed307669d9f57b53d534eb294f684c56aecd2..0000000000000000000000000000000000000000 --- a/funasr_detach/tokenizer/word_tokenizer.py +++ /dev/null @@ -1,56 +0,0 @@ -from pathlib import Path -from typing import Iterable -from typing import List -from typing import Union -import warnings - - -from funasr_detach.tokenizer.abs_tokenizer import AbsTokenizer - - -class WordTokenizer(AbsTokenizer): - def __init__( - self, - delimiter: str = None, - non_linguistic_symbols: Union[Path, str, Iterable[str]] = None, - remove_non_linguistic_symbols: bool = False, - ): - self.delimiter = delimiter - - if not remove_non_linguistic_symbols and non_linguistic_symbols is not None: - warnings.warn( - "non_linguistic_symbols is only used " - "when remove_non_linguistic_symbols = True" - ) - - if non_linguistic_symbols is None: - self.non_linguistic_symbols = set() - elif isinstance(non_linguistic_symbols, (Path, str)): - non_linguistic_symbols = Path(non_linguistic_symbols) - try: - with non_linguistic_symbols.open("r", encoding="utf-8") as f: - self.non_linguistic_symbols = set(line.rstrip() for line in f) - except FileNotFoundError: - warnings.warn(f"{non_linguistic_symbols} doesn't exist.") - self.non_linguistic_symbols = set() - else: - self.non_linguistic_symbols = set(non_linguistic_symbols) - self.remove_non_linguistic_symbols = remove_non_linguistic_symbols - - def __repr__(self): - return f'{self.__class__.__name__}(delimiter="{self.delimiter}")' - - def text2tokens(self, line: str) -> List[str]: - tokens = [] - for t in line.split(self.delimiter): - if self.remove_non_linguistic_symbols and t in self.non_linguistic_symbols: - continue - tokens.append(t) - return tokens - - def tokens2text(self, tokens: Iterable[str]) -> str: - if self.delimiter is None: - delimiter = " " - else: - delimiter = self.delimiter - return delimiter.join(tokens) diff --git a/funasr_detach/train_utils/__init__.py b/funasr_detach/train_utils/__init__.py deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/funasr_detach/train_utils/add_gradient_noise.py b/funasr_detach/train_utils/add_gradient_noise.py deleted file mode 100644 index 654e928ec003b9592120451230c1e2253b2b09c0..0000000000000000000000000000000000000000 --- a/funasr_detach/train_utils/add_gradient_noise.py +++ /dev/null @@ -1,31 +0,0 @@ -import torch - - -def add_gradient_noise( - model: torch.nn.Module, - iteration: int, - duration: float = 100, - eta: float = 1.0, - scale_factor: float = 0.55, -): - """Adds noise from a standard normal distribution to the gradients. - - The standard deviation (`sigma`) is controlled - by the three hyper-parameters below. - `sigma` goes to zero (no noise) with more iterations. - - Args: - model: Model. - iteration: Number of iterations. - duration: {100, 1000}: Number of durations to control - the interval of the `sigma` change. - eta: {0.01, 0.3, 1.0}: The magnitude of `sigma`. - scale_factor: {0.55}: The scale of `sigma`. - """ - interval = (iteration // duration) + 1 - sigma = eta / interval**scale_factor - for param in model.parameters(): - if param.grad is not None: - _shape = param.grad.size() - noise = sigma * torch.randn(_shape).to(param.device) - param.grad += noise diff --git a/funasr_detach/train_utils/average_nbest_models.py b/funasr_detach/train_utils/average_nbest_models.py deleted file mode 100644 index bfccd84116af84f1718e249bc84e82d122ae06f9..0000000000000000000000000000000000000000 --- a/funasr_detach/train_utils/average_nbest_models.py +++ /dev/null @@ -1,187 +0,0 @@ -import logging -from pathlib import Path -from typing import Optional -from typing import Sequence -from typing import Union -import warnings -import os -from io import BytesIO - -import torch -from typing import Collection -import os -import torch -import re -from collections import OrderedDict -from functools import cmp_to_key - - -# @torch.no_grad() -# def average_nbest_models( -# output_dir: Path, -# best_model_criterion: Sequence[Sequence[str]], -# nbest: Union[Collection[int], int], -# suffix: Optional[str] = None, -# oss_bucket=None, -# pai_output_dir=None, -# ) -> None: -# """Generate averaged model from n-best models -# -# Args: -# output_dir: The directory contains the model file for each epoch -# reporter: Reporter instance -# best_model_criterion: Give criterions to decide the best model. -# e.g. [("valid", "loss", "min"), ("train", "acc", "max")] -# nbest: Number of best model files to be averaged -# suffix: A suffix added to the averaged model file name -# """ -# if isinstance(nbest, int): -# nbests = [nbest] -# else: -# nbests = list(nbest) -# if len(nbests) == 0: -# warnings.warn("At least 1 nbest values are required") -# nbests = [1] -# if suffix is not None: -# suffix = suffix + "." -# else: -# suffix = "" -# -# # 1. Get nbests: List[Tuple[str, str, List[Tuple[epoch, value]]]] -# nbest_epochs = [ -# (ph, k, reporter.sort_epochs_and_values(ph, k, m)[: max(nbests)]) -# for ph, k, m in best_model_criterion -# if reporter.has(ph, k) -# ] -# -# _loaded = {} -# for ph, cr, epoch_and_values in nbest_epochs: -# _nbests = [i for i in nbests if i <= len(epoch_and_values)] -# if len(_nbests) == 0: -# _nbests = [1] -# -# for n in _nbests: -# if n == 0: -# continue -# elif n == 1: -# # The averaged model is same as the best model -# e, _ = epoch_and_values[0] -# op = output_dir / f"{e}epoch.pb" -# sym_op = output_dir / f"{ph}.{cr}.ave_1best.{suffix}pb" -# if sym_op.is_symlink() or sym_op.exists(): -# sym_op.unlink() -# sym_op.symlink_to(op.name) -# else: -# op = output_dir / f"{ph}.{cr}.ave_{n}best.{suffix}pb" -# logging.info( -# f"Averaging {n}best models: " f'criterion="{ph}.{cr}": {op}' -# ) -# -# avg = None -# # 2.a. Averaging model -# for e, _ in epoch_and_values[:n]: -# if e not in _loaded: -# if oss_bucket is None: -# _loaded[e] = torch.load( -# output_dir / f"{e}epoch.pb", -# map_location="cpu", -# ) -# else: -# buffer = BytesIO( -# oss_bucket.get_object(os.path.join(pai_output_dir, f"{e}epoch.pb")).read()) -# _loaded[e] = torch.load(buffer) -# states = _loaded[e] -# -# if avg is None: -# avg = states -# else: -# # Accumulated -# for k in avg: -# avg[k] = avg[k] + states[k] -# for k in avg: -# if str(avg[k].dtype).startswith("torch.int"): -# # For int type, not averaged, but only accumulated. -# # e.g. BatchNorm.num_batches_tracked -# # (If there are any cases that requires averaging -# # or the other reducing method, e.g. max/min, for integer type, -# # please report.) -# pass -# else: -# avg[k] = avg[k] / n -# -# # 2.b. Save the ave model and create a symlink -# if oss_bucket is None: -# torch.save(avg, op) -# else: -# buffer = BytesIO() -# torch.save(avg, buffer) -# oss_bucket.put_object(os.path.join(pai_output_dir, f"{ph}.{cr}.ave_{n}best.{suffix}pb"), -# buffer.getvalue()) -# -# # 3. *.*.ave.pb is a symlink to the max ave model -# if oss_bucket is None: -# op = output_dir / f"{ph}.{cr}.ave_{max(_nbests)}best.{suffix}pb" -# sym_op = output_dir / f"{ph}.{cr}.ave.{suffix}pb" -# if sym_op.is_symlink() or sym_op.exists(): -# sym_op.unlink() -# sym_op.symlink_to(op.name) - - -def _get_checkpoint_paths(output_dir: str, last_n: int = 5): - """ - Get the paths of the last 'last_n' checkpoints by parsing filenames - in the output directory. - """ - # List all files in the output directory - files = os.listdir(output_dir) - # Filter out checkpoint files and extract epoch numbers - checkpoint_files = [f for f in files if f.startswith("model.pt.e")] - # Sort files by epoch number in descending order - checkpoint_files.sort( - key=lambda x: int(re.search(r"(\d+)", x).group()), reverse=True - ) - # Get the last 'last_n' checkpoint paths - checkpoint_paths = [os.path.join(output_dir, f) for f in checkpoint_files[:last_n]] - return checkpoint_paths - - -@torch.no_grad() -def average_checkpoints(output_dir: str, last_n: int = 5): - """ - Average the last 'last_n' checkpoints' model state_dicts. - If a tensor is of type torch.int, perform sum instead of average. - """ - checkpoint_paths = _get_checkpoint_paths(output_dir, last_n) - state_dicts = [] - - # Load state_dicts from checkpoints - for path in checkpoint_paths: - if os.path.isfile(path): - state_dicts.append(torch.load(path, map_location="cpu")["state_dict"]) - else: - print(f"Checkpoint file {path} not found.") - continue - - # Check if we have any state_dicts to average - if not state_dicts: - raise RuntimeError("No checkpoints found for averaging.") - - # Average or sum weights - avg_state_dict = OrderedDict() - for key in state_dicts[0].keys(): - tensors = [state_dict[key].cpu() for state_dict in state_dicts] - # Check the type of the tensor - if str(tensors[0].dtype).startswith("torch.int"): - # Perform sum for integer tensors - summed_tensor = sum(tensors) - avg_state_dict[key] = summed_tensor - else: - # Perform average for other types of tensors - stacked_tensors = torch.stack(tensors) - avg_state_dict[key] = torch.mean(stacked_tensors, dim=0) - - torch.save( - {"state_dict": avg_state_dict}, - os.path.join(output_dir, f"model.pt.avg{last_n}"), - ) - return avg_state_dict diff --git a/funasr_detach/train_utils/device_funcs.py b/funasr_detach/train_utils/device_funcs.py deleted file mode 100644 index 7919e7d923200e0deb67bc1a8f83ad56c599d274..0000000000000000000000000000000000000000 --- a/funasr_detach/train_utils/device_funcs.py +++ /dev/null @@ -1,71 +0,0 @@ -import dataclasses -import warnings - -import numpy as np -import torch - - -def to_device(data, device=None, dtype=None, non_blocking=False, copy=False): - """Change the device of object recursively""" - if isinstance(data, dict): - return { - k: to_device(v, device, dtype, non_blocking, copy) for k, v in data.items() - } - elif dataclasses.is_dataclass(data) and not isinstance(data, type): - return type(data)( - *[ - to_device(v, device, dtype, non_blocking, copy) - for v in dataclasses.astuple(data) - ] - ) - # maybe namedtuple. I don't know the correct way to judge namedtuple. - elif isinstance(data, tuple) and type(data) is not tuple: - return type(data)( - *[to_device(o, device, dtype, non_blocking, copy) for o in data] - ) - elif isinstance(data, (list, tuple)): - return type(data)(to_device(v, device, dtype, non_blocking, copy) for v in data) - elif isinstance(data, np.ndarray): - return to_device(torch.from_numpy(data), device, dtype, non_blocking, copy) - elif isinstance(data, torch.Tensor): - return data.to(device, dtype, non_blocking, copy) - else: - return data - - -def force_gatherable(data, device): - """Change object to gatherable in torch.nn.DataParallel recursively - - The difference from to_device() is changing to torch.Tensor if float or int - value is found. - - The restriction to the returned value in DataParallel: - The object must be - - torch.cuda.Tensor - - 1 or more dimension. 0-dimension-tensor sends warning. - or a list, tuple, dict. - - """ - if isinstance(data, dict): - return {k: force_gatherable(v, device) for k, v in data.items()} - # DataParallel can't handle NamedTuple well - elif isinstance(data, tuple) and type(data) is not tuple: - return type(data)(*[force_gatherable(o, device) for o in data]) - elif isinstance(data, (list, tuple, set)): - return type(data)(force_gatherable(v, device) for v in data) - elif isinstance(data, np.ndarray): - return force_gatherable(torch.from_numpy(data), device) - elif isinstance(data, torch.Tensor): - if data.dim() == 0: - # To 1-dim array - data = data[None] - return data.to(device) - elif isinstance(data, float): - return torch.tensor([data], dtype=torch.float, device=device) - elif isinstance(data, int): - return torch.tensor([data], dtype=torch.long, device=device) - elif data is None: - return None - else: - warnings.warn(f"{type(data)} may not be gatherable by DataParallel") - return data diff --git a/funasr_detach/train_utils/forward_adaptor.py b/funasr_detach/train_utils/forward_adaptor.py deleted file mode 100644 index eb6da2bb189690e85bc9192b5bd1958854a1353c..0000000000000000000000000000000000000000 --- a/funasr_detach/train_utils/forward_adaptor.py +++ /dev/null @@ -1,31 +0,0 @@ -import torch - - -class ForwardAdaptor(torch.nn.Module): - """Wrapped module to parallelize specified method - - torch.nn.DataParallel parallelizes only "forward()" - and, maybe, the method having the other name can't be applied - except for wrapping the module just like this class. - - Examples: - >>> class A(torch.nn.Module): - ... def foo(self, x): - ... ... - >>> model = A() - >>> model = ForwardAdaptor(model, "foo") - >>> model = torch.nn.DataParallel(model, device_ids=[0, 1]) - >>> x = torch.randn(2, 10) - >>> model(x) - """ - - def __init__(self, module: torch.nn.Module, name: str): - super().__init__() - self.module = module - self.name = name - if not hasattr(module, name): - raise ValueError(f"{module} doesn't have {name}") - - def forward(self, *args, **kwargs): - func = getattr(self.module, self.name) - return func(*args, **kwargs) diff --git a/funasr_detach/train_utils/initialize.py b/funasr_detach/train_utils/initialize.py deleted file mode 100644 index 667585a27412838823da5b63626097bdf1706f67..0000000000000000000000000000000000000000 --- a/funasr_detach/train_utils/initialize.py +++ /dev/null @@ -1,55 +0,0 @@ -#!/usr/bin/env python3 - -"""Initialize modules for espnet2 neural networks.""" - -import math -import torch - - -def initialize(model: torch.nn.Module, init: str): - """Initialize weights of a neural network module. - - Parameters are initialized using the given method or distribution. - - Custom initialization routines can be implemented into submodules - as function `espnet_initialization_fn` within the custom module. - - Args: - model: Target. - init: Method of initialization. - """ - - # weight init - for p in model.parameters(): - if p.dim() > 1: - if init == "xavier_uniform": - torch.nn.init.xavier_uniform_(p.data) - elif init == "xavier_normal": - torch.nn.init.xavier_normal_(p.data) - elif init == "kaiming_uniform": - torch.nn.init.kaiming_uniform_(p.data, nonlinearity="relu") - elif init == "kaiming_normal": - torch.nn.init.kaiming_normal_(p.data, nonlinearity="relu") - else: - raise ValueError("Unknown initialization: " + init) - # bias init - for p in model.parameters(): - if p.dim() == 1: - p.data.zero_() - - # reset some modules with default init - for m in model.modules(): - if isinstance(m, (torch.nn.Embedding, torch.nn.LayerNorm, torch.nn.GroupNorm)): - m.reset_parameters() - if hasattr(m, "espnet_initialization_fn"): - m.espnet_initialization_fn() - - # TODO(xkc): Hacking s3prl_frontend and wav2vec2encoder initialization - if getattr(model, "encoder", None) and getattr( - model.encoder, "reload_pretrained_parameters", None - ): - model.encoder.reload_pretrained_parameters() - if getattr(model, "frontend", None) and getattr( - model.frontend, "reload_pretrained_parameters", None - ): - model.frontend.reload_pretrained_parameters() diff --git a/funasr_detach/train_utils/load_pretrained_model.py b/funasr_detach/train_utils/load_pretrained_model.py deleted file mode 100644 index 0c7eee4e071b162c84eb22e91abc660ca6a3929b..0000000000000000000000000000000000000000 --- a/funasr_detach/train_utils/load_pretrained_model.py +++ /dev/null @@ -1,172 +0,0 @@ -from typing import Any -from typing import Dict -from typing import Union -from io import BytesIO - -import logging -import torch -import torch.nn -import torch.optim - - -def filter_state_dict( - dst_state: Dict[str, Union[float, torch.Tensor]], - src_state: Dict[str, Union[float, torch.Tensor]], -): - """Filter name, size mismatch instances between dicts. - - Args: - dst_state: reference state dict for filtering - src_state: target state dict for filtering - - """ - match_state = {} - for key, value in src_state.items(): - if key in dst_state and (dst_state[key].size() == src_state[key].size()): - match_state[key] = value - else: - if key not in dst_state: - logging.warning( - f"Filter out {key} from pretrained dict" - + " because of name not found in target dict" - ) - else: - logging.warning( - f"Filter out {key} from pretrained dict" - + " because of size mismatch" - + f"({dst_state[key].size()}-{src_state[key].size()})" - ) - return match_state - - -def assigment_scope_map(dst_state: dict, src_state: dict, scope_map: str = None): - """Compute the union of the current variables and checkpoint variables.""" - import collections - import re - - # current model variables - name_to_variable = collections.OrderedDict() - for name, var in dst_state.items(): - name_to_variable[name] = var - - scope_map_num = 0 - if scope_map is not None: - scope_map = scope_map.split(",") - scope_map_num = len(scope_map) // 2 - for scope_map_idx in range(scope_map_num): - scope_map_id = scope_map_idx * 2 - logging.info( - "assignment_map from scope {} to {}".format( - scope_map[scope_map_id], scope_map[scope_map_id + 1] - ) - ) - - assignment_map = {} - for name, var in src_state.items(): - - if scope_map: - for scope_map_idx in range(scope_map_num): - scope_map_id = scope_map_idx * 2 - try: - idx = name.index(scope_map[scope_map_id]) - new_name = ( - scope_map[scope_map_id + 1] - + name[idx + len(scope_map[scope_map_id]) :] - ) - if new_name in name_to_variable: - assignment_map[name] = var - except: - continue - else: - if name in name_to_variable: - assignment_map[name] = var - - return assignment_map - - -def load_pretrained_model( - path: str, - model: torch.nn.Module, - ignore_init_mismatch: bool, - map_location: str = "cpu", - oss_bucket=None, - scope_map=None, - excludes=None, -): - """Load a model state and set it to the model. - - Args: - init_param: ::: - - Examples: - - """ - - obj = model - dst_state = obj.state_dict() - # import pdb; - # pdb.set_trace() - print(f"ckpt: {path}") - if oss_bucket is None: - src_state = torch.load(path, map_location=map_location) - else: - buffer = BytesIO(oss_bucket.get_object(path).read()) - src_state = torch.load(buffer, map_location=map_location) - if "state_dict" in src_state: - src_state = src_state["state_dict"] - - for k in dst_state.keys(): - if not k.startswith("module.") and "module." + k in src_state.keys(): - k_ddp = "module." + k - else: - k_ddp = k - if k_ddp in src_state: - dst_state[k] = src_state[k_ddp] - else: - print(f"Miss key in ckpt: model: {k}, ckpt: {k_ddp}") - - flag = obj.load_state_dict(dst_state, strict=True) - # print(flag) - - -# def load_pretrained_model( -# path: str, -# model: torch.nn.Module, -# ignore_init_mismatch: bool, -# map_location: str = "cpu", -# oss_bucket=None, -# scope_map=None, -# excludes=None, -# ): -# """Load a model state and set it to the model. -# -# Args: -# init_param: ::: -# -# Examples: -# -# """ -# -# obj = model -# -# if oss_bucket is None: -# src_state = torch.load(path, map_location=map_location) -# else: -# buffer = BytesIO(oss_bucket.get_object(path).read()) -# src_state = torch.load(buffer, map_location=map_location) -# src_state = src_state["model"] if "model" in src_state else src_state -# -# if excludes is not None: -# for e in excludes.split(","): -# src_state = {k: v for k, v in src_state.items() if not k.startswith(e)} -# -# dst_state = obj.state_dict() -# src_state = assigment_scope_map(dst_state, src_state, scope_map) -# -# if ignore_init_mismatch: -# src_state = filter_state_dict(dst_state, src_state) -# -# logging.debug("Loaded src_state keys: {}".format(src_state.keys())) -# logging.debug("Loaded dst_state keys: {}".format(dst_state.keys())) -# dst_state.update(src_state) -# obj.load_state_dict(dst_state, strict=True) diff --git a/funasr_detach/train_utils/model_summary.py b/funasr_detach/train_utils/model_summary.py deleted file mode 100644 index 1001160a9dafd4c3235c7477d88616976686ff3b..0000000000000000000000000000000000000000 --- a/funasr_detach/train_utils/model_summary.py +++ /dev/null @@ -1,64 +0,0 @@ -import numpy as np -import torch - - -def get_human_readable_count(number: int) -> str: - """Return human_readable_count - - Originated from: - https://github.com/PyTorchLightning/pytorch-lightning/blob/master/pytorch_lightning/core/memory.py - - Abbreviates an integer number with K, M, B, T for thousands, millions, - billions and trillions, respectively. - Examples: - >>> get_human_readable_count(123) - '123 ' - >>> get_human_readable_count(1234) # (one thousand) - '1 K' - >>> get_human_readable_count(2e6) # (two million) - '2 M' - >>> get_human_readable_count(3e9) # (three billion) - '3 B' - >>> get_human_readable_count(4e12) # (four trillion) - '4 T' - >>> get_human_readable_count(5e15) # (more than trillion) - '5,000 T' - Args: - number: a positive integer number - Return: - A string formatted according to the pattern described above. - """ - assert number >= 0 - labels = [" ", "K", "M", "B", "T"] - num_digits = int(np.floor(np.log10(number)) + 1 if number > 0 else 1) - num_groups = int(np.ceil(num_digits / 3)) - num_groups = min(num_groups, len(labels)) # don't abbreviate beyond trillions - shift = -3 * (num_groups - 1) - number = number * (10**shift) - index = num_groups - 1 - return f"{number:.2f} {labels[index]}" - - -def to_bytes(dtype) -> int: - # torch.float16 -> 16 - return int(str(dtype)[-2:]) // 8 - - -def model_summary(model: torch.nn.Module) -> str: - message = "Model structure:\n" - message += str(model) - tot_params = sum(p.numel() for p in model.parameters()) - num_params = sum(p.numel() for p in model.parameters() if p.requires_grad) - percent_trainable = "{:.1f}".format(num_params * 100.0 / tot_params) - tot_params = get_human_readable_count(tot_params) - num_params = get_human_readable_count(num_params) - message += "\n\nModel summary:\n" - message += f" Class Name: {model.__class__.__name__}\n" - message += f" Total Number of model parameters: {tot_params}\n" - message += ( - f" Number of trainable parameters: {num_params} ({percent_trainable}%)\n" - ) - - dtype = next(iter(model.parameters())).dtype - message += f" Type: {dtype}" - return message diff --git a/funasr_detach/train_utils/recursive_op.py b/funasr_detach/train_utils/recursive_op.py deleted file mode 100644 index 1c2a92fd83b6efd96b9b42d906418c8301c3259c..0000000000000000000000000000000000000000 --- a/funasr_detach/train_utils/recursive_op.py +++ /dev/null @@ -1,48 +0,0 @@ -"""Torch utility module.""" - -import torch - -if torch.distributed.is_available(): - from torch.distributed import ReduceOp - - -def recursive_sum(obj, weight: torch.Tensor, distributed: bool = False): - assert weight.dim() == 1, weight.size() - if isinstance(obj, (tuple, list)): - return type(obj)(recursive_sum(v, weight, distributed) for v in obj) - elif isinstance(obj, dict): - return {k: recursive_sum(v, weight, distributed) for k, v in obj.items()} - elif isinstance(obj, torch.Tensor): - assert obj.size() == weight.size(), (obj.size(), weight.size()) - obj = (obj * weight.type(obj.dtype)).sum() - if distributed: - torch.distributed.all_reduce(obj, op=ReduceOp.SUM) - return obj - elif obj is None: - return None - else: - raise ValueError(type(obj)) - - -def recursive_divide(a, b: torch.Tensor): - if isinstance(a, (tuple, list)): - return type(a)(recursive_divide(v, b) for v in a) - elif isinstance(a, dict): - return {k: recursive_divide(v, b) for k, v in a.items()} - elif isinstance(a, torch.Tensor): - assert a.size() == b.size(), (a.size(), b.size()) - return a / b.type(a.dtype) - elif a is None: - return None - else: - raise ValueError(type(a)) - - -def recursive_average(obj, weight: torch.Tensor, distributed: bool = False): - obj = recursive_sum(obj, weight, distributed) - weight = weight.sum() - if distributed: - torch.distributed.all_reduce(weight, op=ReduceOp.SUM) - # Normalize weight to be sum-to-1 - obj = recursive_divide(obj, weight) - return obj, weight diff --git a/funasr_detach/train_utils/set_all_random_seed.py b/funasr_detach/train_utils/set_all_random_seed.py deleted file mode 100644 index ebdca3f537aac53bdc6e6cea168c49805bdf2d2f..0000000000000000000000000000000000000000 --- a/funasr_detach/train_utils/set_all_random_seed.py +++ /dev/null @@ -1,10 +0,0 @@ -import random - -import numpy as np -import torch - - -def set_all_random_seed(seed: int): - random.seed(seed) - np.random.seed(seed) - torch.random.manual_seed(seed) diff --git a/funasr_detach/train_utils/trainer.py b/funasr_detach/train_utils/trainer.py deleted file mode 100644 index d6f218efe9f735fdb1f2903f031d856f62a3e573..0000000000000000000000000000000000000000 --- a/funasr_detach/train_utils/trainer.py +++ /dev/null @@ -1,417 +0,0 @@ -import os -import time -import torch -import logging -from tqdm import tqdm -from datetime import datetime -import torch.distributed as dist -from contextlib import nullcontext - -# from torch.utils.tensorboard import SummaryWriter -from tensorboardX import SummaryWriter -from pathlib import Path - -from funasr_detach.train_utils.device_funcs import to_device -from funasr_detach.train_utils.recursive_op import recursive_average -from funasr_detach.train_utils.average_nbest_models import average_checkpoints - - -class Trainer: - """ - A simple trainer class for training a PyTorch model, saving checkpoints at the end of each epoch, - and optionally resuming from a saved checkpoint. - - Attributes: - max_epoch (int): Maximum number of epochs for training. - model (torch.nn.Module): The model to be trained. - optim (torch.optim.Optimizer): The optimizer to use for training. - scheduler (torch.optim.lr_scheduler._LRScheduler): The learning rate scheduler. - dataloader_train (torch.utils.data.DataLoader): DataLoader for the training dataset. - dataloader_val (torch.utils.data.DataLoader): DataLoader for the validation dataset. - output_dir (str): Directory where model checkpoints will be saved. - resume (str, optional): Path to a checkpoint to resume training from. - """ - - def __init__( - self, - model, - optim, - scheduler, - dataloader_train, - dataloader_val, - local_rank, - use_ddp=False, - use_fsdp=False, - output_dir: str = "./", - **kwargs, - ): - """ - Initializes the Trainer class with the model, optimizer, scheduler, dataloaders, and other settings. - - Args: - model (torch.nn.Module): The model to be trained. - optim (torch.optim.Optimizer): The optimizer to use for training. - scheduler (torch.optim.lr_scheduler._LRScheduler): The learning rate scheduler. - dataloader_train (torch.utils.data.DataLoader): The DataLoader for the training dataset. - dataloader_val (torch.utils.data.DataLoader): The DataLoader for the validation dataset. - **kwargs: Additional keyword arguments: - max_epoch (int): The maximum number of epochs for training. - output_dir (str): The directory where model checkpoints will be saved. Default is './'. - resume (str, optional): The file path to a checkpoint to resume training from. - """ - - self.model = model - self.optim = optim - self.scheduler = scheduler - self.dataloader_train = dataloader_train - self.dataloader_val = dataloader_val - self.output_dir = output_dir - self.resume = kwargs.get("resume", True) - self.start_epoch = 0 - self.max_epoch = kwargs.get("max_epoch", 100) - self.local_rank = local_rank - self.use_ddp = use_ddp - self.use_fsdp = use_fsdp - self.device = next(model.parameters()).device - self.avg_nbest_model = kwargs.get("avg_nbest_model", 5) - self.kwargs = kwargs - self.log_interval = kwargs.get("log_interval", 50) - self.batch_total = 0 - - try: - rank = dist.get_rank() - world_size = dist.get_world_size() - except: - rank = 0 - world_size = 1 - logging.warning("distributed is not initialized, only single shard") - self.rank = rank - self.world_size = world_size - - os.makedirs(os.path.join(self.output_dir, "tensorboard"), exist_ok=True) - self.writer = ( - SummaryWriter(os.path.join(self.output_dir, "tensorboard")) - if rank == 0 - else None - ) - - def _save_checkpoint(self, epoch): - """ - Saves a checkpoint containing the model's state, the optimizer's state, - and the scheduler's state at the end of the given epoch. This method is - intended to be called at the end of each epoch to save the training progress. - - Args: - epoch (int): The epoch number at which the checkpoint is being saved. - """ - state = { - "epoch": epoch, - "state_dict": self.model.state_dict(), - "optimizer": self.optim.state_dict(), - "scheduler": self.scheduler.state_dict(), - } - # Create output directory if it does not exist - os.makedirs(self.output_dir, exist_ok=True) - filename = os.path.join(self.output_dir, f"model.pt.ep{epoch}") - torch.save(state, filename) - - print(f"\nCheckpoint saved to {filename}\n") - latest = Path(os.path.join(self.output_dir, f"model.pt")) - torch.save(state, latest) - - def _resume_checkpoint(self, resume_path): - """ - Resumes training from a checkpoint at the given file path. - Loads the model's state, the optimizer's state, and the scheduler's state. - - Args: - resume_path (str): The file path to the checkpoint to resume from. - """ - ckpt = os.path.join(resume_path, "model.pt") - if os.path.isfile(ckpt): - checkpoint = torch.load(ckpt) - self.start_epoch = checkpoint["epoch"] + 1 - # self.model.load_state_dict(checkpoint['state_dict']) - src_state = checkpoint["state_dict"] - dst_state = self.model.state_dict() - for k in dst_state.keys(): - if not k.startswith("module.") and "module." + k in src_state.keys(): - k_ddp = "module." + k - else: - k_ddp = k - if k_ddp in src_state.keys(): - dst_state[k] = src_state[k_ddp] - else: - print(f"Miss key in ckpt: model: {k}, ckpt: {k_ddp}") - - self.model.load_state_dict(dst_state) - self.optim.load_state_dict(checkpoint["optimizer"]) - self.scheduler.load_state_dict(checkpoint["scheduler"]) - print(f"Checkpoint loaded successfully from '{ckpt}'") - else: - print(f"No checkpoint found at '{ckpt}', starting from scratch") - - if self.use_ddp or self.use_fsdp: - dist.barrier() - - def run(self): - """ - Starts the training process, iterating over epochs, training the model, - and saving checkpoints at the end of each epoch. - """ - if self.resume: - self._resume_checkpoint(self.output_dir) - - for epoch in range(self.start_epoch, self.max_epoch + 1): - time1 = time.perf_counter() - self._train_epoch(epoch) - - if self.use_ddp or self.use_fsdp: - dist.barrier() - - self._validate_epoch(epoch) - - if self.use_ddp or self.use_fsdp: - dist.barrier() - - if self.rank == 0: - self._save_checkpoint(epoch) - - if self.use_ddp or self.use_fsdp: - dist.barrier() - - self.scheduler.step() - - time2 = time.perf_counter() - time_escaped = (time2 - time1) / 3600.0 - print( - f"\nrank: {self.local_rank}, time_escaped_epoch: {time_escaped:.3f} hours, estimated to finish {self.max_epoch} epoch: {(self.max_epoch-epoch)*time_escaped:.3f}\n" - ) - - if self.rank == 0: - average_checkpoints(self.output_dir, self.avg_nbest_model) - - if self.use_ddp or self.use_fsdp: - dist.barrier() - - if self.writer: - self.writer.close() - - def _train_epoch(self, epoch): - """ - Defines the training process for a single epoch with gradient accumulation. - Args: - epoch (int): The current epoch number. - """ - self.model.train() - pbar = tqdm( - colour="blue", - desc=f"rank: {self.local_rank}, Training Epoch: {epoch + 1}", - total=len(self.dataloader_train), - dynamic_ncols=True, - ) - - # Set the number of steps for gradient accumulation - accum_grad = self.kwargs.get("accum_grad", 1) - # Initialize the gradient accumulation - self.optim.zero_grad() - speed_stats = {} - time5 = time.perf_counter() - - for batch_idx, batch in enumerate(self.dataloader_train): - self.batch_total += 1 - time1 = time.perf_counter() - speed_stats["data_load"] = f"{time1-time5:0.3f}" - - batch = to_device(batch, self.device) - - my_context = ( - self.model.no_sync if batch_idx % accum_grad != 0 else nullcontext - ) - with my_context(): - time2 = time.perf_counter() - - retval = self.model(**batch) - torch.cuda.empty_cache() - - time3 = time.perf_counter() - speed_stats["forward_time"] = f"{time3 - time2:0.3f}" - loss, stats, weight = retval - stats = {k: v for k, v in stats.items() if v is not None} - if self.use_ddp or self.use_fsdp: - # Apply weighted averaging for loss and stats - loss = (loss * weight.type(loss.dtype)).sum() - # if distributed, this method can also apply all_reduce() - stats, weight = recursive_average(stats, weight, distributed=True) - # Now weight is summation over all workers - loss /= weight - # Multiply world_size because DistributedDataParallel - # automatically normalizes the gradient by world_size. - loss *= self.world_size - # Scale the loss since we're not updating for every mini-batch - loss = loss / accum_grad - loss.backward() - time4 = time.perf_counter() - speed_stats["backward_time"] = f"{time4 - time3:0.3f}" - - # Perform an optimizer step only after accumulating enough gradients - if (batch_idx + 1) % accum_grad == 0 or (batch_idx + 1) == len( - self.dataloader_train - ): - # Perform gradient clipping if it is set - if self.kwargs.get("grad_clip", None) is not None: - grad_norm = torch.nn.utils.clip_grad_norm_( - self.model.parameters(), - max_norm=self.kwargs.get("grad_clip", 10.0), - norm_type=self.kwargs.get("grad_clip_type", 2.0), - ) - if not torch.isfinite(grad_norm): - logging.warning( - f"The grad norm is {grad_norm}. Skipping updating the model." - ) - self.optim.zero_grad() # Reset gradients - continue - - # Execute an optimization step (update model parameters) - if self.use_ddp or self.use_fsdp: - dist.barrier() - self.optim.step() - self.scheduler.step() - # Clear gradients for the next accumulation stage - self.optim.zero_grad() - total_time = f"{time.perf_counter() - time5:0.3f}" - time5 = time.perf_counter() - speed_stats["optim_time"] = f"{time5 - time4:0.3f}" - - speed_stats["total_time"] = total_time - - if (batch_idx + 1) % self.log_interval == 0 or (batch_idx + 1) == len( - self.dataloader_train - ): - pbar.update(self.log_interval) - gpu_info = ( - "GPU, memory: {:.3f} GB, " - "{:.3f} GB, " - "{:.3f} GB, " - "{:.3f} GB".format( - torch.cuda.memory_allocated() / 1024 / 1024 / 1024, - torch.cuda.max_memory_allocated() / 1024 / 1024 / 1024, - torch.cuda.memory_reserved() / 1024 / 1024 / 1024, - torch.cuda.max_memory_reserved() / 1024 / 1024 / 1024, - ) - ) - lr = self.scheduler.get_last_lr()[0] - time_now = datetime.now() - time_now = time_now.strftime("%Y-%m-%d %H:%M:%S") - description = ( - f"{time_now}, " - f"rank: {self.local_rank}, " - f"epoch: {epoch}/{self.max_epoch}, " - f"step: {batch_idx+1}/{len(self.dataloader_train)}, total: {self.batch_total}, " - f"(loss: {loss.detach().cpu().item():.3f}), " - f"(lr: {lr:.3e}), " - f"{[(k, round(v.cpu().item(), 3)) for k, v in stats.items()]}, " - f"{speed_stats}, " - f"{gpu_info}" - ) - pbar.set_description(description) - if self.writer: - self.writer.add_scalar( - f"rank{self.local_rank}_Loss/train", - loss.item(), - self.batch_total, - ) - self.writer.add_scalar( - f"rank{self.local_rank}_lr/train", lr, self.batch_total - ) - for key, var in stats.items(): - self.writer.add_scalar( - f"rank{self.local_rank}_{key}/train", - var.item(), - self.batch_total, - ) - for key, var in speed_stats.items(): - self.writer.add_scalar( - f"rank{self.local_rank}_{key}/train", - eval(var), - self.batch_total, - ) - - pbar.close() - - def _validate_epoch(self, epoch): - """ - Defines the validation process for a single epoch. - Should be implemented with the actual model validation steps. - - Args: - epoch (int): The current epoch number. - """ - self.model.eval() - with torch.no_grad(): - pbar = tqdm( - colour="red", - desc=f"rank: {self.local_rank}, Validation Epoch: {epoch + 1}", - total=len(self.dataloader_val), - dynamic_ncols=True, - ) - speed_stats = {} - time5 = time.perf_counter() - for batch_idx, batch in enumerate(self.dataloader_val): - time1 = time.perf_counter() - speed_stats["data_load"] = f"{time1 - time5:0.3f}" - batch = to_device(batch, self.device) - time2 = time.perf_counter() - retval = self.model(**batch) - time3 = time.perf_counter() - speed_stats["forward_time"] = f"{time3 - time2:0.3f}" - loss, stats, weight = retval - stats = {k: v for k, v in stats.items() if v is not None} - if self.use_ddp or self.use_fsdp: - # Apply weighted averaging for loss and stats - loss = (loss * weight.type(loss.dtype)).sum() - # if distributed, this method can also apply all_reduce() - stats, weight = recursive_average(stats, weight, distributed=True) - # Now weight is summation over all workers - loss /= weight - # Multiply world_size because DistributedDataParallel - # automatically normalizes the gradient by world_size. - loss *= self.world_size - # Scale the loss since we're not updating for every mini-batch - loss = loss - time4 = time.perf_counter() - - if (batch_idx + 1) % self.log_interval == 0 or (batch_idx + 1) == len( - self.dataloader_val - ): - pbar.update(self.log_interval) - time_now = datetime.now() - time_now = time_now.strftime("%Y-%m-%d %H:%M:%S") - description = ( - f"{time_now}, " - f"rank: {self.local_rank}, " - f"validation epoch: {epoch}/{self.max_epoch}, " - f"step: {batch_idx+1}/{len(self.dataloader_val)}, " - f"(loss: {loss.detach().cpu().item():.3f}), " - f"{[(k, round(v.cpu().item(), 3)) for k, v in stats.items()]}, " - f"{speed_stats}, " - ) - pbar.set_description(description) - if self.writer: - self.writer.add_scalar( - f"rank{self.local_rank}_Loss/val", - loss.item(), - epoch * len(self.dataloader_val) + batch_idx, - ) - for key, var in stats.items(): - self.writer.add_scalar( - f"rank{self.local_rank}_{key}/val", - var.item(), - epoch * len(self.dataloader_val) + batch_idx, - ) - for key, var in speed_stats.items(): - self.writer.add_scalar( - f"rank{self.local_rank}_{key}/val", - eval(var), - epoch * len(self.dataloader_val) + batch_idx, - ) diff --git a/funasr_detach/utils/__init__.py b/funasr_detach/utils/__init__.py deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/funasr_detach/utils/datadir_writer.py b/funasr_detach/utils/datadir_writer.py deleted file mode 100644 index dd64018e9feffa40d4f8e29f21f5aaf4f3eb113e..0000000000000000000000000000000000000000 --- a/funasr_detach/utils/datadir_writer.py +++ /dev/null @@ -1,71 +0,0 @@ -from pathlib import Path -from typing import Union -import warnings - - -class DatadirWriter: - """Writer class to create kaldi like data directory. - - Examples: - >>> with DatadirWriter("output") as writer: - ... # output/sub.txt is created here - ... subwriter = writer["sub.txt"] - ... # Write "uttidA some/where/a.wav" - ... subwriter["uttidA"] = "some/where/a.wav" - ... subwriter["uttidB"] = "some/where/b.wav" - - """ - - def __init__(self, p: Union[Path, str]): - self.path = Path(p) - self.chilidren = {} - self.fd = None - self.has_children = False - self.keys = set() - - def __enter__(self): - return self - - def __getitem__(self, key: str) -> "DatadirWriter": - if self.fd is not None: - raise RuntimeError("This writer points out a file") - - if key not in self.chilidren: - w = DatadirWriter((self.path / key)) - self.chilidren[key] = w - self.has_children = True - - retval = self.chilidren[key] - return retval - - def __setitem__(self, key: str, value: str): - if self.has_children: - raise RuntimeError("This writer points out a directory") - if key in self.keys: - warnings.warn(f"Duplicated: {key}") - - if self.fd is None: - self.path.parent.mkdir(parents=True, exist_ok=True) - self.fd = self.path.open("w", encoding="utf-8") - - self.keys.add(key) - self.fd.write(f"{key} {value}\n") - self.fd.flush() - - def __exit__(self, exc_type, exc_val, exc_tb): - self.close() - - def close(self): - if self.has_children: - prev_child = None - for child in self.chilidren.values(): - child.close() - if prev_child is not None and prev_child.keys != child.keys: - warnings.warn( - f"Ids are mismatching between " - f"{prev_child.path} and {child.path}" - ) - prev_child = child - - elif self.fd is not None: - self.fd.close() diff --git a/funasr_detach/utils/load_utils.py b/funasr_detach/utils/load_utils.py deleted file mode 100644 index 548f41a6a0a1f1a2365149cbeff99f55d94175d5..0000000000000000000000000000000000000000 --- a/funasr_detach/utils/load_utils.py +++ /dev/null @@ -1,148 +0,0 @@ -import os -import io -import torch -import numpy as np -import torchaudio -from torch.nn.utils.rnn import pad_sequence - -try: - from funasr_detach.download.file import download_from_url -except: - print("urllib is not installed, if you infer from url, please install it first.") - - -def load_audio_text_image_video( - data_or_path_or_list, - fs: int = 16000, - audio_fs: int = 16000, - data_type="sound", - tokenizer=None, - **kwargs -): - if isinstance(data_or_path_or_list, (list, tuple)): - if data_type is not None and isinstance(data_type, (list, tuple)): - - data_types = [data_type] * len(data_or_path_or_list) - data_or_path_or_list_ret = [[] for d in data_type] - for i, (data_type_i, data_or_path_or_list_i) in enumerate( - zip(data_types, data_or_path_or_list) - ): - - for j, (data_type_j, data_or_path_or_list_j) in enumerate( - zip(data_type_i, data_or_path_or_list_i) - ): - - data_or_path_or_list_j = load_audio_text_image_video( - data_or_path_or_list_j, - fs=fs, - audio_fs=audio_fs, - data_type=data_type_j, - tokenizer=tokenizer, - **kwargs - ) - data_or_path_or_list_ret[j].append(data_or_path_or_list_j) - - return data_or_path_or_list_ret - else: - return [ - load_audio_text_image_video( - audio, fs=fs, audio_fs=audio_fs, data_type=data_type, **kwargs - ) - for audio in data_or_path_or_list - ] - - if isinstance(data_or_path_or_list, str) and data_or_path_or_list.startswith( - "http" - ): # download url to local file - data_or_path_or_list = download_from_url(data_or_path_or_list) - - if isinstance(data_or_path_or_list, io.BytesIO): - data_or_path_or_list, audio_fs = torchaudio.load(data_or_path_or_list) - if kwargs.get("reduce_channels", True): - data_or_path_or_list = data_or_path_or_list.mean(0) - elif isinstance(data_or_path_or_list, str) and os.path.exists( - data_or_path_or_list - ): # local file - if data_type is None or data_type == "sound": - data_or_path_or_list, audio_fs = torchaudio.load(data_or_path_or_list) - if kwargs.get("reduce_channels", True): - data_or_path_or_list = data_or_path_or_list.mean(0) - elif data_type == "text" and tokenizer is not None: - data_or_path_or_list = tokenizer.encode(data_or_path_or_list) - elif data_type == "image": # undo - pass - elif data_type == "video": # undo - pass - - # if data_in is a file or url, set is_final=True - if "cache" in kwargs: - kwargs["cache"]["is_final"] = True - kwargs["cache"]["is_streaming_input"] = False - elif ( - isinstance(data_or_path_or_list, str) - and data_type == "text" - and tokenizer is not None - ): - data_or_path_or_list = tokenizer.encode(data_or_path_or_list) - elif isinstance(data_or_path_or_list, np.ndarray): # audio sample point - data_or_path_or_list = torch.from_numpy( - data_or_path_or_list - ).squeeze() # [n_samples,] - else: - pass - # print(f"unsupport data type: {data_or_path_or_list}, return raw data") - - if audio_fs != fs and data_type != "text": - resampler = torchaudio.transforms.Resample(audio_fs, fs) - data_or_path_or_list = resampler(data_or_path_or_list[None, :])[0, :] - return data_or_path_or_list - - -def load_bytes(input): - middle_data = np.frombuffer(input, dtype=np.int16) - middle_data = np.asarray(middle_data) - if middle_data.dtype.kind not in "iu": - raise TypeError("'middle_data' must be an array of integers") - dtype = np.dtype("float32") - if dtype.kind != "f": - raise TypeError("'dtype' must be a floating point type") - - i = np.iinfo(middle_data.dtype) - abs_max = 2 ** (i.bits - 1) - offset = i.min + abs_max - array = np.frombuffer( - (middle_data.astype(dtype) - offset) / abs_max, dtype=np.float32 - ) - return array - - -def extract_fbank( - data, data_len=None, data_type: str = "sound", frontend=None, **kwargs -): - # import pdb; - # pdb.set_trace() - if isinstance(data, np.ndarray): - data = torch.from_numpy(data) - if len(data.shape) < 2: - data = data[None, :] # data: [batch, N] - data_len = [data.shape[1]] if data_len is None else data_len - elif isinstance(data, torch.Tensor): - if len(data.shape) < 2: - data = data[None, :] # data: [batch, N] - data_len = [data.shape[1]] if data_len is None else data_len - elif isinstance(data, (list, tuple)): - data_list, data_len = [], [] - for data_i in data: - if isinstance(data_i, np.ndarray): - data_i = torch.from_numpy(data_i) - data_list.append(data_i) - data_len.append(data_i.shape[0]) - data = pad_sequence(data_list, batch_first=True) # data: [batch, N] - # import pdb; - # pdb.set_trace() - # if data_type == "sound": - data, data_len = frontend(data, data_len, **kwargs) - - if isinstance(data_len, (list, tuple)): - data_len = torch.tensor([data_len]) - return data.to(torch.float32), data_len.to(torch.int32) diff --git a/funasr_detach/utils/misc.py b/funasr_detach/utils/misc.py deleted file mode 100644 index b8aaa75333be867754934ba85b27b2cfbcccb351..0000000000000000000000000000000000000000 --- a/funasr_detach/utils/misc.py +++ /dev/null @@ -1,50 +0,0 @@ -import io -from collections import OrderedDict -import numpy as np - - -def statistic_model_parameters(model, prefix=None): - var_dict = model.state_dict() - numel = 0 - for i, key in enumerate( - sorted(list([x for x in var_dict.keys() if "num_batches_tracked" not in x])) - ): - if prefix is None or key.startswith(prefix): - numel += var_dict[key].numel() - return numel - - -def int2vec(x, vec_dim=8, dtype=np.int32): - b = ("{:0" + str(vec_dim) + "b}").format(x) - # little-endian order: lower bit first - return (np.array(list(b)[::-1]) == "1").astype(dtype) - - -def seq2arr(seq, vec_dim=8): - return np.row_stack([int2vec(int(x), vec_dim) for x in seq]) - - -def load_scp_as_dict(scp_path, value_type="str", kv_sep=" "): - with io.open(scp_path, "r", encoding="utf-8") as f: - ret_dict = OrderedDict() - for one_line in f.readlines(): - one_line = one_line.strip() - pos = one_line.find(kv_sep) - key, value = one_line[:pos], one_line[pos + 1 :] - if value_type == "list": - value = value.split(" ") - ret_dict[key] = value - return ret_dict - - -def load_scp_as_list(scp_path, value_type="str", kv_sep=" "): - with io.open(scp_path, "r", encoding="utf8") as f: - ret_dict = [] - for one_line in f.readlines(): - one_line = one_line.strip() - pos = one_line.find(kv_sep) - key, value = one_line[:pos], one_line[pos + 1 :] - if value_type == "list": - value = value.split(" ") - ret_dict.append((key, value)) - return ret_dict diff --git a/funasr_detach/utils/postprocess_utils.py b/funasr_detach/utils/postprocess_utils.py deleted file mode 100644 index 34f933c967d4add7680ad371cd2565af708d461b..0000000000000000000000000000000000000000 --- a/funasr_detach/utils/postprocess_utils.py +++ /dev/null @@ -1,301 +0,0 @@ -# Copyright (c) Alibaba, Inc. and its affiliates. - -import string -import logging -from typing import Any, List, Union - - -def isChinese(ch: str): - if "\u4e00" <= ch <= "\u9fff" or "\u0030" <= ch <= "\u0039" or ch == "@": - return True - return False - - -def isAllChinese(word: Union[List[Any], str]): - word_lists = [] - for i in word: - cur = i.replace(" ", "") - cur = cur.replace("", "") - cur = cur.replace("", "") - cur = cur.replace("", "") - cur = cur.replace("", "") - word_lists.append(cur) - - if len(word_lists) == 0: - return False - - for ch in word_lists: - if isChinese(ch) is False: - return False - return True - - -def isAllAlpha(word: Union[List[Any], str]): - word_lists = [] - for i in word: - cur = i.replace(" ", "") - cur = cur.replace("", "") - cur = cur.replace("", "") - cur = cur.replace("", "") - cur = cur.replace("", "") - word_lists.append(cur) - - if len(word_lists) == 0: - return False - - for ch in word_lists: - if ch.isalpha() is False and ch != "'": - return False - elif ch.isalpha() is True and isChinese(ch) is True: - return False - - return True - - -# def abbr_dispose(words: List[Any]) -> List[Any]: -def abbr_dispose(words: List[Any], time_stamp: List[List] = None) -> List[Any]: - words_size = len(words) - word_lists = [] - abbr_begin = [] - abbr_end = [] - last_num = -1 - ts_lists = [] - ts_nums = [] - ts_index = 0 - for num in range(words_size): - if num <= last_num: - continue - - if len(words[num]) == 1 and words[num].encode("utf-8").isalpha(): - if ( - num + 1 < words_size - and words[num + 1] == " " - and num + 2 < words_size - and len(words[num + 2]) == 1 - and words[num + 2].encode("utf-8").isalpha() - ): - # found the begin of abbr - abbr_begin.append(num) - num += 2 - abbr_end.append(num) - # to find the end of abbr - while True: - num += 1 - if num < words_size and words[num] == " ": - num += 1 - if ( - num < words_size - and len(words[num]) == 1 - and words[num].encode("utf-8").isalpha() - ): - abbr_end.pop() - abbr_end.append(num) - last_num = num - else: - break - else: - break - - for num in range(words_size): - if words[num] == " ": - ts_nums.append(ts_index) - else: - ts_nums.append(ts_index) - ts_index += 1 - last_num = -1 - for num in range(words_size): - if num <= last_num: - continue - - if num in abbr_begin: - if time_stamp is not None: - begin = time_stamp[ts_nums[num]][0] - abbr_word = words[num].upper() - num += 1 - while num < words_size: - if num in abbr_end: - abbr_word += words[num].upper() - last_num = num - break - else: - if words[num].encode("utf-8").isalpha(): - abbr_word += words[num].upper() - num += 1 - word_lists.append(abbr_word) - if time_stamp is not None: - end = time_stamp[ts_nums[num]][1] - ts_lists.append([begin, end]) - else: - word_lists.append(words[num]) - if time_stamp is not None and words[num] != " ": - begin = time_stamp[ts_nums[num]][0] - end = time_stamp[ts_nums[num]][1] - ts_lists.append([begin, end]) - begin = end - - if time_stamp is not None: - return word_lists, ts_lists - else: - return word_lists - - -def sentence_postprocess(words: List[Any], time_stamp: List[List] = None): - middle_lists = [] - word_lists = [] - word_item = "" - ts_lists = [] - - # wash words lists - for i in words: - word = "" - if isinstance(i, str): - word = i - else: - word = i.decode("utf-8") - - if word in ["", "", "", ""]: - continue - else: - middle_lists.append(word) - - # all chinese characters - if isAllChinese(middle_lists): - for i, ch in enumerate(middle_lists): - word_lists.append(ch.replace(" ", "")) - if time_stamp is not None: - ts_lists = time_stamp - - # all alpha characters - elif isAllAlpha(middle_lists): - ts_flag = True - for i, ch in enumerate(middle_lists): - if ts_flag and time_stamp is not None: - begin = time_stamp[i][0] - end = time_stamp[i][1] - word = "" - if "@@" in ch: - word = ch.replace("@@", "") - word_item += word - if time_stamp is not None: - ts_flag = False - end = time_stamp[i][1] - else: - word_item += ch - word_lists.append(word_item) - word_lists.append(" ") - word_item = "" - if time_stamp is not None: - ts_flag = True - end = time_stamp[i][1] - ts_lists.append([begin, end]) - begin = end - - # mix characters - else: - alpha_blank = False - ts_flag = True - begin = -1 - end = -1 - for i, ch in enumerate(middle_lists): - if ts_flag and time_stamp is not None: - begin = time_stamp[i][0] - end = time_stamp[i][1] - word = "" - if isAllChinese(ch): - if alpha_blank is True: - word_lists.pop() - word_lists.append(ch) - alpha_blank = False - if time_stamp is not None: - ts_flag = True - ts_lists.append([begin, end]) - begin = end - elif "@@" in ch: - word = ch.replace("@@", "") - word_item += word - alpha_blank = False - if time_stamp is not None: - ts_flag = False - end = time_stamp[i][1] - elif isAllAlpha(ch): - word_item += ch - word_lists.append(word_item) - word_lists.append(" ") - word_item = "" - alpha_blank = True - if time_stamp is not None: - ts_flag = True - end = time_stamp[i][1] - ts_lists.append([begin, end]) - begin = end - else: - word_lists.append(ch) - - if time_stamp is not None: - word_lists, ts_lists = abbr_dispose(word_lists, ts_lists) - real_word_lists = [] - for ch in word_lists: - if ch != " ": - real_word_lists.append(ch) - sentence = " ".join(real_word_lists).strip() - return sentence, ts_lists, real_word_lists - else: - word_lists = abbr_dispose(word_lists) - real_word_lists = [] - for ch in word_lists: - if ch != " ": - real_word_lists.append(ch) - sentence = "".join(word_lists).strip() - return sentence, real_word_lists - - -def sentence_postprocess_sentencepiece(words): - middle_lists = [] - word_lists = [] - word_item = "" - - # wash words lists - for i in words: - word = "" - if isinstance(i, str): - word = i - else: - word = i.decode("utf-8") - - if word in ["", "", "", ""]: - continue - else: - middle_lists.append(word) - - # all alpha characters - for i, ch in enumerate(middle_lists): - word = "" - if "\u2581" in ch and i == 0: - word_item = "" - word = ch.replace("\u2581", "") - word_item += word - elif "\u2581" in ch and i != 0: - word_lists.append(word_item) - word_lists.append(" ") - word_item = "" - word = ch.replace("\u2581", "") - word_item += word - else: - word_item += ch - if word_item is not None: - word_lists.append(word_item) - # word_lists = abbr_dispose(word_lists) - real_word_lists = [] - for ch in word_lists: - if ch != " ": - if ch == "i": - ch = ch.replace("i", "I") - elif ch == "i'm": - ch = ch.replace("i'm", "I'm") - elif ch == "i've": - ch = ch.replace("i've", "I've") - elif ch == "i'll": - ch = ch.replace("i'll", "I'll") - real_word_lists.append(ch) - sentence = "".join(word_lists) - return sentence, real_word_lists diff --git a/funasr_detach/utils/prepare_data.py b/funasr_detach/utils/prepare_data.py deleted file mode 100644 index 356cd3651a1690cd9cb46409a97498c2f6ea972e..0000000000000000000000000000000000000000 --- a/funasr_detach/utils/prepare_data.py +++ /dev/null @@ -1,308 +0,0 @@ -import logging -import os -import shutil -from multiprocessing import Pool - -import kaldiio -import numpy as np -import librosa -import torch.distributed as dist -import torchaudio - - -def filter_wav_text(data_dir, dataset): - wav_file = os.path.join(data_dir, dataset, "wav.scp") - text_file = os.path.join(data_dir, dataset, "text") - with open(wav_file) as f_wav, open(text_file) as f_text: - wav_lines = f_wav.readlines() - text_lines = f_text.readlines() - os.rename(wav_file, "{}.bak".format(wav_file)) - os.rename(text_file, "{}.bak".format(text_file)) - wav_dict = {} - for line in wav_lines: - parts = line.strip().split() - if len(parts) < 2: - continue - wav_dict[parts[0]] = parts[1] - text_dict = {} - for line in text_lines: - parts = line.strip().split() - if len(parts) < 2: - continue - text_dict[parts[0]] = " ".join(parts[1:]) - filter_count = 0 - with open(wav_file, "w") as f_wav, open(text_file, "w") as f_text: - for sample_name, wav_path in wav_dict.items(): - if sample_name in text_dict.keys(): - f_wav.write(sample_name + " " + wav_path + "\n") - f_text.write(sample_name + " " + text_dict[sample_name] + "\n") - else: - filter_count += 1 - logging.info( - "{}/{} samples in {} are filtered because of the mismatch between wav.scp and text".format( - filter_count, len(wav_lines), dataset - ) - ) - - -def wav2num_frame(wav_path, frontend_conf): - try: - waveform, sampling_rate = torchaudio.load(wav_path) - except: - waveform, sampling_rate = librosa.load(wav_path) - waveform = np.expand_dims(waveform, axis=0) - n_frames = (waveform.shape[1] * 1000.0) / ( - sampling_rate * frontend_conf["frame_shift"] * frontend_conf["lfr_n"] - ) - feature_dim = frontend_conf["n_mels"] * frontend_conf["lfr_m"] - return n_frames, feature_dim - - -def calc_shape_core(root_path, args, idx): - file_name = args.data_file_names.split(",")[0] - data_name = args.dataset_conf.get("data_names", "speech,text").split(",")[0] - scp_file = os.path.join(root_path, "{}.{}".format(file_name, idx)) - shape_file = os.path.join(root_path, "{}_shape.{}".format(data_name, idx)) - with open(scp_file) as f: - lines = f.readlines() - data_type = args.dataset_conf.get("data_types", "sound,text").split(",")[0] - if data_type == "sound": - frontend_conf = args.frontend_conf - dataset_conf = args.dataset_conf - length_min = ( - dataset_conf.speech_length_min - if hasattr(dataset_conf, "{}_length_min".format(data_name)) - else -1 - ) - length_max = ( - dataset_conf.speech_length_max - if hasattr(dataset_conf, "{}_length_max".format(data_name)) - else -1 - ) - with open(shape_file, "w") as f: - for line in lines: - sample_name, wav_path = line.strip().split() - n_frames, feature_dim = wav2num_frame(wav_path, frontend_conf) - write_flag = True - if n_frames > 0 and length_min > 0: - write_flag = n_frames >= length_min - if n_frames > 0 and length_max > 0: - write_flag = n_frames <= length_max - if write_flag: - f.write( - "{} {},{}\n".format( - sample_name, - str(int(np.ceil(n_frames))), - str(int(feature_dim)), - ) - ) - f.flush() - elif data_type == "kaldi_ark": - dataset_conf = args.dataset_conf - length_min = ( - dataset_conf.speech_length_min - if hasattr(dataset_conf, "{}_length_min".format(data_name)) - else -1 - ) - length_max = ( - dataset_conf.speech_length_max - if hasattr(dataset_conf, "{}_length_max".format(data_name)) - else -1 - ) - with open(shape_file, "w") as f: - for line in lines: - sample_name, feature_path = line.strip().split() - feature = kaldiio.load_mat(feature_path) - n_frames, feature_dim = feature.shape - write_flag = True - if n_frames > 0 and length_min > 0: - write_flag = n_frames >= length_min - if n_frames > 0 and length_max > 0: - write_flag = n_frames <= length_max - if write_flag: - f.write( - "{} {},{}\n".format( - sample_name, - str(int(np.ceil(n_frames))), - str(int(feature_dim)), - ) - ) - f.flush() - elif data_type == "text": - with open(shape_file, "w") as f: - for line in lines: - sample_name, text = line.strip().split(maxsplit=1) - n_tokens = len(text.split()) - f.write("{} {}\n".format(sample_name, str(int(np.ceil(n_tokens))))) - f.flush() - else: - raise RuntimeError("Unsupported data_type: {}".format(data_type)) - - -def calc_shape(args, dataset, nj=64): - data_name = args.dataset_conf.get("data_names", "speech,text").split(",")[0] - shape_path = os.path.join(args.data_dir, dataset, "{}_shape".format(data_name)) - if os.path.exists(shape_path): - logging.info("Shape file for small dataset already exists.") - return - - split_shape_path = os.path.join( - args.data_dir, dataset, "{}_shape_files".format(data_name) - ) - if os.path.exists(split_shape_path): - shutil.rmtree(split_shape_path) - os.mkdir(split_shape_path) - - # split - file_name = args.data_file_names.split(",")[0] - scp_file = os.path.join(args.data_dir, dataset, file_name) - with open(scp_file) as f: - lines = f.readlines() - num_lines = len(lines) - num_job_lines = num_lines // nj - start = 0 - for i in range(nj): - end = start + num_job_lines - file = os.path.join(split_shape_path, "{}.{}".format(file_name, str(i + 1))) - with open(file, "w") as f: - if i == nj - 1: - f.writelines(lines[start:]) - else: - f.writelines(lines[start:end]) - start = end - - p = Pool(nj) - for i in range(nj): - p.apply_async(calc_shape_core, args=(split_shape_path, args, str(i + 1))) - logging.info("Generating shape files, please wait a few minutes...") - p.close() - p.join() - - # combine - with open(shape_path, "w") as f: - for i in range(nj): - job_file = os.path.join( - split_shape_path, "{}_shape.{}".format(data_name, str(i + 1)) - ) - with open(job_file) as job_f: - lines = job_f.readlines() - f.writelines(lines) - logging.info("Generating shape files done.") - - -def generate_data_list(args, data_dir, dataset, nj=64): - data_names = args.dataset_conf.get("data_names", "speech,text").split(",") - file_names = args.data_file_names.split(",") - concat_data_name = "_".join(data_names) - list_file = os.path.join(data_dir, dataset, "{}_data.list".format(concat_data_name)) - if os.path.exists(list_file): - logging.info("Data list for large dataset already exists.") - return - split_path = os.path.join(data_dir, dataset, "split") - if os.path.exists(split_path): - shutil.rmtree(split_path) - os.mkdir(split_path) - - data_lines_list = [] - for file_name in file_names: - with open(os.path.join(data_dir, dataset, file_name)) as f: - lines = f.readlines() - data_lines_list.append(lines) - num_lines = len(data_lines_list[0]) - num_job_lines = num_lines // nj - start = 0 - for i in range(nj): - end = start + num_job_lines - split_path_nj = os.path.join(split_path, str(i + 1)) - os.mkdir(split_path_nj) - for file_id, file_name in enumerate(file_names): - file = os.path.join(split_path_nj, file_name) - with open(file, "w") as f: - if i == nj - 1: - f.writelines(data_lines_list[file_id][start:]) - else: - f.writelines(data_lines_list[file_id][start:end]) - start = end - - with open(list_file, "w") as f_data: - for i in range(nj): - path = "" - for file_name in file_names: - path = path + " " + os.path.join(split_path, str(i + 1), file_name) - f_data.write(path + "\n") - - -def prepare_data(args, distributed_option): - data_names = args.dataset_conf.get("data_names", "speech,text").split(",") - data_types = args.dataset_conf.get("data_types", "sound,text").split(",") - file_names = args.data_file_names.split(",") - batch_type = args.dataset_conf["batch_conf"]["batch_type"] - print( - "data_names: {}, data_types: {}, file_names: {}".format( - data_names, data_types, file_names - ) - ) - assert len(data_names) == len(data_types) == len(file_names) - if args.dataset_type == "small": - args.train_shape_file = [ - os.path.join( - args.data_dir, args.train_set, "{}_shape".format(data_names[0]) - ) - ] - args.valid_shape_file = [ - os.path.join( - args.data_dir, args.valid_set, "{}_shape".format(data_names[0]) - ) - ] - ( - args.train_data_path_and_name_and_type, - args.valid_data_path_and_name_and_type, - ) = ([], []) - for file_name, data_name, data_type in zip(file_names, data_names, data_types): - args.train_data_path_and_name_and_type.append( - [ - "{}/{}/{}".format(args.data_dir, args.train_set, file_name), - data_name, - data_type, - ] - ) - args.valid_data_path_and_name_and_type.append( - [ - "{}/{}/{}".format(args.data_dir, args.valid_set, file_name), - data_name, - data_type, - ] - ) - if os.path.exists(args.train_shape_file[0]): - assert os.path.exists(args.valid_shape_file[0]) - print("shape file for small dataset already exists.") - return - else: - concat_data_name = "_".join(data_names) - args.train_data_file = os.path.join( - args.data_dir, args.train_set, "{}_data.list".format(concat_data_name) - ) - args.valid_data_file = os.path.join( - args.data_dir, args.valid_set, "{}_data.list".format(concat_data_name) - ) - if os.path.exists(args.train_data_file): - assert os.path.exists(args.valid_data_file) - print("data list for large dataset already exists.") - return - - distributed = distributed_option.distributed - if not distributed or distributed_option.dist_rank == 0: - if hasattr(args, "filter_input") and args.filter_input: - filter_wav_text(args.data_dir, args.train_set) - filter_wav_text(args.data_dir, args.valid_set) - - if args.dataset_type == "small" and batch_type != "unsorted": - calc_shape(args, args.train_set) - calc_shape(args, args.valid_set) - - if args.dataset_type == "large": - generate_data_list(args, args.data_dir, args.train_set) - generate_data_list(args, args.data_dir, args.valid_set) - - if distributed: - dist.barrier() diff --git a/funasr_detach/utils/speaker_utils.py b/funasr_detach/utils/speaker_utils.py deleted file mode 100644 index 7a7b27f9fcb735796c0df6533c17ae3613ddeb95..0000000000000000000000000000000000000000 --- a/funasr_detach/utils/speaker_utils.py +++ /dev/null @@ -1,200 +0,0 @@ -# Copyright (c) Alibaba, Inc. and its affiliates. -"""Some implementations are adapted from https://github.com/yuyq96/D-TDNN""" - -import io -from typing import Union - -import librosa as sf -import numpy as np -import torch -import torch.nn.functional as F -import torchaudio.compliance.kaldi as Kaldi -from torch import nn - -from funasr_detach.utils.modelscope_file import File - - -def check_audio_list(audio: list): - audio_dur = 0 - for i in range(len(audio)): - seg = audio[i] - assert seg[1] >= seg[0], "modelscope error: Wrong time stamps." - assert isinstance(seg[2], np.ndarray), "modelscope error: Wrong data type." - assert ( - int(seg[1] * 16000) - int(seg[0] * 16000) == seg[2].shape[0] - ), "modelscope error: audio data in list is inconsistent with time length." - if i > 0: - assert seg[0] >= audio[i - 1][1], "modelscope error: Wrong time stamps." - audio_dur += seg[1] - seg[0] - return audio_dur - # assert audio_dur > 5, 'modelscope error: The effective audio duration is too short.' - - -def sv_preprocess(inputs: Union[np.ndarray, list]): - output = [] - for i in range(len(inputs)): - if isinstance(inputs[i], str): - file_bytes = File.read(inputs[i]) - data, fs = sf.load(io.BytesIO(file_bytes), dtype="float32") - if len(data.shape) == 2: - data = data[:, 0] - data = torch.from_numpy(data).unsqueeze(0) - data = data.squeeze(0) - elif isinstance(inputs[i], np.ndarray): - assert ( - len(inputs[i].shape) == 1 - ), "modelscope error: Input array should be [N, T]" - data = inputs[i] - if data.dtype in ["int16", "int32", "int64"]: - data = (data / (1 << 15)).astype("float32") - else: - data = data.astype("float32") - data = torch.from_numpy(data) - else: - raise ValueError( - "modelscope error: The input type is restricted to audio address and nump array." - ) - output.append(data) - return output - - -def sv_chunk(vad_segments: list, fs=16000) -> list: - config = { - "seg_dur": 1.5, - "seg_shift": 0.75, - } - - def seg_chunk(seg_data): - seg_st = seg_data[0] - data = seg_data[2] - chunk_len = int(config["seg_dur"] * fs) - chunk_shift = int(config["seg_shift"] * fs) - last_chunk_ed = 0 - seg_res = [] - for chunk_st in range(0, data.shape[0], chunk_shift): - chunk_ed = min(chunk_st + chunk_len, data.shape[0]) - if chunk_ed <= last_chunk_ed: - break - last_chunk_ed = chunk_ed - chunk_st = max(0, chunk_ed - chunk_len) - chunk_data = data[chunk_st:chunk_ed] - if chunk_data.shape[0] < chunk_len: - chunk_data = np.pad( - chunk_data, (0, chunk_len - chunk_data.shape[0]), "constant" - ) - seg_res.append([chunk_st / fs + seg_st, chunk_ed / fs + seg_st, chunk_data]) - return seg_res - - segs = [] - for i, s in enumerate(vad_segments): - segs.extend(seg_chunk(s)) - - return segs - - -def extract_feature(audio): - features = [] - for au in audio: - feature = Kaldi.fbank(au.unsqueeze(0), num_mel_bins=80) - feature = feature - feature.mean(dim=0, keepdim=True) - features.append(feature.unsqueeze(0)) - features = torch.cat(features) - return features - - -def postprocess( - segments: list, vad_segments: list, labels: np.ndarray, embeddings: np.ndarray -) -> list: - assert len(segments) == len(labels) - labels = correct_labels(labels) - distribute_res = [] - for i in range(len(segments)): - distribute_res.append([segments[i][0], segments[i][1], labels[i]]) - # merge the same speakers chronologically - distribute_res = merge_seque(distribute_res) - - # accquire speaker center - spk_embs = [] - for i in range(labels.max() + 1): - spk_emb = embeddings[labels == i].mean(0) - spk_embs.append(spk_emb) - spk_embs = np.stack(spk_embs) - - def is_overlapped(t1, t2): - if t1 > t2 + 1e-4: - return True - return False - - # distribute the overlap region - for i in range(1, len(distribute_res)): - if is_overlapped(distribute_res[i - 1][1], distribute_res[i][0]): - p = (distribute_res[i][0] + distribute_res[i - 1][1]) / 2 - distribute_res[i][0] = p - distribute_res[i - 1][1] = p - - # smooth the result - distribute_res = smooth(distribute_res) - - return distribute_res - - -def correct_labels(labels): - labels_id = 0 - id2id = {} - new_labels = [] - for i in labels: - if i not in id2id: - id2id[i] = labels_id - labels_id += 1 - new_labels.append(id2id[i]) - return np.array(new_labels) - - -def merge_seque(distribute_res): - res = [distribute_res[0]] - for i in range(1, len(distribute_res)): - if distribute_res[i][2] != res[-1][2] or distribute_res[i][0] > res[-1][1]: - res.append(distribute_res[i]) - else: - res[-1][1] = distribute_res[i][1] - return res - - -def smooth(res, mindur=1): - # short segments are assigned to nearest speakers. - for i in range(len(res)): - res[i][0] = round(res[i][0], 2) - res[i][1] = round(res[i][1], 2) - if res[i][1] - res[i][0] < mindur: - if i == 0: - res[i][2] = res[i + 1][2] - elif i == len(res) - 1: - res[i][2] = res[i - 1][2] - elif res[i][0] - res[i - 1][1] <= res[i + 1][0] - res[i][1]: - res[i][2] = res[i - 1][2] - else: - res[i][2] = res[i + 1][2] - # merge the speakers - res = merge_seque(res) - - return res - - -def distribute_spk(sentence_list, sd_time_list): - sd_sentence_list = [] - for d in sentence_list: - sentence_start = d["ts_list"][0][0] - sentence_end = d["ts_list"][-1][1] - sentence_spk = 0 - max_overlap = 0 - for sd_time in sd_time_list: - spk_st, spk_ed, spk = sd_time - spk_st = spk_st * 1000 - spk_ed = spk_ed * 1000 - overlap = max(min(sentence_end, spk_ed) - max(sentence_start, spk_st), 0) - if overlap > max_overlap: - max_overlap = overlap - sentence_spk = spk - d["spk"] = sentence_spk - sd_sentence_list.append(d) - return sd_sentence_list diff --git a/funasr_detach/utils/timestamp_tools.py b/funasr_detach/utils/timestamp_tools.py deleted file mode 100644 index 6bc8cb1f8c4d01c5a4f8059d06303d053d8738d9..0000000000000000000000000000000000000000 --- a/funasr_detach/utils/timestamp_tools.py +++ /dev/null @@ -1,201 +0,0 @@ -import torch -import codecs -import logging -import argparse -import numpy as np - -# import edit_distance -from itertools import zip_longest - - -def cif_wo_hidden(alphas, threshold): - batch_size, len_time = alphas.size() - # loop varss - integrate = torch.zeros([batch_size], device=alphas.device) - # intermediate vars along time - list_fires = [] - for t in range(len_time): - alpha = alphas[:, t] - integrate += alpha - list_fires.append(integrate) - fire_place = integrate >= threshold - integrate = torch.where( - fire_place, - integrate - torch.ones([batch_size], device=alphas.device) * threshold, - integrate, - ) - fires = torch.stack(list_fires, 1) - return fires - - -def ts_prediction_lfr6_standard( - us_alphas, - us_peaks, - char_list, - vad_offset=0.0, - force_time_shift=-1.5, - sil_in_str=True, -): - if not len(char_list): - return "", [] - START_END_THRESHOLD = 5 - MAX_TOKEN_DURATION = 12 - TIME_RATE = 10.0 * 6 / 1000 / 3 # 3 times upsampled - if len(us_alphas.shape) == 2: - alphas, peaks = us_alphas[0], us_peaks[0] # support inference batch_size=1 only - else: - alphas, peaks = us_alphas, us_peaks - if char_list[-1] == "": - char_list = char_list[:-1] - fire_place = ( - torch.where(peaks > 1.0 - 1e-4)[0].cpu().numpy() + force_time_shift - ) # total offset - if len(fire_place) != len(char_list) + 1: - alphas /= alphas.sum() / (len(char_list) + 1) - alphas = alphas.unsqueeze(0) - peaks = cif_wo_hidden(alphas, threshold=1.0 - 1e-4)[0] - fire_place = ( - torch.where(peaks > 1.0 - 1e-4)[0].cpu().numpy() + force_time_shift - ) # total offset - num_frames = peaks.shape[0] - timestamp_list = [] - new_char_list = [] - # for bicif model trained with large data, cif2 actually fires when a character starts - # so treat the frames between two peaks as the duration of the former token - fire_place = ( - torch.where(peaks > 1.0 - 1e-4)[0].cpu().numpy() + force_time_shift - ) # total offset - # assert num_peak == len(char_list) + 1 # number of peaks is supposed to be number of tokens + 1 - # begin silence - if fire_place[0] > START_END_THRESHOLD: - # char_list.insert(0, '') - timestamp_list.append([0.0, fire_place[0] * TIME_RATE]) - new_char_list.append("") - # tokens timestamp - for i in range(len(fire_place) - 1): - new_char_list.append(char_list[i]) - if ( - MAX_TOKEN_DURATION < 0 - or fire_place[i + 1] - fire_place[i] <= MAX_TOKEN_DURATION - ): - timestamp_list.append( - [fire_place[i] * TIME_RATE, fire_place[i + 1] * TIME_RATE] - ) - else: - # cut the duration to token and sil of the 0-weight frames last long - _split = fire_place[i] + MAX_TOKEN_DURATION - timestamp_list.append([fire_place[i] * TIME_RATE, _split * TIME_RATE]) - timestamp_list.append([_split * TIME_RATE, fire_place[i + 1] * TIME_RATE]) - new_char_list.append("") - # tail token and end silence - # new_char_list.append(char_list[-1]) - if num_frames - fire_place[-1] > START_END_THRESHOLD: - _end = (num_frames + fire_place[-1]) * 0.5 - # _end = fire_place[-1] - timestamp_list[-1][1] = _end * TIME_RATE - timestamp_list.append([_end * TIME_RATE, num_frames * TIME_RATE]) - new_char_list.append("") - else: - timestamp_list[-1][1] = num_frames * TIME_RATE - if vad_offset: # add offset time in model with vad - for i in range(len(timestamp_list)): - timestamp_list[i][0] = timestamp_list[i][0] + vad_offset / 1000.0 - timestamp_list[i][1] = timestamp_list[i][1] + vad_offset / 1000.0 - res_txt = "" - for char, timestamp in zip(new_char_list, timestamp_list): - # if char != '': - if not sil_in_str and char == "": - continue - res_txt += "{} {} {};".format( - char, str(timestamp[0] + 0.0005)[:5], str(timestamp[1] + 0.0005)[:5] - ) - res = [] - for char, timestamp in zip(new_char_list, timestamp_list): - if char != "": - res.append([int(timestamp[0] * 1000), int(timestamp[1] * 1000)]) - return res_txt, res - - -def timestamp_sentence( - punc_id_list, timestamp_postprocessed, text_postprocessed, return_raw_text=False -): - punc_list = ["，", "。", "？", "、"] - res = [] - if text_postprocessed is None: - return res - if timestamp_postprocessed is None: - return res - if len(timestamp_postprocessed) == 0: - return res - if len(text_postprocessed) == 0: - return res - - if punc_id_list is None or len(punc_id_list) == 0: - res.append( - { - "text": text_postprocessed.split(), - "start": timestamp_postprocessed[0][0], - "end": timestamp_postprocessed[-1][1], - "timestamp": timestamp_postprocessed, - } - ) - return res - if len(punc_id_list) != len(timestamp_postprocessed): - logging.warning("length mismatch between punc and timestamp") - sentence_text = "" - sentence_text_seg = "" - ts_list = [] - sentence_start = timestamp_postprocessed[0][0] - sentence_end = timestamp_postprocessed[0][1] - texts = text_postprocessed.split() - punc_stamp_text_list = list( - zip_longest(punc_id_list, timestamp_postprocessed, texts, fillvalue=None) - ) - for punc_stamp_text in punc_stamp_text_list: - punc_id, timestamp, text = punc_stamp_text - # sentence_text += text if text is not None else '' - if text is not None: - if "a" <= text[0] <= "z" or "A" <= text[0] <= "Z": - sentence_text += " " + text - elif len(sentence_text) and ( - "a" <= sentence_text[-1] <= "z" or "A" <= sentence_text[-1] <= "Z" - ): - sentence_text += " " + text - else: - sentence_text += text - sentence_text_seg += text + " " - ts_list.append(timestamp) - - punc_id = int(punc_id) if punc_id is not None else 1 - sentence_end = timestamp[1] if timestamp is not None else sentence_end - sentence_text_seg = ( - sentence_text_seg[:-1] - if sentence_text_seg[-1] == " " - else sentence_text_seg - ) - if punc_id > 1: - sentence_text += punc_list[punc_id - 2] - if return_raw_text: - res.append( - { - "text": sentence_text, - "start": sentence_start, - "end": sentence_end, - "timestamp": ts_list, - "raw_text": sentence_text_seg, - } - ) - else: - res.append( - { - "text": sentence_text, - "start": sentence_start, - "end": sentence_end, - "timestamp": ts_list, - } - ) - sentence_text = "" - sentence_text_seg = "" - ts_list = [] - sentence_start = sentence_end - return res diff --git a/funasr_detach/utils/types.py b/funasr_detach/utils/types.py deleted file mode 100644 index 6b36f9c4b87ed9258a5d1e254ba298ed5dbc01d2..0000000000000000000000000000000000000000 --- a/funasr_detach/utils/types.py +++ /dev/null @@ -1,149 +0,0 @@ -from distutils.util import strtobool -from typing import Optional -from typing import Tuple -from typing import Union - -import humanfriendly - - -def str2bool(value: str) -> bool: - return bool(strtobool(value)) - - -def remove_parenthesis(value: str): - value = value.strip() - if value.startswith("(") and value.endswith(")"): - value = value[1:-1] - elif value.startswith("[") and value.endswith("]"): - value = value[1:-1] - return value - - -def remove_quotes(value: str): - value = value.strip() - if value.startswith('"') and value.endswith('"'): - value = value[1:-1] - elif value.startswith("'") and value.endswith("'"): - value = value[1:-1] - return value - - -def int_or_none(value: str) -> Optional[int]: - """int_or_none. - - Examples: - >>> import argparse - >>> parser = argparse.ArgumentParser() - >>> _ = parser.add_argument('--foo', type=int_or_none) - >>> parser.parse_args(['--foo', '456']) - Namespace(foo=456) - >>> parser.parse_args(['--foo', 'none']) - Namespace(foo=None) - >>> parser.parse_args(['--foo', 'null']) - Namespace(foo=None) - >>> parser.parse_args(['--foo', 'nil']) - Namespace(foo=None) - - """ - if value.strip().lower() in ("none", "null", "nil"): - return None - return int(value) - - -def float_or_none(value: str) -> Optional[float]: - """float_or_none. - - Examples: - >>> import argparse - >>> parser = argparse.ArgumentParser() - >>> _ = parser.add_argument('--foo', type=float_or_none) - >>> parser.parse_args(['--foo', '4.5']) - Namespace(foo=4.5) - >>> parser.parse_args(['--foo', 'none']) - Namespace(foo=None) - >>> parser.parse_args(['--foo', 'null']) - Namespace(foo=None) - >>> parser.parse_args(['--foo', 'nil']) - Namespace(foo=None) - - """ - if value.strip().lower() in ("none", "null", "nil"): - return None - return float(value) - - -def humanfriendly_parse_size_or_none(value) -> Optional[float]: - if value.strip().lower() in ("none", "null", "nil"): - return None - return humanfriendly.parse_size(value) - - -def str_or_int(value: str) -> Union[str, int]: - try: - return int(value) - except ValueError: - return value - - -def str_or_none(value: str) -> Optional[str]: - """str_or_none. - - Examples: - >>> import argparse - >>> parser = argparse.ArgumentParser() - >>> _ = parser.add_argument('--foo', type=str_or_none) - >>> parser.parse_args(['--foo', 'aaa']) - Namespace(foo='aaa') - >>> parser.parse_args(['--foo', 'none']) - Namespace(foo=None) - >>> parser.parse_args(['--foo', 'null']) - Namespace(foo=None) - >>> parser.parse_args(['--foo', 'nil']) - Namespace(foo=None) - - """ - if value.strip().lower() in ("none", "null", "nil"): - return None - return value - - -def str2pair_str(value: str) -> Tuple[str, str]: - """str2pair_str. - - Examples: - >>> import argparse - >>> str2pair_str('abc,def ') - ('abc', 'def') - >>> parser = argparse.ArgumentParser() - >>> _ = parser.add_argument('--foo', type=str2pair_str) - >>> parser.parse_args(['--foo', 'abc,def']) - Namespace(foo=('abc', 'def')) - - """ - value = remove_parenthesis(value) - a, b = value.split(",") - - # Workaround for configargparse issues: - # If the list values are given from yaml file, - # the value givent to type() is shaped as python-list, - # e.g. ['a', 'b', 'c'], - # so we need to remove double quotes from it. - return remove_quotes(a), remove_quotes(b) - - -def str2triple_str(value: str) -> Tuple[str, str, str]: - """str2triple_str. - - Examples: - >>> str2triple_str('abc,def ,ghi') - ('abc', 'def', 'ghi') - """ - value = remove_parenthesis(value) - a, b, c = value.split(",") - - # Workaround for configargparse issues: - # If the list values are given from yaml file, - # the value givent to type() is shaped as python-list, - # e.g. ['a', 'b', 'c'], - # so we need to remove quotes from it. - return remove_quotes(a), remove_quotes(b), remove_quotes(c) diff --git a/funasr_detach/utils/vad_utils.py b/funasr_detach/utils/vad_utils.py deleted file mode 100644 index eb9cb6f9a9cb7585cd466f764ac340e73c27fb1b..0000000000000000000000000000000000000000 --- a/funasr_detach/utils/vad_utils.py +++ /dev/null @@ -1,32 +0,0 @@ -import torch -from torch.nn.utils.rnn import pad_sequence - - -def slice_padding_fbank(speech, speech_lengths, vad_segments): - speech_list = [] - speech_lengths_list = [] - for i, segment in enumerate(vad_segments): - - bed_idx = int(segment[0][0] * 16) - end_idx = min(int(segment[0][1] * 16), speech_lengths[0]) - speech_i = speech[0, bed_idx:end_idx] - speech_lengths_i = end_idx - bed_idx - speech_list.append(speech_i) - speech_lengths_list.append(speech_lengths_i) - feats_pad = pad_sequence(speech_list, batch_first=True, padding_value=0.0) - speech_lengths_pad = torch.Tensor(speech_lengths_list).int() - return feats_pad, speech_lengths_pad - - -def slice_padding_audio_samples(speech, speech_lengths, vad_segments): - speech_list = [] - speech_lengths_list = [] - for i, segment in enumerate(vad_segments): - bed_idx = int(segment[0][0] * 16) - end_idx = min(int(segment[0][1] * 16), speech_lengths) - speech_i = speech[bed_idx:end_idx] - speech_lengths_i = end_idx - bed_idx - speech_list.append(speech_i) - speech_lengths_list.append(speech_lengths_i) - - return speech_list, speech_lengths_list diff --git a/funasr_detach/version.txt b/funasr_detach/version.txt deleted file mode 100644 index b0f3d96f877256ed9ae03858ecc5185a989b1d1b..0000000000000000000000000000000000000000 --- a/funasr_detach/version.txt +++ /dev/null @@ -1 +0,0 @@ -1.0.8 diff --git a/requirements.txt b/requirements.txt index de15a8c8140990e63558048f384bf28e08dc0d47..b40c7c55423893940d357281016a129256e463b5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,18 +1,2 @@ -torch==2.3.1 -torchaudio==2.3.1 -transformers==4.48.3 -accelerate==1.3.0 -openai-whisper==20231117 -onnxruntime-gpu==1.17.0 -omegaconf==2.3.0 -librosa==0.10.2.post1 -sox==1.5.0 -modelscope -numpy==1.26.4 -six==1.16.0 -hyperpyyaml -conformer==0.3.2 -diffusers -pillow -sentencepiece -funasr>=1.1.3 \ No newline at end of file +openai +gradio \ No newline at end of file diff --git a/speakers/TingtingRAP_prompt.wav b/speakers/TingtingRAP_prompt.wav deleted file mode 100644 index 55474b9252779ee572650dd4a68be6011d18f3e7..0000000000000000000000000000000000000000 --- a/speakers/TingtingRAP_prompt.wav +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:36d260f1a62d1d96b8d5a01b14a197ad967e7652738d6d4cee25816121c4c730 -size 1042174 diff --git a/speakers/TingtingVOCAL_prompt.wav b/speakers/TingtingVOCAL_prompt.wav deleted file mode 100644 index 3d6ded0476f42c31a552f52a546fa4478d00d5b4..0000000000000000000000000000000000000000 --- a/speakers/TingtingVOCAL_prompt.wav +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:6b8455e31e51ce0cafd78f36f3b726f8dfdfc6171419c68ebde1ab72c3621581 -size 1801518 diff --git a/speakers/Tingting_prompt.wav b/speakers/Tingting_prompt.wav deleted file mode 100644 index e3f1183166690080fb00ff9cbcf3d0b44deb84de..0000000000000000000000000000000000000000 --- a/speakers/Tingting_prompt.wav +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:27ecbef5f3fb6609174ec9556623825867ec5fad485c9641cca41893c4966fc2 -size 394196 diff --git a/speakers/speakers_info.json b/speakers/speakers_info.json deleted file mode 100644 index ba8d5a1f4323e0f4d619ffa7882ec7257252875a..0000000000000000000000000000000000000000 --- a/speakers/speakers_info.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "TingtingRAP": "（RAP）远远甩开的笑他是陆行龟 他曾跌倒也曾吃过灰 他说有福的人才会多吃亏 他的爸爸让他小心交友可他偏偏钻进个垃圾堆 他说他明白How to play", - "TingtingVOCAL": "（哼唱）你从一座叫 我 的小镇经过 刚好屋顶的雪化成雨飘落", - "Tingting": "那等我们到海洋馆之后，给妈妈买个礼物，好不好呀？" -} diff --git a/start_app.sh b/start_app.sh deleted file mode 100755 index 876402f621c31d72ea5b39f91f1317601ccf708a..0000000000000000000000000000000000000000 --- a/start_app.sh +++ /dev/null @@ -1,58 +0,0 @@ -#!/bin/bash - -if [ -z "$HF_USER_NAME" ]; then - echo "错误：环境变量 HF_USER_NAME 未设置！" - exit 1 -fi - -if [ -z "$HF_USER_TOKEN" ]; then - echo "错误：环境变量 HF_USER_TOKEN 未设置！" - exit 1 -fi - -# 启用Git LFS支持 -git lfs install --force - -# 定义需要克隆的仓库列表 -BASE_REPO_URL="https://${HF_USER_NAME}:${HF_USER_TOKEN}@huggingface.co/stepfun-ai" -REPOSITORIES=( - "Step-Audio-Tokenizer" - "Step-Audio-TTS-3B" -) - -# 定义本地存放仓库的目录，默认为当前目录 -LOCAL_DIR="${1:-$(pwd)}" - -# 克隆函数（带无限重试机制） -clone_with_retry() { - local repo_name=$1 - local repo_url="${BASE_REPO_URL}/${repo_name}" - local target_dir="${LOCAL_DIR}/${repo_name}" - - # 检查是否已存在目录 - if [ -d "${target_dir}" ]; then - echo "目录 ${target_dir} 已存在，跳过克隆。" - return 0 - fi - - # 无限重试循环 - while true; do - echo "正在尝试克隆 ${repo_name} 到 ${target_dir}..." - if git clone "${repo_url}" "${target_dir}"; then - echo "成功克隆 ${repo_name} 到 ${target_dir}" - return 0 - else - echo "克隆失败, 5秒后重试..." - sleep 5 - fi - done -} - -# 遍历所有仓库进行克隆 -for repo in "${REPOSITORIES[@]}"; do - clone_with_retry "${repo}" -done - -echo "所有仓库已成功下载！" -export LD_LIBRARY_PATH=/home/user/.local/lib/python3.10/site-packages/nvidia/cuda_runtime/lib:/home/user/.local/lib/python3.10/site-packages/nvidia/cuda_nvrtc/lib:/home/user/.local/lib/python3.10/site-packages/nvidia/cublas/lib:$LD_LIBRARY_PATH -python app.py --model $LOCAL_DIR diff --git a/tokenizer.py b/tokenizer.py deleted file mode 100644 index ad606bcb0c00930e6f0464787f5140d88f9a422f..0000000000000000000000000000000000000000 --- a/tokenizer.py +++ /dev/null @@ -1,197 +0,0 @@ -import io -import threading -import time -import os - -import numpy as np -import torch -import torchaudio -import onnxruntime -import whisper - -from funasr_detach import AutoModel -from utils import resample_audio, energy_norm_fn, trim_silence - - -class StepAudioTokenizer: - def __init__( - self, - encoder_path, - ): - funasr_model_path = os.path.join( - encoder_path, - "dengcunqin/speech_paraformer-large_asr_nat-zh-cantonese-en-16k-vocab8501-online", - ) - kms_path = os.path.join(encoder_path, "linguistic_tokenizer.npy") - cosy_tokenizer_path = os.path.join(encoder_path, "speech_tokenizer_v1.onnx") - self.funasr_model = AutoModel(model=funasr_model_path, model_revision="master") - self.kms = torch.tensor(np.load(kms_path)) - - providers = ["CUDAExecutionProvider"] - session_option = onnxruntime.SessionOptions() - session_option.graph_optimization_level = ( - onnxruntime.GraphOptimizationLevel.ORT_ENABLE_ALL - ) - session_option.intra_op_num_threads = 1 - self.ort_session = onnxruntime.InferenceSession( - cosy_tokenizer_path, sess_options=session_option, providers=providers - ) - self.chunk_size = [0, 4, 5] - self.encoder_chunk_look_back = 4 - self.decoder_chunk_look_back = 1 - - self.vq02_sessions = {} - self.vq02_lock = threading.Lock() - self.vq06_lock = threading.Lock() - - def __call__(self, audio, sr): - _, vq02, vq06 = self.wav2token(audio, sr, False) - text = self.merge_vq0206_to_token_str(vq02, vq06) - return text - - def preprocess_wav(self, audio, sample_rate, enable_trim=True, energy_norm=True): - audio = resample_audio(audio, sample_rate, 16000) - if energy_norm: - audio = energy_norm_fn(audio) - - if enable_trim: - audio = audio.cpu().numpy().squeeze(0) - audio = trim_silence(audio, 16000) - audio = torch.from_numpy(audio) - audio = audio.unsqueeze(0) - return audio - - def wav2token(self, audio, sample_rate, enable_trim=True, energy_norm=True): - audio = self.preprocess_wav( - audio, sample_rate, enable_trim=enable_trim, energy_norm=energy_norm - ) - - vq02_ori = self.get_vq02_code(audio) - vq02 = [int(x) + 65536 for x in vq02_ori] - vq06_ori = self.get_vq06_code(audio) - vq06 = [int(x) + 65536 + 1024 for x in vq06_ori] - - chunk = 1 - chunk_nums = min(len(vq06) // (3 * chunk), len(vq02) // (2 * chunk)) - speech_tokens = [] - for idx in range(chunk_nums): - speech_tokens += vq02[idx * chunk * 2 : (idx + 1) * chunk * 2] - speech_tokens += vq06[idx * chunk * 3 : (idx + 1) * chunk * 3] - return speech_tokens, vq02_ori, vq06_ori - - def get_vq02_code(self, audio, session_id=None, is_final=True): - _tmp_wav = io.BytesIO() - torchaudio.save(_tmp_wav, audio, 16000, format="wav") - _tmp_wav.seek(0) - - with self.vq02_lock: - cache = {} - if session_id in self.vq02_sessions: - cache = self.vq02_sessions[session_id].get("cache", {}) - - res, new_cache = self.funasr_model.infer_encoder( - input=[_tmp_wav], - chunk_size=self.chunk_size, - encoder_chunk_look_back=self.encoder_chunk_look_back, - decoder_chunk_look_back=self.decoder_chunk_look_back, - device=0, - is_final=is_final, - cache=cache, - ) - c_list = [] - for j, res_ in enumerate(res): - feat = res_["enc_out"] - if len(feat) > 0: - c_list = self.dump_label([feat], self.kms)[0] - - if is_final: - if session_id in self.vq02_sessions: - self.vq02_sessions.pop(session_id) - else: - if isinstance(session_id, str) and len(session_id) > 0: - self.vq02_sessions[session_id] = { - "cache": new_cache, - "update_time": time.time(), - } - - return c_list - - def get_vq06_code(self, audio): - - def split_audio(audio, chunk_duration=480000): - start = 0 - chunks = [] - while start < len(audio): - end = min(start + chunk_duration, len(audio)) - chunk = audio[start:end] - if len(chunk) < 480: - pass - else: - chunks.append(chunk) - start = end - return chunks - - with self.vq06_lock: - audio = audio.squeeze(0) - chunk_audios = split_audio(audio, chunk_duration=30 * 16000) # 最大支持30s - speech_tokens = [] - for chunk in chunk_audios: - duration = round(chunk.shape[0] / 16000, 2) - feat = whisper.log_mel_spectrogram(chunk, n_mels=128) - feat = feat.unsqueeze(0) - feat_len = np.array([feat.shape[2]], dtype=np.int32) - chunk_token = ( - self.ort_session.run( - None, - { - self.ort_session.get_inputs()[0] - .name: feat.detach() - .cpu() - .numpy(), - self.ort_session.get_inputs()[1].name: feat_len, - }, - )[0] - .flatten() - .tolist() - ) - assert abs(len(chunk_token) - duration * 25) <= 2 - speech_tokens += chunk_token - - return speech_tokens - - def kmean_cluster(self, samples, means): - dists = torch.cdist(samples, means) - indices = dists.argmin(dim=1).cpu().numpy() - return indices.tolist() - - def dump_label(self, samples, mean): - dims = samples[0].shape[-1] - x_lens = [x.shape[1] for x in samples] - total_len = sum(x_lens) - x_sel = torch.FloatTensor(1, total_len, dims) - start_len = 0 - for sample in samples: - sample_len = sample.shape[1] - end_len = start_len + sample_len - x_sel[:, start_len:end_len] = sample - start_len = end_len - dense_x = x_sel.squeeze(0) - indices = self.kmean_cluster(dense_x, mean) - indices_list = [] - start_len = 0 - for x_len in x_lens: - end_len = start_len + end_len - indices_list.append(indices[start_len:end_len]) - return indices_list - - def merge_vq0206_to_token_str(self, vq02, vq06): - _vq06 = [1024 + x for x in vq06] - result = [] - i = 0 - j = 0 - while i < len(vq02) - 1 and j < len(_vq06) - 2: - sublist = vq02[i : i + 2] + _vq06[j : j + 3] - result.extend(sublist) - i += 2 - j += 3 - return "".join([f"" for x in result]) diff --git a/tts.py b/tts.py deleted file mode 100644 index aca7f7455e123323e2c025ccebd16d84bc340eff..0000000000000000000000000000000000000000 --- a/tts.py +++ /dev/null @@ -1,218 +0,0 @@ -import os -import re -import json - -import torchaudio -import torch -from transformers import AutoModelForCausalLM, AutoTokenizer -from transformers.generation.logits_process import LogitsProcessor -from transformers.generation.utils import LogitsProcessorList - -from cosyvoice.cli.cosyvoice import CosyVoice - - -class RepetitionAwareLogitsProcessor(LogitsProcessor): - def __call__( - self, input_ids: torch.LongTensor, scores: torch.FloatTensor - ) -> torch.FloatTensor: - window_size = 10 - threshold = 0.1 - - window = input_ids[:, -window_size:] - if window.shape[1] < window_size: - return scores - - last_tokens = window[:, -1].unsqueeze(-1) - repeat_counts = (window == last_tokens).sum(dim=1) - repeat_ratios = repeat_counts.float() / window_size - - mask = repeat_ratios > threshold - scores[mask, last_tokens[mask].squeeze(-1)] = float("-inf") - return scores - - -class StepAudioTTS: - def __init__( - self, - model_path, - encoder, - ): - # load optimus_ths for flash attention, make sure LD_LIBRARY_PATH has `nvidia/cuda_nvrtc/lib` - # if not, please manually set LD_LIBRARY_PATH=xxx/python3.10/site-packages/nvidia/cuda_nvrtc/lib - try: - if torch.__version__ >= "2.5": - torch.ops.load_library(os.path.join(model_path, 'lib/liboptimus_ths-torch2.5-cu124.cpython-310-x86_64-linux-gnu.so')) - elif torch.__version__ >= "2.3": - torch.ops.load_library(os.path.join(model_path, 'lib/liboptimus_ths-torch2.3-cu121.cpython-310-x86_64-linux-gnu.so')) - elif torch.__version__ >= "2.2": - torch.ops.load_library(os.path.join(model_path, 'lib/liboptimus_ths-torch2.2-cu121.cpython-310-x86_64-linux-gnu.so')) - print("Load optimus_ths successfully and flash attn would be enabled") - except Exception as err: - print(f"Fail to load optimus_ths and flash attn is disabled: {err}") - - self.llm = AutoModelForCausalLM.from_pretrained( - model_path, - torch_dtype=torch.bfloat16, - device_map="cuda", - trust_remote_code=True, - ) - self.tokenizer = AutoTokenizer.from_pretrained( - model_path, trust_remote_code=True - ) - self.common_cosy_model = CosyVoice( - os.path.join(model_path, "CosyVoice-300M-25Hz") - ) - self.music_cosy_model = CosyVoice( - os.path.join(model_path, "CosyVoice-300M-25Hz-Music") - ) - self.encoder = encoder - self.sys_prompt_dict = { - "sys_prompt_for_rap": "请参考对话历史里的音色，用RAP方式将文本内容大声说唱出来。", - "sys_prompt_for_vocal": "请参考对话历史里的音色，用哼唱的方式将文本内容大声唱出来。", - "sys_prompt_wo_spk": '作为一名卓越的声优演员，你的任务是根据文本中（）或()括号内标注的情感、语种或方言、音乐哼唱、语音调整等标签，以丰富细腻的情感和自然顺畅的语调来朗读文本。\n# 情感标签涵盖了多种情绪状态，包括但不限于：\n- "高兴1"\n- "高兴2"\n- "高兴3"\n- "生气1"\n- "生气2"\n- "生气3"\n- "悲伤1"\n- "惊讶"\n- "厌恶"\n- "恐惧"\n- "中立"\n- "低语1"\n- "撒娇1"\n- "疲惫"\n\n# 语种或方言标签包含多种语言或方言，包括但不限于：\n- "中文"\n- "英文"\n- "韩语"\n- "日语"\n- "四川话"\n- "粤语"\n- "广东话"\n\n# 音乐哼唱标签包含多种类型歌曲哼唱，包括但不限于：\n- "RAP"\n- "哼唱"\n\n# 语音调整标签，包括但不限于：\n- "慢速1"\n- "慢速2"\n- "慢速3"\n- "快速1"\n- "快速2"\n- "快速3"\n\n请在朗读时，根据这些情感标签的指示，调整你的情感、语气、语调和哼唱节奏，以确保文本的情感和意义得到准确而生动的传达，如果没有()或（）括号，则根据文本语义内容自由演绎。', - "sys_prompt_with_spk": '作为一名卓越的声优演员，你的任务是根据文本中（）或()括号内标注的情感、语种或方言、音乐哼唱、语音调整等标签，以丰富细腻的情感和自然顺畅的语调来朗读文本。\n# 情感标签涵盖了多种情绪状态，包括但不限于：\n- "高兴1"\n- "高兴2"\n- "高兴3"\n- "生气1"\n- "生气2"\n- "生气3"\n- "悲伤1"\n- "惊讶"\n- "厌恶"\n- "恐惧"\n- "中立"\n- "低语1"\n- "撒娇1"\n- "疲惫"\n\n# 语种或方言标签包含多种语言或方言，包括但不限于：\n- "中文"\n- "英文"\n- "韩语"\n- "日语"\n- "四川话"\n- "粤语"\n- "广东话"\n\n# 音乐哼唱标签包含多种类型歌曲哼唱，包括但不限于：\n- "RAP"\n- "哼唱"\n\n# 语音调整标签，包括但不限于：\n- "慢速1"\n- "慢速2"\n- "慢速3"\n- "快速1"\n- "快速2"\n- "快速3"\n\n请在朗读时，使用[{}]的声音，根据这些情感标签的指示，调整你的情感、语气、语调和哼唱节奏，以确保文本的情感和意义得到准确而生动的传达，如果没有()或（）括号，则根据文本语义内容自由演绎。', - } - self.register_speakers() - - def __call__(self, text: str, prompt_speaker: str): - instruction_name = self.detect_instruction_name(text) - if instruction_name in ("RAP", "VOCAL"): - prompt_speaker_info = self.speakers_info[ - f"{prompt_speaker}{instruction_name}" - ] - cosy_model = self.music_cosy_model - else: - prompt_speaker_info = self.speakers_info[prompt_speaker] - cosy_model = self.common_cosy_model - - token_ids = self.tokenize( - text, - prompt_speaker_info["prompt_text"], - prompt_speaker, - prompt_speaker_info["prompt_code"], - ) - output_ids = self.llm.generate( - torch.tensor([token_ids]).to(torch.long).to("cuda"), - max_length=8192, - temperature=0.7, - do_sample=True, - logits_processor=LogitsProcessorList([RepetitionAwareLogitsProcessor()]), - ) - output_ids = output_ids[:, len(token_ids) : -1] # skip eos token - return ( - cosy_model.token_to_wav_offline( - output_ids - 65536, - prompt_speaker_info["cosy_speech_feat"].to(torch.bfloat16), - prompt_speaker_info["cosy_speech_feat_len"], - prompt_speaker_info["cosy_prompt_token"], - prompt_speaker_info["cosy_prompt_token_len"], - prompt_speaker_info["cosy_speech_embedding"].to(torch.bfloat16), - ), - 22050, - ) - - def register_speakers(self): - self.speakers_info = {} - - with open("speakers/speakers_info.json", "r") as f: - speakers_info = json.load(f) - - for speaker_id, prompt_text in speakers_info.items(): - prompt_wav_path = f"speakers/{speaker_id}_prompt.wav" - prompt_wav, prompt_wav_sr = torchaudio.load(prompt_wav_path) - _, _, speech_feat, speech_feat_len, speech_embedding = ( - self.preprocess_prompt_wav(prompt_wav, prompt_wav_sr) - ) - prompt_code, _, _ = self.encoder.wav2token(prompt_wav, prompt_wav_sr) - prompt_token = torch.tensor([prompt_code], dtype=torch.long) - 65536 - prompt_token_len = torch.tensor([prompt_token.shape[1]], dtype=torch.long) - self.speakers_info[speaker_id] = { - "prompt_text": prompt_text, - "prompt_code": prompt_code, - "cosy_speech_feat": speech_feat.to(torch.bfloat16), - "cosy_speech_feat_len": speech_feat_len, - "cosy_speech_embedding": speech_embedding.to(torch.bfloat16), - "cosy_prompt_token": prompt_token, - "cosy_prompt_token_len": prompt_token_len, - } - print(f"Registered speaker: {speaker_id}") - - def detect_instruction_name(self, text): - instruction_name = "" - match_group = re.match(r"^([（\(][^\(\)()]*[）\)]).*$", text, re.DOTALL) - if match_group is not None: - instruction = match_group.group(1) - instruction_name = instruction.strip("()（）") - return instruction_name - - def tokenize( - self, text: str, prompt_text: str, prompt_speaker: str, prompt_code: list - ): - rap_or_vocal = self.detect_instruction_name(text) in ("RAP", "VOCAL") - - if rap_or_vocal: - if "哼唱" in text: - prompt = self.sys_prompt_dict["sys_prompt_for_vocal"] - else: - prompt = self.sys_prompt_dict["sys_prompt_for_rap"] - elif prompt_speaker: - prompt = self.sys_prompt_dict["sys_prompt_with_spk"].format(prompt_speaker) - else: - prompt = self.sys_prompt_dict["sys_prompt_wo_spk"] - - sys_tokens = self.tokenizer.encode(f"system\n{prompt}") - - history = [1] - history.extend([4] + sys_tokens + [3]) - - _prefix_tokens = self.tokenizer.encode("\n") - part_tokens1 = self.tokenizer.encode("\n" + prompt_text) - question1_tokens = part_tokens1[len(_prefix_tokens) :] - - part_tokens2 = self.tokenizer.encode("\n" + text) - question2_tokens = part_tokens2[len(_prefix_tokens) :] - - qrole_toks = self.tokenizer.encode("human\n") - arole_toks = self.tokenizer.encode("assistant\n") - - history.extend( - [4] - + qrole_toks - + question1_tokens - + [3] - + [4] - + arole_toks - + prompt_code - + [3] - + [4] - + qrole_toks - + question2_tokens - + [3] - + [4] - + arole_toks - ) - return history - - def preprocess_prompt_wav(self, prompt_wav: torch.Tensor, prompt_wav_sr: int): - prompt_wav_16k = torchaudio.transforms.Resample( - orig_freq=prompt_wav_sr, new_freq=16000 - )(prompt_wav) - prompt_wav_22k = torchaudio.transforms.Resample( - orig_freq=prompt_wav_sr, new_freq=22050 - )(prompt_wav) - prompt_token, prompt_token_len = ( - self.common_cosy_model.frontend._extract_speech_token(prompt_wav_16k) - ) - speech_feat, speech_feat_len = ( - self.common_cosy_model.frontend._extract_speech_feat(prompt_wav_22k) - ) - speech_embedding = self.common_cosy_model.frontend._extract_spk_embedding( - prompt_wav_16k - ) - return ( - prompt_token, - prompt_token_len, - speech_feat, - speech_feat_len, - speech_embedding, - ) diff --git a/utils.py b/utils.py deleted file mode 100644 index 2c985135cecd7796ba0a60c22517099648c78029..0000000000000000000000000000000000000000 --- a/utils.py +++ /dev/null @@ -1,154 +0,0 @@ -import io -import base64 - -import librosa -import numpy as np -import math -import torch -import torchaudio -import torchaudio -import sox -import tempfile - - -def encode_wav(wav, sr, rep_format="wav"): - with io.BytesIO() as wavio: - torchaudio.save(wavio, wav, sr, format=rep_format) - audio_bytes = wavio.getvalue() - encoded_wav = base64.b64encode(audio_bytes).decode("ascii") - return encoded_wav - - -def trim_silence(audio, sr, keep_left_time=0.05, keep_right_time=0.22, hop_size=240): - _, index = librosa.effects.trim(audio, top_db=20, frame_length=512, hop_length=128) - num_frames = int(math.ceil((index[1] - index[0]) / hop_size)) # 300 - - left_sil_samples = int(keep_left_time * sr) - right_sil_samples = int(keep_right_time * sr) - - wav_len = len(audio) - start_idx = index[0] - left_sil_samples - trim_wav = audio - - if start_idx > 0: - trim_wav = trim_wav[start_idx:] - else: - trim_wav = np.pad( - trim_wav, (abs(start_idx), 0), mode="constant", constant_values=0.0 - ) - wav_len = len(trim_wav) - out_len = int(num_frames * hop_size + (keep_left_time + keep_right_time) * sr) - - if out_len < wav_len: - trim_wav = trim_wav[:out_len] - else: - trim_wav = np.pad( - trim_wav, (0, (out_len - wav_len)), mode="constant", constant_values=0.0 - ) - return trim_wav - - -def volumn_adjust(audio16bit_torch, sr, volumn_ratio): - """使用sox进行音频音量调整 - Args: - audio16bit_torch (Tensor): 输入音频张量 [1, samples] - volume_ratio (float): 音量比率，>1增大音量，<1降低音量 - - Returns: - Tensor: 调整音量后的音频张量 - """ - # 创建临时文件 - with tempfile.NamedTemporaryFile( - suffix=".wav", delete=True - ) as temp_in, tempfile.NamedTemporaryFile(suffix=".wav", delete=True) as temp_out: - # 保存输入音频到临时文件 - torchaudio.save(temp_in.name, audio16bit_torch, sr) # 假设采样率为16000 - # 创建sox转换器 - tfm = sox.Transformer() - tfm.vol(volumn_ratio) # 设置音量调整比率 - # 应用音量调整 - tfm.build_file(temp_in.name, temp_out.name) - # 读取处理后的音频 - audio_changed, _ = torchaudio.load(temp_out.name) - return audio_changed - - -def speech_adjust(audio16bit_torch, sr, speed_ratio): - """使用sox进行音频变速处理 - Args: - audio16bit_torch (Tensor): 输入音频张量 [1, samples] - speed_ratio (float): 速度比率，>1加速，<1减速 - - Returns: - Tensor: 变速后的音频张量 - """ - # 创建临时文件 - with tempfile.NamedTemporaryFile( - suffix=".wav", delete=True - ) as temp_in, tempfile.NamedTemporaryFile(suffix=".wav", delete=True) as temp_out: - # 保存输入音频到临时文件 - torchaudio.save(temp_in.name, audio16bit_torch, sr) # 假设采样率为16000 - # 创建sox转换器 - tfm = sox.Transformer() - tfm.tempo(speed_ratio) # 设置变速比率 - # 应用变速处理 - tfm.build_file(temp_in.name, temp_out.name) - # 读取处理后的音频 - audio_changed, _ = torchaudio.load(temp_out.name) - return audio_changed - - -def audio_resample(audio16bit_torch, result_sr, target_sample_rate): - audio16bit_torch = torchaudio.transforms.Resample( - orig_freq=result_sr, new_freq=target_sample_rate - )(audio16bit_torch) - result_sr = target_sample_rate - return audio16bit_torch, result_sr - - -def norm_audio(audio16bit_torch): - # 直接 归一化处理。 - audio16bit_torch = audio16bit_torch.numpy() - audio16bit_torch = ( - audio16bit_torch / np.abs(audio16bit_torch).max() * 32767 - ).astype(np.int16) - audio16bit_torch = torch.from_numpy(audio16bit_torch) - return audio16bit_torch - - -def resample_audio(wav, original_sample_rate, target_sample_rate): - if original_sample_rate != target_sample_rate: - assert ( - original_sample_rate > target_sample_rate - ), "wav sample rate {} must be greater than {}".format( - original_sample_rate, target_sample_rate - ) - wav = torchaudio.transforms.Resample( - orig_freq=original_sample_rate, new_freq=target_sample_rate - )(wav) - return wav - - -def energy_norm_fn(wav): - if type(wav) is np.ndarray: - max_data = np.max(np.abs(wav)) - wav = wav / max(max_data, 0.01) * 0.999 - else: - max_data = torch.max(torch.abs(wav)) - wav = wav / max(max_data, 0.01) * 0.999 - return wav - - -def get_audio_tokens(audio_tokens: str) -> list[int]: - audio_tokens = audio_tokens.split(">", "")) + 65536 - for token in audio_tokens - ] - return audio_tokens - - -def load_audio(audio_path: str): - audio_wav, sr = torchaudio.load(audio_path) - audio_wav = audio_wav.mean(dim=0, keepdim=True) - return audio_wav, sr diff --git a/yuewen_api.py b/yuewen_api.py index 12c356823e6e4abfad1817b41397889d878677aa..b3c1b4a0c8a4bc5c1688c085c4e5670df1a42f1e 100644 --- a/yuewen_api.py +++ b/yuewen_api.py @@ -26,7 +26,7 @@ def call_audiochat(messages): raise e -def call_tts(text, audio_path, voice="qinqienvsheng"): +def call_tts(text, audio_path, voice="jingdiannvsheng"): response = client.audio.speech.create(model=TTS_MODEL, voice=voice, input=text) response.stream_to_file(audio_path) return True @@ -41,8 +41,9 @@ def call_asr(audio_path): ) return response.text + if __name__ == "__main__": - messages = [{"role":"user", "content": "介绍下你自己"}] + messages = [{"role": "user", "content": "介绍下你自己"}] res = call_audiochat(messages) print("call audiochat: ", res)