Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,6 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+TalkingHead-1KH/data_list/train_video_tubes.txt filter=lfs diff=lfs merge=lfs -text
+TalkingHead-1KH/teaser.gif filter=lfs diff=lfs merge=lfs -text
+data_gen/utils/mp_feature_extractors/face_landmarker.task filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,199 @@
+# big files
+data_util/face_tracking/3DMM/01_MorphableModel.mat
+data_util/face_tracking/3DMM/3DMM_info.npy
+
+!/deep_3drecon/BFM/.gitkeep
+deep_3drecon/BFM/Exp_Pca.bin
+deep_3drecon/BFM/01_MorphableModel.mat
+deep_3drecon/BFM/BFM_model_front.mat
+deep_3drecon/network/FaceReconModel.pb
+deep_3drecon/checkpoints/*
+
+.vscode
+### Project ignore
+/checkpoints/*
+!/checkpoints/.gitkeep
+/data/*
+!/data/.gitkeep
+infer_out
+rsync
+.idea
+.DS_Store
+bak
+tmp
+*.tar.gz
+mos
+nbs
+/configs_usr/*
+!/configs_usr/.gitkeep
+/egs_usr/*
+!/egs_usr/.gitkeep
+/rnnoise
+#/usr/*
+#!/usr/.gitkeep
+scripts_usr
+
+# Created by .ignore support plugin (hsz.mobi)
+### Python template
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+data_util/deepspeech_features/deepspeech-0.9.2-models.pbmm
+deep_3drecon/mesh_renderer/bazel-bin
+deep_3drecon/mesh_renderer/bazel-mesh_renderer
+deep_3drecon/mesh_renderer/bazel-out
+deep_3drecon/mesh_renderer/bazel-testlogs
+
+.nfs*
+infer_outs/*
+
+*.pth
+venv_113/*
+*.pt
+experiments/trials
+flame_3drecon/*
+
+temp/
+/kill.sh
+/datasets
+data_util/imagenet_classes.txt
+process_data_May.sh
+/env_prepare_reproduce.md
+/my_debug.py
+
+utils/metrics/shape_predictor_68_face_landmarks.dat
+*.mp4
+_torchshow/
+*.png
+*.jpg
+
+*.mrc
+
+deep_3drecon/BFM/BFM_exp_idx.mat
+deep_3drecon/BFM/BFM_front_idx.mat
+deep_3drecon/BFM/facemodel_info.mat
+deep_3drecon/BFM/index_mp468_from_mesh35709.npy
+deep_3drecon/BFM/mediapipe_in_bfm53201.npy
+deep_3drecon/BFM/std_exp.txt
+!data/raw/examples/*

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 ZhenhuiYe
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README-zh.md

@@ -0,0 +1,144 @@
+# Real3D-Portrait: One-shot Realistic 3D Talking Portrait Synthesis | ICLR 2024 Spotlight
+[![arXiv](https://img.shields.io/badge/arXiv-Paper-%3CCOLOR%3E.svg)](https://arxiv.org/abs/2401.08503)| [![GitHub Stars](https://img.shields.io/github/stars/yerfor/Real3DPortrait
+)](https://github.com/yerfor/Real3DPortrait) | [English Readme](./README.md)
+
+这个仓库是Real3D-Portrait的官方PyTorch实现, 用于实现单参考图(one-shot)、高视频真实度(video reality)的虚拟人视频合成。您可以访问我们的[项目页面](https://real3dportrait.github.io/)以观看Demo视频, 阅读我们的[论文](https://arxiv.org/pdf/2401.08503.pdf)以了解技术细节。
+
+<p align="center">
+    <br>
+    <img src="assets/real3dportrait.png" width="100%"/>
+    <br>
+</p>
+
+## 您可能同样感兴趣
+- 我们发布了GeneFace++([https://github.com/yerfor/GeneFacePlusPlus](https://github.com/yerfor/GeneFacePlusPlus)), 一个专注于提升单个特定说话人效果的说话人合成系统，它实现了高嘴形对齐、高视频质量和高系统效率。
+
+
+# 快速上手！
+## 安装环境
+请参照[环境配置文档](docs/prepare_env/install_guide-zh.md)，配置Conda环境`real3dportrait`
+## 下载预训练与第三方模型
+### 3DMM BFM模型
+下载3DMM BFM模型：[Google Drive](https://drive.google.com/drive/folders/1o4t5YIw7w4cMUN4bgU9nPf6IyWVG1bEk?usp=sharing) 或 [BaiduYun Disk](https://pan.baidu.com/s/1aqv1z_qZ23Vp2VP4uxxblQ?pwd=m9q5 ) 提取码: m9q5
+
+
+下载完成后，放置全部的文件到`deep_3drecon/BFM`里，文件结构如下：
+```
+deep_3drecon/BFM/
+├── 01_MorphableModel.mat
+├── BFM_exp_idx.mat
+├── BFM_front_idx.mat
+├── BFM_model_front.mat
+├── Exp_Pca.bin
+├── facemodel_info.mat
+├── index_mp468_from_mesh35709.npy
+├── mediapipe_in_bfm53201.npy
+└── std_exp.txt
+```
+
+### 预训练模型
+下载预训练的Real3D-Portrait：[Google Drive](https://drive.google.com/drive/folders/1MAveJf7RvJ-Opg1f5qhLdoRoC_Gc6nD9?usp=sharing) 或 [BaiduYun Disk](https://pan.baidu.com/s/1Mjmbn0UtA1Zm9owZ7zWNgQ?pwd=6x4f ) 提取码: 6x4f
+  
+下载完成后，放置全部的文件到`checkpoints`里并解压，文件结构如下：
+```
+checkpoints/
+├── 240210_real3dportrait_orig
+│   ├── audio2secc_vae
+│   │   ├── config.yaml
+│   │   └── model_ckpt_steps_400000.ckpt
+│   └── secc2plane_torso_orig
+│       ├── config.yaml
+│       └── model_ckpt_steps_100000.ckpt
+└── pretrained_ckpts
+    └── mit_b0.pth
+```
+
+## 推理测试
+我们目前提供了**命令行（CLI）**, **Gradio WebUI**与**Google Colab**推理方式。我们同时支持音频驱动（Audio-Driven）与视频驱动（Video-Driven）：
+
+- 音频驱动场景下，需要至少提供`source image`与`driving audio`
+- 视频驱动场景下，需要至少提供`source image`与`driving expression video`
+
+### Gradio WebUI推理
+启动Gradio WebUI，按照提示上传素材，点击`Generate`按钮即可推理：
+```bash
+python inference/app_real3dportrait.py
+```
+
+### Google Colab推理
+运行这个[Colab](https://colab.research.google.com/github/yerfor/Real3DPortrait/blob/main/inference/real3dportrait_demo.ipynb)中的所有cell。
+
+### 命令行推理
+首先，切换至项目根目录并启用Conda环境：
+```bash
+cd <Real3DPortraitRoot>
+conda activate real3dportrait
+export PYTHON_PATH=./
+```
+音频驱动场景下，需要至少提供source image与driving audio，推理指令：
+```bash
+python inference/real3d_infer.py \
+--src_img <PATH_TO_SOURCE_IMAGE> \
+--drv_aud <PATH_TO_AUDIO> \
+--drv_pose <PATH_TO_POSE_VIDEO, OPTIONAL> \
+--bg_img <PATH_TO_BACKGROUND_IMAGE, OPTIONAL> \
+--out_name <PATH_TO_OUTPUT_VIDEO, OPTIONAL>
+```
+视频驱动场景下，需要至少提供source image与driving expression video（作为drv_aud参数），推理指令：
+```bash
+python inference/real3d_infer.py \
+--src_img <PATH_TO_SOURCE_IMAGE> \
+--drv_aud <PATH_TO_EXP_VIDEO> \
+--drv_pose <PATH_TO_POSE_VIDEO, OPTIONAL> \
+--bg_img <PATH_TO_BACKGROUND_IMAGE, OPTIONAL> \
+--out_name <PATH_TO_OUTPUT_VIDEO, OPTIONAL>
+```
+一些可选参数注释：
+- `--drv_pose` 指定时提供了运动pose信息，不指定则为静态运动
+- `--bg_img` 指定时提供了背景信息，不指定则为source image提取的背景
+- `--mouth_amp` 嘴部张幅参数，值越大张幅越大
+- `--map_to_init_pose` 值为`True`时，首帧的pose将被映射到source pose，后续帧也作相同变换
+- `--temperature` 代表audio2motion的采样温度，值越大结果越多样，但同时精确度越低
+- `--out_name` 不指定时，结果将保存在`infer_out/tmp/`中
+- `--out_mode` 值为`final`时，只输出说话人视频；值为`concat_debug`时，同时输出一些可视化的中间结果
+
+指令示例：
+```bash
+python inference/real3d_infer.py \
+--src_img data/raw/examples/Macron.png \
+--drv_aud data/raw/examples/Obama_5s.wav \
+--drv_pose data/raw/examples/May_5s.mp4 \
+--bg_img data/raw/examples/bg.png \
+--out_name output.mp4 \
+--out_mode concat_debug
+```
+
+## ToDo
+- [x] **Release Pre-trained weights of Real3D-Portrait.**
+- [x] **Release Inference Code of Real3D-Portrait.**
+- [x] **Release Gradio Demo of Real3D-Portrait..**
+- [x] **Release Google Colab of Real3D-Portrait..**
+- [ ] **Release Training Code of Real3D-Portrait.**
+
+# 引用我们
+如果这个仓库对你有帮助，请考虑引用我们的工作：
+```
+@article{ye2024real3d,
+  title={Real3D-Portrait: One-shot Realistic 3D Talking Portrait Synthesis},
+  author={Ye, Zhenhui and Zhong, Tianyun and Ren, Yi and Yang, Jiaqi and Li, Weichuang and Huang, Jiawei and Jiang, Ziyue and He, Jinzheng and Huang, Rongjie and Liu, Jinglin and others},
+  journal={arXiv preprint arXiv:2401.08503},
+  year={2024}
+}
+@article{ye2023geneface++,
+  title={GeneFace++: Generalized and Stable Real-Time Audio-Driven 3D Talking Face Generation},
+  author={Ye, Zhenhui and He, Jinzheng and Jiang, Ziyue and Huang, Rongjie and Huang, Jiawei and Liu, Jinglin and Ren, Yi and Yin, Xiang and Ma, Zejun and Zhao, Zhou},
+  journal={arXiv preprint arXiv:2305.00787},
+  year={2023}
+}
+@article{ye2023geneface,
+  title={GeneFace: Generalized and High-Fidelity Audio-Driven 3D Talking Face Synthesis},
+  author={Ye, Zhenhui and Jiang, Ziyue and Ren, Yi and Liu, Jinglin and He, Jinzheng and Zhao, Zhou},
+  journal={arXiv preprint arXiv:2301.13430},
+  year={2023}
+}
+```

README.md

@@ -0,0 +1,149 @@
+# Real3D-Portrait: One-shot Realistic 3D Talking Portrait Synthesis | ICLR 2024 Spotlight
+[![arXiv](https://img.shields.io/badge/arXiv-Paper-%3CCOLOR%3E.svg)](https://arxiv.org/abs/2401.08503)| [![GitHub Stars](https://img.shields.io/github/stars/yerfor/Real3DPortrait
+)](https://github.com/yerfor/Real3DPortrait) | [中文文档](./README-zh.md)
+
+This is the official repo of Real3D-Portrait with Pytorch implementation, for one-shot and high video reality talking portrait synthesis. You can visit our [Demo Page](https://real3dportrait.github.io/) for watching demo videos, and read our [Paper](https://arxiv.org/pdf/2401.08503.pdf) for technical details.
+
+<p align="center">
+    <br>
+    <img src="assets/real3dportrait.png" width="100%"/>
+    <br>
+</p>
+
+## 🔥 Update
+- \[2024.07.02\] We release the training code of the whole system, including audio-to-motion model, image-to-plane model, secc2plane model, and the secc2plane_torso model, please refer to `docs/train_models`. We also release the code to preprocess and binarize the dataset, please refer to `docs/process_data`. Thanks for your patience!
+
+## You may also interested in 
+- We release the code of GeneFace++, ([https://github.com/yerfor/GeneFacePlusPlus](https://github.com/yerfor/GeneFacePlusPlus)), a NeRF-based person-specific talking face system, which aims at producing high-quality talking face videos with extreme idenetity-similarity of the target person.
+
+# Quick Start!
+## Environment Installation
+Please refer to [Installation Guide](docs/prepare_env/install_guide.md), prepare a Conda environment `real3dportrait`.
+## Download Pre-trained & Third-Party Models
+### 3DMM BFM Model
+Download 3DMM BFM Model from [Google Drive](https://drive.google.com/drive/folders/1o4t5YIw7w4cMUN4bgU9nPf6IyWVG1bEk?usp=sharing) or [BaiduYun Disk](https://pan.baidu.com/s/1aqv1z_qZ23Vp2VP4uxxblQ?pwd=m9q5 ) with Password m9q5. 
+
+
+Put all the files in `deep_3drecon/BFM`, the file structure will be like this:
+```
+deep_3drecon/BFM/
+├── 01_MorphableModel.mat
+├── BFM_exp_idx.mat
+├── BFM_front_idx.mat
+├── BFM_model_front.mat
+├── Exp_Pca.bin
+├── facemodel_info.mat
+├── index_mp468_from_mesh35709.npy
+├── mediapipe_in_bfm53201.npy
+└── std_exp.txt
+```
+
+### Pre-trained Real3D-Portrait
+Download Pre-trained Real3D-Portrait：[Google Drive](https://drive.google.com/drive/folders/1MAveJf7RvJ-Opg1f5qhLdoRoC_Gc6nD9?usp=sharing) or [BaiduYun Disk](https://pan.baidu.com/s/1Mjmbn0UtA1Zm9owZ7zWNgQ?pwd=6x4f ) with Password 6x4f
+  
+Put the zip files in `checkpoints` and unzip them, the file structure will be like this:
+```
+checkpoints/
+├── 240210_real3dportrait_orig
+│   ├── audio2secc_vae
+│   │   ├── config.yaml
+│   │   └── model_ckpt_steps_400000.ckpt
+│   └── secc2plane_torso_orig
+│       ├── config.yaml
+│       └── model_ckpt_steps_100000.ckpt
+└── pretrained_ckpts
+    └── mit_b0.pth
+```
+
+## Inference
+Currently, we provide **CLI**, **Gradio WebUI** and **Google Colab** for inference. We support both Audio-Driven and Video-Driven methods:
+
+- For audio-driven, at least prepare `source image` and `driving audio`
+- For video-driven, at least prepare `source image` and `driving expression video`
+
+### Gradio WebUI
+Run Gradio WebUI demo, upload resouces in webpage，click `Generate` button to inference：
+```bash
+python inference/app_real3dportrait.py
+```
+
+### Google Colab
+Run all the cells in this [Colab](https://colab.research.google.com/github/yerfor/Real3DPortrait/blob/main/inference/real3dportrait_demo.ipynb).
+
+### CLI Inference
+Firstly, switch to project folder and activate conda environment:
+```bash
+cd <Real3DPortraitRoot>
+conda activate real3dportrait
+export PYTHONPATH=./
+```
+For audio-driven, provide source image and driving audio:
+```bash
+python inference/real3d_infer.py \
+--src_img <PATH_TO_SOURCE_IMAGE> \
+--drv_aud <PATH_TO_AUDIO> \
+--drv_pose <PATH_TO_POSE_VIDEO, OPTIONAL> \
+--bg_img <PATH_TO_BACKGROUND_IMAGE, OPTIONAL> \
+--out_name <PATH_TO_OUTPUT_VIDEO, OPTIONAL>
+```
+For video-driven, provide source image and driving expression video(as `--drv_aud` parameter):
+```bash
+python inference/real3d_infer.py \
+--src_img <PATH_TO_SOURCE_IMAGE> \
+--drv_aud <PATH_TO_EXP_VIDEO> \
+--drv_pose <PATH_TO_POSE_VIDEO, OPTIONAL> \
+--bg_img <PATH_TO_BACKGROUND_IMAGE, OPTIONAL> \
+--out_name <PATH_TO_OUTPUT_VIDEO, OPTIONAL>
+```
+Some optional parameters：
+- `--drv_pose` provide motion pose information, default to be static poses
+- `--bg_img` provide background information, default to be image extracted from source
+- `--mouth_amp` mouth amplitude, higher value leads to wider mouth
+- `--map_to_init_pose` when set to `True`, the initial pose will be mapped to source pose, and other poses will be equally transformed
+- `--temperature` stands for the sampling temperature of audio2motion, higher for more diverse results at the expense of lower accuracy
+- `--out_name` When not assigned, the results will be stored at `infer_out/tmp/`.
+- `--out_mode` When `final`, only outputs the final result; when `concat_debug`, also outputs visualization of several intermediate process.
+
+Commandline example:
+```bash
+python inference/real3d_infer.py \
+--src_img data/raw/examples/Macron.png \
+--drv_aud data/raw/examples/Obama_5s.wav \
+--drv_pose data/raw/examples/May_5s.mp4 \
+--bg_img data/raw/examples/bg.png \
+--out_name output.mp4 \
+--out_mode concat_debug
+```
+
+# ToDo
+- [x] **Release Pre-trained weights of Real3D-Portrait.**
+- [x] **Release Inference Code of Real3D-Portrait.**
+- [x] **Release Gradio Demo of Real3D-Portrait..**
+- [x] **Release Google Colab of Real3D-Portrait..**
+- [ ] **Release Training Code of Real3D-Portrait.**
+
+# Disclaimer
+Any organization or individual is prohibited from using any technology mentioned in this paper to generate someone's talking video without his/her consent, including but not limited to government leaders, political figures, and celebrities. If you do not comply with this item, you could be in violation of copyright laws.
+
+# Citation
+If you found this repo helpful to your work, please consider cite us:
+```
+@article{ye2024real3d,
+  title={Real3D-Portrait: One-shot Realistic 3D Talking Portrait Synthesis},
+  author={Ye, Zhenhui and Zhong, Tianyun and Ren, Yi and Yang, Jiaqi and Li, Weichuang and Huang, Jiawei and Jiang, Ziyue and He, Jinzheng and Huang, Rongjie and Liu, Jinglin and others},
+  journal={arXiv preprint arXiv:2401.08503},
+  year={2024}
+}
+@article{ye2023geneface++,
+  title={GeneFace++: Generalized and Stable Real-Time Audio-Driven 3D Talking Face Generation},
+  author={Ye, Zhenhui and He, Jinzheng and Jiang, Ziyue and Huang, Rongjie and Huang, Jiawei and Liu, Jinglin and Ren, Yi and Yin, Xiang and Ma, Zejun and Zhao, Zhou},
+  journal={arXiv preprint arXiv:2305.00787},
+  year={2023}
+}
+@article{ye2023geneface,
+  title={GeneFace: Generalized and High-Fidelity Audio-Driven 3D Talking Face Synthesis},
+  author={Ye, Zhenhui and Jiang, Ziyue and Ren, Yi and Liu, Jinglin and He, Jinzheng and Zhao, Zhou},
+  journal={arXiv preprint arXiv:2301.13430},
+  year={2023}
+}
+```

TalkingHead-1KH/.gitignore

@@ -0,0 +1,2 @@
+data/
+.DS_Store

TalkingHead-1KH/LICENSE.txt

@@ -0,0 +1,42 @@
+# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: MIT
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the "Software"),
+# to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense,
+# and/or sell copies of the Software, and to permit persons to whom the
+# Software is furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+# DEALINGS IN THE SOFTWARE.
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */

TalkingHead-1KH/README.md

@@ -0,0 +1,84 @@
+## TalkingHead-1KH Dataset
+![Python 3.7](https://img.shields.io/badge/python-3.7-green.svg?style=plastic)
+![License CC](https://img.shields.io/badge/license-CC-green.svg?style=plastic)
+![Format MP4](https://img.shields.io/badge/format-MP4-green.svg?style=plastic)
+![Resolution 512×512](https://img.shields.io/badge/resolution-512×512-green.svg?style=plastic)
+![Videos 500k](https://img.shields.io/badge/videos-500,000-green.svg?style=plastic)
+
+<img src='teaser.gif' width='800'/>
+
+
+TalkingHead-1KH is a talking-head dataset consisting of YouTube videos, originally created as a benchmark for face-vid2vid:
+
+> **One-Shot Free-View Neural Talking-Head Synthesis for Video Conferencing**<br>
+> Ting-Chun Wang (NVIDIA), Arun Mallya (NVIDIA), Ming-Yu Liu (NVIDIA)<br>
+> https://nvlabs.github.io/face-vid2vid/<br>
+> https://arxiv.org/abs/2011.15126.pdf
+
+The dataset consists of 500k video clips, of which about 80k are greater than 512x512 resolution. Only videos under permissive licenses are included. Note that the number of videos differ from that in the original paper because a more robust preprocessing script was used to split the videos.
+For business inquiries, please visit our website and submit the form: [NVIDIA Research Licensing](https://www.nvidia.com/en-us/research/inquiries/).
+
+
+## Download
+### Unzip the video metadata
+First, unzip the metadata and put it under the root directory:
+```bash
+unzip data_list.zip
+```
+
+### Unit test
+This step downloads a small subset of the dataset to verify the scripts are working on your computer. You can also skip this step if you want to directly download the entire dataset.
+```bash
+bash videos_download_and_crop.sh small
+```
+The processed clips should appear in `small/cropped_clips`.
+
+### Download the entire dataset
+Please run
+```bash
+bash videos_download_and_crop.sh train
+```
+The script will automatically download the YouTube videos, split them into short clips, and then crop and trim them to include only the face regions. The final processed clips should appear in `train/cropped_clips`.
+
+
+## Evaluation
+To download the evaluation set which consists of only 1080p videos, please run
+```bash
+bash videos_download_and_crop.sh val
+```
+The processed clips should appear in `val/cropped_clips`.
+
+We also provide the reconstruction results synthesized by our model [here](https://drive.google.com/file/d/1BX9zaNL_zowTDruvRB3KvebaSUi3WHWc/view?usp=sharing).
+For each video, we use only the first frame to reconstruct all the following frames.
+
+Furthermore, for models trained using the VoxCeleb2 dataset, we also provide comparisons using another model trained on the VoxCeleb2 dataset.
+Please find the reconstruction results [here](https://drive.google.com/file/d/1HVCFj7WOy9KHP1J76wn-ZExh-nQnff9g/view?usp=sharing).
+
+
+## Licenses
+The individual videos were published in YouTube by their respective authors under [Creative Commons BY 3.0](https://creativecommons.org/licenses/by/3.0/legalcode) license.
+The metadata file, the download script file, the processing script file, and the documentation file are made available under [MIT](LICENSE.txt) license. You can **use, redistribute, and adapt it**, as long as you (a) give appropriate credit by **citing our paper**, (b) **indicate any changes** that you've made, and (c) distribute any derivative works **under the same license**.
+
+
+## Privacy
+When collecting the data, we were careful to only include videos that &ndash; to the best of our knowledge &ndash; were intended for free use and redistribution by their respective authors. That said, we are committed to protecting the privacy of individuals who do not wish their videos to be included.
+
+If you would like to remove your video from the dataset, you can either
+
+1. Go to YouTube and change the license of your video, or remove your video entirely.
+2. Contact [[email protected]](mailto:[email protected]). Please include your YouTube video link in the email.
+
+
+## Acknowledgements
+This webpage borrows heavily from the [FFHQ-dataset](https://github.com/NVlabs/ffhq-dataset) page.
+
+## Citation
+If you use this dataset for your work, please cite
+```
+@inproceedings{wang2021facevid2vid,
+  title={One-Shot Free-View Neural Talking-Head Synthesis for Video Conferencing},
+  author={Ting-Chun Wang and Arun Mallya and Ming-Yu Liu},
+  booktitle={CVPR},
+  year={2021}
+}
+```

TalkingHead-1KH/data_list.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fb6179ecac9f08c7c2d960cb8a36354e9fa2edde7215d1fc5a20d6a342050f1a
+size 6822806

TalkingHead-1KH/data_list/small_video_ids.txt

@@ -0,0 +1,2 @@
+--Y9imYnfBw
+-7TMJtnhiPM

TalkingHead-1KH/data_list/small_video_tubes.txt

@@ -0,0 +1,4 @@
+--Y9imYnfBw_0000, 720, 1280, 0, 271, 504, 63, 792, 351
+--Y9imYnfBw_0000, 720, 1280, 1015, 1107, 488, 23, 824, 359
+-7TMJtnhiPM_0000, 720, 1280, 1202, 1607, 345, 26, 857, 538
+-7TMJtnhiPM_0000, 720, 1280, 1608, 1674, 467, 52, 851, 436

TalkingHead-1KH/data_list/train_video_ids.txt

@@ -0,0 +1,2872 @@
+--Y9imYnfBw
+-5Lm-oibjJQ
+-7TMJtnhiPM
+-8lgFak3RwU
+-9qRhqjD7PY
+-A0jrCGS_TE
+-B1Z9vrjpgg
+-B9PTg7XynE
+-BNR218UFX4
+-BPwyGVD2ec
+-DiIOp80LOo
+-EM2xi5Jnd0
+-FbPk4SmV0M
+-GZFS7r2GJ0
+-GtLyhxJ4V4
+-II0hSGU52I
+-J3Z1m6vrCs
+-J9ECsMrNkY
+-KBiHKx54sg
+-KL3wSSFx10
+-Qi6_EcnYEI
+-RuxUoQQaT0
+-TNWOtcCbOU
+-UrrtxGsmoU
+-ZqVCfsj3yg
+-_A67wxucLo
+-_KOlZjGD9c
+-a7oWdSyePY
+-acp9Dk7Ve4
+-axp0KRQJYc
+-ayP3gFLy9s
+-cNe_z2qsGQ
+-dvajjXM2dg
+-dxGq0Mu0oY
+-flEiFmuh64
+-gnJtZFyzZA
+-hLsqlKm-A4
+-mingboa4sM
+-n-HQiT-mkw
+-o8Ns4hvhr0
+-qsTrNdfd1w
+-r-ckuF3JSg
+-tIO8GnrSJM
+-xlo-qTzC7s
+-yhJf04a1mI
+-zqAjBpG0Jw
+0-FQbhkWYuY
+04-JxYnzcq0
+04BN6UOVKOM
+04WJEEb33CY
+04fidgplUXU
+05e4nROsPHM
+06RuVZUbUc8
+07F4x2eeD1k
+0B5ojfWryqA
+0Fz4oferM-c
+0H-cSZFZq_4
+0ND-w_eEOQw
+0NLQfXOo0dM
+0OaM12UIeVA
+0OzxymzEo-A
+0P-9yxO2df4
+0PffDSC_cLU
+0PtUxdA35Zs
+0RQl9e11aeE
+0S269HFjKx0
+0SKHozldsNM
+0S_QNLx2S7E
+0X6ZN_eRaSI
+0_Jci9-d2VY
+0_udiqKlYuU
+0aHSV7nbEvE
+0aZJJHF4K8U
+0aiKZGqjvL4
+0bA8z8qYmdo
+0beHbJDqlOs
+0cSDEZ6NxM8
+0cfM5xNaaSQ
+0ez7IiBfCGE
+0hhbwkQ06EI
+0hmQbZUYG4o
+0kLlrDNq9RA
+0lOkJkh2Hwg
+0lcaZupVMmo
+0oE11MGQRZM
+0pQvRqU43eU
+0pe-7C_c-fY
+0rZJhc1FeU4
+0t9j7CfXTcM
+0uGzSXRi7qQ
+0ueGFLVFi80
+0zTkQXcIaqc
+1-YOlGoKxeM
+13FSE5dCvvM
+1653Kz-SnxA
+16uK2Gbyk2k
+18Kl4qkvGUU
+18Ts-YxUIts
+19baUOGVR2M
+1ADqe6vt0xE
+1BQNyk_BgDQ
+1BuXth0dz5s
+1C7do4tIrfo
+1DgBEF9zbvc
+1FBVcxb4PQA
+1FmSxmpitBA
+1HIHu4mxDLE
+1IJ3ISINhnU
+1JSocaAw--A
+1JwEvzz-kf0
+1MibRcX7KUM
+1MvtBdLyAKc
+1O3ghiyirvU
+1OUkXFsyyP0
+1PDs1mL7TG0
+1PV7Hy_8fhA
+1R5dPw4sYrE
+1SLkDhvOnNQ
+1UHlG0puRUM
+1UyhO1hMhrU
+1WOs6S0VrlA
+1_rRguH_Vx4
+1aQGw_-I25I
+1aY5n5H0ugE
+1bMYn6Vb-Mk
+1bwUe85Mg4U
+1h3J3fDsupc
+1hd5majTxeQ
+1i8607wMy68
+1iXuJgqZhik
+1izPOehU6zA
+1k3DrJOQ008
+1lPU8Tw928Q
+1mAMa9hn-18
+1oPgMsUmC6Y
+1q7yGJbX1sI
+1rxyrZUtm_g
+1tZaTf21D64
+1wKWC0EzK_s
+1wsOU17LXTU
+1yHWhXaXxFk
+1yP_MElLiKM
+1z-0ranXCyc
+1z2ZTPZEx1E
+1z8sqW5xIrA
+216-fEvtTag
+2250kOXVaGE
+22AUNaHGGz8
+22tF3y_epZs
+23eZ2Hjh3gY
+24paFjOVKoE
+264aaUUvB9U
+2CQgBtUJNBc
+2CRIlwxj6uo
+2ChRXYtCCXs
+2DXm-gBAbDQ
+2EgGqTDAiC4
+2H7qsNMyxvU
+2HPH4qtE95k
+2Ix4h6gaa1w
+2JgN7AZO_6Y
+2KTA78QK5-A
+2KmXllrBNHw
+2L-QZfbDm_s
+2LdAF2dHHFU
+2Li-PLW3Aew
+2NV_nn8apOA
+2O_SN4pg-lk
+2ObBpwabNmg
+2OfVzz2aZ8c
+2PejNMim-YQ
+2PqqZcC31ys
+2R3GNE3ob-4
+2Rtp8zWE6nQ
+2T2f2cENuiM
+2TmfeHf8Rqw
+2VLez1fkVq4
+2VdwqgI2l2o
+2bAuuK1uU1M
+2d9BcdLC4dA
+2dt6XZlI-Lg
+2e6a4eog8jc
+2eZgkkbNq54
+2eaxjEpQPkk
+2ep9Eb36OCY
+2gIkmyxa2ds
+2gMaIjvjCFo
+2k7VhqEg7ws
+2llLx_WkF3Q
+2llMolwKs88
+2mLsOuOve8k
+2nhDjEnUslE
+2nj6tpVKUxQ
+2qDH4bPvfMw
+2qVqaA1hn7A
+2rxU553pKgs
+30QwIdz8Vjc
+30gEiweaAVQ
+32DCEzpcVEY
+35ELLdyGmpc
+37QWwuNFVmU
+39Axc3FIu9A
+3AJdjwSsKoI
+3AoYXlMCioI
+3BkL0UXgNDs
+3C0T230n-mo
+3CGECvfakGM
+3EFiRh-y4L0
+3EJtdIIh43s
+3EYU3VTI3IU
+3EsqpF-W_wQ
+3FKX7kcU4hA
+3GUcVRz_JyA
+3I2jJpnw_xY
+3KNGPFM4d6c
+3KtIBPwaRkk
+3NJN-C1R9gA
+3Qlr9YoHBkg
+3RITVJy7ogI
+3Tyan5xgQ4Y
+3UYOtcC00L4
+3YE8YLldLu8
+3Yepui-bWyw
+3ZLYs6Wj51s
+3ZQL-3ElxpY
+3ZlXiEZb3k0
+3_celBcyBJE
+3fyphrZ-yUc
+3gCA0Z4y7wA
+3hPHW74cBRM
+3jESlqRuLw0
+3kyzHQL4s1c
+3l_puRAIDDU
+3lpnoDrt5Tw
+3nJexBvb5UY
+3oV_cljdDs8
+3pIR5nfFNQM
+3qBW_x5fatI
+3tKwbLJEYZg
+3v1tt9mD6cg
+3viekW3AnRA
+3vudNvflufc
+3ytLt3BDqFU
+3zhCbaVBEjg
+4-1Nci8AkAc
+4-7ABAFvCZE
+41Q97FiM2n8
+43wXnwHo_qU
+465Wt-eX2RY
+47OtSe4dffo
+47slfR-Knq0
+4A9OeOEnFyA
+4BPibf6C35E
+4DdTNSKleK4
+4Hg2Eu-F9mU
+4NXUGnLbl5Y
+4Q6NknbHNiQ
+4QQyjqtHwlY
+4QgmM1dcHMw
+4R4aN98Qrbo
+4RN-c0HdJOI
+4cz4pq_N--g
+4gxMVLXmBiE
+4kR-Fev28po
+4l515yNhqxg
+4lB_x0kLffU
+4lOnYQCxbOM
+4neIRMYUT0U
+4nqMZX542bU
+4oXZCuXwrXo
+4q7lHy-1U6o
+4qGWz9v-UXk
+4rJ_RB3Iwws
+4tbkGu8boyo
+4u7y4RPfV5c
+4udsKgSP7UA
+4ujhmvsSE9c
+4x234xeqtHg
+4xlKjVPFFv4
+4zzVjonyHcQ
+50zJWKotPP4
+51Qp9Z3ZlHs
+58GcsaJhTzM
+58K7XpkuMCM
+5JG5xwguT6I
+5K4dtC2A_bw
+5OXy29bFfPk
+5PGVFUADgGY
+5PUTMuJxqZQ
+5TfPoq3lkso
+5WZmSJYYxPk
+5Yz7ssYCa8Y
+5Z8x06jnjSk
+5ZYrc-3gQVo
+5db6EORa_0U
+5dsxC8M7vCM
+5gx3WNfa56g
+5icAa3G2X8E
+5kas2jBObUY
+5kuqWp4gjV4
+5l2j8k759-4
+5leH4t1V9LY
+5llwjtJDqMo
+5meC4Z61qGg
+5okG5zh9ePY
+5oybklNuCZU
+5qaBD538C9k
+5r2Y6QsFACY
+5s9UdpT0TOo
+5shU5ZVQuEc
+5t4cwPdsVgQ
+5ty6o1t2emk
+5u-Aw6NOIy0
+5vAj_wY8c7s
+5xeVIgWq7s4
+5yZEtG_r-wI
+6-6m2H_aE-s
+6-O4gWLQkgk
+60bkwhHzols
+60dSEN1RIbE
+627Ufg0PVkg
+62OgF2Pw09o
+62aDb2JgFQc
+63gG5qmwREo
+63iL92MjDT4
+63stDIiwraM
+64Yx3Odqo9c
+67zz9OzY07c
+681IyvC_JAg
+685CK3xr0jg
+68pNYnDuoQA
+69pi63RgeJ4
+6CIMXUDzKtQ
+6CS50XMGV0o
+6Dtn8MfMPVE
+6Fd-CsvZQX8
+6IA7AuXVSxQ
+6IdeRRqG1ak
+6JIUlIwX3Pw
+6JmuCmI7Iqo
+6KKDqqV8OUk
+6LljU1cDSbI
+6MhEveLVeO4
+6Nyz_yd_GJw
+6QgRw4lDN10
+6R08SayU3bU
+6SA9lGH3JN4
+6SsIE00duz0
+6T5iJ6TjWj0
+6V3kI3QBWK8
+6VVCVtxeq1g
+6YBDqpgpVck
+6aAa8M4vg2k
+6bm6Y9TaX1w
+6bnxY8W5otQ
+6frz5TaAIto
+6gyKLNQH44I
+6iXGqEq_cpI
+6lMj2CGW6u0
+6lzxVmBJIlk
+6mL4rjxEnbo
+6nDOJdyJDk0
+6nPPyCrkwSE
+6ng7_H8pdBo
+6oHmiJLrEAk
+6oLStjIxffE
+6p6boxrvbZk
+6pCBqmPozE4
+6ptI5B4a-ag
+6rKSZPwHTf8
+6tx3pU5F1x8
+6wyDTrAPV7s
+6xSxXiHwMrg
+6yt3pqCQn6s
+6zXMtU6jgc8
+7-AfS9rehcM
+7-ByRppD-EE
+7-QzoS-dW-c
+702SXH0JdaQ
+73hls2GdB5o
+75cv2PgOmX4
+75mMPO0x4Gs
+78UIF4JCcYc
+798S4UbhNE4
+7AjsnSEhZ-w
+7DJdGbYHll4
+7KcRJyXmuzo
+7MBqGLvoQG0
+7ONrAflL5Oc
+7PQ0QmUGpvw
+7Q2Qe-zid24
+7Qu_ETu3vi0
+7Rgh8v8Qmzg
+7STjD4eWMs8
+7UQMqOkGD8M
+7W9ACrwLn1A
+7_ppXSABYLY
+7bAPKKE_tzA
+7bC_8QTdbHM
+7bXAZfRTZQk
+7c0wYvntu8M
+7c7ccOHMK8o
+7dLu8wqYpJw
+7dmNo3X-Lus
+7eOwm6nBBGA
+7elaTVxAEX4
+7f6b7b_yzQY
+7flG_V2SHc0
+7g7RpRl-Pi8
+7hHOLdgvG-4
+7hfFxTbLFWs
+7jyokhjUCyk
+7l7GryUZGvY
+7mHwsV3Mb-Y
+7o8NcTiXGYI
+7oKjW1OIjuw
+7pg_Dgs0wUU
+7s_Sb4-mwes
+7uwUj4aX2YA
+7w68Up6F9ZI
+7zWswXer8i4
+8-RmR1XmaxE
+8-hAwTTVYFM
+80It12pD4j8
+829gaOsl7Z8
+86OqcZtLtgE
+8BZC5UBidm0
+8C5cn_Qj0G8
+8CTEl-Zhv38
+8DzVNa_BjOI
+8HbsSkuiPmg
+8J6FseCtEXc
+8JtWrFwQ-h0
+8KD2cPzxF3U
+8Kux1TQWdLU
+8LSK5wIjZMk
+8MPjG8RMYJY
+8N-x9QC50m0
+8NJ74aVTZMc
+8P3rViI8Xw8
+8SIWlQ0ZUqY
+8ToIwnP2a-g
+8TunLMoE9Xw
+8Uij8BYDuf8
+8WfV93go5TM
+8XJp2c05iVk
+8XlrNlfd-9M
+8_yb_nW5x6I
+8aafXYh_gHA
+8bQPi0ssTLw
+8dI7AzzZLXw
+8e1BMiU951c
+8gg-oKufUo4
+8gtpnlVb31U
+8lrlXoXGQo0
+8pcELaZV2b0
+8qUQwmwC7Oc
+8quGD9W7B2I
+8rwcfIrAXtA
+8uyPpy2ejA4
+8vzKZVqXmo4
+8xhqhC_PHzE
+8xo4s6tYzzs
+9-EtiWDJbJw
+9-a3sSZeXDE
+9-nsXlNXRDw
+90dka2zrP1Y
+90sJHDwKmSg
+91Zu4JRnfxc
+91z62p7t-AU
+95hYfRw1aHA
+95zvFw1VkqQ
+97C_sZ2821s
+9Dm1Sekkcdw
+9G4s_qIxJYk
+9I8mQpFAJ50
+9IGRsXq9Wis
+9ITi5C8vHpw
+9JJ3ullABD8
+9K4A7e3clMM
+9KPyflyHP6s
+9KisGZnflBc
+9MOleOgz5To
+9O2D9K7l-FU
+9Pj-QPlN2CU
+9QeqkUN0bNU
+9S9O2T1B6xE
+9SflFku0eKo
+9UaAyI-uI30
+9Wb409Nlhlw
+9WkYKiltzJU
+9XYx1vUd-v8
+9Zb8e6nE5QI
+9Zuk9Huqdrg
+9_VLbfRXTss
+9awtQRbMhG4
+9c7zctYTBLA
+9dCHp07it-Q
+9eHXo_KFvJ0
+9f0eB75r-Y8
+9h3m1jzWWEU
+9kUljb9G-MY
+9kgiApzKDMw
+9mAiyn6gMJw
+9mnTCYsbKfw
+9n7hn-M4GpI
+9oMmZIJijgY
+9p8z1A3TsxU
+9r8yZ-68pkY
+9s3BPDNEJek
+9t7ujBSH3WM
+9ujM2nAMK0g
+9uyYxs79EcY
+9ychKZIG8ms
+9ydk7zFmOxU
+9ydusnvBSys
+A-QOg-tFApA
+A0AViKj8EGk
+A2B-b-nfCqk
+A2DdRsUdFeU
+A2lEI0kaf3k
+A3uNIgDmqwI
+A6xjN8BqDjk
+A7ktYbVwr90
+AAprf4PLDM4
+ACsoFzXDE3Q
+ADQSqUBZjvE
+AEKZERIDiUk
+AHKGGtJ15o8
+AIXQngEnJgY
+AJa2DO_woJ4
+AKkYU-fExWc
+AL1pISpcG2Q
+AP0S21vT3Co
+AQe2ANirwW4
+AQuAGO9ceIU
+ARXYGhV5VFg
+ATgNzwHSfjw
+AUtRgfFUCl8
+AVc5fXa0oMs
+AWJcd3F-HPY
+AXfRrHD4Cps
+Ac8Fsu0WVKg
+AccPbM4JhFI
+Ad5je4UNgDw
+Adgif4D3ujk
+AfZNZ3bfJvA
+Afy7H04X9Us
+Ag-zqjX1TV8
+Ag1AKIl_2GM
+AglvA1tduMA
+AkU94JdXbQ0
+AnxrJiS5uKU
+AptPjGKdaDU
+Aqu7R_7vFKM
+AuLoMmjFONE
+AwvReatHB2g
+AygCTeXnJ6o
+AzECoalJ4WU
+AziRcPo6rm0
+B-8ovk81nNM
+B-qxGhkRojc
+B0KlNLkO3qE
+B5uSsp0Rbbc
+B7HzMw9rSMs
+BAj3fHStRGI
+BB4bpEKrOlM
+BBWIPL66Fpc
+BDLtYpLZfbU
+BE4Y5Uc53Nc
+BElqXjOG5Gk
+BErluP3jDjw
+BF7hcsKb1WA
+BFWbo17t-Ig
+BFat39XKT2E
+BFoqBnl5XNw
+BGJuAODr8Ks
+BH7-eZYnJkE
+BIuRA0GGIgk
+BK1VxSDsCu8
+BLcZAhQzQF0
+BM2mqrIXY2w
+BNgmYFwUjjw
+BNyCZJOTRZA
+BPLCiXRSBNk
+BQL5wkJS4y0
+BUouNsjhTTc
+BXT72YlkQrA
+BY2ADlTxdZI
+BYgCS4of0TI
+BYvQek24Kbs
+Bb3XWac-WuM
+Bc7WoDXhcjM
+BcTOSxcv2_o
+BcjJjiE4Ivs
+BfdcdAIJh4g
+BhEio_W1LU4
+Bio8ZpEFlqY
+BjrDmB15S-M
+BkTiMi0Owuw
+BlCoAfks8kk
+BoWXd-LNnm4
+BoXR8KrAfIQ
+Boj9eD0Wug8
+BowyM_Wlsd8
+Bq0vohH2pL4
+BsDYAcOsWqk
+BsgJuviWgJI
+Bv3taVzLZZU
+Bv7wdHRhifM
+BzAzXDgqKzc
+Bzb0FhMqaU8
+C2TLW8MS33E
+C2rVYklWl8I
+C3589eDY5rU
+C4bevJm-MbI
+C5WNr5vzUPs
+C7c1LAUbSho
+CA-0cn2Dbgo
+CBYhVcO4WgI
+CEJTBzMKcuk
+CFUGsvVC9mQ
+CG0OnKUqziA
+CIIsZt9c8nA
+CMAiUAvqIh4
+CNFTLmMYY1c
+CPckick_ioM
+CQUP1LhDLPM
+CWLEHmYHNro
+CZjbKqbYS-A
+CaD5TRQQNsI
+Cd4ZxGNbwTw
+CfA30p4X9g4
+CfKy85LA_bs
+CjnVq4zLT6s
+Ckrrk5oBneA
+ClVtccpCs6g
+Clsy0PuGl2A
+Cptzvn3nM0Q
+Cr0cBSpnn40
+Cr2VggKQrQg
+CsG0Or6-SiI
+CtpXTXjQk4o
+CuZnbR4fb_M
+CuoCrLWcsjI
+CuyAI82HFe0
+Cymmi8L0O1E
+CyyImnREpbg
+D0TgSpsBabY
+D17AYqYPFDk
+D2UT1AmyZFE
+D4T0Ffg1I_Y
+D5ZZUHKPC10
+D5gW_X-Db74
+D7wvFZjtVOw
+D7zgzpc_PVI
+D8TFETlLRdA
+D9aRZXIOX5k
+D9ocoySPGOk
+DA2mx0793uI
+DCUnruZponA
+DChlO5fNMGw
+DFqDbrTGTnY
+DGKY7K-pyqw
+DGoTdntvKfs
+DKnHYcKfz6Q
+DMVduTyjp1k
+DPDF4odJrq8
+DRHNdWF4Kho
+DSoKEFb8R_w
+DStiXbf9Mk0
+D_obsdDVv20
+Ddltu_Cq4E0
+DdtEInQEQ-s
+DeeRqZd7sCE
+DgGKe3kCx74
+DgpbuHAOgf8
+Di9AwmKtblo
+DjHPXW6Crac
+DnOLvKEYIQI
+DrDOMMmwPvI
+Dtfi2BHWwaY
+DtuJ55tmjps
+DuNW0KQ_GZM
+E-hVDqrQq6M
+E-qQNDCVSnk
+E1QbVtkza54
+E1gXzYA0tFA
+E2uUfyT64VY
+E3Q61rKXhrM
+E4zPYb7O2EU
+E7XkvbCu-jU
+E9tUCuAZ-LU
+EAc4WhNQZ30
+EB_d1jK1R44
+ECfLtssUZa0
+EEq8OK4BUyM
+EFLh9Vqr-YU
+EGCqtu8qujE
+EH6q2YGx45M
+EHp_I9ETmtQ
+EHrbAhrbw9Y
+EJMlBT6jptE
+EJZvz09LVa4
+EK3WoT1Gqvs
+ENXP9HEul98
+EQJN03a6M28
+EQLg-kHxwCA
+EQ_VlFQT9hg
+EUS1m5MSt9k
+EUqWrqb9Oug
+EVBOSk6kEwg
+EWSp2QMzKv8
+EYaKFlWd2MY
+E_LYrCtoTIA
+EaHZLUWwxfQ
+EaQOEXTkKYg
+EckpALTiYhE
+Ed4bxiP1RPM
+EeqwFjqFvJA
+EiW4lKrMXQ4
+EjB9J20nulw
+EjBkTt0LHbU
+EnPJk_9Ug7I
+EnU6HVRC4s0
+EoHwvsJcBNg
+Eojazns82hw
+Eotj0EeepoQ
+Eoxazjg1NUA
+Erso0HgtV5A
+Esu34JYC2YQ
+EtctGvH92Ww
+EuVBpqBgmvM
+Eup2Ca9Kiis
+EwXXD0uLj8I
+Eylcb4rbLSo
+F-GzNvvs-lU
+F-nsVjM7FmU
+F07fXd4vVlg
+F0InXG0ln4Y
+F2G-buBtp7w
+F2yK5VkHRaA
+F4xgvj4kSnU
+F5KV-iaMKK0
+F63B6wWXGtA
+F6ShWjU7GaI
+F7oit5SKxdw
+FAf0YtSelug
+FBQDiiEbknE
+FBbfCBOJOt4
+FEc-U45TzKM
+FGgMrNSmMn4
+FICT79cA3U4
+FL9qpSH5eKw
+FLI0WmBWWv4
+FLIippwrXSc
+FMIDAWVPq7c
+FNErh9EogUg
+FTN_93Px-Qc
+FUSU_WYPwx4
+FV4aEpanJ2I
+FV7tKSeGr3Y
+FXbC_3_8tGM
+FYo5E7zT-vM
+FZLwYiceIOU
+FaoVpVXcZsA
+FlCNvBBqIyU
+FloFzFl0jZc
+FnKhFaijBBI
+FoYce_3oUGs
+Fpo6nvSZirI
+Ft1Nw-Hy8Ao
+Fwh0r8YNLIU
+FxBLtp7UpTI
+FxQchGBQpZA
+FxoOE2dTCHE
+FxrCNf8utsE
+FyPFdBhklEw
+FzXWP7ZHs-Q
+G-6OXtSMyNI
+G0n970JRNII
+G2tfebkUbPo
+G4SMtaNDtfk
+G5hOJXZmqPc
+G66ClBEmWdY
+G7GI04txkOM
+G7T6dTs5YKw
+G9CHdvWwzQ0
+GAS_7760FSo
+GBvfnfwGq5Q
+GG0F0uXuIqQ
+GGIZzL-1gZ0
+GHjjXhd6WAc
+GJar81QVmQk
+GKCxRcuLm8o
+GLUdt99wUY8
+GMndqLvTqhA
+GNx4EbTu10w
+GOQWmjLkqOU
+GPEavB9GXHc
+GS1UT0mSks4
+GTsSe03hPxY
+GUZ4T_xFtwQ
+GVFq0_6imAA
+GXldrjxDZqQ
+GZRbKJMEdk0
+GZVxh8CQFkg
+GbUfu1wF02s
+GcNJkyYSmW8
+GcayBgPOr04
+Gd_zypjbv9E
+Gf8lUImdL3g
+GiShqIyw-_0
+GnwqktjWrVM
+GoZ6KwuAdT0
+GpAOMD6Z_Cs
+GsYl_thySnM
+GtzbWxb9nuQ
+GuVNl_oEMuQ
+GwU1VXsDXbc
+GxDeY_UiGBg
+GxRgKs4TpWo
+Gzo5PEGQpe8
+H2LhBAi-Q8I
+H450lZb-Mdg
+H4TYEoft_rM
+H6L04OEm71w
+H6ODJtcqyTg
+H8t_snz8B5A
+H91qCYIfZuQ
+HA8PjarK2mo
+HAFKRtBHFlQ
+HC-zhWIIC5w
+HCMZ4s2A_k8
+HFCmCsxt1xw
+HFTVP9qIMPE
+HLXDpFYmyqo
+HNdh6Valoys
+HOpKzDhCFtE
+HPHNakev2ss
+HQpZQj3TY80
+HSjvtpwKyhU
+HSu21_qc2kA
+HTJyZwYPQOI
+HUoUVlK_bHI
+HVMqm9jlUDM
+HWNwvBrUUGQ
+HXSxHJO6Srg
+HZ0pn4ijwnQ
+HZuhPDbZtcg
+HbKvcTZqNA0
+Hb_SVDUmWzo
+Hc6K7g7wqrs
+Hcy27nbeNWY
+HdzDCZ28cI4
+Hg0qN4cNJfY
+HhFPSCGRFHY
+HhNo_IOPOtU
+HhljUdMUbs0
+HkfLT86wPkM
+HmexkGBB428
+HsGbFpi3xtk
+Hy6QRP3ENl8
+HyzD8pNlpwI
+HztoBDblr8o
+HzzUW9y9FCQ
+I-9uLKZmxOw
+I-HFjHKJJ7E
+I0lqvxqEKhU
+I1fZdwFStnY
+I25TYNMclKk
+I2Z6LNkwijk
+I3RMF_9xW1o
+I68lZ9jptWU
+I8tkl9kVfaI
+I9taZpV2JfU
+IAYJhZS231s
+IB6_L9xsnAo
+IBESpBTIQTQ
+IC3EX6ipxFo
+IGEJo3QHvSI
+IGgpoI_0oPs
+IKMjg2fEGgE
+ILQii0-r-bE
+IM-TQHJKefA
+IM1xpnkmG7o
+IMEIBu0uULg
+IMFI8waM8rs
+IPPVWl-jPqk
+IRBAZJ4lF0U
+ISip9JRbYNs
+ITBbGDndjGM
+ITjHgkTTX_s
+IaHioK3Ljz0
+IahmVXN7xEQ
+Iam-aEiQOeI
+IatKu7sngG8
+IdZPa5vWdtc
+If2Fw0z6uxY
+IfnTz7vZyVg
+Iib7x8rYE7E
+ImCLzPvVKTI
+InL3YA_6P6s
+InP5DEpeVSU
+InSvBuHK4vI
+Io6JjgckHbg
+IsMFdiLsqbg
+Is_C4-xmayE
+IuuRvopzIf8
+IxYu1FAY5qc
+J1OGqF5Eo1k
+J3-ySSl7ceM
+J44SPYSVAAc
+J4vKu-s3OqM
+J71SYNCcRQI
+J78srd3-odQ
+J7BLQbvZyrU
+J7ei1-rYHMU
+J8ifUEgXF-o
+JC6pZ92y-hk
+JFkqEW-sz1c
+JGwMIlpgR2A
+JIEgN3las5E
+JIvGXG4z9X4
+JKg4o6SHbCY
+JLEPOAlZ7LU
+JP5ywOknF-8
+JPP_3u8gD-U
+JPmZtJ8vgAo
+JQTxw8OdBKw
+JTq75Se8vRA
+JUM_s4uQDW4
+JUSVl1JXYl0
+JVSwiQ0tNLM
+JZTlzyHnQeA
+Je69HPxSd_c
+JeZ5gAUnlFk
+JfjLKBO27nw
+JgwRwOWyR0Y
+JjSIRfrDKF4
+JmWnjvHEM38
+JmjxtqnhzHI
+JpL__knumpM
+JqQtOGCWrQM
+JrIwV6YniCg
+JrXU1owDEVs
+Jriq9eOSu7g
+JsFEIvtKCns
+Jtu5eFsIPmw
+Juen1iIJQTE
+Juv1TqsYiV4
+JvpxC406_jA
+JxR5EZ_GY1o
+JxUqtbpjpqg
+JxWjxAqCrCQ
+K-FG2oWl-2k
+K1hPyYEMp3s
+K2POkUf2EUQ
+K2QS3ZvjPMM
+K2bh3ZJOFnI
+K2pUtcVSXEo
+K3c6AMXsam8
+KAsvI2qAzlc
+KBQRetXolA8
+KCcenWMXQQ8
+KD6bh5ZfS2k
+KFDlZxR4yG4
+KIiKNpySv6w
+KK5120o36GM
+KNjgy1o65SA
+KP3ToVHnOZY
+KQQbMdsFsdQ
+KQkK8ThNsOY
+KTxhy419vto
+KZvz22uAVM4
+K_vUbBQzFjw
+KcdTad5fztE
+Kd9RcLW7knw
+Kgs0QbthCEU
+Khnx2cNTiu0
+Ki9AHohJpIc
+KihglmOX7j0
+KjMe0TXyQzM
+KjWVlz6cAyY
+KlC5HJFI40M
+KlcZwSsceL0
+KmmMz3NfoU4
+KoqaUANGvpA
+KqygzQmEuhA
+Kr3OvLakOgk
+Kr4Xe-BghFU
+KrEzYXCjJDQ
+KtHgPTAdfYM
+KtVfCgNJdeg
+KuqcVxRqQHI
+KxbRf-ZlSSk
+KxcNu6WZSCY
+KyBD2AeGXIE
+KyDTB0i_wQ0
+KzVEGlNNXuI
+KzVzRN5XoJs
+Kz_wvavZp6c
+L-bQmqP82oU
+L0LZDwNDqRQ
+L13WvYq8G68
+L37uYJnDxVY
+L5M93LmOmqw
+L6SiuhBZWDk
+LCsx9rVQjwE
+LDXrXC0cPBo
+LFBC0d4i3jE
+LG8TfyiQr7w
+LGumhl8-kiY
+LH0IOQrB-NU
+LH1WrGpM7p8
+LHFY1Vg97AQ
+LHNNYQ57V2c
+LJJgAqdxBdA
+LL-SFPRBBFw
+LObC_A4G-_c
+LR7ZZ5gw984
+LRXrEiBeOXo
+LTZPvLi3Hdc
+LUhyxjNub8I
+LUolzAltwKI
+LVYVlHr2FKU
+LXAcG9mITz0
+Lawz1Mc16Vo
+LbQo2tKGxV4
+LbR6jjbgbis
+Lbx1F4V_-8Q
+LgT5uYkkPE4
+Lgo_bkq9SWU
+LhEzvegA-Dg
+LhF_sfDfBd4
+LiCmCW6EGzA
+LiWszVY2lW8
+Li_m7BVja44
+Liv3vT9dGhU
+LkCrGs-XeYI
+Llq5mmhLy8s
+LosFY2otV8E
+Lq2icL5Y_FM
+Lv3WAxgaZqU
+Lx48775nwWs
+LylMvKFdwJU
+M5axFzT2_u0
+M5zDCmXSejU
+M6ZdYNFo6gM
+M76UHFsQp2U
+M8SkT5nE-0s
+M8ayDH5DuJA
+M9Xsci4JUy4
+MCQ3H2jfCBs
+MDyjY3uiWp0
+MFcNY_CyXk4
+MGGCuJdo8xs
+MJnxzdG5QwM
+MLUJVpk0BM8
+MLay5YHp48w
+MMoBEZ_d2g8
+MMwKA-Ku1mM
+MSr3hE3z2jw
+MT6uJni993A
+MTBfv2io-pQ
+MVI_yEqA2RE
+Makb_p6HcxE
+Me95iJdHO18
+MeM6r8Nj8G0
+MfS4oDLBpp0
+MgLHVw0tUBg
+Mh3Dvs7DwdM
+MimJZypAiy8
+Misd5Qrx_CI
+Ml1I6WEYSAY
+MmfiHdQ4Wfs
+MousuD_jX24
+MpGxOR50sn4
+Mrcn4Q50j5s
+Mro6gxnOfus
+Ms9K0eZLnFQ
+MsA2OJiYApw
+MsBU3uGpUGw
+N15YpxEHjVs
+N1cJakFhjNo
+N4t59MjWdsg
+N5esEarb5MQ
+N8XGCs0js30
+N9JXjNCR1EM
+NDsl1_vHHTc
+NISKpzp_QAM
+NKJfBsc5kHk
+NKKaWmoFdZA
+NL6zaIPoU-Y
+NMWqBL_Uhr4
+NM_SJwSMRT8
+NMpf6HNYIzc
+NNK4pvyOhAU
+NNp4yt9dHns
+NNwM_OMLa10
+NOYFz4DIfh8
+NSZvUu8Q8ZQ
+NUKIA9I6gRA
+N_kIz8R84jU
+Nbn1NJjbqoc
+Nc-HP2vyKoE
+Nchrj-dzVgs
+NdKNfaWpCj0
+Ndnq9Ofs2eA
+NhOwdlKHcAE
+NhfsI4jbWgk
+NjN07qsdh0w
+Nk6N7ieiphs
+Nm2nt6dxVv0
+NmBcsOMtKqM
+NnPyqGW2w2Y
+NoIHCm4wrpk
+NoZ7ujJhb3k
+Nom2-9WmsWU
+NpskXvrCNA0
+NysbSdox6zM
+O0D0E42AA4I
+O0hP4Hrek4s
+O1UBCHWPIqE
+O1d4dHwSZqs
+O1j79d0IuhU
+O2AnvMTbDCw
+O2YHi_g2JuY
+O3iPo__LYZQ
+O3jxAN3j_P4
+O3sFnc87STU
+O42jXkaQtQU
+O7dLxPQvIkI
+O8G_glgmBcA
+O8gZs9BCr4Y
+OBlw3eBxHvA
+OCQd02hORJQ
+ODx9C7kHmWs
+OGhvz1fwacA
+OGzPe8LXHeA
+OHHqegRBDWg
+OI4sCSsyS4s
+OLbdw1imnKY
+OLe9xNH4G6s
+OLfHpvJKNg0
+OMeIMC_s0GQ
+OMw8kl2kcZY
+ON3YD52Df5s
+ONvg9SbauMg
+OOJ7OYp1gjg
+OPzyc9rXx-I
+OR6qP-X5fcs
+ORObgzbi8Fc
+OSrCb8eDWWk
+OT_0JLIALxk
+OUE8rKxYGLY
+OUoQ4c1YyJM
+OWdqgZQdMgw
+OX0OARBqBp0
+OYJh1xJbQys
+OZWiY3xcr1w
+OcPW_rtcbio
+Odb1XiXpRTg
+Odw7NOmXeaU
+OgzTnofR1WY
+OhHBPhb8a1I
+OhsXUPDBX90
+Oj22JVi762E
+OmdRZ6ZM_pc
+OnPdfl9qKRg
+OpaFC283wJE
+Opx0cWUuaBk
+OqTI0KYkeJQ
+OsBgWxoAOf0
+OsLj4GyNvYQ
+Osp20p7mHLw
+OtTAH-lcO_M
+OuP7vIfN3GI
+OufFx-XmsLM
+OyNBayVulb4
+Oysv7K93B-w
+OzIUFdCRm4o
+Ozk5w3I6wlU
+P-9Z6WeojxU
+P2-SOq0SrmU
+P9M__yYbsZ4
+P9S15gsSywg
+PAFybFU4fzI
+PGQTyzsX7V8
+PJXZQrwDPdQ
+PJdbIALsMYQ
+PNJghHAUlLg
+PRgoisHRmUE
+PT0KwTjsMUo
+PTOZrIogdhs
+PTaeZggiMrM
+PUYoRT2EA5Q
+PVLxoTzL31U
+PXNdbOr8f9s
+PXf59S2kFag
+Pa-5lbobZpk
+PaEO7DFyzZY
+PiHMIYoV3OE
+PjlEL4poXaU
+Pnfh3Bxo4mE
+PnxpH92CJOU
+PpE-wfM5NhU
+PtV9uxQHnGY
+Puu_PXPw_H0
+Pw_4_f6PQno
+Pw_JdBkki_I
+PxVZGdKMmvc
+Pxg5mHeIoTA
+PxiRvHqbQoo
+PyNVMlDyNtg
+PzzNuCk-e0Y
+Q1MYDq7GgBc
+Q5iKAqZ9yVU
+Q71FNI--3vk
+Q9W5Lxr-7v4
+QEKRkIbCZEg
+QEz1caz0loM
+QFJZwvkJsGk
+QFbp5scBzys
+QGbL4LlrcIs
+QHeYQalwm8Q
+QIz-y9-jywM
+QJj7WiwcadU
+QKxRCmpAFKE
+QL2Hb-v1r0A
+QLunLzt4r4k
+QMcdgAriqy0
+QO7Jhl-r-BY
+QP2zJO0AtlA
+QPKKQnijnsM
+QTp2snIa-cU
+QWgbM9DoA7A
+QX1d52gmEZ4
+QZhzZtExnfc
+QaqyAdedM0Q
+Qd3DEZud65I
+QgscBSUsuNU
+Qh7rX2S4lFs
+Qi4P0mLkkLQ
+QkmGfY9iY9Y
+QkwYl6HJHdc
+Qlzca3efn6E
+Qmp2Z7wPR4Y
+QowV8kgwHX8
+QpXUjqjewGU
+QqsPM1rd688
+Qru-q3ykC48
+QspDZ-DYs0Y
+R0XyMhCdSkY
+R3QabWdSsxY
+R5XePwAO4m0
+R7ZX64ASSe8
+R9KOp8PKhpY
+R9P4_3GEjS8
+RAMbIz3Y2JA
+RCrTM0fHg-o
+RDeLBp_-3sM
+RDkMkH4drhc
+REUvXBK7ypQ
+REzffEzpiUM
+RFM-LCECtmk
+RPTk00TE3Ak
+RQXCRoVV9Hc
+RSJQ7iFntRA
+RVG1EXFbRb0
+RVQcpNgh6sI
+RVluy0cjHbA
+RWjDH6Od5xU
+RWoOC8KvHEA
+RXEUWvpmJaI
+RY8e2Ivu4Ak
+Raiw3nozIoc
+RaqHo26ohYs
+Rb2fCxGGcHE
+RdGOK7ZAHMc
+RdhwDd8PW0Y
+Rdi5ExhmqHM
+RdlMCh2idHI
+RfP1AjOOtSE
+RfmgkgzNhYU
+Rh5qNYU-_jI
+RhcnVBgKxEg
+Rhn0fatp9PI
+Ri-M_Vo3w5A
+RjzuefWqVY8
+Rm5MIya_48o
+RovZoquZGn4
+RsC_d7GnZtI
+RsX1lwPnPPQ
+RvjlBL4A_8U
+RweXbb_OzBU
+RyiqQnCd7qQ
+RyszHongpf0
+S-imxWoyMD0
+S0MWAAykFuc
+S1MV9j0dPAQ
+S4Fje5FUgfw
+S4nfcw632Oo
+S5OzDdLlsUI
+SBUwUUIVwHM
+SCjA7BJDnEM
+SD-LjOboaE0
+SDCICtm9zXQ
+SEDDSCUJxK8
+SFE7NNxfbM4
+SH80ZuySDW4
+SLaZEauuEU8
+SNBMdDaYhZA
+SNcjWH6ZhPI
+SOUME9xzIxk
+SSGEoCsFoH4
+SSfhcpWRUrM
+SSnj8kkmNDI
+STKRq8VXzjw
+STKb-ai6874
+STWrgFYmkL4
+SVR3ZmdAV-A
+SYriZ4xtdDM
+SaznCPVAiJc
+Sd3QWZ76IZg
+SfaKxqo1NfQ
+SfkSpHzhZb8
+Sg-HIZ3qgtk
+SjNRtrZjkfE
+SjbFjIeSCf0
+SlJh09MsJ7I
+SnwbCVxeEVU
+SoeKlf4DcSE
+SqRfNG6yLEk
+SrLPH5590RU
+SsIDNPoW7q8
+Su8Q8XMQzIM
+SuYeKcei7Zo
+SvWCVOGF6vs
+SzjmpNTVH6U
+Szx43_ah4ys
+T-CAP-ULW_A
+T0qjkPboQXY
+T4kVhUwJZdo
+T6P9TCdWE64
+T9X3YhUWsDc
+TAdw9R0ku2o
+TFPS-iX0L4s
+TGCJQR2BZhc
+TIgRgGQ2azQ
+TJXW9hpOlnI
+TMjlO7UUubU
+TNEWst_1m4s
+TOT4eRF4oCU
+TOzkrCUSlFg
+TP2wD0LX4T8
+TQ2NfO8grLs
+TQ7NqpFMbFs
+TQHxvPooKZc
+TQzJKL_4l44
+TS5BXMeG890
+TUVeg-x9Za4
+TX0v0pIVKUY
+TXjwmCoRmhM
+TckoXQoHj7c
+Tg2yHK2Hnag
+Tm1RbCh9YeA
+Tq-AsJ8M5yw
+Trq6vcUOeQE
+Ts04-23URYA
+Tu22Y0kIzJ0
+TuoGVwBkTEA
+TxC0dIBPzZg
+TzzhAQLRwT8
+U-kFZbOf6Nk
+U1VkpKvSn5w
+U2LvmqmOEZI
+U2eqvs_MZGg
+U4auzU8E2ms
+U5Ze75nn72M
+UA-U6m9O5OI
+UBVzrTJEbS0
+UDvCqeXCI-o
+UFXt6O5cxjw
+UGRFE3vTcRA
+UIV2gt1Jzno
+UIr0uGUWN6g
+UJeKNl461d8
+UJjixKei0ag
+UKBlZt_JL-I
+UKc54igtdXI
+UL42vZliknk
+UOGLLo60dzI
+UPRvrLsqN8U
+UPU_yi9Nv3M
+UQO5Qcl8aAk
+URC125wpMS4
+UT7n0WY_6Ww
+UTYuVdJskY0
+UUT_nhIlR_U
+UUoBfrbl1Cs
+UV1luTyPOiY
+UVTsXdLyIsk
+UVaN02jNqgM
+UXEr-xcC46E
+UXRKQjlYUZU
+UXZm1KDE_qA
+UbyxFZSZZ90
+UcwU4ghl9Gg
+UdpTELIW7bU
+UeMQ_al9lDQ
+UeO44STvnJw
+UjAxgnvxIJE
+UlKZ83REIkA
+Ul_ZfzfHRek
+UlsgLGCVKks
+UmA3FpFowh4
+UoVJllDh6rg
+UpScMViXT1s
+Ux-ExovbpsE
+V1frWTXGVN8
+V2DvDDzrFqI
+V2XB2l3aFvc
+V2x33UdHq4w
+V3nPA6doMBM
+V5FsNyk9rDo
+V5blsv5pn60
+V5uV1vR2M6g
+V6egGoCrbIo
+V7uG1tWVqHM
+VC7lRZTxDng
+VDLUQlOR_nI
+VF0hNn-Yfn0
+VFJsMQnqZZ4
+VFeOyT16oEI
+VFf_xowsMcQ
+VGOslZT-f-I
+VJjiUcZCkzw
+VKfVhH35RL4
+VKnNVP23toY
+VNtYIBI3BFc
+VOFQb8sVnxs
+VOomWcgrHis
+VRu86oG1hVY
+VSRuncwwJyQ
+VTsKS3ccd5M
+VVV3XeAevc8
+VWc3ezusNTg
+VWrYbF797LI
+VYE91m6Rli4
+VYKXayozHFU
+VZ0-gccLaNU
+VZki-LyHI0E
+V_lZ61OB0EI
+VcLWubfmJcM
+VcYUWSBo4i4
+Vcvl5piGlYg
+VePpQBCbKBw
+VfLSoyinXmo
+VfmcO0gQ8B4
+VfxI-6_LL54
+VgrW-fB3EXI
+Vm2mKdh_VUc
+VmPl5QClxYk
+VpKnSOS6NGo
+Vpw09qKQal0
+VsOismDNYjE
+VvXMMtldJU0
+Vw7hC4jpglg
+Vwxft_tqrcg
+Vy6PKQy2OI0
+VyJCLVAj-vE
+VzHAbg24fRs
+W-7t8Zho4AI
+W-_X1HZM7ys
+W0Mw5dfZoyg
+W0caVAMB7LQ
+W1tOTi9L7Hs
+W354uz6KPJE
+W3H5b9yZcOs
+W3XADagE6P8
+W42cARRDRe4
+W4PKfrQ2J5Q
+W8mQ41XaZvo
+W9okU7bAE-U
+WA1L8vXkSKQ
+WA9sMdQzdiA
+WAl7wIw5ReQ
+WBZ3das2pd0
+WDialA8RHEg
+WIiuU7Fd-KA
+WJBgKbe1ZyU
+WJFdyqLo-pM
+WJaW32ZTyKE
+WJlK8D9Vy1k
+WL5IJ4JVjrg
+WLVKRBitiY4
+WMl7MSsJMGI
+WP4jOqfpJx8
+WPIIgluc2vA
+WQ2QwrfZHTk
+WQ4W-UqaaMo
+WQXM3hU-7vk
+WQiGCGxYMm4
+WR4vkITJckY
+WT2aDvMsZF8
+WTxTntSi6Jw
+WUx-OqSLBak
+WUzV_mDcBmk
+WXKNBguF0a0
+WZQi3LOqiNc
+WZw5Vz_imPM
+WcLq2oABDhA
+WeHTlU8efMA
+WgWkOiDD4n4
+WiILKjGKveI
+WiNgIQ1pOlE
+Wj6MkLWm8dQ
+Wmt5FzPTuXE
+WnQCWE22AYs
+Wq5ZIu0UZmE
+Wqb2ZPi6KyI
+Wrb8tfqYPzw
+WsQ7ysVt-0A
+WssviVcyncg
+Wx9v_J34Fyo
+Wxp1QG7GIBs
+WysUzNnT4i0
+Wz6DxjLZSbc
+WzyI1gx_PHI
+X-fZ5fMfAv4
+X-irPLY6oLk
+X2ac5fHYZ3I
+X3bcvqrTTJI
+X3tI7SRuvhw
+X3ztiC2hUXM
+X7Jd6mI9iHM
+X7WI5_UUpys
+XAJMmm6sLQs
+XAhljQgieEs
+XAlAgC7rYug
+XBF-Pd_asag
+XEHxXWpF6qI
+XIXA-Kqb_Os
+XKCWJuWup08
+XL6m6Zl2ejc
+XLSJ7NIldNo
+XPgdPEYr648
+XPwYTyTbCag
+XQy5b5mlVZ4
+XUxgC9KYoPE
+XVtVVuER3W4
+XXKVwqZS1R8
+XXgJZXUPZXM
+XZ70Li8v68o
+XZxl6h-TxQ4
+XaGss5gNQEM
+XabAlfY1TyQ
+XaxSxkApUQc
+Xb32P_VWh7w
+XbDyn-2_xIc
+XbNcsgEX2jc
+XbtdjM8wGv4
+XcZaQG7wUpA
+Xdlo2HW5jwk
+Xf6xJiXV3AM
+XfMoY5WSlvc
+XkNofrvAt2s
+XkX-zJa_9Wo
+XlQzQ55BA0c
+XlYXYqwEeJw
+XoVZxH4CwMQ
+Xq-YIJA61W4
+XrGuIdJ0Ckg
+XvfpyrdeZ0s
+Xw2PEuhphPk
+Xwu5PSZfocc
+XxqpGCkrF8g
+XyvlWUQAkxM
+Y05wSSdslyI
+Y1QnypwCPZo
+Y2zmCgmDVqk
+Y3sbt-6ndq0
+Y65tbaBs88M
+Y6Mn-d9QMxY
+Y9anPwwkuU4
+Y9rkKtK1b44
+YC3Hhxqbof0
+YECIYrmXH0o
+YEgE-m8kzpM
+YGdNSqa_clI
+YGwLb86Uso8
+YIDhGzp8Z2s
+YJkS7QqeYPw
+YMoa5JpjEtM
+YMqKHYZHQD0
+YO-oamNqsA4
+YOD6jIFD5aw
+YP_35HEWcfc
+YUJa7_i3Cn8
+YUxb5mw96eQ
+YWpX7tlXLB0
+YXPEXwjYpGA
+YXfpaJVsVa4
+YYTcoNCWy8c
+YZBLFRe-G0A
+YZLGkfy0_oM
+YZME4lTWBHY
+YZp4fNMECVg
+YcGOV1iUBlE
+YeIn1rFbTuw
+Yex9CPZzezk
+Yf_jdF01azE
+YgCBzXR63l8
+YhoYie6_la0
+YiPEZWaxSXs
+YjTCns1fFmM
+YjwTPflZm70
+YlIl427ZdHc
+YnRgf2UNBXA
+Yo-CEXgHwkk
+YpmKbkU-X_U
+YtiVmh7UKOw
+YuLSfLZ9apM
+Yui2Msy8X1E
+YvRRXziXpMY
+YwqTTdUEJGc
+Z12xymgfH9k
+Z3r9tnbK_iU
+Z5SpCVtoOdw
+Z75PmkL_UaQ
+Z8iLMnTX6OQ
+ZAX4LKUeAVE
+ZAYK9cMiSbE
+ZBcOyv8LZ8s
+ZDfRJiIODMs
+ZDv9njERj0s
+ZEHCzjk0Hrk
+ZFnr_mTTL0E
+ZGzQUBDGd-g
+ZH6357PHRCc
+ZHLsDRxEMGs
+ZPHfpA_4uhY
+ZQPrD6cbqHQ
+ZQQwgYaQh1w
+ZSEV1ivNUWY
+ZSTo8stxfG0
+ZUlkRGgSLfI
+ZXMx5C9oFk8
+ZYSjPZUqLdk
+ZZIjgZYnP6Q
+Z_TNQsWm8SM
+ZcO5PLLXLy0
+Zg4zFxyeLMQ
+Zi2RoikofDs
+Zj7U7R-2fcs
+ZjmynEwugnk
+Zn8iU3-RNL0
+ZnpqfpPzQAE
+Zot-rd9KoYE
+ZoyUo4ZY-70
+ZpCK7G0LQAc
+ZpNQS6CX7FA
+ZpbEtTNDQBM
+ZptziXSxpx0
+ZsMJ8wO1YKs
+ZsY6kAHzdwM
+ZtXr7bckLyc
+ZvAaTKbPzH8
+ZyOqaUFxojc
+ZyUCcfMcmyE
+ZzVhwsetzDk
+_1B60L4m60M
+_4bzRoRn260
+_4r4LIX84TM
+_5O0_4kkKSg
+_5r3yqpaNuM
+_6AK5vp6nDE
+_6ck3EDlssw
+_7RMd0g9FtQ
+_A-pZH-S4jk
+_AvNT3vyzr0
+_BxhdmIGNys
+_CKFycGHzLo
+_F6bq0l18Ng
+_FcVS3nf3Qs
+_H71S7ar21o
+_ITluulB74Q
+_K-JEM9RNeA
+_KIjLRuf4NQ
+_LR1NtZOISg
+_OhqC6w4C28
+_QRahNSxQfc
+_SYvYBxt_Dg
+_UVuXcclWM8
+_WN-6t58HdM
+_Xo-Cy_okCE
+_ZZVgjbePRI
+_ZdfwflsC3U
+_aOcddJCrSQ
+_ak7BooJlnk
+_bBrj6QBPW0
+_cg2nSAfeUY
+_czQyx6JYgc
+_f2hDpHsQlg
+_gJN7I0a9XU
+_gR3p72XEMg
+_gm8Jp_4a5E
+_hE7CQmHPNI
+_hYhPQTHOMg
+_kUyGB9O_Hs
+_nwF3zYpfQ4
+_piu_0WXnOA
+_rWxLXmPdc0
+_sWO1yvsmmo
+_t61OZ5dN9Q
+_uV1LuL6yDQ
+_y9aHEd08LM
+_yKGZu_8gyI
+_zYWKwpARf0
+a3Ah-REb5vc
+a3mNe6zQS7Y
+a5CAVvzL2GA
+a5lUNW9wlpY
+a8-ySFmij_I
+aA1H4iLap44
+aB0Ku9J6jHg
+aDURTM8MwIE
+aGPO0OB2_Ak
+aJAQVletzdY
+aLPYGw9HubM
+aN5gtrkwgb4
+aQM3Hbvospw
+aSDRWUh7nqQ
+aSF8LfYNlSM
+aSp8Pf6zXvg
+aSuaW_0lLYY
+aTigt0mmyiM
+aValNp9ADko
+aVstEGR5jt0
+aVtOmrULE2w
+aZn-H9uoYk8
+abAitsu2-iA
+acWlXpQu83s
+ad7p5nSZbBg
+adXUEwBrrl0
+ae527aiePIg
+afqjRWOtCzM
+ajI8zq2SedY
+aluYFRd96Kk
+aqz-KE-bpKQ
+asFNxO3cXMY
+asyKEIv1pZw
+atd0dgRkl2k
+au5oFT_zkYU
+avITb-TZWOQ
+ays4JisCVAw
+azPa3uccg2I
+b-lZA6gHWg8
+b0dX-hS1mUY
+b3wy7s_QvOY
+b44IhiCuNw4
+b4jlwhPxTpo
+b552H9Gclo0
+b6DszEFrkSQ
+bBpQgQ416dY
+bCbKYU8D2QA
+bCqq2y9Ge-A
+bDjCnFvz11A
+bHeksIj7dpM
+bI0kawdOOXs
+bKqwRavOIVY
+bL2qA6Gt6Ko
+bMtTAQHLc6A
+bPnX7JwHQEg
+bQ5gacQY7y8
+bRh_KJ6yyGU
+bSySiLIcnzs
+bUAg8d3aQZU
+bWh4bNRqaK0
+b_PbouAwYIU
+babTVMWIosM
+bc15oBT5Nec
+be0sJh-pxOg
+bebrJ-o4d2E
+bg6f32E9vN0
+bhMSZQLSp2M
+bi3iqJykwEo
+bj77Ljq7kbA
+bjNvrJwE9Dc
+bk4b1P_IHZ8
+bkUDUnE6aP8
+bkZBYQ7mnrQ
+blCUScrxzdo
+bn5OSGM8oKc
+bqfA6lRrlaU
+bqfAc6X_uUY
+bsNztQbe4Ic
+bugT5lGfl9M
+bvCbqmHHQD8
+bzDZY35Vacw
+c0IrO0Rnlls
+c0XSC8b_9hU
+c1Fjrbt7peU
+c1H__IGGs8E
+c30s6OnCApw
+c41Mm_I_pBc
+c62KFa9jY8E
+c6gVQ2e1zEs
+c7baTdyHv8g
+cBXqfQzrsEY
+cCLOtOW3mIk
+cDJ_DVILBuk
+cFhONVldyE0
+cG_MFoGh_EM
+cH3DOOXFE24
+cIpXNRmZtVc
+cLMUQZr0TOg
+cLVQSb0l2Pw
+cLcmBWC4FBI
+cLofgTNATIQ
+cOtoozzRy8g
+cRiOZTv0dzs
+cSEqa8cLqss
+cSp0k6jEvY4
+cTFi0WEdtvo
+cUkZLttONGM
+cVezZhj7sC0
+cWZD_1CxIJw
+cWdvVQ7hIa8
+cXpb2qzMlJ8
+c_Ok8VutSbw
+caSPzISFhiM
+ca_kO_J5rWg
+cafMvJshI1c
+cbqshch5yAA
+ciV5Vi-3Adw
+cmA7Ze4KPd8
+cngWph0hiVo
+co-tsFhaeRQ
+coswCgQ_Zwg
+cpAUXdNPnQs
+crJ3KpoMxdk
+cvlLKmzsEBc
+cy9rx19dujU
+d0am7-lXaus
+d0npGPi3UPw
+d2b921R6Q7U
+d4gLpU_fqmE
+d9ShdxJRArw
+dBHgd6MNyyE
+dCmfHeROKIE
+dFxMzUUWLeM
+dGwu92AU4TU
+dHx4lFPqPiI
+dIFsZ9uTrpE
+dKfNJ2umK4o
+dMN2CeulM6o
+dMjan7yZ4gU
+dMxd2IrTF7w
+dMxrAgNbTRk
+dMyTRV4x50w
+dP4U1yI1WZ0
+dR4CGlYvT7g
+dSfIxFW_X1c
+dUKtnNKWV44
+dVeMgCpj-4c
+dZCCdLDE8gI
+d_CTrbVqWW0
+d_oCEre1NJ0
+daIt5KbxFNY
+daTulcsI2RU
+dbkgiCgapGc
+dd03Lj8KOjA
+dd9EimkKEZw
+dk2FVz_B5ls
+dnQnF9apUF8
+dnYUpixA8m0
+do2lClbRBmA
+dpHBJkblp2Y
+dt1MS1YmxyY
+dt1XiYVENuk
+dt9To7mMjl0
+dt9tqCG8neM
+dw6X5-D3tmY
+dwd_Z917XGI
+dxWIZ0ODlZE
+e0Q9GzQzBhE
+e1kPOlccTbM
+e3rjGvtVXeo
+e6MyukRaKOc
+e7jMlBiYy98
+e7z4uzyeRzA
+e82tT1ge9Ak
+e89XopekEJI
+eAQBVytibCc
+eCKD8fP2aao
+eDGFEV4CtB0
+eF7TdzaX_Jc
+eFUL4yP0vqo
+eFdjgis1E5I
+eGYkD0e411I
+eILksASDf9w
+eIXNlBbw54g
+eKl16S8gUXk
+eLX1KG3FnBg
+eNtpsXBrhk4
+eO8hhdMv7P4
+eOJ7XVQlnB8
+eObAEb5SBmg
+eQY39NuSXHU
+eQcmzGIKrzg
+eQjsCvqh1_Y
+eRU1XIwXCAE
+eRvOQ8MsyV0
+eSR9swuM61I
+eT5IGtWmQ-M
+eVXJaSgEf74
+eVz41Qm-5aU
+eX3PtJN52g8
+eXeWNtyjgPk
+eY5c3ce48Rs
+e_7T20B65hQ
+edRk4JmnLkM
+efIVRw-eVzU
+efooj84xalQ
+eg0bYuuHzkI
+ehMbGtUiYh4
+ehoNb1i6vY0
+ejrSSx2FMhQ
+eke0xp4rprM
+ekxjCWMq8Z0
+emqCYr2xVGw
+eoHw0W_CzX8
+eoN2Xrs6cxc
+eoWIjARiH6Y
+eowka-cMoxM
+epfVelQrqX8
+eppNMltmm-o
+err7vaMQ4z4
+esVois0s3qg
+esnMNkFAG1U
+estUu_XdB7k
+etnFlNJ-uys
+eub0JuAMSiI
+evORVtwC0zk
+evnEcyr9dIw
+ewQ3FI_u5nY
+exsaT4HrbhA
+exvJz3JPHuI
+eyBEIx7fCfY
+ezQjNJUSraY
+ezxm9w77JKI
+f5kTVt7QnK4
+f5mk7tpuEs4
+f5ud_4pbXPk
+f7HNmm8MHMI
+f9oet4cvYf4
+fAa51WqtOT0
+fAtWwadP6CY
+fDZ8fFOesDQ
+fJrNkspUMno
+fKHcxXH3f6k
+fLQ-pj8lteY
+fLc6CPIre_4
+fO1dl3Kzz40
+fREa-rfs8aU
+fSSzNXPVvYc
+fUQ-OTqjp9c
+fUzh2Hu4JoY
+fV1xRX3XGKk
+fWnDlFMfOA0
+fX_sxgqAKkg
+fXuqsnqsfGE
+fYHqUK0tqH8
+fZ8S5N2akUE
+f_LHuvCZLf0
+fb8d5RZqzA0
+fceagC5I1Yk
+fdANKfoxm6s
+ferxnIAT2Go
+ffVL9wvWfgs
+fgIfH_85nBw
+fgzKbL4HjCs
+fhXPsdrWJiA
+fhrwGiZrvio
+fjW_WL3KLUg
+fk3Cq0mR6_4
+fkJt1i4aO_8
+fkYqJFYFYXI
+fk_RkZortw8
+fnArfARx9wI
+fnW2Qy7uY9Y
+fo65xXZqfoA
+fsUkET-XhzM
+fu1ZVnk2Gqk
+fuRU10ocuds
+fuZ88PJ7Lsg
+fvtbdq3WiyU
+fxIAaegID9M
+fyFOwJRr3I0
+g0UDwv1vPgw
+g0cy2szFR4Y
+g3pERE3VsX8
+g4zf6FeVoxM
+g8DbS9Ochow
+gAjMBetApiA
+gB-ZlM-aUTE
+gB6vLAjfMdc
+gE9soy7r-Ms
+gFZ8NPmkPpg
+gFZj_2voVyA
+gGZq_aR_jyg
+gMZp4NeJq5w
+gOprZJ2CbR0
+gRO6JH35Cdc
+gROnZPk4dnk
+gTDprJn164c
+gTPaMD7qhek
+gTmG636eAYY
+gTzhAelL1go
+gV7XWdt72Vo
+gV8icTgcuyU
+gX0AEmOrKsY
+gX6r9BWwCOI
+gZyG2_WKQvc
+gaUt3gTwwzU
+gb13T2EYzvM
+gcsL5WUXoYU
+gdb3IbS2SGQ
+gksr08xM7dc
+gmKD-PSkATI
+gn_UhhjygOk
+gqxjTyxDC-Q
+gs2r0YmX7rk
+gvYPX1ja9gU
+gwysz5CtZLg
+gxYTI3KmFk0
+gzfKeagBy1k
+h-h1ceF9ToI
+h0Jgoc3tosQ
+h16CeeTSV2Y
+h3sdAzf02O8
+h5ZU5xukqyE
+h7HDtT_cOs4
+hB5cFvlTYWk
+hCc4qd-6h_Y
+hFj2tLk_poo
+hHHYqKqySy8
+hIkJ1Q69eJI
+hK4VPzJYU94
+hMmYNZgItKc
+hN4A9M4qXy8
+hNhlfyoVlLQ
+hO4aTFDeokA
+hOloWe3oYR4
+hPUs3kB39sA
+hPhJlvpNz2Y
+hPt5JG9kGUc
+hQyEUb5DS2o
+hQzC6_3S6xg
+hS-re7JJ8ts
+hS2fdP1bNV0
+hSUDE3-e5_0
+hUKXRN4RzpA
+hXtL9Oe2OTI
+halJEOj2RP4
+hasaLmOLJ8k
+hcl4vYMfvYQ
+hfMHcdPo7fk
+hihIrNRfnpQ
+hnAqGpkvYIM
+hoAGRdA2Zpo
+hpK6FZbxjgs
+hq-1BIaFjGc
+hqbW-EaOcUY
+hr5Px8dD_iw
+hrb-HL0UUpM
+hrhBvOz8ius
+hsfgj1OWX0Q
+htCPtKDBmKw
+htZAga_nSW8
+hv4NNsPaL44
+hw0VaaAhfjM
+hx0rraflBpg
+hxHjQZSANwI
+hxa7Kd8P-oM
+hykLNP-Cpsc
+i-vmCEgFW5g
+i0CP1dmr4ng
+i1-9WDuXJGY
+i17JKd_eCAU
+i1srQA7ocZs
+i2l72jO-otU
+i3DQ9UOs8nY
+i4jX18-l9pI
+i4oLqHRxoZ8
+i5HYNnHtcdY
+i5z06C9Mi00
+i9WbGqPeY8k
+iAWKPOyYdXo
+iCPxwkaCXLk
+iCd5W4gwJsI
+iDJVid5u0Mo
+iDWMIMCMNiQ
+iFJ2pJph-Do
+iIw0Hfm9h7E
+iJJxd-XGQMo
+iKAYJ-makHU
+iN0gsOIWQgw
+iNTSOGM5rCY
+iNhHBJkMz58
+iPDuJS2QUrc
+iSYvDl5iBFQ
+iT5-jVpP_TQ
+iTk4dutvfsE
+iVBD4cmI3_E
+iVbefrog1uI
+iVu7DNjFgWk
+iWRrLD7H98s
+iX1P6XAvtlM
+iXUwLs4kNvc
+iY2XmAG6Rl0
+i_QZJaPorpU
+ib2KHcsUS18
+id1m2sLzwrE
+id8Uq_339xM
+idAzpl13_sc
+ihpS0LTtWjw
+ijPyu-jUJ5Y
+ijlyytCZ_RI
+ikDYpq4uESA
+io3xwOBD4f0
+ioFH4kyF0kY
+ip0yFdCqbkY
+iqm-XEqpayc
+iqmkldsIxkI
+irKLUxa7jkA
+it0EYBBl5LI
+iuqe74uT7fw
+iy-uKpGZmrQ
+j-0L20FH2Zc
+j-nCvuPZoWo
+j2WVRXdVu_4
+j2k6EQOoRzU
+j3BVnXHPjyw
+j6FjjenQyko
+j7LhySzyh7k
+j7XDYxypFb0
+j921XOqiQ7o
+j9Q2J2MDctA
+jA4WY877mTk
+jBox8_bAoTM
+jHVnt6VROM8
+jLLVBH5ZZG0
+jLRyB9Hz7U4
+jNyPJ1LQmiA
+jOj_NSDXMz0
+jPxamG3f5TE
+jTaFavHwbiI
+jXqb7dbUBHs
+jYEIkpTjpwI
+jZa3FK_T_Z4
+j_k1TZLK2mQ
+jbzpyRmU3YA
+jcFgCdlsPjI
+jec9rBFqcwc
+jfcrY85C_-k
+jgT5X37XW3w
+jhUnM7jekNo
+jhaa9bIPuOY
+jpAig_5u60Y
+jv-dZw3ITJs
+jvRleA3lWBw
+jvr7UJI47UM
+jwBi_Q9ffys
+jwS75RtJJBc
+jzbqbu-0bxc
+k-pmfynqbko
+k5QJ8s3qUyA
+k6Hf0WGquNM
+k9Kb4Pwu3ok
+k9zYM3XqNps
+kBLs0CsEWEg
+kEQyvL6El-M
+kHBSmEfwZZs
+kLdPQfLv1g0
+kM8iZqP4GDk
+kPUqlB13Kn4
+kQLhEtSx0NA
+kYVd_HIkGw4
+kYXiegTXsEs
+kaBI3TvA7vA
+kdIG_ZsrdUs
+kdQPXCcV2K0
+kdo0b2eiXVI
+kfssQHnlHjM
+khm7702744s
+kl5eVK9kY_c
+klh8JFoo4Og
+kmTRmjeoKYA
+kp33ZprO0Ck
+kqFzRSwX3Fs
+kqsmCUo3xEQ
+kr5Ot5x8mmc
+kvOzjIRylsM
+kzgohxtf3hg
+l1wHRYkFhZg
+l29q7OxKMmM
+l5bkGI-CGFI
+l5hgKlBq2aE
+l5tn1sJ0fKg
+l6iQSCD9UeI
+l7OIqpr56NY
+l82uiWKCgF0
+lAzL_47Wkbc
+lB98olcaEm0
+lClz0KqwrFU
+lDJEC7OX6AE
+lHakoK0j9qA
+lIRpgkLK7r0
+lJYwN2X7IbI
+lK44daVVh2w
+lM1Zmq0Lcd4
+lOwZ3mwlenw
+lSGfxPD4nR0
+lZZgjJz-SEw
+ldiJbsXZgvg
+le3cBRlWSE8
+lfrBVdmeUE0
+lgWY-gjEYW0
+lh_ZUt-O34Y
+li7HdK2vIlw
+liKbDWr0RzI
+liVQZnSAv28
+llj7LzTULog
+losLXh9sDc8
+lpShGGv6iJI
+lqbSxAvUNpc
+ltPAsp71rmI
+lxBYJOEbU_g
+lxE9lqRrcuo
+lxQjwbUiM9w
+lz1RY_CNWeU
+lza-61vEfcw
+lzniYTLmFkc
+m03Bn3jgP48
+m0EwquC6wBU
+m0svbKxpLhI
+m0ttpozYW14
+m1lhGqNCZlA
+m3ZogsjvmLA
+m3q4bKFCkio
+m3q7itlxq14
+m4yEAVUxsFM
+m5_41I_BLc8
+m9qPsbp-l00
+m9y0Kt9UFYc
+mAUNqn8QAFU
+mCghaYzVDxw
+mD24h-bbdMU
+mEP7QCH2oYc
+mETDq4gwIkU
+mFRkTyvuamE
+mFu85lirRfw
+mLTM_vEz1jU
+mMiiyP2W-qQ
+mQu-g8K7qtc
+mQyL3LgJwXA
+mRzqtElhGsY
+mXBHUYIXQAU
+mXNd6zUVwP0
+m_D83abmA4Q
+mbKfbtQvUgQ
+mcZfJHqf4KY
+meka3_pUVqc
+mexPA0ocnnQ
+mf8vuUJoNaI
+mfuwTh61hMA
+mfzWMIEVrFk
+mg7CQGLjRzE
+mhZJxmbSSt4
+mhxEcl-85pM
+miGwjQa0txo
+mjI_IyzUye0
+mp_aq18zg38
+mqSucLmIFeg
+mtXfzd53wRQ
+mu1XN7ABANM
+mw4GQ5jhL6g
+myDLLgryIs8
+myGaZRw-0oU
+myaOG-5N12M
+n-EpKQ6xIJs
+n-Ptyqo4lE0
+n-mmxFdva4w
+n1WR4pPT1Jo
+n7YJE14tLLQ
+n7Z6KyTeouY
+n9Gl1hBgjIU
+nAyLvwIcnRc
+nHEr8-LtfUo
+nHU621TjCi0
+nISa7ahsQVY
+nMIE9IKTV6Y
+nN4fDhAcGTM
+nNgIi4eJduY
+nNuk96Rw2Ks
+nQeMM3bqM1M
+nQgkkH-o-aE
+nQmB8u7aBZs
+nQpuGwWyFQ0
+nSsQFG8QzcE
+nWo--dSu9bs
+nXcU8x_xK18
+nZQ_jnOVFeU
+n__HtO28X8M
+naFR4znnS_0
+ngdwHwnLCt8
+niKRw8zeYa8
+nihu7deKG9E
+nj-YK3JJCIU
+nkCgvEXj7FQ
+nky_VSE43Hs
+noYQhtNG-RA
+nor0P6jwoeg
+nsqOggafySQ
+nsrEmJ_19v4
+nu3p9JQ4ykM
+nujU66hnemE
+nvGGjCIIrm0
+nwyoo3oYYpE
+nxk8alj07sY
+nyes4M2CbtI
+nyxYzQVqo20
+nyyS0FSztKc
+o0lvhDX6DXc
+o1zo4BYFdtU
+o3z2CwnY_5M
+o4R1-TLkxBs
+o5q1ne_uNE0
+o5ufkJMr24c
+oBRZ8sD-OjE
+oCM304tbwcM
+oKC8ikzEFIA
+oMbvC_siQyc
+oNGKZrLIO0U
+oOdNHtF_s5o
+oRvkMz0FXtw
+oSBD9lej7oA
+oWVfSS5m5LY
+oWqLRywhozk
+oYKlQPSZr9A
+o_EvSqIz1EE
+oarLQY6VJzs
+oe45d8WFc20
+oe4VZ1YDL9Y
+oeFSoHWiLBI
+ofWA7ERRwzs
+ogOqpmvv6eU
+ol3tAxnNccY
+om1FNUWg4yo
+oo5gIrWn9tI
+oonc4u-Adbc
+oun9ZWMYYVQ
+owZYdzNJSUo
+owqMWjBPCW4
+oxQHiGgFQ2A
+oxsRS9fialY
+p-xcYP1ef6M
+p1lHlsF1LeM
+p2tdlkmZ8D4
+p4Q2mwsvlpQ
+p9ijo4bNOJI
+pBoDJqyONC4
+pD8z8Dior4A
+pD9cAOBZDj4
+pDySXCjwXJg
+pFXjD9J-JE0
+pHgSyR3qUas
+pI1feWHeUq4
+pIeCH_unCd4
+pJznoC1fZ_U
+pKu6GJdrKOM
+pL3u5ztU9kA
+pOBuggPY9_c
+pQ4C6LaLV48
+pQMiwYP8mq0
+pQcXM9HJXjs
+pRm187D8-MQ
+pSeYDrDjV48
+pUayltZ4PRk
+pXg1P_wX79Q
+pZ3OMf01xJs
+pb85Bw9sHY0
+pcOdWPkjUzY
+peYcx-xDiwg
+pfC0yuJtMc0
+phnXyHYku8k
+pidRKmq2iGI
+pk5WuA39qzc
+pkSnKgRHjU8
+pnmOhTbhKhc
+po-Dx2CgFGs
+pp71BH0UlvE
+ppPbjVv2XeQ
+pru-95YczT4
+psfAc33Ok94
+puDuigN0_I8
+puZZoKkv2Nk
+pujqknUWycM
+px-H1yOAQUo
+pzIQGk5nWDw
+pzev9CB3vuM
+q-abd-Tjf_g
+q0whherUIeo
+q1NTjsXYpK0
+q1SYOXJWXNg
+q2S5mFVGVec
+q3kU8MyAHhE
+q3zhx8M7mgI
+q5uELNh_Fro
+q6irtmJxniE
+q7Pfap5IRMc
+q7qZd-5PQec
+q8G-VO3gy5o
+q8PoT8bhElc
+qDzY7_qAP9E
+qEHexC4KaLA
+qGl8JMXmwL0
+qH8uMr3qb_M
+qKMd70Jr-ws
+qKnrUrYLlj0
+qLciglFWSBY
+qLxgp5VIx4A
+qLyCi_ARgfM
+qMoypa93-_M
+qMroyKN05zQ
+qOxbcVscVuQ
+qRiBE2DuGA4
+qTPgNaT-IyY
+qUbu1hTOKQQ
+qWd6VN9sseU
+qXBgNj7d2aI
+qYW_VBZoTsM
+qZdAl2dAL7k
+q_EJdzfnPSg
+qatNSkb_O3Y
+qav1y7G15JQ
+qb4Kx1MoxGQ
+qbaocv8MUJI
+qckQShckJI0
+qfKmOf3d0fc
+qj63Fyah8Jw
+qmwgjonregk
+qnQhGZNOIzE
+qnRbz82xzWk
+qnboyP15mi8
+qo_ZzjhbWLk
+qrmmPQC6o4I
+qsOjHdZtUM4
+qwMVYILJ7bc
+qxabzkWQ744
+r0i10UvjWBY
+r28QlaYxRDY
+r2REC5k-2AE
+r2vFxIWtQ-E
+r341ehyHft4
+r40wnEAEozQ
+r4yqeuWlJqM
+r59GRI81jQU
+r6xt8HZy1-k
+r7vzgexzXOk
+rA2g4mWai58
+rA4jmcefefQ
+rAnvXFlvV3M
+rEJBmcJlEy0
+rFEg1iwdHBM
+rFOs4b_WTYY
+rGBU08IrdrI
+rGqAwF16IOk
+rHZlSM-m_Xo
+rHgcNbOwnas
+rI8ccs3k0kE
+rJXB9VhK1eM
+rJc0Pf2B7Mg
+rKCT1mmxfb0
+rKvnvavLhKg
+rLUoN3LY6Fk
+rLnNaewOAbY
+rNOyRIU9WUk
+rNqiDGKZiZE
+rPa_Q-v7iPU
+rRexKBeW-LY
+rRsjDRqRvgA
+rS8oFARouuA
+rTgVxBHEKJY
+rWMEyWjaDug
+rZ5iulZ3rvk
+rc744Z9IjhY
+rcdFlgadpYQ
+rfzHohoFoZc
+rgj6AQ0KiXI
+rhFDM4YtAjM
+rizm_pzU7xo
+rjjFQ6onAJM
+rjlNd5leBDY
+rl4qZtDSLCs
+rl6K4-2q4JI
+rnzWJv_L13Y
+rpC49dRxPkM
+rqqgH8fMsOs
+rrzJQMo6QQ4
+ruAnF4N_8y4
+rw0AUfItOss
+ryJxN-ZRAGk
+rztwo4iUnqo
+s1QCigF6JOg
+s2Ya_pPoKQw
+s322l02OzWM
+s4b9dN-X32Q
+s5eZtpsX2Uw
+s6LlkCsQSq0
+s8i4K8MvQjg
+s9aNVQP9Bi8
+s9zkSyuL8eQ
+sCk_3RNWdZI
+sEnf_XAMuso
+sGvmS07G46I
+sIesHZHrHck
+sJIvQ5imN20
+sK-QPNTRj24
+sKFiaAcqTMY
+sKRcy61BFfM
+sNHDCXNLLlw
+sS7O_qMN_pM
+sSrSorPjUvU
+sTKAL3OfgSQ
+s_A2wZB7ZfA
+s_gSNpE1Z5Y
+sb7TL-252e0
+sboMcBqlShY
+sdeLg0on1Go
+sdnbXzi5yDQ
+sebF4SCrhgY
+sgGPHfZ6O00
+sgRTFPJ1MdE
+shkbyqqkkTA
+si4zS_Jx_uY
+skyd1JiJiYs
+slAdtDXiNgI
+sp4yAT08rqw
+sphx939ru4U
+spi7TCSQqns
+sqseuym5HKI
+ssHyZRRz6ek
+stqdcOSqbnI
+suK34prc56o
+swP3fNDUD0M
+sy-JNEKRe6w
+t-N3In2rLI4
+t1hyNFQq1Qc
+t1ok0e9gTRo
+t3SPY13b64M
+t43zNbKooJs
+t4xtn8b1Nk0
+t7caALtdVBM
+t7f-iQf3PRA
+t9nyOewqrU0
+tAbhaguKARw
+tBvd7OSDGgQ
+tC0vzGbKXWs
+tCskNV4zvIA
+tDNEWvgVQU8
+tE9_u7slTrc
+tFHxf3tCRr4
+tGQgcHMIq1g
+tHGLoklGaE0
+tIjr1ZMPEkc
+tMTZDgGaWqE
+tPX6j_22umA
+tQRaArBTpu0
+tRBf2g6XqCU
+tSUX6HReL6o
+tSf72G2vTU8
+tTyFNcuR8dw
+tUPzZFsH9NM
+tXwwOXxj3Wk
+tYS3l-lh39Q
+tZMO74qrrJk
+tZZyNqO9bQ0
+tZzyiJC7pqs
+teIlbfF2SE0
+tgKPxj4-0QE
+thhRZXvb0fE
+tj46eMPD8jw
+tj79bCk8Y7E
+tlPIi5sLOoI
+tlSl7JSQ35w
+tm50929wbaA
+tmAx6umELmU
+tnX_ibkICWk
+tng9I_Yfhg8
+trGNjNAxMMA
+tssS0XyUIi0
+tsxCu5Fom2A
+tu3g6qTyWiU
+tvsBIX44mQM
+tygH_zAKRvc
+u2gB0EGQxxA
+u5TYUHIGqzA
+u5eti_-WK9g
+u61kdTMLJTQ
+u7bLXr675gw
+u8B3lvaw9tw
+u9QPtZEhH3o
+uAWBQI7PVKA
+uAkCPrihEzQ
+uAsqM8rL8GA
+uB9UEkO77MU
+uCQMZfnM-a4
+uGUee74VHAc
+uHmQzfPWBG0
+uK0u0BHCF3s
+uKbvq15Pjw8
+uLycRSuHSAs
+uM9zA0QAvhA
+uOWT7-wqKS4
+uQ2I4dZh9O4
+uQEUqgEVQvU
+uQz8Y_jYzy0
+uSs1HeOShlY
+uTUY9W-eDZk
+uW6e50NYlWE
+uZEfW4QJot0
+ufnnbfJSZ3M
+ulMrIHCeLQU
+un5l34RRYR4
+uomUPcl17N8
+upw774ewNjg
+uqCecnqjL1g
+uqav8KWIBPM
+uvNun-YAzwk
+uwJtuNmBb90
+uxBzmTylbwg
+uxQbWbWVWug
+uxy82zQnQpo
+v-2yFMzxqwU
+v1DXBLkiqFM
+v1RFw8FpEJQ
+v1wFRiV9_v4
+v2DjHVhnjQI
+v2_u0yV5ua0
+v5vIyfkumj0
+v62wDFbK7iM
+v7-eoymQp3o
+v79U6LcewMc
+v7iWP-4rYg4
+v8R4spPNofA
+vC-uE8RAzbo
+vC9i_jB5h1k
+vKHi1XtBPQE
+vLb2GXb-bsE
+vM39qhXle4g
+vO87PpdQKV4
+vOFoltimf5o
+vQtLms02PFM
+vT4z4TCf4uk
+vTKx76zG23U
+vXPaPV2Ii-Y
+v_bumG-g0Dc
+vblot9RiULI
+vcL75ICzL0w
+vf4CrrsnVWs
+vhZqXOG9Sfo
+vi9kXfl4Nzo
+vjjNnix_rGQ
+vlddfDNoCS8
+vmGW-kulgy8
+vmhYHtGvqCI
+vmtEmebxSwo
+vpAyvSQdqvU
+vqqv18miTHs
+vsrK3zDsT4c
+vt6evh25rpY
+vtBXhiKZjWk
+vuHbp02YfKA
+vuThpe-Rgxs
+vueMpDhosM8
+vviufcserks
+vwuBZEo9eAE
+vyUiZfxuXXk
+vyWMhZ-NiHg
+vyrHk4_-epU
+w09mGrJy_h8
+w0ztlIAYTCU
+w4ZSe355UKY
+w4fzfZkZ2Ng
+w7zqBnrLxiw
+w9V5LKdTPRY
+w9rXlL5iJNk
+wAaHY_SXdgY
+wB1IOUjeiXU
+wB5lKhN4uoY
+wD-jLNmRVfw
+wDT446J2XME
+wF17z_DX7DA
+wFS3088FEUk
+wFkig7Vh8YU
+wHJB9u61btI
+wJKn7xElol8
+wKnbAj-2Hnw
+wLvCU_o362k
+wN-xfvQKBgc
+wNcTvkR4u8s
+wRQGTMig_P8
+wSDR6ymotHk
+wSO5y2Vq8Ss
+wTHo7j8Ruww
+wTYpjp9Bzrs
+wUnPxBv2h1E
+wV7TB-8PyhU
+wVnhS8BuE4U
+w_Ms7re3lYU
+waHHFLAet8A
+wbokl5PhXEg
+wfCZYvqVFKM
+whbyuy2nHBg
+wkYeYfLt8P4
+wm9QvbgryRg
+wmKPIm2HuXM
+wnH9X7Y8-fw
+wonfEPTNA0w
+wrtkQSU_Ekc
+wrtrwY2caSI
+wwBfqXf-1Xg
+wxNELCR-TMk
+wyM-7ra8uY4
+wz3TSWrZVjM
+x-YG6j9ZYsY
+x-ty9x6-P1o
+x1HMmnAI5sc
+x2Sh6TOWjkM
+x47V2e-4NmY
+x7jZG8cbqBI
+x8LIzP4vBrQ
+x8jdx-lf2Dw
+x99gUDpak34
+xCDQAlQfzNs
+xD1mRhx68eE
+xFFs9UgOAlE
+xGoCF5bk56A
+xITRYnvl0oc
+xIyfkI0jJMs
+xJUlD1q4bRw
+xL32w7DMwHM
+xLSzvPQpo5A
+xMhd22uQusw
+xSaGl8fiiYk
+xVuZZS93qtc
+xWrJD9-1o44
+xZ0Xdew--H4
+x_0F7lFtPRY
+xe21-4hVA20
+xfXkzZ62y8g
+xgpgHZ9iTMo
+xjwSr2qv1cY
+xkf9Gk1QuYo
+xmXiws6G8Mc
+xnLoToJVQH4
+xog_GEISGDE
+xokbf-np33Q
+xt7bRz9dX4A
+xuVTCXSW6WM
+xz7Ilex4McU
+y0jwPOra9Jg
+y2tCX54cDx4
+y5yxjABbdVQ
+y6Dh74INR_I
+y6Ge4RZsq4I
+y6zsW14YDBY
+y7bKFMO0d-w
+y9L5dA7pQxk
+yAe2kgW47G0
+yBEfb0XnMec
+yF8dBlsLn6k
+yHaY_d3QQR8
+yLkqI2UiZJU
+yMSj47Xs2_g
+yPqieOw41gc
+yRV2bb-tRMk
+yV6ZXGQDFfo
+yWJDUISGUp4
+yX5g1OHKE-E
+yXRXk5ubwW8
+yXhj9XnXpbI
+yXrJCyJBugg
+yY9QkbpDLaE
+yZ6vSn7PaPI
+y_-a5D-1_mE
+ya8cSGAiEyw
+yc0wPSjL8rc
+yc3l8Xsk4k4
+yeVl67bOTD4
+yew316-hSiM
+yg49p-i94rM
+yjD-_z6P75s
+yn_0j3hDnPI
+yqCnnSfXrRg
+yrqpVREbHfs
+ysCHqf1HfEQ
+ysogiWIjy10
+yu2iuSrpjHo
+yu6gBfUQFNI
+yuGOVGNkjAA
+yuNg5JugqrE
+yumyfeToWyg
+yvGUNb80FTI
+yvWV56LXTh8
+yvcUoZqSvbk
+yw--rqjGfI8
+ywljr9RKExQ
+yxPk1s6kjls
+yxuTqbqzzpM
+yz4H6fx15Uk
+z-KziTO_5so
+z2-YMvHXvkk
+z6kC2r9UrF8
+z83CZnYt72o
+z8X4DB-ecWM
+z8gNQpsYIco
+z93Kr9LmMCo
+zArkP0YCXw0
+zESeeaFDVSw
+zF-pekRLJ3M
+zF9VM-voMFQ
+zJIKjeOLOFw
+zJahR4BmskA
+zJsr4Tie-tk
+zJuuBn8mTP8
+zKkJB8Etq54
+zKwgR9IBovs
+zL_d7b1bsUE
+zLaE0SmZdMo
+zQpGf1gPY7M
+zUDEten_j9o
+zUkN2ILNflA
+zYYWeXjQn8M
+zZb8HVaO4Nc
+z_cAYz0Q5DI
+zb8B-vxmjTg
+zdIcTkGHFac
+zeO8n9byqg0
+zesVY6sbjoM
+zgs82z9t7Mw
+zhESYHHbzsc
+zhI__xPhoW4
+zhzlGFKXV2g
+zjnOeTL7Bz8
+zlHe6erNcQA
+zmn50KVgQak
+zotrHY6f1-4
+zpRuDsk_0vk
+zqFEtAUGItg
+zqVaXf-E8us
+zucFlVCiV-g
+zuk7TtRuR-U
+zwGcVuZCM4Y
+zxN1HYq4iEg
+zygTUJZ93dE
+zzQoGtjkKuU
+zzpYxy7lveg

TalkingHead-1KH/data_list/train_video_tubes.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:86b9a6514dc4abdc9484c5fa93a3ad5db3c5ecb965181e92d16b9fd95ff8222a
+size 29153463

TalkingHead-1KH/data_list/val_video_ids.txt

@@ -0,0 +1,28 @@
+1lSejjfNHpw
+2Xu56MEC91w
+3y6Vjr45I34
+4hQi42Q9mcY
+5crEV5DbRyc
+85UEFVcmIjI
+A2800grpOzU
+c1DRo3tPDG4
+d_7s4huYOD4
+EGGsK7po68c
+eKFlMKp9Gs0
+EWKJprUrnPE
+gp4fg9PWuhM
+HBlkinewdHM
+jpCrKYWjYD8
+jxi_Cjc8T1w
+kMXhWN71Ar0
+m2ZmZflLryo
+npEcenV-Y08
+nUe3F8jYoZo
+NXpWIephX1o
+PAaWZTFRP9Q
+SmtJ5Cy4jCM
+SU8NSkuBkb0
+VkKnOEQlwl4
+WigprYZPaLc
+YsrzvkG5_KI
+Zel-zag38mQ

TalkingHead-1KH/data_list/val_video_tubes.txt

@@ -0,0 +1,38 @@
+1lSejjfNHpw_0075, 1080, 1920, 0, 728, 671, 47, 1471, 847
+1lSejjfNHpw_0075, 1080, 1920, 728, 1456, 671, 47, 1471, 847
+2Xu56MEC91w_0046, 1080, 1920, 80, 1105, 586, 86, 1314, 814
+3y6Vjr45I34_0004, 1080, 1920, 287, 1254, 568, 0, 1464, 896
+4hQi42Q9mcY_0002, 1080, 1920, 0, 605, 443, 0, 1515, 992
+4hQi42Q9mcY_0002, 1080, 1920, 605, 1209, 443, 0, 1515, 992
+5crEV5DbRyc_0009, 1080, 1920, 208, 1152, 1058, 102, 1712, 756
+85UEFVcmIjI_0014, 1080, 1920, 92, 627, 558, 134, 1294, 870
+85UEFVcmIjI_0014, 1080, 1920, 627, 1162, 558, 134, 1294, 870
+A2800grpOzU_0002, 1080, 1920, 812, 1407, 227, 7, 1139, 919
+EGGsK7po68c_0007, 1080, 1920, 0, 1024, 786, 50, 1598, 862
+EWKJprUrnPE_0005, 1080, 1920, 0, 1024, 84, 168, 702, 786
+HBlkinewdHM_0000, 1080, 1920, 319, 1344, 807, 149, 1347, 689
+NXpWIephX1o_0031, 1080, 1920, 0, 632, 357, 0, 1493, 1072
+NXpWIephX1o_0031, 1080, 1920, 632, 1264, 357, 0, 1493, 1072
+PAaWZTFRP9Q_0001, 1080, 1920, 0, 672, 624, 42, 1376, 794
+PAaWZTFRP9Q_0001, 1080, 1920, 926, 1425, 696, 101, 1464, 869
+SU8NSkuBkb0_0015, 1080, 1920, 826, 1397, 347, 69, 1099, 821
+SmtJ5Cy4jCM_0006, 1080, 1920, 0, 523, 524, 50, 1388, 914
+SmtJ5Cy4jCM_0006, 1080, 1920, 546, 1134, 477, 42, 1357, 922
+VkKnOEQlwl4_0010, 1080, 1920, 98, 818, 821, 22, 1733, 934
+VkKnOEQlwl4_0010, 1080, 1920, 818, 1537, 821, 22, 1733, 934
+WigprYZPaLc_0002, 1080, 1920, 234, 877, 802, 25, 1490, 713
+WigprYZPaLc_0002, 1080, 1920, 877, 1519, 802, 25, 1490, 713
+YsrzvkG5_KI_0018, 1080, 1920, 36, 1061, 591, 100, 1055, 564
+Zel-zag38mQ_0001, 1080, 1920, 0, 733, 591, 12, 1439, 860
+Zel-zag38mQ_0001, 1080, 1920, 733, 1466, 591, 12, 1439, 860
+c1DRo3tPDG4_0010, 1080, 1920, 0, 865, 432, 33, 1264, 865
+c1DRo3tPDG4_0010, 1080, 1920, 865, 1730, 432, 33, 1264, 865
+eKFlMKp9Gs0_0005, 1080, 1920, 0, 1024, 705, 118, 1249, 662
+gp4fg9PWuhM_0003, 1080, 1920, 0, 858, 526, 0, 1310, 768
+jpCrKYWjYD8_0002, 1080, 1920, 0, 768, 527, 68, 1215, 756
+jpCrKYWjYD8_0002, 1080, 1920, 768, 1535, 527, 68, 1215, 756
+jxi_Cjc8T1w_0061, 1080, 1920, 0, 1024, 660, 102, 1286, 728
+kMXhWN71Ar0_0001, 1080, 1920, 0, 656, 60, 0, 940, 832
+kMXhWN71Ar0_0001, 1080, 1920, 656, 1311, 60, 0, 940, 832
+m2ZmZflLryo_0009, 1080, 1920, 0, 1024, 678, 51, 1390, 763
+npEcenV-Y08_0011, 1080, 1920, 99, 1087, 625, 69, 1425, 869

TalkingHead-1KH/requirements.txt

@@ -0,0 +1,4 @@
+ffmpeg-python
+imageio
+git+https://github.com/nficano/pytube
+tqdm

TalkingHead-1KH/videos_crop.py

@@ -0,0 +1,79 @@
+# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
+#
+# This script is licensed under the MIT License.
+
+import argparse
+import multiprocessing as mp
+import os
+from functools import partial
+from time import time as timer
+
+import ffmpeg
+from tqdm import tqdm
+
+
+parser = argparse.ArgumentParser()
+parser.add_argument('--input_dir', type=str, required=True,
+                    help='Dir containing youtube clips.')
+parser.add_argument('--clip_info_file', type=str, required=True,
+                    help='File containing clip information.')
+parser.add_argument('--output_dir', type=str, required=True,
+                    help='Location to dump outputs.')
+parser.add_argument('--num_workers', type=int, default=8,
+                    help='How many multiprocessing workers?')
+args = parser.parse_args()
+
+
+def get_h_w(filepath):
+    probe = ffmpeg.probe(filepath)
+    video_stream = next((stream for stream in probe['streams'] if stream['codec_type'] == 'video'), None)
+    height = int(video_stream['height'])
+    width = int(video_stream['width'])
+    return height, width
+
+
+def trim_and_crop(input_dir, output_dir, clip_params):
+    video_name, H, W, S, E, L, T, R, B = clip_params.strip().split(',')
+    H, W, S, E, L, T, R, B = int(H), int(W), int(S), int(E), int(L), int(T), int(R), int(B)
+    output_filename = '{}_S{}_E{}_L{}_T{}_R{}_B{}.mp4'.format(video_name, S, E, L, T, R, B)
+    output_filepath = os.path.join(output_dir, output_filename)
+    if os.path.exists(output_filepath):
+        print('Output file %s exists, skipping' % (output_filepath))
+        return
+
+    input_filepath = os.path.join(input_dir, video_name + '.mp4')
+    if not os.path.exists(input_filepath):
+        print('Input file %s does not exist, skipping' % (input_filepath))
+        return
+
+    h, w = get_h_w(input_filepath)
+    t = int(T / H * h)
+    b = int(B / H * h)
+    l = int(L / W * w)
+    r = int(R / W * w)
+    stream = ffmpeg.input(input_filepath)
+    stream = ffmpeg.trim(stream, start_frame=S, end_frame=E+1)
+    stream = ffmpeg.crop(stream, l, t, r-l, b-t)
+    stream = ffmpeg.output(stream, output_filepath)
+    ffmpeg.run(stream)
+
+
+if __name__ == '__main__':
+    # Read list of videos.
+    clip_info = []
+    with open(args.clip_info_file) as fin:
+        for line in fin:
+            clip_info.append(line.strip())
+
+    # Create output folder.
+    os.makedirs(args.output_dir, exist_ok=True)
+
+    # Download videos.
+    downloader = partial(trim_and_crop, args.input_dir, args.output_dir)
+
+    start = timer()
+    pool_size = args.num_workers
+    print('Using pool size of %d' % (pool_size))
+    with mp.Pool(processes=pool_size) as p:
+        _ = list(tqdm(p.imap_unordered(downloader, clip_info), total=len(clip_info)))
+    print('Elapsed time: %.2f' % (timer() - start))

TalkingHead-1KH/videos_download.py

@@ -0,0 +1,56 @@
+import argparse
+import multiprocessing as mp
+import os
+from functools import partial
+from time import time as timer
+
+from pytube import YouTube
+from tqdm import tqdm
+
+parser = argparse.ArgumentParser()
+parser.add_argument('--input_list', type=str, required=True,
+                    help='List of youtube video ids')
+parser.add_argument('--output_dir', type=str, default='data/youtube_videos',
+                    help='Location to download videos')
+parser.add_argument('--num_workers', type=int, default=8,
+                    help='How many multiprocessing workers?')
+args = parser.parse_args()
+
+
+def download_video(output_dir, video_id):
+    r"""Download video."""
+    video_path = '%s/%s.mp4' % (output_dir, video_id)
+    if not os.path.isfile(video_path):
+        try:
+            # Download the highest quality mp4 stream.
+            yt = YouTube('https://www.youtube.com/watch?v=%s' % (video_id))
+            stream = yt.streams.filter(subtype='mp4', only_video=True, adaptive=True).first()
+            if stream is None:
+                stream = yt.streams.filter(subtype='mp4').first()
+            stream.download(output_path=output_dir, filename=video_id + '.mp4')
+        except Exception as e:
+            print(e)
+            print('Failed to download %s' % (video_id))
+    else:
+        print('File exists: %s' % (video_id))
+
+
+if __name__ == '__main__':
+    # Read list of videos.
+    video_ids = []
+    with open(args.input_list) as fin:
+        for line in fin:
+            video_ids.append(line.strip())
+
+    # Create output folder.
+    os.makedirs(args.output_dir, exist_ok=True)
+
+    # Download videos.
+    downloader = partial(download_video, args.output_dir)
+
+    start = timer()
+    pool_size = args.num_workers
+    print('Using pool size of %d' % (pool_size))
+    with mp.Pool(processes=pool_size) as p:
+        _ = list(tqdm(p.imap_unordered(downloader, video_ids), total=len(video_ids)))
+    print('Elapsed time: %.2f' % (timer() - start))

TalkingHead-1KH/videos_download_and_crop.sh

@@ -0,0 +1,10 @@
+dataset=$1
+
+# Download the videos.
+python videos_download.py --input_list data_list/${dataset}_video_ids.txt --output_dir ${dataset}/raw_videos
+
+# Split the videos into 1-min chunks.
+./videos_split.sh ${dataset}/raw_videos ${dataset}/1min_clips
+
+# Extract the talking head clips.
+python videos_crop.py --input_dir ${dataset}/1min_clips/ --output_dir ${dataset}/cropped_clips --clip_info_file data_list/${dataset}_video_tubes.txt

TalkingHead-1KH/videos_split.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+in_dir=$1
+out_dir=$2
+
+mkdir $out_dir;
+for f in $in_dir/*.mp4
+do
+  y=${f##*/};
+  ffmpeg -i $f -c copy -map 0 -segment_time 00:01:00 -f segment $out_dir/${y/.mp4}_%04d.mp4;
+done

data_gen/eg3d/convert_to_eg3d_convention.py

@@ -0,0 +1,146 @@
+import numpy as np
+import torch
+import copy
+from utils.commons.tensor_utils import convert_to_tensor, convert_to_np
+from deep_3drecon.deep_3drecon_models.bfm import ParametricFaceModel
+
+
+def _fix_intrinsics(intrinsics):
+    """
+    intrinsics: [3,3], not batch-wise
+    """
+    # unnormalized                                normalized
+
+    # [[ f_x, s=0,    x_0]             [[ f_x/size_x,   s=0,            x_0/size_x=0.5]
+    #  [ 0,   f_y,  y_0]      ->      [ 0,            f_y/size_y,   y_0/size_y=0.5]
+    #  [ 0,   0,    1  ]]             [ 0,            0,            1         ]]
+    intrinsics = np.array(intrinsics).copy()
+    assert intrinsics.shape == (3, 3), intrinsics
+    intrinsics[0,0] = 2985.29/700
+    intrinsics[1,1] = 2985.29/700
+    intrinsics[0,2] = 1/2
+    intrinsics[1,2] = 1/2
+    assert intrinsics[0,1] == 0
+    assert intrinsics[2,2] == 1
+    assert intrinsics[1,0] == 0
+    assert intrinsics[2,0] == 0
+    assert intrinsics[2,1] == 0
+    return intrinsics
+
+# Used in original submission
+def _fix_pose_orig(pose):
+    """
+    pose: [4,4], not batch-wise
+    """
+    pose = np.array(pose).copy()
+    location = pose[:3, 3]
+    radius = np.linalg.norm(location)
+    pose[:3, 3] = pose[:3, 3]/radius * 2.7
+    return pose
+
+
+def get_eg3d_convention_camera_pose_intrinsic(item):
+    """
+    item: a dict during binarize
+
+    """
+    if item['euler'].ndim == 1:
+        angle = convert_to_tensor(copy.copy(item['euler']))
+        trans = copy.deepcopy(item['trans'])
+
+        # handle the difference of euler axis between eg3d and ours
+        # see data_gen/process_ffhq_for_eg3d/transplant_eg3d_ckpt_into_our_convention.ipynb
+        # angle += torch.tensor([0, 3.1415926535, 3.1415926535], device=angle.device)
+        R = ParametricFaceModel.compute_rotation(angle.unsqueeze(0))[0].cpu().numpy()
+        trans[2] += -10
+        c = -np.dot(R, trans)
+        pose = np.eye(4)
+        pose[:3,:3] = R
+        c *= 0.27 # normalize camera radius
+        c[1] += 0.006 # additional offset used in submission
+        c[2] += 0.161 # additional offset used in submission
+        pose[0,3] = c[0]
+        pose[1,3] = c[1]
+        pose[2,3] = c[2]
+
+        focal = 2985.29 # = 1015*1024/224*(300/466.285),
+        # todo： 如果修改了fit 3dmm阶段的camera intrinsic，这里也要跟着改
+        pp = 512#112
+        w = 1024#224
+        h = 1024#224
+
+        K = np.eye(3)
+        K[0][0] = focal
+        K[1][1] = focal
+        K[0][2] = w/2.0
+        K[1][2] = h/2.0
+        convention_K = _fix_intrinsics(K)
+
+        Rot = np.eye(3)
+        Rot[0, 0] = 1
+        Rot[1, 1] = -1
+        Rot[2, 2] = -1        
+        pose[:3, :3] = np.dot(pose[:3, :3], Rot) # permute axes
+        convention_pose = _fix_pose_orig(pose)
+
+        item['c2w'] = pose
+        item['convention_c2w'] = convention_pose
+        item['intrinsics'] = convention_K
+        return item
+    else:
+        num_samples = len(item['euler'])
+        eulers_all = convert_to_tensor(copy.deepcopy(item['euler'])) # [B, 3]
+        trans_all = copy.deepcopy(item['trans']) # [B, 3]
+
+        # handle the difference of euler axis between eg3d and ours
+        # see data_gen/process_ffhq_for_eg3d/transplant_eg3d_ckpt_into_our_convention.ipynb
+        # eulers_all += torch.tensor([0, 3.1415926535, 3.1415926535], device=eulers_all.device).unsqueeze(0).repeat([eulers_all.shape[0],1])
+
+        intrinsics = []
+        poses = []
+        convention_poses = []
+        for i in range(num_samples):
+            angle = eulers_all[i]
+            trans = trans_all[i]
+            R = ParametricFaceModel.compute_rotation(angle.unsqueeze(0))[0].cpu().numpy()
+            trans[2] += -10
+            c = -np.dot(R, trans)
+            pose = np.eye(4)
+            pose[:3,:3] = R
+            c *= 0.27 # normalize camera radius
+            c[1] += 0.006 # additional offset used in submission
+            c[2] += 0.161 # additional offset used in submission
+            pose[0,3] = c[0]
+            pose[1,3] = c[1]
+            pose[2,3] = c[2]
+
+            focal = 2985.29 # = 1015*1024/224*(300/466.285),
+            # todo： 如果修改了fit 3dmm阶段的camera intrinsic，这里也要跟着改
+            pp = 512#112
+            w = 1024#224
+            h = 1024#224
+
+            K = np.eye(3)
+            K[0][0] = focal
+            K[1][1] = focal
+            K[0][2] = w/2.0
+            K[1][2] = h/2.0
+            convention_K = _fix_intrinsics(K)
+            intrinsics.append(convention_K)
+
+            Rot = np.eye(3)
+            Rot[0, 0] = 1
+            Rot[1, 1] = -1
+            Rot[2, 2] = -1        
+            pose[:3, :3] = np.dot(pose[:3, :3], Rot)
+            convention_pose = _fix_pose_orig(pose)
+            convention_poses.append(convention_pose)
+            poses.append(pose)
+
+        intrinsics = np.stack(intrinsics) # [B, 3, 3]
+        poses = np.stack(poses) # [B, 4, 4]
+        convention_poses = np.stack(convention_poses) # [B, 4, 4]
+        item['intrinsics'] = intrinsics
+        item['c2w'] = poses
+        item['convention_c2w'] = convention_poses
+        return item

data_gen/runs/binarizer_nerf.py

@@ -0,0 +1,335 @@
+import os
+import numpy as np
+import math
+import json
+import imageio
+import torch
+import tqdm
+import cv2
+
+from data_util.face3d_helper import Face3DHelper
+from utils.commons.euler2rot import euler_trans_2_c2w, c2w_to_euler_trans
+from data_gen.utils.process_video.euler2quaterion import euler2quaterion, quaterion2euler
+from deep_3drecon.deep_3drecon_models.bfm import ParametricFaceModel
+
+
+def euler2rot(euler_angle):
+    batch_size = euler_angle.shape[0]
+    theta = euler_angle[:, 0].reshape(-1, 1, 1)
+    phi = euler_angle[:, 1].reshape(-1, 1, 1)
+    psi = euler_angle[:, 2].reshape(-1, 1, 1)
+    one = torch.ones(batch_size, 1, 1).to(euler_angle.device)
+    zero = torch.zeros(batch_size, 1, 1).to(euler_angle.device)
+    rot_x = torch.cat((
+        torch.cat((one, zero, zero), 1),
+        torch.cat((zero, theta.cos(), theta.sin()), 1),
+        torch.cat((zero, -theta.sin(), theta.cos()), 1),
+    ), 2)
+    rot_y = torch.cat((
+        torch.cat((phi.cos(), zero, -phi.sin()), 1),
+        torch.cat((zero, one, zero), 1),
+        torch.cat((phi.sin(), zero, phi.cos()), 1),
+    ), 2)
+    rot_z = torch.cat((
+        torch.cat((psi.cos(), -psi.sin(), zero), 1),
+        torch.cat((psi.sin(), psi.cos(), zero), 1),
+        torch.cat((zero, zero, one), 1)
+    ), 2)
+    return torch.bmm(rot_x, torch.bmm(rot_y, rot_z))
+
+
+def rot2euler(rot_mat):
+    batch_size = len(rot_mat)
+    # we assert that y in in [-0.5pi, 0.5pi]
+    cos_y = torch.sqrt(rot_mat[:, 1, 2] * rot_mat[:, 1, 2] + rot_mat[:, 2, 2] * rot_mat[:, 2, 2])
+    theta_x = torch.atan2(-rot_mat[:, 1, 2], rot_mat[:, 2, 2])
+    theta_y = torch.atan2(rot_mat[:, 2, 0], cos_y)
+    theta_z = torch.atan2(rot_mat[:, 0, 1], rot_mat[:, 0, 0])
+    euler_angles = torch.zeros([batch_size, 3])
+    euler_angles[:, 0] = theta_x
+    euler_angles[:, 1] = theta_y
+    euler_angles[:, 2] = theta_z
+    return euler_angles
+
+index_lm68_from_lm468 = [127,234,93,132,58,136,150,176,152,400,379,365,288,361,323,454,356,70,63,105,66,107,336,296,334,293,300,168,197,5,4,75,97,2,326,305,
+                         33,160,158,133,153,144,362,385,387,263,373,380,61,40,37,0,267,270,291,321,314,17,84,91,78,81,13,311,308,402,14,178]
+
+def plot_lm2d(lm2d):
+    WH = 512
+    img = np.ones([WH, WH, 3], dtype=np.uint8) * 255
+    
+    for i in range(len(lm2d)):
+        x, y = lm2d[i]
+        color = (255,0,0)
+        img = cv2.circle(img, center=(int(x),int(y)), radius=3, color=color, thickness=-1)
+        font = cv2.FONT_HERSHEY_SIMPLEX
+    for i in range(len(lm2d)):
+        x, y = lm2d[i]
+        img = cv2.putText(img, f"{i}", org=(int(x),int(y)), fontFace=font, fontScale=0.3, color=(255,0,0))
+    return img
+
+def get_face_rect(lms, h, w):
+    """
+    lms: [68, 2]
+    h, w: int
+    return: [4,]
+    """
+    assert len(lms) == 68
+    # min_x, max_x = np.min(lms, 0)[0], np.max(lms, 0)[0]
+    min_x, max_x = np.min(lms[:, 0]), np.max(lms[:, 0])
+    cx = int((min_x+max_x)/2.0)
+    cy = int(lms[27, 1])
+    h_w = int((max_x-cx)*1.5)
+    h_h = int((lms[8, 1]-cy)*1.15)
+    rect_x = cx - h_w
+    rect_y = cy - h_h
+    if rect_x < 0:
+        rect_x = 0
+    if rect_y < 0:
+        rect_y = 0
+    rect_w = min(w-1-rect_x, 2*h_w)
+    rect_h = min(h-1-rect_y, 2*h_h)
+    # rect = np.array((rect_x, rect_y, rect_w, rect_h), dtype=np.int32)
+    # rect = [rect_x, rect_y, rect_w, rect_h]
+    rect = [rect_x, rect_x + rect_w, rect_y, rect_y + rect_h] # min_j,  max_j, min_i, max_i
+    return rect # this x is width, y is height
+
+def get_lip_rect(lms, h, w):
+    """
+    lms: [68, 2]
+    h, w: int
+    return: [4,]
+    """
+    # this x is width, y is height
+    # for lms, lms[:, 0] is width, lms[:, 1] is height
+    assert len(lms) == 68
+    lips = slice(48, 60)
+    lms = lms[lips]
+    min_x, max_x = np.min(lms[:, 0]), np.max(lms[:, 0])
+    min_y, max_y = np.min(lms[:, 1]), np.max(lms[:, 1])
+    cx = int((min_x+max_x)/2.0)
+    cy = int((min_y+max_y)/2.0)
+    h_w = int((max_x-cx)*1.2)
+    h_h = int((max_y-cy)*1.2)
+    
+    h_w = max(h_w, h_h)
+    h_h = h_w
+
+    rect_x = cx - h_w
+    rect_y = cy - h_h
+    rect_w = 2*h_w
+    rect_h = 2*h_h
+    if rect_x < 0:
+        rect_x = 0
+    if rect_y < 0:
+        rect_y = 0
+    
+    if rect_x + rect_w > w:
+        rect_x = w - rect_w
+    if rect_y + rect_h > h:
+        rect_y = h - rect_h
+
+    rect = [rect_x, rect_x + rect_w, rect_y, rect_y + rect_h] # min_j,  max_j, min_i, max_i
+    return rect # this x is width, y is height
+
+
+# def get_lip_rect(lms, h, w):
+#     """
+#     lms: [68, 2]
+#     h, w: int
+#     return: [4,]
+#     """
+#     assert len(lms) == 68
+#     lips = slice(48, 60)
+#     # this x is width, y is height
+#     xmin, xmax = int(lms[lips, 1].min()), int(lms[lips, 1].max())
+#     ymin, ymax = int(lms[lips, 0].min()), int(lms[lips, 0].max())
+#     # padding to H == W
+#     cx = (xmin + xmax) // 2
+#     cy = (ymin + ymax) // 2
+#     l = max(xmax - xmin, ymax - ymin) // 2
+#     xmin = max(0, cx - l)
+#     xmax = min(h, cx + l)
+#     ymin = max(0, cy - l)
+#     ymax = min(w, cy + l)
+#     lip_rect = [xmin, xmax, ymin, ymax]
+#     return lip_rect
+
+def get_win_conds(conds, idx, smo_win_size=8, pad_option='zero'):
+    """
+    conds: [b, t=16, h=29]
+    idx: long, time index of the selected frame
+    """
+    idx = max(0, idx)
+    idx = min(idx, conds.shape[0]-1)
+    smo_half_win_size = smo_win_size//2
+    left_i = idx - smo_half_win_size
+    right_i = idx + (smo_win_size - smo_half_win_size)
+    pad_left, pad_right = 0, 0
+    if left_i < 0:
+        pad_left = -left_i
+        left_i = 0
+    if right_i > conds.shape[0]:
+        pad_right = right_i - conds.shape[0]
+        right_i = conds.shape[0]
+    conds_win = conds[left_i:right_i]
+    if pad_left > 0:
+        if pad_option == 'zero':
+            conds_win = np.concatenate([np.zeros_like(conds_win)[:pad_left], conds_win], axis=0)
+        elif pad_option == 'edge':
+            edge_value = conds[0][np.newaxis, ...]
+            conds_win = np.concatenate([edge_value] * pad_left + [conds_win], axis=0)
+        else: 
+            raise NotImplementedError
+    if pad_right > 0:
+        if pad_option == 'zero':
+            conds_win = np.concatenate([conds_win, np.zeros_like(conds_win)[:pad_right]], axis=0)
+        elif pad_option == 'edge':
+            edge_value = conds[-1][np.newaxis, ...]
+            conds_win = np.concatenate([conds_win] + [edge_value] * pad_right , axis=0)
+        else: 
+            raise NotImplementedError
+    assert conds_win.shape[0] == smo_win_size
+    return conds_win
+
+
+def load_processed_data(processed_dir):
+    # load necessary files
+    background_img_name = os.path.join(processed_dir, "bg.jpg")
+    assert os.path.exists(background_img_name)
+    head_img_dir = os.path.join(processed_dir, "head_imgs")
+    torso_img_dir = os.path.join(processed_dir, "inpaint_torso_imgs")
+    gt_img_dir = os.path.join(processed_dir, "gt_imgs")
+
+    hubert_npy_name = os.path.join(processed_dir, "aud_hubert.npy")
+    mel_f0_npy_name = os.path.join(processed_dir, "aud_mel_f0.npy")
+    coeff_npy_name = os.path.join(processed_dir, "coeff_fit_mp.npy")
+    lm2d_npy_name = os.path.join(processed_dir, "lms_2d.npy")
+    
+    ret_dict = {}
+
+    ret_dict['bg_img'] = imageio.imread(background_img_name)
+    ret_dict['H'], ret_dict['W'] = ret_dict['bg_img'].shape[:2]
+    ret_dict['focal'], ret_dict['cx'], ret_dict['cy'] = face_model.focal, face_model.center, face_model.center
+
+    print("loading lm2d coeff ...")
+    lm2d_arr = np.load(lm2d_npy_name)
+    face_rect_lst = []
+    lip_rect_lst = []
+    for lm2d in lm2d_arr:
+        if len(lm2d) in [468, 478]:
+            lm2d = lm2d[index_lm68_from_lm468]
+        face_rect = get_face_rect(lm2d, ret_dict['H'], ret_dict['W'])
+        lip_rect = get_lip_rect(lm2d, ret_dict['H'], ret_dict['W'])
+        face_rect_lst.append(face_rect)
+        lip_rect_lst.append(lip_rect)
+    face_rects = np.stack(face_rect_lst, axis=0) # [T, 4]
+
+    print("loading fitted 3dmm coeff ...")
+    coeff_dict = np.load(coeff_npy_name, allow_pickle=True).tolist()
+    identity_arr = coeff_dict['id']
+    exp_arr = coeff_dict['exp']
+    ret_dict['id'] = identity_arr
+    ret_dict['exp'] = exp_arr
+    euler_arr = ret_dict['euler'] = coeff_dict['euler']
+    trans_arr = ret_dict['trans'] = coeff_dict['trans']
+    print("calculating lm3d ...")
+    idexp_lm3d_arr = face3d_helper.reconstruct_idexp_lm3d(torch.from_numpy(identity_arr), torch.from_numpy(exp_arr)).cpu().numpy().reshape([-1, 68*3])
+    len_motion = len(idexp_lm3d_arr)
+    video_idexp_lm3d_mean = idexp_lm3d_arr.mean(axis=0)
+    video_idexp_lm3d_std = idexp_lm3d_arr.std(axis=0)
+    ret_dict['idexp_lm3d'] = idexp_lm3d_arr
+    ret_dict['idexp_lm3d_mean'] = video_idexp_lm3d_mean
+    ret_dict['idexp_lm3d_std'] = video_idexp_lm3d_std
+    
+    # now we convert the euler_trans from deep3d convention to adnerf convention
+    eulers = torch.FloatTensor(euler_arr)
+    trans = torch.FloatTensor(trans_arr)
+    rots = face_model.compute_rotation(eulers) # rotation matrix is a better intermediate for convention-transplan than euler
+
+    # handle the camera pose to geneface's convention
+    trans[:, 2] = 10 - trans[:, 2] # 抵消fit阶段的to_camera操作，即trans[...,2] = 10 - trans[...,2]
+    rots = rots.permute(0, 2, 1)
+    trans[:, 2] = - trans[:,2] # 因为intrinsic proj不同
+    # below is the NeRF camera preprocessing strategy, see `save_transforms` in data_util/process.py 
+    trans = trans / 10.0
+    rots_inv = rots.permute(0, 2, 1)
+    trans_inv = - torch.bmm(rots_inv, trans.unsqueeze(2))
+
+    pose = torch.eye(4, dtype=torch.float32).unsqueeze(0).repeat([len_motion, 1, 1]) # [T, 4, 4]
+    pose[:, :3, :3] = rots_inv
+    pose[:, :3, 3] = trans_inv[:, :, 0]
+    c2w_transform_matrices = pose.numpy()
+
+    # process the audio features used for postnet training
+    print("loading hubert ...")
+    hubert_features = np.load(hubert_npy_name)
+    print("loading Mel and F0 ...")
+    mel_f0_features = np.load(mel_f0_npy_name, allow_pickle=True).tolist()
+
+    ret_dict['hubert'] = hubert_features
+    ret_dict['mel'] = mel_f0_features['mel']
+    ret_dict['f0'] = mel_f0_features['f0']
+
+    # obtaining train samples
+    frame_indices = list(range(len_motion))
+    num_train = len_motion // 11 * 10
+    train_indices = frame_indices[:num_train]
+    val_indices = frame_indices[num_train:]
+
+    for split in ['train', 'val']:
+        if split == 'train':
+            indices = train_indices
+            samples = []
+            ret_dict['train_samples'] = samples
+        elif split == 'val':
+            indices = val_indices
+            samples = []
+            ret_dict['val_samples'] = samples
+        
+        for idx in indices:
+            sample = {}
+            sample['idx'] = idx
+            sample['head_img_fname'] = os.path.join(head_img_dir,f"{idx:08d}.png")
+            sample['torso_img_fname'] = os.path.join(torso_img_dir,f"{idx:08d}.png")
+            sample['gt_img_fname'] = os.path.join(gt_img_dir,f"{idx:08d}.jpg")
+            # assert os.path.exists(sample['head_img_fname']) and os.path.exists(sample['torso_img_fname']) and os.path.exists(sample['gt_img_fname'])
+            sample['face_rect'] = face_rects[idx]
+            sample['lip_rect'] = lip_rect_lst[idx]
+            sample['c2w'] = c2w_transform_matrices[idx]
+            samples.append(sample)
+    return ret_dict
+
+
+class Binarizer:
+    def __init__(self):
+        self.data_dir = 'data/'
+        
+    def parse(self, video_id):
+        processed_dir = os.path.join(self.data_dir, 'processed/videos', video_id)
+        binary_dir = os.path.join(self.data_dir, 'binary/videos', video_id)
+        out_fname = os.path.join(binary_dir, "trainval_dataset.npy")
+        os.makedirs(binary_dir, exist_ok=True)
+        ret = load_processed_data(processed_dir)
+        mel_name = os.path.join(processed_dir, 'aud_mel_f0.npy')
+        mel_f0_dict = np.load(mel_name, allow_pickle=True).tolist()
+        ret.update(mel_f0_dict)
+        np.save(out_fname, ret, allow_pickle=True)
+
+
+
+if __name__ == '__main__':
+    from argparse import ArgumentParser
+    parser = ArgumentParser()
+    parser.add_argument('--video_id', type=str, default='May', help='')
+    args = parser.parse_args()
+    ### Process Single Long Audio for NeRF dataset
+    video_id = args.video_id
+    face_model = ParametricFaceModel(bfm_folder='deep_3drecon/BFM', 
+                camera_distance=10, focal=1015)
+    face_model.to("cpu")
+    face3d_helper = Face3DHelper()
+
+    binarizer = Binarizer()
+    binarizer.parse(video_id)
+    print(f"Binarization for {video_id} Done!")

data_gen/runs/binarizer_th1kh.py

@@ -0,0 +1,100 @@
+import os
+import numpy as np
+from scipy.misc import face
+import torch
+from tqdm import trange
+import pickle
+from copy import deepcopy
+
+from data_util.face3d_helper import Face3DHelper
+from utils.commons.indexed_datasets import IndexedDataset, IndexedDatasetBuilder
+
+
+def load_video_npy(fn):
+    assert fn.endswith("_coeff_fit_mp.npy")
+    ret_dict = np.load(fn,allow_pickle=True).item()
+    video_dict = {
+        'euler': ret_dict['euler'], # [T, 3]
+        'trans': ret_dict['trans'], # [T, 3]
+        'id': ret_dict['id'], # [T, 80]
+        'exp': ret_dict['exp'], # [T, 64]
+    }
+    return video_dict
+
+def cal_lm3d_in_video_dict(video_dict, face3d_helper):
+    identity = video_dict['id']
+    exp = video_dict['exp']
+    idexp_lm3d = face3d_helper.reconstruct_idexp_lm3d(identity, exp).cpu().numpy()
+    video_dict['idexp_lm3d'] = idexp_lm3d
+
+
+def load_audio_npy(fn):
+    assert fn.endswith(".npy")
+    ret_dict = np.load(fn,allow_pickle=True).item()
+    audio_dict = {
+        "mel": ret_dict['mel'], # [T, 80]
+        "f0": ret_dict['f0'], # [T,1]
+    }
+    return audio_dict
+
+
+if __name__ == '__main__':
+    face3d_helper = Face3DHelper(use_gpu=False)
+    
+    import glob,tqdm
+    prefixs = ['val', 'train']
+    binarized_ds_path = "data/binary/th1kh"
+    os.makedirs(binarized_ds_path, exist_ok=True)
+    for prefix in prefixs:
+        databuilder = IndexedDatasetBuilder(os.path.join(binarized_ds_path, prefix), gzip=False, default_idx_size=1024*1024*1024*2)
+        raw_base_dir =  '/mnt/bn/ailabrenyi/entries/yezhenhui/datasets/raw/TH1KH_512/video'
+        mp4_names = glob.glob(os.path.join(raw_base_dir, '*.mp4'))
+        mp4_names = mp4_names[:1000]
+        cnt = 0
+        scnt = 0
+        pbar = tqdm.tqdm(enumerate(mp4_names), total=len(mp4_names))
+        for i, mp4_name in pbar:
+            cnt += 1
+            if prefix == 'train':
+                if i % 100 == 0:
+                    continue
+            else:
+                if i % 100 != 0:
+                    continue
+            hubert_npy_name = mp4_name.replace("/video/", "/hubert/").replace(".mp4", "_hubert.npy")
+            audio_npy_name = mp4_name.replace("/video/", "/mel_f0/").replace(".mp4", "_mel_f0.npy")
+            video_npy_name = mp4_name.replace("/video/", "/coeff_fit_mp/").replace(".mp4", "_coeff_fit_mp.npy")
+            if not os.path.exists(audio_npy_name):
+                print(f"Skip item for audio npy not found.")
+                continue
+            if not os.path.exists(video_npy_name):
+                print(f"Skip item for video npy not found.")
+                continue
+            if (not os.path.exists(hubert_npy_name)):
+                print(f"Skip item for hubert_npy not found.")
+                continue
+            audio_dict = load_audio_npy(audio_npy_name)
+            hubert = np.load(hubert_npy_name)
+            video_dict = load_video_npy(video_npy_name)
+            com_img_dir = mp4_name.replace("/video/", "/com_imgs/").replace(".mp4", "")
+            num_com_imgs = len(glob.glob(os.path.join(com_img_dir, '*')))
+            num_frames = len(video_dict['exp'])
+            if num_com_imgs != num_frames:
+                print(f"Skip item for length mismatch.")
+                continue
+            mel = audio_dict['mel']
+            if mel.shape[0] < 32: # the video is shorter than 0.6s
+                print(f"Skip item for too short.")
+                continue
+            
+            audio_dict.update(video_dict)
+            audio_dict['item_id'] = os.path.basename(mp4_name)[:-4]
+            audio_dict['hubert'] = hubert # [T_x, hid=1024]
+            audio_dict['img_dir'] = com_img_dir
+
+
+            databuilder.add_item(audio_dict)
+            scnt += 1
+            pbar.set_postfix({'success': scnt, 'success rate': scnt / cnt})
+        databuilder.finalize()
+        print(f"{prefix} set has {cnt} samples!")

data_gen/runs/nerf/process_guide.md

@@ -0,0 +1,49 @@
+# 温馨提示：第一次执行可以先一步步跑完下面的命令行，把环境跑通后，之后可以直接运行同目录的run.sh，一键完成下面的所有步骤。
+
+# Step0. 将视频Crop到512x512分辨率，25FPS，确保每一帧都有目标人脸
+```
+ffmpeg -i data/raw/videos/${VIDEO_ID}.mp4 -vf fps=25,scale=w=512:h=512 -qmin 1 -q:v 1 data/raw/videos/${VIDEO_ID}_512.mp4
+mv data/raw/videos/${VIDEO_ID}.mp4 data/raw/videos/${VIDEO_ID}_to_rm.mp4
+mv data/raw/videos/${VIDEO_ID}_512.mp4 data/raw/videos/${VIDEO_ID}.mp4
+```
+# step1: 提取音频特征, 如mel, f0, hubuert, esperanto
+```
+export CUDA_VISIBLE_DEVICES=0
+export VIDEO_ID=May
+mkdir -p data/processed/videos/${VIDEO_ID}
+ffmpeg -i data/raw/videos/${VIDEO_ID}.mp4 -f wav -ar 16000 data/processed/videos/${VIDEO_ID}/aud.wav 
+python data_gen/utils/process_audio/extract_hubert.py --video_id=${VIDEO_ID}
+python data_gen/utils/process_audio/extract_mel_f0.py --video_id=${VIDEO_ID}
+```
+
+# Step2. 提取图片
+```
+export VIDEO_ID=May
+export CUDA_VISIBLE_DEVICES=0
+mkdir -p data/processed/videos/${VIDEO_ID}/gt_imgs
+ffmpeg -i data/raw/videos/${VIDEO_ID}.mp4 -vf fps=25,scale=w=512:h=512 -qmin 1 -q:v 1 -start_number 0 data/processed/videos/${VIDEO_ID}/gt_imgs/%08d.jpg
+python data_gen/utils/process_video/extract_segment_imgs.py --ds_name=nerf --vid_dir=data/raw/videos/${VIDEO_ID}.mp4 # extract image, segmap, and background
+```
+
+# Step3. 提取lm2d_mediapipe
+### 提取2D landmark用于之后Fit 3DMM
+### num_workers是本机上的CPU worker数量；total_process是使用的机器数；process_id是本机的编号
+
+```
+export VIDEO_ID=May
+python data_gen/utils/process_video/extract_lm2d.py --ds_name=nerf --vid_dir=data/raw/videos/${VIDEO_ID}.mp4
+```
+
+# Step3. fit 3dmm
+```
+export VIDEO_ID=May
+export CUDA_VISIBLE_DEVICES=0
+python data_gen/utils/process_video/fit_3dmm_landmark.py --ds_name=nerf --vid_dir=data/raw/videos/${VIDEO_ID}.mp4 --reset  --debug --id_mode=global
+```
+
+# Step4. Binarize
+```
+export VIDEO_ID=May
+python data_gen/runs/binarizer_nerf.py --video_id=${VIDEO_ID}
+```
+可以看到在`data/binary/videos/Mayssss`目录下得到了数据集。

data_gen/runs/nerf/run.sh

@@ -0,0 +1,51 @@
+# usage: CUDA_VISIBLE_DEVICES=0 bash data_gen/runs/nerf/run.sh <VIDEO_ID>
+# please place video to data/raw/videos/${VIDEO_ID}.mp4 
+VIDEO_ID=$1
+echo Processing $VIDEO_ID
+
+echo Resizing the video to 512x512
+ffmpeg -i data/raw/videos/${VIDEO_ID}.mp4 -vf fps=25,scale=w=512:h=512 -qmin 1 -q:v 1 -y data/raw/videos/${VIDEO_ID}_512.mp4
+mv data/raw/videos/${VIDEO_ID}.mp4 data/raw/videos/${VIDEO_ID}_to_rm.mp4
+mv data/raw/videos/${VIDEO_ID}_512.mp4 data/raw/videos/${VIDEO_ID}.mp4
+echo Done
+echo The old video is moved to data/raw/videos/${VIDEO_ID}.mp4 data/raw/videos/${VIDEO_ID}_to_rm.mp4
+
+echo mkdir -p data/processed/videos/${VIDEO_ID}
+mkdir -p data/processed/videos/${VIDEO_ID}
+echo Done
+
+# extract audio file from the training video
+echo ffmpeg -i data/raw/videos/${VIDEO_ID}.mp4 -f wav -ar 16000 -v quiet -y data/processed/videos/${VIDEO_ID}/aud.wav 
+ffmpeg -i data/raw/videos/${VIDEO_ID}.mp4 -f wav -ar 16000 -v quiet -y data/processed/videos/${VIDEO_ID}/aud.wav 
+echo Done
+
+# extract hubert_mel_f0 from audio
+echo python data_gen/utils/process_audio/extract_hubert.py --video_id=${VIDEO_ID}
+python data_gen/utils/process_audio/extract_hubert.py --video_id=${VIDEO_ID}
+echo python data_gen/utils/process_audio/extract_mel_f0.py --video_id=${VIDEO_ID}
+python data_gen/utils/process_audio/extract_mel_f0.py --video_id=${VIDEO_ID}
+echo Done
+
+# extract segment images
+echo mkdir -p data/processed/videos/${VIDEO_ID}/gt_imgs
+mkdir -p data/processed/videos/${VIDEO_ID}/gt_imgs
+echo ffmpeg -i data/raw/videos/${VIDEO_ID}.mp4 -vf fps=25,scale=w=512:h=512 -qmin 1 -q:v 1 -start_number 0 -v quiet data/processed/videos/${VIDEO_ID}/gt_imgs/%08d.jpg
+ffmpeg -i data/raw/videos/${VIDEO_ID}.mp4 -vf fps=25,scale=w=512:h=512 -qmin 1 -q:v 1 -start_number 0 -v quiet data/processed/videos/${VIDEO_ID}/gt_imgs/%08d.jpg
+echo Done
+
+echo python data_gen/utils/process_video/extract_segment_imgs.py --ds_name=nerf --vid_dir=data/raw/videos/${VIDEO_ID}.mp4 # extract image, segmap, and background
+python data_gen/utils/process_video/extract_segment_imgs.py --ds_name=nerf --vid_dir=data/raw/videos/${VIDEO_ID}.mp4 # extract image, segmap, and background
+echo Done
+
+echo python data_gen/utils/process_video/extract_lm2d.py --ds_name=nerf --vid_dir=data/raw/videos/${VIDEO_ID}.mp4
+python data_gen/utils/process_video/extract_lm2d.py --ds_name=nerf --vid_dir=data/raw/videos/${VIDEO_ID}.mp4
+echo Done
+
+pkill -f void*
+echo python data_gen/utils/process_video/fit_3dmm_landmark.py --ds_name=nerf --vid_dir=data/raw/videos/${VIDEO_ID}.mp4 --reset --debug --id_mode=global
+python data_gen/utils/process_video/fit_3dmm_landmark.py --ds_name=nerf --vid_dir=data/raw/videos/${VIDEO_ID}.mp4 --reset --debug --id_mode=global
+echo Done
+
+echo python data_gen/runs/binarizer_nerf.py --video_id=${VIDEO_ID}
+python data_gen/runs/binarizer_nerf.py --video_id=${VIDEO_ID}
+echo Done

data_gen/utils/mp_feature_extractors/face_landmarker.py

@@ -0,0 +1,130 @@
+import mediapipe as mp
+from mediapipe.tasks import python
+from mediapipe.tasks.python import vision
+import numpy as np
+import cv2
+import os
+import copy
+
+# simplified mediapipe ldm at https://github.com/k-m-irfan/simplified_mediapipe_face_landmarks
+index_lm141_from_lm478 = [70,63,105,66,107,55,65,52,53,46] + [300,293,334,296,336,285,295,282,283,276] + [33,246,161,160,159,158,157,173,133,155,154,153,145,144,163,7] + [263,466,388,387,386,385,384,398,362,382,381,380,374,373,390,249] + [78,191,80,81,82,13,312,311,310,415,308,324,318,402,317,14,87,178,88,95] + [61,185,40,39,37,0,267,269,270,409,291,375,321,405,314,17,84,181,91,146] + [10,338,297,332,284,251,389,356,454,323,361,288,397,365,379,378,400,377,152,148,176,149,150,136,172,58,132,93,234,127,162,21,54,103,67,109] + [468,469,470,471,472] + [473,474,475,476,477] + [64,4,294]
+# lm141 without iris
+index_lm131_from_lm478 = [70,63,105,66,107,55,65,52,53,46] + [300,293,334,296,336,285,295,282,283,276] + [33,246,161,160,159,158,157,173,133,155,154,153,145,144,163,7] + [263,466,388,387,386,385,384,398,362,382,381,380,374,373,390,249] + [78,191,80,81,82,13,312,311,310,415,308,324,318,402,317,14,87,178,88,95] + [61,185,40,39,37,0,267,269,270,409,291,375,321,405,314,17,84,181,91,146] + [10,338,297,332,284,251,389,356,454,323,361,288,397,365,379,378,400,377,152,148,176,149,150,136,172,58,132,93,234,127,162,21,54,103,67,109] + [64,4,294]
+
+# face alignment lm68
+index_lm68_from_lm478 = [127,234,93,132,58,136,150,176,152,400,379,365,288,361,323,454,356,70,63,105,66,107,336,296,334,293,300,168,197,5,4,75,97,2,326,305,
+                         33,160,158,133,153,144,362,385,387,263,373,380,61,40,37,0,267,270,291,321,314,17,84,91,78,81,13,311,308,402,14,178]
+# used for weights for key parts
+unmatch_mask_from_lm478 = [ 93, 127, 132, 234, 323, 356, 361, 454]
+index_eye_from_lm478 = [33,246,161,160,159,158,157,173,133,155,154,153,145,144,163,7] + [263,466,388,387,386,385,384,398,362,382,381,380,374,373,390,249]
+index_innerlip_from_lm478 = [78,191,80,81,82,13,312,311,310,415,308,324,318,402,317,14,87,178,88,95]
+index_outerlip_from_lm478 = [61,185,40,39,37,0,267,269,270,409,291,375,321,405,314,17,84,181,91,146]
+index_withinmouth_from_lm478 = [76, 62] + [184, 183, 74, 72, 73, 41, 72, 38, 11, 12, 302, 268, 303, 271, 304, 272, 408, 407] + [292, 306] +  [325, 307, 319, 320, 403, 404, 316, 315, 15, 16, 86, 85, 179, 180, 89, 90, 96, 77]
+index_mouth_from_lm478 = index_innerlip_from_lm478 + index_outerlip_from_lm478 + index_withinmouth_from_lm478
+
+index_yaw_from_lm68 = list(range(0, 17))
+index_brow_from_lm68 = list(range(17, 27))
+index_nose_from_lm68 = list(range(27, 36))
+index_eye_from_lm68 = list(range(36, 48))
+index_mouth_from_lm68 = list(range(48, 68))
+
+
+def read_video_to_frames(video_name):
+    frames = []
+    cap = cv2.VideoCapture(video_name)
+    while cap.isOpened():
+        ret, frame_bgr = cap.read()
+        if frame_bgr is None:
+            break
+        frames.append(frame_bgr)
+    frames = np.stack(frames)
+    frames = np.flip(frames, -1) # BGR ==> RGB
+    return frames
+
+class MediapipeLandmarker:
+    def __init__(self):
+        model_path = 'data_gen/utils/mp_feature_extractors/face_landmarker.task'
+        if not os.path.exists(model_path):
+            os.makedirs(os.path.dirname(model_path), exist_ok=True)
+            print("downloading face_landmarker model from mediapipe...")
+            model_url = 'https://storage.googleapis.com/mediapipe-models/face_landmarker/face_landmarker/float16/latest/face_landmarker.task'
+            os.system(f"wget {model_url}")
+            os.system(f"mv face_landmarker.task {model_path}")
+            print("download success")
+        base_options = python.BaseOptions(model_asset_path=model_path)
+        self.image_mode_options = vision.FaceLandmarkerOptions(base_options=base_options, 
+                        running_mode=vision.RunningMode.IMAGE, # IMAGE, VIDEO, LIVE_STREAM
+                        num_faces=1)
+        self.video_mode_options = vision.FaceLandmarkerOptions(base_options=base_options, 
+                        running_mode=vision.RunningMode.VIDEO, # IMAGE, VIDEO, LIVE_STREAM
+                        num_faces=1)
+
+    def extract_lm478_from_img_name(self, img_name):
+        img = cv2.imread(img_name)
+        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
+        img_lm478 = self.extract_lm478_from_img(img)
+        return img_lm478
+    
+    def extract_lm478_from_img(self, img):
+        img_landmarker = vision.FaceLandmarker.create_from_options(self.image_mode_options)
+        frame = mp.Image(image_format=mp.ImageFormat.SRGB, data=img.astype(np.uint8))
+        img_face_landmarker_result = img_landmarker.detect(image=frame)
+        img_ldm_i = img_face_landmarker_result.face_landmarks[0]
+        img_face_landmarks = np.array([[l.x, l.y, l.z] for l in img_ldm_i])
+        H, W, _ = img.shape
+        img_lm478 = np.array(img_face_landmarks)[:, :2] * np.array([W, H]).reshape([1,2]) # [478, 2]
+        return img_lm478
+
+    def extract_lm478_from_video_name(self, video_name, fps=25, anti_smooth_factor=2):
+        frames = read_video_to_frames(video_name)
+        img_lm478, vid_lm478 = self.extract_lm478_from_frames(frames, fps, anti_smooth_factor)
+        return img_lm478, vid_lm478
+
+    def extract_lm478_from_frames(self, frames, fps=25, anti_smooth_factor=20):
+        """
+        frames: RGB, uint8
+        anti_smooth_factor: float, 对video模式的interval进行修改, 1代表无修改, 越大越接近image mode
+        """
+        img_mpldms = []
+        vid_mpldms = []
+        img_landmarker = vision.FaceLandmarker.create_from_options(self.image_mode_options)
+        vid_landmarker = vision.FaceLandmarker.create_from_options(self.video_mode_options)
+
+        for i in range(len(frames)):
+            frame = mp.Image(image_format=mp.ImageFormat.SRGB, data=frames[i].astype(np.uint8))
+            img_face_landmarker_result = img_landmarker.detect(image=frame)
+            vid_face_landmarker_result = vid_landmarker.detect_for_video(image=frame, timestamp_ms=int((1000/fps)*anti_smooth_factor*i))
+            try:
+                img_ldm_i = img_face_landmarker_result.face_landmarks[0]
+                vid_ldm_i = vid_face_landmarker_result.face_landmarks[0]
+            except:
+                print(f"Warning: failed detect ldm in idx={i}, use previous frame results.")
+            img_face_landmarks = np.array([[l.x, l.y, l.z] for l in img_ldm_i])
+            vid_face_landmarks = np.array([[l.x, l.y, l.z] for l in vid_ldm_i])
+            img_mpldms.append(img_face_landmarks)
+            vid_mpldms.append(vid_face_landmarks)
+        img_lm478 = np.stack(img_mpldms)[..., :2]
+        vid_lm478 = np.stack(vid_mpldms)[..., :2]
+        bs, H, W, _ = frames.shape
+        img_lm478 = np.array(img_lm478)[..., :2] * np.array([W, H]).reshape([1,1,2]) # [T, 478, 2]
+        vid_lm478 = np.array(vid_lm478)[..., :2] * np.array([W, H]).reshape([1,1,2]) # [T, 478, 2]
+        return img_lm478, vid_lm478
+
+    def combine_vid_img_lm478_to_lm68(self, img_lm478, vid_lm478):
+        img_lm68 = img_lm478[:, index_lm68_from_lm478]
+        vid_lm68 = vid_lm478[:, index_lm68_from_lm478]
+        combined_lm68 = copy.deepcopy(img_lm68)
+        combined_lm68[:, index_yaw_from_lm68] = vid_lm68[:, index_yaw_from_lm68]
+        combined_lm68[:, index_brow_from_lm68] = vid_lm68[:, index_brow_from_lm68]
+        combined_lm68[:, index_nose_from_lm68] = vid_lm68[:, index_nose_from_lm68]
+        return combined_lm68
+     
+    def combine_vid_img_lm478_to_lm478(self, img_lm478, vid_lm478):
+        combined_lm478 = copy.deepcopy(vid_lm478)
+        combined_lm478[:, index_mouth_from_lm478] = img_lm478[:, index_mouth_from_lm478]
+        combined_lm478[:, index_eye_from_lm478] = img_lm478[:, index_eye_from_lm478]
+        return combined_lm478
+
+if __name__ == '__main__':
+    landmarker = MediapipeLandmarker()
+    ret = landmarker.extract_lm478_from_video_name("00000.mp4")

data_gen/utils/mp_feature_extractors/face_landmarker.task

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:64184e229b263107bc2b804c6625db1341ff2bb731874b0bcc2fe6544e0bc9ff
+size 3758596

data_gen/utils/mp_feature_extractors/mp_segmenter.py

@@ -0,0 +1,303 @@
+import os
+import copy
+import numpy as np
+import tqdm
+import mediapipe as mp
+import torch
+from mediapipe.tasks import python
+from mediapipe.tasks.python import vision
+from utils.commons.multiprocess_utils import multiprocess_run_tqdm, multiprocess_run
+from utils.commons.tensor_utils import convert_to_np
+from sklearn.neighbors import NearestNeighbors
+
+def scatter_np(condition_img, classSeg=5):
+# def scatter(condition_img, classSeg=19, label_size=(512, 512)):
+    batch, c, height, width = condition_img.shape
+    # if height != label_size[0] or width != label_size[1]:
+        # condition_img= F.interpolate(condition_img, size=label_size, mode='nearest')
+    input_label = np.zeros([batch, classSeg, condition_img.shape[2], condition_img.shape[3]]).astype(np.int_)
+    # input_label = torch.zeros(batch, classSeg, *label_size, device=condition_img.device)
+    np.put_along_axis(input_label, condition_img, 1, 1)
+    return input_label
+
+def scatter(condition_img, classSeg=19):
+# def scatter(condition_img, classSeg=19, label_size=(512, 512)):
+    batch, c, height, width = condition_img.size()
+    # if height != label_size[0] or width != label_size[1]:
+        # condition_img= F.interpolate(condition_img, size=label_size, mode='nearest')
+    input_label = torch.zeros(batch, classSeg, condition_img.shape[2], condition_img.shape[3], device=condition_img.device)
+    # input_label = torch.zeros(batch, classSeg, *label_size, device=condition_img.device)
+    return input_label.scatter_(1, condition_img.long(), 1)
+
+def encode_segmap_mask_to_image(segmap):
+    # rgb
+    _,h,w = segmap.shape
+    encoded_img = np.ones([h,w,3],dtype=np.uint8) * 255
+    colors = [(255,255,255),(255,255,0),(255,0,255),(0,255,255),(255,0,0),(0,255,0)]
+    for i, color in enumerate(colors):
+        mask = segmap[i].astype(int)
+        index = np.where(mask != 0)
+        encoded_img[index[0], index[1], :] = np.array(color)
+    return encoded_img.astype(np.uint8)
+        
+def decode_segmap_mask_from_image(encoded_img):
+    # rgb
+    colors = [(255,255,255),(255,255,0),(255,0,255),(0,255,255),(255,0,0),(0,255,0)]
+    bg = (encoded_img[..., 0] == 255) & (encoded_img[..., 1] == 255) & (encoded_img[..., 2] == 255)
+    hair = (encoded_img[..., 0] == 255) & (encoded_img[..., 1] == 255) & (encoded_img[..., 2] == 0)
+    body_skin = (encoded_img[..., 0] == 255) & (encoded_img[..., 1] == 0) & (encoded_img[..., 2] == 255)
+    face_skin = (encoded_img[..., 0] == 0) & (encoded_img[..., 1] == 255) & (encoded_img[..., 2] == 255)
+    clothes = (encoded_img[..., 0] == 255) & (encoded_img[..., 1] == 0) & (encoded_img[..., 2] == 0)
+    others = (encoded_img[..., 0] == 0) & (encoded_img[..., 1] == 255) & (encoded_img[..., 2] == 0)
+    segmap = np.stack([bg, hair, body_skin, face_skin, clothes, others], axis=0)
+    return segmap.astype(np.uint8)
+
+def read_video_frame(video_name, frame_id):
+    # https://blog.csdn.net/bby1987/article/details/108923361
+    # frame_num = video_capture.get(cv2.CAP_PROP_FRAME_COUNT) # ==> 总帧数
+    # fps = video_capture.get(cv2.CAP_PROP_FPS)               # ==> 帧率
+    # width = video_capture.get(cv2.CAP_PROP_FRAME_WIDTH)     # ==> 视频宽度
+    # height = video_capture.get(cv2.CAP_PROP_FRAME_HEIGHT)   # ==> 视频高度
+    # pos = video_capture.get(cv2.CAP_PROP_POS_FRAMES)        # ==> 句柄位置
+    # video_capture.set(cv2.CAP_PROP_POS_FRAMES, 1000)        # ==> 设置句柄位置
+    # pos = video_capture.get(cv2.CAP_PROP_POS_FRAMES)        # ==> 此时 pos = 1000.0
+    # video_capture.release()
+    vr = cv2.VideoCapture(video_name)
+    vr.set(cv2.CAP_PROP_POS_FRAMES, frame_id)
+    _, frame = vr.read()
+    return frame
+
+def decode_segmap_mask_from_segmap_video_frame(video_frame):
+    # video_frame: 0~255 BGR, obtained by read_video_frame
+    def assign_values(array):
+        remainder = array % 40  # 计算数组中每个值与40的余数
+        assigned_values = np.where(remainder <= 20, array - remainder, array + (40 - remainder))
+        return assigned_values
+    segmap = video_frame.mean(-1)
+    segmap = assign_values(segmap) // 40 # [H, W] with value 0~5 
+    segmap_mask = scatter_np(segmap[None, None, ...], classSeg=6)[0] # [6, H, W]
+    return segmap.astype(np.uint8)
+
+def extract_background(img_lst, segmap_lst=None):
+    """
+    img_lst: list of rgb ndarray
+    """
+    # only use 1/20 images
+    num_frames = len(img_lst)
+    img_lst = img_lst[::20] if num_frames > 20 else img_lst[0:1]
+        
+    if segmap_lst is not None:
+        segmap_lst = segmap_lst[::20] if num_frames > 20 else segmap_lst[0:1]
+        assert len(img_lst) == len(segmap_lst)
+    # get H/W
+    h, w = img_lst[0].shape[:2]
+
+    # nearest neighbors
+    all_xys = np.mgrid[0:h, 0:w].reshape(2, -1).transpose()
+    distss = []
+    for idx, img in enumerate(img_lst):
+        if segmap_lst is not None:
+            segmap = segmap_lst[idx]
+        else:
+            segmap = seg_model._cal_seg_map(img)
+        bg = (segmap[0]).astype(bool)
+        fg_xys = np.stack(np.nonzero(~bg)).transpose(1, 0)
+        nbrs = NearestNeighbors(n_neighbors=1, algorithm='kd_tree').fit(fg_xys)
+        dists, _ = nbrs.kneighbors(all_xys)
+        distss.append(dists)
+
+    distss = np.stack(distss)
+    max_dist = np.max(distss, 0)
+    max_id = np.argmax(distss, 0)
+
+    bc_pixs = max_dist > 10 # 5
+    bc_pixs_id = np.nonzero(bc_pixs)
+    bc_ids = max_id[bc_pixs]
+
+    num_pixs = distss.shape[1]
+    imgs = np.stack(img_lst).reshape(-1, num_pixs, 3)
+
+    bg_img = np.zeros((h*w, 3), dtype=np.uint8)
+    bg_img[bc_pixs_id, :] = imgs[bc_ids, bc_pixs_id, :]
+    bg_img = bg_img.reshape(h, w, 3)
+
+    max_dist = max_dist.reshape(h, w)
+    bc_pixs = max_dist > 10 # 5
+    bg_xys = np.stack(np.nonzero(~bc_pixs)).transpose()
+    fg_xys = np.stack(np.nonzero(bc_pixs)).transpose()
+    nbrs = NearestNeighbors(n_neighbors=1, algorithm='kd_tree').fit(fg_xys)
+    distances, indices = nbrs.kneighbors(bg_xys)
+    bg_fg_xys = fg_xys[indices[:, 0]]
+    bg_img[bg_xys[:, 0], bg_xys[:, 1], :] = bg_img[bg_fg_xys[:, 0], bg_fg_xys[:, 1], :]
+    return bg_img
+
+
+global_segmenter = None
+def job_cal_seg_map_for_image(img, segmenter_options=None, segmenter=None):
+    """
+    被 MediapipeSegmenter.multiprocess_cal_seg_map_for_a_video所使用, 专门用来处理单个长视频.
+    """
+    global global_segmenter
+    if segmenter is not None:
+        segmenter_actual = segmenter
+    else:
+        global_segmenter = vision.ImageSegmenter.create_from_options(segmenter_options) if global_segmenter is None else global_segmenter
+        segmenter_actual = global_segmenter
+    mp_image = mp.Image(image_format=mp.ImageFormat.SRGB, data=img)
+    out = segmenter_actual.segment(mp_image)
+    segmap = out.category_mask.numpy_view().copy() # [H, W]
+
+    segmap_mask = scatter_np(segmap[None, None, ...], classSeg=6)[0] # [6, H, W]
+    segmap_image = segmap[:, :, None].repeat(3, 2).astype(float)
+    segmap_image = (segmap_image * 40).astype(np.uint8)
+
+    return segmap_mask, segmap_image
+
+class MediapipeSegmenter:
+    def __init__(self):
+        model_path = 'data_gen/utils/mp_feature_extractors/selfie_multiclass_256x256.tflite'
+        if not os.path.exists(model_path):
+            os.makedirs(os.path.dirname(model_path), exist_ok=True)
+            print("downloading segmenter model from mediapipe...")
+            os.system(f"wget https://storage.googleapis.com/mediapipe-models/image_segmenter/selfie_multiclass_256x256/float32/latest/selfie_multiclass_256x256.tflite")
+            os.system(f"mv selfie_multiclass_256x256.tflite {model_path}")
+            print("download success")
+        base_options = python.BaseOptions(model_asset_path=model_path)
+        self.options = vision.ImageSegmenterOptions(base_options=base_options,running_mode=vision.RunningMode.IMAGE, output_category_mask=True)
+        self.video_options = vision.ImageSegmenterOptions(base_options=base_options,running_mode=vision.RunningMode.VIDEO, output_category_mask=True)
+    
+    def multiprocess_cal_seg_map_for_a_video(self, imgs, num_workers=4):
+        """
+        并行处理单个长视频
+        imgs: list of rgb array in 0~255
+        """
+        segmap_masks = []
+        segmap_images = []
+        img_lst = [(self.options, imgs[i]) for i in range(len(imgs))]
+        for (i, res) in multiprocess_run_tqdm(job_cal_seg_map_for_image, args=img_lst, num_workers=num_workers, desc='extracting from a video in multi-process'):
+            segmap_mask, segmap_image = res
+            segmap_masks.append(segmap_mask)
+            segmap_images.append(segmap_image)
+        return segmap_masks, segmap_images
+        
+    def _cal_seg_map_for_video(self, imgs, segmenter=None, return_onehot_mask=True, return_segmap_image=True):
+        segmenter = vision.ImageSegmenter.create_from_options(self.video_options) if segmenter is None else segmenter
+        assert return_onehot_mask or return_segmap_image # you should at least return one
+        segmap_masks = []
+        segmap_images = []
+        for i in tqdm.trange(len(imgs), desc="extracting segmaps from a video..."):
+            img = imgs[i]
+            mp_image = mp.Image(image_format=mp.ImageFormat.SRGB, data=img)
+            out = segmenter.segment_for_video(mp_image, 40 * i)
+            segmap = out.category_mask.numpy_view().copy() # [H, W]
+
+            if return_onehot_mask:
+                segmap_mask = scatter_np(segmap[None, None, ...], classSeg=6)[0] # [6, H, W]
+                segmap_masks.append(segmap_mask)
+            if return_segmap_image:
+                segmap_image = segmap[:, :, None].repeat(3, 2).astype(float)
+                segmap_image = (segmap_image * 40).astype(np.uint8)
+                segmap_images.append(segmap_image)
+        
+        if return_onehot_mask and return_segmap_image:
+            return segmap_masks, segmap_images
+        elif return_onehot_mask:
+            return segmap_masks
+        elif return_segmap_image:
+            return segmap_images
+    
+    def _cal_seg_map(self, img, segmenter=None, return_onehot_mask=True):
+        """
+        segmenter: vision.ImageSegmenter.create_from_options(options)
+        img: numpy, [H, W, 3], 0~255
+        segmap: [C, H, W]
+        0 - background
+        1 - hair
+        2 - body-skin
+        3 - face-skin
+        4 - clothes
+        5 - others (accessories)
+        """
+        assert img.ndim == 3
+        segmenter = vision.ImageSegmenter.create_from_options(self.options) if segmenter is None else segmenter 
+        image = mp.Image(image_format=mp.ImageFormat.SRGB, data=img)
+        out = segmenter.segment(image) 
+        segmap = out.category_mask.numpy_view().copy() # [H, W]
+        if return_onehot_mask:
+            segmap = scatter_np(segmap[None, None, ...], classSeg=6)[0] # [6, H, W]
+        return segmap
+
+    def _seg_out_img_with_segmap(self, img, segmap, mode='head'):
+        """
+        img: [h,w,c], img is in 0~255, np
+        """
+        # 
+        img = copy.deepcopy(img)
+        if mode == 'head':
+            selected_mask = segmap[[1,3,5] , :, :].sum(axis=0)[None,:] > 0.5 # glasses 也属于others
+            img[~selected_mask.repeat(3,axis=0).transpose(1,2,0)] = 0 # (-1,-1,-1) denotes black in our [-1,1] convention
+            # selected_mask = segmap[[1,3] , :, :].sum(dim=0, keepdim=True) > 0.5
+        elif mode == 'person':
+            selected_mask = segmap[[1,2,3,4,5], :, :].sum(axis=0)[None,:] > 0.5 
+            img[~selected_mask.repeat(3,axis=0).transpose(1,2,0)] = 0 # (-1,-1,-1) denotes black in our [-1,1] convention
+        elif mode == 'torso':
+            selected_mask = segmap[[2,4], :, :].sum(axis=0)[None,:] > 0.5
+            img[~selected_mask.repeat(3,axis=0).transpose(1,2,0)] = 0 # (-1,-1,-1) denotes black in our [-1,1] convention
+        elif mode == 'torso_with_bg':
+            selected_mask = segmap[[0, 2,4], :, :].sum(axis=0)[None,:] > 0.5
+            img[~selected_mask.repeat(3,axis=0).transpose(1,2,0)] = 0 # (-1,-1,-1) denotes black in our [-1,1] convention
+        elif mode == 'bg':
+            selected_mask = segmap[[0], :, :].sum(axis=0)[None,:] > 0.5  # only seg out 0, which means background
+            img[~selected_mask.repeat(3,axis=0).transpose(1,2,0)] = 0 # (-1,-1,-1) denotes black in our [-1,1] convention
+        elif mode == 'full':
+            pass
+        else:
+            raise NotImplementedError()
+        return img, selected_mask
+    
+    def _seg_out_img(self, img, segmenter=None, mode='head'):
+        """
+        imgs [H, W, 3] 0-255
+        return : person_img [B, 3, H, W]
+        """
+        segmenter = vision.ImageSegmenter.create_from_options(self.options) if segmenter is None else segmenter 
+        segmap = self._cal_seg_map(img, segmenter=segmenter, return_onehot_mask=True) # [B, 19, H, W]
+        return self._seg_out_img_with_segmap(img, segmap, mode=mode)
+
+    def seg_out_imgs(self, img, mode='head'):
+        """
+        api for pytorch img, -1~1
+        img: [B, 3, H, W], -1~1
+        """
+        device = img.device
+        img = convert_to_np(img.permute(0, 2, 3, 1)) # [B, H, W, 3]
+        img = ((img + 1) * 127.5).astype(np.uint8)
+        img_lst = [copy.deepcopy(img[i]) for i in range(len(img))]
+        out_lst = []
+        for im in img_lst:
+            out = self._seg_out_img(im, mode=mode)
+            out_lst.append(out)
+        seg_imgs = np.stack(out_lst) # [B, H, W, 3]
+        seg_imgs = (seg_imgs - 127.5) / 127.5
+        seg_imgs = torch.from_numpy(seg_imgs).permute(0, 3, 1, 2).to(device)
+        return seg_imgs
+
+if __name__ == '__main__':
+    import imageio, cv2, tqdm
+    import torchshow as ts
+    img = imageio.imread("1.png")
+    img = cv2.resize(img, (512,512))
+
+    seg_model = MediapipeSegmenter()
+    img = torch.tensor(img).unsqueeze(0).repeat([1, 1, 1, 1]).permute(0, 3,1,2)
+    img = (img-127.5)/127.5
+    out = seg_model.seg_out_imgs(img, 'torso')
+    ts.save(out,"torso.png")
+    out = seg_model.seg_out_imgs(img, 'head')
+    ts.save(out,"head.png")
+    out = seg_model.seg_out_imgs(img, 'bg')
+    ts.save(out,"bg.png")
+    img = convert_to_np(img.permute(0, 2, 3, 1)) # [B, H, W, 3]
+    img = ((img + 1) * 127.5).astype(np.uint8)
+    bg = extract_background(img)
+    ts.save(bg,"bg2.png")

data_gen/utils/mp_feature_extractors/selfie_multiclass_256x256.tflite

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c6748b1253a99067ef71f7e26ca71096cd449baefa8f101900ea23016507e0e0
+size 16371837

data_gen/utils/path_converter.py

@@ -0,0 +1,24 @@
+import os
+
+
+class PathConverter():
+    def __init__(self):
+        self.prefixs = {
+            "vid": "/video/",
+            "gt": "/gt_imgs/",
+            "head": "/head_imgs/", 
+            "torso": "/torso_imgs/", 
+            "person": "/person_imgs/", 
+            "torso_with_bg": "/torso_with_bg_imgs/", 
+            "single_bg": "/bg_img/",
+            "bg": "/bg_imgs/",
+            "segmaps": "/segmaps/",
+            "inpaint_torso": "/inpaint_torso_imgs/",
+            "com": "/com_imgs/",
+            "inpaint_torso_with_com_bg": "/inpaint_torso_with_com_bg_imgs/",
+        }
+        
+    def to(self, path: str, old_pattern: str, new_pattern: str):
+        return path.replace(self.prefixs[old_pattern], self.prefixs[new_pattern], 1)
+
+pc = PathConverter()

data_gen/utils/process_audio/extract_hubert.py

@@ -0,0 +1,95 @@
+from transformers import Wav2Vec2Processor, HubertModel
+import soundfile as sf
+import numpy as np
+import torch
+import os
+from utils.commons.hparams import set_hparams, hparams
+
+
+wav2vec2_processor = None
+hubert_model = None
+
+
+def get_hubert_from_16k_wav(wav_16k_name):
+    speech_16k, _ = sf.read(wav_16k_name)
+    hubert = get_hubert_from_16k_speech(speech_16k)
+    return hubert
+
+@torch.no_grad()
+def get_hubert_from_16k_speech(speech, device="cuda:0"):
+    global hubert_model, wav2vec2_processor
+    local_path = '/home/tiger/.cache/huggingface/hub/models--facebook--hubert-large-ls960-ft/snapshots/ece5fabbf034c1073acae96d5401b25be96709d8'
+    if hubert_model is None:
+        print("Loading the HuBERT Model...")
+        if os.path.exists(local_path):
+            hubert_model = HubertModel.from_pretrained(local_path)
+        else:
+            hubert_model = HubertModel.from_pretrained("facebook/hubert-large-ls960-ft")
+    hubert_model = hubert_model.to(device)
+    if wav2vec2_processor is None:
+        print("Loading the Wav2Vec2 Processor...")
+        if os.path.exists(local_path):
+            wav2vec2_processor = Wav2Vec2Processor.from_pretrained(local_path)
+        else:
+            wav2vec2_processor = Wav2Vec2Processor.from_pretrained("facebook/hubert-large-ls960-ft")
+
+    if speech.ndim ==2:
+        speech = speech[:, 0] # [T, 2] ==> [T,]
+    
+    input_values_all = wav2vec2_processor(speech, return_tensors="pt", sampling_rate=16000).input_values # [1, T]
+    input_values_all = input_values_all.to(device)
+    # For long audio sequence, due to the memory limitation, we cannot process them in one run
+    # HuBERT process the wav with a CNN of stride [5,2,2,2,2,2], making a stride of 320
+    # Besides, the kernel is [10,3,3,3,3,2,2], making 400 a fundamental unit to get 1 time step.
+    # So the CNN is euqal to a big Conv1D with kernel k=400 and stride s=320
+    # We have the equation to calculate out time step: T = floor((t-k)/s)
+    # To prevent overlap, we set each clip length of (K+S*(N-1)), where N is the expected length T of this clip
+    # The start point of next clip should roll back with a length of (kernel-stride) so it is stride * N
+    kernel = 400
+    stride = 320
+    clip_length = stride * 1000
+    num_iter = input_values_all.shape[1] // clip_length
+    expected_T = (input_values_all.shape[1] - (kernel-stride)) // stride
+    res_lst = []
+    for i in range(num_iter):
+        if i == 0:
+            start_idx = 0
+            end_idx = clip_length - stride + kernel
+        else:
+            start_idx = clip_length * i
+            end_idx = start_idx + (clip_length - stride + kernel)
+        input_values = input_values_all[:, start_idx: end_idx]
+        hidden_states = hubert_model.forward(input_values).last_hidden_state # [B=1, T=pts//320, hid=1024]
+        res_lst.append(hidden_states[0])
+    if num_iter > 0:
+        input_values = input_values_all[:, clip_length * num_iter:]
+    else:
+        input_values = input_values_all
+
+    if input_values.shape[1] >= kernel: # if the last batch is shorter than kernel_size, skip it            
+        hidden_states = hubert_model(input_values).last_hidden_state # [B=1, T=pts//320, hid=1024]
+        res_lst.append(hidden_states[0])
+    ret = torch.cat(res_lst, dim=0).cpu() # [T, 1024]
+
+    assert abs(ret.shape[0] - expected_T) <= 1
+    if ret.shape[0] < expected_T: # if skipping the last short 
+        ret = torch.cat([ret, ret[:, -1:, :].repeat([1,expected_T-ret.shape[0],1])], dim=1)
+    else:
+        ret = ret[:expected_T]
+
+    return ret
+
+
+if __name__ == '__main__':
+    from argparse import ArgumentParser
+    parser = ArgumentParser()
+    parser.add_argument('--video_id', type=str, default='May', help='')
+    args = parser.parse_args()
+    ### Process Single Long Audio for NeRF dataset
+    person_id = args.video_id
+    wav_16k_name = f"data/processed/videos/{person_id}/aud.wav"
+    hubert_npy_name = f"data/processed/videos/{person_id}/aud_hubert.npy"
+    speech_16k, _ = sf.read(wav_16k_name)
+    hubert_hidden = get_hubert_from_16k_speech(speech_16k)
+    np.save(hubert_npy_name, hubert_hidden.detach().numpy())
+    print(f"Saved at {hubert_npy_name}")

data_gen/utils/process_audio/extract_mel_f0.py

@@ -0,0 +1,148 @@
+import numpy as np
+import torch
+import glob
+import os
+import tqdm
+import librosa
+import parselmouth
+from utils.commons.pitch_utils import f0_to_coarse
+from utils.commons.multiprocess_utils import multiprocess_run_tqdm
+from utils.commons.os_utils import multiprocess_glob
+from utils.audio.io import save_wav
+
+from moviepy.editor import VideoFileClip
+from utils.commons.hparams import hparams, set_hparams
+
+def resample_wav(wav_name, out_name, sr=16000):
+    wav_raw, sr = librosa.core.load(wav_name, sr=sr)
+    save_wav(wav_raw, out_name, sr)
+    
+def split_wav(mp4_name, wav_name=None):
+    if wav_name is None:
+        wav_name = mp4_name.replace(".mp4", ".wav").replace("/video/", "/audio/")
+    if os.path.exists(wav_name):
+        return wav_name
+    os.makedirs(os.path.dirname(wav_name), exist_ok=True)
+    
+    video = VideoFileClip(mp4_name,verbose=False)
+    dur = video.duration
+    audio = video.audio 
+    assert audio is not None
+    audio.write_audiofile(wav_name,fps=16000,verbose=False,logger=None)
+    return wav_name
+
+def librosa_pad_lr(x, fsize, fshift, pad_sides=1):
+    '''compute right padding (final frame) or both sides padding (first and final frames)
+    '''
+    assert pad_sides in (1, 2)
+    # return int(fsize // 2)
+    pad = (x.shape[0] // fshift + 1) * fshift - x.shape[0]
+    if pad_sides == 1:
+        return 0, pad
+    else:
+        return pad // 2, pad // 2 + pad % 2
+
+def extract_mel_from_fname(wav_path,
+                      fft_size=512,
+                      hop_size=320,
+                      win_length=512,
+                      window="hann",
+                      num_mels=80,
+                      fmin=80,
+                      fmax=7600,
+                      eps=1e-6,
+                      sample_rate=16000,
+                      min_level_db=-100):
+    if isinstance(wav_path, str):
+        wav, _ = librosa.core.load(wav_path, sr=sample_rate)
+    else:
+        wav = wav_path
+
+    # get amplitude spectrogram
+    x_stft = librosa.stft(wav, n_fft=fft_size, hop_length=hop_size,
+                          win_length=win_length, window=window, center=False)
+    spc = np.abs(x_stft)  # (n_bins, T)
+
+    # get mel basis
+    fmin = 0 if fmin == -1 else fmin
+    fmax = sample_rate / 2 if fmax == -1 else fmax
+    mel_basis = librosa.filters.mel(sr=sample_rate, n_fft=fft_size, n_mels=num_mels, fmin=fmin, fmax=fmax)
+    mel = mel_basis @ spc
+
+    mel = np.log10(np.maximum(eps, mel))  # (n_mel_bins, T)
+    mel = mel.T
+
+    l_pad, r_pad = librosa_pad_lr(wav, fft_size, hop_size, 1)
+    wav = np.pad(wav, (l_pad, r_pad), mode='constant', constant_values=0.0)
+
+    return wav.T, mel
+
+def extract_f0_from_wav_and_mel(wav, mel,
+                        hop_size=320,
+                        audio_sample_rate=16000,
+                        ):
+    time_step = hop_size / audio_sample_rate * 1000
+    f0_min = 80
+    f0_max = 750
+    f0 = parselmouth.Sound(wav, audio_sample_rate).to_pitch_ac(
+        time_step=time_step / 1000, voicing_threshold=0.6,
+        pitch_floor=f0_min, pitch_ceiling=f0_max).selected_array['frequency']
+
+    delta_l = len(mel) - len(f0)
+    assert np.abs(delta_l) <= 8
+    if delta_l > 0:
+        f0 = np.concatenate([f0, [f0[-1]] * delta_l], 0)
+    f0 = f0[:len(mel)]
+    pitch_coarse = f0_to_coarse(f0)
+    return f0, pitch_coarse
+
+
+def extract_mel_f0_from_fname(wav_name=None, out_name=None):
+    try:
+        out_name = wav_name.replace(".wav", "_mel_f0.npy").replace("/audio/", "/mel_f0/")
+        os.makedirs(os.path.dirname(out_name), exist_ok=True)
+
+        wav, mel = extract_mel_from_fname(wav_name)
+        f0, f0_coarse = extract_f0_from_wav_and_mel(wav, mel)
+        out_dict = {
+            "mel": mel, # [T, 80]
+            "f0": f0,
+        }
+        np.save(out_name, out_dict)
+    except Exception as e:
+        print(e)
+
+def extract_mel_f0_from_video_name(mp4_name, wav_name=None, out_name=None):
+    if mp4_name.endswith(".mp4"):
+        wav_name = split_wav(mp4_name, wav_name)
+        if out_name is None:
+            out_name = mp4_name.replace(".mp4", "_mel_f0.npy").replace("/video/", "/mel_f0/")
+    elif mp4_name.endswith(".wav"):
+        wav_name = mp4_name
+        if out_name is None:
+            out_name = mp4_name.replace(".wav", "_mel_f0.npy").replace("/audio/", "/mel_f0/")
+
+    os.makedirs(os.path.dirname(out_name), exist_ok=True)
+
+    wav, mel = extract_mel_from_fname(wav_name)
+
+    f0, f0_coarse = extract_f0_from_wav_and_mel(wav, mel)
+    out_dict = {
+        "mel": mel, # [T, 80]
+        "f0": f0,
+    }
+    np.save(out_name, out_dict)
+
+
+if __name__ == '__main__':
+    from argparse import ArgumentParser
+    parser = ArgumentParser()
+    parser.add_argument('--video_id', type=str, default='May', help='')
+    args = parser.parse_args()
+    ### Process Single Long Audio for NeRF dataset
+    person_id = args.video_id
+
+    wav_16k_name = f"data/processed/videos/{person_id}/aud.wav"
+    out_name = f"data/processed/videos/{person_id}/aud_mel_f0.npy"
+    extract_mel_f0_from_video_name(wav_16k_name, out_name)
+    print(f"Saved at {out_name}")

data_gen/utils/process_audio/resample_audio_to_16k.py

@@ -0,0 +1,49 @@
+import os, glob
+from utils.commons.os_utils import multiprocess_glob
+from utils.commons.multiprocess_utils import multiprocess_run_tqdm
+
+
+def extract_wav16k_job(audio_name:str):
+    out_path = audio_name.replace("/audio_raw/","/audio/",1)
+    assert out_path != audio_name # prevent inplace
+    os.makedirs(os.path.dirname(out_path), exist_ok=True)
+    ffmpeg_path = "/usr/bin/ffmpeg"
+
+    cmd = f'{ffmpeg_path} -i {audio_name} -ar 16000 -v quiet -y {out_path}'
+    os.system(cmd)
+
+if __name__ == '__main__':
+    import argparse, glob, tqdm, random
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--aud_dir", default='/home/tiger/datasets/raw/CMLR/audio_raw/')
+    parser.add_argument("--ds_name", default='CMLR')
+    parser.add_argument("--num_workers", default=64, type=int)
+    parser.add_argument("--process_id", default=0, type=int)
+    parser.add_argument("--total_process", default=1, type=int)
+    args = parser.parse_args()
+    print(f"args {args}")
+
+    aud_dir = args.aud_dir
+    ds_name = args.ds_name
+    if ds_name in ['CMLR']:
+        aud_name_pattern = os.path.join(aud_dir, "*/*/*.wav")
+        aud_names = multiprocess_glob(aud_name_pattern)
+    else:
+        raise NotImplementedError()
+    aud_names = sorted(aud_names)
+    print(f"total audio number : {len(aud_names)}")
+    print(f"first {aud_names[0]} last {aud_names[-1]}")
+    # exit()
+    process_id = args.process_id
+    total_process = args.total_process
+    if total_process > 1:
+        assert process_id <= total_process -1
+        num_samples_per_process = len(aud_names) // total_process
+        if process_id == total_process:
+            aud_names = aud_names[process_id * num_samples_per_process : ]
+        else:
+            aud_names = aud_names[process_id * num_samples_per_process : (process_id+1) * num_samples_per_process]
+    
+    for i, res in multiprocess_run_tqdm(extract_wav16k_job, aud_names, num_workers=args.num_workers, desc="resampling videos"):
+        pass
+

data_gen/utils/process_image/extract_lm2d.py

@@ -0,0 +1,197 @@
+import os
+os.environ["OMP_NUM_THREADS"] = "1"
+import sys
+
+import glob
+import cv2
+import tqdm
+import numpy as np
+from data_gen.utils.mp_feature_extractors.face_landmarker import MediapipeLandmarker
+from utils.commons.multiprocess_utils import multiprocess_run_tqdm
+import warnings
+warnings.filterwarnings('ignore')
+
+import random
+random.seed(42)
+
+import pickle
+import json
+import gzip
+from typing import Any
+
+def load_file(filename, is_gzip: bool = False, is_json: bool = False) -> Any:
+    if is_json:
+        if is_gzip:
+            with gzip.open(filename, "r", encoding="utf-8") as f:
+                loaded_object = json.load(f)
+                return loaded_object
+        else:
+            with open(filename, "r", encoding="utf-8") as f:
+                loaded_object = json.load(f)
+                return loaded_object
+    else:
+        if is_gzip:
+            with gzip.open(filename, "rb") as f:
+                loaded_object = pickle.load(f)
+                return loaded_object
+        else:
+            with open(filename, "rb") as f:
+                loaded_object = pickle.load(f)
+                return loaded_object
+        
+def save_file(filename, content, is_gzip: bool = False, is_json: bool = False) -> None:
+    if is_json:
+        if is_gzip:
+            with gzip.open(filename, "w", encoding="utf-8") as f:
+                json.dump(content, f)
+        else:
+            with open(filename, "w", encoding="utf-8") as f:
+                json.dump(content, f)
+    else:
+        if is_gzip:
+            with gzip.open(filename, "wb") as f:
+                pickle.dump(content, f)
+        else:
+            with open(filename, "wb") as f:
+                pickle.dump(content, f)
+
+face_landmarker = None
+
+def extract_lms_mediapipe_job(img):
+    if img is None:
+        return None
+    global face_landmarker
+    if face_landmarker is None:
+        face_landmarker = MediapipeLandmarker()
+    lm478 = face_landmarker.extract_lm478_from_img(img)
+    return lm478
+    
+def extract_landmark_job(img_name):
+    try:
+        # if img_name == 'datasets/PanoHeadGen/raw/images/multi_view/chunk_0/seed0000002.png':
+            # print(1)
+            # input()
+        out_name = img_name.replace("/images_512/", "/lms_2d/").replace(".png","_lms.npy")
+        if os.path.exists(out_name):
+            print("out exists, skip...")
+            return
+        try:
+            os.makedirs(os.path.dirname(out_name), exist_ok=True)
+        except:
+            pass
+        img = cv2.imread(img_name)[:,:,::-1]
+
+        if img is not None:
+            lm468 = extract_lms_mediapipe_job(img)
+            if lm468 is not None:
+                np.save(out_name, lm468)
+        # print("Hahaha, solve one item!!!")
+    except Exception as e:
+        print(e)
+        pass
+        
+def out_exist_job(img_name):
+    out_name = img_name.replace("/images_512/", "/lms_2d/").replace(".png","_lms.npy") 
+    if  os.path.exists(out_name):
+        return None
+    else:
+        return img_name
+
+# def get_todo_img_names(img_names):
+#     todo_img_names = []
+#     for i, res in multiprocess_run_tqdm(out_exist_job, img_names, num_workers=64):
+#         if res is not None:
+#             todo_img_names.append(res)
+#     return todo_img_names
+
+
+if __name__ == '__main__':
+    import argparse, glob, tqdm, random
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--img_dir", default='/home/tiger/datasets/raw/FFHQ/images_512/')
+    parser.add_argument("--ds_name", default='FFHQ')
+    parser.add_argument("--num_workers", default=64, type=int)
+    parser.add_argument("--process_id", default=0, type=int)
+    parser.add_argument("--total_process", default=1, type=int)
+    parser.add_argument("--reset", action='store_true')
+    parser.add_argument("--img_names_file", default="img_names.pkl", type=str)
+    parser.add_argument("--load_img_names", action="store_true")
+
+    args = parser.parse_args()
+    print(f"args {args}")
+    img_dir = args.img_dir
+    img_names_file = os.path.join(img_dir, args.img_names_file)
+    if args.load_img_names:
+        img_names = load_file(img_names_file)
+        print(f"load image names from {img_names_file}")
+    else:
+        if args.ds_name == 'FFHQ_MV':
+            img_name_pattern1 = os.path.join(img_dir, "ref_imgs/*.png")
+            img_names1 = glob.glob(img_name_pattern1)
+            img_name_pattern2 = os.path.join(img_dir, "mv_imgs/*.png")
+            img_names2 = glob.glob(img_name_pattern2)
+            img_names = img_names1 + img_names2
+            img_names = sorted(img_names)
+        elif args.ds_name == 'FFHQ':
+            img_name_pattern = os.path.join(img_dir, "*.png")
+            img_names = glob.glob(img_name_pattern)
+            img_names = sorted(img_names)
+        elif args.ds_name == "PanoHeadGen":
+            # img_name_patterns = ["ref/*/*.png", "multi_view/*/*.png", "reverse/*/*.png"]
+            img_name_patterns = ["ref/*/*.png"]
+            img_names = []
+            for img_name_pattern in img_name_patterns:
+                img_name_pattern_full = os.path.join(img_dir, img_name_pattern)
+                img_names_part = glob.glob(img_name_pattern_full)
+                img_names.extend(img_names_part)
+            img_names = sorted(img_names)
+        
+    # save image names
+    if not args.load_img_names:
+        save_file(img_names_file, img_names)
+        print(f"save image names in {img_names_file}")
+        
+    print(f"total images number: {len(img_names)}")
+        
+        
+    process_id = args.process_id
+    total_process = args.total_process
+    if total_process > 1:
+        assert process_id <= total_process -1
+        num_samples_per_process = len(img_names) // total_process
+        if process_id == total_process:
+            img_names = img_names[process_id * num_samples_per_process : ]
+        else:
+            img_names = img_names[process_id * num_samples_per_process : (process_id+1) * num_samples_per_process]
+    
+    # if not args.reset:
+        # img_names = get_todo_img_names(img_names)
+        
+
+    print(f"todo_image {img_names[:10]}")
+    print(f"processing images number in this process: {len(img_names)}")
+    # print(f"todo images number: {len(img_names)}")
+    # input()
+    # exit()
+
+    if args.num_workers == 1:
+        index = 0
+        for img_name in tqdm.tqdm(img_names, desc=f"Root process {args.process_id}: extracting MP-based landmark2d"):
+            try:
+                extract_landmark_job(img_name)
+            except Exception as e:
+                print(e)
+                pass
+            if index % max(1, int(len(img_names) * 0.003)) == 0:
+                print(f"processed {index} / {len(img_names)}")
+                sys.stdout.flush()
+            index += 1
+    else:
+        for i, res in multiprocess_run_tqdm(
+            extract_landmark_job, img_names, 
+            num_workers=args.num_workers, 
+            desc=f"Root {args.process_id}: extracing MP-based landmark2d"): 
+            # if index % max(1, int(len(img_names) * 0.003)) == 0:
+            print(f"processed {i+1} / {len(img_names)}")
+            sys.stdout.flush()
+        print(f"Root {args.process_id}: Finished extracting.")

data_gen/utils/process_image/extract_segment_imgs.py

@@ -0,0 +1,114 @@
+import os
+os.environ["OMP_NUM_THREADS"] = "1"
+
+import glob
+import cv2
+import tqdm
+import numpy as np
+import PIL
+from utils.commons.tensor_utils import convert_to_np
+import torch
+import mediapipe as mp
+from utils.commons.multiprocess_utils import multiprocess_run_tqdm
+from data_gen.utils.mp_feature_extractors.mp_segmenter import MediapipeSegmenter
+from data_gen.utils.process_video.extract_segment_imgs import inpaint_torso_job, extract_background, save_rgb_image_to_path
+seg_model = MediapipeSegmenter()
+
+
+def extract_segment_job(img_name):
+    try:
+        img = cv2.imread(img_name)
+        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
+
+        segmap = seg_model._cal_seg_map(img)
+        bg_img = extract_background([img], [segmap])
+        out_img_name = img_name.replace("/images_512/",f"/bg_img/").replace(".mp4", ".jpg")
+        save_rgb_image_to_path(bg_img, out_img_name)
+
+        com_img = img.copy()
+        bg_part = segmap[0].astype(bool)[..., None].repeat(3,axis=-1)
+        com_img[bg_part] = bg_img[bg_part]
+        out_img_name = img_name.replace("/images_512/",f"/com_imgs/")
+        save_rgb_image_to_path(com_img, out_img_name)
+
+        for mode in ['head', 'torso', 'person', 'torso_with_bg', 'bg']:
+            out_img, _ = seg_model._seg_out_img_with_segmap(img, segmap, mode=mode)
+            out_img_name = img_name.replace("/images_512/",f"/{mode}_imgs/")
+            out_img = cv2.cvtColor(out_img, cv2.COLOR_RGB2BGR)
+            try: os.makedirs(os.path.dirname(out_img_name), exist_ok=True)
+            except: pass
+            cv2.imwrite(out_img_name, out_img)
+
+        inpaint_torso_img, inpaint_torso_with_bg_img, _, _ = inpaint_torso_job(img, segmap)
+        out_img_name = img_name.replace("/images_512/",f"/inpaint_torso_imgs/")
+        save_rgb_image_to_path(inpaint_torso_img, out_img_name)
+        inpaint_torso_with_bg_img[bg_part] = bg_img[bg_part]
+        out_img_name = img_name.replace("/images_512/",f"/inpaint_torso_with_com_bg_imgs/")
+        save_rgb_image_to_path(inpaint_torso_with_bg_img, out_img_name)
+        return 0
+    except Exception as e:
+        print(e)
+        return 1
+
+def out_exist_job(img_name):
+    out_name1 = img_name.replace("/images_512/", "/head_imgs/")
+    out_name2 = img_name.replace("/images_512/", "/com_imgs/")
+    out_name3 = img_name.replace("/images_512/", "/inpaint_torso_with_com_bg_imgs/")
+    
+    if  os.path.exists(out_name1) and os.path.exists(out_name2) and os.path.exists(out_name3):
+        return None
+    else:
+        return img_name
+
+def get_todo_img_names(img_names):
+    todo_img_names = []
+    for i, res in multiprocess_run_tqdm(out_exist_job, img_names, num_workers=64):
+        if res is not None:
+            todo_img_names.append(res)
+    return todo_img_names
+
+
+if __name__ == '__main__':
+    import argparse, glob, tqdm, random
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--img_dir", default='./images_512')
+    # parser.add_argument("--img_dir", default='/home/tiger/datasets/raw/FFHQ/images_512')
+    parser.add_argument("--ds_name", default='FFHQ')
+    parser.add_argument("--num_workers", default=1, type=int)
+    parser.add_argument("--seed", default=0, type=int)
+    parser.add_argument("--process_id", default=0, type=int)
+    parser.add_argument("--total_process", default=1, type=int)
+    parser.add_argument("--reset", action='store_true')
+
+    args = parser.parse_args()
+    img_dir = args.img_dir
+    if args.ds_name == 'FFHQ_MV':
+        img_name_pattern1 = os.path.join(img_dir, "ref_imgs/*.png")
+        img_names1 = glob.glob(img_name_pattern1)
+        img_name_pattern2 = os.path.join(img_dir, "mv_imgs/*.png")
+        img_names2 = glob.glob(img_name_pattern2)
+        img_names = img_names1 + img_names2
+    elif args.ds_name == 'FFHQ':
+        img_name_pattern = os.path.join(img_dir, "*.png")
+        img_names = glob.glob(img_name_pattern)
+    
+    img_names = sorted(img_names)
+    random.seed(args.seed)
+    random.shuffle(img_names)
+
+    process_id = args.process_id
+    total_process = args.total_process
+    if total_process > 1:
+        assert process_id <= total_process -1
+        num_samples_per_process = len(img_names) // total_process
+        if process_id == total_process:
+            img_names = img_names[process_id * num_samples_per_process : ]
+        else:
+            img_names = img_names[process_id * num_samples_per_process : (process_id+1) * num_samples_per_process]
+    
+    if not args.reset:
+        img_names = get_todo_img_names(img_names)
+    print(f"todo images number: {len(img_names)}")
+
+    for vid_name in multiprocess_run_tqdm(extract_segment_job ,img_names, desc=f"Root process {args.process_id}: extracting segment images", num_workers=args.num_workers):
+        pass

data_gen/utils/process_image/fit_3dmm_landmark.py

@@ -0,0 +1,369 @@
+from numpy.core.numeric import require
+from numpy.lib.function_base import quantile
+import torch
+import torch.nn.functional as F
+import copy
+import numpy as np
+
+import os
+import sys
+import cv2
+import argparse
+import tqdm
+from utils.commons.multiprocess_utils import multiprocess_run_tqdm
+from data_gen.utils.mp_feature_extractors.face_landmarker import MediapipeLandmarker
+
+from deep_3drecon.deep_3drecon_models.bfm import ParametricFaceModel
+import pickle
+
+face_model = ParametricFaceModel(bfm_folder='deep_3drecon/BFM', 
+            camera_distance=10, focal=1015, keypoint_mode='mediapipe')
+face_model.to("cuda")
+     
+
+index_lm68_from_lm468 = [127,234,93,132,58,136,150,176,152,400,379,365,288,361,323,454,356,70,63,105,66,107,336,296,334,293,300,168,197,5,4,75,97,2,326,305,
+                         33,160,158,133,153,144,362,385,387,263,373,380,61,40,37,0,267,270,291,321,314,17,84,91,78,81,13,311,308,402,14,178]
+
+dir_path = os.path.dirname(os.path.realpath(__file__))
+
+LAMBDA_REG_ID = 0.3
+LAMBDA_REG_EXP = 0.05
+
+def save_file(name, content):
+    with open(name, "wb") as f:
+        pickle.dump(content, f) 
+        
+def load_file(name):
+    with open(name, "rb") as f:
+        content = pickle.load(f)
+    return content
+
+def cal_lan_loss_mp(proj_lan, gt_lan):
+    # [B, 68, 2]
+    loss = (proj_lan - gt_lan).pow(2)
+    # loss = (proj_lan - gt_lan).abs()
+    unmatch_mask = [ 93, 127, 132, 234, 323, 356, 361, 454]
+    eye = [33,246,161,160,159,158,157,173,133,155,154,153,145,144,163,7] + [263,466,388,387,386,385,384,398,362,382,381,380,374,373,390,249]
+    inner_lip = [78,191,80,81,82,13,312,311,310,415,308,324,318,402,317,14,87,178,88,95]
+    outer_lip = [61,185,40,39,37,0,267,269,270,409,291,375,321,405,314,17,84,181,91,146]
+    weights = torch.ones_like(loss)
+    weights[:, eye] = 5
+    weights[:, inner_lip] = 2
+    weights[:, outer_lip] = 2
+    weights[:, unmatch_mask] = 0
+    loss = loss * weights
+    return torch.mean(loss)
+ 
+def cal_lan_loss(proj_lan, gt_lan):
+    # [B, 68, 2]
+    loss = (proj_lan - gt_lan)** 2
+    # use the ldm weights from deep3drecon, see deep_3drecon/deep_3drecon_models/losses.py
+    weights = torch.zeros_like(loss)
+    weights = torch.ones_like(loss)
+    weights[:, 36:48, :] = 3 # eye 12 points
+    weights[:, -8:, :] =  3 # inner lip 8 points
+    weights[:, 28:31, :] =  3 # nose 3 points
+    loss = loss * weights
+    return torch.mean(loss)
+
+def set_requires_grad(tensor_list):
+    for tensor in tensor_list:
+        tensor.requires_grad = True
+
+def read_video_to_frames(img_name):
+    frames = []
+    cap = cv2.VideoCapture(img_name)
+    while cap.isOpened():
+        ret, frame_bgr = cap.read()
+        if frame_bgr is None:
+            break
+        frame_rgb = cv2.cvtColor(frame_bgr, cv2.COLOR_BGR2RGB)
+        frames.append(frame_rgb)
+    return np.stack(frames)
+    
+@torch.enable_grad()
+def fit_3dmm_for_a_image(img_name, debug=False, keypoint_mode='mediapipe', device="cuda:0", save=True):
+    img = cv2.imread(img_name)
+    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
+    img_h, img_w = img.shape[0], img.shape[0]
+    assert img_h == img_w
+    num_frames = 1
+
+    lm_name = img_name.replace("/images_512/", "/lms_2d/").replace(".png", "_lms.npy")
+    if lm_name.endswith('_lms.npy') and os.path.exists(lm_name):
+        lms = np.load(lm_name)
+    else:
+        # print("lms_2d file not found, try to extract it from image...")
+        try:
+            landmarker = MediapipeLandmarker()
+            lms = landmarker.extract_lm478_from_img_name(img_name)
+            # lms = landmarker.extract_lm478_from_img(img)
+        except Exception as e:
+            print(e)
+            return
+        if lms is None:
+            print("get None lms_2d, please check whether each frame has one head, exiting...")
+            return
+    lms = lms[:468].reshape([468,2])
+    lms = torch.FloatTensor(lms).to(device=device)
+    lms[..., 1] = img_h - lms[..., 1] # flip the height axis
+
+    if keypoint_mode == 'mediapipe':
+        cal_lan_loss_fn = cal_lan_loss_mp
+        out_name = img_name.replace("/images_512/", "/coeff_fit_mp/").replace(".png", "_coeff_fit_mp.npy")
+    else:
+        cal_lan_loss_fn = cal_lan_loss
+        out_name = img_name.replace("/images_512/", "/coeff_fit_lm68/").replace(".png", "_coeff_fit_lm68.npy")
+    try:
+        os.makedirs(os.path.dirname(out_name), exist_ok=True)
+    except:
+        pass
+
+    id_dim, exp_dim = 80, 64
+    sel_ids = np.arange(0, num_frames, 40)
+    sel_num = sel_ids.shape[0]
+    arg_focal = face_model.focal
+
+    h = w = face_model.center * 2
+    img_scale_factor = img_h / h
+    lms /= img_scale_factor
+    cxy = torch.tensor((w / 2.0, h / 2.0), dtype=torch.float).to(device=device)
+
+    id_para = lms.new_zeros((num_frames, id_dim), requires_grad=True) # lms.new_zeros((1, id_dim), requires_grad=True)
+    exp_para = lms.new_zeros((num_frames, exp_dim), requires_grad=True)
+    euler_angle = lms.new_zeros((num_frames, 3), requires_grad=True)
+    trans = lms.new_zeros((num_frames, 3), requires_grad=True)
+
+    focal_length = lms.new_zeros(1, requires_grad=True)
+    focal_length.data += arg_focal
+
+    set_requires_grad([id_para, exp_para, euler_angle, trans])
+
+    optimizer_idexp = torch.optim.Adam([id_para, exp_para], lr=.1)
+    optimizer_frame = torch.optim.Adam([euler_angle, trans], lr=.1)
+
+    # 其他参数初始化，先训练euler和trans
+    for _ in range(200):
+        proj_geo = face_model.compute_for_landmark_fit(
+            id_para, exp_para, euler_angle, trans)
+        loss_lan = cal_lan_loss_fn(proj_geo[:, :, :2], lms.detach())
+        loss = loss_lan
+        optimizer_frame.zero_grad()
+        loss.backward()
+        optimizer_frame.step()
+    # print(f"loss_lan: {loss_lan.item():.2f}, euler_abs_mean: {euler_angle.abs().mean().item():.4f}, euler_std: {euler_angle.std().item():.4f}, euler_min: {euler_angle.min().item():.4f}, euler_max: {euler_angle.max().item():.4f}")
+    # print(f"trans_z_mean: {trans[...,2].mean().item():.4f}, trans_z_std: {trans[...,2].std().item():.4f}, trans_min: {trans[...,2].min().item():.4f}, trans_max: {trans[...,2].max().item():.4f}")
+
+    for param_group in optimizer_frame.param_groups:
+        param_group['lr'] = 0.1
+
+    # "jointly roughly training id exp euler trans"
+    for _ in range(200):
+        proj_geo = face_model.compute_for_landmark_fit(
+            id_para, exp_para, euler_angle, trans)
+        loss_lan = cal_lan_loss_fn(
+            proj_geo[:, :, :2], lms.detach())
+        loss_regid = torch.mean(id_para*id_para) # 正则化
+        loss_regexp = torch.mean(exp_para * exp_para)
+
+        loss = loss_lan  + loss_regid * LAMBDA_REG_ID + loss_regexp * LAMBDA_REG_EXP
+        optimizer_idexp.zero_grad()
+        optimizer_frame.zero_grad()
+        loss.backward()
+        optimizer_idexp.step()
+        optimizer_frame.step()
+    # print(f"loss_lan: {loss_lan.item():.2f}, loss_reg_id: {loss_regid.item():.2f},loss_reg_exp: {loss_regexp.item():.2f},")
+    # print(f"euler_abs_mean: {euler_angle.abs().mean().item():.4f}, euler_std: {euler_angle.std().item():.4f}, euler_min: {euler_angle.min().item():.4f}, euler_max: {euler_angle.max().item():.4f}")
+    # print(f"trans_z_mean: {trans[...,2].mean().item():.4f}, trans_z_std: {trans[...,2].std().item():.4f}, trans_min: {trans[...,2].min().item():.4f}, trans_max: {trans[...,2].max().item():.4f}")
+
+    # start fine training, intialize from the roughly trained results
+    id_para_ = lms.new_zeros((num_frames, id_dim), requires_grad=True)
+    id_para_.data = id_para.data.clone()
+    id_para = id_para_
+    exp_para_ = lms.new_zeros((num_frames, exp_dim), requires_grad=True)
+    exp_para_.data = exp_para.data.clone()
+    exp_para = exp_para_
+    euler_angle_ = lms.new_zeros((num_frames, 3), requires_grad=True)
+    euler_angle_.data = euler_angle.data.clone()
+    euler_angle = euler_angle_
+    trans_ = lms.new_zeros((num_frames, 3), requires_grad=True)
+    trans_.data = trans.data.clone()
+    trans = trans_
+
+    batch_size = 1
+
+    # "fine fitting the 3DMM in batches"
+    for i in range(int((num_frames-1)/batch_size+1)):
+        if (i+1)*batch_size > num_frames:
+            start_n = num_frames-batch_size
+            sel_ids = np.arange(max(num_frames-batch_size,0), num_frames)
+        else:
+            start_n = i*batch_size
+            sel_ids = np.arange(i*batch_size, i*batch_size+batch_size)
+        sel_lms = lms[sel_ids]
+
+        sel_id_para = id_para.new_zeros(
+            (batch_size, id_dim), requires_grad=True)
+        sel_id_para.data = id_para[sel_ids].clone()
+        sel_exp_para = exp_para.new_zeros(
+            (batch_size, exp_dim), requires_grad=True)
+        sel_exp_para.data = exp_para[sel_ids].clone()
+        sel_euler_angle = euler_angle.new_zeros(
+            (batch_size, 3), requires_grad=True)
+        sel_euler_angle.data = euler_angle[sel_ids].clone()
+        sel_trans = trans.new_zeros((batch_size, 3), requires_grad=True)
+        sel_trans.data = trans[sel_ids].clone()
+        
+        set_requires_grad([sel_id_para, sel_exp_para, sel_euler_angle, sel_trans])
+        optimizer_cur_batch = torch.optim.Adam(
+            [sel_id_para, sel_exp_para, sel_euler_angle, sel_trans], lr=0.005)
+
+        for j in range(50):
+            proj_geo = face_model.compute_for_landmark_fit(
+                sel_id_para, sel_exp_para, sel_euler_angle, sel_trans)
+            loss_lan = cal_lan_loss_fn(
+                proj_geo[:, :, :2], lms.unsqueeze(0).detach())
+
+            loss_regid = torch.mean(sel_id_para*sel_id_para) # 正则化
+            loss_regexp = torch.mean(sel_exp_para*sel_exp_para)
+            loss = loss_lan + loss_regid * LAMBDA_REG_ID + loss_regexp * LAMBDA_REG_EXP 
+            optimizer_cur_batch.zero_grad()
+            loss.backward()
+            optimizer_cur_batch.step()
+        print(f"batch {i} | loss_lan: {loss_lan.item():.2f}, loss_reg_id: {loss_regid.item():.2f},loss_reg_exp: {loss_regexp.item():.2f}")
+        id_para[sel_ids].data = sel_id_para.data.clone()
+        exp_para[sel_ids].data = sel_exp_para.data.clone()
+        euler_angle[sel_ids].data = sel_euler_angle.data.clone()
+        trans[sel_ids].data = sel_trans.data.clone()
+
+    coeff_dict = {'id': id_para.detach().cpu().numpy(), 'exp': exp_para.detach().cpu().numpy(),
+                'euler': euler_angle.detach().cpu().numpy(), 'trans': trans.detach().cpu().numpy()}
+    if save:
+        np.save(out_name, coeff_dict, allow_pickle=True)
+    
+    if debug:
+        import imageio
+        debug_name = img_name.replace("/images_512/", "/coeff_fit_mp_debug/").replace(".png", "_debug.png").replace(".jpg", "_debug.jpg")
+        try: os.makedirs(os.path.dirname(debug_name), exist_ok=True)
+        except: pass
+        proj_geo = face_model.compute_for_landmark_fit(id_para, exp_para, euler_angle, trans)
+        lm68s = proj_geo[:,:,:2].detach().cpu().numpy()  # [T, 68,2]
+        lm68s = lm68s * img_scale_factor
+        lms = lms * img_scale_factor
+        lm68s[..., 1] = img_h - lm68s[..., 1] # flip the height axis
+        lms[..., 1] = img_h - lms[..., 1] # flip the height axis
+        lm68s = lm68s.astype(int)
+        lm68s = lm68s.reshape([-1,2])
+        lms = lms.cpu().numpy().astype(int).reshape([-1,2])
+        for lm in lm68s:
+            img = cv2.circle(img, lm, 1, (0, 0, 255), thickness=-1)
+        for gt_lm in lms:
+            img = cv2.circle(img, gt_lm, 2, (255, 0, 0), thickness=1)
+        imageio.imwrite(debug_name, img)
+        print(f"debug img saved at {debug_name}")
+    return coeff_dict
+
+def out_exist_job(vid_name):
+    out_name = vid_name.replace("/images_512/", "/coeff_fit_mp/").replace(".png","_coeff_fit_mp.npy") 
+    # if os.path.exists(out_name) or not os.path.exists(lms_name):
+    if os.path.exists(out_name):
+        return None
+    else:
+        return vid_name
+
+def get_todo_img_names(img_names):
+    todo_img_names = []
+    for i, res in multiprocess_run_tqdm(out_exist_job, img_names, num_workers=16):
+        if res is not None:
+            todo_img_names.append(res)
+    return todo_img_names
+
+
+if __name__ == '__main__':
+    import argparse, glob, tqdm
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--img_dir", default='/home/tiger/datasets/raw/FFHQ/images_512')
+    parser.add_argument("--ds_name", default='FFHQ')
+    parser.add_argument("--seed", default=0, type=int)
+    parser.add_argument("--process_id", default=0, type=int)
+    parser.add_argument("--total_process", default=1, type=int)
+    parser.add_argument("--keypoint_mode", default='mediapipe', type=str)
+    parser.add_argument("--debug", action='store_true')
+    parser.add_argument("--reset", action='store_true')
+    parser.add_argument("--device", default="cuda:0", type=str)
+    parser.add_argument("--output_log", action='store_true')
+    parser.add_argument("--load_names", action="store_true")
+
+    args = parser.parse_args()
+    img_dir = args.img_dir
+    load_names = args.load_names
+    
+    print(f"args {args}")
+    
+    if args.ds_name == 'single_img':
+        img_names = [img_dir]
+    else:
+        img_names_path = os.path.join(img_dir, "img_dir.pkl")
+        if os.path.exists(img_names_path) and load_names:
+            print(f"loading vid names from {img_names_path}")
+            img_names = load_file(img_names_path)
+        else:
+            if args.ds_name == 'FFHQ_MV':
+                img_name_pattern1 = os.path.join(img_dir, "ref_imgs/*.png")
+                img_names1 = glob.glob(img_name_pattern1)
+                img_name_pattern2 = os.path.join(img_dir, "mv_imgs/*.png")
+                img_names2 = glob.glob(img_name_pattern2)
+                img_names = img_names1 + img_names2
+                img_names = sorted(img_names)
+            elif args.ds_name == 'FFHQ':
+                img_name_pattern = os.path.join(img_dir, "*.png")
+                img_names = glob.glob(img_name_pattern)
+                img_names = sorted(img_names)
+            elif args.ds_name == "PanoHeadGen":
+                img_name_patterns = ["ref/*/*.png"]
+                img_names = []
+                for img_name_pattern in img_name_patterns:
+                    img_name_pattern_full = os.path.join(img_dir, img_name_pattern)
+                    img_names_part = glob.glob(img_name_pattern_full)
+                    img_names.extend(img_names_part)
+                img_names = sorted(img_names)
+            print(f"saving image names to {img_names_path}")
+            save_file(img_names_path, img_names)
+            
+    # import random
+    # random.seed(args.seed)
+    # random.shuffle(img_names)
+
+    face_model = ParametricFaceModel(bfm_folder='deep_3drecon/BFM', 
+                camera_distance=10, focal=1015, keypoint_mode=args.keypoint_mode)
+    face_model.to(torch.device(args.device))
+     
+    process_id = args.process_id
+    total_process = args.total_process
+    if total_process > 1:
+        assert process_id <= total_process -1 and process_id >= 0
+        num_samples_per_process = len(img_names) // total_process
+        if process_id == total_process:
+            img_names = img_names[process_id * num_samples_per_process : ]
+        else:
+            img_names = img_names[process_id * num_samples_per_process : (process_id+1) * num_samples_per_process]
+    print(f"image names number (before fileter): {len(img_names)}")
+
+
+    if not args.reset:
+        img_names = get_todo_img_names(img_names)
+
+    print(f"image names number (after  fileter): {len(img_names)}")
+    for i in tqdm.trange(len(img_names), desc=f"process {process_id}: fitting 3dmm ..."):
+        img_name = img_names[i]
+        try:
+            fit_3dmm_for_a_image(img_name, args.debug, device=args.device)
+        except Exception as e:
+            print(img_name, e)
+        if args.output_log and i % max(int(len(img_names) * 0.003), 1) == 0:
+            print(f"process {process_id}: {i + 1} / {len(img_names)} done")
+            sys.stdout.flush()
+            sys.stderr.flush()
+            
+    print(f"process {process_id}: fitting 3dmm all done")
+

data_gen/utils/process_video/euler2quaterion.py

@@ -0,0 +1,35 @@
+import numpy as np
+import torch
+import math
+import numba
+from scipy.spatial.transform import Rotation as R
+
+def euler2quaterion(euler, use_radian=True):
+    """
+    euler: np.array, [batch, 3]
+    return: the quaterion, np.array, [batch, 4]
+    """
+    r = R.from_euler('xyz',euler, degrees=not use_radian)
+    return r.as_quat()
+
+def quaterion2euler(quat, use_radian=True):
+    """
+    quat: np.array, [batch, 4]
+    return: the euler, np.array, [batch, 3]
+    """
+    r = R.from_quat(quat)
+    return r.as_euler('xyz', degrees=not use_radian)
+
+def rot2quaterion(rot):
+    r = R.from_matrix(rot)
+    return r.as_quat()
+
+def quaterion2rot(quat):
+    r = R.from_quat(quat)
+    return r.as_matrix()
+
+if __name__ == '__main__':
+    euler = np.array([89.999,89.999,89.999] * 100).reshape([100,3])
+    q = euler2quaterion(euler, use_radian=False)
+    e = quaterion2euler(q, use_radian=False)
+    print(" ")

data_gen/utils/process_video/extract_blink.py

@@ -0,0 +1,50 @@
+import numpy as np
+from data_util.face3d_helper import Face3DHelper
+from utils.commons.tensor_utils import convert_to_tensor
+
+def polygon_area(x, y):
+    """
+    x: [T, K=6]
+    y: [T, K=6]
+    return: [T,]
+    """
+    x_ = x - x.mean(axis=-1, keepdims=True)
+    y_ = y - y.mean(axis=-1, keepdims=True)
+    correction = x_[:,-1] * y_[:,0] - y_[:,-1]* x_[:,0]
+    main_area = (x_[:,:-1] * y_[:,1:]).sum(axis=-1) - (y_[:,:-1] * x_[:,1:]).sum(axis=-1)
+    return 0.5 * np.abs(main_area + correction)
+
+def get_eye_area_percent(id, exp, face3d_helper):
+    id = convert_to_tensor(id)
+    exp = convert_to_tensor(exp)
+    cano_lm3d = face3d_helper.reconstruct_cano_lm3d(id, exp)
+    cano_lm2d = (cano_lm3d[..., :2] + 1) / 2
+    lms = cano_lm2d.cpu().numpy()
+    eyes_left = slice(36, 42)
+    eyes_right = slice(42, 48)
+    area_left = polygon_area(lms[:, eyes_left, 0], lms[:, eyes_left, 1])
+    area_right = polygon_area(lms[:, eyes_right, 0], lms[:, eyes_right, 1])
+    # area percentage of two eyes of the whole image...
+    area_percent = (area_left + area_right) / 1 * 100 # recommend threshold is 0.25%
+    return area_percent # [T,]
+
+
+if __name__ == '__main__':
+    import numpy as np
+    import imageio
+    import cv2
+    import torch
+    from data_gen.utils.process_video.extract_lm2d import extract_lms_mediapipe_job, read_video_to_frames, index_lm68_from_lm468
+    from data_gen.utils.process_video.fit_3dmm_landmark import fit_3dmm_for_a_video
+    from data_util.face3d_helper import Face3DHelper
+
+    face3d_helper = Face3DHelper()
+    video_name = 'data/raw/videos/May_10s.mp4'
+    frames = read_video_to_frames(video_name)
+    coeff = fit_3dmm_for_a_video(video_name, save=False)
+    area_percent = get_eye_area_percent(torch.tensor(coeff['id']), torch.tensor(coeff['exp']), face3d_helper)
+    writer = imageio.get_writer("1.mp4", fps=25)
+    for idx, frame in enumerate(frames):
+        frame = cv2.putText(frame, f"{area_percent[idx]:.2f}", org=(128,128), fontFace=cv2.FONT_HERSHEY_COMPLEX, fontScale=1, color=(255,0,0), thickness=1)
+        writer.append_data(frame)
+    writer.close()

data_gen/utils/process_video/extract_lm2d.py

@@ -0,0 +1,164 @@
+import os
+os.environ["OMP_NUM_THREADS"] = "1"
+import sys
+import glob
+import cv2
+import pickle
+import tqdm
+import numpy as np
+import mediapipe as mp
+from utils.commons.multiprocess_utils import multiprocess_run_tqdm
+from utils.commons.os_utils import multiprocess_glob
+from data_gen.utils.mp_feature_extractors.face_landmarker import MediapipeLandmarker
+import warnings
+import traceback
+
+warnings.filterwarnings('ignore')
+
+"""
+基于Face_aligment的lm68已被弃用,因为其：
+1. 对眼睛部位的预测精度极低
+2. 无法在大偏转角度时准确预测被遮挡的下颚线, 导致大角度时3dmm的GT label就是有问题的, 从而影响性能
+我们目前转而使用基于mediapipe的lm68
+"""
+# def extract_landmarks(ori_imgs_dir):
+
+#     print(f'[INFO] ===== extract face landmarks from {ori_imgs_dir} =====')
+
+#     fa = face_alignment.FaceAlignment(face_alignment.LandmarksType._2D, flip_input=False)
+#     image_paths = glob.glob(os.path.join(ori_imgs_dir, '*.png'))
+#     for image_path in tqdm.tqdm(image_paths):
+#         out_name = image_path.replace("/images_512/", "/lms_2d/").replace(".png",".lms")
+#         if os.path.exists(out_name):
+#             continue
+#         input = cv2.imread(image_path, cv2.IMREAD_UNCHANGED) # [H, W, 3]
+#         input = cv2.cvtColor(input, cv2.COLOR_BGR2RGB)
+#         preds = fa.get_landmarks(input)
+#         if preds is None:
+#             print(f"Skip {image_path} for no face detected")
+#             continue
+#         if len(preds) > 0:
+#             lands = preds[0].reshape(-1, 2)[:,:2]
+#             os.makedirs(os.path.dirname(out_name), exist_ok=True)
+#             np.savetxt(out_name, lands, '%f')
+#     del fa
+#     print(f'[INFO] ===== extracted face landmarks =====')
+
+def save_file(name, content):
+    with open(name, "wb") as f:
+        pickle.dump(content, f) 
+        
+def load_file(name):
+    with open(name, "rb") as f:
+        content = pickle.load(f)
+    return content
+
+
+face_landmarker = None
+    
+def extract_landmark_job(video_name, nerf=False):
+    try:
+        if nerf:
+            out_name = video_name.replace("/raw/", "/processed/").replace(".mp4","/lms_2d.npy")
+        else:
+            out_name = video_name.replace("/video/", "/lms_2d/").replace(".mp4","_lms.npy")
+        if os.path.exists(out_name):
+            # print("out exists, skip...")
+            return
+        try:
+            os.makedirs(os.path.dirname(out_name), exist_ok=True)
+        except:
+            pass
+        global face_landmarker
+        if face_landmarker is None:
+            face_landmarker = MediapipeLandmarker()
+        img_lm478, vid_lm478 = face_landmarker.extract_lm478_from_video_name(video_name)
+        lm478 = face_landmarker.combine_vid_img_lm478_to_lm478(img_lm478, vid_lm478)
+        np.save(out_name, lm478)
+        return True
+        # print("Hahaha, solve one item!!!")
+    except Exception as e:
+        traceback.print_exc()
+        return False
+        
+def out_exist_job(vid_name):
+    out_name = vid_name.replace("/video/", "/lms_2d/").replace(".mp4","_lms.npy") 
+    if os.path.exists(out_name):
+        return None
+    else:
+        return vid_name
+    
+def get_todo_vid_names(vid_names):
+    if len(vid_names) == 1: # nerf
+        return vid_names
+    todo_vid_names = []
+    for i, res in multiprocess_run_tqdm(out_exist_job, vid_names, num_workers=128):
+        if res is not None:
+            todo_vid_names.append(res)
+    return todo_vid_names
+
+if __name__ == '__main__':
+    import argparse, glob, tqdm, random
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--vid_dir", default='nerf')
+    parser.add_argument("--ds_name", default='data/raw/videos/May.mp4')
+    parser.add_argument("--num_workers", default=2, type=int)
+    parser.add_argument("--process_id", default=0, type=int)
+    parser.add_argument("--total_process", default=1, type=int)
+    parser.add_argument("--reset", action="store_true")
+    parser.add_argument("--load_names", action="store_true")
+
+    args = parser.parse_args()
+    vid_dir = args.vid_dir
+    ds_name = args.ds_name
+    load_names = args.load_names
+
+    if ds_name.lower() == 'nerf': # 处理单个视频
+        vid_names = [vid_dir]
+        out_names = [video_name.replace("/raw/", "/processed/").replace(".mp4","/lms_2d.npy") for video_name in vid_names]
+    else: # 处理整个数据集
+        if ds_name in ['lrs3_trainval']:
+            vid_name_pattern = os.path.join(vid_dir, "*/*.mp4")
+        elif ds_name in ['TH1KH_512', 'CelebV-HQ']:
+            vid_name_pattern = os.path.join(vid_dir, "*.mp4")
+        elif ds_name in ['lrs2', 'lrs3', 'voxceleb2', 'CMLR']:
+            vid_name_pattern = os.path.join(vid_dir, "*/*/*.mp4")
+        elif ds_name in ["RAVDESS", 'VFHQ']:
+            vid_name_pattern = os.path.join(vid_dir, "*/*/*/*.mp4")
+        else:
+            raise NotImplementedError()
+        
+        vid_names_path = os.path.join(vid_dir, "vid_names.pkl")
+        if os.path.exists(vid_names_path) and load_names:
+            print(f"loading vid names from {vid_names_path}")
+            vid_names = load_file(vid_names_path)
+        else:
+            vid_names = multiprocess_glob(vid_name_pattern)
+        vid_names = sorted(vid_names)
+        if not load_names:
+            print(f"saving vid names to {vid_names_path}")
+            save_file(vid_names_path, vid_names)
+        out_names = [video_name.replace("/video/", "/lms_2d/").replace(".mp4","_lms.npy") for video_name in vid_names]
+
+    process_id = args.process_id
+    total_process = args.total_process
+    if total_process > 1:
+        assert process_id <= total_process -1
+        num_samples_per_process = len(vid_names) // total_process
+        if process_id == total_process:
+            vid_names = vid_names[process_id * num_samples_per_process : ]
+        else:
+            vid_names = vid_names[process_id * num_samples_per_process : (process_id+1) * num_samples_per_process]
+    
+    if not args.reset:
+        vid_names = get_todo_vid_names(vid_names)
+    print(f"todo videos number: {len(vid_names)}")
+
+    fail_cnt = 0
+    job_args = [(vid_name, ds_name=='nerf') for vid_name in vid_names]
+    for (i, res) in multiprocess_run_tqdm(extract_landmark_job, job_args, num_workers=args.num_workers, desc=f"Root {args.process_id}: extracing MP-based landmark2d"): 
+        if res is False:
+            fail_cnt += 1
+        print(f"finished {i + 1} / {len(vid_names)} = {(i + 1) / len(vid_names):.4f}, failed {fail_cnt} / {i + 1} = {fail_cnt / (i + 1):.4f}")
+        sys.stdout.flush()
+        pass

data_gen/utils/process_video/extract_segment_imgs.py

@@ -0,0 +1,494 @@
+import os
+os.environ["OMP_NUM_THREADS"] = "1"
+import random
+import glob
+import cv2
+import tqdm
+import numpy as np
+from typing import Union
+from utils.commons.tensor_utils import convert_to_np
+from utils.commons.os_utils import multiprocess_glob
+import pickle
+import traceback
+import multiprocessing
+from utils.commons.multiprocess_utils import multiprocess_run_tqdm
+from scipy.ndimage import binary_erosion, binary_dilation
+from sklearn.neighbors import NearestNeighbors
+from mediapipe.tasks.python import vision
+from data_gen.utils.mp_feature_extractors.mp_segmenter import MediapipeSegmenter, encode_segmap_mask_to_image, decode_segmap_mask_from_image, job_cal_seg_map_for_image
+
+seg_model   = None
+segmenter   = None
+mat_model   = None
+lama_model  = None
+lama_config = None
+
+from data_gen.utils.process_video.split_video_to_imgs import extract_img_job
+
+BG_NAME_MAP = {
+    "knn": "",
+}
+FRAME_SELECT_INTERVAL = 5
+SIM_METHOD = "mse"
+SIM_THRESHOLD = 3
+
+def save_file(name, content):
+    with open(name, "wb") as f:
+        pickle.dump(content, f) 
+        
+def load_file(name):
+    with open(name, "rb") as f:
+        content = pickle.load(f)
+    return content
+
+def save_rgb_alpha_image_to_path(img, alpha, img_path):
+    try: os.makedirs(os.path.dirname(img_path), exist_ok=True)
+    except: pass
+    cv2.imwrite(img_path, np.concatenate([cv2.cvtColor(img, cv2.COLOR_RGB2BGR), alpha], axis=-1))
+
+def save_rgb_image_to_path(img, img_path):
+    try: os.makedirs(os.path.dirname(img_path), exist_ok=True)
+    except: pass
+    cv2.imwrite(img_path, cv2.cvtColor(img, cv2.COLOR_RGB2BGR))
+
+def load_rgb_image_to_path(img_path):
+    return cv2.cvtColor(cv2.imread(img_path), cv2.COLOR_BGR2RGB)
+
+def image_similarity(x: np.ndarray, y: np.ndarray, method="mse"):
+    if method == "mse":
+        return np.mean((x - y) ** 2)
+    else:
+        raise NotImplementedError
+
+def extract_background(img_lst, segmap_mask_lst=None, method="knn", device='cpu', mix_bg=True):
+    """
+    img_lst: list of rgb ndarray
+    method: "knn"
+    """
+    global segmenter
+    global seg_model
+    global mat_model
+    global lama_model
+    global lama_config
+    
+    assert len(img_lst) > 0
+    if segmap_mask_lst is not None:
+        assert len(segmap_mask_lst) == len(img_lst)
+    else:
+        del segmenter
+        del seg_model
+        seg_model = MediapipeSegmenter()
+        segmenter = vision.ImageSegmenter.create_from_options(seg_model.video_options)
+        
+    def get_segmap_mask(img_lst, segmap_mask_lst, index):
+        if segmap_mask_lst is not None:
+            segmap = refresh_segment_mask(segmap_mask_lst[index])
+        else:
+            segmap = seg_model._cal_seg_map(refresh_image(img_lst[index]), segmenter=segmenter)
+        return segmap
+        
+    if method == "knn":
+        num_frames = len(img_lst)
+        if num_frames < 100:
+            FRAME_SELECT_INTERVAL = 5
+        elif num_frames < 10000:
+            FRAME_SELECT_INTERVAL = 20
+        else:
+            FRAME_SELECT_INTERVAL = num_frames // 500
+
+        img_lst = img_lst[::FRAME_SELECT_INTERVAL] if num_frames > FRAME_SELECT_INTERVAL else img_lst[0:1]
+            
+        if segmap_mask_lst is not None:
+            segmap_mask_lst = segmap_mask_lst[::FRAME_SELECT_INTERVAL] if num_frames > FRAME_SELECT_INTERVAL else segmap_mask_lst[0:1]
+            assert len(img_lst) == len(segmap_mask_lst)
+        # get H/W
+        h, w = refresh_image(img_lst[0]).shape[:2]
+
+        # nearest neighbors
+        all_xys = np.mgrid[0:h, 0:w].reshape(2, -1).transpose() # [512*512, 2] coordinate grid
+        distss = []
+        for idx, img in tqdm.tqdm(enumerate(img_lst), desc='combining backgrounds...', total=len(img_lst)):
+            segmap = get_segmap_mask(img_lst=img_lst, segmap_mask_lst=segmap_mask_lst, index=idx)
+            bg = (segmap[0]).astype(bool) # [h,w] bool mask
+            fg_xys = np.stack(np.nonzero(~bg)).transpose(1, 0) # [N_nonbg,2] coordinate of non-bg pixels
+            nbrs = NearestNeighbors(n_neighbors=1, algorithm='kd_tree').fit(fg_xys)
+            dists, _ = nbrs.kneighbors(all_xys) # [512*512, 1] distance to nearest non-bg pixel
+            distss.append(dists)
+
+        distss = np.stack(distss) # [B, 512*512, 1]
+        max_dist = np.max(distss, 0) # [512*512, 1]
+        max_id = np.argmax(distss, 0) # id of frame
+
+        bc_pixs = max_dist > 10 # 在各个frame有一个出现过是bg的pixel，bg标准是离最近的non-bg pixel距离大于10
+        bc_pixs_id = np.nonzero(bc_pixs)
+        bc_ids = max_id[bc_pixs]
+
+        # TODO: maybe we should reimplement here to avoid memory costs?
+        # though there is upper limits of images here
+        num_pixs = distss.shape[1]
+        bg_img = np.zeros((h*w, 3), dtype=np.uint8)
+        img_lst = [refresh_image(img) for img in img_lst] 
+        imgs = np.stack(img_lst).reshape(-1, num_pixs, 3) 
+        bg_img[bc_pixs_id, :] = imgs[bc_ids, bc_pixs_id, :] # 对那些铁bg的pixel，直接去对应的image里面采样
+        bg_img = bg_img.reshape(h, w, 3)
+
+        max_dist = max_dist.reshape(h, w)
+        bc_pixs = max_dist > 10 # 5
+        bg_xys = np.stack(np.nonzero(~bc_pixs)).transpose()
+        fg_xys = np.stack(np.nonzero(bc_pixs)).transpose()
+        nbrs = NearestNeighbors(n_neighbors=1, algorithm='kd_tree').fit(fg_xys)
+        distances, indices = nbrs.kneighbors(bg_xys) # 对non-bg img，用KNN找最近的bg pixel
+        bg_fg_xys = fg_xys[indices[:, 0]]
+        bg_img[bg_xys[:, 0], bg_xys[:, 1], :] = bg_img[bg_fg_xys[:, 0], bg_fg_xys[:, 1], :]
+    else:
+        raise NotImplementedError # deperated
+    
+    return bg_img
+
+def inpaint_torso_job(gt_img, segmap):
+    bg_part = (segmap[0]).astype(bool)
+    head_part = (segmap[1] + segmap[3] + segmap[5]).astype(bool)
+    neck_part = (segmap[2]).astype(bool)
+    torso_part = (segmap[4]).astype(bool) 
+    img = gt_img.copy()
+    img[head_part] = 0
+
+    # torso part "vertical" in-painting...
+    L = 8 + 1
+    torso_coords = np.stack(np.nonzero(torso_part), axis=-1) # [M, 2]
+    # lexsort: sort 2D coords first by y then by x, 
+    # ref: https://stackoverflow.com/questions/2706605/sorting-a-2d-numpy-array-by-multiple-axes
+    inds = np.lexsort((torso_coords[:, 0], torso_coords[:, 1]))
+    torso_coords = torso_coords[inds]
+    # choose the top pixel for each column
+    u, uid, ucnt = np.unique(torso_coords[:, 1], return_index=True, return_counts=True)
+    top_torso_coords = torso_coords[uid] # [m, 2]
+    # only keep top-is-head pixels
+    top_torso_coords_up = top_torso_coords.copy() - np.array([1, 0]) # [N, 2]
+    mask = head_part[tuple(top_torso_coords_up.T)] 
+    if mask.any():
+        top_torso_coords = top_torso_coords[mask]
+        # get the color
+        top_torso_colors = gt_img[tuple(top_torso_coords.T)] # [m, 3]
+        # construct inpaint coords (vertically up, or minus in x)
+        inpaint_torso_coords = top_torso_coords[None].repeat(L, 0) # [L, m, 2]
+        inpaint_offsets = np.stack([-np.arange(L), np.zeros(L, dtype=np.int32)], axis=-1)[:, None] # [L, 1, 2]
+        inpaint_torso_coords += inpaint_offsets
+        inpaint_torso_coords = inpaint_torso_coords.reshape(-1, 2) # [Lm, 2]
+        inpaint_torso_colors = top_torso_colors[None].repeat(L, 0) # [L, m, 3]
+        darken_scaler = 0.98 ** np.arange(L).reshape(L, 1, 1) # [L, 1, 1]
+        inpaint_torso_colors = (inpaint_torso_colors * darken_scaler).reshape(-1, 3) # [Lm, 3]
+        # set color
+        img[tuple(inpaint_torso_coords.T)] = inpaint_torso_colors
+        inpaint_torso_mask = np.zeros_like(img[..., 0]).astype(bool)
+        inpaint_torso_mask[tuple(inpaint_torso_coords.T)] = True
+    else:
+        inpaint_torso_mask = None
+    
+    # neck part "vertical" in-painting...
+    push_down = 4
+    L = 48 + push_down + 1
+    neck_part = binary_dilation(neck_part, structure=np.array([[0, 1, 0], [0, 1, 0], [0, 1, 0]], dtype=bool), iterations=3)
+    neck_coords = np.stack(np.nonzero(neck_part), axis=-1) # [M, 2]
+    # lexsort: sort 2D coords first by y then by x, 
+    # ref: https://stackoverflow.com/questions/2706605/sorting-a-2d-numpy-array-by-multiple-axes
+    inds = np.lexsort((neck_coords[:, 0], neck_coords[:, 1]))
+    neck_coords = neck_coords[inds]
+    # choose the top pixel for each column
+    u, uid, ucnt = np.unique(neck_coords[:, 1], return_index=True, return_counts=True)
+    top_neck_coords = neck_coords[uid] # [m, 2]
+    # only keep top-is-head pixels
+    top_neck_coords_up = top_neck_coords.copy() - np.array([1, 0])
+    mask = head_part[tuple(top_neck_coords_up.T)] 
+    top_neck_coords = top_neck_coords[mask]
+    # push these top down for 4 pixels to make the neck inpainting more natural...
+    offset_down = np.minimum(ucnt[mask] - 1, push_down)
+    top_neck_coords += np.stack([offset_down, np.zeros_like(offset_down)], axis=-1)
+    # get the color
+    top_neck_colors = gt_img[tuple(top_neck_coords.T)] # [m, 3]
+    # construct inpaint coords (vertically up, or minus in x)
+    inpaint_neck_coords = top_neck_coords[None].repeat(L, 0) # [L, m, 2]
+    inpaint_offsets = np.stack([-np.arange(L), np.zeros(L, dtype=np.int32)], axis=-1)[:, None] # [L, 1, 2]
+    inpaint_neck_coords += inpaint_offsets
+    inpaint_neck_coords = inpaint_neck_coords.reshape(-1, 2) # [Lm, 2]
+    inpaint_neck_colors = top_neck_colors[None].repeat(L, 0) # [L, m, 3]
+    darken_scaler = 0.98 ** np.arange(L).reshape(L, 1, 1) # [L, 1, 1]
+    inpaint_neck_colors = (inpaint_neck_colors * darken_scaler).reshape(-1, 3) # [Lm, 3]
+    # set color
+    img[tuple(inpaint_neck_coords.T)] = inpaint_neck_colors
+    # apply blurring to the inpaint area to avoid vertical-line artifects...
+    inpaint_mask = np.zeros_like(img[..., 0]).astype(bool)
+    inpaint_mask[tuple(inpaint_neck_coords.T)] = True
+
+    blur_img = img.copy()
+    blur_img = cv2.GaussianBlur(blur_img, (5, 5), cv2.BORDER_DEFAULT)
+    img[inpaint_mask] = blur_img[inpaint_mask]
+
+    # set mask
+    torso_img_mask = (neck_part | torso_part | inpaint_mask)
+    torso_with_bg_img_mask = (bg_part | neck_part | torso_part | inpaint_mask)
+    if inpaint_torso_mask is not None:
+        torso_img_mask = torso_img_mask | inpaint_torso_mask
+        torso_with_bg_img_mask = torso_with_bg_img_mask | inpaint_torso_mask
+    
+    torso_img = img.copy()
+    torso_img[~torso_img_mask] = 0
+    torso_with_bg_img = img.copy()
+    torso_img[~torso_with_bg_img_mask] = 0
+
+    return torso_img, torso_img_mask, torso_with_bg_img, torso_with_bg_img_mask
+
+def load_segment_mask_from_file(filename: str):
+    encoded_segmap = load_rgb_image_to_path(filename)
+    segmap_mask = decode_segmap_mask_from_image(encoded_segmap)
+    return segmap_mask
+
+# load segment mask to memory if not loaded yet
+def refresh_segment_mask(segmap_mask: Union[str, np.ndarray]):
+    if isinstance(segmap_mask, str):
+        segmap_mask = load_segment_mask_from_file(segmap_mask)
+    return segmap_mask
+
+# load segment mask to memory if not loaded yet
+def refresh_image(image: Union[str, np.ndarray]):
+    if isinstance(image, str):
+        image = load_rgb_image_to_path(image)
+    return image
+
+def generate_segment_imgs_job(img_name, segmap, img):
+    out_img_name = segmap_name = img_name.replace("/gt_imgs/", "/segmaps/").replace(".jpg", ".png") # 存成jpg的话，pixel value会有误差
+    try: os.makedirs(os.path.dirname(out_img_name), exist_ok=True)
+    except: pass
+    encoded_segmap = encode_segmap_mask_to_image(segmap)
+    save_rgb_image_to_path(encoded_segmap, out_img_name)
+
+    for mode in ['head', 'torso', 'person', 'bg']:
+        out_img, mask = seg_model._seg_out_img_with_segmap(img, segmap, mode=mode)
+        img_alpha = 255 * np.ones((img.shape[0], img.shape[1], 1), dtype=np.uint8) # alpha
+        mask = mask[0][..., None]
+        img_alpha[~mask] = 0
+        out_img_name = img_name.replace("/gt_imgs/", f"/{mode}_imgs/").replace(".jpg", ".png")
+        save_rgb_alpha_image_to_path(out_img, img_alpha, out_img_name)
+    
+    inpaint_torso_img, inpaint_torso_img_mask, inpaint_torso_with_bg_img, inpaint_torso_with_bg_img_mask = inpaint_torso_job(img, segmap)
+    img_alpha = 255 * np.ones((img.shape[0], img.shape[1], 1), dtype=np.uint8) # alpha
+    img_alpha[~inpaint_torso_img_mask[..., None]] = 0
+    out_img_name = img_name.replace("/gt_imgs/", f"/inpaint_torso_imgs/").replace(".jpg", ".png")
+    save_rgb_alpha_image_to_path(inpaint_torso_img, img_alpha, out_img_name)
+    return segmap_name
+    
+def segment_and_generate_for_image_job(img_name, img, segmenter_options=None, segmenter=None, store_in_memory=False):
+    img = refresh_image(img)
+    segmap_mask, segmap_image = job_cal_seg_map_for_image(img, segmenter_options=segmenter_options, segmenter=segmenter)
+    segmap_name = generate_segment_imgs_job(img_name=img_name, segmap=segmap_mask, img=img)
+    if store_in_memory:
+        return segmap_mask
+    else:
+        return segmap_name
+    
+def extract_segment_job(
+    video_name, 
+    nerf=False, 
+    background_method='knn', 
+    device="cpu",
+    total_gpus=0, 
+    mix_bg=True,
+    store_in_memory=False, # set to True to speed up a bit of preprocess, but leads to HUGE memory costs (100GB for 5-min video)
+    force_single_process=False, # turn this on if you find multi-process does not work on your environment
+):
+    global segmenter
+    global seg_model
+    del segmenter
+    del seg_model
+    seg_model = MediapipeSegmenter()
+    segmenter = vision.ImageSegmenter.create_from_options(seg_model.options)
+    # nerf means that we extract only one video, so can enable multi-process acceleration
+    multiprocess_enable = nerf and not force_single_process 
+    try:
+        if "cuda" in device:
+            # determine which cuda index from subprocess id
+            pname = multiprocessing.current_process().name
+            pid = int(pname.rsplit("-", 1)[-1]) - 1
+            cuda_id = pid % total_gpus
+            device = f"cuda:{cuda_id}"
+
+        if nerf: # single video
+            raw_img_dir = video_name.replace(".mp4", "/gt_imgs/").replace("/raw/","/processed/")
+        else: # whole dataset
+            raw_img_dir = video_name.replace(".mp4", "").replace("/video/", "/gt_imgs/")
+        if not os.path.exists(raw_img_dir):
+            extract_img_job(video_name, raw_img_dir) # use ffmpeg to split video into imgs
+        
+        img_names = glob.glob(os.path.join(raw_img_dir, "*.jpg"))
+
+        img_lst = []
+
+        for img_name in img_names:
+            if store_in_memory:
+                img = load_rgb_image_to_path(img_name)
+            else:
+                img = img_name
+            img_lst.append(img)
+
+        print("| Extracting Segmaps && Saving...")
+        args = []
+        segmap_mask_lst = []
+        # preparing parameters for segment
+        for i in range(len(img_lst)):
+            img_name = img_names[i]
+            img = img_lst[i]
+            if multiprocess_enable: # create seg_model in subprocesses here
+                options = seg_model.options
+                segmenter_arg = None
+            else: # use seg_model of this process
+                options = None
+                segmenter_arg = segmenter
+            arg = (img_name, img, options, segmenter_arg, store_in_memory)
+            args.append(arg)
+            
+        if multiprocess_enable:
+            for (_, res) in multiprocess_run_tqdm(segment_and_generate_for_image_job, args=args, num_workers=16, desc='generating segment images in multi-processes...'):
+                segmap_mask = res
+                segmap_mask_lst.append(segmap_mask)
+        else:
+            for index in tqdm.tqdm(range(len(img_lst)), desc="generating segment images in single-process..."):
+                segmap_mask = segment_and_generate_for_image_job(*args[index])
+                segmap_mask_lst.append(segmap_mask)
+        print("| Extracted Segmaps Done.")
+        
+        print("| Extracting background...")
+        bg_prefix_name = f"bg{BG_NAME_MAP[background_method]}"
+        bg_img = extract_background(img_lst, segmap_mask_lst, method=background_method, device=device, mix_bg=mix_bg)
+        if nerf:
+            out_img_name = video_name.replace("/raw/", "/processed/").replace(".mp4", f"/{bg_prefix_name}.jpg")
+        else:
+            out_img_name = video_name.replace("/video/", f"/{bg_prefix_name}_img/").replace(".mp4", ".jpg")
+        save_rgb_image_to_path(bg_img, out_img_name)
+        print("| Extracted background done.")
+        
+        print("| Extracting com_imgs...")
+        com_prefix_name = f"com{BG_NAME_MAP[background_method]}"
+        for i in tqdm.trange(len(img_names), desc='extracting com_imgs'):
+            img_name = img_names[i]
+            com_img = refresh_image(img_lst[i]).copy()
+            segmap = refresh_segment_mask(segmap_mask_lst[i])
+            bg_part = segmap[0].astype(bool)[..., None].repeat(3,axis=-1)
+            com_img[bg_part] = bg_img[bg_part]
+            out_img_name = img_name.replace("/gt_imgs/", f"/{com_prefix_name}_imgs/")
+            save_rgb_image_to_path(com_img, out_img_name)
+        print("| Extracted com_imgs done.")
+        
+        return 0
+    except Exception as e:
+        print(str(type(e)), e)
+        traceback.print_exc(e)
+        return 1
+
+def out_exist_job(vid_name, background_method='knn'):
+    com_prefix_name = f"com{BG_NAME_MAP[background_method]}"
+    img_dir = vid_name.replace("/video/", "/gt_imgs/").replace(".mp4", "")
+    out_dir1 = img_dir.replace("/gt_imgs/", "/head_imgs/")
+    out_dir2 = img_dir.replace("/gt_imgs/", f"/{com_prefix_name}_imgs/")
+    
+    if os.path.exists(img_dir) and os.path.exists(out_dir1) and os.path.exists(out_dir1) and os.path.exists(out_dir2) :
+        num_frames = len(os.listdir(img_dir))
+        if len(os.listdir(out_dir1)) == num_frames and len(os.listdir(out_dir2)) == num_frames:
+            return None
+        else:
+            return vid_name
+    else:
+        return vid_name
+
+def get_todo_vid_names(vid_names, background_method='knn'):
+    if len(vid_names) == 1: # nerf
+        return vid_names
+    todo_vid_names = []
+    fn_args = [(vid_name, background_method) for vid_name in vid_names]
+    for i, res in multiprocess_run_tqdm(out_exist_job, fn_args, num_workers=16, desc="checking todo videos..."):
+        if res is not None:
+            todo_vid_names.append(res)
+    return todo_vid_names
+
+if __name__ == '__main__':
+    import argparse, glob, tqdm, random
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--vid_dir", default='/home/tiger/datasets/raw/TH1KH_512/video')
+    parser.add_argument("--ds_name", default='TH1KH_512')
+    parser.add_argument("--num_workers", default=48, type=int)
+    parser.add_argument("--seed", default=0, type=int)
+    parser.add_argument("--process_id", default=0, type=int)
+    parser.add_argument("--total_process", default=1, type=int)
+    parser.add_argument("--reset", action='store_true')
+    parser.add_argument("--load_names", action="store_true")
+    parser.add_argument("--background_method", choices=['knn', 'mat', 'ddnm', 'lama'], type=str, default='knn')
+    parser.add_argument("--total_gpus", default=0, type=int) # zero gpus means utilizing cpu
+    parser.add_argument("--no_mix_bg", action="store_true")
+    parser.add_argument("--store_in_memory", action="store_true") # set to True to speed up preprocess, but leads to high memory costs
+    parser.add_argument("--force_single_process", action="store_true") # turn this on if you find multi-process does not work on your environment
+
+    args = parser.parse_args()
+    vid_dir = args.vid_dir
+    ds_name = args.ds_name
+    load_names = args.load_names
+    background_method = args.background_method
+    total_gpus = args.total_gpus
+    mix_bg = not args.no_mix_bg
+    store_in_memory = args.store_in_memory
+    force_single_process = args.force_single_process
+
+    devices = os.environ.get('CUDA_VISIBLE_DEVICES', '').split(",")
+    for d in devices[:total_gpus]:
+        os.system(f'pkill -f "voidgpu{d}"')
+        
+    if ds_name.lower() == 'nerf': # 处理单个视频
+        vid_names = [vid_dir]
+        out_names = [video_name.replace("/raw/", "/processed/").replace(".mp4","_lms.npy") for video_name in vid_names]
+    else: # 处理整个数据集
+        if ds_name in ['lrs3_trainval']:
+            vid_name_pattern = os.path.join(vid_dir, "*/*.mp4")
+        elif ds_name in ['TH1KH_512', 'CelebV-HQ']:
+            vid_name_pattern = os.path.join(vid_dir, "*.mp4")
+        elif ds_name in ['lrs2', 'lrs3', 'voxceleb2']:
+            vid_name_pattern = os.path.join(vid_dir, "*/*/*.mp4")
+        elif ds_name in ["RAVDESS", 'VFHQ']:
+            vid_name_pattern = os.path.join(vid_dir, "*/*/*/*.mp4")
+        else:
+            raise NotImplementedError()
+        
+        vid_names_path = os.path.join(vid_dir, "vid_names.pkl")
+        if os.path.exists(vid_names_path) and load_names:
+            print(f"loading vid names from {vid_names_path}")
+            vid_names = load_file(vid_names_path)
+        else:
+            vid_names = multiprocess_glob(vid_name_pattern)
+        vid_names = sorted(vid_names)
+        print(f"saving vid names to {vid_names_path}")
+        save_file(vid_names_path, vid_names)
+
+    vid_names = sorted(vid_names)
+    random.seed(args.seed)
+    random.shuffle(vid_names)
+
+    process_id = args.process_id
+    total_process = args.total_process
+    if total_process > 1:
+        assert process_id <= total_process -1
+        num_samples_per_process = len(vid_names) // total_process
+        if process_id == total_process:
+            vid_names = vid_names[process_id * num_samples_per_process : ]
+        else:
+            vid_names = vid_names[process_id * num_samples_per_process : (process_id+1) * num_samples_per_process]
+    
+    if not args.reset:
+        vid_names = get_todo_vid_names(vid_names, background_method)
+    print(f"todo videos number: {len(vid_names)}")
+
+    device = "cuda" if total_gpus > 0 else "cpu"
+    extract_job = extract_segment_job
+    fn_args = [(vid_name, ds_name=='nerf', background_method, device, total_gpus, mix_bg, store_in_memory, force_single_process) for i, vid_name in enumerate(vid_names)]
+        
+    if ds_name == 'nerf': # 处理单个视频
+        extract_job(*fn_args[0])
+    else:
+        for vid_name in multiprocess_run_tqdm(extract_job, fn_args, desc=f"Root process {args.process_id}:  segment images", num_workers=args.num_workers):
+            pass

data_gen/utils/process_video/fit_3dmm_landmark.py

@@ -0,0 +1,565 @@
+# This is a script for efficienct 3DMM coefficient extraction.
+# It could reconstruct accurate 3D face in real-time.
+# It is built upon BFM 2009 model and mediapipe landmark extractor.
+# It is authored by ZhenhuiYe ([email protected]), free to contact him for any suggestion on improvement!
+
+from numpy.core.numeric import require
+from numpy.lib.function_base import quantile
+import torch
+import torch.nn.functional as F
+import copy
+import numpy as np
+
+import random
+import pickle
+import os
+import sys
+import cv2
+import argparse
+import tqdm
+from utils.commons.multiprocess_utils import multiprocess_run_tqdm
+from data_gen.utils.mp_feature_extractors.face_landmarker import MediapipeLandmarker, read_video_to_frames
+from deep_3drecon.deep_3drecon_models.bfm import ParametricFaceModel
+from deep_3drecon.secc_renderer import SECC_Renderer
+from utils.commons.os_utils import multiprocess_glob
+
+
+face_model = ParametricFaceModel(bfm_folder='deep_3drecon/BFM', 
+            camera_distance=10, focal=1015, keypoint_mode='mediapipe')
+face_model.to(torch.device("cuda:0"))
+
+dir_path = os.path.dirname(os.path.realpath(__file__))
+
+
+def draw_axes(img, pitch, yaw, roll, tx, ty, size=50):
+    # yaw = -yaw
+    pitch = - pitch
+    roll = - roll
+    rotation_matrix = cv2.Rodrigues(np.array([pitch, yaw, roll]))[0].astype(np.float64)
+    axes_points = np.array([
+        [1, 0, 0, 0],
+        [0, 1, 0, 0],
+        [0, 0, 1, 0]
+    ], dtype=np.float64)
+    axes_points = rotation_matrix @ axes_points
+    axes_points = (axes_points[:2, :] * size).astype(int)
+    axes_points[0, :] = axes_points[0, :] + tx
+    axes_points[1, :] = axes_points[1, :] + ty
+    
+    new_img = img.copy()
+    cv2.line(new_img, tuple(axes_points[:, 3].ravel()), tuple(axes_points[:, 0].ravel()), (255, 0, 0), 3)    
+    cv2.line(new_img, tuple(axes_points[:, 3].ravel()), tuple(axes_points[:, 1].ravel()), (0, 255, 0), 3)    
+    cv2.line(new_img, tuple(axes_points[:, 3].ravel()), tuple(axes_points[:, 2].ravel()), (0, 0, 255), 3)
+    return new_img
+
+def save_file(name, content):
+    with open(name, "wb") as f:
+        pickle.dump(content, f) 
+        
+def load_file(name):
+    with open(name, "rb") as f:
+        content = pickle.load(f)
+    return content
+
+def cal_lap_loss(in_tensor):
+    # [T, 68, 2]
+    t = in_tensor.shape[0]
+    in_tensor = in_tensor.reshape([t, -1]).permute(1,0).unsqueeze(1) # [c, 1, t]
+    in_tensor = torch.cat([in_tensor[:, :, 0:1], in_tensor, in_tensor[:, :, -1:]], dim=-1)
+    lap_kernel = torch.Tensor((-0.5, 1.0, -0.5)).reshape([1,1,3]).float().to(in_tensor.device) # [1, 1, kw]
+    loss_lap = 0
+
+    out_tensor = F.conv1d(in_tensor, lap_kernel)
+    loss_lap += torch.mean(out_tensor**2)
+    return loss_lap
+
+def cal_vel_loss(ldm):
+    # [B, 68, 2]
+    vel = ldm[1:] - ldm[:-1]
+    return torch.mean(torch.abs(vel))
+
+def cal_lan_loss(proj_lan, gt_lan):
+    # [B, 68, 2]
+    loss = (proj_lan - gt_lan)** 2
+    # use the ldm weights from deep3drecon, see deep_3drecon/deep_3drecon_models/losses.py
+    weights = torch.zeros_like(loss)
+    weights = torch.ones_like(loss)
+    weights[:, 36:48, :] = 3 # eye 12 points
+    weights[:, -8:, :] =  3 # inner lip 8 points
+    weights[:, 28:31, :] =  3 # nose 3 points
+    loss = loss * weights
+    return torch.mean(loss)
+
+def cal_lan_loss_mp(proj_lan, gt_lan, mean:bool=True):
+    # [B, 68, 2]
+    loss = (proj_lan - gt_lan).pow(2)
+    # loss = (proj_lan - gt_lan).abs()
+    unmatch_mask = [ 93, 127, 132, 234, 323, 356, 361, 454]
+    upper_eye = [161,160,159,158,157] + [388,387,386,385,384]
+    eye = [33,246,161,160,159,158,157,173,133,155,154,153,145,144,163,7] + [263,466,388,387,386,385,384,398,362,382,381,380,374,373,390,249]
+    inner_lip = [78,191,80,81,82,13,312,311,310,415,308,324,318,402,317,14,87,178,88,95]
+    outer_lip = [61,185,40,39,37,0,267,269,270,409,291,375,321,405,314,17,84,181,91,146]
+    weights = torch.ones_like(loss)
+    weights[:, eye] = 3
+    weights[:, upper_eye] = 20
+    weights[:, inner_lip] = 5
+    weights[:, outer_lip] = 5
+    weights[:, unmatch_mask] = 0
+    loss = loss * weights
+    if mean:
+        loss = torch.mean(loss)
+    return loss
+
+def cal_acceleration_loss(trans):
+    vel = trans[1:] - trans[:-1]
+    acc = vel[1:] - vel[:-1]
+    return torch.mean(torch.abs(acc))
+
+def cal_acceleration_ldm_loss(ldm):
+    # [B, 68, 2]
+    vel = ldm[1:] - ldm[:-1]
+    acc = vel[1:] - vel[:-1]
+    lip_weight = 0.25 # we dont want smooth the lip too much
+    acc[48:68] *= lip_weight
+    return torch.mean(torch.abs(acc))
+ 
+def set_requires_grad(tensor_list):
+    for tensor in tensor_list:
+        tensor.requires_grad = True
+
+@torch.enable_grad()
+def fit_3dmm_for_a_video(
+    video_name, 
+    nerf=False, # use the file name convention for GeneFace++
+    id_mode='global', 
+    debug=False, 
+    keypoint_mode='mediapipe',
+    large_yaw_threshold=9999999.9,
+    save=True
+) -> bool: # True: good, False: bad 
+    assert video_name.endswith(".mp4"), "this function only support video as input"
+    if id_mode == 'global':
+        LAMBDA_REG_ID = 0.2
+        LAMBDA_REG_EXP = 0.6
+        LAMBDA_REG_LAP = 1.0
+        LAMBDA_REG_VEL_ID = 0.0 # laplcaian is all you need for temporal consistency
+        LAMBDA_REG_VEL_EXP = 0.0 # laplcaian is all you need for temporal consistency
+    else:
+        LAMBDA_REG_ID = 0.3
+        LAMBDA_REG_EXP = 0.05
+        LAMBDA_REG_LAP = 1.0
+        LAMBDA_REG_VEL_ID = 0.0 # laplcaian is all you need for temporal consistency
+        LAMBDA_REG_VEL_EXP = 0.0 # laplcaian is all you need for temporal consistency
+
+    frames = read_video_to_frames(video_name) # [T, H, W, 3]
+    img_h, img_w = frames.shape[1], frames.shape[2]
+    assert img_h == img_w
+    num_frames = len(frames)
+
+    if nerf: # single video
+        lm_name = video_name.replace("/raw/", "/processed/").replace(".mp4","/lms_2d.npy")
+    else:
+        lm_name = video_name.replace("/video/", "/lms_2d/").replace(".mp4", "_lms.npy")
+
+    if os.path.exists(lm_name):
+        lms = np.load(lm_name)
+    else:
+        print(f"lms_2d file not found, try to extract it from video... {lm_name}")
+        try:
+            landmarker = MediapipeLandmarker()
+            img_lm478, vid_lm478 = landmarker.extract_lm478_from_frames(frames, anti_smooth_factor=20)
+            lms = landmarker.combine_vid_img_lm478_to_lm478(img_lm478, vid_lm478)
+        except Exception as e:
+            print(e)
+            return False
+        if lms is None:
+            print(f"get None lms_2d, please check whether each frame has one head, exiting... {lm_name}")
+            return False
+    lms = lms[:, :468, :]
+    lms = torch.FloatTensor(lms).cuda()
+    lms[..., 1] = img_h - lms[..., 1] # flip the height axis
+
+    if keypoint_mode == 'mediapipe':
+        # default
+        cal_lan_loss_fn = cal_lan_loss_mp
+        if nerf: # single video
+            out_name = video_name.replace("/raw/", "/processed/").replace(".mp4", "/coeff_fit_mp.npy")
+        else:
+            out_name = video_name.replace("/video/", "/coeff_fit_mp/").replace(".mp4", "_coeff_fit_mp.npy")
+    else:
+        # lm68 is less accurate than mp
+        cal_lan_loss_fn = cal_lan_loss
+        if nerf: # single video
+            out_name = video_name.replace("/raw/", "/processed/").replace(".mp4", "_coeff_fit_lm68.npy")
+        else:
+            out_name = video_name.replace("/video/", "/coeff_fit_lm68/").replace(".mp4", "_coeff_fit_lm68.npy")
+    try:
+        os.makedirs(os.path.dirname(out_name), exist_ok=True)
+    except:
+        pass
+
+    id_dim, exp_dim = 80, 64
+    sel_ids = np.arange(0, num_frames, 40)
+
+    h = w = face_model.center * 2
+    img_scale_factor = img_h / h
+    lms /= img_scale_factor # rescale lms into [0,224]
+
+    if id_mode == 'global':
+        # default choice by GeneFace++ and later works
+        id_para = lms.new_zeros((1, id_dim), requires_grad=True)
+    elif id_mode == 'finegrained':
+        # legacy choice by GeneFace1 (ICLR 2023)
+        id_para = lms.new_zeros((num_frames, id_dim), requires_grad=True)
+    else: raise NotImplementedError(f"id mode {id_mode} not supported! we only support global or finegrained.")
+    exp_para = lms.new_zeros((num_frames, exp_dim), requires_grad=True)
+    euler_angle = lms.new_zeros((num_frames, 3), requires_grad=True)
+    trans = lms.new_zeros((num_frames, 3), requires_grad=True)
+
+    set_requires_grad([id_para, exp_para, euler_angle, trans])
+
+    optimizer_idexp = torch.optim.Adam([id_para, exp_para], lr=.1)
+    optimizer_frame = torch.optim.Adam([euler_angle, trans], lr=.1)
+
+    # 其他参数初始化，先训练euler和trans
+    for _ in range(200):
+        if id_mode == 'global':
+            proj_geo = face_model.compute_for_landmark_fit(
+                id_para.expand((num_frames, id_dim)), exp_para, euler_angle, trans)
+        else:
+            proj_geo = face_model.compute_for_landmark_fit(
+                id_para, exp_para, euler_angle, trans)
+        loss_lan = cal_lan_loss_fn(proj_geo[:, :, :2], lms.detach())
+        loss = loss_lan
+        optimizer_frame.zero_grad()
+        loss.backward()
+        optimizer_frame.step()
+
+    # print(f"loss_lan: {loss_lan.item():.2f}, euler_abs_mean: {euler_angle.abs().mean().item():.4f}, euler_std: {euler_angle.std().item():.4f}, euler_min: {euler_angle.min().item():.4f}, euler_max: {euler_angle.max().item():.4f}")
+    # print(f"trans_z_mean: {trans[...,2].mean().item():.4f}, trans_z_std: {trans[...,2].std().item():.4f}, trans_min: {trans[...,2].min().item():.4f}, trans_max: {trans[...,2].max().item():.4f}")
+
+    for param_group in optimizer_frame.param_groups:
+        param_group['lr'] = 0.1
+
+    # "jointly roughly training id exp euler trans"
+    for _ in range(200):
+        ret = {}
+        if id_mode == 'global':
+            proj_geo = face_model.compute_for_landmark_fit(
+                id_para.expand((num_frames, id_dim)), exp_para, euler_angle, trans, ret)
+        else:
+            proj_geo = face_model.compute_for_landmark_fit(
+                id_para, exp_para, euler_angle, trans, ret)
+        loss_lan = cal_lan_loss_fn(
+            proj_geo[:, :, :2], lms.detach())
+        # loss_lap = cal_lap_loss(proj_geo)
+        # laplacian对euler影响不大，但是对trans的提升很大
+        loss_lap = cal_lap_loss(id_para) + cal_lap_loss(exp_para) + cal_lap_loss(euler_angle) * 0.3 + cal_lap_loss(trans) * 0.3
+
+        loss_regid = torch.mean(id_para*id_para) # 正则化
+        loss_regexp = torch.mean(exp_para * exp_para)
+
+        loss_vel_id = cal_vel_loss(id_para)
+        loss_vel_exp = cal_vel_loss(exp_para)
+        loss = loss_lan  + loss_regid * LAMBDA_REG_ID + loss_regexp * LAMBDA_REG_EXP  + loss_vel_id * LAMBDA_REG_VEL_ID + loss_vel_exp * LAMBDA_REG_VEL_EXP + loss_lap * LAMBDA_REG_LAP
+        optimizer_idexp.zero_grad()
+        optimizer_frame.zero_grad()
+        loss.backward()
+        optimizer_idexp.step()
+        optimizer_frame.step()
+
+    # print(f"loss_lan: {loss_lan.item():.2f}, loss_reg_id: {loss_regid.item():.2f},loss_reg_exp: {loss_regexp.item():.2f},")
+    # print(f"euler_abs_mean: {euler_angle.abs().mean().item():.4f}, euler_std: {euler_angle.std().item():.4f}, euler_min: {euler_angle.min().item():.4f}, euler_max: {euler_angle.max().item():.4f}")
+    # print(f"trans_z_mean: {trans[...,2].mean().item():.4f}, trans_z_std: {trans[...,2].std().item():.4f}, trans_min: {trans[...,2].min().item():.4f}, trans_max: {trans[...,2].max().item():.4f}")
+
+    # start fine training, intialize from the roughly trained results
+    if id_mode == 'global':
+        id_para_ = lms.new_zeros((1, id_dim), requires_grad=False)
+    else:
+        id_para_ = lms.new_zeros((num_frames, id_dim), requires_grad=True)
+    id_para_.data = id_para.data.clone()
+    id_para = id_para_
+    exp_para_ = lms.new_zeros((num_frames, exp_dim), requires_grad=True)
+    exp_para_.data = exp_para.data.clone()
+    exp_para = exp_para_
+    euler_angle_ = lms.new_zeros((num_frames, 3), requires_grad=True)
+    euler_angle_.data = euler_angle.data.clone()
+    euler_angle = euler_angle_
+    trans_ = lms.new_zeros((num_frames, 3), requires_grad=True)
+    trans_.data = trans.data.clone()
+    trans = trans_
+    
+    batch_size = 50
+    # "fine fitting the 3DMM in batches"
+    for i in range(int((num_frames-1)/batch_size+1)):
+        if (i+1)*batch_size > num_frames:
+            start_n = num_frames-batch_size
+            sel_ids = np.arange(max(num_frames-batch_size,0), num_frames)
+        else:
+            start_n = i*batch_size
+            sel_ids = np.arange(i*batch_size, i*batch_size+batch_size)
+        sel_lms = lms[sel_ids]
+
+        if id_mode == 'global':
+            sel_id_para = id_para.expand((sel_ids.shape[0], id_dim))
+        else:
+            sel_id_para = id_para.new_zeros((batch_size, id_dim), requires_grad=True)
+            sel_id_para.data = id_para[sel_ids].clone()
+        sel_exp_para = exp_para.new_zeros(
+            (batch_size, exp_dim), requires_grad=True)
+        sel_exp_para.data = exp_para[sel_ids].clone()
+        sel_euler_angle = euler_angle.new_zeros(
+            (batch_size, 3), requires_grad=True)
+        sel_euler_angle.data = euler_angle[sel_ids].clone()
+        sel_trans = trans.new_zeros((batch_size, 3), requires_grad=True)
+        sel_trans.data = trans[sel_ids].clone()
+        
+        if id_mode == 'global':
+            set_requires_grad([sel_exp_para, sel_euler_angle, sel_trans])
+            optimizer_cur_batch = torch.optim.Adam(
+                [sel_exp_para, sel_euler_angle, sel_trans], lr=0.005)
+        else:
+            set_requires_grad([sel_id_para, sel_exp_para, sel_euler_angle, sel_trans])
+            optimizer_cur_batch = torch.optim.Adam(
+                [sel_id_para, sel_exp_para, sel_euler_angle, sel_trans], lr=0.005)
+
+        for j in range(50):
+            ret = {}
+            proj_geo = face_model.compute_for_landmark_fit(
+                sel_id_para, sel_exp_para, sel_euler_angle, sel_trans, ret)
+            loss_lan = cal_lan_loss_fn(
+                proj_geo[:, :, :2], lms[sel_ids].detach())
+            
+            # loss_lap = cal_lap_loss(proj_geo)
+            loss_lap = cal_lap_loss(sel_id_para) + cal_lap_loss(sel_exp_para) + cal_lap_loss(sel_euler_angle) * 0.3 + cal_lap_loss(sel_trans) * 0.3
+            loss_vel_id = cal_vel_loss(sel_id_para)
+            loss_vel_exp = cal_vel_loss(sel_exp_para)
+            log_dict = {
+                'loss_vel_id': loss_vel_id,
+                'loss_vel_exp': loss_vel_exp,
+                'loss_vel_euler': cal_vel_loss(sel_euler_angle),
+                'loss_vel_trans': cal_vel_loss(sel_trans),
+            }
+            loss_regid = torch.mean(sel_id_para*sel_id_para) # 正则化
+            loss_regexp = torch.mean(sel_exp_para*sel_exp_para)
+            loss = loss_lan + loss_regid * LAMBDA_REG_ID + loss_regexp * LAMBDA_REG_EXP + loss_lap * LAMBDA_REG_LAP + loss_vel_id * LAMBDA_REG_VEL_ID + loss_vel_exp * LAMBDA_REG_VEL_EXP
+
+            optimizer_cur_batch.zero_grad()
+            loss.backward()
+            optimizer_cur_batch.step()
+            
+        if debug:
+            print(f"batch {i} | loss_lan: {loss_lan.item():.2f}, loss_reg_id: {loss_regid.item():.2f},loss_reg_exp: {loss_regexp.item():.2f},loss_lap_ldm:{loss_lap.item():.4f}")
+            print("|--------" + ', '.join([f"{k}: {v:.4f}" for k,v in log_dict.items()]))
+        if id_mode != 'global':
+            id_para[sel_ids].data = sel_id_para.data.clone()
+        exp_para[sel_ids].data = sel_exp_para.data.clone()
+        euler_angle[sel_ids].data = sel_euler_angle.data.clone()
+        trans[sel_ids].data = sel_trans.data.clone()
+
+    coeff_dict = {'id': id_para.detach().cpu().numpy(), 'exp': exp_para.detach().cpu().numpy(),
+                'euler': euler_angle.detach().cpu().numpy(), 'trans': trans.detach().cpu().numpy()}
+
+    # filter data by side-view pose    
+    # bad_yaw = False
+    # yaws = [] # not so accurate
+    # for index in range(coeff_dict["trans"].shape[0]):
+    #     yaw = coeff_dict["euler"][index][1]
+    #     yaw = np.abs(yaw)
+    #     yaws.append(yaw)
+    #     if yaw > large_yaw_threshold:
+    #         bad_yaw = True
+    
+    if debug:
+        import imageio
+        from utils.visualization.vis_cam3d.camera_pose_visualizer import CameraPoseVisualizer
+        from data_util.face3d_helper import Face3DHelper
+        from data_gen.utils.process_video.extract_blink import get_eye_area_percent
+        face3d_helper = Face3DHelper('deep_3drecon/BFM', keypoint_mode='mediapipe')
+
+        t = coeff_dict['exp'].shape[0]
+        if len(coeff_dict['id']) == 1:
+            coeff_dict['id'] = np.repeat(coeff_dict['id'], t, axis=0)
+        idexp_lm3d = face3d_helper.reconstruct_idexp_lm3d_np(coeff_dict['id'], coeff_dict['exp']).reshape([t, -1])
+        cano_lm3d = idexp_lm3d / 10 + face3d_helper.key_mean_shape.squeeze().reshape([1, -1]).cpu().numpy()
+        cano_lm3d = cano_lm3d.reshape([t, -1, 3])
+        WH = 512
+        cano_lm3d = (cano_lm3d * WH/2 + WH/2).astype(int)
+
+        with torch.no_grad():
+            rot = ParametricFaceModel.compute_rotation(euler_angle)
+            extrinsic = torch.zeros([rot.shape[0], 4, 4]).to(rot.device)
+            extrinsic[:, :3,:3] = rot
+            extrinsic[:, :3, 3] = trans # / 10
+            extrinsic[:, 3, 3] = 1
+        extrinsic = extrinsic.cpu().numpy()
+
+        xy_camera_visualizer = CameraPoseVisualizer(xlim=[extrinsic[:,0,3].min().item()-0.5,extrinsic[:,0,3].max().item()+0.5],ylim=[extrinsic[:,1,3].min().item()-0.5,extrinsic[:,1,3].max().item()+0.5], zlim=[extrinsic[:,2,3].min().item()-0.5,extrinsic[:,2,3].max().item()+0.5], view_mode='xy')
+        xz_camera_visualizer = CameraPoseVisualizer(xlim=[extrinsic[:,0,3].min().item()-0.5,extrinsic[:,0,3].max().item()+0.5],ylim=[extrinsic[:,1,3].min().item()-0.5,extrinsic[:,1,3].max().item()+0.5], zlim=[extrinsic[:,2,3].min().item()-0.5,extrinsic[:,2,3].max().item()+0.5], view_mode='xz')
+
+        if nerf:
+            debug_name = video_name.replace("/raw/", "/processed/").replace(".mp4", "/debug_fit_3dmm.mp4")
+        else:
+            debug_name = video_name.replace("/video/", "/coeff_fit_debug/").replace(".mp4", "_debug.mp4")
+        try:
+            os.makedirs(os.path.dirname(debug_name), exist_ok=True)
+        except: pass
+        writer = imageio.get_writer(debug_name, fps=25)
+        if id_mode == 'global':
+            id_para = id_para.repeat([exp_para.shape[0], 1])
+        proj_geo = face_model.compute_for_landmark_fit(id_para, exp_para, euler_angle, trans)
+        lm68s = proj_geo[:,:,:2].detach().cpu().numpy()  # [T, 68,2]
+        lm68s = lm68s * img_scale_factor
+        lms = lms * img_scale_factor
+        lm68s[..., 1] = img_h - lm68s[..., 1] # flip the height axis
+        lms[..., 1] = img_h - lms[..., 1] # flip the height axis
+        lm68s = lm68s.astype(int)
+        for i in tqdm.trange(min(250, len(frames)), desc=f'rendering debug video to {debug_name}..'):
+            xy_cam3d_img = xy_camera_visualizer.extrinsic2pyramid(extrinsic[i], focal_len_scaled=0.25)
+            xy_cam3d_img = cv2.resize(xy_cam3d_img, (512,512))
+            xz_cam3d_img = xz_camera_visualizer.extrinsic2pyramid(extrinsic[i], focal_len_scaled=0.25)
+            xz_cam3d_img = cv2.resize(xz_cam3d_img, (512,512))
+            
+            img = copy.deepcopy(frames[i])
+            img2 = copy.deepcopy(frames[i])
+
+            img = draw_axes(img, euler_angle[i,0].item(), euler_angle[i,1].item(), euler_angle[i,2].item(), lm68s[i][4][0].item(), lm68s[i, 4][1].item(), size=50)
+
+            gt_lm_color = (255, 0, 0)
+                
+            for lm in lm68s[i]:
+                img = cv2.circle(img, lm, 1, (0, 0, 255), thickness=-1) # blue
+            for gt_lm in lms[i]:
+                img2 = cv2.circle(img2, gt_lm.cpu().numpy().astype(int), 2, gt_lm_color, thickness=1)
+            
+            cano_lm3d_img = np.ones([WH, WH, 3], dtype=np.uint8) * 255
+            for j in range(len(cano_lm3d[i])):
+                x, y, _ = cano_lm3d[i, j]
+                color = (255,0,0)
+                cano_lm3d_img = cv2.circle(cano_lm3d_img, center=(x,y), radius=3, color=color, thickness=-1)
+            cano_lm3d_img = cv2.flip(cano_lm3d_img, 0)
+
+            _, secc_img = secc_renderer(id_para[0:1], exp_para[i:i+1], euler_angle[i:i+1]*0, trans[i:i+1]*0)
+            secc_img = (secc_img +1)*127.5
+            secc_img = F.interpolate(secc_img, size=(img_h, img_w))
+            secc_img = secc_img.permute(0, 2,3,1).int().cpu().numpy()[0]
+            out_img1 = np.concatenate([img, img2, secc_img], axis=1).astype(np.uint8)
+            font = cv2.FONT_HERSHEY_SIMPLEX
+            out_img2 = np.concatenate([xy_cam3d_img, xz_cam3d_img, cano_lm3d_img], axis=1).astype(np.uint8)
+            out_img = np.concatenate([out_img1, out_img2], axis=0)
+            writer.append_data(out_img)
+        writer.close()
+        
+    # if bad_yaw:
+    #     print(f"Skip {video_name} due to TOO LARGE YAW")
+    #     return False
+
+    if save:
+        np.save(out_name, coeff_dict, allow_pickle=True) 
+    return coeff_dict
+
+def out_exist_job(vid_name):
+    out_name = vid_name.replace("/video/", "/coeff_fit_mp/").replace(".mp4","_coeff_fit_mp.npy") 
+    lms_name = vid_name.replace("/video/", "/lms_2d/").replace(".mp4","_lms.npy") 
+    if os.path.exists(out_name) or not os.path.exists(lms_name):
+        return None
+    else:
+        return vid_name
+
+def get_todo_vid_names(vid_names):
+    if len(vid_names) == 1: # single video, nerf
+        return vid_names
+    todo_vid_names = []
+    for i, res in multiprocess_run_tqdm(out_exist_job, vid_names, num_workers=16):
+        if res is not None:
+            todo_vid_names.append(res)
+    return todo_vid_names
+
+
+if __name__ == '__main__':
+    import argparse, glob, tqdm
+    parser = argparse.ArgumentParser()
+    # parser.add_argument("--vid_dir", default='/home/tiger/datasets/raw/CelebV-HQ/video')
+    parser.add_argument("--vid_dir", default='data/raw/videos/May_10s.mp4')
+    parser.add_argument("--ds_name", default='nerf') # 'nerf' | 'CelebV-HQ' | 'TH1KH_512' | etc
+    parser.add_argument("--seed", default=0, type=int)
+    parser.add_argument("--process_id", default=0, type=int)
+    parser.add_argument("--total_process", default=1, type=int)
+    parser.add_argument("--id_mode", default='global', type=str) # global | finegrained
+    parser.add_argument("--keypoint_mode", default='mediapipe', type=str)
+    parser.add_argument("--large_yaw_threshold", default=9999999.9, type=float) # could be 0.7
+    parser.add_argument("--debug", action='store_true')
+    parser.add_argument("--reset", action='store_true')
+    parser.add_argument("--load_names", action="store_true")
+
+    args = parser.parse_args()
+    vid_dir = args.vid_dir
+    ds_name = args.ds_name
+    load_names = args.load_names
+    
+    print(f"args {args}")
+    
+    if ds_name.lower() == 'nerf': # 处理单个视频
+        vid_names = [vid_dir]
+        out_names = [video_name.replace("/raw/", "/processed/").replace(".mp4","_coeff_fit_mp.npy") for video_name in vid_names]
+    else: # 处理整个数据集
+        if ds_name in ['lrs3_trainval']:
+            vid_name_pattern = os.path.join(vid_dir, "*/*.mp4")
+        elif ds_name in ['TH1KH_512', 'CelebV-HQ']:
+            vid_name_pattern = os.path.join(vid_dir, "*.mp4")
+        elif ds_name in ['lrs2', 'lrs3', 'voxceleb2', 'CMLR']:
+            vid_name_pattern = os.path.join(vid_dir, "*/*/*.mp4")
+        elif ds_name in ["RAVDESS", 'VFHQ']:
+            vid_name_pattern = os.path.join(vid_dir, "*/*/*/*.mp4")
+        else:
+            raise NotImplementedError()
+        
+        vid_names_path = os.path.join(vid_dir, "vid_names.pkl")
+        if os.path.exists(vid_names_path) and load_names:
+            print(f"loading vid names from {vid_names_path}")
+            vid_names = load_file(vid_names_path)
+        else:
+            vid_names = multiprocess_glob(vid_name_pattern)
+        vid_names = sorted(vid_names)
+        print(f"saving vid names to {vid_names_path}")
+        save_file(vid_names_path, vid_names)
+        out_names = [video_name.replace("/video/", "/coeff_fit_mp/").replace(".mp4","_coeff_fit_mp.npy") for video_name in vid_names]
+
+    print(vid_names[:10])
+    random.seed(args.seed)
+    random.shuffle(vid_names)
+
+    face_model = ParametricFaceModel(bfm_folder='deep_3drecon/BFM', 
+                camera_distance=10, focal=1015, keypoint_mode=args.keypoint_mode)
+    face_model.to(torch.device("cuda:0"))
+    secc_renderer = SECC_Renderer(512)
+    secc_renderer.to("cuda:0")
+    
+    process_id = args.process_id
+    total_process = args.total_process
+    if total_process > 1:
+        assert process_id <= total_process -1
+        num_samples_per_process = len(vid_names) // total_process
+        if process_id == total_process:
+            vid_names = vid_names[process_id * num_samples_per_process : ]
+        else:
+            vid_names = vid_names[process_id * num_samples_per_process : (process_id+1) * num_samples_per_process]
+
+    if not args.reset:
+        vid_names = get_todo_vid_names(vid_names)
+
+    failed_img_names = []
+    for i in tqdm.trange(len(vid_names), desc=f"process {process_id}: fitting 3dmm ..."):
+        img_name = vid_names[i]
+        try:
+            is_person_specific_data = ds_name=='nerf'
+            success = fit_3dmm_for_a_video(img_name, is_person_specific_data, args.id_mode, args.debug, large_yaw_threshold=args.large_yaw_threshold)
+            if not success:
+                failed_img_names.append(img_name)   
+        except Exception as e:
+            print(img_name, e)
+            failed_img_names.append(img_name)
+        print(f"finished {i + 1} / {len(vid_names)} = {(i + 1) / len(vid_names):.4f}, failed {len(failed_img_names)} / {i + 1} = {len(failed_img_names) / (i + 1):.4f}")
+        sys.stdout.flush()
+    print(f"all failed image names: {failed_img_names}")
+    print(f"All finished!")

data_gen/utils/process_video/inpaint_torso_imgs.py

@@ -0,0 +1,193 @@
+import cv2
+import os
+import numpy as np
+from utils.commons.multiprocess_utils import multiprocess_run_tqdm
+from scipy.ndimage import binary_erosion, binary_dilation
+
+from tasks.eg3ds.loss_utils.segment_loss.mp_segmenter import MediapipeSegmenter
+seg_model = MediapipeSegmenter()
+
+def inpaint_torso_job(video_name, idx=None, total=None):
+    raw_img_dir = video_name.replace(".mp4", "").replace("/video/","/gt_imgs/")
+    img_names = glob.glob(os.path.join(raw_img_dir, "*.jpg"))
+
+    for image_path in tqdm.tqdm(img_names):
+        # read ori image
+        ori_image = cv2.imread(image_path, cv2.IMREAD_UNCHANGED) # [H, W, 3]
+        segmap = seg_model._cal_seg_map(cv2.cvtColor(ori_image, cv2.COLOR_BGR2RGB))
+        head_part = (segmap[1] + segmap[3] + segmap[5]).astype(np.bool)
+        torso_part = (segmap[4]).astype(np.bool)
+        neck_part = (segmap[2]).astype(np.bool)
+        bg_part = segmap[0].astype(np.bool)
+        head_image = cv2.imread(image_path.replace("/gt_imgs/", "/head_imgs/"), cv2.IMREAD_UNCHANGED) # [H, W, 3]
+        torso_image = cv2.imread(image_path.replace("/gt_imgs/", "/torso_imgs/"), cv2.IMREAD_UNCHANGED) # [H, W, 3]
+        bg_image = cv2.imread(image_path.replace("/gt_imgs/", "/bg_imgs/"), cv2.IMREAD_UNCHANGED) # [H, W, 3]
+
+        # head_part = (head_image[...,0] != 0) & (head_image[...,1] != 0) & (head_image[...,2] != 0)
+        # torso_part = (torso_image[...,0] != 0) & (torso_image[...,1] != 0) & (torso_image[...,2] != 0)
+        # bg_part = (bg_image[...,0] != 0) & (bg_image[...,1] != 0) & (bg_image[...,2] != 0)
+
+        # get gt image
+        gt_image = ori_image.copy()
+        gt_image[bg_part] = bg_image[bg_part]
+        cv2.imwrite(image_path.replace('ori_imgs', 'gt_imgs'), gt_image)
+
+        # get torso image
+        torso_image = gt_image.copy() # rgb
+        torso_image[head_part] = 0
+        torso_alpha = 255 * np.ones((gt_image.shape[0], gt_image.shape[1], 1), dtype=np.uint8) # alpha
+        
+        # torso part "vertical" in-painting...
+        L = 8 + 1
+        torso_coords = np.stack(np.nonzero(torso_part), axis=-1) # [M, 2]
+        # lexsort: sort 2D coords first by y then by x, 
+        # ref: https://stackoverflow.com/questions/2706605/sorting-a-2d-numpy-array-by-multiple-axes
+        inds = np.lexsort((torso_coords[:, 0], torso_coords[:, 1]))
+        torso_coords = torso_coords[inds]
+        # choose the top pixel for each column
+        u, uid, ucnt = np.unique(torso_coords[:, 1], return_index=True, return_counts=True)
+        top_torso_coords = torso_coords[uid] # [m, 2]
+        # only keep top-is-head pixels
+        top_torso_coords_up = top_torso_coords.copy() - np.array([1, 0]) # [N, 2]
+        mask = head_part[tuple(top_torso_coords_up.T)] 
+        if mask.any():
+            top_torso_coords = top_torso_coords[mask]
+            # get the color
+            top_torso_colors = gt_image[tuple(top_torso_coords.T)] # [m, 3]
+            # construct inpaint coords (vertically up, or minus in x)
+            inpaint_torso_coords = top_torso_coords[None].repeat(L, 0) # [L, m, 2]
+            inpaint_offsets = np.stack([-np.arange(L), np.zeros(L, dtype=np.int32)], axis=-1)[:, None] # [L, 1, 2]
+            inpaint_torso_coords += inpaint_offsets
+            inpaint_torso_coords = inpaint_torso_coords.reshape(-1, 2) # [Lm, 2]
+            inpaint_torso_colors = top_torso_colors[None].repeat(L, 0) # [L, m, 3]
+            darken_scaler = 0.98 ** np.arange(L).reshape(L, 1, 1) # [L, 1, 1]
+            inpaint_torso_colors = (inpaint_torso_colors * darken_scaler).reshape(-1, 3) # [Lm, 3]
+            # set color
+            torso_image[tuple(inpaint_torso_coords.T)] = inpaint_torso_colors
+
+            inpaint_torso_mask = np.zeros_like(torso_image[..., 0]).astype(bool)
+            inpaint_torso_mask[tuple(inpaint_torso_coords.T)] = True
+        else:
+            inpaint_torso_mask = None
+            
+        # neck part "vertical" in-painting...
+        push_down = 4
+        L = 48 + push_down + 1
+
+        neck_part = binary_dilation(neck_part, structure=np.array([[0, 1, 0], [0, 1, 0], [0, 1, 0]], dtype=bool), iterations=3)
+
+        neck_coords = np.stack(np.nonzero(neck_part), axis=-1) # [M, 2]
+        # lexsort: sort 2D coords first by y then by x, 
+        # ref: https://stackoverflow.com/questions/2706605/sorting-a-2d-numpy-array-by-multiple-axes
+        inds = np.lexsort((neck_coords[:, 0], neck_coords[:, 1]))
+        neck_coords = neck_coords[inds]
+        # choose the top pixel for each column
+        u, uid, ucnt = np.unique(neck_coords[:, 1], return_index=True, return_counts=True)
+        top_neck_coords = neck_coords[uid] # [m, 2]
+        # only keep top-is-head pixels
+        top_neck_coords_up = top_neck_coords.copy() - np.array([1, 0])
+        mask = head_part[tuple(top_neck_coords_up.T)] 
+        
+        top_neck_coords = top_neck_coords[mask]
+        # push these top down for 4 pixels to make the neck inpainting more natural...
+        offset_down = np.minimum(ucnt[mask] - 1, push_down)
+        top_neck_coords += np.stack([offset_down, np.zeros_like(offset_down)], axis=-1)
+        # get the color
+        top_neck_colors = gt_image[tuple(top_neck_coords.T)] # [m, 3]
+        # construct inpaint coords (vertically up, or minus in x)
+        inpaint_neck_coords = top_neck_coords[None].repeat(L, 0) # [L, m, 2]
+        inpaint_offsets = np.stack([-np.arange(L), np.zeros(L, dtype=np.int32)], axis=-1)[:, None] # [L, 1, 2]
+        inpaint_neck_coords += inpaint_offsets
+        inpaint_neck_coords = inpaint_neck_coords.reshape(-1, 2) # [Lm, 2]
+        inpaint_neck_colors = top_neck_colors[None].repeat(L, 0) # [L, m, 3]
+        darken_scaler = 0.98 ** np.arange(L).reshape(L, 1, 1) # [L, 1, 1]
+        inpaint_neck_colors = (inpaint_neck_colors * darken_scaler).reshape(-1, 3) # [Lm, 3]
+        # set color
+        torso_image[tuple(inpaint_neck_coords.T)] = inpaint_neck_colors
+
+        # apply blurring to the inpaint area to avoid vertical-line artifects...
+        inpaint_mask = np.zeros_like(torso_image[..., 0]).astype(bool)
+        inpaint_mask[tuple(inpaint_neck_coords.T)] = True
+
+        blur_img = torso_image.copy()
+        blur_img = cv2.GaussianBlur(blur_img, (5, 5), cv2.BORDER_DEFAULT)
+
+        torso_image[inpaint_mask] = blur_img[inpaint_mask]
+
+        # set mask
+        mask = (neck_part | torso_part | inpaint_mask)
+        if inpaint_torso_mask is not None:
+            mask = mask | inpaint_torso_mask
+        torso_image[~mask] = 0
+        torso_alpha[~mask] = 0
+
+        cv2.imwrite("0.png", np.concatenate([torso_image, torso_alpha], axis=-1))
+
+    print(f'[INFO] ===== extracted torso and gt images =====')
+
+
+def out_exist_job(vid_name):
+    out_dir1 = vid_name.replace("/video/", "/inpaint_torso_imgs/").replace(".mp4","") 
+    out_dir2 = vid_name.replace("/video/", "/inpaint_torso_with_bg_imgs/").replace(".mp4","") 
+    out_dir3 = vid_name.replace("/video/", "/torso_imgs/").replace(".mp4","") 
+    out_dir4 = vid_name.replace("/video/", "/torso_with_bg_imgs/").replace(".mp4","") 
+    
+    if os.path.exists(out_dir1) and os.path.exists(out_dir1) and os.path.exists(out_dir2) and os.path.exists(out_dir3) and os.path.exists(out_dir4):
+        num_frames = len(os.listdir(out_dir1))
+        if len(os.listdir(out_dir1)) == num_frames and len(os.listdir(out_dir2)) == num_frames and len(os.listdir(out_dir3)) == num_frames and len(os.listdir(out_dir4)) == num_frames:
+            return None
+        else:
+            return vid_name
+    else:
+        return vid_name
+    
+def get_todo_vid_names(vid_names):
+    todo_vid_names = []
+    for i, res in multiprocess_run_tqdm(out_exist_job, vid_names, num_workers=16):
+        if res is not None:
+            todo_vid_names.append(res)
+    return todo_vid_names
+
+if __name__ == '__main__':
+    import argparse, glob, tqdm, random
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--vid_dir", default='/home/tiger/datasets/raw/CelebV-HQ/video')
+    parser.add_argument("--ds_name", default='CelebV-HQ')
+    parser.add_argument("--num_workers", default=48, type=int)
+    parser.add_argument("--seed", default=0, type=int)
+    parser.add_argument("--process_id", default=0, type=int)
+    parser.add_argument("--total_process", default=1, type=int)
+    parser.add_argument("--reset", action='store_true')
+    
+    inpaint_torso_job('/home/tiger/datasets/raw/CelebV-HQ/video/dgdEr-mXQT4_8.mp4')
+    # args = parser.parse_args()
+    # vid_dir = args.vid_dir
+    # ds_name = args.ds_name
+    # if ds_name in ['lrs3_trainval']:
+    #     mp4_name_pattern = os.path.join(vid_dir, "*/*.mp4")
+    # if ds_name in ['TH1KH_512', 'CelebV-HQ']:
+    #     vid_names = glob.glob(os.path.join(vid_dir, "*.mp4"))
+    # elif ds_name in ['lrs2', 'lrs3', 'voxceleb2']:
+    #     vid_name_pattern = os.path.join(vid_dir, "*/*/*.mp4")
+    #     vid_names = glob.glob(vid_name_pattern)
+    # vid_names = sorted(vid_names)
+    # random.seed(args.seed)
+    # random.shuffle(vid_names)
+
+    # process_id = args.process_id
+    # total_process = args.total_process
+    # if total_process > 1:
+    #     assert process_id <= total_process -1
+    #     num_samples_per_process = len(vid_names) // total_process
+    #     if process_id == total_process:
+    #         vid_names = vid_names[process_id * num_samples_per_process : ]
+    #     else:
+    #         vid_names = vid_names[process_id * num_samples_per_process : (process_id+1) * num_samples_per_process]
+    
+    # if not args.reset:
+    #     vid_names = get_todo_vid_names(vid_names)
+    # print(f"todo videos number: {len(vid_names)}")
+
+    # fn_args = [(vid_name,i,len(vid_names)) for i, vid_name in enumerate(vid_names)]
+    # for vid_name in multiprocess_run_tqdm(inpaint_torso_job ,fn_args, desc=f"Root process {args.process_id}: extracting segment images", num_workers=args.num_workers):
+    #     pass

data_gen/utils/process_video/resample_video_to_25fps_resize_to_512.py

@@ -0,0 +1,87 @@
+import os, glob
+import cv2
+from utils.commons.os_utils import multiprocess_glob
+from utils.commons.multiprocess_utils import multiprocess_run_tqdm
+
+def get_video_infos(video_path):
+    vid_cap = cv2.VideoCapture(video_path)
+    height = int(vid_cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
+    width = int(vid_cap.get(cv2.CAP_PROP_FRAME_WIDTH))
+    fps = vid_cap.get(cv2.CAP_PROP_FPS)
+    total_frames = int(vid_cap.get(cv2.CAP_PROP_FRAME_COUNT))
+    return {'height': height, 'width': width, 'fps': fps, 'total_frames':total_frames}
+
+def extract_img_job(video_name:str):
+    out_path = video_name.replace("/video_raw/","/video/",1)
+    os.makedirs(os.path.dirname(out_path), exist_ok=True)
+    ffmpeg_path = "/usr/bin/ffmpeg"
+    vid_info = get_video_infos(video_name)
+    assert vid_info['width'] == vid_info['height']
+    cmd = f'{ffmpeg_path} -i {video_name} -vf fps={25},scale=w=512:h=512 -q:v 1 -c:v libx264 -pix_fmt yuv420p -b:v 2000k -v quiet -y {out_path}'
+    os.system(cmd)
+
+def extract_img_job_crop(video_name:str):
+    out_path = video_name.replace("/video_raw/","/video/",1)
+    os.makedirs(os.path.dirname(out_path), exist_ok=True)
+    ffmpeg_path = "/usr/bin/ffmpeg"
+    vid_info = get_video_infos(video_name)
+    wh = min(vid_info['width'], vid_info['height'])
+    cmd = f'{ffmpeg_path} -i {video_name} -vf fps={25},crop={wh}:{wh},scale=w=512:h=512 -q:v 1 -c:v libx264 -pix_fmt yuv420p -b:v 2000k -v quiet -y {out_path}'
+    os.system(cmd)
+
+def extract_img_job_crop_ravdess(video_name:str):
+    out_path = video_name.replace("/video_raw/","/video/",1)
+    os.makedirs(os.path.dirname(out_path), exist_ok=True)
+    ffmpeg_path = "/usr/bin/ffmpeg"
+    cmd = f'{ffmpeg_path} -i {video_name} -vf fps={25},crop=720:720,scale=w=512:h=512 -q:v 1 -c:v libx264 -pix_fmt yuv420p -b:v 2000k -v quiet -y {out_path}'
+    os.system(cmd)
+
+if __name__ == '__main__':
+    import argparse, glob, tqdm, random
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--vid_dir", default='/home/tiger/datasets/raw/CelebV-HQ/video_raw/')
+    parser.add_argument("--ds_name", default='CelebV-HQ')
+    parser.add_argument("--num_workers", default=32, type=int)
+    parser.add_argument("--process_id", default=0, type=int)
+    parser.add_argument("--total_process", default=1, type=int)
+    args = parser.parse_args()
+    print(f"args {args}")
+
+    vid_dir = args.vid_dir
+    ds_name = args.ds_name
+    if ds_name in ['lrs3_trainval']:
+        mp4_name_pattern = os.path.join(vid_dir, "*/*.mp4")
+    elif ds_name in ['TH1KH_512', 'CelebV-HQ']:
+        vid_names = multiprocess_glob(os.path.join(vid_dir, "*.mp4"))
+    elif ds_name in ['lrs2', 'lrs3', 'voxceleb2', 'CMLR']:
+        vid_name_pattern = os.path.join(vid_dir, "*/*/*.mp4")
+        vid_names = multiprocess_glob(vid_name_pattern)
+    elif ds_name in ["RAVDESS", 'VFHQ']:
+        vid_name_pattern = os.path.join(vid_dir, "*/*/*/*.mp4")
+        vid_names = multiprocess_glob(vid_name_pattern)
+    else:
+        raise NotImplementedError()
+    vid_names = sorted(vid_names)
+    print(f"total video number : {len(vid_names)}")
+    print(f"first {vid_names[0]} last {vid_names[-1]}")
+    # exit()
+    process_id = args.process_id
+    total_process = args.total_process
+    if total_process > 1:
+        assert process_id <= total_process -1
+        num_samples_per_process = len(vid_names) // total_process
+        if process_id == total_process:
+            vid_names = vid_names[process_id * num_samples_per_process : ]
+        else:
+            vid_names = vid_names[process_id * num_samples_per_process : (process_id+1) * num_samples_per_process]
+    
+    if ds_name == "RAVDESS":
+        for i, res in multiprocess_run_tqdm(extract_img_job_crop_ravdess, vid_names, num_workers=args.num_workers, desc="resampling videos"):
+            pass
+    elif ds_name == "CMLR":
+        for i, res in multiprocess_run_tqdm(extract_img_job_crop, vid_names, num_workers=args.num_workers, desc="resampling videos"):
+            pass
+    else:
+        for i, res in multiprocess_run_tqdm(extract_img_job, vid_names, num_workers=args.num_workers, desc="resampling videos"):
+            pass
+

data_gen/utils/process_video/split_video_to_imgs.py

@@ -0,0 +1,53 @@
+import os, glob
+from utils.commons.multiprocess_utils import multiprocess_run_tqdm
+
+from data_gen.utils.path_converter import PathConverter, pc
+
+# mp4_names = glob.glob("/home/tiger/datasets/raw/CelebV-HQ/video/*.mp4")
+
+def extract_img_job(video_name, raw_img_dir=None):
+    if raw_img_dir is not None:
+        out_path = raw_img_dir
+    else:
+        out_path = pc.to(video_name.replace(".mp4", ""), "vid", "gt")
+    os.makedirs(out_path, exist_ok=True)
+    ffmpeg_path = "/usr/bin/ffmpeg"
+    cmd = f'{ffmpeg_path} -i {video_name} -vf fps={25},scale=w=512:h=512 -qmin 1 -q:v 1 -start_number 0 -v quiet {os.path.join(out_path, "%8d.jpg")}'
+    os.system(cmd)
+
+if __name__ == '__main__':
+    import argparse, glob, tqdm, random
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--vid_dir", default='/home/tiger/datasets/raw/CelebV-HQ/video')
+    parser.add_argument("--ds_name", default='CelebV-HQ')
+    parser.add_argument("--num_workers", default=64, type=int)
+    parser.add_argument("--process_id", default=0, type=int)
+    parser.add_argument("--total_process", default=1, type=int)
+    args = parser.parse_args()
+    vid_dir = args.vid_dir
+    ds_name = args.ds_name
+    if ds_name in ['lrs3_trainval']:
+        mp4_name_pattern = os.path.join(vid_dir, "*/*.mp4")
+    elif ds_name in ['TH1KH_512', 'CelebV-HQ']:
+        vid_names = glob.glob(os.path.join(vid_dir, "*.mp4"))
+    elif ds_name in ['lrs2', 'lrs3', 'voxceleb2']:
+        vid_name_pattern = os.path.join(vid_dir, "*/*/*.mp4")
+        vid_names = glob.glob(vid_name_pattern)
+    elif ds_name in ["RAVDESS", 'VFHQ']:
+        vid_name_pattern = os.path.join(vid_dir, "*/*/*/*.mp4")
+        vid_names = glob.glob(vid_name_pattern)
+    vid_names = sorted(vid_names)
+    
+    process_id = args.process_id
+    total_process = args.total_process
+    if total_process > 1:
+        assert process_id <= total_process -1
+        num_samples_per_process = len(vid_names) // total_process
+        if process_id == total_process:
+            vid_names = vid_names[process_id * num_samples_per_process : ]
+        else:
+            vid_names = vid_names[process_id * num_samples_per_process : (process_id+1) * num_samples_per_process]
+    
+    for i, res in multiprocess_run_tqdm(extract_img_job, vid_names, num_workers=args.num_workers, desc="extracting images"):
+        pass
+

data_util/face3d_helper.py

@@ -0,0 +1,309 @@
+import os
+import numpy as np
+import torch
+import torch.nn as nn
+from scipy.io import loadmat
+
+from deep_3drecon.deep_3drecon_models.bfm import perspective_projection
+
+
+class Face3DHelper(nn.Module):
+    def __init__(self, bfm_dir='deep_3drecon/BFM', keypoint_mode='lm68', use_gpu=True):
+        super().__init__()
+        self.keypoint_mode = keypoint_mode # lm68 | mediapipe
+        self.bfm_dir = bfm_dir
+        self.load_3dmm()
+        if use_gpu: self.to("cuda")
+            
+    def load_3dmm(self):
+        model = loadmat(os.path.join(self.bfm_dir, "BFM_model_front.mat"))
+        self.register_buffer('mean_shape',torch.from_numpy(model['meanshape'].transpose()).float()) # mean face shape. [3*N, 1], N=35709, xyz=3, ==> 3*N=107127
+        mean_shape = self.mean_shape.reshape([-1, 3])
+        # re-center
+        mean_shape = mean_shape - torch.mean(mean_shape, dim=0, keepdims=True)
+        self.mean_shape = mean_shape.reshape([-1, 1])
+        self.register_buffer('id_base',torch.from_numpy(model['idBase']).float()) # identity basis. [3*N,80], we have 80 eigen faces for identity
+        self.register_buffer('exp_base',torch.from_numpy(model['exBase']).float()) # expression basis. [3*N,64], we have 64 eigen faces for expression
+        
+        self.register_buffer('mean_texure',torch.from_numpy(model['meantex'].transpose()).float()) # mean face texture. [3*N,1] (0-255)
+        self.register_buffer('tex_base',torch.from_numpy(model['texBase']).float()) # texture basis. [3*N,80], rgb=3
+        
+        self.register_buffer('point_buf',torch.from_numpy(model['point_buf']).float()) # triangle indices for each vertex that lies in. starts from 1. [N,8] (1-F)
+        self.register_buffer('face_buf',torch.from_numpy(model['tri']).float()) # vertex indices in each triangle. starts from 1. [F,3] (1-N)
+        if self.keypoint_mode == 'mediapipe':
+            self.register_buffer('key_points', torch.from_numpy(np.load("deep_3drecon/BFM/index_mp468_from_mesh35709.npy").astype(np.int64)))
+            unmatch_mask = self.key_points < 0
+            self.key_points[unmatch_mask] = 0
+        else:
+            self.register_buffer('key_points',torch.from_numpy(model['keypoints'].squeeze().astype(np.int_)).long()) # vertex indices of 68 facial landmarks. starts from 1. [68,1]
+        
+
+        self.register_buffer('key_mean_shape',self.mean_shape.reshape([-1,3])[self.key_points,:])
+        self.register_buffer('key_id_base', self.id_base.reshape([-1,3,80])[self.key_points, :, :].reshape([-1,80])) 
+        self.register_buffer('key_exp_base', self.exp_base.reshape([-1,3,64])[self.key_points, :, :].reshape([-1,64])) 
+        self.key_id_base_np = self.key_id_base.cpu().numpy()
+        self.key_exp_base_np = self.key_exp_base.cpu().numpy()
+
+        self.register_buffer('persc_proj', torch.tensor(perspective_projection(focal=1015, center=112))) 
+    def split_coeff(self, coeff):
+        """
+        coeff: Tensor[B, T, c=257] or [T, c=257]
+        """
+        ret_dict = {
+            'identity': coeff[..., :80],  # identity, [b, t, c=80] 
+            'expression': coeff[..., 80:144],  # expression, [b, t, c=80]
+            'texture': coeff[..., 144:224],  # texture, [b, t, c=80]
+            'euler': coeff[..., 224:227],  # euler euler for pose, [b, t, c=3]
+            'translation':  coeff[..., 254:257], # translation, [b, t, c=3]
+            'gamma': coeff[..., 227:254] # lighting, [b, t, c=27]
+        }
+        return ret_dict
+    
+    def reconstruct_face_mesh(self, id_coeff, exp_coeff):
+        """
+        Generate a pose-independent 3D face mesh!
+        id_coeff: Tensor[T, c=80]
+        exp_coeff: Tensor[T, c=64]
+        """
+        id_coeff = id_coeff.to(self.key_id_base.device)
+        exp_coeff = exp_coeff.to(self.key_id_base.device)
+        mean_face = self.mean_shape.squeeze().reshape([1, -1]) # [3N, 1] ==> [1, 3N]
+        id_base, exp_base = self.id_base, self.exp_base # [3*N, C]
+        identity_diff_face = torch.matmul(id_coeff, id_base.transpose(0,1)) # [t,c],[c,3N] ==> [t,3N]
+        expression_diff_face = torch.matmul(exp_coeff, exp_base.transpose(0,1)) # [t,c],[c,3N] ==> [t,3N]
+        
+        face = mean_face + identity_diff_face + expression_diff_face # [t,3N]
+        face = face.reshape([face.shape[0], -1, 3]) # [t,N,3]
+        # re-centering the face with mean_xyz, so the face will be in [-1, 1]
+        # mean_xyz = self.mean_shape.squeeze().reshape([-1,3]).mean(dim=0) # [1, 3]
+        # face_mesh = face - mean_xyz.unsqueeze(0) # [t,N,3]
+        return face
+
+    def reconstruct_cano_lm3d(self, id_coeff, exp_coeff):
+        """
+        Generate 3D landmark with keypoint base!
+        id_coeff: Tensor[T, c=80]
+        exp_coeff: Tensor[T, c=64]
+        """
+        id_coeff = id_coeff.to(self.key_id_base.device)
+        exp_coeff = exp_coeff.to(self.key_id_base.device)
+        mean_face = self.key_mean_shape.squeeze().reshape([1, -1]) # [3*68, 1] ==> [1, 3*68]
+        id_base, exp_base = self.key_id_base, self.key_exp_base # [3*68, C]
+        identity_diff_face = torch.matmul(id_coeff, id_base.transpose(0,1)) # [t,c],[c,3*68] ==> [t,3*68]
+        expression_diff_face = torch.matmul(exp_coeff, exp_base.transpose(0,1)) # [t,c],[c,3*68] ==> [t,3*68]
+        
+        face = mean_face + identity_diff_face + expression_diff_face # [t,3N]
+        face = face.reshape([face.shape[0], -1, 3]) # [t,N,3]
+        # re-centering the face with mean_xyz, so the face will be in [-1, 1]
+        # mean_xyz = self.key_mean_shape.squeeze().reshape([-1,3]).mean(dim=0) # [1, 3]
+        # lm3d = face - mean_xyz.unsqueeze(0) # [t,N,3]
+        return face
+
+    def reconstruct_lm3d(self, id_coeff, exp_coeff, euler, trans, to_camera=True):
+        """
+        Generate 3D landmark with keypoint base!
+        id_coeff: Tensor[T, c=80]
+        exp_coeff: Tensor[T, c=64]
+        """
+        id_coeff = id_coeff.to(self.key_id_base.device)
+        exp_coeff = exp_coeff.to(self.key_id_base.device)
+        mean_face = self.key_mean_shape.squeeze().reshape([1, -1]) # [3*68, 1] ==> [1, 3*68]
+        id_base, exp_base = self.key_id_base, self.key_exp_base # [3*68, C]
+        identity_diff_face = torch.matmul(id_coeff, id_base.transpose(0,1)) # [t,c],[c,3*68] ==> [t,3*68]
+        expression_diff_face = torch.matmul(exp_coeff, exp_base.transpose(0,1)) # [t,c],[c,3*68] ==> [t,3*68]
+        
+        face = mean_face + identity_diff_face + expression_diff_face # [t,3N]
+        face = face.reshape([face.shape[0], -1, 3]) # [t,N,3]
+        # re-centering the face with mean_xyz, so the face will be in [-1, 1]
+        rot = self.compute_rotation(euler)
+        # transform
+        lm3d = face @ rot + trans.unsqueeze(1) # [t, N, 3]
+        # to camera
+        if to_camera:
+            lm3d[...,-1] = 10 - lm3d[...,-1] 
+        return lm3d
+
+    def reconstruct_lm2d_nerf(self, id_coeff, exp_coeff, euler, trans):
+        lm2d = self.reconstruct_lm2d(id_coeff, exp_coeff, euler, trans, to_camera=False)
+        lm2d[..., 0] = 1 - lm2d[..., 0]
+        lm2d[..., 1] = 1 - lm2d[..., 1]
+        return lm2d
+
+    def reconstruct_lm2d(self, id_coeff, exp_coeff, euler, trans, to_camera=True):
+        """
+        Generate 3D landmark with keypoint base!
+        id_coeff: Tensor[T, c=80]
+        exp_coeff: Tensor[T, c=64]
+        """
+        is_btc_flag = True if id_coeff.ndim == 3 else False
+        if is_btc_flag:
+            b,t,_ = id_coeff.shape
+            id_coeff = id_coeff.reshape([b*t,-1])
+            exp_coeff = exp_coeff.reshape([b*t,-1])
+            euler = euler.reshape([b*t,-1])
+            trans = trans.reshape([b*t,-1])
+        id_coeff = id_coeff.to(self.key_id_base.device)
+        exp_coeff = exp_coeff.to(self.key_id_base.device)
+        mean_face = self.key_mean_shape.squeeze().reshape([1, -1]) # [3*68, 1] ==> [1, 3*68]
+        id_base, exp_base = self.key_id_base, self.key_exp_base # [3*68, C]
+        identity_diff_face = torch.matmul(id_coeff, id_base.transpose(0,1)) # [t,c],[c,3*68] ==> [t,3*68]
+        expression_diff_face = torch.matmul(exp_coeff, exp_base.transpose(0,1)) # [t,c],[c,3*68] ==> [t,3*68]
+        
+        face = mean_face + identity_diff_face + expression_diff_face # [t,3N]
+        face = face.reshape([face.shape[0], -1, 3]) # [t,N,3]
+        # re-centering the face with mean_xyz, so the face will be in [-1, 1]
+        rot = self.compute_rotation(euler)
+        # transform
+        lm3d = face @ rot + trans.unsqueeze(1) # [t, N, 3]
+        # to camera
+        if to_camera:
+            lm3d[...,-1] = 10 - lm3d[...,-1] 
+        # to image_plane
+        lm3d = lm3d @ self.persc_proj
+        lm2d = lm3d[..., :2] / lm3d[..., 2:]
+        # flip
+        lm2d[..., 1] = 224 - lm2d[..., 1]
+        lm2d /= 224
+        if is_btc_flag:
+            return lm2d.reshape([b,t,-1,2])
+        return lm2d
+    
+    def compute_rotation(self, euler):
+        """
+        Return:
+            rot              -- torch.tensor, size (B, 3, 3) pts @ trans_mat
+
+        Parameters:
+            euler           -- torch.tensor, size (B, 3), radian
+        """
+
+        batch_size = euler.shape[0]
+        euler = euler.to(self.key_id_base.device)
+        ones = torch.ones([batch_size, 1]).to(self.key_id_base.device)
+        zeros = torch.zeros([batch_size, 1]).to(self.key_id_base.device)
+        x, y, z = euler[:, :1], euler[:, 1:2], euler[:, 2:],
+        
+        rot_x = torch.cat([
+            ones, zeros, zeros,
+            zeros, torch.cos(x), -torch.sin(x), 
+            zeros, torch.sin(x), torch.cos(x)
+        ], dim=1).reshape([batch_size, 3, 3])
+        
+        rot_y = torch.cat([
+            torch.cos(y), zeros, torch.sin(y),
+            zeros, ones, zeros,
+            -torch.sin(y), zeros, torch.cos(y)
+        ], dim=1).reshape([batch_size, 3, 3])
+
+        rot_z = torch.cat([
+            torch.cos(z), -torch.sin(z), zeros,
+            torch.sin(z), torch.cos(z), zeros,
+            zeros, zeros, ones
+        ], dim=1).reshape([batch_size, 3, 3])
+
+        rot = rot_z @ rot_y @ rot_x
+        return rot.permute(0, 2, 1)
+    
+    def reconstruct_idexp_lm3d(self, id_coeff, exp_coeff):
+        """
+        Generate 3D landmark with keypoint base!
+        id_coeff: Tensor[T, c=80]
+        exp_coeff: Tensor[T, c=64]
+        """
+        id_coeff = id_coeff.to(self.key_id_base.device)
+        exp_coeff = exp_coeff.to(self.key_id_base.device)
+        id_base, exp_base = self.key_id_base, self.key_exp_base # [3*68, C]
+        identity_diff_face = torch.matmul(id_coeff, id_base.transpose(0,1)) # [t,c],[c,3*68] ==> [t,3*68]
+        expression_diff_face = torch.matmul(exp_coeff, exp_base.transpose(0,1)) # [t,c],[c,3*68] ==> [t,3*68]
+        
+        face = identity_diff_face + expression_diff_face # [t,3N]
+        face = face.reshape([face.shape[0], -1, 3]) # [t,N,3]
+        lm3d = face * 10
+        return lm3d
+    
+    def reconstruct_idexp_lm3d_np(self, id_coeff, exp_coeff):
+        """
+        Generate 3D landmark with keypoint base!
+        id_coeff: Tensor[T, c=80]
+        exp_coeff: Tensor[T, c=64]
+        """
+        id_base, exp_base = self.key_id_base_np, self.key_exp_base_np # [3*68, C]
+        identity_diff_face = np.dot(id_coeff, id_base.T) # [t,c],[c,3*68] ==> [t,3*68]
+        expression_diff_face = np.dot(exp_coeff, exp_base.T) # [t,c],[c,3*68] ==> [t,3*68]
+        
+        face = identity_diff_face + expression_diff_face # [t,3N]
+        face = face.reshape([face.shape[0], -1, 3]) # [t,N,3]
+        lm3d = face * 10
+        return lm3d
+    
+    def get_eye_mouth_lm_from_lm3d(self, lm3d):
+        eye_lm = lm3d[:, 17:48] # [T, 31, 3]
+        mouth_lm = lm3d[:, 48:68] # [T, 20, 3]
+        return eye_lm, mouth_lm
+    
+    def get_eye_mouth_lm_from_lm3d_batch(self, lm3d):
+        eye_lm = lm3d[:, :, 17:48] # [T, 31, 3]
+        mouth_lm = lm3d[:, :, 48:68] # [T, 20, 3]
+        return eye_lm, mouth_lm
+    
+    def close_mouth_for_idexp_lm3d(self, idexp_lm3d, freeze_as_first_frame=True):
+        idexp_lm3d = idexp_lm3d.reshape([-1, 68,3])
+        num_frames = idexp_lm3d.shape[0]
+        eps = 0.0
+        # [n_landmarks=68,xyz=3], x 代表左右，y代表上下，z代表深度
+        idexp_lm3d[:,49:54, 1] = (idexp_lm3d[:,49:54, 1] + idexp_lm3d[:,range(59,54,-1), 1])/2 + eps * 2
+        idexp_lm3d[:,range(59,54,-1), 1] = (idexp_lm3d[:,49:54, 1] + idexp_lm3d[:,range(59,54,-1), 1])/2 - eps * 2
+
+        idexp_lm3d[:,61:64, 1] = (idexp_lm3d[:,61:64, 1] + idexp_lm3d[:,range(67,64,-1), 1])/2 + eps
+        idexp_lm3d[:,range(67,64,-1), 1] = (idexp_lm3d[:,61:64, 1] + idexp_lm3d[:,range(67,64,-1), 1])/2 - eps
+
+        idexp_lm3d[:,49:54, 1] += (0.03 - idexp_lm3d[:,49:54, 1].mean(dim=1) + idexp_lm3d[:,61:64, 1].mean(dim=1)).unsqueeze(1).repeat([1,5])
+        idexp_lm3d[:,range(59,54,-1), 1] += (-0.03 - idexp_lm3d[:,range(59,54,-1), 1].mean(dim=1) + idexp_lm3d[:,range(67,64,-1), 1].mean(dim=1)).unsqueeze(1).repeat([1,5])
+
+        if freeze_as_first_frame:
+            idexp_lm3d[:, 48:68,] = idexp_lm3d[0, 48:68].unsqueeze(0).clone().repeat([num_frames, 1,1])*0
+        return idexp_lm3d.cpu()
+
+    def close_eyes_for_idexp_lm3d(self, idexp_lm3d):
+        idexp_lm3d = idexp_lm3d.reshape([-1, 68,3])
+        eps = 0.003
+        idexp_lm3d[:,37:39, 1] = (idexp_lm3d[:,37:39, 1] + idexp_lm3d[:,range(41,39,-1), 1])/2 + eps
+        idexp_lm3d[:,range(41,39,-1), 1] = (idexp_lm3d[:,37:39, 1] + idexp_lm3d[:,range(41,39,-1), 1])/2 - eps
+
+        idexp_lm3d[:,43:45, 1] = (idexp_lm3d[:,43:45, 1] + idexp_lm3d[:,range(47,45,-1), 1])/2 + eps
+        idexp_lm3d[:,range(47,45,-1), 1] = (idexp_lm3d[:,43:45, 1] + idexp_lm3d[:,range(47,45,-1), 1])/2 - eps
+        
+        return idexp_lm3d
+
+if __name__ == '__main__':
+    import cv2
+    
+    font = cv2.FONT_HERSHEY_SIMPLEX
+
+    face_mesh_helper = Face3DHelper('deep_3drecon/BFM')
+    coeff_npy = 'data/coeff_fit_mp/crop_nana_003_coeff_fit_mp.npy'
+    coeff_dict = np.load(coeff_npy, allow_pickle=True).tolist()
+    lm3d = face_mesh_helper.reconstruct_lm2d(torch.tensor(coeff_dict['id']).cuda(), torch.tensor(coeff_dict['exp']).cuda(), torch.tensor(coeff_dict['euler']).cuda(), torch.tensor(coeff_dict['trans']).cuda() )
+
+    WH = 512
+    lm3d = (lm3d * WH).cpu().int().numpy()
+    eye_idx = list(range(36,48))
+    mouth_idx = list(range(48,68))
+    import imageio
+    debug_name = 'debug_lm3d.mp4'
+    writer = imageio.get_writer(debug_name, fps=25)
+    for i_img in range(len(lm3d)):
+        lm2d = lm3d[i_img ,:, :2] # [68, 2]
+        img = np.ones([WH, WH, 3], dtype=np.uint8) * 255
+        for i in range(len(lm2d)):
+            x, y = lm2d[i]
+            if i in eye_idx:
+                color = (0,0,255)
+            elif i in mouth_idx:
+                color = (0,255,0)
+            else:
+                color = (255,0,0)
+            img = cv2.circle(img, center=(x,y), radius=3, color=color, thickness=-1)
+            img = cv2.putText(img, f"{i}", org=(x,y), fontFace=font, fontScale=0.3, color=(255,0,0))
+        writer.append_data(img)
+    writer.close()

deep_3drecon/BFM/index_mp468_from_mesh35709_v1.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0d238a90df0c55075c9cea43dab76348421379a75c204931e34dbd2c11fb4b65
+size 3872