Upload 82 files

.gitattributes

@@ -46,3 +46,16 @@ src/pixel3dmm/preprocessing/facer/samples/data/weirdface.jpg filter=lfs diff=lfs
 src/pixel3dmm/preprocessing/facer/samples/example_output/alignment.png filter=lfs diff=lfs merge=lfs -text
 src/pixel3dmm/preprocessing/facer/samples/example_output/detect.png filter=lfs diff=lfs merge=lfs -text
 src/pixel3dmm/preprocessing/facer/samples/example_output/parsing.png filter=lfs diff=lfs merge=lfs -text
+src/pixel3dmm/preprocessing/MICA/data/FLAME2020/FLAME_masks/FLAME_masks.gif filter=lfs diff=lfs merge=lfs -text
+src/pixel3dmm/preprocessing/MICA/demo/input/connelly.jpg filter=lfs diff=lfs merge=lfs -text
+src/pixel3dmm/preprocessing/MICA/demo/input/justin.png filter=lfs diff=lfs merge=lfs -text
+src/pixel3dmm/preprocessing/MICA/demo/input/lawrence.jpg filter=lfs diff=lfs merge=lfs -text
+src/pixel3dmm/preprocessing/MICA/documents/BP4D.gif filter=lfs diff=lfs merge=lfs -text
+src/pixel3dmm/preprocessing/MICA/documents/D3DFACS.gif filter=lfs diff=lfs merge=lfs -text
+src/pixel3dmm/preprocessing/MICA/documents/FACEWAREHOUSE.gif filter=lfs diff=lfs merge=lfs -text
+src/pixel3dmm/preprocessing/MICA/documents/FLORENCE.gif filter=lfs diff=lfs merge=lfs -text
+src/pixel3dmm/preprocessing/MICA/documents/FRGC.gif filter=lfs diff=lfs merge=lfs -text
+src/pixel3dmm/preprocessing/MICA/documents/LYHM.gif filter=lfs diff=lfs merge=lfs -text
+src/pixel3dmm/preprocessing/MICA/documents/STIRLING.gif filter=lfs diff=lfs merge=lfs -text
+src/pixel3dmm/preprocessing/MICA/documents/teaser.jpg filter=lfs diff=lfs merge=lfs -text
+src/pixel3dmm/preprocessing/MICA/documents/voxceleb.gif filter=lfs diff=lfs merge=lfs -text

src/pixel3dmm/preprocessing/MICA/.gitignore

@@ -0,0 +1,71 @@
+# Compiled source #
+###################
+.idea
+
+datasets/creation/template/*
+statistics/*
+*.o
+*.so
+
+# Packages #
+############
+# it's better to unpack these files and commit the raw source
+# git has its own built in compression methods
+*.7z
+*.dmg
+*.gz
+*.iso
+*.jar
+*.rar
+*.tar
+*.zip
+
+# OS generated files #
+######################
+.DS_Store
+.DS_Store?
+._*
+.Spotlight-V100
+.Trashes
+ehthumbs.db
+Thumbs.db
+
+# 3D data #
+############
+*.mat
+*.obj
+*.dat
+*.npz
+*.pkl
+
+# python file #
+############
+*.pyc
+__pycache__
+
+## deca data
+data/FLAME2020/generic_model.pkl
+data/FLAME2020/female_model.pkl
+data/FLAME2020/male_model.pkl
+data/FLAME2020/FLAME_albedo_from_BFM.npz
+results
+output
+TestSamples
+
+## dump files
+__dump
+
+## visual code files
+.vscode
+render_dataset.py
+shapes.pt
+partial
+images
+*.pt
+testing/now/jobs
+testing/now/logs
+testing/stirling/logs
+testing/stirling/jobs
+
+demo/arcface
+demo/output

src/pixel3dmm/preprocessing/MICA/LICENSE

@@ -0,0 +1,88 @@
+License
+
+Software Copyright License for non-commercial scientific research purposes
+Please read carefully the following terms and conditions and any accompanying documentation before you download
+and/or use the MICA model, data and software, (the "Model & Software"), including 3D meshes, software, and scripts.
+By downloading and/or using the Model & Software (including downloading, cloning, installing, and any other use
+of this github repository), you acknowledge that you have read these terms and conditions, understand them, and
+agree to be bound by them. If you do not agree with these terms and conditions, you must not download and/or use
+the Model & Software. Any infringement of the terms of this agreement will automatically terminate your rights
+under this License
+
+Ownership / Licensees
+The Model & Software and the associated materials has been developed at the
+Max Planck Institute for Intelligent Systems (hereinafter "MPI").
+
+Any copyright or patent right is owned by and proprietary material of the
+Max-Planck-Gesellschaft zur Förderung der Wissenschaften e.V. (hereinafter “MPG”; MPI and MPG hereinafter
+collectively “Max-Planck”) hereinafter the “Licensor”.
+
+License Grant
+Licensor grants you (Licensee) personally a single-user, non-exclusive, non-transferable, free of charge right:
+
+	• To install the Model & Software on computers owned, leased or otherwise controlled by you and/or your organization.
+	• To use the Model & Software for the sole purpose of performing peaceful non-commercial scientific research, 
+    non-commercial education, or non-commercial artistic projects.
+
+Any other use, in particular any use for commercial, pornographic, military, or surveillance purposes is prohibited. 
+This includes, without limitation, incorporation in a commercial product, use in a commercial service, 
+or production of other artefacts for commercial purposes. 
+
+The Model & Software may not be used to create fake, libelous, misleading, or defamatory content of any kind, excluding 
+analyses in peer-reviewed scientific research. 
+
+The Model & Software may not be reproduced, modified and/or made available in any form to any third party 
+without Max-Planck’s prior written permission.
+
+The Model & Software may not be used for pornographic purposes or to generate pornographic material whether
+commercial or not. This license also prohibits the use of the Model & Software to train methods/algorithms/neural
+networks/etc. for commercial use of any kind. By downloading the Model & Software, you agree not to reverse engineer it.
+
+No Distribution
+The Model & Software and the license herein granted shall not be copied, shared, distributed, re-sold, offered
+for re-sale, transferred or sub-licensed in whole or in part except that you may make one copy for archive
+purposes only.
+
+Disclaimer of Representations and Warranties
+You expressly acknowledge and agree that the Model & Software results from basic research, is provided “AS IS”,
+may contain errors, and that any use of the Model & Software is at your sole risk.
+LICENSOR MAKES NO REPRESENTATIONS
+OR WARRANTIES OF ANY KIND CONCERNING THE MODEL & SOFTWARE, NEITHER EXPRESS NOR IMPLIED, AND THE ABSENCE OF ANY
+LEGAL OR ACTUAL DEFECTS, WHETHER DISCOVERABLE OR NOT. Specifically, and not to limit the foregoing, licensor
+makes no representations or warranties (i) regarding the merchantability or fitness for a particular purpose of
+the Model & Software, (ii) that the use of the Model & Software will not infringe any patents, copyrights or other
+intellectual property rights of a third party, and (iii) that the use of the Model & Software will not cause any
+damage of any kind to you or a third party.
+
+Limitation of Liability
+Because this Model & Software License Agreement qualifies as a donation, according to Section 521 of the German
+Civil Code (Bürgerliches Gesetzbuch – BGB) Licensor as a donor is liable for intent and gross negligence only.
+If the Licensor fraudulently conceals a legal or material defect, they are obliged to compensate the Licensee
+for the resulting damage.
+
+Licensor shall be liable for loss of data only up to the amount of typical recovery costs which would have
+arisen had proper and regular data backup measures been taken. For the avoidance of doubt Licensor shall be
+liable in accordance with the German Product Liability Act in the event of product liability. The foregoing
+applies also to Licensor’s legal representatives or assistants in performance. Any further liability shall 
+be excluded. Patent claims generated through the usage of the Model & Software cannot be directed towards the copyright holders.
+The Model & Software is provided in the state of development the licensor defines. If modified or extended by
+Licensee, the Licensor makes no claims about the fitness of the Model & Software and is not responsible
+for any problems such modifications cause.
+
+No Maintenance Services
+You understand and agree that Licensor is under no obligation to provide either maintenance services,
+update services, notices of latent defects, or corrections of defects with regard to the Model & Software.
+Licensor nevertheless reserves the right to update, modify, or discontinue the Model & Software at any time.
+
+Defects of the Model & Software must be notified in writing to the Licensor with a comprehensible description
+of the error symptoms. The notification of the defect should enable the reproduction of the error.
+The Licensee is encouraged to communicate any use, results, modification or publication.
+
+Publications using the Model & Software
+You acknowledge that the Model & Software is a valuable scientific resource and agree to appropriately reference
+the following paper in any publication making use of the Model & Software.
+
+Commercial licensing opportunities
+For commercial uses of the Model & Software, please send email to [email protected]
+
+This Agreement shall be governed by the laws of the Federal Republic of Germany except for the UN Sales Convention.

src/pixel3dmm/preprocessing/MICA/README.md

@@ -0,0 +1,72 @@
+<h2 align="center"><b>MICA - Towards Metrical Reconstruction of Human Faces</b></h2>
+
+<h4 align="center"><b><a href="https://zielon.github.io/" target="_blank">Wojciech Zielonka</a>, <a href="https://sites.google.com/site/bolkartt/" target="_blank">Timo Bolkart</a>, <a href="https://justusthies.github.io/" target="_blank">Justus Thies</a></b></h4>
+
+<h6 align="center"><i>Max Planck Institute for Intelligent Systems, Tübingen, Germany</i></h6>
+
+<h4 align="center">
+<a href="https://youtu.be/vzzEbvv08VA" target="_blank">Video&nbsp</a>
+<a href="https://arxiv.org/pdf/2204.06607.pdf" target="_blank">Paper&nbsp</a>
+<a href="https://zielon.github.io/mica/" target="_blank">Project Website&nbsp</a>
+<a href="https://github.com/Zielon/metrical-tracker" target="_blank">Face Tracker&nbsp</a>
+<a href="https://github.com/Zielon/MICA/tree/master/datasets/" target="_blank"><b>Dataset&nbsp</b></a>
+<a href="https://keeper.mpdl.mpg.de/f/6b12c44378e64738b993/" target="_blank">Supplemental&nbsp</a>
+<a href="mailto:&#109;&#105;&#099;&#097;&#064;&#116;&#117;&#101;&#046;&#109;&#112;&#103;&#046;&#100;&#101;">Email</a>
+</h4>
+
+<div align="center"> 
+<img src="documents/teaser.jpg">
+<i style="font-size: 1.05em;">Official Repository for ECCV 2022 paper Towards Metrical Reconstruction of Human Faces</i>
+</div>
+<br>
+
+<div align="center"> 
+&#x26A0 The face tracker is now available under <a href="https://github.com/Zielon/metrical-tracker" target="_blank">Metrical Photometric Tracker&nbsp</a> &#x26A0
+</div>
+
+### Installation
+
+After cloning the repository please install the environment by using attached conda `environment.yml` file with the command 
+``conda env create -f environment.yml``. Additionally, the FLAME2020 model is needed. To obtain it please create an account at the [website](https://flame.is.tue.mpg.de/) download the model and place it in the `/data/pretrained/FLAME2020/` folder. 
+
+You can also simply run the `install.sh` script:
+
+```shell
+git clone https://github.com/Zielon/MICA.git
+cd MICA
+./install.sh
+```
+you will be asked to provide `{flame_user}` and `{flame_password}` for your FLAME account in order to access the file server.
+
+### Pre-trained Models
+
+If you decide to not use the installation script, the pretrained model can be found under the [link](https://drive.google.com/file/d/1bYsI_spptzyuFmfLYqYkcJA6GZWZViNt/view?usp=sharing). After downloading, please place it in the `/data/pretrained/mica.tar` location. Additionally, you will need to provide models for `inisghtface`:
+1) [antelopev2](https://drive.google.com/file/d/16PWKI_RjjbE4_kqpElG-YFqe8FpXjads/view?usp=sharing)
+2) [buffalo_l](https://drive.google.com/file/d/1navJMy0DTr1_DHjLWu1i48owCPvXWfYc/view?usp=sharing)
+
+then you need to unzip them and place in `~/.insightface/models/`. The `install.sh` script does it for you.
+
+### How To Use
+
+To use MICA you can simply run the `demo.py` file. It will process all the images from `demo/input/` folder and create the output destination for each subject with `.ply` mesh, rendered image, and `.npy` FLAME parameters.
+
+### Dataset and Training
+
+The MICA dataset consists of eight smaller datasets for about 2300 subjects under a common FLAME topology. Read more information about how to obtain and use it under the [link](https://github.com/Zielon/MICA/tree/master/datasets/). To train MICA the images from all eight datasets are needed. The repository contains scripts how to generate the Arcface input images as well as the complete list of all the images used for the training. More information can be found [here](https://github.com/Zielon/MICA/tree/master/datasets).
+
+When you train from scratch for Arcface model initialization please download [Glint360K](https://github.com/deepinsight/insightface/tree/master/recognition/arcface_torch) and specify the path to it in the config as `cfg.model.arcface_pretrained_model`.
+
+### Testing
+
+The testing was done using two datasets, [Stirling](http://pics.stir.ac.uk/ESRC/) and [NoW](https://now.is.tue.mpg.de/). In the [model folder](https://github.com/Zielon/MICA/tree/master/models) you can find the corresponding scripts to run testing routine, which generates the meshes. To calculate the NoW challenge error you can use the following [repository](https://github.com/soubhiksanyal/now_evaluation).   
+
+### Citation
+If you use this project in your research please cite MICA:
+```bibtex
+@proceedings{zielonka22mica,
+  author = {Zielonka, Wojciech and Bolkart, Timo and Thies, Justus},
+  title = {Towards Metrical Reconstruction of Human Faces},
+  journal = {European Conference on Computer Vision},
+  year = {2022}
+}
+```

src/pixel3dmm/preprocessing/MICA/condor.sh

@@ -0,0 +1,44 @@
+#!/bin/sh
+
+# bash condor.sh 100 ./configs/mica.yml 1
+
+# default parameters
+BID=3
+CONFIG=./configs/mica.yml
+NODE_CONFIG=condor/config.sub
+NODE_SCRIPT=./condor/job.sh
+GPUS=1
+GPU_TYPE=0
+
+# set parameters
+if [ -n "$1" ]; then BID=${1}; fi
+if [ -n "$2" ]; then CONFIG=${2}; fi
+if [ -n "$3" ]; then GPU_TYPE=${3}; fi
+if [ -n "$4" ]; then GPUS=${4}; fi
+if [ -n "$5" ]; then NODE_CONFIG=${5}; fi
+if [ -n "$6" ]; then NODE_SCRIPT=${6}; fi
+
+mkdir -p output/condor_logs
+cp -nf ${NODE_CONFIG}{,.bak}
+
+GPU_NAME=Error
+
+if [ $GPU_TYPE -eq 0 ]; then GPU_NAME='Quadro RTX 6000'; fi
+if [ $GPU_TYPE -eq 1 ]; then GPU_NAME='Tesla V100-SXM2-32GB'; fi
+if [ $GPU_TYPE -eq 2 ]; then GPU_NAME='NVIDIA GeForce RTX 2080 Ti'; fi
+
+NAME=$(basename ${CONFIG} .yml)
+sed -i "s/{errorfile}/${NAME}/" ${NODE_CONFIG}.bak
+sed -i "s/{outfile}/${NAME}/" ${NODE_CONFIG}.bak
+sed -i "s/{logfile}/${NAME}/" ${NODE_CONFIG}.bak
+sed -i "s/{gpus}/${GPUS}/" ${NODE_CONFIG}.bak
+sed -i "s/{gpu_name}/${GPU_NAME}/" ${NODE_CONFIG}.bak
+
+# start node and execute script
+echo 'Executing:' ${NODE_SCRIPT} ${CONFIG}
+echo '# BID:' ${BID}
+echo '# GPUS:' ${GPUS}
+echo '# GPU NAME:' ${GPU_NAME}
+
+condor_submit_bid ${BID} ${NODE_CONFIG}.bak -append "arguments = ${NODE_SCRIPT} ${CONFIG}"
+rm ${NODE_CONFIG}.bak

src/pixel3dmm/preprocessing/MICA/condor/config.sub

@@ -0,0 +1,16 @@
+executable = /bin/bash
+error  = ./output/condor_logs/{errorfile}_$(ClusterId).$(ProcId).err
+output = ./output/condor_logs/{outfile}_$(ClusterId).$(ProcId).out
+log    = ./output/condor_logs/{logfile}_$(ClusterId).$(ProcId).log
+request_memory = 32768
+request_cpus = 6
+request_gpus = {gpus}
++WantGPUStats = true
+requirements = (TARGET.CUDADeviceName=="{gpu_name}")
+# EXIT SETTINGS
+on_exit_hold = (ExitCode =?= 3)
+on_exit_hold_reason = "Checkpointed, will resume"
+on_exit_hold_subcode = 2
+periodic_release = ( (JobStatus =?= 5) && (HoldReasonCode =?= 3) && (HoldReasonSubCode =?= 2) )
++RunningPriceExceededAction = "kill"
+queue

src/pixel3dmm/preprocessing/MICA/condor/dataset.sh

@@ -0,0 +1,21 @@
+#!/bin/sh
+
+CONFIG=${1}
+
+PYTHON_ENV=/home/wzielonka/miniconda3/etc/profile.d/conda.sh
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
+export PATH=/usr/local/bin:/usr/bin:/bin:/usr/sbin:$PATH
+export PYTHONPATH="${PYTHONPATH}:/home/wzielonka/projects/OnFlame-internal/"
+
+echo 'START JOB (dataset generation)'
+
+module load cuda/10.1
+module load gcc/4.9
+
+echo 'ACTIVATE MICA'
+source ${PYTHON_ENV}
+conda activate MICA
+
+echo 'RUN SCRIPT'
+cd ${SCRIPT_DIR}/../datasets/creation
+python ./main.py

src/pixel3dmm/preprocessing/MICA/condor/job.sh

@@ -0,0 +1,22 @@
+#!/bin/sh
+
+CONFIG=${1}
+
+PYTHON_ENV=/home/wzielonka/miniconda3/etc/profile.d/conda.sh
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
+export PATH=/usr/local/bin:/usr/bin:/bin:/usr/sbin:$PATH
+
+echo 'START JOB (MICA training)'
+
+module load cuda/10.1
+module load gcc/4.9
+
+echo 'ACTIVATE MICA'
+source ${PYTHON_ENV}
+conda activate MICA
+
+echo 'RUN SCRIPT'
+#echo 'ScriptDir' ${SCRIPT_DIR}
+echo 'CONFIG: ' ${CONFIG}
+cd ${SCRIPT_DIR}/..
+python ./train.py --cfg ${CONFIG}

src/pixel3dmm/preprocessing/MICA/configs/config.py

@@ -0,0 +1,128 @@
+# -*- coding: utf-8 -*-
+
+# Max-Planck-Gesellschaft zur Förderung der Wissenschaften e.V. (MPG) is
+# holder of all proprietary rights on this computer program.
+# You can only use this computer program if you have closed
+# a license agreement with MPG or you get the right to use the computer
+# program from someone who is authorized to grant you that right.
+# Any use of the computer program without a valid license is prohibited and
+# liable to prosecution.
+#
+# Copyright©2023 Max-Planck-Gesellschaft zur Förderung
+# der Wissenschaften e.V. (MPG). acting on behalf of its Max Planck Institute
+# for Intelligent Systems. All rights reserved.
+#
+# Contact: [email protected]
+
+
+import argparse
+import os
+
+from yacs.config import CfgNode as CN
+
+cfg = CN()
+
+abs_mica_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
+cfg.mica_dir = abs_mica_dir
+cfg.device = 'cuda'
+cfg.device_id = '0'
+cfg.pretrained_model_path = os.path.join(cfg.mica_dir, 'data/pretrained', 'mica.tar')
+cfg.output_dir = ''
+
+# ---------------------------------------------------------------------------- #
+# Options for Face model
+# ---------------------------------------------------------------------------- #
+cfg.model = CN()
+cfg.model.testing = False
+cfg.model.name = 'mica'
+
+cfg.model.topology_path = os.path.join(cfg.mica_dir, 'data/FLAME2020', 'head_template.obj')
+cfg.model.flame_model_path = os.path.join(cfg.mica_dir, 'data/FLAME2020', 'generic_model.pkl')
+cfg.model.flame_lmk_embedding_path = os.path.join(cfg.mica_dir, 'data/FLAME2020', 'landmark_embedding.npy')
+cfg.model.n_shape = 300
+cfg.model.layers = 8
+cfg.model.hidden_layers_size = 256
+cfg.model.mapping_layers = 3
+cfg.model.use_pretrained = True
+cfg.model.arcface_pretrained_model = '/scratch/is-rg-ncs/models_weights/arcface-torch/backbone100.pth'
+
+# ---------------------------------------------------------------------------- #
+# Options for Dataset
+# ---------------------------------------------------------------------------- #
+cfg.dataset = CN()
+cfg.dataset.training_data = ['LYHM']
+cfg.dataset.eval_data = ['FLORENCE']
+cfg.dataset.batch_size = 2
+cfg.dataset.K = 4
+cfg.dataset.n_train = 100000
+cfg.dataset.num_workers = 4
+cfg.dataset.root = '/datasets/MICA/'
+
+# ---------------------------------------------------------------------------- #
+# Mask weights
+# ---------------------------------------------------------------------------- #
+cfg.mask_weights = CN()
+cfg.mask_weights.face = 150.0
+cfg.mask_weights.nose = 50.0
+cfg.mask_weights.lips = 50.0
+cfg.mask_weights.forehead = 50.0
+cfg.mask_weights.lr_eye_region = 50.0
+cfg.mask_weights.eye_region = 50.0
+
+cfg.mask_weights.whole = 1.0
+cfg.mask_weights.ears = 0.01
+cfg.mask_weights.eyes = 0.01
+
+cfg.running_average = 7
+
+# ---------------------------------------------------------------------------- #
+# Options for training
+# ---------------------------------------------------------------------------- #
+cfg.train = CN()
+cfg.train.use_mask = False
+cfg.train.max_epochs = 50
+cfg.train.max_steps = 100000
+cfg.train.lr = 1e-4
+cfg.train.arcface_lr = 1e-3
+cfg.train.weight_decay = 0.0
+cfg.train.lr_update_step = 100000000
+cfg.train.log_dir = 'logs'
+cfg.train.log_steps = 10
+cfg.train.vis_dir = 'train_images'
+cfg.train.vis_steps = 200
+cfg.train.write_summary = True
+cfg.train.checkpoint_steps = 1000
+cfg.train.checkpoint_epochs_steps = 2
+cfg.train.val_steps = 1000
+cfg.train.val_vis_dir = 'val_images'
+cfg.train.eval_steps = 5000
+cfg.train.reset_optimizer = False
+cfg.train.val_save_img = 5000
+cfg.test_dataset = 'now'
+
+
+def get_cfg_defaults():
+    return cfg.clone()
+
+
+def update_cfg(cfg, cfg_file):
+    cfg.merge_from_file(cfg_file)
+    return cfg.clone()
+
+
+def parse_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--cfg', type=str, help='cfg file path', required=True)
+    parser.add_argument('--test_dataset', type=str, help='Test dataset type', default='')
+    parser.add_argument('--checkpoint', type=str, help='Checkpoint to load', default='')
+
+    args = parser.parse_args()
+    print(args, end='\n\n')
+
+    cfg = get_cfg_defaults()
+    if args.cfg is not None:
+        cfg_file = args.cfg
+        cfg = update_cfg(cfg, args.cfg)
+        cfg.cfg_file = cfg_file
+
+    return cfg, args

src/pixel3dmm/preprocessing/MICA/configs/mica.yml

@@ -0,0 +1,30 @@
+# Mica config
+
+pretrained_model_path: ''
+
+dataset:
+  root: '/scratch/NFC/MICA/dataset/'
+  training_data: [ 'LYHM', 'D3DFACS', 'BU3DFE', 'FRGC', 'Stirling', 'FaceWarehouse', 'BP4D' ]
+  eval_data: [ 'FLORENCE' ]
+  num_workers: 4
+  batch_size: 8
+  K: 2
+
+train:
+  lr: 1e-5
+  arcface_lr: 1e-5
+  weight_decay: 2e-4
+  use_mask: True
+  reset_optimizer: False
+  max_steps: 160000
+  log_steps: 50
+  val_steps: 300
+  vis_steps: 1200
+  val_save_img: 1200
+  checkpoint_steps: 1000
+  checkpoint_epochs_steps: 10000
+
+model:
+  use_pretrained: False
+  n_shape: 300
+  name: 'mica'

src/pixel3dmm/preprocessing/MICA/data/FLAME2020/FLAME_masks/FLAME_masks.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ccefbe1ac0774ff78c68caf2c627b4abc067a6555ebeb0be5d5b0812366ab492
+size 215062

src/pixel3dmm/preprocessing/MICA/data/FLAME2020/FLAME_masks/readme

@@ -0,0 +1,2 @@
+Dictionary with vertex indices for different masks for the publicly available FLAME head model (https://flame.is.tue.mpg.de/).
+See the gif for a visualization of all masks.

src/pixel3dmm/preprocessing/MICA/data/FLAME2020/landmark_embedding.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8095348eeafce5a02f6bd8765146307f9567a3f03b316d788a2e47336d667954
+size 31292

src/pixel3dmm/preprocessing/MICA/datasets/README.md

@@ -0,0 +1,128 @@
+<h2 align="center"><b>MICA - Dataset</b></h2>
+
+The <b>MICA</b> dataset consists of eight smaller datasets for about 2315 subjects, built by unifying existing small- and medium-scale datasets under a common FLAME topology. It consists of shape geometry only, therefore, to obtain images for each subject please refer to the primary dataset.
+
+This dataset contains registration meshes together with corresponding fitted FLAME parameters. Actors are split between individual folders with a unique identifier based on the original dataset. The folder name of the parameters and mesh is the same as the ones in the analogous dataset with images. 
+
+To obtain the dataset please follow each link separately and request the given subset.
+
+In the case of any questions feel free to <a href="mailto:&#109;&#105;&#099;&#097;&#064;&#116;&#117;&#101;&#046;&#109;&#112;&#103;&#046;&#100;&#101;">email us</a>.
+
+<div align="center">
+<table cellspacing="0" cellpadding="0" bgcolor="#ffffff" border="0">
+  <tr>
+    <th>FLAME Renderings</th>
+    <th>Dataset</th>
+    <th>Subjects</th>
+    <th>Source</th>
+  </tr>
+  <tr></tr>
+  <tr>
+    <td><img src="../documents/STIRLING.gif"></td>
+    <td><b>Stirling</b></td>
+    <td>133</td>
+    <td><a href="http://pics.stir.ac.uk/ESRC/" target="_blank">pics.stir.ac.uk</a></td>
+  </tr>
+  <tr></tr>
+  <tr>
+    <td><img src="../documents/D3DFACS.gif"></td>
+    <td><b>D3DFACS</b></td>
+    <td>10</td>
+    <td><a href="https://www.cs.bath.ac.uk/~dpc/D3DFACS/" target="_blank">cs.bath.ac.uk</a></td>
+  </tr>
+  <tr></tr>
+  <tr>
+    <td><img src="../documents/FLORENCE.gif"></td>
+    <td><b>Florence 2D/3D</b></td>
+    <td>53</td>
+    <td><a href="https://www.micc.unifi.it/resources/datasets/florence-3d-faces/" target="_blank">micc.unifi.it</a></td>
+  </tr>
+  <tr></tr>
+  <tr>
+    <td><img src="../documents/LYHM.gif"></td>
+    <td><b>LYHM</b></td>
+    <td>1211</td>
+    <td><a href="https://www-users.york.ac.uk/~np7/research/Headspace/" target="_blank">cs.york.ac.uk</a></td>
+  </tr>
+  <tr></tr>
+  <tr>
+    <td><img src="../documents/FACEWAREHOUSE.gif"></td>
+    <td><b>FaceWarehouse</b></td>
+    <td>150</td>
+    <td><a href="http://kunzhou.net/zjugaps/facewarehouse/" target="_blank">kunzhou.net</a></td>
+  </tr>
+  <tr></tr>
+  <tr>
+    <td><img src="../documents/FRGC.gif"></td>
+    <td><b><a href="https://github.com/Zielon/MICA/tree/master/datasets/#frgc-subset">*FRGC</a></b></td>
+    <td>531</td>
+    <td><a href="https://cvrl.nd.edu/projects/data/#face-recognition-grand-challenge-frgc-v20-data-collection" target="_blank">cvrl.nd.edu</a</td>
+  </tr>
+  <tr></tr>
+  <tr>
+    <td><img src="../documents/BP4D.gif"></td>
+    <td><b><a href="https://github.com/Zielon/MICA/tree/master/datasets/#frgc-subset">*BP4D+</a></b></td>
+    <td>127</td>
+    <td><a href="http://www.cs.binghamton.edu/~lijun/Research/3DFE/3DFE_Analysis.html" target="_blank">cs.binghamton.edu</a></td>
+  </tr>
+</table>
+</div>
+
+Each subset zip file has the following structure:
+```shell
+root\
+    FLAME_parameters\
+        actor_id\
+            *.npz
+    registrations\
+        actor_id\
+            *.obj
+```
+
+To retrieve FLAME2020 parameters you can simply do:
+```python
+import numpy as np
+import torch
+
+params = np.load('path.npz', allow_pickle=True)
+pose = torch.tensor(params['pose']).float()
+betas = torch.tensor(params['betas']).float()
+
+flame = {
+    'shape_params': betas[:300],
+    'expression_params': betas[300:],
+    'pose_params': torch.cat([pose[:3], pose[6:9]]),
+}
+```
+
+### MICA Training Dataset Preparation
+
+To prepare the MICA training dataset you can follow the scripts from the [creation](https://github.com/Zielon/MICA/tree/master/datasets/creation) folder. Additionally, the complete list of images used for the training can be found in [image_paths](https://github.com/Zielon/MICA/tree/master/datasets/image_paths) folder. It contains the name of the FLAME parameters file `.npz` and a list of all images used for the training with their original name stored as dictionary.
+
+### Licence
+This dataset is for academic, non-commercial usage only. Moreover, it is an extension of already existing datasets, therefore, the license is shared and applies equivalently to both, the original and the corresponding derived one. Please read the original license of each original dataset for more information, especially in the context of data privacy.
+
+### Additional Information
+
+Please note that some subsets do not have an identical amount of subjects due to registration errors. 
+
+#### BP4D+ Subset
+
+The original dataset contains 140 subjects from where we selected scans in neutral pose and successfully registered 127 FLAME meshes for them.
+
+#### FRGC Subset
+
+1) A group desiring to obtain FLAME results on the FRGC data must license the FRGC 2.0 data set from CVRL using the licensing procedure at the website: http://cvrl.nd.edu .
+2) Once the license form is completed and CVRL has approved it, an email will be sent from Globus indicating that you have been authorized to retrieve FRGC 2.0.
+3) That authorization will serve as authorization to receive any derivative work, therefore, please send a confirmation to mica [AT] tue.mpg.de address to receive the MICA dataset.
+
+### Citation
+If you use this dataset in your research please cite MICA:
+```bibtex
+@proceedings{MICA:ECCV2022,
+  author = {Zielonka, Wojciech and Bolkart, Timo and Thies, Justus},
+  title = {Towards Metrical Reconstruction of Human Faces},
+  journal = {European Conference on Computer Vision},
+  year = {2022}
+}
+```

src/pixel3dmm/preprocessing/MICA/datasets/__init__.py

@@ -0,0 +1,55 @@
+# -*- coding: utf-8 -*-
+
+# Max-Planck-Gesellschaft zur Förderung der Wissenschaften e.V. (MPG) is
+# holder of all proprietary rights on this computer program.
+# You can only use this computer program if you have closed
+# a license agreement with MPG or you get the right to use the computer
+# program from someone who is authorized to grant you that right.
+# Any use of the computer program without a valid license is prohibited and
+# liable to prosecution.
+#
+# Copyright©2023 Max-Planck-Gesellschaft zur Förderung
+# der Wissenschaften e.V. (MPG). acting on behalf of its Max Planck Institute
+# for Intelligent Systems. All rights reserved.
+#
+# Contact: [email protected]
+
+
+import numpy as np
+from torch.utils.data import ConcatDataset
+
+from datasets.base import BaseDataset
+
+
+def build_train(config, device):
+    data_list = []
+    total_images = 0
+    for dataset in config.training_data:
+        dataset_name = dataset.upper()
+        config.n_train = np.Inf
+        if type(dataset) is list:
+            dataset_name, n_train = dataset
+            config.n_train = n_train
+
+        dataset = BaseDataset(name=dataset_name, config=config, device=device, isEval=False)
+        data_list.append(dataset)
+        total_images += dataset.total_images
+
+    return ConcatDataset(data_list), total_images
+
+
+def build_val(config, device):
+    data_list = []
+    total_images = 0
+    for dataset in config.eval_data:
+        dataset_name = dataset.upper()
+        config.n_train = np.Inf
+        if type(dataset) is list:
+            dataset_name, n_train = dataset
+            config.n_train = n_train
+
+        dataset = BaseDataset(name=dataset_name, config=config, device=device, isEval=True)
+        data_list.append(dataset)
+        total_images += dataset.total_images
+
+    return ConcatDataset(data_list), total_images

src/pixel3dmm/preprocessing/MICA/datasets/base.py

@@ -0,0 +1,125 @@
+# -*- coding: utf-8 -*-
+
+# Max-Planck-Gesellschaft zur Förderung der Wissenschaften e.V. (MPG) is
+# holder of all proprietary rights on this computer program.
+# You can only use this computer program if you have closed
+# a license agreement with MPG or you get the right to use the computer
+# program from someone who is authorized to grant you that right.
+# Any use of the computer program without a valid license is prohibited and
+# liable to prosecution.
+#
+# Copyright©2023 Max-Planck-Gesellschaft zur Förderung
+# der Wissenschaften e.V. (MPG). acting on behalf of its Max Planck Institute
+# for Intelligent Systems. All rights reserved.
+#
+# Contact: [email protected]
+
+
+import os
+import re
+from abc import ABC
+from functools import reduce
+from pathlib import Path
+
+import loguru
+import numpy as np
+import torch
+from loguru import logger
+from skimage.io import imread
+from torch.utils.data import Dataset
+from torchvision import transforms
+
+
+class BaseDataset(Dataset, ABC):
+    def __init__(self, name, config, device, isEval):
+        self.K = config.K
+        self.isEval = isEval
+        self.n_train = np.Inf
+        self.imagepaths = []
+        self.face_dict = {}
+        self.name = name
+        self.device = device
+        self.min_max_K = 0
+        self.cluster = False
+        self.dataset_root = config.root
+        self.total_images = 0
+        self.image_folder = 'arcface_input'
+        self.flame_folder = 'FLAME_parameters'
+        self.initialize()
+
+    def initialize(self):
+        logger.info(f'[{self.name}] Initialization')
+        image_list = f'{os.path.abspath(os.path.dirname(__file__))}/image_paths/{self.name}.npy'
+        logger.info(f'[{self.name}] Load cached file list: ' + image_list)
+        self.face_dict = np.load(image_list, allow_pickle=True).item()
+        self.imagepaths = list(self.face_dict.keys())
+        logger.info(f'[Dataset {self.name}] Total {len(self.imagepaths)} actors loaded!')
+        self.set_smallest_k()
+
+    def set_smallest_k(self):
+        self.min_max_K = np.Inf
+        max_min_k = -np.Inf
+        for key in self.face_dict.keys():
+            length = len(self.face_dict[key][0])
+            if length < self.min_max_K:
+                self.min_max_K = length
+            if length > max_min_k:
+                max_min_k = length
+
+        self.total_images = reduce(lambda k, l: l + k, map(lambda e: len(self.face_dict[e][0]), self.imagepaths))
+        loguru.logger.info(f'Dataset {self.name} with min K = {self.min_max_K} max K = {max_min_k} length = {len(self.face_dict)} total images = {self.total_images}')
+        return self.min_max_K
+
+    def compose_transforms(self, *args):
+        self.transforms = transforms.Compose([t for t in args])
+
+    def get_arcface_path(self, image_path):
+        return re.sub('png|jpg', 'npy', str(image_path))
+
+    def __len__(self):
+        return len(self.imagepaths)
+
+    def __getitem__(self, index):
+        actor = self.imagepaths[index]
+        images, params_path = self.face_dict[actor]
+        images = [Path(self.dataset_root, self.name, self.image_folder, path) for path in images]
+        sample_list = np.array(np.random.choice(range(len(images)), size=self.K, replace=False))
+
+        K = self.K
+        if self.isEval:
+            K = max(0, min(200, self.min_max_K))
+            sample_list = np.array(range(len(images))[:K])
+
+        params = np.load(os.path.join(self.dataset_root, self.name, self.flame_folder, params_path), allow_pickle=True)
+        pose = torch.tensor(params['pose']).float()
+        betas = torch.tensor(params['betas']).float()
+
+        flame = {
+            'shape_params': torch.cat(K * [betas[:300][None]], dim=0),
+            'expression_params': torch.cat(K * [betas[300:][None]], dim=0),
+            'pose_params': torch.cat(K * [torch.cat([pose[:3], pose[6:9]])[None]], dim=0),
+        }
+
+        images_list = []
+        arcface_list = []
+
+        for i in sample_list:
+            image_path = images[i]
+            image = np.array(imread(image_path))
+            image = image / 255.
+            image = image.transpose(2, 0, 1)
+            arcface_image = np.load(self.get_arcface_path(image_path), allow_pickle=True)
+
+            images_list.append(image)
+            arcface_list.append(torch.tensor(arcface_image))
+
+        images_array = torch.from_numpy(np.array(images_list)).float()
+        arcface_array = torch.stack(arcface_list).float()
+
+        return {
+            'image': images_array,
+            'arcface': arcface_array,
+            'imagename': actor,
+            'dataset': self.name,
+            'flame': flame,
+        }

src/pixel3dmm/preprocessing/MICA/datasets/creation/generator.py

@@ -0,0 +1,98 @@
+# -*- coding: utf-8 -*-
+
+# Max-Planck-Gesellschaft zur Förderung der Wissenschaften e.V. (MPG) is
+# holder of all proprietary rights on this computer program.
+# You can only use this computer program if you have closed
+# a license agreement with MPG or you get the right to use the computer
+# program from someone who is authorized to grant you that right.
+# Any use of the computer program without a valid license is prohibited and
+# liable to prosecution.
+#
+# Copyright©2023 Max-Planck-Gesellschaft zur Förderung
+# der Wissenschaften e.V. (MPG). acting on behalf of its Max Planck Institute
+# for Intelligent Systems. All rights reserved.
+#
+# Contact: [email protected]
+
+
+import os
+from glob import glob
+from multiprocessing import Pool
+from pathlib import Path
+from typing import List
+
+import cv2
+import numpy as np
+from insightface.app import FaceAnalysis
+from insightface.app.common import Face
+from insightface.utils import face_align
+from loguru import logger
+from tqdm import tqdm
+
+from datasets.creation.instances.instance import Instance
+from datasets.creation.util import get_image, get_center, get_arcface_input
+
+
+def _transfer(src, dst):
+    src.parent.mkdir(parents=True, exist_ok=True)
+    dst.parent.mkdir(parents=True, exist_ok=True)
+    os.system(f'cp {str(src)} {str(dst)}')
+
+
+def _copy(payload):
+    instance, func, target, transform_path = payload
+    files = func()
+    for actor in files.keys():
+        for file in files[actor]:
+            _transfer(Path(file), Path(instance.get_dst(), target, actor, transform_path(file)))
+
+
+class Generator:
+    def __init__(self, instances):
+        self.instances: List[Instance] = instances
+        self.ARCFACE = 'arcface_input'
+
+    def copy(self):
+        logger.info('Start copying...')
+        for instance in tqdm(self.instances):
+            payloads = [(instance, instance.get_images, 'images', instance.transform_path)]
+            with Pool(processes=len(payloads)) as pool:
+                for _ in tqdm(pool.imap_unordered(_copy, payloads), total=len(payloads)):
+                    pass
+
+    def preprocess(self):
+        logger.info('Start preprocessing...')
+        for instance in tqdm(self.instances):
+            instance.preprocess()
+
+    def arcface(self):
+        app = FaceAnalysis(name='antelopev2', providers=['CUDAExecutionProvider'])
+        app.prepare(ctx_id=0, det_size=(224, 224))
+
+        logger.info('Start arcface...')
+        for instance in tqdm(self.instances):
+            src = instance.get_dst()
+            for image_path in tqdm(sorted(glob(f'{src}/images/*/*'))):
+                dst = image_path.replace('images', self.ARCFACE)
+                Path(dst).parent.mkdir(exist_ok=True, parents=True)
+                for img in instance.transform_image(get_image(image_path[0:-4])):
+                    bboxes, kpss = app.det_model.detect(img, max_num=0, metric='default')
+                    if bboxes.shape[0] == 0:
+                        continue
+                    i = get_center(bboxes, img)
+                    bbox = bboxes[i, 0:4]
+                    det_score = bboxes[i, 4]
+                    if det_score < instance.get_min_det_score():
+                        continue
+                    kps = None
+                    if kpss is not None:
+                        kps = kpss[i]
+                    face = Face(bbox=bbox, kps=kps, det_score=det_score)
+                    blob, aimg = get_arcface_input(face, img)
+                    np.save(dst[0:-4], blob)
+                    cv2.imwrite(dst, face_align.norm_crop(img, landmark=face.kps, image_size=224))
+
+    def run(self):
+        self.copy()
+        self.preprocess()
+        self.arcface()

src/pixel3dmm/preprocessing/MICA/datasets/creation/instances/bu3dfe.py

@@ -0,0 +1,71 @@
+# -*- coding: utf-8 -*-
+
+# Max-Planck-Gesellschaft zur Förderung der Wissenschaften e.V. (MPG) is
+# holder of all proprietary rights on this computer program.
+# You can only use this computer program if you have closed
+# a license agreement with MPG or you get the right to use the computer
+# program from someone who is authorized to grant you that right.
+# Any use of the computer program without a valid license is prohibited and
+# liable to prosecution.
+#
+# Copyright©2023 Max-Planck-Gesellschaft zur Förderung
+# der Wissenschaften e.V. (MPG). acting on behalf of its Max Planck Institute
+# for Intelligent Systems. All rights reserved.
+#
+# Contact: [email protected]
+
+
+from abc import ABC
+from glob import glob
+from pathlib import Path
+
+from pytorch3d.io import load_objs_as_meshes
+
+from datasets.creation.instances.instance import Instance
+
+
+class BU3DFE(Instance, ABC):
+    def __init__(self):
+        super(BU3DFE, self).__init__()
+        self.dst = '/scratch/NFC/OnFlame/BU3DFE/'
+        self.src = '/scratch/NFC/BU-3DFE/'
+
+    def get_images(self):
+        images = {}
+        for actor in sorted(glob(self.get_src().replace('BU-3DFE', 'BU-3DFE_clean') + 'images/*')):
+            images[Path(actor).name] = glob(f'{actor}/*.jpg')
+
+        return images
+
+    def get_flame_params(self):
+        prams = {}
+        for actor in sorted(glob(self.get_src() + 'FLAME_parameters/iter2/*')):
+            prams[Path(actor).name] = glob(f'{actor}/*.npz')
+
+        return prams
+
+    def get_registrations(self):
+        registrations = {}
+        for actor in sorted(glob(self.get_src() + 'registrations/iter2/neutral_align/*')):
+            registrations[Path(actor).name] = glob(f'{actor}/*.obj')
+
+        return registrations
+
+    def get_meshes(self):
+        meshes = {}
+        files = sorted(glob(self.get_src() + 'raw_ne_data/*'))
+        actors = set(map(lambda f: Path(f).name[0:5], files))
+        for actor in actors:
+            meshes[Path(actor).name] = next(filter(lambda f: actor in f and 'obj' in f, files))
+
+        return meshes
+
+    def transform_mesh(self, path):
+        self.update_obj(path)
+        mesh = load_objs_as_meshes([path], device=self.device)
+        vertices = mesh._verts_list[0]
+        center = vertices.mean(0)
+        mesh._verts_list = [vertices - center]
+        mesh.scale_verts_(0.01)
+
+        return mesh.clone()

src/pixel3dmm/preprocessing/MICA/datasets/creation/instances/d3dfacs.py

@@ -0,0 +1,53 @@
+# -*- coding: utf-8 -*-
+
+# Max-Planck-Gesellschaft zur Förderung der Wissenschaften e.V. (MPG) is
+# holder of all proprietary rights on this computer program.
+# You can only use this computer program if you have closed
+# a license agreement with MPG or you get the right to use the computer
+# program from someone who is authorized to grant you that right.
+# Any use of the computer program without a valid license is prohibited and
+# liable to prosecution.
+#
+# Copyright©2023 Max-Planck-Gesellschaft zur Förderung
+# der Wissenschaften e.V. (MPG). acting on behalf of its Max Planck Institute
+# for Intelligent Systems. All rights reserved.
+#
+# Contact: [email protected]
+
+
+from abc import ABC
+from glob import glob
+from pathlib import Path
+
+from datasets.creation.instances.instance import Instance
+
+
+class D3DFACS(Instance, ABC):
+    def __init__(self):
+        super(D3DFACS, self).__init__()
+        self.dst = '/scratch/NFC/OnFlame/D3DFACS/'
+        self.src = '/home/wzielonka/datasets/D3DFACS/'
+
+    def get_images(self):
+        images = {}
+        for file in sorted(glob(self.get_src() + 'processed/images/*')):
+            actor = Path(file).stem
+            images[actor] = glob(f'{file}/*.jpg')
+
+        return images
+
+    def get_flame_params(self):
+        params = {}
+        for file in sorted(glob(self.get_src() + 'processed/FLAME/*.npz')):
+            actor = Path(file).stem
+            params[actor] = [file]
+
+        return params
+
+    def get_registrations(self):
+        registrations = {}
+        for file in sorted(glob(self.get_src() + 'processed/registrations/*')):
+            actor = Path(file).stem.split('_')[0]
+            registrations[actor] = [file]
+
+        return registrations

src/pixel3dmm/preprocessing/MICA/datasets/creation/instances/facewarehouse.py

@@ -0,0 +1,50 @@
+# -*- coding: utf-8 -*-
+
+# Max-Planck-Gesellschaft zur Förderung der Wissenschaften e.V. (MPG) is
+# holder of all proprietary rights on this computer program.
+# You can only use this computer program if you have closed
+# a license agreement with MPG or you get the right to use the computer
+# program from someone who is authorized to grant you that right.
+# Any use of the computer program without a valid license is prohibited and
+# liable to prosecution.
+#
+# Copyright©2023 Max-Planck-Gesellschaft zur Förderung
+# der Wissenschaften e.V. (MPG). acting on behalf of its Max Planck Institute
+# for Intelligent Systems. All rights reserved.
+#
+# Contact: [email protected]
+
+
+from abc import ABC
+from glob import glob
+from pathlib import Path
+
+from datasets.creation.instances.instance import Instance
+
+
+class FaceWarehouse(Instance, ABC):
+    def __init__(self):
+        super(FaceWarehouse, self).__init__()
+        self.dst = '/scratch/NFC/OnFlame/FACEWAREHOUSE/'
+        self.src = '/scratch/NFC/FaceWarehouse/'
+
+    def get_images(self):
+        images = {}
+        for actor in sorted(glob(self.get_src() + 'Images/*')):
+            images[Path(actor).stem] = glob(f'{actor}/*.png')
+
+        return images
+
+    def get_flame_params(self):
+        params = {}
+        for actor in sorted(glob(self.get_src() + 'FLAME_fits/*')):
+            params[Path(actor).stem] = [sorted(glob(f'{actor}/*.npz'))[0]]
+
+        return params
+
+    def get_registrations(self):
+        registrations = {}
+        for actor in sorted(glob(self.get_src() + 'FLAME_fits/*')):
+            registrations[Path(actor).stem] = [f'{actor}/tmp/pose_0__def_trafo_fit.obj']
+
+        return registrations

src/pixel3dmm/preprocessing/MICA/datasets/creation/instances/florence.py

@@ -0,0 +1,59 @@
+# -*- coding: utf-8 -*-
+
+# Max-Planck-Gesellschaft zur Förderung der Wissenschaften e.V. (MPG) is
+# holder of all proprietary rights on this computer program.
+# You can only use this computer program if you have closed
+# a license agreement with MPG or you get the right to use the computer
+# program from someone who is authorized to grant you that right.
+# Any use of the computer program without a valid license is prohibited and
+# liable to prosecution.
+#
+# Copyright©2023 Max-Planck-Gesellschaft zur Förderung
+# der Wissenschaften e.V. (MPG). acting on behalf of its Max Planck Institute
+# for Intelligent Systems. All rights reserved.
+#
+# Contact: [email protected]
+
+
+from abc import ABC
+from glob import glob
+from pathlib import Path
+
+import numpy as np
+
+from datasets.creation.instances.instance import Instance
+
+
+class Florence(Instance, ABC):
+    def __init__(self):
+        super(Florence, self).__init__()
+        self.dst = '/scratch/NFC/OnFlame/FLORENCE/'
+        self.src = '/scratch/NFC/MICC_Florence/'
+
+    def get_min_det_score(self):
+        return 0.85
+
+    def get_images(self):
+        images = {}
+        for actor in sorted(glob(self.get_src() + 'images/*')):
+            imgs = sorted(list(filter(lambda f: 'PTZ-Outdoor' not in f, glob(f'{actor}/*/*.jpg'))))
+            indecies = np.random.choice(len(imgs), 1000, replace=False)
+            images[Path(actor).stem] = [imgs[i] for i in indecies]
+
+        return images
+
+    def get_flame_params(self):
+        params = {}
+        for actor in sorted(glob(self.get_src() + 'FLAME_parameters/iter1/*')):
+            params[Path(actor).stem] = glob(f'{actor}/*.npz')
+
+        return params
+
+    def get_registrations(self):
+        registrations = {}
+        for actor in sorted(glob(self.get_src() + 'registrations/iter1/*')):
+            if 'rendering' in actor:
+                continue
+            registrations[Path(actor).stem] = glob(f'{actor}/*.obj')
+
+        return registrations

src/pixel3dmm/preprocessing/MICA/datasets/creation/instances/frgc.py

@@ -0,0 +1,84 @@
+# -*- coding: utf-8 -*-
+
+# Max-Planck-Gesellschaft zur Förderung der Wissenschaften e.V. (MPG) is
+# holder of all proprietary rights on this computer program.
+# You can only use this computer program if you have closed
+# a license agreement with MPG or you get the right to use the computer
+# program from someone who is authorized to grant you that right.
+# Any use of the computer program without a valid license is prohibited and
+# liable to prosecution.
+#
+# Copyright©2023 Max-Planck-Gesellschaft zur Förderung
+# der Wissenschaften e.V. (MPG). acting on behalf of its Max Planck Institute
+# for Intelligent Systems. All rights reserved.
+#
+# Contact: [email protected]
+
+
+from abc import ABC
+from glob import glob
+from pathlib import Path
+
+import numpy as np
+from pytorch3d.io import load_objs_as_meshes
+
+from datasets.creation.instances.instance import Instance
+
+
+class FRGC(Instance, ABC):
+    def __init__(self):
+        super(FRGC, self).__init__()
+        self.dst = '/scratch/NFC/OnFlame/FRGC/'
+        self.src = '/scratch/NFC/FRGC_v2/'
+
+    def get_images(self):
+        images = {}
+        for actor in sorted(glob(self.get_src() + 'images/*')):
+            imgs = list(filter(lambda f: 'Spring2003range' not in f, glob(f'/{actor}/*/*.jpg')))
+            images[Path(actor).name] = imgs
+
+        return images
+
+    def get_flame_params(self):
+        prams = {}
+        for actor in sorted(glob(self.get_src() + 'FLAME_parameters/*')):
+            prams[Path(actor).name] = glob(f'/{actor}/*.npz')
+
+        return prams
+
+    def get_registrations(self):
+        registrations = {}
+        for actor in sorted(glob(self.get_src() + 'registrations/*')):
+            registrations[Path(actor).name] = glob(f'/{actor}/*.obj')
+
+        return registrations
+
+    def get_meshes(self):
+        meshes = {}
+        for file in sorted(glob(self.get_src() + 'registrations_tmp_new/*')):
+            meshes[Path(file).name] = glob(f'/{file}/*.obj')
+
+        sessions = np.load('/home/wzielonka/documents/scans_to_session.npy', allow_pickle=True)[()]
+        valid = []
+        for key in sessions.keys():
+            if 'Spring2003range' not in sessions[key]:
+                valid.append(key)
+
+        filtered = {}
+        for actor in meshes.keys():
+            files = meshes[actor]
+            selected = list(filter(lambda f: Path(f).stem in valid, files))
+            if len(selected) > 0:
+                filtered[actor] = selected
+
+        return filtered
+
+    def transform_mesh(self, path):
+        self.update_obj(path[0])
+        mesh = load_objs_as_meshes(path, device=self.device)
+        mesh.scale_verts_(10.0)
+        vertices = mesh._verts_list[0]
+        center = vertices.mean(0)
+        mesh._verts_list = [vertices - center]
+
+        return mesh.clone()

src/pixel3dmm/preprocessing/MICA/datasets/creation/instances/instance.py

@@ -0,0 +1,127 @@
+# -*- coding: utf-8 -*-
+
+# Max-Planck-Gesellschaft zur Förderung der Wissenschaften e.V. (MPG) is
+# holder of all proprietary rights on this computer program.
+# You can only use this computer program if you have closed
+# a license agreement with MPG or you get the right to use the computer
+# program from someone who is authorized to grant you that right.
+# Any use of the computer program without a valid license is prohibited and
+# liable to prosecution.
+#
+# Copyright©2023 Max-Planck-Gesellschaft zur Förderung
+# der Wissenschaften e.V. (MPG). acting on behalf of its Max Planck Institute
+# for Intelligent Systems. All rights reserved.
+#
+# Contact: [email protected]
+
+
+import os
+from abc import abstractmethod
+from pathlib import Path
+
+from pytorch3d.transforms import RotateAxisAngle
+
+
+class Instance:
+    def __init__(self):
+        self.mount = '/home/wzielonka/Cluster/lustre'
+        self.dst = 'empty'
+        self.src = 'empty'
+        self.device = 'cuda:0'
+        self.actors = []
+        self.use_mount = os.path.exists(self.mount)
+
+    def get_dst(self):
+        return self.dst if not self.use_mount else self.mount + self.dst
+
+    def get_src(self):
+        return self.src if not self.use_mount else self.mount + self.src
+
+    @abstractmethod
+    def get_min_det_score(self):
+        return 0
+
+    @abstractmethod
+    def preprocess(self):
+        pass
+
+    @abstractmethod
+    def get_images(self):
+        return {}
+
+    @abstractmethod
+    def get_flame_params(self):
+        return {}
+
+    @abstractmethod
+    def get_registrations(self):
+        return {}
+
+    @abstractmethod
+    def get_meshes(self):
+        return {}
+
+    @abstractmethod
+    def transform_mesh(self, path):
+        return None
+
+    @abstractmethod
+    def transform_image(self, img):
+        return [img]
+
+    @abstractmethod
+    def transform_path(self, file):
+        return Path(file).name
+
+    @abstractmethod
+    def get_rotations(self):
+        rots = {}
+        degree = 2.5
+        step = int(15 / degree / 2)
+        X = range(-step, step + 1)
+        degree = 8.0
+        step = int(144 / degree / 2)
+        Y = range(-step, step + 1)
+        for a, angles in [('X', X), ('Y', Y)]:
+            r = []
+            for i in angles:
+                r.append((RotateAxisAngle(float(degree * i), axis=a, device=self.device), float(degree * i)))
+            rots[a] = r
+        return rots
+
+    @abstractmethod
+    def update_obj(self, path, fix_mtl=False):
+        mesh = Path(path).stem
+        with open(path, 'r') as file:
+            filedata = file.readlines()
+
+        input = []
+        for line in filedata:
+            if 'usemtl' in line or 'newmtl' in line:
+                continue
+            input.append(line)
+
+        output = []
+        for line in input:
+            if 'mtllib' in line:
+                mtl = line.split(' ')[-1].split('.')[0]
+                line += f'usemtl {mtl}\n'
+            output.append(line)
+        with open(path, 'w') as file:
+            file_lines = "".join(output)
+            file.write(file_lines)
+
+        if not fix_mtl:
+            return
+
+        with open(path.replace('obj', 'mtl'), 'r') as file:
+            filedata = file.readlines()
+
+        output = []
+        for line in filedata:
+            if 'newmtl' in line:
+                line = 'newmtl ' + mesh + '\n'
+            output.append(line)
+        with open(path.replace('obj', 'mtl'), 'w') as file:
+            file_lines = "".join(output)
+            file.write(file_lines)

src/pixel3dmm/preprocessing/MICA/datasets/creation/instances/lyhm.py

@@ -0,0 +1,78 @@
+# -*- coding: utf-8 -*-
+
+# Max-Planck-Gesellschaft zur Förderung der Wissenschaften e.V. (MPG) is
+# holder of all proprietary rights on this computer program.
+# You can only use this computer program if you have closed
+# a license agreement with MPG or you get the right to use the computer
+# program from someone who is authorized to grant you that right.
+# Any use of the computer program without a valid license is prohibited and
+# liable to prosecution.
+#
+# Copyright©2023 Max-Planck-Gesellschaft zur Förderung
+# der Wissenschaften e.V. (MPG). acting on behalf of its Max Planck Institute
+# for Intelligent Systems. All rights reserved.
+#
+# Contact: [email protected]
+
+
+from abc import ABC
+from glob import glob
+from pathlib import Path
+
+from PIL import ImageFile
+
+ImageFile.LOAD_TRUNCATED_IMAGES = True
+from pytorch3d.io import load_objs_as_meshes
+from pytorch3d.transforms import RotateAxisAngle
+
+from datasets.creation.instances.instance import Instance
+
+
+class LYHM(Instance, ABC):
+    def __init__(self):
+        super(LYHM, self).__init__()
+        self.dst = '/scratch/NFC/MICA/LYHM/'
+        self.src = '/scratch/NFC/LYHM/'
+
+    def get_images(self):
+        images = {}
+        for actor in sorted(glob(self.get_src() + '/*')):
+            images[Path(actor).name] = glob(f'/{actor}/*.png')
+
+        return images
+
+    def get_flame_params(self):
+        prams = {}
+        for actor in sorted(glob(self.get_src() + '/*')):
+            prams[Path(actor).name] = glob(f'/{actor}/*.npz')
+
+        return prams
+
+    def get_registrations(self):
+        registrations = {}
+        for actor in sorted(glob(self.get_src() + '/*')):
+            all = glob(f'/{actor}/*.obj')
+            registrations[Path(actor).name] = list(filter(lambda m: 'model_fit' not in m, all))
+
+        return registrations
+
+    def get_meshes(self):
+        meshes = {}
+        for actor in sorted(glob(self.get_src() + '/*')):
+            meshes[Path(actor).name] = glob(f'/{actor}/scan/*.obj')
+
+        return meshes
+
+    def transform_mesh(self, path):
+        mesh = load_objs_as_meshes(path, device=self.device)
+        vertices = mesh._verts_list[0]
+        center = vertices.mean(0)
+        mesh._verts_list = [vertices - center]
+        mesh.scale_verts_(0.01)
+
+        rot = RotateAxisAngle(-45, axis='X', device=self.device)
+        mesh._verts_list = [rot.transform_points(mesh.verts_list()[0])]
+        rot = RotateAxisAngle(-45, axis='Y', device=self.device)
+        mesh._verts_list = [rot.transform_points(mesh.verts_list()[0])]
+
+        return mesh.clone()

src/pixel3dmm/preprocessing/MICA/datasets/creation/instances/pb4d.py

@@ -0,0 +1,71 @@
+# -*- coding: utf-8 -*-
+
+# Max-Planck-Gesellschaft zur Förderung der Wissenschaften e.V. (MPG) is
+# holder of all proprietary rights on this computer program.
+# You can only use this computer program if you have closed
+# a license agreement with MPG or you get the right to use the computer
+# program from someone who is authorized to grant you that right.
+# Any use of the computer program without a valid license is prohibited and
+# liable to prosecution.
+#
+# Copyright©2023 Max-Planck-Gesellschaft zur Förderung
+# der Wissenschaften e.V. (MPG). acting on behalf of its Max Planck Institute
+# for Intelligent Systems. All rights reserved.
+#
+# Contact: [email protected]
+
+
+from abc import ABC
+from glob import glob
+from pathlib import Path
+
+import numpy as np
+from pytorch3d.io import load_objs_as_meshes
+
+from datasets.creation.instances.instance import Instance
+
+
+class PB4D(Instance, ABC):
+    def __init__(self):
+        super(PB4D, self).__init__()
+        self.dst = '/scratch/NFC/OnFlame/BP4D/'
+        self.src = '/scratch/NFC/BP4D/'
+
+    def get_images(self):
+        images = {}
+        for actor in sorted(glob(self.get_src() + 'images/*')):
+            imgs = sorted(glob(f'/{actor}/*.jpg'))
+            indecies = np.random.choice(len(imgs), 100, replace=False)
+            images[Path(actor).name] = [imgs[i] for i in indecies]
+
+        return images
+
+    def get_flame_params(self):
+        prams = {}
+        for file in sorted(glob(self.get_src() + 'FLAME_parameters/*.npz')):
+            prams[Path(file).stem] = [file]
+
+        return prams
+
+    def get_registrations(self):
+        registrations = {}
+        for file in sorted(glob(self.get_src() + 'registrations/*')):
+            registrations[Path(file).stem] = [file]
+
+        return registrations
+
+    def get_meshes(self):
+        meshes = {}
+        for file in sorted(glob(self.get_src() + 'scans/*.obj')):
+            meshes[Path(file).stem] = [file]
+
+        return meshes
+
+    def transform_mesh(self, path):
+        mesh = load_objs_as_meshes(path, device=self.device)
+        mesh.scale_verts_(0.01)
+        vertices = mesh._verts_list[0]
+        center = vertices.mean(0)
+        mesh._verts_list = [vertices - center]
+
+        return mesh.clone()

src/pixel3dmm/preprocessing/MICA/datasets/creation/instances/stirling.py

@@ -0,0 +1,85 @@
+# -*- coding: utf-8 -*-
+
+# Max-Planck-Gesellschaft zur Förderung der Wissenschaften e.V. (MPG) is
+# holder of all proprietary rights on this computer program.
+# You can only use this computer program if you have closed
+# a license agreement with MPG or you get the right to use the computer
+# program from someone who is authorized to grant you that right.
+# Any use of the computer program without a valid license is prohibited and
+# liable to prosecution.
+#
+# Copyright©2023 Max-Planck-Gesellschaft zur Förderung
+# der Wissenschaften e.V. (MPG). acting on behalf of its Max Planck Institute
+# for Intelligent Systems. All rights reserved.
+#
+# Contact: [email protected]
+
+
+from abc import ABC
+from glob import glob
+from pathlib import Path
+
+from pytorch3d.io import load_objs_as_meshes
+
+from datasets.creation.instances.instance import Instance
+
+
+class Stirling(Instance, ABC):
+    def __init__(self):
+        super(Stirling, self).__init__()
+        self.dst = '/scratch/NFC/OnFlame/STIRLING/'
+        self.src = '/scratch/NFC/Stirling/'
+
+    def get_min_det_score(self):
+        return 0.75
+
+    def get_images(self):
+        images = {}
+        for file in sorted(glob(self.get_src() + 'images/Real_images__Subset_2D_FG2018/HQ/*')):
+            actor = Path(file).stem.split('_')[0].upper()
+            if actor not in images:
+                images[actor] = []
+            images[actor].append(file)
+
+        return images
+
+    def get_flame_params(self):
+        prams = {}
+        for file in sorted(glob(self.get_src() + 'FLAME_parameters/iter1/*/*.npz')):
+            actor = Path(file).stem[0:5].upper()
+            prams[Path(actor).name] = [file]
+
+        return prams
+
+    def get_registrations(self):
+        registrations = {}
+        for file in sorted(glob(self.get_src() + 'registrations/iter1/*/*')):
+            if 'obj' not in file:
+                continue
+            actor = Path(file).stem[0:5].upper()
+            registrations[Path(actor).name] = [file]
+
+        return registrations
+
+    def get_meshes(self):
+        meshes = {}
+        for file in sorted(glob(self.get_src() + 'scans/*/*.obj')):
+            actor = Path(file).stem[0:5].upper()
+            if 'obj' in file:
+                meshes[actor] = file
+
+        return meshes
+
+    def transform_mesh(self, path):
+        self.update_obj(path, fix_mtl=True)
+        mesh = load_objs_as_meshes([path], device=self.device)
+        vertices = mesh._verts_list[0]
+        center = vertices.mean(0)
+        mesh._verts_list = [vertices - center]
+        mesh.scale_verts_(0.01)
+
+        return mesh.clone()
+
+    def transform_path(self, file):
+        name = Path(file).name
+        return name

src/pixel3dmm/preprocessing/MICA/datasets/creation/main.py

@@ -0,0 +1,39 @@
+# -*- coding: utf-8 -*-
+
+# Max-Planck-Gesellschaft zur Förderung der Wissenschaften e.V. (MPG) is
+# holder of all proprietary rights on this computer program.
+# You can only use this computer program if you have closed
+# a license agreement with MPG or you get the right to use the computer
+# program from someone who is authorized to grant you that right.
+# Any use of the computer program without a valid license is prohibited and
+# liable to prosecution.
+#
+# Copyright©2023 Max-Planck-Gesellschaft zur Förderung
+# der Wissenschaften e.V. (MPG). acting on behalf of its Max Planck Institute
+# for Intelligent Systems. All rights reserved.
+#
+# Contact: [email protected]
+
+
+import numpy as np
+import torch
+
+from datasets.creation.generator import Generator
+from datasets.creation.instances.bu3dfe import BU3DFE
+from datasets.creation.instances.d3dfacs import D3DFACS
+from datasets.creation.instances.facewarehouse import FaceWarehouse
+from datasets.creation.instances.florence import Florence
+from datasets.creation.instances.frgc import FRGC
+from datasets.creation.instances.lyhm import LYHM
+from datasets.creation.instances.pb4d import PB4D
+from datasets.creation.instances.stirling import Stirling
+
+np.random.seed(42)
+
+if __name__ == '__main__':
+    torch.multiprocessing.set_start_method('spawn')
+
+    datasets = [FaceWarehouse(), LYHM(), D3DFACS(), FRGC(), Florence(), Stirling(), BU3DFE(), PB4D()]
+    generator = Generator([FaceWarehouse()])
+
+    generator.run()

src/pixel3dmm/preprocessing/MICA/datasets/creation/util.py

@@ -0,0 +1,146 @@
+# -*- coding: utf-8 -*-
+
+# Max-Planck-Gesellschaft zur Förderung der Wissenschaften e.V. (MPG) is
+# holder of all proprietary rights on this computer program.
+# You can only use this computer program if you have closed
+# a license agreement with MPG or you get the right to use the computer
+# program from someone who is authorized to grant you that right.
+# Any use of the computer program without a valid license is prohibited and
+# liable to prosecution.
+#
+# Copyright©2023 Max-Planck-Gesellschaft zur Förderung
+# der Wissenschaften e.V. (MPG). acting on behalf of its Max Planck Institute
+# for Intelligent Systems. All rights reserved.
+#
+# Contact: [email protected]
+
+
+import os
+import os.path as osp
+from pathlib import Path
+
+import cv2
+import numpy as np
+from insightface.utils import face_align
+from numpy.lib import math
+
+input_mean = 127.5
+input_std = 127.5
+
+
+def create_folders(folders):
+    if not type(folders) is list:
+        folders = folders.split('/')
+    parents = '/'
+    for folder in folders:
+        parents = os.path.join(parents, folder)
+        if os.path.exists(parents):
+            continue
+        Path(parents).mkdir(exist_ok=True)
+
+
+def get_arcface_input(face, img):
+    aimg = face_align.norm_crop(img, landmark=face.kps)
+    blob = cv2.dnn.blobFromImages([aimg], 1.0 / input_std, (112, 112), (input_mean, input_mean, input_mean), swapRB=True)
+    return blob[0], aimg
+
+
+def get_image(name, to_rgb=False):
+    images_dir = osp.join(Path(__file__).parent.absolute(), '../images')
+    ext_names = ['.jpg', '.png', '.jpeg']
+    image_file = None
+    for ext_name in ext_names:
+        _image_file = osp.join(images_dir, "%s%s" % (name, ext_name))
+        if osp.exists(_image_file):
+            image_file = _image_file
+            break
+    assert image_file is not None, '%s not found' % name
+    img = cv2.imread(image_file)
+    if to_rgb:
+        img = img[:, :, ::-1]
+    return img
+
+
+# from the original insightface.app.face_analysis.py file
+def draw_on(img, faces):
+    import cv2
+    dimg = img.copy()
+    for i in range(len(faces)):
+        face = faces[i]
+        box = face.bbox.astype(np.int)
+        color = (0, 0, 255)
+        cv2.rectangle(dimg, (box[0], box[1]), (box[2], box[3]), color, 2)
+        if face.kps is not None:
+            kps = face.kps.astype(np.int)
+            # print(landmark.shape)
+            for l in range(kps.shape[0]):
+                color = (0, 0, 255)
+                if l == 0 or l == 3:
+                    color = (0, 255, 0)
+                cv2.circle(dimg, (kps[l][0], kps[l][1]), 1, color,
+                           2)
+        if face.gender is not None and face.age is not None:
+            cv2.putText(dimg, '%s,%d' % (face.sex, face.age), (box[0] - 1, box[1] - 4), cv2.FONT_HERSHEY_COMPLEX, 0.7, (0, 255, 0), 1)
+
+    return dimg
+
+
+def dist(p1, p2):
+    return math.sqrt(((p1[0] - p2[0]) ** 2) + ((p1[1] - p2[1]) ** 2))
+
+
+def get_center(bboxes, img):
+    img_center = img.shape[1] // 2, img.shape[0] // 2
+    size = bboxes.shape[0]
+    distance = np.Inf
+    j = 0
+    for i in range(size):
+        x1, y1, x2, y2 = bboxes[i, 0:4]
+        dx = abs(x2 - x1) / 2.0
+        dy = abs(y2 - y1) / 2.0
+        current = dist((x1 + dx, y1 + dy), img_center)
+        if current < distance:
+            distance = current
+            j = i
+
+    return j
+
+
+def bbox2point(left, right, top, bottom, type='bbox'):
+    if type == 'kpt68':
+        old_size = (right - left + bottom - top) / 2 * 1.1
+        center = np.array([right - (right - left) / 2.0, bottom - (bottom - top) / 2.0])
+    elif type == 'bbox':
+        old_size = (right - left + bottom - top) / 2
+        center = np.array([right - (right - left) / 2.0, bottom - (bottom - top) / 2.0 + old_size * 0.12])
+    else:
+        raise NotImplementedError
+    return old_size, center
+
+
+def get_bbox(image, lmks, bb_scale=1.0):
+    h, w, c = image.shape
+    bbox = []
+    for i in range(lmks.shape[0]):
+        lmks = lmks.astype(np.int32)
+        x_min, x_max, y_min, y_max = np.min(lmks[i, :, 0]), np.max(lmks[i, :, 0]), np.min(lmks[i, :, 1]), np.max(lmks[i, :, 1])
+        x_center, y_center = int((x_max + x_min) / 2.0), int((y_max + y_min) / 2.0)
+        size = int(bb_scale * 2 * max(x_center - x_min, y_center - y_min))
+        xb_min, xb_max, yb_min, yb_max = max(x_center - size // 2, 0), min(x_center + size // 2, w - 1), \
+            max(y_center - size // 2, 0), min(y_center + size // 2, h - 1)
+
+        yb_max = min(yb_max, h - 1)
+        xb_max = min(xb_max, w - 1)
+        yb_min = max(yb_min, 0)
+        xb_min = max(xb_min, 0)
+
+        if (xb_max - xb_min) % 2 != 0:
+            xb_min += 1
+
+        if (yb_max - yb_min) % 2 != 0:
+            yb_min += 1
+
+        # x1, y1, x2, y2
+        bbox.append(np.array([xb_min, yb_min, xb_max, yb_max, 0]))
+
+    return np.stack(bbox)

src/pixel3dmm/preprocessing/MICA/datasets/image_paths/BP4D.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e625d679f72e3aa2e00ac4faed0b07d6957c94986a96746a4f8de29997e77bba
+size 284773

src/pixel3dmm/preprocessing/MICA/datasets/image_paths/BU3DFE.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3f144740162e25f9067a73f298e8229de60a17eccf5ba5e56ab8f0789688518e
+size 196059

src/pixel3dmm/preprocessing/MICA/datasets/image_paths/D3DFACS.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0270c09ae15eec72a94cfbc3a1502b970330698e290a10f2c4b8176341f4ae87
+size 207899

src/pixel3dmm/preprocessing/MICA/datasets/image_paths/FACEWAREHOUSE.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2e7e9f2bce6fce23e4f5861ebe71e0402bafdf6383217a9d20e1130787b01d3b
+size 101433

src/pixel3dmm/preprocessing/MICA/datasets/image_paths/FLORENCE.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:794583b78501cda2658aa1fac4061ac7f75271ee35d71f090127f3935491ce42
+size 500567

src/pixel3dmm/preprocessing/MICA/datasets/image_paths/FRGC.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dfa3e47a66f6f79b2a44091322c5b5680a32018774c2121d7e53f0f72a358c28
+size 1232221

src/pixel3dmm/preprocessing/MICA/datasets/image_paths/LYHM.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ab68b388abd4b5cf2dbc8cf46058ba8c26a171b8bbbff06d20082ba1b2780974
+size 126716

src/pixel3dmm/preprocessing/MICA/datasets/image_paths/STIRLING.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4fcd43ec4af033f908c660a825a948621d071ac9fa7f0292943c3519dfd6805c
+size 23761

src/pixel3dmm/preprocessing/MICA/demo.py

@@ -0,0 +1,156 @@
+# -*- coding: utf-8 -*-
+
+# Max-Planck-Gesellschaft zur Förderung der Wissenschaften e.V. (MPG) is
+# holder of all proprietary rights on this computer program.
+# You can only use this computer program if you have closed
+# a license agreement with MPG or you get the right to use the computer
+# program from someone who is authorized to grant you that right.
+# Any use of the computer program without a valid license is prohibited and
+# liable to prosecution.
+#
+# Copyright©2023 Max-Planck-Gesellschaft zur Förderung
+# der Wissenschaften e.V. (MPG). acting on behalf of its Max Planck Institute
+# for Intelligent Systems. All rights reserved.
+#
+# Contact: [email protected]
+
+
+import argparse
+import os
+import random
+from glob import glob
+from pathlib import Path
+
+import cv2
+import numpy as np
+import torch
+import torch.backends.cudnn as cudnn
+import trimesh
+from insightface.app.common import Face
+from insightface.utils import face_align
+from loguru import logger
+from skimage.io import imread
+from tqdm import tqdm
+
+from configs.config import get_cfg_defaults
+from datasets.creation.util import get_arcface_input, get_center, draw_on
+from utils import util
+from utils.landmark_detector import LandmarksDetector, detectors
+
+
+def deterministic(rank):
+    torch.manual_seed(rank)
+    torch.cuda.manual_seed(rank)
+    np.random.seed(rank)
+    random.seed(rank)
+
+    cudnn.deterministic = True
+    cudnn.benchmark = False
+
+
+def process(args, app, image_size=224, draw_bbox=False):
+    dst = Path(args.a)
+    dst.mkdir(parents=True, exist_ok=True)
+    processes = []
+    image_paths = sorted(glob(args.i + '/*.*'))
+    for image_path in tqdm(image_paths):
+        name = Path(image_path).stem
+        img = cv2.imread(image_path)
+        bboxes, kpss = app.detect(img)
+        if bboxes.shape[0] == 0:
+            logger.error(f'[ERROR] Face not detected for {image_path}')
+            continue
+        i = get_center(bboxes, img)
+        bbox = bboxes[i, 0:4]
+        det_score = bboxes[i, 4]
+        kps = None
+        if kpss is not None:
+            kps = kpss[i]
+        face = Face(bbox=bbox, kps=kps, det_score=det_score)
+        blob, aimg = get_arcface_input(face, img)
+        file = str(Path(dst, name))
+        np.save(file, blob)
+        processes.append(file + '.npy')
+        cv2.imwrite(file + '.jpg', face_align.norm_crop(img, landmark=face.kps, image_size=image_size))
+        if draw_bbox:
+            dimg = draw_on(img, [face])
+            cv2.imwrite(file + '_bbox.jpg', dimg)
+
+    return processes
+
+
+def to_batch(path):
+    src = path.replace('npy', 'jpg')
+    if not os.path.exists(src):
+        src = path.replace('npy', 'png')
+
+    image = imread(src)[:, :, :3]
+    image = image / 255.
+    image = cv2.resize(image, (224, 224)).transpose(2, 0, 1)
+    image = torch.tensor(image).cuda()[None]
+
+    arcface = np.load(path)
+    arcface = torch.tensor(arcface).cuda()[None]
+
+    return image, arcface
+
+
+def load_checkpoint(args, mica):
+    checkpoint = torch.load(args.m)
+    if 'arcface' in checkpoint:
+        mica.arcface.load_state_dict(checkpoint['arcface'])
+    if 'flameModel' in checkpoint:
+        mica.flameModel.load_state_dict(checkpoint['flameModel'])
+
+
+def main(cfg, args):
+    device = 'cuda:0'
+    cfg.model.testing = True
+    mica = util.find_model_using_name(model_dir='micalib.models', model_name=cfg.model.name)(cfg, device)
+    load_checkpoint(args, mica)
+    mica.eval()
+
+    faces = mica.flameModel.generator.faces_tensor.cpu()
+    Path(args.o).mkdir(exist_ok=True, parents=True)
+
+    app = LandmarksDetector(model=detectors.RETINAFACE)
+
+    with torch.no_grad():
+        logger.info(f'Processing has started...')
+        paths = process(args, app, draw_bbox=False)
+        for path in tqdm(paths):
+            name = Path(path).stem
+            images, arcface = to_batch(path)
+            codedict = mica.encode(images, arcface)
+            opdict = mica.decode(codedict)
+            meshes = opdict['pred_canonical_shape_vertices']
+            code = opdict['pred_shape_code']
+            lmk = mica.flame.compute_landmarks(meshes)
+
+            mesh = meshes[0]
+            landmark_51 = lmk[0, 17:]
+            landmark_7 = landmark_51[[19, 22, 25, 28, 16, 31, 37]]
+
+            dst = Path(args.o, name)
+            dst.mkdir(parents=True, exist_ok=True)
+            trimesh.Trimesh(vertices=mesh.cpu() * 1000.0, faces=faces, process=False).export(f'{dst}/mesh.ply')  # save in millimeters
+            trimesh.Trimesh(vertices=mesh.cpu() * 1000.0, faces=faces, process=False).export(f'{dst}/mesh.obj')
+            np.save(f'{dst}/identity', code[0].cpu().numpy())
+            np.save(f'{dst}/kpt7', landmark_7.cpu().numpy() * 1000.0)
+            np.save(f'{dst}/kpt68', lmk.cpu().numpy() * 1000.0)
+
+        logger.info(f'Processing finished. Results has been saved in {args.o}')
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description='MICA - Towards Metrical Reconstruction of Human Faces')
+    parser.add_argument('-i', default='demo/input', type=str, help='Input folder with images')
+    parser.add_argument('-o', default='demo/output', type=str, help='Output folder')
+    parser.add_argument('-a', default='demo/arcface', type=str, help='Processed images for MICA input')
+    parser.add_argument('-m', default='data/pretrained/mica.tar', type=str, help='Pretrained model path')
+
+    args = parser.parse_args()
+    cfg = get_cfg_defaults()
+
+    deterministic(42)
+    main(cfg, args)