diff --git a/LHM/__pycache__/__init__.cpython-310.pyc b/LHM/__pycache__/__init__.cpython-310.pyc
index cc52b0cc1a132cd412aa072a490811866d9d7315..b83f6a4e47b0804f6aaa497d6f823e7884dc82b6 100644
Binary files a/LHM/__pycache__/__init__.cpython-310.pyc and b/LHM/__pycache__/__init__.cpython-310.pyc differ
diff --git a/LHM/datasets/__pycache__/__init__.cpython-310.pyc b/LHM/datasets/__pycache__/__init__.cpython-310.pyc
index 88338f4b204be95c7e383d2be143ed2d80c03143..39c1225eecdf6eb4179445585991de4e1368ead1 100644
Binary files a/LHM/datasets/__pycache__/__init__.cpython-310.pyc and b/LHM/datasets/__pycache__/__init__.cpython-310.pyc differ
diff --git a/LHM/datasets/__pycache__/cam_utils.cpython-310.pyc b/LHM/datasets/__pycache__/cam_utils.cpython-310.pyc
index 2443ed5b8c2a069ff2237cc847b950749254844c..d940b38ecb01d6a0057be58eb5efbd1a6560d1a6 100644
Binary files a/LHM/datasets/__pycache__/cam_utils.cpython-310.pyc and b/LHM/datasets/__pycache__/cam_utils.cpython-310.pyc differ
diff --git a/LHM/datasets/__pycache__/mixer.cpython-310.pyc b/LHM/datasets/__pycache__/mixer.cpython-310.pyc
index 2423c2c9f7d912d68b4304d2ed8fdbb107467bce..dc02da6b78c093386e7dfee1074fc8633de450b6 100644
Binary files a/LHM/datasets/__pycache__/mixer.cpython-310.pyc and b/LHM/datasets/__pycache__/mixer.cpython-310.pyc differ
diff --git a/LHM/models/__pycache__/ESRGANer_utils.cpython-310.pyc b/LHM/models/__pycache__/ESRGANer_utils.cpython-310.pyc
index 2cfd24013195f64992751c18f5ecc5cbb1242b5e..a09def05505b36397353dfbea1d1bea4877ceb5b 100644
Binary files a/LHM/models/__pycache__/ESRGANer_utils.cpython-310.pyc and b/LHM/models/__pycache__/ESRGANer_utils.cpython-310.pyc differ
diff --git a/LHM/models/__pycache__/__init__.cpython-310.pyc b/LHM/models/__pycache__/__init__.cpython-310.pyc
index 75db055bfde003fe8e78043389cecc456c79c9e8..20cd2e4a73911b0050744e088e050b728915d8ae 100644
Binary files a/LHM/models/__pycache__/__init__.cpython-310.pyc and b/LHM/models/__pycache__/__init__.cpython-310.pyc differ
diff --git a/LHM/models/__pycache__/arcface_utils.cpython-310.pyc b/LHM/models/__pycache__/arcface_utils.cpython-310.pyc
index a722fa489400aca9f8d8544512c22c8219f001d7..93878135c01b3366a349c08a78c750ee56a98452 100644
Binary files a/LHM/models/__pycache__/arcface_utils.cpython-310.pyc and b/LHM/models/__pycache__/arcface_utils.cpython-310.pyc differ
diff --git a/LHM/models/__pycache__/embedder.cpython-310.pyc b/LHM/models/__pycache__/embedder.cpython-310.pyc
index a13e2049fb3577b250594d7b76f148bf07a64509..d20fc19419bba05bbd8d56f9d4b1f27b35b77ae3 100644
Binary files a/LHM/models/__pycache__/embedder.cpython-310.pyc and b/LHM/models/__pycache__/embedder.cpython-310.pyc differ
diff --git a/LHM/models/__pycache__/modeling_human_lrm.cpython-310.pyc b/LHM/models/__pycache__/modeling_human_lrm.cpython-310.pyc
index f4366e9060678346c1044d8a1d9a00ccd496dbc6..49ca7f3cb3d6ce851730349dcabbdd67110cb5a7 100644
Binary files a/LHM/models/__pycache__/modeling_human_lrm.cpython-310.pyc and b/LHM/models/__pycache__/modeling_human_lrm.cpython-310.pyc differ
diff --git a/LHM/models/__pycache__/transformer.cpython-310.pyc b/LHM/models/__pycache__/transformer.cpython-310.pyc
index b397320cec01f53dced38e0092745a9b53d290d9..d8c27e3af742a0801ef41dabbc86684b386cfec3 100644
Binary files a/LHM/models/__pycache__/transformer.cpython-310.pyc and b/LHM/models/__pycache__/transformer.cpython-310.pyc differ
diff --git a/LHM/models/__pycache__/transformer_dit.cpython-310.pyc b/LHM/models/__pycache__/transformer_dit.cpython-310.pyc
index 46070a6fbcfc3d32c42775797ae3ae037a64c20d..cb2c3fc8d03ccdf00f57b824e5c67d6b9b796270 100644
Binary files a/LHM/models/__pycache__/transformer_dit.cpython-310.pyc and b/LHM/models/__pycache__/transformer_dit.cpython-310.pyc differ
diff --git a/LHM/models/__pycache__/utils.cpython-310.pyc b/LHM/models/__pycache__/utils.cpython-310.pyc
index 4e5d516f11edfb6f701828a78fd7415e193b1ba3..a61a1acd9fc3bb0fedd7982fa31c5704db43c826 100644
Binary files a/LHM/models/__pycache__/utils.cpython-310.pyc and b/LHM/models/__pycache__/utils.cpython-310.pyc differ
diff --git a/LHM/models/encoders/__pycache__/__init__.cpython-310.pyc b/LHM/models/encoders/__pycache__/__init__.cpython-310.pyc
index 9b09815cd5aefd7fadc8b81ddcd057e1b4178e9b..ca7b6220647f2cc68fd03b772e1e8fe255a856da 100644
Binary files a/LHM/models/encoders/__pycache__/__init__.cpython-310.pyc and b/LHM/models/encoders/__pycache__/__init__.cpython-310.pyc differ
diff --git a/LHM/models/encoders/__pycache__/dinov2_fusion_wrapper.cpython-310.pyc b/LHM/models/encoders/__pycache__/dinov2_fusion_wrapper.cpython-310.pyc
index 3b3c9887e1e5f5935deb40461b3ee3640d55a729..9270bead6376979fe27f6fb47949b742b0a04b40 100644
Binary files a/LHM/models/encoders/__pycache__/dinov2_fusion_wrapper.cpython-310.pyc and b/LHM/models/encoders/__pycache__/dinov2_fusion_wrapper.cpython-310.pyc differ
diff --git a/LHM/models/encoders/__pycache__/sapiens_warpper.cpython-310.pyc b/LHM/models/encoders/__pycache__/sapiens_warpper.cpython-310.pyc
index 8258d4134d92e1e7509151887538eb201f95ba8a..f84af12f83097150c368ce0df4b2f32cd853fd37 100644
Binary files a/LHM/models/encoders/__pycache__/sapiens_warpper.cpython-310.pyc and b/LHM/models/encoders/__pycache__/sapiens_warpper.cpython-310.pyc differ
diff --git a/LHM/models/encoders/dinov2/__pycache__/__init__.cpython-310.pyc b/LHM/models/encoders/dinov2/__pycache__/__init__.cpython-310.pyc
index 702e9badf855a6652022ca9d14100aec007da065..d729c1bd460616a164f69baa2198e7bd91fd3f53 100644
Binary files a/LHM/models/encoders/dinov2/__pycache__/__init__.cpython-310.pyc and b/LHM/models/encoders/dinov2/__pycache__/__init__.cpython-310.pyc differ
diff --git a/LHM/models/encoders/dinov2/hub/__pycache__/__init__.cpython-310.pyc b/LHM/models/encoders/dinov2/hub/__pycache__/__init__.cpython-310.pyc
index c5a95a43b82ceef1ecc6a268b490bca1c2c693a9..950112853e8227083fe2a894b248e7e813eaa534 100644
Binary files a/LHM/models/encoders/dinov2/hub/__pycache__/__init__.cpython-310.pyc and b/LHM/models/encoders/dinov2/hub/__pycache__/__init__.cpython-310.pyc differ
diff --git a/LHM/models/encoders/dinov2/hub/__pycache__/backbones.cpython-310.pyc b/LHM/models/encoders/dinov2/hub/__pycache__/backbones.cpython-310.pyc
index 006821ad89164ec3ff5dd5b77bef8e81be528b50..6aee238eee1fb44bbd0289f2f8ead53c5f16d8cc 100644
Binary files a/LHM/models/encoders/dinov2/hub/__pycache__/backbones.cpython-310.pyc and b/LHM/models/encoders/dinov2/hub/__pycache__/backbones.cpython-310.pyc differ
diff --git a/LHM/models/encoders/dinov2/hub/__pycache__/utils.cpython-310.pyc b/LHM/models/encoders/dinov2/hub/__pycache__/utils.cpython-310.pyc
index 3bcd18f1b1d2b92fc9f520f4ab8ad175e8cfb068..82feac8fe5138eee998243d7f468091499c4adb5 100644
Binary files a/LHM/models/encoders/dinov2/hub/__pycache__/utils.cpython-310.pyc and b/LHM/models/encoders/dinov2/hub/__pycache__/utils.cpython-310.pyc differ
diff --git a/LHM/models/encoders/dinov2/layers/__pycache__/__init__.cpython-310.pyc b/LHM/models/encoders/dinov2/layers/__pycache__/__init__.cpython-310.pyc
index b4116b99d2e6f9a1effca6d1f4872b1d5b16363e..b71139935419e1eaf7258f4b7397dfc7f0677463 100644
Binary files a/LHM/models/encoders/dinov2/layers/__pycache__/__init__.cpython-310.pyc and b/LHM/models/encoders/dinov2/layers/__pycache__/__init__.cpython-310.pyc differ
diff --git a/LHM/models/encoders/dinov2/layers/__pycache__/attention.cpython-310.pyc b/LHM/models/encoders/dinov2/layers/__pycache__/attention.cpython-310.pyc
index 59dfd78b60de93feebfa8e454cbcfed9d5636a1f..ee33ee458a4873efd9b7d6d045a53b724cbf92bd 100644
Binary files a/LHM/models/encoders/dinov2/layers/__pycache__/attention.cpython-310.pyc and b/LHM/models/encoders/dinov2/layers/__pycache__/attention.cpython-310.pyc differ
diff --git a/LHM/models/encoders/dinov2/layers/__pycache__/block.cpython-310.pyc b/LHM/models/encoders/dinov2/layers/__pycache__/block.cpython-310.pyc
index 8b31a679e2e12df593f8a1a7eb3c04e309ed211c..56dc30384f23f5b72058d482d843c890b469ec63 100644
Binary files a/LHM/models/encoders/dinov2/layers/__pycache__/block.cpython-310.pyc and b/LHM/models/encoders/dinov2/layers/__pycache__/block.cpython-310.pyc differ
diff --git a/LHM/models/encoders/dinov2/layers/__pycache__/dino_head.cpython-310.pyc b/LHM/models/encoders/dinov2/layers/__pycache__/dino_head.cpython-310.pyc
index 86bc66622f883e4b9202e62084f9ea494e833d92..f0750ac5556b54409e78c2d50facf407f4d65a9f 100644
Binary files a/LHM/models/encoders/dinov2/layers/__pycache__/dino_head.cpython-310.pyc and b/LHM/models/encoders/dinov2/layers/__pycache__/dino_head.cpython-310.pyc differ
diff --git a/LHM/models/encoders/dinov2/layers/__pycache__/drop_path.cpython-310.pyc b/LHM/models/encoders/dinov2/layers/__pycache__/drop_path.cpython-310.pyc
index c821a1cfbb7434e731db30e747193f5186e3ee73..36d820882dbb9918f93416633712f353aadd9e0d 100644
Binary files a/LHM/models/encoders/dinov2/layers/__pycache__/drop_path.cpython-310.pyc and b/LHM/models/encoders/dinov2/layers/__pycache__/drop_path.cpython-310.pyc differ
diff --git a/LHM/models/encoders/dinov2/layers/__pycache__/layer_scale.cpython-310.pyc b/LHM/models/encoders/dinov2/layers/__pycache__/layer_scale.cpython-310.pyc
index de0f4a535bbc5da200fd319bb7ffa69995cfcef6..0b483e7f39eab372476bcd818aba2cbd437a7278 100644
Binary files a/LHM/models/encoders/dinov2/layers/__pycache__/layer_scale.cpython-310.pyc and b/LHM/models/encoders/dinov2/layers/__pycache__/layer_scale.cpython-310.pyc differ
diff --git a/LHM/models/encoders/dinov2/layers/__pycache__/mlp.cpython-310.pyc b/LHM/models/encoders/dinov2/layers/__pycache__/mlp.cpython-310.pyc
index 10caa21786883e68e28437ca9c3fea0e1956a800..f4be17f5a1e9dacc6066689cf63d6984a5904b36 100644
Binary files a/LHM/models/encoders/dinov2/layers/__pycache__/mlp.cpython-310.pyc and b/LHM/models/encoders/dinov2/layers/__pycache__/mlp.cpython-310.pyc differ
diff --git a/LHM/models/encoders/dinov2/layers/__pycache__/patch_embed.cpython-310.pyc b/LHM/models/encoders/dinov2/layers/__pycache__/patch_embed.cpython-310.pyc
index eff7797a91991e490cfee7dacc2de0f262ba3d34..dd119fa71a1c9767d97f211225b934e47cb406f9 100644
Binary files a/LHM/models/encoders/dinov2/layers/__pycache__/patch_embed.cpython-310.pyc and b/LHM/models/encoders/dinov2/layers/__pycache__/patch_embed.cpython-310.pyc differ
diff --git a/LHM/models/encoders/dinov2/layers/__pycache__/swiglu_ffn.cpython-310.pyc b/LHM/models/encoders/dinov2/layers/__pycache__/swiglu_ffn.cpython-310.pyc
index 9ccf19307cc48b14bad4816190bf512665cc0de0..0717c07b70d40c68c2485c950fc6bc6bbed42969 100644
Binary files a/LHM/models/encoders/dinov2/layers/__pycache__/swiglu_ffn.cpython-310.pyc and b/LHM/models/encoders/dinov2/layers/__pycache__/swiglu_ffn.cpython-310.pyc differ
diff --git a/LHM/models/encoders/dinov2/models/__pycache__/__init__.cpython-310.pyc b/LHM/models/encoders/dinov2/models/__pycache__/__init__.cpython-310.pyc
index 0486c0e29f1ec58f15134faf8c4a5fdecf266ba3..568675bc5dd99502f657a18d1d4a28c4d474c32e 100644
Binary files a/LHM/models/encoders/dinov2/models/__pycache__/__init__.cpython-310.pyc and b/LHM/models/encoders/dinov2/models/__pycache__/__init__.cpython-310.pyc differ
diff --git a/LHM/models/encoders/dinov2/models/__pycache__/vision_transformer.cpython-310.pyc b/LHM/models/encoders/dinov2/models/__pycache__/vision_transformer.cpython-310.pyc
index 3918d49bba479911bb06f7a0027b0586ecd4387a..11faf57754f31361f64a87b5c37460c38341bfc1 100644
Binary files a/LHM/models/encoders/dinov2/models/__pycache__/vision_transformer.cpython-310.pyc and b/LHM/models/encoders/dinov2/models/__pycache__/vision_transformer.cpython-310.pyc differ
diff --git a/LHM/models/encoders/dinov2_fusion_wrapper.py b/LHM/models/encoders/dinov2_fusion_wrapper.py
index 3f387c4b33d3d487871b838ac423caefd9ca216b..e4455528a399831445f9f7a951184ceb1319fe04 100644
--- a/LHM/models/encoders/dinov2_fusion_wrapper.py
+++ b/LHM/models/encoders/dinov2_fusion_wrapper.py
@@ -19,9 +19,9 @@ import kornia
import torch
import torch.nn as nn
import torch.nn.functional as F
-from accelerate.logging import get_logger
+# from accelerate.logging import get_logger
-logger = get_logger(__name__)
+# logger = get_logger(__name__)
class DPTHead(nn.Module):
@@ -126,7 +126,7 @@ class Dinov2FusionWrapper(nn.Module):
self._freeze()
def _freeze(self):
- logger.warning(f"======== Freezing Dinov2FusionWrapper ========")
+ # logger.warning(f"======== Freezing Dinov2FusionWrapper ========")
self.model.eval()
for name, param in self.model.named_parameters():
param.requires_grad = False
@@ -170,7 +170,7 @@ class Dinov2FusionWrapper(nn.Module):
dinov2_hub = import_module(".dinov2.hub.backbones", package=__package__)
model_fn = getattr(dinov2_hub, model_name)
- logger.debug(f"Modulation dim for Dinov2 is {modulation_dim}.")
+ # logger.debug(f"Modulation dim for Dinov2 is {modulation_dim}.")
model = model_fn(modulation_dim=modulation_dim, pretrained=pretrained)
return model
diff --git a/LHM/models/encoders/sapiens_warpper.py b/LHM/models/encoders/sapiens_warpper.py
index 1e6998ec38108bcb5d32097c1140755cb01cd9bd..49926ecc58addcc83309c46046e8c0b91362876f 100644
--- a/LHM/models/encoders/sapiens_warpper.py
+++ b/LHM/models/encoders/sapiens_warpper.py
@@ -18,10 +18,10 @@ import torch
import torch.nn as nn
import torch.nn.functional as F
import torchvision
-from accelerate.logging import get_logger
+# from accelerate.logging import get_logger
from tqdm import tqdm
-logger = get_logger(__name__)
+# logger = get_logger(__name__)
timings = {}
BATCH_SIZE = 64
@@ -188,7 +188,7 @@ class SapiensWrapper(nn.Module):
@staticmethod
def _build_sapiens(model_name: str, pretrained: bool = True):
- logger.debug(f"Using Sapiens model: {model_name}")
+ # logger.debug(f"Using Sapiens model: {model_name}")
USE_TORCHSCRIPT = "_torchscript" in model_name
# build the model from a checkpoint file
@@ -201,7 +201,7 @@ class SapiensWrapper(nn.Module):
return model
def _freeze(self):
- logger.warning(f"======== Freezing Sapiens Model ========")
+ # logger.warning(f"======== Freezing Sapiens Model ========")
self.model.eval()
for name, param in self.model.named_parameters():
param.requires_grad = False
diff --git a/LHM/models/modeling_human_lrm.py b/LHM/models/modeling_human_lrm.py
index ab2bdbdee0f090fc5305cf635a24e3ac83a62ce4..eeca307d8f3ace9321e563bcb048c06b28ff582d 100644
--- a/LHM/models/modeling_human_lrm.py
+++ b/LHM/models/modeling_human_lrm.py
@@ -14,7 +14,7 @@ import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F
-from accelerate.logging import get_logger
+# from accelerate.logging import get_logger
from diffusers.utils import is_torch_version
from LHM.models.arcface_utils import ResNetArcFace
@@ -29,7 +29,7 @@ from .embedder import CameraEmbedder
from .rendering.synthesizer import TriplaneSynthesizer
from .transformer import TransformerDecoder
-logger = get_logger(__name__)
+# logger = get_logger(__name__)
class ModelHumanLRM(nn.Module):
@@ -212,42 +212,42 @@ class ModelHumanLRM(nn.Module):
if encoder_type == "dino":
from .encoders.dino_wrapper import DinoWrapper
- logger.info("Using DINO as the encoder")
+ # logger.info("Using DINO as the encoder")
return DinoWrapper
elif encoder_type == "dinov2":
from .encoders.dinov2_wrapper import Dinov2Wrapper
- logger.info("Using DINOv2 as the encoder")
+ # logger.info("Using DINOv2 as the encoder")
return Dinov2Wrapper
elif encoder_type == "dinov2_unet":
from .encoders.dinov2_unet_wrapper import Dinov2UnetWrapper
- logger.info("Using Dinov2Unet as the encoder")
+ # logger.info("Using Dinov2Unet as the encoder")
return Dinov2UnetWrapper
elif encoder_type == "resunet":
from .encoders.xunet_wrapper import XnetWrapper
- logger.info("Using XnetWrapper as the encoder")
+ # logger.info("Using XnetWrapper as the encoder")
return XnetWrapper
elif encoder_type == "dinov2_featup":
from .encoders.dinov2_featup_wrapper import Dinov2FeatUpWrapper
- logger.info("Using Dinov2FeatUpWrapper as the encoder")
+ # logger.info("Using Dinov2FeatUpWrapper as the encoder")
return Dinov2FeatUpWrapper
elif encoder_type == "dinov2_dpt":
from .encoders.dinov2_dpt_wrapper import Dinov2DPTWrapper
- logger.info("Using Dinov2DPTWrapper as the encoder")
+ # logger.info("Using Dinov2DPTWrapper as the encoder")
return Dinov2DPTWrapper
elif encoder_type == "dinov2_fusion":
from .encoders.dinov2_fusion_wrapper import Dinov2FusionWrapper
- logger.info("Using Dinov2FusionWrapper as the encoder")
+ # logger.info("Using Dinov2FusionWrapper as the encoder")
return Dinov2FusionWrapper
elif encoder_type == "sapiens":
from .encoders.sapiens_warpper import SapiensWrapper
- logger.info("Using Sapiens as the encoder")
+ # logger.info("Using Sapiens as the encoder")
return SapiensWrapper
def forward_transformer(self, image_feats, camera_embeddings, query_points):
@@ -560,10 +560,10 @@ class ModelHumanLRM(nn.Module):
},
]
- logger.info("======== Weight Decay Parameters ========")
- logger.info(f"Total: {len(decay_params)}")
- logger.info("======== No Weight Decay Parameters ========")
- logger.info(f"Total: {len(no_decay_params)}")
+ # logger.info("======== Weight Decay Parameters ========")
+ # logger.info(f"Total: {len(decay_params)}")
+ # logger.info("======== No Weight Decay Parameters ========")
+ # logger.info(f"Total: {len(no_decay_params)}")
print(f"Total Params: {len(no_decay_params) + len(decay_params)}")
@@ -936,10 +936,10 @@ class ModelHumanLRMSapdinoBodyHeadSD3_5(ModelHumanLRMSapdinoBodyHeadSD3):
},
]
- logger.info("======== Weight Decay Parameters ========")
- logger.info(f"Total: {len(decay_params)}")
- logger.info("======== No Weight Decay Parameters ========")
- logger.info(f"Total: {len(no_decay_params)}")
+ # logger.info("======== Weight Decay Parameters ========")
+ # logger.info(f"Total: {len(decay_params)}")
+ # logger.info("======== No Weight Decay Parameters ========")
+ # logger.info(f"Total: {len(no_decay_params)}")
print(f"Total Params: {len(no_decay_params) + len(decay_params)}")
diff --git a/LHM/models/rendering/__pycache__/__init__.cpython-310.pyc b/LHM/models/rendering/__pycache__/__init__.cpython-310.pyc
index 3b59b919cc6cd0c353ae91802e88b202802bdbee..a839ad14ee0e7e9e3153e460f442319463934ea9 100644
Binary files a/LHM/models/rendering/__pycache__/__init__.cpython-310.pyc and b/LHM/models/rendering/__pycache__/__init__.cpython-310.pyc differ
diff --git a/LHM/models/rendering/__pycache__/gs_renderer.cpython-310.pyc b/LHM/models/rendering/__pycache__/gs_renderer.cpython-310.pyc
index 6e6253f19580a52bcaa1887ad25eb15f831866d0..4f2faa8d119be30c7f95fce51ce6c0b25479299f 100644
Binary files a/LHM/models/rendering/__pycache__/gs_renderer.cpython-310.pyc and b/LHM/models/rendering/__pycache__/gs_renderer.cpython-310.pyc differ
diff --git a/LHM/models/rendering/__pycache__/gsplat_renderer.cpython-310.pyc b/LHM/models/rendering/__pycache__/gsplat_renderer.cpython-310.pyc
index 06d2d9a06d98665d9d02717e0357e7de6fb6d793..8d7feec0a4bd5d3a52fa802deb2b98de202808eb 100644
Binary files a/LHM/models/rendering/__pycache__/gsplat_renderer.cpython-310.pyc and b/LHM/models/rendering/__pycache__/gsplat_renderer.cpython-310.pyc differ
diff --git a/LHM/models/rendering/__pycache__/mesh_utils.cpython-310.pyc b/LHM/models/rendering/__pycache__/mesh_utils.cpython-310.pyc
index 9b4d34e13ca0191436c16d1f7f0ce754cce8e348..aa1a8a2342c53d349149fbc8a6e9db1314320518 100644
Binary files a/LHM/models/rendering/__pycache__/mesh_utils.cpython-310.pyc and b/LHM/models/rendering/__pycache__/mesh_utils.cpython-310.pyc differ
diff --git a/LHM/models/rendering/__pycache__/smpl_x.cpython-310.pyc b/LHM/models/rendering/__pycache__/smpl_x.cpython-310.pyc
index e78611d65030fb63d150bd381331ad2311f03568..fc996ce1cce4b342aefff8696f0c5d2344b4de00 100644
Binary files a/LHM/models/rendering/__pycache__/smpl_x.cpython-310.pyc and b/LHM/models/rendering/__pycache__/smpl_x.cpython-310.pyc differ
diff --git a/LHM/models/rendering/__pycache__/smpl_x_voxel_dense_sampling.cpython-310.pyc b/LHM/models/rendering/__pycache__/smpl_x_voxel_dense_sampling.cpython-310.pyc
index f2243ad8c51b8f28400ca027480db3627846489c..5d7942710d22c20d71bb6e92e03c88ca8dad4ae2 100644
Binary files a/LHM/models/rendering/__pycache__/smpl_x_voxel_dense_sampling.cpython-310.pyc and b/LHM/models/rendering/__pycache__/smpl_x_voxel_dense_sampling.cpython-310.pyc differ
diff --git a/LHM/models/rendering/__pycache__/synthesizer.cpython-310.pyc b/LHM/models/rendering/__pycache__/synthesizer.cpython-310.pyc
index 4cd9a8aca607c68cbbca938f4c0cd965b751bd1b..0345f81d305c159747937093112860b5b0de92fe 100644
Binary files a/LHM/models/rendering/__pycache__/synthesizer.cpython-310.pyc and b/LHM/models/rendering/__pycache__/synthesizer.cpython-310.pyc differ
diff --git a/LHM/models/rendering/smpl_x_voxel_dense_sampling.py b/LHM/models/rendering/smpl_x_voxel_dense_sampling.py
index 9bb715793a7d77815c01b5da43a082970f877d03..7b23ef17bc11b8a59acc1661c3a58f1fea42355f 100644
--- a/LHM/models/rendering/smpl_x_voxel_dense_sampling.py
+++ b/LHM/models/rendering/smpl_x_voxel_dense_sampling.py
@@ -318,8 +318,10 @@ class SMPLX_Mesh(object):
return joint_offset
def get_subdivider(self, subdivide_num):
- vert = self.layer["neutral"].v_template.float().cuda()
- face = torch.LongTensor(self.face).cuda()
+ # vert = self.layer["neutral"].v_template.float().cuda()
+ # face = torch.LongTensor(self.face).cuda()
+ vert = self.layer["neutral"].v_template.float()
+ face = torch.LongTensor(self.face)
mesh = Meshes(vert[None, :, :], face[None, :, :])
if subdivide_num > 0:
@@ -419,7 +421,8 @@ class SMPLX_Mesh(object):
normal = (
Meshes(
verts=mesh_neutral_pose[None, :, :],
- faces=torch.LongTensor(self.face_upsampled).cuda()[None, :, :],
+ # faces=torch.LongTensor(self.face_upsampled).cuda()[None, :, :],
+ faces=torch.LongTensor(self.face_upsampled)[None, :, :],
)
.verts_normals_packed()
.reshape(self.vertex_num_upsampled, 3)
@@ -537,11 +540,14 @@ class SMPLXVoxelMeshModel(nn.Module):
):
"""Smooth KNN to handle skirt deformation."""
- lbs_weights = lbs_weights.cuda()
+ # lbs_weights = lbs_weights.cuda()
+ lbs_weights = lbs_weights
dist = knn_points(
- voxel_v.unsqueeze(0).cuda(),
- template_v.unsqueeze(0).cuda(),
+ # voxel_v.unsqueeze(0).cuda(),
+ # template_v.unsqueeze(0).cuda(),
+ voxel_v.unsqueeze(0),
+ template_v.unsqueeze(0),
K=1,
return_nn=True,
)
@@ -555,8 +561,10 @@ class SMPLXVoxelMeshModel(nn.Module):
# Smooth Skinning
knn_dis = knn_points(
- voxel_v.unsqueeze(0).cuda(),
- voxel_v.unsqueeze(0).cuda(),
+ # voxel_v.unsqueeze(0).cuda(),
+ # voxel_v.unsqueeze(0).cuda(),
+ voxel_v.unsqueeze(0),
+ voxel_v.unsqueeze(0),
K=smooth_k + 1,
return_nn=True,
)
@@ -667,7 +675,7 @@ class SMPLXVoxelMeshModel(nn.Module):
)
coordinates = coordinates.view(-1, 3).float()
- coordinates = coordinates.cuda()
+ # coordinates = coordinates.cuda()
if os.path.exists(f"./pretrained_models/voxel_grid/voxel_{voxel_size}.pth"):
print(f"load voxel_grid voxel_{voxel_size}.pth")
@@ -722,12 +730,15 @@ class SMPLXVoxelMeshModel(nn.Module):
smpl_x = self.smpl_x
# using KNN to query subdivided mesh
- dense_pts = self.dense_pts.cuda()
+ # dense_pts = self.dense_pts.cuda()
+ dense_pts = self.dense_pts
template_verts = self.smplx_layer.v_template
nn_vertex_idxs = knn_points(
- dense_pts.unsqueeze(0).cuda(),
- template_verts.unsqueeze(0).cuda(),
+ # dense_pts.unsqueeze(0).cuda(),
+ # template_verts.unsqueeze(0).cuda(),
+ dense_pts.unsqueeze(0),
+ template_verts.unsqueeze(0),
K=1,
return_nn=True,
).idx
@@ -1046,7 +1057,8 @@ class SMPLXVoxelMeshModel(nn.Module):
) # [B, 54, 3]
# smplx pose-dependent vertex offset
pose = (
- axis_angle_to_matrix(pose) - torch.eye(3)[None, None, :, :].float().cuda()
+ # axis_angle_to_matrix(pose) - torch.eye(3)[None, None, :, :].float().cuda()
+ axis_angle_to_matrix(pose) - torch.eye(3)[None, None, :, :].float()
).view(batch_size, (self.smpl_x.joint_num - 1) * 9)
# (B, 54 * 9) x (54*9, V)
@@ -1499,7 +1511,8 @@ def read_smplx_param(smplx_data_root, shape_param_file, batch_size=1, device="cu
osp.join(data_root_path, "cam_params", str(frame_idx) + ".json")
) as f:
cam_param = {
- k: torch.FloatTensor(v).cuda() for k, v in json.load(f).items()
+ # k: torch.FloatTensor(v).cuda() for k, v in json.load(f).items()
+ k: torch.FloatTensor(v) for k, v in json.load(f).items()
}
cam_param_list.append(cam_param)
@@ -1668,7 +1681,8 @@ def generate_smplx_point():
for k, v in data.items():
if k in smplx_keys:
# print(k, v.shape)
- smplx_params[k] = data[k].unsqueeze(0).cuda()
+ # smplx_params[k] = data[k].unsqueeze(0).cuda()
+ smplx_params[k] = data[k].unsqueeze(0)
return smplx_params
def sample_one(data):
diff --git a/LHM/models/rendering/utils/__pycache__/__init__.cpython-310.pyc b/LHM/models/rendering/utils/__pycache__/__init__.cpython-310.pyc
index 480ac7f4668b7bdd4650c99d2f66f23345d5c12d..86d53d7f66535bd8f2715ac17f58821172c3ab7b 100644
Binary files a/LHM/models/rendering/utils/__pycache__/__init__.cpython-310.pyc and b/LHM/models/rendering/utils/__pycache__/__init__.cpython-310.pyc differ
diff --git a/LHM/models/rendering/utils/__pycache__/math_utils.cpython-310.pyc b/LHM/models/rendering/utils/__pycache__/math_utils.cpython-310.pyc
index 7a1faac72c01a13751596085abee8a8c38fbe61b..f7bb59c659cb7c006aea61a693a2701ffe90bea2 100644
Binary files a/LHM/models/rendering/utils/__pycache__/math_utils.cpython-310.pyc and b/LHM/models/rendering/utils/__pycache__/math_utils.cpython-310.pyc differ
diff --git a/LHM/models/rendering/utils/__pycache__/ray_marcher.cpython-310.pyc b/LHM/models/rendering/utils/__pycache__/ray_marcher.cpython-310.pyc
index 4996f47b6f7599aaa6ed79a22cf12b1cd6cb6ed4..7ad31b6c234d46ba0b4da597d6f8e2960086e722 100644
Binary files a/LHM/models/rendering/utils/__pycache__/ray_marcher.cpython-310.pyc and b/LHM/models/rendering/utils/__pycache__/ray_marcher.cpython-310.pyc differ
diff --git a/LHM/models/rendering/utils/__pycache__/ray_sampler.cpython-310.pyc b/LHM/models/rendering/utils/__pycache__/ray_sampler.cpython-310.pyc
index 0e475cb0c3d77de6f6cd8dbe540d3809ebd1de35..c52b5c78a54bfa687065fd0d8979d6363acc00a8 100644
Binary files a/LHM/models/rendering/utils/__pycache__/ray_sampler.cpython-310.pyc and b/LHM/models/rendering/utils/__pycache__/ray_sampler.cpython-310.pyc differ
diff --git a/LHM/models/rendering/utils/__pycache__/renderer.cpython-310.pyc b/LHM/models/rendering/utils/__pycache__/renderer.cpython-310.pyc
index 45c3a1a5e554c6f51ed8bf8faf93a80b14c3467e..3777e1230781993b1a97cc655e126fa2b710a869 100644
Binary files a/LHM/models/rendering/utils/__pycache__/renderer.cpython-310.pyc and b/LHM/models/rendering/utils/__pycache__/renderer.cpython-310.pyc differ
diff --git a/LHM/models/rendering/utils/__pycache__/sh_utils.cpython-310.pyc b/LHM/models/rendering/utils/__pycache__/sh_utils.cpython-310.pyc
index 163b86e3f7daf424d9b4b86112fcb0282a67fefc..886755afc23f68d479d94641a4e2f4b2512471cc 100644
Binary files a/LHM/models/rendering/utils/__pycache__/sh_utils.cpython-310.pyc and b/LHM/models/rendering/utils/__pycache__/sh_utils.cpython-310.pyc differ
diff --git a/LHM/models/rendering/utils/__pycache__/typing.cpython-310.pyc b/LHM/models/rendering/utils/__pycache__/typing.cpython-310.pyc
index f19f118b02df9bf45b4e68e58a9f19e7217ac7bf..b1e5f4ffc8ba2b4dcebc713445af8ab65ca796cd 100644
Binary files a/LHM/models/rendering/utils/__pycache__/typing.cpython-310.pyc and b/LHM/models/rendering/utils/__pycache__/typing.cpython-310.pyc differ
diff --git a/LHM/models/rendering/utils/__pycache__/utils.cpython-310.pyc b/LHM/models/rendering/utils/__pycache__/utils.cpython-310.pyc
index 9328383d08e0d0a85230c7b60a6984bdc8eb437e..3ab405e490d4fbffa567723334c8a68e258095ca 100644
Binary files a/LHM/models/rendering/utils/__pycache__/utils.cpython-310.pyc and b/LHM/models/rendering/utils/__pycache__/utils.cpython-310.pyc differ
diff --git a/LHM/models/transformer.py b/LHM/models/transformer.py
index 18738a8adb32bccf273a238165acf21eb368cf87..3a72472d7e4996838e0f552394e60e583288c62e 100644
--- a/LHM/models/transformer.py
+++ b/LHM/models/transformer.py
@@ -11,10 +11,10 @@ from typing import Any, Dict, Optional, Tuple, Union
import torch
import torch.nn as nn
-from accelerate.logging import get_logger
+# from accelerate.logging import get_logger
from diffusers.utils import is_torch_version
-logger = get_logger(__name__)
+# logger = get_logger(__name__)
class TransformerDecoder(nn.Module):
@@ -106,7 +106,7 @@ class TransformerDecoder(nn.Module):
), f"Condition and modulation are not supported for BasicBlock"
from .block import BasicBlock
- logger.debug(f"Using BasicBlock")
+ # logger.debug(f"Using BasicBlock")
return partial(BasicBlock, inner_dim=inner_dim)
elif self.block_type == "cond":
assert (
@@ -117,10 +117,10 @@ class TransformerDecoder(nn.Module):
), f"Modulation dimension is not supported for ConditionBlock"
from .block import ConditionBlock
- logger.debug(f"Using ConditionBlock")
+ # logger.debug(f"Using ConditionBlock")
return partial(ConditionBlock, inner_dim=inner_dim, cond_dim=cond_dim)
elif self.block_type == "mod":
- logger.error(f"modulation without condition is not implemented")
+ # logger.error(f"modulation without condition is not implemented")
raise NotImplementedError(
f"modulation without condition is not implemented"
)
@@ -130,7 +130,7 @@ class TransformerDecoder(nn.Module):
), f"Condition and modulation dimensions must be specified for ConditionModulationBlock"
from .block import ConditionModulationBlock
- logger.debug(f"Using ConditionModulationBlock")
+ # logger.debug(f"Using ConditionModulationBlock")
return partial(
ConditionModulationBlock,
inner_dim=inner_dim,
@@ -138,25 +138,25 @@ class TransformerDecoder(nn.Module):
mod_dim=mod_dim,
)
elif self.block_type == "cogvideo_cond":
- logger.debug(f"Using CogVideoXBlock")
+ # logger.debug(f"Using CogVideoXBlock")
from LHM.models.transformer_dit import CogVideoXBlock
# assert inner_dim == cond_dim, f"inner_dim:{inner_dim}, cond_dim:{cond_dim}"
return partial(CogVideoXBlock, dim=inner_dim, attention_bias=True)
elif self.block_type == "sd3_cond":
- logger.debug(f"Using SD3JointTransformerBlock")
+ # logger.debug(f"Using SD3JointTransformerBlock")
from LHM.models.transformer_dit import SD3JointTransformerBlock
return partial(SD3JointTransformerBlock, dim=inner_dim, qk_norm="rms_norm")
elif self.block_type == "sd3_mm_cond":
- logger.debug(f"Using SD3MMJointTransformerBlock")
+ # logger.debug(f"Using SD3MMJointTransformerBlock")
from LHM.models.transformer_dit import SD3MMJointTransformerBlock
return partial(
SD3MMJointTransformerBlock, dim=inner_dim, qk_norm="rms_norm"
)
elif self.block_type == "sd3_mm_bh_cond":
- logger.debug(f"Using SD3MMJointTransformerBlock")
+ # logger.debug(f"Using SD3MMJointTransformerBlock")
from LHM.models.transformer_dit import SD3BodyHeadMMJointTransformerBlock
return partial(
diff --git a/LHM/outputs/__pycache__/base.cpython-310.pyc b/LHM/outputs/__pycache__/base.cpython-310.pyc
index c5ffba683d2b79956acf1c11ad8f3daa344fe52f..1ade3afa0551b9bd6cba621ad8472e8fd5c0db6e 100644
Binary files a/LHM/outputs/__pycache__/base.cpython-310.pyc and b/LHM/outputs/__pycache__/base.cpython-310.pyc differ
diff --git a/LHM/outputs/__pycache__/output.cpython-310.pyc b/LHM/outputs/__pycache__/output.cpython-310.pyc
index 32ce5f5560d509cda5f747cc8ab3b0dcba7d6600..f28675c82cf5300ca0053f383d59821c89911e28 100644
Binary files a/LHM/outputs/__pycache__/output.cpython-310.pyc and b/LHM/outputs/__pycache__/output.cpython-310.pyc differ
diff --git a/LHM/runners/__pycache__/__init__.cpython-310.pyc b/LHM/runners/__pycache__/__init__.cpython-310.pyc
index d8c02b352b2a621ceb830e4679fac736e10eb8e2..b93346a1ccd75cf20b8f91f195e6b41401a79bf6 100644
Binary files a/LHM/runners/__pycache__/__init__.cpython-310.pyc and b/LHM/runners/__pycache__/__init__.cpython-310.pyc differ
diff --git a/LHM/runners/__pycache__/abstract.cpython-310.pyc b/LHM/runners/__pycache__/abstract.cpython-310.pyc
index 0376115339cb1f417f48fe7fc5ed7ab60c33fe1f..d8b13f0d52bca34b1a9748f8f6089a51e13778a5 100644
Binary files a/LHM/runners/__pycache__/abstract.cpython-310.pyc and b/LHM/runners/__pycache__/abstract.cpython-310.pyc differ
diff --git a/LHM/runners/infer/__pycache__/__init__.cpython-310.pyc b/LHM/runners/infer/__pycache__/__init__.cpython-310.pyc
index f53e9d46c430fab9bd6c3198fcd478bc296966fc..2cfbc7b62e098234f84da2edf3ee90bfcf099a4d 100644
Binary files a/LHM/runners/infer/__pycache__/__init__.cpython-310.pyc and b/LHM/runners/infer/__pycache__/__init__.cpython-310.pyc differ
diff --git a/LHM/runners/infer/__pycache__/base_inferrer.cpython-310.pyc b/LHM/runners/infer/__pycache__/base_inferrer.cpython-310.pyc
index c8ced6612f74e9215c6b15a4aed679a920359846..6502e33b9f702d0eb2b9ab023938232b0b3d4d05 100644
Binary files a/LHM/runners/infer/__pycache__/base_inferrer.cpython-310.pyc and b/LHM/runners/infer/__pycache__/base_inferrer.cpython-310.pyc differ
diff --git a/LHM/runners/infer/__pycache__/human_lrm.cpython-310.pyc b/LHM/runners/infer/__pycache__/human_lrm.cpython-310.pyc
index cc92d140440823d0e52fdc35855cb59d8f3f210c..47825032243df2f2f10d91ffd80ee8ccb46771cb 100644
Binary files a/LHM/runners/infer/__pycache__/human_lrm.cpython-310.pyc and b/LHM/runners/infer/__pycache__/human_lrm.cpython-310.pyc differ
diff --git a/LHM/runners/infer/__pycache__/utils.cpython-310.pyc b/LHM/runners/infer/__pycache__/utils.cpython-310.pyc
index 5008c1501009fe5c4efd859ea651205bb79c458b..668995e3430f9439d8853fbf6375d1d1602723d4 100644
Binary files a/LHM/runners/infer/__pycache__/utils.cpython-310.pyc and b/LHM/runners/infer/__pycache__/utils.cpython-310.pyc differ
diff --git a/LHM/runners/infer/base_inferrer.py b/LHM/runners/infer/base_inferrer.py
index 8ee26b27969a0de94a9f47864037d4d1f0d53401..46545bf5fb0029c98c2bee65a23cf9c94e1b10c7 100644
--- a/LHM/runners/infer/base_inferrer.py
+++ b/LHM/runners/infer/base_inferrer.py
@@ -16,12 +16,12 @@
from abc import abstractmethod
import torch
-from accelerate import Accelerator
-from accelerate.logging import get_logger
+# from accelerate import Accelerator
+# from accelerate.logging import get_logger
from LHM.runners.abstract import Runner
-logger = get_logger(__name__)
+# logger = get_logger(__name__)
class Inferrer(Runner):
@@ -32,7 +32,7 @@ class Inferrer(Runner):
super().__init__()
torch._dynamo.config.disable = True
- self.accelerator = Accelerator()
+ # self.accelerator = Accelerator()
self.model: torch.nn.Module = None
@@ -44,8 +44,8 @@ class Inferrer(Runner):
@property
def device(self):
- return self.accelerator.device
-
+ # return self.accelerator.device
+ return torch.device('cpu')
@abstractmethod
def _build_model(self, cfg):
pass
diff --git a/LHM/runners/infer/human_lrm.py b/LHM/runners/infer/human_lrm.py
index 85b95b1f82d755b66bcfd90cfbe77d975cdd677f..beca453d49b2275529747efcb4bea0f76e6ae550 100644
--- a/LHM/runners/infer/human_lrm.py
+++ b/LHM/runners/infer/human_lrm.py
@@ -12,6 +12,7 @@ import time
import cv2
import numpy as np
+import spaces
import torch
from accelerate.logging import get_logger
from omegaconf import OmegaConf
@@ -19,7 +20,7 @@ from PIL import Image
from tqdm.auto import tqdm
from engine.pose_estimation.pose_estimator import PoseEstimator
-from engine.SegmentAPI.SAM import Bbox, SAM2Seg
+# from engine.SegmentAPI.SAM import Bbox, SAM2Seg
from LHM.datasets.cam_utils import (
build_camera_principle,
build_camera_standard,
@@ -46,12 +47,12 @@ logger = get_logger(__name__)
def avaliable_device():
- if torch.cuda.is_available():
- current_device_id = torch.cuda.current_device()
- device = f"cuda:{current_device_id}"
- else:
- device = "cpu"
-
+ # if torch.cuda.is_available():
+ # current_device_id = torch.cuda.current_device()
+ # device = f"cuda:{current_device_id}"
+ # else:
+ # device = "cpu"
+ device = "cpu"
return device
@@ -297,7 +298,7 @@ class HumanLRMInferrer(Inferrer):
EXP_TYPE: str = "human_lrm_sapdino_bh_sd3_5"
# EXP_TYPE: str = "human_lrm_sd3"
-
+
def __init__(self):
super().__init__()
@@ -315,7 +316,11 @@ class HumanLRMInferrer(Inferrer):
self.pose_estimator = PoseEstimator(
"./pretrained_models/human_model_files/", device=avaliable_device()
)
- self.parsingnet = SAM2Seg()
+
+ # self.parsingnet = SAM2Seg()
+
+ # print("self.device:", self.device)
+ # self.device = 'cpu'
self.model: ModelHumanLRM = self._build_model(self.cfg).to(self.device)
@@ -678,13 +683,9 @@ class HumanLRMInferrer(Inferrer):
image_paths.append(gradio_demo_image)
# alloc to each DDP worker
- image_paths = image_paths[
- self.accelerator.process_index :: self.accelerator.num_processes
- ]
+ image_paths = image_paths
- for image_path in tqdm(image_paths,
- disable=not self.accelerator.is_local_main_process,
- ):
+ for image_path in tqdm(image_paths):
# prepare dump paths
image_name = os.path.basename(image_path)
@@ -729,7 +730,8 @@ class HumanLRMInferrer(Inferrer):
os.makedirs(dump_image_dir, exist_ok=True)
# os.makedirs(dump_tmp_dir, exist_ok=True)
print(image_path, motion_seqs_dir, dump_image_dir, dump_video_path)
- # try:
+
+
shape_pose = self.pose_estimator(image_path)
assert shape_pose.is_full_body, f"The input image is illegal, {shape_pose.msg}"
@@ -750,235 +752,232 @@ class HumanLRMInferrer(Inferrer):
# if gradio_video_save_path is not None:
# os.system("cp {} {}".format(dump_video_path, gradio_video_save_path))
return True
- # except:
- # print("no find human or not full body!")
- # return False
-
-
-@REGISTRY_RUNNERS.register("infer.human_lrm_video")
-class HumanLRMVideoInferrer(HumanLRMInferrer):
- """video reconstruction for in the wild data"""
-
- EXP_TYPE: str = "human_lrm_sapdino_bh_sd3_5"
-
- def infer_single(
- self,
- image_path: str,
- motion_seqs_dir,
- motion_img_dir,
- motion_video_read_fps,
- export_video: bool,
- export_mesh: bool,
- dump_tmp_dir: str, # require by extracting motion seq from video, to save some results
- dump_image_dir: str,
- dump_video_path: str,
- ):
- source_size = self.cfg.source_size
- render_size = self.cfg.render_size
- # render_views = self.cfg.render_views
- render_fps = self.cfg.render_fps
- # mesh_size = self.cfg.mesh_size
- # mesh_thres = self.cfg.mesh_thres
- # frame_size = self.cfg.frame_size
- # source_cam_dist = self.cfg.source_cam_dist if source_cam_dist is None else source_cam_dist
- aspect_standard = 5.0 / 3
- motion_img_need_mask = self.cfg.get("motion_img_need_mask", False) # False
- vis_motion = self.cfg.get("vis_motion", False) # False
-
- parsing_mask = self.parsing(image_path)
-
- save_dir = os.path.join(dump_image_dir, "rgb")
- if os.path.exists(save_dir):
- return
-
- # prepare reference image
- image, _, _ = infer_preprocess_image(
- image_path,
- mask=parsing_mask,
- intr=None,
- pad_ratio=0,
- bg_color=1.0,
- max_tgt_size=896,
- aspect_standard=aspect_standard,
- enlarge_ratio=[1.0, 1.0],
- render_tgt_size=source_size,
- multiply=14,
- need_mask=True,
- )
- src_head_rgb = self.crop_face_image(image_path)
-
- import cv2
-
- try:
- src_head_rgb = cv2.resize(
- src_head_rgb,
- dsize=(self.cfg.src_head_size, self.cfg.src_head_size),
- interpolation=cv2.INTER_AREA,
- ) # resize to dino size
- except:
- src_head_rgb = np.zeros(
- (self.cfg.src_head_size, self.cfg.src_head_size, 3), dtype=np.uint8
- )
-
- src_head_rgb = (
- torch.from_numpy(src_head_rgb / 255.0).float().permute(2, 0, 1).unsqueeze(0)
- ) # [1, 3, H, W]
-
- # save masked image for vis
- save_ref_img_path = os.path.join(
- dump_tmp_dir, "refer_" + os.path.basename(image_path)
- )
- vis_ref_img = (image[0].permute(1, 2, 0).cpu().detach().numpy() * 255).astype(
- np.uint8
- )
-
- Image.fromarray(vis_ref_img).save(save_ref_img_path)
-
- # read motion seq
-
- if not os.path.exists(motion_seqs_dir):
- return
-
- motion_seq = prepare_motion_seqs(
- motion_seqs_dir,
- os.path.basename(image_path),
- save_root=dump_tmp_dir,
- fps=motion_video_read_fps,
- bg_color=1.0,
- aspect_standard=aspect_standard,
- enlarge_ratio=[1.0, 1, 0],
- render_image_res=render_size,
- multiply=16,
- need_mask=motion_img_need_mask,
- vis_motion=vis_motion,
- )
- motion_seqs = motion_seq["motion_seqs"]
-
- device = "cuda"
- dtype = torch.float32
- self.model.to(dtype)
-
- start_time = time.time()
-
- with torch.no_grad():
- # TODO check device and dtype
- # dict_keys(['comp_rgb', 'comp_rgb_bg', 'comp_mask', 'comp_depth', '3dgs'])
- render_intrs = motion_seq["render_intrs"].to(device)
- render_intrs[..., 0, 0] *= 2
- render_intrs[..., 1, 1] *= 2
- render_intrs[..., 0, 2] *= 2
- render_intrs[..., 1, 2] *= 2
- # smplx_params["focal"] *= 2
- # smplx_params["princpt"] *= 2
- # smplx_params["img_size_wh"] *= 2
-
- res = self.model.infer_single_view(
- image.unsqueeze(0).to(device, dtype),
- src_head_rgb.unsqueeze(0).to(device, dtype),
- None,
- None,
- render_c2ws=motion_seq["render_c2ws"].to(device),
- render_intrs=render_intrs,
- render_bg_colors=motion_seq["render_bg_colors"].to(device),
- smplx_params={
- k: v.to(device) for k, v in motion_seq["smplx_params"].items()
- },
- )
-
- print(f"time elapsed: {time.time() - start_time}")
- rgb = res["comp_rgb"].detach().cpu().numpy() # [Nv, H, W, 3], 0-1
- mask = res["comp_mask"].detach().cpu().numpy() # [Nv, H, W, 3], 0-1
- # mask[mask > 0.5] = 1.0
- # mask[mask < 0.4] = 0.0
- rgb = rgb * mask + (1 - mask) * 1
-
- rgb = np.clip(rgb * 255, 0, 255).astype(np.uint8)
- mask = np.clip(mask * 255, 0, 255).astype(np.uint8)
- rgba_numpy = np.concatenate([rgb, mask], axis=-1)
-
- for rgb_i, (rgba, motion_seq) in enumerate(zip(rgba_numpy, motion_seqs)):
-
- rgb_i = int(os.path.basename(motion_seq).replace(".json", ""))
- save_file = os.path.join(dump_image_dir, "rgb", f"{rgb_i:05d}.png")
- os.makedirs(os.path.dirname(save_file), exist_ok=True)
- Image.fromarray(rgba).save(save_file)
-
- def infer(self):
-
- image_paths = []
-
- omit_prefix = self.cfg.image_input
- suffixes = (".jpg", ".jpeg", ".png", ".webp")
-
- front_view_dict = dict()
- with open(os.path.join(self.cfg.image_input, "front_view.txt"), "r") as f:
- for line in f.readlines():
- name, idx = line.strip().split(" ")
- idx = int(idx)
- front_view_dict[name] = idx
-
- for root, dirs, files in os.walk(self.cfg.image_input):
- for dir in dirs:
- if dir in front_view_dict:
- idx = front_view_dict[dir]
- else:
- raise ValueError("no front view")
- img_path = os.path.join(root, dir, f"{idx:06d}.png")
- if dir in front_view_dict:
- print(img_path)
- image_paths.append(img_path)
-
- image_paths.sort()
-
- # alloc to each DDP worke
- image_paths = image_paths[
- self.accelerator.process_index :: self.accelerator.num_processes
- ]
-
- for image_path in tqdm(
- image_paths, disable=not self.accelerator.is_local_main_process
- ):
-
- # prepare dump paths
- image_name = os.path.basename(image_path)
- uid = image_name.split(".")[0]
- subdir_path = os.path.dirname(image_path).replace(omit_prefix, "")
- subdir_path = (
- subdir_path[1:] if subdir_path.startswith("/") else subdir_path
- )
- print("subdir_path and uid:", subdir_path, uid)
-
- # setting config
- motion_seqs_dir = self.cfg.motion_seqs_dir
- motion_name = os.path.dirname(
- motion_seqs_dir[:-1] if motion_seqs_dir[-1] == "/" else motion_seqs_dir
- )
- motion_name = os.path.basename(motion_name)
- dump_video_path = os.path.join(
- self.cfg.video_dump.replace("videos", "videos_benchmark"),
- subdir_path,
- motion_name,
- f"{uid}.mp4",
- )
- dump_image_dir = os.path.join(
- self.cfg.image_dump.replace("images", "images_benchmark"),
- subdir_path,
- )
-
- dump_tmp_dir = os.path.join(self.cfg.image_dump, subdir_path, "tmp_res")
- os.makedirs(dump_image_dir, exist_ok=True)
- os.makedirs(dump_tmp_dir, exist_ok=True)
- item_name = os.path.basename(os.path.dirname(image_path))
- self.infer_single(
- image_path,
- motion_seqs_dir=os.path.join(self.cfg.motion_seqs_dir, item_name),
- motion_img_dir=self.cfg.motion_img_dir,
- motion_video_read_fps=self.cfg.motion_video_read_fps,
- export_video=self.cfg.export_video,
- export_mesh=self.cfg.export_mesh,
- dump_tmp_dir=dump_tmp_dir,
- dump_image_dir=dump_image_dir,
- dump_video_path=dump_video_path,
- )
+# @REGISTRY_RUNNERS.register("infer.human_lrm_video")
+# class HumanLRMVideoInferrer(HumanLRMInferrer):
+# """video reconstruction for in the wild data"""
+
+# EXP_TYPE: str = "human_lrm_sapdino_bh_sd3_5"
+
+# def infer_single(
+# self,
+# image_path: str,
+# motion_seqs_dir,
+# motion_img_dir,
+# motion_video_read_fps,
+# export_video: bool,
+# export_mesh: bool,
+# dump_tmp_dir: str, # require by extracting motion seq from video, to save some results
+# dump_image_dir: str,
+# dump_video_path: str,
+# ):
+# source_size = self.cfg.source_size
+# render_size = self.cfg.render_size
+# # render_views = self.cfg.render_views
+# render_fps = self.cfg.render_fps
+# # mesh_size = self.cfg.mesh_size
+# # mesh_thres = self.cfg.mesh_thres
+# # frame_size = self.cfg.frame_size
+# # source_cam_dist = self.cfg.source_cam_dist if source_cam_dist is None else source_cam_dist
+# aspect_standard = 5.0 / 3
+# motion_img_need_mask = self.cfg.get("motion_img_need_mask", False) # False
+# vis_motion = self.cfg.get("vis_motion", False) # False
+
+# parsing_mask = self.parsing(image_path)
+
+# save_dir = os.path.join(dump_image_dir, "rgb")
+# if os.path.exists(save_dir):
+# return
+
+# # prepare reference image
+# image, _, _ = infer_preprocess_image(
+# image_path,
+# mask=parsing_mask,
+# intr=None,
+# pad_ratio=0,
+# bg_color=1.0,
+# max_tgt_size=896,
+# aspect_standard=aspect_standard,
+# enlarge_ratio=[1.0, 1.0],
+# render_tgt_size=source_size,
+# multiply=14,
+# need_mask=True,
+# )
+# src_head_rgb = self.crop_face_image(image_path)
+
+# import cv2
+
+# try:
+# src_head_rgb = cv2.resize(
+# src_head_rgb,
+# dsize=(self.cfg.src_head_size, self.cfg.src_head_size),
+# interpolation=cv2.INTER_AREA,
+# ) # resize to dino size
+# except:
+# src_head_rgb = np.zeros(
+# (self.cfg.src_head_size, self.cfg.src_head_size, 3), dtype=np.uint8
+# )
+
+# src_head_rgb = (
+# torch.from_numpy(src_head_rgb / 255.0).float().permute(2, 0, 1).unsqueeze(0)
+# ) # [1, 3, H, W]
+
+# # save masked image for vis
+# save_ref_img_path = os.path.join(
+# dump_tmp_dir, "refer_" + os.path.basename(image_path)
+# )
+# vis_ref_img = (image[0].permute(1, 2, 0).cpu().detach().numpy() * 255).astype(
+# np.uint8
+# )
+
+# Image.fromarray(vis_ref_img).save(save_ref_img_path)
+
+# # read motion seq
+
+# if not os.path.exists(motion_seqs_dir):
+# return
+
+# motion_seq = prepare_motion_seqs(
+# motion_seqs_dir,
+# os.path.basename(image_path),
+# save_root=dump_tmp_dir,
+# fps=motion_video_read_fps,
+# bg_color=1.0,
+# aspect_standard=aspect_standard,
+# enlarge_ratio=[1.0, 1, 0],
+# render_image_res=render_size,
+# multiply=16,
+# need_mask=motion_img_need_mask,
+# vis_motion=vis_motion,
+# )
+# motion_seqs = motion_seq["motion_seqs"]
+
+# device = "cuda"
+# dtype = torch.float32
+# self.model.to(dtype)
+
+# start_time = time.time()
+
+# with torch.no_grad():
+# # TODO check device and dtype
+# # dict_keys(['comp_rgb', 'comp_rgb_bg', 'comp_mask', 'comp_depth', '3dgs'])
+# render_intrs = motion_seq["render_intrs"].to(device)
+# render_intrs[..., 0, 0] *= 2
+# render_intrs[..., 1, 1] *= 2
+# render_intrs[..., 0, 2] *= 2
+# render_intrs[..., 1, 2] *= 2
+# # smplx_params["focal"] *= 2
+# # smplx_params["princpt"] *= 2
+# # smplx_params["img_size_wh"] *= 2
+
+# res = self.model.infer_single_view(
+# image.unsqueeze(0).to(device, dtype),
+# src_head_rgb.unsqueeze(0).to(device, dtype),
+# None,
+# None,
+# render_c2ws=motion_seq["render_c2ws"].to(device),
+# render_intrs=render_intrs,
+# render_bg_colors=motion_seq["render_bg_colors"].to(device),
+# smplx_params={
+# k: v.to(device) for k, v in motion_seq["smplx_params"].items()
+# },
+# )
+
+# print(f"time elapsed: {time.time() - start_time}")
+# rgb = res["comp_rgb"].detach().cpu().numpy() # [Nv, H, W, 3], 0-1
+# mask = res["comp_mask"].detach().cpu().numpy() # [Nv, H, W, 3], 0-1
+# # mask[mask > 0.5] = 1.0
+# # mask[mask < 0.4] = 0.0
+# rgb = rgb * mask + (1 - mask) * 1
+
+# rgb = np.clip(rgb * 255, 0, 255).astype(np.uint8)
+# mask = np.clip(mask * 255, 0, 255).astype(np.uint8)
+# rgba_numpy = np.concatenate([rgb, mask], axis=-1)
+
+# for rgb_i, (rgba, motion_seq) in enumerate(zip(rgba_numpy, motion_seqs)):
+
+# rgb_i = int(os.path.basename(motion_seq).replace(".json", ""))
+# save_file = os.path.join(dump_image_dir, "rgb", f"{rgb_i:05d}.png")
+# os.makedirs(os.path.dirname(save_file), exist_ok=True)
+# Image.fromarray(rgba).save(save_file)
+
+# def infer(self):
+
+# image_paths = []
+
+# omit_prefix = self.cfg.image_input
+# suffixes = (".jpg", ".jpeg", ".png", ".webp")
+
+# front_view_dict = dict()
+# with open(os.path.join(self.cfg.image_input, "front_view.txt"), "r") as f:
+# for line in f.readlines():
+# name, idx = line.strip().split(" ")
+# idx = int(idx)
+# front_view_dict[name] = idx
+
+# for root, dirs, files in os.walk(self.cfg.image_input):
+# for dir in dirs:
+# if dir in front_view_dict:
+# idx = front_view_dict[dir]
+# else:
+# raise ValueError("no front view")
+# img_path = os.path.join(root, dir, f"{idx:06d}.png")
+# if dir in front_view_dict:
+# print(img_path)
+# image_paths.append(img_path)
+
+# image_paths.sort()
+
+# # alloc to each DDP worke
+# image_paths = image_paths[
+# self.accelerator.process_index :: self.accelerator.num_processes
+# ]
+
+# for image_path in tqdm(
+# image_paths, disable=not self.accelerator.is_local_main_process
+# ):
+
+# # prepare dump paths
+# image_name = os.path.basename(image_path)
+# uid = image_name.split(".")[0]
+# subdir_path = os.path.dirname(image_path).replace(omit_prefix, "")
+# subdir_path = (
+# subdir_path[1:] if subdir_path.startswith("/") else subdir_path
+# )
+# print("subdir_path and uid:", subdir_path, uid)
+
+# # setting config
+# motion_seqs_dir = self.cfg.motion_seqs_dir
+# motion_name = os.path.dirname(
+# motion_seqs_dir[:-1] if motion_seqs_dir[-1] == "/" else motion_seqs_dir
+# )
+# motion_name = os.path.basename(motion_name)
+# dump_video_path = os.path.join(
+# self.cfg.video_dump.replace("videos", "videos_benchmark"),
+# subdir_path,
+# motion_name,
+# f"{uid}.mp4",
+# )
+# dump_image_dir = os.path.join(
+# self.cfg.image_dump.replace("images", "images_benchmark"),
+# subdir_path,
+# )
+
+# dump_tmp_dir = os.path.join(self.cfg.image_dump, subdir_path, "tmp_res")
+# os.makedirs(dump_image_dir, exist_ok=True)
+# os.makedirs(dump_tmp_dir, exist_ok=True)
+
+# item_name = os.path.basename(os.path.dirname(image_path))
+
+# self.infer_single(
+# image_path,
+# motion_seqs_dir=os.path.join(self.cfg.motion_seqs_dir, item_name),
+# motion_img_dir=self.cfg.motion_img_dir,
+# motion_video_read_fps=self.cfg.motion_video_read_fps,
+# export_video=self.cfg.export_video,
+# export_mesh=self.cfg.export_mesh,
+# dump_tmp_dir=dump_tmp_dir,
+# dump_image_dir=dump_image_dir,
+# dump_video_path=dump_video_path,
+# )
diff --git a/LHM/utils/__pycache__/__init__.cpython-310.pyc b/LHM/utils/__pycache__/__init__.cpython-310.pyc
index 8a7f7ac9bcb0ae0daf696271addc6128d40a1ac4..d96fcf34b136bbdce0d5216bb4a58c134ae93f60 100644
Binary files a/LHM/utils/__pycache__/__init__.cpython-310.pyc and b/LHM/utils/__pycache__/__init__.cpython-310.pyc differ
diff --git a/LHM/utils/__pycache__/face_detector.cpython-310.pyc b/LHM/utils/__pycache__/face_detector.cpython-310.pyc
index 31b392a23b51a74593a15010077f384deb9e0ae4..78724f5b073dbfb9f2cb8ca1c4fcf4a7678456de 100644
Binary files a/LHM/utils/__pycache__/face_detector.cpython-310.pyc and b/LHM/utils/__pycache__/face_detector.cpython-310.pyc differ
diff --git a/LHM/utils/__pycache__/ffmpeg_utils.cpython-310.pyc b/LHM/utils/__pycache__/ffmpeg_utils.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..b19e4efc277cba6a9ea16a97593ee1f4988d61ad
Binary files /dev/null and b/LHM/utils/__pycache__/ffmpeg_utils.cpython-310.pyc differ
diff --git a/LHM/utils/__pycache__/hf_hub.cpython-310.pyc b/LHM/utils/__pycache__/hf_hub.cpython-310.pyc
index 8de585c0a864c2545683fc4c92320390fbdd66c4..dcd25905a6255c252135059f48bff7d184612b57 100644
Binary files a/LHM/utils/__pycache__/hf_hub.cpython-310.pyc and b/LHM/utils/__pycache__/hf_hub.cpython-310.pyc differ
diff --git a/LHM/utils/__pycache__/logging.cpython-310.pyc b/LHM/utils/__pycache__/logging.cpython-310.pyc
index b3f6b48a08a989a038fe23b25b5b26434496f602..99c7007a27dda6aa691c8b6789ab2f3496be3e3e 100644
Binary files a/LHM/utils/__pycache__/logging.cpython-310.pyc and b/LHM/utils/__pycache__/logging.cpython-310.pyc differ
diff --git a/LHM/utils/__pycache__/registry.cpython-310.pyc b/LHM/utils/__pycache__/registry.cpython-310.pyc
index 1472df0de26ccc65d2493b8ddcd30dc303eb75c4..b33645e03b527f932153a2f1d95a0bf48d592f99 100644
Binary files a/LHM/utils/__pycache__/registry.cpython-310.pyc and b/LHM/utils/__pycache__/registry.cpython-310.pyc differ
diff --git a/app.py b/app.py
index 04cc31aa8d0e06aeaac3b59bb361ed71d831e43f..f8d7397c3fba4c759a40026c05608be4518f68de 100644
--- a/app.py
+++ b/app.py
@@ -1,7 +1,282 @@
+# # Copyright (c) 2023-2024, Qi Zuo
+# #
+# # Licensed under the Apache License, Version 2.0 (the "License");
+# # you may not use this file except in compliance with the License.
+# # You may obtain a copy of the License at
+# #
+# # https://www.apache.org/licenses/LICENSE-2.0
+# #
+# # Unless required by applicable law or agreed to in writing, software
+# # distributed under the License is distributed on an "AS IS" BASIS,
+# # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# # See the License for the specific language governing permissions and
+# # limitations under the License.
+
+
+# import os
+# from PIL import Image
+# import numpy as np
+# import gradio as gr
+# import base64
+# import spaces
+# import subprocess
+# import os
+
+# # def install_cuda_toolkit():
+# # # CUDA_TOOLKIT_URL = "https://developer.download.nvidia.com/compute/cuda/11.8.0/local_installers/cuda_11.8.0_520.61.05_linux.run"
+# # # # CUDA_TOOLKIT_URL = "https://developer.download.nvidia.com/compute/cuda/12.2.0/local_installers/cuda_12.2.0_535.54.03_linux.run"
+# # # CUDA_TOOLKIT_FILE = "/tmp/%s" % os.path.basename(CUDA_TOOLKIT_URL)
+# # # subprocess.call(["wget", "-q", CUDA_TOOLKIT_URL, "-O", CUDA_TOOLKIT_FILE])
+# # # subprocess.call(["chmod", "+x", CUDA_TOOLKIT_FILE])
+# # # subprocess.call([CUDA_TOOLKIT_FILE, "--silent", "--toolkit"])
+
+# # os.environ["CUDA_HOME"] = "/usr/local/cuda"
+# # os.environ["PATH"] = "%s/bin:%s" % (os.environ["CUDA_HOME"], os.environ["PATH"])
+# # os.environ["LD_LIBRARY_PATH"] = "%s/lib:%s" % (
+# # os.environ["CUDA_HOME"],
+# # "" if "LD_LIBRARY_PATH" not in os.environ else os.environ["LD_LIBRARY_PATH"],
+# # )
+# # # Fix: arch_list[-1] += '+PTX'; IndexError: list index out of range
+# # os.environ["TORCH_CUDA_ARCH_LIST"] = "8.0;8.6"
+
+# # install_cuda_toolkit()
+
+# def launch_pretrained():
+# from huggingface_hub import snapshot_download, hf_hub_download
+# hf_hub_download(repo_id="DyrusQZ/LHM_Runtime", repo_type='model', filename='assets.tar', local_dir="./")
+# os.system("tar -xvf assets.tar && rm assets.tar")
+# hf_hub_download(repo_id="DyrusQZ/LHM_Runtime", repo_type='model', filename='LHM-0.5B.tar', local_dir="./")
+# os.system("tar -xvf LHM-0.5B.tar && rm LHM-0.5B.tar")
+# hf_hub_download(repo_id="DyrusQZ/LHM_Runtime", repo_type='model', filename='LHM_prior_model.tar', local_dir="./")
+# os.system("tar -xvf LHM_prior_model.tar && rm LHM_prior_model.tar")
+
+# def launch_env_not_compile_with_cuda():
+# os.system("pip install chumpy")
+# os.system("pip uninstall -y basicsr")
+# os.system("pip install git+https://github.com/hitsz-zuoqi/BasicSR/")
+# # os.system("pip install -e ./third_party/sam2")
+# os.system("pip install numpy==1.23.0")
+# # os.system("pip install git+https://github.com/hitsz-zuoqi/sam2/")
+# # os.system("pip install git+https://github.com/ashawkey/diff-gaussian-rasterization/")
+# # os.system("pip install git+https://github.com/camenduru/simple-knn/")
+# os.system("pip install --no-index --no-cache-dir pytorch3d -f https://dl.fbaipublicfiles.com/pytorch3d/packaging/wheels/py310_cu121_pyt251/download.html")
+
+# # def launch_env_compile_with_cuda():
+# # # simple_knn
+# # os.system("wget oss://virutalbuy-public/share/aigc3d/data/for_lingteng/LHM/simple_knn.zip && wget oss://virutalbuy-public/share/aigc3d/data/for_lingteng/LHM/simple_knn-0.0.0.dist-info.zip")
+# # os.system("unzip simple_knn.zip && unzip simple_knn-0.0.0.dist-info.zip")
+# # os.system("mv simple_knn /usr/local/lib/python3.10/site-packages/")
+# # os.system("mv simple_knn-0.0.0.dist-info /usr/local/lib/python3.10/site-packages/")
+
+# # # diff_gaussian
+# # os.system("wget oss://virutalbuy-public/share/aigc3d/data/for_lingteng/LHM/diff_gaussian_rasterization.zip && wget oss://virutalbuy-public/share/aigc3d/data/for_lingteng/LHM/diff_gaussian_rasterization-0.0.0.dist-info.zip")
+# # os.system("unzip diff_gaussian_rasterization.zip && unzip diff_gaussian_rasterization-0.0.0.dist-info.zip")
+# # os.system("mv diff_gaussian_rasterization /usr/local/lib/python3.10/site-packages/")
+# # os.system("mv diff_gaussian_rasterization-0.0.0.dist-info /usr/local/lib/python3.10/site-packages/")
+
+# # # pytorch3d
+# # os.system("wget oss://virutalbuy-public/share/aigc3d/data/for_lingteng/LHM/pytorch3d.zip && wget oss://virutalbuy-public/share/aigc3d/data/for_lingteng/LHM/pytorch3d-0.7.8.dist-info.zip")
+# # os.system("unzip pytorch3d.zip && unzip pytorch3d-0.7.8.dist-info.zip")
+# # os.system("mv pytorch3d /usr/local/lib/python3.10/site-packages/")
+# # os.system("mv pytorch3d-0.7.8.dist-info /usr/local/lib/python3.10/site-packages/")
+
+
+# # launch_env_compile_with_cuda()
+
+# def assert_input_image(input_image):
+# if input_image is None:
+# raise gr.Error("No image selected or uploaded!")
+
+# def prepare_working_dir():
+# import tempfile
+# working_dir = tempfile.TemporaryDirectory()
+# return working_dir
+
+# def init_preprocessor():
+# from LHM.utils.preprocess import Preprocessor
+# global preprocessor
+# preprocessor = Preprocessor()
+
+# def preprocess_fn(image_in: np.ndarray, remove_bg: bool, recenter: bool, working_dir):
+# image_raw = os.path.join(working_dir.name, "raw.png")
+# with Image.fromarray(image_in) as img:
+# img.save(image_raw)
+# image_out = os.path.join(working_dir.name, "rembg.png")
+# success = preprocessor.preprocess(image_path=image_raw, save_path=image_out, rmbg=remove_bg, recenter=recenter)
+# assert success, f"Failed under preprocess_fn!"
+# return image_out
+
+# def get_image_base64(path):
+# with open(path, "rb") as image_file:
+# encoded_string = base64.b64encode(image_file.read()).decode()
+# return f"data:image/png;base64,{encoded_string}"
+
+
+# def demo_lhm(infer_impl):
+
+# def core_fn(image: str, video_params, working_dir):
+# image_raw = os.path.join(working_dir.name, "raw.png")
+# with Image.fromarray(image) as img:
+# img.save(image_raw)
+
+# base_vid = os.path.basename(video_params).split("_")[0]
+# smplx_params_dir = os.path.join("./assets/sample_motion", base_vid, "smplx_params")
+
+# dump_video_path = os.path.join(working_dir.name, "output.mp4")
+# dump_image_path = os.path.join(working_dir.name, "output.png")
+
+# status = spaces.GPU(infer_impl(
+# gradio_demo_image=image_raw,
+# gradio_motion_file=smplx_params_dir,
+# gradio_masked_image=dump_image_path,
+# gradio_video_save_path=dump_video_path
+# ))
+# if status:
+# return dump_image_path, dump_video_path
+# else:
+# return None, None
+
+# _TITLE = '''LHM: Large Animatable Human Model'''
+
+# _DESCRIPTION = '''
+# Reconstruct a human avatar in 0.2 seconds with A100!
+# '''
+
+# with gr.Blocks(analytics_enabled=False) as demo:
+
+# #
+# logo_url = "./assets/rgba_logo_new.png"
+# logo_base64 = get_image_base64(logo_url)
+# gr.HTML(
+# f"""
+#
+#
+#
Large Animatable Human Model
+#
+#
Notes: Please input full-body image in case of detection errors.
"""
+# )
+
+# # DISPLAY
+# with gr.Row():
+
+# with gr.Column(variant='panel', scale=1):
+# with gr.Tabs(elem_id="openlrm_input_image"):
+# with gr.TabItem('Input Image'):
+# with gr.Row():
+# input_image = gr.Image(label="Input Image", image_mode="RGBA", height=480, width=270, sources="upload", type="numpy", elem_id="content_image")
+# # EXAMPLES
+# with gr.Row():
+# examples = [
+# ['assets/sample_input/joker.jpg'],
+# ['assets/sample_input/anime.png'],
+# ['assets/sample_input/basket.png'],
+# ['assets/sample_input/ai_woman1.JPG'],
+# ['assets/sample_input/anime2.JPG'],
+# ['assets/sample_input/anime3.JPG'],
+# ['assets/sample_input/boy1.png'],
+# ['assets/sample_input/choplin.jpg'],
+# ['assets/sample_input/eins.JPG'],
+# ['assets/sample_input/girl1.png'],
+# ['assets/sample_input/girl2.png'],
+# ['assets/sample_input/robot.jpg'],
+# ]
+# gr.Examples(
+# examples=examples,
+# inputs=[input_image],
+# examples_per_page=20,
+# )
+
+# with gr.Column():
+# with gr.Tabs(elem_id="openlrm_input_video"):
+# with gr.TabItem('Input Video'):
+# with gr.Row():
+# video_input = gr.Video(label="Input Video",height=480, width=270, interactive=False)
+
+# examples = [
+# # './assets/sample_motion/danaotiangong/danaotiangong_origin.mp4',
+# './assets/sample_motion/ex5/ex5_origin.mp4',
+# './assets/sample_motion/girl2/girl2_origin.mp4',
+# './assets/sample_motion/jntm/jntm_origin.mp4',
+# './assets/sample_motion/mimo1/mimo1_origin.mp4',
+# './assets/sample_motion/mimo2/mimo2_origin.mp4',
+# './assets/sample_motion/mimo4/mimo4_origin.mp4',
+# './assets/sample_motion/mimo5/mimo5_origin.mp4',
+# './assets/sample_motion/mimo6/mimo6_origin.mp4',
+# './assets/sample_motion/nezha/nezha_origin.mp4',
+# './assets/sample_motion/taiji/taiji_origin.mp4'
+# ]
+
+# gr.Examples(
+# examples=examples,
+# inputs=[video_input],
+# examples_per_page=20,
+# )
+# with gr.Column(variant='panel', scale=1):
+# with gr.Tabs(elem_id="openlrm_processed_image"):
+# with gr.TabItem('Processed Image'):
+# with gr.Row():
+# processed_image = gr.Image(label="Processed Image", image_mode="RGBA", type="filepath", elem_id="processed_image", height=480, width=270, interactive=False)
+
+# with gr.Column(variant='panel', scale=1):
+# with gr.Tabs(elem_id="openlrm_render_video"):
+# with gr.TabItem('Rendered Video'):
+# with gr.Row():
+# output_video = gr.Video(label="Rendered Video", format="mp4", height=480, width=270, autoplay=True)
+
+# # SETTING
+# with gr.Row():
+# with gr.Column(variant='panel', scale=1):
+# submit = gr.Button('Generate', elem_id="openlrm_generate", variant='primary')
+
+
+# working_dir = gr.State()
+# submit.click(
+# fn=assert_input_image,
+# inputs=[input_image],
+# queue=False,
+# ).success(
+# fn=prepare_working_dir,
+# outputs=[working_dir],
+# queue=False,
+# ).success(
+# fn=core_fn,
+# inputs=[input_image, video_input, working_dir], # video_params refer to smpl dir
+# outputs=[processed_image, output_video],
+# )
+
+# demo.queue()
+# demo.launch()
+
+
+# def launch_gradio_app():
+
+# os.environ.update({
+# "APP_ENABLED": "1",
+# "APP_MODEL_NAME": "./exps/releases/video_human_benchmark/human-lrm-500M/step_060000/",
+# "APP_INFER": "./configs/inference/human-lrm-500M.yaml",
+# "APP_TYPE": "infer.human_lrm",
+# "NUMBA_THREADING_LAYER": 'omp',
+# })
+
+# from LHM.runners import REGISTRY_RUNNERS
+# RunnerClass = REGISTRY_RUNNERS[os.getenv("APP_TYPE")]
+# with RunnerClass() as runner:
+# demo_lhm(infer_impl=runner.infer)
+
+
+# if __name__ == '__main__':
+# # launch_pretrained()
+# # launch_env_not_compile_with_cuda()
+# launch_gradio_app()
+
import gradio as gr
def greet(name):
return "Hello " + name + "!!"
demo = gr.Interface(fn=greet, inputs="text", outputs="text")
-demo.launch()
+demo.launch()
\ No newline at end of file
diff --git a/app_lhm.py b/app_lhm.py
index 24adb21e6a94a0e8ebb6d3c59e23c74191268184..9695da2e2126e5125c6233824aade99e7ae128cf 100644
--- a/app_lhm.py
+++ b/app_lhm.py
@@ -18,7 +18,7 @@ from PIL import Image
import numpy as np
import gradio as gr
import base64
-
+import spaces
import subprocess
import os
@@ -54,7 +54,8 @@ def launch_env_not_compile_with_cuda():
os.system("pip install chumpy")
os.system("pip uninstall -y basicsr")
os.system("pip install git+https://github.com/hitsz-zuoqi/BasicSR/")
- os.system("pip install -e ./third_party/sam2")
+ # os.system("pip install -e ./third_party/sam2")
+ os.system("pip install numpy==1.23.0")
# os.system("pip install git+https://github.com/hitsz-zuoqi/sam2/")
# os.system("pip install git+https://github.com/ashawkey/diff-gaussian-rasterization/")
# os.system("pip install git+https://github.com/camenduru/simple-knn/")
@@ -123,13 +124,13 @@ def demo_lhm(infer_impl):
dump_video_path = os.path.join(working_dir.name, "output.mp4")
dump_image_path = os.path.join(working_dir.name, "output.png")
- # print(video_params)
- status = infer_impl(
+
+ status = spaces.GPU(infer_impl(
gradio_demo_image=image_raw,
gradio_motion_file=smplx_params_dir,
gradio_masked_image=dump_image_path,
gradio_video_save_path=dump_video_path
- )
+ ))
if status:
return dump_image_path, dump_video_path
else:
@@ -264,10 +265,12 @@ def launch_gradio_app():
from LHM.runners import REGISTRY_RUNNERS
RunnerClass = REGISTRY_RUNNERS[os.getenv("APP_TYPE")]
with RunnerClass() as runner:
+ runner.pose_estimator.device = torch.device('cuda')
+ runner.pose_estimator.mhmr_model.cuda()
demo_lhm(infer_impl=runner.infer)
if __name__ == '__main__':
launch_pretrained()
launch_env_not_compile_with_cuda()
- launch_gradio_app()
+ # launch_gradio_app()
diff --git a/engine/SegmentAPI/__pycache__/SAM.cpython-310.pyc b/engine/SegmentAPI/__pycache__/SAM.cpython-310.pyc
index e3dcc91debe6011913dbc1199d266a2dffa88d02..d1000d5f932b60a9205687bf655e161d2021b594 100644
Binary files a/engine/SegmentAPI/__pycache__/SAM.cpython-310.pyc and b/engine/SegmentAPI/__pycache__/SAM.cpython-310.pyc differ
diff --git a/engine/__pycache__/__init__.cpython-310.pyc b/engine/__pycache__/__init__.cpython-310.pyc
index e4e2507cac763f542e0c9f3678a9ac4e87fc143b..d5a14354bdaed7ffd41d806c9ef0132917fec343 100644
Binary files a/engine/__pycache__/__init__.cpython-310.pyc and b/engine/__pycache__/__init__.cpython-310.pyc differ
diff --git a/engine/__pycache__/ouputs.cpython-310.pyc b/engine/__pycache__/ouputs.cpython-310.pyc
index ae681e9b5d9e68eb762e92f75c354a181049efd0..cfaf8b2b0ec829e2d4b40282bf436bcd0be54a6d 100644
Binary files a/engine/__pycache__/ouputs.cpython-310.pyc and b/engine/__pycache__/ouputs.cpython-310.pyc differ
diff --git a/engine/pose_estimation/pose_estimator.py b/engine/pose_estimation/pose_estimator.py
index 89aa6c8da859495e898163eb86aa70539ca015ff..b019dac725ba4cd373686af4079c71214497243b 100644
--- a/engine/pose_estimation/pose_estimator.py
+++ b/engine/pose_estimation/pose_estimator.py
@@ -40,7 +40,7 @@ def normalize_rgb_tensor(img, imgenet_normalization=True):
) / torch.tensor(IMG_NORM_STD, device=img.device).view(1, 3, 1, 1)
return img
-
+# @spaces.GPU()
def load_model(ckpt_path, model_path, device=torch.device("cuda")):
"""Open a checkpoint, build Multi-HMR using saved arguments, load the model weigths."""
# Model
@@ -173,6 +173,9 @@ class PoseEstimator:
@torch.no_grad()
def __call__(self, img_path):
# image_tensor H W C
+
+ # self.device = torch.device('cuda')
+ # self.mhmr_model.to(self.device)
img_np = np.asarray(Image.open(img_path).convert("RGB"))
diff --git a/requirements_lhm.txt b/requirements_lhm.txt
index 51bb15f775d1e695b644a7db22898d564db10924..ffdefc9e4a67210ddbb6555be8c66890384795ef 100644
--- a/requirements_lhm.txt
+++ b/requirements_lhm.txt
@@ -37,8 +37,8 @@ setuptools==74.0.0
taming_transformers_rom1504==0.0.6
timm==1.0.15
-# https://download.pytorch.org/whl/cu121/torch-2.5.1%2Bcu121-cp310-cp310-linux_x86_64.whl#sha256=92af92c569de5da937dd1afb45ecfdd598ec1254cf2e49e3d698cb24d71aae14
-# https://download.pytorch.org/whl/cu121/torchvision-0.20.1%2Bcu121-cp310-cp310-linux_x86_64.whl#sha256=304937b82c933d5155bd04d771f4b187273f67a76050bb4276b521f7e9b4c4e7
+https://download.pytorch.org/whl/cu121/torch-2.5.1%2Bcu121-cp310-cp310-linux_x86_64.whl#sha256=92af92c569de5da937dd1afb45ecfdd598ec1254cf2e49e3d698cb24d71aae14
+https://download.pytorch.org/whl/cu121/torchvision-0.20.1%2Bcu121-cp310-cp310-linux_x86_64.whl#sha256=304937b82c933d5155bd04d771f4b187273f67a76050bb4276b521f7e9b4c4e7
# https://download.pytorch.org/whl/cu121/xformers-0.0.29.post1-cp310-cp310-manylinux_2_28_x86_64.whl#sha256=e213ff8123e20602bd486739ffee4013338b02f9d2e0e4635a2912750854fdbe
tqdm==4.66.4
@@ -48,6 +48,5 @@ typeguard==2.13.3
xatlas==0.0.9
imageio-ffmpeg
-https://huggingface.co/spaces/DyrusQZ/LHM_ZeroDebug/resolve/main/wheels/diff_gaussian_rasterization-0.0.0-cp310-cp310-linux_x86_64.whl?download=true
-https://huggingface.co/spaces/DyrusQZ/LHM_ZeroDebug/resolve/main/wheels/simple_knn-0.0.0-cp310-cp310-linux_x86_64.whl?download=true
-
+./wheels/diff_gaussian_rasterization-0.0.0-cp310-cp310-linux_x86_64.whl
+./wheels/simple_knn-0.0.0-cp310-cp310-linux_x86_64.whl
\ No newline at end of file