app.py

import gradio as gr
import spaces
from gradio_litmodel3d import LitModel3D

import os
import shutil
os.environ['SPCONV_ALGO'] = 'native'
from typing import *
import torch
import numpy as np
import imageio
import uuid
from easydict import EasyDict as edict
from PIL import Image
from trellis.pipelines import TrellisImageTo3DPipeline
from trellis.representations import Gaussian, MeshExtractResult
from trellis.utils import render_utils, postprocessing_utils

# Constants
MAX_SEED = np.iinfo(np.int32).max
TMP_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'tmp')
os.makedirs(TMP_DIR, exist_ok=True)

# Session Management Functions
def start_session(req: gr.Request):
    user_dir = os.path.join(TMP_DIR, str(req.session_hash))
    print(f'Creating user directory: {user_dir}')
    os.makedirs(user_dir, exist_ok=True)

def end_session(req: gr.Request):
    user_dir = os.path.join(TMP_DIR, str(req.session_hash))
    print(f'Removing user directory: {user_dir}')
    shutil.rmtree(user_dir)

# Image Preprocessing Function
def preprocess_image(image: Image.Image) -> Image.Image:
    """
    Preprocess the input image.

    Args:
        image (Image.Image): The input image.

    Returns:
        Image.Image: The preprocessed image.
    """
    processed_image = pipeline.preprocess_image(image)
    return processed_image

# State Packing and Unpacking Functions
def pack_state(gs: Gaussian, mesh: MeshExtractResult, trial_id: str) -> dict:
    return {
        'gaussian': {
            **gs.init_params,
            '_xyz': gs._xyz.cpu().numpy(),
            '_features_dc': gs._features_dc.cpu().numpy(),
            '_scaling': gs._scaling.cpu().numpy(),
            '_rotation': gs._rotation.cpu().numpy(),
            '_opacity': gs._opacity.cpu().numpy(),
        },
        'mesh': {
            'vertices': mesh.vertices.cpu().numpy(),
            'faces': mesh.faces.cpu().numpy(),
        },
        'trial_id': trial_id,
    }

def unpack_state(state: dict) -> Tuple[Gaussian, edict, str]:
    gs = Gaussian(
        aabb=state['gaussian']['aabb'],
        sh_degree=state['gaussian']['sh_degree'],
        mininum_kernel_size=state['gaussian']['mininum_kernel_size'],
        scaling_bias=state['gaussian']['scaling_bias'],
        opacity_bias=state['gaussian']['opacity_bias'],
        scaling_activation=state['gaussian']['scaling_activation'],
    )
    gs._xyz = torch.tensor(state['gaussian']['_xyz'], device='cuda')
    gs._features_dc = torch.tensor(state['gaussian']['_features_dc'], device='cuda')
    gs._scaling = torch.tensor(state['gaussian']['_scaling'], device='cuda')
    gs._rotation = torch.tensor(state['gaussian']['_rotation'], device='cuda')
    gs._opacity = torch.tensor(state['gaussian']['_opacity'], device='cuda')
    
    mesh = edict(
        vertices=torch.tensor(state['mesh']['vertices'], device='cuda'),
        faces=torch.tensor(state['mesh']['faces'], device='cuda'),
    )
    
    return gs, mesh, state['trial_id']

# Seed Management Function
def get_seed(randomize_seed: bool, seed: int) -> int:
    """
    Get the random seed.

    Args:
        randomize_seed (bool): Whether to randomize the seed.
        seed (int): The provided seed value.

    Returns:
        int: The final seed to use.
    """
    return np.random.randint(0, MAX_SEED) if randomize_seed else seed

# Core 3D Generation Function
@spaces.GPU
def image_to_3d(
    image: Image.Image,
    seed: int,
    ss_guidance_strength: float,
    ss_sampling_steps: int,
    slat_guidance_strength: float,
    slat_sampling_steps: int,
    req: gr.Request,
) -> Tuple[dict, str]:
    """
    Convert an image to a 3D model.

    Args:
        image (Image.Image): The input image.
        seed (int): The random seed.
        ss_guidance_strength (float): The guidance strength for sparse structure generation.
        ss_sampling_steps (int): The number of sampling steps for sparse structure generation.
        slat_guidance_strength (float): The guidance strength for structured latent generation.
        slat_sampling_steps (int): The number of sampling steps for structured latent generation.
        req (gr.Request): Gradio request object.

    Returns:
        Tuple[dict, str]: The state dictionary and the path to the generated video.
    """
    user_dir = os.path.join(TMP_DIR, str(req.session_hash))
    outputs = pipeline.run(
        image,
        seed=seed,
        formats=["gaussian", "mesh"],
        preprocess_image=False,
        sparse_structure_sampler_params={
            "steps": ss_sampling_steps,
            "cfg_strength": ss_guidance_strength,
        },
        slat_sampler_params={
            "steps": slat_sampling_steps,
            "cfg_strength": slat_guidance_strength,
        },
    )
    video = render_utils.render_video(outputs['gaussian'][0], num_frames=120)['color']
    video_geo = render_utils.render_video(outputs['mesh'][0], num_frames=120)['normal']
    video = [np.concatenate([video[i], video_geo[i]], axis=1) for i in range(len(video))]
    trial_id = uuid.uuid4()
    video_path = os.path.join(user_dir, f"{trial_id}.mp4")
    imageio.mimsave(video_path, video, fps=15)
    state = pack_state(outputs['gaussian'][0], outputs['mesh'][0], trial_id)
    torch.cuda.empty_cache()
    return state, video_path

# Existing GLB Extraction Function
@spaces.GPU
def extract_glb(
    state: dict,
    mesh_simplify: float,
    texture_size: int,
    req: gr.Request,
) -> Tuple[dict, bytes]:
    """
    Extract a GLB file from the 3D model.

    Args:
        state (dict): The state of the generated 3D model.
        mesh_simplify (float): The mesh simplification factor.
        texture_size (int): The texture resolution.
        req (gr.Request): Gradio request object.

    Returns:
        Tuple[dict, bytes]: The model state and the GLB file bytes.
    """
    user_dir = os.path.join(TMP_DIR, str(req.session_hash))
    gs, mesh, trial_id = unpack_state(state)
    glb = postprocessing_utils.to_glb(gs, mesh, simplify=mesh_simplify, texture_size=texture_size, verbose=False)
    glb_path = os.path.join(user_dir, f"{trial_id}.glb")
    glb.export(glb_path)
    # Read the GLB file as bytes
    with open(glb_path, "rb") as f:
        glb_bytes = f.read()
    torch.cuda.empty_cache()
    return state, glb_bytes

# New High-Quality GLB Extraction Function
@spaces.GPU
def extract_glb_high_quality(
    state: dict,
    req: gr.Request,
) -> Tuple[dict, bytes]:
    """
    Extract a high-quality GLB file from the 3D model without polygon reduction.

    Args:
        state (dict): The state of the generated 3D model.
        req (gr.Request): Gradio request object.

    Returns:
        Tuple[dict, bytes]: The model state and the high-quality GLB file bytes.
    """
    user_dir = os.path.join(TMP_DIR, str(req.session_hash))
    gs, mesh, trial_id = unpack_state(state)
    # Set simplify to 0.0 to disable polygon reduction
    # Set texture_size to 2048 for maximum texture quality
    glb = postprocessing_utils.to_glb(gs, mesh, simplify=0.0, texture_size=2048, verbose=False)
    glb_path = os.path.join(user_dir, f"{trial_id}_high_quality.glb")
    glb.export(glb_path)
    # Read the GLB file as bytes
    with open(glb_path, "rb") as f:
        glb_bytes = f.read()
    torch.cuda.empty_cache()
    return state, glb_bytes

# Gradio Interface Definition
with gr.Blocks(delete_cache=(600, 600)) as demo:
    gr.Markdown("""
    ## Image to 3D Asset with [TRELLIS](https://trellis3d.github.io/)
    * Upload an image and click "Generate" to create a 3D asset. If the image has an alpha channel, it will be used as the mask. Otherwise, the background will be removed automatically.
    * If you find the generated 3D asset satisfactory, click "Extract GLB" to extract the GLB file and download it.
    * **New:** Click "Download High Quality GLB" to download the GLB file without any polygon reduction and with maximum texture quality.
    """)

    with gr.Row():
        with gr.Column():
            # Image Input
            image_prompt = gr.Image(
                label="Image Prompt",
                format="png",
                image_mode="RGBA",
                type="pil",
                height=300
            )
            
            # Generation Settings Accordion
            with gr.Accordion(label="Generation Settings", open=False):
                seed = gr.Slider(
                    0,
                    MAX_SEED,
                    label="Seed",
                    value=0,
                    step=1
                )
                randomize_seed = gr.Checkbox(
                    label="Randomize Seed",
                    value=True
                )
                gr.Markdown("### Stage 1: Sparse Structure Generation")
                with gr.Row():
                    ss_guidance_strength = gr.Slider(
                        0.0,
                        10.0,
                        label="Guidance Strength",
                        value=7.5,
                        step=0.1
                    )
                    ss_sampling_steps = gr.Slider(
                        1,
                        500,
                        label="Sampling Steps",
                        value=12,
                        step=1
                    )
                gr.Markdown("### Stage 2: Structured Latent Generation")
                with gr.Row():
                    slat_guidance_strength = gr.Slider(
                        0.0,
                        10.0,
                        label="Guidance Strength",
                        value=3.0,
                        step=0.1
                    )
                    slat_sampling_steps = gr.Slider(
                        1,
                        500,
                        label="Sampling Steps",
                        value=12,
                        step=1
                    )

            # Generate Button
            generate_btn = gr.Button("Generate")
            
            # GLB Extraction Settings Accordion
            with gr.Accordion(label="GLB Extraction Settings", open=False):
                mesh_simplify = gr.Slider(
                    0.0,
                    0.98,
                    label="Simplify",
                    value=0.95,
                    step=0.01
                )
                texture_size = gr.Slider(
                    512,
                    2048,
                    label="Texture Size",
                    value=1024,
                    step=512
                )
            
            # Existing Extract GLB Button
            extract_glb_btn = gr.Button("Extract GLB", interactive=False)
            
            # New Extract High Quality GLB Button
            extract_glb_high_quality_btn = gr.Button("Download High Quality GLB", interactive=False)

        with gr.Column():
            # Video Output
            video_output = gr.Video(
                label="Generated 3D Asset",
                autoplay=True,
                loop=True,
                height=300
            )
            # 3D Model Display
            model_output = LitModel3D(
                label="Extracted GLB",
                exposure=20.0,
                height=300
            )
            # Existing Download GLB Button
            download_glb = gr.DownloadButton(
                label="Download GLB",
                interactive=False  # Initially disabled
            )
            # New Download High Quality GLB Button
            download_high_quality_glb = gr.DownloadButton(
                label="Download High Quality GLB",
                interactive=False  # Initially disabled
            )

    # State Variables
    output_buf = gr.State()
    glb_bytes_state = gr.State()  # For standard GLB
    glb_high_quality_bytes_state = gr.State()  # For high-quality GLB

    # Example Images
    with gr.Row():
        examples = gr.Examples(
            examples=[
                f'assets/example_image/{image}'
                for image in os.listdir("assets/example_image")
            ],
            inputs=[image_prompt],
            fn=preprocess_image,
            outputs=[image_prompt],
            run_on_click=True,
            examples_per_page=64,
        )

    # Event Handlers
    demo.load(start_session)
    demo.unload(end_session)
    
    # Image Upload Handler
    image_prompt.upload(
        preprocess_image,
        inputs=[image_prompt],
        outputs=[image_prompt],
    )

    # Generate Button Click Handler
    generate_btn.click(
        get_seed,
        inputs=[randomize_seed, seed],
        outputs=[seed],
    ).then(
        image_to_3d,
        inputs=[
            image_prompt,
            seed,
            ss_guidance_strength,
            ss_sampling_steps,
            slat_guidance_strength,
            slat_sampling_steps
        ],
        outputs=[output_buf, video_output],
    ).then(
        lambda: (gr.Button.update(interactive=True), gr.Button.update(interactive=True)),
        outputs=[extract_glb_btn, extract_glb_high_quality_btn],
    )

    # Existing Extract GLB Button Click Handler
    extract_glb_btn.click(
        extract_glb,
        inputs=[output_buf, mesh_simplify, texture_size],
        outputs=[model_output, glb_bytes_state],
    ).then(
        lambda glb_bytes: (glb_bytes, ),
        inputs=[glb_bytes_state],
        outputs=[download_glb],
    ).then(
        lambda: gr.DownloadButton.update(interactive=True),
        outputs=[download_glb],
    )

    # New Extract High Quality GLB Button Click Handler
    extract_glb_high_quality_btn.click(
        extract_glb_high_quality,
        inputs=[output_buf],
        outputs=[model_output, glb_high_quality_bytes_state],
    ).then(
        lambda glb_bytes: (glb_bytes, ),
        inputs=[glb_high_quality_bytes_state],
        outputs=[download_high_quality_glb],
    ).then(
        lambda: gr.DownloadButton.update(interactive=True),
        outputs=[download_high_quality_glb],
    )

    # Handle Clearing of Video Output
    video_output.clear(
        lambda: (gr.Button.update(interactive=False), gr.Button.update(interactive=False)),
        outputs=[extract_glb_btn, extract_glb_high_quality_btn],
    )

    # Handle Clearing of Model Output
    model_output.clear(
        lambda: (gr.File.update(value=None), gr.File.update(value=None)),
        outputs=[download_glb, download_high_quality_glb],
    )

# Launch the Gradio app
if __name__ == "__main__":
    # Initialize the pipeline
    pipeline = TrellisImageTo3DPipeline.from_pretrained("JeffreyXiang/TRELLIS-image-large")
    pipeline.cuda()
    try:
        pipeline.preprocess_image(Image.fromarray(np.zeros((512, 512, 3), dtype=np.uint8)))    # Preload rembg
    except:
        pass
    demo.launch()