app.py

import jax
import jax.numpy as jnp
from flax.jax_utils import replicate
from flax.training import train_state
import optax
from diffusers import FlaxStableDiffusionPipeline, FlaxUNet2DConditionModel
from diffusers.schedulers import FlaxPNDMScheduler
from datasets import load_dataset
from tqdm.auto import tqdm
import os
import pickle
from PIL import Image
import numpy as np

# Custom Scheduler
class CustomFlaxPNDMScheduler(FlaxPNDMScheduler):
    def add_noise(self, state, original_samples, noise, timesteps):
        timesteps = timesteps.astype(jnp.int32)
        return super().add_noise(state, original_samples, noise, timesteps)

# Set up cache directories
cache_dir = "/tmp/huggingface_cache"
model_cache_dir = os.path.join(cache_dir, "stable_diffusion_model")
os.makedirs(model_cache_dir, exist_ok=True)

print(f"Cache directory: {cache_dir}")
print(f"Model cache directory: {model_cache_dir}")

# Function to load or download the model
def get_model(model_id, revision):
    model_cache_file = os.path.join(model_cache_dir, f"{model_id.replace('/', '_')}_{revision}.pkl")
    print(f"Model cache file: {model_cache_file}")
    if os.path.exists(model_cache_file):
        print("Loading model from cache...")
        with open(model_cache_file, 'rb') as f:
            return pickle.load(f)
    else:
        print("Downloading model...")
        pipeline, params = FlaxStableDiffusionPipeline.from_pretrained(
            model_id, 
            revision=revision,
            dtype=jnp.float32,
        )
        with open(model_cache_file, 'wb') as f:
            pickle.dump((pipeline, params), f)
        return pipeline, params

# Load the pre-trained model
model_id = "CompVis/stable-diffusion-v1-4"
pipeline, params = get_model(model_id, "flax")

# Use custom scheduler
custom_scheduler = CustomFlaxPNDMScheduler.from_config(pipeline.scheduler.config)
pipeline.scheduler = custom_scheduler

# Extract UNet from pipeline
unet = pipeline.unet

# Print UNet configuration
print("UNet configuration:")
print(unet.config)

# Adjust the input layer of the UNet
def adjust_unet_input_layer(params):
    conv_in_weight = params['unet']['conv_in']['kernel']
    print(f"Original conv_in weight shape: {conv_in_weight.shape}")
    new_conv_in_weight = jnp.zeros((3, 3, 4, 320), dtype=jnp.float32)
    new_conv_in_weight = new_conv_in_weight.at[:, :, :3, :].set(conv_in_weight[:, :, :3, :])
    params['unet']['conv_in']['kernel'] = new_conv_in_weight
    print(f"New conv_in weight shape: {params['unet']['conv_in']['kernel'].shape}")
    return params

params = adjust_unet_input_layer(params)

# Load and preprocess your dataset
def preprocess_images(examples):
    def process_image(image):
        if isinstance(image, str):
            if not image.lower().endswith('.jpg') and not image.lower().endswith('.jpeg'):
                return None
            image = Image.open(image)
        if not isinstance(image, Image.Image):
            return None
        image = image.convert("RGB").resize((512, 512))
        image = np.array(image).astype(np.float32) / 255.0
        return image

    processed = [process_image(img) for img in examples["image"]]
    return {"pixel_values": [img for img in processed if img is not None]}

# Load dataset from Hugging Face
dataset_name = "uruguayai/montevideo"
dataset_cache_file = os.path.join(cache_dir, "montevideo_dataset.pkl")

print(f"Dataset name: {dataset_name}")
print(f"Dataset cache file: {dataset_cache_file}")

if os.path.exists(dataset_cache_file):
    print("Loading dataset from cache...")
    with open(dataset_cache_file, 'rb') as f:
        processed_dataset = pickle.load(f)
else:
    print("Processing dataset...")
    dataset = load_dataset(dataset_name)
    processed_dataset = dataset["train"].map(preprocess_images, batched=True, remove_columns=dataset["train"].column_names)
    processed_dataset = processed_dataset.filter(lambda example: len(example['pixel_values']) > 0)
    with open(dataset_cache_file, 'wb') as f:
        pickle.dump(processed_dataset, f)

print(f"Processed dataset size: {len(processed_dataset)}")

# Print sample input shape
sample_batch = next(iter(processed_dataset.batch(1)))
print(f"Sample batch keys: {sample_batch.keys()}")
print(f"Sample pixel_values type: {type(sample_batch['pixel_values'])}")
print(f"Sample pixel_values length: {len(sample_batch['pixel_values'])}")
if len(sample_batch['pixel_values']) > 0:
    print(f"Sample pixel_values[0] shape: {np.array(sample_batch['pixel_values'][0]).shape}")

# Training function
def train_step(state, batch, rng):
    def compute_loss(params, pixel_values, rng):
        pixel_values = jnp.array(pixel_values, dtype=jnp.float32)
        pixel_values = jnp.expand_dims(pixel_values, axis=0)  # Add batch dimension
        print(f"pixel_values shape in compute_loss: {pixel_values.shape}")
        
        latents = pipeline.vae.apply(
            {"params": params["vae"]},
            pixel_values,
            method=pipeline.vae.encode
        ).latent_dist.sample(rng)
        latents = latents * jnp.float32(0.18215)
        print(f"latents shape: {latents.shape}")

        noise = jax.random.normal(rng, latents.shape, dtype=jnp.float32)
        
        timesteps = jax.random.randint(
            rng, (latents.shape[0],), 0, pipeline.scheduler.config.num_train_timesteps
        )
        
        noisy_latents = pipeline.scheduler.add_noise(
            pipeline.scheduler.create_state(),
            original_samples=latents,
            noise=noise,
            timesteps=timesteps
        )
        
        encoder_hidden_states = jax.random.normal(
            rng, 
            (latents.shape[0], pipeline.text_encoder.config.hidden_size),
            dtype=jnp.float32
        )
        
        print(f"noisy_latents shape: {noisy_latents.shape}")
        print(f"timesteps shape: {timesteps.shape}")
        print(f"encoder_hidden_states shape: {encoder_hidden_states.shape}")
        
        # Use the correct method to call the UNet
        model_output = unet.apply(
            {'params': params["unet"]},
            noisy_latents,
            jnp.array(timesteps, dtype=jnp.int32),
            encoder_hidden_states,
            train=True,
        ).sample
        
        return jnp.mean((model_output - noise) ** 2)

    grad_fn = jax.grad(compute_loss, argnums=0, allow_int=True)
    rng, step_rng = jax.random.split(rng)
    
    grads = grad_fn(state.params, batch["pixel_values"], step_rng)
    loss = compute_loss(state.params, batch["pixel_values"], step_rng)
    state = state.apply_gradients(grads=grads)
    return state, loss

# Initialize training state
learning_rate = 1e-5
optimizer = optax.adam(learning_rate)
float32_params = jax.tree_util.tree_map(lambda x: x.astype(jnp.float32) if x.dtype != jnp.int32 else x, params)
state = train_state.TrainState.create(
    apply_fn=unet.apply,
    params=float32_params,
    tx=optimizer,
)

# Training loop
num_epochs = 3
batch_size = 1
rng = jax.random.PRNGKey(0)

for epoch in range(num_epochs):
    epoch_loss = 0
    num_batches = 0
    for batch in tqdm(processed_dataset.batch(batch_size)):
        batch['pixel_values'] = jnp.array(batch['pixel_values'][0], dtype=jnp.float32)
        rng, step_rng = jax.random.split(rng)
        state, loss = train_step(state, batch, step_rng)
        epoch_loss += loss
        num_batches += 1
        
        if num_batches % 10 == 0:
            jax.clear_caches()
    
    avg_loss = epoch_loss / num_batches
    print(f"Epoch {epoch+1}/{num_epochs}, Average Loss: {avg_loss}")
    jax.clear_caches()

# Save the fine-tuned model
output_dir = "/tmp/montevideo_fine_tuned_model"
os.makedirs(output_dir, exist_ok=True)
unet.save_pretrained(output_dir, params=state.params["unet"])

print(f"Model saved to {output_dir}")