startup.sh

#!/bin/bash
set -e  # Exit on error
set -x  # Print commands for debugging

echo "Starting environment setup..."

# Enable device-side assertions
export TORCH_USE_CUDA_DSA=1
export CUDA_LAUNCH_BLOCKING=1

# Hugging Face cache settings (recommended)
export HF_HOME=/cache
export TRANSFORMERS_CACHE=/cache/transformers
export BITSANDBYTES_NOWELCOME=1
export HF_HUB_ENABLE_HF_TRANSFER=1

# Debugging info (before launching the model)
echo "CUDA Info:"
nvidia-smi
python -c "import torch; print('CUDA Available:', torch.cuda.is_available())"

# Upgrade bitsandbytes (before launching to avoid runtime issues)
pip install --upgrade bitsandbytes

# Start the model (this should be the last command)
exec text-generation-launcher