| set -e # Exit on error | |
| set -x # Print commands for debugging | |
| echo "Starting environment setup..." | |
| # Enable device-side assertions | |
| export TORCH_USE_CUDA_DSA=1 | |
| export CUDA_LAUNCH_BLOCKING=1 | |
| # Hugging Face cache settings (recommended) | |
| export HF_HOME=/cache | |
| export TRANSFORMERS_CACHE=/cache/transformers | |
| export BITSANDBYTES_NOWELCOME=1 | |
| export HF_HUB_ENABLE_HF_TRANSFER=1 | |
| # Debugging info (before launching the model) | |
| echo "CUDA Info:" | |
| nvidia-smi | |
| python -c "import torch; print('CUDA Available:', torch.cuda.is_available())" | |
| # Upgrade bitsandbytes (before launching to avoid runtime issues) | |
| pip install --upgrade bitsandbytes | |
| # Start the model (this should be the last command) | |
| exec text-generation-launcher |