Upload 6 files

adapter_utils.py

@@ -0,0 +1,14 @@
+from peft import LoraConfig
+
+import config
+
+def load_adapter(target_modules):
+    peft_config = LoraConfig(
+        lora_alpha=config.LORA_ALPHA,
+        lora_dropout=config.LORA_DROPOUT,
+        r = config.LORA_RANK,
+        bias="none",
+        task_type=config.TASK_TYPE,
+        target_modules=target_modules
+    )
+    return peft_config

config.py

@@ -0,0 +1,53 @@
+'''
+Data Configuration
+'''
+DATASET = "OpenAssistant/oasst1"
+DATASET_TEXT_FIELD = "prompt_response"
+
+'''
+Model Configuration
+'''
+MODEL_NAME = "microsoft/phi-2"
+TRUST_REMOTE_CODE = True
+ENABLE_MODEL_CONFIG_CACHE = False
+
+'''
+Quantization Configuration
+'''
+ENABLE_4BIT = True
+QUANTIZATION_TYPE = "nf4"
+
+'''
+Adapter Configuration
+'''
+LORA_ALPHA = 16
+LORA_DROPOUT = 0.1
+LORA_RANK = 64
+TASK_TYPE = "CAUSAL_LM"
+
+'''
+Model Training Configuration
+'''
+MODEL_OUTPUT_DIR = "results/"
+PER_DEVICE_TRAIN_BATCH_SIZE = 4
+GRADIENT_ACCUMULATION_STEPS = 4
+OPTIM = "paged_adamw_32bit"
+SAVE_STEPS = 100
+LOGGING_STEPS = 10
+LEARNING_RATE = 2e-4
+MAX_GRAD_NORM = 0.3
+MAX_STEPS = 700
+WARMUP_RATIO = 0.05
+LR_SCHEDULER_TYPE = "constant"
+ENABLE_FP_16 = True
+ENABLE_GRADIENT_CHECKPOINTING=False
+
+'''
+Model Trainer Configuration
+'''
+MAX_SEQ_LENGTH = 512
+
+'''
+Inference Configuration
+'''
+TASK = "text-generation"

data_utils.py

@@ -0,0 +1,35 @@
+import datasets
+from datasets import load_dataset
+
+import config
+
+def download(mode):
+    print("Downloading Dataset - ", config.DATASET, "...")
+    dataset = load_dataset(config.DATASET, split=mode)
+    return dataset
+
+def prepare_prompts_responses(dataset):
+    print("Preparing Prompt and Assistant....")
+    dataset_df = dataset.to_pandas()
+    user_prompters = dataset_df[(dataset_df.role=="prompter")]
+    user_prompters = user_prompters.set_index("message_id")
+    assistants = dataset_df[(dataset_df.role=="assistant") & (dataset_df["rank"] == 0.0)]
+    
+    prompts_responses = []
+    for _,record in assistants.iterrows():
+        prompt_text = user_prompters.loc[record.parent_id,'text']
+        prompt_response = "### Human: " + prompt_text + " ### Assistant: " + record['text']
+        prompts_responses.append(prompt_response)
+    assistants[config.DATASET_TEXT_FIELD] = prompts_responses
+    
+    return assistants
+
+def preparedata(mode):
+    print("Preparing data for - ", mode, "...")
+    dataset = download(mode=mode)
+    prompts_responses = prepare_prompts_responses(dataset)
+    prompts_responses_dataset = datasets.Dataset.from_pandas(prompts_responses)
+    return prompts_responses_dataset
+    
+
+

model_utils.py

@@ -0,0 +1,48 @@
+from trl import SFTTrainer
+from transformers import AutoModelForCausalLM, AutoTokenizer, TrainingArguments
+
+import config
+
+def load_model(quantization_config):
+    model = AutoModelForCausalLM.from_pretrained(
+        config.MODEL_NAME,
+        quantization_config = quantization_config,
+        trust_remote_code = config.TRUST_REMOTE_CODE
+    )
+    model.config.use_cache = config.ENABLE_MODEL_CONFIG_CACHE
+    return model
+
+def load_tokenizers():
+    tokenizer = AutoTokenizer.from_pretrained(
+        config.MODEL_NAME, 
+        trust_remote_code=config.TRUST_REMOTE_CODE)
+    return tokenizer
+
+def load_training_arguments():
+    training_arguments = TrainingArguments(
+        output_dir=config.MODEL_OUTPUT_DIR,
+        per_device_train_batch_size=config.PER_DEVICE_TRAIN_BATCH_SIZE,
+        gradient_accumulation_steps=config.GRADIENT_ACCUMULATION_STEPS,
+        optim=config.OPTIM,
+        save_steps=config.SAVE_STEPS,
+        logging_steps=config.LOGGING_STEPS,
+        learning_rate=config.LEARNING_RATE,
+        fp16=config.ENABLE_FP_16,
+        max_grad_norm=config.MAX_GRAD_NORM,
+        max_steps=config.MAX_STEPS,
+        warmup_ratio=config.WARMUP_RATIO,
+        gradient_checkpointing=config.ENABLE_GRADIENT_CHECKPOINTING
+    )
+    return training_arguments
+
+def load_trainer(model, training_dataset, peft_config, tokenizer, training_arguments):
+    trainer = SFTTrainer(
+        model = model,
+        train_dataset = training_dataset,
+        peft_config = peft_config,
+        dataset_text_field = config.DATASET_TEXT_FIELD,
+        max_seq_length = config.MAX_SEQ_LENGTH,
+        tokenizer = tokenizer,
+        args = training_arguments
+    )
+    return trainer

quantization_utils.py

@@ -0,0 +1,13 @@
+import torch
+from transformers import BitsAndBytesConfig
+
+import config
+
+def load_bits_and_bytes_config():
+    bnb_config = BitsAndBytesConfig(
+        load_in_4bit=config.ENABLE_4BIT,
+        bnb_4bit_quant_type=config.QUANTIZATION_TYPE,
+        bnb_4bit_compute_dtype=torch.float16
+    )
+    
+    return bnb_config

requirements.txt

@@ -0,0 +1,10 @@
+torch
+torchvision
+trl
+transformers
+accelerate
+peft
+einops
+datasets
+bitsandbytes
+scipy