Update app.py

app.py

@@ -1,212 +1,212 @@
-# import torch
-# from torch.utils.data import DataLoader
-# from torch.utils.tensorboard import SummaryWriter
-# from transformers import AutoModelForCausalLM, AutoTokenizer
-# from peft import LoraConfig, get_peft_model, TaskType
-# import pandas as pd
-# from qa_dataset import QADataset
-# from tqdm import tqdm
-# import os, time, sys
-#
-#
-# def train_model(model, train_loader, val_loader, optimizer, gradient_accumulation_steps,
-#                 device, num_epochs, model_output_dir, writer):
-#     batch_step = 0
-#     for epoch in range(num_epochs):
-#         time1 = time.time()
-#         model.train()
-#         for index, data in enumerate(tqdm(train_loader, file=sys.stdout, desc="Train Epoch: " + str(epoch))):
-#             input_ids = data['input_ids'].to(device, dtype=torch.long)
-#             attention_mask = data['attention_mask'].to(device, dtype=torch.long)
-#             labels = data['labels'].to(device, dtype=torch.long)
-#             # 前向传播
-#             outputs = model(
-#                 input_ids=input_ids,
-#                 attention_mask=attention_mask,
-#                 labels=labels,
-#             )
-#             loss = outputs.loss
-#             # 反向传播，计算当前梯度
-#             loss.backward()
-#             # 梯度累积步数
-#             if (index % gradient_accumulation_steps == 0 and index != 0) or index == len(train_loader) - 1:
-#                 # 更新网络参数
-#                 optimizer.step()
-#                 # 清空过往梯度
-#                 optimizer.zero_grad()
-#                 writer.add_scalar('Loss/train', loss, batch_step)
-#                 batch_step += 1
-#             # 100轮打印一次 loss
-#             if index % 100 == 0 or index == len(train_loader) - 1:
-#                 time2 = time.time()
-#                 tqdm.write(
-#                     f"{index}, epoch: {epoch} -loss: {str(loss)} ; each step's time spent: {(str(float(time2 - time1) / float(index + 0.0001)))}")
-#         # 验证
-#         model.eval()
-#         val_loss = validate_model(model, val_loader, device)
-#         writer.add_scalar('Loss/val', val_loss, epoch)
-#         print(f"val loss: {val_loss} , epoch: {epoch}")
-#         print("Save Model To ", model_output_dir)
-#         model.save_pretrained(model_output_dir)
-#
-#
-# def validate_model(model, device, val_loader):
-#     running_loss = 0.0
-#     with torch.no_grad():
-#         for _, data in enumerate(tqdm(val_loader, file=sys.stdout, desc="Validation Data")):
-#             input_ids = data['input_ids'].to(device, dtype=torch.long)
-#             attention_mask = data['attention_mask'].to(device, dtype=torch.long)
-#             labels = data['labels'].to(device, dtype=torch.long)
-#             outputs = model(
-#                 input_ids=input_ids,
-#                 attention_mask=attention_mask,
-#                 labels=labels,
-#             )
-#             loss = outputs.loss
-#             running_loss += loss.item()
-#     return running_loss / len(val_loader)
-#
-#
-# def main():
-#     # 基础模型位置
-#     model_name = "model/Qwen2-1.5B-Instruct"
-#     # 训练集
-#     train_json_path = "./data/train.json"
-#     # 验证集
-#     val_json_path = "./data/val.json"
-#     max_source_length = 128
-#     max_target_length = 256
-#     epochs = 10
-#     batch_size = 1
-#     lr = 1e-4
-#     gradient_accumulation_steps = 16
-#     lora_rank = 8
-#     lora_alpha = 32
-#     model_output_dir = "output"
-#     logs_dir = "logs"
-#     # 设备
-#     device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
-#     # 加载分词器和模型
-#     tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
-#     model = AutoModelForCausalLM.from_pretrained(model_name, trust_remote_code=True)
-#     # setup peft
-#     peft_config = LoraConfig(
-#         task_type=TaskType.CAUSAL_LM,
-#         target_modules=["q_proj", "k_proj", "v_proj", "o_proj", "gate_proj", "up_proj", "down_proj"],
-#         inference_mode=False,
-#         r=lora_rank,
-#         lora_alpha=lora_alpha,
-#         lora_dropout=0.1
-#     )
-#     model = get_peft_model(model, peft_config)
-#     model.is_parallelizable = True
-#     model.model_parallel = True
-#     model.print_trainable_parameters()
-#     print("Start Load Train Data...")
-#     train_params = {
-#         "batch_size": batch_size,
-#         "shuffle": True,
-#         "num_workers": 0,
-#     }
-#     training_set = QADataset(train_json_path, tokenizer, max_source_length, max_target_length)
-#     training_loader = DataLoader(training_set, **train_params)
-#     print("Start Load Validation Data...")
-#     val_params = {
-#         "batch_size": batch_size,
-#         "shuffle": False,
-#         "num_workers": 0,
-#     }
-#     val_set = QADataset(val_json_path, tokenizer, max_source_length, max_target_length)
-#     val_loader = DataLoader(val_set, **val_params)
-#     # 日志记录
-#     writer = SummaryWriter(logs_dir)
-#     # 优化器
-#     optimizer = torch.optim.AdamW(params=model.parameters(), lr=lr)
-#     model = model.to(device)
-#     # 开始训练
-#     print("Start Training...")
-#     train_model(
-#         model=model,
-#         train_loader=training_loader,
-#         val_loader=val_loader,
-#         optimizer=optimizer,
-#         gradient_accumulation_steps=gradient_accumulation_steps,
-#         device=device,
-#         num_epochs=epochs,
-#         model_output_dir=model_output_dir,
-#         writer=writer
-#     )
-#     writer.close()
-#
-#
-# if __name__ == '__main__':
-#     main()
-#
-from transformers import AutoModelForCausalLM, AutoTokenizer, Trainer, TrainingArguments
-from peft import LoraConfig, get_peft_model, TaskType
-from datasets import load_dataset
-from torch.utils.tensorboard import SummaryWriter
-import os
-import torch
-def main():
-    # 基础模型位置
-    model_name = "model/Qwen2-1.5B-Instruct"
-    # 设备
-    device = "cuda" if torch.cuda.is_available() else "cpu"
-
-    # 加载分词器和模型
-    tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
-    model = AutoModelForCausalLM.from_pretrained(model_name, trust_remote_code=True)
-
-    # Setup PEFT (Low-Rank Adaption)
-    peft_config = LoraConfig(
-        task_type=TaskType.CAUSAL_LM,
-        target_modules=["q_proj", "k_proj", "v_proj", "o_proj", "gate_proj", "up_proj", "down_proj"],
-        inference_mode=False,
-        r=8,  # 低秩矩阵的秩
-        lora_alpha=32,  # LoRA的alpha超参数
-        lora_dropout=0.1
-    )
-    model = get_peft_model(model, peft_config)
-
-    # Load Dataset
-    train_dataset = load_dataset('json', data_files='./data/train.json', split='train')
-    val_dataset = load_dataset('json', data_files='./data/val.json', split='validation')
-
-    # Tokenize the datasets
-    def tokenize_function(examples):
-        return tokenizer(examples['input_text'], padding='max_length', truncation=True, max_length=128)
-
-    train_dataset = train_dataset.map(tokenize_function, batched=True)
-    val_dataset = val_dataset.map(tokenize_function, batched=True)
-
-    # Define Training Arguments
-    training_args = TrainingArguments(
-        output_dir="./output",             # 保存模型和日志的路径
-        evaluation_strategy="epoch",       # 每个epoch后进行验证
-        per_device_train_batch_size=1,     # 每个设备的batch size
-        per_device_eval_batch_size=1,      # 验证时的batch size
-        logging_dir="./logs",              # 日志目录
-        logging_steps=10,                  # 每10步记录一次日志
-        save_steps=100,                    # 每100步保存一次模型
-        num_train_epochs=10,               # 训练的epoch数
-        save_total_limit=2,                # 最大保存模型数
-    )
-
-    # Define the Trainer
-    trainer = Trainer(
-        model=model,                       # 训练的模型
-        args=training_args,                # 训练的参数
-        train_dataset=train_dataset,       # 训练数据集
-        eval_dataset=val_dataset,          # 验证数据集
-        tokenizer=tokenizer,               # 用于预处理的分词器
-    )
-
-    # Start Training
-    trainer.train()
-
-    # Save the model
-    model.save_pretrained('./output')
-
-if __name__ == '__main__':
-    main()
+# import torch
+# from torch.utils.data import DataLoader
+# from torch.utils.tensorboard import SummaryWriter
+# from transformers import AutoModelForCausalLM, AutoTokenizer
+# from peft import LoraConfig, get_peft_model, TaskType
+# import pandas as pd
+# from qa_dataset import QADataset
+# from tqdm import tqdm
+# import os, time, sys
+#
+#
+# def train_model(model, train_loader, val_loader, optimizer, gradient_accumulation_steps,
+#                 device, num_epochs, model_output_dir, writer):
+#     batch_step = 0
+#     for epoch in range(num_epochs):
+#         time1 = time.time()
+#         model.train()
+#         for index, data in enumerate(tqdm(train_loader, file=sys.stdout, desc="Train Epoch: " + str(epoch))):
+#             input_ids = data['input_ids'].to(device, dtype=torch.long)
+#             attention_mask = data['attention_mask'].to(device, dtype=torch.long)
+#             labels = data['labels'].to(device, dtype=torch.long)
+#             # 前向传播
+#             outputs = model(
+#                 input_ids=input_ids,
+#                 attention_mask=attention_mask,
+#                 labels=labels,
+#             )
+#             loss = outputs.loss
+#             # 反向传播，计算当前梯度
+#             loss.backward()
+#             # 梯度累积步数
+#             if (index % gradient_accumulation_steps == 0 and index != 0) or index == len(train_loader) - 1:
+#                 # 更新网络参数
+#                 optimizer.step()
+#                 # 清空过往梯度
+#                 optimizer.zero_grad()
+#                 writer.add_scalar('Loss/train', loss, batch_step)
+#                 batch_step += 1
+#             # 100轮打印一次 loss
+#             if index % 100 == 0 or index == len(train_loader) - 1:
+#                 time2 = time.time()
+#                 tqdm.write(
+#                     f"{index}, epoch: {epoch} -loss: {str(loss)} ; each step's time spent: {(str(float(time2 - time1) / float(index + 0.0001)))}")
+#         # 验证
+#         model.eval()
+#         val_loss = validate_model(model, val_loader, device)
+#         writer.add_scalar('Loss/val', val_loss, epoch)
+#         print(f"val loss: {val_loss} , epoch: {epoch}")
+#         print("Save Model To ", model_output_dir)
+#         model.save_pretrained(model_output_dir)
+#
+#
+# def validate_model(model, device, val_loader):
+#     running_loss = 0.0
+#     with torch.no_grad():
+#         for _, data in enumerate(tqdm(val_loader, file=sys.stdout, desc="Validation Data")):
+#             input_ids = data['input_ids'].to(device, dtype=torch.long)
+#             attention_mask = data['attention_mask'].to(device, dtype=torch.long)
+#             labels = data['labels'].to(device, dtype=torch.long)
+#             outputs = model(
+#                 input_ids=input_ids,
+#                 attention_mask=attention_mask,
+#                 labels=labels,
+#             )
+#             loss = outputs.loss
+#             running_loss += loss.item()
+#     return running_loss / len(val_loader)
+#
+#
+# def main():
+#     # 基础模型位置
+#     model_name = "model/Qwen2-1.5B-Instruct"
+#     # 训练集
+#     train_json_path = "./data/train.json"
+#     # 验证集
+#     val_json_path = "./data/val.json"
+#     max_source_length = 128
+#     max_target_length = 256
+#     epochs = 10
+#     batch_size = 1
+#     lr = 1e-4
+#     gradient_accumulation_steps = 16
+#     lora_rank = 8
+#     lora_alpha = 32
+#     model_output_dir = "output"
+#     logs_dir = "logs"
+#     # 设备
+#     device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
+#     # 加载分词器和模型
+#     tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
+#     model = AutoModelForCausalLM.from_pretrained(model_name, trust_remote_code=True)
+#     # setup peft
+#     peft_config = LoraConfig(
+#         task_type=TaskType.CAUSAL_LM,
+#         target_modules=["q_proj", "k_proj", "v_proj", "o_proj", "gate_proj", "up_proj", "down_proj"],
+#         inference_mode=False,
+#         r=lora_rank,
+#         lora_alpha=lora_alpha,
+#         lora_dropout=0.1
+#     )
+#     model = get_peft_model(model, peft_config)
+#     model.is_parallelizable = True
+#     model.model_parallel = True
+#     model.print_trainable_parameters()
+#     print("Start Load Train Data...")
+#     train_params = {
+#         "batch_size": batch_size,
+#         "shuffle": True,
+#         "num_workers": 0,
+#     }
+#     training_set = QADataset(train_json_path, tokenizer, max_source_length, max_target_length)
+#     training_loader = DataLoader(training_set, **train_params)
+#     print("Start Load Validation Data...")
+#     val_params = {
+#         "batch_size": batch_size,
+#         "shuffle": False,
+#         "num_workers": 0,
+#     }
+#     val_set = QADataset(val_json_path, tokenizer, max_source_length, max_target_length)
+#     val_loader = DataLoader(val_set, **val_params)
+#     # 日志记录
+#     writer = SummaryWriter(logs_dir)
+#     # 优化器
+#     optimizer = torch.optim.AdamW(params=model.parameters(), lr=lr)
+#     model = model.to(device)
+#     # 开始训练
+#     print("Start Training...")
+#     train_model(
+#         model=model,
+#         train_loader=training_loader,
+#         val_loader=val_loader,
+#         optimizer=optimizer,
+#         gradient_accumulation_steps=gradient_accumulation_steps,
+#         device=device,
+#         num_epochs=epochs,
+#         model_output_dir=model_output_dir,
+#         writer=writer
+#     )
+#     writer.close()
+#
+#
+# if __name__ == '__main__':
+#     main()
+#
+from transformers import AutoModelForCausalLM, AutoTokenizer, Trainer, TrainingArguments
+from peft import LoraConfig, get_peft_model, TaskType
+from datasets import load_dataset
+from torch.utils.tensorboard import SummaryWriter
+import os
+import torch
+def main():
+    # 基础模型位置
+    model_name = "D:\\Qwen2-1.5B-Instruct"
+    # 设备
+    device = "cuda" if torch.cuda.is_available() else "cpu"
+
+    # 加载分词器和模型
+    tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
+    model = AutoModelForCausalLM.from_pretrained(model_name, trust_remote_code=True)
+
+    # Setup PEFT (Low-Rank Adaption)
+    peft_config = LoraConfig(
+        task_type=TaskType.CAUSAL_LM,
+        target_modules=["q_proj", "k_proj", "v_proj", "o_proj", "gate_proj", "up_proj", "down_proj"],
+        inference_mode=False,
+        r=8,  # 低秩矩阵的秩
+        lora_alpha=32,  # LoRA的alpha超参数
+        lora_dropout=0.1
+    )
+    model = get_peft_model(model, peft_config)
+
+    # Load Dataset
+    train_dataset = load_dataset('json', data_files='./data/train.json', split='train')
+    val_dataset = load_dataset('json', data_files='./data/val.json', split='validation')
+
+    # Tokenize the datasets
+    def tokenize_function(examples):
+        return tokenizer(examples['input_text'], padding='max_length', truncation=True, max_length=128)
+
+    train_dataset = train_dataset.map(tokenize_function, batched=True)
+    val_dataset = val_dataset.map(tokenize_function, batched=True)
+
+    # Define Training Arguments
+    training_args = TrainingArguments(
+        output_dir="./output",             # 保存模型和日志的路径
+        evaluation_strategy="epoch",       # 每个epoch后进行验证
+        per_device_train_batch_size=1,     # 每个设备的batch size
+        per_device_eval_batch_size=1,      # 验证时的batch size
+        logging_dir="./logs",              # 日志目录
+        logging_steps=10,                  # 每10步记录一次日志
+        save_steps=100,                    # 每100步保存一次模型
+        num_train_epochs=10,               # 训练的epoch数
+        save_total_limit=2,                # 最大保存模型数
+    )
+
+    # Define the Trainer
+    trainer = Trainer(
+        model=model,                       # 训练的模型
+        args=training_args,                # 训练的参数
+        train_dataset=train_dataset,       # 训练数据集
+        eval_dataset=val_dataset,          # 验证数据集
+        tokenizer=tokenizer,               # 用于预处理的分词器
+    )
+
+    # Start Training
+    trainer.train()
+
+    # Save the model
+    model.save_pretrained('./output')
+
+if __name__ == '__main__':
+    main()