Upload llama-3-2-3b-appreciation-full-inference-test.ipynb
Browse files
llama-3-2-3b-appreciation-full-inference-test.ipynb
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[],"dockerImageVersionId":30787,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"%pip install bitsandbytes","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-28T08:23:02.211622Z","iopub.execute_input":"2024-11-28T08:23:02.211906Z","iopub.status.idle":"2024-11-28T08:23:15.556282Z","shell.execute_reply.started":"2024-11-28T08:23:02.211879Z","shell.execute_reply":"2024-11-28T08:23:15.555209Z"}},"outputs":[{"name":"stdout","text":"Collecting bitsandbytes\n Downloading bitsandbytes-0.44.1-py3-none-manylinux_2_24_x86_64.whl.metadata (3.5 kB)\nRequirement already satisfied: torch in /opt/conda/lib/python3.10/site-packages (from bitsandbytes) (2.4.0)\nRequirement already satisfied: numpy in /opt/conda/lib/python3.10/site-packages (from bitsandbytes) (1.26.4)\nRequirement already satisfied: filelock in /opt/conda/lib/python3.10/site-packages (from torch->bitsandbytes) (3.15.1)\nRequirement already satisfied: typing-extensions>=4.8.0 in /opt/conda/lib/python3.10/site-packages (from torch->bitsandbytes) (4.12.2)\nRequirement already satisfied: sympy in /opt/conda/lib/python3.10/site-packages (from torch->bitsandbytes) (1.13.3)\nRequirement already satisfied: networkx in /opt/conda/lib/python3.10/site-packages (from torch->bitsandbytes) (3.3)\nRequirement already satisfied: jinja2 in /opt/conda/lib/python3.10/site-packages (from torch->bitsandbytes) (3.1.4)\nRequirement already satisfied: fsspec in /opt/conda/lib/python3.10/site-packages (from torch->bitsandbytes) (2024.6.1)\nRequirement already satisfied: MarkupSafe>=2.0 in /opt/conda/lib/python3.10/site-packages (from jinja2->torch->bitsandbytes) (2.1.5)\nRequirement already satisfied: mpmath<1.4,>=1.1.0 in /opt/conda/lib/python3.10/site-packages (from sympy->torch->bitsandbytes) (1.3.0)\nDownloading bitsandbytes-0.44.1-py3-none-manylinux_2_24_x86_64.whl (122.4 MB)\n\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m122.4/122.4 MB\u001b[0m \u001b[31m14.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n\u001b[?25hInstalling collected packages: bitsandbytes\nSuccessfully installed bitsandbytes-0.44.1\nNote: you may need to restart the kernel to use updated packages.\n","output_type":"stream"}],"execution_count":1},{"cell_type":"markdown","source":"# Llama-3.2-3B-appreciation-full inference test","metadata":{}},{"cell_type":"markdown","source":"## Load model directly","metadata":{}},{"cell_type":"code","source":"from transformers import AutoTokenizer, AutoModelForCausalLM\n\ntokenizer = AutoTokenizer.from_pretrained(\"eltorio/Llama-3.2-3B-appreciation-full\")\nmodel = AutoModelForCausalLM.from_pretrained(\"eltorio/Llama-3.2-3B-appreciation-full\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-28T08:23:15.558374Z","iopub.execute_input":"2024-11-28T08:23:15.559124Z","iopub.status.idle":"2024-11-28T08:24:17.654413Z","shell.execute_reply.started":"2024-11-28T08:23:15.559082Z","shell.execute_reply":"2024-11-28T08:24:17.653473Z"}},"outputs":[{"output_type":"display_data","data":{"text/plain":"tokenizer_config.json: 0%| | 0.00/54.7k [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"cac20a492713460ba585931dccfad33b"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"tokenizer.json: 0%| | 0.00/17.2M [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"6d23a2d7216d4c129d07dad5fd168403"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"special_tokens_map.json: 0%| | 0.00/454 [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"858a86477560490ea127a4b090bbe8f9"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"config.json: 0%| | 0.00/1.42k [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"5dbb3026776c4b7381714d49dcec4528"}},"metadata":{}},{"name":"stderr","text":"Unused kwargs: ['_load_in_4bit', '_load_in_8bit', 'quant_method']. These kwargs are not used in <class 'transformers.utils.quantization_config.BitsAndBytesConfig'>.\n`low_cpu_mem_usage` was None, now set to True since model is quantized.\n","output_type":"stream"},{"output_type":"display_data","data":{"text/plain":"model.safetensors: 0%| | 0.00/2.24G [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"508bb44e09814ed497009306dff88a06"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"generation_config.json: 0%| | 0.00/189 [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"bf4fd1f8a1864039a1ebc0b5288b0728"}},"metadata":{}}],"execution_count":2},{"cell_type":"markdown","source":"## Define a function for getting a multiturn conversation","metadata":{}},{"cell_type":"code","source":"# Define a function to infer a evaluation from the incoming parameters\ndef infere(trimestre: str, moyenne_1: float,moyenne_2: float,moyenne_3: float, comportement: float, participation: float, travail: float) -> str:\n\n if trimestre == \"1\":\n trimestre_full = \"premier trimestre\"\n user_question = f\"Veuillez rédiger une appréciation en moins de 40 mots pour le {trimestre_full} pour cet élève qui a eu {moyenne_1} de moyenne, j'ai évalué son comportement à {comportement}/10, sa participation à {participation}/10 et son travail à {travail}/10. Les notes ne doivent pas apparaître dans l'appréciation.\"\n elif trimestre == \"2\":\n trimestre_full = \"deuxième trimestre\"\n user_question = f\"Veuillez rédiger une appréciation en moins de 40 mots pour le {trimestre_full} pour cet élève qui a eu {moyenne_2} de moyenne ce trimestre et {moyenne_1} au premier trimestre, j'ai évalué son comportement à {comportement}/10, sa participation à {participation}/10 et son travail à {travail}/10. Les notes ne doivent pas apparaître dans l'appréciation.\"\n elif trimestre == \"3\":\n trimestre_full = \"troisième trimestre\"\n user_question= f\"Veuillez rédiger une appréciation en moins de 40 mots pour le {trimestre_full} pour cet élève qui a eu {moyenne_3} de moyenne ce trimestre, {moyenne_2} au deuxième trimestre et {moyenne_1} au premier trimestre, j'ai évalué son comportement à {comportement}/10, sa participation à {participation}/10 et son travail à {travail}/10. Les notes ne doivent pas apparaître dans l'appréciation.\"\n messages = [\n {\n \"role\": \"system\",\n \"content\": \"Vous êtes une IA assistant les enseignants d'histoire-géographie en rédigeant à leur place une appréciation personnalisée pour leur élève en fonction de ses performances. Votre appreciation doit être en français, et doit aider l'élève à comprendre ses points forts et les axes d'amélioration. Votre appréciation doit comporter de 1 à 40 mots. Votre appréciation ne doit jamais comporter la valeur de la note. Votre appréciation doit utiliser le style impersonnel.Attention l'élément le plus important de votre analyse doit rester la moyenne du trimestre\"},\n {\n \"role\": \"user\",\n \"content\": user_question},\n ]\n return messages","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2024-11-28T08:24:17.655615Z","iopub.execute_input":"2024-11-28T08:24:17.655990Z","iopub.status.idle":"2024-11-28T08:24:17.662434Z","shell.execute_reply.started":"2024-11-28T08:24:17.655964Z","shell.execute_reply":"2024-11-28T08:24:17.661672Z"}},"outputs":[],"execution_count":3},{"cell_type":"code","source":"messages = infere(\"1\", 3, float('nan'), float('nan'), 10, 10, 10)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-28T08:24:17.664762Z","iopub.execute_input":"2024-11-28T08:24:17.665103Z","iopub.status.idle":"2024-11-28T08:24:17.677115Z","shell.execute_reply.started":"2024-11-28T08:24:17.665060Z","shell.execute_reply":"2024-11-28T08:24:17.676354Z"}},"outputs":[],"execution_count":4},{"cell_type":"markdown","source":"## Tokenize the input","metadata":{}},{"cell_type":"code","source":"inputs = tokenizer.apply_chat_template(\n messages,\n tokenize = True,\n add_generation_prompt = True,\n return_tensors = \"pt\",).to(\"cuda\")\ninputs","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-28T08:24:17.678000Z","iopub.execute_input":"2024-11-28T08:24:17.678293Z","iopub.status.idle":"2024-11-28T08:24:17.719967Z","shell.execute_reply.started":"2024-11-28T08:24:17.678268Z","shell.execute_reply":"2024-11-28T08:24:17.719187Z"}},"outputs":[{"execution_count":5,"output_type":"execute_result","data":{"text/plain":"tensor([[128000, 128006, 9125, 128007, 271, 38766, 1303, 33025, 2696,\n 25, 6790, 220, 2366, 18, 198, 15724, 2696, 25,\n 220, 1591, 4723, 220, 2366, 19, 271, 43273, 62299,\n 6316, 44190, 18328, 3625, 68061, 625, 1821, 294, 6,\n 90446, 2427, 978, 3257, 648, 665, 9517, 67, 7404,\n 519, 3869, 28130, 2035, 6316, 917, 43711, 5979, 367,\n 97252, 35965, 8047, 5019, 28130, 33013, 79351, 665, 34501,\n 409, 15907, 24601, 13, 650, 52262, 35996, 42182, 23761,\n 665, 55467, 11, 1880, 42182, 91878, 326, 6, 19010,\n 79351, 3869, 60946, 265, 15907, 3585, 75652, 1880, 3625,\n 25776, 294, 58591, 73511, 7769, 13, 650, 52262, 917,\n 43711, 5979, 367, 42182, 52962, 261, 409, 220, 16,\n 3869, 220, 1272, 78199, 13, 650, 52262, 917, 43711,\n 5979, 367, 841, 42182, 56316, 52962, 261, 1208, 51304,\n 409, 1208, 5296, 13, 650, 52262, 917, 43711, 5979,\n 367, 42182, 75144, 514, 1742, 60849, 8301, 47472, 3012,\n 326, 6, 29982, 479, 514, 5636, 3062, 409, 15265,\n 49586, 42182, 2800, 261, 1208, 52138, 26193, 3930, 75110,\n 265, 128009, 128006, 882, 128007, 271, 53, 89025, 9517,\n 67, 7420, 6316, 917, 43711, 5979, 367, 665, 40970,\n 409, 220, 1272, 78199, 5019, 514, 21134, 75110, 265,\n 5019, 42067, 33013, 79351, 7930, 264, 15925, 220, 18,\n 409, 52138, 26193, 11, 503, 34155, 4046, 26591, 978,\n 4538, 52962, 1133, 3869, 220, 605, 14, 605, 11,\n 829, 20852, 3869, 220, 605, 14, 605, 1880, 4538,\n 42775, 3869, 220, 605, 14, 605, 13, 11876, 8554,\n 841, 97569, 6502, 917, 5169, 66014, 7010, 326, 6,\n 391, 652, 978, 5979, 367, 13, 128009, 128006, 78191,\n 128007, 271]], device='cuda:0')"},"metadata":{}}],"execution_count":5},{"cell_type":"markdown","source":"## Generate the output","metadata":{}},{"cell_type":"code","source":"outputs = model.generate(input_ids = inputs, \n max_new_tokens = 90, \n use_cache = True,\n temperature = 1.5,\n min_p = 0.1,\n pad_token_id=tokenizer.eos_token_id,)\ndecoded_sequences = tokenizer.batch_decode(outputs[:, inputs.shape[1]:],skip_special_tokens=True)[0]\ndecoded_sequences","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-28T08:24:17.720836Z","iopub.execute_input":"2024-11-28T08:24:17.721094Z","iopub.status.idle":"2024-11-28T08:24:34.250620Z","shell.execute_reply.started":"2024-11-28T08:24:17.721070Z","shell.execute_reply":"2024-11-28T08:24:34.249701Z"}},"outputs":[{"name":"stderr","text":"The attention mask is not set and cannot be inferred from input because pad token is same as eos token. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\nStarting from v4.46, the `logits` model output will have the same type as the model (except at train time, where it will always be FP32)\n","output_type":"stream"},{"execution_count":6,"output_type":"execute_result","data":{"text/plain":"\"Pour le trimestre, vous avez démontré un engagement constant et un travail attentif en matière d'histoire-géographie. Votre comportement et votre participation à la classe ont été remarquables, montrant une grande qualité professionnelle.\""},"metadata":{}}],"execution_count":6}]}
|