{ "cells": [ { "cell_type": "code", "execution_count": 1, "id": "8495bede-ab8f-416b-b5f2-6a76b1e63935", "metadata": { "tags": [] }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "D:\\Projects\\LLMs\\venv\\lib\\site-packages\\tqdm\\auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", " from .autonotebook import tqdm as notebook_tqdm\n" ] } ], "source": [ "from tqdm import tqdm\n", "from sentence_transformers import SentenceTransformer, util" ] }, { "cell_type": "code", "execution_count": 2, "id": "2b8cae6d-547b-4018-9f68-b0a45284b4b4", "metadata": { "tags": [] }, "outputs": [], "source": [ "# model = SentenceTransformer('DMetaSoul/sbert-chinese-general-v2')\n", "model = SentenceTransformer('TintinMeimei/menglang_yongtulv_aimatch_v1')" ] }, { "cell_type": "code", "execution_count": 3, "id": "d3907a6f-f8ab-40fe-8702-c8cb81e189c6", "metadata": { "tags": [] }, "outputs": [], "source": [ "def sim(text1, text2):\n", " emb1 = model.encode(text1, convert_to_tensor=True)\n", " emb2 = model.encode(text2, convert_to_tensor=True)\n", " score = util.cos_sim(emb1, emb2)\n", " return score" ] }, { "cell_type": "code", "execution_count": 24, "id": "3cec9f05-4ea9-46f8-a393-950c67a0150a", "metadata": { "tags": [] }, "outputs": [], "source": [ "text1 = '挂机空调'\n", "# text2 = '1.1.11 高效节能家用电器制造\\n包括节能型房间空调器、空调机组、电冰箱、电动洗衣机、平板电视机、电风扇等家用电器制造。房间空气调节器能效优于《房间空气调节器能效限定值及能效等级》（GB 12021.3）标准1级能效水平；转速可控型房间空气调节器能效优于《转速可控型房间空气调节器能效限定值及能效等级》（GB 21455）标准1级能效水平；多联式空调（热泵）机组能效比优于《多联式空调（热泵）机组能效限定值及能源效率等级》（GB 21454）标准1级能效水平；家用电冰箱能效优于《家用电冰箱耗电量限定值及能效等级》（GB 12021.2）标准1级能效水平；电动洗衣机能效优于《电动洗衣机能效水效限定值及等级》（GB 12021.4）标准1级能效水平；电饭煲能效优于《电饭锅能效限定值及能效等级》（GB 12021.6）标准1级能效水平；平板电视机能效优于《平板电视能效限定值及能效等级》（GB 24850）标准1级能效水平；交流电风扇能效优于《交流电风扇能效限定值及能效等级》（GB 12021.9）标准1级能效水平。其他高效节能家用电器能效均优于相应国家强制性标准1级能效水平。'\n", "# text2 = '包括节能泵、节能型真空干燥设备、节能型真空炉等设备制造。清水离心泵能效指标优于《清水离心泵能效限定值及节能评价值》（GB 19762）标准中节能评价值水平；石油化工离心泵能效优于《石油化工离心泵能效限定值及能效等级》（GB 32284）标准中1级能效水平；潜水电泵能效优于《井用潜水电泵能效限定值及能效等级》（GB 32030）、《小型潜水电泵能效限定值及能效等级》（GB 32029）、《污水污物潜水电泵能效限定值及能效等级》（GB 32031）标准中1级能效水平。'\n", "text2 = '退耕还林'" ] }, { "cell_type": "code", "execution_count": 25, "id": "d570bf57-2518-4306-a7ae-712e81199460", "metadata": { "tags": [] }, "outputs": [ { "data": { "text/plain": [ "tensor([[-0.5000]], device='cuda:0')" ] }, "execution_count": 25, "metadata": {}, "output_type": "execute_result" } ], "source": [ "sim(text1, text2)" ] }, { "cell_type": "markdown", "id": "040cc794-9bb0-4c22-986c-933ca55ee637", "metadata": {}, "source": [ "### Process Data" ] }, { "cell_type": "code", "execution_count": 6, "id": "d46e4e74-f7c2-4339-b009-4ba77f1b2f9a", "metadata": { "tags": [] }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
X1X2YSplit
0中新制药厂空调末端送回风系统改造-询价公示1.1.11 高效节能家用电器制造\\n包括节能型房间空调器、空调机组、电冰箱、电动洗衣机、平...1train
1中新制药厂空调末端送回风系统改造-询价公示1.5.1 锅炉（窑炉）节能改造和能效提升\\n包括燃煤锅炉“以大代小”，采用先进燃煤锅炉、节...0train
\n", "
" ], "text/plain": [ " X1 X2 \\\n", "0 中新制药厂空调末端送回风系统改造-询价公示 1.1.11 高效节能家用电器制造\\n包括节能型房间空调器、空调机组、电冰箱、电动洗衣机、平... \n", "1 中新制药厂空调末端送回风系统改造-询价公示 1.5.1 锅炉（窑炉）节能改造和能效提升\\n包括燃煤锅炉“以大代小”，采用先进燃煤锅炉、节... \n", "\n", " Y Split \n", "0 1 train \n", "1 0 train " ] }, "execution_count": 6, "metadata": {}, "output_type": "execute_result" } ], "source": [ "import pandas as pd\n", "\n", "df_data = pd.read_excel('AI匹配算法样本.xlsx', sheet_name='Sheet1', dtype=str)\n", "df_data.head(2)" ] }, { "cell_type": "code", "execution_count": 7, "id": "673ce0e0-2801-4bb3-8e5d-5c4aff3ac725", "metadata": { "tags": [] }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "C:\\Users\\vermouth\\AppData\\Local\\Temp\\ipykernel_22160\\3257358016.py:1: FutureWarning: Using short name for 'orient' is deprecated. Only the options: ('dict', list, 'series', 'split', 'records', 'index') will be used in a future version. Use one of the above to silence this warning.\n", " train_data = [item for item in df_data.to_dict(orient='record') if item['Split']=='train']\n", "C:\\Users\\vermouth\\AppData\\Local\\Temp\\ipykernel_22160\\3257358016.py:2: FutureWarning: Using short name for 'orient' is deprecated. Only the options: ('dict', list, 'series', 'split', 'records', 'index') will be used in a future version. Use one of the above to silence this warning.\n", " eval_data = [item for item in df_data.to_dict(orient='record') if item['Split']=='eval']\n", "C:\\Users\\vermouth\\AppData\\Local\\Temp\\ipykernel_22160\\3257358016.py:3: FutureWarning: Using short name for 'orient' is deprecated. Only the options: ('dict', list, 'series', 'split', 'records', 'index') will be used in a future version. Use one of the above to silence this warning.\n", " test_data = [item for item in df_data.to_dict(orient='record') if item['Split']=='test']\n" ] } ], "source": [ "train_data = [item for item in df_data.to_dict(orient='record') if item['Split']=='train']\n", "eval_data = [item for item in df_data.to_dict(orient='record') if item['Split']=='eval']\n", "test_data = [item for item in df_data.to_dict(orient='record') if item['Split']=='test']" ] }, { "cell_type": "markdown", "id": "5037803d-980d-48a1-a61d-528bb9508ce0", "metadata": {}, "source": [ "### Model 1 - Fine tune a Sentence Transformer" ] }, { "cell_type": "code", "execution_count": 8, "id": "773429e9-57ce-418f-ad44-3c35d1b31a74", "metadata": {}, "outputs": [], "source": [ "# from sentence_transformers import InputExample, losses\n", "# from torch.utils.data import DataLoader\n", "\n", "# # Prepare data\n", "# train_data_sbert = []\n", "# eval_data_sbert = []\n", "# test_data_sbert = []\n", "\n", "# for item in train_data:\n", "# label = 1.0 if float(item.get('Y')) == 1 else -1.0\n", "# train_data_sbert.append(InputExample(texts=[item.get('X1'), item.get('X2')], label=label))\n", "# train_dataloader = DataLoader(train_data_sbert, shuffle=True, batch_size=2)\n", "# train_loss = losses.CosineSimilarityLoss(model)" ] }, { "cell_type": "code", "execution_count": 9, "id": "ec1b68cb-bec3-4896-b196-ec31b1132ad1", "metadata": {}, "outputs": [], "source": [ "# from sentence_transformers import evaluation\n", "# evaluator = evaluation.EmbeddingSimilarityEvaluator([item.get('X1') for item in eval_data], [item.get('X2') for item in eval_data], [1.0 if float(item.get('Y'))==1 else -1.0 for item in eval_data])" ] }, { "cell_type": "code", "execution_count": 10, "id": "7c05c6ef-c5e7-416b-b797-9f8735ae5436", "metadata": {}, "outputs": [], "source": [ "# model.fit(train_objectives=[(train_dataloader, train_loss)], epochs=1, warmup_steps=100, evaluator=evaluator, evaluation_steps=500)" ] }, { "cell_type": "code", "execution_count": 11, "id": "7de1e5f0-4b83-4d34-8385-77cdaa0ef08f", "metadata": {}, "outputs": [], "source": [ "# model.save('./tmp_model')" ] }, { "cell_type": "markdown", "id": "fdd686b1-c654-4135-8989-05f23c914afa", "metadata": { "tags": [] }, "source": [ "### Model 2 - No Fine Tune + Threshold Tuning" ] }, { "cell_type": "code", "execution_count": 12, "id": "a0247889-577d-4a92-8c0f-9c923748df93", "metadata": { "tags": [] }, "outputs": [], "source": [ "def sim(text1, text2):\n", " emb1 = model.encode(text1, convert_to_tensor=True)\n", " emb2 = model.encode(text2, convert_to_tensor=True)\n", " score = util.cos_sim(emb1, emb2)\n", " return score\n", "\n", "def _acc_thres(scores, thres):\n", " correct = 0\n", " total = len(scores)\n", " for score, truth in scores:\n", " truth = float(truth)\n", " pred = 1 if score >= thres else 0\n", " if pred == truth:\n", " correct += 1\n", " return round(correct/total, 3)\n", "\n", "def model_train(train_data, eval_data):\n", " score_train = []\n", " score_eval = []\n", " for item in tqdm(train_data):\n", " score = sim(item['X1'], item['X2'])\n", " score_train.append((score, item['Y']))\n", " for item in tqdm(eval_data):\n", " score = sim(item['X1'], item['X2'])\n", " score_eval.append((score, item['Y']))\n", " # find threshold that minize train error\n", " score_train = sorted(score_train, reverse=True)\n", " win_acc = -1\n", " win_thres = -1\n", " for thres in range(5, 100, 5):\n", " thres = thres*0.01\n", " acc = _acc_thres(score_train, thres)\n", " if acc > win_acc:\n", " win_acc = acc\n", " win_thres = thres\n", " eval_acc = _acc_thres(score_eval, win_thres)\n", " return {'thres': win_thres, 'train_accuracy': win_acc, 'eval_accuracy': eval_acc}" ] }, { "cell_type": "code", "execution_count": 13, "id": "4e943ef9-ad40-494e-9d53-db9ccbf48bb4", "metadata": { "tags": [] }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 12256/12256 [13:54<00:00, 14.69it/s]\n", "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 4248/4248 [04:44<00:00, 14.94it/s]\n" ] } ], "source": [ "r = model_train(train_data, eval_data)" ] }, { "cell_type": "code", "execution_count": 14, "id": "9cd38cc9-fe71-45b9-a22e-977a2e787fb5", "metadata": { "tags": [] }, "outputs": [ { "data": { "text/plain": [ "{'thres': 0.25, 'train_accuracy': 0.831, 'eval_accuracy': 0.816}" ] }, "execution_count": 14, "metadata": {}, "output_type": "execute_result" } ], "source": [ "r" ] }, { "cell_type": "code", "execution_count": 15, "id": "53622ff1-7465-4663-a9f0-0c18df37b93e", "metadata": { "tags": [] }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 4468/4468 [04:58<00:00, 14.98it/s]\n" ] } ], "source": [ "score_test = []\n", "for item in tqdm(test_data):\n", " score = sim(item['X1'], item['X2'])\n", " score_test.append((score, item['Y']))" ] }, { "cell_type": "code", "execution_count": 17, "id": "47411f71-c774-4274-a1af-2a128589b559", "metadata": { "tags": [] }, "outputs": [ { "data": { "text/plain": [ "0.815" ] }, "execution_count": 17, "metadata": {}, "output_type": "execute_result" } ], "source": [ "_acc_thres(score_test, r['thres'])\n", "#_acc_thres(score_test, 0.25)" ] }, { "cell_type": "code", "execution_count": null, "id": "59b741bc-7a20-4ed0-bc9d-b82ec3edff34", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.0" } }, "nbformat": 4, "nbformat_minor": 5 }