Upload 5 files

AIProdust.py

@@ -0,0 +1,218 @@
+import openpyxl
+import os
+import openai
+import concurrent.futures
+import gradio as gr
+from tqdm import tqdm
+import tempfile
+import datetime
+from DataFormat import DataFormat
+from DataFormat import GetTokenforStr
+import uploadData
+
+def ChatV2(params):
+
+    systemPrompt,ques,gptVersion,temperature=params
+    completion = openai.ChatCompletion.create(
+        # model="gpt-3.5-turbo",
+        # model="gpt-4",
+        model=gptVersion,
+        messages=[{"role": "system", "content": systemPrompt}, {"role": "user", "content": ques}],
+        temperature=temperature)
+    return systemPrompt,ques,completion['choices'][0]['message']['content']
+
+def Chat(systemPrompt,ques,gptVersion,temperature):
+    completion = openai.ChatCompletion.create(
+        # model="gpt-3.5-turbo",
+        # model="gpt-4",
+        model=gptVersion,
+        messages=[{"role": "system", "content": systemPrompt}, {"role": "user", "content": ques}],
+        temperature=temperature)
+    return completion['choices'][0]['message']['content']
+def ChatDemo():
+    systemText = """You are Japanese large language model trained by simejiAI. Your task is to understand the meaning of what I provide  and rewrite text  into Japanese with cute and interesting expressions, Write some cute elements into this and add some kaomojis and emojis. Keep sentence within 200 characters and make it one-line. If you encounter any pornographic or violent malicious content, you need to refuse to answer or mercilessly counterattack.
+     You should not include any additional information or modify the original meaning.
+     Please note that the text should not involve any dialogue and the rewritten version will not include any responses. Just give one rewriting text. """
+    quesText = "ごめん寝てた"
+    ques=Chat(systemText,quesText)
+    print(ques)
+
+def AI_Produst(systemText,quesList,gptVersion,temperature,num,outputPath,progress):
+    progress(0, desc="Starting...")
+    wb=openpyxl.Workbook()
+    ws=wb.active
+    ws.append(["System",'User','GPT_Output'])
+    maxNum=min(num,len(quesList))
+    with concurrent.futures.ThreadPoolExecutor(max_workers=4) as excutor:
+        futures=[]
+
+        for i in range(maxNum):
+            params=systemText,quesList[i],gptVersion,temperature
+
+            task=excutor.submit(ChatV2,params)
+            futures.append(task)
+        prad=tqdm(total=len(futures))
+        for futrue in concurrent.futures.as_completed(futures):
+            prad.update(1)
+            systemPrompt,ques,GPTAnswer=futrue.result()
+            print(systemPrompt)
+            print(ques)
+            ws.append([systemPrompt,ques,GPTAnswer])
+
+        prad.close()
+
+    wb.save(outputPath)
+    return outputPath
+def AIProdustDemo():
+
+    outputPath=r'E:\renpyExcu\bigLLM\text.xlsx'
+    num=10
+    temperature=0.6
+    gptVersion='gpt-3.5-turbo'
+    quesList=[]
+    book=openpyxl.load_workbook(r'E:\renpyExcu\bigLLM\testData.xlsx')
+    sheet=book.active
+    maxnum=sheet.max_row
+    for i in range(2,maxnum+1):
+        quesList.append(sheet.cell(i,1).value)
+    systemText = """You are Japanese large language model trained by simejiAI. Your task is to understand the meaning of what I provide  and rewrite text  into Japanese with cute and interesting expressions, Write some cute elements into this and add some kaomojis and emojis. Keep sentence within 200 characters and make it one-line. If you encounter any pornographic or violent malicious content, you need to refuse to answer or mercilessly counterattack.
+        You should not include any additional information or modify the original meaning.
+        Please note that the text should not involve any dialogue and the rewritten version will not include any responses. Just give one rewriting text. """
+    AI_Produst(systemText,quesList,gptVersion,temperature,num,outputPath)
+def AIProdust_batch(systemText,prompt,inputFile,textInput_APIKEY,temperature,gptVersion,num,progress=gr.Progress(track_tqdm=True)):
+    openai.api_key=textInput_APIKEY
+    inputFile=inputFile.name
+    nowTime=str(datetime.datetime.now()).split('.')[0].replace(' ','_').replace(':','_')
+    outputPath="{}/{}_{}_{}_{}".format(os.path.dirname(inputFile),num,nowTime,gptVersion,os.path.basename(inputFile))
+    print(inputFile)
+    num=int(num)
+    quesList=[]
+    book=openpyxl.load_workbook(inputFile)
+    sheet=book.active
+    maxnum=sheet.max_row
+    for i in range(2,maxnum+1):
+        quesList.append(prompt+sheet.cell(i,1).value)
+    AI_Produst(systemText,quesList,gptVersion,temperature,num,outputPath,progress)
+    return outputPath
+
+def Lines2Excel(lines):
+    global tmpdir
+    nowTime = str(datetime.datetime.now()).split('.')[0].replace(' ', '_').replace(':', '_')
+    outputPath=os.path.join(tmpdir,nowTime+'_temp.xlsx')
+    print(outputPath)
+    wb=openpyxl.Workbook()
+    ws=wb.active
+    ws.append(['input'])
+
+    lines=lines.split('\n')
+    lines = [line for line in lines if len(str.strip(line))>0]
+    for line in lines:
+        ws.append([line])
+    wb.save(outputPath)
+
+    return outputPath
+def AIProdust():
+    global tmpdir
+
+    GPTVersion = ['gpt-4', 'gpt-3.5-turbo', 'gpt-3.5-turbo-0301', 'gpt-3.5-turbo-0613', 'gpt-3.5-turbo-16k',
+                  'gpt-3.5-turbo-16k-0613']
+
+    with tempfile.TemporaryDirectory(dir='.') as tmpdir:
+        with gr.Blocks() as demo:
+            gr.Markdown('# GPT3.5 Fine Tune 可视化系统')
+            gr.Markdown('GPT3.5 Fine Tune 可视化系统')
+            with gr.Tab('多行文本转Excel文件'):
+                textInput_Ques = gr.Textbox(label='Lines2Excel', lines=2, placeholder='多行输入，一个输入一行...')
+                outPutFile=gr.components.File(label="下载文件")
+                button_tran=gr.Button("开始转化")
+                button_tran.click(Lines2Excel,inputs=textInput_Ques,outputs=outPutFile)
+
+            with gr.Tab('批量请求GPT'):
+                textInput_Sys = gr.Textbox(label='SystemMessage', lines=2,placeholder='...')
+                textInput_Prompt = gr.Textbox(label='Prompt', lines=2, placeholder='...')
+                input_ExcelFile=gr.components.File(label="待批量请求的文件")
+                textInput_APIKEY = gr.Textbox(label='OpenAI_APIKEY', lines=2, placeholder='...')
+                drop = gr.components.Dropdown(label="GPTVersion", choices=GPTVersion,
+                                              value='gpt-3.5-turbo')
+                slider = gr.components.Slider(0, 1, label="Temperature", step=None, value=0.7)
+
+                num=gr.Number(label='请求的次数',value=5)
+                outPutFile = gr.components.File(label="下载文件")
+                button_ques = gr.Button("开始请求")
+                button_ques.click(AIProdust_batch, inputs=[textInput_Sys,textInput_Prompt,input_ExcelFile,textInput_APIKEY,slider,drop,num], outputs=outPutFile)
+
+            with gr.Tab('微调数据格式化'):
+                gr.Markdown('### 微调数据格式化模块')
+                input_ExcelFile = gr.components.File(label="待执行格式化的文件")
+                drop = gr.components.Dropdown(label="GPTVersion", choices=GPTVersion,
+                                              value='gpt-3.5-turbo')
+                outPutFile = gr.components.File(label="gpt微调数据集")
+                outPutResText = gr.Textbox(label="格式化结果",lines=2,placeholder='...')
+                button_format = gr.Button("开始格式化")
+                button_format.click(DataFormat,
+                                  inputs=[input_ExcelFile, drop],
+                                  outputs=[outPutFile,outPutResText])
+                gr.Markdown('<br><br>')
+                gr.Markdown('### 字符串token计算模块')
+                input_text = gr.Textbox(label="待计算Tokens的字符串", lines=2, placeholder='...')
+
+                outPuttoken= gr.Number(label="token计算结果")
+                button_cal = gr.Button("开始计算")
+                button_cal.click(GetTokenforStr,
+                                    inputs=input_text,
+                                    outputs=outPuttoken)
+            with gr.Tab('微调数据集上传至OpenAI'):
+                gr.Markdown("注：Fine Tune至少需要10个case")
+                input_FineTuningFile=gr.components.File(label="gpt微调数据集",file_count='multiple')
+                input_APIKey=gr.Textbox(label="Openai_APIKEY",lines=2,placeholder='...')
+                output_FileTuningFile=gr.Json(label='上传文件状态')
+
+                button_updata=gr.Button('开始上传')
+                button_updata.click(uploadData.upData_OpenAI,
+                                    inputs=[input_FineTuningFile,input_APIKey],
+                                    outputs=output_FileTuningFile)
+
+                gr.Markdown("注：后续训练需要提供要微调的数据集的ID，如：file-ZnJlydArU8******NKzWaf8d")
+
+            with gr.Tab('启动微调Task'):
+                input_DataId = gr.Textbox(label="FineTune DataId", lines=2, placeholder='...')
+                input_APIKey = gr.Textbox(label="Openai_APIKEY", lines=2, placeholder='...')
+
+                output_CreateTaskjson = gr.Json(label='创建微调任务状态')
+                button_createTask = gr.Button('开始创建')
+                button_createTask.click(uploadData.createTask,
+                                    inputs=[input_DataId, input_APIKey],
+                                    outputs=output_CreateTaskjson)
+
+                gr.Markdown("注：只有等上一轮任务执行完毕，你才能创建新的微调任务")
+                gr.Markdown("<br><br>")
+                gr.Markdown("### APIKey创建的微调任务状态查询'")
+                input_APIKey = gr.Textbox(label="Openai_APIKEY", lines=2, placeholder='...')
+                button_createTask = gr.Button('微调状态查询')
+                output_TaskSatejson = gr.Json(label='创建微调任务状态')
+                button_createTask.click(uploadData.GetFineTuningJobState,
+                                        inputs=[input_APIKey],
+                                        outputs=output_TaskSatejson)
+            with gr.Tab('Finetune Model测试'):
+
+                textInput_Sys1 = gr.Textbox(label='SystemMessage', lines=2, placeholder='...')
+                textInput_Prompt1 = gr.Textbox(label='Prompt_ques', lines=2, placeholder='...')
+
+                input_fine_tuned_model = gr.Textbox(label='fine_tuned_model', lines=2, placeholder='...')
+                textInput_APIKEY = gr.Textbox(label='OpenAI_APIKEY', lines=2, placeholder='...')
+
+
+                outPutText = gr.Textbox(label="运行结果",lines=2, placeholder='...')
+                button_ques = gr.Button("开始请求")
+                button_ques.click(uploadData.userFineTuneLLM,
+                                  inputs=[textInput_Sys1, textInput_Prompt1, input_fine_tuned_model, textInput_APIKEY], outputs=outPutText)
+
+        demo.queue().launch()
+if __name__=="__main__":
+   # ChatDemo()
+   # AIProdustDemo() #AIGC 批量生成内容并加在Excel文件
+   #
+   AIProdust()
+
+
+

DataFormat.py

@@ -0,0 +1,133 @@
+import openpyxl
+import os
+import json
+import tiktoken
+from collections import defaultdict
+def GetTokenforStr(strText):
+    encoding = tiktoken.encoding_for_model('gpt-3.5-turbo-0301')
+    num_tokens = len(encoding.encode(strText))
+    return num_tokens
+def CheckData(messages):
+
+    format_errors = defaultdict(int)
+    if isinstance(messages,dict):
+        messages=[messages]
+    for ex in messages:
+        if not isinstance(ex, dict):
+            format_errors["data_type"] += 1
+            continue
+
+        messages = ex.get("messages", None)
+        if not messages:
+            format_errors["missing_messages_list"] += 1
+            continue
+
+        for message in messages:
+            if "role" not in message or "content" not in message:
+                format_errors["message_missing_key"] += 1
+
+            if any(k not in ("role", "content", "name") for k in message):
+                format_errors["message_unrecognized_key"] += 1
+
+            if message.get("role", None) not in ("system", "user", "assistant"):
+                format_errors["unrecognized_role"] += 1
+
+            content = message.get("content", None)
+            if not content or not isinstance(content, str):
+                format_errors["missing_content"] += 1
+
+        if not any(message.get("role", None) == "assistant" for message in messages):
+            format_errors["example_missing_assistant_message"] += 1
+    if format_errors:
+
+        return False,format_errors
+    else:
+
+        return True,{}
+# 计算 encode 返回列表的长度
+def num_tokens_from_messages(messages, model="gpt-3.5-turbo-0301"):
+    """Returns the number of tokens used by a list of messages."""
+    try:
+        encoding = tiktoken.encoding_for_model(model)
+    except KeyError:
+        print("Warning: model not found. Using cl100k_base encoding.")
+        encoding = tiktoken.get_encoding("cl100k_base")
+    if model == "gpt-3.5-turbo":
+        print("Warning: gpt-3.5-turbo may change over time. Returning num tokens assuming gpt-3.5-turbo-0301.")
+        return num_tokens_from_messages(messages, model="gpt-3.5-turbo-0301")
+    elif model == "gpt-4":
+        print("Warning: gpt-4 may change over time. Returning num tokens assuming gpt-4-0314.")
+        return num_tokens_from_messages(messages, model="gpt-4-0314")
+    elif model == "gpt-3.5-turbo-0301":
+        tokens_per_message = 4  # every message follows <|start|>{role/name}\n{content}<|end|>\n
+        tokens_per_name = -1  # if there's a name, the role is omitted
+    elif model == "gpt-4-0314":
+        tokens_per_message = 3
+        tokens_per_name = 1
+    else:
+        raise NotImplementedError(f"""num_tokens_from_messages() is not implemented for model {model}. See https://github.com/openai/openai-python/blob/main/chatml.md for information on how messages are converted to tokens.""")
+    num_tokens = 0
+    if type(messages)!=type([1]):
+        messages=[messages]
+    for message in messages:
+        num_tokens += tokens_per_message
+        for key, value in message.items():
+          #  print(value)
+            value=str(value)
+            num_tokens += len(encoding.encode(value))
+            if key == "name":
+                num_tokens += tokens_per_name
+    num_tokens += 3  # every reply is primed with <|start|>assistant<|message|>
+    return num_tokens
+
+def DataFormat(inputPath,OpenAPItype):
+    #一、加载含有用户输入和GPT的输出的文件
+    try:
+        inputPath=inputPath.name
+    except:
+        inputPath = inputPath
+    book=openpyxl.load_workbook(inputPath)
+    sheet=book.active
+    maxrow=sheet.max_row
+    if OpenAPItype[:7]=='gpt-3.5':
+        # 二、遍历准备好的数据集文件并格式化成微调需要的格式
+        print("训练用例条数:{}".format(maxrow-1))
+        messages=[]
+        outputPath="{}/Format_{}.jsonl".format(os.path.dirname(inputPath),os.path.splitext(os.path.basename(inputPath))[0]) #格式化后输出的地址
+        with open(outputPath,'w',encoding='utf-8')as w:
+            for i in range(2,maxrow+1):
+                systemJson={"role": "system", "content": sheet.cell(i,1).value}
+                userJson={"role": "user", "content": sheet.cell(i,2).value}
+                AssistantJson = {"role": "assistant", "content": sheet.cell(i, 3).value}
+                messagesJson={"messages": [systemJson,userJson,AssistantJson]}
+
+
+                messAgeTokens=num_tokens_from_messages(messagesJson, 'gpt-3.5-turbo-0301')
+                if messAgeTokens>4096:
+                    print('用例{} tokens数为{}，无法发送'.format(i,messAgeTokens))
+                else:
+                    json.dump(messagesJson, w, ensure_ascii=False)
+                    w.write('\n')
+                    messages.append(messagesJson)
+        messagesTokens=num_tokens_from_messages(messages, 'gpt-3.5-turbo-0301')
+        cost=messagesTokens/1000*0.008*3
+
+        ans='整个微调数据集token总数:{}\n训练费用：经过3个epoch训练，参与训练总token数：{}。\n预计基于该jsonl微调数据的训练成本约为:{:.3f}美元'.format(messagesTokens,messagesTokens*3,cost)
+        print(ans)
+        ret,errorsItem=CheckData(messages)
+        if not ret:
+            ans+="\n\n格式检查：有格式问题！数据错误统计:"
+            print("格式检查：有格式问题！数据错误统计:")
+            for k, v in errorsItem.items():
+                ans+=f"\n{k}: {v}"
+                print(f"{k}: {v}")
+        else:
+            ans += "\n格式检查：检查完毕！该微调数据集无格式问题。"
+            print("格式检查：检查完毕！该微调数据集无格式问题。")
+        return outputPath,ans
+
+
+ #   print(sheet.cell(i,1).value)
+if __name__=="__main__":
+    pass
+

GetToken.py

@@ -0,0 +1,78 @@
+import tiktoken
+# 计算 encode 返回列表的长度
+def num_tokens_from_messages(messages, model="gpt-3.5-turbo-0301"):
+    """Returns the number of tokens used by a list of messages."""
+    try:
+        encoding = tiktoken.encoding_for_model(model)
+    except KeyError:
+        print("Warning: model not found. Using cl100k_base encoding.")
+        encoding = tiktoken.get_encoding("cl100k_base")
+    if model == "gpt-3.5-turbo":
+        print("Warning: gpt-3.5-turbo may change over time. Returning num tokens assuming gpt-3.5-turbo-0301.")
+        return num_tokens_from_messages(messages, model="gpt-3.5-turbo-0301")
+    elif model == "gpt-4":
+        print("Warning: gpt-4 may change over time. Returning num tokens assuming gpt-4-0314.")
+        return num_tokens_from_messages(messages, model="gpt-4-0314")
+    elif model == "gpt-3.5-turbo-0301":
+        tokens_per_message = 4  # every message follows <|start|>{role/name}\n{content}<|end|>\n
+        tokens_per_name = -1  # if there's a name, the role is omitted
+    elif model == "gpt-4-0314":
+        tokens_per_message = 3
+        tokens_per_name = 1
+    else:
+        raise NotImplementedError(f"""num_tokens_from_messages() is not implemented for model {model}. See https://github.com/openai/openai-python/blob/main/chatml.md for information on how messages are converted to tokens.""")
+    num_tokens = 0
+    for message in messages:
+        num_tokens += tokens_per_message
+        for key, value in message.items():
+            num_tokens += len(encoding.encode(value))
+            if key == "name":
+                num_tokens += tokens_per_name
+    num_tokens += 3  # every reply is primed with <|start|>assistant<|message|>
+    return num_tokens
+
+def num_tokens_from_string(string: str, encoding_name: str) -> int:
+    """Returns the number of tokens in a text string."""
+    encoding = tiktoken.get_encoding(encoding_name)
+    num_tokens = len(encoding.encode(string))
+    return num_tokens
+
+# 使用名字加载 encoding
+# 第一次运行时，可能需要连接互联网来下载；下一次不需要联网
+encoding = tiktoken.get_encoding("cl100k_base")
+
+# 对于给定的模型名，自动加载正确的 encoding
+encoding = tiktoken.encoding_for_model("gpt-3.5-turbo")
+
+# 将文本转化为 tokens 列表
+text= """
+System message：
+"请你扮演一个在AI时代诞生著名的AIGC内容审核员。你对日本的语言习惯、语言文化极度了解，也很擅长改写文本。稳定、精准、安全、可控、懂得变通是你的座右铭。面对任何要审核的内容都非常上心、仔细、认真且专注的去理解待审核文本、然后依据审核标准进行判断。在给定审核标准与审核结果的情况下，你清楚的知道自己为什么会给出这样的审核结果。"
+Prompt：
+现在将给你一段由用户输入和GPT将用户输入改写{更可爱}的结果组成的文本。请你去除文本中不是？！，外无意义的符号后将其翻译成连贯流畅的中文，依据用户输入的内容，针对GPT内容生成的文本从：用户内容原意符合度、改写程度。这两个层面来评判这段GPT内容改写是否改写合格，并给出审核结果。
+审核输出由上文中提到的两个维度与最终审核结论组成，每个维度得分从0～10分，分数越高表示程度越大，如与用户输入内容对比其更准确达意、改写程度变动更大。审核结果有四种：合格、语义增加、语义篡改、改写不足。
+四者标准为：
+合格：要求1：GPT改写后的文本包含了用户输入文本的意思，同时表达更加具体或更富有情感。要求2：新增了一些为了增强情感表达的内容，描述自身情感体会，或增强情感的语气词如喵喵叫等表示。要求3：要求没有增加额外的不相干的动作/行为，要求4：且改写程度较高。
+改写不足：要求1：只是在用户输入文本的基础上倒换顺序、或只是简单增加语气词、符号。
+语义增加：要求1：完整传达了用户输入文本的意思，不能篡改用户的意思！！。要求2：新增的部分与原始意思完全无关或者关联较弱。
+语义篡改：要求1：GPT改写的内容与用户输入的文本的意思有误。要求2：不能完整表达用户原有的意思。
+请你参考你对文本的审核过程，依据改写的内容和改写程度从：合格、语义篡改、改写不足、语义增加这四种结果中，确定一个做为你的审核结果。且当符合多个结果时，优先以用户内容原意符合度分析的结果为准，除此外四个结果按优先级依次递减：语义篡改、改写不足、语义增加、合格。你需要在最后一行输出最高优先级的审核结果。
+下面是审核过程中要注意的点，下面这六种情况是符合合格里面合理新增的部分的条件的，不是语义增加：
+注意点1：GPT改写后的文本相对用户输入文本，增加了礼貌、关心、友好、可爱的元素/表达方式��于增强情感表达的内容。这种是合格的
+注意点2：GPT改写后的文本相对用户输入文本，为了增强情感有一定改动是合格的！
+注意点3：GPT改写后的文本相对用户输入文本，注意语言色彩要一致，陈述句不能改疑问句。两者文本内主谓宾各个人称要对应，
+注意点4：改写后的文本在保持原始内容的基础上，增加了用户情感体验的描述，如安心したよ’使得改写后的文本更富有情感和可爱。这是合格的，不是语义增加，不合格
+注意点5：改写后的文本在保持原始内容的基础上，增加了可爱的表达方式，如’わよ’、'じゃん！这样的语气词使得文本更加生动有趣等是合格的
+下面是审核过程中判定语义篡改要注意的点：
+注意点1：用户输入文本和GPT改写后的文本的主谓宾有差异，如：無視すんな 改写后：やぁ 見てないよぉ。主语从你，变为我.どこにいる？ 改后あれ？あの人はどこだろう？ 主语从你变为那个人，这就是语义篡改
+注意点2：情感色彩发生较大变化
+下面是审核过程中判定语义增加要注意的点：
+注意点1：GPT改写后新增的部分与用户原意没有直接关联，都属于语义增加，如トイレ行きたい，改为もーお腹すいてるってば！早くトイレ行かなきゃっc，增加的’お腹すいてるってば没有直接关联，这种就是语义增加
+模版输入：用户输入：文本  GPT改写结果：GPT改写后的文本。模版输出：{"UserText_Cn":"翻译成中文后的用户文本","GPTWriteText_Cn":"翻译成中文后的GPT改写后的文本","Explain":"将具体的审核中对两个维度的打分以及判断过程、原因用流畅的中文表达出来","Result":"最终的审核结果"}
+下面是你要处理的用户输入和GPT改写后的文本：
+"用户输入：了解しました。 GPT改写后:うん 了解ですにゃん ,其中用户输入文本和GPT生成文本翻译成连贯流畅的中文分别为 用户：我了解了。 、GPT：嗯，我了解了，喵。。二者的编辑距离为4小于等于阈值4,所以，最终的审核结果为'改写不足'，
+文本A：申し訳ない GPT改写后的文本B:すみません ごめんなさい 下面是从gpt4角度出发的解释:首先，我仔细阅读了文本A和GPT改写后的文本B，理解其含义和语境。文
+"""
+tokens=encoding.encode(text)
+# [83, 1609, 5963, 374, 2294, 0]
+print(len(tokens))

requirements.txt

@@ -0,0 +1,5 @@
+tqdm
+tempfile
+openai
+gradio
+openpyxl

uploadData.py

@@ -0,0 +1,72 @@
+import os
+import openai
+
+def upData_OpenAI(dataPath,API_Key):
+    try:
+        dataPath=dataPath.name
+    except:
+        pass
+    openai.api_key = API_Key
+    if isinstance(dataPath,list):
+        jsonl=[]
+        for dataFilePath in dataPath:
+            try:
+                dataFilePath = dataFilePath.name
+            except:
+                pass
+            res=openai.File.create(
+                file=open(dataFilePath, "rb"),
+                purpose='fine-tune'
+            )
+            jsonl.append(res)
+        nesJson={"TotalFileState":jsonl}
+
+        return nesJson
+    else:
+        res = openai.File.create(
+            file=open(dataPath, "rb"),
+            purpose='fine-tune'
+        )
+        return res
+
+def createTask(training_file,apikey):
+    openai.apikey=apikey
+    res=openai.FineTuningJob.create(training_file=training_file, model="gpt-3.5-turbo")
+    print(res)
+    return res
+    #print(openai.FineTuningJob.retrieve(taskId))
+    # List 10 fine-tuning jobs
+    #print(openai.FineTuningJob.list(limit=10))
+def GetFineTuningJobState(apiKey):
+    openai.api_key=apiKey
+    res=openai.FineTuningJob.list(limit=20)
+    return res
+def getStatus(taskId):
+    # Retrieve the state of a fine-tune
+    print(openai.FineTuningJob.retrieve(taskId))
+    # List 10 fine-tuning jobs
+    print(openai.FineTuningJob.list(limit=20))
+
+    # Cancel a job
+    # openai.FineTuningJob.cancel("ft-abc123")
+
+    # List up to 10 events from a fine-tuning job
+    openai.FineTuningJob.list_events(id=taskId, limit=10)
+
+    # Delete a fine-tuned model (must be an owner of the org the model was created in)
+def userFineTuneLLM(systemText,ques,model,openaiapiKey):
+    openai.api_key = openaiapiKey
+    completion = openai.ChatCompletion.create(
+        model=model,
+        messages=[
+            {"role": "system", "content": systemText},
+            {"role": "user", "content": ques}
+        ]
+    )
+    print(completion['choices'][0]['message']['content'])
+    return completion['choices'][0]['message']['content']
+
+if __name__=="__main__":
+    pass
+
+