Thamed-Chowdhury commited on
Commit
100ec2a
·
verified ·
1 Parent(s): b9a63eb

Update LLM_automation_GPT.py

Browse files
Files changed (1) hide show
  1. LLM_automation_GPT.py +126 -125
LLM_automation_GPT.py CHANGED
@@ -1,126 +1,127 @@
1
- def create_data(description):
2
- print("Running THis Script")
3
- print("Length of description is: ", len(description))
4
- from langchain_core.prompts import ChatPromptTemplate ### To create a chatbot, chatprompttemplate used
5
- from langchain_openai import ChatOpenAI ##### For using chat openai features
6
- from langchain_core.output_parsers import StrOutputParser ### Default output parser. Custom parser can also be created
7
- from langchain_community.llms import ollama ### Importing ollama
8
-
9
-
10
- import os
11
- from dotenv import load_dotenv
12
- import pandas as pd
13
-
14
-
15
- load_dotenv()
16
-
17
- ### Set all api keys:
18
- os.environ["OPENAI_API_KEY"]="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"
19
- ### Create Prompt Template:
20
- prompt=ChatPromptTemplate.from_messages(
21
- {
22
- ("system", "You are a helpful assistant, please respond to the queries"), ### We need both system and users in prompt
23
- ("user","question: {question}")
24
- }
25
- )
26
-
27
- #### Create OpenAI llm:
28
- llm=ChatOpenAI(model="gpt-4o")
29
-
30
- ### Create an output parser:
31
- output_parser=StrOutputParser()
32
-
33
- #### Creating chain: The concept is- output of action before | symbol will be passed as input in action after the symbol.
34
- #### Here we have created three actions: The prompt, llm and output parser:
35
- chain=prompt|llm|output_parser
36
-
37
- df = description
38
- df = df.fillna(0)
39
- dj=[]
40
- for i in range(len(df)):
41
- dj.append(chain.invoke({"question" : df['Description'][i]+" Is the news about road accident? If no, then reply 'General'. Else if the news is about road accident then check if the news is referring to a specific accident incident or accident in general? Answer only in a word: Either specific or general."}))
42
-
43
- df2=df.copy()
44
- df2['Report Type']=dj
45
- def drp(p):
46
- df2.drop([p],inplace=True)
47
-
48
- ### Removing the general accident types:
49
- for p in range(len(df)):
50
- if "General" in df2['Report Type'][p]:
51
- drp(p)
52
-
53
- ### Reseting index of df3:
54
- df2.reset_index(drop=True,inplace=True)
55
-
56
- ### Now finding column values using llm:
57
- ### A function to invoke the llm. For some reason phi3 doesn't give accurate result sometimes if used directly in dj.append()
58
- def res(i):
59
- response=chain.invoke({"question" : f"""I will give you two strings. 1st string will contain a publish date of a news and the 2nd string will contain the accident news itself.
60
- If the 2nd string contains more than one accident incidents, only consider the 1st incident. Based on these two strings, you have to answer the following questions. Remember your answer must contain ONLY THE ANSWERS WITHOUT ANY EXTRA WORDS OR SENTENCES:
61
- what is the date (Day-Month-Year numerical format) of accident occurrence? ;
62
- Time of Accident occured; How many people were killed in the accident?;
63
- How many people were injured in the accident?;
64
- Location of the accident;
65
- Type of road where accident occured;
66
- Was there any pedestrian involved?;
67
- Do not include any extra words or sentences except the answers seperated by semicolons only. Your reply cannot contain sentences such as - 'Here are the answers to the questions'
68
- string 1 = {df2['Publish Date'][i]}
69
- string 2 = {df2['Description'][i]}""" })
70
- return response
71
- #### dj2 list contains all column values seperated by comma:
72
- dj2=[]
73
-
74
- for i in range(len(df2)):
75
- dj2.append(res(i))
76
- ### Finding vehicle
77
- def res2(i):
78
- response=chain.invoke({"question" : df2['Description'][i]+" Only name the type of vehicles involved in the accident. If multiple vehicles are involved, seperate them by hyphens(-). Example answers: Bus, Truck-Bus etc. If no vehicles are mentioned, your answer will be: Not Available. Your answer should only contain the vehicle name, do not include any extra sentences"})
79
- return response
80
- #### vehicle list contains all vehicles involved:
81
- vehicles=[]
82
-
83
- for i in range(len(df2)):
84
- vehicles.append(res2(i))
85
-
86
-
87
-
88
-
89
- ### Splitting dj2 string based on comma position:
90
- Date=[]
91
- Time=[]
92
- Killed=[]
93
- Injured=[]
94
- Location=[]
95
- Road_Characteristic=[]
96
- Pedestrian_Involved=[]
97
- #Vehicles_involved=[]
98
-
99
- for i in range(len(dj2)):
100
- words = dj2[i].split(";") # Splitting at the semicolon delimiter
101
- #print(f"Date: {words[0]}")
102
- Date.append(words[0])
103
-
104
- #print(f"Time: {words[1]}")
105
- Time.append(words[1])
106
-
107
- #print(f"Casualities: {words[2]}")
108
- Killed.append(words[2])
109
- Injured.append(words[3])
110
- Location.append(words[4])
111
- Road_Characteristic.append(words[5])
112
- Pedestrian_Involved.append(words[6])
113
- #Vehicles_involved.append(words[7])
114
-
115
- #### Probable type of final dataframe:
116
- df2["Accident Date"]=Date
117
- df2["Time"]=Time
118
- df2["Killed"]=Killed
119
- df2["Injured"]=Injured
120
- df2["Location"]=Location
121
- df2["Road_Characteristic"]=Road_Characteristic
122
- df2["Pedestrian_Involved"]=Pedestrian_Involved
123
- df2["Vehicles Involved"]=vehicles
124
- df3=df2.drop(columns=['Description','Report Type'])
125
- return df3
 
126
 
 
1
+ def create_data(description):
2
+ print("Running THis Script")
3
+ print("Length of description is: ", len(description))
4
+ from langchain_core.prompts import ChatPromptTemplate ### To create a chatbot, chatprompttemplate used
5
+ from langchain_openai import ChatOpenAI ##### For using chat openai features
6
+ from langchain_core.output_parsers import StrOutputParser ### Default output parser. Custom parser can also be created
7
+ from langchain_community.llms import ollama ### Importing ollama
8
+
9
+
10
+ import os
11
+ from dotenv import load_dotenv
12
+ import pandas as pd
13
+
14
+
15
+ load_dotenv()
16
+
17
+ ### Set all api keys:
18
+ api_key = os.getenv("OPENAI_API")
19
+ os.environ["OPENAI_API_KEY"]= api_key
20
+ ### Create Prompt Template:
21
+ prompt=ChatPromptTemplate.from_messages(
22
+ {
23
+ ("system", "You are a helpful assistant, please respond to the queries"), ### We need both system and users in prompt
24
+ ("user","question: {question}")
25
+ }
26
+ )
27
+
28
+ #### Create OpenAI llm:
29
+ llm=ChatOpenAI(model="gpt-4o")
30
+
31
+ ### Create an output parser:
32
+ output_parser=StrOutputParser()
33
+
34
+ #### Creating chain: The concept is- output of action before | symbol will be passed as input in action after the symbol.
35
+ #### Here we have created three actions: The prompt, llm and output parser:
36
+ chain=prompt|llm|output_parser
37
+
38
+ df = description
39
+ df = df.fillna(0)
40
+ dj=[]
41
+ for i in range(len(df)):
42
+ dj.append(chain.invoke({"question" : df['Description'][i]+" Is the news about road accident? If no, then reply 'General'. Else if the news is about road accident then check if the news is referring to a specific accident incident or accident in general? Answer only in a word: Either specific or general."}))
43
+
44
+ df2=df.copy()
45
+ df2['Report Type']=dj
46
+ def drp(p):
47
+ df2.drop([p],inplace=True)
48
+
49
+ ### Removing the general accident types:
50
+ for p in range(len(df)):
51
+ if "General" in df2['Report Type'][p]:
52
+ drp(p)
53
+
54
+ ### Reseting index of df3:
55
+ df2.reset_index(drop=True,inplace=True)
56
+
57
+ ### Now finding column values using llm:
58
+ ### A function to invoke the llm. For some reason phi3 doesn't give accurate result sometimes if used directly in dj.append()
59
+ def res(i):
60
+ response=chain.invoke({"question" : f"""I will give you two strings. 1st string will contain a publish date of a news and the 2nd string will contain the accident news itself.
61
+ If the 2nd string contains more than one accident incidents, only consider the 1st incident. Based on these two strings, you have to answer the following questions. Remember your answer must contain ONLY THE ANSWERS WITHOUT ANY EXTRA WORDS OR SENTENCES:
62
+ what is the date (Day-Month-Year numerical format) of accident occurrence? ;
63
+ Time of Accident occured; How many people were killed in the accident?;
64
+ How many people were injured in the accident?;
65
+ Location of the accident;
66
+ Type of road where accident occured;
67
+ Was there any pedestrian involved?;
68
+ Do not include any extra words or sentences except the answers seperated by semicolons only. Your reply cannot contain sentences such as - 'Here are the answers to the questions'
69
+ string 1 = {df2['Publish Date'][i]}
70
+ string 2 = {df2['Description'][i]}""" })
71
+ return response
72
+ #### dj2 list contains all column values seperated by comma:
73
+ dj2=[]
74
+
75
+ for i in range(len(df2)):
76
+ dj2.append(res(i))
77
+ ### Finding vehicle
78
+ def res2(i):
79
+ response=chain.invoke({"question" : df2['Description'][i]+" Only name the type of vehicles involved in the accident. If multiple vehicles are involved, seperate them by hyphens(-). Example answers: Bus, Truck-Bus etc. If no vehicles are mentioned, your answer will be: Not Available. Your answer should only contain the vehicle name, do not include any extra sentences"})
80
+ return response
81
+ #### vehicle list contains all vehicles involved:
82
+ vehicles=[]
83
+
84
+ for i in range(len(df2)):
85
+ vehicles.append(res2(i))
86
+
87
+
88
+
89
+
90
+ ### Splitting dj2 string based on comma position:
91
+ Date=[]
92
+ Time=[]
93
+ Killed=[]
94
+ Injured=[]
95
+ Location=[]
96
+ Road_Characteristic=[]
97
+ Pedestrian_Involved=[]
98
+ #Vehicles_involved=[]
99
+
100
+ for i in range(len(dj2)):
101
+ words = dj2[i].split(";") # Splitting at the semicolon delimiter
102
+ #print(f"Date: {words[0]}")
103
+ Date.append(words[0])
104
+
105
+ #print(f"Time: {words[1]}")
106
+ Time.append(words[1])
107
+
108
+ #print(f"Casualities: {words[2]}")
109
+ Killed.append(words[2])
110
+ Injured.append(words[3])
111
+ Location.append(words[4])
112
+ Road_Characteristic.append(words[5])
113
+ Pedestrian_Involved.append(words[6])
114
+ #Vehicles_involved.append(words[7])
115
+
116
+ #### Probable type of final dataframe:
117
+ df2["Accident Date"]=Date
118
+ df2["Time"]=Time
119
+ df2["Killed"]=Killed
120
+ df2["Injured"]=Injured
121
+ df2["Location"]=Location
122
+ df2["Road_Characteristic"]=Road_Characteristic
123
+ df2["Pedestrian_Involved"]=Pedestrian_Involved
124
+ df2["Vehicles Involved"]=vehicles
125
+ df3=df2.drop(columns=['Description','Report Type'])
126
+ return df3
127