Update app.py

app.py

@@ -58,7 +58,7 @@ def upload_blob(pdf_name, json_data, pdf_data_jobdescription,pdf_data_cvs, pre_g
 
             cv_nr_for_id = i+1
             cv_session_state_string = "cv-"+str(cv_nr_for_id)
-            session_state_name = st.session_state["final_candidates"][i].metadata["name"]
+            session_state_name = st.session_state["final_candidates"][i][0].metadata["name"]
             names.append(session_state_name)
             cv_id = pdf_name + "-cv-nr-" + str(cv_nr_for_id)+str(calendar.timegm(time.gmtime()))
             upload_db_item(session_state_name, json.loads(json_data), pdf_name, cv_id)
@@ -171,26 +171,68 @@ st.markdown(
 
 with open("sys_prompt_frontend.txt") as f:
     sys_prompt = f.read()
+with open("sys_prompt_job_optimization.txt") as j:
+    sys_prompt_optimization = j.read()
 
 def adjust_numbering(lst):
     return [f"{i + 1}. {item.split('. ', 1)[1]}" for i, item in enumerate(lst)]
 
 def generate_candidate_mail(candidate, chat_link)-> str:
-    prompt = "You are a professional recruiter who has selected a suitable candidate based on a job description. Your task is to write two to three sentences about the candidate and why we think they are suitable for the job. The text will then be used in an email to the candidate, so address the candidate."
-    res = openai.ChatCompletion.create(
-        engine="gpt-4",
-        temperature=0.2,
-        messages=[
-            {
-                "role": "system",
-                "content": prompt,
-            },
-            {"role": "system", "content": "Job description: "+st.session_state["job_string"]+"; Resume: "+candidate.page_content}
-        ],
-    )
-    # print(res.choices[0]["message"]["content"])
-    output_string = f"""{res.choices[0]["message"]["content"]}
+    candidate_first_name = candidate[0].metadata["name"].split(" ")[0]
+    prompt = f"You are a professional recruiter who has selected a suitable candidate on the basis of a job description. Your task is to write two to three sentences about the applicant and explain why we think they are suitable for the job. The text will then be used in an e-mail to the applicant, so please address it to them. Please start the e-mail with 'Dear {candidate_first_name}'. I'll write the end of the mail myself."
+    try:
+        res = openai.ChatCompletion.create(
+            engine="gpt-4",
+            temperature=0.2,
+            messages=[
+                {
+                    "role": "system",
+                    "content": prompt,
+                },
+                {"role": "system", "content": "Job description: "+st.session_state["job_string"]+"; Resume: "+candidate[0].page_content}
+            ],
+        )
+        # print(res.choices[0]["message"]["content"])
+    except Exception as e:
+        # Iterativ die Anfrage wiederholen und 200 Chars von hinten vom Resume weglassen
+        max_retries = 5
+        retries = 0
+        while retries < max_retries:
+            try:
+                # Reduziere die Länge des Resume um 200 Chars von hinten
+                candidate[0].page_content = candidate[0].page_content[:-200]
+
+                # Neue Anfrage senden
+                res = openai.ChatCompletion.create(
+                    engine="gpt-4",
+                    temperature=0.2,
+                    messages=[
+                        {
+                            "role": "system",
+                            "content": prompt,
+                        },
+                        {"role": "system", "content": "Job description: " + st.session_state["job_string"] + "; Resume: " + candidate[0].page_content}
+                    ],
+                )
+                # print(res.choices[0]["message"]["content"])
+
+                # Wenn die Anfrage erfolgreich ist, den Schleifen-Iterator beenden
+                break
 
+            except Exception as e:
+                # Bei erneuter Ausnahme die Schleife fortsetzen
+                retries += 1
+                if retries == max_retries:
+                    # Falls die maximale Anzahl von Wiederholungen erreicht ist, handle die Ausnahme entsprechend
+                    print("Max retries reached. Unable to get a valid response.")
+                    return "The CV was too long to generate a Mail"
+                    # Hier kannst du zusätzlichen Code für den Fall implementieren, dass die maximale Anzahl von Wiederholungen erreicht wurde.
+
+            # Optional: Füge eine Wartezeit zwischen den Anfragen hinzu, um API-Beschränkungen zu respektieren
+            time.sleep(1)
+        
+    output_string = f"""{res.choices[0]["message"]["content"]}
+    
 We have added the job description to the mail attachment. 
 If you are interested in the position, please click on the following link, answer a few questions from our chatbot for about 10-15 minutes and we will get back to you.
 
@@ -204,21 +246,24 @@ WorkGenius
 
 def generate_job_bullets(job)->str:
     prompt = "You are a professional recruiter whose task is to summarize the provided job description in the most important 5 key points. The key points should have a maximum of 8 words. The only thing you should return are the bullet points."
-    res = openai.ChatCompletion.create(
-        engine="gpt-4",
-        temperature=0.2,
-        messages=[
-            {
-                "role": "system",
-                "content": prompt,
-            },
-            {"role": "system", "content": "Job description: "+st.session_state["job_string"]}
-        ],
-    )
-    # print(res.choices[0]["message"]["content"])
-    output_string = f"""{res.choices[0]["message"]["content"]}"""
-    # print(output_string)
-    return output_string
+    try:
+        res = openai.ChatCompletion.create(
+            engine="gpt-4",
+            temperature=0.2,
+            messages=[
+                {
+                    "role": "system",
+                    "content": prompt,
+                },
+                {"role": "system", "content": "Job description: "+job}
+            ],
+        )
+        # print(res.choices[0]["message"]["content"])
+        output_string = f"""{res.choices[0]["message"]["content"]}"""
+        # print(output_string)
+        return output_string
+    except Exception as e:
+        print(f"Fehler beim generieren der Bullets: {str(e)}")
 
 def check_keywords_in_content(database_path, table_name, input_id, keywords):
     # Verbindung zur Datenbank herstellen
@@ -248,6 +293,100 @@ def check_keywords_in_content(database_path, table_name, input_id, keywords):
 
     return contains_keywords
 
+def load_candidates(fillup):
+    with st.spinner("Load the candidates, this may take a moment..."):
+        # print(st.session_state["job_string"])
+        filter_string = ""
+        query_string = "The following keywords must be included: " + text_area_params + " " + st.session_state["job_string"]
+        checked_candidates = []
+        db_path = 'cvdb.db'
+        table_name = 'files'
+        candidates_per_search = 100
+        target_candidates_count = 10
+        current_offset = 0
+
+        if st.session_state["screened"]:
+            filter_string = "amount_screenings gt 0 "
+        if st.session_state["handed"]:
+            if len(filter_string) > 0:
+                filter_string += "and amount_handoffs gt 0 "
+            else:
+                filter_string += "amount_handoffs gt 0 "
+        if st.session_state["placed"]:
+            if len(filter_string) > 0:
+                filter_string += "and amount_placed gt 0"
+            else:
+                filter_string += "amount_placed gt 0"
+        # print(filter_string)
+        if not fillup:
+            while len(checked_candidates) < target_candidates_count:
+                # Führe eine similarity search durch und erhalte 100 Kandidaten
+                if st.session_state["search_type"]:
+                    print("hybrid")
+                    # raw_candidates = st.session_state["db"].hybrid_search(query_string, k=candidates_per_search+current_offset, filters=filter_string)
+                    raw_candidates = st.session_state["db"].hybrid_search_with_score(query_string, k=candidates_per_search+current_offset, filters=filter_string)
+                else:
+                    print("similarity")
+                    # raw_candidates = st.session_state["db"].similarity_search(query_string, k=candidates_per_search+current_offset, filters=filter_string)
+                    raw_candidates = st.session_state["db"].similarity_search_with_relevance_scores(query_string, k=candidates_per_search+current_offset, filters=filter_string)
+
+                for candidate in raw_candidates[current_offset:]:
+                    candidates_id = candidate[0].metadata["source"].split("/")[-1]
+                    keyword_bool = check_keywords_in_content(db_path, table_name, candidates_id, text_area_params.split(','))
+                    
+                    if keyword_bool:
+                        checked_candidates.append(candidate)
+
+                        # Überprüfe, ob die Zielanzahl erreicht wurde und breche die Schleife ab, wenn ja
+                        if len(checked_candidates) >= target_candidates_count:
+                            break
+
+                current_offset += candidates_per_search
+                if current_offset == 600:
+                    break
+            # Setze die Ergebnisse in der Session State Variable
+            st.session_state["docs_res"] = checked_candidates
+            st.session_state["candidate_offset"] = current_offset
+            if len(checked_candidates) == 0:
+                st.error("No candidates can be found with these keywords. Please adjust the keywords and try again.", icon="🚨")
+        else:
+            # Setze die Zielanzahl auf 10
+            target_candidates_count = 10
+
+            current_offset = st.session_state["candidate_offset"]
+            
+            # Solange die Anzahl der überprüften Kandidaten kleiner als die Zielanzahl ist
+            while len(st.session_state["docs_res"]) < target_candidates_count:
+                # Führe eine similarity search durch und erhalte 100 Kandidaten
+                if st.session_state["search_type"]:
+                    print("hybrid")
+                    # raw_candidates = st.session_state["db"].hybrid_search(query_string, k=candidates_per_search+current_offset, filters=filter_string)
+                    raw_candidates = st.session_state["db"].hybrid_search_with_score(query_string, k=candidates_per_search+current_offset, filters=filter_string)
+                else:
+                    print("similarity")
+                    # raw_candidates = st.session_state["db"].similarity_search(query_string, k=candidates_per_search+current_offset, filters=filter_string)
+                    raw_candidates = st.session_state["db"].similarity_search_with_relevance_scores(query_string, k=candidates_per_search+current_offset, filters=filter_string)
+                temp_offset_add = 0 
+                for candidate in raw_candidates[current_offset:]:
+                    candidates_id = candidate[0].metadata["source"].split("/")[-1]
+                    keyword_bool = check_keywords_in_content(db_path, table_name, candidates_id, text_area_params.split(','))
+
+                    if keyword_bool:
+                        st.session_state["docs_res"].append(candidate)
+                        temp_offset_add += 1
+                        # Überprüfe, ob die Zielanzahl erreicht wurde und breche die Schleife ab, wenn ja
+                        if len(st.session_state["docs_res"]) >= target_candidates_count:
+                            st.session_state["candidate_offset"] = current_offset+temp_offset_add
+                            break
+
+                current_offset += candidates_per_search
+                if current_offset == 900:
+                    break
+
+            # Wenn die Liste immer noch leer ist, zeige eine Fehlermeldung an
+            if len(st.session_state["docs_res"]) == 0:
+                st.warning("No more candidates can be found.", icon="🔥")
+
 if "similarity_search_string" not in st.session_state:
     st.session_state["similarity_search_string"] = None
 if "job_string" not in st.session_state:
@@ -260,6 +399,12 @@ if "final_question_string" not in st.session_state:
     st.session_state["final_question_string"] = []
 if "ai_questions" not in st.session_state:
     st.session_state["ai_questions"] = None
+if "raw_job" not in st.session_state:
+    st.session_state["raw_job"] = None
+if "optimized_job" not in st.session_state:
+    st.session_state["optimized_job"] = None
+if "candidate_offset" not in st.session_state:
+    st.session_state["candidate_offset"] = 0
 if "db" not in st.session_state:
     embedder = OpenAIEmbeddings(deployment="text-embedding-ada-002", chunk_size=1)
     embedding_function = embedder.embed_query
@@ -288,6 +433,43 @@ with col_clear:
     if st.button("Clear", use_container_width=True):
         streamlit_js_eval(js_expressions="parent.window.location.reload()")
 
+if st.session_state["job"]:
+    if not st.session_state["job_string"]:
+        if not st.session_state["optimized_job"]:
+            with st.spinner("Optimizing the job description. This may take a moment..."):
+                pdf_data_jobdescription = st.session_state["job"].read()
+                pdf_data_jobdescription_string = ""
+                pdf_reader_job = PdfReader(io.BytesIO(pdf_data_jobdescription))
+                for page_num in range(len(pdf_reader_job.pages)):
+                        page = pdf_reader_job.pages[page_num]
+                        pdf_data_jobdescription_string += page.extract_text()
+                # st.session_state["pdf_data_jobdescription"] = pdf_data_jobdescription activate and add sessio state if data is needed
+                system_prompt_job = sys_prompt_optimization.format(job=pdf_data_jobdescription_string)
+                try:
+                    res = openai.ChatCompletion.create(
+                        engine="gpt-4",
+                        temperature=0.2,
+                        messages=[
+                            {
+                                "role": "system",
+                                "content": system_prompt_job,
+                            },
+                        ],
+                    )
+                    # print(res.choices[0]["message"]["content"])
+                    output_string = f"""{res.choices[0]["message"]["content"]}"""
+                    st.session_state["optimized_job"] = output_string
+                    st.rerun()
+                except Exception as e:
+                    print(f"Fehler beim generieren der optimierten JD: {str(e)}")
+                    st.error("An error has occurred. Please reload the page or contact the admin.", icon="🚨")
+                # st.session_state["job_string"] = pdf_data_jobdescription_string
+                # print(output_string)
+        st.text_area("This is the AI-generated optimized job description. If necessary, change something to your liking:", value=st.session_state["optimized_job"], height=700, key="optimized_job_edited")
+        if st.button("Accept the job description"):
+            st.session_state["job_string"] = st.session_state["optimized_job_edited"]
+            st.rerun()
+
 st.write("Switch from a similarity search (default) to a hybrid search (activated)")
 st.toggle("Switch Search", key="search_type")
 
@@ -304,74 +486,23 @@ text_area_params = st.text_area(label="Add additional search parameters, which a
 
 submit = st.button("Search candidates",disabled= True if st.session_state["final_candidates"] else False)
 
-def load_candidates():
-    with st.spinner("Load the candidates, this may take a moment..."):
-        filter_string = ""
-        query_string = "The following keywords must be included: " + text_area_params + " " + st.session_state["job_string"]
-        checked_candidates = []
-        db_path = 'cvdb.db'
-        table_name = 'files'
-        candidates_per_search = 100
-        target_candidates_count = 10
-        current_offset = 0
-
-        if st.session_state["screened"]:
-            filter_string = "amount_screenings gt 0 "
-        if st.session_state["handed"]:
-            if len(filter_string) > 0:
-                filter_string += "and amount_handoffs gt 0 "
-            else:
-                filter_string += "amount_handoffs gt 0 "
-        if st.session_state["placed"]:
-            if len(filter_string) > 0:
-                filter_string += "and amount_placed gt 0"
-            else:
-                filter_string += "amount_placed gt 0"
-        print(filter_string)
-        while len(checked_candidates) < target_candidates_count:
-            # Führe eine similarity search durch und erhalte 100 Kandidaten
-            if st.session_state["search_type"]:
-                print("hybrid")
-                raw_candidates = st.session_state["db"].hybrid_search(query_string, k=candidates_per_search+current_offset, filters=filter_string)
-            else:
-                print("similarity")
-                raw_candidates = st.session_state["db"].similarity_search(query_string, k=candidates_per_search+current_offset, filters=filter_string)
-
-            for candidate in raw_candidates[current_offset:]:
-                candidates_id = candidate.metadata["source"].split("/")[-1]
-                keyword_bool = check_keywords_in_content(db_path, table_name, candidates_id, text_area_params.split(','))
-                
-                if keyword_bool:
-                    checked_candidates.append(candidate)
-
-                    # Überprüfe, ob die Zielanzahl erreicht wurde und breche die Schleife ab, wenn ja
-                    if len(checked_candidates) >= target_candidates_count:
-                        break
-
-            current_offset += candidates_per_search
-            if current_offset == 600:
-                break
-        # Setze die Ergebnisse in der Session State Variable
-        st.session_state["docs_res"] = checked_candidates
-        if len(checked_candidates) == 0:
-            st.error("No candidates can be found with these keywords. Please adjust the keywords and try again.", icon="🚨")
 
 if not st.session_state["job"] and submit:
     st.error("Please upload a job description to search for candidates")
 if st.session_state["docs_res"] and submit:
-    load_candidates()
-if (st.session_state["job"] and submit) or st.session_state["docs_res"]:
-    if not st.session_state["job_string"]:
-        pdf_data_jobdescription = st.session_state["job"].read()
-        pdf_data_jobdescription_string = ""
-        pdf_reader_job = PdfReader(io.BytesIO(pdf_data_jobdescription))
-        for page_num in range(len(pdf_reader_job.pages)):
-                page = pdf_reader_job.pages[page_num]
-                pdf_data_jobdescription_string += page.extract_text()
-        # st.session_state["pdf_data_jobdescription"] = pdf_data_jobdescription activate and add sessio state if data is needed
-        st.session_state["job_string"] = pdf_data_jobdescription_string
+    load_candidates(False)
+if (st.session_state["job_string"] and submit) or st.session_state["docs_res"]:
+    # if not st.session_state["job_string"]:
+    #     pdf_data_jobdescription = st.session_state["job"].read()
+    #     pdf_data_jobdescription_string = ""
+    #     pdf_reader_job = PdfReader(io.BytesIO(pdf_data_jobdescription))
+    #     for page_num in range(len(pdf_reader_job.pages)):
+    #             page = pdf_reader_job.pages[page_num]
+    #             pdf_data_jobdescription_string += page.extract_text()
+    #     # st.session_state["pdf_data_jobdescription"] = pdf_data_jobdescription activate and add sessio state if data is needed
+    #     st.session_state["job_string"] = pdf_data_jobdescription_string
     if not st.session_state["docs_res"]:
-        load_candidates()
+        load_candidates(False)
     if not st.session_state["final_candidates"]:
         for i,doc in enumerate(st.session_state["docs_res"]):
             # print(doc)
@@ -383,21 +514,34 @@ if (st.session_state["job"] and submit) or st.session_state["docs_res"]:
                     st.rerun()
             with cols_final[0]:
                 # st.subheader(doc.metadata["source"])
-                with st.expander(doc.metadata["name"]):
-                    st.write(doc.page_content)
-        if st.button("Accept candidates", key="accept_candidates_btn"):       
-            print("hello")
-            st.session_state["final_candidates"] = st.session_state["docs_res"].copy()
-            st.rerun()
+                with st.expander(doc[0].metadata["name"]+" with a search score of: "+str(round(doc[1] * 100, 3))+"%"):
+                    st.write(doc[0].page_content)
+        if len(st.session_state["docs_res"])>=10:    
+            if st.button("Accept candidates", key="accept_candidates_btn"):       
+                print("hello")
+                st.session_state["final_candidates"] = st.session_state["docs_res"].copy()
+                st.rerun()
+        else:
+            col_accept, col_empty ,col_load_new = st.columns([2, 3, 2])
+            with col_accept:
+                if st.button("Accept candidates", key="accept_candidates_btn"):       
+                    print("hello")
+                    st.session_state["final_candidates"] = st.session_state["docs_res"].copy()
+                    st.rerun()
+            with col_load_new:
+                if st.button("Load new candidates", key="load_new_candidates"):
+                    print("loading new candidates")
+                    load_candidates(True)
+                    st.rerun()
     else:
         print("Now Questions")
         st.subheader("Your Candidates:")
-        st.write(", ".join(candidate.metadata["name"] for candidate in st.session_state["final_candidates"]))
+        st.write(", ".join(candidate[0].metadata["name"] for candidate in st.session_state["final_candidates"]))
         # for i,candidate in enumerate(st.session_state["final_candidates"]):
         #     st.write(candidate.metadata["source"])
-        cv_strings = "; Next CV: ".join(candidate.page_content for candidate in st.session_state["final_candidates"])
+        cv_strings = "; Next CV: ".join(candidate[0].page_content for candidate in st.session_state["final_candidates"])
         # print(len(cv_strings))
-        system = sys_prompt.format(job=st.session_state["job_string"], resume=st.session_state["final_candidates"][0], n=15)
+        system = sys_prompt.format(job=st.session_state["job_string"], resume=st.session_state["final_candidates"][0][0].page_content, n=15)
         if not st.session_state["ai_questions"]:
             try:
                 # st.write("The questions are generated. This may take a short moment...")
@@ -517,7 +661,7 @@ Your Candidate-Search-Tool
                         for candidate in st.session_state["final_candidates"]:
                             styles = getSampleStyleSheet()
                             pdf = SimpleDocTemplate(temp_pdf_file)
-                            flowables = [Paragraph(candidate.page_content, styles['Normal'])]
+                            flowables = [Paragraph(candidate[0].page_content, styles['Normal'])]
                             pdf.build(flowables)
                             with open(temp_pdf_file, 'rb') as pdf_file:
                                 bytes_data = pdf_file.read()
@@ -527,9 +671,9 @@ Your Candidate-Search-Tool
                         mail_txt_string = ""
                         for i, candidate in enumerate(st.session_state["final_candidates"]):
                             if i > 0:
-                                mail_txt_string += "\n\nMail to the "+str(i+1)+". candidate: "+candidate.metadata["name"]+" "+candidate.metadata["candidateId"]+" \n\n"
+                                mail_txt_string += "\n\nMail to the "+str(i+1)+". candidate: "+candidate[0].metadata["name"]+" "+candidate[0].metadata["candidateId"]+" \n\n"
                             else:
-                                mail_txt_string += "Mail to the "+str(i+1)+". candidate: "+candidate.metadata["name"]+" "+candidate.metadata["candidateId"]+" \n\n"
+                                mail_txt_string += "Mail to the "+str(i+1)+". candidate: "+candidate[0].metadata["name"]+" "+candidate[0].metadata["candidateId"]+" \n\n"
                             mail_txt_string += generate_candidate_mail(candidate,candidate_links[i])
                         # Summary in eine TXT Datei schreiben
                         mail_txt_path = "mailattachment.txt"
@@ -565,6 +709,4 @@ Your Candidate-Search-Tool
                             st.success('The dispatch and the upload of the data was successful')
                         except Exception as e:
                             st.error("Unfortunately the SMS dispatch did not work. Please reload the page and try again or contact the administrator. ", icon="🚨")
-                            print("Fehler beim Senden der SMS:", str(e))
-
-        
+                            print("Fehler beim Senden der SMS:", str(e))