Update with filters and mail dispatch

app.py

@@ -3,17 +3,154 @@ import os
 import openai
 import re
 import sqlite3
+import base64
+import calendar
+import json
+import time
+import uuid
+from reportlab.platypus import SimpleDocTemplate, Paragraph
+from reportlab.lib.styles import getSampleStyleSheet
 import streamlit as st
 from streamlit_js_eval import streamlit_js_eval
 from langchain.embeddings.openai import OpenAIEmbeddings
 from langchain.vectorstores.azuresearch import AzureSearch
+from azure.storage.blob import BlobServiceClient
+from azure.cosmos import CosmosClient, exceptions
 from PyPDF2 import PdfReader
+import openai
+import sendgrid
+from sendgrid.helpers.mail import Mail, Attachment, FileContent, FileName, FileType, Disposition
+import ssl
+ssl._create_default_https_context = ssl._create_unverified_context
 
 openai.api_key = os.getenv("OPENAI_API_KEY")
 openai.api_base = "https://tensora-oai.openai.azure.com/"
 openai.api_type = "azure"
 openai.api_version = "2023-05-15"
 
+connection_string = os.getenv("CONNECTION")
+blob_service_client = BlobServiceClient.from_connection_string(connection_string)
+
+def upload_blob(pdf_name, json_data, pdf_data_jobdescription,pdf_data_cvs, pre_generated_bool, custom_questions):
+    try:
+        container_name = "jobdescriptions"
+        # json_blob_name = f"{pdf_name}_jsondata.json"
+        pdf_blob_name_jobdescription = f"{pdf_name}.pdf"
+
+        container_client = blob_service_client.get_container_client(container_name)
+
+        # json_blob_client = container_client.get_blob_client(json_blob_name)
+        # json_blob_client.upload_blob(json_data.encode('utf-8'), overwrite=True)
+
+        pdf_blob_client = container_client.get_blob_client(pdf_blob_name_jobdescription)
+        pdf_blob_client.upload_blob(pdf_data_jobdescription, overwrite=True)
+
+        upload_job_db_item(pdf_name,len(pdf_data_cvs),json.loads(json_data),pre_generated_bool, custom_questions)
+        if pre_generated_bool:
+            for i,question in enumerate(custom_questions):
+                question_nr_for_id = i+1
+                question_id = pdf_name + "-question-nr-" + str(question_nr_for_id)+str(calendar.timegm(time.gmtime()))
+                upload_question_db_item(question_id, pdf_name, question,st.session_state["job_string"])
+        links = []
+        names = []
+        for i,cv in enumerate(pdf_data_cvs):
+
+            cv_nr_for_id = i+1
+            cv_session_state_string = "cv-"+str(cv_nr_for_id)
+            session_state_name = st.session_state["final_candidates"][i].metadata["name"]
+            names.append(session_state_name)
+            cv_id = pdf_name + "-cv-nr-" + str(cv_nr_for_id)+str(calendar.timegm(time.gmtime()))
+            upload_db_item(session_state_name, json.loads(json_data), pdf_name, cv_id)
+            pdf_blob_name_cv = f"{cv_id}.pdf"
+            pdf_blob_client = container_client.get_blob_client(pdf_blob_name_cv)
+            pdf_blob_client.upload_blob(pdf_data_cvs[i], overwrite=True)
+            links.append("https://tensora.ai/workgenius/cv-evaluation2/?job="+cv_id)
+
+        return links
+    except Exception as e:
+        print(f"Fehler beim Hochladen der Daten: {str(e)}")
+        return []
+
+def upload_job_db_item(id, number_of_applicants, data, pre_generated_bool, custom_questions):
+    endpoint = "https://wg-candidate-data.documents.azure.com:443/"
+    key = os.getenv("CONNECTION_DB")
+    client = CosmosClient(endpoint, key)
+    database = client.get_database_client("ToDoList")
+    container = database.get_container_client("JobData")
+    job_item = {
+        "id": id,
+        'partitionKey' : 'wg-job-data-v1',
+        "title": data["title"],
+        "number_of_applicants": number_of_applicants,
+        "every_interview_conducted": False,
+        "evaluation_email": data["email"],
+        "question_one": data["question_one"],
+        "question_two": data["question_two"],
+        "question_three": data["question_three"],
+        "pre_generated": pre_generated_bool,
+        "custom_questions": custom_questions
+    }
+    try:
+        # Fügen Sie das Element in den Container ein
+        container.create_item(body=job_item)
+        print("Eintrag erfolgreich in die Cosmos DB eingefügt. Container: Job Data")
+    except exceptions.CosmosHttpResponseError as e:
+        print(f"Fehler beim Schreiben in die Cosmos DB: {str(e)}")
+    except Exception as e:
+        print(f"Allgemeiner Fehler: {str(e)}")
+
+def upload_db_item(name, data, job_description_id, cv_id):
+
+    endpoint = "https://wg-candidate-data.documents.azure.com:443/"
+    key = os.getenv("CONNECTION_DB")
+    client = CosmosClient(endpoint, key)
+    database = client.get_database_client("ToDoList")
+    container = database.get_container_client("Items")
+    candidate_item = {
+        "id": cv_id,
+        'partitionKey' : 'wg-candidate-data-v1',
+        "name": name,
+        "title": data["title"],
+        "interview_conducted": False,
+        "ai_summary": "",
+        "evaluation_email": data["email"],
+        "question_one": data["question_one"],
+        "question_two": data["question_two"],
+        "question_three": data["question_three"],
+        "job_description_id": job_description_id,
+    }
+
+    try:
+        # Fügen Sie das Element in den Container ein
+        container.create_item(body=candidate_item)
+        print("Eintrag erfolgreich in die Cosmos DB eingefügt. Container: Items(candidate Data)")
+    except exceptions.CosmosHttpResponseError as e:
+        print(f"Fehler beim Schreiben in die Cosmos DB: {str(e)}")
+    except Exception as e:
+        print(f"Allgemeiner Fehler: {str(e)}")
+
+def upload_question_db_item(id, job_id, question, job_content):
+    endpoint = "https://wg-candidate-data.documents.azure.com:443/"
+    key = os.getenv("CONNECTION_DB")
+    client = CosmosClient(endpoint, key)
+    database = client.get_database_client("ToDoList")
+    container = database.get_container_client("Questions")
+    question_item = {
+        "id": id,
+        "partitionKey" : "wg-question-data-v1",
+        "job_id": job_id,
+        "question_content": question,
+        "job_description": job_content,
+    }
+    try:
+        # Fügen Sie das Element in den Container ein
+        container.create_item(body=question_item)
+        print("Eintrag erfolgreich in die Cosmos DB eingefügt. Container: Questions(Question Data)")
+    except exceptions.CosmosHttpResponseError as e:
+        print(f"Fehler beim Schreiben in die Cosmos DB: {str(e)}")
+    except Exception as e:
+        print(f"Allgemeiner Fehler: {str(e)}")
+
 st.markdown(
 """
 <style>
@@ -37,6 +174,33 @@ with open("sys_prompt_frontend.txt") as f:
 def adjust_numbering(lst):
     return [f"{i + 1}. {item.split('. ', 1)[1]}" for i, item in enumerate(lst)]
 
+def generate_candidate_mail(candidate, chat_link)-> str:
+    prompt = "You are a professional recruiter who has selected a suitable candidate based on a job description. Your task is to write two to three sentences about the candidate and why we think they are suitable for the job. The text will then be used in an email to the candidate, so address the candidate."
+    res = openai.ChatCompletion.create(
+        engine="gpt-4",
+        temperature=0.2,
+        messages=[
+            {
+                "role": "system",
+                "content": prompt,
+            },
+            {"role": "system", "content": "Job description: "+st.session_state["job_string"]+"; Resume: "+candidate.page_content}
+        ],
+    )
+    # print(res.choices[0]["message"]["content"])
+    output_string = f"""{res.choices[0]["message"]["content"]}
+
+We have added the job description to the mail attachment. 
+If you are interested in the position, please click on the following link, answer a few questions from our chatbot for about 10-15 minutes and we will get back to you.
+
+Link to the interview chatbot: {chat_link}
+
+Sincerely,
+WorkGenius
+"""
+    print("Mail generated")
+    return output_string
+
 def check_keywords_in_content(database_path, table_name, input_id, keywords):
     # Verbindung zur Datenbank herstellen
     conn = sqlite3.connect(database_path)
@@ -81,12 +245,12 @@ if "db" not in st.session_state:
     embedder = OpenAIEmbeddings(deployment="text-embedding-ada-002", chunk_size=1)
     embedding_function = embedder.embed_query
 
-
     db = AzureSearch(
-        index_name="wg-cvs",
+        index_name="wg-cvs-data",
         azure_search_endpoint=os.environ.get("AZURE_SEARCH_ENDPOINT"),
         azure_search_key=os.environ.get("AZURE_SEARCH_KEY"),
         embedding_function=embedding_function,
+        # fields=fields
     )
     st.session_state["db"] = db
 
@@ -105,13 +269,25 @@ with col_clear:
     if st.button("Clear", use_container_width=True):
         streamlit_js_eval(js_expressions="parent.window.location.reload()")
 
+st.write("Switch from a similarity search (default) to a hybrid search (activated)")
+st.toggle("Switch Search", key="search_type")
+
+st.write("Activate the following toggles to filter according to the respective properties:")
+col_screening, col_handoff, col_placed = st.columns([1,1,1])
+with col_screening:
+    st.toggle("Screened", key="screened")
+with col_handoff:
+    st.toggle("Handed over", key="handed")
+with col_placed:
+    st.toggle("Placed", key="placed")
+
 text_area_params = st.text_area(label="Add additional search parameters, which are separated by commas (e.g. master, phd, web developer, spanish)")
 
 submit = st.button("Search candidates",disabled= True if st.session_state["final_candidates"] else False)
-if not st.session_state["job"] and submit:
-    st.error("Please upload a job description to search for candidates")
-if st.session_state["docs_res"] and submit:
+
+def load_candidates():
     with st.spinner("Load the candidates, this may take a moment..."):
+        filter_string = ""
         query_string = "The following keywords must be included: " + text_area_params + " " + st.session_state["job_string"]
         checked_candidates = []
         db_path = 'cvdb.db'
@@ -120,9 +296,27 @@ if st.session_state["docs_res"] and submit:
         target_candidates_count = 10
         current_offset = 0
 
+        if st.session_state["screened"]:
+            filter_string = "amount_screenings gt 0 "
+        if st.session_state["handed"]:
+            if len(filter_string) > 0:
+                filter_string += "and amount_handoffs gt 0 "
+            else:
+                filter_string += "amount_handoffs gt 0 "
+        if st.session_state["placed"]:
+            if len(filter_string) > 0:
+                filter_string += "and amount_placed gt 0"
+            else:
+                filter_string += "amount_placed gt 0"
+        print(filter_string)
         while len(checked_candidates) < target_candidates_count:
             # Führe eine similarity search durch und erhalte 100 Kandidaten
-            raw_candidates = st.session_state["db"].similarity_search(query_string, k=candidates_per_search+current_offset)
+            if st.session_state["search_type"]:
+                print("hybrid")
+                raw_candidates = st.session_state["db"].hybrid_search(query_string, k=candidates_per_search+current_offset, filters=filter_string)
+            else:
+                print("similarity")
+                raw_candidates = st.session_state["db"].similarity_search(query_string, k=candidates_per_search+current_offset, filters=filter_string)
 
             for candidate in raw_candidates[current_offset:]:
                 candidates_id = candidate.metadata["source"].split("/")[-1]
@@ -138,11 +332,15 @@ if st.session_state["docs_res"] and submit:
             current_offset += candidates_per_search
             if current_offset == 600:
                 break
-
         # Setze die Ergebnisse in der Session State Variable
         st.session_state["docs_res"] = checked_candidates
         if len(checked_candidates) == 0:
             st.error("No candidates can be found with these keywords. Please adjust the keywords and try again.", icon="🚨")
+
+if not st.session_state["job"] and submit:
+    st.error("Please upload a job description to search for candidates")
+if st.session_state["docs_res"] and submit:
+    load_candidates()
 if (st.session_state["job"] and submit) or st.session_state["docs_res"]:
     if not st.session_state["job_string"]:
         pdf_data_jobdescription = st.session_state["job"].read()
@@ -154,104 +352,7 @@ if (st.session_state["job"] and submit) or st.session_state["docs_res"]:
         # st.session_state["pdf_data_jobdescription"] = pdf_data_jobdescription activate and add sessio state if data is needed
         st.session_state["job_string"] = pdf_data_jobdescription_string
     if not st.session_state["docs_res"]:
-        # print("ich bin im spinner")
-        # print(st.session_state["job_string"]+" "+text_area_params)
-        with st.spinner("Load the candidates, this may take a moment..."):
-            #Use this line if you just want to perform one similarity search
-            # st.session_state["docs_res"] = st.session_state["db"].similarity_search(text_area_params+" "+st.session_state["job_string"], k=100)
-            
-            query_string = "The following keywords must be included: " + text_area_params + " " + st.session_state["job_string"]
-            checked_candidates = []
-            db_path = 'cvdb.db'
-            table_name = 'files'
-            candidates_per_search = 100
-            target_candidates_count = 10
-            current_offset = 0
-
-            while len(checked_candidates) < target_candidates_count:
-                # Führe eine similarity search durch und erhalte 100 Kandidaten
-                raw_candidates = st.session_state["db"].similarity_search(query_string, k=candidates_per_search+current_offset)
-
-                for candidate in raw_candidates[current_offset:]:
-                    candidates_id = candidate.metadata["source"].split("/")[-1]
-                    keyword_bool = check_keywords_in_content(db_path, table_name, candidates_id, text_area_params.split(','))
-                    
-                    if keyword_bool:
-                        checked_candidates.append(candidate)
-
-                        # Überprüfe, ob die Zielanzahl erreicht wurde und breche die Schleife ab, wenn ja
-                        if len(checked_candidates) >= target_candidates_count:
-                            break
-
-                current_offset += candidates_per_search
-                if current_offset == 600:
-                    break
-
-            # Setze die Ergebnisse in der Session State Variable
-            st.session_state["docs_res"] = checked_candidates
-            if len(checked_candidates) == 0:
-                st.error("No candidates can be found with these keywords. Please adjust the keywords and try again.", icon="🚨")
-            # query_string = "The following keywords must be included: "+text_area_params+" "+st.session_state["job_string"]
-            # raw_candidates = st.session_state["db"].similarity_search(query_string, k=100)
-            # checked_candidates = []
-            # db_path = 'cvdb.db'
-            # table_name = 'files'
-            # for candidate in raw_candidates:
-            #     candidates_id = candidate.metadata["source"].split("/")[-1]
-            #     keyword_bool = check_keywords_in_content(db_path,table_name,candidates_id,text_area_params.split(','))
-            #     print(keyword_bool)
-            #     if check_keywords_in_content(db_path,table_name,candidates_id,text_area_params.split(',')):
-            #         if len(checked_candidates)<15:
-            #             checked_candidates.append(candidate)
-            #         else:
-            #             break
-            
-            # st.session_state["docs_res"] = checked_candidates
-            #This Code is creating a new Index based on the raw candidates
-            # raw_candidates = st.session_state["db"].similarity_search(text_area_params+" "+st.session_state["job_string"], k=50)
-            # raw_candidates_embeddings = []
-            # for candidate in raw_candidates:
-            #     raw_candidates_embeddings.append(embedding_function(candidate.page_content))
-            
-            # st.session_state["docs_res"] = st.session_state["db"].similarity_search_by_vector(embedding=raw_candidates_embeddings,k=10,query="Every candidate needs to be proficient in spanish")
-            # db_temp = AzureSearch.from_documents(
-            #     raw_candidates,
-            #     embedding=embedder,
-            #     index_name="wg-cvs-temp",
-            #     azure_search_endpoint=os.environ.get("AZURE_SEARCH_ENDPOINT"),
-            #     azure_search_key=os.environ.get("AZURE_SEARCH_KEY"),
-            # )
-
-            # st.session_state["docs_res"] = db_temp.similarity_search(query="Every candidate needs to be proficient in spanish", k=10)
-            
-            #Use this code to check candidates with gpt-4 
-            # raw_candidates = st.session_state["db"].similarity_search(text_area_params+" "+st.session_state["job_string"], k=15)
-            # temp_candidates = []
-            # for candidate in raw_candidates:
-            #     res_approve = openai.ChatCompletion.create(
-            #             engine="gpt-4",
-            #             temperature=0.1,
-            #             messages=[
-            #                 {
-            #                     "role": "system",
-            #                     "content": "You are a professional recruiter who receives a resume and a set of requirements. The only thing you have to do is to say whether the requirements are fulfilled or not. you should not explain yourself and simply answer '1' if the requirements are fulfilled and '0' if not.",
-            #                 },
-            #                 {
-            #                     "role": "system",
-            #                     "content": "The candidate needs to be located in New York"
-            #                 },
-            #                 {
-            #                     "role": "system",
-            #                     "content": candidate.page_content
-            #                 }
-            #             ],
-            #             )
-            #     print(res_approve.choices[0]["message"]["content"])
-            #     if res_approve.choices[0]["message"]["content"] == "1":
-            #         temp_candidates.append(candidate)
-            # st.session_state["docs_res"]  = temp_candidates
-
-
+        load_candidates()
     if not st.session_state["final_candidates"]:
         for i,doc in enumerate(st.session_state["docs_res"]):
             # print(doc)
@@ -263,7 +364,7 @@ if (st.session_state["job"] and submit) or st.session_state["docs_res"]:
                     st.rerun()
             with cols_final[0]:
                 # st.subheader(doc.metadata["source"])
-                with st.expander(doc.metadata["source"]):
+                with st.expander(doc.metadata["name"]):
                     st.write(doc.page_content)
         if st.button("Accept candidates", key="accept_candidates_btn"):       
             print("hello")
@@ -272,7 +373,7 @@ if (st.session_state["job"] and submit) or st.session_state["docs_res"]:
     else:
         print("Now Questions")
         st.subheader("Your Candidates:")
-        st.write(", ".join(candidate.metadata["source"] for candidate in st.session_state["final_candidates"]))
+        st.write(", ".join(candidate.metadata["name"] for candidate in st.session_state["final_candidates"]))
         # for i,candidate in enumerate(st.session_state["final_candidates"]):
         #     st.write(candidate.metadata["source"])
         cv_strings = "; Next CV: ".join(candidate.page_content for candidate in st.session_state["final_candidates"])
@@ -350,7 +451,86 @@ if (st.session_state["job"] and submit) or st.session_state["docs_res"]:
                 with cols_final[0]:
                     st.write(final_q)
             if len(st.session_state["final_question_string"])>0:
+                st.text_input("Enter the email address to which the test emails should be sent:",key="recruiter_mail")
+                st.text_input("Enter the job title:", key="job_title")
                 if st.button("Submit", use_container_width=True):
-                    st.success('Successful search for candidates and generation of questions')
+                    with st.spinner("Generation and dispatch of mails. This process may take a few minutes..."):
+                        sg = sendgrid.SendGridAPIClient(api_key=os.environ.get('SENDGRID_API'))
+                        # Sender- und Empfänger-E-Mail-Adressen
+                        sender_email = "[email protected]"
+                        receiver_email = st.session_state["recruiter_mail"]
+                        print(receiver_email)
+                        subject = "Mails for potential candidates for the following position: "+st.session_state["job_title"]
+                        message = f"""Dear Recruiter,
+                        
+enclosed in the text file you will find the e-mails that are sent to the potential candidates. 
+
+The subject of the mail would be the following: Are you interested in a new position as a {st.session_state["job_title"]}? 
+
+Sincerely,
+Your Candidate-Search-Tool
+"""
+                        # SendGrid-E-Mail erstellen
+                        message = Mail(
+                            from_email=sender_email,
+                            to_emails=receiver_email,
+                            subject=subject,
+                            plain_text_content=message,
+                        )
+                        data = {
+                            "title": st.session_state["job_title"],
+                            "email": st.session_state["recruiter_mail"],
+                            "question_one": "",
+                            "question_two": "",
+                            "question_three": "",
+                        }
+                        json_data = json.dumps(data, ensure_ascii=False)
+                        # Eine zufällige UUID generieren
+                        random_uuid = uuid.uuid4()
+
+                        # Die UUID als String darstellen
+                        uuid_string = str(random_uuid)
+
+                        pdf_name = uuid_string
+                        cvs_data = []
+                        temp_pdf_file = "candidate_pdf.pdf"
+                        for candidate in st.session_state["final_candidates"]:
+                            styles = getSampleStyleSheet()
+                            pdf = SimpleDocTemplate(temp_pdf_file)
+                            flowables = [Paragraph(candidate.page_content, styles['Normal'])]
+                            pdf.build(flowables)
+                            with open(temp_pdf_file, 'rb') as pdf_file:
+                                bytes_data = pdf_file.read()
+                            cvs_data.append(bytes_data)
+                        os.remove(temp_pdf_file)
+                        candidate_links = upload_blob(pdf_name, json_data, st.session_state["job"].read(),cvs_data,True,st.session_state["final_question_string"])
+                        mail_txt_string = ""
+                        for i, candidate in enumerate(st.session_state["final_candidates"]):
+                            if i > 0:
+                                mail_txt_string += "\n\nMail to the "+str(i+1)+". candidate: "+candidate.metadata["name"]+" "+candidate.metadata["candidateId"]+" \n\n"
+                            else:
+                                mail_txt_string += "Mail to the "+str(i+1)+". candidate: "+candidate.metadata["name"]+" "+candidate.metadata["candidateId"]+" \n\n"
+                            mail_txt_string += generate_candidate_mail(candidate,candidate_links[i])
+                        # Summary in eine TXT Datei schreiben
+                        mail_txt_path = "mailattachment.txt"
+                        with open(mail_txt_path, 'wb') as summary_file:
+                            summary_file.write(mail_txt_string.encode('utf-8'))
+                        # Resume als Anhang hinzufügen
+                        with open(mail_txt_path, 'rb') as summary_file:
+                            encode_file_summary = base64.b64encode(summary_file.read()).decode()
+                            summary_attachment = Attachment()
+                            summary_attachment.file_content = FileContent(encode_file_summary)
+                            summary_attachment.file_name = FileName('candidate_mails.txt')
+                            summary_attachment.file_type = FileType('text/plain')
+                            summary_attachment.disposition = Disposition('attachment')
+                            message.attachment = summary_attachment
+                        try:
+                            response = sg.send(message)
+                            print("E-Mail wurde erfolgreich gesendet. Statuscode:", response.status_code)
+                            st.success('The mail dispatch and the upload of the data was successful')
+                            os.remove("mailattachment.txt")
+                        except Exception as e:
+                            print("Fehler beim Senden der E-Mail:", str(e))
+                            st.error("Unfortunately the mail dispatch did not work. Please reload the page and try again or contact the administrator. ", icon="🚨")