Spaces:
Running
Running
| import streamlit as st | |
| import pandas as pd | |
| import numpy as np | |
| import fitz # PyMuPDF | |
| import re | |
| import json | |
| def extract_pdf_text(pdf_path): | |
| """Extract text from a PDF file.""" | |
| with fitz.open(pdf_path) as pdf_document: | |
| content_text = "" | |
| for page_num in range(len(pdf_document)): | |
| page = pdf_document.load_page(page_num) | |
| content_text += page.get_text() + "\n" | |
| return content_text | |
| # Streamlit Application | |
| st.title("PDF Data Extractor") | |
| uploaded_file = st.file_uploader("Upload a PDF File", type="pdf") | |
| if uploaded_file is not None: | |
| with open("temp.pdf", "wb") as f: | |
| f.write(uploaded_file.getbuffer()) | |
| pdf_text = extract_pdf_text("temp.pdf") | |
| # Step 2: Extract relevant information from the text using regex | |
| pattern = r"2\s*3\s*4\s*5\s*\n-1,5\s*0([\s\S]*?)\n\nTrainer & Berater-Feedback" | |
| matches = re.findall(pattern, pdf_text) | |
| json_chunks = [] | |
| for match in matches: | |
| match = match.replace(",", ".") | |
| values = [value.strip() for value in match.split("\n") if value.strip()] | |
| if len(values) == 22: | |
| json_chunks.append({"current": values}) | |
| else: | |
| current = values[1::2] | |
| json_chunks.append({"current": current}) | |
| json_output = json.dumps(json_chunks, indent=2) | |
| json_data = json.loads(json_output) | |
| # Define the original data structure | |
| original_data = { | |
| 'Title': [ | |
| "Personal Competence", "Personal Competence", "Personal Competence", "Personal Competence", "Personal Competence", "Personal Competence", | |
| "Personal Competence", "Personal Competence", "Personal Competence", "Personal Competence", "Personal Competence", | |
| "Personal Competence", "Personal Competence", "Business Competence", "Business Competence", "Business Competence", "Business Competence", | |
| "Business Competence", "Management Competence", "Management Competence", "Management Competence", "Management Competence", | |
| ], | |
| 'Code': ["P1", "P2", "P3", "P4", "P5", "P6", "P7", "P8", "P9", "P10", "P11", "P12", | |
| "P13", "B1", "B2", "B3", "B4", "B5", "M1", "M2", "M3", "M4"], | |
| 'Dimensions': [ | |
| "Integrity/ Reliability", "Appearance", "Enthusiasm/Passion", "Learning Motivation/ Self-Development", "Ability to Adapt/Flexibility", | |
| "Communication/Information", "Cooperation/ Team spirit", "Handling of Complex Situations", "Coolness/Handling of Unclear Situations", | |
| "Self-reliance/Initiative", "Conflict Management", "Ability to Assert Oneself/ Negotiation Skills", "Tact and Sensitivity", | |
| "Quality Orientation", "Client Orientation", "Specialized Knowledge", "Methodology/ Didactics/ Language", "Creativity/ Conceptional Skills", | |
| "Project Management", "Result Orientation", "Leadership Skills", "Coach and Advisor" | |
| ] | |
| } | |
| df = pd.DataFrame(original_data) | |
| # Add extracted scores to the DataFrame | |
| score_columns = ['Boss_score', 'Colleague_score', 'Colleague_other_score', 'Report_score', 'Customer_score'] | |
| for idx, col in enumerate(score_columns): | |
| df[col] = json_data[idx]['current'] + [None] * (len(df) - len(json_data[idx]['current'])) | |
| score_pattern = r"\d{1,2},\d{2}" | |
| code_pattern = r"[A-Z]\.[0-9]{1,2}" | |
| all_scores = re.findall(score_pattern, pdf_text) | |
| all_codes = re.findall(code_pattern, pdf_text) | |
| scores = [float(score.replace(",", ".")) for score in all_scores] | |
| codes = [code.strip() for code in all_codes] | |
| if len(codes) >= 44: | |
| codes = codes[22:44] | |
| if len(scores) >= 22: | |
| scores = scores[0:22] | |
| df1 = pd.DataFrame({'Code': [code.replace('.', '') for code in codes], 'All_raters_Score': scores}) | |
| df_combined = pd.merge(df, df1, on="Code", how="inner") | |
| feature_cols = ['Boss_score', 'Colleague_score', 'Report_score', 'Customer_score', 'Colleague_other_score'] | |
| df_combined[feature_cols] = df_combined[feature_cols].astype(float) | |
| def calculate_self_score(row): | |
| valid_features = [val for val in row[feature_cols] if pd.notna(val)] | |
| num_features = len(valid_features) | |
| if num_features > 1: | |
| sum_features = sum(valid_features) - row['All_raters_Score'] | |
| return (row['All_raters_Score'] * num_features) - sum_features | |
| return np.nan | |
| df_combined['Self_score'] = df_combined.apply(calculate_self_score, axis=1) | |
| #Step 7 : Picking strengths and weaknesses | |
| # List of keywords/phrases to capture | |
| keywords = [ | |
| 'Integrity/ Reliability', 'Appearance', 'Enthusiasm/Passion', | |
| 'Learning Motivation/ Self-Development', 'Ability to Adapt/Flexibility', | |
| 'Communication/Information', 'Cooperation/ Team spirit', | |
| 'Handling of Complex Situations', 'Coolness/Handling of Unclear Situations', 'Self-reliance/Initiative', | |
| 'Conflict Management', 'Ability to Assert Oneself/ Negotiation Skills', | |
| 'Tact and Sensitivity', 'Quality Orientation', 'Client Orientation', | |
| 'Specialized Knowledge', 'Methodology/ Didactics/ Language', | |
| 'Creativity/ Conceptional Skills', 'Project Management', | |
| 'Result Orientation', 'Leadership Skills', 'Coach and Advisor' | |
| ] | |
| # Extract phrases between "Topics I would like to discuss... " and "Schedule for the follow-up meeting" | |
| phrases_pattern = r"Please use the form at the end of the section to finalize your development planning\.\s*(.*?)\s*Schedule for the follow-up meeting" | |
| phrases_matches = re.findall(phrases_pattern, pdf_text, re.DOTALL) | |
| # Extract the word after "The biggest strengths and room for improvements perceived by:" | |
| label_pattern = r"The biggest strengths and room for improvements perceived by:\s*(\w+)" | |
| labels = re.findall(label_pattern, pdf_text) | |
| # Process each match and extract only the required keywords | |
| json_output = [] | |
| for i, phrases_text in enumerate(phrases_matches): | |
| extracted_phrases = [ | |
| phrase for phrase in keywords if phrase in phrases_text | |
| ] | |
| if extracted_phrases: | |
| label = labels[i] if i < len(labels) else f"Phrases_{i+1}" | |
| json_output.append({label: extracted_phrases}) | |
| # Convert to JSON format | |
| json_output_str = json.dumps(json_output, indent=2) | |
| # Print the JSON result | |
| #print(json_output_str) | |
| json_data = df.to_json(orient='records') | |
| data = [] | |
| for item in json_output: | |
| for label, phrases in item.items(): | |
| for phrase in phrases: | |
| data.append({'Rater': label, 'Dimensions': phrase}) | |
| df4 = pd.DataFrame(data) | |
| #Step 9: Converting Streangths and Weaknesses with scores into json | |
| # Filter dataframes based on 'Label' value | |
| boss, direct, colleague, other_colleague = [df4[df4['Rater'] == label].copy() for label in ['Boss', 'Direct', 'Colleagues', 'Colleague (o']] | |
| # Create mapping dictionaries from df3 | |
| mappings = { | |
| 'Boss_score': df_combined.set_index('Dimensions')['Boss_score'].to_dict(), | |
| 'Report_score': df_combined.set_index('Dimensions')['Report_score'].to_dict(), | |
| 'Colleague_score': df_combined.set_index('Dimensions')['Colleague_score'].to_dict(), | |
| 'Other_colleague_score': df_combined.set_index('Dimensions')['Colleague_other_score'].to_dict() | |
| } | |
| # Map the values from df3 to the appropriate DataFrames | |
| boss['Boss_score'] = boss['Dimensions'].map(mappings['Boss_score']) | |
| direct['Report_score'] = direct['Dimensions'].map(mappings['Report_score']) | |
| colleague['Colleague_score'] = colleague['Dimensions'].map(mappings['Colleague_score']) | |
| other_colleague['Other_colleague_score'] = other_colleague['Dimensions'].map(mappings['Other_colleague_score']) | |
| boss = boss.sort_values(by = 'Boss_score', ascending = False).reset_index(drop = True) | |
| direct = direct.sort_values(by = 'Report_score', ascending = False).reset_index(drop = True) | |
| colleague = colleague.sort_values(by = 'Colleague_score', ascending = False).reset_index(drop = True) | |
| other_colleague = other_colleague.sort_values(by = 'Other_colleague_score', ascending = False).reset_index(drop = True) | |
| def assign_strength_weakness(df): | |
| df['Strength/Weakness'] = np.nan | |
| df.loc[df.index.isin([0, 1, 2]) & df['Score'].notna(), 'Strength/Weakness'] = 'S' | |
| df.loc[df.index.isin([3, 4, 5]) & df['Score'].notna(), 'Strength/Weakness'] = 'W' | |
| return df | |
| # Apply the function to each DataFrame | |
| boss = assign_strength_weakness(boss) | |
| direct = assign_strength_weakness(direct) | |
| colleague = assign_strength_weakness(colleague) | |
| other_colleague = assign_strength_weakness(other_colleague) | |
| df5 = pd.concat([boss, direct, colleague, other_colleague], axis = 0) | |
| df5 = df5.dropna() | |
| st.write("## Output:") | |
| st.write("### 1. Extracted dataset: Dimensions, Compentency Cluster, Raters and Scores by Raters") | |
| st.dataframe(df_combined) | |
| st.write("### 2. Extracted list of Strengths and Weaknesses rated by each Rater") | |
| st.write(df5) | |