import PyPDF2 import pandas as pd import os import ast import streamlit as st import pandas as pd import os from google.oauth2.credentials import Credentials from googleapiclient.discovery import build from googleapiclient.http import MediaIoBaseDownload,MediaFileUpload # Load credentials from environment variables credentials_dict = { "token": os.environ.get("token"), "refresh_token": os.environ.get("refresh_token"), "token_uri": os.environ.get("token_uri"), "client_id": os.environ.get("client_id"), "client_secret": os.environ.get("client_secret"), "scopes": os.environ.get("scopes") } MAPPING_FILENAME = "Data Mapping with ItemCode.xlsx" def convert_pdf_to_excel(pdf_file): inputpdf = PyPDF2.PdfReader(pdf_file) pages_no = len(inputpdf.pages) whole_data = [] for i in range(pages_no): inputpdf = PyPDF2.PdfReader(pdf_file) # output = PyPDF2.PdfWriter() # output.add_page(inputpdf.pages[i]) pageObj = inputpdf.pages[i] page_content = pageObj.extract_text() for each_table in [i for i in page_content.split('Delivery Schedule Sheet') if i]: data = each_table.split('\n') each_table_data = [] date_qty = [] row_start_index = 0 row_stop_index = 0 year = "" for index in range(len(data)): if data[index].strip() == 'Part No.': each_table_data.append(data[index+1].replace('Part Color Code',"")) if 'Part Name' not in data[index+2]: each_table_data.append(data[index+2].replace('Part Color Code',"")) else: each_table_data.append("") if data[index].strip()=='MORIROKU TECHNOLOGY': try: year = data[index+1].split(' ')[0].split('/')[1] except Exception as e: print(e) year = "" if 'Part Name' in data[index].strip(): each_table_data.append(data[index+1]) if data[index].strip() == 'ADJ': row_start_index = index + 1 if data[index].strip() == 'Total': row_stop_index = index if row_start_index>0 and row_stop_index>0: for index in range(row_start_index,row_stop_index): if '/' in data[index].strip(): date_qty.append([data[index].strip()[-5:].strip() + "/"+year,data[index+1].strip()]) if not date_qty: date_qty = [["",""]] each_table_data.append(date_qty) whole_data.append(each_table_data) whole_data = pd.DataFrame(whole_data) whole_data.columns = ["Part No.","Part Color Code","Part Name",'Date Qty'] extracted_file = "Data Extracted.xlsx" data_for_mapping = "Data Mapping.xlsx" extracted_data_for_mapping = whole_data.drop('Date Qty',axis=1) extracted_data_for_mapping = extracted_data_for_mapping.drop_duplicates(subset=["Part No.","Part Color Code","Part Name"]) whole_data.to_excel(extracted_file, index=False) extracted_data_for_mapping.to_excel(data_for_mapping, index=False) return extracted_file,data_for_mapping def map_data_to_template(excel_file, mapping_file): # Load Excel file and mapping file extracted_data = pd.read_excel(excel_file) mapping_data = pd.read_excel(mapping_file) mapping_data.to_excel(MAPPING_FILENAME) save_mapping_file_to_drive() mapping_data = mapping_data.rename(columns = {'Customer Part no as per pdf':'Part No.'}) # Perform mapping extracted_data['Date Qty'] = extracted_data['Date Qty'].apply(lambda x: ast.literal_eval(x)) extracted_data = extracted_data.explode('Date Qty') extracted_data[['SchDate','Qty']]= pd.DataFrame(extracted_data['Date Qty'].to_list(), index= extracted_data.index) extracted_data = extracted_data.drop('Date Qty',axis=1) extracted_data = extracted_data[~extracted_data['SchDate'].isna()] mapped_data = extracted_data.merge(mapping_data, on =['Part No.'],how='outer')#[['Item Code','SchDate','Qty']] mapped_data['SOType'] = "R" mapped_data = mapped_data[~mapped_data["SchDate"].isna()] return mapped_data def save_mapping_file_to_drive(): creds = Credentials.from_authorized_user_info(credentials_dict) # Authenticate with Google Drive API service = build('drive', 'v3', credentials=creds) folder_id = "1HBRUZePST0D0buyU9MxeYg2vQyEL4wLF" # List all files in the folder results = service.files().list( q=f"'{folder_id}' in parents and mimeType='application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'", fields="files(id, name)").execute() files = results.get('files', []) files = [i for i in files if i.get('name')=='Data Mapping with ItemCode.xlsx'] if not files: print('No Excel Mapping files found in the folder.') else: for file in files: # Get the ID and name of the first Excel file found in the folder existing_file_id = file['id'] existing_file_name = file['name'] # Delete the existing file service.files().delete(fileId=existing_file_id).execute() file_metadata = {'name': MAPPING_FILENAME, 'parents': [folder_id]} media = MediaFileUpload(MAPPING_FILENAME, mimetype='application/vnd.openxmlformats-officedocument.spreadsheetml.sheet') service.files().create(body=file_metadata, media_body=media, fields='id').execute() def pull_mapping_file_from_drive(): creds = Credentials.from_authorized_user_info(credentials_dict) # Authenticate with Google Drive API service = build('drive', 'v3', credentials=creds) results = service.files().list( q="mimeType='application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'", fields="files(id, name)").execute() files = results.get('files', []) files = [i for i in files if i.get('name')=='Data Mapping with ItemCode.xlsx'] if files: file_id = files[0]['id'] file_name = files[0]['name'] request = service.files().get_media(fileId=file_id) fh = open(file_name, 'wb') downloader = MediaIoBaseDownload(fh, request) # Execute the download done = False while not done: status, done = downloader.next_chunk() fh.close() return 1 return 0 def main(): st.title("PDF to Excel Converter") # File uploader uploaded_file = st.file_uploader("Upload a PDF file", type=["pdf"]) if uploaded_file is not None: st.write("Uploaded PDF file:", uploaded_file.name) # Convert PDF to Excel extracted_file,data_for_mapping = convert_pdf_to_excel(uploaded_file) # Download link for the Excel file # st.markdown(f"Download the extracted data in Excel file [here](/{excel_file})") if os.path.exists(data_for_mapping): with open(data_for_mapping, "rb") as f: excel_bytes = f.read() st.download_button( label="Download Excel file", data=excel_bytes, file_name=data_for_mapping, mime="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" ) else: st.error("Error: Converted Excel file not found") file_present = pull_mapping_file_from_drive() if not os.path.exists("Data Mapping with ItemCode.xlsx"): st.markdown("## Upload the Data Master file with Item Code mapping") mapping_uploaded_file = st.file_uploader("Upload the Data Master file with Item Code mapping", type=["xlsx","ods"]) else: mapping_data = pd.read_excel("Data Mapping with ItemCode.xlsx") mapping_data = mapping_data.rename(columns = {'Customer Part no as per pdf':'Part No.'}) data_for_mapping = "Data Mapping.xlsx" extracted_data_for_mapping = pd.read_excel(data_for_mapping) extracted_data_for_mapping = extracted_data_for_mapping[~extracted_data_for_mapping['Part No.'].isin(mapping_data['Part No.'])] unmapped_part_no = extracted_data_for_mapping['Part No.'].nunique() if unmapped_part_no>0: st.markdown("#### There are {} Part No. with No ItemCode present. Upload a new file after mapping them".format(unmapped_part_no)) mapping_uploaded_file = st.file_uploader("Upload the Data Master file with Item Code mapping", type=["xlsx","ods"]) else: st.markdown("#### Using the Mapping file available in Google Drive") mapping_uploaded_file = "Data Mapping with ItemCode.xlsx" if mapping_uploaded_file is not None: # st.write("Uploaded Mapping Excel file:", mapping_uploaded_file.name) # Perform data mapping mapped_data = map_data_to_template(extracted_file, mapping_uploaded_file) # Provide a link to download the final Excel file after mapping st.markdown("### Final Excel File After Mapping") final_excel_file = 'Final Data.xlsx' mapped_data.to_excel(final_excel_file, index=False) if os.path.exists(final_excel_file): with open(final_excel_file, "rb") as f: excel_bytes = f.read() st.download_button( label="Download Excel file", data=excel_bytes, file_name=final_excel_file, mime="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" ) else: st.error("Error: Converted Excel file not found") if __name__ == "__main__": main()