# --- Library Imports --- from collections import defaultdict from datetime import datetime from azure.storage.blob import BlobServiceClient from fastapi import BackgroundTasks, HTTPException, UploadFile, status from fastapi.responses import StreamingResponse from sqlalchemy import text from sqlalchemy.exc import SQLAlchemyError from sqlalchemy.orm import Session # --- # --- User Imports --- from app.config.env import env from app.models.application import Application from app.models.document_type import DocumentSubType, DocumentType from app.models.onboarding_profile import OnboardingProfile from app.models.user_document import UserDocument from app.models.user_profile import UserProfile from app.schema.response_schema import ResponseSchema from app.services.helpers.document_helper import * from ..enums.document_status_enum import DocumentStatusEnum from ..services.background_services.verify_document import verify_documents from ..utils.utility import get_current_datetime, update_table # --- # --- Constants --- CONNECTION_STRING = env.CONNECTION_STRING CONTAINER_NAME = env.CONTAINER_NAME MISCELLANEOUS_DOCUMENT_TYPE_ID = 7 LENDERS_DOCUMENT_TYPE_ID = 9 VERIFY_DOC_QUEUE_NAME = "verify_doc_queue" # --- # Initiate connection to blob storage blob_service_client = BlobServiceClient.from_connection_string(CONNECTION_STRING) container_client = blob_service_client.get_container_client(container=CONTAINER_NAME) class DocumentService: async def upload_file_to_blob_storage(file: UploadFile, user, payload, db, background_tasks: BackgroundTasks): """ Uploads a file to Azure Blob Storage and stores metadata in the database. Parameters ---------- - file: The file to upload. - user: User information. - payload: Additional metadata for the file, including document type and subtype. - db: SQLAlchemy database session. - background_tasks: library to run tasks in background Returns ------- - ResponseSchema: A schema containing the status of the upload operation. """ try: # Write file for processing it later for doc verification file_path = await write_file(file) file_name = file.filename file_size = file.size user_id = user["user_id"] # checking if user Id is present in body or not for loan officer if payload["user_id"] is not None: user_id = payload["user_id"] # Upload file blob_url = await upload_file(user_id, payload, file) # Create a subtype in case of miscellaneous document if payload["document_type_id"] is None or payload["document_subtype_id"] is None: document_title = payload["document_title"] if document_title is None: document_title = "_" subtype_dict = { "documentTypeId": 7, "name": document_title, "code": generate_unique_code(document_title), "applicationId": payload["application_id"], } subtype = DocumentSubType(**subtype_dict) db.add(subtype) db.commit() # Store file metadata in database user_document_data = UserDocument( userId=user_id, documentTypeId=payload["document_type_id"] or MISCELLANEOUS_DOCUMENT_TYPE_ID, documentSubTypeId=payload["document_subtype_id"] or subtype.documentSubTypeId, documentStatusId=3, # Pending Verification date=datetime.now(), URL=blob_url, applicationId=payload["application_id"], userProfileId=payload["user_profile_id"], documentName=file_name, documentSize=file_size, ) db.add(user_document_data) db.commit() # Refresh the data to get the auto-generated userDocumentId db.refresh(user_document_data) user_document_id = user_document_data.userDocumentId document_type_id = user_document_data.documentTypeId document_subtype_id = user_document_data.documentSubTypeId doc_dict = { "file_path": file_path, "user_profile_id": payload["user_profile_id"], "user_document_id": user_document_id, "document_type_id": document_type_id, "document_subtype_id": document_subtype_id, } # Send document to queue for verification background_tasks.add_task(verify_documents, doc_dict) # Return success response return ResponseSchema( status=200, success=True, message="File uploaded succesfully!", ) # Handle database errors except SQLAlchemyError as e: db.rollback() raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=e.args[0]) # Handle other exceptions except Exception as e: raise HTTPException(status_code=500, detail=e.args[0]) async def update_file(file: UploadFile, user, payload, db): """ Updates a file in Azure Blob Storage and updates metadata in the database. Parameters ---------- - file: The file to upload. - user: User information. - payload: Additional metadata for the file, including document type and subtype. - db: SQLAlchemy database session. Returns ------- - ResponseSchema: A schema containing the status of the update operation. """ try: user_id = user["user_id"] # checking if user Id is present in body or not if payload["user_id"] is not None: user_id = payload["user_id"] # Check if document exists existing_document = ( db.query(UserDocument).filter_by(userDocumentId=payload["document_id"], userId=user_id).first() ) if not existing_document: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="Document does not exist!", ) file_name = file.filename file_size = file.size # Upload file blob_url = await upload_file(user_id, payload, file) # Update metadata in DB update_dict = { "URL": blob_url, "date": get_current_datetime(), "documentName": file_name, "documentSize": file_size, } update_table(db, existing_document, update_dict) # Return success response return ResponseSchema( status=200, success=True, message="File updated succesfully!", ) # Handle database errors except SQLAlchemyError as e: db.rollback() raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=e.args[0]) # Handle other exceptions except Exception as e: raise HTTPException(status_code=500, detail=e.args[0]) async def read_file_from_blob_storage(document_id, user, user_id, db): """ Reads a file from Azure Blob Storage and returns it as a streaming response. Parameters ---------- - document_id (str): The ID of the document to be read. - user (dict): The user object containing user information. - db: SQLAlchemy database session. Returns ------- - StreamingResponse: A streaming response containing the file content. """ try: final_user_id = user["user_id"] if user_id is not None: final_user_id = user_id # Fetch document data document_data = db.query(UserDocument).filter_by(userDocumentId=document_id, userId=final_user_id).first() if document_data is None: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="Document does not exist!", ) document_data = document_data.dict() # Specify file path parts = document_data["URL"].split("/") index = parts.index(CONTAINER_NAME) file_path = "/".join(parts[index + 1 :]) file_extension = file_path.split(".")[-1] # Get file stream from blob storage file_stream = get_download_file_stream_from_blob_storage(file_path) # Return a StreamingResponse with the file content return StreamingResponse( iter([file_stream]), media_type="application/octet-stream", headers={"Content-Disposition": f"fileType={file_extension}"}, ) except Exception as e: raise HTTPException(status_code=500, detail=str(e)) async def get_document_types(application_id, user, db: Session): """ Retrieves document types along with their corresponding subtypes. Parameters ---------- - application_id: The Loan Application Id - db: SQLAlchemy database session Returns ------- - ResponseSchema: Response object containing document types and subtypes """ try: # Query to get all details by joining DocumentType and DocumentSubType result = ( db.query(DocumentType, DocumentSubType) .outerjoin( DocumentSubType, DocumentType.documentTypeId == DocumentSubType.documentTypeId, ) .all() ) # Initialize an empty dictionary to store document types and their subtypes result_dicts = {} document_dict_list = [] # Iterate through the result for document_type, sub_type in result: # Convert DocumentType and DocumentSubType to dictionaries document_type_dict = document_type.dict() sub_type_dict = sub_type.dict() if sub_type else {} # Extract document type ID document_type_id = document_type_dict.get("documentTypeId") # Check if document type ID already exists in result_dicts if document_type_id in result_dicts: # Append subtype to the list of subtypes for the existing document type result_dicts[document_type_id]["sub_types"].append(sub_type_dict) else: # Add a new entry for the document type and initialize its sub_types list result_dicts[document_type_id] = document_type_dict result_dicts[document_type_id]["sub_types"] = [sub_type_dict] if sub_type_dict else [] for documents in result_dicts: document_dict_list.append(result_dicts[documents]) # Filter list according to application data application_details = fetch_application_details(db, application_id) # Get loan purpose id and employment id from onboarding data in case they are empty if not len(application_details["employments"]) or application_details.get("loanPurposeId", None) is None: user_id = user if isinstance(user, str) else user["user_id"] onboarding_details = ( db.query( OnboardingProfile.loanPurposeId, OnboardingProfile.employmentTypeId, ) .filter_by(userId=user_id) .first() ) application_details["loanPurposeId"] = ( application_details["loanPurposeId"] if application_details["loanPurposeId"] is not None else onboarding_details[0] ) if not len(application_details["employments"]): application_details["employments"].append( { "employmentTypeId": onboarding_details[1], "otherIncome": 0, "bonusIncome": 0, "OTIncome": 0, } ) # Get filtered documents according to application data filtered_document_dict_list = filter_documents(document_dict_list, application_details) # Fetch all documents associated with the given application_id user_documents = ( db.query( UserDocument.userId, UserDocument.userDocumentId, UserDocument.documentStatusId, UserDocument.documentTypeId, UserDocument.documentSubTypeId, UserDocument.date, UserDocument.remarks, UserDocument.documentName, UserDocument.documentSize, ) .filter_by(applicationId=application_id) .all() ) # Create a lookup dictionary from the queried data # This dictionary will map documentTypeId -> documentSubTypeId -> document object document_lookup = defaultdict(dict) # Initialize an empty dictionary to store the results all_documents_dict = {} for doc in user_documents: document_lookup[doc.documentTypeId][doc.documentSubTypeId] = doc document_sub_type_id = doc.documentSubTypeId # If the user_id is not already in the dictionary, add it with an empty list if document_sub_type_id not in all_documents_dict: all_documents_dict[document_sub_type_id] = [] # Append the document details to the list for the corresponding user_id all_documents_dict[document_sub_type_id].append( { "userDocumentId": doc.userDocumentId, "timestamp": int(datetime.timestamp(doc.date)), "status": doc.documentStatusId, "remarks": doc.remarks, "name": doc.documentName if doc.documentName else "", "size": format_bytes(doc.documentSize) if doc.documentSize else None, } ) # Iterate through each document type in the filtered list for doc_type in filtered_document_dict_list: document_type_id = doc_type["documentTypeId"] # Iterate through each subtype within the current document type for sub_type in doc_type.get("sub_types", []): document_subtype_id = sub_type["documentSubTypeId"] sub_type["applicationId"] = application_id # Retrieve the corresponding document from the lookup dictionary result = document_lookup.get(document_type_id, {}).get(document_subtype_id) # Set the 'uploaded' flag to True if the document exists, otherwise False sub_type["uploaded"] = result is not None # fetching data as per documentSubTypeId sub_type["documents"] = all_documents_dict.get(sub_type["documentSubTypeId"], []) if sub_type["uploaded"]: sub_type["documentStatusId"] = DocumentStatusEnum.UPLOADED.value # Check if 'documents' is not empty if sub_type["documents"]: for document in sub_type["documents"]: if document.get("status") in [ DocumentStatusEnum.PENDING_VERIFICATION.value, DocumentStatusEnum.VERIFICATION_FAILED.value, ]: sub_type["documentStatusId"] = DocumentStatusEnum.PENDING_VERIFICATION.value break # sorted the list in order to be latest first # only for LENDERS_DOCUMENT_TYPE_ID and MISCELLANEOUS_DOCUMENT_TYPE_ID for document_type in filtered_document_dict_list: if ( document_type.get("documentTypeId") == LENDERS_DOCUMENT_TYPE_ID or document_type.get("documentTypeId") == MISCELLANEOUS_DOCUMENT_TYPE_ID ): document_type.get("sub_types").sort(key=lambda x: x.get("documentSubTypeId"), reverse=True) # Return success response return ResponseSchema( status=200, success=True, message="Document types retrieved successfully", data=filtered_document_dict_list, ) except Exception as e: # Raise HTTPException if an error occurs raise HTTPException(status_code=500, detail=e.args[0]) async def delete_document(db, document_id, user, user_id): """ Deletes a user document. Parameters ---------- - db: SQLAlchemy database session - document_id: ID of the document to be deleted Returns ------- - ResponseSchema: Response object indicating success or failure of the deletion operation """ try: final_user_id = user["user_id"] if user_id is not None: final_user_id = user_id # Query to retrieve user documents result = db.query(UserDocument).filter_by(userDocumentId=document_id, userId=final_user_id).first() if not result: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="No documents found", ) document_type_id = result.dict()["documentTypeId"] document_subtype_id = result.dict()["documentSubTypeId"] # Delete the document db.delete(result) db.commit() # Delete the document subtype in case of misc docunment if document_type_id in (MISCELLANEOUS_DOCUMENT_TYPE_ID, LENDERS_DOCUMENT_TYPE_ID): subtype_data = ( db.query(DocumentSubType).filter(DocumentSubType.documentSubTypeId == document_subtype_id).first() ) db.delete(subtype_data) db.commit() # Return success response return ResponseSchema( status=200, success=True, message="User Document deleted successfully", ) except HTTPException as e: # Raise HTTPException if an error occurs raise e except Exception as e: # Raise HTTPException if an error occurs raise HTTPException(status_code=500, detail=str(e)) async def get_user_info(db, user): """ Retrieves user information including application details and user profile. Parameters ---------- - db: SQLAlchemy database session - user: Dictionary containing user information with 'user_id' key Returns ------- - ResponseSchema: Response object containing user information """ try: result = ( db.query( Application, UserProfile, ) .filter_by(userId=user["user_id"]) .outerjoin(UserProfile, Application.applicationId == UserProfile.applicationId) .all() ) if not result: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="Applications not found", ) data = [] for app in result: application = app[0] app = app[1:] user_profile_details = None for index, value in enumerate(app): if value is None: break if index == 0: user_profile_details = value data_dict = { "user_id": application.userId, "application_id": application.applicationId, "user_profile_id": user_profile_details.userProfileId if user_profile_details else None, "first_name": user_profile_details.firstName if user_profile_details else None, "middle_name": user_profile_details.middleName if user_profile_details else None, "last_name": user_profile_details.lastName if user_profile_details else None, "property_address": application.propertyAddress, } data.append(data_dict) # Return success response return ResponseSchema( status=200, success=True, message="User info fetched successfully", data=data, ) except HTTPException as e: # Raise HTTPException if an error occurs raise e except Exception as e: # Raise HTTPException if an error occurs raise HTTPException(status_code=500, detail=str(e)) def approve_document(db, user_document_id, user_id): existing_document = db.query(UserDocument).filter_by(userDocumentId=user_document_id, userId=user_id).first() if not existing_document: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="Document does not exist!", ) existing_document.documentStatusId = DocumentStatusEnum.UPLOADED.value db.commit() return ResponseSchema( status=200, success=True, message="User document approved successfully", ) def reject_document(db, user_document_id, user_id): existing_document = db.query(UserDocument).filter_by(userDocumentId=user_document_id, userId=user_id).first() if not existing_document: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="Document does not exist!", ) existing_document.documentStatusId = DocumentStatusEnum.VERIFICATION_FAILED.value db.commit() return ResponseSchema( status=200, success=True, message="User document rejected successfully", ) def fetch_application_details(db, application_id): try: query = f""" SELECT "Applications"."applicationId" AS "applicationId", "Applications"."loanPurposeId" AS "loanPurposeId", "UserEmploymentAndIncomes"."userEmploymentAndIncomeId" AS "userEmploymentAndIncomeId", "UserEmploymentAndIncomes"."employmentTypeId" AS "employmentTypeId", "UserEmploymentAndIncomes"."OTIncome" AS "OTIncome", "UserEmploymentAndIncomes"."bonusIncome" AS "bonusIncome", "UserEmploymentAndIncomes"."otherIncome" AS "otherIncome", "UserProfiles"."ownershipId" AS "ownershipId", "UserProfiles"."citizenshipStatusId" AS "citizenshipStatusId", "UserAssets"."userAssetId" AS "userAssetId", "UserAssets"."lenderCredit" AS "lenderCredit", "UserAssets"."realtorCredit" AS "realtorCredit", "UserAssets"."otherCredit" AS "otherCredit", "UserLiabilities"."userLiabilityId" AS "userLiabilityId", "UserLiabilities"."isPayoffRequired" AS "isPayoffRequired", "UserLiabilities"."isExclude" AS "isExclude", "UserREOs"."userREOId" AS "userREOId", "UserREOs"."escrowedPayment" AS "escrowedPayment" FROM "Applications" LEFT OUTER JOIN "UserProfiles" ON "UserProfiles"."applicationId" = "Applications"."applicationId" LEFT OUTER JOIN "UserLiabilities" ON "UserLiabilities"."applicationId" = "Applications"."applicationId" LEFT OUTER JOIN "UserAssets" ON "UserAssets"."applicationId" = "Applications"."applicationId" LEFT OUTER JOIN "UserEmploymentAndIncomes" ON "UserEmploymentAndIncomes"."applicationId" = "Applications"."applicationId" LEFT OUTER JOIN "UserREOs" ON "UserREOs"."applicationId" = "Applications"."applicationId" WHERE "Applications"."applicationId" = {application_id}; """ application_result = db.execute(text(query)) column_names = application_result.keys() # Initialize the response object application_data = { "applicationId": None, "loanPurposeId": None, "ownershipId": None, "citizenshipStatusId": None, "employments": [], "liabilities": [], "assets": [], "reos": [], } # Initialize sets to track unique employment, liabilities, assets, and REOs employment_set = set() liability_set = set() asset_set = set() reo_set = set() # Iterate over each row in the result set for row in application_result.fetchall(): application_details = dict(zip(column_names, row)) # Populate application-level data (only once) if application_data["applicationId"] is None: application_data["applicationId"] = application_details["applicationId"] application_data["loanPurposeId"] = application_details["loanPurposeId"] application_data["ownershipId"] = application_details["ownershipId"] application_data["citizenshipStatusId"] = application_details["citizenshipStatusId"] # Append unique employment data employment_id = application_details["userEmploymentAndIncomeId"] if employment_id is not None and employment_id not in employment_set: employment_set.add(employment_id) employment_data = { "employmentTypeId": application_details["employmentTypeId"], "OTIncome": application_details["OTIncome"], "bonusIncome": application_details["bonusIncome"], "otherIncome": application_details["otherIncome"], } application_data["employments"].append(employment_data) # Append unique liability data liability_id = application_details["userLiabilityId"] if liability_id is not None and liability_id not in liability_set: liability_set.add(liability_id) liability_data = { "isPayoffRequired": application_details["isPayoffRequired"], "isExclude": application_details["isExclude"], } application_data["liabilities"].append(liability_data) # Append unique asset data asset_id = application_details["userAssetId"] if asset_id is not None and asset_id not in asset_set: asset_set.add(asset_id) asset_data = { "lenderCredit": application_details["lenderCredit"], "realtorCredit": application_details["realtorCredit"], "otherCredit": application_details["otherCredit"], } application_data["assets"].append(asset_data) # Append unique REO data reo_id = application_details["userREOId"] if reo_id is not None and reo_id not in reo_set: reo_set.add(reo_id) reo_data = {"escrowedPayment": application_details["escrowedPayment"]} application_data["reos"].append(reo_data) # Return application data return application_data except Exception as e: # Raise HTTPException if an error occurs raise HTTPException(status_code=500, detail=str(e))