app.py

from flask import Flask, render_template, request, redirect, url_for, send_file
import os
import shutil
import pandas as pd
from werkzeug.utils import secure_filename
from joblib import load, dump
import numpy as np
from sklearn.preprocessing import LabelEncoder
from time import time
from huggingface_hub import hf_hub_download
import pickle
import uuid
from pathlib import Path
import numpy as np 
import pandas as pd 
import seaborn as sns
import matplotlib as mpl
import matplotlib.pyplot as plt
import matplotlib.pylab as pylab
from sklearn.preprocessing import OneHotEncoder, LabelEncoder
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
from sklearn.decomposition import PCA
from sklearn.pipeline import Pipeline
from sklearn.tree import DecisionTreeRegressor
from sklearn.ensemble import RandomForestRegressor
from sklearn.linear_model import LinearRegression
from xgboost import XGBRegressor
from sklearn.neighbors import KNeighborsRegressor
from sklearn.model_selection import cross_val_score
from sklearn.metrics import mean_squared_error
from sklearn import metrics
from sklearn.model_selection import train_test_split
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import PowerTransformer, StandardScaler
from sklearn.ensemble import RandomForestRegressor
from sklearn.model_selection import train_test_split, cross_val_score, RandomizedSearchCV
import lightgbm as lgb
from catboost import CatBoostRegressor
from sklearn.ensemble import StackingRegressor

app = Flask(__name__)

# Set the secret key for session management
app.secret_key = os.urandom(24)

# Configurations
UPLOAD_FOLDER = "uploads/"
DATA_FOLDER = "data/"
MODEL_FOLDER = "models/"

os.makedirs(MODEL_FOLDER, exist_ok=True)

# Define the model directory and label encoder directory
MODEL_DIR = r'./Model'
LABEL_ENCODER_DIR = r'./Label_encoders'  # Renamed for clarity

# Global file names for outputs; these will be updated per prediction.
# Note: we now include a unique id to avoid overwriting.
PRED_OUTPUT_FILE = None
CLASS_OUTPUT_FILE = None

ALLOWED_EXTENSIONS = {'csv', 'xlsx'}

# Create directories if they do not exist.
app.config['UPLOAD_FOLDER'] = UPLOAD_FOLDER
os.makedirs(app.config['UPLOAD_FOLDER'], exist_ok=True)

app.config['DATA_FOLDER'] = DATA_FOLDER
os.makedirs(app.config['DATA_FOLDER'], exist_ok=True)

os.makedirs("data", exist_ok=True)

app.config['MODEL_FOLDER'] = MODEL_FOLDER
os.makedirs(app.config['MODEL_FOLDER'], exist_ok=True)


# ------------------------------
# Load Models and Label Encoders
# ------------------------------

# Prediction analysis models loaded from Hugging Face.
src_path = hf_hub_download(
    repo_id="WebashalarForML/Diamond_model_",
    filename="models_list/mkble/StackingRegressor_best_pipeline_mkble_0_to_1.01.pkl",
    cache_dir=MODEL_FOLDER
)
dst_path = os.path.join(MODEL_FOLDER, "StackingRegressor_best_pipeline_mkble_0_to_1.01.pkl")
shutil.copy(src_path, dst_path)
makable_model = load(dst_path)

src_path = hf_hub_download(
    repo_id="WebashalarForML/Diamond_model_",
    filename="models_list/grd/StackingRegressor_best_pipeline_grd_0_to_1.01.pkl",
    cache_dir=MODEL_FOLDER
)
dst_path = os.path.join(MODEL_FOLDER, "StackingRegressor_best_pipeline_grd_0_to_1.01.pkl")
shutil.copy(src_path, dst_path)
grade_model = load(dst_path)
 

src_path = hf_hub_download(
    repo_id="WebashalarForML/Diamond_model_",
    filename="models_list/bygrad/StackingRegressor_best_pipeline_bygrad_0_to_1.01.pkl",
    cache_dir=MODEL_FOLDER
)
dst_path = os.path.join(MODEL_FOLDER, "StackingRegressor_best_pipeline_bygrad_0_to_1.01.pkl")
shutil.copy(src_path, dst_path)
bygrade_model = load(dst_path)


src_path = hf_hub_download(
    repo_id="WebashalarForML/Diamond_model_",
    filename="models_list/gia/StackingRegressor_best_pipeline_gia_0_to_1.01.pkl",
    cache_dir=MODEL_FOLDER
)
dst_path = os.path.join(MODEL_FOLDER, "StackingRegressor_best_pipeline_gia_0_to_1.01.pkl")
shutil.copy(src_path, dst_path)
gia_model = load(dst_path)



print("makable_model type:", type(makable_model))
print("grade_model type:", type(grade_model))
print("bygrade_model type:", type(bygrade_model))
print("gia_model type:", type(gia_model))

#gia_model = load("models/StackingRegressor_best_pipeline_mkble_0_to_1.01.pkl")
#grade_model = load("models/StackingRegressor_best_pipeline_grd_0_to_1.01.pkl")
#bygrade_model = load("models/StackingRegressor_best_pipeline_bygrad_0_to_1.01.pkl")
#makable_model = load("models/StackingRegressor_best_pipeline_gia_0_to_1.01.pkl")
    

# Classification models loaded using joblib.
col_model = load(os.path.join(MODEL_DIR, 'classification_LogisticRegression_col.joblib'))
cts_model = load(os.path.join(MODEL_DIR, 'classification_LogisticRegression_cts.joblib'))
cut_model = load(os.path.join(MODEL_DIR, 'classification_LogisticRegression_cut.joblib'))
qua_model = load(os.path.join(MODEL_DIR, 'classification_LogisticRegression_qua.joblib'))
shp_model = load(os.path.join(MODEL_DIR, 'classification_LogisticRegression_shp.joblib'))

blk_eng_to_mkbl_model = load(os.path.join(MODEL_DIR, 'classification_LogisticRegression_mkbl_blk.joblib'))
wht_eng_to_mkbl_model = load(os.path.join(MODEL_DIR, 'classification_LogisticRegression_mkbl_wht.joblib'))
open_eng_to_mkbl_model = load(os.path.join(MODEL_DIR, 'classification_LogisticRegression_mkbl_open.joblib'))
pav_eng_to_mkbl_model = load(os.path.join(MODEL_DIR, 'classification_LogisticRegression_mkbl_pav.joblib'))
blk_eng_to_grade_model = load(os.path.join(MODEL_DIR, 'classification_LogisticRegression_grade_blk.joblib'))
wht_eng_to_grade_model = load(os.path.join(MODEL_DIR, 'classification_LogisticRegression_grade_wht.joblib'))
open_eng_to_grade_model = load(os.path.join(MODEL_DIR, 'classification_LogisticRegression_grade_open.joblib'))
pav_eng_to_grade_model = load(os.path.join(MODEL_DIR, 'classification_LogisticRegression_grade_pav.joblib'))
blk_eng_to_bygrade_model = load(os.path.join(MODEL_DIR, 'classification_LogisticRegression_bygrade_blk.joblib'))
wht_eng_to_bygrade_model = load(os.path.join(MODEL_DIR, 'classification_LogisticRegression_bygrade_wht.joblib'))
open_eng_to_bygrade_model = load(os.path.join(MODEL_DIR, 'classification_LogisticRegression_bygrade_open.joblib'))
pav_eng_to_bygrade_model = load(os.path.join(MODEL_DIR, 'classification_LogisticRegression_bygrade_pav.joblib'))
blk_eng_to_gia_model = load(os.path.join(MODEL_DIR, 'classification_LogisticRegression_gia_blk.joblib'))
wht_eng_to_gia_model = load(os.path.join(MODEL_DIR, 'classification_LogisticRegression_gia_wht.joblib'))
open_eng_to_gia_model = load(os.path.join(MODEL_DIR, 'classification_LogisticRegression_gia_open.joblib'))
pav_eng_to_gia_model = load(os.path.join(MODEL_DIR, 'classification_LogisticRegression_gia_pav.joblib'))

# List of label encoder names.
encoder_list = [
    'Tag', 'EngShp', 'EngQua', 'EngCol', 'EngCut', 'EngPol', 'EngSym', 'EngFlo',
    'EngNts', 'EngMikly', 'EngLab','EngBlk', 'EngWht', 'EngOpen','EngPav',
    'Change_cts_value', 'Change_shape_value', 'Change_quality_value', 'Change_color_value',
    'Change_cut_value', 'Change_Blk_Eng_to_Mkbl_value', 'Change_Wht_Eng_to_Mkbl_value',
    'Change_Open_Eng_to_Mkbl_value', 'Change_Pav_Eng_to_Mkbl_value', 'Change_Blk_Eng_to_Grd_value',
    'Change_Wht_Eng_to_Grd_value', 'Change_Open_Eng_to_Grd_value', 'Change_Pav_Eng_to_Grd_value',
    'Change_Blk_Eng_to_ByGrd_value', 'Change_Wht_Eng_to_ByGrd_value', 'Change_Open_Eng_to_ByGrd_value',
    'Change_Pav_Eng_to_ByGrd_value', 'Change_Blk_Eng_to_Gia_value', 'Change_Wht_Eng_to_Gia_value',
    'Change_Open_Eng_to_Gia_value', 'Change_Pav_Eng_to_Gia_value'
]

# Load label encoders using pathlib for cleaner path management.
loaded_label_encoder = {}
enc_path = Path(LABEL_ENCODER_DIR)
for val in encoder_list:
    encoder_file = enc_path / f"label_encoder_{val}.joblib"
    loaded_label_encoder[val] = load(encoder_file)

# ------------------------------
# Utility: Allowed File Check
# ------------------------------
def allowed_file(filename):
    return '.' in filename and filename.rsplit('.', 1)[1].lower() in ALLOWED_EXTENSIONS

# ------------------------------
# Routes
# ------------------------------
@app.route('/')
def index():
    return render_template('index.html')

@app.route('/predict', methods=['POST'])
def predict():
    if 'file' not in request.files:
        print('No file part', 'error')
        return redirect(url_for('index'))
    
    file = request.files['file']
    if file.filename == '':
        print('No selected file', 'error')
        return redirect(url_for('index'))
    
    if file and allowed_file(file.filename):
        filename = secure_filename(file.filename)
        filepath = os.path.join(app.config['UPLOAD_FOLDER'], filename)
        file.save(filepath)
        
        # Convert file to DataFrame
        try:
            if filename.endswith('.csv'):
                df = pd.read_csv(filepath)
            else:
                df = pd.read_excel(filepath)
        except Exception as e:
            print(f'Error reading file: {e}', 'error')
            return redirect(url_for('index'))
        
        # Process the DataFrame and generate predictions and classification analysis.
        df_pred, dx_class = process_dataframe(df)
        if df_pred.empty:
            print("Processed prediction DataFrame is empty. Check the input file and processing logic.", "error")
            return redirect(url_for('index'))
        
        # Save output files with a timestamp and unique id.
        current_date = pd.Timestamp.now().strftime("%Y-%m-%d")
        unique_id = uuid.uuid4().hex[:8]
        global PRED_OUTPUT_FILE, CLASS_OUTPUT_FILE
        PRED_OUTPUT_FILE = f'data/prediction_output_{current_date}_{unique_id}.csv'
        CLASS_OUTPUT_FILE = f'data/classification_output_{current_date}_{unique_id}.csv'
        df_pred.to_csv(PRED_OUTPUT_FILE, index=False)
        dx_class.to_csv(CLASS_OUTPUT_FILE, index=False)
        
        # Redirect to report view; default to prediction report, page 1.
        return redirect(url_for('report_view', report_type='pred', page=1))
    else:
        print('Invalid file type. Only CSV and Excel files are allowed.', 'error')
        return redirect(url_for('index'))

def process_dataframe(df):
    try:
        # Define the columns needed for two parts.
        required_columns = ['Tag', 'EngCts', 'EngShp', 'EngQua', 'EngCol', 'EngCut', 'EngPol',
                           'EngSym', 'EngFlo', 'EngNts', 'EngMikly', 'EngBlk', 'EngWht', 'EngOpen',
                           'EngPav', 'EngAmt']
        required_columns_2 = ['Tag', 'EngCts', 'EngShp', 'EngQua', 'EngCol', 'EngCut', 'EngPol',
                           'EngSym', 'EngFlo', 'EngNts', 'EngMikly', 'EngAmt']
        
        # Create two DataFrames: one for prediction and one for classification.
        df_pred = df[required_columns].copy()
        df_pred[['EngBlk', 'EngWht', 'EngOpen', 'EngPav']]=df_pred[['EngBlk', 'EngWht', 'EngOpen', 'EngPav']].fillna("NA")
        df_class = df[required_columns_2].fillna("NA").copy()
        
        # Transform categorical columns for prediction DataFrame using the label encoders.
        for col in ['Tag', 'EngShp', 'EngQua', 'EngCol', 'EngCut', 'EngPol', 'EngSym', 'EngFlo', 'EngNts', 'EngMikly','EngBlk', 'EngWht', 'EngOpen', 'EngPav']:
            try:
                df_pred[col] = loaded_label_encoder[col].transform(df_pred[col])
            except ValueError as e:
                print(f'Invalid value in column {col}: {e}', 'error')
                return pd.DataFrame(), pd.DataFrame()
        
        # Update the classification DataFrame with the transformed prediction columns.
        for col in ['Tag', 'EngShp', 'EngQua', 'EngCol', 'EngCut', 'EngPol', 'EngSym', 'EngFlo', 'EngNts', 'EngMikly']:
            df_class[col] = df_pred[col]
        
        # Transform the extra columns in the classification DataFrame.
        #for col in ['EngBlk', 'EngWht', 'EngOpen', 'EngPav']:
        #    try:
        #        df_class[col] = loaded_label_encoder[col].transform(df_class[col])
        #    except ValueError as e:
        #        print(f'Invalid value in column {col}: {e}', 'error')
        #        return pd.DataFrame(), pd.DataFrame()
        
        # Convert both DataFrames to float.
        df_pred = df_pred.astype(float)
        df_class = df_class.astype(float)
        
        # -------------------------
        # Prediction Report Section
        # -------------------------
        try:
            x = df_pred.copy()
            df_pred['Makable_Predicted'] = pd.DataFrame(np.expm1(makable_model.predict(x)), columns=["Predicted"])
            df_pred['Grade_Predicted'] = pd.DataFrame(np.expm1(grade_model.predict(x)), columns=["Predicted"])
            df_pred['ByGrade_Predicted'] = pd.DataFrame(np.expm1(bygrade_model.predict(x)), columns=["Predicted"])
            df_pred['GIA_Predicted'] = pd.DataFrame(np.expm1(gia_model.predict(x)), columns=["Predicted"])
            #df_pred['GIA_Diff'] = df_pred['EngAmt'] - df_pred['GIA_Predicted']
            #df_pred['Grade_Diff'] = df_pred['EngAmt'] - df_pred['Grade_Predicted']
            #df_pred['ByGrade_Diff'] = df_pred['EngAmt'] - df_pred['ByGrade_Predicted']
            #df_pred['Makable_Diff'] = df_pred['EngAmt'] - df_pred['Makable_Predicted']
            
            for col in ['Tag', 'EngShp', 'EngQua', 'EngCol', 'EngCut', 'EngPol', 'EngSym', 'EngFlo', 'EngNts', 'EngMikly','EngBlk', 'EngWht', 'EngOpen', 'EngPav']:
                try:
                    df_pred[col] = loaded_label_encoder[col].inverse_transform(df_pred[col].astype(int))
                except ValueError as e:
                    print(f'inverse transform fails value in column {col}: {e}', 'error')
                    
        except ValueError as e:
            print(f'pred model error----->: {e}', 'error')
        
        # -------------------------
        # Classification Report Section
        # -------------------------
        try:
            x2 = df_class.copy()
            dx = df_pred.copy()  # Start with the prediction data.
            dx['col_change'] = col_model.predict(x2)
            dx['cts_change'] = cts_model.predict(x2)
            dx['cut_change'] = cut_model.predict(x2)
            dx['qua_change'] = qua_model.predict(x2)
            dx['shp_change'] = shp_model.predict(x2)
        except ValueError as e:
            print(f'class model error----->: {e}', 'error')

        try:
            dx['Change_Blk_Eng_to_Mkbl_value'] = blk_eng_to_mkbl_model.predict(x)
            dx['Change_Wht_Eng_to_Mkbl_value'] = wht_eng_to_mkbl_model.predict(x)
            dx['Change_Open_Eng_to_Mkbl_value'] = open_eng_to_mkbl_model.predict(x)
            dx['Change_Pav_Eng_to_Mkbl_value'] = pav_eng_to_mkbl_model.predict(x)
            dx['Change_Blk_Eng_to_Grd_value'] = blk_eng_to_grade_model.predict(x)
            dx['Change_Wht_Eng_to_Grd_value'] = wht_eng_to_grade_model.predict(x)
            dx['Change_Open_Eng_to_Grd_value'] = open_eng_to_grade_model.predict(x)
            dx['Change_Pav_Eng_to_Grd_value'] = pav_eng_to_grade_model.predict(x)
            dx['Change_Blk_Eng_to_ByGrd_value'] = blk_eng_to_bygrade_model.predict(x)
            dx['Change_Wht_Eng_to_ByGrd_value'] = wht_eng_to_bygrade_model.predict(x)
            dx['Change_Open_Eng_to_ByGrd_value'] = open_eng_to_bygrade_model.predict(x)
            dx['Change_Pav_Eng_to_ByGrd_value'] = pav_eng_to_bygrade_model.predict(x)
            dx['Change_Blk_Eng_to_Gia_value'] = blk_eng_to_gia_model.predict(x)
            dx['Change_Wht_Eng_to_Gia_value'] = wht_eng_to_gia_model.predict(x)
            dx['Change_Open_Eng_to_Gia_value'] = open_eng_to_gia_model.predict(x)
            dx['Change_Pav_Eng_to_Gia_value'] = pav_eng_to_gia_model.predict(x)
        except ValueError as e:
            print(f'grade_code model error----->: {e}', 'error')
        
        # Inverse transform classification predictions.
        dx['col_change'] = loaded_label_encoder['Change_color_value'].inverse_transform(dx['col_change'])
        dx['cts_change'] = loaded_label_encoder['Change_cts_value'].inverse_transform(dx['cts_change'])
        dx['cut_change'] = loaded_label_encoder['Change_cut_value'].inverse_transform(dx['cut_change'])
        dx['qua_change'] = loaded_label_encoder['Change_quality_value'].inverse_transform(dx['qua_change'])
        dx['shp_change'] = loaded_label_encoder['Change_shape_value'].inverse_transform(dx['shp_change'])
        dx['Change_Blk_Eng_to_Mkbl_value'] = loaded_label_encoder['Change_Blk_Eng_to_Mkbl_value'].inverse_transform(dx['Change_Blk_Eng_to_Mkbl_value'])
        dx['Change_Wht_Eng_to_Mkbl_value'] = loaded_label_encoder['Change_Wht_Eng_to_Mkbl_value'].inverse_transform(dx['Change_Wht_Eng_to_Mkbl_value'])
        dx['Change_Open_Eng_to_Mkbl_value'] = loaded_label_encoder['Change_Open_Eng_to_Mkbl_value'].inverse_transform(dx['Change_Open_Eng_to_Mkbl_value'])
        dx['Change_Pav_Eng_to_Mkbl_value'] = loaded_label_encoder['Change_Pav_Eng_to_Mkbl_value'].inverse_transform(dx['Change_Pav_Eng_to_Mkbl_value'])
        dx['Change_Blk_Eng_to_Grd_value'] = loaded_label_encoder['Change_Blk_Eng_to_Grd_value'].inverse_transform(dx['Change_Blk_Eng_to_Grd_value'])
        dx['Change_Wht_Eng_to_Grd_value'] = loaded_label_encoder['Change_Wht_Eng_to_Grd_value'].inverse_transform(dx['Change_Wht_Eng_to_Grd_value'])
        dx['Change_Open_Eng_to_Grd_value'] = loaded_label_encoder['Change_Open_Eng_to_Grd_value'].inverse_transform(dx['Change_Open_Eng_to_Grd_value'])
        dx['Change_Pav_Eng_to_Grd_value'] = loaded_label_encoder['Change_Pav_Eng_to_Grd_value'].inverse_transform(dx['Change_Pav_Eng_to_Grd_value'])
        dx['Change_Blk_Eng_to_ByGrd_value'] = loaded_label_encoder['Change_Blk_Eng_to_ByGrd_value'].inverse_transform(dx['Change_Blk_Eng_to_ByGrd_value'])
        dx['Change_Wht_Eng_to_ByGrd_value'] = loaded_label_encoder['Change_Wht_Eng_to_ByGrd_value'].inverse_transform(dx['Change_Wht_Eng_to_ByGrd_value'])
        dx['Change_Open_Eng_to_ByGrd_value'] = loaded_label_encoder['Change_Open_Eng_to_ByGrd_value'].inverse_transform(dx['Change_Open_Eng_to_ByGrd_value'])
        dx['Change_Pav_Eng_to_ByGrd_value'] = loaded_label_encoder['Change_Pav_Eng_to_ByGrd_value'].inverse_transform(dx['Change_Pav_Eng_to_ByGrd_value'])
        dx['Change_Blk_Eng_to_Gia_value'] = loaded_label_encoder['Change_Blk_Eng_to_Gia_value'].inverse_transform(dx['Change_Blk_Eng_to_Gia_value'])
        dx['Change_Wht_Eng_to_Gia_value'] = loaded_label_encoder['Change_Wht_Eng_to_Gia_value'].inverse_transform(dx['Change_Wht_Eng_to_Gia_value'])
        dx['Change_Open_Eng_to_Gia_value'] = loaded_label_encoder['Change_Open_Eng_to_Gia_value'].inverse_transform(dx['Change_Open_Eng_to_Gia_value'])
        dx['Change_Pav_Eng_to_Gia_value'] = loaded_label_encoder['Change_Pav_Eng_to_Gia_value'].inverse_transform(dx['Change_Pav_Eng_to_Gia_value'])
        
        # Final return with full data for pagination.
        return df_pred, dx.head(len(df_pred))
    except Exception as e:
        print(f'Error processing file: {e}', 'error')
        return pd.DataFrame(), pd.DataFrame()

# ------------------------------
# Report View Route with Pagination & Toggle
# ------------------------------
@app.route('/report')
def report_view():
    report_type = request.args.get('report_type', 'pred')
    try:
        page = int(request.args.get('page', 1))
    except ValueError:
        page = 1
    per_page = 15  # records per page

    # Read the appropriate CSV file.
    if report_type == 'pred':
        df = pd.read_csv(PRED_OUTPUT_FILE)
    else:
        df = pd.read_csv(CLASS_OUTPUT_FILE)

    start_idx = (page - 1) * per_page
    end_idx = start_idx + per_page
    total_records = len(df)
    
    df_page = df.iloc[start_idx:end_idx]
    table_html = df_page.to_html(classes="data-table", index=False)
    
    has_prev = page > 1
    has_next = end_idx < total_records
    
    return render_template('output.html',
                           table_html=table_html,
                           report_type=report_type,
                           page=page,
                           has_prev=has_prev,
                           has_next=has_next)

# ------------------------------
# Download Routes
# ------------------------------
@app.route('/download_pred', methods=['GET'])
def download_pred():
    return send_file(PRED_OUTPUT_FILE, as_attachment=True)

@app.route('/download_class', methods=['GET'])
def download_class():
    return send_file(CLASS_OUTPUT_FILE, as_attachment=True)

if __name__ == "__main__":
    app.run(debug=True)