from typing import List from pydantic import BaseModel from fastapi import FastAPI, Response, BackgroundTasks from fastapi.middleware.cors import CORSMiddleware # 跨域 from sklearn.metrics import mean_absolute_error from sklearn.metrics import mean_squared_error from sklearn.metrics import r2_score from sklearn.model_selection import train_test_split import sklearn.preprocessing as preproc from sklearn.preprocessing import StandardScaler import lightgbm as lgb from xgboost import XGBRegressor from sklearn.neural_network import MLPRegressor from sklearn.ensemble import RandomForestRegressor from sklearn.linear_model import LinearRegression import matplotlib.pyplot as plt import io import json import numpy as np import pandas as pd import matplotlib matplotlib.use('AGG') app = FastAPI() # set cross-domain whitelist origins = [ "http://127.0.0.1:5500", "http://localhost:8081", "http://mlca.coycs.com", "https://mlca.coycs.com", "http://celadon-lebkuchen-cc4bb0.netlify.app", "https://celadon-lebkuchen-cc4bb0.netlify.app" ] app.add_middleware( CORSMiddleware, allow_origins=origins, allow_credentials=True, allow_methods=["POST", "GET"], allow_headers=["*"], ) # 工具函数 def json2df(json): # 字符串转数值 def str2num(x): if isinstance(x, str): return eval(x) else: return x df = pd.DataFrame(json) # 空白符转None，且是"None"让eval能解析成功 df.replace(to_replace=r"^\s*$", value="None", regex=True, inplace=True) # 科学计数法转数值 df = df.applymap(str2num) return df # def process_abnormal(df, detect, method): # 异常值处理 # if detect == 1: # IQR检测方式 # for coloum in df.columns: # q1 = df[coloum].quantile(0.75) # q3 = df[coloum].quantile(0.25) # iqr = q1-q3 # if method == 1: # 删除异常值 # df.drop( # df.loc[lambda x:x[coloum] > q1 + 1.5 * iqr].index, inplace=True) # df.drop( # df.loc[lambda x:x[coloum] < q3 - 1.5 * iqr].index, inplace=True) # elif method == 2: # 均值替换 # df.loc[lambda x:x[coloum] > q1 + 1.5 * # iqr, coloum]=df[coloum].mean() # df.loc[lambda x:x[coloum] < q3 - 1.5 * # iqr, coloum]=df[coloum].mean() # elif method == 3: # 中位数替换 # df.loc[lambda x:x[coloum] > q1 + 1.5 * # iqr, coloum]=df[coloum].median() # df.loc[lambda x:x[coloum] < q3 - 1.5 * # iqr, coloum]=df[coloum].median() # elif method == 4: # 众数替换 # df.loc[lambda x:x[coloum] > q1 + 1.5 * # iqr, coloum]=df[coloum].mode().iloc[0] # df.loc[lambda x:x[coloum] < q3 - 1.5 * # iqr, coloum]=df[coloum].mode().iloc[0] # elif method == 5: # 边界替换 # df.loc[lambda x:x[coloum] > q1 + # 1.5 * iqr, coloum]=q1 + 1.5 * iqr # df.loc[lambda x:x[coloum] < q3 - # 1.5 * iqr, coloum]=q3 - 1.5 * iqr # elif detect == 2: # Z-score检测方式 # for coloum in df.columns: # mean = df[coloum].mean() # std = df[coloum].std() # df.drop( # df.loc[lambda x:x[coloum] > mean + 3 * std].index, inplace=True) # df.drop( # df.loc[lambda x:x[coloum] < mean - 3 * std].index, inplace=True) # if method == 1: # 删除异常值 # df.drop( # df.loc[lambda x:x[coloum] > mean + 3 * std].index, inplace=True) # df.drop( # df.loc[lambda x:x[coloum] < mean - 3 * std].index, inplace=True) # elif method == 2: # 均值替换 # df.loc[lambda x:x[coloum] > mean + # 3 * std, coloum]=df[coloum].mean() # df.loc[lambda x:x[coloum] < mean - # 3 * std, coloum]=df[coloum].mean() # elif method == 3: # 中位数替换 # df.loc[lambda x:x[coloum] > mean + 3 * # std, coloum]=df[coloum].median() # df.loc[lambda x:x[coloum] < mean - 3 * # std, coloum]=df[coloum].median() # elif method == 4: # 众数替换 # df.loc[lambda x:x[coloum] > mean + 3 * # std, coloum]=df[coloum].mode().iloc[0] # df.loc[lambda x:x[coloum] < mean - 3 * # std, coloum]=df[coloum].mode().iloc[0] # elif method == 5: # 边界替换 # df.loc[lambda x:x[coloum] > mean + # 3 * std, coloum]=mean + 3 * std # df.loc[lambda x:x[coloum] < mean - # 3 * std, coloum]=mean - 3 * std # return df def process_miss(df, method): # 缺失值处理 # 舍弃全为空的行 df = df.dropna(how='all') # 舍弃全为空的列 df = df.dropna(axis=1, how='all') if method == 1: # 均值 df = df.fillna(df.mean()) elif method == 2: # 中位数 df = df.fillna(df.median()) elif method == 3: # 众数 df = df.fillna(df.mode().iloc[0]) elif method == 4: # 线性 df = df.fillna(df.interpolate( method='linear', limit_direction='forward', axis=0)) elif method == 5: # 前值 df = df.fillna(method="ffill") elif method == 6: # 后值 df = df.fillna(method="bfill") return df def process_abnormal(df_inside, df_user, detect, method): # 异常值处理 df = pd.concat([df_inside, df_user], axis=0, ignore_index=True) # 合并的dataframe df_features = df.iloc[:, :12] # 取所有的特征列为dataframe # print(df) if detect == 1: # IQR检测方式 for coloum in df_features.columns: q1 = df_features[coloum].quantile(0.75) q3 = df_features[coloum].quantile(0.25) iqr = q1-q3 if method == 1: # 删除异常值 df_features.drop( df_features.loc[lambda x:x[coloum] > q1 + 1.5 * iqr].index, inplace=True) df_features.drop( df_features.loc[lambda x:x[coloum] < q3 - 1.5 * iqr].index, inplace=True) elif method == 2: # 均值替换 df_features.loc[lambda x:x[coloum] > q1 + 1.5 * iqr, coloum]=df_features[coloum].mean() df_features.loc[lambda x:x[coloum] < q3 - 1.5 * iqr, coloum]=df_features[coloum].mean() elif method == 3: # 中位数替换 df_features.loc[lambda x:x[coloum] > q1 + 1.5 * iqr, coloum]=df_features[coloum].median() df_features.loc[lambda x:x[coloum] < q3 - 1.5 * iqr, coloum]=df_features[coloum].median() elif method == 4: # 众数替换 df_features.loc[lambda x:x[coloum] > q1 + 1.5 * iqr, coloum]=df_features[coloum].mode().iloc[0] df_features.loc[lambda x:x[coloum] < q3 - 1.5 * iqr, coloum]=df_features[coloum].mode().iloc[0] elif method == 5: # 边界替换 df_features.loc[lambda x:x[coloum] > q1 + 1.5 * iqr, coloum]=q1 + 1.5 * iqr df_features.loc[lambda x:x[coloum] < q3 - 1.5 * iqr, coloum]=q3 - 1.5 * iqr elif detect == 2: # Z-score检测方式 for coloum in df_features.columns: mean = df_features[coloum].mean() std = df_features[coloum].std() df_features.drop( df_features.loc[lambda x:x[coloum] > mean + 3 * std].index, inplace=True) df_features.drop( df_features.loc[lambda x:x[coloum] < mean - 3 * std].index, inplace=True) if method == 1: # 删除异常值 df_features.drop( df_features.loc[lambda x:x[coloum] > mean + 3 * std].index, inplace=True) df_features.drop( df_features.loc[lambda x:x[coloum] < mean - 3 * std].index, inplace=True) elif method == 2: # 均值替换 df_features.loc[lambda x:x[coloum] > mean + 3 * std, coloum]=df_features[coloum].mean() df_features.loc[lambda x:x[coloum] < mean - 3 * std, coloum]=df_features[coloum].mean() elif method == 3: # 中位数替换 df_features.loc[lambda x:x[coloum] > mean + 3 * std, coloum]=df_features[coloum].median() df_features.loc[lambda x:x[coloum] < mean - 3 * std, coloum]=df_features[coloum].median() elif method == 4: # 众数替换 df_features.loc[lambda x:x[coloum] > mean + 3 * std, coloum]=df_features[coloum].mode().iloc[0] df_features.loc[lambda x:x[coloum] < mean - 3 * std, coloum]=df_features[coloum].mode().iloc[0] elif method == 5: # 边界替换 df_features.loc[lambda x:x[coloum] > mean + 3 * std, coloum]=mean + 3 * std df_features.loc[lambda x:x[coloum] < mean - 3 * std, coloum]=mean - 3 * std df.iloc[:, :12] = df_features df_inside = df.iloc[:df_inside.shape[0], :] df_user = df.iloc[df_inside.shape[0]:, :12] return {"df_inside": df_inside, "df_user": df_user} def process_standard(df_inside, df_user, method): # 标准化处理 df = pd.concat([df_inside, df_user], axis=0, ignore_index=True) # 合并的dataframe df_features = df.iloc[:, :12] # 取所有的特征列为dataframe columns = df_features.columns # 列名 if method == 1: # Min-max df_features = preproc.minmax_scale(df_features) elif method == 2: # Z-Score df_features = preproc.StandardScaler().fit_transform(df_features) elif method == 3: # MaxAbs df_features = preproc.maxabs_scale(df_features, axis=0) elif method == 4: # RobustScaler df_features = preproc.RobustScaler().fit_transform(df_features) elif method == 5: # 正则化 df_features = preproc.normalize(df_features, axis=0) df_features = pd.DataFrame( data=df_features[0:, 0:], columns=columns) # 补充列名 df.iloc[:, :12] = df_features df_inside = df.iloc[:df_inside.shape[0], :] df_user = df.iloc[df_inside.shape[0]:, :12] return {"df_inside": df_inside, "df_user": df_user} def train_model(x, y, test_size, algorithm, paras): # 模型训练 # 划分数据集 x_train, x_test, y_train, y_test = train_test_split( x, y, test_size=test_size, random_state=0) # 机器学习 model = None results = {} if algorithm == 1: # 最小二乘法线性回归 model = LinearRegression(fit_intercept=paras["fit_intercept"]) if algorithm == 2: # 随机森林回归 model = RandomForestRegressor(n_estimators=paras["n_estimators"], criterion=paras["criterion"], max_depth=paras["max_depth"], random_state=0) if algorithm == 3: # BP神经网络回归 model = MLPRegressor(hidden_layer_sizes=(paras["hidden_layer_sizes_1"], paras["hidden_layer_sizes_2"]), activation=paras["activation"], solver='lbfgs', random_state=paras["random_state"]) if algorithm == 4: # XGBoost回归 model = XGBRegressor( max_depth=paras["max_depth"], learning_rate=paras["learning_rate"], n_estimators=paras["n_estimators"]) if algorithm == 5: # LightGBM回归 # model = lgb.LGBMRegressor(objective='regression',boosting_type="dart",num_leaves=30, max_depth=-1,n_estimators=20,learning_rate=1) model = lgb.LGBMRegressor(objective='regression', max_depth=paras["max_depth"], learning_rate=paras["learning_rate"], random_state=paras["random_state"], n_estimators=paras["n_estimators"]) # 返回数据 if model != None: model.fit(x_train, y_train) if algorithm == 1: # 最小二乘法线性回归 # 保留小数点后三位 # results["coef"] = model.coef_.tolist() # 模型斜率 results["coef"] = [float('{:.4f}'.format(i)) for i in model.coef_.tolist()] # 模型斜率 results["intercept"] = round(model.intercept_, 3) # 模型截距 y_pred = model.predict(x_test) # 预测值 # y_test = y_test.values # 误差，用round保留三位小数且四舍五入 mae = round(mean_absolute_error(y_test, y_pred), 3) rmse = round(np.sqrt(mean_squared_error(y_test, y_pred)), 3) r2 = round(r2_score(y_test, y_pred), 3) # y_test = [x[0] for x in np.array(y_test).tolist()] # y_pred = [x[0] for x in y_pred.tolist()] y_test = np.array(y_test).tolist() y_pred = y_pred.tolist() res = {"y_test": y_test, "y_pred": y_pred, "error": { "MAE": mae, "RMSE": rmse, "R2": r2}, "results": results} print(res) return res # return {"y_test": y_test, "y_pred": y_pred, "error": {"MAE": mae, "RMSE": rmse, "R2": r2}, "results": results} else: return "模型训练出错" def predict_connectivity(x, x1, y, test_size, algorithm, paras): # 划分数据集 x_train, x_test, y_train, y_test = train_test_split( x, y, test_size=test_size, random_state=0) # 机器学习 model = None results = {} if algorithm == 1: # 最小二乘法线性回归 model = LinearRegression(fit_intercept=paras["fit_intercept"]) if algorithm == 2: # 随机森林回归 model = RandomForestRegressor(n_estimators=paras["n_estimators"], criterion=paras["criterion"], max_depth=paras["max_depth"], random_state=0) if algorithm == 3: # BP神经网络回归 model = MLPRegressor(hidden_layer_sizes=(paras["hidden_layer_sizes_1"], paras["hidden_layer_sizes_2"]), activation=paras["activation"], solver='lbfgs', random_state=paras["random_state"]) if algorithm == 4: # XGBoost回归 model = XGBRegressor( max_depth=paras["max_depth"], learning_rate=paras["learning_rate"], n_estimators=paras["n_estimators"]) if algorithm == 5: # LightGBM回归 model = lgb.LGBMRegressor(objective='regression', max_depth=paras["max_depth"], learning_rate=paras["learning_rate"], random_state=paras["random_state"], n_estimators=paras["n_estimators"]) # 返回数据 if model != None: model.fit(x_train, y_train) y_pred = model.predict(x1).tolist() # 预测值 return y_pred else: return "预测连通性出错" # 登录验证 class Login(BaseModel): # 接口数据类型 username: str password: str @app.post("/login") # 接口 async def login(login: Login): username = login.username password = login.password if username == "admin" and password == "123456": return True return False # 处理用户数据 class Process_user(BaseModel): # 接口数据类型 mode: int data: List miss: List abnormal: List standard: List @app.post("/process/user") # 接口 async def process_user(user: Process_user): mode = user.mode # 选择的井间连通模式 df_inside = pd.read_csv( "./mode_{}.csv".format(mode)).dropna(axis=0) # 连通模式对应的内置数据 df_user = json2df(user.data) abnormal = user.abnormal[0] miss = user.miss[0] standard = user.standard[0] # 异常值处理 if abnormal["state"]: abnormaled = process_abnormal( df_inside, df_user, abnormal["detect"], abnormal["method"]) df_inside = abnormaled["df_inside"] df_user = abnormaled["df_user"] # 缺失值处理 if miss["state"]: df_user = process_miss(df_user, miss["method"]) # 标准化处理 if standard["state"]: standarded = process_standard(df_inside, df_user, standard["method"]) df_inside = standarded["df_inside"] df_user = standarded["df_user"] # 用astype将数值转科学计数法 return {"inside": df_inside.astype('str').to_json(orient='records'), "user": df_user.astype('str').to_json(orient='records')} # # 用astype将数值转科学计数法 # return df.astype('str').to_json(orient='records') # 处理内置数据 class Process_inside(BaseModel): # 接口数据类型 data: List abnormal: List standard: List @app.post("/process/inside") # 接口 async def process_inside(inside: Process_inside): df = json2df(inside.data) abnormal = inside.abnormal[0] standard = inside.standard[0] # 异常值处理 if abnormal["state"]: df = process_abnormal(df, abnormal["detect"], abnormal["method"]) # 标准化处理：只对特征进行标准化，不包括标签（后三列） if standard["state"]: df = pd.concat([process_standard(df.iloc[:, :12], standard["method"]), df.iloc[:, 12:]], axis=1) # 用astype将数值转科学计数法 return df.astype('str').to_json(orient='records') # 训练模型 class Train(BaseModel): # 接口数据类型 data: List test_size: float algorithm: int paras: List @app.post("/train") # 接口 async def train(train: Train): # 解析数据 df = json2df(train.data) test_size = train.test_size algorithm = train.algorithm paras = train.paras[0] x = df.iloc[:, :12] y1 = df.loc[:, "BSR"] y2 = df.loc[:, "SBR"] y3 = df.loc[:, "D"] bsr = train_model(x, y1, test_size, algorithm, paras) sbr = train_model(x, y2, test_size, algorithm, paras) x_train, x_test, y_train, y_test = train_test_split( x, y3, test_size=test_size, random_state=0) d = {"y_test": np.array(y_test).tolist(), "y_pred": np.sum( [bsr["y_pred"], sbr["y_pred"]], axis=0).tolist()} return {"bsr": bsr, "sbr": sbr, "d": d} # 预测连通性 class Predict(BaseModel): # 接口数据类型 data_train: List data_predict: List test_size: float algorithm: int paras: List @app.post("/predict") # 接口 async def predict(predict: Predict): # 解析数据 df_train = json2df(predict.data_train) df_predict = json2df(predict.data_predict) test_size = predict.test_size algorithm = predict.algorithm paras = predict.paras[0] x = df_train.iloc[:, :12] y1 = df_train.loc[:, "BSR"] y2 = df_train.loc[:, "SBR"] # 预测连通性 bsr = predict_connectivity(x, df_predict, y1, test_size, algorithm, paras) sbr = predict_connectivity(x, df_predict, y2, test_size, algorithm, paras) d = np.sum([bsr, sbr], axis=0).tolist() # 合并为一个list后转dataframe再转json实现前端表格数据格式 data = [] data.append(bsr) data.append(sbr) data.append(d) df_result = pd.concat([pd.DataFrame(predict.data_predict), pd.DataFrame(data=np.array( data).T.tolist(), columns=["BSR", "SBR", "D"])], axis=1) return df_result.to_json(orient='records') # return pd.DataFrame(data=np.array(data).T.tolist(), columns=["BSR", "SBR", "D"]).to_json(orient='records') # # 图片测试 # def create_img(): # plt.rcParams['figure.figsize'] = [7.50, 3.50] # plt.rcParams['figure.autolayout'] = True # plt.plot([1, 2]) # img_buf = io.BytesIO() # plt.savefig(img_buf, format='png') # plt.close() # return img_buf # @app.get('/png') # async def get_img(background_tasks: BackgroundTasks): # img_buf = create_img() # # get the entire buffer content # # because of the async, this will await the loading of all content # bufContents: bytes = img_buf.getvalue() # background_tasks.add_task(img_buf.close) # headers = {'Content-Disposition': 'inline; filename="out.png"'} # return Response(bufContents, headers=headers, media_type='image/png')