from typing import List
from pydantic import BaseModel
from fastapi import FastAPI, Response, BackgroundTasks
from fastapi.middleware.cors import CORSMiddleware # 跨域
from sklearn.metrics import mean_absolute_error
from sklearn.metrics import mean_squared_error
from sklearn.metrics import r2_score
from sklearn.model_selection import train_test_split
import sklearn.preprocessing as preproc
from sklearn.preprocessing import StandardScaler
import lightgbm as lgb
from xgboost import XGBRegressor
from sklearn.neural_network import MLPRegressor
from sklearn.ensemble import RandomForestRegressor
from sklearn.linear_model import LinearRegression
import matplotlib.pyplot as plt
import io
import json
import numpy as np
import pandas as pd
import matplotlib
matplotlib.use('AGG')
app = FastAPI()
# set cross-domain whitelist
origins = [
"http://127.0.0.1:5500",
"http://localhost:8081",
"http://mlca.coycs.com",
"https://mlca.coycs.com"
]
app.add_middleware(
CORSMiddleware,
allow_origins=origins,
allow_credentials=True,
allow_methods=["POST", "GET"],
allow_headers=["*"],
)
# 工具函数
def json2df(json):
# 字符串转数值
def str2num(x):
if isinstance(x, str):
return eval(x)
else:
return x
df = pd.DataFrame(json)
# 空白符转None，且是"None"让eval能解析成功
df.replace(to_replace=r"^\s*$", value="None", regex=True, inplace=True)
# 科学计数法转数值
df = df.applymap(str2num)
return df
# def process_abnormal(df, detect, method): # 异常值处理
# if detect == 1: # IQR检测方式
# for coloum in df.columns:
# q1 = df[coloum].quantile(0.75)
# q3 = df[coloum].quantile(0.25)
# iqr = q1-q3
# if method == 1: # 删除异常值
# df.drop(
# df.loc[lambda x:x[coloum] > q1 + 1.5 * iqr].index, inplace=True)
# df.drop(
# df.loc[lambda x:x[coloum] < q3 - 1.5 * iqr].index, inplace=True)
# elif method == 2: # 均值替换
# df.loc[lambda x:x[coloum] > q1 + 1.5 *
# iqr, coloum]=df[coloum].mean()
# df.loc[lambda x:x[coloum] < q3 - 1.5 *
# iqr, coloum]=df[coloum].mean()
# elif method == 3: # 中位数替换
# df.loc[lambda x:x[coloum] > q1 + 1.5 *
# iqr, coloum]=df[coloum].median()
# df.loc[lambda x:x[coloum] < q3 - 1.5 *
# iqr, coloum]=df[coloum].median()
# elif method == 4: # 众数替换
# df.loc[lambda x:x[coloum] > q1 + 1.5 *
# iqr, coloum]=df[coloum].mode().iloc[0]
# df.loc[lambda x:x[coloum] < q3 - 1.5 *
# iqr, coloum]=df[coloum].mode().iloc[0]
# elif method == 5: # 边界替换
# df.loc[lambda x:x[coloum] > q1 +
# 1.5 * iqr, coloum]=q1 + 1.5 * iqr
# df.loc[lambda x:x[coloum] < q3 -
# 1.5 * iqr, coloum]=q3 - 1.5 * iqr
# elif detect == 2: # Z-score检测方式
# for coloum in df.columns:
# mean = df[coloum].mean()
# std = df[coloum].std()
# df.drop(
# df.loc[lambda x:x[coloum] > mean + 3 * std].index, inplace=True)
# df.drop(
# df.loc[lambda x:x[coloum] < mean - 3 * std].index, inplace=True)
# if method == 1: # 删除异常值
# df.drop(
# df.loc[lambda x:x[coloum] > mean + 3 * std].index, inplace=True)
# df.drop(
# df.loc[lambda x:x[coloum] < mean - 3 * std].index, inplace=True)
# elif method == 2: # 均值替换
# df.loc[lambda x:x[coloum] > mean +
# 3 * std, coloum]=df[coloum].mean()
# df.loc[lambda x:x[coloum] < mean -
# 3 * std, coloum]=df[coloum].mean()
# elif method == 3: # 中位数替换
# df.loc[lambda x:x[coloum] > mean + 3 *
# std, coloum]=df[coloum].median()
# df.loc[lambda x:x[coloum] < mean - 3 *
# std, coloum]=df[coloum].median()
# elif method == 4: # 众数替换
# df.loc[lambda x:x[coloum] > mean + 3 *
# std, coloum]=df[coloum].mode().iloc[0]
# df.loc[lambda x:x[coloum] < mean - 3 *
# std, coloum]=df[coloum].mode().iloc[0]
# elif method == 5: # 边界替换
# df.loc[lambda x:x[coloum] > mean +
# 3 * std, coloum]=mean + 3 * std
# df.loc[lambda x:x[coloum] < mean -
# 3 * std, coloum]=mean - 3 * std
# return df
def process_miss(df, method): # 缺失值处理
# 舍弃全为空的行
df = df.dropna(how='all')
# 舍弃全为空的列
df = df.dropna(axis=1, how='all')
if method == 1: # 均值
df = df.fillna(df.mean())
elif method == 2: # 中位数
df = df.fillna(df.median())
elif method == 3: # 众数
df = df.fillna(df.mode().iloc[0])
elif method == 4: # 线性
df = df.fillna(df.interpolate(
method='linear', limit_direction='forward', axis=0))
elif method == 5: # 前值
df = df.fillna(method="ffill")
elif method == 6: # 后值
df = df.fillna(method="bfill")
return df
def process_abnormal(df_inside, df_user, detect, method): # 异常值处理
df = pd.concat([df_inside, df_user], axis=0,
ignore_index=True) # 合并的dataframe
df_features = df.iloc[:, :12] # 取所有的特征列为dataframe
# print(df)
if detect == 1: # IQR检测方式
for coloum in df_features.columns:
q1 = df_features[coloum].quantile(0.75)
q3 = df_features[coloum].quantile(0.25)
iqr = q1-q3
if method == 1: # 删除异常值
df_features.drop(
df_features.loc[lambda x:x[coloum] > q1 + 1.5 * iqr].index, inplace=True)
df_features.drop(
df_features.loc[lambda x:x[coloum] < q3 - 1.5 * iqr].index, inplace=True)
elif method == 2: # 均值替换
df_features.loc[lambda x:x[coloum] > q1 + 1.5 *
iqr, coloum]=df_features[coloum].mean()
df_features.loc[lambda x:x[coloum] < q3 - 1.5 *
iqr, coloum]=df_features[coloum].mean()
elif method == 3: # 中位数替换
df_features.loc[lambda x:x[coloum] > q1 + 1.5 *
iqr, coloum]=df_features[coloum].median()
df_features.loc[lambda x:x[coloum] < q3 - 1.5 *
iqr, coloum]=df_features[coloum].median()
elif method == 4: # 众数替换
df_features.loc[lambda x:x[coloum] > q1 + 1.5 *
iqr, coloum]=df_features[coloum].mode().iloc[0]
df_features.loc[lambda x:x[coloum] < q3 - 1.5 *
iqr, coloum]=df_features[coloum].mode().iloc[0]
elif method == 5: # 边界替换
df_features.loc[lambda x:x[coloum] > q1 +
1.5 * iqr, coloum]=q1 + 1.5 * iqr
df_features.loc[lambda x:x[coloum] < q3 -
1.5 * iqr, coloum]=q3 - 1.5 * iqr
elif detect == 2: # Z-score检测方式
for coloum in df_features.columns:
mean = df_features[coloum].mean()
std = df_features[coloum].std()
df_features.drop(
df_features.loc[lambda x:x[coloum] > mean + 3 * std].index, inplace=True)
df_features.drop(
df_features.loc[lambda x:x[coloum] < mean - 3 * std].index, inplace=True)
if method == 1: # 删除异常值
df_features.drop(
df_features.loc[lambda x:x[coloum] > mean + 3 * std].index, inplace=True)
df_features.drop(
df_features.loc[lambda x:x[coloum] < mean - 3 * std].index, inplace=True)
elif method == 2: # 均值替换
df_features.loc[lambda x:x[coloum] > mean +
3 * std, coloum]=df_features[coloum].mean()
df_features.loc[lambda x:x[coloum] < mean -
3 * std, coloum]=df_features[coloum].mean()
elif method == 3: # 中位数替换
df_features.loc[lambda x:x[coloum] > mean + 3 *
std, coloum]=df_features[coloum].median()
df_features.loc[lambda x:x[coloum] < mean - 3 *
std, coloum]=df_features[coloum].median()
elif method == 4: # 众数替换
df_features.loc[lambda x:x[coloum] > mean + 3 *
std, coloum]=df_features[coloum].mode().iloc[0]
df_features.loc[lambda x:x[coloum] < mean - 3 *
std, coloum]=df_features[coloum].mode().iloc[0]
elif method == 5: # 边界替换
df_features.loc[lambda x:x[coloum] > mean +
3 * std, coloum]=mean + 3 * std
df_features.loc[lambda x:x[coloum] < mean -
3 * std, coloum]=mean - 3 * std
df.iloc[:, :12] = df_features
df_inside = df.iloc[:df_inside.shape[0], :]
df_user = df.iloc[df_inside.shape[0]:, :12]
return {"df_inside": df_inside, "df_user": df_user}
def process_standard(df_inside, df_user, method): # 标准化处理
df = pd.concat([df_inside, df_user], axis=0,
ignore_index=True) # 合并的dataframe
df_features = df.iloc[:, :12] # 取所有的特征列为dataframe
columns = df_features.columns # 列名
if method == 1: # Min-max
df_features = preproc.minmax_scale(df_features)
elif method == 2: # Z-Score
df_features = preproc.StandardScaler().fit_transform(df_features)
elif method == 3: # MaxAbs
df_features = preproc.maxabs_scale(df_features, axis=0)
elif method == 4: # RobustScaler
df_features = preproc.RobustScaler().fit_transform(df_features)
elif method == 5: # 正则化
df_features = preproc.normalize(df_features, axis=0)
df_features = pd.DataFrame(
data=df_features[0:, 0:], columns=columns) # 补充列名
df.iloc[:, :12] = df_features
df_inside = df.iloc[:df_inside.shape[0], :]
df_user = df.iloc[df_inside.shape[0]:, :12]
return {"df_inside": df_inside, "df_user": df_user}
def train_model(x, y, test_size, algorithm, paras): # 模型训练
# 划分数据集
x_train, x_test, y_train, y_test = train_test_split(
x, y, test_size=test_size, random_state=0)
# 机器学习
model = None
results = {}
if algorithm == 1: # 最小二乘法线性回归
model = LinearRegression(fit_intercept=paras["fit_intercept"])
if algorithm == 2: # 随机森林回归
model = RandomForestRegressor(n_estimators=paras["n_estimators"],
criterion=paras["criterion"], max_depth=paras["max_depth"], random_state=0)
if algorithm == 3: # BP神经网络回归
model = MLPRegressor(hidden_layer_sizes=(paras["hidden_layer_sizes_1"], paras["hidden_layer_sizes_2"]),
activation=paras["activation"], solver='lbfgs', random_state=paras["random_state"])
if algorithm == 4: # XGBoost回归
model = XGBRegressor(
max_depth=paras["max_depth"], learning_rate=paras["learning_rate"], n_estimators=paras["n_estimators"])
if algorithm == 5: # LightGBM回归
model = lgb.LGBMRegressor(objective='regression', max_depth=paras["max_depth"],
learning_rate=paras["learning_rate"], random_state=paras["random_state"], n_estimators=paras["n_estimators"])
# 返回数据
if model != None:
model.fit(x_train, y_train)
if algorithm == 1: # 最小二乘法线性回归
# 保留小数点后三位
# results["coef"] = model.coef_.tolist() # 模型斜率
results["coef"] = [float('{:.4f}'.format(i))
for i in model.coef_.tolist()] # 模型斜率
results["intercept"] = round(model.intercept_, 3) # 模型截距
y_pred = model.predict(x_test) # 预测值
# y_test = y_test.values
# 误差，用round保留三位小数且四舍五入
mae = round(mean_absolute_error(y_test, y_pred), 3)
rmse = round(np.sqrt(mean_squared_error(y_test, y_pred)), 3)
r2 = round(r2_score(y_test, y_pred), 3)
# y_test = [x[0] for x in np.array(y_test).tolist()]
# y_pred = [x[0] for x in y_pred.tolist()]
y_test = np.array(y_test).tolist()
y_pred = y_pred.tolist()
return {"y_test": y_test, "y_pred": y_pred, "error": {"MAE": mae, "RMSE": rmse, "R2": r2}, "results": results}
else:
return "模型训练出错"
def predict_connectivity(x, x1, y, test_size, algorithm, paras):
# 划分数据集
x_train, x_test, y_train, y_test = train_test_split(
x, y, test_size=test_size, random_state=0)
# 机器学习
model = None
results = {}
if algorithm == 1: # 最小二乘法线性回归
model = LinearRegression(fit_intercept=paras["fit_intercept"])
if algorithm == 2: # 随机森林回归
model = RandomForestRegressor(n_estimators=paras["n_estimators"],
criterion=paras["criterion"], max_depth=paras["max_depth"], random_state=0)
if algorithm == 3: # BP神经网络回归
model = MLPRegressor(hidden_layer_sizes=(paras["hidden_layer_sizes_1"], paras["hidden_layer_sizes_2"]),
activation=paras["activation"], solver='lbfgs', random_state=paras["random_state"])
if algorithm == 4: # XGBoost回归
model = XGBRegressor(
max_depth=paras["max_depth"], learning_rate=paras["learning_rate"], n_estimators=paras["n_estimators"])
if algorithm == 5: # LightGBM回归
model = lgb.LGBMRegressor(objective='regression', max_depth=paras["max_depth"],
learning_rate=paras["learning_rate"], random_state=paras["random_state"], n_estimators=paras["n_estimators"])
# 返回数据
if model != None:
model.fit(x_train, y_train)
y_pred = model.predict(x1).tolist() # 预测值
return y_pred
else:
return "预测连通性出错"
# 登录验证
class Login(BaseModel): # 接口数据类型
username: str
password: str
@app.post("/login") # 接口
async def login(login: Login):
username = login.username
password = login.password
if username == "admin" and password == "123456":
return True
return False
# 处理用户数据
class Process_user(BaseModel): # 接口数据类型
mode: int
data: List
miss: List
abnormal: List
standard: List
@app.post("/process/user") # 接口
async def process_user(user: Process_user):
mode = user.mode # 选择的井间连通模式
df_inside = pd.read_csv(
"./data/mode_{}.csv".format(mode)).dropna(axis=0) # 连通模式对应的内置数据
df_user = json2df(user.data)
abnormal = user.abnormal[0]
miss = user.miss[0]
standard = user.standard[0]
# 异常值处理
if abnormal["state"]:
abnormaled = process_abnormal(
df_inside, df_user, abnormal["detect"], abnormal["method"])
df_inside = abnormaled["df_inside"]
df_user = abnormaled["df_user"]
# 缺失值处理
if miss["state"]:
df_user = process_miss(df_user, miss["method"])
# 标准化处理
if standard["state"]:
standarded = process_standard(df_inside, df_user, standard["method"])
df_inside = standarded["df_inside"]
df_user = standarded["df_user"]
# 用astype将数值转科学计数法
return {"inside": df_inside.astype('str').to_json(orient='records'), "user": df_user.astype('str').to_json(orient='records')}
# # 用astype将数值转科学计数法
# return df.astype('str').to_json(orient='records')
# 处理内置数据
class Process_inside(BaseModel): # 接口数据类型
data: List
abnormal: List
standard: List
@app.post("/process/inside") # 接口
async def process_inside(inside: Process_inside):
df = json2df(inside.data)
abnormal = inside.abnormal[0]
standard = inside.standard[0]
# 异常值处理
if abnormal["state"]:
df = process_abnormal(df, abnormal["detect"], abnormal["method"])
# 标准化处理：只对特征进行标准化，不包括标签（后三列）
if standard["state"]:
df = pd.concat([process_standard(df.iloc[:, :12],
standard["method"]), df.iloc[:, 12:]], axis=1)
# 用astype将数值转科学计数法
return df.astype('str').to_json(orient='records')
# 训练模型
class Train(BaseModel): # 接口数据类型
data: List
test_size: float
algorithm: int
paras: List
@app.post("/train") # 接口
async def train(train: Train):
# 解析数据
df = json2df(train.data)
test_size = train.test_size
algorithm = train.algorithm
paras = train.paras[0]
x = df.iloc[:, :12]
y1 = df.loc[:, "BSR"]
y2 = df.loc[:, "SBR"]
y3 = df.loc[:, "D"]
bsr = train_model(x, y1, test_size, algorithm, paras)
sbr = train_model(x, y2, test_size, algorithm, paras)
x_train, x_test, y_train, y_test = train_test_split(
x, y3, test_size=test_size, random_state=0)
d = {"y_test": np.array(y_test).tolist(), "y_pred": np.sum(
[bsr["y_pred"], sbr["y_pred"]], axis=0).tolist()}
return {"bsr": bsr, "sbr": sbr, "d": d}
# 预测连通性
class Predict(BaseModel): # 接口数据类型
data_train: List
data_predict: List
test_size: float
algorithm: int
paras: List
@app.post("/predict") # 接口
async def predict(predict: Predict):
# 解析数据
df_train = json2df(predict.data_train)
df_predict = json2df(predict.data_predict)
test_size = predict.test_size
algorithm = predict.algorithm
paras = predict.paras[0]
x = df_train.iloc[:, :12]
y1 = df_train.loc[:, "BSR"]
y2 = df_train.loc[:, "SBR"]
# 预测连通性
bsr = predict_connectivity(x, df_predict, y1, test_size, algorithm, paras)
sbr = predict_connectivity(x, df_predict, y2, test_size, algorithm, paras)
d = np.sum([bsr, sbr], axis=0).tolist()
# 合并为一个list后转dataframe再转json实现前端表格数据格式
data = []
data.append(bsr)
data.append(sbr)
data.append(d)
df_result = pd.concat([pd.DataFrame(predict.data_predict), pd.DataFrame(data=np.array(
data).T.tolist(), columns=["BSR", "SBR", "D"])], axis=1)
return df_result.to_json(orient='records')
# return pd.DataFrame(data=np.array(data).T.tolist(), columns=["BSR", "SBR", "D"]).to_json(orient='records')
# # 图片测试
# def create_img():
# plt.rcParams['figure.figsize'] = [7.50, 3.50]
# plt.rcParams['figure.autolayout'] = True
# plt.plot([1, 2])
# img_buf = io.BytesIO()
# plt.savefig(img_buf, format='png')
# plt.close()
# return img_buf
# @app.get('/png')
# async def get_img(background_tasks: BackgroundTasks):
# img_buf = create_img()
# # get the entire buffer content
# # because of the async, this will await the loading of all content
# bufContents: bytes = img_buf.getvalue()
# background_tasks.add_task(img_buf.close)
# headers = {'Content-Disposition': 'inline; filename="out.png"'}
# return Response(bufContents, headers=headers, media_type='image/png')