Kould
commited on
Commit
·
f4d1c72
1
Parent(s):
6858ec5
support S3 for file storage (#15)
Browse files* feat: support S3 for file storage
* doc: fmt env template
- .env +0 -3
- .env.template +7 -1
- Cargo.toml +1 -0
- src/api/doc_info.rs +40 -39
- src/errors.rs +32 -26
- src/main.rs +38 -19
.env
DELETED
@@ -1,3 +0,0 @@
|
|
1 |
-
HOST=127.0.0.1
|
2 |
-
PORT=8000
|
3 |
-
DATABASE_URL="postgresql://infiniflow:infiniflow@localhost/docgpt"
|
|
|
|
|
|
|
|
.env.template
CHANGED
@@ -1,3 +1,9 @@
|
|
|
|
1 |
HOST=127.0.0.1
|
2 |
PORT=8000
|
3 |
-
DATABASE_URL="postgresql://infiniflow:infiniflow@localhost/docgpt"
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
# Database
|
2 |
HOST=127.0.0.1
|
3 |
PORT=8000
|
4 |
+
DATABASE_URL="postgresql://infiniflow:infiniflow@localhost/docgpt"
|
5 |
+
|
6 |
+
# S3 Storage
|
7 |
+
S3_BASE_URL="https://play.min.io"
|
8 |
+
S3_ACCESS_KEY="Q3AM3UQ867SPQQA43P2F"
|
9 |
+
S3_SECRET_KEY="zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG"
|
Cargo.toml
CHANGED
@@ -23,6 +23,7 @@ dotenvy = "0.15.7"
|
|
23 |
listenfd = "1.0.1"
|
24 |
chrono = "0.4.31"
|
25 |
migration = { path = "./migration" }
|
|
|
26 |
futures-util = "0.3.29"
|
27 |
actix-multipart-extract = "0.1.5"
|
28 |
|
|
|
23 |
listenfd = "1.0.1"
|
24 |
chrono = "0.4.31"
|
25 |
migration = { path = "./migration" }
|
26 |
+
minio = "0.1.0"
|
27 |
futures-util = "0.3.29"
|
28 |
actix-multipart-extract = "0.1.5"
|
29 |
|
src/api/doc_info.rs
CHANGED
@@ -1,8 +1,9 @@
|
|
1 |
use std::collections::HashMap;
|
2 |
use std::io::Write;
|
3 |
-
use actix_multipart_extract::{
|
4 |
-
use actix_web::{ HttpResponse, post, web
|
5 |
-
use chrono::{
|
|
|
6 |
use sea_orm::DbConn;
|
7 |
use crate::api::JsonResponse;
|
8 |
use crate::AppState;
|
@@ -11,6 +12,9 @@ use crate::errors::AppError;
|
|
11 |
use crate::service::doc_info::{ Mutation, Query };
|
12 |
use serde::Deserialize;
|
13 |
|
|
|
|
|
|
|
14 |
fn now() -> chrono::DateTime<FixedOffset> {
|
15 |
Utc::now().with_timezone(&FixedOffset::east_opt(3600 * 8).unwrap())
|
16 |
}
|
@@ -69,57 +73,54 @@ async fn upload(
|
|
69 |
data: web::Data<AppState>
|
70 |
) -> Result<HttpResponse, AppError> {
|
71 |
let uid = payload.uid;
|
72 |
-
|
73 |
-
|
74 |
-
conn: &DbConn,
|
75 |
-
uid: i64,
|
76 |
-
parent_id: i64
|
77 |
-
) -> String {
|
78 |
let mut i = 0;
|
79 |
let mut new_file_name = file_name.to_string();
|
80 |
let arr: Vec<&str> = file_name.split(".").collect();
|
81 |
-
let suffix = String::from(arr[arr.len()
|
82 |
-
let preffix = arr[..arr.len()
|
83 |
-
let mut docs = Query::find_doc_infos_by_name(
|
84 |
-
|
85 |
-
uid,
|
86 |
-
&new_file_name,
|
87 |
-
Some(parent_id)
|
88 |
-
).await.unwrap();
|
89 |
-
while docs.len() > 0 {
|
90 |
i += 1;
|
91 |
new_file_name = format!("{}_{}.{}", preffix, i, suffix);
|
92 |
-
docs = Query::find_doc_infos_by_name(
|
93 |
-
conn,
|
94 |
-
uid,
|
95 |
-
&new_file_name,
|
96 |
-
Some(parent_id)
|
97 |
-
).await.unwrap();
|
98 |
}
|
99 |
new_file_name
|
100 |
}
|
101 |
-
let fnm = add_number_to_filename(
|
102 |
-
|
103 |
-
|
104 |
-
|
105 |
-
|
106 |
-
|
107 |
-
|
108 |
-
|
109 |
-
|
110 |
-
|
111 |
-
|
112 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
113 |
let doc = Mutation::create_doc_info(&data.conn, Model {
|
114 |
-
did:
|
115 |
-
uid:
|
116 |
doc_name: fnm,
|
117 |
size: payload.file_field.bytes.len() as i64,
|
118 |
-
location
|
119 |
r#type: "doc".to_string(),
|
120 |
created_at: now(),
|
121 |
updated_at: now(),
|
122 |
-
is_deleted:
|
123 |
}).await?;
|
124 |
|
125 |
let _ = Mutation::place_doc(&data.conn, payload.did, doc.did.unwrap()).await?;
|
|
|
1 |
use std::collections::HashMap;
|
2 |
use std::io::Write;
|
3 |
+
use actix_multipart_extract::{File, Multipart, MultipartForm};
|
4 |
+
use actix_web::{get, HttpResponse, post, web};
|
5 |
+
use chrono::{Utc, FixedOffset};
|
6 |
+
use minio::s3::args::{BucketExistsArgs, MakeBucketArgs, UploadObjectArgs};
|
7 |
use sea_orm::DbConn;
|
8 |
use crate::api::JsonResponse;
|
9 |
use crate::AppState;
|
|
|
12 |
use crate::service::doc_info::{ Mutation, Query };
|
13 |
use serde::Deserialize;
|
14 |
|
15 |
+
const BUCKET_NAME: &'static str = "docgpt-upload";
|
16 |
+
|
17 |
+
|
18 |
fn now() -> chrono::DateTime<FixedOffset> {
|
19 |
Utc::now().with_timezone(&FixedOffset::east_opt(3600 * 8).unwrap())
|
20 |
}
|
|
|
73 |
data: web::Data<AppState>
|
74 |
) -> Result<HttpResponse, AppError> {
|
75 |
let uid = payload.uid;
|
76 |
+
let file_name = payload.file_field.name.as_str();
|
77 |
+
async fn add_number_to_filename(file_name: &str, conn:&DbConn, uid:i64, parent_id:i64) -> String {
|
|
|
|
|
|
|
|
|
78 |
let mut i = 0;
|
79 |
let mut new_file_name = file_name.to_string();
|
80 |
let arr: Vec<&str> = file_name.split(".").collect();
|
81 |
+
let suffix = String::from(arr[arr.len()-1]);
|
82 |
+
let preffix = arr[..arr.len()-1].join(".");
|
83 |
+
let mut docs = Query::find_doc_infos_by_name(conn, uid, &new_file_name, Some(parent_id)).await.unwrap();
|
84 |
+
while docs.len()>0 {
|
|
|
|
|
|
|
|
|
|
|
85 |
i += 1;
|
86 |
new_file_name = format!("{}_{}.{}", preffix, i, suffix);
|
87 |
+
docs = Query::find_doc_infos_by_name(conn, uid, &new_file_name, Some(parent_id)).await.unwrap();
|
|
|
|
|
|
|
|
|
|
|
88 |
}
|
89 |
new_file_name
|
90 |
}
|
91 |
+
let fnm = add_number_to_filename(file_name, &data.conn, uid, payload.did).await;
|
92 |
+
|
93 |
+
let s3_client = &data.s3_client;
|
94 |
+
let buckets_exists = s3_client
|
95 |
+
.bucket_exists(&BucketExistsArgs::new(BUCKET_NAME)?)
|
96 |
+
.await?;
|
97 |
+
if !buckets_exists {
|
98 |
+
s3_client
|
99 |
+
.make_bucket(&MakeBucketArgs::new(BUCKET_NAME)?)
|
100 |
+
.await?;
|
101 |
+
}
|
102 |
|
103 |
+
s3_client
|
104 |
+
.upload_object(
|
105 |
+
&mut UploadObjectArgs::new(
|
106 |
+
BUCKET_NAME,
|
107 |
+
fnm.as_str(),
|
108 |
+
format!("/{}/{}-{}", payload.uid, payload.did, fnm).as_str()
|
109 |
+
)?
|
110 |
+
)
|
111 |
+
.await?;
|
112 |
+
|
113 |
+
let location = format!("/{}/{}", BUCKET_NAME, fnm);
|
114 |
let doc = Mutation::create_doc_info(&data.conn, Model {
|
115 |
+
did:Default::default(),
|
116 |
+
uid: uid,
|
117 |
doc_name: fnm,
|
118 |
size: payload.file_field.bytes.len() as i64,
|
119 |
+
location,
|
120 |
r#type: "doc".to_string(),
|
121 |
created_at: now(),
|
122 |
updated_at: now(),
|
123 |
+
is_deleted:Default::default(),
|
124 |
}).await?;
|
125 |
|
126 |
let _ = Mutation::place_doc(&data.conn, payload.did, doc.did.unwrap()).await?;
|
src/errors.rs
CHANGED
@@ -1,17 +1,25 @@
|
|
1 |
-
use actix_web::{
|
2 |
use thiserror::Error;
|
3 |
|
4 |
#[derive(Debug, Error)]
|
5 |
pub(crate) enum AppError {
|
6 |
-
#[error("`{0}`")]
|
|
|
7 |
|
8 |
-
#[error("`{0}`")]
|
|
|
9 |
|
10 |
-
#[error("`{0}`")]
|
|
|
11 |
|
12 |
-
#[error("`{0}`")]
|
|
|
13 |
|
14 |
-
#[error("`{0}`")]
|
|
|
|
|
|
|
|
|
15 |
}
|
16 |
|
17 |
#[derive(Debug, Error)]
|
@@ -28,7 +36,8 @@ pub(crate) enum UserError {
|
|
28 |
#[error("`password` field of `User` cannot contain whitespaces!")]
|
29 |
PasswordInvalidCharacter,
|
30 |
|
31 |
-
#[error("Could not find any `User` for id: `{0}`!")]
|
|
|
32 |
|
33 |
#[error("Failed to login user!")]
|
34 |
LoginFailed,
|
@@ -46,26 +55,23 @@ pub(crate) enum UserError {
|
|
46 |
impl ResponseError for AppError {
|
47 |
fn status_code(&self) -> actix_web::http::StatusCode {
|
48 |
match self {
|
49 |
-
AppError::User(user_error) =>
|
50 |
-
|
51 |
-
|
52 |
-
|
53 |
-
|
54 |
-
|
55 |
-
|
56 |
-
|
57 |
-
actix_web::http::StatusCode::UNPROCESSABLE_ENTITY
|
58 |
-
}
|
59 |
-
UserError::NotFound(_) => actix_web::http::StatusCode::NOT_FOUND,
|
60 |
-
UserError::NotLoggedIn => actix_web::http::StatusCode::UNAUTHORIZED,
|
61 |
-
UserError::Empty => actix_web::http::StatusCode::NOT_FOUND,
|
62 |
-
UserError::LoginFailed => actix_web::http::StatusCode::NOT_FOUND,
|
63 |
-
UserError::InvalidToken => actix_web::http::StatusCode::UNAUTHORIZED,
|
64 |
}
|
65 |
-
|
|
|
|
|
|
|
|
|
|
|
66 |
AppError::Actix(fail) => fail.as_response_error().status_code(),
|
67 |
-
|
68 |
-
AppError::Std(_) => actix_web::http::StatusCode::INTERNAL_SERVER_ERROR,
|
69 |
}
|
70 |
}
|
71 |
|
@@ -74,4 +80,4 @@ impl ResponseError for AppError {
|
|
74 |
let response = HttpResponse::build(status_code).body(self.to_string());
|
75 |
response
|
76 |
}
|
77 |
-
}
|
|
|
1 |
+
use actix_web::{HttpResponse, ResponseError};
|
2 |
use thiserror::Error;
|
3 |
|
4 |
#[derive(Debug, Error)]
|
5 |
pub(crate) enum AppError {
|
6 |
+
#[error("`{0}`")]
|
7 |
+
User(#[from] UserError),
|
8 |
|
9 |
+
#[error("`{0}`")]
|
10 |
+
Json(#[from] serde_json::Error),
|
11 |
|
12 |
+
#[error("`{0}`")]
|
13 |
+
Actix(#[from] actix_web::Error),
|
14 |
|
15 |
+
#[error("`{0}`")]
|
16 |
+
Db(#[from] sea_orm::DbErr),
|
17 |
|
18 |
+
#[error("`{0}`")]
|
19 |
+
MinioS3(#[from] minio::s3::error::Error),
|
20 |
+
|
21 |
+
#[error("`{0}`")]
|
22 |
+
Std(#[from] std::io::Error),
|
23 |
}
|
24 |
|
25 |
#[derive(Debug, Error)]
|
|
|
36 |
#[error("`password` field of `User` cannot contain whitespaces!")]
|
37 |
PasswordInvalidCharacter,
|
38 |
|
39 |
+
#[error("Could not find any `User` for id: `{0}`!")]
|
40 |
+
NotFound(i64),
|
41 |
|
42 |
#[error("Failed to login user!")]
|
43 |
LoginFailed,
|
|
|
55 |
impl ResponseError for AppError {
|
56 |
fn status_code(&self) -> actix_web::http::StatusCode {
|
57 |
match self {
|
58 |
+
AppError::User(user_error) => match user_error {
|
59 |
+
UserError::EmptyUsername => actix_web::http::StatusCode::UNPROCESSABLE_ENTITY,
|
60 |
+
UserError::UsernameInvalidCharacter => {
|
61 |
+
actix_web::http::StatusCode::UNPROCESSABLE_ENTITY
|
62 |
+
}
|
63 |
+
UserError::EmptyPassword => actix_web::http::StatusCode::UNPROCESSABLE_ENTITY,
|
64 |
+
UserError::PasswordInvalidCharacter => {
|
65 |
+
actix_web::http::StatusCode::UNPROCESSABLE_ENTITY
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
66 |
}
|
67 |
+
UserError::NotFound(_) => actix_web::http::StatusCode::NOT_FOUND,
|
68 |
+
UserError::NotLoggedIn => actix_web::http::StatusCode::UNAUTHORIZED,
|
69 |
+
UserError::Empty => actix_web::http::StatusCode::NOT_FOUND,
|
70 |
+
UserError::LoginFailed => actix_web::http::StatusCode::NOT_FOUND,
|
71 |
+
UserError::InvalidToken => actix_web::http::StatusCode::UNAUTHORIZED,
|
72 |
+
},
|
73 |
AppError::Actix(fail) => fail.as_response_error().status_code(),
|
74 |
+
_ => actix_web::http::StatusCode::INTERNAL_SERVER_ERROR,
|
|
|
75 |
}
|
76 |
}
|
77 |
|
|
|
80 |
let response = HttpResponse::build(status_code).body(self.to_string());
|
81 |
response
|
82 |
}
|
83 |
+
}
|
src/main.rs
CHANGED
@@ -5,29 +5,33 @@ mod errors;
|
|
5 |
|
6 |
use std::env;
|
7 |
use actix_files::Files;
|
8 |
-
use actix_identity::{
|
9 |
use actix_session::CookieSession;
|
10 |
-
use actix_web::{
|
11 |
use actix_web::cookie::time::Duration;
|
12 |
use actix_web::dev::ServiceRequest;
|
13 |
use actix_web::error::ErrorUnauthorized;
|
14 |
use actix_web_httpauth::extractors::bearer::BearerAuth;
|
15 |
use listenfd::ListenFd;
|
16 |
-
use
|
17 |
-
use
|
18 |
-
use
|
|
|
|
|
|
|
19 |
|
20 |
#[derive(Debug, Clone)]
|
21 |
struct AppState {
|
22 |
conn: DatabaseConnection,
|
|
|
23 |
}
|
24 |
|
25 |
pub(crate) async fn validator(
|
26 |
req: ServiceRequest,
|
27 |
-
credentials: BearerAuth
|
28 |
) -> Result<ServiceRequest, Error> {
|
29 |
if let Some(token) = req.get_identity() {
|
30 |
-
println!("{}, {}",
|
31 |
(credentials.token() == token)
|
32 |
.then(|| req)
|
33 |
.ok_or(ErrorUnauthorized(UserError::InvalidToken))
|
@@ -37,7 +41,7 @@ pub(crate) async fn validator(
|
|
37 |
}
|
38 |
|
39 |
#[actix_web::main]
|
40 |
-
async fn main() ->
|
41 |
std::env::set_var("RUST_LOG", "debug");
|
42 |
tracing_subscriber::fmt::init();
|
43 |
|
@@ -48,12 +52,29 @@ async fn main() -> std::io::Result<()> {
|
|
48 |
let port = env::var("PORT").expect("PORT is not set in .env file");
|
49 |
let server_url = format!("{host}:{port}");
|
50 |
|
|
|
|
|
|
|
|
|
51 |
// establish connection to database and apply migrations
|
52 |
// -> create post table if not exists
|
53 |
let conn = Database::connect(&db_url).await.unwrap();
|
54 |
Migrator::up(&conn, None).await.unwrap();
|
55 |
|
56 |
-
let
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
57 |
|
58 |
// create server and try to serve over socket if possible
|
59 |
let mut listenfd = ListenFd::from_env();
|
@@ -61,20 +82,18 @@ async fn main() -> std::io::Result<()> {
|
|
61 |
App::new()
|
62 |
.service(Files::new("/static", "./static"))
|
63 |
.app_data(web::Data::new(state.clone()))
|
64 |
-
.wrap(
|
65 |
-
|
66 |
-
|
67 |
-
|
68 |
-
|
69 |
-
|
70 |
-
)
|
71 |
-
)
|
72 |
.wrap(
|
73 |
CookieSession::signed(&[0; 32])
|
74 |
.name("session-cookie")
|
75 |
.secure(false)
|
76 |
// WARNING(alex): This uses the `time` crate, not `std::time`!
|
77 |
-
.expires_in_time(Duration::seconds(60))
|
78 |
)
|
79 |
.wrap(middleware::Logger::default())
|
80 |
.configure(init)
|
@@ -118,4 +137,4 @@ fn init(cfg: &mut web::ServiceConfig) {
|
|
118 |
cfg.service(api::user_info::login);
|
119 |
cfg.service(api::user_info::register);
|
120 |
cfg.service(api::user_info::setting);
|
121 |
-
}
|
|
|
5 |
|
6 |
use std::env;
|
7 |
use actix_files::Files;
|
8 |
+
use actix_identity::{CookieIdentityPolicy, IdentityService, RequestIdentity};
|
9 |
use actix_session::CookieSession;
|
10 |
+
use actix_web::{web, App, HttpServer, middleware, Error};
|
11 |
use actix_web::cookie::time::Duration;
|
12 |
use actix_web::dev::ServiceRequest;
|
13 |
use actix_web::error::ErrorUnauthorized;
|
14 |
use actix_web_httpauth::extractors::bearer::BearerAuth;
|
15 |
use listenfd::ListenFd;
|
16 |
+
use minio::s3::client::Client;
|
17 |
+
use minio::s3::creds::StaticProvider;
|
18 |
+
use minio::s3::http::BaseUrl;
|
19 |
+
use sea_orm::{Database, DatabaseConnection};
|
20 |
+
use migration::{Migrator, MigratorTrait};
|
21 |
+
use crate::errors::{AppError, UserError};
|
22 |
|
23 |
#[derive(Debug, Clone)]
|
24 |
struct AppState {
|
25 |
conn: DatabaseConnection,
|
26 |
+
s3_client: Client,
|
27 |
}
|
28 |
|
29 |
pub(crate) async fn validator(
|
30 |
req: ServiceRequest,
|
31 |
+
credentials: BearerAuth,
|
32 |
) -> Result<ServiceRequest, Error> {
|
33 |
if let Some(token) = req.get_identity() {
|
34 |
+
println!("{}, {}",credentials.token(), token);
|
35 |
(credentials.token() == token)
|
36 |
.then(|| req)
|
37 |
.ok_or(ErrorUnauthorized(UserError::InvalidToken))
|
|
|
41 |
}
|
42 |
|
43 |
#[actix_web::main]
|
44 |
+
async fn main() -> Result<(), AppError> {
|
45 |
std::env::set_var("RUST_LOG", "debug");
|
46 |
tracing_subscriber::fmt::init();
|
47 |
|
|
|
52 |
let port = env::var("PORT").expect("PORT is not set in .env file");
|
53 |
let server_url = format!("{host}:{port}");
|
54 |
|
55 |
+
let s3_base_url = env::var("S3_BASE_URL").expect("S3_BASE_URL is not set in .env file");
|
56 |
+
let s3_access_key = env::var("S3_ACCESS_KEY").expect("S3_ACCESS_KEY is not set in .env file");;
|
57 |
+
let s3_secret_key = env::var("S3_SECRET_KEY").expect("S3_SECRET_KEY is not set in .env file");;
|
58 |
+
|
59 |
// establish connection to database and apply migrations
|
60 |
// -> create post table if not exists
|
61 |
let conn = Database::connect(&db_url).await.unwrap();
|
62 |
Migrator::up(&conn, None).await.unwrap();
|
63 |
|
64 |
+
let static_provider = StaticProvider::new(
|
65 |
+
s3_access_key.as_str(),
|
66 |
+
s3_secret_key.as_str(),
|
67 |
+
None,
|
68 |
+
);
|
69 |
+
|
70 |
+
let s3_client = Client::new(
|
71 |
+
s3_base_url.parse::<BaseUrl>()?,
|
72 |
+
Some(Box::new(static_provider)),
|
73 |
+
None,
|
74 |
+
None,
|
75 |
+
)?;
|
76 |
+
|
77 |
+
let state = AppState { conn, s3_client };
|
78 |
|
79 |
// create server and try to serve over socket if possible
|
80 |
let mut listenfd = ListenFd::from_env();
|
|
|
82 |
App::new()
|
83 |
.service(Files::new("/static", "./static"))
|
84 |
.app_data(web::Data::new(state.clone()))
|
85 |
+
.wrap(IdentityService::new(
|
86 |
+
CookieIdentityPolicy::new(&[0; 32])
|
87 |
+
.name("auth-cookie")
|
88 |
+
.login_deadline(Duration::seconds(120))
|
89 |
+
.secure(false),
|
90 |
+
))
|
|
|
|
|
91 |
.wrap(
|
92 |
CookieSession::signed(&[0; 32])
|
93 |
.name("session-cookie")
|
94 |
.secure(false)
|
95 |
// WARNING(alex): This uses the `time` crate, not `std::time`!
|
96 |
+
.expires_in_time(Duration::seconds(60)),
|
97 |
)
|
98 |
.wrap(middleware::Logger::default())
|
99 |
.configure(init)
|
|
|
137 |
cfg.service(api::user_info::login);
|
138 |
cfg.service(api::user_info::register);
|
139 |
cfg.service(api::user_info::setting);
|
140 |
+
}
|