Kould commited on
Commit
f4d1c72
·
1 Parent(s): 6858ec5

support S3 for file storage (#15)

Browse files

* feat: support S3 for file storage

* doc: fmt env template

Files changed (6) hide show
  1. .env +0 -3
  2. .env.template +7 -1
  3. Cargo.toml +1 -0
  4. src/api/doc_info.rs +40 -39
  5. src/errors.rs +32 -26
  6. src/main.rs +38 -19
.env DELETED
@@ -1,3 +0,0 @@
1
- HOST=127.0.0.1
2
- PORT=8000
3
- DATABASE_URL="postgresql://infiniflow:infiniflow@localhost/docgpt"
 
 
 
 
.env.template CHANGED
@@ -1,3 +1,9 @@
 
1
  HOST=127.0.0.1
2
  PORT=8000
3
- DATABASE_URL="postgresql://infiniflow:infiniflow@localhost/docgpt"
 
 
 
 
 
 
1
+ # Database
2
  HOST=127.0.0.1
3
  PORT=8000
4
+ DATABASE_URL="postgresql://infiniflow:infiniflow@localhost/docgpt"
5
+
6
+ # S3 Storage
7
+ S3_BASE_URL="https://play.min.io"
8
+ S3_ACCESS_KEY="Q3AM3UQ867SPQQA43P2F"
9
+ S3_SECRET_KEY="zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG"
Cargo.toml CHANGED
@@ -23,6 +23,7 @@ dotenvy = "0.15.7"
23
  listenfd = "1.0.1"
24
  chrono = "0.4.31"
25
  migration = { path = "./migration" }
 
26
  futures-util = "0.3.29"
27
  actix-multipart-extract = "0.1.5"
28
 
 
23
  listenfd = "1.0.1"
24
  chrono = "0.4.31"
25
  migration = { path = "./migration" }
26
+ minio = "0.1.0"
27
  futures-util = "0.3.29"
28
  actix-multipart-extract = "0.1.5"
29
 
src/api/doc_info.rs CHANGED
@@ -1,8 +1,9 @@
1
  use std::collections::HashMap;
2
  use std::io::Write;
3
- use actix_multipart_extract::{ File, Multipart, MultipartForm };
4
- use actix_web::{ HttpResponse, post, web };
5
- use chrono::{ Utc, FixedOffset };
 
6
  use sea_orm::DbConn;
7
  use crate::api::JsonResponse;
8
  use crate::AppState;
@@ -11,6 +12,9 @@ use crate::errors::AppError;
11
  use crate::service::doc_info::{ Mutation, Query };
12
  use serde::Deserialize;
13
 
 
 
 
14
  fn now() -> chrono::DateTime<FixedOffset> {
15
  Utc::now().with_timezone(&FixedOffset::east_opt(3600 * 8).unwrap())
16
  }
@@ -69,57 +73,54 @@ async fn upload(
69
  data: web::Data<AppState>
70
  ) -> Result<HttpResponse, AppError> {
71
  let uid = payload.uid;
72
- async fn add_number_to_filename(
73
- file_name: String,
74
- conn: &DbConn,
75
- uid: i64,
76
- parent_id: i64
77
- ) -> String {
78
  let mut i = 0;
79
  let mut new_file_name = file_name.to_string();
80
  let arr: Vec<&str> = file_name.split(".").collect();
81
- let suffix = String::from(arr[arr.len() - 1]);
82
- let preffix = arr[..arr.len() - 1].join(".");
83
- let mut docs = Query::find_doc_infos_by_name(
84
- conn,
85
- uid,
86
- &new_file_name,
87
- Some(parent_id)
88
- ).await.unwrap();
89
- while docs.len() > 0 {
90
  i += 1;
91
  new_file_name = format!("{}_{}.{}", preffix, i, suffix);
92
- docs = Query::find_doc_infos_by_name(
93
- conn,
94
- uid,
95
- &new_file_name,
96
- Some(parent_id)
97
- ).await.unwrap();
98
  }
99
  new_file_name
100
  }
101
- let fnm = add_number_to_filename(
102
- payload.file_field.name.clone(),
103
- &data.conn,
104
- uid,
105
- payload.did
106
- ).await;
107
-
108
- std::fs::create_dir_all(format!("./upload/{}/", uid));
109
- let filepath = format!("./upload/{}/{}-{}", payload.uid, payload.did, fnm.clone());
110
- let mut f = std::fs::File::create(&filepath)?;
111
- f.write(&payload.file_field.bytes)?;
112
 
 
 
 
 
 
 
 
 
 
 
 
113
  let doc = Mutation::create_doc_info(&data.conn, Model {
114
- did: Default::default(),
115
- uid: uid,
116
  doc_name: fnm,
117
  size: payload.file_field.bytes.len() as i64,
118
- location: filepath,
119
  r#type: "doc".to_string(),
120
  created_at: now(),
121
  updated_at: now(),
122
- is_deleted: Default::default(),
123
  }).await?;
124
 
125
  let _ = Mutation::place_doc(&data.conn, payload.did, doc.did.unwrap()).await?;
 
1
  use std::collections::HashMap;
2
  use std::io::Write;
3
+ use actix_multipart_extract::{File, Multipart, MultipartForm};
4
+ use actix_web::{get, HttpResponse, post, web};
5
+ use chrono::{Utc, FixedOffset};
6
+ use minio::s3::args::{BucketExistsArgs, MakeBucketArgs, UploadObjectArgs};
7
  use sea_orm::DbConn;
8
  use crate::api::JsonResponse;
9
  use crate::AppState;
 
12
  use crate::service::doc_info::{ Mutation, Query };
13
  use serde::Deserialize;
14
 
15
+ const BUCKET_NAME: &'static str = "docgpt-upload";
16
+
17
+
18
  fn now() -> chrono::DateTime<FixedOffset> {
19
  Utc::now().with_timezone(&FixedOffset::east_opt(3600 * 8).unwrap())
20
  }
 
73
  data: web::Data<AppState>
74
  ) -> Result<HttpResponse, AppError> {
75
  let uid = payload.uid;
76
+ let file_name = payload.file_field.name.as_str();
77
+ async fn add_number_to_filename(file_name: &str, conn:&DbConn, uid:i64, parent_id:i64) -> String {
 
 
 
 
78
  let mut i = 0;
79
  let mut new_file_name = file_name.to_string();
80
  let arr: Vec<&str> = file_name.split(".").collect();
81
+ let suffix = String::from(arr[arr.len()-1]);
82
+ let preffix = arr[..arr.len()-1].join(".");
83
+ let mut docs = Query::find_doc_infos_by_name(conn, uid, &new_file_name, Some(parent_id)).await.unwrap();
84
+ while docs.len()>0 {
 
 
 
 
 
85
  i += 1;
86
  new_file_name = format!("{}_{}.{}", preffix, i, suffix);
87
+ docs = Query::find_doc_infos_by_name(conn, uid, &new_file_name, Some(parent_id)).await.unwrap();
 
 
 
 
 
88
  }
89
  new_file_name
90
  }
91
+ let fnm = add_number_to_filename(file_name, &data.conn, uid, payload.did).await;
92
+
93
+ let s3_client = &data.s3_client;
94
+ let buckets_exists = s3_client
95
+ .bucket_exists(&BucketExistsArgs::new(BUCKET_NAME)?)
96
+ .await?;
97
+ if !buckets_exists {
98
+ s3_client
99
+ .make_bucket(&MakeBucketArgs::new(BUCKET_NAME)?)
100
+ .await?;
101
+ }
102
 
103
+ s3_client
104
+ .upload_object(
105
+ &mut UploadObjectArgs::new(
106
+ BUCKET_NAME,
107
+ fnm.as_str(),
108
+ format!("/{}/{}-{}", payload.uid, payload.did, fnm).as_str()
109
+ )?
110
+ )
111
+ .await?;
112
+
113
+ let location = format!("/{}/{}", BUCKET_NAME, fnm);
114
  let doc = Mutation::create_doc_info(&data.conn, Model {
115
+ did:Default::default(),
116
+ uid: uid,
117
  doc_name: fnm,
118
  size: payload.file_field.bytes.len() as i64,
119
+ location,
120
  r#type: "doc".to_string(),
121
  created_at: now(),
122
  updated_at: now(),
123
+ is_deleted:Default::default(),
124
  }).await?;
125
 
126
  let _ = Mutation::place_doc(&data.conn, payload.did, doc.did.unwrap()).await?;
src/errors.rs CHANGED
@@ -1,17 +1,25 @@
1
- use actix_web::{ HttpResponse, ResponseError };
2
  use thiserror::Error;
3
 
4
  #[derive(Debug, Error)]
5
  pub(crate) enum AppError {
6
- #[error("`{0}`")] User(#[from] UserError),
 
7
 
8
- #[error("`{0}`")] Json(#[from] serde_json::Error),
 
9
 
10
- #[error("`{0}`")] Actix(#[from] actix_web::Error),
 
11
 
12
- #[error("`{0}`")] Db(#[from] sea_orm::DbErr),
 
13
 
14
- #[error("`{0}`")] Std(#[from] std::io::Error),
 
 
 
 
15
  }
16
 
17
  #[derive(Debug, Error)]
@@ -28,7 +36,8 @@ pub(crate) enum UserError {
28
  #[error("`password` field of `User` cannot contain whitespaces!")]
29
  PasswordInvalidCharacter,
30
 
31
- #[error("Could not find any `User` for id: `{0}`!")] NotFound(i64),
 
32
 
33
  #[error("Failed to login user!")]
34
  LoginFailed,
@@ -46,26 +55,23 @@ pub(crate) enum UserError {
46
  impl ResponseError for AppError {
47
  fn status_code(&self) -> actix_web::http::StatusCode {
48
  match self {
49
- AppError::User(user_error) =>
50
- match user_error {
51
- UserError::EmptyUsername => actix_web::http::StatusCode::UNPROCESSABLE_ENTITY,
52
- UserError::UsernameInvalidCharacter => {
53
- actix_web::http::StatusCode::UNPROCESSABLE_ENTITY
54
- }
55
- UserError::EmptyPassword => actix_web::http::StatusCode::UNPROCESSABLE_ENTITY,
56
- UserError::PasswordInvalidCharacter => {
57
- actix_web::http::StatusCode::UNPROCESSABLE_ENTITY
58
- }
59
- UserError::NotFound(_) => actix_web::http::StatusCode::NOT_FOUND,
60
- UserError::NotLoggedIn => actix_web::http::StatusCode::UNAUTHORIZED,
61
- UserError::Empty => actix_web::http::StatusCode::NOT_FOUND,
62
- UserError::LoginFailed => actix_web::http::StatusCode::NOT_FOUND,
63
- UserError::InvalidToken => actix_web::http::StatusCode::UNAUTHORIZED,
64
  }
65
- AppError::Json(_) => actix_web::http::StatusCode::INTERNAL_SERVER_ERROR,
 
 
 
 
 
66
  AppError::Actix(fail) => fail.as_response_error().status_code(),
67
- AppError::Db(_) => actix_web::http::StatusCode::INTERNAL_SERVER_ERROR,
68
- AppError::Std(_) => actix_web::http::StatusCode::INTERNAL_SERVER_ERROR,
69
  }
70
  }
71
 
@@ -74,4 +80,4 @@ impl ResponseError for AppError {
74
  let response = HttpResponse::build(status_code).body(self.to_string());
75
  response
76
  }
77
- }
 
1
+ use actix_web::{HttpResponse, ResponseError};
2
  use thiserror::Error;
3
 
4
  #[derive(Debug, Error)]
5
  pub(crate) enum AppError {
6
+ #[error("`{0}`")]
7
+ User(#[from] UserError),
8
 
9
+ #[error("`{0}`")]
10
+ Json(#[from] serde_json::Error),
11
 
12
+ #[error("`{0}`")]
13
+ Actix(#[from] actix_web::Error),
14
 
15
+ #[error("`{0}`")]
16
+ Db(#[from] sea_orm::DbErr),
17
 
18
+ #[error("`{0}`")]
19
+ MinioS3(#[from] minio::s3::error::Error),
20
+
21
+ #[error("`{0}`")]
22
+ Std(#[from] std::io::Error),
23
  }
24
 
25
  #[derive(Debug, Error)]
 
36
  #[error("`password` field of `User` cannot contain whitespaces!")]
37
  PasswordInvalidCharacter,
38
 
39
+ #[error("Could not find any `User` for id: `{0}`!")]
40
+ NotFound(i64),
41
 
42
  #[error("Failed to login user!")]
43
  LoginFailed,
 
55
  impl ResponseError for AppError {
56
  fn status_code(&self) -> actix_web::http::StatusCode {
57
  match self {
58
+ AppError::User(user_error) => match user_error {
59
+ UserError::EmptyUsername => actix_web::http::StatusCode::UNPROCESSABLE_ENTITY,
60
+ UserError::UsernameInvalidCharacter => {
61
+ actix_web::http::StatusCode::UNPROCESSABLE_ENTITY
62
+ }
63
+ UserError::EmptyPassword => actix_web::http::StatusCode::UNPROCESSABLE_ENTITY,
64
+ UserError::PasswordInvalidCharacter => {
65
+ actix_web::http::StatusCode::UNPROCESSABLE_ENTITY
 
 
 
 
 
 
 
66
  }
67
+ UserError::NotFound(_) => actix_web::http::StatusCode::NOT_FOUND,
68
+ UserError::NotLoggedIn => actix_web::http::StatusCode::UNAUTHORIZED,
69
+ UserError::Empty => actix_web::http::StatusCode::NOT_FOUND,
70
+ UserError::LoginFailed => actix_web::http::StatusCode::NOT_FOUND,
71
+ UserError::InvalidToken => actix_web::http::StatusCode::UNAUTHORIZED,
72
+ },
73
  AppError::Actix(fail) => fail.as_response_error().status_code(),
74
+ _ => actix_web::http::StatusCode::INTERNAL_SERVER_ERROR,
 
75
  }
76
  }
77
 
 
80
  let response = HttpResponse::build(status_code).body(self.to_string());
81
  response
82
  }
83
+ }
src/main.rs CHANGED
@@ -5,29 +5,33 @@ mod errors;
5
 
6
  use std::env;
7
  use actix_files::Files;
8
- use actix_identity::{ CookieIdentityPolicy, IdentityService, RequestIdentity };
9
  use actix_session::CookieSession;
10
- use actix_web::{ web, App, HttpServer, middleware, Error };
11
  use actix_web::cookie::time::Duration;
12
  use actix_web::dev::ServiceRequest;
13
  use actix_web::error::ErrorUnauthorized;
14
  use actix_web_httpauth::extractors::bearer::BearerAuth;
15
  use listenfd::ListenFd;
16
- use sea_orm::{ Database, DatabaseConnection };
17
- use migration::{ Migrator, MigratorTrait };
18
- use crate::errors::UserError;
 
 
 
19
 
20
  #[derive(Debug, Clone)]
21
  struct AppState {
22
  conn: DatabaseConnection,
 
23
  }
24
 
25
  pub(crate) async fn validator(
26
  req: ServiceRequest,
27
- credentials: BearerAuth
28
  ) -> Result<ServiceRequest, Error> {
29
  if let Some(token) = req.get_identity() {
30
- println!("{}, {}", credentials.token(), token);
31
  (credentials.token() == token)
32
  .then(|| req)
33
  .ok_or(ErrorUnauthorized(UserError::InvalidToken))
@@ -37,7 +41,7 @@ pub(crate) async fn validator(
37
  }
38
 
39
  #[actix_web::main]
40
- async fn main() -> std::io::Result<()> {
41
  std::env::set_var("RUST_LOG", "debug");
42
  tracing_subscriber::fmt::init();
43
 
@@ -48,12 +52,29 @@ async fn main() -> std::io::Result<()> {
48
  let port = env::var("PORT").expect("PORT is not set in .env file");
49
  let server_url = format!("{host}:{port}");
50
 
 
 
 
 
51
  // establish connection to database and apply migrations
52
  // -> create post table if not exists
53
  let conn = Database::connect(&db_url).await.unwrap();
54
  Migrator::up(&conn, None).await.unwrap();
55
 
56
- let state = AppState { conn };
 
 
 
 
 
 
 
 
 
 
 
 
 
57
 
58
  // create server and try to serve over socket if possible
59
  let mut listenfd = ListenFd::from_env();
@@ -61,20 +82,18 @@ async fn main() -> std::io::Result<()> {
61
  App::new()
62
  .service(Files::new("/static", "./static"))
63
  .app_data(web::Data::new(state.clone()))
64
- .wrap(
65
- IdentityService::new(
66
- CookieIdentityPolicy::new(&[0; 32])
67
- .name("auth-cookie")
68
- .login_deadline(Duration::seconds(120))
69
- .secure(false)
70
- )
71
- )
72
  .wrap(
73
  CookieSession::signed(&[0; 32])
74
  .name("session-cookie")
75
  .secure(false)
76
  // WARNING(alex): This uses the `time` crate, not `std::time`!
77
- .expires_in_time(Duration::seconds(60))
78
  )
79
  .wrap(middleware::Logger::default())
80
  .configure(init)
@@ -118,4 +137,4 @@ fn init(cfg: &mut web::ServiceConfig) {
118
  cfg.service(api::user_info::login);
119
  cfg.service(api::user_info::register);
120
  cfg.service(api::user_info::setting);
121
- }
 
5
 
6
  use std::env;
7
  use actix_files::Files;
8
+ use actix_identity::{CookieIdentityPolicy, IdentityService, RequestIdentity};
9
  use actix_session::CookieSession;
10
+ use actix_web::{web, App, HttpServer, middleware, Error};
11
  use actix_web::cookie::time::Duration;
12
  use actix_web::dev::ServiceRequest;
13
  use actix_web::error::ErrorUnauthorized;
14
  use actix_web_httpauth::extractors::bearer::BearerAuth;
15
  use listenfd::ListenFd;
16
+ use minio::s3::client::Client;
17
+ use minio::s3::creds::StaticProvider;
18
+ use minio::s3::http::BaseUrl;
19
+ use sea_orm::{Database, DatabaseConnection};
20
+ use migration::{Migrator, MigratorTrait};
21
+ use crate::errors::{AppError, UserError};
22
 
23
  #[derive(Debug, Clone)]
24
  struct AppState {
25
  conn: DatabaseConnection,
26
+ s3_client: Client,
27
  }
28
 
29
  pub(crate) async fn validator(
30
  req: ServiceRequest,
31
+ credentials: BearerAuth,
32
  ) -> Result<ServiceRequest, Error> {
33
  if let Some(token) = req.get_identity() {
34
+ println!("{}, {}",credentials.token(), token);
35
  (credentials.token() == token)
36
  .then(|| req)
37
  .ok_or(ErrorUnauthorized(UserError::InvalidToken))
 
41
  }
42
 
43
  #[actix_web::main]
44
+ async fn main() -> Result<(), AppError> {
45
  std::env::set_var("RUST_LOG", "debug");
46
  tracing_subscriber::fmt::init();
47
 
 
52
  let port = env::var("PORT").expect("PORT is not set in .env file");
53
  let server_url = format!("{host}:{port}");
54
 
55
+ let s3_base_url = env::var("S3_BASE_URL").expect("S3_BASE_URL is not set in .env file");
56
+ let s3_access_key = env::var("S3_ACCESS_KEY").expect("S3_ACCESS_KEY is not set in .env file");;
57
+ let s3_secret_key = env::var("S3_SECRET_KEY").expect("S3_SECRET_KEY is not set in .env file");;
58
+
59
  // establish connection to database and apply migrations
60
  // -> create post table if not exists
61
  let conn = Database::connect(&db_url).await.unwrap();
62
  Migrator::up(&conn, None).await.unwrap();
63
 
64
+ let static_provider = StaticProvider::new(
65
+ s3_access_key.as_str(),
66
+ s3_secret_key.as_str(),
67
+ None,
68
+ );
69
+
70
+ let s3_client = Client::new(
71
+ s3_base_url.parse::<BaseUrl>()?,
72
+ Some(Box::new(static_provider)),
73
+ None,
74
+ None,
75
+ )?;
76
+
77
+ let state = AppState { conn, s3_client };
78
 
79
  // create server and try to serve over socket if possible
80
  let mut listenfd = ListenFd::from_env();
 
82
  App::new()
83
  .service(Files::new("/static", "./static"))
84
  .app_data(web::Data::new(state.clone()))
85
+ .wrap(IdentityService::new(
86
+ CookieIdentityPolicy::new(&[0; 32])
87
+ .name("auth-cookie")
88
+ .login_deadline(Duration::seconds(120))
89
+ .secure(false),
90
+ ))
 
 
91
  .wrap(
92
  CookieSession::signed(&[0; 32])
93
  .name("session-cookie")
94
  .secure(false)
95
  // WARNING(alex): This uses the `time` crate, not `std::time`!
96
+ .expires_in_time(Duration::seconds(60)),
97
  )
98
  .wrap(middleware::Logger::default())
99
  .configure(init)
 
137
  cfg.service(api::user_info::login);
138
  cfg.service(api::user_info::register);
139
  cfg.service(api::user_info::setting);
140
+ }