feat: File api (#70)

* feat: s3 minio wip

* feat: s3 minio bucket create idempotent

* feat: put storage setting into configurations

* chore: clippy lint

* feat: add setting to base config

* feat: add configuration for prod

* fix: allow use different minio host

* feat: add server file storage

* fix: add missing file

* feat: add code template

* feat: add http api

* feat: added file upload to client api

* feat: database metadata impl

* fix: added missing files

* feat: added test cases and improve impl

---------

Co-authored-by: nathan <nathan@appflowy.io>
This commit is contained in:
Zack 2023-09-27 10:21:37 +08:00 committed by GitHub
parent b729e3529d
commit ff6a8e1eaf
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
25 changed files with 559 additions and 6 deletions

View File

@ -0,0 +1,47 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT * FROM af_file_metadata\n WHERE owner_uid = (\n SELECT uid\n FROM af_user\n WHERE uuid = $1\n ) AND path = $2\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "owner_uid",
"type_info": "Int8"
},
{
"ordinal": 1,
"name": "path",
"type_info": "Varchar"
},
{
"ordinal": 2,
"name": "file_type",
"type_info": "Varchar"
},
{
"ordinal": 3,
"name": "file_size",
"type_info": "Int8"
},
{
"ordinal": 4,
"name": "created_at",
"type_info": "Timestamptz"
}
],
"parameters": {
"Left": [
"Uuid",
"Text"
]
},
"nullable": [
false,
false,
false,
false,
false
]
},
"hash": "1f98d40728c1bda6841418970cac282651f06207b3b4681c3f15cd158a052f45"
}

View File

@ -0,0 +1,47 @@
{
"db_name": "PostgreSQL",
"query": "\n DELETE FROM af_file_metadata\n WHERE owner_uid = (\n SELECT uid\n FROM af_user\n WHERE uuid = $1\n ) AND path = $2\n RETURNING *\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "owner_uid",
"type_info": "Int8"
},
{
"ordinal": 1,
"name": "path",
"type_info": "Varchar"
},
{
"ordinal": 2,
"name": "file_type",
"type_info": "Varchar"
},
{
"ordinal": 3,
"name": "file_size",
"type_info": "Int8"
},
{
"ordinal": 4,
"name": "created_at",
"type_info": "Timestamptz"
}
],
"parameters": {
"Left": [
"Uuid",
"Text"
]
},
"nullable": [
false,
false,
false,
false,
false
]
},
"hash": "5567487eeaaa4ed1f725f48d89f51f9ee509f964b84bbc10a12e7299faa26d33"
}

View File

@ -0,0 +1,49 @@
{
"db_name": "PostgreSQL",
"query": "\n INSERT INTO af_file_metadata (owner_uid, path, file_type, file_size)\n SELECT uid, $2, $3, $4\n FROM af_user\n WHERE uuid = $1\n ON CONFLICT (owner_uid, path) DO UPDATE SET\n file_type = $3,\n file_size = $4\n RETURNING *\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "owner_uid",
"type_info": "Int8"
},
{
"ordinal": 1,
"name": "path",
"type_info": "Varchar"
},
{
"ordinal": 2,
"name": "file_type",
"type_info": "Varchar"
},
{
"ordinal": 3,
"name": "file_size",
"type_info": "Int8"
},
{
"ordinal": 4,
"name": "created_at",
"type_info": "Timestamptz"
}
],
"parameters": {
"Left": [
"Uuid",
"Varchar",
"Varchar",
"Int8"
]
},
"nullable": [
false,
false,
false,
false,
false
]
},
"hash": "f63968cfcf5a7c9bd3517573213dcf40388fe754fc3204789cf4aff3da5d00b2"
}

3
Cargo.lock generated
View File

@ -467,6 +467,7 @@ dependencies = [
"infra",
"jsonwebtoken",
"lazy_static",
"mime",
"once_cell",
"openssl",
"rand 0.8.5",
@ -837,6 +838,7 @@ dependencies = [
"futures-util",
"gotrue",
"gotrue-entity",
"mime",
"opener",
"parking_lot",
"reqwest",
@ -3370,6 +3372,7 @@ dependencies = [
"gotrue-entity",
"opener",
"reqwest",
"rust-s3",
"serde",
"serde_json",
"serde_repr",

View File

@ -48,6 +48,7 @@ validator = "0.16.0"
bytes = "1.4.0"
rcgen = { version = "0.10.0", features = ["pem", "x509-parser"] }
jsonwebtoken = "8.3.0"
mime = "0.3.17"
# aws-config = "0.56.1"
# aws-sdk-s3 = "0.31.1"
rust-s3 = "0.33.0"

View File

@ -30,4 +30,4 @@ bytes = "1.0"
uuid = "1.4.1"
collab-sync-protocol = { version = "0.1.0" }
scraper = "0.17.1"
mime = "0.3.17"

View File

@ -1,11 +1,14 @@
use anyhow::anyhow;
use bytes::Bytes;
use gotrue::grant::Grant;
use gotrue::grant::PasswordGrant;
use gotrue::grant::RefreshTokenGrant;
use gotrue::params::{AdminUserParams, GenerateLinkParams};
use gotrue_entity::OAuthProvider;
use gotrue_entity::SignUpResponse::{Authenticated, NotAuthenticated};
use mime::Mime;
use parking_lot::RwLock;
use reqwest::header;
use reqwest::Method;
use reqwest::RequestBuilder;
use scraper::{Html, Selector};
@ -443,6 +446,53 @@ impl Client {
Ok(extract_sign_in_url(&resp_text)?)
}
pub async fn put_file_storage_object(
&self,
path: &str,
data: Bytes,
mime: &Mime,
) -> Result<(), AppError> {
let url = format!("{}/api/file_storage/{}", self.base_url, path);
let resp = self
.http_client_with_auth(Method::PUT, &url)
.await?
.header(header::CONTENT_TYPE, mime.to_string())
.body(data)
.send()
.await?;
AppResponse::<()>::from_response(resp).await?.into_error()
}
pub async fn get_file_storage_object(&self, path: &str) -> Result<Bytes, AppError> {
let url = format!("{}/api/file_storage/{}", self.base_url, path);
let resp = self
.http_client_with_auth(Method::GET, &url)
.await?
.send()
.await?;
match resp.status() {
reqwest::StatusCode::OK => {
let bytes = resp.bytes().await?;
Ok(bytes)
},
reqwest::StatusCode::NOT_FOUND => Err(ErrorCode::FileNotFound.into()),
c => Err(AppError::new(
ErrorCode::Unhandled,
format!("status code: {}, message: {}", c, resp.text().await?),
)),
}
}
pub async fn delete_file_storage_object(&self, path: &str) -> Result<(), AppError> {
let url = format!("{}/api/file_storage/{}", self.base_url, path);
let resp = self
.http_client_with_auth(Method::DELETE, &url)
.await?
.send()
.await?;
AppResponse::<()>::from_response(resp).await?.into_error()
}
async fn http_client_with_auth(
&self,
method: Method,

9
libs/s3/Cargo.toml Normal file
View File

@ -0,0 +1,9 @@
[package]
name = "storage"
version = "0.1.0"
edition = "2021"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
rust-s3 = "0.33"

1
libs/s3/src/lib.rs Normal file
View File

@ -0,0 +1 @@

View File

@ -20,6 +20,7 @@ sqlx = { version = "0.7", default-features = false, features = ["postgres"], opt
validator = { version = "0.16", features = ["validator_derive", "derive"], optional = true }
opener = "0.6.1"
url = "2.4.1"
rust-s3 = "0.33.0"
[features]
cloud = ["actix-web", "sqlx", "validator"]

View File

@ -137,3 +137,10 @@ impl From<SystemTimeError> for AppError {
AppError::new(ErrorCode::Unhandled, value.to_string())
}
}
#[cfg(feature = "cloud")]
impl From<s3::error::S3Error> for AppError {
fn from(value: s3::error::S3Error) -> Self {
AppError::new(ErrorCode::S3Error, value.to_string())
}
}

View File

@ -57,6 +57,12 @@ pub enum ErrorCode {
#[error("User name cannot be empty")]
UserNameIsEmpty = 1013,
#[error("S3 Error")]
S3Error = 1014,
#[error("File Not Found")]
FileNotFound = 1015,
}
/// Implements conversion from `anyhow::Error` to `ErrorCode`.

View File

@ -205,3 +205,18 @@ pub struct AFWorkspaceMember {
pub email: String,
pub role: AFRole,
}
#[derive(FromRow, Serialize, Deserialize)]
pub struct AFFileMetadata {
pub owner_uid: i64,
pub path: String,
pub file_type: String,
pub file_size: i64,
pub created_at: DateTime<Utc>,
}
impl AFFileMetadata {
pub fn s3_path(&self) -> String {
format!("{}/{}", self.owner_uid, self.path)
}
}

View File

@ -0,0 +1,75 @@
use sqlx::{PgPool, Transaction};
use storage_entity::AFFileMetadata;
pub async fn insert_file_metadata(
trans: &mut Transaction<'_, sqlx::Postgres>,
user: &uuid::Uuid,
path: &str,
file_type: &str,
file_size: i64,
) -> Result<AFFileMetadata, sqlx::Error> {
sqlx::query_as!(
AFFileMetadata,
r#"
INSERT INTO af_file_metadata (owner_uid, path, file_type, file_size)
SELECT uid, $2, $3, $4
FROM af_user
WHERE uuid = $1
ON CONFLICT (owner_uid, path) DO UPDATE SET
file_type = $3,
file_size = $4
RETURNING *
"#,
user,
path,
file_type,
file_size
)
.fetch_one(trans.as_mut())
.await
}
pub async fn delete_file_metadata(
trans: &mut Transaction<'_, sqlx::Postgres>,
user: &uuid::Uuid,
path: &str,
) -> Result<AFFileMetadata, sqlx::Error> {
sqlx::query_as!(
AFFileMetadata,
r#"
DELETE FROM af_file_metadata
WHERE owner_uid = (
SELECT uid
FROM af_user
WHERE uuid = $1
) AND path = $2
RETURNING *
"#,
user,
path,
)
.fetch_one(trans.as_mut())
.await
}
pub async fn get_file_metadata(
pg_pool: &PgPool,
user: &uuid::Uuid,
path: &str,
) -> Result<AFFileMetadata, sqlx::Error> {
sqlx::query_as!(
AFFileMetadata,
r#"
SELECT * FROM af_file_metadata
WHERE owner_uid = (
SELECT uid
FROM af_user
WHERE uuid = $1
) AND path = $2
"#,
user,
path,
)
.fetch_one(pg_pool)
.await
}

View File

@ -1,3 +1,4 @@
pub mod collab;
pub mod error;
pub mod file_storage;
pub mod workspace;

View File

@ -0,0 +1,8 @@
CREATE TABLE IF NOT EXISTS af_file_metadata (
owner_uid BIGINT REFERENCES af_user(uid) ON DELETE CASCADE NOT NULL,
path VARCHAR NOT NULL,
file_type VARCHAR NOT NULL,
file_size BIGINT NOT NULL,
created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP NOT NULL,
UNIQUE (owner_uid, path)
);

67
src/api/file_storage.rs Normal file
View File

@ -0,0 +1,67 @@
use actix_web::http::header::ContentType;
use actix_web::Result;
use actix_web::{
web::{self, Data},
Scope,
};
use bytes::Bytes;
use shared_entity::data::{AppResponse, JsonAppResponse};
use shared_entity::error_code::ErrorCode;
use crate::biz::file_storage;
use crate::{component::auth::jwt::UserUuid, state::AppState};
pub fn file_storage_scope() -> Scope {
web::scope("/api/file_storage").service(
web::resource("/{path}")
.route(web::get().to(get_handler))
.route(web::put().to(put_handler))
.route(web::delete().to(delete_handler)),
)
}
async fn put_handler(
user_uuid: UserUuid,
state: Data<AppState>,
path: web::Path<String>,
file_data: Bytes,
content_type: web::Header<ContentType>,
) -> Result<JsonAppResponse<()>> {
let file_path = path.into_inner();
let mime = content_type.into_inner().0;
file_storage::put_object(
&state.pg_pool,
&state.s3_bucket,
&user_uuid,
&file_path,
&file_data,
mime,
)
.await?;
Ok(AppResponse::Ok().into())
}
async fn delete_handler(
user_uuid: UserUuid,
state: Data<AppState>,
path: web::Path<String>,
) -> Result<JsonAppResponse<()>> {
let file_path = path.into_inner();
file_storage::delete_object(&state.pg_pool, &state.s3_bucket, &user_uuid, &file_path).await?;
Ok(AppResponse::Ok().into())
}
async fn get_handler(
user_uuid: UserUuid,
state: Data<AppState>,
path: web::Path<String>,
) -> Result<Bytes> {
let file_path = path.into_inner();
match file_storage::get_object(&state.pg_pool, &state.s3_bucket, &user_uuid, &file_path).await {
Ok(data) => Ok(data),
Err(e) => match e.code {
ErrorCode::FileNotFound => Err(actix_web::error::ErrorNotFound(e)),
_ => Err(actix_web::error::ErrorInternalServerError(e)),
},
}
}

View File

@ -1,9 +1,11 @@
mod collaborate;
mod file_storage;
mod user;
mod workspace;
mod ws;
pub use collaborate::collab_scope;
pub use file_storage::file_storage_scope;
pub use user::user_scope;
pub use workspace::workspace_scope;
pub use ws::ws_scope;

View File

@ -3,7 +3,9 @@ use crate::component::auth::{
change_password, logged_user_from_request, login, logout, register, ChangePasswordRequest,
RegisterRequest,
};
use crate::component::auth::{InputParamsError, LoginRequest};
use crate::component::token_state::SessionToken;
use crate::domain::{UserEmail, UserName, UserPassword};
use crate::state::AppState;

View File

@ -1,4 +1,4 @@
use crate::api::{collab_scope, user_scope, workspace_scope, ws_scope};
use crate::api::{collab_scope, file_storage_scope, user_scope, workspace_scope, ws_scope};
use crate::component::auth::HEADER_TOKEN;
use crate::config::config::{Config, DatabaseSetting, GoTrueSetting, S3Setting, TlsConfig};
use crate::middleware::cors::default_cors;
@ -100,6 +100,7 @@ where
.service(workspace_scope())
.service(ws_scope())
.service(collab_scope())
.service(file_storage_scope())
.app_data(Data::new(collab_server.clone()))
.app_data(Data::new(state.clone()))
.app_data(Data::new(storage.clone()))
@ -127,7 +128,7 @@ pub async fn init_state(config: &Config) -> AppState {
let pg_pool = get_connection_pool(&config.database).await;
migrate(&pg_pool).await;
let s3_bucket = get_aws_s3_client(&config.s3).await;
let s3_bucket = get_aws_s3_bucket(&config.s3).await;
let gotrue_client = get_gotrue_client(&config.gotrue).await;
setup_admin_account(&gotrue_client, &pg_pool, &config.gotrue).await;
@ -167,7 +168,7 @@ async fn setup_admin_account(
.unwrap();
}
async fn get_aws_s3_client(s3_setting: &S3Setting) -> s3::Bucket {
async fn get_aws_s3_bucket(s3_setting: &S3Setting) -> s3::Bucket {
let region = {
match s3_setting.use_minio {
true => s3::Region::Custom {
@ -201,7 +202,9 @@ async fn get_aws_s3_client(s3_setting: &S3Setting) -> s3::Bucket {
},
}
s3::Bucket::new(&s3_setting.bucket, region.clone(), cred.clone()).unwrap()
s3::Bucket::new(&s3_setting.bucket, region.clone(), cred.clone())
.unwrap()
.with_path_style()
}
// async fn get_aws_s3_client() -> aws_sdk_s3::Client {

87
src/biz/file_storage.rs Normal file
View File

@ -0,0 +1,87 @@
use bytes::Bytes;
use s3::request::ResponseData;
use shared_entity::{error::AppError, error_code::ErrorCode};
use sqlx::types::uuid;
use storage::file_storage;
// todo: user writer
pub async fn put_object(
pg_pool: &sqlx::PgPool,
s3_bucket: &s3::Bucket,
user_uuid: &uuid::Uuid,
path: &str,
data: &[u8],
mime: mime::Mime,
) -> Result<(), AppError> {
// TODO: access control
let size = data.len() as i64;
let mut trans = pg_pool.begin().await?;
let file_type = mime.to_string();
let metadata =
file_storage::insert_file_metadata(&mut trans, user_uuid, path, &file_type, size).await?;
let resp = s3_bucket.put_object(metadata.s3_path(), data).await?;
check_s3_status(&resp)?;
trans.commit().await?;
Ok(())
}
pub async fn delete_object(
pg_pool: &sqlx::PgPool,
s3_bucket: &s3::Bucket,
user_uuid: &uuid::Uuid,
path: &str,
) -> Result<(), AppError> {
// TODO: access control
let mut trans = pg_pool.begin().await?;
match file_storage::delete_file_metadata(&mut trans, user_uuid, path).await {
Ok(metadata) => {
let resp = s3_bucket.delete_object(metadata.s3_path()).await?;
check_s3_status(&resp)?;
trans.commit().await?;
Ok(())
},
Err(e) => match e {
sqlx::Error::RowNotFound => Err(ErrorCode::FileNotFound.into()),
e => Err(e.into()),
},
}
}
// user reader
pub async fn get_object(
pg_pool: &sqlx::PgPool,
s3_bucket: &s3::Bucket,
user_uuid: &uuid::Uuid,
path: &str,
) -> Result<Bytes, AppError> {
// TODO: access control
match file_storage::get_file_metadata(pg_pool, user_uuid, path).await {
Ok(metadata) => {
let resp = s3_bucket.get_object(metadata.s3_path()).await?;
check_s3_status(&resp)?;
Ok(resp.bytes().to_owned())
},
Err(e) => match e {
sqlx::Error::RowNotFound => Err(ErrorCode::FileNotFound.into()),
e => Err(e.into()),
},
}
}
fn check_s3_status(resp: &ResponseData) -> Result<(), AppError> {
let status_code = resp.status_code();
match status_code {
200..=299 => Ok(()),
error_code => {
let text = resp.bytes();
let s = String::from_utf8_lossy(text);
Err(AppError::new(
ErrorCode::S3Error,
format!("{}: {}", error_code, s),
))
},
}
}

View File

@ -1,2 +1,3 @@
pub mod file_storage;
pub mod user;
pub mod workspace;

View File

@ -16,7 +16,7 @@ pub async fn token_verify(
let user = gotrue_client.user_info(access_token).await?;
let user_uuid = uuid::Uuid::parse_str(&user.id)?;
let name = name_from_user_metadata(&user.user_metadata);
let is_new: bool = create_user_if_not_exists(pg_pool, &user_uuid, &user.email, &name).await?;
let is_new = create_user_if_not_exists(pg_pool, &user_uuid, &user.email, &name).await?;
Ok(is_new)
}

View File

@ -0,0 +1,70 @@
use shared_entity::error_code::ErrorCode;
use crate::client::utils::generate_unique_registered_user_client;
#[tokio::test]
async fn get_but_not_exists() {
let (c1, _user1) = generate_unique_registered_user_client().await;
let err = c1
.get_file_storage_object("not_exists_file")
.await
.unwrap_err();
assert_eq!(err.code, ErrorCode::FileNotFound);
}
#[tokio::test]
async fn put_and_get() {
let (c1, _user1) = generate_unique_registered_user_client().await;
let mime = mime::TEXT_PLAIN_UTF_8;
let data = "hello world";
let path = "mydata";
c1.put_file_storage_object(path, data.into(), &mime)
.await
.unwrap();
let got_data = c1.get_file_storage_object(path).await.unwrap();
assert_eq!(got_data, data.as_bytes());
}
#[tokio::test]
async fn put_and_put_and_get() {
let (c1, _user1) = generate_unique_registered_user_client().await;
let mime = mime::TEXT_PLAIN_UTF_8;
let data1 = "my content 1";
let data2 = "my content 2";
let path = "mydata";
c1.put_file_storage_object(path, data1.into(), &mime)
.await
.unwrap();
c1.put_file_storage_object(path, data2.into(), &mime)
.await
.unwrap();
let got_data = c1.get_file_storage_object(path).await.unwrap();
assert_eq!(got_data, data2.as_bytes());
}
#[tokio::test]
async fn delete_but_not_exists() {
let (c1, _user1) = generate_unique_registered_user_client().await;
let err = c1
.delete_file_storage_object("not_exists_file")
.await
.unwrap_err();
assert_eq!(err.code, ErrorCode::FileNotFound);
}
#[tokio::test]
async fn put_delete_get() {
let (c1, _user1) = generate_unique_registered_user_client().await;
let mime = mime::TEXT_PLAIN_UTF_8;
let data = "my contents";
let path = "mydata";
c1.put_file_storage_object(path, data.into(), &mime)
.await
.unwrap();
c1.delete_file_storage_object(path).await.unwrap();
let err = c1.get_file_storage_object(path).await.unwrap_err();
assert_eq!(err.code, ErrorCode::FileNotFound);
}

View File

@ -1,4 +1,5 @@
pub mod constants;
mod file_storage;
mod refresh;
mod sign_in;
mod sign_out;