feat: File api (#68)

* feat: s3 minio wip

* feat: s3 minio bucket create idempotent

* feat: put storage setting into configurations

* chore: clippy lint

* feat: add setting to base config

* feat: add configuration for prod

* fix: allow use different minio host
This commit is contained in:
Zack 2023-09-22 15:10:41 +08:00 committed by GitHub
parent df9c71edb2
commit 67706f9e8e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 353 additions and 10 deletions

250
Cargo.lock generated
View File

@ -70,7 +70,7 @@ dependencies = [
"actix-service",
"actix-tls",
"actix-utils",
"ahash",
"ahash 0.8.3",
"base64 0.21.4",
"bitflags 2.4.0",
"brotli",
@ -243,7 +243,7 @@ dependencies = [
"actix-tls",
"actix-utils",
"actix-web-codegen",
"ahash",
"ahash 0.8.3",
"bytes",
"bytestring",
"cfg-if",
@ -359,6 +359,17 @@ dependencies = [
"subtle",
]
[[package]]
name = "ahash"
version = "0.7.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fcb51a0695d8f838b1ee009b3fbf66bda078cd64590202a864a8f3e8c4315c47"
dependencies = [
"getrandom 0.2.10",
"once_cell",
"version_check",
]
[[package]]
name = "ahash"
version = "0.8.3"
@ -462,6 +473,7 @@ dependencies = [
"rcgen",
"realtime",
"reqwest",
"rust-s3",
"secrecy",
"serde",
"serde-aux",
@ -577,12 +589,52 @@ version = "0.1.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "112ef6b3f6cb3cb6fc5b6b494ef7a848492cff1ab0ef4de10b0f7d572861c905"
[[package]]
name = "attohttpc"
version = "0.22.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1fcf00bc6d5abb29b5f97e3c61a90b6d3caa12f3faf897d4a3e3607c050a35a7"
dependencies = [
"http",
"log",
"native-tls",
"serde",
"serde_json",
"url",
]
[[package]]
name = "autocfg"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
[[package]]
name = "aws-creds"
version = "0.34.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3776743bb68d4ad02ba30ba8f64373f1be4e082fe47651767171ce75bb2f6cf5"
dependencies = [
"attohttpc",
"dirs",
"log",
"quick-xml",
"rust-ini",
"serde",
"thiserror",
"time",
"url",
]
[[package]]
name = "aws-region"
version = "0.25.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "056557a61427d0e5ba29dd931031c8ffed4ee7a550e7cd55692a9d8deb0a9dba"
dependencies = [
"thiserror",
]
[[package]]
name = "backtrace"
version = "0.3.69"
@ -1082,6 +1134,9 @@ name = "deranged"
version = "0.3.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f2696e8a945f658fd14dc3b87242e6b80cd0f36ff04ea560fa39082368847946"
dependencies = [
"serde",
]
[[package]]
name = "derive_more"
@ -1108,6 +1163,26 @@ dependencies = [
"subtle",
]
[[package]]
name = "dirs"
version = "4.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ca3aa72a6f96ea37bbc5aa912f6788242832f75369bdfdadcb0e38423f100059"
dependencies = [
"dirs-sys",
]
[[package]]
name = "dirs-sys"
version = "0.3.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1b1d1d91c932ef41c0f2663aa8b0ca0342d444d842c06914aa0a7e352d0bada6"
dependencies = [
"libc",
"redox_users",
"winapi",
]
[[package]]
name = "displaydoc"
version = "0.2.4"
@ -1119,6 +1194,12 @@ dependencies = [
"syn 2.0.32",
]
[[package]]
name = "dlv-list"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0688c2a7f92e427f44895cd63841bff7b29f8d7a1648b9e7e07a4a365b2e1257"
[[package]]
name = "dotenv"
version = "0.15.0"
@ -1473,6 +1554,9 @@ name = "hashbrown"
version = "0.12.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"
dependencies = [
"ahash 0.7.6",
]
[[package]]
name = "hashbrown"
@ -1480,7 +1564,7 @@ version = "0.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2c6201b9ff9fd90a5a3bac2e56a830d0caa509576f0e503818ee82c181b3437a"
dependencies = [
"ahash",
"ahash 0.8.3",
"allocator-api2",
]
@ -1909,6 +1993,17 @@ version = "0.1.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2532096657941c2fea9c289d370a250971c689d4f143798ff67113ec042024a5"
[[package]]
name = "maybe-async"
version = "0.2.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0f1b8c13cb1f814b634a96b2c725449fe7ed464a7b8781de8688be5ffbd3f305"
dependencies = [
"proc-macro2",
"quote",
"syn 1.0.109",
]
[[package]]
name = "md-5"
version = "0.10.5"
@ -1946,6 +2041,15 @@ dependencies = [
"unicase",
]
[[package]]
name = "minidom"
version = "0.15.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f45614075738ce1b77a1768912a60c0227525971b03e09122a05b8a34a2a6278"
dependencies = [
"rxml",
]
[[package]]
name = "minimal-lexical"
version = "0.2.1"
@ -2174,6 +2278,16 @@ dependencies = [
"vcpkg",
]
[[package]]
name = "ordered-multimap"
version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ccd746e37177e1711c20dd619a1620f34f5c8b569c53590a72dedd5344d8924a"
dependencies = [
"dlv-list",
"hashbrown 0.12.3",
]
[[package]]
name = "overload"
version = "0.1.1"
@ -2198,7 +2312,7 @@ checksum = "93f00c865fe7cabf650081affecd3871070f26767e7b2070a3ffae14c654b447"
dependencies = [
"cfg-if",
"libc",
"redox_syscall",
"redox_syscall 0.3.5",
"smallvec",
"windows-targets",
]
@ -2376,6 +2490,16 @@ dependencies = [
"psl-types",
]
[[package]]
name = "quick-xml"
version = "0.26.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7f50b1c63b38611e7d4d7f68b82d3ad0cc71a2ad2e7f61fc10f1328d917c93cd"
dependencies = [
"memchr",
"serde",
]
[[package]]
name = "quote"
version = "1.0.33"
@ -2534,6 +2658,15 @@ dependencies = [
"url",
]
[[package]]
name = "redox_syscall"
version = "0.2.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a"
dependencies = [
"bitflags 1.3.2",
]
[[package]]
name = "redox_syscall"
version = "0.3.5"
@ -2543,6 +2676,17 @@ dependencies = [
"bitflags 1.3.2",
]
[[package]]
name = "redox_users"
version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b"
dependencies = [
"getrandom 0.2.10",
"redox_syscall 0.2.16",
"thiserror",
]
[[package]]
name = "regex"
version = "1.9.5"
@ -2623,10 +2767,12 @@ dependencies = [
"tokio",
"tokio-native-tls",
"tokio-rustls",
"tokio-util",
"tower-service",
"url",
"wasm-bindgen",
"wasm-bindgen-futures",
"wasm-streams",
"web-sys",
"webpki-roots 0.25.2",
"winreg",
@ -2669,6 +2815,49 @@ dependencies = [
"zeroize",
]
[[package]]
name = "rust-ini"
version = "0.18.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f6d5f2436026b4f6e79dc829837d467cc7e9a55ee40e750d716713540715a2df"
dependencies = [
"cfg-if",
"ordered-multimap",
]
[[package]]
name = "rust-s3"
version = "0.33.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1b2ac5ff6acfbe74226fa701b5ef793aaa054055c13ebb7060ad36942956e027"
dependencies = [
"async-trait",
"aws-creds",
"aws-region",
"base64 0.13.1",
"bytes",
"cfg-if",
"futures",
"hex",
"hmac",
"http",
"log",
"maybe-async",
"md5",
"minidom",
"percent-encoding",
"quick-xml",
"reqwest",
"serde",
"serde_derive",
"sha2",
"thiserror",
"time",
"tokio",
"tokio-stream",
"url",
]
[[package]]
name = "rustc-demangle"
version = "0.1.23"
@ -2737,6 +2926,23 @@ dependencies = [
"untrusted",
]
[[package]]
name = "rxml"
version = "0.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a98f186c7a2f3abbffb802984b7f1dfd65dac8be1aafdaabbca4137f53f0dff7"
dependencies = [
"bytes",
"rxml_validation",
"smartstring",
]
[[package]]
name = "rxml_validation"
version = "0.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "22a197350ece202f19a166d1ad6d9d6de145e1d2a8ef47db299abe164dbd7530"
[[package]]
name = "ryu"
version = "1.0.15"
@ -2983,6 +3189,17 @@ version = "1.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "62bb4feee49fdd9f707ef802e22365a35de4b7b299de4763d44bfea899442ff9"
[[package]]
name = "smartstring"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3fb72c633efbaa2dd666986505016c32c3044395ceaf881518399d2f4127ee29"
dependencies = [
"autocfg",
"static_assertions",
"version_check",
]
[[package]]
name = "snowflake"
version = "0.1.0"
@ -3062,7 +3279,7 @@ version = "0.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dd4cef4251aabbae751a3710927945901ee1d97ee96d757f6880ebb9a79bfd53"
dependencies = [
"ahash",
"ahash 0.8.3",
"atoi",
"byteorder",
"bytes",
@ -3249,6 +3466,12 @@ dependencies = [
"uuid",
]
[[package]]
name = "static_assertions"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f"
[[package]]
name = "storage"
version = "0.1.0"
@ -3341,7 +3564,7 @@ checksum = "cb94d2f3cc536af71caac6b6fcebf65860b347e7ce0cc9ebe8f70d3e521054ef"
dependencies = [
"cfg-if",
"fastrand",
"redox_syscall",
"redox_syscall 0.3.5",
"rustix",
"windows-sys",
]
@ -3590,7 +3813,7 @@ version = "0.3.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b5c266b9ac83dedf0e0385ad78514949e6d89491269e7065bee51d2bb8ec7373"
dependencies = [
"ahash",
"ahash 0.8.3",
"gethostname",
"log",
"serde",
@ -3923,6 +4146,19 @@ version = "0.2.87"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ca6ad05a4870b2bf5fe995117d3728437bd27d7cd5f06f13c17443ef369775a1"
[[package]]
name = "wasm-streams"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b4609d447824375f43e1ffbc051b50ad8f4b3ae8219680c94452ea05eb240ac7"
dependencies = [
"futures-util",
"js-sys",
"wasm-bindgen",
"wasm-bindgen-futures",
"web-sys",
]
[[package]]
name = "web-sys"
version = "0.3.64"

View File

@ -47,6 +47,9 @@ validator = "0.16.0"
bytes = "1.4.0"
rcgen = { version = "0.10.0", features = ["pem", "x509-parser"] }
jsonwebtoken = "8.3.0"
# aws-config = "0.56.1"
# aws-sdk-s3 = "0.31.1"
rust-s3 = "0.33.0"
# tracing
tracing = { version = "0.1.37" }

View File

@ -17,3 +17,10 @@ redis_uri: "redis://127.0.0.1:6379"
gotrue:
base_url: "http://127.0.0.1:9999"
ext_url: "http://127.0.0.1:9999"
s3:
use_minio: true
minio_url: http://localhost:9000
access_key: minioadmin
secret_key: minioadmin
bucket: appflowy
region: us-east-1

View File

@ -8,4 +8,6 @@ database:
redis_uri: "redis://redis:6379"
gotrue:
base_url: "http://gotrue:9999"
ext_url: "http://gotrue:9999"
ext_url: "http://gotrue:9999"
s3:
minio_url: http://minio:9000

View File

@ -39,3 +39,10 @@ GOTRUE_EXTERNAL_GOOGLE_CLIENT_ID=
GOTRUE_EXTERNAL_GOOGLE_SECRET=
GOTRUE_EXTERNAL_GOOGLE_REDIRECT_URI=http://localhost:9998/callback
# File Storage
USE_MINIO=true
# MINIO_URL=http://localhost:9000 # change this if you are using a different address for minio
AWS_ACCESS_KEY_ID=minioadmin
AWS_SECRET_ACCESS_KEY=minioadmin
AWS_S3_BUCKET=appflowy
AWS_REGION=us-east-1

View File

@ -1,5 +1,12 @@
version: '3'
services:
minio:
image: minio/minio
ports:
- 9000:9000
- 9001:9001
command: server /data --console-address ":9001"
postgres:
build:
context: .

View File

@ -13,6 +13,13 @@ services:
- ./nginx/ssl/certificate.crt:/etc/nginx/ssl/certificate.crt
- ./nginx/ssl/private_key.key:/etc/nginx/ssl/private_key.key
minio:
image: minio/minio
ports:
- 9000:9000
- 9001:9001
command: server /data --console-address ":9001"
postgres:
build:
context: .
@ -67,6 +74,12 @@ services:
- APP_ENVIRONMENT=production
- APP__GOTRUE__JWT_SECRET=${GOTRUE_JWT_SECRET}
- APP__GOTRUE__EXT_URL=${API_EXTERNAL_URL}
- APP__S3__USE_MINIO=${USE_MINIO}
- APP__S3__MINIO_URL=${MINIO_URL:-http://minio:9000}
- APP__S3__AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}
- APP__S3__AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY}
- APP__S3__AWS_S3_BUCKET=${AWS_S3_BUCKET}
- APP__S3__AWS_REGION=${AWS_REGION}
build:
context: .
dockerfile: Dockerfile

View File

@ -1,6 +1,6 @@
use crate::api::{collab_scope, user_scope, workspace_scope, ws_scope};
use crate::component::auth::HEADER_TOKEN;
use crate::config::config::{Config, DatabaseSetting, GoTrueSetting, TlsConfig};
use crate::config::config::{Config, DatabaseSetting, GoTrueSetting, S3Setting, TlsConfig};
use crate::middleware::cors::default_cors;
use crate::self_signed::create_self_signed_certificate;
use crate::state::{AppState, Storage};
@ -11,7 +11,6 @@ use actix_web::cookie::Key;
use actix_web::{dev::Server, web, web::Data, App, HttpServer};
use actix::Actor;
use openssl::ssl::{SslAcceptor, SslAcceptorBuilder, SslFiletype, SslMethod};
use openssl::x509::X509;
use secrecy::{ExposeSecret, Secret};
@ -127,6 +126,7 @@ fn get_certificate_and_server_key(config: &Config) -> Option<(Secret<String>, Se
pub async fn init_state(config: &Config) -> AppState {
let pg_pool = get_connection_pool(&config.database).await;
migrate(&pg_pool).await;
let s3_bucket = get_aws_s3_client(&config.s3).await;
let gotrue_client = get_gotrue_client(&config.gotrue).await;
@ -136,9 +136,65 @@ pub async fn init_state(config: &Config) -> AppState {
user: Arc::new(Default::default()),
id_gen: Arc::new(RwLock::new(Snowflake::new(1))),
gotrue_client,
s3_bucket,
}
}
async fn get_aws_s3_client(s3_setting: &S3Setting) -> s3::Bucket {
let region = {
match s3_setting.use_minio {
true => s3::Region::Custom {
region: "".to_owned(),
endpoint: s3_setting.minio_url.to_owned(),
},
false => s3_setting.region.parse::<s3::Region>().unwrap(),
}
};
let cred = s3::creds::Credentials {
access_key: Some(s3_setting.access_key.to_owned()),
secret_key: Some(s3_setting.secret_key.to_owned()),
security_token: None,
session_token: None,
expiration: None,
};
match s3::Bucket::create_with_path_style(
&s3_setting.bucket,
region.clone(),
cred.clone(),
s3::BucketConfiguration::default(),
)
.await
{
Ok(_) => {},
Err(e) => match e {
s3::error::S3Error::Http(409, _) => {}, // Bucket already exists
_ => panic!("Failed to create bucket: {:?}", e),
},
}
s3::Bucket::new(&s3_setting.bucket, region.clone(), cred.clone()).unwrap()
}
// async fn get_aws_s3_client() -> aws_sdk_s3::Client {
// let credentials = Credentials::new("minioadmin", "minioadmin", None, None, "none");
// let config = aws_config::SdkConfig::builder()
// .set_region(Region {})
// .endpoint_url("http://localhost:9000")
// .credentials_provider(Some(credentials))
// .build();
//
// let client = aws_sdk_s3::Client::new(&config);
// client
// .create_bucket()
// .bucket("hellothisisme")
// .send()
// .await
// .unwrap();
// client
// }
async fn get_connection_pool(setting: &DatabaseSetting) -> PgPool {
PgPoolOptions::new()
.acquire_timeout(std::time::Duration::from_secs(5))

View File

@ -12,6 +12,17 @@ pub struct Config {
pub application: ApplicationSetting,
pub websocket: WebsocketSetting,
pub redis_uri: Secret<String>,
pub s3: S3Setting,
}
#[derive(serde::Deserialize, Clone, Debug)]
pub struct S3Setting {
pub use_minio: bool,
pub minio_url: String,
pub access_key: String,
pub secret_key: String,
pub bucket: String,
pub region: String,
}
#[derive(serde::Deserialize, Clone, Debug)]

View File

@ -16,6 +16,7 @@ pub struct AppState {
pub user: Arc<RwLock<UserCache>>,
pub id_gen: Arc<RwLock<Snowflake>>,
pub gotrue_client: gotrue::api::Client,
pub s3_bucket: s3::Bucket,
}
impl AppState {