diff --git a/Cargo.lock b/Cargo.lock index 8765c229..801dc66a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -70,7 +70,7 @@ dependencies = [ "actix-service", "actix-tls", "actix-utils", - "ahash", + "ahash 0.8.3", "base64 0.21.4", "bitflags 2.4.0", "brotli", @@ -243,7 +243,7 @@ dependencies = [ "actix-tls", "actix-utils", "actix-web-codegen", - "ahash", + "ahash 0.8.3", "bytes", "bytestring", "cfg-if", @@ -359,6 +359,17 @@ dependencies = [ "subtle", ] +[[package]] +name = "ahash" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcb51a0695d8f838b1ee009b3fbf66bda078cd64590202a864a8f3e8c4315c47" +dependencies = [ + "getrandom 0.2.10", + "once_cell", + "version_check", +] + [[package]] name = "ahash" version = "0.8.3" @@ -462,6 +473,7 @@ dependencies = [ "rcgen", "realtime", "reqwest", + "rust-s3", "secrecy", "serde", "serde-aux", @@ -577,12 +589,52 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "112ef6b3f6cb3cb6fc5b6b494ef7a848492cff1ab0ef4de10b0f7d572861c905" +[[package]] +name = "attohttpc" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fcf00bc6d5abb29b5f97e3c61a90b6d3caa12f3faf897d4a3e3607c050a35a7" +dependencies = [ + "http", + "log", + "native-tls", + "serde", + "serde_json", + "url", +] + [[package]] name = "autocfg" version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +[[package]] +name = "aws-creds" +version = "0.34.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3776743bb68d4ad02ba30ba8f64373f1be4e082fe47651767171ce75bb2f6cf5" +dependencies = [ + "attohttpc", + "dirs", + "log", + "quick-xml", + "rust-ini", + "serde", + "thiserror", + "time", + "url", +] + +[[package]] +name = "aws-region" +version = "0.25.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "056557a61427d0e5ba29dd931031c8ffed4ee7a550e7cd55692a9d8deb0a9dba" +dependencies = [ + "thiserror", +] + [[package]] name = "backtrace" version = "0.3.69" @@ -1082,6 +1134,9 @@ name = "deranged" version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2696e8a945f658fd14dc3b87242e6b80cd0f36ff04ea560fa39082368847946" +dependencies = [ + "serde", +] [[package]] name = "derive_more" @@ -1108,6 +1163,26 @@ dependencies = [ "subtle", ] +[[package]] +name = "dirs" +version = "4.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca3aa72a6f96ea37bbc5aa912f6788242832f75369bdfdadcb0e38423f100059" +dependencies = [ + "dirs-sys", +] + +[[package]] +name = "dirs-sys" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b1d1d91c932ef41c0f2663aa8b0ca0342d444d842c06914aa0a7e352d0bada6" +dependencies = [ + "libc", + "redox_users", + "winapi", +] + [[package]] name = "displaydoc" version = "0.2.4" @@ -1119,6 +1194,12 @@ dependencies = [ "syn 2.0.32", ] +[[package]] +name = "dlv-list" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0688c2a7f92e427f44895cd63841bff7b29f8d7a1648b9e7e07a4a365b2e1257" + [[package]] name = "dotenv" version = "0.15.0" @@ -1473,6 +1554,9 @@ name = "hashbrown" version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" +dependencies = [ + "ahash 0.7.6", +] [[package]] name = "hashbrown" @@ -1480,7 +1564,7 @@ version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2c6201b9ff9fd90a5a3bac2e56a830d0caa509576f0e503818ee82c181b3437a" dependencies = [ - "ahash", + "ahash 0.8.3", "allocator-api2", ] @@ -1909,6 +1993,17 @@ version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2532096657941c2fea9c289d370a250971c689d4f143798ff67113ec042024a5" +[[package]] +name = "maybe-async" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f1b8c13cb1f814b634a96b2c725449fe7ed464a7b8781de8688be5ffbd3f305" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "md-5" version = "0.10.5" @@ -1946,6 +2041,15 @@ dependencies = [ "unicase", ] +[[package]] +name = "minidom" +version = "0.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f45614075738ce1b77a1768912a60c0227525971b03e09122a05b8a34a2a6278" +dependencies = [ + "rxml", +] + [[package]] name = "minimal-lexical" version = "0.2.1" @@ -2174,6 +2278,16 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "ordered-multimap" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccd746e37177e1711c20dd619a1620f34f5c8b569c53590a72dedd5344d8924a" +dependencies = [ + "dlv-list", + "hashbrown 0.12.3", +] + [[package]] name = "overload" version = "0.1.1" @@ -2198,7 +2312,7 @@ checksum = "93f00c865fe7cabf650081affecd3871070f26767e7b2070a3ffae14c654b447" dependencies = [ "cfg-if", "libc", - "redox_syscall", + "redox_syscall 0.3.5", "smallvec", "windows-targets", ] @@ -2376,6 +2490,16 @@ dependencies = [ "psl-types", ] +[[package]] +name = "quick-xml" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f50b1c63b38611e7d4d7f68b82d3ad0cc71a2ad2e7f61fc10f1328d917c93cd" +dependencies = [ + "memchr", + "serde", +] + [[package]] name = "quote" version = "1.0.33" @@ -2534,6 +2658,15 @@ dependencies = [ "url", ] +[[package]] +name = "redox_syscall" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" +dependencies = [ + "bitflags 1.3.2", +] + [[package]] name = "redox_syscall" version = "0.3.5" @@ -2543,6 +2676,17 @@ dependencies = [ "bitflags 1.3.2", ] +[[package]] +name = "redox_users" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b" +dependencies = [ + "getrandom 0.2.10", + "redox_syscall 0.2.16", + "thiserror", +] + [[package]] name = "regex" version = "1.9.5" @@ -2623,10 +2767,12 @@ dependencies = [ "tokio", "tokio-native-tls", "tokio-rustls", + "tokio-util", "tower-service", "url", "wasm-bindgen", "wasm-bindgen-futures", + "wasm-streams", "web-sys", "webpki-roots 0.25.2", "winreg", @@ -2669,6 +2815,49 @@ dependencies = [ "zeroize", ] +[[package]] +name = "rust-ini" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6d5f2436026b4f6e79dc829837d467cc7e9a55ee40e750d716713540715a2df" +dependencies = [ + "cfg-if", + "ordered-multimap", +] + +[[package]] +name = "rust-s3" +version = "0.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b2ac5ff6acfbe74226fa701b5ef793aaa054055c13ebb7060ad36942956e027" +dependencies = [ + "async-trait", + "aws-creds", + "aws-region", + "base64 0.13.1", + "bytes", + "cfg-if", + "futures", + "hex", + "hmac", + "http", + "log", + "maybe-async", + "md5", + "minidom", + "percent-encoding", + "quick-xml", + "reqwest", + "serde", + "serde_derive", + "sha2", + "thiserror", + "time", + "tokio", + "tokio-stream", + "url", +] + [[package]] name = "rustc-demangle" version = "0.1.23" @@ -2737,6 +2926,23 @@ dependencies = [ "untrusted", ] +[[package]] +name = "rxml" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a98f186c7a2f3abbffb802984b7f1dfd65dac8be1aafdaabbca4137f53f0dff7" +dependencies = [ + "bytes", + "rxml_validation", + "smartstring", +] + +[[package]] +name = "rxml_validation" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22a197350ece202f19a166d1ad6d9d6de145e1d2a8ef47db299abe164dbd7530" + [[package]] name = "ryu" version = "1.0.15" @@ -2983,6 +3189,17 @@ version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "62bb4feee49fdd9f707ef802e22365a35de4b7b299de4763d44bfea899442ff9" +[[package]] +name = "smartstring" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fb72c633efbaa2dd666986505016c32c3044395ceaf881518399d2f4127ee29" +dependencies = [ + "autocfg", + "static_assertions", + "version_check", +] + [[package]] name = "snowflake" version = "0.1.0" @@ -3062,7 +3279,7 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dd4cef4251aabbae751a3710927945901ee1d97ee96d757f6880ebb9a79bfd53" dependencies = [ - "ahash", + "ahash 0.8.3", "atoi", "byteorder", "bytes", @@ -3249,6 +3466,12 @@ dependencies = [ "uuid", ] +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + [[package]] name = "storage" version = "0.1.0" @@ -3341,7 +3564,7 @@ checksum = "cb94d2f3cc536af71caac6b6fcebf65860b347e7ce0cc9ebe8f70d3e521054ef" dependencies = [ "cfg-if", "fastrand", - "redox_syscall", + "redox_syscall 0.3.5", "rustix", "windows-sys", ] @@ -3590,7 +3813,7 @@ version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5c266b9ac83dedf0e0385ad78514949e6d89491269e7065bee51d2bb8ec7373" dependencies = [ - "ahash", + "ahash 0.8.3", "gethostname", "log", "serde", @@ -3923,6 +4146,19 @@ version = "0.2.87" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ca6ad05a4870b2bf5fe995117d3728437bd27d7cd5f06f13c17443ef369775a1" +[[package]] +name = "wasm-streams" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4609d447824375f43e1ffbc051b50ad8f4b3ae8219680c94452ea05eb240ac7" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + [[package]] name = "web-sys" version = "0.3.64" diff --git a/Cargo.toml b/Cargo.toml index 0b6e016d..0e309108 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -47,6 +47,9 @@ validator = "0.16.0" bytes = "1.4.0" rcgen = { version = "0.10.0", features = ["pem", "x509-parser"] } jsonwebtoken = "8.3.0" +# aws-config = "0.56.1" +# aws-sdk-s3 = "0.31.1" +rust-s3 = "0.33.0" # tracing tracing = { version = "0.1.37" } diff --git a/configuration/base.yaml b/configuration/base.yaml index 966328ed..cc5d0bbf 100644 --- a/configuration/base.yaml +++ b/configuration/base.yaml @@ -17,3 +17,10 @@ redis_uri: "redis://127.0.0.1:6379" gotrue: base_url: "http://127.0.0.1:9999" ext_url: "http://127.0.0.1:9999" +s3: + use_minio: true + minio_url: http://localhost:9000 + access_key: minioadmin + secret_key: minioadmin + bucket: appflowy + region: us-east-1 diff --git a/configuration/production.yaml b/configuration/production.yaml index 6059d050..af28061e 100644 --- a/configuration/production.yaml +++ b/configuration/production.yaml @@ -8,4 +8,6 @@ database: redis_uri: "redis://redis:6379" gotrue: base_url: "http://gotrue:9999" - ext_url: "http://gotrue:9999" \ No newline at end of file + ext_url: "http://gotrue:9999" +s3: + minio_url: http://minio:9000 diff --git a/dev.env b/dev.env index c88be601..f93f02ee 100644 --- a/dev.env +++ b/dev.env @@ -39,3 +39,10 @@ GOTRUE_EXTERNAL_GOOGLE_CLIENT_ID= GOTRUE_EXTERNAL_GOOGLE_SECRET= GOTRUE_EXTERNAL_GOOGLE_REDIRECT_URI=http://localhost:9998/callback +# File Storage +USE_MINIO=true +# MINIO_URL=http://localhost:9000 # change this if you are using a different address for minio +AWS_ACCESS_KEY_ID=minioadmin +AWS_SECRET_ACCESS_KEY=minioadmin +AWS_S3_BUCKET=appflowy +AWS_REGION=us-east-1 diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index 6b54ae68..2e44f601 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -1,5 +1,12 @@ version: '3' services: + minio: + image: minio/minio + ports: + - 9000:9000 + - 9001:9001 + command: server /data --console-address ":9001" + postgres: build: context: . diff --git a/docker-compose.yml b/docker-compose.yml index 73f36f8b..32ca864f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -13,6 +13,13 @@ services: - ./nginx/ssl/certificate.crt:/etc/nginx/ssl/certificate.crt - ./nginx/ssl/private_key.key:/etc/nginx/ssl/private_key.key + minio: + image: minio/minio + ports: + - 9000:9000 + - 9001:9001 + command: server /data --console-address ":9001" + postgres: build: context: . @@ -67,6 +74,12 @@ services: - APP_ENVIRONMENT=production - APP__GOTRUE__JWT_SECRET=${GOTRUE_JWT_SECRET} - APP__GOTRUE__EXT_URL=${API_EXTERNAL_URL} + - APP__S3__USE_MINIO=${USE_MINIO} + - APP__S3__MINIO_URL=${MINIO_URL:-http://minio:9000} + - APP__S3__AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} + - APP__S3__AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} + - APP__S3__AWS_S3_BUCKET=${AWS_S3_BUCKET} + - APP__S3__AWS_REGION=${AWS_REGION} build: context: . dockerfile: Dockerfile diff --git a/src/application.rs b/src/application.rs index f55ec9f1..14e1793c 100644 --- a/src/application.rs +++ b/src/application.rs @@ -1,6 +1,6 @@ use crate::api::{collab_scope, user_scope, workspace_scope, ws_scope}; use crate::component::auth::HEADER_TOKEN; -use crate::config::config::{Config, DatabaseSetting, GoTrueSetting, TlsConfig}; +use crate::config::config::{Config, DatabaseSetting, GoTrueSetting, S3Setting, TlsConfig}; use crate::middleware::cors::default_cors; use crate::self_signed::create_self_signed_certificate; use crate::state::{AppState, Storage}; @@ -11,7 +11,6 @@ use actix_web::cookie::Key; use actix_web::{dev::Server, web, web::Data, App, HttpServer}; use actix::Actor; - use openssl::ssl::{SslAcceptor, SslAcceptorBuilder, SslFiletype, SslMethod}; use openssl::x509::X509; use secrecy::{ExposeSecret, Secret}; @@ -127,6 +126,7 @@ fn get_certificate_and_server_key(config: &Config) -> Option<(Secret, Se pub async fn init_state(config: &Config) -> AppState { let pg_pool = get_connection_pool(&config.database).await; migrate(&pg_pool).await; + let s3_bucket = get_aws_s3_client(&config.s3).await; let gotrue_client = get_gotrue_client(&config.gotrue).await; @@ -136,9 +136,65 @@ pub async fn init_state(config: &Config) -> AppState { user: Arc::new(Default::default()), id_gen: Arc::new(RwLock::new(Snowflake::new(1))), gotrue_client, + s3_bucket, } } +async fn get_aws_s3_client(s3_setting: &S3Setting) -> s3::Bucket { + let region = { + match s3_setting.use_minio { + true => s3::Region::Custom { + region: "".to_owned(), + endpoint: s3_setting.minio_url.to_owned(), + }, + false => s3_setting.region.parse::().unwrap(), + } + }; + + let cred = s3::creds::Credentials { + access_key: Some(s3_setting.access_key.to_owned()), + secret_key: Some(s3_setting.secret_key.to_owned()), + security_token: None, + session_token: None, + expiration: None, + }; + + match s3::Bucket::create_with_path_style( + &s3_setting.bucket, + region.clone(), + cred.clone(), + s3::BucketConfiguration::default(), + ) + .await + { + Ok(_) => {}, + Err(e) => match e { + s3::error::S3Error::Http(409, _) => {}, // Bucket already exists + _ => panic!("Failed to create bucket: {:?}", e), + }, + } + + s3::Bucket::new(&s3_setting.bucket, region.clone(), cred.clone()).unwrap() +} + +// async fn get_aws_s3_client() -> aws_sdk_s3::Client { +// let credentials = Credentials::new("minioadmin", "minioadmin", None, None, "none"); +// let config = aws_config::SdkConfig::builder() +// .set_region(Region {}) +// .endpoint_url("http://localhost:9000") +// .credentials_provider(Some(credentials)) +// .build(); +// +// let client = aws_sdk_s3::Client::new(&config); +// client +// .create_bucket() +// .bucket("hellothisisme") +// .send() +// .await +// .unwrap(); +// client +// } + async fn get_connection_pool(setting: &DatabaseSetting) -> PgPool { PgPoolOptions::new() .acquire_timeout(std::time::Duration::from_secs(5)) diff --git a/src/config/config.rs b/src/config/config.rs index c793c431..d0884506 100644 --- a/src/config/config.rs +++ b/src/config/config.rs @@ -12,6 +12,17 @@ pub struct Config { pub application: ApplicationSetting, pub websocket: WebsocketSetting, pub redis_uri: Secret, + pub s3: S3Setting, +} + +#[derive(serde::Deserialize, Clone, Debug)] +pub struct S3Setting { + pub use_minio: bool, + pub minio_url: String, + pub access_key: String, + pub secret_key: String, + pub bucket: String, + pub region: String, } #[derive(serde::Deserialize, Clone, Debug)] diff --git a/src/state.rs b/src/state.rs index 360365e3..51a07fcb 100644 --- a/src/state.rs +++ b/src/state.rs @@ -16,6 +16,7 @@ pub struct AppState { pub user: Arc>, pub id_gen: Arc>, pub gotrue_client: gotrue::api::Client, + pub s3_bucket: s3::Bucket, } impl AppState {