feat: File api (#68)

* feat: s3 minio wip

* feat: s3 minio bucket create idempotent

* feat: put storage setting into configurations

* chore: clippy lint

* feat: add setting to base config

* feat: add configuration for prod

* fix: allow use different minio host

Cargo.lock

@@ -70,7 +70,7 @@ dependencies = [
  "actix-service",
  "actix-tls",
  "actix-utils",
- "ahash",
+ "ahash 0.8.3",
  "base64 0.21.4",
  "bitflags 2.4.0",
  "brotli",
@@ -243,7 +243,7 @@ dependencies = [
  "actix-tls",
  "actix-utils",
  "actix-web-codegen",
- "ahash",
+ "ahash 0.8.3",
  "bytes",
  "bytestring",
  "cfg-if",
@@ -359,6 +359,17 @@ dependencies = [
  "subtle",
 ]
 
+[[package]]
+name = "ahash"
+version = "0.7.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fcb51a0695d8f838b1ee009b3fbf66bda078cd64590202a864a8f3e8c4315c47"
+dependencies = [
+ "getrandom 0.2.10",
+ "once_cell",
+ "version_check",
+]
+
 [[package]]
 name = "ahash"
 version = "0.8.3"
@@ -462,6 +473,7 @@ dependencies = [
  "rcgen",
  "realtime",
  "reqwest",
+ "rust-s3",
  "secrecy",
  "serde",
  "serde-aux",
@@ -577,12 +589,52 @@ version = "0.1.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "112ef6b3f6cb3cb6fc5b6b494ef7a848492cff1ab0ef4de10b0f7d572861c905"
 
+[[package]]
+name = "attohttpc"
+version = "0.22.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1fcf00bc6d5abb29b5f97e3c61a90b6d3caa12f3faf897d4a3e3607c050a35a7"
+dependencies = [
+ "http",
+ "log",
+ "native-tls",
+ "serde",
+ "serde_json",
+ "url",
+]
+
 [[package]]
 name = "autocfg"
 version = "1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
 
+[[package]]
+name = "aws-creds"
+version = "0.34.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3776743bb68d4ad02ba30ba8f64373f1be4e082fe47651767171ce75bb2f6cf5"
+dependencies = [
+ "attohttpc",
+ "dirs",
+ "log",
+ "quick-xml",
+ "rust-ini",
+ "serde",
+ "thiserror",
+ "time",
+ "url",
+]
+
+[[package]]
+name = "aws-region"
+version = "0.25.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "056557a61427d0e5ba29dd931031c8ffed4ee7a550e7cd55692a9d8deb0a9dba"
+dependencies = [
+ "thiserror",
+]
+
 [[package]]
 name = "backtrace"
 version = "0.3.69"
@@ -1082,6 +1134,9 @@ name = "deranged"
 version = "0.3.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f2696e8a945f658fd14dc3b87242e6b80cd0f36ff04ea560fa39082368847946"
+dependencies = [
+ "serde",
+]
 
 [[package]]
 name = "derive_more"
@@ -1108,6 +1163,26 @@ dependencies = [
  "subtle",
 ]
 
+[[package]]
+name = "dirs"
+version = "4.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ca3aa72a6f96ea37bbc5aa912f6788242832f75369bdfdadcb0e38423f100059"
+dependencies = [
+ "dirs-sys",
+]
+
+[[package]]
+name = "dirs-sys"
+version = "0.3.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1b1d1d91c932ef41c0f2663aa8b0ca0342d444d842c06914aa0a7e352d0bada6"
+dependencies = [
+ "libc",
+ "redox_users",
+ "winapi",
+]
+
 [[package]]
 name = "displaydoc"
 version = "0.2.4"
@@ -1119,6 +1194,12 @@ dependencies = [
  "syn 2.0.32",
 ]
 
+[[package]]
+name = "dlv-list"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0688c2a7f92e427f44895cd63841bff7b29f8d7a1648b9e7e07a4a365b2e1257"
+
 [[package]]
 name = "dotenv"
 version = "0.15.0"
@@ -1473,6 +1554,9 @@ name = "hashbrown"
 version = "0.12.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"
+dependencies = [
+ "ahash 0.7.6",
+]
 
 [[package]]
 name = "hashbrown"
@@ -1480,7 +1564,7 @@ version = "0.14.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2c6201b9ff9fd90a5a3bac2e56a830d0caa509576f0e503818ee82c181b3437a"
 dependencies = [
- "ahash",
+ "ahash 0.8.3",
  "allocator-api2",
 ]
 
@@ -1909,6 +1993,17 @@ version = "0.1.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2532096657941c2fea9c289d370a250971c689d4f143798ff67113ec042024a5"
 
+[[package]]
+name = "maybe-async"
+version = "0.2.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0f1b8c13cb1f814b634a96b2c725449fe7ed464a7b8781de8688be5ffbd3f305"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 1.0.109",
+]
+
 [[package]]
 name = "md-5"
 version = "0.10.5"
@@ -1946,6 +2041,15 @@ dependencies = [
  "unicase",
 ]
 
+[[package]]
+name = "minidom"
+version = "0.15.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f45614075738ce1b77a1768912a60c0227525971b03e09122a05b8a34a2a6278"
+dependencies = [
+ "rxml",
+]
+
 [[package]]
 name = "minimal-lexical"
 version = "0.2.1"
@@ -2174,6 +2278,16 @@ dependencies = [
  "vcpkg",
 ]
 
+[[package]]
+name = "ordered-multimap"
+version = "0.4.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ccd746e37177e1711c20dd619a1620f34f5c8b569c53590a72dedd5344d8924a"
+dependencies = [
+ "dlv-list",
+ "hashbrown 0.12.3",
+]
+
 [[package]]
 name = "overload"
 version = "0.1.1"
@@ -2198,7 +2312,7 @@ checksum = "93f00c865fe7cabf650081affecd3871070f26767e7b2070a3ffae14c654b447"
 dependencies = [
  "cfg-if",
  "libc",
- "redox_syscall",
+ "redox_syscall 0.3.5",
  "smallvec",
  "windows-targets",
 ]
@@ -2376,6 +2490,16 @@ dependencies = [
  "psl-types",
 ]
 
+[[package]]
+name = "quick-xml"
+version = "0.26.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7f50b1c63b38611e7d4d7f68b82d3ad0cc71a2ad2e7f61fc10f1328d917c93cd"
+dependencies = [
+ "memchr",
+ "serde",
+]
+
 [[package]]
 name = "quote"
 version = "1.0.33"
@@ -2534,6 +2658,15 @@ dependencies = [
  "url",
 ]
 
+[[package]]
+name = "redox_syscall"
+version = "0.2.16"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a"
+dependencies = [
+ "bitflags 1.3.2",
+]
+
 [[package]]
 name = "redox_syscall"
 version = "0.3.5"
@@ -2543,6 +2676,17 @@ dependencies = [
  "bitflags 1.3.2",
 ]
 
+[[package]]
+name = "redox_users"
+version = "0.4.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b"
+dependencies = [
+ "getrandom 0.2.10",
+ "redox_syscall 0.2.16",
+ "thiserror",
+]
+
 [[package]]
 name = "regex"
 version = "1.9.5"
@@ -2623,10 +2767,12 @@ dependencies = [
  "tokio",
  "tokio-native-tls",
  "tokio-rustls",
+ "tokio-util",
  "tower-service",
  "url",
  "wasm-bindgen",
  "wasm-bindgen-futures",
+ "wasm-streams",
  "web-sys",
  "webpki-roots 0.25.2",
  "winreg",
@@ -2669,6 +2815,49 @@ dependencies = [
  "zeroize",
 ]
 
+[[package]]
+name = "rust-ini"
+version = "0.18.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f6d5f2436026b4f6e79dc829837d467cc7e9a55ee40e750d716713540715a2df"
+dependencies = [
+ "cfg-if",
+ "ordered-multimap",
+]
+
+[[package]]
+name = "rust-s3"
+version = "0.33.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1b2ac5ff6acfbe74226fa701b5ef793aaa054055c13ebb7060ad36942956e027"
+dependencies = [
+ "async-trait",
+ "aws-creds",
+ "aws-region",
+ "base64 0.13.1",
+ "bytes",
+ "cfg-if",
+ "futures",
+ "hex",
+ "hmac",
+ "http",
+ "log",
+ "maybe-async",
+ "md5",
+ "minidom",
+ "percent-encoding",
+ "quick-xml",
+ "reqwest",
+ "serde",
+ "serde_derive",
+ "sha2",
+ "thiserror",
+ "time",
+ "tokio",
+ "tokio-stream",
+ "url",
+]
+
 [[package]]
 name = "rustc-demangle"
 version = "0.1.23"
@@ -2737,6 +2926,23 @@ dependencies = [
  "untrusted",
 ]
 
+[[package]]
+name = "rxml"
+version = "0.9.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a98f186c7a2f3abbffb802984b7f1dfd65dac8be1aafdaabbca4137f53f0dff7"
+dependencies = [
+ "bytes",
+ "rxml_validation",
+ "smartstring",
+]
+
+[[package]]
+name = "rxml_validation"
+version = "0.9.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "22a197350ece202f19a166d1ad6d9d6de145e1d2a8ef47db299abe164dbd7530"
+
 [[package]]
 name = "ryu"
 version = "1.0.15"
@@ -2983,6 +3189,17 @@ version = "1.11.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "62bb4feee49fdd9f707ef802e22365a35de4b7b299de4763d44bfea899442ff9"
 
+[[package]]
+name = "smartstring"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3fb72c633efbaa2dd666986505016c32c3044395ceaf881518399d2f4127ee29"
+dependencies = [
+ "autocfg",
+ "static_assertions",
+ "version_check",
+]
+
 [[package]]
 name = "snowflake"
 version = "0.1.0"
@@ -3062,7 +3279,7 @@ version = "0.7.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "dd4cef4251aabbae751a3710927945901ee1d97ee96d757f6880ebb9a79bfd53"
 dependencies = [
- "ahash",
+ "ahash 0.8.3",
  "atoi",
  "byteorder",
  "bytes",
@@ -3249,6 +3466,12 @@ dependencies = [
  "uuid",
 ]
 
+[[package]]
+name = "static_assertions"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f"
+
 [[package]]
 name = "storage"
 version = "0.1.0"
@@ -3341,7 +3564,7 @@ checksum = "cb94d2f3cc536af71caac6b6fcebf65860b347e7ce0cc9ebe8f70d3e521054ef"
 dependencies = [
  "cfg-if",
  "fastrand",
- "redox_syscall",
+ "redox_syscall 0.3.5",
  "rustix",
  "windows-sys",
 ]
@@ -3590,7 +3813,7 @@ version = "0.3.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b5c266b9ac83dedf0e0385ad78514949e6d89491269e7065bee51d2bb8ec7373"
 dependencies = [
- "ahash",
+ "ahash 0.8.3",
  "gethostname",
  "log",
  "serde",
@@ -3923,6 +4146,19 @@ version = "0.2.87"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ca6ad05a4870b2bf5fe995117d3728437bd27d7cd5f06f13c17443ef369775a1"
 
+[[package]]
+name = "wasm-streams"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b4609d447824375f43e1ffbc051b50ad8f4b3ae8219680c94452ea05eb240ac7"
+dependencies = [
+ "futures-util",
+ "js-sys",
+ "wasm-bindgen",
+ "wasm-bindgen-futures",
+ "web-sys",
+]
+
 [[package]]
 name = "web-sys"
 version = "0.3.64"

Cargo.toml

@@ -47,6 +47,9 @@ validator = "0.16.0"
 bytes = "1.4.0"
 rcgen = { version = "0.10.0", features = ["pem", "x509-parser"] }
 jsonwebtoken = "8.3.0"
+# aws-config = "0.56.1"
+# aws-sdk-s3 = "0.31.1"
+rust-s3 = "0.33.0"
 
 # tracing
 tracing = { version = "0.1.37" }

configuration/base.yaml

@@ -17,3 +17,10 @@ redis_uri: "redis://127.0.0.1:6379"
 gotrue:
   base_url: "http://127.0.0.1:9999"
   ext_url: "http://127.0.0.1:9999"
+s3:
+  use_minio: true
+  minio_url: http://localhost:9000
+  access_key: minioadmin
+  secret_key: minioadmin
+  bucket: appflowy
+  region: us-east-1

configuration/production.yaml

@@ -9,3 +9,5 @@ redis_uri: "redis://redis:6379"
 gotrue:
   base_url: "http://gotrue:9999"
   ext_url: "http://gotrue:9999"
+s3:
+  minio_url: http://minio:9000

dev.env

@@ -39,3 +39,10 @@ GOTRUE_EXTERNAL_GOOGLE_CLIENT_ID=
 GOTRUE_EXTERNAL_GOOGLE_SECRET=
 GOTRUE_EXTERNAL_GOOGLE_REDIRECT_URI=http://localhost:9998/callback
 
+# File Storage
+USE_MINIO=true
+# MINIO_URL=http://localhost:9000 # change this if you are using a different address for minio
+AWS_ACCESS_KEY_ID=minioadmin
+AWS_SECRET_ACCESS_KEY=minioadmin
+AWS_S3_BUCKET=appflowy
+AWS_REGION=us-east-1

docker-compose-dev.yml

@@ -1,5 +1,12 @@
 version: '3'
 services:
+  minio:
+    image: minio/minio
+    ports:
+      - 9000:9000
+      - 9001:9001
+    command: server /data --console-address ":9001"
+
   postgres:
     build:
       context: .

docker-compose.yml

@@ -13,6 +13,13 @@ services:
       - ./nginx/ssl/certificate.crt:/etc/nginx/ssl/certificate.crt
       - ./nginx/ssl/private_key.key:/etc/nginx/ssl/private_key.key
 
+  minio:
+    image: minio/minio
+    ports:
+      - 9000:9000
+      - 9001:9001
+    command: server /data --console-address ":9001"
+
   postgres:
     build:
       context: .
@@ -67,6 +74,12 @@ services:
       - APP_ENVIRONMENT=production
       - APP__GOTRUE__JWT_SECRET=${GOTRUE_JWT_SECRET}
       - APP__GOTRUE__EXT_URL=${API_EXTERNAL_URL}
+      - APP__S3__USE_MINIO=${USE_MINIO}
+      - APP__S3__MINIO_URL=${MINIO_URL:-http://minio:9000}
+      - APP__S3__AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}
+      - APP__S3__AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY}
+      - APP__S3__AWS_S3_BUCKET=${AWS_S3_BUCKET}
+      - APP__S3__AWS_REGION=${AWS_REGION}
     build:
       context: .
       dockerfile: Dockerfile

src/application.rs

@@ -1,6 +1,6 @@
 use crate::api::{collab_scope, user_scope, workspace_scope, ws_scope};
 use crate::component::auth::HEADER_TOKEN;
-use crate::config::config::{Config, DatabaseSetting, GoTrueSetting, TlsConfig};
+use crate::config::config::{Config, DatabaseSetting, GoTrueSetting, S3Setting, TlsConfig};
 use crate::middleware::cors::default_cors;
 use crate::self_signed::create_self_signed_certificate;
 use crate::state::{AppState, Storage};
@@ -11,7 +11,6 @@ use actix_web::cookie::Key;
 use actix_web::{dev::Server, web, web::Data, App, HttpServer};
 
 use actix::Actor;
-
 use openssl::ssl::{SslAcceptor, SslAcceptorBuilder, SslFiletype, SslMethod};
 use openssl::x509::X509;
 use secrecy::{ExposeSecret, Secret};
@@ -127,6 +126,7 @@ fn get_certificate_and_server_key(config: &Config) -> Option<(Secret<String>, Se
 pub async fn init_state(config: &Config) -> AppState {
   let pg_pool = get_connection_pool(&config.database).await;
   migrate(&pg_pool).await;
+  let s3_bucket = get_aws_s3_client(&config.s3).await;
 
   let gotrue_client = get_gotrue_client(&config.gotrue).await;
 
@@ -136,9 +136,65 @@ pub async fn init_state(config: &Config) -> AppState {
     user: Arc::new(Default::default()),
     id_gen: Arc::new(RwLock::new(Snowflake::new(1))),
     gotrue_client,
+    s3_bucket,
   }
 }
 
+async fn get_aws_s3_client(s3_setting: &S3Setting) -> s3::Bucket {
+  let region = {
+    match s3_setting.use_minio {
+      true => s3::Region::Custom {
+        region: "".to_owned(),
+        endpoint: s3_setting.minio_url.to_owned(),
+      },
+      false => s3_setting.region.parse::<s3::Region>().unwrap(),
+    }
+  };
+
+  let cred = s3::creds::Credentials {
+    access_key: Some(s3_setting.access_key.to_owned()),
+    secret_key: Some(s3_setting.secret_key.to_owned()),
+    security_token: None,
+    session_token: None,
+    expiration: None,
+  };
+
+  match s3::Bucket::create_with_path_style(
+    &s3_setting.bucket,
+    region.clone(),
+    cred.clone(),
+    s3::BucketConfiguration::default(),
+  )
+  .await
+  {
+    Ok(_) => {},
+    Err(e) => match e {
+      s3::error::S3Error::Http(409, _) => {}, // Bucket already exists
+      _ => panic!("Failed to create bucket: {:?}", e),
+    },
+  }
+
+  s3::Bucket::new(&s3_setting.bucket, region.clone(), cred.clone()).unwrap()
+}
+
+// async fn get_aws_s3_client() -> aws_sdk_s3::Client {
+//   let credentials = Credentials::new("minioadmin", "minioadmin", None, None, "none");
+//   let config = aws_config::SdkConfig::builder()
+//     .set_region(Region {})
+//     .endpoint_url("http://localhost:9000")
+//     .credentials_provider(Some(credentials))
+//     .build();
+//
+//   let client = aws_sdk_s3::Client::new(&config);
+//   client
+//     .create_bucket()
+//     .bucket("hellothisisme")
+//     .send()
+//     .await
+//     .unwrap();
+//   client
+// }
+
 async fn get_connection_pool(setting: &DatabaseSetting) -> PgPool {
   PgPoolOptions::new()
     .acquire_timeout(std::time::Duration::from_secs(5))

src/config/config.rs

@@ -12,6 +12,17 @@ pub struct Config {
   pub application: ApplicationSetting,
   pub websocket: WebsocketSetting,
   pub redis_uri: Secret<String>,
+  pub s3: S3Setting,
+}
+
+#[derive(serde::Deserialize, Clone, Debug)]
+pub struct S3Setting {
+  pub use_minio: bool,
+  pub minio_url: String,
+  pub access_key: String,
+  pub secret_key: String,
+  pub bucket: String,
+  pub region: String,
 }
 
 #[derive(serde::Deserialize, Clone, Debug)]

src/state.rs

@@ -16,6 +16,7 @@ pub struct AppState {
   pub user: Arc<RwLock<UserCache>>,
   pub id_gen: Arc<RwLock<Snowflake>>,
   pub gotrue_client: gotrue::api::Client,
+  pub s3_bucket: s3::Bucket,
 }
 
 impl AppState {