Merge pull request #257 from AppFlowy-IO/hander_large_blob

fix: large file test case

deploy.env

@@ -51,13 +51,14 @@ GOTRUE_EXTERNAL_DISCORD_ENABLED=false
 GOTRUE_EXTERNAL_DISCORD_CLIENT_ID=
 GOTRUE_EXTERNAL_DISCORD_SECRET=
 GOTRUE_EXTERNAL_DISCORD_REDIRECT_URI=http://your-host/callback
+
 # File Storage
-USE_MINIO=true
+APPFLOWY_S3_USE_MINIO=true
-# MINIO_URL=http://localhost:9000 # change this if you are using a different address for minio
+APPFLOWY_S3_MINIO_URL=http://minio:9000 # change this if you are using a different address for minio
-AWS_ACCESS_KEY_ID=minioadmin
+APPFLOWY_S3_ACCESS_KEY=minioadmin
-AWS_SECRET_ACCESS_KEY=minioadmin
+APPFLOWY_S3_SECRET_KEY=minioadmin
-AWS_S3_BUCKET=appflowy
+APPFLOWY_S3_BUCKET=appflowy
-#AWS_REGION=us-east-1
+#APPFLOWY_S3_REGION=us-east-1
 
 RUST_LOG=info
 
@@ -72,9 +73,5 @@ PORTAINER_PASSWORD=password1234
 CLOUDFLARE_TUNNEL_TOKEN=
 
 # If you are using a different postgres database, change the following values
-# POSTGRES_USER=
+# GOTRUE_DATABASE_URL=postgres://supabase_auth_admin:root@<host>:<port>/$POSTGRES_DB
-# POSTGRES_DB=
-# POSTGRES_PASSWORD=
-# POSTGRES_HOST=
-# GOTRUE_DATABASE_URL=postgres://supabase_auth_admin:$POSTGRES_PASSWORD@$POSTGRES_HOST:5432/$POSTGRES_DB
 # APPFLOWY_DATABASE_URL=postgres://POSTGRES_USER:$POSTGRES_PASSWORD@$POSTGRES_HOST:5432/$POSTGRES_DB

dev.env

@@ -56,13 +56,14 @@ GOTRUE_EXTERNAL_DISCORD_ENABLED=false
 GOTRUE_EXTERNAL_DISCORD_CLIENT_ID=
 GOTRUE_EXTERNAL_DISCORD_SECRET=
 GOTRUE_EXTERNAL_DISCORD_REDIRECT_URI=http://localhost:9999/callback
+
 # File Storage
-USE_MINIO=true
+APPFLOWY_S3_USE_MINIO=true
-# MINIO_URL=http://localhost:9000 # change this if you are using a different address for minio
+APPFLOWY_S3_MINIO_URL=http://localhost:9000 # change this if you are using a different address for minio
-AWS_ACCESS_KEY_ID=minioadmin
+APPFLOWY_S3_ACCESS_KEY=minioadmin
-AWS_SECRET_ACCESS_KEY=minioadmin
+APPFLOWY_S3_SECRET_KEY=minioadmin
-AWS_S3_BUCKET=appflowy
+APPFLOWY_S3_BUCKET=appflowy
-#AWS_REGION=us-east-1
+#APPFLOWY_S3_REGION=us-east-1
 
 RUST_LOG=info
 

doc/DEPLOYMENT.md

@@ -179,7 +179,20 @@ with your own in `nginx/ssl/` directory
 
 ## 5. FAQ
 - How do I use a different `postgres`?
-1. You need set `APPFLOWY_DATABASE_URL` to another postgres url.
-You also need to set `DATABASE_URL` to use the same postgres database.
 The default url is using the postgres in docker compose, in service `appflowy_cloud` and `gotrue` respectively.
-2. You would need to run the initialization sql file from `migrations/before` in your hosted postgres.
+However it is possible to change the database storage for it. The following steps are listed below.
+
+1. You need set `APPFLOWY_DATABASE_URL` to another postgres url.
+```
+APPFLOWY_DATABASE_URL=postgres://<postgres_user>:<password>@<host>:<port>/<dbname>
+```
+
+2. You also need to set `GOTRUE_DATABASE_URL` to use the same postgres database.
+```
+GOTRUE_DATABASE_URL=postgres://supabase_auth_admin:root@<host>:<port>/<dbname>
+```
+- `supabase_auth_admin` and `root` must be kept in sync with the init migration scripts from `migrations/before`.
+Currently it's possible to change the password, but probably can't change the username.
+- `dbname` for `appflowy_cloud` and `gotrue` must be the same.
+
+3. You would need to run the initialization sql file from `migrations/before` in your hosted postgres.

docker-compose.yml

@@ -96,12 +96,12 @@ services:
       - APPFLOWY_GOTRUE_EXT_URL=${API_EXTERNAL_URL}
       - APPFLOWY_GOTRUE_ADMIN_EMAIL=${GOTRUE_ADMIN_EMAIL}
       - APPFLOWY_GOTRUE_ADMIN_PASSWORD=${GOTRUE_ADMIN_PASSWORD}
-      - APPFLOWY_S3_USE_MINIO=${USE_MINIO}
+      - APPFLOWY_S3_USE_MINIO=${APPFLOWY_S3_USE_MINIO}
-      - APPFLOWY_S3_MINIO_URL=${MINIO_URL:-http://minio:9000}
+      - APPFLOWY_S3_MINIO_URL=${APPFLOWY_S3_MINIO_URL}
-      - APPFLOWY_S3_ACCESS_KEY=${AWS_ACCESS_KEY_ID}
+      - APPFLOWY_S3_ACCESS_KEY=${APPFLOWY_S3_ACCESS_KEY}
-      - APPFLOWY_S3_SECRET_KEY=${AWS_SECRET_ACCESS_KEY}
+      - APPFLOWY_S3_SECRET_KEY=${APPFLOWY_S3_SECRET_KEY}
-      - APPFLOWY_S3_BUCKET=${AWS_S3_BUCKET}
+      - APPFLOWY_S3_BUCKET=${APPFLOWY_S3_BUCKET}
-      - APPFLOWY_S3_REGION=${AWS_REGION}
+      - APPFLOWY_S3_REGION=${APPFLOWY_S3_REGION}
     build:
       context: .
       dockerfile: Dockerfile

libs/client-api/src/http.rs

@@ -25,7 +25,7 @@ use gotrue::params::{AdminUserParams, GenerateLinkParams};
 use mime::Mime;
 use parking_lot::RwLock;
 use realtime_entity::EncodedCollab;
-use reqwest::{header, Body};
+use reqwest::{header, Body, StatusCode};
 
 use collab_entity::CollabType;
 use reqwest::header::HeaderValue;
@@ -1103,7 +1103,12 @@ impl Client {
       .send()
       .await?;
     log_request_id(&resp);
-    Ok(())
+    if resp.status() == StatusCode::PAYLOAD_TOO_LARGE {
+      return Err(AppResponseError::from(AppError::PayloadTooLarge(
+        StatusCode::PAYLOAD_TOO_LARGE.to_string(),
+      )));
+    }
+    AppResponse::<()>::from_response(resp).await?.into_error()
   }
 
   /// Only expose this method for testing

src/api/file_storage.rs

@@ -13,7 +13,6 @@ use app_error::AppError;
 use chrono::DateTime;
 use database::file::{MAX_BLOB_SIZE, MAX_USAGE};
 use database::resource_usage::{get_all_workspace_blob_metadata, get_workspace_usage_size};
-use database_entity::dto::AFBlobRecord;
 use shared_entity::dto::workspace_dto::{BlobMetadata, RepeatedBlobMetaData, WorkspaceSpaceUsage};
 use shared_entity::response::{AppResponse, AppResponseError, JsonAppResponse};
 use sqlx::types::Uuid;
@@ -53,7 +52,7 @@ async fn put_blob_handler(
   content_type: web::Header<ContentType>,
   content_length: web::Header<ContentLength>,
   payload: Payload,
-) -> Result<JsonAppResponse<AFBlobRecord>> {
+) -> Result<JsonAppResponse<()>> {
   let (workspace_id, file_id) = path.into_inner();
   let content_length = content_length.into_inner().into_inner();
   let content_type = content_type.into_inner().to_string();
@@ -103,9 +102,7 @@ async fn put_blob_handler(
     .await
     .map_err(AppResponseError::from)?;
 
-  let record = AFBlobRecord::new(file_id);
+  Ok(AppResponse::Ok().into())
-  event!(tracing::Level::TRACE, "did put blob: {:?}", record);
-  Ok(Json(AppResponse::Ok().with_data(record)))
 }
 
 #[instrument(level = "debug", skip(state), err)]

tests/workspace/blob/put_and_get.rs

@@ -42,19 +42,19 @@ async fn put_and_get() {
 }
 
 // TODO: fix inconsistent behavior due to different error handling with nginx
-// #[tokio::test]
+#[tokio::test]
-// async fn put_giant_file() {
+async fn put_giant_file() {
-//   let (c1, _user1) = generate_unique_registered_user_client().await;
+  let (c1, _user1) = generate_unique_registered_user_client().await;
-//   let workspace_id = workspace_id_from_client(&c1).await;
+  let workspace_id = workspace_id_from_client(&c1).await;
-//   let mime = mime::TEXT_PLAIN_UTF_8;
+  let mime = mime::TEXT_PLAIN_UTF_8;
-//   let file_id = uuid::Uuid::new_v4().to_string();
+  let file_id = uuid::Uuid::new_v4().to_string();
-//
+
-//   let url = c1.get_blob_url(&workspace_id, &file_id);
+  let url = c1.get_blob_url(&workspace_id, &file_id);
-//   let data = vec![0; 10 * 1024 * 1024 * 1024];
+  let data = vec![0; 10 * 1024 * 1024 * 1024];
-//   let error = c1.put_blob(&url, data, &mime).await.unwrap_err();
+  let error = c1.put_blob(&url, data, &mime).await.unwrap_err();
-//
+
-//   assert_eq!(error.code, ErrorCode::PayloadTooLarge);
+  assert_eq!(error.code, ErrorCode::PayloadTooLarge);
-// }
+}
 
 #[tokio::test]
 async fn put_and_put_and_get() {