From 0f4d4e4b3c95dd4e2f44913e03fa54664b4248d2 Mon Sep 17 00:00:00 2001 From: Fu Zi Xiang Date: Fri, 12 Jan 2024 17:59:25 +0800 Subject: [PATCH 1/6] fix: large file test case --- libs/client-api/src/http.rs | 2 +- tests/workspace/blob/put_and_get.rs | 26 +++++++++++++------------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/libs/client-api/src/http.rs b/libs/client-api/src/http.rs index b303e432..a90126a7 100644 --- a/libs/client-api/src/http.rs +++ b/libs/client-api/src/http.rs @@ -1096,7 +1096,7 @@ impl Client { .send() .await?; log_request_id(&resp); - Ok(()) + AppResponse::<()>::from_response(resp).await?.into_error() } /// Only expose this method for testing diff --git a/tests/workspace/blob/put_and_get.rs b/tests/workspace/blob/put_and_get.rs index 6c2b7686..8e83bac6 100644 --- a/tests/workspace/blob/put_and_get.rs +++ b/tests/workspace/blob/put_and_get.rs @@ -42,19 +42,19 @@ async fn put_and_get() { } // TODO: fix inconsistent behavior due to different error handling with nginx -// #[tokio::test] -// async fn put_giant_file() { -// let (c1, _user1) = generate_unique_registered_user_client().await; -// let workspace_id = workspace_id_from_client(&c1).await; -// let mime = mime::TEXT_PLAIN_UTF_8; -// let file_id = uuid::Uuid::new_v4().to_string(); -// -// let url = c1.get_blob_url(&workspace_id, &file_id); -// let data = vec![0; 10 * 1024 * 1024 * 1024]; -// let error = c1.put_blob(&url, data, &mime).await.unwrap_err(); -// -// assert_eq!(error.code, ErrorCode::PayloadTooLarge); -// } +#[tokio::test] +async fn put_giant_file() { + let (c1, _user1) = generate_unique_registered_user_client().await; + let workspace_id = workspace_id_from_client(&c1).await; + let mime = mime::TEXT_PLAIN_UTF_8; + let file_id = uuid::Uuid::new_v4().to_string(); + + let url = c1.get_blob_url(&workspace_id, &file_id); + let data = vec![0; 10 * 1024 * 1024 * 1024]; + let error = c1.put_blob(&url, data, &mime).await.unwrap_err(); + + assert_eq!(error.code, ErrorCode::PayloadTooLarge); +} #[tokio::test] async fn put_and_put_and_get() { From c9d2f4f9bde9d389dba627f5b19a810d00574afe Mon Sep 17 00:00:00 2001 From: Fu Zi Xiang Date: Fri, 12 Jan 2024 17:59:25 +0800 Subject: [PATCH 2/6] fix: update file storage configs --- deploy.env | 13 +++++++------ dev.env | 13 +++++++------ 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/deploy.env b/deploy.env index fb329289..25472afd 100644 --- a/deploy.env +++ b/deploy.env @@ -51,13 +51,14 @@ GOTRUE_EXTERNAL_DISCORD_ENABLED=false GOTRUE_EXTERNAL_DISCORD_CLIENT_ID= GOTRUE_EXTERNAL_DISCORD_SECRET= GOTRUE_EXTERNAL_DISCORD_REDIRECT_URI=http://your-host/callback + # File Storage -USE_MINIO=true -# MINIO_URL=http://localhost:9000 # change this if you are using a different address for minio -AWS_ACCESS_KEY_ID=minioadmin -AWS_SECRET_ACCESS_KEY=minioadmin -AWS_S3_BUCKET=appflowy -#AWS_REGION=us-east-1 +APPFLOWY_S3_USE_MINIO=true +# APPFLOWY_S3_MINIO_URL=http://localhost:9000 # change this if you are using a different address for minio +APPFLOWY_S3_ACCESS_KEY=minioadmin +APPFLOWY_S3_SECRET_KEY=minioadmin +APPFLOWY_S3_BUCKET=appflowy +#APPFLOWY_S3_REGION=us-east-1 RUST_LOG=info diff --git a/dev.env b/dev.env index c7b30e31..d60200d8 100644 --- a/dev.env +++ b/dev.env @@ -56,13 +56,14 @@ GOTRUE_EXTERNAL_DISCORD_ENABLED=false GOTRUE_EXTERNAL_DISCORD_CLIENT_ID= GOTRUE_EXTERNAL_DISCORD_SECRET= GOTRUE_EXTERNAL_DISCORD_REDIRECT_URI=http://localhost:9999/callback + # File Storage -USE_MINIO=true -# MINIO_URL=http://localhost:9000 # change this if you are using a different address for minio -AWS_ACCESS_KEY_ID=minioadmin -AWS_SECRET_ACCESS_KEY=minioadmin -AWS_S3_BUCKET=appflowy -#AWS_REGION=us-east-1 +APPFLOWY_S3_USE_MINIO=true +# APPFLOWY_S3_MINIO_URL=http://localhost:9000 # change this if you are using a different address for minio +APPFLOWY_S3_ACCESS_KEY=minioadmin +APPFLOWY_S3_SECRET_KEY=minioadmin +APPFLOWY_S3_BUCKET=appflowy +#APPFLOWY_S3_REGION=us-east-1 RUST_LOG=info From 1b18c82dfaa1588ee0631c173dca89e830e414ad Mon Sep 17 00:00:00 2001 From: Fu Zi Xiang Date: Fri, 12 Jan 2024 22:06:05 +0800 Subject: [PATCH 3/6] fix: set nginx 413 error as payload too large --- libs/client-api/src/http.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/libs/client-api/src/http.rs b/libs/client-api/src/http.rs index a90126a7..db4646b9 100644 --- a/libs/client-api/src/http.rs +++ b/libs/client-api/src/http.rs @@ -25,7 +25,7 @@ use gotrue::params::{AdminUserParams, GenerateLinkParams}; use mime::Mime; use parking_lot::RwLock; use realtime_entity::EncodedCollab; -use reqwest::{header, Body}; +use reqwest::{header, Body, StatusCode}; use collab_entity::CollabType; use reqwest::header::HeaderValue; @@ -1096,6 +1096,11 @@ impl Client { .send() .await?; log_request_id(&resp); + if resp.status() == StatusCode::PAYLOAD_TOO_LARGE { + return Err(AppResponseError::from(AppError::PayloadTooLarge( + StatusCode::PAYLOAD_TOO_LARGE.to_string(), + ))); + } AppResponse::<()>::from_response(resp).await?.into_error() } From 851cd6929e9f5ea1ffeed38322aab27323bdf06a Mon Sep 17 00:00:00 2001 From: Fu Zi Xiang Date: Fri, 12 Jan 2024 22:50:25 +0800 Subject: [PATCH 4/6] fix: minio url configs --- deploy.env | 2 +- dev.env | 2 +- docker-compose.yml | 12 ++++++------ 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/deploy.env b/deploy.env index 25472afd..5ed243cc 100644 --- a/deploy.env +++ b/deploy.env @@ -54,7 +54,7 @@ GOTRUE_EXTERNAL_DISCORD_REDIRECT_URI=http://your-host/callback # File Storage APPFLOWY_S3_USE_MINIO=true -# APPFLOWY_S3_MINIO_URL=http://localhost:9000 # change this if you are using a different address for minio +APPFLOWY_S3_MINIO_URL=http://minio:9000 # change this if you are using a different address for minio APPFLOWY_S3_ACCESS_KEY=minioadmin APPFLOWY_S3_SECRET_KEY=minioadmin APPFLOWY_S3_BUCKET=appflowy diff --git a/dev.env b/dev.env index d60200d8..2503722d 100644 --- a/dev.env +++ b/dev.env @@ -59,7 +59,7 @@ GOTRUE_EXTERNAL_DISCORD_REDIRECT_URI=http://localhost:9999/callback # File Storage APPFLOWY_S3_USE_MINIO=true -# APPFLOWY_S3_MINIO_URL=http://localhost:9000 # change this if you are using a different address for minio +APPFLOWY_S3_MINIO_URL=http://localhost:9000 # change this if you are using a different address for minio APPFLOWY_S3_ACCESS_KEY=minioadmin APPFLOWY_S3_SECRET_KEY=minioadmin APPFLOWY_S3_BUCKET=appflowy diff --git a/docker-compose.yml b/docker-compose.yml index 2dfa7a13..70070b23 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -96,12 +96,12 @@ services: - APPFLOWY_GOTRUE_EXT_URL=${API_EXTERNAL_URL} - APPFLOWY_GOTRUE_ADMIN_EMAIL=${GOTRUE_ADMIN_EMAIL} - APPFLOWY_GOTRUE_ADMIN_PASSWORD=${GOTRUE_ADMIN_PASSWORD} - - APPFLOWY_S3_USE_MINIO=${USE_MINIO} - - APPFLOWY_S3_MINIO_URL=${MINIO_URL:-http://minio:9000} - - APPFLOWY_S3_ACCESS_KEY=${AWS_ACCESS_KEY_ID} - - APPFLOWY_S3_SECRET_KEY=${AWS_SECRET_ACCESS_KEY} - - APPFLOWY_S3_BUCKET=${AWS_S3_BUCKET} - - APPFLOWY_S3_REGION=${AWS_REGION} + - APPFLOWY_S3_USE_MINIO=${APPFLOWY_S3_USE_MINIO} + - APPFLOWY_S3_MINIO_URL=${APPFLOWY_S3_MINIO_URL} + - APPFLOWY_S3_ACCESS_KEY=${APPFLOWY_S3_ACCESS_KEY} + - APPFLOWY_S3_SECRET_KEY=${APPFLOWY_S3_SECRET_KEY} + - APPFLOWY_S3_BUCKET=${APPFLOWY_S3_BUCKET} + - APPFLOWY_S3_REGION=${APPFLOWY_S3_REGION} build: context: . dockerfile: Dockerfile From 8ba4559cbd19a34e0803b0c056740e513a6f19e0 Mon Sep 17 00:00:00 2001 From: Fu Zi Xiang Date: Sat, 13 Jan 2024 00:53:41 +0800 Subject: [PATCH 5/6] fix: result return from put blob --- src/api/file_storage.rs | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/api/file_storage.rs b/src/api/file_storage.rs index 13272e9f..61fcfa04 100644 --- a/src/api/file_storage.rs +++ b/src/api/file_storage.rs @@ -13,7 +13,6 @@ use app_error::AppError; use chrono::DateTime; use database::file::{MAX_BLOB_SIZE, MAX_USAGE}; use database::resource_usage::{get_all_workspace_blob_metadata, get_workspace_usage_size}; -use database_entity::dto::AFBlobRecord; use shared_entity::dto::workspace_dto::{BlobMetadata, RepeatedBlobMetaData, WorkspaceSpaceUsage}; use shared_entity::response::{AppResponse, AppResponseError, JsonAppResponse}; use sqlx::types::Uuid; @@ -53,7 +52,7 @@ async fn put_blob_handler( content_type: web::Header, content_length: web::Header, payload: Payload, -) -> Result> { +) -> Result> { let (workspace_id, file_id) = path.into_inner(); let content_length = content_length.into_inner().into_inner(); let content_type = content_type.into_inner().to_string(); @@ -103,9 +102,7 @@ async fn put_blob_handler( .await .map_err(AppResponseError::from)?; - let record = AFBlobRecord::new(file_id); - event!(tracing::Level::TRACE, "did put blob: {:?}", record); - Ok(Json(AppResponse::Ok().with_data(record))) + Ok(AppResponse::Ok().into()) } #[instrument(level = "debug", skip(state), err)] From d8c03055cdbb6561b587aa13e407e09abe38e931 Mon Sep 17 00:00:00 2001 From: Fu Zi Xiang Date: Sun, 14 Jan 2024 18:46:15 +0800 Subject: [PATCH 6/6] feat: update deployment docs --- deploy.env | 6 +----- doc/DEPLOYMENT.md | 19 ++++++++++++++++--- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/deploy.env b/deploy.env index 5ed243cc..469fd2db 100644 --- a/deploy.env +++ b/deploy.env @@ -73,9 +73,5 @@ PORTAINER_PASSWORD=password1234 CLOUDFLARE_TUNNEL_TOKEN= # If you are using a different postgres database, change the following values -# POSTGRES_USER= -# POSTGRES_DB= -# POSTGRES_PASSWORD= -# POSTGRES_HOST= -# GOTRUE_DATABASE_URL=postgres://supabase_auth_admin:$POSTGRES_PASSWORD@$POSTGRES_HOST:5432/$POSTGRES_DB +# GOTRUE_DATABASE_URL=postgres://supabase_auth_admin:root@:/$POSTGRES_DB # APPFLOWY_DATABASE_URL=postgres://POSTGRES_USER:$POSTGRES_PASSWORD@$POSTGRES_HOST:5432/$POSTGRES_DB diff --git a/doc/DEPLOYMENT.md b/doc/DEPLOYMENT.md index cba34390..51d6c8dd 100644 --- a/doc/DEPLOYMENT.md +++ b/doc/DEPLOYMENT.md @@ -179,7 +179,20 @@ with your own in `nginx/ssl/` directory ## 5. FAQ - How do I use a different `postgres`? -1. You need set `APPFLOWY_DATABASE_URL` to another postgres url. -You also need to set `DATABASE_URL` to use the same postgres database. The default url is using the postgres in docker compose, in service `appflowy_cloud` and `gotrue` respectively. -2. You would need to run the initialization sql file from `migrations/before` in your hosted postgres. +However it is possible to change the database storage for it. The following steps are listed below. + +1. You need set `APPFLOWY_DATABASE_URL` to another postgres url. +``` +APPFLOWY_DATABASE_URL=postgres://:@:/ +``` + +2. You also need to set `GOTRUE_DATABASE_URL` to use the same postgres database. +``` +GOTRUE_DATABASE_URL=postgres://supabase_auth_admin:root@:/ +``` +- `supabase_auth_admin` and `root` must be kept in sync with the init migration scripts from `migrations/before`. +Currently it's possible to change the password, but probably can't change the username. +- `dbname` for `appflowy_cloud` and `gotrue` must be the same. + +3. You would need to run the initialization sql file from `migrations/before` in your hosted postgres.