From 6d0612ffaf1b3f90d20d57f0f767d369a5fbc564 Mon Sep 17 00:00:00 2001 From: "Nathan.fooo" <86001920+appflowy@users.noreply.github.com> Date: Wed, 3 Jan 2024 06:07:21 +0800 Subject: [PATCH] feat: encode batch create collab data as binary (#242) --- Cargo.lock | 1 + libs/client-api/src/http.rs | 7 ++++++- libs/database-entity/Cargo.toml | 1 + libs/database-entity/src/dto.rs | 17 +++++++++++++++ nginx/nginx.conf | 2 +- src/api/workspace.rs | 8 +++++-- src/middleware/request_id.rs | 36 +++++++++++++++++++------------- tests/collab/collab_curd_test.rs | 2 +- 8 files changed, 55 insertions(+), 19 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 54b165bb..b7bf3ccd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1751,6 +1751,7 @@ version = "0.1.0" dependencies = [ "anyhow", "app-error", + "bincode", "chrono", "collab-entity", "serde", diff --git a/libs/client-api/src/http.rs b/libs/client-api/src/http.rs index 4d5e7018..d180a54e 100644 --- a/libs/client-api/src/http.rs +++ b/libs/client-api/src/http.rs @@ -729,10 +729,15 @@ impl Client { workspace_id: workspace_id.to_string(), params_list: params, }; + let resp = self .http_client_with_auth(Method::POST, &url) .await? - .json(&payload) + .body( + payload + .to_bytes() + .map_err(|err| AppError::Internal(err.into()))?, + ) .send() .await?; log_request_id(&resp); diff --git a/libs/database-entity/Cargo.toml b/libs/database-entity/Cargo.toml index d631284c..7211bfde 100644 --- a/libs/database-entity/Cargo.toml +++ b/libs/database-entity/Cargo.toml @@ -18,3 +18,4 @@ anyhow = "1.0.75" tracing = "0.1" serde_repr = "0.1.16" app-error = { workspace = true } +bincode = "1.3.3" \ No newline at end of file diff --git a/libs/database-entity/src/dto.rs b/libs/database-entity/src/dto.rs index ee291143..5d9e87c4 100644 --- a/libs/database-entity/src/dto.rs +++ b/libs/database-entity/src/dto.rs @@ -31,6 +31,13 @@ impl CreateCollabParams { workspace_id: workspace_id.to_string(), } } + + pub fn to_bytes(&self) -> Result, bincode::Error> { + bincode::serialize(self) + } + pub fn from_bytes(bytes: &[u8]) -> Result { + bincode::deserialize(bytes) + } } impl Deref for CreateCollabParams { @@ -81,6 +88,16 @@ pub struct BatchCreateCollabParams { pub params_list: Vec, } +impl BatchCreateCollabParams { + pub fn to_bytes(&self) -> Result, bincode::Error> { + bincode::serialize(self) + } + + pub fn from_bytes(bytes: &[u8]) -> Result { + bincode::deserialize(bytes) + } +} + #[derive(Debug, Clone, Validate, Serialize, Deserialize)] pub struct DeleteCollabParams { #[validate(custom = "validate_not_empty_str")] diff --git a/nginx/nginx.conf b/nginx/nginx.conf index 8a7fe24b..19bd81c4 100644 --- a/nginx/nginx.conf +++ b/nginx/nginx.conf @@ -27,6 +27,7 @@ http { listen 80; listen 443 ssl; + client_max_body_size 5M; underscores_in_headers on; @@ -55,7 +56,6 @@ http { location /api { proxy_set_header X-Request-Id $request_id; proxy_pass http://appflowy_cloud:8000; - client_max_body_size 6M; } # Minio Web UI diff --git a/src/api/workspace.rs b/src/api/workspace.rs index 117e5357..f3812da5 100644 --- a/src/api/workspace.rs +++ b/src/api/workspace.rs @@ -253,14 +253,18 @@ async fn create_collab_handler( #[instrument(skip(state, payload), err)] async fn batch_create_collab_handler( user_uuid: UserUuid, - payload: Json, + payload: Bytes, state: Data, ) -> Result>> { + let payload = BatchCreateCollabParams::from_bytes(&payload).map_err(|err| { + AppError::InvalidRequest(format!("Failed to parse BatchCreateCollabParams: {}", err)) + })?; + payload.validate().map_err(AppError::from)?; let BatchCreateCollabParams { workspace_id, params_list, - } = payload.into_inner(); + } = payload; if params_list.is_empty() { return Err(AppError::InvalidRequest("Empty collab params list".to_string()).into()); diff --git a/src/middleware/request_id.rs b/src/middleware/request_id.rs index c882e916..43df4ce0 100644 --- a/src/middleware/request_id.rs +++ b/src/middleware/request_id.rs @@ -1,6 +1,6 @@ use actix_http::header::HeaderName; use std::future::{ready, Ready}; -use tracing::{Instrument, Level}; +use tracing::{span, Instrument, Level}; use actix_service::{forward_ready, Service, Transform}; use actix_web::dev::{ServiceRequest, ServiceResponse}; @@ -49,22 +49,22 @@ where let fut = self.service.call(req); Box::pin(fut) } else { - let request_id = match get_request_id(&req) { - Some(request_id) => request_id, - None => { - let request_id = uuid::Uuid::new_v4().to_string(); - if let Ok(header_value) = HeaderValue::from_str(&request_id) { - req - .headers_mut() - .insert(HeaderName::from_static(X_REQUEST_ID), header_value); - } - request_id - }, + let request_id = get_request_id(&req).unwrap_or_else(|| { + let request_id = uuid::Uuid::new_v4().to_string(); + if let Ok(header_value) = HeaderValue::from_str(&request_id) { + req + .headers_mut() + .insert(HeaderName::from_static(X_REQUEST_ID), header_value); + } + request_id + }); + + let span = match get_payload_size(&req) { + Some(size) => span!(Level::INFO, "request", request_id = %request_id, payload_size = size), + None => span!(Level::INFO, "request", request_id = %request_id), }; - let span = tracing::span!(Level::INFO, "request_id", request_id = %request_id); let fut = self.service.call(req); - Box::pin(async move { let mut res = fut.instrument(span).await?; @@ -92,3 +92,11 @@ pub fn get_request_id(req: &ServiceRequest) -> Option { None => None, } } + +fn get_payload_size(req: &ServiceRequest) -> Option { + req + .headers() + .get("content-length") + .and_then(|val| val.to_str().ok()) + .and_then(|val| val.parse::().ok()) +} diff --git a/tests/collab/collab_curd_test.rs b/tests/collab/collab_curd_test.rs index a4d1873b..ab2f096d 100644 --- a/tests/collab/collab_curd_test.rs +++ b/tests/collab/collab_curd_test.rs @@ -29,7 +29,7 @@ async fn batch_insert_collab_success_test() { let params_list = (0..5) .map(|_| CollabParams { object_id: Uuid::new_v4().to_string(), - encoded_collab_v1: vec![0, 200], + encoded_collab_v1: vec![0u8; 1024 * 1024], collab_type: CollabType::Document, override_if_exist: false, })