feat: encode batch create collab data as binary (#242)

Cargo.lock

@@ -1751,6 +1751,7 @@ version = "0.1.0"
 dependencies = [
  "anyhow",
  "app-error",
+ "bincode",
  "chrono",
  "collab-entity",
  "serde",

libs/client-api/src/http.rs

@@ -729,10 +729,15 @@ impl Client {
       workspace_id: workspace_id.to_string(),
       params_list: params,
     };
+
     let resp = self
       .http_client_with_auth(Method::POST, &url)
       .await?
-      .json(&payload)
+      .body(
+        payload
+          .to_bytes()
+          .map_err(|err| AppError::Internal(err.into()))?,
+      )
       .send()
       .await?;
     log_request_id(&resp);

libs/database-entity/Cargo.toml

@@ -18,3 +18,4 @@ anyhow = "1.0.75"
 tracing = "0.1"
 serde_repr = "0.1.16"
 app-error = { workspace = true }
+bincode = "1.3.3"

libs/database-entity/src/dto.rs

@@ -31,6 +31,13 @@ impl CreateCollabParams {
       workspace_id: workspace_id.to_string(),
     }
   }
+
+  pub fn to_bytes(&self) -> Result<Vec<u8>, bincode::Error> {
+    bincode::serialize(self)
+  }
+  pub fn from_bytes(bytes: &[u8]) -> Result<Self, bincode::Error> {
+    bincode::deserialize(bytes)
+  }
 }
 
 impl Deref for CreateCollabParams {
@@ -81,6 +88,16 @@ pub struct BatchCreateCollabParams {
   pub params_list: Vec<CollabParams>,
 }
 
+impl BatchCreateCollabParams {
+  pub fn to_bytes(&self) -> Result<Vec<u8>, bincode::Error> {
+    bincode::serialize(self)
+  }
+
+  pub fn from_bytes(bytes: &[u8]) -> Result<Self, bincode::Error> {
+    bincode::deserialize(bytes)
+  }
+}
+
 #[derive(Debug, Clone, Validate, Serialize, Deserialize)]
 pub struct DeleteCollabParams {
   #[validate(custom = "validate_not_empty_str")]

nginx/nginx.conf

@@ -27,6 +27,7 @@ http {
 
         listen 80;
         listen 443 ssl;
+        client_max_body_size 5M;
 
         underscores_in_headers on;
 
@@ -55,7 +56,6 @@ http {
         location /api {
             proxy_set_header X-Request-Id $request_id;
             proxy_pass http://appflowy_cloud:8000;
-            client_max_body_size 6M;
         }
 
         # Minio Web UI

src/api/workspace.rs

@@ -253,14 +253,18 @@ async fn create_collab_handler(
 #[instrument(skip(state, payload), err)]
 async fn batch_create_collab_handler(
   user_uuid: UserUuid,
-  payload: Json<BatchCreateCollabParams>,
+  payload: Bytes,
   state: Data<AppState>,
 ) -> Result<Json<AppResponse<()>>> {
+  let payload = BatchCreateCollabParams::from_bytes(&payload).map_err(|err| {
+    AppError::InvalidRequest(format!("Failed to parse BatchCreateCollabParams: {}", err))
+  })?;
+
   payload.validate().map_err(AppError::from)?;
   let BatchCreateCollabParams {
     workspace_id,
     params_list,
-  } = payload.into_inner();
+  } = payload;
 
   if params_list.is_empty() {
     return Err(AppError::InvalidRequest("Empty collab params list".to_string()).into());

src/middleware/request_id.rs

@@ -1,6 +1,6 @@
 use actix_http::header::HeaderName;
 use std::future::{ready, Ready};
-use tracing::{Instrument, Level};
+use tracing::{span, Instrument, Level};
 
 use actix_service::{forward_ready, Service, Transform};
 use actix_web::dev::{ServiceRequest, ServiceResponse};
@@ -49,22 +49,22 @@ where
       let fut = self.service.call(req);
       Box::pin(fut)
     } else {
-      let request_id = match get_request_id(&req) {
+      let request_id = get_request_id(&req).unwrap_or_else(|| {
-        Some(request_id) => request_id,
+        let request_id = uuid::Uuid::new_v4().to_string();
-        None => {
+        if let Ok(header_value) = HeaderValue::from_str(&request_id) {
-          let request_id = uuid::Uuid::new_v4().to_string();
+          req
-          if let Ok(header_value) = HeaderValue::from_str(&request_id) {
+            .headers_mut()
-            req
+            .insert(HeaderName::from_static(X_REQUEST_ID), header_value);
-              .headers_mut()
+        }
-              .insert(HeaderName::from_static(X_REQUEST_ID), header_value);
+        request_id
-          }
+      });
-          request_id
+
-        },
+      let span = match get_payload_size(&req) {
+        Some(size) => span!(Level::INFO, "request", request_id = %request_id, payload_size = size),
+        None => span!(Level::INFO, "request", request_id = %request_id),
       };
 
-      let span = tracing::span!(Level::INFO, "request_id", request_id = %request_id);
       let fut = self.service.call(req);
-
       Box::pin(async move {
         let mut res = fut.instrument(span).await?;
 
@@ -92,3 +92,11 @@ pub fn get_request_id(req: &ServiceRequest) -> Option<String> {
     None => None,
   }
 }
+
+fn get_payload_size(req: &ServiceRequest) -> Option<usize> {
+  req
+    .headers()
+    .get("content-length")
+    .and_then(|val| val.to_str().ok())
+    .and_then(|val| val.parse::<usize>().ok())
+}

tests/collab/collab_curd_test.rs

@@ -29,7 +29,7 @@ async fn batch_insert_collab_success_test() {
   let params_list = (0..5)
     .map(|_| CollabParams {
       object_id: Uuid::new_v4().to_string(),
-      encoded_collab_v1: vec![0, 200],
+      encoded_collab_v1: vec![0u8; 1024 * 1024],
       collab_type: CollabType::Document,
       override_if_exist: false,
     })