chore: bump collab (#884)

Cargo.lock

@@ -829,15 +829,6 @@ dependencies = [
  "uuid",
 ]
 
-[[package]]
-name = "arbitrary"
-version = "1.3.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7d5a26814d8dcb93b0e5a0ff3c6d80a8843bafb21b39e8e18a6f05471870e110"
-dependencies = [
- "derive_arbitrary",
-]
-
 [[package]]
 name = "arc-swap"
 version = "1.7.1"
@@ -2236,7 +2227,7 @@ dependencies = [
 [[package]]
 name = "collab"
 version = "0.2.0"
-source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=8987e035accc81bd5ca0acc9812da8e4c6023eba#8987e035accc81bd5ca0acc9812da8e4c6023eba"
+source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=270cd376ee4bfbedd9146105dea4e9e8dabc9359#270cd376ee4bfbedd9146105dea4e9e8dabc9359"
 dependencies = [
  "anyhow",
  "arc-swap",
@@ -2261,7 +2252,7 @@ dependencies = [
 [[package]]
 name = "collab-database"
 version = "0.2.0"
-source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=8987e035accc81bd5ca0acc9812da8e4c6023eba#8987e035accc81bd5ca0acc9812da8e4c6023eba"
+source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=270cd376ee4bfbedd9146105dea4e9e8dabc9359#270cd376ee4bfbedd9146105dea4e9e8dabc9359"
 dependencies = [
  "anyhow",
  "async-trait",
@@ -2300,7 +2291,7 @@ dependencies = [
 [[package]]
 name = "collab-document"
 version = "0.2.0"
-source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=8987e035accc81bd5ca0acc9812da8e4c6023eba#8987e035accc81bd5ca0acc9812da8e4c6023eba"
+source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=270cd376ee4bfbedd9146105dea4e9e8dabc9359#270cd376ee4bfbedd9146105dea4e9e8dabc9359"
 dependencies = [
  "anyhow",
  "arc-swap",
@@ -2321,7 +2312,7 @@ dependencies = [
 [[package]]
 name = "collab-entity"
 version = "0.2.0"
-source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=8987e035accc81bd5ca0acc9812da8e4c6023eba#8987e035accc81bd5ca0acc9812da8e4c6023eba"
+source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=270cd376ee4bfbedd9146105dea4e9e8dabc9359#270cd376ee4bfbedd9146105dea4e9e8dabc9359"
 dependencies = [
  "anyhow",
  "bytes",
@@ -2341,7 +2332,7 @@ dependencies = [
 [[package]]
 name = "collab-folder"
 version = "0.2.0"
-source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=8987e035accc81bd5ca0acc9812da8e4c6023eba#8987e035accc81bd5ca0acc9812da8e4c6023eba"
+source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=270cd376ee4bfbedd9146105dea4e9e8dabc9359#270cd376ee4bfbedd9146105dea4e9e8dabc9359"
 dependencies = [
  "anyhow",
  "arc-swap",
@@ -2363,11 +2354,12 @@ dependencies = [
 [[package]]
 name = "collab-importer"
 version = "0.1.0"
-source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=8987e035accc81bd5ca0acc9812da8e4c6023eba#8987e035accc81bd5ca0acc9812da8e4c6023eba"
+source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=270cd376ee4bfbedd9146105dea4e9e8dabc9359#270cd376ee4bfbedd9146105dea4e9e8dabc9359"
 dependencies = [
  "anyhow",
  "async-recursion",
  "async-trait",
+ "async_zip",
  "base64 0.22.1",
  "chrono",
  "collab",
@@ -2377,21 +2369,23 @@ dependencies = [
  "collab-folder",
  "fancy-regex 0.13.0",
  "futures",
+ "futures-lite",
  "futures-util",
  "fxhash",
  "hex",
  "markdown",
  "percent-encoding",
  "rayon",
+ "sanitize-filename",
  "serde",
  "serde_json",
  "sha2",
  "thiserror",
  "tokio",
+ "tokio-util",
  "tracing",
  "uuid",
  "walkdir",
- "zip",
 ]
 
 [[package]]
@@ -2461,7 +2455,7 @@ dependencies = [
 [[package]]
 name = "collab-user"
 version = "0.2.0"
-source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=8987e035accc81bd5ca0acc9812da8e4c6023eba#8987e035accc81bd5ca0acc9812da8e4c6023eba"
+source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=270cd376ee4bfbedd9146105dea4e9e8dabc9359#270cd376ee4bfbedd9146105dea4e9e8dabc9359"
 dependencies = [
  "anyhow",
  "collab",
@@ -2539,12 +2533,6 @@ dependencies = [
  "tiny-keccak",
 ]
 
-[[package]]
-name = "constant_time_eq"
-version = "0.3.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6"
-
 [[package]]
 name = "convert_case"
 version = "0.4.0"
@@ -3061,17 +3049,6 @@ dependencies = [
  "powerfmt",
 ]
 
-[[package]]
-name = "derive_arbitrary"
-version = "1.3.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "67e77553c4162a157adbf834ebae5b415acbecbeafc7a74b0e886657506a7611"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn 2.0.72",
-]
-
 [[package]]
 name = "derive_more"
 version = "0.99.18"
@@ -4431,12 +4408,6 @@ dependencies = [
  "scopeguard",
 ]
 
-[[package]]
-name = "lockfree-object-pool"
-version = "0.1.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9374ef4228402d4b7e403e5838cb880d9ee663314b0a900d5a6aabf0c213552e"
-
 [[package]]
 name = "log"
 version = "0.4.22"
@@ -4452,16 +4423,6 @@ dependencies = [
  "hashbrown 0.14.5",
 ]
 
-[[package]]
-name = "lzma-rs"
-version = "0.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "297e814c836ae64db86b36cf2a557ba54368d03f6afcd7d947c266692f71115e"
-dependencies = [
- "byteorder",
- "crc",
-]
-
 [[package]]
 name = "lzma-sys"
 version = "0.1.20"
@@ -5008,16 +4969,6 @@ version = "1.0.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a"
 
-[[package]]
-name = "pbkdf2"
-version = "0.12.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2"
-dependencies = [
- "digest",
- "hmac",
-]
-
 [[package]]
 name = "pem"
 version = "1.1.1"
@@ -6323,6 +6274,16 @@ dependencies = [
  "winapi-util",
 ]
 
+[[package]]
+name = "sanitize-filename"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2ed72fbaf78e6f2d41744923916966c4fbe3d7c74e3037a8ee482f1115572603"
+dependencies = [
+ "lazy_static",
+ "regex",
+]
+
 [[package]]
 name = "scc"
 version = "2.1.6"
@@ -6695,12 +6656,6 @@ dependencies = [
  "rand_core 0.6.4",
 ]
 
-[[package]]
-name = "simd-adler32"
-version = "0.3.7"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d66dc143e6b11c1eddc06d5c423cfc97062865baf299914ab64caa38182078fe"
-
 [[package]]
 name = "simdutf8"
 version = "0.1.4"
@@ -8622,49 +8577,6 @@ dependencies = [
  "syn 2.0.72",
 ]
 
-[[package]]
-name = "zip"
-version = "2.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dc5e4288ea4057ae23afc69a4472434a87a2495cafce6632fd1c4ec9f5cf3494"
-dependencies = [
- "aes",
- "arbitrary",
- "bzip2",
- "constant_time_eq",
- "crc32fast",
- "crossbeam-utils",
- "deflate64",
- "displaydoc",
- "flate2",
- "hmac",
- "indexmap 2.3.0",
- "lzma-rs",
- "memchr",
- "pbkdf2",
- "rand 0.8.5",
- "sha1",
- "thiserror",
- "time",
- "zeroize",
- "zopfli",
- "zstd",
-]
-
-[[package]]
-name = "zopfli"
-version = "0.8.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e5019f391bac5cf252e93bbcc53d039ffd62c7bfb7c150414d61369afe57e946"
-dependencies = [
- "bumpalo",
- "crc32fast",
- "lockfree-object-pool",
- "log",
- "once_cell",
- "simd-adler32",
-]
-
 [[package]]
 name = "zstd"
 version = "0.13.2"

Cargo.toml

@@ -293,13 +293,13 @@ debug = true
 [patch.crates-io]
 # It's diffcult to resovle different version with the same crate used in AppFlowy Frontend and the Client-API crate.
 # So using patch to workaround this issue.
-collab = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "8987e035accc81bd5ca0acc9812da8e4c6023eba" }
+collab = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "270cd376ee4bfbedd9146105dea4e9e8dabc9359" }
-collab-entity = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "8987e035accc81bd5ca0acc9812da8e4c6023eba" }
+collab-entity = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "270cd376ee4bfbedd9146105dea4e9e8dabc9359" }
-collab-folder = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "8987e035accc81bd5ca0acc9812da8e4c6023eba" }
+collab-folder = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "270cd376ee4bfbedd9146105dea4e9e8dabc9359" }
-collab-document = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "8987e035accc81bd5ca0acc9812da8e4c6023eba" }
+collab-document = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "270cd376ee4bfbedd9146105dea4e9e8dabc9359" }
-collab-user = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "8987e035accc81bd5ca0acc9812da8e4c6023eba" }
+collab-user = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "270cd376ee4bfbedd9146105dea4e9e8dabc9359" }
-collab-database = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "8987e035accc81bd5ca0acc9812da8e4c6023eba" }
+collab-database = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "270cd376ee4bfbedd9146105dea4e9e8dabc9359" }
-collab-importer = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "8987e035accc81bd5ca0acc9812da8e4c6023eba" }
+collab-importer = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "270cd376ee4bfbedd9146105dea4e9e8dabc9359" }
 
 [features]
 history = []

services/appflowy-worker/src/import_worker/mod.rs

@@ -1,4 +1,3 @@
 pub mod email_notifier;
 pub mod report;
-pub mod unzip;
 pub mod worker;

services/appflowy-worker/src/import_worker/unzip.rs

@@ -1,101 +0,0 @@
-use anyhow::{Context, Result};
-use async_zip::base::read::stream::{Ready, ZipFileReader};
-use async_zip::{StringEncoding, ZipString};
-use futures::io::{AsyncBufRead, AsyncReadExt};
-use std::ffi::OsString;
-use std::os::unix::ffi::OsStringExt;
-use std::path::PathBuf;
-use tokio::fs;
-use tokio::fs::File;
-use tokio::io::AsyncWriteExt;
-
-use tracing::error;
-
-pub struct UnzipFile {
-  pub file_name: String,
-  pub unzip_dir_path: PathBuf,
-}
-
-pub async fn unzip_async<R: AsyncBufRead + Unpin>(
-  mut zip_reader: ZipFileReader<Ready<R>>,
-  out_dir: PathBuf,
-) -> Result<UnzipFile, anyhow::Error> {
-  let mut real_file_name = None;
-  while let Some(mut next_reader) = zip_reader.next_with_entry().await? {
-    let entry_reader = next_reader.reader_mut();
-    let filename = get_filename(entry_reader.entry().filename())
-      .with_context(|| "Failed to extract filename from entry".to_string())?;
-
-    // Save the root folder name if we haven't done so yet
-    if real_file_name.is_none() && filename.ends_with('/') {
-      real_file_name = Some(filename.split('/').next().unwrap_or(&filename).to_string());
-    }
-
-    let output_path = out_dir.join(&filename);
-    if filename.ends_with('/') {
-      fs::create_dir_all(&output_path)
-        .await
-        .with_context(|| format!("Failed to create directory: {}", output_path.display()))?;
-    } else {
-      // Ensure parent directories exist
-      if let Some(parent) = output_path.parent() {
-        if !parent.exists() {
-          fs::create_dir_all(parent)
-            .await
-            .with_context(|| format!("Failed to create parent directory: {}", parent.display()))?;
-        }
-      }
-
-      // Write file contents
-      let mut outfile = File::create(&output_path)
-        .await
-        .with_context(|| format!("Failed to create file: {}", output_path.display()))?;
-      let mut buffer = vec![];
-      match entry_reader.read_to_end(&mut buffer).await {
-        Ok(_) => {
-          outfile
-            .write_all(&buffer)
-            .await
-            .with_context(|| format!("Failed to write data to file: {}", output_path.display()))?;
-        },
-        Err(err) => {
-          error!(
-            "Failed to read entry: {:?}. Error: {:?}",
-            entry_reader.entry(),
-            err,
-          );
-          return Err(anyhow::anyhow!(
-            "Unexpected EOF while reading: {}",
-            filename
-          ));
-        },
-      }
-    }
-
-    // Move to the next file in the zip
-    zip_reader = next_reader.done().await?;
-  }
-
-  match real_file_name {
-    None => Err(anyhow::anyhow!("No files found in the zip archive")),
-    Some(file_name) => Ok(UnzipFile {
-      file_name: file_name.clone(),
-      unzip_dir_path: out_dir.join(file_name),
-    }),
-  }
-}
-
-pub fn get_filename(zip_string: &ZipString) -> Result<String, anyhow::Error> {
-  match zip_string.encoding() {
-    StringEncoding::Utf8 => match zip_string.as_str() {
-      Ok(valid_str) => Ok(valid_str.to_string()),
-      Err(err) => Err(err.into()),
-    },
-
-    StringEncoding::Raw => {
-      let raw_bytes = zip_string.as_bytes();
-      let os_string = OsString::from_vec(raw_bytes.to_vec());
-      Ok(os_string.to_string_lossy().into_owned())
-    },
-  }
-}

services/appflowy-worker/src/import_worker/worker.rs

@@ -1,5 +1,4 @@
 use crate::import_worker::report::{ImportNotifier, ImportProgress, ImportResult};
-use crate::import_worker::unzip::unzip_async;
 use crate::s3_client::S3StreamResponse;
 use anyhow::anyhow;
 use async_zip::base::read::stream::ZipFileReader;
@@ -15,6 +14,7 @@ use collab_importer::imported_collab::ImportType;
 use collab_importer::notion::page::CollabResource;
 use collab_importer::notion::NotionImporter;
 use collab_importer::util::FileId;
+use collab_importer::zip_tool::unzip_stream;
 use database::collab::{insert_into_af_collab_bulk_for_user, select_blob_from_af_collab};
 use database::workspace::{
   delete_from_workspace, select_workspace_database_storage_id, update_import_task_status,
@@ -320,7 +320,7 @@ async fn download_and_unzip_file(
       ImportError::Internal(anyhow!("Failed to set permissions for temp dir: {:?}", err))
     })?;
 
-  let unzip_file = unzip_async(zip_reader, output_file_path)
+  let unzip_file = unzip_stream(zip_reader, output_file_path)
     .await
     .map_err(ImportError::Internal)?;
   Ok(unzip_file.unzip_dir_path)