From 2b02a8d12bf87b2bf77f3f051702b76f93cfed0d Mon Sep 17 00:00:00 2001 From: "Nathan.fooo" <86001920+appflowy@users.noreply.github.com> Date: Tue, 15 Oct 2024 15:56:14 +0800 Subject: [PATCH] chore: bump collab (#884) --- Cargo.lock | 130 +++--------------- Cargo.toml | 14 +- .../appflowy-worker/src/import_worker/mod.rs | 1 - .../src/import_worker/unzip.rs | 101 -------------- .../src/import_worker/worker.rs | 4 +- 5 files changed, 30 insertions(+), 220 deletions(-) delete mode 100644 services/appflowy-worker/src/import_worker/unzip.rs diff --git a/Cargo.lock b/Cargo.lock index dd002731..1ebf89ab 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -829,15 +829,6 @@ dependencies = [ "uuid", ] -[[package]] -name = "arbitrary" -version = "1.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d5a26814d8dcb93b0e5a0ff3c6d80a8843bafb21b39e8e18a6f05471870e110" -dependencies = [ - "derive_arbitrary", -] - [[package]] name = "arc-swap" version = "1.7.1" @@ -2236,7 +2227,7 @@ dependencies = [ [[package]] name = "collab" version = "0.2.0" -source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=8987e035accc81bd5ca0acc9812da8e4c6023eba#8987e035accc81bd5ca0acc9812da8e4c6023eba" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=270cd376ee4bfbedd9146105dea4e9e8dabc9359#270cd376ee4bfbedd9146105dea4e9e8dabc9359" dependencies = [ "anyhow", "arc-swap", @@ -2261,7 +2252,7 @@ dependencies = [ [[package]] name = "collab-database" version = "0.2.0" -source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=8987e035accc81bd5ca0acc9812da8e4c6023eba#8987e035accc81bd5ca0acc9812da8e4c6023eba" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=270cd376ee4bfbedd9146105dea4e9e8dabc9359#270cd376ee4bfbedd9146105dea4e9e8dabc9359" dependencies = [ "anyhow", "async-trait", @@ -2300,7 +2291,7 @@ dependencies = [ [[package]] name = "collab-document" version = "0.2.0" -source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=8987e035accc81bd5ca0acc9812da8e4c6023eba#8987e035accc81bd5ca0acc9812da8e4c6023eba" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=270cd376ee4bfbedd9146105dea4e9e8dabc9359#270cd376ee4bfbedd9146105dea4e9e8dabc9359" dependencies = [ "anyhow", "arc-swap", @@ -2321,7 +2312,7 @@ dependencies = [ [[package]] name = "collab-entity" version = "0.2.0" -source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=8987e035accc81bd5ca0acc9812da8e4c6023eba#8987e035accc81bd5ca0acc9812da8e4c6023eba" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=270cd376ee4bfbedd9146105dea4e9e8dabc9359#270cd376ee4bfbedd9146105dea4e9e8dabc9359" dependencies = [ "anyhow", "bytes", @@ -2341,7 +2332,7 @@ dependencies = [ [[package]] name = "collab-folder" version = "0.2.0" -source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=8987e035accc81bd5ca0acc9812da8e4c6023eba#8987e035accc81bd5ca0acc9812da8e4c6023eba" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=270cd376ee4bfbedd9146105dea4e9e8dabc9359#270cd376ee4bfbedd9146105dea4e9e8dabc9359" dependencies = [ "anyhow", "arc-swap", @@ -2363,11 +2354,12 @@ dependencies = [ [[package]] name = "collab-importer" version = "0.1.0" -source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=8987e035accc81bd5ca0acc9812da8e4c6023eba#8987e035accc81bd5ca0acc9812da8e4c6023eba" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=270cd376ee4bfbedd9146105dea4e9e8dabc9359#270cd376ee4bfbedd9146105dea4e9e8dabc9359" dependencies = [ "anyhow", "async-recursion", "async-trait", + "async_zip", "base64 0.22.1", "chrono", "collab", @@ -2377,21 +2369,23 @@ dependencies = [ "collab-folder", "fancy-regex 0.13.0", "futures", + "futures-lite", "futures-util", "fxhash", "hex", "markdown", "percent-encoding", "rayon", + "sanitize-filename", "serde", "serde_json", "sha2", "thiserror", "tokio", + "tokio-util", "tracing", "uuid", "walkdir", - "zip", ] [[package]] @@ -2461,7 +2455,7 @@ dependencies = [ [[package]] name = "collab-user" version = "0.2.0" -source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=8987e035accc81bd5ca0acc9812da8e4c6023eba#8987e035accc81bd5ca0acc9812da8e4c6023eba" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=270cd376ee4bfbedd9146105dea4e9e8dabc9359#270cd376ee4bfbedd9146105dea4e9e8dabc9359" dependencies = [ "anyhow", "collab", @@ -2539,12 +2533,6 @@ dependencies = [ "tiny-keccak", ] -[[package]] -name = "constant_time_eq" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6" - [[package]] name = "convert_case" version = "0.4.0" @@ -3061,17 +3049,6 @@ dependencies = [ "powerfmt", ] -[[package]] -name = "derive_arbitrary" -version = "1.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67e77553c4162a157adbf834ebae5b415acbecbeafc7a74b0e886657506a7611" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.72", -] - [[package]] name = "derive_more" version = "0.99.18" @@ -4431,12 +4408,6 @@ dependencies = [ "scopeguard", ] -[[package]] -name = "lockfree-object-pool" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9374ef4228402d4b7e403e5838cb880d9ee663314b0a900d5a6aabf0c213552e" - [[package]] name = "log" version = "0.4.22" @@ -4452,16 +4423,6 @@ dependencies = [ "hashbrown 0.14.5", ] -[[package]] -name = "lzma-rs" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "297e814c836ae64db86b36cf2a557ba54368d03f6afcd7d947c266692f71115e" -dependencies = [ - "byteorder", - "crc", -] - [[package]] name = "lzma-sys" version = "0.1.20" @@ -5008,16 +4969,6 @@ version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" -[[package]] -name = "pbkdf2" -version = "0.12.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2" -dependencies = [ - "digest", - "hmac", -] - [[package]] name = "pem" version = "1.1.1" @@ -6323,6 +6274,16 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "sanitize-filename" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ed72fbaf78e6f2d41744923916966c4fbe3d7c74e3037a8ee482f1115572603" +dependencies = [ + "lazy_static", + "regex", +] + [[package]] name = "scc" version = "2.1.6" @@ -6695,12 +6656,6 @@ dependencies = [ "rand_core 0.6.4", ] -[[package]] -name = "simd-adler32" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d66dc143e6b11c1eddc06d5c423cfc97062865baf299914ab64caa38182078fe" - [[package]] name = "simdutf8" version = "0.1.4" @@ -8622,49 +8577,6 @@ dependencies = [ "syn 2.0.72", ] -[[package]] -name = "zip" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc5e4288ea4057ae23afc69a4472434a87a2495cafce6632fd1c4ec9f5cf3494" -dependencies = [ - "aes", - "arbitrary", - "bzip2", - "constant_time_eq", - "crc32fast", - "crossbeam-utils", - "deflate64", - "displaydoc", - "flate2", - "hmac", - "indexmap 2.3.0", - "lzma-rs", - "memchr", - "pbkdf2", - "rand 0.8.5", - "sha1", - "thiserror", - "time", - "zeroize", - "zopfli", - "zstd", -] - -[[package]] -name = "zopfli" -version = "0.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5019f391bac5cf252e93bbcc53d039ffd62c7bfb7c150414d61369afe57e946" -dependencies = [ - "bumpalo", - "crc32fast", - "lockfree-object-pool", - "log", - "once_cell", - "simd-adler32", -] - [[package]] name = "zstd" version = "0.13.2" diff --git a/Cargo.toml b/Cargo.toml index 7d254e4c..42b05875 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -293,13 +293,13 @@ debug = true [patch.crates-io] # It's diffcult to resovle different version with the same crate used in AppFlowy Frontend and the Client-API crate. # So using patch to workaround this issue. -collab = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "8987e035accc81bd5ca0acc9812da8e4c6023eba" } -collab-entity = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "8987e035accc81bd5ca0acc9812da8e4c6023eba" } -collab-folder = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "8987e035accc81bd5ca0acc9812da8e4c6023eba" } -collab-document = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "8987e035accc81bd5ca0acc9812da8e4c6023eba" } -collab-user = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "8987e035accc81bd5ca0acc9812da8e4c6023eba" } -collab-database = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "8987e035accc81bd5ca0acc9812da8e4c6023eba" } -collab-importer = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "8987e035accc81bd5ca0acc9812da8e4c6023eba" } +collab = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "270cd376ee4bfbedd9146105dea4e9e8dabc9359" } +collab-entity = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "270cd376ee4bfbedd9146105dea4e9e8dabc9359" } +collab-folder = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "270cd376ee4bfbedd9146105dea4e9e8dabc9359" } +collab-document = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "270cd376ee4bfbedd9146105dea4e9e8dabc9359" } +collab-user = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "270cd376ee4bfbedd9146105dea4e9e8dabc9359" } +collab-database = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "270cd376ee4bfbedd9146105dea4e9e8dabc9359" } +collab-importer = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "270cd376ee4bfbedd9146105dea4e9e8dabc9359" } [features] history = [] diff --git a/services/appflowy-worker/src/import_worker/mod.rs b/services/appflowy-worker/src/import_worker/mod.rs index a432eae0..bea95379 100644 --- a/services/appflowy-worker/src/import_worker/mod.rs +++ b/services/appflowy-worker/src/import_worker/mod.rs @@ -1,4 +1,3 @@ pub mod email_notifier; pub mod report; -pub mod unzip; pub mod worker; diff --git a/services/appflowy-worker/src/import_worker/unzip.rs b/services/appflowy-worker/src/import_worker/unzip.rs deleted file mode 100644 index c34ba794..00000000 --- a/services/appflowy-worker/src/import_worker/unzip.rs +++ /dev/null @@ -1,101 +0,0 @@ -use anyhow::{Context, Result}; -use async_zip::base::read::stream::{Ready, ZipFileReader}; -use async_zip::{StringEncoding, ZipString}; -use futures::io::{AsyncBufRead, AsyncReadExt}; -use std::ffi::OsString; -use std::os::unix::ffi::OsStringExt; -use std::path::PathBuf; -use tokio::fs; -use tokio::fs::File; -use tokio::io::AsyncWriteExt; - -use tracing::error; - -pub struct UnzipFile { - pub file_name: String, - pub unzip_dir_path: PathBuf, -} - -pub async fn unzip_async( - mut zip_reader: ZipFileReader>, - out_dir: PathBuf, -) -> Result { - let mut real_file_name = None; - while let Some(mut next_reader) = zip_reader.next_with_entry().await? { - let entry_reader = next_reader.reader_mut(); - let filename = get_filename(entry_reader.entry().filename()) - .with_context(|| "Failed to extract filename from entry".to_string())?; - - // Save the root folder name if we haven't done so yet - if real_file_name.is_none() && filename.ends_with('/') { - real_file_name = Some(filename.split('/').next().unwrap_or(&filename).to_string()); - } - - let output_path = out_dir.join(&filename); - if filename.ends_with('/') { - fs::create_dir_all(&output_path) - .await - .with_context(|| format!("Failed to create directory: {}", output_path.display()))?; - } else { - // Ensure parent directories exist - if let Some(parent) = output_path.parent() { - if !parent.exists() { - fs::create_dir_all(parent) - .await - .with_context(|| format!("Failed to create parent directory: {}", parent.display()))?; - } - } - - // Write file contents - let mut outfile = File::create(&output_path) - .await - .with_context(|| format!("Failed to create file: {}", output_path.display()))?; - let mut buffer = vec![]; - match entry_reader.read_to_end(&mut buffer).await { - Ok(_) => { - outfile - .write_all(&buffer) - .await - .with_context(|| format!("Failed to write data to file: {}", output_path.display()))?; - }, - Err(err) => { - error!( - "Failed to read entry: {:?}. Error: {:?}", - entry_reader.entry(), - err, - ); - return Err(anyhow::anyhow!( - "Unexpected EOF while reading: {}", - filename - )); - }, - } - } - - // Move to the next file in the zip - zip_reader = next_reader.done().await?; - } - - match real_file_name { - None => Err(anyhow::anyhow!("No files found in the zip archive")), - Some(file_name) => Ok(UnzipFile { - file_name: file_name.clone(), - unzip_dir_path: out_dir.join(file_name), - }), - } -} - -pub fn get_filename(zip_string: &ZipString) -> Result { - match zip_string.encoding() { - StringEncoding::Utf8 => match zip_string.as_str() { - Ok(valid_str) => Ok(valid_str.to_string()), - Err(err) => Err(err.into()), - }, - - StringEncoding::Raw => { - let raw_bytes = zip_string.as_bytes(); - let os_string = OsString::from_vec(raw_bytes.to_vec()); - Ok(os_string.to_string_lossy().into_owned()) - }, - } -} diff --git a/services/appflowy-worker/src/import_worker/worker.rs b/services/appflowy-worker/src/import_worker/worker.rs index 89ba1b97..25755b1a 100644 --- a/services/appflowy-worker/src/import_worker/worker.rs +++ b/services/appflowy-worker/src/import_worker/worker.rs @@ -1,5 +1,4 @@ use crate::import_worker::report::{ImportNotifier, ImportProgress, ImportResult}; -use crate::import_worker::unzip::unzip_async; use crate::s3_client::S3StreamResponse; use anyhow::anyhow; use async_zip::base::read::stream::ZipFileReader; @@ -15,6 +14,7 @@ use collab_importer::imported_collab::ImportType; use collab_importer::notion::page::CollabResource; use collab_importer::notion::NotionImporter; use collab_importer::util::FileId; +use collab_importer::zip_tool::unzip_stream; use database::collab::{insert_into_af_collab_bulk_for_user, select_blob_from_af_collab}; use database::workspace::{ delete_from_workspace, select_workspace_database_storage_id, update_import_task_status, @@ -320,7 +320,7 @@ async fn download_and_unzip_file( ImportError::Internal(anyhow!("Failed to set permissions for temp dir: {:?}", err)) })?; - let unzip_file = unzip_async(zip_reader, output_file_path) + let unzip_file = unzip_stream(zip_reader, output_file_path) .await .map_err(ImportError::Internal)?; Ok(unzip_file.unzip_dir_path)