chore: bump collab (#884)
This commit is contained in:
Nathan.fooo
GitHub
parent
2a15c8e4b5
commit
2b02a8d12b
|
|
@ -829,15 +829,6 @@ dependencies = [
|
|||
"uuid",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "arbitrary"
|
||||
version = "1.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7d5a26814d8dcb93b0e5a0ff3c6d80a8843bafb21b39e8e18a6f05471870e110"
|
||||
dependencies = [
|
||||
"derive_arbitrary",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "arc-swap"
|
||||
version = "1.7.1"
|
||||
|
|
@ -2236,7 +2227,7 @@ dependencies = [
|
|||
[[package]]
|
||||
name = "collab"
|
||||
version = "0.2.0"
|
||||
source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=8987e035accc81bd5ca0acc9812da8e4c6023eba#8987e035accc81bd5ca0acc9812da8e4c6023eba"
|
||||
source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=270cd376ee4bfbedd9146105dea4e9e8dabc9359#270cd376ee4bfbedd9146105dea4e9e8dabc9359"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"arc-swap",
|
||||
|
|
@ -2261,7 +2252,7 @@ dependencies = [
|
|||
[[package]]
|
||||
name = "collab-database"
|
||||
version = "0.2.0"
|
||||
source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=8987e035accc81bd5ca0acc9812da8e4c6023eba#8987e035accc81bd5ca0acc9812da8e4c6023eba"
|
||||
source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=270cd376ee4bfbedd9146105dea4e9e8dabc9359#270cd376ee4bfbedd9146105dea4e9e8dabc9359"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"async-trait",
|
||||
|
|
@ -2300,7 +2291,7 @@ dependencies = [
|
|||
[[package]]
|
||||
name = "collab-document"
|
||||
version = "0.2.0"
|
||||
source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=8987e035accc81bd5ca0acc9812da8e4c6023eba#8987e035accc81bd5ca0acc9812da8e4c6023eba"
|
||||
source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=270cd376ee4bfbedd9146105dea4e9e8dabc9359#270cd376ee4bfbedd9146105dea4e9e8dabc9359"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"arc-swap",
|
||||
|
|
@ -2321,7 +2312,7 @@ dependencies = [
|
|||
[[package]]
|
||||
name = "collab-entity"
|
||||
version = "0.2.0"
|
||||
source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=8987e035accc81bd5ca0acc9812da8e4c6023eba#8987e035accc81bd5ca0acc9812da8e4c6023eba"
|
||||
source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=270cd376ee4bfbedd9146105dea4e9e8dabc9359#270cd376ee4bfbedd9146105dea4e9e8dabc9359"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"bytes",
|
||||
|
|
@ -2341,7 +2332,7 @@ dependencies = [
|
|||
[[package]]
|
||||
name = "collab-folder"
|
||||
version = "0.2.0"
|
||||
source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=8987e035accc81bd5ca0acc9812da8e4c6023eba#8987e035accc81bd5ca0acc9812da8e4c6023eba"
|
||||
source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=270cd376ee4bfbedd9146105dea4e9e8dabc9359#270cd376ee4bfbedd9146105dea4e9e8dabc9359"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"arc-swap",
|
||||
|
|
@ -2363,11 +2354,12 @@ dependencies = [
|
|||
[[package]]
|
||||
name = "collab-importer"
|
||||
version = "0.1.0"
|
||||
source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=8987e035accc81bd5ca0acc9812da8e4c6023eba#8987e035accc81bd5ca0acc9812da8e4c6023eba"
|
||||
source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=270cd376ee4bfbedd9146105dea4e9e8dabc9359#270cd376ee4bfbedd9146105dea4e9e8dabc9359"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"async-recursion",
|
||||
"async-trait",
|
||||
"async_zip",
|
||||
"base64 0.22.1",
|
||||
"chrono",
|
||||
"collab",
|
||||
|
|
@ -2377,21 +2369,23 @@ dependencies = [
|
|||
"collab-folder",
|
||||
"fancy-regex 0.13.0",
|
||||
"futures",
|
||||
"futures-lite",
|
||||
"futures-util",
|
||||
"fxhash",
|
||||
"hex",
|
||||
"markdown",
|
||||
"percent-encoding",
|
||||
"rayon",
|
||||
"sanitize-filename",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"sha2",
|
||||
"thiserror",
|
||||
"tokio",
|
||||
"tokio-util",
|
||||
"tracing",
|
||||
"uuid",
|
||||
"walkdir",
|
||||
"zip",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -2461,7 +2455,7 @@ dependencies = [
|
|||
[[package]]
|
||||
name = "collab-user"
|
||||
version = "0.2.0"
|
||||
source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=8987e035accc81bd5ca0acc9812da8e4c6023eba#8987e035accc81bd5ca0acc9812da8e4c6023eba"
|
||||
source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=270cd376ee4bfbedd9146105dea4e9e8dabc9359#270cd376ee4bfbedd9146105dea4e9e8dabc9359"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"collab",
|
||||
|
|
@ -2539,12 +2533,6 @@ dependencies = [
|
|||
"tiny-keccak",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "constant_time_eq"
|
||||
version = "0.3.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6"
|
||||
|
||||
[[package]]
|
||||
name = "convert_case"
|
||||
version = "0.4.0"
|
||||
|
|
@ -3061,17 +3049,6 @@ dependencies = [
|
|||
"powerfmt",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "derive_arbitrary"
|
||||
version = "1.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "67e77553c4162a157adbf834ebae5b415acbecbeafc7a74b0e886657506a7611"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.72",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "derive_more"
|
||||
version = "0.99.18"
|
||||
|
|
@ -4431,12 +4408,6 @@ dependencies = [
|
|||
"scopeguard",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "lockfree-object-pool"
|
||||
version = "0.1.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9374ef4228402d4b7e403e5838cb880d9ee663314b0a900d5a6aabf0c213552e"
|
||||
|
||||
[[package]]
|
||||
name = "log"
|
||||
version = "0.4.22"
|
||||
|
|
@ -4452,16 +4423,6 @@ dependencies = [
|
|||
"hashbrown 0.14.5",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "lzma-rs"
|
||||
version = "0.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "297e814c836ae64db86b36cf2a557ba54368d03f6afcd7d947c266692f71115e"
|
||||
dependencies = [
|
||||
"byteorder",
|
||||
"crc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "lzma-sys"
|
||||
version = "0.1.20"
|
||||
|
|
@ -5008,16 +4969,6 @@ version = "1.0.15"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a"
|
||||
|
||||
[[package]]
|
||||
name = "pbkdf2"
|
||||
version = "0.12.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2"
|
||||
dependencies = [
|
||||
"digest",
|
||||
"hmac",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pem"
|
||||
version = "1.1.1"
|
||||
|
|
@ -6323,6 +6274,16 @@ dependencies = [
|
|||
"winapi-util",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "sanitize-filename"
|
||||
version = "0.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2ed72fbaf78e6f2d41744923916966c4fbe3d7c74e3037a8ee482f1115572603"
|
||||
dependencies = [
|
||||
"lazy_static",
|
||||
"regex",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "scc"
|
||||
version = "2.1.6"
|
||||
|
|
@ -6695,12 +6656,6 @@ dependencies = [
|
|||
"rand_core 0.6.4",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "simd-adler32"
|
||||
version = "0.3.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d66dc143e6b11c1eddc06d5c423cfc97062865baf299914ab64caa38182078fe"
|
||||
|
||||
[[package]]
|
||||
name = "simdutf8"
|
||||
version = "0.1.4"
|
||||
|
|
@ -8622,49 +8577,6 @@ dependencies = [
|
|||
"syn 2.0.72",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zip"
|
||||
version = "2.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "dc5e4288ea4057ae23afc69a4472434a87a2495cafce6632fd1c4ec9f5cf3494"
|
||||
dependencies = [
|
||||
"aes",
|
||||
"arbitrary",
|
||||
"bzip2",
|
||||
"constant_time_eq",
|
||||
"crc32fast",
|
||||
"crossbeam-utils",
|
||||
"deflate64",
|
||||
"displaydoc",
|
||||
"flate2",
|
||||
"hmac",
|
||||
"indexmap 2.3.0",
|
||||
"lzma-rs",
|
||||
"memchr",
|
||||
"pbkdf2",
|
||||
"rand 0.8.5",
|
||||
"sha1",
|
||||
"thiserror",
|
||||
"time",
|
||||
"zeroize",
|
||||
"zopfli",
|
||||
"zstd",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zopfli"
|
||||
version = "0.8.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e5019f391bac5cf252e93bbcc53d039ffd62c7bfb7c150414d61369afe57e946"
|
||||
dependencies = [
|
||||
"bumpalo",
|
||||
"crc32fast",
|
||||
"lockfree-object-pool",
|
||||
"log",
|
||||
"once_cell",
|
||||
"simd-adler32",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zstd"
|
||||
version = "0.13.2"
|
||||
|
|
|
|||
14
Cargo.toml
14
Cargo.toml
|
|
@ -293,13 +293,13 @@ debug = true
|
|||
[patch.crates-io]
|
||||
# It's diffcult to resovle different version with the same crate used in AppFlowy Frontend and the Client-API crate.
|
||||
# So using patch to workaround this issue.
|
||||
collab = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "8987e035accc81bd5ca0acc9812da8e4c6023eba" }
|
||||
collab-entity = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "8987e035accc81bd5ca0acc9812da8e4c6023eba" }
|
||||
collab-folder = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "8987e035accc81bd5ca0acc9812da8e4c6023eba" }
|
||||
collab-document = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "8987e035accc81bd5ca0acc9812da8e4c6023eba" }
|
||||
collab-user = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "8987e035accc81bd5ca0acc9812da8e4c6023eba" }
|
||||
collab-database = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "8987e035accc81bd5ca0acc9812da8e4c6023eba" }
|
||||
collab-importer = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "8987e035accc81bd5ca0acc9812da8e4c6023eba" }
|
||||
collab = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "270cd376ee4bfbedd9146105dea4e9e8dabc9359" }
|
||||
collab-entity = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "270cd376ee4bfbedd9146105dea4e9e8dabc9359" }
|
||||
collab-folder = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "270cd376ee4bfbedd9146105dea4e9e8dabc9359" }
|
||||
collab-document = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "270cd376ee4bfbedd9146105dea4e9e8dabc9359" }
|
||||
collab-user = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "270cd376ee4bfbedd9146105dea4e9e8dabc9359" }
|
||||
collab-database = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "270cd376ee4bfbedd9146105dea4e9e8dabc9359" }
|
||||
collab-importer = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "270cd376ee4bfbedd9146105dea4e9e8dabc9359" }
|
||||
|
||||
[features]
|
||||
history = []
|
||||
|
|
|
|||
|
|
@ -1,4 +1,3 @@
|
|||
pub mod email_notifier;
|
||||
pub mod report;
|
||||
pub mod unzip;
|
||||
pub mod worker;
|
||||
|
|
|
|||
|
|
@ -1,101 +0,0 @@
|
|||
use anyhow::{Context, Result};
|
||||
use async_zip::base::read::stream::{Ready, ZipFileReader};
|
||||
use async_zip::{StringEncoding, ZipString};
|
||||
use futures::io::{AsyncBufRead, AsyncReadExt};
|
||||
use std::ffi::OsString;
|
||||
use std::os::unix::ffi::OsStringExt;
|
||||
use std::path::PathBuf;
|
||||
use tokio::fs;
|
||||
use tokio::fs::File;
|
||||
use tokio::io::AsyncWriteExt;
|
||||
|
||||
use tracing::error;
|
||||
|
||||
pub struct UnzipFile {
|
||||
pub file_name: String,
|
||||
pub unzip_dir_path: PathBuf,
|
||||
}
|
||||
|
||||
pub async fn unzip_async<R: AsyncBufRead + Unpin>(
|
||||
mut zip_reader: ZipFileReader<Ready<R>>,
|
||||
out_dir: PathBuf,
|
||||
) -> Result<UnzipFile, anyhow::Error> {
|
||||
let mut real_file_name = None;
|
||||
while let Some(mut next_reader) = zip_reader.next_with_entry().await? {
|
||||
let entry_reader = next_reader.reader_mut();
|
||||
let filename = get_filename(entry_reader.entry().filename())
|
||||
.with_context(|| "Failed to extract filename from entry".to_string())?;
|
||||
|
||||
// Save the root folder name if we haven't done so yet
|
||||
if real_file_name.is_none() && filename.ends_with('/') {
|
||||
real_file_name = Some(filename.split('/').next().unwrap_or(&filename).to_string());
|
||||
}
|
||||
|
||||
let output_path = out_dir.join(&filename);
|
||||
if filename.ends_with('/') {
|
||||
fs::create_dir_all(&output_path)
|
||||
.await
|
||||
.with_context(|| format!("Failed to create directory: {}", output_path.display()))?;
|
||||
} else {
|
||||
// Ensure parent directories exist
|
||||
if let Some(parent) = output_path.parent() {
|
||||
if !parent.exists() {
|
||||
fs::create_dir_all(parent)
|
||||
.await
|
||||
.with_context(|| format!("Failed to create parent directory: {}", parent.display()))?;
|
||||
}
|
||||
}
|
||||
|
||||
// Write file contents
|
||||
let mut outfile = File::create(&output_path)
|
||||
.await
|
||||
.with_context(|| format!("Failed to create file: {}", output_path.display()))?;
|
||||
let mut buffer = vec![];
|
||||
match entry_reader.read_to_end(&mut buffer).await {
|
||||
Ok(_) => {
|
||||
outfile
|
||||
.write_all(&buffer)
|
||||
.await
|
||||
.with_context(|| format!("Failed to write data to file: {}", output_path.display()))?;
|
||||
},
|
||||
Err(err) => {
|
||||
error!(
|
||||
"Failed to read entry: {:?}. Error: {:?}",
|
||||
entry_reader.entry(),
|
||||
err,
|
||||
);
|
||||
return Err(anyhow::anyhow!(
|
||||
"Unexpected EOF while reading: {}",
|
||||
filename
|
||||
));
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// Move to the next file in the zip
|
||||
zip_reader = next_reader.done().await?;
|
||||
}
|
||||
|
||||
match real_file_name {
|
||||
None => Err(anyhow::anyhow!("No files found in the zip archive")),
|
||||
Some(file_name) => Ok(UnzipFile {
|
||||
file_name: file_name.clone(),
|
||||
unzip_dir_path: out_dir.join(file_name),
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn get_filename(zip_string: &ZipString) -> Result<String, anyhow::Error> {
|
||||
match zip_string.encoding() {
|
||||
StringEncoding::Utf8 => match zip_string.as_str() {
|
||||
Ok(valid_str) => Ok(valid_str.to_string()),
|
||||
Err(err) => Err(err.into()),
|
||||
},
|
||||
|
||||
StringEncoding::Raw => {
|
||||
let raw_bytes = zip_string.as_bytes();
|
||||
let os_string = OsString::from_vec(raw_bytes.to_vec());
|
||||
Ok(os_string.to_string_lossy().into_owned())
|
||||
},
|
||||
}
|
||||
}
|
||||
|
|
@ -1,5 +1,4 @@
|
|||
use crate::import_worker::report::{ImportNotifier, ImportProgress, ImportResult};
|
||||
use crate::import_worker::unzip::unzip_async;
|
||||
use crate::s3_client::S3StreamResponse;
|
||||
use anyhow::anyhow;
|
||||
use async_zip::base::read::stream::ZipFileReader;
|
||||
|
|
@ -15,6 +14,7 @@ use collab_importer::imported_collab::ImportType;
|
|||
use collab_importer::notion::page::CollabResource;
|
||||
use collab_importer::notion::NotionImporter;
|
||||
use collab_importer::util::FileId;
|
||||
use collab_importer::zip_tool::unzip_stream;
|
||||
use database::collab::{insert_into_af_collab_bulk_for_user, select_blob_from_af_collab};
|
||||
use database::workspace::{
|
||||
delete_from_workspace, select_workspace_database_storage_id, update_import_task_status,
|
||||
|
|
@ -320,7 +320,7 @@ async fn download_and_unzip_file(
|
|||
ImportError::Internal(anyhow!("Failed to set permissions for temp dir: {:?}", err))
|
||||
})?;
|
||||
|
||||
let unzip_file = unzip_async(zip_reader, output_file_path)
|
||||
let unzip_file = unzip_stream(zip_reader, output_file_path)
|
||||
.await
|
||||
.map_err(ImportError::Internal)?;
|
||||
Ok(unzip_file.unzip_dir_path)
|
||||
|
|
|
|||
Loading…
Reference in New Issue