From ac8d2d6e257c31bebab66cd19ccd8a27f0093307 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Sun, 15 Dec 2024 01:30:54 +0100 Subject: [PATCH] improve backend zip lib --- backend/Cargo.lock | 66 +++++++++----------------- backend/Cargo.toml | 2 +- backend/windmill-api/src/workspaces.rs | 29 ++++++----- 3 files changed, 40 insertions(+), 57 deletions(-) diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 8e9206433c..1edfc85db3 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -455,23 +455,6 @@ dependencies = [ "futures-core", ] -[[package]] -name = "async-compression" -version = "0.3.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "942c7cd7ae39e91bde4820d74132e9862e62c2f386c3aa90ccf55949f5bad63a" -dependencies = [ - "bzip2", - "flate2", - "futures-core", - "memchr", - "pin-project-lite", - "tokio", - "xz2", - "zstd 0.11.2+zstd.1.5.2", - "zstd-safe 5.0.2+zstd.1.5.2", -] - [[package]] name = "async-compression" version = "0.4.18" @@ -581,17 +564,18 @@ dependencies = [ [[package]] name = "async_zip" -version = "0.0.11" +version = "0.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c50d29ab7e2f9e808cca1a69ea56a36f4ff216f54a41a23aae1fd4afc05cc020" +checksum = "00b9f7252833d5ed4b00aa9604b563529dd5e11de9c23615de2dcdf91eb87b52" dependencies = [ - "async-compression 0.3.15", + "async-compression", "chrono", "crc32fast", - "log", + "futures-lite 2.5.0", "pin-project", "thiserror 1.0.69", "tokio", + "tokio-util", ] [[package]] @@ -2194,7 +2178,7 @@ dependencies = [ "arrow-array", "arrow-ipc", "arrow-schema", - "async-compression 0.4.18", + "async-compression", "async-trait", "bytes", "bzip2", @@ -3547,6 +3531,19 @@ dependencies = [ "waker-fn", ] +[[package]] +name = "futures-lite" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cef40d21ae2c515b51041df9ed313ed21e572df340ea58a922a0aefe7e8891a1" +dependencies = [ + "fastrand 2.3.0", + "futures-core", + "futures-io", + "parking", + "pin-project-lite", +] + [[package]] name = "futures-macro" version = "0.3.31" @@ -4185,7 +4182,7 @@ dependencies = [ "anyhow", "async-channel", "base64 0.13.1", - "futures-lite", + "futures-lite 1.13.0", "http 0.2.12", "infer", "pin-project-lite", @@ -7023,7 +7020,7 @@ version = "0.12.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a77c62af46e79de0a562e1a9849205ffcb7fc1238876e9bd743357570e04046f" dependencies = [ - "async-compression 0.4.18", + "async-compression", "base64 0.22.1", "bytes", "encoding_rs", @@ -9829,7 +9826,7 @@ version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "403fa3b783d4b626a8ad51d766ab03cb6d2dbfc46b1c5d4448395e6628dc9697" dependencies = [ - "async-compression 0.4.18", + "async-compression", "bitflags 2.6.0", "bytes", "futures-core", @@ -11719,15 +11716,6 @@ dependencies = [ "crossbeam-utils", ] -[[package]] -name = "zstd" -version = "0.11.2+zstd.1.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20cc960326ece64f010d2d2107537f26dc589a6573a316bd5b1dba685fa5fde4" -dependencies = [ - "zstd-safe 5.0.2+zstd.1.5.2", -] - [[package]] name = "zstd" version = "0.12.4" @@ -11746,16 +11734,6 @@ dependencies = [ "zstd-safe 7.2.1", ] -[[package]] -name = "zstd-safe" -version = "5.0.2+zstd.1.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d2a5585e04f9eea4b2a3d1eca508c4dee9592a89ef6f450c11719da0726f4db" -dependencies = [ - "libc", - "zstd-sys", -] - [[package]] name = "zstd-safe" version = "6.0.6" diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 0dd9f2df1a..9a614b6776 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -244,7 +244,7 @@ async-stripe = { version = "0.39.1", features = [ "checkout", "billing", ] } -async_zip = { version = "0.0.11", features = ["full"] } +async_zip = { version = "0.0.17", features = ["tokio", "tokio-fs", "deflate", "chrono"] } once_cell = "1.17.1" gosyn = "0.2.6" bytes = "1.4.0" diff --git a/backend/windmill-api/src/workspaces.rs b/backend/windmill-api/src/workspaces.rs index e7558b0767..1519b9d46f 100644 --- a/backend/windmill-api/src/workspaces.rs +++ b/backend/windmill-api/src/workspaces.rs @@ -2116,7 +2116,7 @@ pub fn is_none_or_false(val: &Option) -> bool { enum ArchiveImpl { #[cfg(feature = "zip")] - Zip(async_zip::write::ZipFileWriter), + Zip(async_zip::tokio::write::ZipFileWriter), Tar(tokio_tar::Builder), } @@ -2136,13 +2136,11 @@ impl ArchiveImpl { } #[cfg(feature = "zip")] ArchiveImpl::Zip(z) => { - let header = async_zip::ZipEntryBuilder::new( - path.to_owned(), - async_zip::Compression::Deflate, - ) - .last_modification_date(Default::default()) - .unix_permissions(0o777) - .build(); + let header = + async_zip::ZipEntryBuilder::new(path.into(), async_zip::Compression::Deflate) + .last_modification_date(Default::default()) + .unix_permissions(0o777) + .build(); z.write_entry_whole(header, content.as_bytes()) .await .map_err(to_anyhow)?; @@ -2154,7 +2152,7 @@ impl ArchiveImpl { match self { ArchiveImpl::Tar(t) => t.into_inner().await?, #[cfg(feature = "zip")] - ArchiveImpl::Zip(z) => z.close().await.map_err(to_anyhow)?, + ArchiveImpl::Zip(z) => z.close().await.map_err(to_anyhow)?.into_inner(), } .sync_all() .await?; @@ -2305,11 +2303,18 @@ async fn tarball_workspace( Some(t) => Err(Error::BadRequest(format!("Invalid Archive Type {t}"))), }?; let file_path = tmp_dir.path().join(&name); - let file = File::create(&file_path).await?; let mut archive = match archive_type.as_deref() { - Some("tar") | None => Ok(ArchiveImpl::Tar(tokio_tar::Builder::new(file))), + Some("tar") | None => { + let file = File::create(&file_path).await?; + Ok(ArchiveImpl::Tar(tokio_tar::Builder::new(file))) + } #[cfg(feature = "zip")] - Some("zip") => Ok(ArchiveImpl::Zip(async_zip::write::ZipFileWriter::new(file))), + Some("zip") => { + let file = tokio::fs::File::create(&file_path).await?; + Ok(ArchiveImpl::Zip( + async_zip::tokio::write::ZipFileWriter::with_tokio(file), + )) + } Some(t) => Err(Error::BadRequest(format!("Invalid Archive Type {t}"))), }?; {