diff --git a/backend/Cargo.lock b/backend/Cargo.lock index f7ea2b4e6f..fb6073a508 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -9981,6 +9981,7 @@ dependencies = [ "bytes", "chrono", "const_format", + "crc", "deno_ast", "deno_console", "deno_core", diff --git a/backend/windmill-worker/Cargo.toml b/backend/windmill-worker/Cargo.toml index d0b080fe2e..b6eb7cdf04 100644 --- a/backend/windmill-worker/Cargo.toml +++ b/backend/windmill-worker/Cargo.toml @@ -14,7 +14,7 @@ prometheus = ["dep:prometheus", "windmill-common/prometheus"] enterprise = ["windmill-queue/enterprise", "windmill-git-sync/enterprise", "windmill-common/enterprise", "dep:gcp_auth", "dep:jsonwebtoken", "dep:pem", "dep:sha2", "dep:tiberius", "dep:tokio-util", "dep:openidconnect"] benchmark = ["windmill-queue/benchmark"] flamegraph = [] -parquet = ["windmill-common/parquet", "dep:object_store", "dep:tar"] +parquet = ["windmill-common/parquet", "dep:object_store", "dep:tar", "dep:crc"] flow_testing = [] [dependencies] @@ -80,6 +80,7 @@ tokio-util = { workspace = true, optional = true } openidconnect = { workspace = true, optional = true} tar = { workspace = true, optional = true} object_store = { workspace = true, optional = true} +crc = { workspace = true, optional = true} [build-dependencies] deno_fetch.workspace = true diff --git a/backend/windmill-worker/src/global_cache.rs b/backend/windmill-worker/src/global_cache.rs index 56aa4c8eae..04a9905556 100644 --- a/backend/windmill-worker/src/global_cache.rs +++ b/backend/windmill-worker/src/global_cache.rs @@ -71,6 +71,9 @@ pub async fn build_tar_and_push( Ok(()) } +#[cfg(all(feature = "enterprise", feature = "parquet"))] +pub const X25: crc::Crc = crc::Crc::::new(&crc::CRC_16_IBM_SDLC); + #[cfg(all(feature = "enterprise", feature = "parquet"))] pub async fn pull_from_tar(client: Arc, folder: String) -> error::Result<()> { use object_store::path::Path; @@ -100,10 +103,9 @@ pub async fn pull_from_tar(client: Arc, folder: String) -> erro .open(&target) .await .unwrap(); - - file.write_all(&object.unwrap().bytes().await.unwrap()) - .await - .unwrap(); + let bytes = object.unwrap().bytes().await.unwrap(); + tracing::info!("{tar_path} checksum: {}", X25.checksum(&bytes)); + file.write_all(&bytes).await.unwrap(); if metadata(&target).await.is_err() { tracing::info!(