mirror of
https://github.com/windmill-labs/windmill.git
synced 2026-10-13 00:03:00 +00:00
chore: bump datafusion, object_store and quick-xml off the vulnerable set
`quick-xml` < 0.41 is affected by RUSTSEC-2026-0194 and RUSTSEC-2026-0195 (both CVSS 7.5). The only `object_store` releases that depend on `quick-xml` >= 0.41 are 0.14.1+, and no published `datafusion` release accepts `object_store` 0.14 yet, so `datafusion` is pinned to the apache/datafusion commit that moved to arrow/parquet 60 and `object_store` 0.14.2. That set also drops `thrift` (parquet >= 59 no longer uses it), closing GHSA-2f9f-gq7v-9h6m. - datafusion 47.0.0 -> git 2cac7132 (arrow/parquet 60, object_store 0.14.2) - object_store git 36752c97 (0.12.0) -> 0.14.2; the pinned rev's Azure query-signing fix (arrow-rs-object-store#334) shipped in 0.12.1, so the `[patch.crates-io]` entry and the git dep are both dropped - parquet 55.2.0 -> 60.0.0, arrow 55.2.0 -> 60.0.0 - quick-xml 0.37 -> 0.41; samael pinned forward to the rev that did the same, since it was the other source of a 0.37 copy - tokio =1.46.1 -> =1.53.1, forced by datafusion's `tokio ^1.52` - `reqwest_object_store` (reqwest 0.12) folded into the workspace `reqwest` 0.13, which object_store 0.14 now builds its clients with Code adaptations: `get`/`get_range`/`head`/`delete`/`copy`/`put`/ `put_multipart` moved to `ObjectStoreExt`, `copy`/`rename` to `copy_opts`/`rename_opts`, `delete` to `delete_stream` with a `'static` stream, `Error::NotImplemented` gained fields, `NdJsonReadOptions` became `JsonReadOptions`, `DFSchema: Into<Schema>` is gone, and `set_max_row_group_size` became `set_max_row_group_row_count`. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Kv72vdCDggCZEjJSmNCwnX
This commit is contained in:
co-authored by
Claude Opus 5
parent
cedd6dc901
commit
6b00d085e5
Generated
+564
-470
File diff suppressed because it is too large
Load Diff
+9
-9
@@ -246,7 +246,6 @@ sqlx-macros-core = { git = "https://github.com/windmill-labs/sqlx", rev = "6bdae
|
||||
sqlx-postgres = { git = "https://github.com/windmill-labs/sqlx", rev = "6bdaee94fa62a01561125646da3f99eb341f2457" }
|
||||
sqlx-mysql = { git = "https://github.com/windmill-labs/sqlx", rev = "6bdaee94fa62a01561125646da3f99eb341f2457" }
|
||||
sqlx-sqlite = { git = "https://github.com/windmill-labs/sqlx", rev = "6bdaee94fa62a01561125646da3f99eb341f2457" }
|
||||
object_store = { git = "https://github.com/apache/arrow-rs-object-store", rev = "36752c975d4f29e20b57c91f81a10872dcd48ae7" }
|
||||
# Use tiberius main branch for libgssapi 0.8.1 fix (https://github.com/prisma/tiberius/issues/343)
|
||||
tiberius = { git = "https://github.com/prisma/tiberius", rev = "59db57960a14b422fb3a1309aa4aa47880896ff8" }
|
||||
# Pin tokio-postgres / postgres-types / postgres-protocol to the
|
||||
@@ -462,7 +461,7 @@ headers = "^0"
|
||||
hyper = { version = "^1", features = ["full"] }
|
||||
hyper-tls = "^0.6"
|
||||
hyper-util = { version = "^0.1", features = ["client-legacy", "http1", "tokio"] }
|
||||
tokio = { version = "=1.46.1", features = ["full", "tracing", "time"] }
|
||||
tokio = { version = "=1.53.1", features = ["full", "tracing", "time"] }
|
||||
tokio-stream = { version = "0.1.17" }
|
||||
tower = "^0"
|
||||
tower-http = { version = "^0.6", features = ["trace", "cors", "catch-panic"] }
|
||||
@@ -507,12 +506,10 @@ php-parser-rs = { git = "https://github.com/php-rust-tools/parser", rev = "ec4cb
|
||||
cron = "^0"
|
||||
mail-send = { version = "0.4.0", features = ["builder"], default-features=false }
|
||||
urlencoding = "^2"
|
||||
quick-xml = { version = "^0.37", features = ["serialize"] }
|
||||
quick-xml = { version = "^0.41", features = ["serialize"] }
|
||||
url = { version = "^2" , features = ["serde"]}
|
||||
async-oauth2 = "0.5.1"
|
||||
reqwest = { version = "^0.13", features = ["json", "stream", "gzip", "multipart", "query", "form"] }
|
||||
# The reqwest object_store builds its HTTP clients with, for a custom `HttpConnector`.
|
||||
reqwest_object_store = { package = "reqwest", version = "0.12", default-features = false, features = ["rustls-tls-native-roots"] }
|
||||
eventsource-stream = "0.2.3"
|
||||
time = "^0"
|
||||
serde_urlencoded = "^0"
|
||||
@@ -636,7 +633,7 @@ native-tls = ">=0.2, <0.2.17"
|
||||
# samael will break compilation on MacOS. Use this fork instead to make it work
|
||||
# samael = { git="https://github.com/njaremko/samael", rev="464d015e3ae393e4b5dd00b4d6baa1b617de0dd6", features = ["xmlsec"] }
|
||||
libxml = { version = "=0.3.3" }
|
||||
samael = { git="https://github.com/njaremko/samael", rev="f879f1942ec1b34b6d3027ce7e4724ad95d15dfa", features = ["xmlsec"] }
|
||||
samael = { git="https://github.com/njaremko/samael", rev="2d27dfe4930c4632b202f32886a85b2d1f9226c2", features = ["xmlsec"] }
|
||||
gcp_auth = "0.9.0"
|
||||
rust_decimal = { version = "^1", features = ["db-postgres", "serde-float"]}
|
||||
# aws_lc_rs, not rust_crypto: the RustCrypto backend refuses RSA keys above 4096 bits, and
|
||||
@@ -672,14 +669,17 @@ process-wrap = { version = "8.2.1", features = ["tokio1"] }
|
||||
|
||||
systemstat = "0.2.4"
|
||||
|
||||
datafusion = "47.0.0"
|
||||
# No datafusion release yet depends on `object_store` 0.14, and only `object_store` >= 0.14.1
|
||||
# carries `quick-xml` >= 0.41 (RUSTSEC-2026-0194, RUSTSEC-2026-0195). Drop the git pin for
|
||||
# `datafusion = "56"` once it is published.
|
||||
datafusion = { git = "https://github.com/apache/datafusion", rev = "2cac713274496ba8ee08e58655e80f0d464f6d48" }
|
||||
# The row API only: a dbt engine's parquet index is six string columns, so this
|
||||
# needs no arrow and no writer. `parquet` is already in the tree with `arrow` for
|
||||
# every shipped edition (`oss_core`), and cargo unifies the features there; this
|
||||
# set is what a build WITHOUT object storage compiles. ZSTD is what the engine
|
||||
# writes today, snap what parquet writers most often default to.
|
||||
parquet = { version = "55.2.0", default-features = false, features = ["snap", "zstd"] }
|
||||
object_store = { git = "https://github.com/apache/arrow-rs-object-store", rev = "36752c975d4f29e20b57c91f81a10872dcd48ae7", features = ["aws", "azure", "gcp"] }
|
||||
parquet = { version = "60.0.0", default-features = false, features = ["snap", "zstd"] }
|
||||
object_store = { version = "0.14.2", features = ["aws", "azure", "gcp"] }
|
||||
openidconnect = { version = "4.0.0-rc.1" }
|
||||
aws-config = "^1"
|
||||
aws-sdk-bedrock = "1.129.0"
|
||||
|
||||
@@ -5,6 +5,11 @@
|
||||
* Please see the included NOTICE for copyright information and
|
||||
* LICENSE-AGPL for a copy of the license.
|
||||
*/
|
||||
|
||||
// `sqlparser`'s AST (reached through datafusion) nests deeply enough that inferring
|
||||
// `Send`/`Sync` for the futures spawned in `main` exceeds the default 128-step limit.
|
||||
#![recursion_limit = "256"]
|
||||
|
||||
use anyhow::Context;
|
||||
use monitor::{
|
||||
flush_pending_log_files_to_object_store, load_base_url, load_otel,
|
||||
|
||||
@@ -1471,6 +1471,8 @@ async fn send_log_file_to_object_store(
|
||||
) -> bool {
|
||||
#[cfg(feature = "parquet")]
|
||||
if let Some(s3_client) = windmill_object_store::get_object_store().await {
|
||||
use windmill_object_store::object_store_reexports::ObjectStoreExt;
|
||||
|
||||
let path = std::path::Path::new(&*TMP_WINDMILL_LOGS_SERVICE)
|
||||
.join(hostname)
|
||||
.join(file_name);
|
||||
|
||||
@@ -284,6 +284,7 @@ pub(crate) async fn pull_from_object_store(_dir: &Path, _key: &str) -> Result<Op
|
||||
{
|
||||
use futures::StreamExt;
|
||||
use tokio::io::AsyncWriteExt;
|
||||
use windmill_object_store::object_store_reexports::ObjectStoreExt;
|
||||
|
||||
let Some(os) = windmill_object_store::get_object_store().await else {
|
||||
return Ok(None);
|
||||
@@ -399,7 +400,7 @@ pub(crate) fn push_to_object_store(_dir: PathBuf, _key: String) {
|
||||
#[cfg(all(feature = "enterprise", feature = "parquet"))]
|
||||
async fn push_inner(dir: &Path, key: &str) -> Result<()> {
|
||||
use tokio::io::AsyncReadExt;
|
||||
use windmill_object_store::object_store_reexports::WriteMultipart;
|
||||
use windmill_object_store::object_store_reexports::{ObjectStoreExt, WriteMultipart};
|
||||
|
||||
let Some(os) = windmill_object_store::get_object_store().await else {
|
||||
return Ok(());
|
||||
|
||||
@@ -42,7 +42,9 @@ use windmill_common::error::{self};
|
||||
use windmill_common::tracing_init::LOGS_AUDIT;
|
||||
use windmill_common::{DB, INSTANCE_NAME};
|
||||
|
||||
use windmill_object_store::object_store_reexports::{ObjectStore, Path as ObjectPath};
|
||||
use windmill_object_store::object_store_reexports::{
|
||||
ObjectStore, ObjectStoreExt, Path as ObjectPath,
|
||||
};
|
||||
|
||||
pub const TASK_NAME: &str = "audit_logs_s3_backfill";
|
||||
|
||||
@@ -423,7 +425,9 @@ mod tests {
|
||||
use futures::stream::StreamExt;
|
||||
use std::sync::atomic::Ordering;
|
||||
use std::sync::Arc;
|
||||
use windmill_object_store::object_store_reexports::{InMemory, ObjectStore, Path as OsPath};
|
||||
use windmill_object_store::object_store_reexports::{
|
||||
InMemory, ObjectStore, ObjectStoreExt, Path as OsPath,
|
||||
};
|
||||
|
||||
/// A private, per-test object store. `run_backfill` takes the store as a parameter,
|
||||
/// so tests use a local one and never touch the process-global `OBJECT_STORE_SETTINGS`
|
||||
|
||||
@@ -302,6 +302,9 @@ pub async fn test_email(
|
||||
#[cfg(feature = "parquet")]
|
||||
use windmill_object_store::ObjectSettings;
|
||||
|
||||
#[cfg(feature = "parquet")]
|
||||
use windmill_object_store::object_store_reexports::ObjectStoreExt;
|
||||
|
||||
#[cfg(feature = "parquet")]
|
||||
use windmill_object_store::{
|
||||
build_object_store_from_settings, build_public_object_store_from_settings,
|
||||
|
||||
@@ -32,7 +32,7 @@ use windmill_common::error::{Error, Result};
|
||||
use windmill_common::utils::calculate_hash;
|
||||
use windmill_common::DB;
|
||||
use windmill_object_store::object_store_reexports::{
|
||||
ObjectStore, ObjectStoreError, Path as ObjectPath,
|
||||
ObjectStore, ObjectStoreError, ObjectStoreExt, Path as ObjectPath,
|
||||
};
|
||||
use windmill_object_store::{
|
||||
object_store_error_to_error, object_store_location, ObjectStoreResource,
|
||||
|
||||
@@ -8011,6 +8011,8 @@ async fn clone_apps(
|
||||
// Clone bundles from S3 for versions not found in DB
|
||||
#[cfg(all(feature = "enterprise", feature = "parquet"))]
|
||||
{
|
||||
use windmill_object_store::object_store_reexports::ObjectStoreExt;
|
||||
|
||||
let object_store = windmill_object_store::get_object_store().await;
|
||||
if let Some(os) = object_store {
|
||||
for (&old_version_id, &new_version_id) in &version_id_mapping {
|
||||
|
||||
@@ -2008,7 +2008,7 @@ async fn delete_fork_ducklake_data(
|
||||
// 1000-object chunks: S3 DeleteObjects caps a batch at 1000 keys.
|
||||
for chunk in locations.chunks(1000) {
|
||||
store
|
||||
.delete_stream(futures::stream::iter(chunk.iter().cloned().map(Ok)).boxed())
|
||||
.delete_stream(futures::stream::iter(chunk.to_vec().into_iter().map(Ok)).boxed())
|
||||
.try_collect::<Vec<_>>()
|
||||
.await
|
||||
.map_err(windmill_object_store::object_store_error_to_error)?;
|
||||
|
||||
@@ -37,7 +37,7 @@ use windmill_common::error::{Error, JsonResult, Result};
|
||||
use windmill_common::utils::calculate_hash;
|
||||
use windmill_common::variables::{crypt_from_key_with_suffix, get_workspace_key};
|
||||
use windmill_object_store::object_store_reexports::{
|
||||
ObjectStore, ObjectStoreError, Path as ObjectPath, PutPayload,
|
||||
ObjectStore, ObjectStoreError, ObjectStoreExt, Path as ObjectPath, PutPayload,
|
||||
};
|
||||
use windmill_object_store::{build_object_store_client, object_store_error_to_error};
|
||||
|
||||
|
||||
@@ -80,7 +80,7 @@ use windmill_common::{
|
||||
HUB_BASE_URL,
|
||||
};
|
||||
#[cfg(feature = "parquet")]
|
||||
use windmill_object_store::object_store_reexports::{Attribute, Attributes};
|
||||
use windmill_object_store::object_store_reexports::{Attribute, Attributes, ObjectStoreExt};
|
||||
use windmill_store::resources::get_resource_value_interpolated_internal;
|
||||
|
||||
use windmill_api_auth::{
|
||||
|
||||
@@ -18,7 +18,7 @@ use windmill_common::db::UserDB;
|
||||
#[cfg(not(feature = "private"))]
|
||||
use windmill_common::error;
|
||||
#[cfg(all(feature = "parquet", not(feature = "private")))]
|
||||
use windmill_object_store::object_store_reexports::{ObjectStore, PutMultipartOpts, PutResult};
|
||||
use windmill_object_store::object_store_reexports::{ObjectStore, PutMultipartOptions, PutResult};
|
||||
#[cfg(not(feature = "private"))]
|
||||
use windmill_object_store::ObjectStoreResource;
|
||||
|
||||
@@ -86,7 +86,7 @@ pub async fn upload_file_from_req(
|
||||
_s3_client: Arc<dyn ObjectStore>,
|
||||
_file_key: &str,
|
||||
_req: axum::extract::Request,
|
||||
_options: PutMultipartOpts,
|
||||
_options: PutMultipartOptions,
|
||||
_max_size: Option<usize>,
|
||||
) -> error::Result<(PutResult, usize)> {
|
||||
Err(error::Error::internal_err(
|
||||
@@ -99,7 +99,7 @@ pub async fn upload_file_internal(
|
||||
_s3_client: Arc<dyn ObjectStore>,
|
||||
_file_key: &str,
|
||||
_stream: impl Stream<Item = Result<Bytes, std::io::Error>> + Unpin,
|
||||
_options: PutMultipartOpts,
|
||||
_options: PutMultipartOptions,
|
||||
_max_size: Option<usize>,
|
||||
) -> error::Result<(PutResult, usize)> {
|
||||
Err(error::Error::internal_err(
|
||||
|
||||
@@ -10217,6 +10217,8 @@ async fn get_log_file(
|
||||
|
||||
#[cfg(all(feature = "enterprise", feature = "parquet"))]
|
||||
if let Some(os) = windmill_object_store::get_object_store().await {
|
||||
use windmill_object_store::object_store_reexports::ObjectStoreExt;
|
||||
|
||||
let file = os
|
||||
.get(&windmill_object_store::object_store_reexports::Path::from(
|
||||
format!("logs/{file_p}"),
|
||||
|
||||
@@ -164,7 +164,7 @@ async fn get_log_file(
|
||||
let s3_client = windmill_object_store::get_object_store().await;
|
||||
#[cfg(feature = "parquet")]
|
||||
if let Some(s3_client) = s3_client {
|
||||
use windmill_object_store::object_store_reexports::ObjectStoreError;
|
||||
use windmill_object_store::object_store_reexports::{ObjectStoreError, ObjectStoreExt};
|
||||
|
||||
// The raw file, for as long as it is there. It outlives its ingestion by
|
||||
// one indexer pass at most, so this covers the most recent minutes of a
|
||||
|
||||
@@ -613,6 +613,8 @@ async fn route_job(
|
||||
#[cfg(feature = "parquet")]
|
||||
if let Some(sqlx::types::Json(config)) = trigger.static_asset_config {
|
||||
let build_static_response_f = async {
|
||||
use windmill_object_store::object_store_reexports::ObjectStoreExt;
|
||||
|
||||
let (_, s3_resource_opt) = get_workspace_s3_resource(
|
||||
&authed,
|
||||
&db,
|
||||
|
||||
@@ -11,7 +11,6 @@ enterprise = []
|
||||
parquet = [
|
||||
"windmill-common/parquet",
|
||||
"dep:object_store",
|
||||
"dep:reqwest_object_store",
|
||||
"dep:aws-sdk-sts",
|
||||
"dep:aws-smithy-types-convert",
|
||||
"dep:datafusion",
|
||||
@@ -49,7 +48,6 @@ globset.workspace = true
|
||||
axum.workspace = true
|
||||
|
||||
object_store = { workspace = true, optional = true }
|
||||
reqwest_object_store = { workspace = true, optional = true }
|
||||
aws-sdk-sts = { workspace = true, optional = true }
|
||||
aws-smithy-types-convert = { workspace = true, optional = true }
|
||||
datafusion = { workspace = true, optional = true }
|
||||
|
||||
@@ -38,6 +38,8 @@ use object_store::CredentialProvider;
|
||||
#[cfg(feature = "parquet")]
|
||||
use object_store::ObjectStore;
|
||||
#[cfg(feature = "parquet")]
|
||||
use object_store::ObjectStoreExt;
|
||||
#[cfg(feature = "parquet")]
|
||||
use object_store::{aws::AmazonS3Builder, ClientOptions};
|
||||
#[cfg(feature = "parquet")]
|
||||
use reqwest::header::HeaderMap;
|
||||
@@ -68,8 +70,8 @@ pub mod object_store_reexports {
|
||||
pub use object_store::path::Path;
|
||||
pub use object_store::{
|
||||
Attribute, Attributes, Error as ObjectStoreError, GetOptions, GetRange, GetResult,
|
||||
ObjectMeta, ObjectStore, PutMode, PutMultipartOpts, PutOptions, PutPayload, PutResult,
|
||||
Result as ObjectStoreResult, UpdateVersion, WriteMultipart,
|
||||
ObjectMeta, ObjectStore, ObjectStoreExt, PutMode, PutMultipartOptions, PutOptions,
|
||||
PutPayload, PutResult, Result as ObjectStoreResult, UpdateVersion, WriteMultipart,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -96,7 +98,10 @@ pub fn object_store_error_to_error(err: object_store::Error) -> error::Error {
|
||||
NotModified { path, source } => {
|
||||
error::Error::ExecutionErr(format!("Not modified at {}: {}", path, source))
|
||||
}
|
||||
NotImplemented => error::Error::BadRequest("Operation not yet implemented.".to_string()),
|
||||
NotImplemented { operation, implementer } => error::Error::BadRequest(format!(
|
||||
"Operation '{}' is not implemented by {}.",
|
||||
operation, implementer
|
||||
)),
|
||||
PermissionDenied { path, source } => {
|
||||
error::Error::PermissionDenied(format!("Permission denied at {}: {}", path, source))
|
||||
}
|
||||
@@ -610,11 +615,11 @@ impl object_store::client::HttpConnector for PublicOnlyConnector {
|
||||
.get_config_value(&object_store::ClientConfigKey::AllowHttp)
|
||||
.is_some_and(|v| v == "true");
|
||||
let proxy_hosts: Arc<[String]> = system_proxy_hosts().into();
|
||||
let client = reqwest_object_store::Client::builder()
|
||||
let client = reqwest::Client::builder()
|
||||
.dns_resolver(Arc::new(PublicOnlyResolver {
|
||||
proxy_hosts: proxy_hosts.clone(),
|
||||
}))
|
||||
.redirect(reqwest_object_store::redirect::Policy::none())
|
||||
.redirect(reqwest::redirect::Policy::none())
|
||||
.connect_timeout(std::time::Duration::from_secs(5))
|
||||
.default_headers(HeaderMap::from_iter([(
|
||||
reqwest::header::ACCEPT_ENCODING,
|
||||
@@ -640,7 +645,7 @@ impl object_store::client::HttpConnector for PublicOnlyConnector {
|
||||
#[cfg(feature = "parquet")]
|
||||
#[derive(Debug)]
|
||||
struct PublicOnlyClient {
|
||||
client: reqwest_object_store::Client,
|
||||
client: reqwest::Client,
|
||||
proxy_hosts: Arc<[String]>,
|
||||
}
|
||||
|
||||
@@ -714,11 +719,8 @@ fn system_proxy_hosts() -> Vec<String> {
|
||||
}
|
||||
|
||||
#[cfg(feature = "parquet")]
|
||||
impl reqwest_object_store::dns::Resolve for PublicOnlyResolver {
|
||||
fn resolve(
|
||||
&self,
|
||||
name: reqwest_object_store::dns::Name,
|
||||
) -> reqwest_object_store::dns::Resolving {
|
||||
impl reqwest::dns::Resolve for PublicOnlyResolver {
|
||||
fn resolve(&self, name: reqwest::dns::Name) -> reqwest::dns::Resolving {
|
||||
let is_proxy = self
|
||||
.proxy_hosts
|
||||
.iter()
|
||||
@@ -737,7 +739,7 @@ impl reqwest_object_store::dns::Resolve for PublicOnlyResolver {
|
||||
)
|
||||
.into());
|
||||
}
|
||||
Ok(Box::new(addrs.into_iter()) as reqwest_object_store::dns::Addrs)
|
||||
Ok(Box::new(addrs.into_iter()) as reqwest::dns::Addrs)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1005,7 +1007,7 @@ impl ObjectStore for FilesystemStoreIgnoringAttributes {
|
||||
async fn put_multipart_opts(
|
||||
&self,
|
||||
location: &object_store::path::Path,
|
||||
mut opts: object_store::PutMultipartOpts,
|
||||
mut opts: object_store::PutMultipartOptions,
|
||||
) -> object_store::Result<Box<dyn object_store::MultipartUpload>> {
|
||||
opts.attributes = Default::default();
|
||||
self.0.put_multipart_opts(location, opts).await
|
||||
@@ -1019,14 +1021,6 @@ impl ObjectStore for FilesystemStoreIgnoringAttributes {
|
||||
self.0.get_opts(location, options).await
|
||||
}
|
||||
|
||||
async fn get_range(
|
||||
&self,
|
||||
location: &object_store::path::Path,
|
||||
range: std::ops::Range<u64>,
|
||||
) -> object_store::Result<Bytes> {
|
||||
self.0.get_range(location, range).await
|
||||
}
|
||||
|
||||
async fn get_ranges(
|
||||
&self,
|
||||
location: &object_store::path::Path,
|
||||
@@ -1035,15 +1029,14 @@ impl ObjectStore for FilesystemStoreIgnoringAttributes {
|
||||
self.0.get_ranges(location, ranges).await
|
||||
}
|
||||
|
||||
async fn head(
|
||||
fn delete_stream(
|
||||
&self,
|
||||
location: &object_store::path::Path,
|
||||
) -> object_store::Result<object_store::ObjectMeta> {
|
||||
self.0.head(location).await
|
||||
}
|
||||
|
||||
async fn delete(&self, location: &object_store::path::Path) -> object_store::Result<()> {
|
||||
self.0.delete(location).await
|
||||
locations: futures::stream::BoxStream<
|
||||
'static,
|
||||
object_store::Result<object_store::path::Path>,
|
||||
>,
|
||||
) -> futures::stream::BoxStream<'static, object_store::Result<object_store::path::Path>> {
|
||||
self.0.delete_stream(locations)
|
||||
}
|
||||
|
||||
fn list(
|
||||
@@ -1068,36 +1061,22 @@ impl ObjectStore for FilesystemStoreIgnoringAttributes {
|
||||
self.0.list_with_delimiter(prefix).await
|
||||
}
|
||||
|
||||
async fn copy(
|
||||
async fn copy_opts(
|
||||
&self,
|
||||
from: &object_store::path::Path,
|
||||
to: &object_store::path::Path,
|
||||
options: object_store::CopyOptions,
|
||||
) -> object_store::Result<()> {
|
||||
self.0.copy(from, to).await
|
||||
self.0.copy_opts(from, to, options).await
|
||||
}
|
||||
|
||||
async fn rename(
|
||||
async fn rename_opts(
|
||||
&self,
|
||||
from: &object_store::path::Path,
|
||||
to: &object_store::path::Path,
|
||||
options: object_store::RenameOptions,
|
||||
) -> object_store::Result<()> {
|
||||
self.0.rename(from, to).await
|
||||
}
|
||||
|
||||
async fn copy_if_not_exists(
|
||||
&self,
|
||||
from: &object_store::path::Path,
|
||||
to: &object_store::path::Path,
|
||||
) -> object_store::Result<()> {
|
||||
self.0.copy_if_not_exists(from, to).await
|
||||
}
|
||||
|
||||
async fn rename_if_not_exists(
|
||||
&self,
|
||||
from: &object_store::path::Path,
|
||||
to: &object_store::path::Path,
|
||||
) -> object_store::Result<()> {
|
||||
self.0.rename_if_not_exists(from, to).await
|
||||
self.0.rename_opts(from, to, options).await
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1840,7 +1819,7 @@ where
|
||||
const WRITE_BUF_CAPACITY: usize = 256 * 1024;
|
||||
const PROGRESS_INTERVAL_SECS: u64 = 10;
|
||||
|
||||
use datafusion::{execution::context::SessionContext, prelude::NdJsonReadOptions};
|
||||
use datafusion::{execution::context::SessionContext, prelude::JsonReadOptions};
|
||||
use futures::StreamExt;
|
||||
use std::path::PathBuf;
|
||||
use std::time::{Duration, Instant};
|
||||
@@ -1935,20 +1914,20 @@ where
|
||||
ctx.register_json(
|
||||
"my_table",
|
||||
path_str,
|
||||
NdJsonReadOptions::default().schema(&inferred_schema),
|
||||
JsonReadOptions::default().schema(&inferred_schema),
|
||||
)
|
||||
.await
|
||||
.map_err(to_anyhow)?;
|
||||
|
||||
let df = ctx.sql("SELECT * FROM my_table").await.map_err(to_anyhow)?;
|
||||
let schema = df.schema().clone().into();
|
||||
let schema = df.schema().inner().clone();
|
||||
let mut datafusion_stream = df.execute_stream().await.map_err(to_anyhow)?;
|
||||
|
||||
let (tx, rx) = tokio::sync::mpsc::channel(MAX_MPSC_SIZE);
|
||||
let writer: Arc<Mutex<Option<RecordBatchWriterEnum>>> =
|
||||
Arc::new(Mutex::new(Some(match output_format {
|
||||
S3ModeFormat::Parquet => RecordBatchWriterEnum::Parquet(
|
||||
ArrowWriter::try_new(ChannelWriter { sender: tx.clone() }, Arc::new(schema), None)
|
||||
ArrowWriter::try_new(ChannelWriter { sender: tx.clone() }, schema, None)
|
||||
.map_err(to_anyhow)?,
|
||||
),
|
||||
|
||||
@@ -2503,7 +2482,10 @@ mod tests {
|
||||
assert!(matches!(mapped, Error::BadRequest(_)));
|
||||
|
||||
// NotImplemented
|
||||
let mapped = object_store_error_to_error(object_store::Error::NotImplemented);
|
||||
let mapped = object_store_error_to_error(object_store::Error::NotImplemented {
|
||||
operation: "put".to_string(),
|
||||
implementer: "LocalFileSystem".to_string(),
|
||||
});
|
||||
assert!(matches!(mapped, Error::BadRequest(_)));
|
||||
|
||||
// Unauthenticated
|
||||
|
||||
@@ -542,7 +542,7 @@ async fn forget_objects(keys: &[Option<String>; 2]) {
|
||||
/// for a project the reader may have no access to.
|
||||
#[cfg(all(feature = "enterprise", feature = "parquet"))]
|
||||
async fn put_object(key: &str, value: String) -> error::Result<bool> {
|
||||
use windmill_object_store::object_store_reexports::Path as ObjectPath;
|
||||
use windmill_object_store::object_store_reexports::{ObjectStoreExt, Path as ObjectPath};
|
||||
let Some(store) = windmill_object_store::get_object_store().await else {
|
||||
return Ok(false);
|
||||
};
|
||||
@@ -565,7 +565,7 @@ async fn get_object(key: &str) -> error::Result<String> {
|
||||
|
||||
#[cfg(all(feature = "enterprise", feature = "parquet"))]
|
||||
async fn delete_object(key: &str) {
|
||||
use windmill_object_store::object_store_reexports::Path as ObjectPath;
|
||||
use windmill_object_store::object_store_reexports::{ObjectStoreExt, Path as ObjectPath};
|
||||
let Some(store) = windmill_object_store::get_object_store().await else {
|
||||
return;
|
||||
};
|
||||
|
||||
@@ -2,7 +2,7 @@ use tokio::time::Instant;
|
||||
use windmill_common::error;
|
||||
|
||||
#[cfg(all(feature = "enterprise", feature = "parquet"))]
|
||||
use windmill_object_store::object_store_reexports::ObjectStore;
|
||||
use windmill_object_store::object_store_reexports::{ObjectStore, ObjectStoreExt};
|
||||
|
||||
#[cfg(all(feature = "enterprise", feature = "parquet"))]
|
||||
use std::sync::Arc;
|
||||
|
||||
Reference in New Issue
Block a user