Compare commits

..

1 Commits

Author SHA1 Message Date
lancedb automation 3e4be3c4b5 chore: update lance dependency to v4.0.0-beta.7 2026-03-04 02:16:34 +00:00
8 changed files with 90 additions and 128 deletions
Generated
+32 -32
View File
@@ -3088,8 +3088,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
[[package]]
name = "fsst"
version = "3.0.0-rc.3"
source = "git+https://github.com/lance-format/lance.git?tag=v3.0.0-rc.3#de393a26a068dd297929ca7d798e43dc31c57337"
version = "4.0.0-beta.7"
source = "git+https://github.com/lance-format/lance.git?tag=v4.0.0-beta.7#e1e5689e9c1e58df75cfdc64e8f029004e98ec71"
dependencies = [
"arrow-array",
"rand 0.9.2",
@@ -4260,8 +4260,8 @@ dependencies = [
[[package]]
name = "lance"
version = "3.0.0-rc.3"
source = "git+https://github.com/lance-format/lance.git?tag=v3.0.0-rc.3#de393a26a068dd297929ca7d798e43dc31c57337"
version = "4.0.0-beta.7"
source = "git+https://github.com/lance-format/lance.git?tag=v4.0.0-beta.7#e1e5689e9c1e58df75cfdc64e8f029004e98ec71"
dependencies = [
"arrow",
"arrow-arith",
@@ -4327,8 +4327,8 @@ dependencies = [
[[package]]
name = "lance-arrow"
version = "3.0.0-rc.3"
source = "git+https://github.com/lance-format/lance.git?tag=v3.0.0-rc.3#de393a26a068dd297929ca7d798e43dc31c57337"
version = "4.0.0-beta.7"
source = "git+https://github.com/lance-format/lance.git?tag=v4.0.0-beta.7#e1e5689e9c1e58df75cfdc64e8f029004e98ec71"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -4348,8 +4348,8 @@ dependencies = [
[[package]]
name = "lance-bitpacking"
version = "3.0.0-rc.3"
source = "git+https://github.com/lance-format/lance.git?tag=v3.0.0-rc.3#de393a26a068dd297929ca7d798e43dc31c57337"
version = "4.0.0-beta.7"
source = "git+https://github.com/lance-format/lance.git?tag=v4.0.0-beta.7#e1e5689e9c1e58df75cfdc64e8f029004e98ec71"
dependencies = [
"arrayref",
"paste",
@@ -4358,8 +4358,8 @@ dependencies = [
[[package]]
name = "lance-core"
version = "3.0.0-rc.3"
source = "git+https://github.com/lance-format/lance.git?tag=v3.0.0-rc.3#de393a26a068dd297929ca7d798e43dc31c57337"
version = "4.0.0-beta.7"
source = "git+https://github.com/lance-format/lance.git?tag=v4.0.0-beta.7#e1e5689e9c1e58df75cfdc64e8f029004e98ec71"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -4396,8 +4396,8 @@ dependencies = [
[[package]]
name = "lance-datafusion"
version = "3.0.0-rc.3"
source = "git+https://github.com/lance-format/lance.git?tag=v3.0.0-rc.3#de393a26a068dd297929ca7d798e43dc31c57337"
version = "4.0.0-beta.7"
source = "git+https://github.com/lance-format/lance.git?tag=v4.0.0-beta.7#e1e5689e9c1e58df75cfdc64e8f029004e98ec71"
dependencies = [
"arrow",
"arrow-array",
@@ -4427,8 +4427,8 @@ dependencies = [
[[package]]
name = "lance-datagen"
version = "3.0.0-rc.3"
source = "git+https://github.com/lance-format/lance.git?tag=v3.0.0-rc.3#de393a26a068dd297929ca7d798e43dc31c57337"
version = "4.0.0-beta.7"
source = "git+https://github.com/lance-format/lance.git?tag=v4.0.0-beta.7#e1e5689e9c1e58df75cfdc64e8f029004e98ec71"
dependencies = [
"arrow",
"arrow-array",
@@ -4446,8 +4446,8 @@ dependencies = [
[[package]]
name = "lance-encoding"
version = "3.0.0-rc.3"
source = "git+https://github.com/lance-format/lance.git?tag=v3.0.0-rc.3#de393a26a068dd297929ca7d798e43dc31c57337"
version = "4.0.0-beta.7"
source = "git+https://github.com/lance-format/lance.git?tag=v4.0.0-beta.7#e1e5689e9c1e58df75cfdc64e8f029004e98ec71"
dependencies = [
"arrow-arith",
"arrow-array",
@@ -4484,8 +4484,8 @@ dependencies = [
[[package]]
name = "lance-file"
version = "3.0.0-rc.3"
source = "git+https://github.com/lance-format/lance.git?tag=v3.0.0-rc.3#de393a26a068dd297929ca7d798e43dc31c57337"
version = "4.0.0-beta.7"
source = "git+https://github.com/lance-format/lance.git?tag=v4.0.0-beta.7#e1e5689e9c1e58df75cfdc64e8f029004e98ec71"
dependencies = [
"arrow-arith",
"arrow-array",
@@ -4517,8 +4517,8 @@ dependencies = [
[[package]]
name = "lance-index"
version = "3.0.0-rc.3"
source = "git+https://github.com/lance-format/lance.git?tag=v3.0.0-rc.3#de393a26a068dd297929ca7d798e43dc31c57337"
version = "4.0.0-beta.7"
source = "git+https://github.com/lance-format/lance.git?tag=v4.0.0-beta.7#e1e5689e9c1e58df75cfdc64e8f029004e98ec71"
dependencies = [
"arrow",
"arrow-arith",
@@ -4581,8 +4581,8 @@ dependencies = [
[[package]]
name = "lance-io"
version = "3.0.0-rc.3"
source = "git+https://github.com/lance-format/lance.git?tag=v3.0.0-rc.3#de393a26a068dd297929ca7d798e43dc31c57337"
version = "4.0.0-beta.7"
source = "git+https://github.com/lance-format/lance.git?tag=v4.0.0-beta.7#e1e5689e9c1e58df75cfdc64e8f029004e98ec71"
dependencies = [
"arrow",
"arrow-arith",
@@ -4623,8 +4623,8 @@ dependencies = [
[[package]]
name = "lance-linalg"
version = "3.0.0-rc.3"
source = "git+https://github.com/lance-format/lance.git?tag=v3.0.0-rc.3#de393a26a068dd297929ca7d798e43dc31c57337"
version = "4.0.0-beta.7"
source = "git+https://github.com/lance-format/lance.git?tag=v4.0.0-beta.7#e1e5689e9c1e58df75cfdc64e8f029004e98ec71"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -4640,8 +4640,8 @@ dependencies = [
[[package]]
name = "lance-namespace"
version = "3.0.0-rc.3"
source = "git+https://github.com/lance-format/lance.git?tag=v3.0.0-rc.3#de393a26a068dd297929ca7d798e43dc31c57337"
version = "4.0.0-beta.7"
source = "git+https://github.com/lance-format/lance.git?tag=v4.0.0-beta.7#e1e5689e9c1e58df75cfdc64e8f029004e98ec71"
dependencies = [
"arrow",
"async-trait",
@@ -4653,8 +4653,8 @@ dependencies = [
[[package]]
name = "lance-namespace-impls"
version = "3.0.0-rc.3"
source = "git+https://github.com/lance-format/lance.git?tag=v3.0.0-rc.3#de393a26a068dd297929ca7d798e43dc31c57337"
version = "4.0.0-beta.7"
source = "git+https://github.com/lance-format/lance.git?tag=v4.0.0-beta.7#e1e5689e9c1e58df75cfdc64e8f029004e98ec71"
dependencies = [
"arrow",
"arrow-ipc",
@@ -4698,8 +4698,8 @@ dependencies = [
[[package]]
name = "lance-table"
version = "3.0.0-rc.3"
source = "git+https://github.com/lance-format/lance.git?tag=v3.0.0-rc.3#de393a26a068dd297929ca7d798e43dc31c57337"
version = "4.0.0-beta.7"
source = "git+https://github.com/lance-format/lance.git?tag=v4.0.0-beta.7#e1e5689e9c1e58df75cfdc64e8f029004e98ec71"
dependencies = [
"arrow",
"arrow-array",
@@ -4738,8 +4738,8 @@ dependencies = [
[[package]]
name = "lance-testing"
version = "3.0.0-rc.3"
source = "git+https://github.com/lance-format/lance.git?tag=v3.0.0-rc.3#de393a26a068dd297929ca7d798e43dc31c57337"
version = "4.0.0-beta.7"
source = "git+https://github.com/lance-format/lance.git?tag=v4.0.0-beta.7#e1e5689e9c1e58df75cfdc64e8f029004e98ec71"
dependencies = [
"arrow-array",
"arrow-schema",
+14 -14
View File
@@ -15,20 +15,20 @@ categories = ["database-implementations"]
rust-version = "1.91.0"
[workspace.dependencies]
lance = { "version" = "=4.0.0-beta.12", default-features = false, "tag" = "v4.0.0-beta.12", "git" = "https://github.com/lance-format/lance.git" }
lance-core = { "version" = "=4.0.0-beta.12", "tag" = "v4.0.0-beta.12", "git" = "https://github.com/lance-format/lance.git" }
lance-datagen = { "version" = "=4.0.0-beta.12", "tag" = "v4.0.0-beta.12", "git" = "https://github.com/lance-format/lance.git" }
lance-file = { "version" = "=4.0.0-beta.12", "tag" = "v4.0.0-beta.12", "git" = "https://github.com/lance-format/lance.git" }
lance-io = { "version" = "=4.0.0-beta.12", default-features = false, "tag" = "v4.0.0-beta.12", "git" = "https://github.com/lance-format/lance.git" }
lance-index = { "version" = "=4.0.0-beta.12", "tag" = "v4.0.0-beta.12", "git" = "https://github.com/lance-format/lance.git" }
lance-linalg = { "version" = "=4.0.0-beta.12", "tag" = "v4.0.0-beta.12", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace = { "version" = "=4.0.0-beta.12", "tag" = "v4.0.0-beta.12", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace-impls = { "version" = "=4.0.0-beta.12", default-features = false, "tag" = "v4.0.0-beta.12", "git" = "https://github.com/lance-format/lance.git" }
lance-table = { "version" = "=4.0.0-beta.12", "tag" = "v4.0.0-beta.12", "git" = "https://github.com/lance-format/lance.git" }
lance-testing = { "version" = "=4.0.0-beta.12", "tag" = "v4.0.0-beta.12", "git" = "https://github.com/lance-format/lance.git" }
lance-datafusion = { "version" = "=4.0.0-beta.12", "tag" = "v4.0.0-beta.12", "git" = "https://github.com/lance-format/lance.git" }
lance-encoding = { "version" = "=4.0.0-beta.12", "tag" = "v4.0.0-beta.12", "git" = "https://github.com/lance-format/lance.git" }
lance-arrow = { "version" = "=4.0.0-beta.12", "tag" = "v4.0.0-beta.12", "git" = "https://github.com/lance-format/lance.git" }
lance = { "version" = "=4.0.0-beta.7", default-features = false, "tag" = "v4.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
lance-core = { "version" = "=4.0.0-beta.7", "tag" = "v4.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
lance-datagen = { "version" = "=4.0.0-beta.7", "tag" = "v4.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
lance-file = { "version" = "=4.0.0-beta.7", "tag" = "v4.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
lance-io = { "version" = "=4.0.0-beta.7", default-features = false, "tag" = "v4.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
lance-index = { "version" = "=4.0.0-beta.7", "tag" = "v4.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
lance-linalg = { "version" = "=4.0.0-beta.7", "tag" = "v4.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace = { "version" = "=4.0.0-beta.7", "tag" = "v4.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace-impls = { "version" = "=4.0.0-beta.7", default-features = false, "tag" = "v4.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
lance-table = { "version" = "=4.0.0-beta.7", "tag" = "v4.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
lance-testing = { "version" = "=4.0.0-beta.7", "tag" = "v4.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
lance-datafusion = { "version" = "=4.0.0-beta.7", "tag" = "v4.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
lance-encoding = { "version" = "=4.0.0-beta.7", "tag" = "v4.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
lance-arrow = { "version" = "=4.0.0-beta.7", "tag" = "v4.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
ahash = "0.8"
# Note that this one does not include pyarrow
arrow = { version = "57.2", optional = false }
+1 -1
View File
@@ -28,7 +28,7 @@
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<arrow.version>15.0.0</arrow.version>
<lance-core.version>3.1.0-beta.2</lance-core.version>
<lance-core.version>4.0.0-beta.7</lance-core.version>
<spotless.skip>false</spotless.skip>
<spotless.version>2.30.0</spotless.version>
<spotless.java.googlejavaformat.version>1.7</spotless.java.googlejavaformat.version>
+18 -10
View File
@@ -66,10 +66,12 @@ impl StorageOptionsProvider for PyStorageOptionsProviderWrapper {
.inner
.bind(py)
.call_method0("fetch_storage_options")
.map_err(|e| lance_core::Error::io_source(Box::new(std::io::Error::other(format!(
"Failed to call fetch_storage_options: {}",
e
)))))?;
.map_err(|e| {
lance_core::Error::io_source(Box::new(std::io::Error::other(format!(
"Failed to call fetch_storage_options: {}",
e
))))
})?;
// If result is None, return None
if result.is_none() {
@@ -79,7 +81,7 @@ impl StorageOptionsProvider for PyStorageOptionsProviderWrapper {
// Extract the result dict - should be a flat Map<String, String>
let result_dict = result.downcast::<PyDict>().map_err(|_| {
lance_core::Error::invalid_input(
"fetch_storage_options() must return a dict of string key-value pairs or None",
"fetch_storage_options() must return None or a dict of string key-value pairs",
)
})?;
@@ -87,10 +89,16 @@ impl StorageOptionsProvider for PyStorageOptionsProviderWrapper {
let mut storage_options = HashMap::new();
for (key, value) in result_dict.iter() {
let key_str: String = key.extract().map_err(|e| {
lance_core::Error::invalid_input(format!("Storage option key must be a string: {}", e))
lance_core::Error::invalid_input(format!(
"Storage option key must be a string: {}",
e
))
})?;
let value_str: String = value.extract().map_err(|e| {
lance_core::Error::invalid_input(format!("Storage option value must be a string: {}", e))
lance_core::Error::invalid_input(format!(
"Storage option value must be a string: {}",
e
))
})?;
storage_options.insert(key_str, value_str);
}
@@ -100,9 +108,9 @@ impl StorageOptionsProvider for PyStorageOptionsProviderWrapper {
})
.await
.map_err(|e| lance_core::Error::io_source(Box::new(std::io::Error::other(format!(
"Task join error: {}",
e
)))))?
"Task join error: {}",
e
)))))?
}
fn provider_id(&self) -> String {
+2 -5
View File
@@ -566,11 +566,8 @@ pub struct ConnectBuilder {
}
#[cfg(feature = "remote")]
const ENV_VARS_TO_STORAGE_OPTS: [(&str, &str); 3] = [
("AZURE_STORAGE_ACCOUNT_NAME", "azure_storage_account_name"),
("AZURE_CLIENT_ID", "azure_client_id"),
("AZURE_TENANT_ID", "azure_tenant_id"),
];
const ENV_VARS_TO_STORAGE_OPTS: [(&str, &str); 1] =
[("AZURE_STORAGE_ACCOUNT_NAME", "azure_storage_account_name")];
impl ConnectBuilder {
/// Create a new [`ConnectOptions`] with the given database URI.
+15 -13
View File
@@ -205,25 +205,27 @@ impl Database for LanceNamespaceDatabase {
let mut table_id = request.namespace.clone();
table_id.push(request.name.clone());
// Try declare_table first, falling back to create_empty_table for backwards
// compatibility with older namespace clients that don't support declare_table
// Declare table metadata through lance-namespace before creating data files.
let declare_request = DeclareTableRequest {
id: Some(table_id.clone()),
..Default::default()
};
let (location, initial_storage_options) = {
let response = self.namespace.declare_table(declare_request).await?;
let loc = response.location.ok_or_else(|| Error::Runtime {
message: "Table location is missing from declare_table response".to_string(),
let declare_response = self
.namespace
.declare_table(declare_request)
.await
.map_err(|e| Error::Runtime {
message: format!("Failed to declare table: {}", e),
})?;
// Use storage options from response, fall back to self.storage_options
let opts = response
.storage_options
.or_else(|| Some(self.storage_options.clone()))
.filter(|o| !o.is_empty());
(loc, opts)
};
let location = declare_response.location.ok_or_else(|| Error::Runtime {
message: "Table location is missing from declare_table response".to_string(),
})?;
// Use storage options from response, fall back to self.storage_options
let initial_storage_options = declare_response
.storage_options
.or_else(|| Some(self.storage_options.clone()))
.filter(|o| !o.is_empty());
let write_params = if let Some(storage_opts) = initial_storage_options {
let mut params = request.write_options.lance_write_params.unwrap_or_default();
+7 -47
View File
@@ -446,23 +446,13 @@ impl<S: HttpSend> RestfulLanceDbClient<S> {
})?,
);
}
// Map azure storage options to x-azure-* headers.
// The option key uses underscores (e.g. "azure_client_id") while the
// header uses hyphens (e.g. "x-azure-client-id").
let azure_opts: [(&str, &str); 3] = [
("azure_storage_account_name", "x-azure-storage-account-name"),
("azure_client_id", "x-azure-client-id"),
("azure_tenant_id", "x-azure-tenant-id"),
];
for (opt_key, header_name) in azure_opts {
if let Some(v) = options.0.get(opt_key) {
headers.insert(
HeaderName::from_static(header_name),
HeaderValue::from_str(v).map_err(|_| Error::InvalidInput {
message: format!("non-ascii value '{}' for option '{}'", v, opt_key),
})?,
);
}
if let Some(v) = options.0.get("azure_storage_account_name") {
headers.insert(
HeaderName::from_static("x-azure-storage-account-name"),
HeaderValue::from_str(v).map_err(|_| Error::InvalidInput {
message: format!("non-ascii storage account name '{}' provided", db_name),
})?,
);
}
for (key, value) in &config.extra_headers {
@@ -1085,34 +1075,4 @@ mod tests {
_ => panic!("Expected Runtime error"),
}
}
#[test]
fn test_default_headers_azure_opts() {
let mut opts = HashMap::new();
opts.insert(
"azure_storage_account_name".to_string(),
"myaccount".to_string(),
);
opts.insert("azure_client_id".to_string(), "my-client-id".to_string());
opts.insert("azure_tenant_id".to_string(), "my-tenant-id".to_string());
let remote_opts = RemoteOptions::new(opts);
let headers = RestfulLanceDbClient::<Sender>::default_headers(
"test-key",
"us-east-1",
"testdb",
false,
&remote_opts,
None,
&ClientConfig::default(),
)
.unwrap();
assert_eq!(
headers.get("x-azure-storage-account-name").unwrap(),
"myaccount"
);
assert_eq!(headers.get("x-azure-client-id").unwrap(), "my-client-id");
assert_eq!(headers.get("x-azure-tenant-id").unwrap(), "my-tenant-id");
}
}
+1 -6
View File
@@ -777,12 +777,7 @@ impl RemoteOptions {
impl From<StorageOptions> for RemoteOptions {
fn from(options: StorageOptions) -> Self {
let supported_opts = vec![
"account_name",
"azure_storage_account_name",
"azure_client_id",
"azure_tenant_id",
];
let supported_opts = vec!["account_name", "azure_storage_account_name"];
let mut filtered = HashMap::new();
for opt in supported_opts {
if let Some(v) = options.0.get(opt) {