mirror of
https://github.com/lancedb/lancedb.git
synced 2026-01-13 23:32:57 +00:00
This PR adds support of multi-level namespace in a LanceDB database,
according to the Lance Namespace spec.
This allows users to create namespace inside a database connection,
perform create, drop, list, list_tables in a namespace. (other
operations like update, describe will be in a follow-up PR)
The 3 types of database connections behave like the following:
1 Local database connections will continue to have just a flat list of
tables for backwards compatibility.
2. Remote database connections will make REST API calls according to the
APIs in the Lance Namespace spec.
3. Lance Namespace connections will invoke the corresponding operations
against the specific namespace implementation which could have different
behaviors regarding these APIs.
All the table APIs now take identifier instead of name, for example
`/v1/table/{name}/create` is now `/v1/table/{id}/create`. If a table is
directly in the root namespace, the API call is identical. If the table
is in a namespace, then the full table ID should be used, with `$` as
the default delimiter (`.` is a special character and creates issues
with URL parsing so `$` is used), for example
`/v1/table/ns1$table1/create`. If a different parameter needs to be
passed in, user can configure the `id_delimiter` in client config and
that becomes a query parameter, for example
`/v1/table/ns1__table1/create?delimiter=__`
The Python and Typescript APIs are kept backwards compatible, but the
following Rust APIs are not:
1. `Connection::drop_table(&self, name: impl AsRef<str>) -> Result<()>`
is now `Connection::drop_table(&self, name: impl AsRef<str>, namespace:
&[String]) -> Result<()>`
2. `Connection::drop_all_tables(&self) -> Result<()>` is now
`Connection::drop_all_tables(&self, name: impl AsRef<str>) ->
Result<()>`
123 lines
5.2 KiB
Rust
123 lines
5.2 KiB
Rust
// SPDX-License-Identifier: Apache-2.0
|
|
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
|
|
|
use std::collections::HashMap;
|
|
|
|
use napi_derive::*;
|
|
|
|
/// Timeout configuration for remote HTTP client.
|
|
#[napi(object)]
|
|
#[derive(Debug)]
|
|
pub struct TimeoutConfig {
|
|
/// The overall timeout for the entire request in seconds. This includes
|
|
/// connection, send, and read time. If the entire request doesn't complete
|
|
/// within this time, it will fail. Default is None (no overall timeout).
|
|
/// This can also be set via the environment variable `LANCE_CLIENT_TIMEOUT`,
|
|
/// as an integer number of seconds.
|
|
pub timeout: Option<f64>,
|
|
/// The timeout for establishing a connection in seconds. Default is 120
|
|
/// seconds (2 minutes). This can also be set via the environment variable
|
|
/// `LANCE_CLIENT_CONNECT_TIMEOUT`, as an integer number of seconds.
|
|
pub connect_timeout: Option<f64>,
|
|
/// The timeout for reading data from the server in seconds. Default is 300
|
|
/// seconds (5 minutes). This can also be set via the environment variable
|
|
/// `LANCE_CLIENT_READ_TIMEOUT`, as an integer number of seconds.
|
|
pub read_timeout: Option<f64>,
|
|
/// The timeout for keeping idle connections in the connection pool in seconds.
|
|
/// Default is 300 seconds (5 minutes). This can also be set via the
|
|
/// environment variable `LANCE_CLIENT_CONNECTION_TIMEOUT`, as an integer
|
|
/// number of seconds.
|
|
pub pool_idle_timeout: Option<f64>,
|
|
}
|
|
|
|
/// Retry configuration for the remote HTTP client.
|
|
#[napi(object)]
|
|
#[derive(Debug)]
|
|
pub struct RetryConfig {
|
|
/// The maximum number of retries for a request. Default is 3. You can also
|
|
/// set this via the environment variable `LANCE_CLIENT_MAX_RETRIES`.
|
|
pub retries: Option<u8>,
|
|
/// The maximum number of retries for connection errors. Default is 3. You
|
|
/// can also set this via the environment variable `LANCE_CLIENT_CONNECT_RETRIES`.
|
|
pub connect_retries: Option<u8>,
|
|
/// The maximum number of retries for read errors. Default is 3. You can also
|
|
/// set this via the environment variable `LANCE_CLIENT_READ_RETRIES`.
|
|
pub read_retries: Option<u8>,
|
|
/// The backoff factor to apply between retries. Default is 0.25. Between each retry
|
|
/// the client will wait for the amount of seconds:
|
|
/// `{backoff factor} * (2 ** ({number of previous retries}))`. So for the default
|
|
/// of 0.25, the first retry will wait 0.25 seconds, the second retry will wait 0.5
|
|
/// seconds, the third retry will wait 1 second, etc.
|
|
///
|
|
/// You can also set this via the environment variable
|
|
/// `LANCE_CLIENT_RETRY_BACKOFF_FACTOR`.
|
|
pub backoff_factor: Option<f64>,
|
|
/// The jitter to apply to the backoff factor, in seconds. Default is 0.25.
|
|
///
|
|
/// A random value between 0 and `backoff_jitter` will be added to the backoff
|
|
/// factor in seconds. So for the default of 0.25 seconds, between 0 and 250
|
|
/// milliseconds will be added to the sleep between each retry.
|
|
///
|
|
/// You can also set this via the environment variable
|
|
/// `LANCE_CLIENT_RETRY_BACKOFF_JITTER`.
|
|
pub backoff_jitter: Option<f64>,
|
|
/// The HTTP status codes for which to retry the request. Default is
|
|
/// [429, 500, 502, 503].
|
|
///
|
|
/// You can also set this via the environment variable
|
|
/// `LANCE_CLIENT_RETRY_STATUSES`. Use a comma-separated list of integers.
|
|
pub statuses: Option<Vec<u16>>,
|
|
}
|
|
|
|
#[napi(object)]
|
|
#[derive(Debug, Default)]
|
|
pub struct ClientConfig {
|
|
pub user_agent: Option<String>,
|
|
pub retry_config: Option<RetryConfig>,
|
|
pub timeout_config: Option<TimeoutConfig>,
|
|
pub extra_headers: Option<HashMap<String, String>>,
|
|
pub id_delimiter: Option<String>,
|
|
}
|
|
|
|
impl From<TimeoutConfig> for lancedb::remote::TimeoutConfig {
|
|
fn from(config: TimeoutConfig) -> Self {
|
|
Self {
|
|
timeout: config.timeout.map(std::time::Duration::from_secs_f64),
|
|
connect_timeout: config
|
|
.connect_timeout
|
|
.map(std::time::Duration::from_secs_f64),
|
|
read_timeout: config.read_timeout.map(std::time::Duration::from_secs_f64),
|
|
pool_idle_timeout: config
|
|
.pool_idle_timeout
|
|
.map(std::time::Duration::from_secs_f64),
|
|
}
|
|
}
|
|
}
|
|
|
|
impl From<RetryConfig> for lancedb::remote::RetryConfig {
|
|
fn from(config: RetryConfig) -> Self {
|
|
Self {
|
|
retries: config.retries,
|
|
connect_retries: config.connect_retries,
|
|
read_retries: config.read_retries,
|
|
backoff_factor: config.backoff_factor.map(|v| v as f32),
|
|
backoff_jitter: config.backoff_jitter.map(|v| v as f32),
|
|
statuses: config.statuses,
|
|
}
|
|
}
|
|
}
|
|
|
|
impl From<ClientConfig> for lancedb::remote::ClientConfig {
|
|
fn from(config: ClientConfig) -> Self {
|
|
Self {
|
|
user_agent: config
|
|
.user_agent
|
|
.unwrap_or(concat!("LanceDB-Node-Client/", env!("CARGO_PKG_VERSION")).to_string()),
|
|
retry_config: config.retry_config.map(Into::into).unwrap_or_default(),
|
|
timeout_config: config.timeout_config.map(Into::into).unwrap_or_default(),
|
|
extra_headers: config.extra_headers.unwrap_or_default(),
|
|
id_delimiter: config.id_delimiter,
|
|
}
|
|
}
|
|
}
|