rust/lancedb
1
0
Fork 0
mirror of https://github.com/lancedb/lancedb.git synced 2026-07-04 11:30:46 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor: replace WAL host routing with x-use-wal header

Browse Source 
Route all traffic through a single host and use an x-use-wal: true
header to signal WAL routing, letting the Pingora router handle the
split instead of maintaining a separate WAL host endpoint.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Daniel Rammer
2026-04-07 15:51:21 -05:00
parent b0d3fadfc0
commit 4446a2972c
5 changed files with 15 additions and 70 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
rust/lancedb/src/connection.rs
View File

@@ -30,10 +30,7 @@ use crate::error::{Error, Result};
#[cfg(feature = "remote")]
use crate::remote::{
client::ClientConfig,
db::{
OPT_REMOTE_API_KEY, OPT_REMOTE_HOST_OVERRIDE, OPT_REMOTE_REGION,
OPT_REMOTE_WAL_HOST_OVERRIDE,
},
db::{OPT_REMOTE_API_KEY, OPT_REMOTE_HOST_OVERRIDE, OPT_REMOTE_REGION},
};
use lance::io::ObjectStoreParams;
pub use lance_encoding::version::LanceFileVersion;
@@ -674,19 +671,6 @@ impl ConnectBuilder {
///
/// This option is only used when connecting to LanceDB Cloud (db:// URIs)
/// and will be ignored for other URIs.
///
/// # Arguments
///
/// * `wal_host_override` - The WAL host override to use for the connection
#[cfg(feature = "remote")]
pub fn wal_host_override(mut self, wal_host_override: &str) -> Self {
self.request.options.insert(
OPT_REMOTE_WAL_HOST_OVERRIDE.to_string(),
wal_host_override.to_string(),
);
self
}
/// Set the database specific options
///
/// See [crate::database::listing::ListingDatabaseOptions] for the options available for
@@ -840,7 +824,6 @@ impl ConnectBuilder {
&api_key,
&region,
options.host_override,
options.wal_host_override,
self.request.client_config,
storage_options.into(),
)?);

22
rust/lancedb/src/remote/client.rs
View File

@@ -190,7 +190,6 @@ pub struct RetryConfig {
pub struct RestfulLanceDbClient<S: HttpSend = Sender> {
client: reqwest::Client,
host: String,
wal_host: String,
pub(crate) retry_config: ResolvedRetryConfig,
pub(crate) sender: S,
pub(crate) id_delimiter: String,
@@ -201,7 +200,6 @@ impl<S: HttpSend> std::fmt::Debug for RestfulLanceDbClient<S> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("RestfulLanceDbClient")
.field("host", &self.host)
.field("wal_host", &self.wal_host)
.field("retry_config", &self.retry_config)
.field("sender", &self.sender)
.field("id_delimiter", &self.id_delimiter)
@@ -287,7 +285,6 @@ impl RestfulLanceDbClient<Sender> {
parsed_url: &ParsedDbUrl,
region: &str,
host_override: Option<String>,
wal_host_override: Option<String>,
default_headers: HeaderMap,
client_config: ClientConfig,
) -> Result<Self> {
@@ -375,16 +372,11 @@ impl RestfulLanceDbClient<Sender> {
Some(host_override) => host_override,
None => format!("https://{}.{}.api.lancedb.com", parsed_url.db_name, region),
};
let wal_host = match wal_host_override {
Some(wal_host_override) => wal_host_override,
None => format!("https://{}.{}.wal.lancedb.com", parsed_url.db_name, region),
};
debug!("Created client for host: {}, wal_host: {}", host, wal_host);
debug!("Created client for host: {}", host);
let retry_config = client_config.retry_config.clone().try_into()?;
Ok(Self {
client,
host,
wal_host,
retry_config,
sender: Sender,
id_delimiter: client_config
@@ -488,8 +480,8 @@ impl<S: HttpSend> RestfulLanceDbClient<S> {
}
pub fn post_wal(&self, uri: &str) -> RequestBuilder {
let full_uri = format!("{}{}", self.wal_host, uri);
let builder = self.client.post(full_uri);
let full_uri = format!("{}{}", self.host, uri);
let builder = self.client.post(full_uri).header("x-use-wal", "true");
self.add_id_delimiter_query_param(builder)
}
@@ -805,7 +797,6 @@ pub mod test_utils {
RestfulLanceDbClient {
client: reqwest::Client::new(),
host: "http://localhost".to_string(),
wal_host: "http://localhost-wal".to_string(),
retry_config: RetryConfig::default().try_into().unwrap(),
sender: MockSender {
f: Arc::new(wrapper),
@@ -830,7 +821,6 @@ pub mod test_utils {
RestfulLanceDbClient {
client: reqwest::Client::new(),
host: "http://localhost".to_string(),
wal_host: "http://localhost-wal".to_string(),
retry_config: config.retry_config.try_into().unwrap(),
sender: MockSender {
f: Arc::new(wrapper),
@@ -998,7 +988,7 @@ mod tests {
let client = RestfulLanceDbClient {
client: reqwest::Client::new(),
host: "https://example.com".to_string(),
wal_host: "https://example.com".to_string(),
retry_config: RetryConfig::default().try_into().unwrap(),
sender: Sender,
id_delimiter: "+".to_string(),
@@ -1034,7 +1024,7 @@ mod tests {
let client = RestfulLanceDbClient {
client: reqwest::Client::new(),
host: "https://example.com".to_string(),
wal_host: "https://example.com".to_string(),
retry_config: RetryConfig::default().try_into().unwrap(),
sender: Sender,
id_delimiter: "+".to_string(),
@@ -1072,7 +1062,7 @@ mod tests {
let client = RestfulLanceDbClient {
client: reqwest::Client::new(),
host: "https://example.com".to_string(),
wal_host: "https://example.com".to_string(),
retry_config: RetryConfig::default().try_into().unwrap(),
sender: Sender,
id_delimiter: "+".to_string(),

28
rust/lancedb/src/remote/db.rs
View File

@@ -82,7 +82,6 @@ pub const OPT_REMOTE_PREFIX: &str = "remote_database_";
pub const OPT_REMOTE_API_KEY: &str = "remote_database_api_key";
pub const OPT_REMOTE_REGION: &str = "remote_database_region";
pub const OPT_REMOTE_HOST_OVERRIDE: &str = "remote_database_host_override";
pub const OPT_REMOTE_WAL_HOST_OVERRIDE: &str = "remote_database_wal_host_override";
// TODO: add support for configuring client config via key/value options
#[derive(Clone, Debug, Default)]
@@ -96,11 +95,6 @@ pub struct RemoteDatabaseOptions {
/// This is required when connecting to LanceDB Enterprise and should be
/// provided if using an on-premises LanceDB Enterprise instance.
pub host_override: Option<String>,
/// The WAL host override
///
/// When set, merge_insert operations using WAL routing will be sent to
/// this host instead of the auto-derived WAL host.
pub wal_host_override: Option<String>,
/// Storage options configure the storage layer (e.g. S3, GCS, Azure, etc.)
///
/// See available options at <https://lancedb.com/docs/storage/>
@@ -119,7 +113,6 @@ impl RemoteDatabaseOptions {
let api_key = map.get(OPT_REMOTE_API_KEY).cloned();
let region = map.get(OPT_REMOTE_REGION).cloned();
let host_override = map.get(OPT_REMOTE_HOST_OVERRIDE).cloned();
let wal_host_override = map.get(OPT_REMOTE_WAL_HOST_OVERRIDE).cloned();
let storage_options = map
.iter()
.filter(|(key, _)| !key.starts_with(OPT_REMOTE_PREFIX))
@@ -129,7 +122,6 @@ impl RemoteDatabaseOptions {
api_key,
region,
host_override,
wal_host_override,
storage_options,
})
}
@@ -149,12 +141,6 @@ impl DatabaseOptions for RemoteDatabaseOptions {
if let Some(host_override) = &self.host_override {
map.insert(OPT_REMOTE_HOST_OVERRIDE.to_string(), host_override.clone());
}
if let Some(wal_host_override) = &self.wal_host_override {
map.insert(
OPT_REMOTE_WAL_HOST_OVERRIDE.to_string(),
wal_host_override.clone(),
);
}
}
}
@@ -200,18 +186,6 @@ impl RemoteDatabaseOptionsBuilder {
self
}
/// Set the WAL host override
///
/// When set, merge_insert operations using WAL routing will be sent to
/// this host instead of the auto-derived WAL host.
///
/// # Arguments
///
/// * `wal_host_override` - The WAL host override
pub fn wal_host_override(mut self, wal_host_override: String) -> Self {
self.options.wal_host_override = Some(wal_host_override);
self
}
}
#[derive(Debug)]
@@ -231,7 +205,6 @@ impl RemoteDatabase {
api_key: &str,
region: &str,
host_override: Option<String>,
wal_host_override: Option<String>,
client_config: ClientConfig,
options: RemoteOptions,
) -> Result<Self> {
@@ -259,7 +232,6 @@ impl RemoteDatabase {
&parsed,
region,
host_override,
wal_host_override,
header_map,
client_config.clone(),
)?;

8
rust/lancedb/src/remote/table.rs
View File

@@ -2723,11 +2723,11 @@ mod tests {
let table = Table::new_with_handler("my_table", move |request| {
if request.url().path() == "/v1/table/my_table/merge_insert/" {
// Verify the request was sent to the WAL host
// Verify the x-use-wal header is set for router-based WAL routing
assert_eq!(
request.url().host_str().unwrap(),
"localhost-wal",
"merge_insert with use_wal should route to WAL host"
request.headers().get("x-use-wal").unwrap(),
"true",
"merge_insert with use_wal should set x-use-wal header"
);
http::Response::builder()

8
rust/lancedb/src/table/merge.rs
View File

@@ -150,11 +150,11 @@ impl MergeInsertBuilder {
self
}
/// Controls whether to route the merge insert operation through the WAL host.
/// Controls whether to route the merge insert operation through the WAL.
///
/// When set to `true`, the operation will be sent to the WAL host instead of
/// the main API host. The WAL host is auto-derived from the database connection
/// or can be explicitly set via [`crate::connection::ConnectBuilder::wal_host_override`].
/// When set to `true`, the request includes an `x-use-wal: true` header,
/// which the router uses to forward the operation to wal-writer instances
/// instead of Phalanx.
///
/// Defaults to `false`.
pub fn use_wal(&mut self, use_wal: bool) -> &mut Self {