mirror of
https://github.com/lancedb/lancedb.git
synced 2026-01-03 18:32:55 +00:00
c9f248b058
Support hybrid search in both rust and node SDKs. - Adds a new rerankers package to rust LanceDB, with the implementation of the default RRF reranker - Adds a new hybrid package to lancedb, with some helper methods related to hybrid search such as normalizing scores and converting score column to rank columns - Adds capability to LanceDB VectorQuery to perform hybrid search if it has both a nearest vector and full text search parameters. - Adds wrappers for reranker implementations to nodejs SDK. Additional rerankers will be added in followup PRs https://github.com/lancedb/lancedb/issues/1921 --- Notes about how the rust rerankers are wrapped for calling from JS: I wanted to keep the core reranker logic, and the invocation of the reranker by the query code, in Rust. This aligns with the philosophy of the new node SDK where it's just a thin wrapper around Rust. However, I also wanted to have support for users who want to add custom rerankers written in Javascript. When we add a reranker to the query from Javascript, it adds a special Rust reranker that has a callback to the Javascript code (which could then turn around and call an underlying Rust reranker implementation if desired). This adds a bit of complexity, but overall I think it moves us in the right direction of having the majority of the query logic in the underlying Rust SDK while keeping the option open to support custom Javascript Rerankers.
90 lines
2.9 KiB
Rust
90 lines
2.9 KiB
Rust
// Copyright 2024 Lance Developers.
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
use std::collections::HashMap;
|
|
|
|
use env_logger::Env;
|
|
use napi_derive::*;
|
|
|
|
mod connection;
|
|
mod error;
|
|
mod index;
|
|
mod iterator;
|
|
pub mod merge;
|
|
mod query;
|
|
pub mod remote;
|
|
mod rerankers;
|
|
mod table;
|
|
mod util;
|
|
|
|
#[napi(object)]
|
|
#[derive(Debug)]
|
|
pub struct ConnectionOptions {
|
|
/// (For LanceDB OSS only): The interval, in seconds, at which to check for
|
|
/// updates to the table from other processes. If None, then consistency is not
|
|
/// checked. For performance reasons, this is the default. For strong
|
|
/// consistency, set this to zero seconds. Then every read will check for
|
|
/// updates from other processes. As a compromise, you can set this to a
|
|
/// non-zero value for eventual consistency. If more than that interval
|
|
/// has passed since the last check, then the table will be checked for updates.
|
|
/// Note: this consistency only applies to read operations. Write operations are
|
|
/// always consistent.
|
|
pub read_consistency_interval: Option<f64>,
|
|
/// (For LanceDB OSS only): configuration for object storage.
|
|
///
|
|
/// The available options are described at https://lancedb.github.io/lancedb/guides/storage/
|
|
pub storage_options: Option<HashMap<String, String>>,
|
|
|
|
/// (For LanceDB cloud only): configuration for the remote HTTP client.
|
|
pub client_config: Option<remote::ClientConfig>,
|
|
/// (For LanceDB cloud only): the API key to use with LanceDB Cloud.
|
|
///
|
|
/// Can also be set via the environment variable `LANCEDB_API_KEY`.
|
|
pub api_key: Option<String>,
|
|
/// (For LanceDB cloud only): the region to use for LanceDB cloud.
|
|
/// Defaults to 'us-east-1'.
|
|
pub region: Option<String>,
|
|
/// (For LanceDB cloud only): the host to use for LanceDB cloud. Used
|
|
/// for testing purposes.
|
|
pub host_override: Option<String>,
|
|
}
|
|
|
|
/// Write mode for writing a table.
|
|
#[napi(string_enum)]
|
|
pub enum WriteMode {
|
|
Create,
|
|
Append,
|
|
Overwrite,
|
|
}
|
|
|
|
/// Write options when creating a Table.
|
|
#[napi(object)]
|
|
pub struct WriteOptions {
|
|
/// Write mode for writing to a table.
|
|
pub mode: Option<WriteMode>,
|
|
}
|
|
|
|
#[napi(object)]
|
|
pub struct OpenTableOptions {
|
|
pub storage_options: Option<HashMap<String, String>>,
|
|
}
|
|
|
|
#[napi::module_init]
|
|
fn init() {
|
|
let env = Env::new()
|
|
.filter_or("LANCEDB_LOG", "warn")
|
|
.write_style("LANCEDB_LOG_STYLE");
|
|
env_logger::init_from_env(env);
|
|
}
|