Format code

Signed-off-by: Xuanwo <github@xuanwo.io>
feat: Expose disableScoringAutoprojection to lancedb
2025-12-25 22:29:58 +00:00 · 2025-09-09 01:08:02 +08:00 · 2025-09-09 01:06:29 +08:00
12 changed files with 101 additions and 128 deletions
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1399,7 +1399,7 @@ version = "0.6.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766"
 dependencies = [
- "nom 7.1.3",
+ "nom",
 ]

 [[package]]
@@ -2717,7 +2717,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "74fef4569247a5f429d9156b9d0a2599914385dd189c539334c625d8099d90ab"
 dependencies = [
 "futures-core",
- "nom 7.1.3",
+ "nom",
 "pin-project-lite",
 ]

@@ -2744,12 +2744,6 @@ version = "0.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "95765f67b4b18863968b4a1bd5bb576f732b29a4a28c7cd84c09fa3e2875f33c"

-[[package]]
-name = "fast-float2"
-version = "0.2.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f8eb564c5c7423d25c886fb561d1e4ee69f72354d16918afa32c08811f6b6a55"
-
 [[package]]
 name = "fastdivide"
 version = "0.4.2"
@@ -2843,8 +2837,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"

 [[package]]
 name = "fsst"
-version = "0.34.0"
-source = "git+https://github.com/lancedb/lance.git?tag=v0.34.0-beta.3#31bc1c9b5c27871e8087726bf136696f22243679"
+version = "0.33.0"
+source = "git+https://github.com/lancedb/lance.git?tag=v0.33.0-beta.4#e37e9df2458e88c37205415bd00d950b7f936061"
 dependencies = [
 "arrow-array",
 "rand 0.9.1",
@@ -3896,12 +3890,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "be1f93b8b1eb69c77f24bbb0afdf66f54b632ee39af40ca21c4365a1d7347e49"
 dependencies = [
 "jiff-static",
- "jiff-tzdb-platform",
 "log",
 "portable-atomic",
 "portable-atomic-util",
 "serde",
- "windows-sys 0.52.0",
 ]

 [[package]]
@@ -3915,21 +3907,6 @@ dependencies = [
 "syn 2.0.103",
 ]

-[[package]]
-name = "jiff-tzdb"
-version = "0.1.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c1283705eb0a21404d2bfd6eef2a7593d240bc42a0bdb39db0ad6fa2ec026524"
-
-[[package]]
-name = "jiff-tzdb-platform"
-version = "0.1.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "875a5a69ac2bab1a891711cf5eccbec1ce0341ea805560dcd90b7a2e925132e8"
-dependencies = [
- "jiff-tzdb",
-]
-
 [[package]]
 name = "jni"
 version = "0.21.1"
@@ -3972,30 +3949,10 @@ dependencies = [
 "wasm-bindgen",
 ]

-[[package]]
-name = "jsonb"
-version = "0.5.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a452366d21e8d3cbca680c41388e01d6a88739afef7877961946a6da409f9ccd"
-dependencies = [
- "byteorder",
- "ethnum",
- "fast-float2",
- "itoa",
- "jiff",
- "nom 8.0.0",
- "num-traits",
- "ordered-float",
- "rand 0.9.1",
- "ryu",
- "serde",
- "serde_json",
-]
-
 [[package]]
 name = "lance"
-version = "0.34.0"
-source = "git+https://github.com/lancedb/lance.git?tag=v0.34.0-beta.3#31bc1c9b5c27871e8087726bf136696f22243679"
+version = "0.33.0"
+source = "git+https://github.com/lancedb/lance.git?tag=v0.33.0-beta.4#e37e9df2458e88c37205415bd00d950b7f936061"
 dependencies = [
 "arrow",
 "arrow-arith",
@@ -4050,7 +4007,6 @@ dependencies = [
 "tantivy",
 "tempfile",
 "tokio",
- "tokio-stream",
 "tracing",
 "url",
 "uuid",
@@ -4058,8 +4014,8 @@ dependencies = [

 [[package]]
 name = "lance-arrow"
-version = "0.34.0"
-source = "git+https://github.com/lancedb/lance.git?tag=v0.34.0-beta.3#31bc1c9b5c27871e8087726bf136696f22243679"
+version = "0.33.0"
+source = "git+https://github.com/lancedb/lance.git?tag=v0.33.0-beta.4#e37e9df2458e88c37205415bd00d950b7f936061"
 dependencies = [
 "arrow-array",
 "arrow-buffer",
@@ -4070,25 +4026,14 @@ dependencies = [
 "bytes",
 "getrandom 0.2.16",
 "half",
- "jsonb",
 "num-traits",
 "rand 0.9.1",
 ]

-[[package]]
-name = "lance-bitpacking"
-version = "0.34.0"
-source = "git+https://github.com/lancedb/lance.git?tag=v0.34.0-beta.3#31bc1c9b5c27871e8087726bf136696f22243679"
-dependencies = [
- "arrayref",
- "paste",
- "seq-macro",
-]
-
 [[package]]
 name = "lance-core"
-version = "0.34.0"
-source = "git+https://github.com/lancedb/lance.git?tag=v0.34.0-beta.3#31bc1c9b5c27871e8087726bf136696f22243679"
+version = "0.33.0"
+source = "git+https://github.com/lancedb/lance.git?tag=v0.33.0-beta.4#e37e9df2458e88c37205415bd00d950b7f936061"
 dependencies = [
 "arrow-array",
 "arrow-buffer",
@@ -4123,8 +4068,8 @@ dependencies = [

 [[package]]
 name = "lance-datafusion"
-version = "0.34.0"
-source = "git+https://github.com/lancedb/lance.git?tag=v0.34.0-beta.3#31bc1c9b5c27871e8087726bf136696f22243679"
+version = "0.33.0"
+source = "git+https://github.com/lancedb/lance.git?tag=v0.33.0-beta.4#e37e9df2458e88c37205415bd00d950b7f936061"
 dependencies = [
 "arrow",
 "arrow-array",
@@ -4138,7 +4083,6 @@ dependencies = [
 "datafusion-functions",
 "datafusion-physical-expr",
 "futures",
- "jsonb",
 "lance-arrow",
 "lance-core",
 "lance-datagen",
@@ -4153,8 +4097,8 @@ dependencies = [

 [[package]]
 name = "lance-datagen"
-version = "0.34.0"
-source = "git+https://github.com/lancedb/lance.git?tag=v0.34.0-beta.3#31bc1c9b5c27871e8087726bf136696f22243679"
+version = "0.33.0"
+source = "git+https://github.com/lancedb/lance.git?tag=v0.33.0-beta.4#e37e9df2458e88c37205415bd00d950b7f936061"
 dependencies = [
 "arrow",
 "arrow-array",
@@ -4171,9 +4115,11 @@ dependencies = [

 [[package]]
 name = "lance-encoding"
-version = "0.34.0"
-source = "git+https://github.com/lancedb/lance.git?tag=v0.34.0-beta.3#31bc1c9b5c27871e8087726bf136696f22243679"
+version = "0.33.0"
+source = "git+https://github.com/lancedb/lance.git?tag=v0.33.0-beta.4#e37e9df2458e88c37205415bd00d950b7f936061"
 dependencies = [
+ "arrayref",
+ "arrow",
 "arrow-arith",
 "arrow-array",
 "arrow-buffer",
@@ -4190,15 +4136,16 @@ dependencies = [
 "hyperloglogplus",
 "itertools 0.13.0",
 "lance-arrow",
- "lance-bitpacking",
 "lance-core",
 "log",
 "lz4",
 "num-traits",
+ "paste",
 "prost",
 "prost-build",
 "prost-types",
 "rand 0.9.1",
+ "seq-macro",
 "snafu",
 "tokio",
 "tracing",
@@ -4208,8 +4155,8 @@ dependencies = [

 [[package]]
 name = "lance-file"
-version = "0.34.0"
-source = "git+https://github.com/lancedb/lance.git?tag=v0.34.0-beta.3#31bc1c9b5c27871e8087726bf136696f22243679"
+version = "0.33.0"
+source = "git+https://github.com/lancedb/lance.git?tag=v0.33.0-beta.4#e37e9df2458e88c37205415bd00d950b7f936061"
 dependencies = [
 "arrow-arith",
 "arrow-array",
@@ -4243,8 +4190,8 @@ dependencies = [

 [[package]]
 name = "lance-index"
-version = "0.34.0"
-source = "git+https://github.com/lancedb/lance.git?tag=v0.34.0-beta.3#31bc1c9b5c27871e8087726bf136696f22243679"
+version = "0.33.0"
+source = "git+https://github.com/lancedb/lance.git?tag=v0.33.0-beta.4#e37e9df2458e88c37205415bd00d950b7f936061"
 dependencies = [
 "arrow",
 "arrow-array",
@@ -4298,8 +4245,8 @@ dependencies = [

 [[package]]
 name = "lance-io"
-version = "0.34.0"
-source = "git+https://github.com/lancedb/lance.git?tag=v0.34.0-beta.3#31bc1c9b5c27871e8087726bf136696f22243679"
+version = "0.33.0"
+source = "git+https://github.com/lancedb/lance.git?tag=v0.33.0-beta.4#e37e9df2458e88c37205415bd00d950b7f936061"
 dependencies = [
 "arrow",
 "arrow-arith",
@@ -4339,8 +4286,8 @@ dependencies = [

 [[package]]
 name = "lance-linalg"
-version = "0.34.0"
-source = "git+https://github.com/lancedb/lance.git?tag=v0.34.0-beta.3#31bc1c9b5c27871e8087726bf136696f22243679"
+version = "0.33.0"
+source = "git+https://github.com/lancedb/lance.git?tag=v0.33.0-beta.4#e37e9df2458e88c37205415bd00d950b7f936061"
 dependencies = [
 "arrow-array",
 "arrow-buffer",
@@ -4363,8 +4310,8 @@ dependencies = [

 [[package]]
 name = "lance-table"
-version = "0.34.0"
-source = "git+https://github.com/lancedb/lance.git?tag=v0.34.0-beta.3#31bc1c9b5c27871e8087726bf136696f22243679"
+version = "0.33.0"
+source = "git+https://github.com/lancedb/lance.git?tag=v0.33.0-beta.4#e37e9df2458e88c37205415bd00d950b7f936061"
 dependencies = [
 "arrow",
 "arrow-array",
@@ -4402,8 +4349,8 @@ dependencies = [

 [[package]]
 name = "lance-testing"
-version = "0.34.0"
-source = "git+https://github.com/lancedb/lance.git?tag=v0.34.0-beta.3#31bc1c9b5c27871e8087726bf136696f22243679"
+version = "0.33.0"
+source = "git+https://github.com/lancedb/lance.git?tag=v0.33.0-beta.4#e37e9df2458e88c37205415bd00d950b7f936061"
 dependencies = [
 "arrow-array",
 "arrow-schema",
@@ -5023,15 +4970,6 @@ dependencies = [
 "minimal-lexical",
 ]

-[[package]]
-name = "nom"
-version = "8.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "df9761775871bdef83bee530e60050f7e54b1105350d6884eb0fb4f46c2f9405"
-dependencies = [
- "memchr",
-]
-
 [[package]]
 name = "now"
 version = "0.1.3"
@@ -5319,15 +5257,6 @@ version = "0.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d"

-[[package]]
-name = "ordered-float"
-version = "5.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e2c1f9f56e534ac6a9b8a4600bdf0f530fb393b5f393e7b4d03489c3cf0c3f01"
-dependencies = [
- "num-traits",
-]
-
 [[package]]
 name = "outref"
 version = "0.5.2"
@@ -6605,7 +6534,7 @@ dependencies = [
 "futures-core",
 "futures-timer",
 "mime",
- "nom 7.1.3",
+ "nom",
 "pin-project-lite",
 "reqwest",
 "thiserror 1.0.69",
@@ -7239,7 +7168,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "5851699c4033c63636f7ea4cf7b7c1f1bf06d0cc03cfb42e711de5a5c46cf326"
 dependencies = [
 "base64 0.13.1",
- "nom 7.1.3",
+ "nom",
 "serde",
 "unicode-segmentation",
 ]
@@ -7576,7 +7505,7 @@ version = "0.24.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e810cdeeebca57fc3f7bfec5f85fdbea9031b2ac9b990eb5ff49b371d52bbe6a"
 dependencies = [
- "nom 7.1.3",
+ "nom",
 "serde",
 "serde_json",
 ]
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -15,14 +15,14 @@ categories = ["database-implementations"]
 rust-version = "1.78.0"

 [workspace.dependencies]
-lance = { "version" = "=0.34.0", default-features = false, "features" = ["dynamodb"], "tag" = "v0.34.0-beta.3", "git" = "https://github.com/lancedb/lance.git" }
-lance-io = { "version" = "=0.34.0", default-features = false, "tag" = "v0.34.0-beta.3", "git" = "https://github.com/lancedb/lance.git" }
-lance-index = { "version" = "=0.34.0", "tag" = "v0.34.0-beta.3", "git" = "https://github.com/lancedb/lance.git" }
-lance-linalg = { "version" = "=0.34.0", "tag" = "v0.34.0-beta.3", "git" = "https://github.com/lancedb/lance.git" }
-lance-table = { "version" = "=0.34.0", "tag" = "v0.34.0-beta.3", "git" = "https://github.com/lancedb/lance.git" }
-lance-testing = { "version" = "=0.34.0", "tag" = "v0.34.0-beta.3", "git" = "https://github.com/lancedb/lance.git" }
-lance-datafusion = { "version" = "=0.34.0", "tag" = "v0.34.0-beta.3", "git" = "https://github.com/lancedb/lance.git" }
-lance-encoding = { "version" = "=0.34.0", "tag" = "v0.34.0-beta.3", "git" = "https://github.com/lancedb/lance.git" }
+lance = { "version" = "=0.33.0", default-features = false, "features" = ["dynamodb"], "tag" = "v0.33.0-beta.4", "git" = "https://github.com/lancedb/lance.git" }
+lance-io = { "version" = "=0.33.0", default-features = false, "tag" = "v0.33.0-beta.4", "git" = "https://github.com/lancedb/lance.git" }
+lance-index = { "version" = "=0.33.0", "tag" = "v0.33.0-beta.4", "git" = "https://github.com/lancedb/lance.git" }
+lance-linalg = { "version" = "=0.33.0", "tag" = "v0.33.0-beta.4", "git" = "https://github.com/lancedb/lance.git" }
+lance-table = { "version" = "=0.33.0", "tag" = "v0.33.0-beta.4", "git" = "https://github.com/lancedb/lance.git" }
+lance-testing = { "version" = "=0.33.0", "tag" = "v0.33.0-beta.4", "git" = "https://github.com/lancedb/lance.git" }
+lance-datafusion = { "version" = "=0.33.0", "tag" = "v0.33.0-beta.4", "git" = "https://github.com/lancedb/lance.git" }
+lance-encoding = { "version" = "=0.33.0", "tag" = "v0.33.0-beta.4", "git" = "https://github.com/lancedb/lance.git" }
 # Note that this one does not include pyarrow
 arrow = { version = "55.1", optional = false }
 arrow-array = "55.1"
--- a/nodejs/lancedb/query.ts
+++ b/nodejs/lancedb/query.ts
@@ -363,6 +363,23 @@ export class StandardQueryBase<
    return this.where(predicate);
  }

+  /**
+   * Disable autoprojection of scoring columns.
+   *
+   * When you specify an explicit projection with {@link select} that does not
+   * include scoring columns (e.g. `_score` for FTS or `_distance` for vector
+   * search), Lance currently auto-includes those columns and emits a
+   * deprecation warning. Calling this method disables that behavior so the
+   * scoring columns are only returned if explicitly selected.
+   */
+  disableScoringAutoprojection(): this {
+    this.doCall((inner: NativeQueryType) => {
+      // @ts-expect-error method is present on Query and VectorQuery only
+      inner.disableScoringAutoprojection();
+    });
+    return this;
+  }
+
  fullTextSearch(
    query: string | FullTextQuery,
    options?: Partial<FullTextSearchOptions>,
--- a/nodejs/src/query.rs
+++ b/nodejs/src/query.rs
@@ -88,6 +88,11 @@ impl Query {
        self.inner = self.inner.clone().with_row_id();
    }

+    #[napi]
+    pub fn disable_scoring_autoprojection(&mut self) {
+        self.inner = self.inner.clone().disable_scoring_autoprojection();
+    }
+
    #[napi(catch_unwind)]
    pub async fn execute(
        &self,
@@ -265,6 +270,11 @@ impl VectorQuery {
        self.inner = self.inner.clone().with_row_id();
    }

+    #[napi]
+    pub fn disable_scoring_autoprojection(&mut self) {
+        self.inner = self.inner.clone().disable_scoring_autoprojection();
+    }
+
    #[napi]
    pub fn rerank(&mut self, callbacks: RerankerCallbacks) {
        self.inner = self
--- a/nodejs/src/session.rs
+++ b/nodejs/src/session.rs
@@ -94,7 +94,7 @@ impl napi::bindgen_prelude::FromNapiValue for Session {
        env: napi::sys::napi_env,
        napi_val: napi::sys::napi_value,
    ) -> napi::Result<Self> {
-        let object: napi::bindgen_prelude::ClassInstance<Self> =
+        let object: napi::bindgen_prelude::ClassInstance<Session> =
            napi::bindgen_prelude::ClassInstance::from_napi_value(env, napi_val)?;
        let copy = object.clone();
        Ok(copy)
--- a/python/.bumpversion.toml
+++ b/python/.bumpversion.toml
@@ -1,5 +1,5 @@
 [tool.bumpversion]
-current_version = "0.25.0-beta.0"
+current_version = "0.24.4-beta.1"
 parse = """(?x)
    (?P<major>0|[1-9]\\d*)\\.
    (?P<minor>0|[1-9]\\d*)\\.
--- a/python/Cargo.toml
+++ b/python/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "lancedb-python"
-version = "0.25.0-beta.0"
+version = "0.24.4-beta.1"
 edition.workspace = true
 description = "Python bindings for LanceDB"
 license.workspace = true
--- a/rust/lancedb/src/io/object_store.rs
+++ b/rust/lancedb/src/io/object_store.rs
@@ -170,11 +170,7 @@ impl MirroringObjectStoreWrapper {
 }

 impl WrappingObjectStore for MirroringObjectStoreWrapper {
-    fn wrap(
-        &self,
-        primary: Arc<dyn ObjectStore>,
-        _storage_options: Option<&std::collections::HashMap<String, String>>,
-    ) -> Arc<dyn ObjectStore> {
+    fn wrap(&self, primary: Arc<dyn ObjectStore>) -> Arc<dyn ObjectStore> {
        Arc::new(MirroringObjectStore {
            primary,
            secondary: self.secondary.clone(),
--- a/rust/lancedb/src/io/object_store/io_tracking.rs
+++ b/rust/lancedb/src/io/object_store/io_tracking.rs
@@ -50,11 +50,7 @@ impl IoStatsHolder {
 }

 impl WrappingObjectStore for IoStatsHolder {
-    fn wrap(
-        &self,
-        target: Arc<dyn ObjectStore>,
-        _storage_options: Option<&std::collections::HashMap<String, String>>,
-    ) -> Arc<dyn ObjectStore> {
+    fn wrap(&self, target: Arc<dyn ObjectStore>) -> Arc<dyn ObjectStore> {
        Arc::new(IoTrackingStore {
            target,
            stats: self.0.clone(),
--- a/rust/lancedb/src/query.rs
+++ b/rust/lancedb/src/query.rs
@@ -448,6 +448,15 @@ pub trait QueryBase {
    /// the scores are converted to ranks and then normalized. If "Score", the
    /// scores are normalized directly.
    fn norm(self, norm: NormalizeMethod) -> Self;
+
+    /// Disable autoprojection of scoring columns.
+    ///
+    /// When an explicit projection is provided that does not include scoring
+    /// columns (e.g. `_score` for FTS or `_distance` for vector search), the
+    /// current default behavior is to auto-include those columns and emit a
+    /// deprecation warning. Calling this adopts the future behavior and avoids
+    /// the warning.
+    fn disable_scoring_autoprojection(self) -> Self;
 }

 pub trait HasQuery {
@@ -507,6 +516,11 @@ impl<T: HasQuery> QueryBase for T {
        self.mut_query().norm = Some(norm);
        self
    }
+
+    fn disable_scoring_autoprojection(mut self) -> Self {
+        self.mut_query().disable_scoring_autoprojection = true;
+        self
+    }
 }

 /// Options for controlling the execution of a query
@@ -645,6 +659,10 @@ pub struct QueryRequest {

    /// Configure how query results are normalized when doing hybrid search
    pub norm: Option<NormalizeMethod>,
+
+    /// If true, do not auto-include scoring columns when they are
+    /// omitted from an explicit projection.
+    pub disable_scoring_autoprojection: bool,
 }

 impl Default for QueryRequest {
@@ -660,6 +678,7 @@ impl Default for QueryRequest {
            prefilter: true,
            reranker: None,
            norm: None,
+            disable_scoring_autoprojection: false,
        }
    }
 }
--- a/rust/lancedb/src/remote/table.rs
+++ b/rust/lancedb/src/remote/table.rs
@@ -372,6 +372,9 @@ impl<S: HttpSend> RemoteTable<S> {
        params: &QueryRequest,
    ) -> Result<()> {
        body["prefilter"] = params.prefilter.into();
+        if params.disable_scoring_autoprojection {
+            body["disable_scoring_autoprojection"] = serde_json::Value::Bool(true);
+        }
        if let Some(offset) = params.offset {
            body["offset"] = serde_json::Value::Number(serde_json::Number::from(offset));
        }
--- a/rust/lancedb/src/table.rs
+++ b/rust/lancedb/src/table.rs
@@ -2331,6 +2331,10 @@ impl BaseTable for NativeTable {
            scanner.full_text_search(fts.clone())?;
        }

+        if query.base.disable_scoring_autoprojection {
+            scanner.disable_scoring_autoprojection();
+        }
+
        if let Some(refine_factor) = query.refine_factor {
            scanner.refine(refine_factor);
        }
@@ -3264,7 +3268,6 @@ mod tests {
        fn wrap(
            &self,
            original: Arc<dyn object_store::ObjectStore>,
-            _storage_options: Option<&std::collections::HashMap<String, String>>,
        ) -> Arc<dyn object_store::ObjectStore> {
            self.called.store(true, Ordering::Relaxed);
            original
Author	SHA1	Message	Date
Xuanwo	a8f57e8043	Format code Signed-off-by: Xuanwo <github@xuanwo.io>	2025-09-09 01:08:02 +08:00
Xuanwo	c36409d3b8	feat: Expose disableScoringAutoprojection to lancedb Signed-off-by: Xuanwo <github@xuanwo.io>	2025-09-09 01:06:29 +08:00