docs(nodejs): correct LangChain result handling

docs/src/js/README.md

@@ -34,6 +34,36 @@ const results = await table.vectorSearch([0.1, 0.3]).limit(20).toArray();
 console.log(results);
 ```
 
+### Use an Existing Table with LangChain
+
+When wrapping an existing table with `@langchain/community`, open the table
+with LanceDB and pass the resulting table handle to LangChain. The LangChain
+`uri` and `tableName` options are used when creating a table; they do not open
+an existing table for search.
+
+```javascript
+import { LanceDB as LangChainLanceDB } from "@langchain/community/vectorstores/lancedb";
+import * as lancedb from "@lancedb/lancedb";
+
+const db = await lancedb.connect("data/sample-lancedb");
+const table = await db.openTable("my_table");
+const vectorStore = new LangChainLanceDB(embeddings, {
+  table,
+  textKey: "item",
+});
+
+const documents = await vectorStore.similaritySearch("foo", 5);
+```
+
+Affected versions of `@langchain/community` do not map LanceDB's `_distance`
+column to the score returned by `similaritySearchVectorWithScore`. Query the
+table directly when you need the numeric vector distance:
+
+```javascript
+const results = await table.vectorSearch(queryVector).limit(5).toArray();
+console.log(results[0]._distance);
+```
+
 The [quickstart](https://docs.lancedb.com/quickstart/) contains more complete examples.
 
 ## Development

nodejs/README.md

@@ -30,6 +30,36 @@ const results = await table.vectorSearch([0.1, 0.3]).limit(20).toArray();
 console.log(results);
 ```
 
+### Use an Existing Table with LangChain
+
+When wrapping an existing table with `@langchain/community`, open the table
+with LanceDB and pass the resulting table handle to LangChain. The LangChain
+`uri` and `tableName` options are used when creating a table; they do not open
+an existing table for search.
+
+```javascript
+import { LanceDB as LangChainLanceDB } from "@langchain/community/vectorstores/lancedb";
+import * as lancedb from "@lancedb/lancedb";
+
+const db = await lancedb.connect("data/sample-lancedb");
+const table = await db.openTable("my_table");
+const vectorStore = new LangChainLanceDB(embeddings, {
+  table,
+  textKey: "item",
+});
+
+const documents = await vectorStore.similaritySearch("foo", 5);
+```
+
+Affected versions of `@langchain/community` do not map LanceDB's `_distance`
+column to the score returned by `similaritySearchVectorWithScore`. Query the
+table directly when you need the numeric vector distance:
+
+```javascript
+const results = await table.vectorSearch(queryVector).limit(5).toArray();
+console.log(results[0]._distance);
+```
+
 The [quickstart](https://docs.lancedb.com/quickstart/) contains more complete examples.
 
 ## Development

python/python/lancedb/_lancedb.pyi

@@ -269,7 +269,6 @@ class Table:
         mode: Literal["append", "overwrite"],
         progress: Optional[Any] = None,
         write_parallelism: Optional[int] = None,
-        on_nan_vectors: Optional[Literal["error", "keep"]] = None,
     ) -> AddResult: ...
     async def update(
         self, updates: Dict[str, str], where: Optional[str]

python/python/lancedb/db.py

@@ -333,9 +333,7 @@ class DBConnection(EnforceOverrides):
             schema that's specified.
         on_bad_vectors: str, default "error"
             What to do if any of the vectors are not the same size or contains NaNs.
-            One of "error", "drop", "fill", "null", or "keep". With "keep",
-            vectors containing NaNs are preserved, but vectors with the wrong
-            dimension still raise an error.
+            One of "error", "drop", "fill".
         fill_value: float
             The value to use when filling vectors. Only used if on_bad_vectors="fill".
         storage_options: dict, optional
@@ -709,9 +707,6 @@ class LanceDBConnection(DBConnection):
         self._namespace_client_properties = namespace_client_properties
         if _inner is not None:
             self._conn = _inner
-            # Native-derived wrappers resolve this in their async reconstruction
-            # path so construction never synchronously re-enters LOOP.
-            self._read_consistency_interval = read_consistency_interval
             self._cached_namespace_client = None
             return
 
@@ -761,14 +756,11 @@ class LanceDBConnection(DBConnection):
         # storage_options.  Also, this class really shouldn't be holding any state
         # beyond _conn.
         self._conn = AsyncConnection(LOOP.run(do_connect()))
-        # Keep property access synchronous so debugger introspection cannot wait on
-        # the background loop while that thread is suspended at a breakpoint.
-        self._read_consistency_interval = read_consistency_interval
         self._cached_namespace_client: Optional[LanceNamespace] = None
 
     @property
     def read_consistency_interval(self) -> Optional[timedelta]:
-        return self._read_consistency_interval
+        return LOOP.run(self._conn.get_read_consistency_interval())
 
     @property
     def session(self) -> Optional[Session]:
@@ -779,16 +771,8 @@ class LanceDBConnection(DBConnection):
         return self._conn.uri
 
     @classmethod
-    def from_inner(
-        cls,
-        inner: LanceDbConnection,
-        read_consistency_interval: Optional[timedelta],
-    ):
-        return cls(
-            None,
-            read_consistency_interval=read_consistency_interval,
-            _inner=inner,
-        )
+    def from_inner(cls, inner: LanceDbConnection):
+        return cls(None, _inner=inner)
 
     def __repr__(self) -> str:
         return f"{self.__class__.__name__}(uri={self._conn.uri!r})"
@@ -1597,9 +1581,7 @@ class AsyncConnection(object):
             schema that's specified.
         on_bad_vectors: str, default "error"
             What to do if any of the vectors are not the same size or contains NaNs.
-            One of "error", "drop", "fill", "null", or "keep". With "keep",
-            vectors containing NaNs are preserved, but vectors with the wrong
-            dimension still raise an error.
+            One of "error", "drop", "fill".
         fill_value: float
             The value to use when filling vectors. Only used if on_bad_vectors="fill".
         storage_options: dict, optional

python/python/lancedb/merge.py

@@ -175,9 +175,7 @@ class LanceMergeInsertBuilder(object):
             can be anything you use for [`add`][lancedb.table.Table.add]
         on_bad_vectors: str, default "error"
             What to do if any of the vectors are not the same size or contains NaNs.
-            One of "error", "drop", "fill", "null", or "keep". With "keep",
-            vectors containing NaNs are preserved, but vectors with the wrong
-            dimension still raise an error.
+            One of "error", "drop", "fill".
         fill_value: float, default 0.
             The value to use when filling vectors. Only used if on_bad_vectors="fill".
         timeout: Optional[timedelta], default None

python/python/lancedb/permutation.py

@@ -226,7 +226,7 @@ class PermutationBuilder:
 
         async def do_execute():
             inner_tbl = await self._async.execute()
-            return await LanceTable.from_inner(inner_tbl)
+            return LanceTable.from_inner(inner_tbl)
 
         return LOOP.run(do_execute())
 

python/python/lancedb/remote/db.py

@@ -543,9 +543,7 @@ class RemoteDBConnection(DBConnection):
             to "exist_ok".
         on_bad_vectors: str, default "error"
             What to do if any of the vectors are not the same size or contains NaNs.
-            One of "error", "drop", "fill", "null", or "keep". With "keep",
-            vectors containing NaNs are preserved, but vectors with the wrong
-            dimension still raise an error.
+            One of "error", "drop", "fill".
         fill_value: float
             The value to use when filling vectors. Only used if on_bad_vectors="fill".
 

python/python/lancedb/remote/table.py

@@ -629,9 +629,7 @@ class RemoteTable(Table):
             "append" and "overwrite".
         on_bad_vectors: str, default "error"
             What to do if any of the vectors are not the same size or contains NaNs.
-            One of "error", "drop", "fill", "null", or "keep". With "keep",
-            vectors containing NaNs are preserved, but vectors with the wrong
-            dimension still raise an error.
+            One of "error", "drop", "fill".
         fill_value: float, default 0.
             The value to use when filling vectors. Only used if on_bad_vectors="fill".
         progress: bool, callable, or tqdm-like, optional

python/python/lancedb/table.py

@@ -350,10 +350,8 @@ def _sanitize_data(
         in the input table before casting.
     metadata : Optional[dict], default None
         The embedding metadata to add to the schema.
-    on_bad_vectors : Literal["error", "drop", "fill", "null", "keep"], default "error"
+    on_bad_vectors : Literal["error", "drop", "fill", "null"], default "error"
         What to do if any of the vectors are not the same size or contains NaNs.
-        With "keep", vectors containing NaNs are preserved, but vectors with the
-        wrong dimension still raise an error.
     fill_value : float, default 0.0
         The value to use when filling vectors. Only used if on_bad_vectors="fill".
         All entries in the vector will be set to this value.
@@ -1249,9 +1247,7 @@ class Table(ABC):
             "append" and "overwrite".
         on_bad_vectors: str, default "error"
             What to do if any of the vectors are not the same size or contains NaNs.
-            One of "error", "drop", "fill", "null", or "keep". With "keep",
-            vectors containing NaNs are preserved but are not indexed for vector
-            search; vectors with the wrong dimension still raise an error.
+            One of "error", "drop", "fill".
         fill_value: float, default 0.
             The value to use when filling vectors. Only used if on_bad_vectors="fill".
         progress: bool, callable, or tqdm-like, optional
@@ -2186,15 +2182,11 @@ class LanceTable(Table):
         return self.name
 
     @classmethod
-    async def from_inner(cls, tbl: LanceDBTable):
-        from .db import AsyncConnection, LanceDBConnection
+    def from_inner(cls, tbl: LanceDBTable):
+        from .db import LanceDBConnection
 
         async_tbl = AsyncTable(tbl)
-        inner_conn = tbl.database()
-        read_consistency_interval = await AsyncConnection(
-            inner_conn
-        ).get_read_consistency_interval()
-        conn = LanceDBConnection.from_inner(inner_conn, read_consistency_interval)
+        conn = LanceDBConnection.from_inner(tbl.database())
         return cls(
             conn,
             async_tbl.name,
@@ -3273,9 +3265,7 @@ class LanceTable(Table):
             "append" and "overwrite".
         on_bad_vectors: str, default "error"
             What to do if any of the vectors are not the same size or contains NaNs.
-            One of "error", "drop", "fill", "null", or "keep". With "keep",
-            vectors containing NaNs are preserved, but vectors with the wrong
-            dimension still raise an error.
+            One of "error", "drop", "fill", "null".
         fill_value: float, default 0.
             The value to use when filling vectors. Only used if on_bad_vectors="fill".
         progress: bool, callable, or tqdm-like, optional
@@ -3588,9 +3578,7 @@ class LanceTable(Table):
             data but will validate against any schema that's specified.
         on_bad_vectors: str, default "error"
             What to do if any of the vectors are not the same size or contains NaNs.
-            One of "error", "drop", "fill", "null", or "keep". With "keep",
-            vectors containing NaNs are preserved, but vectors with the wrong
-            dimension still raise an error.
+            One of "error", "drop", "fill", "null".
         fill_value: float, default 0.
             The value to use when filling vectors. Only used if on_bad_vectors="fill".
         embedding_functions: list of EmbeddingFunctionModel, default None
@@ -4026,7 +4014,7 @@ class LanceTable(Table):
 
 def _handle_bad_vectors(
     reader: pa.RecordBatchReader,
-    on_bad_vectors: OnBadVectorsType = "error",
+    on_bad_vectors: Literal["error", "drop", "fill", "null"] = "error",
     fill_value: float = 0.0,
     target_schema: Optional[pa.Schema] = None,
     metadata: Optional[dict] = None,
@@ -4200,9 +4188,7 @@ def _handle_bad_vector_column(
         The name of the vector column.
     on_bad_vectors: str, default "error"
         What to do if any of the vectors are not the same size or contains NaNs.
-        One of "error", "drop", "fill", "null", or "keep". With "keep",
-        vectors containing NaNs are preserved, but vectors with the wrong dimension
-        still raise an error.
+        One of "error", "drop", "fill", "null".
     fill_value: float, default 0.0
         The value to use when filling vectors. Only used if on_bad_vectors="fill".
     """
@@ -4267,8 +4253,7 @@ def _handle_bad_vector_column(
                     f"Vector column '{vector_column_name}' has NaNs. "
                     "Set on_bad_vectors='drop' to remove them, "
                     "set on_bad_vectors='fill' and fill_value=<value> to replace them, "
-                    "set on_bad_vectors='null' to replace them with null, "
-                    "or set on_bad_vectors='keep' to preserve them."
+                    "or set on_bad_vectors='null' to replace them with null."
                 )
         elif on_bad_vectors == "null":
             vec_arr = pc.if_else(
@@ -4285,16 +4270,6 @@ def _handle_bad_vector_column(
                     "`fill_value` must not be None if `on_bad_vectors` is 'fill'"
                 )
             vec_arr = _fill_bad_vector_values(vec_arr, dim, fill_value)
-        elif on_bad_vectors == "keep":
-            if pc.any(has_wrong_dim).as_py():
-                raise ValueError(
-                    f"Vector column '{vector_column_name}' has variable length "
-                    "vectors. on_bad_vectors='keep' only preserves vectors "
-                    "containing NaNs. Set on_bad_vectors='drop' to remove "
-                    "wrong-size vectors, set on_bad_vectors='fill' and "
-                    "fill_value=<value> to replace them, or set "
-                    "on_bad_vectors='null' to replace them with null."
-                )
         else:
             raise ValueError(f"Invalid value for on_bad_vectors: {on_bad_vectors}")
 
@@ -5139,9 +5114,7 @@ class AsyncTable:
             "append" and "overwrite".
         on_bad_vectors: str, default "error"
             What to do if any of the vectors are not the same size or contains NaNs.
-            One of "error", "drop", "fill", "null", or "keep". With "keep",
-            vectors containing NaNs are preserved but are not indexed for vector
-            search; vectors with the wrong dimension still raise an error.
+            One of "error", "drop", "fill", "null".
         fill_value: float, default 0.
             The value to use when filling vectors. Only used if on_bad_vectors="fill".
         progress: callable or tqdm-like, optional
@@ -5189,7 +5162,6 @@ class AsyncTable:
                 mode or "append",
                 progress=progress,
                 write_parallelism=write_parallelism,
-                on_nan_vectors="keep" if on_bad_vectors == "keep" else None,
             )
         except RuntimeError as e:
             if "Cast error" in str(e):

python/python/lancedb/types.py

@@ -24,7 +24,7 @@ DistanceType = Literal["l2", "cosine", "dot"]
 DistanceTypeWithHamming = Literal["l2", "cosine", "dot", "hamming"]
 
 # Vector handling literals
-OnBadVectorsType = Literal["error", "drop", "fill", "null", "keep"]
+OnBadVectorsType = Literal["error", "drop", "fill", "null"]
 
 # Mode literals
 AddMode = Literal["append", "overwrite"]

python/python/tests/test_db.py

@@ -77,23 +77,6 @@ def test_sync_repr_does_not_use_background_loop(tmp_path, monkeypatch):
     assert repr(table) == f"LanceTable(name='test', _conn={db!r})"
 
 
-def test_read_consistency_interval_does_not_use_background_loop(tmp_path, monkeypatch):
-    from lancedb.background_loop import LOOP
-    from lancedb.db import LanceDBConnection
-
-    consistency_interval = timedelta(seconds=5)
-    db = lancedb.connect(tmp_path, read_consistency_interval=consistency_interval)
-    db_from_inner = LanceDBConnection.from_inner(db._inner, consistency_interval)
-
-    def fail_run(*args, **kwargs):
-        raise AssertionError("properties should not use the Python background loop")
-
-    monkeypatch.setattr(LOOP, "run", fail_run)
-
-    assert db.read_consistency_interval == consistency_interval
-    assert db_from_inner.read_consistency_interval == consistency_interval
-
-
 def test_ingest_pd(tmp_path):
     db = lancedb.connect(tmp_path)
 

python/python/tests/test_permutation.py

@@ -6,7 +6,6 @@ import math
 import pytest
 
 from lancedb import DBConnection, Table, connect
-from lancedb.background_loop import LOOP
 from lancedb.permutation import Permutation, Permutations, permutation_builder
 
 
@@ -32,25 +31,6 @@ def test_split_random_ratios(mem_db):
     assert 65 <= split_1_count <= 75  # ~70% ± tolerance
 
 
-def test_execute_does_not_reenter_background_loop(tmp_path, monkeypatch):
-    import threading
-
-    db = connect(tmp_path)
-    tbl = db.create_table("test_table", pa.table({"x": range(10)}))
-    original_run = LOOP.run
-
-    def fail_on_reentry(future):
-        assert threading.current_thread() is not LOOP.thread
-        return original_run(future)
-
-    monkeypatch.setattr(LOOP, "run", fail_on_reentry)
-
-    permutation_tbl = permutation_builder(tbl).execute()
-
-    assert permutation_tbl.count_rows() == 10
-    assert permutation_tbl._conn.read_consistency_interval is None
-
-
 def test_split_random_counts(mem_db):
     """Test random splitting with absolute counts."""
     tbl = mem_db.create_table(

python/python/tests/test_table.py

@@ -6,7 +6,6 @@ import os
 import sys
 import threading
 import warnings
-from concurrent.futures import ThreadPoolExecutor
 from datetime import date, datetime, timedelta
 from time import sleep
 from typing import List
@@ -1767,40 +1766,6 @@ def test_add_with_nans(mem_db: DBConnection):
     assert np.allclose(filled_vectors[22.0], np.array([5.0, 0.0]))
 
 
-def test_add_with_non_finite_values_keep(mem_db: DBConnection):
-    schema = pa.schema([pa.field("data", pa.list_(pa.float32(), 4))])
-    table = mem_db.create_table("test", schema=schema)
-    batch = pa.table(
-        {
-            "data": pa.array(
-                [[np.nan, np.inf, -np.inf, -0.0]],
-                type=schema.field("data").type,
-            )
-        },
-        schema=schema,
-    )
-
-    with pytest.raises(ValueError, match="NaN"):
-        table.add(batch)
-
-    table.add(batch, on_bad_vectors="keep")
-
-    values = table.to_arrow()["data"][0].as_py()
-    assert np.isnan(values[0])
-    assert np.isposinf(values[1])
-    assert np.isneginf(values[2])
-    assert values[3] == 0.0
-    assert np.signbit(values[3])
-
-
-def test_add_keep_rejects_wrong_dimension(mem_db: DBConnection):
-    schema = pa.schema([pa.field("vector", pa.list_(pa.float32(), 2))])
-    table = mem_db.create_table("test", schema=schema)
-
-    with pytest.raises((ValueError, RuntimeError), match="variable length"):
-        table.add([{"vector": [1.0]}], on_bad_vectors="keep")
-
-
 def test_add_with_empty_fixed_size_list_drops_bad_rows(mem_db: DBConnection):
     class Schema(LanceModel):
         text: str
@@ -2159,27 +2124,6 @@ def test_delete(mem_db: DBConnection):
     assert table.to_arrow()["id"].to_pylist() == [1]
 
 
-def test_concurrent_deletes_are_thread_safe(mem_db: DBConnection):
-    num_workers = 8
-    table = mem_db.create_table(
-        "my_table", data=[{"id": row_id} for row_id in range(num_workers)]
-    )
-    barrier = threading.Barrier(num_workers)
-
-    def delete(row_id: int):
-        barrier.wait()
-        return table.delete(f"id = {row_id}")
-
-    with ThreadPoolExecutor(max_workers=num_workers) as pool:
-        results = list(pool.map(delete, range(num_workers)))
-
-    assert all(result.num_deleted_rows == 1 for result in results)
-    assert sorted(result.version for result in results) == list(
-        range(2, num_workers + 2)
-    )
-    assert table.count_rows() == 0
-
-
 def test_delete_expr(mem_db: DBConnection):
     table = mem_db.create_table(
         "my_table",

python/python/tests/test_util.py

@@ -2,7 +2,6 @@
 # SPDX-FileCopyrightText: Copyright The LanceDB Authors
 
 
-import math
 import os
 import pathlib
 from typing import Optional
@@ -365,7 +364,7 @@ def test_fill_bad_vector_values_arrow_types(vector_type, vectors, expected):
     assert actual.to_pylist() == expected
 
 
-@pytest.mark.parametrize("on_bad_vectors", ["error", "drop", "fill", "null", "keep"])
+@pytest.mark.parametrize("on_bad_vectors", ["error", "drop", "fill", "null"])
 def test_handle_bad_vectors_nan(on_bad_vectors):
     vector = pa.array([[1.0, float("nan")], [3.0, 4.0]])
     data = pa.table({"vector": vector})
@@ -380,9 +379,8 @@ def test_handle_bad_vectors_nan(on_bad_vectors):
         assert output == (
             "ValueError: Vector column 'vector' has NaNs. Set "
             "on_bad_vectors='drop' to remove them, set on_bad_vectors='fill' "
-            "and fill_value=<value> to replace them, set on_bad_vectors='null' "
-            "to replace them with null, or set on_bad_vectors='keep' to preserve "
-            "them."
+            "and fill_value=<value> to replace them, or set on_bad_vectors='null' "
+            "to replace them with null."
         )
         return
     else:
@@ -398,26 +396,10 @@ def test_handle_bad_vectors_nan(on_bad_vectors):
         expected = pa.array([[1.0, 42.0], [3.0, 4.0]])
     elif on_bad_vectors == "null":
         expected = pa.array([None, [3.0, 4.0]])
-    elif on_bad_vectors == "keep":
-        actual = output["vector"].to_pylist()
-        assert actual[0][0] == 1.0
-        assert math.isnan(actual[0][1])
-        assert actual[1] == [3.0, 4.0]
-        return
 
     assert output["vector"].combine_chunks() == expected
 
 
-def test_handle_bad_vectors_keep_rejects_wrong_dimension():
-    data = pa.table({"vector": [[1.0, 2.0], [3.0]]})
-
-    with pytest.raises(ValueError, match="only preserves vectors containing NaNs"):
-        _handle_bad_vectors(
-            data.to_reader(),
-            on_bad_vectors="keep",
-        ).read_all()
-
-
 def test_handle_bad_vectors_noop():
     # ChunkedArray should be preserved as-is
     vector = pa.chunked_array(

python/src/table.rs

@@ -21,8 +21,7 @@ use lancedb::blob::{BlobFile, BlobRangeRequest};
 use lancedb::index::scalar::FtsIndexBuilder;
 use lancedb::table::{
     AddDataMode, ColumnAlteration, Duration, FieldMetadataUpdate, FtsToken as LanceDbFtsToken,
-    NaNVectorBehavior, NewColumnTransform, OptimizeAction, OptimizeOptions, Ref,
-    Table as LanceDbTable,
+    NewColumnTransform, OptimizeAction, OptimizeOptions, Ref, Table as LanceDbTable,
 };
 use lancedb::tokenize as lancedb_tokenize;
 use pyo3::{
@@ -643,14 +642,13 @@ impl Table {
         })
     }
 
-    #[pyo3(signature = (data, mode, progress=None, write_parallelism=None, on_nan_vectors=None))]
+    #[pyo3(signature = (data, mode, progress=None, write_parallelism=None))]
     pub fn add<'a>(
         self_: PyRef<'a, Self>,
         data: PyScannable,
         mode: String,
         progress: Option<Py<PyAny>>,
         write_parallelism: Option<usize>,
-        on_nan_vectors: Option<String>,
     ) -> PyResult<Bound<'a, PyAny>> {
         let mut op = self_.inner_ref()?.add(data);
         if mode == "append" {
@@ -660,18 +658,6 @@ impl Table {
         } else {
             return Err(PyValueError::new_err(format!("Invalid mode: {}", mode)));
         }
-        match on_nan_vectors.as_deref() {
-            None | Some("error") => {}
-            Some("keep") => {
-                op = op.on_nan_vectors(NaNVectorBehavior::Keep);
-            }
-            Some(other) => {
-                return Err(PyValueError::new_err(format!(
-                    "Invalid on_nan_vectors: {}",
-                    other
-                )));
-            }
-        }
         if let Some(write_parallelism) = write_parallelism {
             op = op.write_parallelism(write_parallelism);
         }
@@ -759,9 +745,6 @@ impl Table {
 
     #[allow(private_interfaces)]
     pub fn delete(self_: PyRef<'_, Self>, condition: PredicateArg) -> PyResult<Bound<'_, PyAny>> {
-        // Do not hold the Python borrow across the await.  The cloned Rust table
-        // handle is thread-safe and allows deletes on the same Python table to
-        // run concurrently without PyO3 reporting "Already borrowed".
         let inner = self_.inner_ref()?.clone();
         future_into_py(self_.py(), async move {
             let result = match &condition {