From 0aa842b17f13d1f8ffd2851bc57c222bc5a376ba Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Fri, 9 Oct 2026 12:48:29 +0800 Subject: [PATCH] feat: support FTS indexes on JSON fields (#4219) `FTS()` rejects JSON columns because validation only considers their `LargeBinary` storage type. Validate the complete Arrow field in embedded and remote creation so `lance.json` columns reach Lance's existing JSON-aware inverted index, while ordinary binary fields remain rejected. JSON searches use path-qualified triples such as `meta.tag,str,nature`; the API documentation describes this syntax and its field-scoping behavior. Remote creation preserves the supplied FTS configuration. Two integration limits remain: malformed triples can still panic in the pinned Lance dependency (tracked by lance-format/lance#8812 and the pending fix in lance-format/lance#8813), and the opt-in server test has not run against a real deployment because no remote database is configured. That test covers index completion, statistics, search results, and index use; HTTP mock coverage only verifies the client request. Refs #3891. --- python/python/lancedb/index.py | 37 +++++++++++- python/python/tests/test_e2e_remote_db.py | 65 +++++++++++++++++++++ python/python/tests/test_fts.py | 69 +++++++++++++++++++++++ python/python/tests/test_remote_db.py | 66 ++++++++++++++++++++++ rust/lancedb/src/index.rs | 11 ++++ rust/lancedb/src/remote/db.rs | 2 +- rust/lancedb/src/remote/table.rs | 21 ++++--- rust/lancedb/src/table/create_index.rs | 6 +- rust/lancedb/src/utils/mod.rs | 57 +++++++++++++++++++ 9 files changed, 321 insertions(+), 13 deletions(-) diff --git a/python/python/lancedb/index.py b/python/python/lancedb/index.py index 2a7731be3..cf3bf8e4f 100644 --- a/python/python/lancedb/index.py +++ b/python/python/lancedb/index.py @@ -112,9 +112,42 @@ class Fm: class FTS: """Describe a FTS index configuration. - `FTS` is a full-text search index that can be used on `String` columns + `FTS` supports string columns, lists of strings, and JSON columns created + from PyArrow ``pa.json_()``. JSON columns use Lance's JSON-aware tokenizer + and are stored as ``LargeBinary`` with the ``lance.json`` logical type. + Arbitrary binary columns are not supported. - For example, it works with `title`, `description`, `content`, etc. + JSON queries use ``path,str,value`` triples. Paths are relative to the JSON + column and use dots for nested object keys. For example, + ``meta.tag,str,nature`` matches the string value at ``$.meta.tag``; + it does not search other paths or JSON key names. The value is processed + with the configured text tokenizer. Use ``str``, not ``string``. + + Create and query a JSON index through the modern API:: + + import pyarrow as pa + from lancedb.index import FTS + + table = db.create_table( + "documents", + pa.table({"doc": pa.array( + ['{"meta":{"tag":"nature"}}'], type=pa.json_() + )}), + ) + table.create_index("doc", config=FTS()) + table.search( + "meta.tag,str,nature", query_type="fts", fts_columns="doc" + ).to_list() + + Create the index on the JSON column itself, not a JSON subpath such as + ``doc.meta.tag``. JSON-path scalar indexes do not provide FTS path scoping. + Plain text such as ``nature`` is not a valid JSON FTS query. The current + Lance tokenizer can panic on malformed triples; see + https://github.com/lance-format/lance/issues/8812. + + Remote tables use the same ``FTS()`` configuration and require a server + that supports JSON FTS. Wait for ``table.wait_for_index(["doc_idx"])`` + before searching a newly created remote index with its default name. Examples -------- diff --git a/python/python/tests/test_e2e_remote_db.py b/python/python/tests/test_e2e_remote_db.py index d4364bc43..74d5cd7de 100644 --- a/python/python/tests/test_e2e_remote_db.py +++ b/python/python/tests/test_e2e_remote_db.py @@ -2,13 +2,78 @@ # SPDX-FileCopyrightText: Copyright The LanceDB Authors +import os +from datetime import timedelta +from uuid import uuid4 + +import lancedb import numpy as np +import pyarrow as pa import pytest from lancedb import LanceDBConnection +from lancedb.index import FTS # TODO: setup integ test mark and script +@pytest.mark.skipif( + not os.environ.get("LANCEDB_URI"), reason="Set LANCEDB_URI to a JSON FTS server" +) +def test_json_fts_against_server(): + """Requires a writable database and capacity to run the distributed indexer. + + Configure LANCEDB_URI, LANCEDB_API_KEY, and optionally LANCEDB_REGION and + LANCEDB_HOST_OVERRIDE before running this test. + """ + uri = os.environ["LANCEDB_URI"] + assert uri.startswith("db://"), "LANCEDB_URI must identify a remote database" + conn = lancedb.connect( + uri, + api_key=os.environ.get("LANCEDB_API_KEY"), + region=os.environ.get("LANCEDB_REGION", "us-east-1"), + host_override=os.environ.get("LANCEDB_HOST_OVERRIDE"), + ) + name = f"json_fts_{uuid4().hex}" + table = conn.create_table( + name, + pa.table( + { + "id": [0, 1, 2], + "doc": pa.array( + [ + '{"meta":{"tag":"nature"},"author":"alice"}', + '{"meta":{"tag":"city"},"author":"nature"}', + '{"meta":{"tag":"nature"},"author":"bob"}', + ], + type=pa.json_(), + ), + "raw": pa.array([b"nature"] * 3, type=pa.large_binary()), + } + ), + ) + try: + table.create_index("doc", config=FTS(), name="doc_fts") + table.wait_for_index(["doc_fts"], timeout=timedelta(minutes=5)) + stats = table.index_stats("doc_fts") + assert stats.num_indexed_rows == 3 + assert stats.num_unindexed_rows == 0 + assert stats.index_type == "FTS" + for query, expected in [ + ("meta.tag,str,nature", [0, 2]), + ("meta.tag,str,alice", []), + ("meta.tag,str,tag", []), + ]: + search = table.search(query, query_type="fts", fts_columns="doc") + assert sorted(row["id"] for row in search.to_list()) == expected + plan = search.explain_plan(True) + assert "MatchQuery" in plan, plan + assert "FlatMatchQuery" not in plan, plan + with pytest.raises(ValueError, match="FTS.*lance.json.*raw binary"): + table.create_index("raw", config=FTS()) + finally: + conn.drop_table(name) + + @pytest.mark.skip(reason="Need to set up a local server") def test_against_local_server(): conn = LanceDBConnection("lancedb+http://localhost:10024") diff --git a/python/python/tests/test_fts.py b/python/python/tests/test_fts.py index e3963559d..d7fa2e0ec 100644 --- a/python/python/tests/test_fts.py +++ b/python/python/tests/test_fts.py @@ -45,6 +45,75 @@ from utils import exception_output TEST_LANGUAGE_MODEL_HOME = Path(__file__).parent / "models" +@pytest.mark.skipif(not hasattr(pa, "json_"), reason="requires PyArrow JSON type") +@pytest.mark.parametrize("block_size", [128, 256]) +def test_json_fts(tmp_path, block_size): + db = ldb.connect(tmp_path) + data = pa.table( + { + "id": [0, 1, 2, 3], + "doc": pa.array( + [ + '{"title":"brown fox","meta":{"tag":"nature"},"author":"alice"}', + '{"title":"blue car","meta":{"tag":"city"}}', + '{"title":"brown bear","meta":{"tag":"nature"}}', + '{"title":"red fox","author":"alice"}', + ], + type=pa.json_(), + ), + } + ) + table = db.create_table("json_fts", data) + table.create_index("doc", config=FTS(block_size=block_size), name="doc_fts") + table = db.open_table("json_fts") + index = table.list_indices()[0] + assert index.name == "doc_fts" + assert index.index_type == "FTS" + assert index.columns == ["doc"] + stats = table.index_stats("doc_fts") + assert stats.num_indexed_rows == 4 + assert stats.num_unindexed_rows == 0 + + for query, expected in [ + ("title,str,brown", [0, 2]), + ("meta.tag,str,nature", [0, 2]), + ("author,str,alice", [0, 3]), + ("title,str,alice", []), + ("title,str,title", []), + ]: + search = table.search(query, query_type="fts", fts_columns="doc") + assert sorted(row["id"] for row in search.to_list()) == expected + plan = search.explain_plan(True) + assert "MatchQuery" in plan, plan + assert "FlatMatchQuery" not in plan, plan + + +@pytest.mark.skipif(not hasattr(pa, "json_"), reason="requires PyArrow JSON type") +@pytest.mark.asyncio +async def test_json_fts_async(tmp_path): + db = await ldb.connect_async(tmp_path) + table = await db.create_table( + "json_fts", + pa.table({"doc": pa.array(['{"meta":{"tag":"nature"}}'], type=pa.json_())}), + ) + await table.create_index("doc", config=FTS()) + search = table.query().nearest_to_text(MatchQuery("meta.tag,str,nature", "doc")) + assert len(await search.to_list()) == 1 + plan = await search.explain_plan(True) + assert "MatchQuery" in plan, plan + assert "FlatMatchQuery" not in plan, plan + + +@pytest.mark.parametrize("dtype", [pa.binary(), pa.large_binary()]) +def test_fts_rejects_raw_binary(tmp_path, dtype): + table = ldb.connect(tmp_path).create_table( + "binary_fts", pa.table({"doc": pa.array([b'{"title":"brown"}'], type=dtype)}) + ) + with pytest.raises(ValueError, match="FTS.*lance.json.*raw binary"): + table.create_index("doc", config=FTS()) + assert table.list_indices() == [] + + @pytest.fixture def table(tmp_path) -> ldb.table.LanceTable: # Use local random state to avoid affecting other tests diff --git a/python/python/tests/test_remote_db.py b/python/python/tests/test_remote_db.py index 00e575e46..81fb3e10e 100644 --- a/python/python/tests/test_remote_db.py +++ b/python/python/tests/test_remote_db.py @@ -1410,6 +1410,72 @@ def test_remote_job_wait_raises_on_failure(): job.wait() +@pytest.mark.parametrize("server_version", ["0.2.0", "0.6.0"]) +def test_remote_json_fts_config(server_version): + from lancedb.index import FTS + + received_requests = [] + + def handler(request): + request.send_response(200) + request.send_header("Content-Type", "application/json") + request.send_header("phalanx-version", server_version) + request.end_headers() + if request.path == "/v1/table/test/describe/": + fields = [ + { + "name": "doc", + "type": {"type": "large_binary"}, + "nullable": True, + "metadata": {"ARROW:extension:name": "lance.json"}, + }, + { + "name": "raw", + "type": {"type": "large_binary"}, + "nullable": True, + }, + ] + request.wfile.write( + json.dumps({"version": 1, "schema": {"fields": fields}}).encode() + ) + elif request.path == "/v1/table/test/create_index/": + received_requests.append(read_json_body(request)) + request.wfile.write(b"{}") + else: + raise AssertionError(f"Unexpected request: {request.path}") + + with mock_lancedb_connection(handler) as db: + table = db.open_table("test") + table.create_index( + "doc", + config=FTS( + with_position=True, + base_tokenizer="whitespace", + stem=False, + remove_stop_words=False, + lower_case=False, + block_size=256, + ), + name="doc_fts", + replace=False, + ) + with pytest.raises(ValueError, match="FTS.*lance.json.*raw binary"): + table.create_index("raw", config=FTS()) + + assert len(received_requests) == 1 + request = received_requests[0] + assert request["column"] == "doc" + assert request["index_type"] == "FTS" + assert request["name"] == "doc_fts" + assert request["replace"] is False + assert request["with_position"] is True + assert request["base_tokenizer"] == "whitespace" + assert request["stem"] is False + assert request["remove_stop_words"] is False + assert request["lower_case"] is False + assert request["block_size"] == 256 + + def test_remote_create_index_new_api(caplog): received_requests = [] diff --git a/rust/lancedb/src/index.rs b/rust/lancedb/src/index.rs index cdc973044..82f8ad845 100644 --- a/rust/lancedb/src/index.rs +++ b/rust/lancedb/src/index.rs @@ -75,6 +75,17 @@ pub enum Index { /// Full text search index using BM25. /// + /// Supports strings, lists of strings, and `lance.json` logical fields + /// stored as `LargeBinary`. Raw binary fields are not supported. Lance + /// selects its JSON-aware tokenizer from the field metadata. + /// + /// JSON queries use `path,str,value` triples, where the dotted path is + /// relative to the indexed column. For example, `meta.tag,str,nature` + /// searches the string value at `$.meta.tag`, not other paths or key names. + /// Create the index on the JSON column, not on a JSON subpath. Plain text + /// queries are invalid for this tokenizer; malformed triples can panic in + /// the current Lance dependency (see [lance#8812](https://github.com/lance-format/lance/issues/8812)). + /// /// The posting block size defaults to 128. Supported values are 128 and 256; /// a value of 256 uses the experimental FTS V3 format and may introduce /// breaking changes. diff --git a/rust/lancedb/src/remote/db.rs b/rust/lancedb/src/remote/db.rs index 1ea11a23d..c6462a834 100644 --- a/rust/lancedb/src/remote/db.rs +++ b/rust/lancedb/src/remote/db.rs @@ -1,7 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The LanceDB Authors -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::sync::Arc; use async_trait::async_trait; diff --git a/rust/lancedb/src/remote/table.rs b/rust/lancedb/src/remote/table.rs index a17aadc9f..f0d1e554d 100644 --- a/rust/lancedb/src/remote/table.rs +++ b/rust/lancedb/src/remote/table.rs @@ -47,6 +47,7 @@ use crate::utils::background_cache::BackgroundCache; use crate::utils::{ MaxBatchLengthStream, TimeoutStream, public_fts_field_path_by_id, resolve_arrow_field_path, resolve_arrow_fts_field_path, supported_btree_data_type, supported_vector_data_type, + validate_fts_field, }; use crate::{DistanceType, Error}; use crate::{ @@ -597,6 +598,7 @@ impl RemoteTable { Index::BloomFilter(p) => ("BLOOM_FILTER", Some(to_json(p)?)), Index::RTree(p) => ("RTREE", Some(to_json(p)?)), Index::FTS(p) => { + validate_fts_field(&field)?; let mut params = to_json(p)?; if p.get_document_granularity().is_list_element() { params["document_granularity"] = "list_element".into(); @@ -7083,19 +7085,24 @@ mod tests { ]; for (index_type, expected_body, index) in cases { + let data_type = match &index { + Index::FTS(params) if params.get_document_granularity().is_list_element() => { + DataType::List(Arc::new(Field::new("item", DataType::Utf8, true))) + } + Index::FTS(_) => DataType::Utf8, + _ => DataType::Int32, + }; + let schema = Schema::new(vec![Field::new("a", data_type, false)]); let table = Table::new_with_handler_version( "my_table", semver::Version::new(0, 6, 0), move |request| { assert_eq!(request.method(), "POST"); match request.url().path() { - "/v1/table/my_table/describe/" => { - let schema = Schema::new(vec![Field::new("a", DataType::Int32, false)]); - http::Response::builder() - .status(200) - .body(describe_response(&schema)) - .unwrap() - } + "/v1/table/my_table/describe/" => http::Response::builder() + .status(200) + .body(describe_response(&schema)) + .unwrap(), "/v1/table/my_table/create_index/" => { assert_eq!( request.headers().get("Content-Type").unwrap(), diff --git a/rust/lancedb/src/table/create_index.rs b/rust/lancedb/src/table/create_index.rs index 916b28ed1..3ac759bdd 100644 --- a/rust/lancedb/src/table/create_index.rs +++ b/rust/lancedb/src/table/create_index.rs @@ -29,8 +29,8 @@ use crate::index::Index; use crate::index::vector::{VectorIndex, suggested_num_sub_vectors}; use crate::utils::{ resolve_lance_fts_field_path, supported_bitmap_data_type, supported_btree_data_type, - supported_fm_data_type, supported_fts_data_type, supported_label_list_data_type, - supported_vector_data_type, supported_zonemap_data_type, + supported_fm_data_type, supported_label_list_data_type, supported_vector_data_type, + supported_zonemap_data_type, validate_fts_field, }; use super::NativeTable; @@ -292,7 +292,7 @@ impl NativeTable { )) } Index::FTS(fts_opts) => { - Self::validate_index_type(field, "FTS", supported_fts_data_type)?; + validate_fts_field(field)?; Ok(Box::new(fts_opts)) } Index::IvfFlat(index) => { diff --git a/rust/lancedb/src/utils/mod.rs b/rust/lancedb/src/utils/mod.rs index 52b7bd5e8..7692f9bb5 100644 --- a/rust/lancedb/src/utils/mod.rs +++ b/rust/lancedb/src/utils/mod.rs @@ -494,6 +494,26 @@ pub fn supported_fts_data_type(dtype: &DataType) -> bool { supported_fts_data_type_impl(dtype, false) } +/// Validate FTS input without losing the logical type in field metadata. +pub(crate) fn validate_fts_field(field: &Field) -> Result<()> { + if lance_arrow::json::is_json_field(field) || supported_fts_data_type(field.data_type()) { + return Ok(()); + } + let logical_type = field + .metadata() + .get("ARROW:extension:name") + .map(|name| format!(" (logical type {name})")) + .unwrap_or_default(); + Err(Error::Schema { + message: format!( + "A FTS index cannot be created on the field `{}` which has data type {}{}. FTS supports strings, lists of strings, and lance.json fields stored as LargeBinary; raw binary is not supported", + field.name(), + field.data_type(), + logical_type, + ), + }) +} + fn supported_fts_data_type_impl(dtype: &DataType, in_list: bool) -> bool { match (dtype, in_list) { (DataType::Utf8 | DataType::LargeUtf8, _) => true, @@ -707,6 +727,43 @@ mod tests { use super::*; + #[test] + fn test_fts_field_logical_type_validation() { + let json = lance_arrow::json::json_field("doc", true); + assert!(validate_fts_field(&json).is_ok()); + + for dtype in [DataType::Binary, DataType::LargeBinary] { + let raw = Field::new("doc", dtype.clone(), true); + let err = validate_fts_field(&raw).unwrap_err().to_string(); + assert!(err.contains("raw binary is not supported"), "{err}"); + assert!(err.contains("lance.json"), "{err}"); + + let other_extension = raw.with_metadata(std::collections::HashMap::from([( + "ARROW:extension:name".into(), + "other.json".into(), + )])); + let err = validate_fts_field(&other_extension) + .unwrap_err() + .to_string(); + assert!(err.contains("logical type other.json"), "{err}"); + } + + // A JSON marker on the wrong binary storage type is insufficient. + let wrong_storage = + Field::new("doc", DataType::Binary, true).with_metadata(json.metadata().clone()); + assert!(validate_fts_field(&wrong_storage).is_err()); + for dtype in [DataType::Utf8, DataType::LargeUtf8] { + let text = Field::new("text", dtype, true); + assert!(validate_fts_field(&text).is_ok()); + for list_type in [ + DataType::List(Arc::new(text.clone())), + DataType::LargeList(Arc::new(text.clone())), + ] { + assert!(validate_fts_field(&Field::new("texts", list_type, true)).is_ok()); + } + } + } + #[test] fn test_public_fts_field_path_prefers_exact_case() { let text_list = || {