diff --git a/docs/src/js/classes/Table.md b/docs/src/js/classes/Table.md index b41ade302..7cf87c405 100644 --- a/docs/src/js/classes/Table.md +++ b/docs/src/js/classes/Table.md @@ -1298,7 +1298,11 @@ Specify exactly one of `column` or `indexName`. Model-backed tokenizers such as `jieba/*` and `lindera/*` are rebuilt in the client process from index metadata. For remote tables, this means the -same tokenizer model files must also exist locally. +same tokenizer model files must also exist locally (under +`LANCE_LANGUAGE_MODEL_HOME`) and must match the files the server uses. A +mismatch cannot be detected: the returned tokens may differ from what the +server's index produces. If the model files are missing, an error naming +the tokenizer is raised. #### Parameters diff --git a/nodejs/lancedb/table.ts b/nodejs/lancedb/table.ts index 0c79e6a7e..4ae1aefb8 100644 --- a/nodejs/lancedb/table.ts +++ b/nodejs/lancedb/table.ts @@ -989,7 +989,11 @@ export abstract class Table { * * Model-backed tokenizers such as `jieba/*` and `lindera/*` are rebuilt in * the client process from index metadata. For remote tables, this means the - * same tokenizer model files must also exist locally. + * same tokenizer model files must also exist locally (under + * `LANCE_LANGUAGE_MODEL_HOME`) and must match the files the server uses. A + * mismatch cannot be detected: the returned tokens may differ from what the + * server's index produces. If the model files are missing, an error naming + * the tokenizer is raised. */ abstract tokenize( query: string, diff --git a/python/python/lancedb/remote/table.py b/python/python/lancedb/remote/table.py index 96b988911..86e8fb8a0 100644 --- a/python/python/lancedb/remote/table.py +++ b/python/python/lancedb/remote/table.py @@ -279,8 +279,12 @@ class RemoteTable(Table): """Tokenize a query using the tokenizer configured on an FTS index. Model-backed tokenizers such as ``jieba/*`` and ``lindera/*`` are - rebuilt in the client process from index metadata, so the same tokenizer - model files must exist locally. + rebuilt in the client process from index metadata. For remote tables, + this means the same tokenizer model files must also exist locally + (under ``LANCE_LANGUAGE_MODEL_HOME``) and must match the files the + server uses. A mismatch cannot be detected: the returned tokens may + differ from what the server's index produces. If the model files are + missing, an error naming the tokenizer is raised. """ return LOOP.run( self._table.tokenize(query, column=column, index_name=index_name) diff --git a/python/python/lancedb/table.py b/python/python/lancedb/table.py index f9cfac6e1..124fa34b5 100644 --- a/python/python/lancedb/table.py +++ b/python/python/lancedb/table.py @@ -2558,7 +2558,11 @@ class Table(ABC): Model-backed tokenizers such as ``jieba/*`` and ``lindera/*`` are rebuilt in the client process from index metadata. For remote tables, - this means the same tokenizer model files must also exist locally. + this means the same tokenizer model files must also exist locally + (under ``LANCE_LANGUAGE_MODEL_HOME``) and must match the files the + server uses. A mismatch cannot be detected: the returned tokens may + differ from what the server's index produces. If the model files are + missing, an error naming the tokenizer is raised. """ @abstractmethod @@ -4813,7 +4817,11 @@ class LanceTable(Table): Model-backed tokenizers such as ``jieba/*`` and ``lindera/*`` are rebuilt in the client process from index metadata. For remote tables, - this means the same tokenizer model files must also exist locally. + this means the same tokenizer model files must also exist locally + (under ``LANCE_LANGUAGE_MODEL_HOME``) and must match the files the + server uses. A mismatch cannot be detected: the returned tokens may + differ from what the server's index produces. If the model files are + missing, an error naming the tokenizer is raised. """ return LOOP.run( self._table.tokenize(query, column=column, index_name=index_name) @@ -7505,7 +7513,11 @@ class AsyncTable: Model-backed tokenizers such as ``jieba/*`` and ``lindera/*`` are rebuilt in the client process from index metadata. For remote tables, - this means the same tokenizer model files must also exist locally. + this means the same tokenizer model files must also exist locally + (under ``LANCE_LANGUAGE_MODEL_HOME``) and must match the files the + server uses. A mismatch cannot be detected: the returned tokens may + differ from what the server's index produces. If the model files are + missing, an error naming the tokenizer is raised. """ return await self._inner.tokenize(query, column=column, index_name=index_name) diff --git a/python/python/tests/test_remote_db.py b/python/python/tests/test_remote_db.py index 80bd10289..a45e1aaef 100644 --- a/python/python/tests/test_remote_db.py +++ b/python/python/tests/test_remote_db.py @@ -1793,6 +1793,79 @@ def test_table_wait_for_index_timeout(): table.wait_for_index(["id_idx"], timedelta(seconds=1)) +def _jieba_fts_handler(request): + """Serve a table with a ``jieba/default`` FTS index and no other state.""" + if request.path == "/v1/table/test/describe/": + body = { + "version": 1, + "schema": { + "fields": [ + {"name": "text", "type": {"type": "string"}, "nullable": True} + ] + }, + } + elif request.path == "/v1/table/test/index/list/": + index_details = { + "base_tokenizer": "jieba/default", + "language": "English", + "with_position": False, + "max_token_length": 40, + "lower_case": True, + "stem": False, + "remove_stop_words": False, + "ascii_folding": False, + } + body = { + "indexes": [ + { + "index_name": "text_idx", + "columns": ["text"], + "index_type": "FTS", + "index_details": json.dumps(index_details), + } + ] + } + else: + request.send_response(404) + request.end_headers() + return + request.send_response(200) + request.send_header("Content-Type", "application/json") + request.end_headers() + request.wfile.write(json.dumps(body).encode()) + + +_MODEL_FILES_ERROR = ( + r"(?s)Invalid directory path.*" + r"full text search index 'text_idx'.*" + r"tokenize\(\) runs on the client.*'jieba/default'.*" + r"LANCE_LANGUAGE_MODEL_HOME.*match the files the server uses" +) + + +def test_remote_tokenize_model_backed_without_model_files(tmp_path, monkeypatch): + monkeypatch.setenv("LANCE_LANGUAGE_MODEL_HOME", str(tmp_path)) + with mock_lancedb_connection(_jieba_fts_handler) as db: + table = db.open_table("test") + with pytest.raises(ValueError, match=_MODEL_FILES_ERROR): + table.tokenize("我们是中国人", index_name="text_idx") + with pytest.raises(ValueError, match=_MODEL_FILES_ERROR): + table.tokenize("我们是中国人", column="text") + + +@pytest.mark.asyncio +async def test_remote_tokenize_model_backed_without_model_files_async( + tmp_path, monkeypatch +): + monkeypatch.setenv("LANCE_LANGUAGE_MODEL_HOME", str(tmp_path)) + async with mock_lancedb_connection_async(_jieba_fts_handler) as db: + table = await db.open_table("test") + with pytest.raises(ValueError, match=_MODEL_FILES_ERROR): + await table.tokenize("我们是中国人", index_name="text_idx") + with pytest.raises(ValueError, match=_MODEL_FILES_ERROR): + await table.tokenize("我们是中国人", column="text") + + def test_stats(): stats = { "total_bytes": 38, diff --git a/rust/lancedb/src/remote/table.rs b/rust/lancedb/src/remote/table.rs index 84c1b3fef..b722dc4a1 100644 --- a/rust/lancedb/src/remote/table.rs +++ b/rust/lancedb/src/remote/table.rs @@ -8871,6 +8871,131 @@ mod tests { ); } + #[tokio::test] + #[serial_test::serial(language_model_env)] + async fn test_tokenize_model_backed_tokenizer_without_model_files() { + let model_home = tempfile::tempdir().unwrap(); + // SAFETY: This is only called in tests, serialized on the env key. + unsafe { + std::env::set_var("LANCE_LANGUAGE_MODEL_HOME", model_home.path()); + } + let schema = Schema::new(vec![Field::new("text", DataType::Utf8, false)]); + let index_details = serde_json::json!({ + "base_tokenizer": "jieba/default", + "language": "English", + "with_position": false, + "max_token_length": 40, + "lower_case": true, + "stem": false, + "remove_stop_words": false, + "ascii_folding": false, + }) + .to_string(); + let table = Table::new_with_handler("my_table", move |request| { + assert_eq!(request.method(), "POST"); + match request.url().path() { + "/v1/table/my_table/describe/" => http::Response::builder() + .status(200) + .body(describe_response(&schema)) + .unwrap(), + "/v1/table/my_table/index/list/" => { + let body = serde_json::json!({ + "indexes": [ + { + "index_name": "text_idx", + "columns": ["text"], + "index_type": "FTS", + "index_details": index_details, + }, + ] + }); + http::Response::builder() + .status(200) + .body(serde_json::to_string(&body).unwrap()) + .unwrap() + } + path => panic!("Unexpected path: {}", path), + } + }); + + let result = table.tokenize("我们是中国人", "text_idx").await; + // SAFETY: This is only called in tests, serialized on the env key. + unsafe { + std::env::remove_var("LANCE_LANGUAGE_MODEL_HOME"); + } + + let err = result.unwrap_err(); + let Error::InvalidInput { message } = err else { + panic!("expected InvalidInput, got {err:?}"); + }; + // The original cause and the index context are preserved... + assert!(message.contains("Invalid directory path"), "{message}"); + assert!( + message.contains("full text search index 'text_idx'"), + "{message}" + ); + // ...and the message explains that this is a client-side limitation. + assert!( + message.contains("tokenize() runs on the client"), + "{message}" + ); + assert!(message.contains("'jieba/default'"), "{message}"); + assert!(message.contains("LANCE_LANGUAGE_MODEL_HOME"), "{message}"); + assert!( + message.contains("match the files the server uses"), + "{message}" + ); + } + + #[tokio::test] + async fn test_tokenize_unknown_tokenizer_error_has_no_model_files_hint() { + let schema = Schema::new(vec![Field::new("text", DataType::Utf8, false)]); + let index_details = serde_json::json!({ + "base_tokenizer": "no_such_tokenizer", + "language": "English", + "with_position": false, + "max_token_length": 40, + "lower_case": true, + "stem": false, + "remove_stop_words": false, + "ascii_folding": false, + }) + .to_string(); + let table = Table::new_with_handler("my_table", move |request| { + assert_eq!(request.method(), "POST"); + match request.url().path() { + "/v1/table/my_table/describe/" => http::Response::builder() + .status(200) + .body(describe_response(&schema)) + .unwrap(), + "/v1/table/my_table/index/list/" => { + let body = serde_json::json!({ + "indexes": [ + { + "index_name": "text_idx", + "columns": ["text"], + "index_type": "FTS", + "index_details": index_details, + }, + ] + }); + http::Response::builder() + .status(200) + .body(serde_json::to_string(&body).unwrap()) + .unwrap() + } + path => panic!("Unexpected path: {}", path), + } + }); + + let err = table.tokenize("hello", "text_idx").await.unwrap_err(); + let message = err.to_string(); + assert!( + !message.contains("tokenize() runs on the client"), + "{message}" + ); + } + #[tokio::test] async fn test_tokenize_requires_existing_index_name() { let schema = Schema::new(vec![Field::new("text", DataType::Utf8, false)]); diff --git a/rust/lancedb/src/table.rs b/rust/lancedb/src/table.rs index 39e6f1911..3123eb776 100644 --- a/rust/lancedb/src/table.rs +++ b/rust/lancedb/src/table.rs @@ -2264,7 +2264,11 @@ impl Table { /// /// Model-backed tokenizers such as `jieba/*` and `lindera/*` are rebuilt in /// the client process from index metadata. For remote tables, this means the - /// same tokenizer model files must also exist locally. + /// same tokenizer model files must also exist locally, and must match the + /// files the server uses: the client cannot detect a mismatch, so it may + /// return tokens the server's index never produces. If the model files are + /// missing, this returns an error naming the tokenizer and + /// `LANCE_LANGUAGE_MODEL_HOME`. pub async fn tokenize(&self, query: &str, index_name: &str) -> Result> { let indices = self.inner.list_indices().await?; let matches = indices @@ -2298,7 +2302,8 @@ impl Table { /// The column must have exactly one FTS index. Model-backed tokenizers such /// as `jieba/*` and `lindera/*` are rebuilt in the client process from /// index metadata. For remote tables, this means the same tokenizer model - /// files must also exist locally. + /// files must also exist locally, and must match the files the server uses + /// (see [`Self::tokenize`]). pub async fn tokenize_with_column(&self, query: &str, column: &str) -> Result> { let schema = self.inner.schema().await?; let (column, _) = resolve_arrow_field_path(schema.as_ref(), column)?; @@ -2354,13 +2359,30 @@ impl Table { ), } })?; + let model_backed_tokenizer = serde_json::from_str::(details) + .ok() + .and_then(|details| { + let tokenizer = details.get("base_tokenizer")?.as_str()?.to_string(); + (tokenizer.starts_with("jieba/") || tokenizer.starts_with("lindera/")) + .then_some(tokenizer) + }); tokenize(query, ¶ms).map_err(|err| match err { - Error::InvalidInput { message } => Error::InvalidInput { - message: format!( + Error::InvalidInput { message } => { + let mut message = format!( "{} for full text search index '{}' for {}", message, index.name, selector_description - ), - }, + ); + if let Some(tokenizer) = model_backed_tokenizer { + message.push_str(&format!( + ". tokenize() runs on the client: the '{}' tokenizer is rebuilt \ + in this process from the index metadata, so its model files must \ + be available under LANCE_LANGUAGE_MODEL_HOME and, for remote \ + tables, match the files the server uses", + tokenizer + )); + } + Error::InvalidInput { message } + } err => err, }) }