fix(rust): explain client-side model files when remote tokenize fails (#4519)

Addresses option 2 of #4440.

tokenize() builds the FTS analyzer on the client. For jieba/* and
lindera/* that needs model files under LANCE_LANGUAGE_MODEL_HOME, and
when they’re missing you only got lance’s raw Invalid directory path:
... error, even though server-side search works.

Now, if a jieba/* or lindera/* tokenizer can’t be built, the error keeps
the original cause and adds that tokenize() runs on the client, which
directory the files need to be in, and that for remote tables they must
match the server’s. I also updated the tokenize docs (Rust, Python, TS,
generated JS docs) to say a mismatch can’t be detected, so tokens may
differ from the server’s index.

Behavior when the files exist is unchanged. Detecting a mismatch needs a
server-side tokenize endpoint (option 1), so I left it out. The hint
isn’t limited to remote tables, since Table::as_native is slated for
removal and the wording fits local tables too.

Tests: Rust test with mocked jieba/default index_details and an empty
model dir, one checking other tokenizer errors don’t get the hint, and
Python sync/async tests.

docs/src/js/classes/Table.md

@@ -1298,7 +1298,11 @@ Specify exactly one of `column` or `indexName`.
 
 Model-backed tokenizers such as `jieba/*` and `lindera/*` are rebuilt in
 the client process from index metadata. For remote tables, this means the
-same tokenizer model files must also exist locally.
+same tokenizer model files must also exist locally (under
+`LANCE_LANGUAGE_MODEL_HOME`) and must match the files the server uses. A
+mismatch cannot be detected: the returned tokens may differ from what the
+server's index produces. If the model files are missing, an error naming
+the tokenizer is raised.
 
 #### Parameters
 

nodejs/lancedb/table.ts

@@ -989,7 +989,11 @@ export abstract class Table {
    *
    * Model-backed tokenizers such as `jieba/*` and `lindera/*` are rebuilt in
    * the client process from index metadata. For remote tables, this means the
-   * same tokenizer model files must also exist locally.
+   * same tokenizer model files must also exist locally (under
+   * `LANCE_LANGUAGE_MODEL_HOME`) and must match the files the server uses. A
+   * mismatch cannot be detected: the returned tokens may differ from what the
+   * server's index produces. If the model files are missing, an error naming
+   * the tokenizer is raised.
    */
   abstract tokenize(
     query: string,

python/python/lancedb/remote/table.py

@@ -279,8 +279,12 @@ class RemoteTable(Table):
         """Tokenize a query using the tokenizer configured on an FTS index.
 
         Model-backed tokenizers such as ``jieba/*`` and ``lindera/*`` are
-        rebuilt in the client process from index metadata, so the same tokenizer
-        model files must exist locally.
+        rebuilt in the client process from index metadata. For remote tables,
+        this means the same tokenizer model files must also exist locally
+        (under ``LANCE_LANGUAGE_MODEL_HOME``) and must match the files the
+        server uses. A mismatch cannot be detected: the returned tokens may
+        differ from what the server's index produces. If the model files are
+        missing, an error naming the tokenizer is raised.
         """
         return LOOP.run(
             self._table.tokenize(query, column=column, index_name=index_name)

python/python/lancedb/table.py

@@ -2558,7 +2558,11 @@ class Table(ABC):
 
         Model-backed tokenizers such as ``jieba/*`` and ``lindera/*`` are
         rebuilt in the client process from index metadata. For remote tables,
-        this means the same tokenizer model files must also exist locally.
+        this means the same tokenizer model files must also exist locally
+        (under ``LANCE_LANGUAGE_MODEL_HOME``) and must match the files the
+        server uses. A mismatch cannot be detected: the returned tokens may
+        differ from what the server's index produces. If the model files are
+        missing, an error naming the tokenizer is raised.
         """
 
     @abstractmethod
@@ -4813,7 +4817,11 @@ class LanceTable(Table):
 
         Model-backed tokenizers such as ``jieba/*`` and ``lindera/*`` are
         rebuilt in the client process from index metadata. For remote tables,
-        this means the same tokenizer model files must also exist locally.
+        this means the same tokenizer model files must also exist locally
+        (under ``LANCE_LANGUAGE_MODEL_HOME``) and must match the files the
+        server uses. A mismatch cannot be detected: the returned tokens may
+        differ from what the server's index produces. If the model files are
+        missing, an error naming the tokenizer is raised.
         """
         return LOOP.run(
             self._table.tokenize(query, column=column, index_name=index_name)
@@ -7505,7 +7513,11 @@ class AsyncTable:
 
         Model-backed tokenizers such as ``jieba/*`` and ``lindera/*`` are
         rebuilt in the client process from index metadata. For remote tables,
-        this means the same tokenizer model files must also exist locally.
+        this means the same tokenizer model files must also exist locally
+        (under ``LANCE_LANGUAGE_MODEL_HOME``) and must match the files the
+        server uses. A mismatch cannot be detected: the returned tokens may
+        differ from what the server's index produces. If the model files are
+        missing, an error naming the tokenizer is raised.
         """
         return await self._inner.tokenize(query, column=column, index_name=index_name)
 

python/python/tests/test_remote_db.py

@@ -1793,6 +1793,79 @@ def test_table_wait_for_index_timeout():
             table.wait_for_index(["id_idx"], timedelta(seconds=1))
 
 
+def _jieba_fts_handler(request):
+    """Serve a table with a ``jieba/default`` FTS index and no other state."""
+    if request.path == "/v1/table/test/describe/":
+        body = {
+            "version": 1,
+            "schema": {
+                "fields": [
+                    {"name": "text", "type": {"type": "string"}, "nullable": True}
+                ]
+            },
+        }
+    elif request.path == "/v1/table/test/index/list/":
+        index_details = {
+            "base_tokenizer": "jieba/default",
+            "language": "English",
+            "with_position": False,
+            "max_token_length": 40,
+            "lower_case": True,
+            "stem": False,
+            "remove_stop_words": False,
+            "ascii_folding": False,
+        }
+        body = {
+            "indexes": [
+                {
+                    "index_name": "text_idx",
+                    "columns": ["text"],
+                    "index_type": "FTS",
+                    "index_details": json.dumps(index_details),
+                }
+            ]
+        }
+    else:
+        request.send_response(404)
+        request.end_headers()
+        return
+    request.send_response(200)
+    request.send_header("Content-Type", "application/json")
+    request.end_headers()
+    request.wfile.write(json.dumps(body).encode())
+
+
+_MODEL_FILES_ERROR = (
+    r"(?s)Invalid directory path.*"
+    r"full text search index 'text_idx'.*"
+    r"tokenize\(\) runs on the client.*'jieba/default'.*"
+    r"LANCE_LANGUAGE_MODEL_HOME.*match the files the server uses"
+)
+
+
+def test_remote_tokenize_model_backed_without_model_files(tmp_path, monkeypatch):
+    monkeypatch.setenv("LANCE_LANGUAGE_MODEL_HOME", str(tmp_path))
+    with mock_lancedb_connection(_jieba_fts_handler) as db:
+        table = db.open_table("test")
+        with pytest.raises(ValueError, match=_MODEL_FILES_ERROR):
+            table.tokenize("我们是中国人", index_name="text_idx")
+        with pytest.raises(ValueError, match=_MODEL_FILES_ERROR):
+            table.tokenize("我们是中国人", column="text")
+
+
+@pytest.mark.asyncio
+async def test_remote_tokenize_model_backed_without_model_files_async(
+    tmp_path, monkeypatch
+):
+    monkeypatch.setenv("LANCE_LANGUAGE_MODEL_HOME", str(tmp_path))
+    async with mock_lancedb_connection_async(_jieba_fts_handler) as db:
+        table = await db.open_table("test")
+        with pytest.raises(ValueError, match=_MODEL_FILES_ERROR):
+            await table.tokenize("我们是中国人", index_name="text_idx")
+        with pytest.raises(ValueError, match=_MODEL_FILES_ERROR):
+            await table.tokenize("我们是中国人", column="text")
+
+
 def test_stats():
     stats = {
         "total_bytes": 38,

rust/lancedb/src/remote/table.rs

@@ -8871,6 +8871,131 @@ mod tests {
         );
     }
 
+    #[tokio::test]
+    #[serial_test::serial(language_model_env)]
+    async fn test_tokenize_model_backed_tokenizer_without_model_files() {
+        let model_home = tempfile::tempdir().unwrap();
+        // SAFETY: This is only called in tests, serialized on the env key.
+        unsafe {
+            std::env::set_var("LANCE_LANGUAGE_MODEL_HOME", model_home.path());
+        }
+        let schema = Schema::new(vec![Field::new("text", DataType::Utf8, false)]);
+        let index_details = serde_json::json!({
+            "base_tokenizer": "jieba/default",
+            "language": "English",
+            "with_position": false,
+            "max_token_length": 40,
+            "lower_case": true,
+            "stem": false,
+            "remove_stop_words": false,
+            "ascii_folding": false,
+        })
+        .to_string();
+        let table = Table::new_with_handler("my_table", move |request| {
+            assert_eq!(request.method(), "POST");
+            match request.url().path() {
+                "/v1/table/my_table/describe/" => http::Response::builder()
+                    .status(200)
+                    .body(describe_response(&schema))
+                    .unwrap(),
+                "/v1/table/my_table/index/list/" => {
+                    let body = serde_json::json!({
+                        "indexes": [
+                            {
+                                "index_name": "text_idx",
+                                "columns": ["text"],
+                                "index_type": "FTS",
+                                "index_details": index_details,
+                            },
+                        ]
+                    });
+                    http::Response::builder()
+                        .status(200)
+                        .body(serde_json::to_string(&body).unwrap())
+                        .unwrap()
+                }
+                path => panic!("Unexpected path: {}", path),
+            }
+        });
+
+        let result = table.tokenize("我们是中国人", "text_idx").await;
+        // SAFETY: This is only called in tests, serialized on the env key.
+        unsafe {
+            std::env::remove_var("LANCE_LANGUAGE_MODEL_HOME");
+        }
+
+        let err = result.unwrap_err();
+        let Error::InvalidInput { message } = err else {
+            panic!("expected InvalidInput, got {err:?}");
+        };
+        // The original cause and the index context are preserved...
+        assert!(message.contains("Invalid directory path"), "{message}");
+        assert!(
+            message.contains("full text search index 'text_idx'"),
+            "{message}"
+        );
+        // ...and the message explains that this is a client-side limitation.
+        assert!(
+            message.contains("tokenize() runs on the client"),
+            "{message}"
+        );
+        assert!(message.contains("'jieba/default'"), "{message}");
+        assert!(message.contains("LANCE_LANGUAGE_MODEL_HOME"), "{message}");
+        assert!(
+            message.contains("match the files the server uses"),
+            "{message}"
+        );
+    }
+
+    #[tokio::test]
+    async fn test_tokenize_unknown_tokenizer_error_has_no_model_files_hint() {
+        let schema = Schema::new(vec![Field::new("text", DataType::Utf8, false)]);
+        let index_details = serde_json::json!({
+            "base_tokenizer": "no_such_tokenizer",
+            "language": "English",
+            "with_position": false,
+            "max_token_length": 40,
+            "lower_case": true,
+            "stem": false,
+            "remove_stop_words": false,
+            "ascii_folding": false,
+        })
+        .to_string();
+        let table = Table::new_with_handler("my_table", move |request| {
+            assert_eq!(request.method(), "POST");
+            match request.url().path() {
+                "/v1/table/my_table/describe/" => http::Response::builder()
+                    .status(200)
+                    .body(describe_response(&schema))
+                    .unwrap(),
+                "/v1/table/my_table/index/list/" => {
+                    let body = serde_json::json!({
+                        "indexes": [
+                            {
+                                "index_name": "text_idx",
+                                "columns": ["text"],
+                                "index_type": "FTS",
+                                "index_details": index_details,
+                            },
+                        ]
+                    });
+                    http::Response::builder()
+                        .status(200)
+                        .body(serde_json::to_string(&body).unwrap())
+                        .unwrap()
+                }
+                path => panic!("Unexpected path: {}", path),
+            }
+        });
+
+        let err = table.tokenize("hello", "text_idx").await.unwrap_err();
+        let message = err.to_string();
+        assert!(
+            !message.contains("tokenize() runs on the client"),
+            "{message}"
+        );
+    }
+
     #[tokio::test]
     async fn test_tokenize_requires_existing_index_name() {
         let schema = Schema::new(vec![Field::new("text", DataType::Utf8, false)]);

rust/lancedb/src/table.rs

@@ -2264,7 +2264,11 @@ impl Table {
     ///
     /// Model-backed tokenizers such as `jieba/*` and `lindera/*` are rebuilt in
     /// the client process from index metadata. For remote tables, this means the
-    /// same tokenizer model files must also exist locally.
+    /// same tokenizer model files must also exist locally, and must match the
+    /// files the server uses: the client cannot detect a mismatch, so it may
+    /// return tokens the server's index never produces. If the model files are
+    /// missing, this returns an error naming the tokenizer and
+    /// `LANCE_LANGUAGE_MODEL_HOME`.
     pub async fn tokenize(&self, query: &str, index_name: &str) -> Result<Vec<FtsToken>> {
         let indices = self.inner.list_indices().await?;
         let matches = indices
@@ -2298,7 +2302,8 @@ impl Table {
     /// The column must have exactly one FTS index. Model-backed tokenizers such
     /// as `jieba/*` and `lindera/*` are rebuilt in the client process from
     /// index metadata. For remote tables, this means the same tokenizer model
-    /// files must also exist locally.
+    /// files must also exist locally, and must match the files the server uses
+    /// (see [`Self::tokenize`]).
     pub async fn tokenize_with_column(&self, query: &str, column: &str) -> Result<Vec<FtsToken>> {
         let schema = self.inner.schema().await?;
         let (column, _) = resolve_arrow_field_path(schema.as_ref(), column)?;
@@ -2354,13 +2359,30 @@ impl Table {
                 ),
             }
         })?;
+        let model_backed_tokenizer = serde_json::from_str::<serde_json::Value>(details)
+            .ok()
+            .and_then(|details| {
+                let tokenizer = details.get("base_tokenizer")?.as_str()?.to_string();
+                (tokenizer.starts_with("jieba/") || tokenizer.starts_with("lindera/"))
+                    .then_some(tokenizer)
+            });
         tokenize(query, &params).map_err(|err| match err {
-            Error::InvalidInput { message } => Error::InvalidInput {
-                message: format!(
+            Error::InvalidInput { message } => {
+                let mut message = format!(
                     "{} for full text search index '{}' for {}",
                     message, index.name, selector_description
-                ),
-            },
+                );
+                if let Some(tokenizer) = model_backed_tokenizer {
+                    message.push_str(&format!(
+                        ". tokenize() runs on the client: the '{}' tokenizer is rebuilt \
+                         in this process from the index metadata, so its model files must \
+                         be available under LANCE_LANGUAGE_MODEL_HOME and, for remote \
+                         tables, match the files the server uses",
+                        tokenizer
+                    ));
+                }
+                Error::InvalidInput { message }
+            }
             err => err,
         })
     }