From 51bdfd95f51199f8fa923617b93914b2771dde95 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ga=C3=A9tan?= <86427581+GaetanVDB07@users.noreply.github.com> Date: Tue, 3 Nov 2026 13:59:09 +0100 Subject: [PATCH] docs: clarify remote index build completion (#4436) On remote tables, `create_index` returns as soon as the build is scheduled. Until it finishes, a vector index is missing from `list_indices`, a scalar or FTS index shows 0 indexed rows, and queries don't use the index. The docs didn't say any of this, so it was easy to assume the index was ready. This adds that to the `create_index` and `list_indices` docs in Python (`Table` and `AsyncTable`) and TypeScript, and points to `wait_timeout` / `waitTimeoutSeconds`, `wait_for_index`, and `create_index_async` for blocking until the index is ready. It also explains that completion tracking is tied to the submitted build only when the server returns a job ID, while legacy name-based waits can return early when replacing an existing index. Closes #4405 --- docs/src/js/classes/Table.md | 28 +++++++++ docs/src/js/interfaces/IndexOptions.md | 5 +- nodejs/lancedb/indices.ts | 5 +- nodejs/lancedb/table.ts | 32 ++++++++++- python/python/lancedb/table.py | 78 +++++++++++++++++++++++++- 5 files changed, 143 insertions(+), 5 deletions(-) diff --git a/docs/src/js/classes/Table.md b/docs/src/js/classes/Table.md index a2a15838a..188769747 100644 --- a/docs/src/js/classes/Table.md +++ b/docs/src/js/classes/Table.md @@ -357,6 +357,24 @@ vector and non-vector searches) We currently don't support custom named indexes. The index name will always be `${column}_idx`. +On LanceDB Embedded tables the index is built before the returned promise +resolves. On LanceDB Enterprise tables it resolves once the build is +scheduled. Until the build finishes, a vector index may be missing from +[Table.listIndices](Table.md#listindices), a scalar or FTS index may be listed with 0 +indexed rows, and queries won't use the index. To wait until the index is +ready, set [IndexOptions.waitTimeoutSeconds](../interfaces/IndexOptions.md#waittimeoutseconds), call +[Table.waitForIndex](Table.md#waitforindex), or use [Table.createIndexAsync](Table.md#createindexasync) and wait +on its job when the server returned a job ID. + +`waitTimeoutSeconds` waits for the submitted build when the server returns +a job ID. On older servers that don't return one, it falls back to polling +the index by name. [Table.waitForIndex](Table.md#waitforindex) always waits by name. When +replacing an existing index, those name-based waits may return before the +new build finishes because the old index can still match. +`createIndexAsync` can track a specific build only when the server returned +a job ID; otherwise it returns an already-complete job and callers must +use [Table.waitForIndex](Table.md#waitforindex) for that build. + #### Parameters * **column**: `string` @@ -413,6 +431,11 @@ Create an index, returning a handle to the indexing job. The job may already be complete when returned; callers must not assume the index exists until [Job.wait](Job.md#wait) resolves. +In LanceDB Enterprise, the returned job tracks this submission only if the +server returns a job ID. If it does not, the job is already complete and +does not wait for the background build; use [Table.waitForIndex](Table.md#waitforindex) and +account for the replacement caveat described in [Table.createIndex](Table.md#createindex). + #### Parameters * **column**: `string` @@ -849,6 +872,11 @@ abstract listIndices(): Promise List all indices that have been created with [Table.createIndex](Table.md#createindex) +On LanceDB Enterprise tables, an index whose build hasn't finished may be +missing (vector indices) or listed with 0 indexed rows (scalar and FTS +indices), and `numUnindexedRows` may be undefined. Use +[Table.waitForIndex](Table.md#waitforindex) to wait until an index is ready. + #### Returns `Promise`<[`IndexConfig`](../interfaces/IndexConfig.md)[]> diff --git a/docs/src/js/interfaces/IndexOptions.md b/docs/src/js/interfaces/IndexOptions.md index eb1a10c8f..e0990fc8d 100644 --- a/docs/src/js/interfaces/IndexOptions.md +++ b/docs/src/js/interfaces/IndexOptions.md @@ -77,4 +77,7 @@ optional waitTimeoutSeconds: number; Timeout in seconds to wait for index creation to complete. -If not specified, the method will return immediately after starting the index creation. +Only applies to LanceDB Enterprise tables, which build the index in the +background. +If not specified, `createIndex` returns once the build is scheduled. +LanceDB Embedded tables always build the index before `createIndex` returns. diff --git a/nodejs/lancedb/indices.ts b/nodejs/lancedb/indices.ts index ea64fc712..53cbf8a4d 100644 --- a/nodejs/lancedb/indices.ts +++ b/nodejs/lancedb/indices.ts @@ -939,7 +939,10 @@ export interface IndexOptions { /** * Timeout in seconds to wait for index creation to complete. * - * If not specified, the method will return immediately after starting the index creation. + * Only applies to LanceDB Enterprise tables, which build the index in the + * background. + * If not specified, `createIndex` returns once the build is scheduled. + * LanceDB Embedded tables always build the index before `createIndex` returns. */ waitTimeoutSeconds?: number; diff --git a/nodejs/lancedb/table.ts b/nodejs/lancedb/table.ts index 3dbf914fe..4861bf3f0 100644 --- a/nodejs/lancedb/table.ts +++ b/nodejs/lancedb/table.ts @@ -365,6 +365,24 @@ export abstract class Table { * We currently don't support custom named indexes. * The index name will always be `${column}_idx`. * + * On LanceDB Embedded tables the index is built before the returned promise + * resolves. On LanceDB Enterprise tables it resolves once the build is + * scheduled. Until the build finishes, a vector index may be missing from + * {@link Table.listIndices}, a scalar or FTS index may be listed with 0 + * indexed rows, and queries won't use the index. To wait until the index is + * ready, set {@link IndexOptions.waitTimeoutSeconds}, call + * {@link Table.waitForIndex}, or use {@link Table.createIndexAsync} and wait + * on its job when the server returned a job ID. + * + * `waitTimeoutSeconds` waits for the submitted build when the server returns + * a job ID. On older servers that don't return one, it falls back to polling + * the index by name. {@link Table.waitForIndex} always waits by name. When + * replacing an existing index, those name-based waits may return before the + * new build finishes because the old index can still match. + * `createIndexAsync` can track a specific build only when the server returned + * a job ID; otherwise it returns an already-complete job and callers must + * use {@link Table.waitForIndex} for that build. + * * @example * // If the column has a vector (fixed size list) data type then * // an IvfPq vector index will be created. @@ -399,6 +417,11 @@ export abstract class Table { * * The job may already be complete when returned; callers must not assume * the index exists until {@link Job.wait} resolves. + * + * In LanceDB Enterprise, the returned job tracks this submission only if the + * server returns a job ID. If it does not, the job is already complete and + * does not wait for the background build; use {@link Table.waitForIndex} and + * account for the replacement caveat described in {@link Table.createIndex}. */ abstract createIndexAsync( column: string, @@ -998,7 +1021,14 @@ export abstract class Table { * modification operations. */ abstract optimize(options?: Partial): Promise; - /** List all indices that have been created with {@link Table.createIndex} */ + /** + * List all indices that have been created with {@link Table.createIndex} + * + * On LanceDB Enterprise tables, an index whose build hasn't finished may be + * missing (vector indices) or listed with 0 indexed rows (scalar and FTS + * indices), and `numUnindexedRows` may be undefined. Use + * {@link Table.waitForIndex} to wait until an index is ready. + */ abstract listIndices(): Promise; /** * Tokenize a full-text search query using the tokenizer configured on an FTS index. diff --git a/python/python/lancedb/table.py b/python/python/lancedb/table.py index 53b5a8971..e19f2e70f 100644 --- a/python/python/lancedb/table.py +++ b/python/python/lancedb/table.py @@ -1901,12 +1901,34 @@ class Table(ABC): replace : bool, default True Whether to replace an existing index on this column. wait_timeout : timedelta, optional - Timeout to wait for async indexing to complete. + LanceDB Enterprise only: how long to wait for the index build to finish. + If not set, the call returns once the build is scheduled. name : str, optional Custom name for the index. train : bool, default True Whether to train the index with existing data. + Notes + ----- + On LanceDB Embedded tables the index is built before this method + returns. On LanceDB Enterprise tables it returns once the build is + scheduled. Until the build finishes, a vector index may be missing + from [list_indices][lancedb.table.Table.list_indices], a scalar or FTS + index may be listed with 0 indexed rows, and queries won't use the + index. To block until the index is ready, pass ``wait_timeout``, call + [wait_for_index][lancedb.table.Table.wait_for_index], or use + [create_index_async][lancedb.table.Table.create_index_async] and wait + on its job when the server returned a job ID. + + ``wait_timeout`` waits for the submitted build when the server returns + a job ID. On older servers that don't return one, it falls back to + polling the index by name. ``wait_for_index`` always waits by name. + When replacing an existing index, those name-based waits may return + before the new build finishes because the old index can still match. + ``create_index_async`` can track a specific build only when the server + returned a job ID; otherwise it returns an already-complete job and + callers must use ``wait_for_index`` for that build. + Examples -------- New API (recommended): @@ -1940,6 +1962,12 @@ class Table(ABC): Takes the same arguments as :meth:`create_index`. The job may already be complete when returned; callers must not assume the index exists until :meth:`Job.wait` returns. + + In LanceDB Enterprise, the returned job tracks this submission only if + the server returns a job ID. If it does not, the job is already + complete and does not wait for the background build; use + ``wait_for_index`` and account for the replacement caveat described in + [create_index][lancedb.table.Table.create_index]. """ raise NotImplementedError( "create_index_async is not supported for this table type" @@ -2962,6 +2990,12 @@ class Table(ABC): """ List all indices that have been created with [Table.create_index][lancedb.table.Table.create_index] + + On LanceDB Enterprise tables, an index whose build hasn't finished may + be missing (vector indices) or listed with 0 indexed rows (scalar and + FTS indices), and ``num_unindexed_rows`` may be ``None``. Use + [wait_for_index][lancedb.table.Table.wait_for_index] to wait until an + index is ready. """ @abstractmethod @@ -4277,6 +4311,12 @@ class LanceTable(Table): The job may already be complete when returned; callers must not assume the index exists until :meth:`Job.wait` returns. + + In LanceDB Enterprise, the returned job tracks this submission only if + the server returns a job ID. If it does not, the job is already + complete and does not wait for the background build; use + ``wait_for_index`` and account for the replacement caveat described in + [create_index][lancedb.table.Table.create_index]. """ return Job( LOOP.run( @@ -6481,12 +6521,34 @@ class AsyncTable: like to create. You can also specify index-specific parameters when creating an index object. wait_timeout: timedelta, optional - The timeout to wait if indexing is asynchronous. + LanceDB Enterprise only: how long to wait for the index build to finish. + If not set, the call returns once the build is scheduled. name: str, optional The name of the index. If not provided, a default name will be generated. train: bool, default True Whether to train the index with existing data. Vector indices always train with existing data. + + Notes + ----- + On LanceDB Embedded tables the index is built before this method + returns. On LanceDB Enterprise tables it returns once the build is + scheduled. Until the build finishes, a vector index may be missing + from [list_indices][lancedb.table.AsyncTable.list_indices], a scalar or + FTS index may be listed with 0 indexed rows, and queries won't use the + index. To block until the index is ready, pass ``wait_timeout``, call + [wait_for_index][lancedb.table.AsyncTable.wait_for_index], or use + [create_index_async][lancedb.table.AsyncTable.create_index_async] and + wait on its job when the server returned a job ID. + + ``wait_timeout`` waits for the submitted build when the server returns + a job ID. On older servers that don't return one, it falls back to + polling the index by name. ``wait_for_index`` always waits by name. + When replacing an existing index, those name-based waits may return + before the new build finishes because the old index can still match. + ``create_index_async`` can track a specific build only when the server + returned a job ID; otherwise it returns an already-complete job and + callers must use ``wait_for_index`` for that build. """ if config is not None: if not isinstance( @@ -6560,6 +6622,12 @@ class AsyncTable: Takes the same arguments as :meth:`create_index`. The job may already be complete when returned; callers must not assume the index exists until :meth:`AsyncJob.wait` resolves. + + In LanceDB Enterprise, the returned job tracks this submission only if + the server returns a job ID. If it does not, the job is already + complete and does not wait for the background build; use + ``wait_for_index`` and account for the replacement caveat described in + [create_index][lancedb.table.AsyncTable.create_index]. """ job = await self._inner.create_index_async( column, @@ -8197,6 +8265,12 @@ class AsyncTable: async def list_indices(self) -> Iterable[IndexConfig]: """ List all indices that have been created with Self::create_index + + On LanceDB Enterprise tables, an index whose build hasn't finished may + be missing (vector indices) or listed with 0 indexed rows (scalar and + FTS indices), and ``num_unindexed_rows`` may be ``None``. Use + [wait_for_index][lancedb.table.AsyncTable.wait_for_index] to wait until + an index is ready. """ return await self._inner.list_indices()