diff --git a/docs/web/sql/ddl/index.mdx b/docs/web/sql/ddl/index.mdx index 425af7376..035424b5f 100644 --- a/docs/web/sql/ddl/index.mdx +++ b/docs/web/sql/ddl/index.mdx @@ -38,6 +38,46 @@ runs as `create_` followed by the algorithm instead. See [Incremental Indexing](/indexing/incremental-indexing). +{/* Draft for the index exist_ok release. + +```sql +CREATE INDEX IF NOT EXISTS text_search ON documents USING fts (text) +``` + +Use `IF NOT EXISTS` to preserve the existing index with the same name. Without an +explicit name, the name is derived from the column (`title_idx`, or +`title_list_element_idx` for list-element FTS). Existing definitions and builds +are kept, even when the new request specifies different settings. + +The Python SDK exposes the same behavior through `exist_ok=True` on +`create_index` and `create_index_async`, including scalar and FTS wrappers. +This option takes precedence over `replace`, whose default behavior is unchanged. + +```python +from lancedb.index import BTree + +table.create_index("title", config=BTree(), name="title_idx", exist_ok=True) +``` + +```rust +use lancedb::index::Index; + +table.create_index(&["title"], Index::BTree(Default::default())) + .name("title_idx".to_string()) + .exist_ok(true) + .execute() + .await?; +``` + +The SDK option is available in Python and Rust. TypeScript support is a separate +follow-up. Remote use requires a server that supports the option. + +An existing background build continues running. A completed submission handle +for a skipped creation does not wait for that build; use `wait_for_index` when +its completion is required. + +*/} + ## DROP INDEX {#drop} diff --git a/python/python/lancedb/_lancedb.pyi b/python/python/lancedb/_lancedb.pyi index e3f00557c..cbc62f772 100644 --- a/python/python/lancedb/_lancedb.pyi +++ b/python/python/lancedb/_lancedb.pyi @@ -591,6 +591,7 @@ class Table: *, name: Optional[str], train: Optional[bool], + exist_ok: bool = False, ): ... async def create_index_async( self, @@ -615,6 +616,7 @@ class Table: *, name: Optional[str], train: Optional[bool], + exist_ok: bool = False, ) -> Job: ... async def list_versions(self) -> List[Dict[str, Any]]: ... async def version(self) -> int: ... diff --git a/python/python/lancedb/remote/table.py b/python/python/lancedb/remote/table.py index 6730e6694..dbfd127ee 100644 --- a/python/python/lancedb/remote/table.py +++ b/python/python/lancedb/remote/table.py @@ -369,6 +369,7 @@ class RemoteTable(Table): index_type: ScalarIndexType = "BTREE", wait_timeout: Optional[timedelta] = None, name: Optional[str] = None, + exist_ok: bool = False, ): """Create a scalar index on a column. @@ -395,6 +396,7 @@ class RemoteTable(Table): self._table.create_index( column, replace=replace, + exist_ok=exist_ok, config=config, wait_timeout=wait_timeout, name=name, @@ -433,6 +435,7 @@ class RemoteTable(Table): document_granularity: DocumentGranularity = DocumentGranularity.ROW, wait_timeout: Optional[timedelta] = None, name: Optional[str] = None, + exist_ok: bool = False, ): """Create a full-text search index on a column. @@ -492,6 +495,7 @@ class RemoteTable(Table): self._table.create_index( field_names, replace=replace, + exist_ok=exist_ok, config=config, name=name, wait_timeout=wait_timeout, @@ -516,6 +520,7 @@ class RemoteTable(Table): replace: bool = ..., wait_timeout: Optional[timedelta] = ..., name: Optional[str] = ..., + exist_ok: bool = ..., train: bool = ..., ) -> None: ... @@ -541,6 +546,7 @@ class RemoteTable(Table): *, wait_timeout: Optional[timedelta] = ..., name: Optional[str] = ..., + exist_ok: bool = ..., train: bool = ..., target_partition_size: Optional[int] = ..., ) -> None: ... @@ -572,6 +578,7 @@ class RemoteTable(Table): config: Optional[IndexConfigType] = None, wait_timeout: Optional[timedelta] = None, name: Optional[str] = None, + exist_ok: bool = False, train: bool = True, target_partition_size: Optional[int] = None, ): @@ -675,6 +682,7 @@ class RemoteTable(Table): self._table.create_index( column, replace=replace, + exist_ok=exist_ok, config=config, wait_timeout=wait_timeout, name=name, @@ -690,6 +698,7 @@ class RemoteTable(Table): replace: Optional[bool] = None, wait_timeout: Optional[timedelta] = None, name: Optional[str] = None, + exist_ok: bool = False, train: bool = True, ) -> Job: """Create an index, returning a handle to the indexing job. @@ -702,6 +711,7 @@ class RemoteTable(Table): self._table.create_index_async( column, replace=replace, + exist_ok=exist_ok, config=config, wait_timeout=wait_timeout, name=name, diff --git a/python/python/lancedb/table.py b/python/python/lancedb/table.py index 5f0af7a66..e0b96e476 100644 --- a/python/python/lancedb/table.py +++ b/python/python/lancedb/table.py @@ -168,6 +168,36 @@ def _validate_index_accelerator(accelerator: Any) -> None: raise RuntimeError(_CUDA_UNAVAILABLE_MESSAGE) from err +def _create_accelerated_index(dataset, *, exist_ok=False, name=None, **options): + if exist_ok: + parts = _parse_field_path(options["column"]) + default_name = ".".join(parts) + if any("." in part for part in parts): + default_name = ".".join( + "`" + part.replace("`", "``") + "`" + if any( + not character.isalnum() and character != "_" for character in part + ) + else part + for part in parts + ) + name = name or f"{default_name}_idx" + dataset.checkout_latest() + if any(index["name"] == name for index in dataset.list_indices()): + return + options["replace"] = False + options["name"] = name + _validate_index_accelerator(options.get("accelerator")) + try: + dataset.create_index(**options) + except Exception: + if exist_ok: + dataset.checkout_latest() + if any(index["name"] == name for index in dataset.list_indices()): + return + raise + + def _polars_predicate_pushdown_barrier(frame: Any) -> Any: """Return a Polars frame unchanged while blocking predicate pushdown.""" return frame @@ -1834,6 +1864,7 @@ class Table(ABC): replace: bool = ..., wait_timeout: Optional[timedelta] = ..., name: Optional[str] = ..., + exist_ok: bool = ..., train: bool = ..., ) -> None: ... @@ -1859,6 +1890,7 @@ class Table(ABC): *, wait_timeout: Optional[timedelta] = ..., name: Optional[str] = ..., + exist_ok: bool = ..., train: bool = ..., target_partition_size: Optional[int] = ..., ) -> None: ... @@ -1890,6 +1922,7 @@ class Table(ABC): config: Optional[IndexConfigType] = None, wait_timeout: Optional[timedelta] = None, name: Optional[str] = None, + exist_ok: bool = False, train: bool = True, target_partition_size: Optional[int] = None, ): @@ -1913,6 +1946,9 @@ class Table(ABC): BTree, Bitmap, LabelList, Fm, NGram, ZoneMap, FTS. replace : bool, default True Whether to replace an existing index on this column. + exist_ok : bool, default False + Preserve the existing index with the requested or derived name. + Takes precedence over replace and keeps its definition and build. wait_timeout : timedelta, optional LanceDB Enterprise only: how long to wait for the index build to finish. If not set, the call returns once the build is scheduled. @@ -1968,6 +2004,7 @@ class Table(ABC): replace: Optional[bool] = None, wait_timeout: Optional[timedelta] = None, name: Optional[str] = None, + exist_ok: bool = False, train: bool = True, ) -> Job: """Create an index, returning a handle to the indexing job. @@ -2060,6 +2097,7 @@ class Table(ABC): index_type: ScalarIndexType = "BTREE", wait_timeout: Optional[timedelta] = None, name: Optional[str] = None, + exist_ok: bool = False, ): """Create a scalar index on a column. @@ -2144,6 +2182,7 @@ class Table(ABC): document_granularity: DocumentGranularity = DocumentGranularity.ROW, wait_timeout: Optional[timedelta] = None, name: Optional[str] = None, + exist_ok: bool = False, ): """Create a full-text search index on the table. @@ -4318,6 +4357,7 @@ class LanceTable(Table): replace: bool = ..., wait_timeout: Optional[timedelta] = ..., name: Optional[str] = ..., + exist_ok: bool = ..., train: bool = ..., ) -> None: ... @@ -4343,6 +4383,7 @@ class LanceTable(Table): *, wait_timeout: Optional[timedelta] = ..., name: Optional[str] = ..., + exist_ok: bool = ..., train: bool = ..., target_partition_size: Optional[int] = ..., ) -> None: ... @@ -4374,6 +4415,7 @@ class LanceTable(Table): config: Optional[IndexConfigType] = None, wait_timeout: Optional[timedelta] = None, name: Optional[str] = None, + exist_ok: bool = False, train: bool = True, target_partition_size: Optional[int] = None, ): @@ -4397,6 +4439,9 @@ class LanceTable(Table): BTree, Bitmap, LabelList, Fm, NGram, ZoneMap, FTS. replace : bool, default True Whether to replace an existing index on this column. + exist_ok : bool, default False + Preserve the existing index with the requested or derived name. + Takes precedence over replace and keeps its definition and build. wait_timeout : timedelta, optional Timeout to wait for async indexing to complete. name : str, optional @@ -4465,8 +4510,10 @@ class LanceTable(Table): # Handle accelerator through pylance if accelerator is not None: dataset = self.to_lance() - _validate_index_accelerator(accelerator) - dataset.create_index( + _create_accelerated_index( + dataset, + exist_ok=exist_ok, + name=name, column=column, index_type=index_type, metric=metric, @@ -4492,7 +4539,6 @@ class LanceTable(Table): if acc is not None: # Dispatch to pylance for GPU acceleration dataset = self.to_lance() - _validate_index_accelerator(acc) index_type_map = { "IvfFlat": "IVF_FLAT", "IvfSq": "IVF_SQ", @@ -4509,7 +4555,10 @@ class LanceTable(Table): if num_bits is not None: quantization_kwargs["num_bits"] = num_bits - dataset.create_index( + _create_accelerated_index( + dataset, + exist_ok=exist_ok, + name=name, column=column, index_type=lance_index_type, metric=getattr(config, "distance_type", "l2"), @@ -4531,6 +4580,7 @@ class LanceTable(Table): self._table.create_index( column, replace=replace, + exist_ok=exist_ok, config=config, wait_timeout=wait_timeout, name=name, @@ -4546,6 +4596,7 @@ class LanceTable(Table): replace: Optional[bool] = None, wait_timeout: Optional[timedelta] = None, name: Optional[str] = None, + exist_ok: bool = False, train: bool = True, ) -> Job: """Create an index, returning a handle to the indexing job. @@ -4564,6 +4615,7 @@ class LanceTable(Table): self._table.create_index_async( column, replace=replace, + exist_ok=exist_ok, config=config, wait_timeout=wait_timeout, name=name, @@ -4684,6 +4736,7 @@ class LanceTable(Table): index_type: ScalarIndexType = "BTREE", wait_timeout: Optional[timedelta] = None, name: Optional[str] = None, + exist_ok: bool = False, ): """Create a scalar index on a column. @@ -4710,6 +4763,7 @@ class LanceTable(Table): self._table.create_index( column, replace=replace, + exist_ok=exist_ok, config=config, wait_timeout=wait_timeout, name=name, @@ -4747,6 +4801,7 @@ class LanceTable(Table): document_granularity: DocumentGranularity = DocumentGranularity.ROW, wait_timeout: Optional[timedelta] = None, name: Optional[str] = None, + exist_ok: bool = False, ): """Create a full-text search index on a column. @@ -4808,6 +4863,7 @@ class LanceTable(Table): self._table.create_index( field_names, replace=replace, + exist_ok=exist_ok, config=config, name=name, wait_timeout=wait_timeout, @@ -6806,6 +6862,7 @@ class AsyncTable: ] = None, wait_timeout: Optional[timedelta] = None, name: Optional[str] = None, + exist_ok: bool = False, train: bool = True, ): """Create an index to speed up queries @@ -6836,6 +6893,9 @@ class AsyncTable: If not set, the call returns once the build is scheduled. name: str, optional The name of the index. If not provided, a default name will be generated. + exist_ok: bool, default False + Preserve an existing index with the requested or derived name. This + takes precedence over replace and keeps its definition and build. train: bool, default True Whether to train the index with existing data. Vector indices always train with existing data. @@ -6891,6 +6951,7 @@ class AsyncTable: column, index=config, replace=replace, + exist_ok=exist_ok, wait_timeout=wait_timeout, name=name, train=train, @@ -6928,6 +6989,7 @@ class AsyncTable: ] = None, wait_timeout: Optional[timedelta] = None, name: Optional[str] = None, + exist_ok: bool = False, train: bool = True, ) -> AsyncJob: """Create an index, returning a handle to the indexing job. @@ -6946,6 +7008,7 @@ class AsyncTable: column, index=config, replace=replace, + exist_ok=exist_ok, wait_timeout=wait_timeout, name=name, train=train, diff --git a/python/python/tests/test_index.py b/python/python/tests/test_index.py index 738a791ad..e658d18e1 100644 --- a/python/python/tests/test_index.py +++ b/python/python/tests/test_index.py @@ -621,3 +621,45 @@ def test_index_statistics_index_type_lists_all_supported_values(): set(get_args(get_type_hints(IndexStatistics)["index_type"])) == expected_index_types ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("background", [False, True]) +@pytest.mark.parametrize( + "column,config", [("id", BTree()), ("vector", IvfFlat(num_partitions=1))] +) +async def test_index_exist_ok_preserves_existing_index( + some_table, background, column, config +): + await some_table.create_index(column, config=config, exist_ok=True) + version = await some_table.version() + original = await some_table.list_indices() + if background: + job = await some_table.create_index_async( + column, config=config, replace=True, exist_ok=True + ) + await job.wait() + else: + await some_table.create_index( + column, config=config, replace=True, exist_ok=True + ) + assert await some_table.version() == version + assert [(i.name, i.index_type) for i in await some_table.list_indices()] == [ + (i.name, i.index_type) for i in original + ] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("background", [False, True]) +async def test_named_index_exist_ok_checks_before_column_validation( + some_table, background +): + await some_table.create_index("id", config=BTree(), name="kept") + version = await some_table.version() + method = some_table.create_index_async if background else some_table.create_index + result = await method("missing", config=BTree(), name="kept", exist_ok=True) + if background: + assert result.id is None + await result.wait() + assert await some_table.version() == version + assert [index.name for index in await some_table.list_indices()] == ["kept"] diff --git a/python/python/tests/test_remote_db.py b/python/python/tests/test_remote_db.py index ca2516fda..ea7f73c71 100644 --- a/python/python/tests/test_remote_db.py +++ b/python/python/tests/test_remote_db.py @@ -45,6 +45,7 @@ from lancedb.query import ( LanceVectorQueryBuilder, MatchQuery, ) +from lancedb.index import BTree from lancedb.remote import ClientConfig from lancedb.remote.errors import ( AlreadyExistsError, @@ -7420,3 +7421,54 @@ async def test_remote_blob_bytes_hybrid_rejects_same_version_different_incarnati if use_async: await result assert not any(path == "fetch_blobs" for path, _ in requests) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("is_async", [False, True]) +@pytest.mark.parametrize("background", [False, True]) +@pytest.mark.parametrize("kwargs", [{}, {"exist_ok": False}, {"exist_ok": True}]) +async def test_remote_index_exist_ok_wire(is_async, background, kwargs): + received = [] + + def handler(request): + parsed = urlparse(request.path) + body = request.rfile.read(int(request.headers.get("Content-Length", 0))) + if parsed.path.endswith("/create_index/"): + received.append((parse_qs(parsed.query), json.loads(body))) + payload = {} + elif parsed.path.endswith("/describe/"): + payload = dict( + version=1, + schema=dict( + fields=[dict(name="id", type={"type": "int64"}, nullable=False)] + ), + ) + elif parsed.path.endswith("/create/"): + payload = {} + else: + raise AssertionError(request.path) + request.send_response(200) + request.send_header("Content-Type", "application/json") + request.end_headers() + request.wfile.write(json.dumps(payload).encode()) + + method = "create_index_async" if background else "create_index" + if is_async: + async with mock_lancedb_connection_async(handler) as db: + table = await db.create_table("test", [{"id": 1}]) + result = await getattr(table, method)("id", config=BTree(), **kwargs) + if background: + await result.wait() + else: + with mock_lancedb_connection(handler) as db: + table = db.create_table("test", [{"id": 1}]) + result = getattr(table, method)("id", config=BTree(), **kwargs) + if background: + result.wait() + query, body = received[0] + assert query == ({"exist_ok": ["true"]} if kwargs.get("exist_ok") else {}) + assert "exist_ok" not in body + if kwargs.get("exist_ok"): + assert body["replace"] is False + else: + assert "replace" not in body diff --git a/python/python/tests/test_table.py b/python/python/tests/test_table.py index e3b69ced1..604d6afb3 100644 --- a/python/python/tests/test_table.py +++ b/python/python/tests/test_table.py @@ -1837,6 +1837,7 @@ def test_create_index_method(mock_create_index, mem_db: DBConnection): wait_timeout=None, name=None, train=True, + exist_ok=False, ) # Test with target_partition_size @@ -1862,6 +1863,7 @@ def test_create_index_method(mock_create_index, mem_db: DBConnection): wait_timeout=None, name=None, train=True, + exist_ok=False, ) # target_partition_size has a default value, @@ -1886,6 +1888,7 @@ def test_create_index_method(mock_create_index, mem_db: DBConnection): wait_timeout=None, name=None, train=True, + exist_ok=False, ) table.create_index( @@ -1902,6 +1905,7 @@ def test_create_index_method(mock_create_index, mem_db: DBConnection): wait_timeout=None, name=None, train=True, + exist_ok=False, ) table.create_index( @@ -1922,6 +1926,7 @@ def test_create_index_method(mock_create_index, mem_db: DBConnection): wait_timeout=None, name=None, train=True, + exist_ok=False, ) table.create_index( @@ -1942,6 +1947,7 @@ def test_create_index_method(mock_create_index, mem_db: DBConnection): wait_timeout=None, name=None, train=True, + exist_ok=False, ) @@ -2073,6 +2079,7 @@ def test_create_index_name_and_train_parameters( wait_timeout=None, name="my_custom_index", train=True, + exist_ok=False, ) # Test with train=False @@ -2084,6 +2091,7 @@ def test_create_index_name_and_train_parameters( wait_timeout=None, name=None, train=False, + exist_ok=False, ) # Test with both name and train @@ -2095,6 +2103,7 @@ def test_create_index_name_and_train_parameters( wait_timeout=None, name="my_index_name", train=True, + exist_ok=False, ) @@ -2132,6 +2141,7 @@ def test_create_index_new_api(mock_create_index, mem_db: DBConnection): wait_timeout=None, name=None, train=True, + exist_ok=False, ) # Scalar index via new API @@ -2143,6 +2153,7 @@ def test_create_index_new_api(mock_create_index, mem_db: DBConnection): wait_timeout=None, name=None, train=True, + exist_ok=False, ) # FTS index via new API @@ -2154,6 +2165,7 @@ def test_create_index_new_api(mock_create_index, mem_db: DBConnection): wait_timeout=None, name=None, train=True, + exist_ok=False, ) @@ -6292,3 +6304,87 @@ async def test_async_computed_auto_refresh_uses_server_interval(tmp_path): metadata = (await table.schema()).field("doubled").metadata assert metadata[b"computed_column.auto_refresh_enabled"] == b"true" assert b"computed_column.auto_refresh_interval_seconds" not in metadata + + +@pytest.mark.parametrize( + "method", + ["create_index", "create_index_async", "create_scalar_index", "create_fts_index"], +) +@pytest.mark.parametrize("name", [None, "kept_index"]) +def test_index_exist_ok_preserves_definition_and_version(mem_db, method, name): + table = mem_db.create_table( + "keep_index", [{"id": i, "text": f"document {i}"} for i in range(20)] + ) + column = "text" if method == "create_fts_index" else "id" + config = FTS() if column == "text" else BTree() + table.create_index(column, config=config, name=name, exist_ok=True) + before = table.version + original = table.list_indices() + kwargs = dict(name=name, exist_ok=True) + if method in ("create_index", "create_index_async"): + kwargs["config"] = config + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + result = getattr(table, method)(column, **kwargs) + if method == "create_index_async": + result.wait() + assert table.version == before + assert [(index.name, index.index_type) for index in table.list_indices()] == [ + (index.name, index.index_type) for index in original + ] + with pytest.raises((ValueError, RuntimeError), match="already exists"): + table.create_index(column, config=config, name=name, replace=False) + table.create_index(column, config=config, name=name) + assert table.version > before + + +@pytest.mark.parametrize( + "column,name,existing", + [ + ("vector", "kept", "kept"), + ("`docs`.`nested.dot`", None, "docs.`nested.dot`_idx"), + ], +) +def test_accelerated_index_exist_ok_skips_gpu_validation(column, name, existing): + dataset = MagicMock() + dataset.list_indices.return_value = [{"name": existing}] + with patch("lancedb.table._validate_index_accelerator") as validate: + table_module._create_accelerated_index( + dataset, column=column, accelerator="cuda", name=name, exist_ok=True + ) + validate.assert_not_called() + dataset.create_index.assert_not_called() + + +def test_accelerated_index_exist_ok_preserves_concurrent_winner(): + dataset = MagicMock() + dataset.list_indices.side_effect = [[], [{"name": "vector_idx"}]] + dataset.create_index.side_effect = RuntimeError("concurrent create") + with patch("lancedb.table._validate_index_accelerator"): + table_module._create_accelerated_index( + dataset, column="vector", accelerator="cuda", replace=True, exist_ok=True + ) + assert dataset.create_index.call_args.kwargs["replace"] is False + assert dataset.create_index.call_args.kwargs["name"] == "vector_idx" + + +@pytest.mark.parametrize("column", ["`user-id`", "`user:id`", "nested.`user-id`"]) +def test_index_exist_ok_matches_lance_default_name(mem_db, column): + table = mem_db.create_table( + "quoted_index", + [{"user-id": 1, "user:id": 2, "nested": {"user-id": 4}}], + ) + table.create_index(column, config=BTree()) + before = table.version + table.create_index(column, config=BTree(), exist_ok=True) + assert table.version == before + assert len(table.list_indices()) == 1 + + +def test_fts_exist_ok_matches_quoted_default_name(mem_db): + table = mem_db.create_table("quoted_fts", [{"body-text": "keep this document"}]) + table.create_index("`body-text`", config=FTS()) + before = table.version + table.create_index("`body-text`", config=FTS(), exist_ok=True) + assert table.version == before + assert len(table.list_indices()) == 1 diff --git a/python/python/tests/test_table_signatures.py b/python/python/tests/test_table_signatures.py index 387a89f31..41a688068 100644 --- a/python/python/tests/test_table_signatures.py +++ b/python/python/tests/test_table_signatures.py @@ -115,7 +115,11 @@ def test_vector_tuning_options(sync_table): }.items(): assert getattr(config, name) == value assert kwargs == dict( - wait_timeout=timeout, name="custom_idx", replace=False, train=False + wait_timeout=timeout, + name="custom_idx", + replace=False, + train=False, + exist_ok=False, ) diff --git a/python/src/table.rs b/python/src/table.rs index 31cf8d25c..f77ff5a7f 100644 --- a/python/src/table.rs +++ b/python/src/table.rs @@ -1628,7 +1628,8 @@ impl Table { }) } - #[pyo3(signature = (column, index=None, replace=None, wait_timeout=None, *, name=None, train=None))] + #[allow(clippy::too_many_arguments)] + #[pyo3(signature = (column, index=None, replace=None, wait_timeout=None, *, name=None, train=None, exist_ok=false))] pub fn create_index<'a>( self_: PyRef<'a, Self>, column: String, @@ -1637,6 +1638,7 @@ impl Table { wait_timeout: Option>, name: Option, train: Option, + exist_ok: bool, ) -> PyResult> { let index = extract_index_params(&index)?; let timeout = wait_timeout @@ -1645,6 +1647,7 @@ impl Table { let mut op = self_ .inner_ref()? .create_index_with_timeout(&[column], index, timeout); + op = op.exist_ok(exist_ok); if let Some(replace) = replace { op = op.replace(replace); } @@ -1661,7 +1664,8 @@ impl Table { }) } - #[pyo3(signature = (column, index=None, replace=None, wait_timeout=None, *, name=None, train=None))] + #[allow(clippy::too_many_arguments)] + #[pyo3(signature = (column, index=None, replace=None, wait_timeout=None, *, name=None, train=None, exist_ok=false))] pub fn create_index_async<'a>( self_: PyRef<'a, Self>, column: String, @@ -1670,6 +1674,7 @@ impl Table { wait_timeout: Option>, name: Option, train: Option, + exist_ok: bool, ) -> PyResult> { let index = extract_index_params(&index)?; let timeout = wait_timeout @@ -1678,6 +1683,7 @@ impl Table { let mut op = self_ .inner_ref()? .create_index_with_timeout(&[column], index, timeout); + op = op.exist_ok(exist_ok); if let Some(replace) = replace { op = op.replace(replace); } diff --git a/rust/lancedb/src/index.rs b/rust/lancedb/src/index.rs index 87eff864d..91c941154 100644 --- a/rust/lancedb/src/index.rs +++ b/rust/lancedb/src/index.rs @@ -220,6 +220,7 @@ pub struct IndexBuilder { pub(crate) index: Index, pub(crate) columns: Vec, pub(crate) replace: bool, + pub(crate) exist_ok: bool, pub(crate) wait_timeout: Option, pub(crate) train: bool, pub(crate) name: Option, @@ -232,6 +233,7 @@ impl IndexBuilder { index, columns, replace: true, + exist_ok: false, train: true, wait_timeout: None, name: None, @@ -324,6 +326,21 @@ impl IndexBuilder { self } + /// Preserve an existing index with the requested name. Defaults to false. + /// This takes precedence over `replace`; a skipped creation does not rebuild. + /// + /// ```no_run + /// # async fn example(table: &lancedb::Table) -> lancedb::Result<()> { + /// table.create_index(&["id"], lancedb::index::Index::Auto) + /// .exist_ok(true).execute().await?; + /// # Ok(()) + /// # } + /// ``` + pub fn exist_ok(mut self, exist_ok: bool) -> Self { + self.exist_ok = exist_ok; + self + } + /// Duration of time to wait for asynchronous indexing to complete. If not set, /// `create_index()` will not wait. /// diff --git a/rust/lancedb/src/remote/table.rs b/rust/lancedb/src/remote/table.rs index f4e5c1e4a..a7cf3f80d 100644 --- a/rust/lancedb/src/remote/table.rs +++ b/rust/lancedb/src/remote/table.rs @@ -701,9 +701,13 @@ impl RemoteTable { async fn submit_create_index(&self, mut index: IndexBuilder) -> Result> { self.check_mutable().await?; - let request = self + let mut request = self .client .post(&format!("/v1/table/{}/create_index/", self.identifier)); + if index.exist_ok { + request = request.query(&[("exist_ok", true)]); + index.replace = false; + } let column = match index.columns.len() { 0 => { @@ -18101,6 +18105,42 @@ mod tests { branch.restore().await.unwrap(); } + #[tokio::test] + async fn test_index_exist_ok_query_and_replacement_policy() { + for exist_ok in [false, true] { + for background in [false, true] { + let table = Table::new_with_handler("my_table", move |request| { + match request.url().path() { + "/v1/table/my_table/describe/" => simple_describe_response(), + "/v1/table/my_table/create_index/" => { + assert_eq!(request.url().query(), exist_ok.then_some("exist_ok=true")); + let body = request_body_json(&request); + assert_eq!( + body.get("replace"), + exist_ok.then_some(&serde_json::Value::Bool(false)) + ); + assert!(body.get("exist_ok").is_none()); + http::Response::builder() + .status(200) + .body("{}".into()) + .unwrap() + } + path => panic!("unexpected request path: {path}"), + } + }); + let builder = table + .create_index(&["id"], Index::BTree(Default::default())) + .replace(true) + .exist_ok(exist_ok); + if background { + builder.execute_async().await.unwrap().wait().await.unwrap(); + } else { + builder.execute().await.unwrap(); + } + } + } + } + #[tokio::test] async fn test_branch_create_index_carries_branch_in_body() { use lance::dataset::refs::Ref; diff --git a/rust/lancedb/src/table.rs b/rust/lancedb/src/table.rs index 4959b2842..ff0ab8b94 100644 --- a/rust/lancedb/src/table.rs +++ b/rust/lancedb/src/table.rs @@ -4145,14 +4145,18 @@ impl BaseTable for NativeTable { } async fn create_index(&self, opts: IndexBuilder) -> Result<()> { - let prepared = self.prepare_index(&opts).await?; + let Some(prepared) = self.prepare_index(&opts).await? else { + return Ok(()); + }; self.build_index(opts, prepared).await } async fn create_index_async(&self, opts: IndexBuilder) -> Result { // Prepare before spawning so bad input is reported by this call rather // than only by the job. - let prepared = self.prepare_index(&opts).await?; + let Some(prepared) = self.prepare_index(&opts).await? else { + return Ok(Job::new_done()); + }; let table = self.clone(); Ok(Job::spawned(tokio::spawn(async move { table.build_index(opts, prepared).await diff --git a/rust/lancedb/src/table/create_index.rs b/rust/lancedb/src/table/create_index.rs index 5f9ae13b4..e0da64029 100644 --- a/rust/lancedb/src/table/create_index.rs +++ b/rust/lancedb/src/table/create_index.rs @@ -115,14 +115,24 @@ impl NativeTable { pub(super) async fn prepare_index( &self, opts: &crate::index::IndexBuilder, - ) -> Result { + ) -> Result> { + self.dataset.ensure_mutable()?; + let dataset = self.dataset.get().await?; + if opts.exist_ok + && let Some(name) = &opts.name + && dataset + .load_indices() + .await? + .iter() + .any(|index| &index.name == name) + { + return Ok(None); + } if opts.columns.len() != 1 { return Err(Error::Schema { message: "Multi-column (composite) indices are not yet supported".to_string(), }); } - self.dataset.ensure_mutable()?; - let dataset = self.dataset.get().await?; let (column, field) = if let Index::FTS(params) = &opts.index { let resolved = resolve_lance_fts_field_path(dataset.schema(), &opts.columns[0])?; if params.get_document_granularity().is_list_element() && resolved.list_depth == 0 { @@ -137,9 +147,48 @@ impl NativeTable { } else { Self::resolve_index_field(dataset.schema(), &opts.columns[0])? }; + if let Some(name) = Self::guarded_index_name(opts, &column)? + && dataset + .load_indices() + .await? + .iter() + .any(|index| index.name == name) + { + return Ok(None); + } let params = self.make_index_params(&field, opts.index.clone()).await?; let index_type = self.get_index_type_for_field(&field, &opts.index); - Ok((column, params, index_type)) + Ok(Some((column, params, index_type))) + } + + fn guarded_index_name( + opts: &crate::index::IndexBuilder, + column: &str, + ) -> Result> { + let name = if opts.exist_ok { + Some(if let Some(name) = &opts.name { + name.clone() + } else { + let names = lance_core::datatypes::parse_field_path(column)?; + let default_name = if matches!(&opts.index, Index::FTS(_)) + || names.iter().any(|name| name.contains('.')) + { + column.to_string() + } else { + names.join(".") + }; + let suffix = if matches!(&opts.index, Index::FTS(params) if params.get_document_granularity().is_list_element()) + { + "_list_element_idx" + } else { + "_idx" + }; + format!("{default_name}{suffix}") + }) + } else { + None + }; + Ok(name) } /// Builds a prepared index and publishes the new dataset version. @@ -150,16 +199,42 @@ impl NativeTable { ) -> Result<()> { let (column, lance_idx_params, index_type) = prepared; let mut dataset = (*self.dataset.get().await?).clone(); + let guarded_name = Self::guarded_index_name(&opts, &column)?; + if let Some(name) = &guarded_name + && dataset + .load_indices() + .await? + .iter() + .any(|index| &index.name == name) + { + return Ok(()); + } let columns = [column.as_str()]; let mut builder = dataset .create_index_builder(&columns, index_type, lance_idx_params.as_ref()) .train(opts.train) - .replace(opts.replace); + .replace(opts.replace && !opts.exist_ok); - if let Some(name) = opts.name { + if let Some(name) = guarded_name.clone().or(opts.name) { builder = builder.name(name); } - builder.await?; + if let Err(error) = builder.await { + if let Some(name) = guarded_name { + self.dataset.reload().await?; + if self + .dataset + .get() + .await? + .load_indices() + .await? + .iter() + .any(|index| index.name == name) + { + return Ok(()); + } + } + return Err(error.into()); + } // A writer already open was built before this index existed, so it has // to pick it up here or a read needing it is refused until it reopens. self.dataset @@ -568,6 +643,63 @@ mod tests { Ok(FixedSizeListArray::from(data)) } + #[tokio::test] + async fn test_concurrent_index_exist_ok_preserves_single_identity() { + let dir = tempdir().unwrap(); + let db = connect(dir.path().to_str().unwrap()) + .execute() + .await + .unwrap(); + let batch = record_batch!(("id", Int32, (0..512).collect::>())).unwrap(); + let table = db.create_table("t", batch).execute().await.unwrap(); + let second = db.open_table("t").execute().await.unwrap(); + let (left, right) = tokio::join!( + table + .create_index(&["id"], Index::BTree(Default::default())) + .exist_ok(true) + .execute(), + second + .create_index(&["id"], Index::BTree(Default::default())) + .exist_ok(true) + .execute() + ); + left.unwrap(); + right.unwrap(); + table.checkout_latest().await.unwrap(); + let indices = table + .as_native() + .unwrap() + .dataset + .get() + .await + .unwrap() + .load_indices() + .await + .unwrap(); + assert_eq!(indices.len(), 1); + let original_uuid = indices[0].uuid; + let version = table.version().await.unwrap(); + table + .create_index(&["id"], Index::BTree(Default::default())) + .replace(true) + .exist_ok(true) + .execute() + .await + .unwrap(); + assert_eq!(table.version().await.unwrap(), version); + let indices = table + .as_native() + .unwrap() + .dataset + .get() + .await + .unwrap() + .load_indices() + .await + .unwrap(); + assert_eq!(indices[0].uuid, original_uuid); + } + #[tokio::test] async fn test_create_index() { use std::iter::repeat_with;