feat: support exist_ok for index creation (#4993)

This commit is contained in:
Jack Ye
2026-11-13 19:21:50 -08:00
committed by GitHub
parent 81d04b8c36
commit f6311ae7e9
13 changed files with 525 additions and 17 deletions
+40
View File
@@ -38,6 +38,46 @@ runs as `create_` followed by the algorithm instead. See
[Incremental Indexing](/indexing/incremental-indexing).
</Note>
{/* Draft for the index exist_ok release.
```sql
CREATE INDEX IF NOT EXISTS text_search ON documents USING fts (text)
```
Use `IF NOT EXISTS` to preserve the existing index with the same name. Without an
explicit name, the name is derived from the column (`title_idx`, or
`title_list_element_idx` for list-element FTS). Existing definitions and builds
are kept, even when the new request specifies different settings.
The Python SDK exposes the same behavior through `exist_ok=True` on
`create_index` and `create_index_async`, including scalar and FTS wrappers.
This option takes precedence over `replace`, whose default behavior is unchanged.
```python
from lancedb.index import BTree
table.create_index("title", config=BTree(), name="title_idx", exist_ok=True)
```
```rust
use lancedb::index::Index;
table.create_index(&["title"], Index::BTree(Default::default()))
.name("title_idx".to_string())
.exist_ok(true)
.execute()
.await?;
```
The SDK option is available in Python and Rust. TypeScript support is a separate
follow-up. Remote use requires a server that supports the option.
An existing background build continues running. A completed submission handle
for a skipped creation does not wait for that build; use `wait_for_index` when
its completion is required.
*/}
## DROP INDEX {#drop}
<CodeGroup>
+2
View File
@@ -591,6 +591,7 @@ class Table:
*,
name: Optional[str],
train: Optional[bool],
exist_ok: bool = False,
): ...
async def create_index_async(
self,
@@ -615,6 +616,7 @@ class Table:
*,
name: Optional[str],
train: Optional[bool],
exist_ok: bool = False,
) -> Job: ...
async def list_versions(self) -> List[Dict[str, Any]]: ...
async def version(self) -> int: ...
+10
View File
@@ -369,6 +369,7 @@ class RemoteTable(Table):
index_type: ScalarIndexType = "BTREE",
wait_timeout: Optional[timedelta] = None,
name: Optional[str] = None,
exist_ok: bool = False,
):
"""Create a scalar index on a column.
@@ -395,6 +396,7 @@ class RemoteTable(Table):
self._table.create_index(
column,
replace=replace,
exist_ok=exist_ok,
config=config,
wait_timeout=wait_timeout,
name=name,
@@ -433,6 +435,7 @@ class RemoteTable(Table):
document_granularity: DocumentGranularity = DocumentGranularity.ROW,
wait_timeout: Optional[timedelta] = None,
name: Optional[str] = None,
exist_ok: bool = False,
):
"""Create a full-text search index on a column.
@@ -492,6 +495,7 @@ class RemoteTable(Table):
self._table.create_index(
field_names,
replace=replace,
exist_ok=exist_ok,
config=config,
name=name,
wait_timeout=wait_timeout,
@@ -516,6 +520,7 @@ class RemoteTable(Table):
replace: bool = ...,
wait_timeout: Optional[timedelta] = ...,
name: Optional[str] = ...,
exist_ok: bool = ...,
train: bool = ...,
) -> None: ...
@@ -541,6 +546,7 @@ class RemoteTable(Table):
*,
wait_timeout: Optional[timedelta] = ...,
name: Optional[str] = ...,
exist_ok: bool = ...,
train: bool = ...,
target_partition_size: Optional[int] = ...,
) -> None: ...
@@ -572,6 +578,7 @@ class RemoteTable(Table):
config: Optional[IndexConfigType] = None,
wait_timeout: Optional[timedelta] = None,
name: Optional[str] = None,
exist_ok: bool = False,
train: bool = True,
target_partition_size: Optional[int] = None,
):
@@ -675,6 +682,7 @@ class RemoteTable(Table):
self._table.create_index(
column,
replace=replace,
exist_ok=exist_ok,
config=config,
wait_timeout=wait_timeout,
name=name,
@@ -690,6 +698,7 @@ class RemoteTable(Table):
replace: Optional[bool] = None,
wait_timeout: Optional[timedelta] = None,
name: Optional[str] = None,
exist_ok: bool = False,
train: bool = True,
) -> Job:
"""Create an index, returning a handle to the indexing job.
@@ -702,6 +711,7 @@ class RemoteTable(Table):
self._table.create_index_async(
column,
replace=replace,
exist_ok=exist_ok,
config=config,
wait_timeout=wait_timeout,
name=name,
+67 -4
View File
@@ -168,6 +168,36 @@ def _validate_index_accelerator(accelerator: Any) -> None:
raise RuntimeError(_CUDA_UNAVAILABLE_MESSAGE) from err
def _create_accelerated_index(dataset, *, exist_ok=False, name=None, **options):
if exist_ok:
parts = _parse_field_path(options["column"])
default_name = ".".join(parts)
if any("." in part for part in parts):
default_name = ".".join(
"`" + part.replace("`", "``") + "`"
if any(
not character.isalnum() and character != "_" for character in part
)
else part
for part in parts
)
name = name or f"{default_name}_idx"
dataset.checkout_latest()
if any(index["name"] == name for index in dataset.list_indices()):
return
options["replace"] = False
options["name"] = name
_validate_index_accelerator(options.get("accelerator"))
try:
dataset.create_index(**options)
except Exception:
if exist_ok:
dataset.checkout_latest()
if any(index["name"] == name for index in dataset.list_indices()):
return
raise
def _polars_predicate_pushdown_barrier(frame: Any) -> Any:
"""Return a Polars frame unchanged while blocking predicate pushdown."""
return frame
@@ -1834,6 +1864,7 @@ class Table(ABC):
replace: bool = ...,
wait_timeout: Optional[timedelta] = ...,
name: Optional[str] = ...,
exist_ok: bool = ...,
train: bool = ...,
) -> None: ...
@@ -1859,6 +1890,7 @@ class Table(ABC):
*,
wait_timeout: Optional[timedelta] = ...,
name: Optional[str] = ...,
exist_ok: bool = ...,
train: bool = ...,
target_partition_size: Optional[int] = ...,
) -> None: ...
@@ -1890,6 +1922,7 @@ class Table(ABC):
config: Optional[IndexConfigType] = None,
wait_timeout: Optional[timedelta] = None,
name: Optional[str] = None,
exist_ok: bool = False,
train: bool = True,
target_partition_size: Optional[int] = None,
):
@@ -1913,6 +1946,9 @@ class Table(ABC):
BTree, Bitmap, LabelList, Fm, NGram, ZoneMap, FTS.
replace : bool, default True
Whether to replace an existing index on this column.
exist_ok : bool, default False
Preserve the existing index with the requested or derived name.
Takes precedence over replace and keeps its definition and build.
wait_timeout : timedelta, optional
LanceDB Enterprise only: how long to wait for the index build to finish.
If not set, the call returns once the build is scheduled.
@@ -1968,6 +2004,7 @@ class Table(ABC):
replace: Optional[bool] = None,
wait_timeout: Optional[timedelta] = None,
name: Optional[str] = None,
exist_ok: bool = False,
train: bool = True,
) -> Job:
"""Create an index, returning a handle to the indexing job.
@@ -2060,6 +2097,7 @@ class Table(ABC):
index_type: ScalarIndexType = "BTREE",
wait_timeout: Optional[timedelta] = None,
name: Optional[str] = None,
exist_ok: bool = False,
):
"""Create a scalar index on a column.
@@ -2144,6 +2182,7 @@ class Table(ABC):
document_granularity: DocumentGranularity = DocumentGranularity.ROW,
wait_timeout: Optional[timedelta] = None,
name: Optional[str] = None,
exist_ok: bool = False,
):
"""Create a full-text search index on the table.
@@ -4318,6 +4357,7 @@ class LanceTable(Table):
replace: bool = ...,
wait_timeout: Optional[timedelta] = ...,
name: Optional[str] = ...,
exist_ok: bool = ...,
train: bool = ...,
) -> None: ...
@@ -4343,6 +4383,7 @@ class LanceTable(Table):
*,
wait_timeout: Optional[timedelta] = ...,
name: Optional[str] = ...,
exist_ok: bool = ...,
train: bool = ...,
target_partition_size: Optional[int] = ...,
) -> None: ...
@@ -4374,6 +4415,7 @@ class LanceTable(Table):
config: Optional[IndexConfigType] = None,
wait_timeout: Optional[timedelta] = None,
name: Optional[str] = None,
exist_ok: bool = False,
train: bool = True,
target_partition_size: Optional[int] = None,
):
@@ -4397,6 +4439,9 @@ class LanceTable(Table):
BTree, Bitmap, LabelList, Fm, NGram, ZoneMap, FTS.
replace : bool, default True
Whether to replace an existing index on this column.
exist_ok : bool, default False
Preserve the existing index with the requested or derived name.
Takes precedence over replace and keeps its definition and build.
wait_timeout : timedelta, optional
Timeout to wait for async indexing to complete.
name : str, optional
@@ -4465,8 +4510,10 @@ class LanceTable(Table):
# Handle accelerator through pylance
if accelerator is not None:
dataset = self.to_lance()
_validate_index_accelerator(accelerator)
dataset.create_index(
_create_accelerated_index(
dataset,
exist_ok=exist_ok,
name=name,
column=column,
index_type=index_type,
metric=metric,
@@ -4492,7 +4539,6 @@ class LanceTable(Table):
if acc is not None:
# Dispatch to pylance for GPU acceleration
dataset = self.to_lance()
_validate_index_accelerator(acc)
index_type_map = {
"IvfFlat": "IVF_FLAT",
"IvfSq": "IVF_SQ",
@@ -4509,7 +4555,10 @@ class LanceTable(Table):
if num_bits is not None:
quantization_kwargs["num_bits"] = num_bits
dataset.create_index(
_create_accelerated_index(
dataset,
exist_ok=exist_ok,
name=name,
column=column,
index_type=lance_index_type,
metric=getattr(config, "distance_type", "l2"),
@@ -4531,6 +4580,7 @@ class LanceTable(Table):
self._table.create_index(
column,
replace=replace,
exist_ok=exist_ok,
config=config,
wait_timeout=wait_timeout,
name=name,
@@ -4546,6 +4596,7 @@ class LanceTable(Table):
replace: Optional[bool] = None,
wait_timeout: Optional[timedelta] = None,
name: Optional[str] = None,
exist_ok: bool = False,
train: bool = True,
) -> Job:
"""Create an index, returning a handle to the indexing job.
@@ -4564,6 +4615,7 @@ class LanceTable(Table):
self._table.create_index_async(
column,
replace=replace,
exist_ok=exist_ok,
config=config,
wait_timeout=wait_timeout,
name=name,
@@ -4684,6 +4736,7 @@ class LanceTable(Table):
index_type: ScalarIndexType = "BTREE",
wait_timeout: Optional[timedelta] = None,
name: Optional[str] = None,
exist_ok: bool = False,
):
"""Create a scalar index on a column.
@@ -4710,6 +4763,7 @@ class LanceTable(Table):
self._table.create_index(
column,
replace=replace,
exist_ok=exist_ok,
config=config,
wait_timeout=wait_timeout,
name=name,
@@ -4747,6 +4801,7 @@ class LanceTable(Table):
document_granularity: DocumentGranularity = DocumentGranularity.ROW,
wait_timeout: Optional[timedelta] = None,
name: Optional[str] = None,
exist_ok: bool = False,
):
"""Create a full-text search index on a column.
@@ -4808,6 +4863,7 @@ class LanceTable(Table):
self._table.create_index(
field_names,
replace=replace,
exist_ok=exist_ok,
config=config,
name=name,
wait_timeout=wait_timeout,
@@ -6806,6 +6862,7 @@ class AsyncTable:
] = None,
wait_timeout: Optional[timedelta] = None,
name: Optional[str] = None,
exist_ok: bool = False,
train: bool = True,
):
"""Create an index to speed up queries
@@ -6836,6 +6893,9 @@ class AsyncTable:
If not set, the call returns once the build is scheduled.
name: str, optional
The name of the index. If not provided, a default name will be generated.
exist_ok: bool, default False
Preserve an existing index with the requested or derived name. This
takes precedence over replace and keeps its definition and build.
train: bool, default True
Whether to train the index with existing data. Vector indices always train
with existing data.
@@ -6891,6 +6951,7 @@ class AsyncTable:
column,
index=config,
replace=replace,
exist_ok=exist_ok,
wait_timeout=wait_timeout,
name=name,
train=train,
@@ -6928,6 +6989,7 @@ class AsyncTable:
] = None,
wait_timeout: Optional[timedelta] = None,
name: Optional[str] = None,
exist_ok: bool = False,
train: bool = True,
) -> AsyncJob:
"""Create an index, returning a handle to the indexing job.
@@ -6946,6 +7008,7 @@ class AsyncTable:
column,
index=config,
replace=replace,
exist_ok=exist_ok,
wait_timeout=wait_timeout,
name=name,
train=train,
+42
View File
@@ -621,3 +621,45 @@ def test_index_statistics_index_type_lists_all_supported_values():
set(get_args(get_type_hints(IndexStatistics)["index_type"]))
== expected_index_types
)
@pytest.mark.asyncio
@pytest.mark.parametrize("background", [False, True])
@pytest.mark.parametrize(
"column,config", [("id", BTree()), ("vector", IvfFlat(num_partitions=1))]
)
async def test_index_exist_ok_preserves_existing_index(
some_table, background, column, config
):
await some_table.create_index(column, config=config, exist_ok=True)
version = await some_table.version()
original = await some_table.list_indices()
if background:
job = await some_table.create_index_async(
column, config=config, replace=True, exist_ok=True
)
await job.wait()
else:
await some_table.create_index(
column, config=config, replace=True, exist_ok=True
)
assert await some_table.version() == version
assert [(i.name, i.index_type) for i in await some_table.list_indices()] == [
(i.name, i.index_type) for i in original
]
@pytest.mark.asyncio
@pytest.mark.parametrize("background", [False, True])
async def test_named_index_exist_ok_checks_before_column_validation(
some_table, background
):
await some_table.create_index("id", config=BTree(), name="kept")
version = await some_table.version()
method = some_table.create_index_async if background else some_table.create_index
result = await method("missing", config=BTree(), name="kept", exist_ok=True)
if background:
assert result.id is None
await result.wait()
assert await some_table.version() == version
assert [index.name for index in await some_table.list_indices()] == ["kept"]
+52
View File
@@ -45,6 +45,7 @@ from lancedb.query import (
LanceVectorQueryBuilder,
MatchQuery,
)
from lancedb.index import BTree
from lancedb.remote import ClientConfig
from lancedb.remote.errors import (
AlreadyExistsError,
@@ -7420,3 +7421,54 @@ async def test_remote_blob_bytes_hybrid_rejects_same_version_different_incarnati
if use_async:
await result
assert not any(path == "fetch_blobs" for path, _ in requests)
@pytest.mark.asyncio
@pytest.mark.parametrize("is_async", [False, True])
@pytest.mark.parametrize("background", [False, True])
@pytest.mark.parametrize("kwargs", [{}, {"exist_ok": False}, {"exist_ok": True}])
async def test_remote_index_exist_ok_wire(is_async, background, kwargs):
received = []
def handler(request):
parsed = urlparse(request.path)
body = request.rfile.read(int(request.headers.get("Content-Length", 0)))
if parsed.path.endswith("/create_index/"):
received.append((parse_qs(parsed.query), json.loads(body)))
payload = {}
elif parsed.path.endswith("/describe/"):
payload = dict(
version=1,
schema=dict(
fields=[dict(name="id", type={"type": "int64"}, nullable=False)]
),
)
elif parsed.path.endswith("/create/"):
payload = {}
else:
raise AssertionError(request.path)
request.send_response(200)
request.send_header("Content-Type", "application/json")
request.end_headers()
request.wfile.write(json.dumps(payload).encode())
method = "create_index_async" if background else "create_index"
if is_async:
async with mock_lancedb_connection_async(handler) as db:
table = await db.create_table("test", [{"id": 1}])
result = await getattr(table, method)("id", config=BTree(), **kwargs)
if background:
await result.wait()
else:
with mock_lancedb_connection(handler) as db:
table = db.create_table("test", [{"id": 1}])
result = getattr(table, method)("id", config=BTree(), **kwargs)
if background:
result.wait()
query, body = received[0]
assert query == ({"exist_ok": ["true"]} if kwargs.get("exist_ok") else {})
assert "exist_ok" not in body
if kwargs.get("exist_ok"):
assert body["replace"] is False
else:
assert "replace" not in body
+96
View File
@@ -1837,6 +1837,7 @@ def test_create_index_method(mock_create_index, mem_db: DBConnection):
wait_timeout=None,
name=None,
train=True,
exist_ok=False,
)
# Test with target_partition_size
@@ -1862,6 +1863,7 @@ def test_create_index_method(mock_create_index, mem_db: DBConnection):
wait_timeout=None,
name=None,
train=True,
exist_ok=False,
)
# target_partition_size has a default value,
@@ -1886,6 +1888,7 @@ def test_create_index_method(mock_create_index, mem_db: DBConnection):
wait_timeout=None,
name=None,
train=True,
exist_ok=False,
)
table.create_index(
@@ -1902,6 +1905,7 @@ def test_create_index_method(mock_create_index, mem_db: DBConnection):
wait_timeout=None,
name=None,
train=True,
exist_ok=False,
)
table.create_index(
@@ -1922,6 +1926,7 @@ def test_create_index_method(mock_create_index, mem_db: DBConnection):
wait_timeout=None,
name=None,
train=True,
exist_ok=False,
)
table.create_index(
@@ -1942,6 +1947,7 @@ def test_create_index_method(mock_create_index, mem_db: DBConnection):
wait_timeout=None,
name=None,
train=True,
exist_ok=False,
)
@@ -2073,6 +2079,7 @@ def test_create_index_name_and_train_parameters(
wait_timeout=None,
name="my_custom_index",
train=True,
exist_ok=False,
)
# Test with train=False
@@ -2084,6 +2091,7 @@ def test_create_index_name_and_train_parameters(
wait_timeout=None,
name=None,
train=False,
exist_ok=False,
)
# Test with both name and train
@@ -2095,6 +2103,7 @@ def test_create_index_name_and_train_parameters(
wait_timeout=None,
name="my_index_name",
train=True,
exist_ok=False,
)
@@ -2132,6 +2141,7 @@ def test_create_index_new_api(mock_create_index, mem_db: DBConnection):
wait_timeout=None,
name=None,
train=True,
exist_ok=False,
)
# Scalar index via new API
@@ -2143,6 +2153,7 @@ def test_create_index_new_api(mock_create_index, mem_db: DBConnection):
wait_timeout=None,
name=None,
train=True,
exist_ok=False,
)
# FTS index via new API
@@ -2154,6 +2165,7 @@ def test_create_index_new_api(mock_create_index, mem_db: DBConnection):
wait_timeout=None,
name=None,
train=True,
exist_ok=False,
)
@@ -6292,3 +6304,87 @@ async def test_async_computed_auto_refresh_uses_server_interval(tmp_path):
metadata = (await table.schema()).field("doubled").metadata
assert metadata[b"computed_column.auto_refresh_enabled"] == b"true"
assert b"computed_column.auto_refresh_interval_seconds" not in metadata
@pytest.mark.parametrize(
"method",
["create_index", "create_index_async", "create_scalar_index", "create_fts_index"],
)
@pytest.mark.parametrize("name", [None, "kept_index"])
def test_index_exist_ok_preserves_definition_and_version(mem_db, method, name):
table = mem_db.create_table(
"keep_index", [{"id": i, "text": f"document {i}"} for i in range(20)]
)
column = "text" if method == "create_fts_index" else "id"
config = FTS() if column == "text" else BTree()
table.create_index(column, config=config, name=name, exist_ok=True)
before = table.version
original = table.list_indices()
kwargs = dict(name=name, exist_ok=True)
if method in ("create_index", "create_index_async"):
kwargs["config"] = config
with warnings.catch_warnings():
warnings.simplefilter("ignore", DeprecationWarning)
result = getattr(table, method)(column, **kwargs)
if method == "create_index_async":
result.wait()
assert table.version == before
assert [(index.name, index.index_type) for index in table.list_indices()] == [
(index.name, index.index_type) for index in original
]
with pytest.raises((ValueError, RuntimeError), match="already exists"):
table.create_index(column, config=config, name=name, replace=False)
table.create_index(column, config=config, name=name)
assert table.version > before
@pytest.mark.parametrize(
"column,name,existing",
[
("vector", "kept", "kept"),
("`docs`.`nested.dot`", None, "docs.`nested.dot`_idx"),
],
)
def test_accelerated_index_exist_ok_skips_gpu_validation(column, name, existing):
dataset = MagicMock()
dataset.list_indices.return_value = [{"name": existing}]
with patch("lancedb.table._validate_index_accelerator") as validate:
table_module._create_accelerated_index(
dataset, column=column, accelerator="cuda", name=name, exist_ok=True
)
validate.assert_not_called()
dataset.create_index.assert_not_called()
def test_accelerated_index_exist_ok_preserves_concurrent_winner():
dataset = MagicMock()
dataset.list_indices.side_effect = [[], [{"name": "vector_idx"}]]
dataset.create_index.side_effect = RuntimeError("concurrent create")
with patch("lancedb.table._validate_index_accelerator"):
table_module._create_accelerated_index(
dataset, column="vector", accelerator="cuda", replace=True, exist_ok=True
)
assert dataset.create_index.call_args.kwargs["replace"] is False
assert dataset.create_index.call_args.kwargs["name"] == "vector_idx"
@pytest.mark.parametrize("column", ["`user-id`", "`user:id`", "nested.`user-id`"])
def test_index_exist_ok_matches_lance_default_name(mem_db, column):
table = mem_db.create_table(
"quoted_index",
[{"user-id": 1, "user:id": 2, "nested": {"user-id": 4}}],
)
table.create_index(column, config=BTree())
before = table.version
table.create_index(column, config=BTree(), exist_ok=True)
assert table.version == before
assert len(table.list_indices()) == 1
def test_fts_exist_ok_matches_quoted_default_name(mem_db):
table = mem_db.create_table("quoted_fts", [{"body-text": "keep this document"}])
table.create_index("`body-text`", config=FTS())
before = table.version
table.create_index("`body-text`", config=FTS(), exist_ok=True)
assert table.version == before
assert len(table.list_indices()) == 1
+5 -1
View File
@@ -115,7 +115,11 @@ def test_vector_tuning_options(sync_table):
}.items():
assert getattr(config, name) == value
assert kwargs == dict(
wait_timeout=timeout, name="custom_idx", replace=False, train=False
wait_timeout=timeout,
name="custom_idx",
replace=False,
train=False,
exist_ok=False,
)
+8 -2
View File
@@ -1628,7 +1628,8 @@ impl Table {
})
}
#[pyo3(signature = (column, index=None, replace=None, wait_timeout=None, *, name=None, train=None))]
#[allow(clippy::too_many_arguments)]
#[pyo3(signature = (column, index=None, replace=None, wait_timeout=None, *, name=None, train=None, exist_ok=false))]
pub fn create_index<'a>(
self_: PyRef<'a, Self>,
column: String,
@@ -1637,6 +1638,7 @@ impl Table {
wait_timeout: Option<Bound<'_, PyAny>>,
name: Option<String>,
train: Option<bool>,
exist_ok: bool,
) -> PyResult<Bound<'a, PyAny>> {
let index = extract_index_params(&index)?;
let timeout = wait_timeout
@@ -1645,6 +1647,7 @@ impl Table {
let mut op = self_
.inner_ref()?
.create_index_with_timeout(&[column], index, timeout);
op = op.exist_ok(exist_ok);
if let Some(replace) = replace {
op = op.replace(replace);
}
@@ -1661,7 +1664,8 @@ impl Table {
})
}
#[pyo3(signature = (column, index=None, replace=None, wait_timeout=None, *, name=None, train=None))]
#[allow(clippy::too_many_arguments)]
#[pyo3(signature = (column, index=None, replace=None, wait_timeout=None, *, name=None, train=None, exist_ok=false))]
pub fn create_index_async<'a>(
self_: PyRef<'a, Self>,
column: String,
@@ -1670,6 +1674,7 @@ impl Table {
wait_timeout: Option<Bound<'_, PyAny>>,
name: Option<String>,
train: Option<bool>,
exist_ok: bool,
) -> PyResult<Bound<'a, PyAny>> {
let index = extract_index_params(&index)?;
let timeout = wait_timeout
@@ -1678,6 +1683,7 @@ impl Table {
let mut op = self_
.inner_ref()?
.create_index_with_timeout(&[column], index, timeout);
op = op.exist_ok(exist_ok);
if let Some(replace) = replace {
op = op.replace(replace);
}
+17
View File
@@ -220,6 +220,7 @@ pub struct IndexBuilder {
pub(crate) index: Index,
pub(crate) columns: Vec<String>,
pub(crate) replace: bool,
pub(crate) exist_ok: bool,
pub(crate) wait_timeout: Option<Duration>,
pub(crate) train: bool,
pub(crate) name: Option<String>,
@@ -232,6 +233,7 @@ impl IndexBuilder {
index,
columns,
replace: true,
exist_ok: false,
train: true,
wait_timeout: None,
name: None,
@@ -324,6 +326,21 @@ impl IndexBuilder {
self
}
/// Preserve an existing index with the requested name. Defaults to false.
/// This takes precedence over `replace`; a skipped creation does not rebuild.
///
/// ```no_run
/// # async fn example(table: &lancedb::Table) -> lancedb::Result<()> {
/// table.create_index(&["id"], lancedb::index::Index::Auto)
/// .exist_ok(true).execute().await?;
/// # Ok(())
/// # }
/// ```
pub fn exist_ok(mut self, exist_ok: bool) -> Self {
self.exist_ok = exist_ok;
self
}
/// Duration of time to wait for asynchronous indexing to complete. If not set,
/// `create_index()` will not wait.
///
+41 -1
View File
@@ -701,9 +701,13 @@ impl<S: HttpSend> RemoteTable<S> {
async fn submit_create_index(&self, mut index: IndexBuilder) -> Result<Option<String>> {
self.check_mutable().await?;
let request = self
let mut request = self
.client
.post(&format!("/v1/table/{}/create_index/", self.identifier));
if index.exist_ok {
request = request.query(&[("exist_ok", true)]);
index.replace = false;
}
let column = match index.columns.len() {
0 => {
@@ -18101,6 +18105,42 @@ mod tests {
branch.restore().await.unwrap();
}
#[tokio::test]
async fn test_index_exist_ok_query_and_replacement_policy() {
for exist_ok in [false, true] {
for background in [false, true] {
let table = Table::new_with_handler("my_table", move |request| {
match request.url().path() {
"/v1/table/my_table/describe/" => simple_describe_response(),
"/v1/table/my_table/create_index/" => {
assert_eq!(request.url().query(), exist_ok.then_some("exist_ok=true"));
let body = request_body_json(&request);
assert_eq!(
body.get("replace"),
exist_ok.then_some(&serde_json::Value::Bool(false))
);
assert!(body.get("exist_ok").is_none());
http::Response::builder()
.status(200)
.body("{}".into())
.unwrap()
}
path => panic!("unexpected request path: {path}"),
}
});
let builder = table
.create_index(&["id"], Index::BTree(Default::default()))
.replace(true)
.exist_ok(exist_ok);
if background {
builder.execute_async().await.unwrap().wait().await.unwrap();
} else {
builder.execute().await.unwrap();
}
}
}
}
#[tokio::test]
async fn test_branch_create_index_carries_branch_in_body() {
use lance::dataset::refs::Ref;
+6 -2
View File
@@ -4145,14 +4145,18 @@ impl BaseTable for NativeTable {
}
async fn create_index(&self, opts: IndexBuilder) -> Result<()> {
let prepared = self.prepare_index(&opts).await?;
let Some(prepared) = self.prepare_index(&opts).await? else {
return Ok(());
};
self.build_index(opts, prepared).await
}
async fn create_index_async(&self, opts: IndexBuilder) -> Result<Job> {
// Prepare before spawning so bad input is reported by this call rather
// than only by the job.
let prepared = self.prepare_index(&opts).await?;
let Some(prepared) = self.prepare_index(&opts).await? else {
return Ok(Job::new_done());
};
let table = self.clone();
Ok(Job::spawned(tokio::spawn(async move {
table.build_index(opts, prepared).await
+139 -7
View File
@@ -115,14 +115,24 @@ impl NativeTable {
pub(super) async fn prepare_index(
&self,
opts: &crate::index::IndexBuilder,
) -> Result<PreparedIndex> {
) -> Result<Option<PreparedIndex>> {
self.dataset.ensure_mutable()?;
let dataset = self.dataset.get().await?;
if opts.exist_ok
&& let Some(name) = &opts.name
&& dataset
.load_indices()
.await?
.iter()
.any(|index| &index.name == name)
{
return Ok(None);
}
if opts.columns.len() != 1 {
return Err(Error::Schema {
message: "Multi-column (composite) indices are not yet supported".to_string(),
});
}
self.dataset.ensure_mutable()?;
let dataset = self.dataset.get().await?;
let (column, field) = if let Index::FTS(params) = &opts.index {
let resolved = resolve_lance_fts_field_path(dataset.schema(), &opts.columns[0])?;
if params.get_document_granularity().is_list_element() && resolved.list_depth == 0 {
@@ -137,9 +147,48 @@ impl NativeTable {
} else {
Self::resolve_index_field(dataset.schema(), &opts.columns[0])?
};
if let Some(name) = Self::guarded_index_name(opts, &column)?
&& dataset
.load_indices()
.await?
.iter()
.any(|index| index.name == name)
{
return Ok(None);
}
let params = self.make_index_params(&field, opts.index.clone()).await?;
let index_type = self.get_index_type_for_field(&field, &opts.index);
Ok((column, params, index_type))
Ok(Some((column, params, index_type)))
}
fn guarded_index_name(
opts: &crate::index::IndexBuilder,
column: &str,
) -> Result<Option<String>> {
let name = if opts.exist_ok {
Some(if let Some(name) = &opts.name {
name.clone()
} else {
let names = lance_core::datatypes::parse_field_path(column)?;
let default_name = if matches!(&opts.index, Index::FTS(_))
|| names.iter().any(|name| name.contains('.'))
{
column.to_string()
} else {
names.join(".")
};
let suffix = if matches!(&opts.index, Index::FTS(params) if params.get_document_granularity().is_list_element())
{
"_list_element_idx"
} else {
"_idx"
};
format!("{default_name}{suffix}")
})
} else {
None
};
Ok(name)
}
/// Builds a prepared index and publishes the new dataset version.
@@ -150,16 +199,42 @@ impl NativeTable {
) -> Result<()> {
let (column, lance_idx_params, index_type) = prepared;
let mut dataset = (*self.dataset.get().await?).clone();
let guarded_name = Self::guarded_index_name(&opts, &column)?;
if let Some(name) = &guarded_name
&& dataset
.load_indices()
.await?
.iter()
.any(|index| &index.name == name)
{
return Ok(());
}
let columns = [column.as_str()];
let mut builder = dataset
.create_index_builder(&columns, index_type, lance_idx_params.as_ref())
.train(opts.train)
.replace(opts.replace);
.replace(opts.replace && !opts.exist_ok);
if let Some(name) = opts.name {
if let Some(name) = guarded_name.clone().or(opts.name) {
builder = builder.name(name);
}
builder.await?;
if let Err(error) = builder.await {
if let Some(name) = guarded_name {
self.dataset.reload().await?;
if self
.dataset
.get()
.await?
.load_indices()
.await?
.iter()
.any(|index| index.name == name)
{
return Ok(());
}
}
return Err(error.into());
}
// A writer already open was built before this index existed, so it has
// to pick it up here or a read needing it is refused until it reopens.
self.dataset
@@ -568,6 +643,63 @@ mod tests {
Ok(FixedSizeListArray::from(data))
}
#[tokio::test]
async fn test_concurrent_index_exist_ok_preserves_single_identity() {
let dir = tempdir().unwrap();
let db = connect(dir.path().to_str().unwrap())
.execute()
.await
.unwrap();
let batch = record_batch!(("id", Int32, (0..512).collect::<Vec<_>>())).unwrap();
let table = db.create_table("t", batch).execute().await.unwrap();
let second = db.open_table("t").execute().await.unwrap();
let (left, right) = tokio::join!(
table
.create_index(&["id"], Index::BTree(Default::default()))
.exist_ok(true)
.execute(),
second
.create_index(&["id"], Index::BTree(Default::default()))
.exist_ok(true)
.execute()
);
left.unwrap();
right.unwrap();
table.checkout_latest().await.unwrap();
let indices = table
.as_native()
.unwrap()
.dataset
.get()
.await
.unwrap()
.load_indices()
.await
.unwrap();
assert_eq!(indices.len(), 1);
let original_uuid = indices[0].uuid;
let version = table.version().await.unwrap();
table
.create_index(&["id"], Index::BTree(Default::default()))
.replace(true)
.exist_ok(true)
.execute()
.await
.unwrap();
assert_eq!(table.version().await.unwrap(), version);
let indices = table
.as_native()
.unwrap()
.dataset
.get()
.await
.unwrap()
.load_indices()
.await
.unwrap();
assert_eq!(indices[0].uuid, original_uuid);
}
#[tokio::test]
async fn test_create_index() {
use std::iter::repeat_with;