feat: support exist_ok for index creation (#4993)

This commit is contained in:
Jack Ye
2026-11-13 19:21:50 -08:00
committed by GitHub
parent 81d04b8c36
commit f6311ae7e9
13 changed files with 525 additions and 17 deletions
+2
View File
@@ -591,6 +591,7 @@ class Table:
*,
name: Optional[str],
train: Optional[bool],
exist_ok: bool = False,
): ...
async def create_index_async(
self,
@@ -615,6 +616,7 @@ class Table:
*,
name: Optional[str],
train: Optional[bool],
exist_ok: bool = False,
) -> Job: ...
async def list_versions(self) -> List[Dict[str, Any]]: ...
async def version(self) -> int: ...
+10
View File
@@ -369,6 +369,7 @@ class RemoteTable(Table):
index_type: ScalarIndexType = "BTREE",
wait_timeout: Optional[timedelta] = None,
name: Optional[str] = None,
exist_ok: bool = False,
):
"""Create a scalar index on a column.
@@ -395,6 +396,7 @@ class RemoteTable(Table):
self._table.create_index(
column,
replace=replace,
exist_ok=exist_ok,
config=config,
wait_timeout=wait_timeout,
name=name,
@@ -433,6 +435,7 @@ class RemoteTable(Table):
document_granularity: DocumentGranularity = DocumentGranularity.ROW,
wait_timeout: Optional[timedelta] = None,
name: Optional[str] = None,
exist_ok: bool = False,
):
"""Create a full-text search index on a column.
@@ -492,6 +495,7 @@ class RemoteTable(Table):
self._table.create_index(
field_names,
replace=replace,
exist_ok=exist_ok,
config=config,
name=name,
wait_timeout=wait_timeout,
@@ -516,6 +520,7 @@ class RemoteTable(Table):
replace: bool = ...,
wait_timeout: Optional[timedelta] = ...,
name: Optional[str] = ...,
exist_ok: bool = ...,
train: bool = ...,
) -> None: ...
@@ -541,6 +546,7 @@ class RemoteTable(Table):
*,
wait_timeout: Optional[timedelta] = ...,
name: Optional[str] = ...,
exist_ok: bool = ...,
train: bool = ...,
target_partition_size: Optional[int] = ...,
) -> None: ...
@@ -572,6 +578,7 @@ class RemoteTable(Table):
config: Optional[IndexConfigType] = None,
wait_timeout: Optional[timedelta] = None,
name: Optional[str] = None,
exist_ok: bool = False,
train: bool = True,
target_partition_size: Optional[int] = None,
):
@@ -675,6 +682,7 @@ class RemoteTable(Table):
self._table.create_index(
column,
replace=replace,
exist_ok=exist_ok,
config=config,
wait_timeout=wait_timeout,
name=name,
@@ -690,6 +698,7 @@ class RemoteTable(Table):
replace: Optional[bool] = None,
wait_timeout: Optional[timedelta] = None,
name: Optional[str] = None,
exist_ok: bool = False,
train: bool = True,
) -> Job:
"""Create an index, returning a handle to the indexing job.
@@ -702,6 +711,7 @@ class RemoteTable(Table):
self._table.create_index_async(
column,
replace=replace,
exist_ok=exist_ok,
config=config,
wait_timeout=wait_timeout,
name=name,
+67 -4
View File
@@ -168,6 +168,36 @@ def _validate_index_accelerator(accelerator: Any) -> None:
raise RuntimeError(_CUDA_UNAVAILABLE_MESSAGE) from err
def _create_accelerated_index(dataset, *, exist_ok=False, name=None, **options):
if exist_ok:
parts = _parse_field_path(options["column"])
default_name = ".".join(parts)
if any("." in part for part in parts):
default_name = ".".join(
"`" + part.replace("`", "``") + "`"
if any(
not character.isalnum() and character != "_" for character in part
)
else part
for part in parts
)
name = name or f"{default_name}_idx"
dataset.checkout_latest()
if any(index["name"] == name for index in dataset.list_indices()):
return
options["replace"] = False
options["name"] = name
_validate_index_accelerator(options.get("accelerator"))
try:
dataset.create_index(**options)
except Exception:
if exist_ok:
dataset.checkout_latest()
if any(index["name"] == name for index in dataset.list_indices()):
return
raise
def _polars_predicate_pushdown_barrier(frame: Any) -> Any:
"""Return a Polars frame unchanged while blocking predicate pushdown."""
return frame
@@ -1834,6 +1864,7 @@ class Table(ABC):
replace: bool = ...,
wait_timeout: Optional[timedelta] = ...,
name: Optional[str] = ...,
exist_ok: bool = ...,
train: bool = ...,
) -> None: ...
@@ -1859,6 +1890,7 @@ class Table(ABC):
*,
wait_timeout: Optional[timedelta] = ...,
name: Optional[str] = ...,
exist_ok: bool = ...,
train: bool = ...,
target_partition_size: Optional[int] = ...,
) -> None: ...
@@ -1890,6 +1922,7 @@ class Table(ABC):
config: Optional[IndexConfigType] = None,
wait_timeout: Optional[timedelta] = None,
name: Optional[str] = None,
exist_ok: bool = False,
train: bool = True,
target_partition_size: Optional[int] = None,
):
@@ -1913,6 +1946,9 @@ class Table(ABC):
BTree, Bitmap, LabelList, Fm, NGram, ZoneMap, FTS.
replace : bool, default True
Whether to replace an existing index on this column.
exist_ok : bool, default False
Preserve the existing index with the requested or derived name.
Takes precedence over replace and keeps its definition and build.
wait_timeout : timedelta, optional
LanceDB Enterprise only: how long to wait for the index build to finish.
If not set, the call returns once the build is scheduled.
@@ -1968,6 +2004,7 @@ class Table(ABC):
replace: Optional[bool] = None,
wait_timeout: Optional[timedelta] = None,
name: Optional[str] = None,
exist_ok: bool = False,
train: bool = True,
) -> Job:
"""Create an index, returning a handle to the indexing job.
@@ -2060,6 +2097,7 @@ class Table(ABC):
index_type: ScalarIndexType = "BTREE",
wait_timeout: Optional[timedelta] = None,
name: Optional[str] = None,
exist_ok: bool = False,
):
"""Create a scalar index on a column.
@@ -2144,6 +2182,7 @@ class Table(ABC):
document_granularity: DocumentGranularity = DocumentGranularity.ROW,
wait_timeout: Optional[timedelta] = None,
name: Optional[str] = None,
exist_ok: bool = False,
):
"""Create a full-text search index on the table.
@@ -4318,6 +4357,7 @@ class LanceTable(Table):
replace: bool = ...,
wait_timeout: Optional[timedelta] = ...,
name: Optional[str] = ...,
exist_ok: bool = ...,
train: bool = ...,
) -> None: ...
@@ -4343,6 +4383,7 @@ class LanceTable(Table):
*,
wait_timeout: Optional[timedelta] = ...,
name: Optional[str] = ...,
exist_ok: bool = ...,
train: bool = ...,
target_partition_size: Optional[int] = ...,
) -> None: ...
@@ -4374,6 +4415,7 @@ class LanceTable(Table):
config: Optional[IndexConfigType] = None,
wait_timeout: Optional[timedelta] = None,
name: Optional[str] = None,
exist_ok: bool = False,
train: bool = True,
target_partition_size: Optional[int] = None,
):
@@ -4397,6 +4439,9 @@ class LanceTable(Table):
BTree, Bitmap, LabelList, Fm, NGram, ZoneMap, FTS.
replace : bool, default True
Whether to replace an existing index on this column.
exist_ok : bool, default False
Preserve the existing index with the requested or derived name.
Takes precedence over replace and keeps its definition and build.
wait_timeout : timedelta, optional
Timeout to wait for async indexing to complete.
name : str, optional
@@ -4465,8 +4510,10 @@ class LanceTable(Table):
# Handle accelerator through pylance
if accelerator is not None:
dataset = self.to_lance()
_validate_index_accelerator(accelerator)
dataset.create_index(
_create_accelerated_index(
dataset,
exist_ok=exist_ok,
name=name,
column=column,
index_type=index_type,
metric=metric,
@@ -4492,7 +4539,6 @@ class LanceTable(Table):
if acc is not None:
# Dispatch to pylance for GPU acceleration
dataset = self.to_lance()
_validate_index_accelerator(acc)
index_type_map = {
"IvfFlat": "IVF_FLAT",
"IvfSq": "IVF_SQ",
@@ -4509,7 +4555,10 @@ class LanceTable(Table):
if num_bits is not None:
quantization_kwargs["num_bits"] = num_bits
dataset.create_index(
_create_accelerated_index(
dataset,
exist_ok=exist_ok,
name=name,
column=column,
index_type=lance_index_type,
metric=getattr(config, "distance_type", "l2"),
@@ -4531,6 +4580,7 @@ class LanceTable(Table):
self._table.create_index(
column,
replace=replace,
exist_ok=exist_ok,
config=config,
wait_timeout=wait_timeout,
name=name,
@@ -4546,6 +4596,7 @@ class LanceTable(Table):
replace: Optional[bool] = None,
wait_timeout: Optional[timedelta] = None,
name: Optional[str] = None,
exist_ok: bool = False,
train: bool = True,
) -> Job:
"""Create an index, returning a handle to the indexing job.
@@ -4564,6 +4615,7 @@ class LanceTable(Table):
self._table.create_index_async(
column,
replace=replace,
exist_ok=exist_ok,
config=config,
wait_timeout=wait_timeout,
name=name,
@@ -4684,6 +4736,7 @@ class LanceTable(Table):
index_type: ScalarIndexType = "BTREE",
wait_timeout: Optional[timedelta] = None,
name: Optional[str] = None,
exist_ok: bool = False,
):
"""Create a scalar index on a column.
@@ -4710,6 +4763,7 @@ class LanceTable(Table):
self._table.create_index(
column,
replace=replace,
exist_ok=exist_ok,
config=config,
wait_timeout=wait_timeout,
name=name,
@@ -4747,6 +4801,7 @@ class LanceTable(Table):
document_granularity: DocumentGranularity = DocumentGranularity.ROW,
wait_timeout: Optional[timedelta] = None,
name: Optional[str] = None,
exist_ok: bool = False,
):
"""Create a full-text search index on a column.
@@ -4808,6 +4863,7 @@ class LanceTable(Table):
self._table.create_index(
field_names,
replace=replace,
exist_ok=exist_ok,
config=config,
name=name,
wait_timeout=wait_timeout,
@@ -6806,6 +6862,7 @@ class AsyncTable:
] = None,
wait_timeout: Optional[timedelta] = None,
name: Optional[str] = None,
exist_ok: bool = False,
train: bool = True,
):
"""Create an index to speed up queries
@@ -6836,6 +6893,9 @@ class AsyncTable:
If not set, the call returns once the build is scheduled.
name: str, optional
The name of the index. If not provided, a default name will be generated.
exist_ok: bool, default False
Preserve an existing index with the requested or derived name. This
takes precedence over replace and keeps its definition and build.
train: bool, default True
Whether to train the index with existing data. Vector indices always train
with existing data.
@@ -6891,6 +6951,7 @@ class AsyncTable:
column,
index=config,
replace=replace,
exist_ok=exist_ok,
wait_timeout=wait_timeout,
name=name,
train=train,
@@ -6928,6 +6989,7 @@ class AsyncTable:
] = None,
wait_timeout: Optional[timedelta] = None,
name: Optional[str] = None,
exist_ok: bool = False,
train: bool = True,
) -> AsyncJob:
"""Create an index, returning a handle to the indexing job.
@@ -6946,6 +7008,7 @@ class AsyncTable:
column,
index=config,
replace=replace,
exist_ok=exist_ok,
wait_timeout=wait_timeout,
name=name,
train=train,
+42
View File
@@ -621,3 +621,45 @@ def test_index_statistics_index_type_lists_all_supported_values():
set(get_args(get_type_hints(IndexStatistics)["index_type"]))
== expected_index_types
)
@pytest.mark.asyncio
@pytest.mark.parametrize("background", [False, True])
@pytest.mark.parametrize(
"column,config", [("id", BTree()), ("vector", IvfFlat(num_partitions=1))]
)
async def test_index_exist_ok_preserves_existing_index(
some_table, background, column, config
):
await some_table.create_index(column, config=config, exist_ok=True)
version = await some_table.version()
original = await some_table.list_indices()
if background:
job = await some_table.create_index_async(
column, config=config, replace=True, exist_ok=True
)
await job.wait()
else:
await some_table.create_index(
column, config=config, replace=True, exist_ok=True
)
assert await some_table.version() == version
assert [(i.name, i.index_type) for i in await some_table.list_indices()] == [
(i.name, i.index_type) for i in original
]
@pytest.mark.asyncio
@pytest.mark.parametrize("background", [False, True])
async def test_named_index_exist_ok_checks_before_column_validation(
some_table, background
):
await some_table.create_index("id", config=BTree(), name="kept")
version = await some_table.version()
method = some_table.create_index_async if background else some_table.create_index
result = await method("missing", config=BTree(), name="kept", exist_ok=True)
if background:
assert result.id is None
await result.wait()
assert await some_table.version() == version
assert [index.name for index in await some_table.list_indices()] == ["kept"]
+52
View File
@@ -45,6 +45,7 @@ from lancedb.query import (
LanceVectorQueryBuilder,
MatchQuery,
)
from lancedb.index import BTree
from lancedb.remote import ClientConfig
from lancedb.remote.errors import (
AlreadyExistsError,
@@ -7420,3 +7421,54 @@ async def test_remote_blob_bytes_hybrid_rejects_same_version_different_incarnati
if use_async:
await result
assert not any(path == "fetch_blobs" for path, _ in requests)
@pytest.mark.asyncio
@pytest.mark.parametrize("is_async", [False, True])
@pytest.mark.parametrize("background", [False, True])
@pytest.mark.parametrize("kwargs", [{}, {"exist_ok": False}, {"exist_ok": True}])
async def test_remote_index_exist_ok_wire(is_async, background, kwargs):
received = []
def handler(request):
parsed = urlparse(request.path)
body = request.rfile.read(int(request.headers.get("Content-Length", 0)))
if parsed.path.endswith("/create_index/"):
received.append((parse_qs(parsed.query), json.loads(body)))
payload = {}
elif parsed.path.endswith("/describe/"):
payload = dict(
version=1,
schema=dict(
fields=[dict(name="id", type={"type": "int64"}, nullable=False)]
),
)
elif parsed.path.endswith("/create/"):
payload = {}
else:
raise AssertionError(request.path)
request.send_response(200)
request.send_header("Content-Type", "application/json")
request.end_headers()
request.wfile.write(json.dumps(payload).encode())
method = "create_index_async" if background else "create_index"
if is_async:
async with mock_lancedb_connection_async(handler) as db:
table = await db.create_table("test", [{"id": 1}])
result = await getattr(table, method)("id", config=BTree(), **kwargs)
if background:
await result.wait()
else:
with mock_lancedb_connection(handler) as db:
table = db.create_table("test", [{"id": 1}])
result = getattr(table, method)("id", config=BTree(), **kwargs)
if background:
result.wait()
query, body = received[0]
assert query == ({"exist_ok": ["true"]} if kwargs.get("exist_ok") else {})
assert "exist_ok" not in body
if kwargs.get("exist_ok"):
assert body["replace"] is False
else:
assert "replace" not in body
+96
View File
@@ -1837,6 +1837,7 @@ def test_create_index_method(mock_create_index, mem_db: DBConnection):
wait_timeout=None,
name=None,
train=True,
exist_ok=False,
)
# Test with target_partition_size
@@ -1862,6 +1863,7 @@ def test_create_index_method(mock_create_index, mem_db: DBConnection):
wait_timeout=None,
name=None,
train=True,
exist_ok=False,
)
# target_partition_size has a default value,
@@ -1886,6 +1888,7 @@ def test_create_index_method(mock_create_index, mem_db: DBConnection):
wait_timeout=None,
name=None,
train=True,
exist_ok=False,
)
table.create_index(
@@ -1902,6 +1905,7 @@ def test_create_index_method(mock_create_index, mem_db: DBConnection):
wait_timeout=None,
name=None,
train=True,
exist_ok=False,
)
table.create_index(
@@ -1922,6 +1926,7 @@ def test_create_index_method(mock_create_index, mem_db: DBConnection):
wait_timeout=None,
name=None,
train=True,
exist_ok=False,
)
table.create_index(
@@ -1942,6 +1947,7 @@ def test_create_index_method(mock_create_index, mem_db: DBConnection):
wait_timeout=None,
name=None,
train=True,
exist_ok=False,
)
@@ -2073,6 +2079,7 @@ def test_create_index_name_and_train_parameters(
wait_timeout=None,
name="my_custom_index",
train=True,
exist_ok=False,
)
# Test with train=False
@@ -2084,6 +2091,7 @@ def test_create_index_name_and_train_parameters(
wait_timeout=None,
name=None,
train=False,
exist_ok=False,
)
# Test with both name and train
@@ -2095,6 +2103,7 @@ def test_create_index_name_and_train_parameters(
wait_timeout=None,
name="my_index_name",
train=True,
exist_ok=False,
)
@@ -2132,6 +2141,7 @@ def test_create_index_new_api(mock_create_index, mem_db: DBConnection):
wait_timeout=None,
name=None,
train=True,
exist_ok=False,
)
# Scalar index via new API
@@ -2143,6 +2153,7 @@ def test_create_index_new_api(mock_create_index, mem_db: DBConnection):
wait_timeout=None,
name=None,
train=True,
exist_ok=False,
)
# FTS index via new API
@@ -2154,6 +2165,7 @@ def test_create_index_new_api(mock_create_index, mem_db: DBConnection):
wait_timeout=None,
name=None,
train=True,
exist_ok=False,
)
@@ -6292,3 +6304,87 @@ async def test_async_computed_auto_refresh_uses_server_interval(tmp_path):
metadata = (await table.schema()).field("doubled").metadata
assert metadata[b"computed_column.auto_refresh_enabled"] == b"true"
assert b"computed_column.auto_refresh_interval_seconds" not in metadata
@pytest.mark.parametrize(
"method",
["create_index", "create_index_async", "create_scalar_index", "create_fts_index"],
)
@pytest.mark.parametrize("name", [None, "kept_index"])
def test_index_exist_ok_preserves_definition_and_version(mem_db, method, name):
table = mem_db.create_table(
"keep_index", [{"id": i, "text": f"document {i}"} for i in range(20)]
)
column = "text" if method == "create_fts_index" else "id"
config = FTS() if column == "text" else BTree()
table.create_index(column, config=config, name=name, exist_ok=True)
before = table.version
original = table.list_indices()
kwargs = dict(name=name, exist_ok=True)
if method in ("create_index", "create_index_async"):
kwargs["config"] = config
with warnings.catch_warnings():
warnings.simplefilter("ignore", DeprecationWarning)
result = getattr(table, method)(column, **kwargs)
if method == "create_index_async":
result.wait()
assert table.version == before
assert [(index.name, index.index_type) for index in table.list_indices()] == [
(index.name, index.index_type) for index in original
]
with pytest.raises((ValueError, RuntimeError), match="already exists"):
table.create_index(column, config=config, name=name, replace=False)
table.create_index(column, config=config, name=name)
assert table.version > before
@pytest.mark.parametrize(
"column,name,existing",
[
("vector", "kept", "kept"),
("`docs`.`nested.dot`", None, "docs.`nested.dot`_idx"),
],
)
def test_accelerated_index_exist_ok_skips_gpu_validation(column, name, existing):
dataset = MagicMock()
dataset.list_indices.return_value = [{"name": existing}]
with patch("lancedb.table._validate_index_accelerator") as validate:
table_module._create_accelerated_index(
dataset, column=column, accelerator="cuda", name=name, exist_ok=True
)
validate.assert_not_called()
dataset.create_index.assert_not_called()
def test_accelerated_index_exist_ok_preserves_concurrent_winner():
dataset = MagicMock()
dataset.list_indices.side_effect = [[], [{"name": "vector_idx"}]]
dataset.create_index.side_effect = RuntimeError("concurrent create")
with patch("lancedb.table._validate_index_accelerator"):
table_module._create_accelerated_index(
dataset, column="vector", accelerator="cuda", replace=True, exist_ok=True
)
assert dataset.create_index.call_args.kwargs["replace"] is False
assert dataset.create_index.call_args.kwargs["name"] == "vector_idx"
@pytest.mark.parametrize("column", ["`user-id`", "`user:id`", "nested.`user-id`"])
def test_index_exist_ok_matches_lance_default_name(mem_db, column):
table = mem_db.create_table(
"quoted_index",
[{"user-id": 1, "user:id": 2, "nested": {"user-id": 4}}],
)
table.create_index(column, config=BTree())
before = table.version
table.create_index(column, config=BTree(), exist_ok=True)
assert table.version == before
assert len(table.list_indices()) == 1
def test_fts_exist_ok_matches_quoted_default_name(mem_db):
table = mem_db.create_table("quoted_fts", [{"body-text": "keep this document"}])
table.create_index("`body-text`", config=FTS())
before = table.version
table.create_index("`body-text`", config=FTS(), exist_ok=True)
assert table.version == before
assert len(table.list_indices()) == 1
+5 -1
View File
@@ -115,7 +115,11 @@ def test_vector_tuning_options(sync_table):
}.items():
assert getattr(config, name) == value
assert kwargs == dict(
wait_timeout=timeout, name="custom_idx", replace=False, train=False
wait_timeout=timeout,
name="custom_idx",
replace=False,
train=False,
exist_ok=False,
)
+8 -2
View File
@@ -1628,7 +1628,8 @@ impl Table {
})
}
#[pyo3(signature = (column, index=None, replace=None, wait_timeout=None, *, name=None, train=None))]
#[allow(clippy::too_many_arguments)]
#[pyo3(signature = (column, index=None, replace=None, wait_timeout=None, *, name=None, train=None, exist_ok=false))]
pub fn create_index<'a>(
self_: PyRef<'a, Self>,
column: String,
@@ -1637,6 +1638,7 @@ impl Table {
wait_timeout: Option<Bound<'_, PyAny>>,
name: Option<String>,
train: Option<bool>,
exist_ok: bool,
) -> PyResult<Bound<'a, PyAny>> {
let index = extract_index_params(&index)?;
let timeout = wait_timeout
@@ -1645,6 +1647,7 @@ impl Table {
let mut op = self_
.inner_ref()?
.create_index_with_timeout(&[column], index, timeout);
op = op.exist_ok(exist_ok);
if let Some(replace) = replace {
op = op.replace(replace);
}
@@ -1661,7 +1664,8 @@ impl Table {
})
}
#[pyo3(signature = (column, index=None, replace=None, wait_timeout=None, *, name=None, train=None))]
#[allow(clippy::too_many_arguments)]
#[pyo3(signature = (column, index=None, replace=None, wait_timeout=None, *, name=None, train=None, exist_ok=false))]
pub fn create_index_async<'a>(
self_: PyRef<'a, Self>,
column: String,
@@ -1670,6 +1674,7 @@ impl Table {
wait_timeout: Option<Bound<'_, PyAny>>,
name: Option<String>,
train: Option<bool>,
exist_ok: bool,
) -> PyResult<Bound<'a, PyAny>> {
let index = extract_index_params(&index)?;
let timeout = wait_timeout
@@ -1678,6 +1683,7 @@ impl Table {
let mut op = self_
.inner_ref()?
.create_index_with_timeout(&[column], index, timeout);
op = op.exist_ok(exist_ok);
if let Some(replace) = replace {
op = op.replace(replace);
}