mirror of
https://github.com/lancedb/lancedb.git
synced 2026-11-16 07:36:50 +00:00
feat: support exist_ok for index creation (#4993)
This commit is contained in:
@@ -591,6 +591,7 @@ class Table:
|
||||
*,
|
||||
name: Optional[str],
|
||||
train: Optional[bool],
|
||||
exist_ok: bool = False,
|
||||
): ...
|
||||
async def create_index_async(
|
||||
self,
|
||||
@@ -615,6 +616,7 @@ class Table:
|
||||
*,
|
||||
name: Optional[str],
|
||||
train: Optional[bool],
|
||||
exist_ok: bool = False,
|
||||
) -> Job: ...
|
||||
async def list_versions(self) -> List[Dict[str, Any]]: ...
|
||||
async def version(self) -> int: ...
|
||||
|
||||
@@ -369,6 +369,7 @@ class RemoteTable(Table):
|
||||
index_type: ScalarIndexType = "BTREE",
|
||||
wait_timeout: Optional[timedelta] = None,
|
||||
name: Optional[str] = None,
|
||||
exist_ok: bool = False,
|
||||
):
|
||||
"""Create a scalar index on a column.
|
||||
|
||||
@@ -395,6 +396,7 @@ class RemoteTable(Table):
|
||||
self._table.create_index(
|
||||
column,
|
||||
replace=replace,
|
||||
exist_ok=exist_ok,
|
||||
config=config,
|
||||
wait_timeout=wait_timeout,
|
||||
name=name,
|
||||
@@ -433,6 +435,7 @@ class RemoteTable(Table):
|
||||
document_granularity: DocumentGranularity = DocumentGranularity.ROW,
|
||||
wait_timeout: Optional[timedelta] = None,
|
||||
name: Optional[str] = None,
|
||||
exist_ok: bool = False,
|
||||
):
|
||||
"""Create a full-text search index on a column.
|
||||
|
||||
@@ -492,6 +495,7 @@ class RemoteTable(Table):
|
||||
self._table.create_index(
|
||||
field_names,
|
||||
replace=replace,
|
||||
exist_ok=exist_ok,
|
||||
config=config,
|
||||
name=name,
|
||||
wait_timeout=wait_timeout,
|
||||
@@ -516,6 +520,7 @@ class RemoteTable(Table):
|
||||
replace: bool = ...,
|
||||
wait_timeout: Optional[timedelta] = ...,
|
||||
name: Optional[str] = ...,
|
||||
exist_ok: bool = ...,
|
||||
train: bool = ...,
|
||||
) -> None: ...
|
||||
|
||||
@@ -541,6 +546,7 @@ class RemoteTable(Table):
|
||||
*,
|
||||
wait_timeout: Optional[timedelta] = ...,
|
||||
name: Optional[str] = ...,
|
||||
exist_ok: bool = ...,
|
||||
train: bool = ...,
|
||||
target_partition_size: Optional[int] = ...,
|
||||
) -> None: ...
|
||||
@@ -572,6 +578,7 @@ class RemoteTable(Table):
|
||||
config: Optional[IndexConfigType] = None,
|
||||
wait_timeout: Optional[timedelta] = None,
|
||||
name: Optional[str] = None,
|
||||
exist_ok: bool = False,
|
||||
train: bool = True,
|
||||
target_partition_size: Optional[int] = None,
|
||||
):
|
||||
@@ -675,6 +682,7 @@ class RemoteTable(Table):
|
||||
self._table.create_index(
|
||||
column,
|
||||
replace=replace,
|
||||
exist_ok=exist_ok,
|
||||
config=config,
|
||||
wait_timeout=wait_timeout,
|
||||
name=name,
|
||||
@@ -690,6 +698,7 @@ class RemoteTable(Table):
|
||||
replace: Optional[bool] = None,
|
||||
wait_timeout: Optional[timedelta] = None,
|
||||
name: Optional[str] = None,
|
||||
exist_ok: bool = False,
|
||||
train: bool = True,
|
||||
) -> Job:
|
||||
"""Create an index, returning a handle to the indexing job.
|
||||
@@ -702,6 +711,7 @@ class RemoteTable(Table):
|
||||
self._table.create_index_async(
|
||||
column,
|
||||
replace=replace,
|
||||
exist_ok=exist_ok,
|
||||
config=config,
|
||||
wait_timeout=wait_timeout,
|
||||
name=name,
|
||||
|
||||
@@ -168,6 +168,36 @@ def _validate_index_accelerator(accelerator: Any) -> None:
|
||||
raise RuntimeError(_CUDA_UNAVAILABLE_MESSAGE) from err
|
||||
|
||||
|
||||
def _create_accelerated_index(dataset, *, exist_ok=False, name=None, **options):
|
||||
if exist_ok:
|
||||
parts = _parse_field_path(options["column"])
|
||||
default_name = ".".join(parts)
|
||||
if any("." in part for part in parts):
|
||||
default_name = ".".join(
|
||||
"`" + part.replace("`", "``") + "`"
|
||||
if any(
|
||||
not character.isalnum() and character != "_" for character in part
|
||||
)
|
||||
else part
|
||||
for part in parts
|
||||
)
|
||||
name = name or f"{default_name}_idx"
|
||||
dataset.checkout_latest()
|
||||
if any(index["name"] == name for index in dataset.list_indices()):
|
||||
return
|
||||
options["replace"] = False
|
||||
options["name"] = name
|
||||
_validate_index_accelerator(options.get("accelerator"))
|
||||
try:
|
||||
dataset.create_index(**options)
|
||||
except Exception:
|
||||
if exist_ok:
|
||||
dataset.checkout_latest()
|
||||
if any(index["name"] == name for index in dataset.list_indices()):
|
||||
return
|
||||
raise
|
||||
|
||||
|
||||
def _polars_predicate_pushdown_barrier(frame: Any) -> Any:
|
||||
"""Return a Polars frame unchanged while blocking predicate pushdown."""
|
||||
return frame
|
||||
@@ -1834,6 +1864,7 @@ class Table(ABC):
|
||||
replace: bool = ...,
|
||||
wait_timeout: Optional[timedelta] = ...,
|
||||
name: Optional[str] = ...,
|
||||
exist_ok: bool = ...,
|
||||
train: bool = ...,
|
||||
) -> None: ...
|
||||
|
||||
@@ -1859,6 +1890,7 @@ class Table(ABC):
|
||||
*,
|
||||
wait_timeout: Optional[timedelta] = ...,
|
||||
name: Optional[str] = ...,
|
||||
exist_ok: bool = ...,
|
||||
train: bool = ...,
|
||||
target_partition_size: Optional[int] = ...,
|
||||
) -> None: ...
|
||||
@@ -1890,6 +1922,7 @@ class Table(ABC):
|
||||
config: Optional[IndexConfigType] = None,
|
||||
wait_timeout: Optional[timedelta] = None,
|
||||
name: Optional[str] = None,
|
||||
exist_ok: bool = False,
|
||||
train: bool = True,
|
||||
target_partition_size: Optional[int] = None,
|
||||
):
|
||||
@@ -1913,6 +1946,9 @@ class Table(ABC):
|
||||
BTree, Bitmap, LabelList, Fm, NGram, ZoneMap, FTS.
|
||||
replace : bool, default True
|
||||
Whether to replace an existing index on this column.
|
||||
exist_ok : bool, default False
|
||||
Preserve the existing index with the requested or derived name.
|
||||
Takes precedence over replace and keeps its definition and build.
|
||||
wait_timeout : timedelta, optional
|
||||
LanceDB Enterprise only: how long to wait for the index build to finish.
|
||||
If not set, the call returns once the build is scheduled.
|
||||
@@ -1968,6 +2004,7 @@ class Table(ABC):
|
||||
replace: Optional[bool] = None,
|
||||
wait_timeout: Optional[timedelta] = None,
|
||||
name: Optional[str] = None,
|
||||
exist_ok: bool = False,
|
||||
train: bool = True,
|
||||
) -> Job:
|
||||
"""Create an index, returning a handle to the indexing job.
|
||||
@@ -2060,6 +2097,7 @@ class Table(ABC):
|
||||
index_type: ScalarIndexType = "BTREE",
|
||||
wait_timeout: Optional[timedelta] = None,
|
||||
name: Optional[str] = None,
|
||||
exist_ok: bool = False,
|
||||
):
|
||||
"""Create a scalar index on a column.
|
||||
|
||||
@@ -2144,6 +2182,7 @@ class Table(ABC):
|
||||
document_granularity: DocumentGranularity = DocumentGranularity.ROW,
|
||||
wait_timeout: Optional[timedelta] = None,
|
||||
name: Optional[str] = None,
|
||||
exist_ok: bool = False,
|
||||
):
|
||||
"""Create a full-text search index on the table.
|
||||
|
||||
@@ -4318,6 +4357,7 @@ class LanceTable(Table):
|
||||
replace: bool = ...,
|
||||
wait_timeout: Optional[timedelta] = ...,
|
||||
name: Optional[str] = ...,
|
||||
exist_ok: bool = ...,
|
||||
train: bool = ...,
|
||||
) -> None: ...
|
||||
|
||||
@@ -4343,6 +4383,7 @@ class LanceTable(Table):
|
||||
*,
|
||||
wait_timeout: Optional[timedelta] = ...,
|
||||
name: Optional[str] = ...,
|
||||
exist_ok: bool = ...,
|
||||
train: bool = ...,
|
||||
target_partition_size: Optional[int] = ...,
|
||||
) -> None: ...
|
||||
@@ -4374,6 +4415,7 @@ class LanceTable(Table):
|
||||
config: Optional[IndexConfigType] = None,
|
||||
wait_timeout: Optional[timedelta] = None,
|
||||
name: Optional[str] = None,
|
||||
exist_ok: bool = False,
|
||||
train: bool = True,
|
||||
target_partition_size: Optional[int] = None,
|
||||
):
|
||||
@@ -4397,6 +4439,9 @@ class LanceTable(Table):
|
||||
BTree, Bitmap, LabelList, Fm, NGram, ZoneMap, FTS.
|
||||
replace : bool, default True
|
||||
Whether to replace an existing index on this column.
|
||||
exist_ok : bool, default False
|
||||
Preserve the existing index with the requested or derived name.
|
||||
Takes precedence over replace and keeps its definition and build.
|
||||
wait_timeout : timedelta, optional
|
||||
Timeout to wait for async indexing to complete.
|
||||
name : str, optional
|
||||
@@ -4465,8 +4510,10 @@ class LanceTable(Table):
|
||||
# Handle accelerator through pylance
|
||||
if accelerator is not None:
|
||||
dataset = self.to_lance()
|
||||
_validate_index_accelerator(accelerator)
|
||||
dataset.create_index(
|
||||
_create_accelerated_index(
|
||||
dataset,
|
||||
exist_ok=exist_ok,
|
||||
name=name,
|
||||
column=column,
|
||||
index_type=index_type,
|
||||
metric=metric,
|
||||
@@ -4492,7 +4539,6 @@ class LanceTable(Table):
|
||||
if acc is not None:
|
||||
# Dispatch to pylance for GPU acceleration
|
||||
dataset = self.to_lance()
|
||||
_validate_index_accelerator(acc)
|
||||
index_type_map = {
|
||||
"IvfFlat": "IVF_FLAT",
|
||||
"IvfSq": "IVF_SQ",
|
||||
@@ -4509,7 +4555,10 @@ class LanceTable(Table):
|
||||
if num_bits is not None:
|
||||
quantization_kwargs["num_bits"] = num_bits
|
||||
|
||||
dataset.create_index(
|
||||
_create_accelerated_index(
|
||||
dataset,
|
||||
exist_ok=exist_ok,
|
||||
name=name,
|
||||
column=column,
|
||||
index_type=lance_index_type,
|
||||
metric=getattr(config, "distance_type", "l2"),
|
||||
@@ -4531,6 +4580,7 @@ class LanceTable(Table):
|
||||
self._table.create_index(
|
||||
column,
|
||||
replace=replace,
|
||||
exist_ok=exist_ok,
|
||||
config=config,
|
||||
wait_timeout=wait_timeout,
|
||||
name=name,
|
||||
@@ -4546,6 +4596,7 @@ class LanceTable(Table):
|
||||
replace: Optional[bool] = None,
|
||||
wait_timeout: Optional[timedelta] = None,
|
||||
name: Optional[str] = None,
|
||||
exist_ok: bool = False,
|
||||
train: bool = True,
|
||||
) -> Job:
|
||||
"""Create an index, returning a handle to the indexing job.
|
||||
@@ -4564,6 +4615,7 @@ class LanceTable(Table):
|
||||
self._table.create_index_async(
|
||||
column,
|
||||
replace=replace,
|
||||
exist_ok=exist_ok,
|
||||
config=config,
|
||||
wait_timeout=wait_timeout,
|
||||
name=name,
|
||||
@@ -4684,6 +4736,7 @@ class LanceTable(Table):
|
||||
index_type: ScalarIndexType = "BTREE",
|
||||
wait_timeout: Optional[timedelta] = None,
|
||||
name: Optional[str] = None,
|
||||
exist_ok: bool = False,
|
||||
):
|
||||
"""Create a scalar index on a column.
|
||||
|
||||
@@ -4710,6 +4763,7 @@ class LanceTable(Table):
|
||||
self._table.create_index(
|
||||
column,
|
||||
replace=replace,
|
||||
exist_ok=exist_ok,
|
||||
config=config,
|
||||
wait_timeout=wait_timeout,
|
||||
name=name,
|
||||
@@ -4747,6 +4801,7 @@ class LanceTable(Table):
|
||||
document_granularity: DocumentGranularity = DocumentGranularity.ROW,
|
||||
wait_timeout: Optional[timedelta] = None,
|
||||
name: Optional[str] = None,
|
||||
exist_ok: bool = False,
|
||||
):
|
||||
"""Create a full-text search index on a column.
|
||||
|
||||
@@ -4808,6 +4863,7 @@ class LanceTable(Table):
|
||||
self._table.create_index(
|
||||
field_names,
|
||||
replace=replace,
|
||||
exist_ok=exist_ok,
|
||||
config=config,
|
||||
name=name,
|
||||
wait_timeout=wait_timeout,
|
||||
@@ -6806,6 +6862,7 @@ class AsyncTable:
|
||||
] = None,
|
||||
wait_timeout: Optional[timedelta] = None,
|
||||
name: Optional[str] = None,
|
||||
exist_ok: bool = False,
|
||||
train: bool = True,
|
||||
):
|
||||
"""Create an index to speed up queries
|
||||
@@ -6836,6 +6893,9 @@ class AsyncTable:
|
||||
If not set, the call returns once the build is scheduled.
|
||||
name: str, optional
|
||||
The name of the index. If not provided, a default name will be generated.
|
||||
exist_ok: bool, default False
|
||||
Preserve an existing index with the requested or derived name. This
|
||||
takes precedence over replace and keeps its definition and build.
|
||||
train: bool, default True
|
||||
Whether to train the index with existing data. Vector indices always train
|
||||
with existing data.
|
||||
@@ -6891,6 +6951,7 @@ class AsyncTable:
|
||||
column,
|
||||
index=config,
|
||||
replace=replace,
|
||||
exist_ok=exist_ok,
|
||||
wait_timeout=wait_timeout,
|
||||
name=name,
|
||||
train=train,
|
||||
@@ -6928,6 +6989,7 @@ class AsyncTable:
|
||||
] = None,
|
||||
wait_timeout: Optional[timedelta] = None,
|
||||
name: Optional[str] = None,
|
||||
exist_ok: bool = False,
|
||||
train: bool = True,
|
||||
) -> AsyncJob:
|
||||
"""Create an index, returning a handle to the indexing job.
|
||||
@@ -6946,6 +7008,7 @@ class AsyncTable:
|
||||
column,
|
||||
index=config,
|
||||
replace=replace,
|
||||
exist_ok=exist_ok,
|
||||
wait_timeout=wait_timeout,
|
||||
name=name,
|
||||
train=train,
|
||||
|
||||
@@ -621,3 +621,45 @@ def test_index_statistics_index_type_lists_all_supported_values():
|
||||
set(get_args(get_type_hints(IndexStatistics)["index_type"]))
|
||||
== expected_index_types
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("background", [False, True])
|
||||
@pytest.mark.parametrize(
|
||||
"column,config", [("id", BTree()), ("vector", IvfFlat(num_partitions=1))]
|
||||
)
|
||||
async def test_index_exist_ok_preserves_existing_index(
|
||||
some_table, background, column, config
|
||||
):
|
||||
await some_table.create_index(column, config=config, exist_ok=True)
|
||||
version = await some_table.version()
|
||||
original = await some_table.list_indices()
|
||||
if background:
|
||||
job = await some_table.create_index_async(
|
||||
column, config=config, replace=True, exist_ok=True
|
||||
)
|
||||
await job.wait()
|
||||
else:
|
||||
await some_table.create_index(
|
||||
column, config=config, replace=True, exist_ok=True
|
||||
)
|
||||
assert await some_table.version() == version
|
||||
assert [(i.name, i.index_type) for i in await some_table.list_indices()] == [
|
||||
(i.name, i.index_type) for i in original
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("background", [False, True])
|
||||
async def test_named_index_exist_ok_checks_before_column_validation(
|
||||
some_table, background
|
||||
):
|
||||
await some_table.create_index("id", config=BTree(), name="kept")
|
||||
version = await some_table.version()
|
||||
method = some_table.create_index_async if background else some_table.create_index
|
||||
result = await method("missing", config=BTree(), name="kept", exist_ok=True)
|
||||
if background:
|
||||
assert result.id is None
|
||||
await result.wait()
|
||||
assert await some_table.version() == version
|
||||
assert [index.name for index in await some_table.list_indices()] == ["kept"]
|
||||
|
||||
@@ -45,6 +45,7 @@ from lancedb.query import (
|
||||
LanceVectorQueryBuilder,
|
||||
MatchQuery,
|
||||
)
|
||||
from lancedb.index import BTree
|
||||
from lancedb.remote import ClientConfig
|
||||
from lancedb.remote.errors import (
|
||||
AlreadyExistsError,
|
||||
@@ -7420,3 +7421,54 @@ async def test_remote_blob_bytes_hybrid_rejects_same_version_different_incarnati
|
||||
if use_async:
|
||||
await result
|
||||
assert not any(path == "fetch_blobs" for path, _ in requests)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("is_async", [False, True])
|
||||
@pytest.mark.parametrize("background", [False, True])
|
||||
@pytest.mark.parametrize("kwargs", [{}, {"exist_ok": False}, {"exist_ok": True}])
|
||||
async def test_remote_index_exist_ok_wire(is_async, background, kwargs):
|
||||
received = []
|
||||
|
||||
def handler(request):
|
||||
parsed = urlparse(request.path)
|
||||
body = request.rfile.read(int(request.headers.get("Content-Length", 0)))
|
||||
if parsed.path.endswith("/create_index/"):
|
||||
received.append((parse_qs(parsed.query), json.loads(body)))
|
||||
payload = {}
|
||||
elif parsed.path.endswith("/describe/"):
|
||||
payload = dict(
|
||||
version=1,
|
||||
schema=dict(
|
||||
fields=[dict(name="id", type={"type": "int64"}, nullable=False)]
|
||||
),
|
||||
)
|
||||
elif parsed.path.endswith("/create/"):
|
||||
payload = {}
|
||||
else:
|
||||
raise AssertionError(request.path)
|
||||
request.send_response(200)
|
||||
request.send_header("Content-Type", "application/json")
|
||||
request.end_headers()
|
||||
request.wfile.write(json.dumps(payload).encode())
|
||||
|
||||
method = "create_index_async" if background else "create_index"
|
||||
if is_async:
|
||||
async with mock_lancedb_connection_async(handler) as db:
|
||||
table = await db.create_table("test", [{"id": 1}])
|
||||
result = await getattr(table, method)("id", config=BTree(), **kwargs)
|
||||
if background:
|
||||
await result.wait()
|
||||
else:
|
||||
with mock_lancedb_connection(handler) as db:
|
||||
table = db.create_table("test", [{"id": 1}])
|
||||
result = getattr(table, method)("id", config=BTree(), **kwargs)
|
||||
if background:
|
||||
result.wait()
|
||||
query, body = received[0]
|
||||
assert query == ({"exist_ok": ["true"]} if kwargs.get("exist_ok") else {})
|
||||
assert "exist_ok" not in body
|
||||
if kwargs.get("exist_ok"):
|
||||
assert body["replace"] is False
|
||||
else:
|
||||
assert "replace" not in body
|
||||
|
||||
@@ -1837,6 +1837,7 @@ def test_create_index_method(mock_create_index, mem_db: DBConnection):
|
||||
wait_timeout=None,
|
||||
name=None,
|
||||
train=True,
|
||||
exist_ok=False,
|
||||
)
|
||||
|
||||
# Test with target_partition_size
|
||||
@@ -1862,6 +1863,7 @@ def test_create_index_method(mock_create_index, mem_db: DBConnection):
|
||||
wait_timeout=None,
|
||||
name=None,
|
||||
train=True,
|
||||
exist_ok=False,
|
||||
)
|
||||
|
||||
# target_partition_size has a default value,
|
||||
@@ -1886,6 +1888,7 @@ def test_create_index_method(mock_create_index, mem_db: DBConnection):
|
||||
wait_timeout=None,
|
||||
name=None,
|
||||
train=True,
|
||||
exist_ok=False,
|
||||
)
|
||||
|
||||
table.create_index(
|
||||
@@ -1902,6 +1905,7 @@ def test_create_index_method(mock_create_index, mem_db: DBConnection):
|
||||
wait_timeout=None,
|
||||
name=None,
|
||||
train=True,
|
||||
exist_ok=False,
|
||||
)
|
||||
|
||||
table.create_index(
|
||||
@@ -1922,6 +1926,7 @@ def test_create_index_method(mock_create_index, mem_db: DBConnection):
|
||||
wait_timeout=None,
|
||||
name=None,
|
||||
train=True,
|
||||
exist_ok=False,
|
||||
)
|
||||
|
||||
table.create_index(
|
||||
@@ -1942,6 +1947,7 @@ def test_create_index_method(mock_create_index, mem_db: DBConnection):
|
||||
wait_timeout=None,
|
||||
name=None,
|
||||
train=True,
|
||||
exist_ok=False,
|
||||
)
|
||||
|
||||
|
||||
@@ -2073,6 +2079,7 @@ def test_create_index_name_and_train_parameters(
|
||||
wait_timeout=None,
|
||||
name="my_custom_index",
|
||||
train=True,
|
||||
exist_ok=False,
|
||||
)
|
||||
|
||||
# Test with train=False
|
||||
@@ -2084,6 +2091,7 @@ def test_create_index_name_and_train_parameters(
|
||||
wait_timeout=None,
|
||||
name=None,
|
||||
train=False,
|
||||
exist_ok=False,
|
||||
)
|
||||
|
||||
# Test with both name and train
|
||||
@@ -2095,6 +2103,7 @@ def test_create_index_name_and_train_parameters(
|
||||
wait_timeout=None,
|
||||
name="my_index_name",
|
||||
train=True,
|
||||
exist_ok=False,
|
||||
)
|
||||
|
||||
|
||||
@@ -2132,6 +2141,7 @@ def test_create_index_new_api(mock_create_index, mem_db: DBConnection):
|
||||
wait_timeout=None,
|
||||
name=None,
|
||||
train=True,
|
||||
exist_ok=False,
|
||||
)
|
||||
|
||||
# Scalar index via new API
|
||||
@@ -2143,6 +2153,7 @@ def test_create_index_new_api(mock_create_index, mem_db: DBConnection):
|
||||
wait_timeout=None,
|
||||
name=None,
|
||||
train=True,
|
||||
exist_ok=False,
|
||||
)
|
||||
|
||||
# FTS index via new API
|
||||
@@ -2154,6 +2165,7 @@ def test_create_index_new_api(mock_create_index, mem_db: DBConnection):
|
||||
wait_timeout=None,
|
||||
name=None,
|
||||
train=True,
|
||||
exist_ok=False,
|
||||
)
|
||||
|
||||
|
||||
@@ -6292,3 +6304,87 @@ async def test_async_computed_auto_refresh_uses_server_interval(tmp_path):
|
||||
metadata = (await table.schema()).field("doubled").metadata
|
||||
assert metadata[b"computed_column.auto_refresh_enabled"] == b"true"
|
||||
assert b"computed_column.auto_refresh_interval_seconds" not in metadata
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"method",
|
||||
["create_index", "create_index_async", "create_scalar_index", "create_fts_index"],
|
||||
)
|
||||
@pytest.mark.parametrize("name", [None, "kept_index"])
|
||||
def test_index_exist_ok_preserves_definition_and_version(mem_db, method, name):
|
||||
table = mem_db.create_table(
|
||||
"keep_index", [{"id": i, "text": f"document {i}"} for i in range(20)]
|
||||
)
|
||||
column = "text" if method == "create_fts_index" else "id"
|
||||
config = FTS() if column == "text" else BTree()
|
||||
table.create_index(column, config=config, name=name, exist_ok=True)
|
||||
before = table.version
|
||||
original = table.list_indices()
|
||||
kwargs = dict(name=name, exist_ok=True)
|
||||
if method in ("create_index", "create_index_async"):
|
||||
kwargs["config"] = config
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter("ignore", DeprecationWarning)
|
||||
result = getattr(table, method)(column, **kwargs)
|
||||
if method == "create_index_async":
|
||||
result.wait()
|
||||
assert table.version == before
|
||||
assert [(index.name, index.index_type) for index in table.list_indices()] == [
|
||||
(index.name, index.index_type) for index in original
|
||||
]
|
||||
with pytest.raises((ValueError, RuntimeError), match="already exists"):
|
||||
table.create_index(column, config=config, name=name, replace=False)
|
||||
table.create_index(column, config=config, name=name)
|
||||
assert table.version > before
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"column,name,existing",
|
||||
[
|
||||
("vector", "kept", "kept"),
|
||||
("`docs`.`nested.dot`", None, "docs.`nested.dot`_idx"),
|
||||
],
|
||||
)
|
||||
def test_accelerated_index_exist_ok_skips_gpu_validation(column, name, existing):
|
||||
dataset = MagicMock()
|
||||
dataset.list_indices.return_value = [{"name": existing}]
|
||||
with patch("lancedb.table._validate_index_accelerator") as validate:
|
||||
table_module._create_accelerated_index(
|
||||
dataset, column=column, accelerator="cuda", name=name, exist_ok=True
|
||||
)
|
||||
validate.assert_not_called()
|
||||
dataset.create_index.assert_not_called()
|
||||
|
||||
|
||||
def test_accelerated_index_exist_ok_preserves_concurrent_winner():
|
||||
dataset = MagicMock()
|
||||
dataset.list_indices.side_effect = [[], [{"name": "vector_idx"}]]
|
||||
dataset.create_index.side_effect = RuntimeError("concurrent create")
|
||||
with patch("lancedb.table._validate_index_accelerator"):
|
||||
table_module._create_accelerated_index(
|
||||
dataset, column="vector", accelerator="cuda", replace=True, exist_ok=True
|
||||
)
|
||||
assert dataset.create_index.call_args.kwargs["replace"] is False
|
||||
assert dataset.create_index.call_args.kwargs["name"] == "vector_idx"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("column", ["`user-id`", "`user:id`", "nested.`user-id`"])
|
||||
def test_index_exist_ok_matches_lance_default_name(mem_db, column):
|
||||
table = mem_db.create_table(
|
||||
"quoted_index",
|
||||
[{"user-id": 1, "user:id": 2, "nested": {"user-id": 4}}],
|
||||
)
|
||||
table.create_index(column, config=BTree())
|
||||
before = table.version
|
||||
table.create_index(column, config=BTree(), exist_ok=True)
|
||||
assert table.version == before
|
||||
assert len(table.list_indices()) == 1
|
||||
|
||||
|
||||
def test_fts_exist_ok_matches_quoted_default_name(mem_db):
|
||||
table = mem_db.create_table("quoted_fts", [{"body-text": "keep this document"}])
|
||||
table.create_index("`body-text`", config=FTS())
|
||||
before = table.version
|
||||
table.create_index("`body-text`", config=FTS(), exist_ok=True)
|
||||
assert table.version == before
|
||||
assert len(table.list_indices()) == 1
|
||||
|
||||
@@ -115,7 +115,11 @@ def test_vector_tuning_options(sync_table):
|
||||
}.items():
|
||||
assert getattr(config, name) == value
|
||||
assert kwargs == dict(
|
||||
wait_timeout=timeout, name="custom_idx", replace=False, train=False
|
||||
wait_timeout=timeout,
|
||||
name="custom_idx",
|
||||
replace=False,
|
||||
train=False,
|
||||
exist_ok=False,
|
||||
)
|
||||
|
||||
|
||||
|
||||
+8
-2
@@ -1628,7 +1628,8 @@ impl Table {
|
||||
})
|
||||
}
|
||||
|
||||
#[pyo3(signature = (column, index=None, replace=None, wait_timeout=None, *, name=None, train=None))]
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
#[pyo3(signature = (column, index=None, replace=None, wait_timeout=None, *, name=None, train=None, exist_ok=false))]
|
||||
pub fn create_index<'a>(
|
||||
self_: PyRef<'a, Self>,
|
||||
column: String,
|
||||
@@ -1637,6 +1638,7 @@ impl Table {
|
||||
wait_timeout: Option<Bound<'_, PyAny>>,
|
||||
name: Option<String>,
|
||||
train: Option<bool>,
|
||||
exist_ok: bool,
|
||||
) -> PyResult<Bound<'a, PyAny>> {
|
||||
let index = extract_index_params(&index)?;
|
||||
let timeout = wait_timeout
|
||||
@@ -1645,6 +1647,7 @@ impl Table {
|
||||
let mut op = self_
|
||||
.inner_ref()?
|
||||
.create_index_with_timeout(&[column], index, timeout);
|
||||
op = op.exist_ok(exist_ok);
|
||||
if let Some(replace) = replace {
|
||||
op = op.replace(replace);
|
||||
}
|
||||
@@ -1661,7 +1664,8 @@ impl Table {
|
||||
})
|
||||
}
|
||||
|
||||
#[pyo3(signature = (column, index=None, replace=None, wait_timeout=None, *, name=None, train=None))]
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
#[pyo3(signature = (column, index=None, replace=None, wait_timeout=None, *, name=None, train=None, exist_ok=false))]
|
||||
pub fn create_index_async<'a>(
|
||||
self_: PyRef<'a, Self>,
|
||||
column: String,
|
||||
@@ -1670,6 +1674,7 @@ impl Table {
|
||||
wait_timeout: Option<Bound<'_, PyAny>>,
|
||||
name: Option<String>,
|
||||
train: Option<bool>,
|
||||
exist_ok: bool,
|
||||
) -> PyResult<Bound<'a, PyAny>> {
|
||||
let index = extract_index_params(&index)?;
|
||||
let timeout = wait_timeout
|
||||
@@ -1678,6 +1683,7 @@ impl Table {
|
||||
let mut op = self_
|
||||
.inner_ref()?
|
||||
.create_index_with_timeout(&[column], index, timeout);
|
||||
op = op.exist_ok(exist_ok);
|
||||
if let Some(replace) = replace {
|
||||
op = op.replace(replace);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user