diff --git a/docs/src/python/python.md b/docs/src/python/python.md index 02ccced2c..4d0dfc87a 100644 --- a/docs/src/python/python.md +++ b/docs/src/python/python.md @@ -297,6 +297,25 @@ listing a storage directory. ## Tables (Synchronous) +Local and remote tables use the same arguments for index creation, deletion, +index statistics, and search. For example, `table.delete(where="id = 1")`, +`table.create_index("cosine", 2)` (the deprecated vector API), and +`table.search([1.0, 1.0], fast_search=True)` work with either backend. +`fast_search=True` excludes rows that have not been indexed. + +`create_scalar_index` defaults to `replace=True` on both backends. Both scalar +and full-text index creation accept `wait_timeout`. Remote maintenance methods +accept the local options: cleanup and compaction warn that they are no-ops, +while `optimize` reports an unsupported operation. Removed Tantivy options +produce the same explanatory errors on both backends. + +The old remote keyword names `predicate` for `delete`, `index_name` for +`drop_index`, `index_uuid` for `index_stats`, and `column` for `create_fts_index` +remain available with deprecation warnings. Use `where`, `name`, `index_name`, +and `field_names`, respectively. For positional calls, use the argument order +documented by `Table` below; specify index options by keyword when migrating +code that used the former remote-only argument order. + ::: lancedb.table.Table ::: lancedb.table.FragmentStatistics diff --git a/python/python/lancedb/remote/table.py b/python/python/lancedb/remote/table.py index dd22dea48..faf15c987 100644 --- a/python/python/lancedb/remote/table.py +++ b/python/python/lancedb/remote/table.py @@ -4,7 +4,7 @@ from datetime import timedelta import deprecation import logging -from functools import cached_property +from functools import cached_property, wraps import os from typing import ( TYPE_CHECKING, @@ -42,19 +42,12 @@ from lancedb.index import ( FTS, BTree, Bitmap, - HnswFlat, - HnswPq, - HnswSq, - IvfFlat, - IvfPq, - IvfRq, - IvfSq, LabelList, ) from lancedb.job import Job from lancedb.functions import FunctionApplication, RefreshColumnResult from lancedb.remote.db import LOOP -from lancedb.table import IndexConfigType, KNOWN_METRICS +from lancedb.table import IndexConfigType, _maybe_add_fts_error_note import pyarrow as pa from lancedb.common import DATA, VEC, VECTOR_COLUMN_NAME @@ -69,7 +62,6 @@ from ..query import ( FullTextQuery, LanceQueryBuilder, LanceTakeQueryBuilder, - LanceVectorQueryBuilder, ) from ..table import ( AsyncTable, @@ -80,7 +72,7 @@ from ..table import ( Table, Tags, ) -from ..types import BaseTokenizerType +from ..types import BaseTokenizerType, ScalarIndexType from ..util import infer_vector_column_name if TYPE_CHECKING: @@ -88,6 +80,28 @@ if TYPE_CHECKING: import polars as pl +def _deprecated_keyword(old_name: str, new_name: str): + """Keep old remote-only keyword names without changing the Table signature.""" + + def decorate(method): + @wraps(method) + def wrapped(self, *args, **kwargs): + if old_name in kwargs: + if new_name in kwargs: + raise TypeError(f"Cannot specify both {old_name} and {new_name}") + warnings.warn( + f"{old_name} is deprecated; use {new_name} instead", + DeprecationWarning, + stacklevel=2, + ) + kwargs[new_name] = kwargs.pop(old_name) + return method(self, *args, **kwargs) + + return wrapped + + return decorate + + class RemoteTable(Table): def __init__( self, @@ -268,7 +282,7 @@ class RemoteTable(Table): """ return LOOP.run(self._table.to_lance(**kwargs)) - def to_polars(self, **kwargs) -> "pl.LazyFrame": + def to_polars(self, batch_size=None) -> "pl.LazyFrame": """Use ``table.search().to_polars()`` to read remote query results.""" raise NotImplementedError( "to_polars() is not supported for remote tables. " @@ -315,9 +329,10 @@ class RemoteTable(Table): self._table.tokenize(query, column=column, index_name=index_name) ) - def index_stats(self, index_uuid: str) -> Optional[IndexStatistics]: + @_deprecated_keyword("index_uuid", "index_name") + def index_stats(self, index_name: str) -> Optional[IndexStatistics]: """List all the stats of a specified index""" - return LOOP.run(self._table.index_stats(index_uuid)) + return LOOP.run(self._table.index_stats(index_name)) @deprecation.deprecated( deprecated_in="0.25.0", @@ -327,43 +342,38 @@ class RemoteTable(Table): def create_scalar_index( self, column: str, - index_type: Literal["BTREE", "BITMAP", "LABEL_LIST", "scalar"] = "scalar", *, - replace: bool = False, + replace: bool = True, + index_type: ScalarIndexType = "BTREE", wait_timeout: Optional[timedelta] = None, name: Optional[str] = None, ): - """Creates a scalar index. + """Create a scalar index on a column. .. deprecated:: 0.25.0 Use :meth:`create_index` with a BTree, Bitmap, or LabelList config instead. Example: ``table.create_index("column", config=BTree())`` - - Parameters - ---------- - column : str - The column to be indexed. Must be a boolean, integer, float, - or string column. - index_type : str - The index type of the scalar index. Must be "scalar" (BTREE), - "BTREE", "BITMAP", or "LABEL_LIST", - replace : bool - If True, replace the existing index with the new one. """ - if index_type == "scalar" or index_type == "BTREE": + if index_type == "scalar": + warnings.warn( + 'index_type="scalar" is deprecated; use "BTREE" instead', + DeprecationWarning, + stacklevel=2, + ) + index_type = "BTREE" + if index_type == "BTREE": config = BTree() elif index_type == "BITMAP": config = Bitmap() elif index_type == "LABEL_LIST": config = LabelList() else: - raise ValueError(f"Unknown index type: {index_type}") - - LOOP.run( + raise ValueError(f"Unknown index type {index_type}") + return LOOP.run( self._table.create_index( column, - config=config, replace=replace, + config=config, wait_timeout=wait_timeout, name=name, ) @@ -374,12 +384,16 @@ class RemoteTable(Table): current_version=__version__, details="Use create_index() with config=FTS() instead.", ) + @_deprecated_keyword("column", "field_names") def create_fts_index( self, - column: str, + field_names: Union[str, List[str]], *, + ordering_field_names: Optional[Union[str, List[str]]] = None, replace: bool = False, - wait_timeout: Optional[timedelta] = None, + writer_heap_size: Optional[int] = 1024 * 1024 * 1024, + use_tantivy: bool = False, + tokenizer_name: Optional[str] = None, with_position: bool = False, # tokenizer configs: base_tokenizer: BaseTokenizerType = "simple", @@ -395,6 +409,7 @@ class RemoteTable(Table): prefix_only: bool = False, block_size: int = 128, document_granularity: DocumentGranularity = DocumentGranularity.ROW, + wait_timeout: Optional[timedelta] = None, name: Optional[str] = None, ): """Create a full-text search index on a column. @@ -403,31 +418,70 @@ class RemoteTable(Table): Use :meth:`create_index` with an FTS config instead. Example: ``table.create_index("text_column", config=FTS())`` """ + if use_tantivy: + raise ValueError( + "Tantivy-based FTS has been removed. " + "Remove use_tantivy and recreate the index with native FTS." + ) + if ordering_field_names is not None: + raise ValueError( + "ordering_field_names was only supported by the removed " + "Tantivy-based FTS implementation." + ) + if writer_heap_size != 1024 * 1024 * 1024: + raise ValueError( + "writer_heap_size was only supported by the removed " + "Tantivy-based FTS implementation." + ) + if not isinstance(field_names, str): + raise ValueError( + "Native FTS indexes can only be created on a single field " + "at a time. To search over multiple text fields, create a " + "separate FTS index for each field." + ) + + if tokenizer_name is None: + tokenizer_configs = { + "base_tokenizer": base_tokenizer, + "language": language, + "with_position": with_position, + "max_token_length": max_token_length, + "lower_case": lower_case, + "stem": stem, + "remove_stop_words": remove_stop_words, + "custom_stop_words": custom_stop_words, + "ascii_folding": ascii_folding, + "ngram_min_length": ngram_min_length, + "ngram_max_length": ngram_max_length, + "prefix_only": prefix_only, + } + else: + tokenizer_configs = self.infer_tokenizer_configs(tokenizer_name) + tokenizer_configs["custom_stop_words"] = custom_stop_words + config = FTS( - with_position=with_position, - base_tokenizer=base_tokenizer, - language=language, - max_token_length=max_token_length, - lower_case=lower_case, - stem=stem, - remove_stop_words=remove_stop_words, - custom_stop_words=custom_stop_words, - ascii_folding=ascii_folding, - ngram_min_length=ngram_min_length, - ngram_max_length=ngram_max_length, - prefix_only=prefix_only, block_size=block_size, document_granularity=document_granularity, + **tokenizer_configs, ) - LOOP.run( - self._table.create_index( - column, - config=config, - replace=replace, - wait_timeout=wait_timeout, - name=name, + + try: + LOOP.run( + self._table.create_index( + field_names, + replace=replace, + config=config, + name=name, + wait_timeout=wait_timeout, + ) ) - ) + except (ValueError, RuntimeError) as e: + _maybe_add_fts_error_note( + e, + base_tokenizer=config.base_tokenizer, + language=config.language, + ) + raise e # New unified API overload @overload @@ -437,6 +491,7 @@ class RemoteTable(Table): /, *, config: IndexConfigType, + replace: bool = ..., wait_timeout: Optional[timedelta] = ..., name: Optional[str] = ..., train: bool = ..., @@ -447,38 +502,56 @@ class RemoteTable(Table): def create_index( self, metric: Literal["l2", "cosine", "dot", "hamming"] = ..., - vector_column_name: str = ..., - index_cache_size: Optional[int] = ..., num_partitions: Optional[int] = ..., num_sub_vectors: Optional[int] = ..., - replace: Optional[bool] = ..., + vector_column_name: str = ..., + replace: bool = ..., accelerator: Optional[str] = ..., - index_type: Literal[ - "VECTOR", "IVF_FLAT", "IVF_SQ", "IVF_PQ", "IVF_HNSW_SQ", "IVF_HNSW_PQ" - ] = ..., - wait_timeout: Optional[timedelta] = ..., - *, + index_cache_size: Optional[int] = ..., num_bits: int = ..., + index_type: Literal[ + "IVF_FLAT", "IVF_SQ", "IVF_PQ", "IVF_RQ", "IVF_HNSW_SQ", "IVF_HNSW_PQ" + ] = ..., + max_iterations: int = ..., + sample_rate: int = ..., + m: int = ..., + ef_construction: int = ..., + *, + wait_timeout: Optional[timedelta] = ..., name: Optional[str] = ..., train: bool = ..., + target_partition_size: Optional[int] = ..., ) -> None: ... def create_index( self, metric: str = "l2", - vector_column_name: str = VECTOR_COLUMN_NAME, - index_cache_size: Optional[int] = None, num_partitions: Optional[int] = None, num_sub_vectors: Optional[int] = None, - replace: Optional[bool] = None, + vector_column_name: str = VECTOR_COLUMN_NAME, + replace: bool = True, accelerator: Optional[str] = None, - index_type="vector", - wait_timeout: Optional[timedelta] = None, - *, + index_cache_size: Optional[int] = None, num_bits: int = 8, + index_type: Literal[ + "IVF_FLAT", + "IVF_SQ", + "IVF_PQ", + "IVF_RQ", + "IVF_HNSW_SQ", + "IVF_HNSW_PQ", + "IVF_HNSW_FLAT", + ] = "IVF_PQ", + max_iterations: int = 50, + sample_rate: int = 256, + m: int = 20, + ef_construction: int = 300, + *, config: Optional[IndexConfigType] = None, + wait_timeout: Optional[timedelta] = None, name: Optional[str] = None, train: bool = True, + target_partition_size: Optional[int] = None, ): """Create an index on a column. @@ -489,6 +562,24 @@ class RemoteTable(Table): argument plus separate ``vector_column_name`` / ``num_partitions`` / etc. parameters, and emits a ``DeprecationWarning``. + Parameters + ---------- + metric : str + For new API: the column name to index. + For legacy API: the distance metric ("l2", "cosine", "dot", "hamming"). + config : IndexConfigType, optional + The index configuration object. If provided, uses the new unified API. + Can be one of: IvfFlat, IvfPq, IvfSq, IvfRq, HnswPq, HnswSq, + BTree, Bitmap, LabelList, Fm, FTS. + replace : bool, default True + Whether to replace an existing index on this column. + wait_timeout : timedelta, optional + Timeout to wait for async indexing to complete. + name : str, optional + Custom name for the index. + train : bool, default True + Whether to train the index with existing data. + Examples -------- New API (recommended): @@ -514,7 +605,6 @@ class RemoteTable(Table): vector_column_name, accelerator, index_cache_size, - replace, ) if is_legacy: @@ -530,6 +620,7 @@ class RemoteTable(Table): stacklevel=2, ) + # Legacy API: first arg is the distance metric column = vector_column_name if accelerator is not None: @@ -538,45 +629,27 @@ class RemoteTable(Table): "and will be ignored." ) - idx_type = index_type.upper() - if idx_type == "VECTOR" or idx_type == "IVF_PQ": - config = IvfPq( - distance_type=metric, - num_partitions=num_partitions, - num_sub_vectors=num_sub_vectors, - num_bits=num_bits, - ) - elif idx_type == "IVF_RQ": - config = IvfRq( - distance_type=metric, - num_partitions=num_partitions, - num_bits=num_bits, - ) - elif idx_type == "IVF_SQ": - config = IvfSq(distance_type=metric, num_partitions=num_partitions) - elif idx_type == "IVF_HNSW_PQ": - config = HnswPq( - distance_type=metric, - num_partitions=num_partitions, - num_sub_vectors=num_sub_vectors, - num_bits=num_bits, - ) - elif idx_type == "IVF_HNSW_SQ": - config = HnswSq(distance_type=metric, num_partitions=num_partitions) - elif idx_type == "IVF_HNSW_FLAT": - config = HnswFlat(distance_type=metric, num_partitions=num_partitions) - elif idx_type == "IVF_FLAT": - config = IvfFlat(distance_type=metric, num_partitions=num_partitions) - else: - raise ValueError( - f"Unknown vector index type: {idx_type}. Valid options are" - " 'IVF_FLAT', 'IVF_PQ', 'IVF_RQ', 'IVF_SQ'," - " 'IVF_HNSW_PQ', 'IVF_HNSW_SQ', 'IVF_HNSW_FLAT'" - ) + # Build config from legacy parameters + config = self._build_vector_config_from_legacy_params( + metric=metric, + index_type="IVF_PQ" + if index_type.upper() == "VECTOR" + else index_type.upper(), + num_partitions=num_partitions, + num_sub_vectors=num_sub_vectors, + num_bits=num_bits, + max_iterations=max_iterations, + sample_rate=sample_rate, + m=m, + ef_construction=ef_construction, + target_partition_size=target_partition_size, + accelerator=None, + ) + else: column = metric - LOOP.run( + return LOOP.run( self._table.create_index( column, replace=replace, @@ -615,37 +688,6 @@ class RemoteTable(Table): ) ) - def _is_legacy_create_index_call( - self, - first_arg: str, - config: Optional[IndexConfigType], - num_partitions: Optional[int], - num_sub_vectors: Optional[int], - vector_column_name: str, - accelerator: Optional[str], - index_cache_size: Optional[int], - replace: Optional[bool], - ) -> bool: - """Detect if this is a legacy create_index call.""" - if config is not None: - return False - if any( - x is not None - for x in ( - num_partitions, - num_sub_vectors, - accelerator, - index_cache_size, - replace, - ) - ): - return True - if vector_column_name != VECTOR_COLUMN_NAME: - return True - if first_arg.lower() in KNOWN_METRICS: - return True - return False - def add( self, data: DATA, @@ -717,9 +759,10 @@ class RemoteTable(Table): query: Union[VEC, str] = None, vector_column_name: Optional[str] = None, query_type="auto", + ordering_field_name: Optional[str] = None, fts_columns: Optional[Union[str, List[str]]] = None, fast_search: Optional[bool] = None, - ) -> LanceVectorQueryBuilder: + ) -> LanceQueryBuilder: """Create a search query to find the nearest neighbors of the given query vector. We currently support [vector search](https://lancedb.com/docs/search/vector-search/) @@ -810,6 +853,7 @@ class RemoteTable(Table): query, query_type, vector_column_name=vector_column_name, + ordering_field_name=ordering_field_name, fts_columns=fts_columns or [], fast_search=fast_search, ) @@ -869,7 +913,8 @@ class RemoteTable(Table): self._table._do_merge(merge, new_data, on_bad_vectors, fill_value) ) - def delete(self, predicate: str) -> DeleteResult: + @_deprecated_keyword("predicate", "where") + def delete(self, where: Union[str, Expr]) -> DeleteResult: """Delete rows from the table. This can be used to delete a single row, many rows, all rows, or @@ -877,7 +922,7 @@ class RemoteTable(Table): Parameters ---------- - predicate: str + where: Union[str, Expr] The SQL where clause to use when deleting rows. - For example, 'x = 2' or 'x IN (1, 2, 3)'. @@ -921,7 +966,7 @@ class RemoteTable(Table): x vector _distance # doctest: +SKIP 0 2 [3.0, 4.0] 85.0 # doctest: +SKIP """ - return LOOP.run(self._table.delete(predicate)) + return LOOP.run(self._table.delete(where)) def update( self, @@ -983,7 +1028,12 @@ class RemoteTable(Table): self._table.update(where=where, updates=values, updates_sql=values_sql) ) - def cleanup_old_versions(self, *_): + def cleanup_old_versions( + self, + older_than: Optional[timedelta] = None, + *, + delete_unverified: bool = False, + ): """ cleanup_old_versions() is a no-op for remote tables. """ @@ -993,7 +1043,7 @@ class RemoteTable(Table): ) pass - def compact_files(self, *_): + def compact_files(self, *args, **kwargs): """ compact_files() is a no-op for remote tables. """ @@ -1008,12 +1058,14 @@ class RemoteTable(Table): *, cleanup_older_than: Optional[timedelta] = None, delete_unverified: bool = False, + retrain: bool = False, ): """optimize() is not supported for remote tables.""" return LOOP.run( self._table.optimize( cleanup_older_than=cleanup_older_than, delete_unverified=delete_unverified, + retrain=retrain, ) ) @@ -1121,8 +1173,9 @@ class RemoteTable(Table): """No-op for remote tables (no local shard writers).""" return LOOP.run(self._table.close_rte_writers()) - def drop_index(self, index_name: str): - return LOOP.run(self._table.drop_index(index_name)) + @_deprecated_keyword("index_name", "name") + def drop_index(self, name: str) -> None: + return LOOP.run(self._table.drop_index(name)) def prewarm_index(self, name: str) -> None: """Prewarm an index in the table. diff --git a/python/python/lancedb/table.py b/python/python/lancedb/table.py index cc455f746..d2ff02683 100644 --- a/python/python/lancedb/table.py +++ b/python/python/lancedb/table.py @@ -265,10 +265,8 @@ if TYPE_CHECKING: OnBadVectorsType, AddMode, CreateMode, - VectorIndexType, ScalarIndexType, BaseTokenizerType, - DistanceType, ) # Type alias for index configuration objects @@ -1489,12 +1487,18 @@ class Table(ABC): """ raise NotImplementedError("to_lance is not supported for this table type") - def to_polars(self, **kwargs) -> "pl.LazyFrame": + def to_polars(self, batch_size=None) -> "pl.LazyFrame": """Return the table as a Polars LazyFrame. For remote tables, use ``table.search().to_polars()`` to convert query results to a Polars DataFrame. + Parameters + ---------- + batch_size: int, optional + Passed to Polars as the maximum row count for scanned PyArrow + record batches. + Note ---- The Polars streaming engine is not supported because it does not currently @@ -1507,6 +1511,193 @@ class Table(ABC): """ raise NotImplementedError("to_polars is not supported for this table type") + def _is_legacy_create_index_call( + self, + first_arg: str, + config: Optional[IndexConfigType], + num_partitions: Optional[int], + num_sub_vectors: Optional[int], + vector_column_name: str, + accelerator: Optional[str], + index_cache_size: Optional[int], + ) -> bool: + """Detect if this is a legacy create_index call.""" + # If config is provided, it's definitely the new API + if config is not None: + return False + + # If old-style parameters were explicitly set, it's legacy + if any( + x is not None + for x in (num_partitions, num_sub_vectors, accelerator, index_cache_size) + ): + return True + + # If vector_column_name differs from default, it's legacy + if vector_column_name != VECTOR_COLUMN_NAME: + return True + + # If first arg is a known metric, assume legacy + if first_arg.lower() in KNOWN_METRICS: + return True + + # Otherwise assume new API + return False + + def _build_vector_config_from_legacy_params( + self, + metric: str, + index_type: str, + num_partitions: Optional[int], + num_sub_vectors: Optional[int], + num_bits: int, + max_iterations: int, + sample_rate: int, + m: int, + ef_construction: int, + target_partition_size: Optional[int], + accelerator: Optional[str], + ) -> IndexConfigType: + """Build an index config object from legacy parameters.""" + if index_type == "IVF_FLAT": + return IvfFlat( + distance_type=metric, + num_partitions=num_partitions, + max_iterations=max_iterations, + sample_rate=sample_rate, + target_partition_size=target_partition_size, + accelerator=accelerator, + ) + elif index_type == "IVF_SQ": + return IvfSq( + distance_type=metric, + num_partitions=num_partitions, + max_iterations=max_iterations, + sample_rate=sample_rate, + target_partition_size=target_partition_size, + accelerator=accelerator, + ) + elif index_type == "IVF_PQ": + return IvfPq( + distance_type=metric, + num_partitions=num_partitions, + num_sub_vectors=num_sub_vectors, + num_bits=num_bits, + max_iterations=max_iterations, + sample_rate=sample_rate, + target_partition_size=target_partition_size, + accelerator=accelerator, + ) + elif index_type == "IVF_RQ": + return IvfRq( + distance_type=metric, + num_partitions=num_partitions, + num_bits=num_bits, + max_iterations=max_iterations, + sample_rate=sample_rate, + target_partition_size=target_partition_size, + accelerator=accelerator, + ) + elif index_type == "IVF_HNSW_PQ": + return HnswPq( + distance_type=metric, + num_partitions=num_partitions, + num_sub_vectors=num_sub_vectors, + num_bits=num_bits, + max_iterations=max_iterations, + sample_rate=sample_rate, + m=m, + ef_construction=ef_construction, + target_partition_size=target_partition_size, + accelerator=accelerator, + ) + elif index_type == "IVF_HNSW_SQ": + return HnswSq( + distance_type=metric, + num_partitions=num_partitions, + max_iterations=max_iterations, + sample_rate=sample_rate, + m=m, + ef_construction=ef_construction, + target_partition_size=target_partition_size, + accelerator=accelerator, + ) + elif index_type == "IVF_HNSW_FLAT": + return HnswFlat( + distance_type=metric, + num_partitions=num_partitions, + max_iterations=max_iterations, + sample_rate=sample_rate, + m=m, + ef_construction=ef_construction, + target_partition_size=target_partition_size, + ) + else: + raise ValueError(f"Unknown index type {index_type}") + + @staticmethod + def infer_tokenizer_configs(tokenizer_name: str) -> dict: + if tokenizer_name == "default": + return { + "base_tokenizer": "simple", + "language": "English", + "max_token_length": 40, + "lower_case": True, + "stem": False, + "remove_stop_words": False, + "ascii_folding": False, + "ngram_min_length": 3, + "ngram_max_length": 3, + "prefix_only": False, + } + elif tokenizer_name == "raw": + return { + "base_tokenizer": "raw", + "language": "English", + "max_token_length": None, + "lower_case": False, + "stem": False, + "remove_stop_words": False, + "ascii_folding": False, + "ngram_min_length": 3, + "ngram_max_length": 3, + "prefix_only": False, + } + elif tokenizer_name == "whitespace": + return { + "base_tokenizer": "whitespace", + "language": "English", + "max_token_length": None, + "lower_case": False, + "stem": False, + "remove_stop_words": False, + "ascii_folding": False, + "ngram_min_length": 3, + "ngram_max_length": 3, + "prefix_only": False, + } + + # or it's with language stemming with pattern like "en_stem" + if len(tokenizer_name) != 7: + raise ValueError(f"Invalid tokenizer name {tokenizer_name}") + lang = tokenizer_name[:2] + if tokenizer_name[-5:] != "_stem": + raise ValueError(f"Invalid tokenizer name {tokenizer_name}") + if lang not in lang_mapping: + raise ValueError(f"Invalid language code {lang}") + return { + "base_tokenizer": "simple", + "language": lang_mapping[lang], + "max_token_length": 40, + "lower_case": True, + "stem": True, + "remove_stop_words": False, + "ascii_folding": False, + "ngram_min_length": 3, + "ngram_max_length": 3, + "prefix_only": False, + } + # New unified API overload @overload def create_index( @@ -1532,14 +1723,16 @@ class Table(ABC): replace: bool = ..., accelerator: Optional[str] = ..., index_cache_size: Optional[int] = ..., - *, - index_type: VectorIndexType = ..., - wait_timeout: Optional[timedelta] = ..., num_bits: int = ..., + index_type: Literal[ + "IVF_FLAT", "IVF_SQ", "IVF_PQ", "IVF_RQ", "IVF_HNSW_SQ", "IVF_HNSW_PQ" + ] = ..., max_iterations: int = ..., sample_rate: int = ..., m: int = ..., ef_construction: int = ..., + *, + wait_timeout: Optional[timedelta] = ..., name: Optional[str] = ..., train: bool = ..., target_partition_size: Optional[int] = ..., @@ -1547,22 +1740,30 @@ class Table(ABC): def create_index( self, - metric: DistanceType = "l2", + metric: str = "l2", num_partitions: Optional[int] = None, num_sub_vectors: Optional[int] = None, vector_column_name: str = VECTOR_COLUMN_NAME, replace: bool = True, accelerator: Optional[str] = None, index_cache_size: Optional[int] = None, - *, - index_type: VectorIndexType = "IVF_PQ", - wait_timeout: Optional[timedelta] = None, num_bits: int = 8, + index_type: Literal[ + "IVF_FLAT", + "IVF_SQ", + "IVF_PQ", + "IVF_RQ", + "IVF_HNSW_SQ", + "IVF_HNSW_PQ", + "IVF_HNSW_FLAT", + ] = "IVF_PQ", max_iterations: int = 50, sample_rate: int = 256, m: int = 20, ef_construction: int = 300, + *, config: Optional[IndexConfigType] = None, + wait_timeout: Optional[timedelta] = None, name: Optional[str] = None, train: bool = True, target_partition_size: Optional[int] = None, @@ -2019,6 +2220,7 @@ class Table(ABC): query_type: QueryType = "auto", ordering_field_name: Optional[str] = None, fts_columns: Optional[Union[str, List[str]]] = None, + fast_search: Optional[bool] = None, ) -> LanceQueryBuilder: """Create a search query to find the nearest neighbors of the given query vector. We currently support [vector search](https://lancedb.com/docs/search/vector-search/) @@ -3816,130 +4018,6 @@ class LanceTable(Table): ) ) - def _is_legacy_create_index_call( - self, - first_arg: str, - config: Optional[IndexConfigType], - num_partitions: Optional[int], - num_sub_vectors: Optional[int], - vector_column_name: str, - accelerator: Optional[str], - index_cache_size: Optional[int], - ) -> bool: - """Detect if this is a legacy create_index call.""" - # If config is provided, it's definitely the new API - if config is not None: - return False - - # If old-style parameters were explicitly set, it's legacy - if any( - x is not None - for x in (num_partitions, num_sub_vectors, accelerator, index_cache_size) - ): - return True - - # If vector_column_name differs from default, it's legacy - if vector_column_name != VECTOR_COLUMN_NAME: - return True - - # If first arg is a known metric, assume legacy - if first_arg.lower() in KNOWN_METRICS: - return True - - # Otherwise assume new API - return False - - def _build_vector_config_from_legacy_params( - self, - metric: str, - index_type: str, - num_partitions: Optional[int], - num_sub_vectors: Optional[int], - num_bits: int, - max_iterations: int, - sample_rate: int, - m: int, - ef_construction: int, - target_partition_size: Optional[int], - accelerator: Optional[str], - ) -> IndexConfigType: - """Build an index config object from legacy parameters.""" - if index_type == "IVF_FLAT": - return IvfFlat( - distance_type=metric, - num_partitions=num_partitions, - max_iterations=max_iterations, - sample_rate=sample_rate, - target_partition_size=target_partition_size, - accelerator=accelerator, - ) - elif index_type == "IVF_SQ": - return IvfSq( - distance_type=metric, - num_partitions=num_partitions, - max_iterations=max_iterations, - sample_rate=sample_rate, - target_partition_size=target_partition_size, - accelerator=accelerator, - ) - elif index_type == "IVF_PQ": - return IvfPq( - distance_type=metric, - num_partitions=num_partitions, - num_sub_vectors=num_sub_vectors, - num_bits=num_bits, - max_iterations=max_iterations, - sample_rate=sample_rate, - target_partition_size=target_partition_size, - accelerator=accelerator, - ) - elif index_type == "IVF_RQ": - return IvfRq( - distance_type=metric, - num_partitions=num_partitions, - num_bits=num_bits, - max_iterations=max_iterations, - sample_rate=sample_rate, - target_partition_size=target_partition_size, - accelerator=accelerator, - ) - elif index_type == "IVF_HNSW_PQ": - return HnswPq( - distance_type=metric, - num_partitions=num_partitions, - num_sub_vectors=num_sub_vectors, - num_bits=num_bits, - max_iterations=max_iterations, - sample_rate=sample_rate, - m=m, - ef_construction=ef_construction, - target_partition_size=target_partition_size, - accelerator=accelerator, - ) - elif index_type == "IVF_HNSW_SQ": - return HnswSq( - distance_type=metric, - num_partitions=num_partitions, - max_iterations=max_iterations, - sample_rate=sample_rate, - m=m, - ef_construction=ef_construction, - target_partition_size=target_partition_size, - accelerator=accelerator, - ) - elif index_type == "IVF_HNSW_FLAT": - return HnswFlat( - distance_type=metric, - num_partitions=num_partitions, - max_iterations=max_iterations, - sample_rate=sample_rate, - m=m, - ef_construction=ef_construction, - target_partition_size=target_partition_size, - ) - else: - raise ValueError(f"Unknown index type {index_type}") - def drop_index(self, name: str) -> None: """ Drops an index from the table @@ -4050,6 +4128,7 @@ class LanceTable(Table): *, replace: bool = True, index_type: ScalarIndexType = "BTREE", + wait_timeout: Optional[timedelta] = None, name: Optional[str] = None, ): """Create a scalar index on a column. @@ -4058,6 +4137,13 @@ class LanceTable(Table): Use :meth:`create_index` with a BTree, Bitmap, or LabelList config instead. Example: ``table.create_index("column", config=BTree())`` """ + if index_type == "scalar": + warnings.warn( + 'index_type="scalar" is deprecated; use "BTREE" instead', + DeprecationWarning, + stacklevel=2, + ) + index_type = "BTREE" if index_type == "BTREE": config = BTree() elif index_type == "BITMAP": @@ -4067,7 +4153,13 @@ class LanceTable(Table): else: raise ValueError(f"Unknown index type {index_type}") return LOOP.run( - self._table.create_index(column, replace=replace, config=config, name=name) + self._table.create_index( + column, + replace=replace, + config=config, + wait_timeout=wait_timeout, + name=name, + ) ) @deprecation.deprecated( @@ -4099,6 +4191,7 @@ class LanceTable(Table): prefix_only: bool = False, block_size: int = 128, document_granularity: DocumentGranularity = DocumentGranularity.ROW, + wait_timeout: Optional[timedelta] = None, name: Optional[str] = None, ): """Create a full-text search index on a column. @@ -4163,6 +4256,7 @@ class LanceTable(Table): replace=replace, config=config, name=name, + wait_timeout=wait_timeout, ) ) except (ValueError, RuntimeError) as e: @@ -4173,69 +4267,6 @@ class LanceTable(Table): ) raise e - @staticmethod - def infer_tokenizer_configs(tokenizer_name: str) -> dict: - if tokenizer_name == "default": - return { - "base_tokenizer": "simple", - "language": "English", - "max_token_length": 40, - "lower_case": True, - "stem": False, - "remove_stop_words": False, - "ascii_folding": False, - "ngram_min_length": 3, - "ngram_max_length": 3, - "prefix_only": False, - } - elif tokenizer_name == "raw": - return { - "base_tokenizer": "raw", - "language": "English", - "max_token_length": None, - "lower_case": False, - "stem": False, - "remove_stop_words": False, - "ascii_folding": False, - "ngram_min_length": 3, - "ngram_max_length": 3, - "prefix_only": False, - } - elif tokenizer_name == "whitespace": - return { - "base_tokenizer": "whitespace", - "language": "English", - "max_token_length": None, - "lower_case": False, - "stem": False, - "remove_stop_words": False, - "ascii_folding": False, - "ngram_min_length": 3, - "ngram_max_length": 3, - "prefix_only": False, - } - - # or it's with language stemming with pattern like "en_stem" - if len(tokenizer_name) != 7: - raise ValueError(f"Invalid tokenizer name {tokenizer_name}") - lang = tokenizer_name[:2] - if tokenizer_name[-5:] != "_stem": - raise ValueError(f"Invalid tokenizer name {tokenizer_name}") - if lang not in lang_mapping: - raise ValueError(f"Invalid language code {lang}") - return { - "base_tokenizer": "simple", - "language": lang_mapping[lang], - "max_token_length": 40, - "lower_case": True, - "stem": True, - "remove_stop_words": False, - "ascii_folding": False, - "ngram_min_length": 3, - "ngram_max_length": 3, - "prefix_only": False, - } - def add( self, data: DATA, @@ -4386,6 +4417,7 @@ class LanceTable(Table): query_type: Literal["vector"] = "vector", ordering_field_name: Optional[str] = None, fts_columns: Optional[Union[str, List[str]]] = None, + fast_search: Optional[bool] = None, ) -> LanceVectorQueryBuilder: ... @overload @@ -4396,6 +4428,7 @@ class LanceTable(Table): query_type: Literal["fts"] = "fts", ordering_field_name: Optional[str] = None, fts_columns: Optional[Union[str, List[str]]] = None, + fast_search: Optional[bool] = None, ) -> LanceFtsQueryBuilder: ... @overload @@ -4408,6 +4441,7 @@ class LanceTable(Table): query_type: Literal["hybrid"] = "hybrid", ordering_field_name: Optional[str] = None, fts_columns: Optional[Union[str, List[str]]] = None, + fast_search: Optional[bool] = None, ) -> LanceHybridQueryBuilder: ... @overload @@ -4418,6 +4452,7 @@ class LanceTable(Table): query_type: QueryType = "auto", ordering_field_name: Optional[str] = None, fts_columns: Optional[Union[str, List[str]]] = None, + fast_search: Optional[bool] = None, ) -> LanceEmptyQueryBuilder: ... def search( @@ -4429,6 +4464,7 @@ class LanceTable(Table): query_type: QueryType = "auto", ordering_field_name: Optional[str] = None, fts_columns: Optional[Union[str, List[str]]] = None, + fast_search: Optional[bool] = None, ) -> LanceQueryBuilder: """Create a search query to find the nearest neighbors of the given query vector. We currently support [vector search](https://lancedb.com/docs/search/vector-search/) @@ -4513,6 +4549,7 @@ class LanceTable(Table): vector_column_name=vector_column_name, ordering_field_name=ordering_field_name, fts_columns=fts_columns or [], + fast_search=fast_search, ) @classmethod diff --git a/python/python/tests/test_remote_db.py b/python/python/tests/test_remote_db.py index 049a5fc58..db3394735 100644 --- a/python/python/tests/test_remote_db.py +++ b/python/python/tests/test_remote_db.py @@ -1155,8 +1155,9 @@ def test_table_create_indices(): # Test create_fts_index with custom name (legacy method) with pytest.warns(DeprecationWarning, match="create_fts_index"): table.create_fts_index( - "text", + field_names="text", wait_timeout=timedelta(seconds=2), + tokenizer_name="en_stem", block_size=256, custom_stop_words=["cloud"], name="custom_fts_idx", @@ -1165,6 +1166,8 @@ def test_table_create_indices(): # Test create_index with custom name (legacy form: vector_column_name kwarg) with pytest.warns(DeprecationWarning, match="create_index"): table.create_index( + "cosine", + 2, vector_column_name="vector", wait_timeout=timedelta(seconds=10), name="custom_vector_idx", @@ -1177,7 +1180,8 @@ def test_table_create_indices(): scalar_req = received_requests[0] assert "name" in scalar_req assert scalar_req["name"] == "custom_scalar_idx" - assert scalar_req["replace"] is False + # The wire protocol omits replace=True (the server default). + assert "replace" not in scalar_req # Check FTS index request has custom name fts_req = received_requests[1] @@ -1186,18 +1190,39 @@ def test_table_create_indices(): assert fts_req["replace"] is False assert fts_req["block_size"] == 256 assert fts_req["custom_stop_words"] == ["cloud"] + assert fts_req["language"] == "English" + assert fts_req["stem"] is True # Check vector index request has custom name vector_req = received_requests[2] assert "name" in vector_req assert vector_req["name"] == "custom_vector_idx" assert "replace" not in vector_req + assert vector_req["metric_type"] == "cosine" + assert vector_req["num_partitions"] == 2 + + with pytest.warns(DeprecationWarning, match="create_index"): + table.create_index( + index_type="IVF_HNSW_SQ", + num_partitions=1, + m=10, + ef_construction=50, + max_iterations=12, + sample_rate=64, + ) + tuned_req = received_requests[-1] + assert tuned_req["index_type"] == "IVF_HNSW_SQ" + assert tuned_req["num_partitions"] == 1 + assert tuned_req["m"] == 10 + assert tuned_req["ef_construction"] == 50 + assert tuned_req["max_iterations"] == 12 + assert tuned_req["sample_rate"] == 64 table.wait_for_index(["custom_scalar_idx"], timedelta(seconds=2)) table.wait_for_index( ["custom_fts_idx", "custom_vector_idx"], timedelta(seconds=2) ) - table.drop_index("custom_vector_idx") + table.drop_index(name="custom_vector_idx") table.drop_index("custom_scalar_idx") table.drop_index("custom_fts_idx") diff --git a/python/python/tests/test_table_signatures.py b/python/python/tests/test_table_signatures.py new file mode 100644 index 000000000..346935b2b --- /dev/null +++ b/python/python/tests/test_table_signatures.py @@ -0,0 +1,253 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright The LanceDB Authors + +from datetime import timedelta +import inspect +from unittest.mock import AsyncMock, MagicMock + +import lancedb +from lancedb.expr import col +from lancedb.index import BTree, FTS, HnswSq, IvfPq +from lancedb.remote.table import RemoteTable +from lancedb.table import LanceTable, Table +import pyarrow as pa +import pytest + + +@pytest.mark.parametrize( + "method", + [ + "delete", + "drop_index", + "index_stats", + "create_index", + "create_scalar_index", + "create_fts_index", + "search", + "cleanup_old_versions", + "compact_files", + "optimize", + "to_polars", + ], +) +def test_sync_table_signatures(method): + """A Table call must bind the same arguments on either backend.""" + + def parameters(cls): + return [ + (p.name, p.kind, p.default) + for p in inspect.signature(getattr(cls, method)).parameters.values() + ] + + assert parameters(Table) == parameters(LanceTable) == parameters(RemoteTable) + + +@pytest.fixture(params=[LanceTable, RemoteTable], ids=["local", "remote"]) +def sync_table(request, monkeypatch): + """Exercise real sync wrappers with a recording async backend.""" + handle = MagicMock() + handle.name = "test" + handle.create_index = AsyncMock() + handle.delete = AsyncMock() + handle.drop_index = AsyncMock() + handle.index_stats = AsyncMock() + if request.param is RemoteTable: + table = RemoteTable(handle, "test_db") + else: + table = LanceTable.__new__(LanceTable) + table._table = handle + monkeypatch.setattr(table, "_ensure_no_legacy_fts_index", lambda: None) + return table, handle + + +def test_sync_keyword_names(sync_table): + table, handle = sync_table + predicate = col("id") == 1 + table.delete(where=predicate) + handle.delete.assert_awaited_once() + expected = predicate._inner if isinstance(table, LanceTable) else predicate + assert handle.delete.await_args.args[0] is expected + table.drop_index(name="vector_idx") + handle.drop_index.assert_awaited_once_with("vector_idx") + table.index_stats(index_name="vector_idx") + handle.index_stats.assert_awaited_once_with("vector_idx") + + +def test_legacy_vector_positional_arguments(sync_table): + table, handle = sync_table + with pytest.warns(DeprecationWarning): + table.create_index("cosine", 2) + args, kwargs = handle.create_index.await_args + assert args == ("vector",) + assert isinstance(kwargs["config"], IvfPq) + assert kwargs["config"].distance_type == "cosine" + assert kwargs["config"].num_partitions == 2 + assert kwargs["replace"] is True + + +def test_vector_tuning_options(sync_table): + table, handle = sync_table + timeout = timedelta(seconds=2) + with pytest.warns(DeprecationWarning): + table.create_index( + index_type="IVF_HNSW_SQ", + num_partitions=1, + m=10, + ef_construction=50, + max_iterations=12, + sample_rate=64, + target_partition_size=1000, + wait_timeout=timeout, + name="custom_idx", + replace=False, + train=False, + ) + kwargs = handle.create_index.await_args.kwargs + config = kwargs.pop("config") + assert isinstance(config, HnswSq) + for name, value in { + "num_partitions": 1, + "m": 10, + "ef_construction": 50, + "max_iterations": 12, + "sample_rate": 64, + "target_partition_size": 1000, + }.items(): + assert getattr(config, name) == value + assert kwargs == dict( + wait_timeout=timeout, name="custom_idx", replace=False, train=False + ) + + +def test_scalar_defaults_and_wait_timeout(sync_table): + table, handle = sync_table + timeout = timedelta(seconds=2) + with pytest.warns(DeprecationWarning): + table.create_scalar_index("id", wait_timeout=timeout) + kwargs = handle.create_index.await_args.kwargs + assert isinstance(kwargs["config"], BTree) + assert kwargs["replace"] is True + assert kwargs["wait_timeout"] == timeout + + +def test_fts_tokenizer_and_wait_timeout(sync_table): + table, handle = sync_table + timeout = timedelta(seconds=2) + with pytest.warns(DeprecationWarning): + table.create_fts_index( + field_names="text", + tokenizer_name="en_stem", + replace=True, + wait_timeout=timeout, + custom_stop_words=["example"], + ) + args, kwargs = handle.create_index.await_args + assert args == ("text",) + assert kwargs["replace"] is True + assert kwargs["wait_timeout"] == timeout + config = kwargs["config"] + assert isinstance(config, FTS) + assert config.language == "English" + assert config.stem is True + assert config.custom_stop_words == ["example"] + + +@pytest.mark.parametrize( + "kwargs, message", + [ + ({"use_tantivy": True}, "Tantivy-based FTS has been removed"), + ({"ordering_field_names": "id"}, "ordering_field_names"), + ({"writer_heap_size": 123}, "writer_heap_size"), + ({"tokenizer_name": "invalid"}, "Invalid tokenizer name"), + ], +) +def test_fts_unsupported_options(sync_table, kwargs, message): + table, handle = sync_table + with pytest.warns(DeprecationWarning), pytest.raises(ValueError, match=message): + table.create_fts_index("text", **kwargs) + handle.create_index.assert_not_awaited() + + +@pytest.mark.parametrize( + "method, kwargs", + [ + ( + "cleanup_old_versions", + {"older_than": timedelta(days=1), "delete_unverified": True}, + ), + ("compact_files", {"target_rows_per_fragment": 1000}), + ], +) +def test_remote_maintenance_options_warn(method, kwargs): + table = RemoteTable(MagicMock(name="test"), "test_db") + with pytest.warns(UserWarning, match="no-op"): + getattr(table, method)(**kwargs) + + +def test_remote_optimize_options(): + handle = MagicMock() + handle.optimize = AsyncMock(side_effect=NotImplementedError("remote optimize")) + table = RemoteTable(handle, "test_db") + age = timedelta(days=1) + with pytest.raises(NotImplementedError, match="remote optimize"): + table.optimize(cleanup_older_than=age, delete_unverified=True, retrain=True) + handle.optimize.assert_awaited_once_with( + cleanup_older_than=age, delete_unverified=True, retrain=True + ) + + +@pytest.mark.parametrize( + "method, old, new, value", + [ + ("delete", "predicate", "where", "id = 1"), + ("drop_index", "index_name", "name", "vector_idx"), + ("index_stats", "index_uuid", "index_name", "vector_idx"), + ("create_fts_index", "column", "field_names", "text"), + ], +) +def test_deprecated_remote_keywords(method, old, new, value): + handle = MagicMock() + handle.name = "test" + target = "create_index" if method == "create_fts_index" else method + setattr(handle, target, AsyncMock()) + table = RemoteTable(handle, "test_db") + with pytest.warns(DeprecationWarning, match=f"{old} is deprecated"): + getattr(table, method)(**{old: value}) + assert getattr(handle, target).await_args.args == (value,) + with pytest.raises(TypeError, match="Cannot specify both"): + getattr(table, method)(**{old: value, new: value}) + + +@pytest.mark.parametrize("remote_wrapper", [False, True], ids=["local", "remote"]) +def test_sync_search_and_polars(tmp_path, remote_wrapper): + """Use the native engine through both sync wrappers, without a cloud service.""" + local = lancedb.connect(tmp_path).create_table( + "test", + [{"id": 1, "text": "quick fox", "vector": [1.0, 1.0]}], + ) + table = RemoteTable(local._table, "test_db") if remote_wrapper else local + table.create_index("text", config=FTS()) + assert table.search("quick", query_type="fts").to_arrow()["id"].to_pylist() == [1] + # Both backends must accept the deprecated ordering argument in the same slot. + with pytest.warns(DeprecationWarning, match="ordering_field_name"): + table.search("quick", None, "fts", "id", ["text"]) + fast = table.search([1.0, 1.0], fast_search=True).to_arrow() + assert len(fast) == 0 # No vector index: fast_search excludes unindexed rows. + assert table.search([1.0, 1.0]).to_arrow()["id"].to_pylist() == [1] + if remote_wrapper: + with pytest.raises(NotImplementedError, match=r"table.search\(\).to_polars"): + table.to_polars(batch_size=1) + assert table.search().to_polars()["id"].to_list() == [1] + else: + assert table.to_polars(batch_size=1).collect()["id"].to_list() == [1] + table.delete(where="id = 1") + assert table.count_rows() == 0 + + +def test_index_replacement_defaults(tmp_path): + local = lancedb.connect(tmp_path).create_table("test", pa.table({"id": [1]})) + table = RemoteTable(local._table, "test_db") + for sync in [local, table]: + with pytest.warns(DeprecationWarning): + sync.create_scalar_index("id") + assert len(list(table.list_indices())) == 1