Merge branch 'main' into gatekeeper/fix-2325-1

# Conflicts:
#	python/python/lancedb/__init__.py

docs/src/python/python.md

@@ -42,6 +42,8 @@ listing a storage directory.
 
 ::: lancedb.table.Table
 
+::: lancedb.table.CompactionOptions
+
 ::: lancedb.table.FragmentStatistics
 
 ::: lancedb.table.FragmentSummaryStats
@@ -223,14 +225,10 @@ tokens = list(
 Blob columns store large binary values out of line so they can be read lazily
 instead of being materialized with the rest of the row.
 
-`lancedb.BlobType` is `lance.blob.BlobType` when pylance is installed. Without
-pylance, LanceDB uses a matching `lance.blob.v2` extension type so blob columns
-still work. Queries return descriptors. Call
-[`fetch_blob_files`][lancedb.table.Table.fetch_blob_files] for lazy reads or
-[`fetch_blobs`][lancedb.table.Table.fetch_blobs] for eager bytes.
-
 ::: lancedb.blob
 
+::: lancedb.BlobType
+
 ::: lancedb._blob.BlobFile
     options:
       show_root_full_path: false

nodejs/__test__/arrow.test.ts

@@ -1,7 +1,5 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright The LanceDB Authors
-import * as fs from "node:fs";
-import * as vm from "node:vm";
 import * as arrow15 from "apache-arrow-15";
 import * as arrow16 from "apache-arrow-16";
 import * as arrow17 from "apache-arrow-17";
@@ -42,41 +40,6 @@ function sampleRecords(): Array<Record<string, any>> {
   ];
 }
 
-it("serializes an Arrow Table created in another JavaScript realm", async () => {
-  const context = vm.createContext({
-    TextDecoder,
-    TextEncoder,
-    console,
-    setTimeout,
-    clearTimeout,
-  });
-  vm.runInContext(
-    fs.readFileSync(
-      require.resolve("apache-arrow-15/Arrow.es2015.min"),
-      "utf8",
-    ),
-    context,
-  );
-  const foreignTable: unknown = vm.runInContext(
-    "Arrow.tableFromArrays({ id: new Int32Array([1, 2, 3]), text: ['foo', 'bar', 'baz'] })",
-    context,
-  );
-
-  const foreignMetadata = (
-    foreignTable as { schema: { metadata: Map<string, string> } }
-  ).schema.metadata;
-  expect(foreignMetadata).not.toBeInstanceOf(Map);
-
-  const buf = await fromDataToBuffer(
-    foreignTable as Parameters<typeof fromDataToBuffer>[0],
-  );
-  const actual = currentTableFromIPC(buf);
-
-  expect(actual.numRows).toBe(3);
-  expect(actual.getChild("id")?.toJSON()).toEqual([1, 2, 3]);
-  expect(actual.getChild("text")?.toJSON()).toEqual(["foo", "bar", "baz"]);
-});
-
 it("preserves field metadata from a provided schema", async function () {
   const jsonMetadata = new Map([["ARROW:extension:name", "lance.json"]]);
   const schema = new CurrentSchema([

nodejs/lancedb/arrow.ts

@@ -72,7 +72,8 @@ export type FieldLike =
     };
 
 export type DataLike =
-  | import("apache-arrow").Data
+  // biome-ignore lint/suspicious/noExplicitAny: <explanation>
+  | import("apache-arrow").Data<Struct<any>>
   | {
       // biome-ignore lint/suspicious/noExplicitAny: <explanation>
       type: any;
@@ -81,7 +82,6 @@ export type DataLike =
       stride: number;
       nullable: boolean;
       children: DataLike[];
-      dictionary?: { data: readonly DataLike[] };
       get nullCount(): number;
       // biome-ignore lint/suspicious/noExplicitAny: <explanation>
       values: Buffers<any>[BufferType.DATA];

nodejs/lancedb/sanitize.ts

@@ -94,24 +94,17 @@ export function sanitizeMetadata(
   if (metadataLike === undefined || metadataLike === null) {
     return undefined;
   }
-
-  let entries: IterableIterator<[unknown, unknown]>;
-  try {
-    entries = Map.prototype.entries.call(metadataLike);
-  } catch {
+  if (!(metadataLike instanceof Map)) {
     throw Error("Expected metadata, if present, to be a Map<string, string>");
   }
-
-  const metadata = new Map<string, string>();
-  for (const [key, value] of entries) {
-    if (typeof key !== "string" || typeof value !== "string") {
+  for (const item of metadataLike) {
+    if (typeof item[0] !== "string" || typeof item[1] !== "string") {
       throw Error(
         "Expected metadata, if present, to be a Map<string, string> but it had non-string keys or values",
       );
     }
-    metadata.set(key, value);
   }
-  return metadata;
+  return metadataLike as Map<string, string>;
 }
 
 export function sanitizeInt(typeLike: object) {

python/python/lancedb/__init__.py

@@ -6,7 +6,7 @@ import importlib.metadata
 import os
 from concurrent.futures import ThreadPoolExecutor
 from datetime import timedelta
-from typing import Dict, Optional, Union, Any, List, Iterable, TYPE_CHECKING
+from typing import Dict, Optional, Union, Any, List, Iterable
 
 __version__ = importlib.metadata.version("lancedb")
 
@@ -20,7 +20,7 @@ from .db import AsyncConnection, DBConnection, LanceDBConnection
 from .remote import ClientConfig
 from .remote.db import RemoteDBConnection
 from .expr import Expr, col, lit, func
-from .schema import blob, vector
+from .schema import blob, vector, BlobType
 from .job import AsyncJob, Job
 from .functions import (
     FunctionArtifactRequest as FunctionArtifactRequest,
@@ -38,7 +38,7 @@ from .materialized_view import (
     MaterializedView,
     MaterializedViewDefinition,
 )
-from .table import AsyncTable, Table
+from .table import AsyncTable, CompactionOptions, Table
 from .types import BaseTokenizerType
 from ._lancedb import Session
 from .namespace import (
@@ -49,19 +49,6 @@ from .namespace import (
 )
 
 
-if TYPE_CHECKING:
-    from lance.blob import BlobType as BlobType
-
-
-def __getattr__(name: str):
-    if name == "BlobType":
-        from .schema import BlobType
-
-        globals()["BlobType"] = BlobType
-        return BlobType
-    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
-
-
 def _check_s3_bucket_with_dots(
     uri: str, storage_options: Optional[Dict[str, str]]
 ) -> None:
@@ -558,6 +545,7 @@ __all__ = [
     "AsyncJob",
     "AsyncLanceNamespaceDBConnection",
     "AsyncTable",
+    "CompactionOptions",
     "FtsToken",
     "col",
     "Expr",

python/python/lancedb/_blob.py

@@ -12,7 +12,7 @@ from typing import TYPE_CHECKING, Optional, Union
 import pyarrow as pa
 
 from .expr import Expr
-from .schema import row_addressable_blob_v2_paths
+from .schema import blob_v2_column_paths
 from .types import BlobMode, QueryProjection, QueryProjectionSpec
 
 if TYPE_CHECKING:
@@ -119,7 +119,7 @@ def blob_v2_projection_sources(
     schema: pa.Schema,
     projection: QueryProjection,
 ) -> dict[str, str]:
-    blob_columns = row_addressable_blob_v2_paths(schema)
+    blob_columns = blob_v2_column_paths(schema)
     if not blob_columns:
         return {}
     columns = set(blob_columns)
@@ -140,9 +140,7 @@ def v2_projection_needs_row_id(
 ) -> bool:
     if with_row_id:
         return False
-    return projection_includes_blob_column(
-        projection, row_addressable_blob_v2_paths(schema)
-    )
+    return projection_includes_blob_column(projection, blob_v2_column_paths(schema))
 
 
 def blob_auto_row_id_for_scan(
@@ -272,8 +270,7 @@ def _iter_projection_pairs(
             if isinstance(expr, str):
                 yield name, expr
             elif isinstance(expr, Expr):
-                source = expr._column_name()
-                yield name, source if source is not None else expr.to_sql()
+                yield name, expr.to_sql()
         return
     for column in projection:
         if isinstance(column, str):
@@ -283,8 +280,7 @@ def _iter_projection_pairs(
             if isinstance(expr, str):
                 yield name, expr
             elif isinstance(expr, Expr):
-                source = expr._column_name()
-                yield name, source if source is not None else expr.to_sql()
+                yield name, expr.to_sql()
 
 
 def _set_blob_column(tbl: pa.Table, output_name: str, blobs: pa.Array) -> pa.Table:

python/python/lancedb/_lancedb.pyi

@@ -87,7 +87,6 @@ class PyExpr:
     def contains(self, substr: "PyExpr") -> "PyExpr": ...
     def isin(self, values: List["PyExpr"]) -> "PyExpr": ...
     def cast(self, data_type: pa.DataType) -> "PyExpr": ...
-    def column_name(self) -> Optional[str]: ...
     def to_sql(self) -> str: ...
 
 def expr_col(name: str) -> PyExpr: ...
@@ -374,6 +373,7 @@ class Table:
         *,
         cleanup_since_ms: Optional[int] = None,
         delete_unverified: Optional[bool] = None,
+        compaction_options: Optional[Dict[str, Any]] = None,
     ) -> OptimizeStats: ...
     async def uri(self) -> str: ...
     async def initial_storage_options(self) -> Optional[Dict[str, str]]: ...
@@ -609,7 +609,6 @@ class PyQueryRequest:
     filter: Optional[Union[str, bytes]]
     full_text_search: Optional[FullTextQuery]
     select: Optional[Union[str, List[str]]]
-    select_source_columns: Optional[Dict[str, str]]
     fast_search: Optional[bool]
     with_row_id: Optional[bool]
     use_lsm: Optional[bool]

python/python/lancedb/db.py

@@ -16,7 +16,6 @@ from typing import (
     Iterable,
     List,
     Literal,
-    Mapping,
     Optional,
     Union,
 )
@@ -688,35 +687,17 @@ class DBConnection(EnforceOverrides):
         """
         raise NotImplementedError("serialize is not supported for this connection type")
 
-    def create_function(
-        self,
-        definition: UdfDefinition,
-        *,
-        secrets: Optional[Mapping[str, str]] = None,
-    ) -> FunctionVersion:
+    def create_function(self, definition: UdfDefinition) -> FunctionVersion:
         """Register a scalar Python UDF and wait for its immutable version.
 
-        ``secrets`` must contain exactly the names declared by
-        ``@udf(secrets=[...])``. Values are sent in the create request and
-        stored server-side in the private execution artifact; returned
-        Function and Job metadata contain only the declared names.
         This is the blocking counterpart of :meth:`create_function_async`.
         Local connections raise ``NotImplementedError``.
         """
-        return self.create_function_async(definition, secrets=secrets).wait()
+        return self.create_function_async(definition).wait()
 
-    def create_function_async(
-        self,
-        definition: UdfDefinition,
-        *,
-        secrets: Optional[Mapping[str, str]] = None,
-    ) -> Job[FunctionVersion]:
+    def create_function_async(self, definition: UdfDefinition) -> Job[FunctionVersion]:
         """Register a scalar Python UDF through the remote Function catalog.
 
-        ``secrets`` must contain exactly the names declared by
-        ``@udf(secrets=[...])``. Values are sent in the create request and
-        stored server-side in the private execution artifact; returned
-        Function and Job metadata contain only the declared names.
         Submission returns a typed job. The immutable Function version becomes
         available only when :meth:`Job.wait` succeeds. Local connections raise
         ``NotImplementedError``.
@@ -1424,13 +1405,8 @@ class LanceDBConnection(DBConnection):
         return Job(self._conn.job(job_id))
 
     @override
-    def create_function_async(
-        self,
-        definition: UdfDefinition,
-        *,
-        secrets: Optional[Mapping[str, str]] = None,
-    ) -> Job[FunctionVersion]:
-        job = LOOP.run(self._conn.create_function_async(definition, secrets=secrets))
+    def create_function_async(self, definition: UdfDefinition) -> Job[FunctionVersion]:
+        job = LOOP.run(self._conn.create_function_async(definition))
         return Job(job)
 
     @override
@@ -2249,24 +2225,17 @@ class AsyncConnection(object):
         return AsyncJob(self._inner.job(job_id))
 
     async def create_function_async(
-        self,
-        definition: UdfDefinition,
-        *,
-        secrets: Optional[Mapping[str, str]] = None,
+        self, definition: UdfDefinition
     ) -> AsyncJob[FunctionVersion]:
         """Register a scalar Python UDF through the remote Function catalog.
 
-        ``secrets`` must contain exactly the names declared by
-        ``@udf(secrets=[...])``. Values are sent in the create request and
-        stored server-side in the private execution artifact; returned
-        Function and Job metadata contain only the declared names.
         The returned typed job resolves to the immutable Function version.
         Local connections raise ``NotImplementedError``.
         """
         if not isinstance(definition, UdfDefinition):
             raise TypeError("create_function_async requires a @udf definition")
         inner = await self._inner.create_function_async(
-            definition._submission_json(secrets)
+            definition.registration_request.to_canonical_json()
         )
         return _typed_job(inner, FunctionVersion.from_json)
 

python/python/lancedb/expr.py

@@ -249,10 +249,6 @@ class Expr:
 
     # ── utilities ────────────────────────────────────────────────────────────
 
-    def _column_name(self) -> str | None:
-        """Return the source name when this is a bare column expression."""
-        return self._inner.column_name()
-
     def to_sql(self) -> str:
         """Render the expression as a SQL string (useful for debugging)."""
         return self._inner.to_sql()
@@ -316,7 +312,7 @@ def func(name: str, *args: ExprLike) -> Expr:
     --------
     >>> from lancedb.expr import col, func
     >>> func("lower", col("name"))
-    Expr(lower(`name`))
+    Expr(lower(name))
     """
     inner_args = [_coerce(a)._inner for a in args]
     return Expr(expr_func(name, inner_args))

python/python/lancedb/functions.py

@@ -4,7 +4,7 @@
 """Canonical Function values exchanged with LanceDB Enterprise services.
 
 These immutable models contain client/wire state only. Catalog persistence,
-environment bake, secret resolution, and execution are owned by Sophon.
+environment bake, and execution are owned by Sophon.
 ``RefreshColumnResult`` is also the backend-neutral result of a local
 expression-backed refresh job.
 """
@@ -229,7 +229,7 @@ class PythonEnvironmentSpec(_RemoteValue):
 
 
 class PythonRuntimeSpec(_RemoteValue):
-    """Remote runtime definition with non-secret environment values.
+    """Remote runtime definition with environment values.
 
     V1 supports ``kind="python"``. Newer runtime kinds remain readable, while
     their unknown payload fields are intentionally not retained by the client.
@@ -268,7 +268,6 @@ class FunctionVersion(_RemoteValue):
     runtime: PythonRuntimeSpec
     runtime_digest: str
     environment_digest: str
-    required_secrets: tuple[str, ...] = ()
     created_at: str
 
     def __call__(self, **inputs: Any) -> FunctionApplication:
@@ -330,17 +329,12 @@ class FunctionVersion(_RemoteValue):
 
 
 class FunctionRegistrationRequest(_RemoteValue):
-    """Stable remote registration envelope produced by :func:`udf`.
-
-    Only secret names are represented. Secret values are supplied separately
-    when the definition is submitted and are not part of this durable value.
-    """
+    """Stable remote registration envelope produced by :func:`udf`."""
 
     name: str
     artifact: FunctionArtifactRequest
     signature: FunctionSignature
     runtime: PythonRuntimeSpec
-    required_secrets: tuple[str, ...] = ()
 
 
 class FunctionVersionRef(_OpenRemoteValue):
@@ -485,27 +479,6 @@ class RefreshColumnResult(_RemoteValue):
 
 
 _FUNCTION_NAME = re.compile(r"^[A-Za-z_][A-Za-z0-9_.-]*$")
-_SECRET_NAME = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
-# Keep this byte limit aligned with Sophon's MAX_FUNCTION_SECRET_VALUE_BYTES.
-_MAX_FUNCTION_SECRET_VALUE_BYTES = 64 * 1024
-_MAX_FUNCTION_SECRET_VALUES_BYTES = 512 * 1024
-
-
-def _validate_secret_value(name: str, value: Any) -> str:
-    """Validate one secret value before building the create request."""
-    if not isinstance(value, str):
-        raise TypeError(f"Function secret {name!r} value must be a string")
-    if not value:
-        raise ValueError(f"Function secret {name!r} value must be non-empty")
-    if "\0" in value:
-        raise ValueError(f"Function secret {name!r} value must not contain NUL")
-    value_bytes = len(value.encode("utf-8"))
-    if value_bytes > _MAX_FUNCTION_SECRET_VALUE_BYTES:
-        raise ValueError(
-            f"Function secret {name!r} value exceeds the "
-            f"{_MAX_FUNCTION_SECRET_VALUE_BYTES}-byte limit"
-        )
-    return value
 
 
 _GRAMMAR_PRIMITIVES = (
@@ -936,7 +909,6 @@ class UdfDefinition:
         output_schema: Optional[pa.DataType | pa.Field | pa.Schema],
         pip: tuple[str, ...],
         env: Mapping[str, str],
-        secrets: tuple[str, ...],
         python_version: Optional[str],
         conda: tuple[str, ...] = (),
         conda_channels: tuple[str, ...] = (),
@@ -963,17 +935,6 @@ class UdfDefinition:
             for key, value in environment.items()
         ):
             raise TypeError("Function env keys and values must be strings")
-        required_secrets = tuple(sorted(set(secrets)))
-        invalid_secrets = [
-            secret for secret in required_secrets if not _SECRET_NAME.fullmatch(secret)
-        ]
-        if invalid_secrets:
-            raise ValueError(f"invalid Function secret names: {invalid_secrets!r}")
-        overlap = set(environment) & set(required_secrets)
-        if overlap:
-            raise ValueError(
-                f"Function env and secret names must be disjoint: {sorted(overlap)!r}"
-            )
         signature = _infer_signature(function, input_schema, output_schema)
         source = _package_source(function)
         digest = f"sha256:{hashlib.sha256(source).hexdigest()}"
@@ -1002,65 +963,14 @@ class UdfDefinition:
             ),
             signature=signature,
             runtime=runtime,
-            required_secrets=required_secrets,
         )
         functools.update_wrapper(self, function)
 
     @property
     def registration_request(self) -> FunctionRegistrationRequest:
-        """The immutable, value-free client model for a Function submission."""
+        """The immutable request sent by ``create_function_async``."""
         return self._request
 
-    def _submission_json(self, secrets: Optional[Mapping[str, str]]) -> str:
-        """Build one registration submission without retaining values on self."""
-        if secrets is None:
-            secret_values: Mapping[str, str] = {}
-        elif not isinstance(secrets, Mapping):
-            raise TypeError("Function secrets must be a mapping of names to strings")
-        else:
-            secret_values = secrets
-
-        if any(not isinstance(name, str) for name in secret_values):
-            raise TypeError("Function secret names must be strings")
-        expected = set(self._request.required_secrets)
-        provided = set(secret_values)
-        if provided != expected:
-            missing = sorted(expected - provided)
-            unexpected = sorted(provided - expected)
-            details = []
-            if missing:
-                details.append(f"missing: {missing!r}")
-            if unexpected:
-                details.append(f"unexpected: {unexpected!r}")
-            raise ValueError(
-                "Function secret values must exactly match the declared secrets ("
-                + "; ".join(details)
-                + ")"
-            )
-
-        canonical_values = {}
-        total_bytes = 0
-        for name in sorted(secret_values):
-            value = _validate_secret_value(name, secret_values[name])
-            total_bytes += len(value.encode("utf-8"))
-            if total_bytes > _MAX_FUNCTION_SECRET_VALUES_BYTES:
-                raise ValueError(
-                    "Function secret values exceed the "
-                    f"{_MAX_FUNCTION_SECRET_VALUES_BYTES}-byte request limit"
-                )
-            canonical_values[name] = value
-
-        submission = self._request._known_dict()
-        if canonical_values:
-            submission["secret_values"] = canonical_values
-        return json.dumps(
-            submission,
-            ensure_ascii=False,
-            allow_nan=False,
-            sort_keys=True,
-            separators=(",", ":"),
-        )
-
     def __call__(self, *args, **kwargs):
         return self._function(*args, **kwargs)
 
@@ -1078,7 +988,6 @@ def udf(
     output_schema: Optional[pa.DataType | pa.Field | pa.Schema] = None,
     pip: tuple[str, ...] | list[str] = (),
     env: Optional[Mapping[str, str]] = None,
-    secrets: tuple[str, ...] | list[str] = (),
     python_version: Optional[str] = None,
     conda: tuple[str, ...] | list[str] = (),
     conda_channels: tuple[str, ...] | list[str] = (),
@@ -1093,7 +1002,6 @@ def udf(
     output_schema: Optional[pa.DataType | pa.Field | pa.Schema] = None,
     pip: tuple[str, ...] | list[str] = (),
     env: Optional[Mapping[str, str]] = None,
-    secrets: tuple[str, ...] | list[str] = (),
     python_version: Optional[str] = None,
     conda: tuple[str, ...] | list[str] = (),
     conda_channels: tuple[str, ...] | list[str] = (),
@@ -1124,10 +1032,7 @@ def udf(
     conda_channels : sequence of str, optional
         Conda channels in priority order; requires ``conda``.
     env : mapping of str to str, optional
-        Non-secret environment variables. Use ``secrets`` for credentials.
-    secrets : sequence of str, optional
-        Names of secrets required by the callable. Supply their values separately
-        to ``create_function`` or ``create_function_async``.
+        Environment variables included in the Function definition.
     python_version : str, optional
         Remote Python major/minor version. Defaults to the client version.
 
@@ -1149,15 +1054,11 @@ def udf(
     Examples
     --------
     >>> from lancedb import udf
-    >>> @udf(pip=["numpy==2.2.0"], secrets=["MODEL_TOKEN"])
+    >>> @udf(pip=["numpy==2.2.0"])
     ... def score(value: float) -> float:
     ...     return value * 2
     >>> score(1.5)
     3.0
-    >>> db.create_function(  # doctest: +SKIP
-    ...     score, secrets={"MODEL_TOKEN": "user-secret-value"}
-    ... )
-
     """
 
     def decorate(target: Callable[..., Any]) -> UdfDefinition:
@@ -1168,7 +1069,6 @@ def udf(
             output_schema=output_schema,
             pip=tuple(pip),
             env={} if env is None else env,
-            secrets=tuple(secrets),
             python_version=python_version,
             conda=tuple(conda),
             conda_channels=tuple(conda_channels),

python/python/lancedb/query.py

@@ -167,12 +167,6 @@ def _projection_to_scanner_kwargs(columns: QueryProjection) -> Dict[str, Any]:
     return {"columns": projection}
 
 
-def _query_request_projection(req: "PyQueryRequest") -> QueryProjection:
-    if req.select_source_columns is not None:
-        return req.select_source_columns
-    return req.select
-
-
 def _scanner_kwargs_for_query(
     query: Query,
     blob_mode: BlobMode,
@@ -2805,16 +2799,15 @@ class AsyncQueryBase(object):
 
         req = self._inner.to_query_request()
         schema = await self._table.schema()
-        projection = _query_request_projection(req)
         self._blob_auto_row_id = blob_auto_row_id_for_scan(
             schema,
-            projection,
+            req.select,
             with_row_id=self._with_row_id,
         )
         if not self._blob_auto_row_id:
             self._blob_paths = ()
             return
-        self._blob_paths = tuple(blob_v2_projection_sources(schema, projection).keys())
+        self._blob_paths = tuple(blob_v2_projection_sources(schema, req.select).keys())
         self._inner.with_row_id()
 
     def select(self, columns: Union[List[str], dict[str, str]]) -> Self:
@@ -3901,15 +3894,14 @@ class AsyncHybridQuery(AsyncStandardQuery, AsyncVectorQueryBase):
         blob_paths: tuple[str, ...] = ()
         if self._table is not None:
             schema = await self._table.schema()
-            projection = _query_request_projection(req)
             blob_auto_row_id = blob_auto_row_id_for_scan(
                 schema,
-                projection,
+                req.select,
                 with_row_id=self._with_row_id,
             )
             if blob_auto_row_id:
                 blob_paths = tuple(
-                    blob_v2_projection_sources(schema, projection).keys()
+                    blob_v2_projection_sources(schema, req.select).keys()
                 )
         self._blob_auto_row_id = blob_auto_row_id
         self._blob_paths = blob_paths

python/python/lancedb/remote/db.py

@@ -7,7 +7,7 @@ import json
 import logging
 from concurrent.futures import ThreadPoolExecutor
 import sys
-from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Mapping, Optional, Union
+from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Optional, Union
 from urllib.parse import urlparse
 import warnings
 
@@ -742,15 +742,8 @@ class RemoteDBConnection(DBConnection):
         return Job(self._conn.job(job_id))
 
     @override
-    def create_function_async(
-        self,
-        definition: UdfDefinition,
-        *,
-        secrets: Optional[Mapping[str, str]] = None,
-    ) -> Job[FunctionVersion]:
-        return Job(
-            LOOP.run(self._conn.create_function_async(definition, secrets=secrets))
-        )
+    def create_function_async(self, definition: UdfDefinition) -> Job[FunctionVersion]:
+        return Job(LOOP.run(self._conn.create_function_async(definition)))
 
     @override
     def get_function(self, name: str, *, version: str) -> FunctionVersion:

python/python/lancedb/remote/table.py

@@ -36,7 +36,6 @@ from lancedb._lancedb import (
     UpdateResult,
 )
 from lancedb.embeddings.base import EmbeddingFunctionConfig
-from lancedb.expr import Expr
 from lancedb.index import (
     FTS,
     BTree,
@@ -67,7 +66,16 @@ from ..query import (
     LanceTakeQueryBuilder,
     LanceVectorQueryBuilder,
 )
-from ..table import AsyncTable, BlobMode, Branches, IndexStatistics, Query, Table, Tags
+from ..table import (
+    AsyncTable,
+    BlobMode,
+    Branches,
+    CompactionOptions,
+    IndexStatistics,
+    Query,
+    Table,
+    Tags,
+)
 from ..types import BaseTokenizerType
 
 
@@ -864,7 +872,7 @@ class RemoteTable(Table):
 
     def update(
         self,
-        where: Optional[Union[str, Expr]] = None,
+        where: Optional[str] = None,
         values: Optional[dict] = None,
         *,
         values_sql: Optional[Dict[str, str]] = None,
@@ -875,11 +883,9 @@ class RemoteTable(Table):
 
         Parameters
         ----------
-        where: str or [Expr][lancedb.expr.Expr], optional
-            The filter condition. Can be a SQL string or a type-safe
-            [Expr][lancedb.expr.Expr] built with [col][lancedb.expr.col] and
-            [lit][lancedb.expr.lit]. The filter must not be empty, or it will
-            error.
+        where: str, optional
+            The SQL where clause to use when updating rows. For example, 'x = 2'
+            or 'x IN (1, 2, 3)'. The filter must not be empty, or it will error.
         values: dict, optional
             The values to update. The keys are the column names and the values
             are the values to set.
@@ -953,6 +959,7 @@ class RemoteTable(Table):
         *,
         cleanup_older_than: Optional[timedelta] = None,
         delete_unverified: bool = False,
+        compaction_options: Optional[CompactionOptions] = None,
     ):
         """
         optimize() is a no-op on LanceDB Cloud.

python/python/lancedb/schema.py

@@ -4,34 +4,30 @@
 
 """Schema helpers for Lance blob columns."""
 
-import importlib
-from typing import TYPE_CHECKING
-
 import pyarrow as pa
-import pyarrow.ipc
-
-if TYPE_CHECKING:
-    from lance.blob import BlobType as BlobType
 
 _BLOB_EXTENSION_NAME = "lance.blob.v2"
 _BLOB_V1_KEY = "lance-encoding:blob"
 _ARROW_EXT_NAME_KEY = "ARROW:extension:name"
-_BLOB_V2_STORAGE_TYPE = pa.struct(
-    [
-        pa.field("data", pa.large_binary(), nullable=True),
-        pa.field("uri", pa.utf8(), nullable=True),
-        pa.field("position", pa.uint64(), nullable=True),
-        pa.field("size", pa.uint64(), nullable=True),
-    ]
-)
-_resolved_blob_type = None
 
 
-class _FallbackBlobType(pa.ExtensionType):
-    """lance.blob.v2 extension type used when pylance is not installed."""
+class BlobType(pa.ExtensionType):
+    """PyArrow extension type for a Lance blob v2 column.
+
+    Queries return descriptors; call :meth:`~lancedb.table.Table.fetch_blob_files`
+    for lazy reads or :meth:`~lancedb.table.Table.fetch_blobs` for eager bytes.
+    """
 
     def __init__(self) -> None:
-        pa.ExtensionType.__init__(self, _BLOB_V2_STORAGE_TYPE, _BLOB_EXTENSION_NAME)
+        storage_type = pa.struct(
+            [
+                pa.field("data", pa.large_binary(), nullable=True),
+                pa.field("uri", pa.utf8(), nullable=True),
+                pa.field("position", pa.uint64(), nullable=True),
+                pa.field("size", pa.uint64(), nullable=True),
+            ]
+        )
+        super().__init__(storage_type, _BLOB_EXTENSION_NAME)
 
     def __arrow_ext_serialize__(self) -> bytes:
         return b""
@@ -39,16 +35,23 @@ class _FallbackBlobType(pa.ExtensionType):
     @classmethod
     def __arrow_ext_deserialize__(
         cls, storage_type: pa.DataType, serialized: bytes
-    ) -> "_FallbackBlobType":
+    ) -> "BlobType":
         return cls()
 
     def __reduce__(self):
+        # Ensure pickle round-trips on older pyarrow (apache/arrow#35599).
         return type(self).__arrow_ext_deserialize__, (
             self.storage_type,
             self.__arrow_ext_serialize__(),
         )
 
 
+try:
+    pa.register_extension_type(BlobType())  # type: ignore[arg-type]
+except pa.ArrowKeyError:
+    pass
+
+
 def _metadata_value(metadata: dict, key: str):
     return metadata.get(key.encode()) or metadata.get(key)
 
@@ -89,105 +92,43 @@ def is_blob_like_field(field: pa.Field) -> bool:
     return is_blob_v2_field(field) or _metadata_marks_legacy_blob(field.metadata or {})
 
 
-def _collect_blob_paths(schema: pa.Schema, is_blob) -> list[tuple[str, bool]]:
-    """Walk the schema and return (path, has_list_ancestor) for each blob field."""
-    paths: list[tuple[str, bool]] = []
+def _collect_blob_paths(schema: pa.Schema, is_blob) -> list[str]:
+    paths: list[str] = []
 
-    def walk(fields, prefix: str, has_list_ancestor: bool) -> None:
+    def walk(fields, prefix: str) -> None:
         for field in fields:
             path = f"{prefix}.{field.name}" if prefix else field.name
             if is_blob(field):
-                paths.append((path, has_list_ancestor))
+                paths.append(path)
             elif pa.types.is_struct(field.type):
-                walk(field.type, path, has_list_ancestor)
+                walk(field.type, path)
             elif (
                 pa.types.is_list(field.type)
                 or pa.types.is_large_list(field.type)
                 or pa.types.is_fixed_size_list(field.type)
             ):
-                walk([field.type.value_field], path, True)
+                walk([field.type.value_field], path)
 
-    walk(schema, "", False)
+    walk(schema, "")
     return paths
 
 
 def blob_column_paths(schema: pa.Schema) -> list[str]:
     """Dotted paths of blob-like columns (v2 extension or legacy metadata)."""
-    return [path for path, _ in _collect_blob_paths(schema, is_blob_like_field)]
+    return _collect_blob_paths(schema, is_blob_like_field)
 
 
 def blob_v2_column_paths(schema: pa.Schema) -> list[str]:
-    return [path for path, _ in _collect_blob_paths(schema, is_blob_v2_field)]
-
-
-def row_addressable_blob_v2_paths(schema: pa.Schema) -> list[str]:
-    """Blob v2 paths with one blob addressable by table row id.
-
-    ``fetch_blobs`` and the descriptor row-id ride-along address one blob per
-    row, so a blob inside a list container has no row-id slot and no fetch
-    path. Those columns still store and query as raw descriptors.
-    """
-    return [
-        path
-        for path, has_list_ancestor in _collect_blob_paths(schema, is_blob_v2_field)
-        if not has_list_ancestor
-    ]
+    return _collect_blob_paths(schema, is_blob_v2_field)
 
 
 def schema_has_blob_field(schema: pa.Schema) -> bool:
     return bool(blob_column_paths(schema))
 
 
-def _deserialize_registered_type(extension_type: pa.ExtensionType) -> pa.DataType:
-    """Return the type Arrow reconstructs for this extension name."""
-    schema = pa.schema([pa.field("value", extension_type)])
-    restored = pa.ipc.read_schema(schema.serialize())
-    return restored.field("value").type
-
-
-def _resolve_blob_type():
-    """Return the BlobType class this process should use.
-
-    pylance's class when it owns the lance.blob.v2 registry entry,
-    otherwise LanceDB's fallback. A different registered class is an error.
-    """
-    global _resolved_blob_type
-    if _resolved_blob_type is not None:
-        return _resolved_blob_type
-    try:
-        blob_module = importlib.import_module("lance.blob")
-    except ModuleNotFoundError as err:
-        if err.name not in ("lance", "lance.blob"):
-            raise
-    else:
-        blob_type = getattr(blob_module, "BlobType", None)
-        if blob_type is not None:
-            registered_type = _deserialize_registered_type(blob_type())
-            if type(registered_type) is not blob_type:
-                registered_cls = type(registered_type)
-                raise ValueError(
-                    "lance.blob.v2 is already registered by "
-                    f"{registered_cls.__module__}.{registered_cls.__qualname__}"
-                )
-            _resolved_blob_type = blob_type
-            return blob_type
-    try:
-        pa.register_extension_type(_FallbackBlobType())  # type: ignore[arg-type]
-    except pa.ArrowKeyError as err:
-        raise ValueError(
-            "lance.blob.v2 is already registered by another extension class"
-        ) from err
-    _resolved_blob_type = _FallbackBlobType
-    return _resolved_blob_type
-
-
 def blob(name: str, nullable: bool = True) -> pa.Field:
-    """Create a Lance blob v2 column field.
-
-    When pylance is installed this is ``lance.blob.BlobType``.
-    """
-    blob_type = _resolve_blob_type()
-    return pa.field(name, blob_type(), nullable=nullable)
+    """Create a Lance blob v2 column field."""
+    return pa.field(name, BlobType(), nullable=nullable)
 
 
 def vector(dimension: int, value_type: pa.DataType = pa.float32()) -> pa.DataType:
@@ -214,11 +155,3 @@ def vector(dimension: int, value_type: pa.DataType = pa.float32()) -> pa.DataTyp
     ... ])
     """
     return pa.list_(value_type, dimension)
-
-
-def __getattr__(name: str):
-    if name == "BlobType":
-        blob_type = _resolve_blob_type()
-        globals()["BlobType"] = blob_type
-        return blob_type
-    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")

python/python/lancedb/table.py

@@ -22,6 +22,7 @@ from typing import (
     Optional,
     Sequence,
     Tuple,
+    TypedDict,
     Union,
     overload,
 )
@@ -104,12 +105,7 @@ from .util import (
     value_to_sql,
 )
 from .index import lang_mapping
-from .schema import (
-    blob_v2_column_paths,
-    is_blob_v2_field,
-    row_addressable_blob_v2_paths,
-    schema_has_blob_field,
-)
+from .schema import blob_v2_column_paths, schema_has_blob_field
 
 
 def _should_push_down_query_table(
@@ -227,6 +223,88 @@ IndexConfigType = Union[
     FTS,
 ]
 
+
+class CompactionOptions(TypedDict, total=False):
+    """Options that control file compaction during table optimization.
+
+    Unspecified options use Lance's defaults.
+
+    Compaction planning is row based. Lowering ``target_rows_per_fragment``
+    based on the expected row size can bound later compaction passes once
+    oversized fragments have been rewritten. It does not split an existing
+    fragment, so the first pass over an oversized fragment is not subject to
+    that bound. ``max_bytes_per_file`` limits output file size, not compaction
+    memory. Source budgets keep whole planned tasks; if the first task exceeds
+    a budget, that run performs no compaction work.
+
+    Examples
+    --------
+    Derive a steady-state row target from the expected row size:
+
+    >>> desired_fragment_bytes = 750 * 1024 * 1024
+    >>> average_row_bytes = 1_500_000
+    >>> options: CompactionOptions = {
+    ...     "target_rows_per_fragment": max(
+    ...         1, desired_fragment_bytes // average_row_bytes
+    ...     ),
+    ... }
+    >>> await table.optimize(compaction_options=options)  # doctest: +SKIP
+    """
+
+    target_rows_per_fragment: int
+    """Target rows per fragment; existing oversized fragments are not split."""
+
+    max_rows_per_group: int
+    """Maximum number of rows per row group (default: 1,024)."""
+
+    max_bytes_per_file: Optional[int]
+    """Maximum output data-file size; this does not bound compaction memory."""
+
+    materialize_deletions: bool
+    """Whether to rewrite fragments containing deleted rows (default: True)."""
+
+    materialize_deletions_threshold: float
+    """Minimum deleted-row fraction that makes a fragment eligible (default: 0.1)."""
+
+    num_threads: Optional[int]
+    """Number of compaction tasks to run in parallel."""
+
+    batch_size: Optional[int]
+    """Number of rows per input scan batch."""
+
+    io_buffer_size: Optional[int]
+    """Maximum number of bytes queued in the input scan I/O buffer."""
+
+    defer_index_remap: bool
+    """Whether to defer index remapping during compaction (default: False)."""
+
+    index_remap_mode: Literal["direct", "compact"]
+    """How to construct the old-to-new row-address mapping."""
+
+    compaction_mode: Optional[
+        Literal["reencode", "try_binary_copy", "force_binary_copy"]
+    ]
+    """Whether compaction re-encodes data or uses binary copying."""
+
+    binary_copy_read_batch_bytes: Optional[int]
+    """Number of bytes read per batch during binary-copy compaction."""
+
+    max_source_fragments: Optional[int]
+    """Maximum number of source fragments compacted in one run."""
+
+    max_source_rows: Optional[int]
+    """Maximum live source rows per run, applied to whole planned tasks."""
+
+    max_source_bytes: Optional[int]
+    """Maximum source bytes per run, applied to whole planned tasks."""
+
+    excluded_fragment_ids: List[int]
+    """Fragment IDs to leave unchanged and use as planning boundaries."""
+
+    max_overlays_per_fragment: Optional[int]
+    """Maximum overlays before a fragment is fully compacted."""
+
+
 # Known distance metrics for legacy API detection
 KNOWN_METRICS = {"l2", "cosine", "dot", "hamming"}
 
@@ -431,7 +509,6 @@ def _cast_to_target_schema(
 
     def gen():
         for batch in reader:
-            batch = _coerce_blob_write_columns(batch, reordered_schema)
             # Table but not RecordBatch has cast.
             cast_batches = (
                 pa.Table.from_batches([batch]).cast(reordered_schema).to_batches()
@@ -444,166 +521,6 @@ def _cast_to_target_schema(
     return pa.RecordBatchReader.from_batches(reordered_schema, gen())
 
 
-def _coerce_blob_write_columns(
-    batch: pa.RecordBatch, target_schema: pa.Schema
-) -> pa.RecordBatch:
-    """Materialize blob storage structs before the stream leaves Python.
-
-    merge_insert requires its source reader to already match the table's
-    physical schema. Unlike add and insert, it does not pass through
-    LanceDB's Rust blob coercion, so preserving binary input here would
-    reach Lance as binary and fail the schema check.
-    """
-    columns = []
-    fields = []
-    changed = False
-    for field, column in zip(batch.schema, batch.columns):
-        target_field = target_schema.field(field.name)
-        coerced = _coerce_blob_value(column, target_field)
-        if coerced is not column:
-            column = coerced
-            field = pa.field(
-                field.name,
-                coerced.type,
-                field.nullable,
-                target_field.metadata,
-            )
-            changed = True
-        columns.append(column)
-        fields.append(field)
-    if not changed:
-        return batch
-    return pa.RecordBatch.from_arrays(
-        columns, schema=pa.schema(fields, metadata=batch.schema.metadata)
-    )
-
-
-def _coerce_blob_value(column: pa.Array, target_field: pa.Field) -> pa.Array:
-    if is_blob_v2_field(target_field) and _can_coerce_to_blob(column.type):
-        return _coerce_value_to_blob(column, target_field)
-
-    target_type = target_field.type
-    if pa.types.is_struct(target_type) and pa.types.is_struct(column.type):
-        children = []
-        fields = []
-        changed = False
-        for source_field in column.type:
-            source_column = column.field(source_field.name)
-            nested_target = next(
-                (field for field in target_type if field.name == source_field.name),
-                None,
-            )
-            if nested_target is None:
-                children.append(source_column)
-                fields.append(source_field)
-                continue
-            coerced = _coerce_blob_value(source_column, nested_target)
-            if coerced is not source_column:
-                changed = True
-            child_array, child_type = _physical_array_and_type(coerced)
-            children.append(child_array)
-            fields.append(
-                pa.field(
-                    source_field.name,
-                    child_type,
-                    source_field.nullable,
-                    nested_target.metadata,
-                )
-            )
-        if not changed:
-            return column
-        return pa.StructArray.from_arrays(
-            children,
-            fields=fields,
-            mask=column.is_null() if column.null_count else None,
-        )
-
-    if _is_list_like(target_type) and _is_list_like(column.type):
-        return _coerce_blob_list_values(column, target_type.value_field)
-
-    return column
-
-
-def _coerce_blob_list_values(
-    column: pa.Array, target_value_field: pa.Field
-) -> pa.Array:
-    """Coerce blob values inside a list column, preserving offsets and nulls.
-
-    Works on the raw child values window instead of ``pc.list_flatten`` because
-    flatten drops values spanned by null slots, which would misalign offsets.
-    """
-    mask = column.is_null() if column.null_count else None
-    if pa.types.is_fixed_size_list(column.type):
-        list_size = column.type.list_size
-        values = column.values.slice(column.offset * list_size, len(column) * list_size)
-        coerced = _coerce_blob_value(values, target_value_field)
-        if coerced is values:
-            return column
-        physical_values, _ = _physical_array_and_type(coerced)
-        return pa.FixedSizeListArray.from_arrays(physical_values, list_size, mask=mask)
-    offsets = column.offsets
-    first_offset = offsets[0].as_py()
-    values = column.values.slice(
-        first_offset,
-        offsets[-1].as_py() - first_offset,
-    )
-    coerced = _coerce_blob_value(values, target_value_field)
-    if coerced is values:
-        return column
-    physical_values, _ = _physical_array_and_type(coerced)
-    if first_offset:
-        offsets = pc.subtract(offsets, pa.scalar(first_offset, offsets.type))
-    if pa.types.is_large_list(column.type):
-        return pa.LargeListArray.from_arrays(offsets, physical_values, mask=mask)
-    return pa.ListArray.from_arrays(offsets, physical_values, mask=mask)
-
-
-def _coerce_value_to_blob(values: pa.Array, target_field: pa.Field) -> pa.Array:
-    if pa.types.is_null(values.type):
-        data = pa.nulls(len(values), type=pa.large_binary())
-    elif pa.types.is_large_binary(values.type):
-        data = values
-    else:
-        data = values.cast(pa.large_binary())
-    length = len(values)
-    storage_type = target_field.type
-    if isinstance(storage_type, pa.ExtensionType):
-        storage_type = storage_type.storage_type
-    storage_fields = list(storage_type)
-    children = []
-    for storage_field in storage_fields:
-        if storage_field.name == "data":
-            children.append(data)
-        else:
-            children.append(pa.nulls(length, type=storage_field.type))
-    storage = pa.StructArray.from_arrays(
-        children,
-        fields=storage_fields,
-        mask=values.is_null() if values.null_count else None,
-    )
-    if isinstance(target_field.type, pa.ExtensionType):
-        return pa.ExtensionArray.from_storage(target_field.type, storage)
-    return storage
-
-
-def _physical_array_and_type(array: pa.Array) -> tuple[pa.Array, pa.DataType]:
-    if isinstance(array.type, pa.ExtensionType):
-        return array.storage, array.type.storage_type
-    return array, array.type
-
-
-def _can_coerce_to_blob(data_type: pa.DataType) -> bool:
-    return _is_binary_like(data_type) or pa.types.is_null(data_type)
-
-
-def _is_binary_like(data_type: pa.DataType) -> bool:
-    return (
-        pa.types.is_binary(data_type)
-        or pa.types.is_large_binary(data_type)
-        or pa.types.is_binary_view(data_type)
-    )
-
-
 def _field_extension_name(field: pa.Field) -> Optional[str]:
     extension_name = getattr(field.type, "extension_name", None)
     if extension_name is not None:
@@ -630,73 +547,65 @@ def _align_field_types(
         target_field = next((f for f in target_fields if f.name == field.name), None)
         if target_field is None:
             raise ValueError(f"Field '{field.name}' not found in target schema")
-        new_fields.append(_align_field(field, target_field))
+        # Preserve arrow.json input until it reaches Lance. LanceDB exposes stored
+        # JSON columns as lance.json (JSONB-backed LargeBinary), but casting the
+        # input to that storage type here merely relabels the raw JSON bytes as
+        # JSONB. Lance must see arrow.json so it can perform the JSONB encoding.
+        if (
+            _field_extension_name(field) == "arrow.json"
+            and _field_extension_name(target_field) == "lance.json"
+        ):
+            new_fields.append(field)
+            continue
+        if pa.types.is_struct(target_field.type):
+            if pa.types.is_struct(field.type):
+                new_type = pa.struct(
+                    _align_field_types(
+                        field.type.fields,
+                        target_field.type.fields,
+                    )
+                )
+            else:
+                new_type = target_field.type
+        elif pa.types.is_list(target_field.type):
+            if _is_list_like(field.type):
+                new_type = pa.list_(
+                    _align_field_types(
+                        [field.type.value_field],
+                        [target_field.type.value_field],
+                    )[0]
+                )
+            else:
+                new_type = target_field.type
+        elif pa.types.is_large_list(target_field.type):
+            if _is_list_like(field.type):
+                new_type = pa.large_list(
+                    _align_field_types(
+                        [field.type.value_field],
+                        [target_field.type.value_field],
+                    )[0]
+                )
+            else:
+                new_type = target_field.type
+        elif pa.types.is_fixed_size_list(target_field.type):
+            if _is_list_like(field.type):
+                new_type = pa.list_(
+                    _align_field_types(
+                        [field.type.value_field],
+                        [target_field.type.value_field],
+                    )[0],
+                    target_field.type.list_size,
+                )
+            else:
+                new_type = target_field.type
+        else:
+            new_type = target_field.type
+        new_fields.append(
+            pa.field(field.name, new_type, field.nullable, target_field.metadata)
+        )
     return new_fields
 
 
-def _align_list_value_field(
-    value_field: pa.Field, target_value_field: pa.Field
-) -> pa.Field:
-    # A list has exactly one child, so the inferred child name ("item") aligns
-    # positionally and adopts the table's child name; pa.Table.cast renames it.
-    return _align_field(value_field, target_value_field).with_name(
-        target_value_field.name
-    )
-
-
-def _align_field(field: pa.Field, target_field: pa.Field) -> pa.Field:
-    # Preserve arrow.json input until it reaches Lance. LanceDB exposes stored
-    # JSON columns as lance.json (JSONB-backed LargeBinary), but casting the
-    # input to that storage type here merely relabels the raw JSON bytes as
-    # JSONB. Lance must see arrow.json so it can perform the JSONB encoding.
-    if (
-        _field_extension_name(field) == "arrow.json"
-        and _field_extension_name(target_field) == "lance.json"
-    ):
-        return field
-    if pa.types.is_struct(target_field.type):
-        if pa.types.is_struct(field.type):
-            new_type = pa.struct(
-                _align_field_types(
-                    field.type.fields,
-                    target_field.type.fields,
-                )
-            )
-        else:
-            new_type = target_field.type
-    elif pa.types.is_list(target_field.type):
-        if _is_list_like(field.type):
-            new_type = pa.list_(
-                _align_list_value_field(
-                    field.type.value_field, target_field.type.value_field
-                )
-            )
-        else:
-            new_type = target_field.type
-    elif pa.types.is_large_list(target_field.type):
-        if _is_list_like(field.type):
-            new_type = pa.large_list(
-                _align_list_value_field(
-                    field.type.value_field, target_field.type.value_field
-                )
-            )
-        else:
-            new_type = target_field.type
-    elif pa.types.is_fixed_size_list(target_field.type):
-        if _is_list_like(field.type):
-            new_type = pa.list_(
-                _align_list_value_field(
-                    field.type.value_field, target_field.type.value_field
-                ),
-                target_field.type.list_size,
-            )
-        else:
-            new_type = target_field.type
-    else:
-        new_type = target_field.type
-    return pa.field(field.name, new_type, field.nullable, target_field.metadata)
-
-
 def _infer_subschema(
     schema: List[pa.Field],
     reference_fields: List[pa.Field],
@@ -763,7 +672,7 @@ def sanitize_create_table(
         schema = data.schema
     else:
         if schema is not None:
-            data = pa.Table.from_batches([], schema=schema)
+            data = pa.Table.from_pylist([], schema)
     if schema is None:
         if data is None:
             raise ValueError("Either data or schema must be provided")
@@ -1918,7 +1827,7 @@ class Table(ABC):
     @abstractmethod
     def update(
         self,
-        where: Optional[Union[str, Expr]] = None,
+        where: Optional[str] = None,
         values: Optional[dict] = None,
         *,
         values_sql: Optional[Dict[str, str]] = None,
@@ -1933,11 +1842,9 @@ class Table(ABC):
 
         Parameters
         ----------
-        where: str or [Expr][lancedb.expr.Expr], optional
-            The filter condition. Can be a SQL string or a type-safe
-            [Expr][lancedb.expr.Expr] built with [col][lancedb.expr.col] and
-            [lit][lancedb.expr.lit]. The filter must not be empty, or it will
-            error.
+        where: str, optional
+            The SQL where clause to use when updating rows. For example, 'x = 2'
+            or 'x IN (1, 2, 3)'. The filter must not be empty, or it will error.
         values: dict, optional
             The values to update. The keys are the column names and the values
             are the values to set.
@@ -1955,7 +1862,6 @@ class Table(ABC):
         Examples
         --------
         >>> import lancedb
-        >>> from lancedb.expr import col
         >>> import pandas as pd
         >>> data = pd.DataFrame({"x": [1, 2, 3], "vector": [[1.0, 2], [3, 4], [5, 6]]})
         >>> db = lancedb.connect("./.lancedb")
@@ -1965,7 +1871,7 @@ class Table(ABC):
         0  1  [1.0, 2.0]
         1  2  [3.0, 4.0]
         2  3  [5.0, 6.0]
-        >>> table.update(where=col("x") == 2, values={"vector": [10.0, 10]})
+        >>> table.update(where="x = 2", values={"vector": [10.0, 10]})
         UpdateResult(rows_updated=1, version=2)
         >>> table.to_pandas()
            x        vector
@@ -2049,6 +1955,7 @@ class Table(ABC):
         cleanup_older_than: Optional[timedelta] = None,
         delete_unverified: bool = False,
         retrain: bool = False,
+        compaction_options: Optional[CompactionOptions] = None,
     ):
         """
         Optimize the on-disk data and indices for better performance.
@@ -2081,6 +1988,11 @@ class Table(ABC):
 
         retrain: bool, default False
             This parameter is no longer used and is deprecated.
+        compaction_options: CompactionOptions, optional
+            Options that control file compaction. For large rows, derive a lower
+            ``target_rows_per_fragment`` from the expected row size to bound later
+            passes. This does not cap the first pass over an existing oversized
+            fragment; see [CompactionOptions][lancedb.table.CompactionOptions].
 
         Notes
         -----
@@ -2872,7 +2784,7 @@ class LanceTable(Table):
             arrow_tbl = self.to_arrow()
             if blob_mode == "descriptions":
                 arrow_tbl = strip_auto_row_ids(
-                    arrow_tbl, row_addressable_blob_v2_paths(self.schema)
+                    arrow_tbl, blob_v2_column_paths(self.schema)
                 )
             return arrow_tbl.to_pandas(**kwargs)
 
@@ -4018,7 +3930,7 @@ class LanceTable(Table):
 
     def update(
         self,
-        where: Optional[Union[str, Expr]] = None,
+        where: Optional[str] = None,
         values: Optional[dict] = None,
         *,
         values_sql: Optional[Dict[str, str]] = None,
@@ -4029,11 +3941,9 @@ class LanceTable(Table):
 
         Parameters
         ----------
-        where: str or [Expr][lancedb.expr.Expr], optional
-            The filter condition. Can be a SQL string or a type-safe
-            [Expr][lancedb.expr.Expr] built with [col][lancedb.expr.col] and
-            [lit][lancedb.expr.lit]. The filter must not be empty, or it will
-            error.
+        where: str, optional
+            The SQL where clause to use when updating rows. For example, 'x = 2'
+            or 'x IN (1, 2, 3)'. The filter must not be empty, or it will error.
         values: dict, optional
             The values to update. The keys are the column names and the values
             are the values to set.
@@ -4051,7 +3961,6 @@ class LanceTable(Table):
         Examples
         --------
         >>> import lancedb
-        >>> from lancedb.expr import col
         >>> import pandas as pd
         >>> data = pd.DataFrame({"x": [1, 2, 3], "vector": [[1.0, 2], [3, 4], [5, 6]]})
         >>> db = lancedb.connect("./.lancedb")
@@ -4061,7 +3970,7 @@ class LanceTable(Table):
         0  1  [1.0, 2.0]
         1  2  [3.0, 4.0]
         2  3  [5.0, 6.0]
-        >>> table.update(where=col("x") == 2, values={"vector": [10.0, 10]})
+        >>> table.update(where="x = 2", values={"vector": [10.0, 10]})
         UpdateResult(rows_updated=1, version=2)
         >>> table.to_pandas()
            x        vector
@@ -4199,6 +4108,7 @@ class LanceTable(Table):
         cleanup_older_than: Optional[timedelta] = None,
         delete_unverified: bool = False,
         retrain: bool = False,
+        compaction_options: Optional[CompactionOptions] = None,
     ):
         """
         Optimize the on-disk data and indices for better performance.
@@ -4231,6 +4141,11 @@ class LanceTable(Table):
 
         retrain: bool, default False
             This parameter is no longer used and is deprecated.
+        compaction_options: CompactionOptions, optional
+            Options that control file compaction. For large rows, derive a lower
+            ``target_rows_per_fragment`` from the expected row size to bound later
+            passes. This does not cap the first pass over an existing oversized
+            fragment; see [CompactionOptions][lancedb.table.CompactionOptions].
 
         Notes
         -----
@@ -4245,6 +4160,7 @@ class LanceTable(Table):
                 cleanup_older_than=cleanup_older_than,
                 delete_unverified=delete_unverified,
                 retrain=retrain,
+                compaction_options=compaction_options,
             )
         )
 
@@ -5276,9 +5192,7 @@ class AsyncTable:
         if blob_mode == "descriptions" or not schema_has_blob_field(schema):
             arrow_tbl = await self.to_arrow()
             if blob_mode == "descriptions":
-                arrow_tbl = strip_auto_row_ids(
-                    arrow_tbl, row_addressable_blob_v2_paths(schema)
-                )
+                arrow_tbl = strip_auto_row_ids(arrow_tbl, blob_v2_column_paths(schema))
             return arrow_tbl.to_pandas(**kwargs)
 
         if blob_mode == "lazy" and get_uri_scheme(await self.uri()) == "memory":
@@ -6177,7 +6091,7 @@ class AsyncTable:
         self,
         updates: Optional[Dict[str, Any]] = None,
         *,
-        where: Optional[Union[str, Expr]] = None,
+        where: Optional[str] = None,
         updates_sql: Optional[Dict[str, str]] = None,
     ) -> UpdateResult:
         """
@@ -6192,11 +6106,9 @@ class AsyncTable:
             The updates to apply.  The keys should be the name of the column to
             update.  The values should be the new values to assign.  This is
             required unless updates_sql is supplied.
-        where: str or [Expr][lancedb.expr.Expr], optional
-            The filter condition. Can be a SQL string or a type-safe
-            [Expr][lancedb.expr.Expr] built with [col][lancedb.expr.col] and
-            [lit][lancedb.expr.lit]. Only rows that satisfy this filter will
-            be updated.
+        where: str, optional
+            An SQL filter that controls which rows are updated. For example, 'x = 2'
+            or 'x IN (1, 2, 3)'.  Only rows that satisfy this filter will be udpated.
         updates_sql: dict, optional
             The updates to apply, expressed as SQL expression strings.  The keys should
             be column names. The values should be SQL expressions.  These can be SQL
@@ -6214,14 +6126,13 @@ class AsyncTable:
         --------
         >>> import asyncio
         >>> import lancedb
-        >>> from lancedb.expr import col
         >>> import pandas as pd
         >>> async def demo_update():
         ...     data = pd.DataFrame({"x": [1, 2], "vector": [[1, 2], [3, 4]]})
         ...     db = await lancedb.connect_async("./.lancedb")
         ...     table = await db.create_table("my_table", data)
         ...     # x is [1, 2], vector is [[1, 2], [3, 4]]
-        ...     await table.update({"vector": [10, 10]}, where=col("x") == 2)
+        ...     await table.update({"vector": [10, 10]}, where="x = 2")
         ...     # x is [1, 2], vector is [[1, 2], [10, 10]]
         ...     await table.update(updates_sql={"x": "x + 1"})
         ...     # x is [2, 3], vector is [[1, 2], [10, 10]]
@@ -6235,8 +6146,7 @@ class AsyncTable:
         if updates is not None:
             updates_sql = {k: value_to_sql(v) for k, v in updates.items()}
 
-        predicate = where.to_sql() if isinstance(where, Expr) else where
-        return await self._inner.update(updates_sql, predicate)
+        return await self._inner.update(updates_sql, where)
 
     async def add_columns(
         self,
@@ -6643,6 +6553,7 @@ class AsyncTable:
         cleanup_older_than: Optional[timedelta] = None,
         delete_unverified: bool = False,
         retrain=False,
+        compaction_options: Optional[CompactionOptions] = None,
     ) -> OptimizeStats:
         """
         Optimize the on-disk data and indices for better performance.
@@ -6675,6 +6586,11 @@ class AsyncTable:
 
         retrain: bool, default False
             This parameter is no longer used and is deprecated.
+        compaction_options: CompactionOptions, optional
+            Options that control file compaction. For large rows, derive a lower
+            ``target_rows_per_fragment`` from the expected row size to bound later
+            passes. This does not cap the first pass over an existing oversized
+            fragment; see [CompactionOptions][lancedb.table.CompactionOptions].
 
         Notes
         -----
@@ -6701,6 +6617,7 @@ class AsyncTable:
         return await self._inner.optimize(
             cleanup_since_ms=cleanup_since_ms,
             delete_unverified=delete_unverified,
+            compaction_options=compaction_options,
         )
 
     async def list_indices(self) -> Iterable[IndexConfig]:

python/python/tests/test_blob.py

@@ -2,41 +2,17 @@
 # SPDX-FileCopyrightText: Copyright The LanceDB Authors
 
 import io
-import subprocess
-import sys
-import textwrap
 
-import lance
 import pyarrow as pa
 import pyarrow.compute as pc
 import pytest
-from lance.blob import BlobType as LanceBlobType
 
 import lancedb
-from lancedb._blob import (
-    blob_v2_projection_sources,
-    read_row_ids_from_hits,
-    stash_auto_row_ids,
-)
-from lancedb.expr import col
+from lancedb._blob import read_row_ids_from_hits, stash_auto_row_ids
 from lancedb.index import FTS
 from lancedb.schema import blob_column_paths, blob_v2_column_paths
 
 
-_HIDE_LANCE_BLOB = """\
-import importlib.abc
-import sys
-
-class _MissingLanceBlob(importlib.abc.MetaPathFinder):
-    def find_spec(self, fullname, path, target=None):
-        if fullname == "lance.blob" or fullname.startswith("lance.blob."):
-            raise ModuleNotFoundError(fullname, name="lance.blob")
-
-sys.modules.pop("lance.blob", None)
-sys.meta_path.insert(0, _MissingLanceBlob())
-"""
-
-
 def _blob_table(name, rows):
     db = lancedb.connect("memory:///")
     schema = pa.schema([pa.field("id", pa.int64()), lancedb.blob("image")])
@@ -70,181 +46,6 @@ def test_blob_factory_declares_v2_field():
     field = lancedb.blob("image")
     assert isinstance(field.type, pa.ExtensionType)
     assert field.type.extension_name == "lance.blob.v2"
-    assert lancedb.BlobType is LanceBlobType
-    assert type(field.type) is LanceBlobType
-
-
-def test_blob_type_works_without_pylance():
-    script = _HIDE_LANCE_BLOB + textwrap.dedent(
-        """\
-        import lancedb
-        import pyarrow as pa
-
-        field = lancedb.blob("image")
-        if not isinstance(field.type, pa.ExtensionType):
-            raise SystemExit("expected an extension type")
-        if field.type.extension_name != "lance.blob.v2":
-            raise SystemExit(field.type.extension_name)
-        if lancedb.BlobType is not type(field.type):
-            raise SystemExit("BlobType is not the field type class")
-        if lancedb.BlobType.__module__ != "lancedb.schema":
-            raise SystemExit(lancedb.BlobType.__module__)
-
-        db = lancedb.connect("memory:///")
-        table = db.create_table(
-            "images",
-            schema=pa.schema([pa.field("id", pa.int64()), field]),
-        )
-        table.add([{"id": 1, "image": b"hello"}])
-        result = (
-            table.merge_insert("id")
-            .when_matched_update_all()
-            .when_not_matched_insert_all()
-            .execute([{"id": 1, "image": b"updated"}, {"id": 2, "image": b"inserted"}])
-        )
-        if result.num_updated_rows != 1 or result.num_inserted_rows != 1:
-            raise SystemExit(
-                f"merge_insert rows updated={result.num_updated_rows} "
-                f"inserted={result.num_inserted_rows}"
-            )
-        """
-    )
-    result = subprocess.run(
-        [sys.executable, "-c", script],
-        capture_output=True,
-        text=True,
-        check=False,
-    )
-    assert result.returncode == 0, result.stderr
-
-
-def test_blob_resolves_pylance_type_without_eager_import():
-    script = textwrap.dedent(
-        """\
-        import sys
-        import lancedb
-
-        if "lance.blob" in sys.modules:
-            raise SystemExit("import lancedb imported lance.blob")
-        field = lancedb.blob("image")
-        from lance.blob import BlobType
-
-        if type(field.type) is not BlobType:
-            raise SystemExit(f"{type(field.type)} is not {BlobType}")
-        import lance
-
-        image = lance.blob_array([b"x"])
-        if type(image.type) is not BlobType:
-            raise SystemExit("blob_array used a different class")
-        if type(image.type) is not type(field.type):
-            raise SystemExit("field and array classes differ")
-        """
-    )
-    result = subprocess.run(
-        [sys.executable, "-c", script],
-        capture_output=True,
-        text=True,
-        check=False,
-    )
-    assert result.returncode == 0, result.stderr
-
-
-def test_blob_fallback_fails_if_name_already_registered():
-    script = _HIDE_LANCE_BLOB + textwrap.dedent(
-        """\
-        import pyarrow as pa
-
-        class OtherBlobType(pa.ExtensionType):
-            def __init__(self):
-                super().__init__(
-                    pa.struct([pa.field("data", pa.large_binary())]),
-                    "lance.blob.v2",
-                )
-
-            def __arrow_ext_serialize__(self):
-                return b""
-
-            @classmethod
-            def __arrow_ext_deserialize__(cls, storage_type, serialized):
-                return cls()
-
-        pa.register_extension_type(OtherBlobType())
-        import lancedb
-
-        try:
-            lancedb.blob("image")
-        except ValueError as err:
-            if "already registered" not in str(err):
-                raise SystemExit(err)
-        else:
-            raise SystemExit("expected ValueError")
-        """
-    )
-    result = subprocess.run(
-        [sys.executable, "-c", script],
-        capture_output=True,
-        text=True,
-        check=False,
-    )
-    assert result.returncode == 0, result.stderr
-
-
-def test_blob_type_rejects_competing_registration_with_pylance():
-    script = textwrap.dedent(
-        """\
-        import pyarrow as pa
-        import pyarrow.ipc
-
-        class OtherBlobType(pa.ExtensionType):
-            def __init__(self):
-                super().__init__(
-                    pa.struct(
-                        [
-                            pa.field("data", pa.large_binary()),
-                            pa.field("uri", pa.utf8()),
-                            pa.field("position", pa.uint64()),
-                            pa.field("size", pa.uint64()),
-                        ]
-                    ),
-                    "lance.blob.v2",
-                )
-
-            def __arrow_ext_serialize__(self):
-                return b""
-
-            @classmethod
-            def __arrow_ext_deserialize__(cls, storage_type, serialized):
-                return cls()
-
-        pa.register_extension_type(OtherBlobType())
-
-        from lance.blob import BlobType
-
-        if BlobType is OtherBlobType:
-            raise SystemExit("pylance BlobType was replaced")
-        schema = pa.schema([pa.field("value", BlobType())])
-        restored = pa.ipc.read_schema(schema.serialize())
-        if type(restored.field("value").type) is not OtherBlobType:
-            raise SystemExit(type(restored.field("value").type))
-
-        import lancedb
-
-        try:
-            lancedb.blob("image")
-        except ValueError as err:
-            if "__main__.OtherBlobType" not in str(err):
-                raise SystemExit(err)
-        else:
-            raise SystemExit("expected ValueError")
-        """
-    )
-    result = subprocess.run(
-        [sys.executable, "-c", script],
-        capture_output=True,
-        text=True,
-        check=False,
-    )
-    assert result.returncode == 0, result.stderr
 
 
 def test_blob_v2_column_paths_include_list_children():
@@ -269,14 +70,6 @@ def test_blob_v2_column_paths_include_list_children():
     ]
 
 
-def test_blob_v2_projection_sources_use_typed_column_name():
-    schema = pa.schema([lancedb.blob("blob")])
-
-    assert blob_v2_projection_sources(schema, {"blob_alias": col("blob")}) == {
-        "blob_alias": "blob"
-    }
-
-
 def _legacy_v1_table(name):
     db = lancedb.connect("memory:///")
     schema = pa.schema(
@@ -373,20 +166,6 @@ async def test_async_table_to_pandas_descriptions_mode_omits_row_id():
     assert set(descriptor.keys()) == {"kind", "position", "size", "blob_id", "blob_uri"}
 
 
-@pytest.mark.asyncio
-async def test_async_typed_blob_projection_preserves_source_column():
-    db = await lancedb.connect_async("memory:///typed_blob_projection")
-    schema = pa.schema([pa.field("id", pa.int64()), lancedb.blob("blob")])
-    table = await db.create_table("typed_blob_projection", schema=schema)
-    await table.add([{"id": 1, "blob": b"alpha"}])
-
-    hits = await table.query().select({"blob_alias": col("blob")}).to_arrow()
-
-    assert "_lance_row_id" in hits.schema.field("blob_alias").type.names
-    blobs = await table.fetch_blobs("blob", hits)
-    assert blobs.to_pylist() == [b"alpha"]
-
-
 def test_fetch_blobs_round_trip():
     table = _blob_table(
         "round_trip",
@@ -397,292 +176,6 @@ def test_fetch_blobs_round_trip():
     assert [blobs[0].as_py(), blobs[1].as_py()] == [b"alpha", b"beta"]
 
 
-def test_merge_insert_writes_python_bytes():
-    table = _blob_table("merge_bytes", [{"id": 1, "image": b"before"}])
-    result = (
-        table.merge_insert("id")
-        .when_matched_update_all()
-        .when_not_matched_insert_all()
-        .execute([{"id": 1, "image": b"updated"}, {"id": 2, "image": b"inserted"}])
-    )
-    assert result.num_updated_rows == 1
-    assert result.num_inserted_rows == 1
-    by_id = _row_ids_by_id(table)
-    blobs = table.fetch_blobs("image", [by_id[1], by_id[2]])
-    assert blobs.to_pylist() == [b"updated", b"inserted"]
-
-
-def test_merge_insert_bytes_after_reopen_without_touching_blob_type(tmp_path):
-    db = lancedb.connect(tmp_path)
-    schema = pa.schema([pa.field("id", pa.int64()), lancedb.blob("image")])
-    table = db.create_table("images", schema=schema)
-    table.add([{"id": 1, "image": b"hello"}])
-
-    script = textwrap.dedent(
-        f"""\
-        import lancedb
-
-        db = lancedb.connect({str(tmp_path)!r})
-        table = db.open_table("images")
-        image_type = table.schema.field("image").type
-        if type(image_type).__name__ != "StructType":
-            raise SystemExit(f"expected StructType, got {{type(image_type)}}")
-        result = (
-            table.merge_insert("id")
-            .when_matched_update_all()
-            .when_not_matched_insert_all()
-            .execute(
-                [{{"id": 1, "image": b"updated"}}, {{"id": 2, "image": b"inserted"}}]
-            )
-        )
-        if result.num_updated_rows != 1 or result.num_inserted_rows != 1:
-            raise SystemExit(
-                f"rows updated={{result.num_updated_rows}} "
-                f"inserted={{result.num_inserted_rows}}"
-            )
-        hits = table.search().with_row_id(True).limit(10).to_arrow()
-        by_id = dict(zip(hits["id"].to_pylist(), hits["_rowid"].to_pylist()))
-        blobs = table.fetch_blobs("image", [by_id[1], by_id[2]])
-        if blobs.to_pylist() != [b"updated", b"inserted"]:
-            raise SystemExit(blobs.to_pylist())
-        """
-    )
-    result = subprocess.run(
-        [sys.executable, "-c", script],
-        capture_output=True,
-        text=True,
-        check=False,
-    )
-    assert result.returncode == 0, result.stderr
-
-
-def test_merge_insert_bytes_after_reopen_without_pylance(tmp_path):
-    db = lancedb.connect(tmp_path)
-    schema = pa.schema([pa.field("id", pa.int64()), lancedb.blob("image")])
-    table = db.create_table("images", schema=schema)
-    table.add([{"id": 1, "image": b"hello"}])
-
-    script = _HIDE_LANCE_BLOB + textwrap.dedent(
-        f"""\
-        import lancedb
-
-        db = lancedb.connect({str(tmp_path)!r})
-        table = db.open_table("images")
-        image_type = table.schema.field("image").type
-        if type(image_type).__name__ != "StructType":
-            raise SystemExit(f"expected StructType, got {{type(image_type)}}")
-        result = (
-            table.merge_insert("id")
-            .when_matched_update_all()
-            .when_not_matched_insert_all()
-            .execute(
-                [{{"id": 1, "image": b"updated"}}, {{"id": 2, "image": b"inserted"}}]
-            )
-        )
-        if result.num_updated_rows != 1 or result.num_inserted_rows != 1:
-            raise SystemExit(
-                f"rows updated={{result.num_updated_rows}} "
-                f"inserted={{result.num_inserted_rows}}"
-            )
-        hits = table.search().with_row_id(True).limit(10).to_arrow()
-        by_id = dict(zip(hits["id"].to_pylist(), hits["_rowid"].to_pylist()))
-        blobs = table.fetch_blobs("image", [by_id[1], by_id[2]])
-        if blobs.to_pylist() != [b"updated", b"inserted"]:
-            raise SystemExit(blobs.to_pylist())
-        """
-    )
-    result = subprocess.run(
-        [sys.executable, "-c", script],
-        capture_output=True,
-        text=True,
-        check=False,
-    )
-    assert result.returncode == 0, result.stderr
-
-
-def test_merge_insert_blob_array_into_reopened_unregistered_table(tmp_path):
-    db = lancedb.connect(tmp_path)
-    schema = pa.schema([pa.field("id", pa.int64()), lancedb.blob("image")])
-    table = db.create_table("images", schema=schema)
-    table.add([{"id": 1, "image": b"before"}])
-
-    script = textwrap.dedent(
-        f"""\
-        import pyarrow as pa
-        import lancedb
-
-        db = lancedb.connect({str(tmp_path)!r})
-        table = db.open_table("images")
-        image_type = table.schema.field("image").type
-        if type(image_type).__name__ != "StructType":
-            raise SystemExit(
-                f"expected StructType before lance import, got {{type(image_type)}}"
-            )
-
-        import lance
-
-        updates = pa.Table.from_arrays(
-            [
-                pa.array([1, 2], type=pa.int64()),
-                lance.blob_array([b"updated", b"inserted"]),
-            ],
-            names=["id", "image"],
-        )
-        result = (
-            table.merge_insert("id")
-            .when_matched_update_all()
-            .when_not_matched_insert_all()
-            .execute(updates)
-        )
-        if result.num_updated_rows != 1 or result.num_inserted_rows != 1:
-            raise SystemExit(
-                f"rows updated={{result.num_updated_rows}} "
-                f"inserted={{result.num_inserted_rows}}"
-            )
-        hits = table.search().with_row_id(True).limit(10).to_arrow()
-        by_id = dict(zip(hits["id"].to_pylist(), hits["_rowid"].to_pylist()))
-        blobs = table.fetch_blobs("image", [by_id[1], by_id[2]])
-        if blobs.to_pylist() != [b"updated", b"inserted"]:
-            raise SystemExit(blobs.to_pylist())
-        """
-    )
-    result = subprocess.run(
-        [sys.executable, "-c", script],
-        capture_output=True,
-        text=True,
-        check=False,
-    )
-    assert result.returncode == 0, result.stderr
-
-
-def test_add_all_null_blob_column():
-    db = lancedb.connect("memory:///")
-    schema = pa.schema([pa.field("id", pa.int64()), lancedb.blob("image")])
-    table = db.create_table("all_null", schema=schema)
-    table.add([{"id": 1, "image": None}, {"id": 2, "image": None}])
-    by_id = _row_ids_by_id(table)
-    blobs = table.fetch_blobs("image", [by_id[1], by_id[2]])
-    assert blobs.to_pylist() == [None, None]
-
-
-def test_create_table_nested_blob_schema_without_rows():
-    db = lancedb.connect("memory:///")
-    schema = pa.schema(
-        [
-            pa.field("id", pa.int64()),
-            pa.field("info", pa.struct([lancedb.blob("blob")])),
-            pa.field("images", pa.list_(lancedb.blob("image"))),
-        ]
-    )
-    table = db.create_table("nested_empty", schema=schema)
-    assert table.count_rows() == 0
-
-
-def test_merge_insert_nested_blob_dicts():
-    db = lancedb.connect("memory:///")
-    info = pa.StructArray.from_arrays(
-        [
-            pa.array(["first"], type=pa.string()),
-            _blob_array("blob", [b"before"]),
-        ],
-        names=["name", "blob"],
-    )
-    data = pa.Table.from_arrays(
-        [pa.array([1], type=pa.int64()), info],
-        names=["id", "info"],
-    )
-    table = db.create_table("nested_merge", data=data)
-    result = (
-        table.merge_insert("id")
-        .when_matched_update_all()
-        .execute([{"id": 1, "info": {"name": "first", "blob": b"after"}}])
-    )
-    assert result.num_updated_rows == 1
-    by_id = _row_ids_by_id(table)
-    blobs = table.fetch_blobs("info.blob", [by_id[1]])
-    assert blobs.to_pylist() == [b"after"]
-
-
-def _list_blob_table(name):
-    db = lancedb.connect("memory:///")
-    blob_field = lancedb.blob("image")
-    images = pa.ListArray.from_arrays(
-        pa.array([0, 1], type=pa.int32()), _blob_array("image", [b"before"])
-    )
-    data = pa.Table.from_arrays(
-        [pa.array([1], type=pa.int64()), images],
-        schema=pa.schema(
-            [pa.field("id", pa.int64()), pa.field("images", pa.list_(blob_field))]
-        ),
-    )
-    return db.create_table(name, data=data)
-
-
-def test_merge_insert_list_blob_dicts():
-    table = _list_blob_table("list_merge")
-    result = (
-        table.merge_insert("id")
-        .when_matched_update_all()
-        .when_not_matched_insert_all()
-        .execute([{"id": 1, "images": [b"one", b"two"]}, {"id": 2, "images": None}])
-    )
-    assert result.num_updated_rows == 1
-    assert result.num_inserted_rows == 1
-    hits = table.search().limit(10).to_arrow()
-    sizes = {
-        row["id"]: None if row["images"] is None else [d["size"] for d in row["images"]]
-        for row in hits.to_pylist()
-    }
-    assert sizes == {1: [3, 3], 2: None}
-
-
-def test_list_blob_column_queries_as_raw_descriptors():
-    table = _list_blob_table("list_query")
-    hits = table.search().limit(10).to_arrow()
-    element = hits.schema.field("images").type.value_type
-    assert pa.types.is_struct(element)
-    assert "_lance_row_id" not in element.names
-    with pytest.raises(ValueError, match="expected struct before segment"):
-        table.fetch_blobs("images.image", [0])
-
-
-def test_row_addressable_paths_exclude_list_children():
-    from lancedb.schema import row_addressable_blob_v2_paths
-
-    schema = pa.schema(
-        [
-            pa.field("id", pa.int64()),
-            pa.field("info", pa.struct([lancedb.blob("blob")])),
-            pa.field("images", pa.list_(lancedb.blob("image"))),
-        ]
-    )
-    assert blob_v2_column_paths(schema) == ["info.blob", "images.image"]
-    assert row_addressable_blob_v2_paths(schema) == ["info.blob"]
-
-
-def test_merge_insert_writes_pylance_blob_array():
-    table = _blob_table("merge_pylance", [{"id": 1, "image": b"before"}])
-    image = lance.blob_array([b"updated", b"inserted"])
-    assert type(image.type) is LanceBlobType
-    assert type(image.type) is type(lancedb.BlobType())
-    updates = pa.Table.from_arrays(
-        [pa.array([1, 2], type=pa.int64()), image], names=["id", "image"]
-    )
-
-    result = (
-        table.merge_insert("id")
-        .when_matched_update_all()
-        .when_not_matched_insert_all()
-        .execute(updates)
-    )
-
-    assert result.num_updated_rows == 1
-    assert result.num_inserted_rows == 1
-    by_id = _row_ids_by_id(table)
-    blobs = table.fetch_blobs("image", [by_id[1], by_id[2]])
-    assert blobs.to_pylist() == [b"updated", b"inserted"]
-
-
 def test_fetch_blobs_accepts_query_result():
     table = _blob_table("from_result", [{"id": 1, "image": b"gamma"}])
     hits = table.search().limit(10).to_arrow()
@@ -910,50 +403,6 @@ async def test_blob_v2_hybrid_fetch_blobs_async():
     assert {blobs[i].as_py() for i in range(len(blobs))} == {b"alpha", b"beta"}
 
 
-@pytest.mark.asyncio
-async def test_async_hybrid_typed_blob_projection_preserves_source_column():
-    db = await lancedb.connect_async("memory:///hybrid_typed_blob")
-    schema = pa.schema(
-        [
-            pa.field("id", pa.int64()),
-            pa.field("text", pa.utf8()),
-            pa.field("vector", pa.list_(pa.float32(), list_size=2)),
-            lancedb.blob("blob"),
-        ]
-    )
-    table = await db.create_table("hybrid_typed_blob", schema=schema)
-    await table.add(
-        [
-            {
-                "id": 1,
-                "text": "hello alpha",
-                "vector": [1.0, 0.0],
-                "blob": b"alpha",
-            },
-            {
-                "id": 2,
-                "text": "hello beta",
-                "vector": [0.9, 0.1],
-                "blob": b"beta",
-            },
-        ]
-    )
-    await table.create_index("text", config=FTS(with_position=False))
-
-    hits = await (
-        table.query()
-        .nearest_to([1.0, 0.0])
-        .nearest_to_text("hello")
-        .select({"blob_alias": col("blob")})
-        .limit(2)
-        .to_arrow()
-    )
-
-    assert "_lance_row_id" in hits.schema.field("blob_alias").type.names
-    blobs = await table.fetch_blobs("blob", hits)
-    assert {blobs[i].as_py() for i in range(len(blobs))} == {b"alpha", b"beta"}
-
-
 def test_blob_file_seek_read_and_read_range():
     payload = _identifiable_payload(1024)
     table = _blob_table("seek_read", [{"id": 1, "image": payload}])

python/python/tests/test_expr.py

@@ -52,7 +52,7 @@ class TestExprConstruction:
     def test_func(self):
         e = func("lower", col("name"))
         assert isinstance(e, Expr)
-        assert e.to_sql() == "lower(`name`)"
+        assert e.to_sql() == "lower(name)"
 
     def test_func_unknown_raises(self):
         with pytest.raises(Exception):
@@ -115,7 +115,7 @@ class TestExprOperators:
     def test_and_operator(self):
         e = (col("age") > lit(18)) & (col("status") == lit("active"))
         assert isinstance(e, Expr)
-        assert e.to_sql() == "((age > 18) AND (`status` = 'active'))"
+        assert e.to_sql() == "((age > 18) AND (status = 'active'))"
 
     def test_or_operator(self):
         e = (col("a") == lit(1)) | (col("b") == lit(2))
@@ -166,7 +166,7 @@ class TestExprOperators:
     def test_coerce_plain_str(self):
         e = col("name") == "alice"
         assert isinstance(e, Expr)
-        assert e.to_sql() == "(`name` = 'alice')"
+        assert e.to_sql() == "(name = 'alice')"
 
     def test_reflexive_comparisons(self):
         # 10 < col("age") swaps to col("age") > 10
@@ -198,85 +198,85 @@ class TestExprBytesLiteral:
 
     def test_bytes_equality_expr_sql(self):
         e = col("data") == lit(b"\xca\xfe")
-        assert e.to_sql() == "(`data` = X'CAFE')"
+        assert e.to_sql() == "(data = X'CAFE')"
 
     def test_bytes_ne_expr_sql(self):
         e = col("data") != lit(b"\xff")
-        assert e.to_sql() == "(`data` <> X'FF')"
+        assert e.to_sql() == "(data <> X'FF')"
 
     def test_bytes_compound_expr_sql(self):
         e = (col("data") == lit(b"\x01")) & (col("id") > lit(5))
-        assert e.to_sql() == "((`data` = X'01') AND (id > 5))"
+        assert e.to_sql() == "((data = X'01') AND (id > 5))"
 
     def test_bytes_in_function_call(self):
         # Regression test: binary literals inside scalar function calls
         # used to fail because DataFusion's unparser does not support Binary
         # scalars.  Now handled via a placeholder-substitution rewrite.
         e = func("contains", col("data"), lit(b"\xff"))
-        assert e.to_sql() == "contains(`data`, X'FF')"
+        assert e.to_sql() == "contains(data, X'FF')"
 
     def test_bytes_in_not(self):
         e = ~(col("data") == lit(b"\xff"))
-        assert e.to_sql() == "NOT (`data` = X'FF')"
+        assert e.to_sql() == "NOT (data = X'FF')"
 
 
 class TestExprStringMethods:
     def test_lower(self):
         e = col("name").lower()
         assert isinstance(e, Expr)
-        assert e.to_sql() == "lower(`name`)"
+        assert e.to_sql() == "lower(name)"
 
     def test_upper(self):
         e = col("name").upper()
         assert isinstance(e, Expr)
-        assert e.to_sql() == "upper(`name`)"
+        assert e.to_sql() == "upper(name)"
 
     def test_contains(self):
         e = col("text").contains(lit("hello"))
         assert isinstance(e, Expr)
-        assert e.to_sql() == "contains(`text`, 'hello')"
+        assert e.to_sql() == "contains(text, 'hello')"
 
     def test_contains_with_str_coerce(self):
         e = col("text").contains("hello")
         assert isinstance(e, Expr)
-        assert e.to_sql() == "contains(`text`, 'hello')"
+        assert e.to_sql() == "contains(text, 'hello')"
 
     def test_chained_lower_eq(self):
         e = col("name").lower() == lit("alice")
         assert isinstance(e, Expr)
-        assert e.to_sql() == "(lower(`name`) = 'alice')"
+        assert e.to_sql() == "(lower(name) = 'alice')"
 
 
 class TestExprCast:
     def test_cast_string(self):
         e = col("id").cast("string")
         assert isinstance(e, Expr)
-        assert e.to_sql() == "arrow_cast(id, 'Utf8')"
+        assert e.to_sql() == "CAST(id AS VARCHAR)"
 
     def test_cast_int32(self):
         e = col("score").cast("int32")
         assert isinstance(e, Expr)
-        assert e.to_sql() == "arrow_cast(score, 'Int32')"
+        assert e.to_sql() == "CAST(score AS INTEGER)"
 
     def test_cast_float64(self):
         e = col("val").cast("float64")
         assert isinstance(e, Expr)
-        assert e.to_sql() == "arrow_cast(val, 'Float64')"
+        assert e.to_sql() == "CAST(val AS DOUBLE)"
 
     def test_cast_pyarrow_type(self):
         e = col("score").cast(pa.int32())
         assert isinstance(e, Expr)
-        assert e.to_sql() == "arrow_cast(score, 'Int32')"
+        assert e.to_sql() == "CAST(score AS INTEGER)"
 
     def test_cast_pyarrow_float64(self):
         e = col("val").cast(pa.float64())
         assert isinstance(e, Expr)
-        assert e.to_sql() == "arrow_cast(val, 'Float64')"
+        assert e.to_sql() == "CAST(val AS DOUBLE)"
 
     def test_cast_pyarrow_string(self):
         e = col("id").cast(pa.string())
         assert isinstance(e, Expr)
-        assert e.to_sql() == "arrow_cast(id, 'Utf8')"
+        assert e.to_sql() == "CAST(id AS VARCHAR)"
 
     def test_cast_pyarrow_and_string_equivalent(self):
         # pa.int32() and "int32" should produce equivalent SQL
@@ -597,14 +597,14 @@ class TestExprIsin:
     def test_isin_strs(self):
         assert (
             col("status").isin(["active", "pending"]).to_sql()
-            == "`status` IN ('active', 'pending')"
+            == "status IN ('active', 'pending')"
         )
 
     def test_isin_coerces_and_mixes(self):
         assert col("id").isin([lit(1), 2]).to_sql() == "id IN (1, 2)"
 
     def test_isin_empty(self):
-        assert col("id").isin([]).to_sql() == "false"
+        assert col("id").isin([]).to_sql() == "id IN ()"
 
     def test_isin_filter(self, simple_table):
         result = simple_table.search().where(col("id").isin([1, 3, 5])).to_arrow()

python/python/tests/test_first_class_function_slice1.py

@@ -37,21 +37,6 @@ def job_result(name: str) -> dict:
     return json.loads(fixture(name))["result"]
 
 
-def assert_no_secret_values(value):
-    if isinstance(value, dict):
-        for key, child in value.items():
-            assert key not in {
-                "secret_value",
-                "secret_values",
-                "resolved_secret",
-                "resolved_secrets",
-            }
-            assert_no_secret_values(child)
-    elif isinstance(value, list):
-        for child in value:
-            assert_no_secret_values(child)
-
-
 def test_public_function_values_are_in_api_reference():
     docs = Path(__file__).parents[3] / "docs" / "src" / "python" / "python.md"
     rendered = docs.read_text()
@@ -109,7 +94,6 @@ def test_function_version_identity_is_immutable_and_exact():
     version = FunctionVersion.from_json(json.dumps(value))
     assert version.name == "embed"
     assert version.version == "fv_01K3EXACT"
-    assert version.required_secrets == ("HF_TOKEN",)
 
     with pytest.raises((TypeError, ValueError)):
         version.version = "fv_changed"
@@ -292,15 +276,6 @@ def test_refresh_result_rejects_non_u64_values(field):
         RefreshColumnResult.from_json(json.dumps(value))
 
 
-def test_canonical_client_values_contain_secret_names_only():
-    version = FunctionVersion.from_json(
-        json.dumps(job_result("remote_function_job.json"))
-    )
-    canonical = json.loads(version.to_canonical_json())
-    assert canonical["required_secrets"] == ["HF_TOKEN"]
-    assert_no_secret_values(canonical)
-
-
 class _FunctionDeclarationInner:
     def __init__(self):
         self.calls = []

python/python/tests/test_first_class_function_slice2.py

@@ -19,13 +19,7 @@ import pyarrow as pa
 import pytest
 
 import lancedb
-from lancedb.functions import (
-    _MAX_FUNCTION_SECRET_VALUE_BYTES,
-    _MAX_FUNCTION_SECRET_VALUES_BYTES,
-    FunctionRegistrationRequest,
-    UdfDefinition,
-    udf,
-)
+from lancedb.functions import UdfDefinition, udf
 
 THRESHOLD = 20
 _CACHE = None
@@ -45,28 +39,12 @@ FIXTURES = (
 @udf(
     pip=["numpy>=2"],
     env={"MODE": "test"},
-    secrets=["API_TOKEN"],
     python_version="3.12",
 )
 def normalize_score(value: float) -> float:
     return value / 100.0
 
 
-def _assert_no_secret_values(value):
-    if isinstance(value, dict):
-        for key, child in value.items():
-            assert key not in {
-                "secret_value",
-                "secret_values",
-                "resolved_secret",
-                "resolved_secrets",
-            }
-            _assert_no_secret_values(child)
-    elif isinstance(value, list):
-        for child in value:
-            _assert_no_secret_values(child)
-
-
 def test_scalar_udf_matches_shared_registration_golden_and_remains_callable():
     assert isinstance(normalize_score, UdfDefinition)
     assert normalize_score(25.0) == 0.25
@@ -81,8 +59,6 @@ def test_scalar_udf_matches_shared_registration_golden_and_remains_callable():
         "kind": "scalar_to_arrow_batch",
         "version": 1,
     }
-    assert request["required_secrets"] == ["API_TOKEN"]
-    _assert_no_secret_values(request)
 
 
 def _run_packaged(definition, *args):
@@ -396,7 +372,6 @@ def test_udf_recursion_versus_a_rebound_module_name(tmp_path):
         output_schema=None,
         pip=(),
         env={},
-        secrets=(),
         python_version=None,
     )
     with pytest.raises(ValueError, match="binds that name to another value"):
@@ -551,54 +526,13 @@ def test_annotation_and_explicit_schema_validation_fail_closed():
             return value
 
 
-def test_secret_names_are_canonical_and_disjoint_from_environment():
-    @udf(secrets=["Z_TOKEN", "A_TOKEN", "Z_TOKEN"])
-    def canonical_secrets(value: int) -> int:
-        return value
-
-    assert canonical_secrets.registration_request.required_secrets == (
-        "A_TOKEN",
-        "Z_TOKEN",
-    )
-
-    with pytest.raises(ValueError, match="must be disjoint"):
-
-        @udf(env={"TOKEN": "plaintext"}, secrets=["TOKEN"])
-        def overlapping(value: int) -> int:
-            return value
-
-
-def test_declared_secret_api_still_requires_explicit_create_values():
-    @udf(secrets=["API_TOKEN"])
-    def declared_secret(value: int) -> int:
-        return value
-
-    with pytest.raises(ValueError, match="missing"):
-        declared_secret._submission_json(None)
-    submission = json.loads(
-        declared_secret._submission_json({"API_TOKEN": "explicit-secret"})
-    )
-    assert submission["required_secrets"] == ["API_TOKEN"]
-    assert submission["secret_values"] == {"API_TOKEN": "explicit-secret"}
-
-
-def test_no_secrets_preserve_canonical_registration_shape():
-    @udf
-    def no_secrets(value: int) -> int:
-        return value
-
-    canonical = json.loads(no_secrets.registration_request.to_canonical_json())
-    assert "required_secrets" not in canonical
-    assert json.loads(no_secrets._submission_json(None)) == canonical
-
-
 def test_local_function_catalog_operations_are_not_supported(tmp_path):
     db = lancedb.connect(tmp_path)
     message = "Function catalog operations are not supported by this database"
     with pytest.raises(NotImplementedError, match=message):
-        db.create_function(normalize_score, secrets={"API_TOKEN": "value"})
+        db.create_function(normalize_score)
     with pytest.raises(NotImplementedError, match=message):
-        db.create_function_async(normalize_score, secrets={"API_TOKEN": "value"})
+        db.create_function_async(normalize_score)
     with pytest.raises(NotImplementedError, match=message):
         db.get_function("normalize_score", version="fv_exact")
 
@@ -628,7 +562,6 @@ def _mock_remote_function_catalog():
                     "runtime": body["runtime"],
                     "runtime_digest": "sha256:runtime",
                     "environment_digest": "sha256:environment",
-                    "required_secrets": body.get("required_secrets", []),
                     "created_at": "2026-08-21T00:00:00Z",
                 }
                 response = {"job_id": "job-register"}
@@ -675,9 +608,7 @@ def test_remote_registration_job_and_exact_version_reopen_round_trip():
             host_override=host,
             client_config={"retry_config": {"retries": 0}},
         )
-        registration = db.create_function_async(
-            normalize_score, secrets={"API_TOKEN": "secret-value"}
-        )
+        registration = db.create_function_async(normalize_score)
         assert registration.id == "job-register"
         created = registration.wait()
         reopened = db.get_function("normalize_score", version=created.version)
@@ -686,18 +617,9 @@ def test_remote_registration_job_and_exact_version_reopen_round_trip():
     assert reopened.name == "normalize_score"
     assert reopened.version == "fv_exact"
     create_request = state["requests"][0][1]
-    expected = json.loads(normalize_score.registration_request.to_canonical_json())
-    expected["secret_values"] = {"API_TOKEN": "secret-value"}
-    assert create_request == expected
-    durable_request = FunctionRegistrationRequest.from_json(json.dumps(create_request))
-    assert not hasattr(durable_request, "secret_values")
-    assert "secret_values" not in json.loads(durable_request.to_canonical_json())
-    assert "secret_values" not in json.loads(
+    assert create_request == json.loads(
         normalize_score.registration_request.to_canonical_json()
     )
-    assert "secret-value" not in repr(normalize_score)
-    assert "secret-value" not in repr(normalize_score.registration_request)
-    assert not hasattr(created, "secret_values")
 
 
 def test_blocking_remote_registration_returns_function_version():
@@ -708,9 +630,7 @@ def test_blocking_remote_registration_returns_function_version():
             host_override=host,
             client_config={"retry_config": {"retries": 0}},
         )
-        created = db.create_function(
-            normalize_score, secrets={"API_TOKEN": "blocking-secret"}
-        )
+        created = db.create_function(normalize_score)
 
     assert created.name == "normalize_score"
     assert created.version == "fv_exact"
@@ -718,121 +638,3 @@ def test_blocking_remote_registration_returns_function_version():
         "/v1/functions/create",
         "/v1/jobs/describe",
     ]
-    assert state["requests"][0][1]["secret_values"] == {"API_TOKEN": "blocking-secret"}
-
-
-@pytest.mark.parametrize(
-    ("secret_values", "error_type", "message"),
-    [
-        (None, ValueError, "missing"),
-        ({}, ValueError, "missing"),
-        ({"OTHER": "value"}, ValueError, "missing.*unexpected"),
-        ({"API_TOKEN": ""}, ValueError, "non-empty"),
-        ({"API_TOKEN": "bad\0value"}, ValueError, "NUL"),
-        ({"API_TOKEN": 123}, TypeError, "must be a string"),
-        ([("API_TOKEN", "value")], TypeError, "must be a mapping"),
-    ],
-)
-def test_secret_values_are_validated_before_remote_request(
-    secret_values, error_type, message
-):
-    with _mock_remote_function_catalog() as (host, state):
-        db = lancedb.connect(
-            "db://dev",
-            api_key="fake",
-            host_override=host,
-            client_config={"retry_config": {"retries": 0}},
-        )
-        with pytest.raises(error_type, match=message):
-            db.create_function_async(normalize_score, secrets=secret_values)
-        assert state["requests"] == []
-
-
-@pytest.mark.parametrize(
-    "value",
-    [
-        "x" * _MAX_FUNCTION_SECRET_VALUE_BYTES,
-        "é" * (_MAX_FUNCTION_SECRET_VALUE_BYTES // len("é".encode("utf-8"))),
-    ],
-    ids=["ascii", "multibyte"],
-)
-def test_secret_value_accepts_exact_utf8_byte_limit(value):
-    submission = json.loads(normalize_score._submission_json({"API_TOKEN": value}))
-    assert submission["secret_values"]["API_TOKEN"] == value
-    assert len(value.encode("utf-8")) == _MAX_FUNCTION_SECRET_VALUE_BYTES
-
-
-@pytest.mark.parametrize(
-    "value",
-    [
-        "x" * (_MAX_FUNCTION_SECRET_VALUE_BYTES + 1),
-        "é" * (_MAX_FUNCTION_SECRET_VALUE_BYTES // len("é".encode("utf-8")) + 1),
-    ],
-    ids=["ascii", "multibyte"],
-)
-def test_secret_value_rejects_over_utf8_byte_limit_before_json_construction(
-    monkeypatch, value
-):
-    def fail_if_json_construction_starts(self):
-        pytest.fail("oversized secret reached JSON construction")
-
-    monkeypatch.setattr(
-        FunctionRegistrationRequest, "_known_dict", fail_if_json_construction_starts
-    )
-    with pytest.raises(ValueError, match=r"exceeds the 65536-byte limit"):
-        normalize_score._submission_json({"API_TOKEN": value})
-
-
-def test_secret_values_accept_exact_aggregate_utf8_byte_limit(monkeypatch):
-    names = tuple(f"SECRET_{index}" for index in range(8))
-    value = "é" * (_MAX_FUNCTION_SECRET_VALUE_BYTES // len("é".encode("utf-8")))
-    values = {name: value for name in names}
-    monkeypatch.setattr(
-        normalize_score,
-        "_request",
-        normalize_score._request._copy(update={"required_secrets": names}),
-    )
-
-    submission = json.loads(normalize_score._submission_json(values))
-
-    assert submission["secret_values"] == values
-    assert sum(len(item.encode("utf-8")) for item in values.values()) == (
-        _MAX_FUNCTION_SECRET_VALUES_BYTES
-    )
-
-
-def test_secret_values_reject_aggregate_over_limit_before_construction(monkeypatch):
-    names = tuple(f"SECRET_{index}" for index in range(9))
-    values = {name: "x" * _MAX_FUNCTION_SECRET_VALUE_BYTES for name in names}
-    monkeypatch.setattr(
-        normalize_score,
-        "_request",
-        normalize_score._request._copy(update={"required_secrets": names}),
-    )
-
-    def fail_if_json_construction_starts(self):
-        pytest.fail("oversized aggregate reached JSON construction")
-
-    monkeypatch.setattr(
-        FunctionRegistrationRequest, "_known_dict", fail_if_json_construction_starts
-    )
-    with pytest.raises(ValueError, match=r"exceed.*524288-byte request limit"):
-        normalize_score._submission_json(values)
-
-
-@pytest.mark.asyncio
-async def test_async_remote_registration_submits_secret_values_only_once():
-    with _mock_remote_function_catalog() as (host, state):
-        db = await lancedb.connect_async(
-            "db://dev",
-            api_key="fake",
-            host_override=host,
-            client_config={"retry_config": {"retries": 0}},
-        )
-        registration = await db.create_function_async(
-            normalize_score, secrets={"API_TOKEN": "async-secret"}
-        )
-        created = await registration.wait()
-
-    assert state["requests"][0][1]["secret_values"] == {"API_TOKEN": "async-secret"}
-    assert not hasattr(created, "secret_values")

python/python/tests/test_table.py

@@ -11,9 +11,8 @@ import warnings
 import weakref
 from concurrent.futures import ThreadPoolExecutor
 from datetime import date, datetime, timedelta
-from decimal import Decimal
 from time import sleep
-from typing import List
+from typing import Any, List
 from unittest.mock import patch
 
 import lancedb
@@ -337,21 +336,6 @@ async def test_update_async(mem_db_async: AsyncConnection):
     assert await table.count_rows("id == 10") == 1
 
 
-@pytest.mark.asyncio
-async def test_update_expr_filter_literals_async(mem_db_async: AsyncConnection):
-    values = ["5", "4.66e-84", "it's"]
-    table = await mem_db_async.create_table(
-        "update_expr_literals",
-        data=[{"field": value, "result": "original"} for value in values],
-    )
-
-    for value in values:
-        update_res = await table.update({"result": value}, where=col("field") == value)
-        assert update_res.rows_updated == 1
-
-    assert (await table.to_arrow())["result"].to_pylist() == values
-
-
 def test_create_table(mem_db: DBConnection):
     schema = pa.schema(
         {
@@ -2359,148 +2343,6 @@ def test_update(mem_db: DBConnection):
     assert np.allclose(v, np.array([[1.2, 1.9], [1.1, 1.1]]))
 
 
-def test_update_expr_filter_literals(mem_db: DBConnection):
-    values = ["5", "4.66e-84", "it's"]
-    table = mem_db.create_table(
-        "update_expr_literals",
-        data=[{"field": value, "result": "original"} for value in values],
-    )
-
-    for value in values:
-        update_res = table.update(where=col("field") == value, values={"result": value})
-        assert update_res.rows_updated == 1
-
-    assert table.to_arrow()["result"].to_pylist() == values
-
-
-def test_update_expr_filter_preserves_typed_semantics(mem_db: DBConnection):
-    low = Decimal("1.234567890123456789")
-    high = Decimal("1.234567890123456790")
-    decimal_schema = pa.schema(
-        [("val", pa.decimal128(19, 18)), ("result", pa.string())]
-    )
-    decimal_table = mem_db.create_table(
-        "update_expr_decimal",
-        pa.table(
-            {"val": [low, high], "result": ["old", "old"]},
-            schema=decimal_schema,
-        ),
-    )
-    predicate = col("val") < lit(high)
-    assert decimal_table.search().where(predicate).to_arrow().num_rows == 1
-    result = decimal_table.update(where=predicate, values={"result": "new"})
-    assert result.rows_updated == 1
-
-    keyword_table = mem_db.create_table(
-        "update_expr_keyword", [{"null": 1, "result": "old"}]
-    )
-    predicate = col("null") == 1
-    assert keyword_table.search().where(predicate).to_arrow().num_rows == 1
-    result = keyword_table.update(where=predicate, values={"result": "new"})
-    assert result.rows_updated == 1
-
-    empty_in_table = mem_db.create_table(
-        "update_expr_empty_in", [{"id": 1, "result": "old"}]
-    )
-    predicate = col("id").isin([])
-    assert empty_in_table.search().where(predicate).to_arrow().num_rows == 0
-    result = empty_in_table.update(where=predicate, values={"result": "new"})
-    assert result.rows_updated == 0
-
-    marker = "__lancedb_binary_placeholder_0__"
-    binary_schema = pa.schema(
-        [("payload", pa.binary()), ("text", pa.string()), ("result", pa.string())]
-    )
-    binary_table = mem_db.create_table(
-        "update_expr_binary",
-        pa.table(
-            {
-                "payload": [b"\x01", b"\x02"],
-                "text": ["other", marker],
-                "result": ["old", "old"],
-            },
-            schema=binary_schema,
-        ),
-    )
-    predicate = (col("payload") == lit(b"\x01")) | (col("text") == marker)
-    assert binary_table.search().where(predicate).to_arrow().num_rows == 2
-    result = binary_table.update(where=predicate, values={"result": "new"})
-    assert result.rows_updated == 2
-
-    nonfinite_table = mem_db.create_table(
-        "update_expr_nonfinite",
-        [{"x": 1.0, "result": "old"}, {"x": 2.0, "result": "old"}],
-    )
-    predicate = col("x") < float("inf")
-    assert nonfinite_table.search().where(predicate).to_arrow().num_rows == 2
-    result = nonfinite_table.update(where=predicate, values={"result": "new"})
-    assert result.rows_updated == 2
-
-    float16_table = mem_db.create_table(
-        "update_expr_float16",
-        [{"x": 1.0, "result": "old"}, {"x": 3.0, "result": "old"}],
-    )
-    predicate = col("x").cast(pa.float16()) < 2.0
-    assert float16_table.search().where(predicate).to_arrow().num_rows == 1
-    result = float16_table.update(where=predicate, values={"result": "new"})
-    assert result.rows_updated == 1
-
-    string_cast_table = mem_db.create_table(
-        "update_expr_string_cast",
-        [{"x": 1, "result": "old"}, {"x": 2, "result": "old"}],
-    )
-    predicate = col("x").cast("string") == "1"
-    assert string_cast_table.search().where(predicate).to_arrow().num_rows == 1
-    result = string_cast_table.update(where=predicate, values={"result": "new"})
-    assert result.rows_updated == 1
-
-    quoted_identifier_schema = pa.schema(
-        [("payload", pa.binary()), ("odd'name", pa.int64()), ("result", pa.string())]
-    )
-    quoted_identifier_table = mem_db.create_table(
-        "update_expr_quoted_identifier",
-        pa.table(
-            {"payload": [b"\x01"], "odd'name": [1], "result": ["old"]},
-            schema=quoted_identifier_schema,
-        ),
-    )
-    predicate = (col("payload") == lit(b"\x01")) & (col("odd'name") == 1)
-    assert quoted_identifier_table.search().where(predicate).to_arrow().num_rows == 1
-    result = quoted_identifier_table.update(where=predicate, values={"result": "new"})
-    assert result.rows_updated == 1
-
-    decimal256_schema = pa.schema(
-        [("val", pa.decimal256(40, 2)), ("result", pa.string())]
-    )
-    decimal256_table = mem_db.create_table(
-        "update_expr_decimal256",
-        pa.table(
-            {
-                "val": [Decimal("1.00"), Decimal("3.00")],
-                "result": ["old", "old"],
-            },
-            schema=decimal256_schema,
-        ),
-    )
-    predicate = col("val") < lit(Decimal("2.00")).cast(pa.decimal256(40, 2))
-    assert decimal256_table.search().where(predicate).to_arrow().num_rows == 1
-    result = decimal256_table.update(where=predicate, values={"result": "new"})
-    assert result.rows_updated == 1
-
-    binary_empty_table = mem_db.create_table(
-        "update_expr_binary_empty",
-        pa.table(
-            {"payload": [b"\x01", b"\x02"], "result": ["old", "old"]},
-            schema=pa.schema([("payload", pa.binary()), ("result", pa.string())]),
-        ),
-    )
-    predicate = (col("payload") == lit(b"\x01")).isin([])
-    assert binary_empty_table.search().where(predicate).to_arrow().num_rows == 0
-    assert predicate.to_sql() == "false"
-    result = binary_empty_table.update(where=predicate, values={"result": "new"})
-    assert result.rows_updated == 0
-
-
 def test_update_with_arrow_scalar(mem_db: DBConnection):
     schema = pa.schema({"id": pa.int64(), "vector": pa.list_(pa.float32(), 4)})
     table = mem_db.create_table("my_table", schema=schema)
@@ -3876,6 +3718,100 @@ async def test_optimize(mem_db_async: AsyncConnection):
     assert await table.query().to_arrow() == pa.table({"x": [[1], [2]]})
 
 
+@pytest.mark.asyncio
+async def test_optimize_compaction_options(mem_db_async: AsyncConnection):
+    table = await mem_db_async.create_table("test", data=[{"x": 1}])
+    await table.add([{"x": 2}])
+
+    stats = await table.optimize(
+        compaction_options={
+            "target_rows_per_fragment": 1,
+            "batch_size": 1,
+            "num_threads": 1,
+        }
+    )
+    assert stats.compaction.fragments_removed == 0
+    assert stats.compaction.fragments_added == 0
+
+    stats = await table.optimize(compaction_options={"target_rows_per_fragment": 3})
+    assert stats.compaction.fragments_removed == 2
+    assert stats.compaction.fragments_added == 1
+
+    with pytest.raises(ValueError, match="Invalid compaction option: unknown"):
+        await table.optimize(compaction_options={"unknown": 1})
+
+
+@pytest.mark.parametrize("option", ["max_source_rows", "max_source_bytes"])
+@pytest.mark.asyncio
+async def test_optimize_compaction_source_limits(
+    mem_db_async: AsyncConnection, option: str
+):
+    table = await mem_db_async.create_table("test", data=[{"x": 1}])
+    await table.add([{"x": 2}])
+
+    stats = await table.optimize(
+        compaction_options={
+            "target_rows_per_fragment": 3,
+            option: 1,
+        }
+    )
+    assert stats.compaction.fragments_removed == 0
+    assert stats.compaction.fragments_added == 0
+
+
+@pytest.mark.asyncio
+async def test_optimize_compaction_excluded_fragments(mem_db_async: AsyncConnection):
+    table = await mem_db_async.create_table("test", data=[{"x": 1}])
+    await table.add([{"x": 2}])
+
+    stats = await table.optimize(
+        compaction_options={
+            "target_rows_per_fragment": 3,
+            "excluded_fragment_ids": [0],
+        }
+    )
+    assert stats.compaction.fragments_removed == 0
+    assert stats.compaction.fragments_added == 0
+
+    stats = await table.optimize(compaction_options={"target_rows_per_fragment": 3})
+    assert stats.compaction.fragments_removed == 2
+    assert stats.compaction.fragments_added == 1
+
+
+@pytest.mark.parametrize(
+    ("option", "value", "message"),
+    [
+        ("target_rows_per_fragment", 0, "must be between 1 and 4294967295"),
+        ("max_rows_per_group", 0, "must be between 1 and 4294967295"),
+        ("batch_size", 0, "must be between 1 and 4294967295"),
+        ("num_threads", 0, "must be greater than 0"),
+        ("target_rows_per_fragment", 2**32, "must be between 1 and 4294967295"),
+        ("max_rows_per_group", 2**32, "must be between 1 and 4294967295"),
+        ("batch_size", 2**32, "must be between 1 and 4294967295"),
+        ("io_buffer_size", 2**63, "must be at most 9223372036854775807"),
+        ("max_source_rows", 0, "must be greater than 0"),
+        ("max_source_bytes", 0, "must be greater than 0"),
+        (
+            "excluded_fragment_ids",
+            [-1],
+            "must contain values between 0 and 4294967295",
+        ),
+        (
+            "excluded_fragment_ids",
+            [2**32],
+            "must contain values between 0 and 4294967295",
+        ),
+    ],
+)
+@pytest.mark.asyncio
+async def test_optimize_compaction_options_validation(
+    mem_db_async: AsyncConnection, option: str, value: Any, message: str
+):
+    table = await mem_db_async.create_table("test", data=[{"x": 1}])
+    with pytest.raises(ValueError, match=message):
+        await table.optimize(compaction_options={option: value})
+
+
 @pytest.mark.asyncio
 async def test_optimize_delete_unverified(tmp_db_async: AsyncConnection, tmp_path):
     table = await tmp_db_async.create_table(

python/python/tests/test_util.py

@@ -7,7 +7,6 @@ import pathlib
 from typing import Optional
 
 import lance
-from lance.blob import BlobType as LanceBlobType
 from lancedb.conftest import MockTextEmbeddingFunction
 from lancedb.embeddings.base import EmbeddingFunctionConfig
 from lancedb.embeddings.registry import EmbeddingFunctionRegistry
@@ -908,165 +907,6 @@ def test_cast_to_target_schema():
     assert output == expected
 
 
-def test_cast_to_target_schema_coerces_binary_to_blob_v2():
-    data = pa.table({"image": pa.array([b"hello", None], type=pa.binary())})
-    target = pa.schema([lancedb.blob("image")])
-
-    output = _cast_to_target_schema(data.to_reader(), target).read_all()
-
-    image = output["image"].chunk(0)
-    assert type(image.type) is lancedb.BlobType
-    assert image.storage.to_pylist() == [
-        {"data": b"hello", "uri": None, "position": None, "size": None},
-        None,
-    ]
-
-
-def test_cast_to_target_schema_coerces_binary_to_metadata_blob_struct():
-    storage = lancedb.blob("image").type.storage_type
-    target = pa.schema(
-        [
-            pa.field(
-                "image",
-                storage,
-                metadata={
-                    b"ARROW:extension:name": b"lance.blob.v2",
-                    b"ARROW:extension:metadata": b"",
-                },
-            )
-        ]
-    )
-    data = pa.table({"image": pa.array([b"hello", None], type=pa.binary())})
-
-    output = _cast_to_target_schema(data.to_reader(), target).read_all()
-
-    image = output["image"].chunk(0)
-    assert not isinstance(image.type, pa.ExtensionType)
-    assert image.to_pylist() == [
-        {"data": b"hello", "uri": None, "position": None, "size": None},
-        None,
-    ]
-
-
-def test_cast_to_target_schema_coerces_nested_binary_blob():
-    data = pa.table(
-        {
-            "info": pa.array(
-                [{"blob": b"hello"}, {"blob": None}],
-                type=pa.struct([pa.field("blob", pa.binary())]),
-            )
-        }
-    )
-    target = pa.schema([pa.field("info", pa.struct([lancedb.blob("blob")]))])
-
-    output = _cast_to_target_schema(data.to_reader(), target).read_all()
-
-    blob = output["info"].chunk(0).field("blob")
-    assert type(blob.type) is lancedb.BlobType
-    assert blob.storage.to_pylist() == [
-        {"data": b"hello", "uri": None, "position": None, "size": None},
-        None,
-    ]
-
-
-def test_cast_to_target_schema_coerces_list_binary_blob_with_inferred_child_name():
-    data = pa.table(
-        {"images": pa.array([[b"a", b"b"], None], type=pa.list_(pa.binary()))}
-    )
-    target = pa.schema([pa.field("images", pa.list_(lancedb.blob("image")))])
-
-    output = _cast_to_target_schema(data.to_reader(), target).read_all()
-
-    images = output["images"].chunk(0)
-    assert images.type.value_field.name == "image"
-    assert type(images.type.value_type) is lancedb.BlobType
-    assert images.to_pylist()[1] is None
-    assert images.values.storage.to_pylist() == [
-        {"data": b"a", "uri": None, "position": None, "size": None},
-        {"data": b"b", "uri": None, "position": None, "size": None},
-    ]
-
-
-def test_list_blob_coercion_preserves_null_slots_with_nonzero_extent():
-    child = pa.field("image", pa.binary())
-    source = pa.ListArray.from_arrays(
-        pa.array([0, 2, 4], type=pa.int32()),
-        pa.array([b"a", b"b", b"dead", b"beef"], type=pa.binary()),
-        mask=pa.array([False, True]),
-    ).cast(pa.list_(child))
-    target = pa.schema([pa.field("images", pa.list_(lancedb.blob("image")))])
-
-    output = _cast_to_target_schema(
-        pa.table({"images": source}).to_reader(), target
-    ).read_all()
-
-    images = output["images"].chunk(0)
-    assert images.to_pylist()[1] is None
-    assert [b["data"] for b in images.to_pylist()[0]] == [b"a", b"b"]
-
-
-def test_fixed_size_list_blob_coercion_keeps_null_rows():
-    child = pa.field("frame", pa.binary())
-    source = (
-        pa.FixedSizeListArray.from_arrays(
-            pa.array([b"a", b"b", b"c", b"d"], type=pa.binary()), 2
-        )
-        .take(pa.array([0, None], type=pa.int32()))
-        .cast(pa.list_(child, 2))
-    )
-    target = pa.schema([pa.field("frames", pa.list_(lancedb.blob("frame"), 2))])
-
-    output = _cast_to_target_schema(
-        pa.table({"frames": source}).to_reader(), target
-    ).read_all()
-
-    frames = output["frames"].chunk(0)
-    assert frames.to_pylist()[1] is None
-    assert [b["data"] for b in frames.to_pylist()[0]] == [b"a", b"b"]
-
-
-def test_cast_to_target_schema_accepts_pylance_blob_v2():
-    target_type = lancedb.BlobType()
-    source = lance.blob_array([b"hello", None])
-    assert type(source.type) is LanceBlobType
-    assert type(source.type) is type(target_type)
-    data = pa.table({"image": source})
-    target = pa.schema([pa.field("image", target_type)])
-
-    output = _cast_to_target_schema(data.to_reader(), target).read_all()
-
-    image = output["image"].chunk(0)
-    assert type(image.type) is LanceBlobType
-    assert image.type == target_type
-    assert image.storage.to_pylist() == [
-        {"data": b"hello", "uri": None, "position": None, "size": None},
-        None,
-    ]
-
-
-def test_cast_to_target_schema_rejects_different_blob_v2_class():
-    class OtherBlobType(pa.ExtensionType):
-        def __init__(self):
-            super().__init__(lancedb.BlobType().storage_type, "lance.blob.v2")
-
-        def __arrow_ext_serialize__(self) -> bytes:
-            return b""
-
-        @classmethod
-        def __arrow_ext_deserialize__(
-            cls, storage_type: pa.DataType, serialized: bytes
-        ) -> "OtherBlobType":
-            return cls()
-
-    storage = lance.blob_array([b"hello"]).storage
-    source = pa.ExtensionArray.from_storage(OtherBlobType(), storage)
-    data = pa.table({"image": source})
-    target = pa.schema([lancedb.blob("image")])
-
-    with pytest.raises(pa.ArrowTypeError, match="different extension type"):
-        _cast_to_target_schema(data.to_reader(), target).read_all()
-
-
 def test_sanitize_data_stream():
     # Make sure we don't collect the whole stream when running sanitize_data
     schema = pa.schema({"a": pa.int32()})

python/src/expr.rs

@@ -130,14 +130,6 @@ impl PyExpr {
 
     // ── utilities ────────────────────────────────────────────────────────────
 
-    /// Return the referenced column name for a bare column expression.
-    fn column_name(&self) -> Option<String> {
-        match &self.0 {
-            DfExpr::Column(column) if column.relation.is_none() => Some(column.name.clone()),
-            _ => None,
-        }
-    }
-
     /// Render the expression as a SQL string (useful for debugging).
     fn to_sql(&self) -> PyResult<String> {
         lancedb::expr::expr_to_sql_string(&self.0).map_err(|e| PyValueError::new_err(e.to_string()))

python/src/query.rs

@@ -1,7 +1,6 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright The LanceDB Authors
 
-use std::collections::HashMap;
 use std::sync::Arc;
 use std::time::Duration;
 
@@ -326,7 +325,6 @@ pub struct PyQueryRequest {
     pub filter: Option<PyQueryFilter>,
     pub full_text_search: Option<PyLanceDB<FtsQuery>>,
     pub select: PySelect,
-    pub select_source_columns: Option<HashMap<String, String>>,
     pub fast_search: Option<bool>,
     pub with_row_id: Option<bool>,
     pub use_lsm: Option<bool>,
@@ -357,7 +355,6 @@ impl From<AnyQuery> for PyQueryRequest {
                 full_text_search: query_request
                     .full_text_search
                     .map(|fts| PyLanceDB(fts.query)),
-                select_source_columns: PySelect::source_columns(&query_request.select),
                 select: PySelect(query_request.select),
                 fast_search: Some(query_request.fast_search),
                 with_row_id: Some(query_request.with_row_id),
@@ -383,7 +380,6 @@ impl From<AnyQuery> for PyQueryRequest {
                 offset: vector_query.base.offset,
                 filter: vector_query.base.filter.map(PyQueryFilter),
                 full_text_search: None,
-                select_source_columns: PySelect::source_columns(&vector_query.base.select),
                 select: PySelect(vector_query.base.select),
                 fast_search: Some(vector_query.base.fast_search),
                 with_row_id: Some(vector_query.base.with_row_id),
@@ -416,25 +412,6 @@ impl From<AnyQuery> for PyQueryRequest {
 #[derive(Clone)]
 pub struct PySelect(Select);
 
-impl PySelect {
-    fn source_columns(select: &Select) -> Option<HashMap<String, String>> {
-        match select {
-            Select::Expr(pairs) => Some(
-                pairs
-                    .iter()
-                    .filter_map(|(output, expr)| match expr {
-                        lancedb::expr::DfExpr::Column(column) if column.relation.is_none() => {
-                            Some((output.clone(), column.name.clone()))
-                        }
-                        _ => None,
-                    })
-                    .collect(),
-            ),
-            _ => None,
-        }
-    }
-}
-
 impl<'py> IntoPyObject<'py> for PySelect {
     type Target = PyAny;
     type Output = Bound<'py, Self::Target>;

python/src/table.rs

@@ -20,8 +20,9 @@ use arrow::{
 use lancedb::blob::{BlobFile, BlobRangeRequest};
 use lancedb::index::scalar::FtsIndexBuilder;
 use lancedb::table::{
-    AddDataMode, ColumnAlteration, Duration, FieldMetadataUpdate, FtsToken as LanceDbFtsToken,
-    NewColumnTransform, OptimizeAction, OptimizeOptions, Ref, Table as LanceDbTable,
+    AddDataMode, ColumnAlteration, CompactionMode, CompactionOptions, Duration,
+    FieldMetadataUpdate, FtsToken as LanceDbFtsToken, IndexRemapMode, NewColumnTransform,
+    OptimizeAction, OptimizeOptions, Ref, Table as LanceDbTable,
 };
 use lancedb::tokenize as lancedb_tokenize;
 use pyo3::{
@@ -100,6 +101,128 @@ enum PredicateArg {
     Sql(String),
 }
 
+fn validate_positive_u32(value: u64, name: &str) -> PyResult<usize> {
+    if !(1..=u32::MAX as u64).contains(&value) {
+        return Err(PyValueError::new_err(format!(
+            "{name} must be between 1 and {}",
+            u32::MAX
+        )));
+    }
+    Ok(value as usize)
+}
+
+fn positive_u32(value: &Bound<'_, PyAny>, name: &str) -> PyResult<usize> {
+    validate_positive_u32(value.extract()?, name)
+}
+
+fn optional_positive_u32(value: &Bound<'_, PyAny>, name: &str) -> PyResult<Option<usize>> {
+    value
+        .extract::<Option<u64>>()?
+        .map(|value| validate_positive_u32(value, name))
+        .transpose()
+}
+
+fn optional_positive_usize(value: &Bound<'_, PyAny>, name: &str) -> PyResult<Option<usize>> {
+    let value: Option<usize> = value.extract()?;
+    if value == Some(0) {
+        return Err(PyValueError::new_err(format!(
+            "{name} must be greater than 0"
+        )));
+    }
+    Ok(value)
+}
+
+fn optional_positive_u64(value: &Bound<'_, PyAny>, name: &str) -> PyResult<Option<u64>> {
+    let value: Option<u64> = value.extract()?;
+    if value == Some(0) {
+        return Err(PyValueError::new_err(format!(
+            "{name} must be greater than 0"
+        )));
+    }
+    Ok(value)
+}
+
+fn u32_list(value: &Bound<'_, PyAny>, name: &str) -> PyResult<Vec<u32>> {
+    value
+        .extract::<Vec<i64>>()?
+        .into_iter()
+        .map(|value| {
+            u32::try_from(value).map_err(|_| {
+                PyValueError::new_err(format!(
+                    "{name} must contain values between 0 and {}",
+                    u32::MAX
+                ))
+            })
+        })
+        .collect()
+}
+
+fn optional_i64_bounded_u64(value: &Bound<'_, PyAny>, name: &str) -> PyResult<Option<u64>> {
+    let value: Option<u64> = value.extract()?;
+    if value.is_some_and(|value| value > i64::MAX as u64) {
+        return Err(PyValueError::new_err(format!(
+            "{name} must be at most {}",
+            i64::MAX
+        )));
+    }
+    Ok(value)
+}
+
+fn parse_compaction_options(options: Option<&Bound<'_, PyDict>>) -> PyResult<CompactionOptions> {
+    let mut parsed = CompactionOptions::default();
+    let Some(options) = options else {
+        return Ok(parsed);
+    };
+
+    for (key, value) in options.iter() {
+        let key: String = key.extract()?;
+        match key.as_str() {
+            "target_rows_per_fragment" => {
+                parsed.target_rows_per_fragment = positive_u32(&value, &key)?
+            }
+            "max_rows_per_group" => parsed.max_rows_per_group = positive_u32(&value, &key)?,
+            "max_bytes_per_file" => parsed.max_bytes_per_file = value.extract()?,
+            "materialize_deletions" => parsed.materialize_deletions = value.extract()?,
+            "materialize_deletions_threshold" => {
+                parsed.materialize_deletions_threshold = value.extract()?
+            }
+            "num_threads" => parsed.num_threads = optional_positive_usize(&value, &key)?,
+            "batch_size" => parsed.batch_size = optional_positive_u32(&value, &key)?,
+            "io_buffer_size" => parsed.io_buffer_size = optional_i64_bounded_u64(&value, &key)?,
+            "defer_index_remap" => parsed.defer_index_remap = value.extract()?,
+            "index_remap_mode" => {
+                let mode: String = value.extract()?;
+                parsed.index_remap_mode = IndexRemapMode::try_from(mode.as_str())
+                    .map_err(|err| PyValueError::new_err(err.to_string()))?;
+            }
+            "compaction_mode" => {
+                let mode: Option<String> = value.extract()?;
+                parsed.compaction_mode = mode
+                    .map(|mode| {
+                        CompactionMode::try_from(mode.as_str())
+                            .map_err(|err| PyValueError::new_err(err.to_string()))
+                    })
+                    .transpose()?;
+            }
+            "binary_copy_read_batch_bytes" => {
+                parsed.binary_copy_read_batch_bytes = value.extract()?
+            }
+            "max_source_fragments" => parsed.max_source_fragments = value.extract()?,
+            "max_source_rows" => parsed.max_source_rows = optional_positive_usize(&value, &key)?,
+            "max_source_bytes" => parsed.max_source_bytes = optional_positive_u64(&value, &key)?,
+            "excluded_fragment_ids" => parsed.excluded_fragment_ids = u32_list(&value, &key)?,
+            "max_overlays_per_fragment" => parsed.max_overlays_per_fragment = value.extract()?,
+            _ => {
+                return Err(PyValueError::new_err(format!(
+                    "Invalid compaction option: {key}"
+                )));
+            }
+        }
+    }
+
+    Ok(parsed)
+}
+
 /// Statistics about a compaction operation.
 #[pyclass(get_all, from_py_object)]
 #[derive(Clone, Debug)]
@@ -1340,13 +1463,15 @@ impl Table {
     }
 
     /// Optimize the on-disk data by compacting and pruning old data, for better performance.
-    #[pyo3(signature = (cleanup_since_ms=None, delete_unverified=None))]
-    pub fn optimize(
-        self_: PyRef<'_, Self>,
+    #[pyo3(signature = (cleanup_since_ms=None, delete_unverified=None, compaction_options=None))]
+    pub fn optimize<'py>(
+        self_: PyRef<'py, Self>,
         cleanup_since_ms: Option<u64>,
         delete_unverified: Option<bool>,
-    ) -> PyResult<Bound<'_, PyAny>> {
+        compaction_options: Option<&Bound<'py, PyDict>>,
+    ) -> PyResult<Bound<'py, PyAny>> {
         let inner = self_.inner_ref()?.clone();
+        let compaction_options = parse_compaction_options(compaction_options)?;
         let older_than = if let Some(ms) = cleanup_since_ms {
             if ms > i64::MAX as u64 {
                 return Err(PyValueError::new_err(format!(
@@ -1362,7 +1487,7 @@ impl Table {
         future_into_py(self_.py(), async move {
             let compaction_stats = inner
                 .optimize(OptimizeAction::Compact {
-                    options: lancedb::table::CompactionOptions::default(),
+                    options: compaction_options,
                     remap_options: None,
                 })
                 .await

rust/lancedb/src/database/listing.rs

@@ -13,7 +13,7 @@ use lance::dataset::{ReadParams, WriteMode, builder::DatasetBuilder};
 use lance::io::{ObjectStore, ObjectStoreParams, WrappingObjectStore};
 use lance_datafusion::utils::StreamingWriteSource;
 use lance_file::version::LanceFileVersion;
-use lance_io::object_store::{ReadDirOptions, StorageOptionsAccessor, StorageOptionsProvider};
+use lance_io::object_store::{StorageOptionsAccessor, StorageOptionsProvider};
 use lance_table::io::commit::commit_handler_from_url;
 use object_store::local::LocalFileSystem;
 use snafu::ResultExt;
@@ -281,22 +281,6 @@ impl std::fmt::Display for ListingDatabase {
 }
 
 const LANCE_EXTENSION: &str = "lance";
-
-/// The table a listed child of the database names, or `None` if the child is not a table.
-///
-/// A table is the directory `<name>.lance`; a loose file or any other directory under the
-/// database prefix belongs to something else. `dir_suffix` is `.lance`, built once by the
-/// caller rather than per child.
-/// The table a listed child directory holds, or `None` if it is not a table at all.
-///
-/// Only directories are considered, so a loose object named like a table is not one.
-fn table_name(location: &object_store::path::Path, dir_suffix: &str) -> Option<String> {
-    location
-        .filename()?
-        .strip_suffix(dir_suffix)
-        .map(String::from)
-        .filter(|name| !name.is_empty())
-}
 const ENGINE: &str = "engine";
 const MIRRORED_STORE: &str = "mirroredStore";
 
@@ -960,72 +944,51 @@ impl Database for ListingDatabase {
         Ok(f)
     }
 
-    /// List the tables in the database, a page at a time.
-    ///
-    /// The page_token is opaque, unlike the `start_after` parameter of [`Self::table_names()`].
-    ///
-    /// When there are no more results, the returned page_token will be None.
-    ///
-    /// `limit` is the maximum number of tables to return in the response. But it is possible
-    /// for the response to contain fewer than `limit` tables, even when there are more tables
-    /// to return. Clients should check the returned page_token to determine if there are
-    /// more results, rather than relying on the number of tables returned.
-    ///
-    /// The order that results are returned in not guaranteed to be stable across calls,
-    /// so clients should not rely on it.
     async fn list_tables(&self, request: ListTablesRequest) -> Result<ListTablesResponse> {
         if request.id.as_ref().map(|v| !v.is_empty()).unwrap_or(false) {
             return self.namespace_database().list_tables(request).await;
         }
-        let limit = request.limit.map(|limit| limit.max(0) as usize);
-        let dir_suffix = format!(".{LANCE_EXTENSION}");
-        let mut tables = Vec::new();
-        let mut page_token = request.page_token.filter(|token| !token.is_empty());
+        let mut f = self
+            .object_store
+            .read_dir(self.base_path.clone())
+            .await?
+            .iter()
+            .map(Path::new)
+            .filter(|path| {
+                let is_lance = path
+                    .extension()
+                    .and_then(|e| e.to_str())
+                    .map(|e| e == LANCE_EXTENSION);
+                is_lance.unwrap_or(false)
+            })
+            .filter_map(|p| p.file_stem().and_then(|s| s.to_str().map(String::from)))
+            .collect::<Vec<String>>();
+        f.sort();
 
-        // A page of nothing: the store rejects a limit of zero, and no table was handed over
-        // for a token to resume after.
-        if limit == Some(0) {
-            return Ok(ListTablesResponse {
-                context: None,
-                tables,
-                page_token: None,
-            });
+        // Handle pagination with page_token
+        if let Some(ref page_token) = request.page_token {
+            let index = f
+                .iter()
+                .position(|name| name.as_str() > page_token.as_str())
+                .unwrap_or(f.len());
+            f.drain(0..index);
         }
 
-        loop {
-            // Ask only for what the page still has room for, so a database holding more
-            // than one page costs one request per page rather than one per table.
-            let listing = self
-                .object_store
-                .read_dir_page(
-                    self.base_path.clone(),
-                    ReadDirOptions {
-                        page_token: page_token.take(),
-                        limit: limit.map(|limit| limit - tables.len()),
-                    },
-                )
-                .await?;
-            page_token = listing.page_token;
-            // Only child directories can be tables, and the store already separates them
-            // out, so the objects in the page are not looked at.
-            tables.extend(
-                listing
-                    .result
-                    .common_prefixes
-                    .iter()
-                    .filter_map(|location| table_name(location, &dir_suffix)),
-            );
-            // Children that are not tables leave the page short of the limit, so keep
-            // going until the page is full or the database runs out.
-            if page_token.is_none() || limit.is_none_or(|limit| tables.len() >= limit) {
-                break;
+        // Determine if there's a next page. The token is the last name of this page,
+        // not the first of the next one: the next page resumes strictly after the
+        // token, so naming the next page's first entry would skip it.
+        let next_page_token = match request.limit {
+            Some(limit) if f.len() > limit as usize => {
+                f.truncate(limit as usize);
+                f.last().cloned()
             }
-        }
+            _ => None,
+        };
 
         Ok(ListTablesResponse {
             context: None,
-            tables,
-            page_token,
+            tables: f,
+            page_token: next_page_token,
         })
     }
 
@@ -1521,182 +1484,6 @@ mod tests {
     use tokio::sync::Barrier;
     use tokio::time::timeout;
 
-    async fn create_tables(db: &ListingDatabase, names: &[&str]) {
-        let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int32, false)]));
-        for name in names {
-            db.create_table(CreateTableRequest {
-                name: name.to_string(),
-                namespace_path: vec![],
-                data: Box::new(RecordBatch::new_empty(schema.clone())) as Box<dyn Scannable>,
-                mode: CreateTableMode::Create,
-                write_options: Default::default(),
-                location: None,
-                namespace_client: None,
-            })
-            .await
-            .unwrap();
-        }
-    }
-
-    /// Every table in the database, taken `limit` at a time, which is how a caller walks a
-    /// listing: the token ends the walk, never a short page.
-    async fn walk(db: &ListingDatabase, limit: Option<i32>) -> Vec<String> {
-        let mut seen = Vec::new();
-        let mut page_token = None;
-        loop {
-            let page = db
-                .list_tables(ListTablesRequest {
-                    limit,
-                    page_token,
-                    ..Default::default()
-                })
-                .await
-                .unwrap();
-            seen.extend(page.tables);
-            page_token = page.page_token;
-            if page_token.is_none() {
-                return seen;
-            }
-            assert!(
-                seen.len() < 100,
-                "the walk is serving tables more than once"
-            );
-        }
-    }
-
-    /// Paging with the returned token has to visit every table exactly once, whatever the
-    /// page size, with nothing lost or repeated at a boundary.
-    #[rstest::rstest]
-    #[tokio::test]
-    async fn test_list_tables_pages_over_every_table_once(#[values(1, 2, 3, 5, 10)] limit: i32) {
-        let (_tempdir, db) = setup_database().await;
-        create_tables(&db, &["a", "b", "c", "d", "e"]).await;
-
-        assert_eq!(walk(&db, Some(limit)).await, vec!["a", "b", "c", "d", "e"]);
-    }
-
-    /// The token is opaque: it is whatever resumes the store the database sits on, not a
-    /// table name. Callers hand it back and nothing else.
-    ///
-    /// Nothing validates a token, so one invented by a caller is read as a position rather
-    /// than refused — which is why the token has to come back from a previous page.
-    #[tokio::test]
-    async fn test_the_page_token_is_not_a_table_name() {
-        let (_tempdir, db) = setup_database().await;
-        create_tables(&db, &["a", "b", "c"]).await;
-
-        let page = db
-            .list_tables(ListTablesRequest {
-                limit: Some(1),
-                ..Default::default()
-            })
-            .await
-            .unwrap();
-
-        assert_eq!(page.tables, vec!["a"]);
-        let token = page.page_token.expect("two tables are still to come");
-        assert_ne!(token, "a");
-
-        // Handing it back is the only thing a caller does with it, and it resumes.
-        let rest = db
-            .list_tables(ListTablesRequest {
-                page_token: Some(token),
-                ..Default::default()
-            })
-            .await
-            .unwrap();
-        assert_eq!(rest.tables, vec!["b", "c"]);
-    }
-
-    /// A limit the listing does not fill leaves no token behind, so a caller paging by token
-    /// stops without asking for an empty page.
-    #[tokio::test]
-    async fn test_a_listing_that_runs_out_has_no_token() {
-        let (_tempdir, db) = setup_database().await;
-        create_tables(&db, &["a", "b"]).await;
-
-        let page = db
-            .list_tables(ListTablesRequest {
-                limit: Some(10),
-                ..Default::default()
-            })
-            .await
-            .unwrap();
-
-        assert_eq!(page.tables, vec!["a", "b"]);
-        assert_eq!(page.page_token, None);
-    }
-
-    /// An empty page token means "from the start", which is how a client looping on a token
-    /// spells its first request.
-    #[tokio::test]
-    async fn test_an_empty_page_token_lists_from_the_start() {
-        let (_tempdir, db) = setup_database().await;
-        create_tables(&db, &["a", "b"]).await;
-
-        let page = db
-            .list_tables(ListTablesRequest {
-                page_token: Some(String::new()),
-                ..Default::default()
-            })
-            .await
-            .unwrap();
-
-        assert_eq!(page.tables, vec!["a", "b"]);
-    }
-
-    /// Listing follows the order the object store lists directories in, so a name that
-    /// extends another comes first: the `-` of `users-archive.lance` sorts below the `.` of
-    /// `users.lance`. Pagination pushes its cursor into the list request, so it cannot report
-    /// an order other than the one it resumes in.
-    #[tokio::test]
-    async fn test_listing_order_follows_the_store_not_the_table_name() {
-        let (_tempdir, db) = setup_database().await;
-        create_tables(&db, &["users", "users-archive", "users.old"]).await;
-
-        assert_eq!(
-            walk(&db, None).await,
-            vec!["users-archive", "users", "users.old"]
-        );
-        // And paging reports the same order, so a walk sees each table once.
-        assert_eq!(
-            walk(&db, Some(1)).await,
-            vec!["users-archive", "users", "users.old"]
-        );
-    }
-
-    /// Only directories named `<name>.lance` are tables; loose files and other directories
-    /// under the database prefix are not. A page spent on them is filled from the next one,
-    /// so a page holding only non-tables does not read as an empty database.
-    #[tokio::test]
-    async fn test_listing_ignores_non_table_children() {
-        let (tempdir, db) = setup_database().await;
-        create_tables(&db, &["real"]).await;
-        std::fs::write(tempdir.path().join("aaa-loose.lance"), b"not a table").unwrap();
-        create_dir_all(tempdir.path().join("aaa-scratch")).unwrap();
-
-        let page = db
-            .list_tables(ListTablesRequest {
-                limit: Some(1),
-                ..Default::default()
-            })
-            .await
-            .unwrap();
-
-        assert_eq!(page.tables, vec!["real"]);
-    }
-
-    #[tokio::test]
-    async fn listing_ignores_empty_table_name() {
-        let (tempdir, db) = setup_database().await;
-        create_dir_all(tempdir.path().join(".lance")).unwrap();
-        let page = db.list_tables(ListTablesRequest::default()).await.unwrap();
-        assert!(
-            page.tables.is_empty(),
-            "invalid empty table name was listed"
-        );
-    }
-
     async fn setup_database() -> (tempfile::TempDir, ListingDatabase) {
         let tempdir = tempdir().unwrap();
         let uri = tempdir.path().to_str().unwrap();

rust/lancedb/src/expr.rs

@@ -157,7 +157,7 @@ mod tests {
         use datafusion_common::ScalarValue;
         let expr = col("data").eq(lit(ScalarValue::Binary(Some(vec![0xca, 0xfe]))));
         let sql = expr_to_sql_string(&expr).unwrap();
-        assert_eq!(sql, "(`data` = X'CAFE')");
+        assert_eq!(sql, "(data = X'CAFE')");
     }
 
     #[test]
@@ -167,7 +167,7 @@ mod tests {
         let int_expr = col("id").gt(lit(5i64));
         let combined = bin_expr.and(int_expr);
         let sql = expr_to_sql_string(&combined).unwrap();
-        assert_eq!(sql, "((`data` = X'01') AND (id > 5))");
+        assert_eq!(sql, "((data = X'01') AND (id > 5))");
     }
 
     #[test]
@@ -185,7 +185,7 @@ mod tests {
         // serialized correctly (regression test for placeholder rewrite path).
         let expr = contains(col("data"), lit(ScalarValue::Binary(Some(vec![0xff]))));
         let sql = expr_to_sql_string(&expr).unwrap();
-        assert_eq!(sql, "contains(`data`, X'FF')");
+        assert_eq!(sql, "contains(data, X'FF')");
     }
 
     #[test]
@@ -196,7 +196,7 @@ mod tests {
             .eq(lit(ScalarValue::Binary(Some(vec![0xab, 0xcd]))))
             .not();
         let sql = expr_to_sql_string(&expr).unwrap();
-        assert_eq!(sql, "NOT (`data` = X'ABCD')");
+        assert_eq!(sql, "NOT (data = X'ABCD')");
     }
 
     #[test]
@@ -206,122 +206,6 @@ mod tests {
         assert!(sql.contains("IN"), "expected IN in: {}", sql);
     }
 
-    #[test]
-    fn test_empty_is_in() {
-        let expr = is_in(col("id"), vec![]);
-        assert_eq!(expr_to_sql_string(&expr).unwrap(), "false");
-    }
-
-    #[test]
-    fn test_empty_is_in_discards_binary_children() {
-        use datafusion_common::ScalarValue;
-
-        let expr = is_in(
-            col("payload").eq(lit(ScalarValue::Binary(Some(vec![0x01])))),
-            vec![],
-        );
-        assert_eq!(expr_to_sql_string(&expr).unwrap(), "false");
-    }
-
-    #[test]
-    fn test_keyword_identifier() {
-        let expr = col("null").eq(lit(1i64));
-        assert_eq!(expr_to_sql_string(&expr).unwrap(), "(`null` = 1)");
-    }
-
-    #[test]
-    fn test_decimal_literal_preserves_type() {
-        use datafusion_common::ScalarValue;
-
-        let expr = col("val").lt(lit(ScalarValue::Decimal128(
-            Some(1_234_567_890_123_456_790),
-            19,
-            18,
-        )));
-        let sql = expr_to_sql_string(&expr).unwrap();
-        assert_eq!(
-            sql,
-            "(val < arrow_cast('1.234567890123456790', 'Decimal128(19, 18)'))"
-        );
-    }
-
-    #[test]
-    fn test_non_finite_float_literal_preserves_type() {
-        let expr = col("x").lt(lit(f64::INFINITY));
-        assert_eq!(
-            expr_to_sql_string(&expr).unwrap(),
-            "(x < arrow_cast('inf', 'Float64'))"
-        );
-    }
-
-    #[test]
-    fn test_cast_uses_arrow_type_name() {
-        let string = expr_cast(col("x"), DataType::Utf8);
-        assert_eq!(
-            expr_to_sql_string(&string).unwrap(),
-            "arrow_cast(x, 'Utf8')"
-        );
-
-        let int32 = expr_cast(col("x"), DataType::Int32);
-        assert_eq!(
-            expr_to_sql_string(&int32).unwrap(),
-            "arrow_cast(x, 'Int32')"
-        );
-
-        let expr = expr_cast(col("x"), DataType::Float16).lt(lit(2.0));
-        assert_eq!(
-            expr_to_sql_string(&expr).unwrap(),
-            "(arrow_cast(x, 'Float16') < 2.0)"
-        );
-
-        let decimal = expr_cast(lit("2.00"), DataType::Decimal256(40, 2));
-        assert_eq!(
-            expr_to_sql_string(&decimal).unwrap(),
-            "arrow_cast('2.00', 'Decimal256(40, 2)')"
-        );
-    }
-
-    #[test]
-    fn test_binary_placeholder_does_not_rewrite_user_string() {
-        use datafusion_common::ScalarValue;
-
-        let marker = "__lancedb_binary_placeholder_0__";
-        let expr = col("payload")
-            .eq(lit(ScalarValue::Binary(Some(vec![0x01]))))
-            .or(col("text").eq(lit(marker)));
-        assert_eq!(
-            expr_to_sql_string(&expr).unwrap(),
-            "((payload = X'01') OR (`text` = '__lancedb_binary_placeholder_0__'))"
-        );
-    }
-
-    #[test]
-    fn test_binary_binding_skips_quoted_identifiers() {
-        use datafusion_common::ScalarValue;
-
-        let expr = col("payload")
-            .eq(lit(ScalarValue::Binary(Some(vec![0x01]))))
-            .and(col("odd'name").eq(lit(1i64)))
-            .and(col("odd`'name").eq(lit(2i64)));
-        assert_eq!(
-            expr_to_sql_string(&expr).unwrap(),
-            "(((payload = X'01') AND (`odd'name` = 1)) AND (`odd``'name` = 2))"
-        );
-    }
-
-    #[test]
-    fn test_binary_placeholder_collision_search_is_linear() {
-        use datafusion_common::ScalarValue;
-
-        let collision_shaped = format!("__lancedb_binary_placeholder_0__{}", "_".repeat(64_000));
-        let expr = col("payload")
-            .eq(lit(ScalarValue::Binary(Some(vec![0x01]))))
-            .and(col("text").eq(lit(collision_shaped.clone())));
-        let sql = expr_to_sql_string(&expr).unwrap();
-        assert!(sql.contains("X'01'"));
-        assert!(sql.contains(&format!("'{collision_shaped}'")));
-    }
-
     #[test]
     fn test_multiple_binary_literals() {
         use datafusion_common::ScalarValue;

rust/lancedb/src/expr/sql.rs

@@ -1,24 +1,13 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright The LanceDB Authors
 
-use std::{
-    any::TypeId,
-    collections::{HashMap, HashSet},
-};
+use std::any::TypeId;
 
-use arrow_array::types::{
-    Decimal32Type, Decimal64Type, Decimal128Type, Decimal256Type, DecimalType,
-};
-use arrow_schema::DataType;
 use datafusion_common::ScalarValue;
 use datafusion_common::tree_node::{Transformed, TreeNode, TreeNodeRecursion};
 use datafusion_expr::Expr;
-use datafusion_functions::core::expr_fn::{
-    arrow_cast as datafusion_arrow_cast, arrow_try_cast as datafusion_arrow_try_cast,
-};
 use datafusion_sql::sqlparser::{
     dialect::{Dialect as SqlParserDialect, GenericDialect},
-    keywords::ALL_KEYWORDS,
     tokenizer::{Token, Tokenizer},
 };
 use datafusion_sql::unparser::{self, dialect::Dialect as UnparserDialect};
@@ -38,13 +27,11 @@ struct LanceSqlDialect;
 
 impl UnparserDialect for LanceSqlDialect {
     fn identifier_quote_style(&self, identifier: &str) -> Option<char> {
-        let identifier_upper = identifier.to_ascii_uppercase();
-        let needs_quote =
-            (identifier_upper != "ID" && ALL_KEYWORDS.contains(&identifier_upper.as_str()))
-                || identifier.chars().any(|c| c.is_ascii_uppercase())
-                || !identifier.chars().enumerate().all(|(i, c)| {
-                    c == '_' || c.is_ascii_alphabetic() || (i > 0 && c.is_ascii_digit())
-                });
+        let needs_quote = identifier.chars().any(|c| c.is_ascii_uppercase())
+            || !identifier
+                .chars()
+                .enumerate()
+                .all(|(i, c)| c == '_' || c.is_ascii_alphabetic() || (i > 0 && c.is_ascii_digit()));
         if needs_quote { Some('`') } else { None }
     }
 }
@@ -113,128 +100,24 @@ fn bytes_to_hex_sql(bytes: &[u8]) -> String {
     format!("X'{hex}'")
 }
 
-fn string_literals(expr: &Expr) -> HashSet<String> {
-    let mut literals = HashSet::new();
+/// Returns true if *expr* contains a `Binary` or `LargeBinary` scalar literal
+/// anywhere in its subtree.  DataFusion's SQL unparser cannot serialize those
+/// variants, so we route such expressions through a placeholder-substitution
+/// path that emits SQL `X'...'` byte-string literals.
+fn has_binary_literal(expr: &Expr) -> bool {
+    let mut found = false;
     let _ = expr.apply(&mut |e: &Expr| {
-        if let Expr::Literal(
-            ScalarValue::Utf8(Some(value))
-            | ScalarValue::LargeUtf8(Some(value))
-            | ScalarValue::Utf8View(Some(value)),
-            _,
-        ) = e
-        {
-            literals.insert(value.clone());
-        }
-        Ok(TreeNodeRecursion::Continue)
-    });
-    literals
-}
-
-fn typed_string_literal(value: String, data_type: DataType) -> Expr {
-    datafusion_arrow_cast(
-        Expr::Literal(ScalarValue::Utf8(Some(value)), None),
-        Expr::Literal(ScalarValue::Utf8(Some(data_type.to_string())), None),
-    )
-}
-
-fn next_binary_placeholder(user_strings: &HashSet<String>, next_id: &mut usize) -> String {
-    loop {
-        let placeholder = format!("{BINARY_PLACEHOLDER_PREFIX}{}__", *next_id);
-        *next_id += 1;
-        if !user_strings.contains(&placeholder) {
-            return placeholder;
-        }
-    }
-}
-
-fn bind_binary_literals(
-    sql: &str,
-    mut bindings: HashMap<String, Vec<u8>>,
-) -> crate::Result<String> {
-    let bytes = sql.as_bytes();
-    let mut output = Vec::with_capacity(bytes.len());
-    let mut index = 0;
-
-    // Walk SQL string tokens once. Placeholders are plain, unescaped string
-    // literals, so this remains linear even when user strings are large or
-    // deliberately resemble the placeholder prefix.
-    while index < bytes.len() {
-        if bytes[index] == b'`' {
-            let identifier_start = index;
-            index += 1;
-            let mut identifier_end = None;
-            while index < bytes.len() {
-                if bytes[index] == b'`' {
-                    if index + 1 < bytes.len() && bytes[index + 1] == b'`' {
-                        index += 2;
-                    } else {
-                        index += 1;
-                        identifier_end = Some(index);
-                        break;
-                    }
-                } else {
-                    index += 1;
-                }
-            }
-
-            let Some(identifier_end) = identifier_end else {
-                return Err(crate::Error::InvalidInput {
-                    message: "unterminated identifier while binding binary literal".to_string(),
-                });
-            };
-            output.extend_from_slice(&bytes[identifier_start..identifier_end]);
-            continue;
-        }
-
-        if bytes[index] != b'\'' {
-            output.push(bytes[index]);
-            index += 1;
-            continue;
-        }
-
-        let literal_start = index;
-        index += 1;
-        let content_start = index;
-        let mut escaped = false;
-        let mut content_end = None;
-        while index < bytes.len() {
-            if bytes[index] == b'\'' {
-                if index + 1 < bytes.len() && bytes[index + 1] == b'\'' {
-                    escaped = true;
-                    index += 2;
-                } else {
-                    content_end = Some(index);
-                    index += 1;
-                    break;
-                }
-            } else {
-                index += 1;
-            }
-        }
-
-        let Some(content_end) = content_end else {
-            return Err(crate::Error::InvalidInput {
-                message: "unterminated string while binding binary literal".to_string(),
-            });
-        };
-
-        let placeholder = &sql[content_start..content_end];
-        if !escaped && let Some(value) = bindings.remove(placeholder) {
-            output.extend_from_slice(bytes_to_hex_sql(&value).as_bytes());
+        if matches!(
+            e,
+            Expr::Literal(ScalarValue::Binary(_) | ScalarValue::LargeBinary(_), _)
+        ) {
+            found = true;
+            Ok(TreeNodeRecursion::Stop)
         } else {
-            output.extend_from_slice(&bytes[literal_start..index]);
+            Ok(TreeNodeRecursion::Continue)
         }
-    }
-
-    if !bindings.is_empty() {
-        return Err(crate::Error::InvalidInput {
-            message: "failed to bind binary literal while serializing expression".to_string(),
-        });
-    }
-
-    String::from_utf8(output).map_err(|e| crate::Error::InvalidInput {
-        message: format!("failed to bind binary literal: {e}"),
-    })
+    });
+    found
 }
 
 fn run_unparser(expr: &Expr) -> crate::Result<String> {
@@ -247,37 +130,25 @@ fn run_unparser(expr: &Expr) -> crate::Result<String> {
 }
 
 pub fn expr_to_sql_string(expr: &Expr) -> crate::Result<String> {
-    // DataFusion's unparser needs a few adaptations before its SQL can be
-    // reparsed by Lance without changing the typed expression's semantics:
-    //
-    // * decimal literals need an explicit cast to preserve precision and scale;
-    // * casts need exact Arrow type names rather than SQL type aliases;
-    // * an empty IN list is valid in DataFusion but invalid SQL;
-    // * binary literals are unsupported by the unparser and need placeholders.
-    // Eliminate empty membership expressions before visiting their children.
-    // Otherwise a discarded binary child could leave behind a stale binding.
+    // Fast path: no binary literals — DataFusion's unparser handles everything.
+    if !has_binary_literal(expr) {
+        return run_unparser(expr);
+    }
+
+    // Slow path: DataFusion's unparser cannot serialize `Binary`/`LargeBinary`
+    // scalars, so we rewrite each one to a unique string-literal placeholder,
+    // let the unparser do the rest of the work, then substitute the SQL
+    // `X'...'` byte-string literal back in.  This keeps the operator/function
+    // serialization logic centralized in DataFusion and works for every
+    // expression node type the unparser supports.
+    let mut bindings: Vec<Vec<u8>> = Vec::new();
     let rewritten = expr
         .clone()
-        .transform(|e: Expr| match e {
-            Expr::InList(in_list) if in_list.list.is_empty() => Ok(Transformed::yes(
-                Expr::Literal(ScalarValue::Boolean(Some(in_list.negated)), None),
-            )),
-            other => Ok(Transformed::no(other)),
-        })
-        .map_err(|e| crate::Error::InvalidInput {
-            message: format!("failed to rewrite expression: {e}"),
-        })?
-        .data;
-
-    let user_strings = string_literals(&rewritten);
-    let mut next_placeholder_id = 0;
-    let mut binary_bindings = HashMap::new();
-    let rewritten = rewritten
         .transform(|e: Expr| match e {
             Expr::Literal(ScalarValue::Binary(Some(bytes)), m)
             | Expr::Literal(ScalarValue::LargeBinary(Some(bytes)), m) => {
-                let placeholder = next_binary_placeholder(&user_strings, &mut next_placeholder_id);
-                binary_bindings.insert(placeholder.clone(), bytes);
+                let placeholder = format!("{}{}__", BINARY_PLACEHOLDER_PREFIX, bindings.len());
+                bindings.push(bytes);
                 Ok(Transformed::yes(Expr::Literal(
                     ScalarValue::Utf8(Some(placeholder)),
                     m,
@@ -287,57 +158,6 @@ pub fn expr_to_sql_string(expr: &Expr) -> crate::Result<String> {
             | Expr::Literal(ScalarValue::LargeBinary(None), m) => {
                 Ok(Transformed::yes(Expr::Literal(ScalarValue::Null, m)))
             }
-            Expr::Literal(ScalarValue::Decimal32(Some(value), precision, scale), _m) => {
-                let value = Decimal32Type::format_decimal(value, precision, scale);
-                Ok(Transformed::yes(typed_string_literal(
-                    value,
-                    DataType::Decimal32(precision, scale),
-                )))
-            }
-            Expr::Literal(ScalarValue::Decimal64(Some(value), precision, scale), _m) => {
-                let value = Decimal64Type::format_decimal(value, precision, scale);
-                Ok(Transformed::yes(typed_string_literal(
-                    value,
-                    DataType::Decimal64(precision, scale),
-                )))
-            }
-            Expr::Literal(ScalarValue::Decimal128(Some(value), precision, scale), _m) => {
-                let value = Decimal128Type::format_decimal(value, precision, scale);
-                Ok(Transformed::yes(typed_string_literal(
-                    value,
-                    DataType::Decimal128(precision, scale),
-                )))
-            }
-            Expr::Literal(ScalarValue::Decimal256(Some(value), precision, scale), _m) => {
-                let value = Decimal256Type::format_decimal(value, precision, scale);
-                Ok(Transformed::yes(typed_string_literal(
-                    value,
-                    DataType::Decimal256(precision, scale),
-                )))
-            }
-            Expr::Literal(ScalarValue::Float16(Some(value)), _m) if !value.is_finite() => Ok(
-                Transformed::yes(typed_string_literal(value.to_string(), DataType::Float16)),
-            ),
-            Expr::Literal(ScalarValue::Float32(Some(value)), _m) if !value.is_finite() => Ok(
-                Transformed::yes(typed_string_literal(value.to_string(), DataType::Float32)),
-            ),
-            Expr::Literal(ScalarValue::Float64(Some(value)), _m) if !value.is_finite() => Ok(
-                Transformed::yes(typed_string_literal(value.to_string(), DataType::Float64)),
-            ),
-            Expr::Cast(cast) => Ok(Transformed::yes(datafusion_arrow_cast(
-                *cast.expr,
-                Expr::Literal(
-                    ScalarValue::Utf8(Some(cast.field.data_type().to_string())),
-                    None,
-                ),
-            ))),
-            Expr::TryCast(cast) => Ok(Transformed::yes(datafusion_arrow_try_cast(
-                *cast.expr,
-                Expr::Literal(
-                    ScalarValue::Utf8(Some(cast.field.data_type().to_string())),
-                    None,
-                ),
-            ))),
             other => Ok(Transformed::no(other)),
         })
         .map_err(|e| crate::Error::InvalidInput {
@@ -345,12 +165,14 @@ pub fn expr_to_sql_string(expr: &Expr) -> crate::Result<String> {
         })?
         .data;
 
-    let sql = run_unparser(&rewritten)?;
-    if binary_bindings.is_empty() {
-        Ok(sql)
-    } else {
-        bind_binary_literals(&sql, binary_bindings)
+    let mut sql = run_unparser(&rewritten)?;
+    for (i, bytes) in bindings.iter().enumerate() {
+        // The unparser quotes string literals with single quotes, so the
+        // placeholder appears as `'__lancedb_binary_placeholder_<i>__'`.
+        let quoted = format!("'{}{}__'", BINARY_PLACEHOLDER_PREFIX, i);
+        sql = sql.replace(&quoted, &bytes_to_hex_sql(bytes));
     }
+    Ok(sql)
 }
 
 #[cfg(test)]

rust/lancedb/src/function.rs

@@ -5,9 +5,9 @@
 //! backend-neutral terminal result of a computed-column refresh.
 //!
 //! This module contains client/wire values only. Catalog persistence,
-//! environment bake, secret resolution, and execution are owned by Sophon.
+//! environment bake, and execution are owned by Sophon.
 
-use std::collections::{BTreeMap, BTreeSet};
+use std::collections::BTreeMap;
 
 use serde::de::{self, DeserializeOwned};
 use serde::{Deserialize, Deserializer, Serialize, Serializer};
@@ -15,16 +15,6 @@ use serde_json::Value;
 
 use crate::{Error, Result};
 
-// Keep these byte limits aligned with Sophon's Function submission validation.
-pub(crate) const MAX_FUNCTION_SECRET_VALUE_BYTES: usize = 64 * 1024;
-const MAX_FUNCTION_SECRET_VALUES_BYTES: usize = 512 * 1024;
-
-fn is_portable_environment_name(name: &str) -> bool {
-    let mut bytes = name.bytes();
-    matches!(bytes.next(), Some(b'A'..=b'Z' | b'a'..=b'z' | b'_'))
-        && bytes.all(|byte| matches!(byte, b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'_'))
-}
-
 fn invalid_json(error: impl std::fmt::Display) -> Error {
     Error::InvalidInput {
         message: format!("invalid remote Function JSON: {error}"),
@@ -208,11 +198,6 @@ pub struct PythonEnvironmentSpec {
 }
 
 /// Reproducible Python runtime definition understood by Sophon.
-///
-/// `env` contains non-secret values. Secret values are submission-only in the
-/// client model and do not become part of this public runtime identity;
-/// [`FunctionVersion::required_secrets`] contains names only. Sophon persists
-/// submitted values separately in the private execution artifact.
 #[derive(Debug, Clone, PartialEq, Eq)]
 #[non_exhaustive]
 pub enum PythonRuntimeSpec {
@@ -254,7 +239,7 @@ impl PythonRuntimeSpec {
         }
     }
 
-    /// Non-secret environment variables, or `None` for an unknown kind.
+    /// Environment variables, or `None` for an unknown kind.
     pub fn env(&self) -> Option<&BTreeMap<String, String>> {
         match self {
             Self::Python { env, .. } => Some(env),
@@ -339,8 +324,6 @@ pub struct FunctionVersion {
     runtime: PythonRuntimeSpec,
     runtime_digest: String,
     environment_digest: String,
-    #[serde(default, skip_serializing_if = "Vec::is_empty")]
-    required_secrets: Vec<String>,
     created_at: String,
 }
 
@@ -373,12 +356,6 @@ impl FunctionVersion {
         &self.environment_digest
     }
 
-    /// Required secret names. Resolved values exist only in Sophon's private
-    /// execution artifact and worker launch path.
-    pub fn required_secrets(&self) -> &[String] {
-        &self.required_secrets
-    }
-
     pub fn created_at(&self) -> &str {
         &self.created_at
     }
@@ -420,115 +397,12 @@ pub struct FunctionArtifactRequest {
 }
 
 /// Stable request envelope for remote immutable Function registration.
-///
-/// Secret values are submission-only in the client model. Sophon persists them
-/// in the database-scoped private execution artifact; returned
-/// [`FunctionVersion`] and Job metadata contain only
-/// [`Self::required_secrets`] names. Debug formatting always redacts values.
-#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)]
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
 pub struct FunctionRegistrationRequest {
     pub name: String,
     pub artifact: FunctionArtifactRequest,
     pub signature: FunctionSignature,
     pub runtime: PythonRuntimeSpec,
-    #[serde(default, skip_serializing_if = "Vec::is_empty")]
-    pub required_secrets: Vec<String>,
-    #[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
-    pub secret_values: BTreeMap<String, String>,
-}
-
-impl FunctionRegistrationRequest {
-    pub(crate) fn validate_secret_values(&self) -> Result<()> {
-        let mut required = BTreeSet::new();
-        for name in &self.required_secrets {
-            if !is_portable_environment_name(name) {
-                return Err(Error::InvalidInput {
-                    message: format!(
-                        "Function secret name {name:?} must be a portable environment variable name"
-                    ),
-                });
-            }
-            if !required.insert(name) {
-                return Err(Error::InvalidInput {
-                    message: format!("Function required_secrets contains duplicate name {name:?}"),
-                });
-            }
-        }
-
-        if let PythonRuntimeSpec::Python { env, .. } = &self.runtime
-            && let Some(name) = required.iter().find(|name| env.contains_key(**name))
-        {
-            return Err(Error::InvalidInput {
-                message: format!(
-                    "Function runtime env and secret names must be disjoint: {name:?}"
-                ),
-            });
-        }
-
-        let provided = self.secret_values.keys().collect::<BTreeSet<_>>();
-        if required != provided {
-            return Err(Error::InvalidInput {
-                message: "Function secret_values keys must exactly match required_secrets"
-                    .to_string(),
-            });
-        }
-
-        let mut total_bytes = 0usize;
-        for (name, value) in &self.secret_values {
-            if value.is_empty() {
-                return Err(Error::InvalidInput {
-                    message: format!("Function secret {name:?} value must be non-empty"),
-                });
-            }
-            if value.contains('\0') {
-                return Err(Error::InvalidInput {
-                    message: format!("Function secret {name:?} value must not contain NUL"),
-                });
-            }
-            if value.len() > MAX_FUNCTION_SECRET_VALUE_BYTES {
-                return Err(Error::InvalidInput {
-                    message: format!(
-                        "Function secret {name:?} value exceeds the \
-                         {MAX_FUNCTION_SECRET_VALUE_BYTES}-byte limit"
-                    ),
-                });
-            }
-            total_bytes =
-                total_bytes
-                    .checked_add(value.len())
-                    .ok_or_else(|| Error::InvalidInput {
-                        message: "Function secret values exceed the request byte limit".to_string(),
-                    })?;
-        }
-        if total_bytes > MAX_FUNCTION_SECRET_VALUES_BYTES {
-            return Err(Error::InvalidInput {
-                message: format!(
-                    "Function secret values exceed the \
-                     {MAX_FUNCTION_SECRET_VALUES_BYTES}-byte request limit"
-                ),
-            });
-        }
-        Ok(())
-    }
-}
-
-impl std::fmt::Debug for FunctionRegistrationRequest {
-    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        let secret_values = self
-            .secret_values
-            .keys()
-            .map(|name| (name, "[REDACTED]"))
-            .collect::<BTreeMap<_, _>>();
-        formatter
-            .debug_struct("FunctionRegistrationRequest")
-            .field("name", &self.name)
-            .field("artifact", &self.artifact)
-            .field("signature", &self.signature)
-            .field("runtime", &self.runtime)
-            .field("required_secrets", &self.required_secrets)
-            .field("secret_values", &secret_values)
-            .finish()
-    }
 }
 
 impl_json!(FunctionRegistrationRequest);
@@ -713,185 +587,6 @@ impl RefreshColumnResult {
 
 impl_json!(RefreshColumnResult);
 
-#[cfg(test)]
-mod secret_value_tests {
-    use super::{
-        FunctionRegistrationRequest, MAX_FUNCTION_SECRET_VALUE_BYTES,
-        MAX_FUNCTION_SECRET_VALUES_BYTES, PythonRuntimeSpec,
-    };
-    use crate::Error;
-
-    fn request() -> FunctionRegistrationRequest {
-        FunctionRegistrationRequest::from_json(include_str!(
-            "../tests/fixtures/first_class_functions/v1/remote_function_registration_request.json"
-        ))
-        .unwrap()
-    }
-
-    #[test]
-    fn validates_secret_name_and_value_invariants() {
-        let missing = request();
-        assert!(matches!(
-            missing.validate_secret_values(),
-            Err(Error::InvalidInput { message }) if message.contains("exactly match")
-        ));
-
-        let mut empty = request();
-        empty
-            .secret_values
-            .insert("API_TOKEN".to_string(), String::new());
-        assert!(matches!(
-            empty.validate_secret_values(),
-            Err(Error::InvalidInput { message }) if message.contains("non-empty")
-        ));
-
-        let mut nul = request();
-        nul.secret_values
-            .insert("API_TOKEN".to_string(), "before\0after".to_string());
-        assert!(matches!(
-            nul.validate_secret_values(),
-            Err(Error::InvalidInput { message }) if message.contains("NUL")
-        ));
-
-        let mut unexpected = request();
-        unexpected
-            .secret_values
-            .insert("OTHER".to_string(), "value".to_string());
-        assert!(matches!(
-            unexpected.validate_secret_values(),
-            Err(Error::InvalidInput { message }) if message.contains("exactly match")
-        ));
-    }
-
-    #[test]
-    fn rejects_invalid_duplicate_and_overlapping_secret_declarations() {
-        let mut invalid_name = request();
-        invalid_name.required_secrets = vec!["BAD=NAME".to_string()];
-        invalid_name
-            .secret_values
-            .insert("BAD=NAME".to_string(), "secret".to_string());
-
-        let mut duplicate = request();
-        duplicate.required_secrets = vec!["API_TOKEN".to_string(), "API_TOKEN".to_string()];
-        duplicate
-            .secret_values
-            .insert("API_TOKEN".to_string(), "secret".to_string());
-
-        let mut overlap = request();
-        overlap
-            .secret_values
-            .insert("API_TOKEN".to_string(), "secret".to_string());
-        if let PythonRuntimeSpec::Python { env, .. } = &mut overlap.runtime {
-            env.insert("API_TOKEN".to_string(), "public".to_string());
-        }
-
-        assert!(matches!(
-            invalid_name.validate_secret_values(),
-            Err(Error::InvalidInput { message }) if message.contains("portable environment variable")
-        ));
-        assert!(matches!(
-            duplicate.validate_secret_values(),
-            Err(Error::InvalidInput { message }) if message.contains("duplicate")
-        ));
-        assert!(matches!(
-            overlap.validate_secret_values(),
-            Err(Error::InvalidInput { message }) if message.contains("must be disjoint")
-        ));
-    }
-
-    #[test]
-    fn enforces_portable_secret_name_boundaries() {
-        for name in ["A", "_", "A0_"] {
-            let mut request = request();
-            request.required_secrets = vec![name.to_string()];
-            request
-                .secret_values
-                .insert(name.to_string(), "secret".to_string());
-            request.validate_secret_values().unwrap();
-        }
-
-        for name in ["", "0TOKEN", "BAD-NAME", "TÖKEN"] {
-            let mut request = request();
-            request.required_secrets = vec![name.to_string()];
-            request
-                .secret_values
-                .insert(name.to_string(), "secret".to_string());
-            assert!(matches!(
-                request.validate_secret_values(),
-                Err(Error::InvalidInput { message })
-                    if message.contains("portable environment variable")
-            ));
-        }
-    }
-
-    #[test]
-    fn accepts_exact_secret_value_utf8_byte_limit() {
-        for value in [
-            "x".repeat(MAX_FUNCTION_SECRET_VALUE_BYTES),
-            "é".repeat(MAX_FUNCTION_SECRET_VALUE_BYTES / "é".len()),
-        ] {
-            assert_eq!(value.len(), MAX_FUNCTION_SECRET_VALUE_BYTES);
-            let mut request = request();
-            request.secret_values.insert("API_TOKEN".to_string(), value);
-            request.validate_secret_values().unwrap();
-        }
-    }
-
-    #[test]
-    fn rejects_secret_value_over_utf8_byte_limit() {
-        for value in [
-            "x".repeat(MAX_FUNCTION_SECRET_VALUE_BYTES + 1),
-            "é".repeat(MAX_FUNCTION_SECRET_VALUE_BYTES / "é".len() + 1),
-        ] {
-            assert!(value.len() > MAX_FUNCTION_SECRET_VALUE_BYTES);
-            let mut request = request();
-            request.secret_values.insert("API_TOKEN".to_string(), value);
-            assert!(matches!(
-                request.validate_secret_values(),
-                Err(Error::InvalidInput { message }) if message.contains("65536-byte limit")
-            ));
-        }
-    }
-
-    #[test]
-    fn rejects_aggregate_secret_value_bytes_over_server_limit() {
-        let mut request = request();
-        request.required_secrets = (0..9).map(|index| format!("SECRET_{index}")).collect();
-        request.secret_values = request
-            .required_secrets
-            .iter()
-            .map(|name| (name.clone(), "x".repeat(MAX_FUNCTION_SECRET_VALUE_BYTES)))
-            .collect();
-
-        assert!(matches!(
-            request.validate_secret_values(),
-            Err(Error::InvalidInput { message })
-                if message.contains(&format!("{MAX_FUNCTION_SECRET_VALUES_BYTES}-byte request limit"))
-        ));
-    }
-
-    #[test]
-    fn accepts_exact_aggregate_secret_value_byte_limit() {
-        let mut request = request();
-        request.required_secrets = (0..8).map(|index| format!("SECRET_{index}")).collect();
-        request.secret_values = request
-            .required_secrets
-            .iter()
-            .map(|name| (name.clone(), "x".repeat(MAX_FUNCTION_SECRET_VALUE_BYTES)))
-            .collect();
-
-        assert_eq!(
-            request
-                .secret_values
-                .values()
-                .map(String::len)
-                .sum::<usize>(),
-            MAX_FUNCTION_SECRET_VALUES_BYTES
-        );
-        request.validate_secret_values().unwrap();
-    }
-}
-
 #[cfg(test)]
 mod conda_environment_tests {
     use super::PythonEnvironmentSpec;

rust/lancedb/src/remote/client.rs

@@ -7,7 +7,6 @@ use reqwest::{
     Body, Request, RequestBuilder, Response,
     header::{HeaderMap, HeaderValue},
 };
-use serde_json::Value;
 use std::{collections::HashMap, future::Future, str::FromStr, sync::Arc, time::Duration};
 
 use crate::error::{Error, Result};
@@ -15,60 +14,6 @@ use crate::remote::db::RemoteOptions;
 use crate::remote::retry::{ResolvedRetryConfig, RetryCounter};
 
 const REQUEST_ID_HEADER: HeaderName = HeaderName::from_static("x-request-id");
-const REDACTED_JSON_VALUE: &str = "[REDACTED]";
-const SUPPRESSED_JSON_BODY: &str = "[JSON BODY SUPPRESSED]";
-
-fn is_sensitive_json_field(name: &str) -> bool {
-    name.to_ascii_lowercase().contains("secret")
-}
-
-fn redact_sensitive_json_fields(value: &mut Value) {
-    match value {
-        Value::Object(fields) => {
-            for (name, child) in fields {
-                if is_sensitive_json_field(name) {
-                    *child = Value::String(REDACTED_JSON_VALUE.to_string());
-                } else {
-                    redact_sensitive_json_fields(child);
-                }
-            }
-        }
-        Value::Array(values) => values.iter_mut().for_each(redact_sensitive_json_fields),
-        _ => {}
-    }
-}
-
-fn redacted_json_body(request: &Request) -> Option<String> {
-    let body = request.body()?.as_bytes()?;
-    let mut value = serde_json::from_slice(body).ok()?;
-    redact_sensitive_json_fields(&mut value);
-    serde_json::to_string(&value).ok()
-}
-
-fn request_log_message(request: &Request, request_id: &str) -> String {
-    let prefix = format!(
-        "Sending request_id={}: {} {}",
-        request_id,
-        request.method(),
-        request.url()
-    );
-    let content_type = request
-        .headers()
-        .get("content-type")
-        .and_then(|value| value.to_str().ok())
-        .and_then(|value| value.split(';').next());
-    if content_type.is_some_and(|value| value.eq_ignore_ascii_case("application/json")) {
-        // Never format the raw Request here: its Debug representation is not a
-        // redaction boundary and may include the original body. If the JSON body
-        // cannot be structurally parsed, suppress it instead of logging raw bytes.
-        let body = redacted_json_body(request).unwrap_or_else(|| SUPPRESSED_JSON_BODY.to_string());
-        format!("{prefix} with body {body}")
-    } else {
-        // Method and URL are sufficient request context. Raw Request formatting
-        // may expose headers or a non-JSON body, so it is never a logging fallback.
-        prefix
-    }
-}
 
 /// Configuration for TLS/mTLS settings.
 #[derive(Clone, Debug)]
@@ -894,9 +839,22 @@ impl<S: HttpSend> RestfulLanceDbClient<S> {
         }
     }
 
-    pub(crate) fn log_request(&self, request: &Request, request_id: &str) {
+    pub(crate) fn log_request(&self, request: &Request, request_id: &String) {
         if log::log_enabled!(log::Level::Debug) {
-            debug!("{}", request_log_message(request, request_id));
+            let content_type = request
+                .headers()
+                .get("content-type")
+                .map(|v| v.to_str().unwrap());
+            if content_type == Some("application/json") {
+                let body = request.body().as_ref().unwrap().as_bytes().unwrap();
+                let body = String::from_utf8_lossy(body);
+                debug!(
+                    "Sending request_id={}: {:?} with body {}",
+                    request_id, request, body
+                );
+            } else {
+                debug!("Sending request_id={}: {:?}", request_id, request);
+            }
         }
     }
 
@@ -1119,49 +1077,6 @@ mod tests {
         ENV_MUTEX.lock().unwrap_or_else(|e| e.into_inner())
     }
 
-    #[test]
-    fn test_request_log_message_redacts_secrets_and_never_formats_raw_requests() {
-        const SECRET_SENTINEL: &str = "udf-secret-log-sentinel-7e4e";
-        const MALFORMED_SENTINEL: &str = "malformed-secret-log-sentinel-b652";
-        const NON_JSON_SENTINEL: &str = "non-json-secret-log-sentinel-7fd1";
-
-        let request = reqwest::Client::new()
-            .post("https://example.com/v1/functions/create")
-            .json(&serde_json::json!({
-                "name": "uses_secret",
-                "nested": {
-                    "secret_values": {"OPENAI_API_KEY": SECRET_SENTINEL},
-                    "safe": "visible-value"
-                }
-            }))
-            .build()
-            .unwrap();
-        let log_message = request_log_message(&request, "valid-json");
-
-        let malformed_request = reqwest::Client::new()
-            .post("https://example.com/v1/functions/create")
-            .header("content-type", "application/json; charset=utf-8")
-            .body(format!(r#"{{"secret_values":"{MALFORMED_SENTINEL}""#))
-            .build()
-            .unwrap();
-        let malformed_log_message = request_log_message(&malformed_request, "malformed-json");
-
-        let non_json_request = reqwest::Client::new()
-            .post("https://example.com/v1/functions/create")
-            .header("content-type", "text/plain")
-            .body(NON_JSON_SENTINEL)
-            .build()
-            .unwrap();
-        let non_json_log_message = request_log_message(&non_json_request, "non-json");
-
-        assert!(log_message.contains("visible-value"));
-        assert!(log_message.contains(REDACTED_JSON_VALUE));
-        assert!(!log_message.contains(SECRET_SENTINEL));
-        assert!(malformed_log_message.contains(SUPPRESSED_JSON_BODY));
-        assert!(!malformed_log_message.contains(MALFORMED_SENTINEL));
-        assert!(!non_json_log_message.contains(NON_JSON_SENTINEL));
-    }
-
     #[test]
     fn test_timeout_config_default() {
         let config = TimeoutConfig::default();

rust/lancedb/src/remote/db.rs

@@ -554,7 +554,6 @@ impl<S: HttpSend> Database for RemoteDatabase<S> {
         &self,
         request: FunctionRegistrationRequest,
     ) -> Result<Job<FunctionVersion>> {
-        request.validate_secret_values()?;
         let req = self.client.post("/v1/functions/create").json(&request);
         let (request_id, response) = self.client.send(req).await?;
         let response = self.client.check_response(&request_id, response).await?;
@@ -2643,8 +2642,7 @@ mod tests {
         );
         const FUNCTION_JOB: &str =
             include_str!("../../tests/fixtures/first_class_functions/v1/remote_function_job.json");
-        let mut expected: serde_json::Value = serde_json::from_str(REQUEST).unwrap();
-        expected["secret_values"] = serde_json::json!({"API_TOKEN": "secret-value"});
+        let expected: serde_json::Value = serde_json::from_str(REQUEST).unwrap();
         let conn = Connection::new_with_handler(move |request| match request.url().path() {
             "/v1/functions/create" => {
                 assert_eq!(request.method(), &reqwest::Method::POST);
@@ -2662,10 +2660,7 @@ mod tests {
                 .unwrap(),
             path => panic!("unexpected path: {path}"),
         });
-        let mut request = crate::function::FunctionRegistrationRequest::from_json(REQUEST).unwrap();
-        request
-            .secret_values
-            .insert("API_TOKEN".to_string(), "secret-value".to_string());
+        let request = crate::function::FunctionRegistrationRequest::from_json(REQUEST).unwrap();
         let job = conn.create_function_async(request).await.unwrap();
         assert_eq!(job.id(), Some("job-function-1"));
         let version = job.wait().await.unwrap();
@@ -2673,32 +2668,6 @@ mod tests {
         assert_eq!(version.version(), "fv_01K3EXACT");
     }
 
-    #[tokio::test]
-    async fn test_create_function_async_validates_secrets_before_serialization_and_send() {
-        const REQUEST: &str = include_str!(
-            "../../tests/fixtures/first_class_functions/v1/remote_function_registration_request.json"
-        );
-        let sends = Arc::new(AtomicUsize::new(0));
-        let sends_ref = sends.clone();
-        let conn = Connection::new_with_handler(move |_| {
-            sends_ref.fetch_add(1, Ordering::SeqCst);
-            http::Response::builder().status(500).body("").unwrap()
-        });
-        let mut request = crate::function::FunctionRegistrationRequest::from_json(REQUEST).unwrap();
-        request.secret_values.insert(
-            "API_TOKEN".to_string(),
-            "x".repeat(crate::function::MAX_FUNCTION_SECRET_VALUE_BYTES + 1),
-        );
-
-        let error = conn.create_function_async(request).await.unwrap_err();
-
-        assert!(matches!(
-            error,
-            Error::InvalidInput { message } if message.contains("65536-byte limit")
-        ));
-        assert_eq!(sends.load(Ordering::SeqCst), 0);
-    }
-
     #[tokio::test]
     async fn test_get_function_requires_and_sends_exact_version() {
         const VERSION: &str = include_str!(

rust/lancedb/src/table.rs

@@ -103,7 +103,9 @@ pub use lance::dataset::refs::{BranchContents, Ref, TagContents, Tags as LanceTa
 pub use lance::dataset::scanner::DatasetRecordBatchStream;
 pub use lance_index::optimize::OptimizeOptions;
 pub use lsm_stats::{BucketStats, GenerationStats, LsmStats, MemtableStats};
-pub use optimize::{CompactionOptions, OptimizeAction, OptimizeStats};
+pub use optimize::{
+    CompactionMode, CompactionOptions, IndexRemapMode, OptimizeAction, OptimizeStats,
+};
 pub use refresh::RefreshColumnResult;
 pub use schema_evolution::{
     AddColumnsResult, AlterColumnsResult, DropColumnsResult, FieldMetadataUpdate,

rust/lancedb/src/table/datafusion/blob_coerce.rs

@@ -36,14 +36,6 @@ pub(super) fn coerce_blob_expr(
     };
 
     let input_shape = match input_field.data_type() {
-        DataType::Null => {
-            let expr: Arc<dyn PhysicalExpr> = Arc::new(CastExpr::new(
-                input_expr,
-                table_field.data_type().clone(),
-                None,
-            ));
-            return Ok((expr, table_field.clone()));
-        }
         DataType::Binary | DataType::LargeBinary | DataType::BinaryView => BlobInputShape::Bytes,
         DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View => BlobInputShape::String,
         DataType::Struct(children) => {
@@ -163,7 +155,7 @@ mod tests {
     use crate::blob::blob;
     use arrow_array::{
         Array, ArrayRef, BinaryArray, BinaryViewArray, Int32Array, Int64Array, LargeBinaryArray,
-        NullArray, RecordBatch, StringArray, StringViewArray, StructArray, UInt8Array, UInt64Array,
+        RecordBatch, StringArray, StringViewArray, StructArray, UInt8Array, UInt64Array,
     };
     use arrow_schema::Schema;
     use datafusion::prelude::SessionContext;
@@ -287,18 +279,6 @@ mod tests {
         assert_eq!(data.value(0), b"view");
     }
 
-    #[tokio::test]
-    async fn null_column_coerces_to_all_null_blob_struct() {
-        let batch = batch_with_image(
-            Field::new("image", DataType::Null, true),
-            Arc::new(NullArray::new(2)),
-        );
-        let coerced = coerce(batch, &blob_table_schema()).await;
-        let image = image_struct(&coerced);
-        assert!(image.is_null(0));
-        assert!(image.is_null(1));
-    }
-
     #[tokio::test]
     async fn binary_nulls_stay_null_after_coercion() {
         let batch = batch_with_image(

rust/lancedb/src/table/optimize.rs

@@ -15,7 +15,7 @@ use lance_index::optimize::OptimizeOptions;
 use log::info;
 
 pub use chrono::Duration;
-pub use lance::dataset::optimize::CompactionOptions;
+pub use lance::dataset::optimize::{CompactionMode, CompactionOptions, IndexRemapMode};
 
 use super::NativeTable;
 use crate::error::Result;

rust/lancedb/tests/first_class_function_slice1.rs

@@ -20,25 +20,6 @@ fn job_result(name: &str) -> Value {
     serde_json::from_str::<Value>(&fixture(name)).expect("remote Job fixture")["result"].clone()
 }
 
-fn assert_no_secret_values(value: &Value) {
-    match value {
-        Value::Object(values) => {
-            for (key, value) in values {
-                assert!(
-                    !matches!(
-                        key.as_str(),
-                        "secret_value" | "secret_values" | "resolved_secret" | "resolved_secrets"
-                    ),
-                    "client canonical value must not model resolved secret material"
-                );
-                assert_no_secret_values(value);
-            }
-        }
-        Value::Array(values) => values.iter().for_each(assert_no_secret_values),
-        _ => {}
-    }
-}
-
 #[test]
 fn function_version_job_result_matches_shared_canonical_golden() {
     let result = job_result("remote_function_job.json");
@@ -47,7 +28,6 @@ fn function_version_job_result_matches_shared_canonical_golden() {
     assert_eq!(version.name(), "embed");
     assert_eq!(version.version(), "fv_01K3EXACT");
     assert_eq!(version.runtime_digest(), "sha256:runtime");
-    assert_eq!(version.required_secrets(), &["HF_TOKEN"]);
     assert_eq!(
         version.to_canonical_json().expect("canonical JSON"),
         fixture("remote_function_version.canonical.json").trim()
@@ -162,21 +142,3 @@ fn floating_point_application_literals_are_rejected_consistently() {
             .contains("floating-point Function literals")
     );
 }
-
-#[test]
-fn canonical_client_values_contain_secret_names_only() {
-    let result = job_result("remote_function_job.json");
-    let version = FunctionVersion::from_json(&result.to_string()).expect("FunctionVersion result");
-    let canonical: Value = serde_json::from_str(
-        &version
-            .to_canonical_json()
-            .expect("canonical FunctionVersion"),
-    )
-    .expect("canonical JSON");
-
-    assert_eq!(
-        canonical["required_secrets"],
-        serde_json::json!(["HF_TOKEN"])
-    );
-    assert_no_secret_values(&canonical);
-}

rust/lancedb/tests/first_class_function_slice2.rs

@@ -6,7 +6,6 @@ use std::path::PathBuf;
 
 use lancedb::Error;
 use lancedb::function::FunctionRegistrationRequest;
-use serde_json::Value;
 
 fn fixture(name: &str) -> String {
     let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
@@ -15,25 +14,6 @@ fn fixture(name: &str) -> String {
     fs::read_to_string(path).expect("fixture must be readable")
 }
 
-fn assert_no_secret_values(value: &Value) {
-    match value {
-        Value::Object(values) => {
-            for (key, value) in values {
-                assert!(
-                    !matches!(
-                        key.as_str(),
-                        "secret_value" | "secret_values" | "resolved_secret" | "resolved_secrets"
-                    ),
-                    "registration requests must not model resolved secret material"
-                );
-                assert_no_secret_values(value);
-            }
-        }
-        Value::Array(values) => values.iter().for_each(assert_no_secret_values),
-        _ => {}
-    }
-}
-
 #[test]
 fn registration_request_matches_shared_canonical_golden() {
     let request = FunctionRegistrationRequest::from_json(&fixture(
@@ -42,33 +22,10 @@ fn registration_request_matches_shared_canonical_golden() {
     .expect("registration request");
     assert_eq!(request.name, "normalize_score");
     assert_eq!(request.artifact.adapter.kind, "scalar_to_arrow_batch");
-    assert_eq!(request.required_secrets, ["API_TOKEN"]);
-    assert!(request.secret_values.is_empty());
     assert_eq!(
         request.to_canonical_json().expect("canonical request"),
         fixture("remote_function_registration_request.canonical.json").trim()
     );
-
-    let value: Value =
-        serde_json::from_str(&request.to_canonical_json().expect("canonical request"))
-            .expect("request JSON");
-    assert_no_secret_values(&value);
-}
-
-#[test]
-fn registration_request_serializes_secret_values_but_redacts_debug_output() {
-    let mut value: Value =
-        serde_json::from_str(&fixture("remote_function_registration_request.json")).unwrap();
-    value["secret_values"] = serde_json::json!({"API_TOKEN": "secret-plaintext"});
-    let request = FunctionRegistrationRequest::from_json(&value.to_string()).unwrap();
-
-    assert_eq!(request.secret_values["API_TOKEN"], "secret-plaintext");
-    let canonical = request.to_canonical_json().unwrap();
-    assert!(canonical.contains("secret-plaintext"));
-    let debug = format!("{request:?}");
-    assert!(debug.contains("API_TOKEN"));
-    assert!(debug.contains("[REDACTED]"));
-    assert!(!debug.contains("secret-plaintext"));
 }
 
 #[tokio::test]

rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_job.json

@@ -24,7 +24,6 @@
     },
     "runtime_digest": "sha256:runtime",
     "environment_digest": "sha256:environment",
-    "required_secrets": ["HF_TOKEN"],
     "created_at": "2026-08-21T00:00:00Z"
   },
   "future_job": {"trace_id": "trace-1"}

rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_registration_request.canonical.json

@@ -1 +1 @@
-{"artifact":{"adapter":{"kind":"scalar_to_arrow_batch","version":1},"content":{"data":"ZnJvbSBfX2Z1dHVyZV9fIGltcG9ydCBhbm5vdGF0aW9ucwoKZGVmIG5vcm1hbGl6ZV9zY29yZSh2YWx1ZTogZmxvYXQpIC0+IGZsb2F0OgogICAgcmV0dXJuIHZhbHVlIC8gMTAwLjAK","encoding":"base64"},"digest":"sha256:760784bdcef57b802f389b97804cc0b618aae39e86044733451bcd0b13089a7f","entrypoint":"normalize_score","kind":"python_callable"},"name":"normalize_score","required_secrets":["API_TOKEN"],"runtime":{"env":{"MODE":"test"},"environment":{"kind":"pip","packages":["numpy>=2"]},"kind":"python","python_version":"3.12"},"signature":{"inputs":[{"arrow_type":"float64","name":"value","nullable":false}],"output":{"arrow_type":"float64","kind":"scalar","nullable":false}}}
+{"artifact":{"adapter":{"kind":"scalar_to_arrow_batch","version":1},"content":{"data":"ZnJvbSBfX2Z1dHVyZV9fIGltcG9ydCBhbm5vdGF0aW9ucwoKZGVmIG5vcm1hbGl6ZV9zY29yZSh2YWx1ZTogZmxvYXQpIC0+IGZsb2F0OgogICAgcmV0dXJuIHZhbHVlIC8gMTAwLjAK","encoding":"base64"},"digest":"sha256:760784bdcef57b802f389b97804cc0b618aae39e86044733451bcd0b13089a7f","entrypoint":"normalize_score","kind":"python_callable"},"name":"normalize_score","runtime":{"env":{"MODE":"test"},"environment":{"kind":"pip","packages":["numpy>=2"]},"kind":"python","python_version":"3.12"},"signature":{"inputs":[{"arrow_type":"float64","name":"value","nullable":false}],"output":{"arrow_type":"float64","kind":"scalar","nullable":false}}}

rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_registration_request.json

@@ -39,8 +39,5 @@
     "env": {
       "MODE": "test"
     }
-  },
-  "required_secrets": [
-    "API_TOKEN"
-  ]
+  }
 }

rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_version.canonical.json

@@ -1 +1 @@
-{"artifact":{"digest":"sha256:code","entrypoint":"embed","kind":"python_callable"},"created_at":"2026-08-21T00:00:00Z","environment_digest":"sha256:environment","name":"embed","required_secrets":["HF_TOKEN"],"runtime":{"env":{"TOKENIZERS_PARALLELISM":"false"},"environment":{"kind":"pip","packages":["sentence-transformers>=3"]},"kind":"python","python_version":"3.12"},"runtime_digest":"sha256:runtime","signature":{"inputs":[{"arrow_type":"utf8","name":"text","nullable":true}],"output":{"arrow_type":"list<float32>","kind":"scalar","nullable":false}},"version":"fv_01K3EXACT"}
+{"artifact":{"digest":"sha256:code","entrypoint":"embed","kind":"python_callable"},"created_at":"2026-08-21T00:00:00Z","environment_digest":"sha256:environment","name":"embed","runtime":{"env":{"TOKENIZERS_PARALLELISM":"false"},"environment":{"kind":"pip","packages":["sentence-transformers>=3"]},"kind":"python","python_version":"3.12"},"runtime_digest":"sha256:runtime","signature":{"inputs":[{"arrow_type":"utf8","name":"text","nullable":true}],"output":{"arrow_type":"list<float32>","kind":"scalar","nullable":false}},"version":"fv_01K3EXACT"}