Revert "fix: pass storage_options in LanceNamespaceDatabase::create_table"

This reverts commit 1078983993.

.github/workflows/build_linux_wheel/action.yml

@@ -3,7 +3,7 @@ name: build-linux-wheel
 description: "Build a manylinux wheel for lance"
 inputs:
   python-minor-version:
-    description: "10, 11, 12, 13"
+    description: "8, 9, 10, 11, 12"
     required: true
   args:
     description: "--release"

.github/workflows/build_mac_wheel/action.yml

@@ -3,7 +3,7 @@ name: build_wheel
 description: "Build a lance wheel"
 inputs:
   python-minor-version:
-    description: "10, 11, 12, 13"
+    description: "8, 9, 10, 11"
     required: true
   args:
     description: "--release"

.github/workflows/build_windows_wheel/action.yml

@@ -3,7 +3,7 @@ name: build_wheel
 description: "Build a lance wheel"
 inputs:
   python-minor-version:
-    description: "10, 11, 12, 13, 14"
+    description: "8, 9, 10, 11"
     required: true
   args:
     description: "--release"

.github/workflows/docs.yml

@@ -41,7 +41,7 @@ jobs:
           sudo apt install -y protobuf-compiler libssl-dev
           rustup update && rustup default
       - name: Set up Python
-        uses: actions/setup-python@v6
+        uses: actions/setup-python@v5
         with:
           python-version: "3.10"
           cache: "pip"

.github/workflows/pypi-publish.yml

@@ -44,12 +44,12 @@ jobs:
           fetch-depth: 0
           lfs: true
       - name: Set up Python
-        uses: actions/setup-python@v6
+        uses: actions/setup-python@v4
         with:
-          python-version: "3.10"
+          python-version: 3.8
       - uses: ./.github/workflows/build_linux_wheel
         with:
-          python-minor-version: 10
+          python-minor-version: 8
           args: "--release --strip ${{ matrix.config.extra_args }}"
           arm-build: ${{ matrix.config.platform == 'aarch64' }}
           manylinux: ${{ matrix.config.manylinux }}
@@ -74,12 +74,12 @@ jobs:
           fetch-depth: 0
           lfs: true
       - name: Set up Python
-        uses: actions/setup-python@v6
+        uses: actions/setup-python@v4
         with:
-          python-version: "3.13"
+          python-version: 3.12
       - uses: ./.github/workflows/build_mac_wheel
         with:
-          python-minor-version: 10
+          python-minor-version: 8
           args: "--release --strip --target ${{ matrix.config.target }} --features fp16kernels"
       - uses: ./.github/workflows/upload_wheel
         if: startsWith(github.ref, 'refs/tags/python-v')
@@ -95,12 +95,12 @@ jobs:
           fetch-depth: 0
           lfs: true
       - name: Set up Python
-        uses: actions/setup-python@v6
+        uses: actions/setup-python@v4
         with:
-          python-version: "3.13"
+          python-version: 3.12
       - uses: ./.github/workflows/build_windows_wheel
         with:
-          python-minor-version: 10
+          python-minor-version: 8
           args: "--release --strip"
           vcpkg_token: ${{ secrets.VCPKG_GITHUB_PACKAGES }}
       - uses: ./.github/workflows/upload_wheel

.github/workflows/python.yml

@@ -25,7 +25,7 @@ jobs:
   lint:
     name: "Lint"
     timeout-minutes: 30
-    runs-on: "ubuntu-24.04"
+    runs-on: "ubuntu-22.04"
     defaults:
       run:
         shell: bash
@@ -36,9 +36,9 @@ jobs:
           fetch-depth: 0
           lfs: true
       - name: Set up Python
-        uses: actions/setup-python@v6
+        uses: actions/setup-python@v5
         with:
-          python-version: "3.13"
+          python-version: "3.12"
       - name: Install ruff
         run: |
           pip install ruff==0.9.9
@@ -61,9 +61,9 @@ jobs:
           fetch-depth: 0
           lfs: true
       - name: Set up Python
-        uses: actions/setup-python@v6
+        uses: actions/setup-python@v5
         with:
-          python-version: "3.13"
+          python-version: "3.12"
       - name: Install protobuf compiler
         run: |
           sudo apt update
@@ -90,9 +90,9 @@ jobs:
           fetch-depth: 0
           lfs: true
       - name: Set up Python
-        uses: actions/setup-python@v6
+        uses: actions/setup-python@v5
         with:
-          python-version: "3.13"
+          python-version: "3.12"
           cache: "pip"
       - name: Install protobuf
         run: |
@@ -110,7 +110,7 @@ jobs:
     timeout-minutes: 30
     strategy:
       matrix:
-        python-minor-version: ["10", "13"]
+        python-minor-version: ["9", "12"]
     runs-on: "ubuntu-24.04"
     defaults:
       run:
@@ -126,7 +126,7 @@ jobs:
           sudo apt update
           sudo apt install -y protobuf-compiler
       - name: Set up Python
-        uses: actions/setup-python@v6
+        uses: actions/setup-python@v5
         with:
           python-version: 3.${{ matrix.python-minor-version }}
       - uses: ./.github/workflows/build_linux_wheel
@@ -156,9 +156,9 @@ jobs:
           fetch-depth: 0
           lfs: true
       - name: Set up Python
-        uses: actions/setup-python@v6
+        uses: actions/setup-python@v5
         with:
-          python-version: "3.13"
+          python-version: "3.12"
       - uses: ./.github/workflows/build_mac_wheel
         with:
           args: --profile ci
@@ -185,9 +185,9 @@ jobs:
           fetch-depth: 0
           lfs: true
       - name: Set up Python
-        uses: actions/setup-python@v6
+        uses: actions/setup-python@v5
         with:
-          python-version: "3.13"
+          python-version: "3.12"
       - uses: ./.github/workflows/build_windows_wheel
         with:
           args: --profile ci
@@ -195,7 +195,7 @@ jobs:
       # Make sure wheels are not included in the Rust cache
       - name: Delete wheels
         run: rm -rf target/wheels
-  min-deps:
+  pydantic1x:
     timeout-minutes: 30
     runs-on: "ubuntu-24.04"
     defaults:
@@ -212,11 +212,12 @@ jobs:
           sudo apt update
           sudo apt install -y protobuf-compiler
       - name: Set up Python
-        uses: actions/setup-python@v6
+        uses: actions/setup-python@v5
         with:
-          python-version: "3.10"
+          python-version: 3.9
       - name: Install lancedb
         run: |
+          pip install "pydantic<2"
           pip install pyarrow==16
           pip install --extra-index-url https://pypi.fury.io/lance-format/ --extra-index-url https://pypi.fury.io/lancedb/ -e .[tests]
           pip install tantivy

nodejs/lancedb/table.ts

@@ -538,6 +538,35 @@ export abstract class Table {
    *
    */
   abstract stats(): Promise<TableStatistics>;
+
+  /**
+   * Get the initial storage options that were passed in when opening this table.
+   *
+   * For dynamically refreshed options (e.g., credential vending), use
+   * {@link Table.latestStorageOptions}.
+   *
+   * Warning: This is an internal API and the return value is subject to change.
+   *
+   * @returns The storage options, or undefined if no storage options were configured.
+   */
+  abstract initialStorageOptions(): Promise<
+    Record<string, string> | null | undefined
+  >;
+
+  /**
+   * Get the latest storage options, refreshing from provider if configured.
+   *
+   * This method is useful for credential vending scenarios where storage options
+   * may be refreshed dynamically. If no dynamic provider is configured, this
+   * returns the initial static options.
+   *
+   * Warning: This is an internal API and the return value is subject to change.
+   *
+   * @returns The storage options, or undefined if no storage options were configured.
+   */
+  abstract latestStorageOptions(): Promise<
+    Record<string, string> | null | undefined
+  >;
 }
 
 export class LocalTable extends Table {
@@ -858,6 +887,18 @@ export class LocalTable extends Table {
     return await this.inner.stats();
   }
 
+  async initialStorageOptions(): Promise<
+    Record<string, string> | null | undefined
+  > {
+    return await this.inner.initialStorageOptions();
+  }
+
+  async latestStorageOptions(): Promise<
+    Record<string, string> | null | undefined
+  > {
+    return await this.inner.latestStorageOptions();
+  }
+
   mergeInsert(on: string | string[]): MergeInsertBuilder {
     on = Array.isArray(on) ? on : [on];
     return new MergeInsertBuilder(this.inner.mergeInsert(on), this.schema());

nodejs/src/table.rs

@@ -166,6 +166,19 @@ impl Table {
         Ok(stats.into())
     }
 
+    #[napi(catch_unwind)]
+    pub async fn initial_storage_options(&self) -> napi::Result<Option<HashMap<String, String>>> {
+        Ok(self.inner_ref()?.initial_storage_options().await)
+    }
+
+    #[napi(catch_unwind)]
+    pub async fn latest_storage_options(&self) -> napi::Result<Option<HashMap<String, String>>> {
+        self.inner_ref()?
+            .latest_storage_options()
+            .await
+            .default_error()
+    }
+
     #[napi(catch_unwind)]
     pub async fn update(
         &self,

python/CONTRIBUTING.md

@@ -16,7 +16,7 @@ The Python package is a wrapper around the Rust library, `lancedb`. We use
 
 To set up your development environment, you will need to install the following:
 
-1. Python 3.10 or later
+1. Python 3.9 or later
 2. Cargo (Rust's package manager). Use [rustup](https://rustup.rs/) to install.
 3. [protoc](https://grpc.io/docs/protoc-installation/) (Protocol Buffers compiler)
 

python/Cargo.toml

@@ -21,7 +21,7 @@ lance-core.workspace = true
 lance-namespace.workspace = true
 lance-io.workspace = true
 env_logger.workspace = true
-pyo3 = { version = "0.25", features = ["extension-module", "abi3-py310"] }
+pyo3 = { version = "0.25", features = ["extension-module", "abi3-py39"] }
 pyo3-async-runtimes = { version = "0.25", features = [
     "attributes",
     "tokio-runtime",
@@ -34,7 +34,7 @@ tokio = { version = "1.40", features = ["sync"] }
 [build-dependencies]
 pyo3-build-config = { version = "0.25", features = [
     "extension-module",
-    "abi3-py310",
+    "abi3-py39",
 ] }
 
 [features]

python/pyproject.toml

@@ -8,7 +8,7 @@ dependencies = [
     "overrides>=0.7; python_version<'3.12'",
     "packaging",
     "pyarrow>=16",
-    "pydantic>=2",
+    "pydantic>=1.10",
     "tqdm>=4.27.0",
     "lance-namespace>=0.3.2"
 ]
@@ -16,7 +16,7 @@ description = "lancedb"
 authors = [{ name = "LanceDB Devs", email = "dev@lancedb.com" }]
 license = { file = "LICENSE" }
 readme = "README.md"
-requires-python = ">=3.10"
+requires-python = ">=3.9"
 keywords = [
     "data-format",
     "data-science",
@@ -33,10 +33,10 @@ classifiers = [
     "Programming Language :: Python",
     "Programming Language :: Python :: 3",
     "Programming Language :: Python :: 3 :: Only",
+    "Programming Language :: Python :: 3.9",
     "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
     "Programming Language :: Python :: 3.12",
-    "Programming Language :: Python :: 3.13",
     "Topic :: Scientific/Engineering",
 ]
 
@@ -137,4 +137,4 @@ include = [
     "python/lancedb/_lancedb.pyi",
 ]
 exclude = ["python/tests/"]
-pythonVersion = "3.13"
+pythonVersion = "3.12"

python/python/lancedb/_lancedb.pyi

@@ -180,6 +180,8 @@ class Table:
         delete_unverified: Optional[bool] = None,
     ) -> OptimizeStats: ...
     async def uri(self) -> str: ...
+    async def initial_storage_options(self) -> Optional[Dict[str, str]]: ...
+    async def latest_storage_options(self) -> Optional[Dict[str, str]]: ...
     @property
     def tags(self) -> Tags: ...
     def query(self) -> Query: ...

python/python/lancedb/pydantic.py

@@ -6,6 +6,7 @@
 from __future__ import annotations
 
 import inspect
+import sys
 import types
 from abc import ABC, abstractmethod
 from datetime import date, datetime
@@ -140,6 +141,14 @@ def Vector(
                 raise TypeError("A list of numbers or numpy.ndarray is needed")
             return cls(v)
 
+        if PYDANTIC_VERSION.major < 2:
+
+            @classmethod
+            def __modify_schema__(cls, field_schema: Dict[str, Any]):
+                field_schema["items"] = {"type": "number"}
+                field_schema["maxItems"] = dim
+                field_schema["minItems"] = dim
+
     return FixedSizeList
 
 
@@ -217,14 +226,26 @@ def MultiVector(
         def __get_validators__(cls) -> Generator[Callable, None, None]:
             yield cls.validate
 
+        # For pydantic v1
         @classmethod
-        def __modify_schema__(cls, field_schema: Dict[str, Any]):
-            field_schema["items"] = {
-                "type": "array",
-                "items": {"type": "number"},
-                "minItems": dim,
-                "maxItems": dim,
-            }
+        def validate(cls, v):
+            if not isinstance(v, (list, range)):
+                raise TypeError("A list of vectors is needed")
+            for vec in v:
+                if not isinstance(vec, (list, range, np.ndarray)) or len(vec) != dim:
+                    raise TypeError(f"Each vector must be a list of {dim} numbers")
+            return cls(v)
+
+        if PYDANTIC_VERSION.major < 2:
+
+            @classmethod
+            def __modify_schema__(cls, field_schema: Dict[str, Any]):
+                field_schema["items"] = {
+                    "type": "array",
+                    "items": {"type": "number"},
+                    "minItems": dim,
+                    "maxItems": dim,
+                }
 
     return MultiVectorList
 
@@ -254,31 +275,35 @@ def _py_type_to_arrow_type(py_type: Type[Any], field: FieldInfo) -> pa.DataType:
         return pa.timestamp("us", tz=tz)
     elif getattr(py_type, "__origin__", None) in (list, tuple):
         child = py_type.__args__[0]
-        return _pydantic_list_child_to_arrow(child, field)
+        return pa.list_(_py_type_to_arrow_type(child, field))
     raise TypeError(
         f"Converting Pydantic type to Arrow Type: unsupported type {py_type}."
     )
 
 
-def _pydantic_model_to_fields(model: pydantic.BaseModel) -> List[pa.Field]:
-    return [
-        _pydantic_to_field(name, field) for name, field in model.model_fields.items()
-    ]
+if PYDANTIC_VERSION.major < 2:
+
+    def _pydantic_model_to_fields(model: pydantic.BaseModel) -> List[pa.Field]:
+        return [
+            _pydantic_to_field(name, field) for name, field in model.__fields__.items()
+        ]
+
+else:
+
+    def _pydantic_model_to_fields(model: pydantic.BaseModel) -> List[pa.Field]:
+        return [
+            _pydantic_to_field(name, field)
+            for name, field in model.model_fields.items()
+        ]
 
 
 def _pydantic_type_to_arrow_type(tp: Any, field: FieldInfo) -> pa.DataType:
-    def _safe_issubclass(candidate: Any, base: type) -> bool:
-        try:
-            return issubclass(candidate, base)
-        except TypeError:
-            return False
-
     if inspect.isclass(tp):
-        if _safe_issubclass(tp, pydantic.BaseModel):
+        if issubclass(tp, pydantic.BaseModel):
             # Struct
             fields = _pydantic_model_to_fields(tp)
             return pa.struct(fields)
-        if _safe_issubclass(tp, FixedSizeListMixin):
+        if issubclass(tp, FixedSizeListMixin):
             if getattr(tp, "is_multi_vector", lambda: False)():
                 return pa.list_(pa.list_(tp.value_arrow_type(), tp.dim()))
             # For regular Vector
@@ -286,67 +311,45 @@ def _pydantic_type_to_arrow_type(tp: Any, field: FieldInfo) -> pa.DataType:
     return _py_type_to_arrow_type(tp, field)
 
 
-def _pydantic_list_child_to_arrow(child: Any, field: FieldInfo) -> pa.DataType:
-    unwrapped = _unwrap_optional_annotation(child)
-    if unwrapped is not None:
-        return pa.list_(
-            pa.field("item", _pydantic_type_to_arrow_type(unwrapped, field), True)
-        )
-    return pa.list_(_pydantic_type_to_arrow_type(child, field))
-
-
-def _unwrap_optional_annotation(annotation: Any) -> Any | None:
-    if isinstance(annotation, (_GenericAlias, GenericAlias)):
-        origin = annotation.__origin__
-        args = annotation.__args__
-        if origin == Union:
-            non_none = [arg for arg in args if arg is not type(None)]
-            if len(non_none) == 1 and len(non_none) != len(args):
-                return non_none[0]
-    elif isinstance(annotation, types.UnionType):
-        args = annotation.__args__
-        non_none = [arg for arg in args if arg is not type(None)]
-        if len(non_none) == 1 and len(non_none) != len(args):
-            return non_none[0]
-    return None
-
-
 def _pydantic_to_arrow_type(field: FieldInfo) -> pa.DataType:
     """Convert a Pydantic FieldInfo to Arrow DataType"""
-    unwrapped = _unwrap_optional_annotation(field.annotation)
-    if unwrapped is not None:
-        return _pydantic_type_to_arrow_type(unwrapped, field)
     if isinstance(field.annotation, (_GenericAlias, GenericAlias)):
         origin = field.annotation.__origin__
         args = field.annotation.__args__
 
         if origin is list:
             child = args[0]
-            return _pydantic_list_child_to_arrow(child, field)
+            return pa.list_(_py_type_to_arrow_type(child, field))
+        elif origin == Union:
+            if len(args) == 2 and args[1] is type(None):
+                return _pydantic_type_to_arrow_type(args[0], field)
+    elif sys.version_info >= (3, 10) and isinstance(field.annotation, types.UnionType):
+        args = field.annotation.__args__
+        if len(args) == 2:
+            for typ in args:
+                if typ is type(None):
+                    continue
+                return _py_type_to_arrow_type(typ, field)
     return _pydantic_type_to_arrow_type(field.annotation, field)
 
 
 def is_nullable(field: FieldInfo) -> bool:
     """Check if a Pydantic FieldInfo is nullable."""
-    if _unwrap_optional_annotation(field.annotation) is not None:
-        return True
     if isinstance(field.annotation, (_GenericAlias, GenericAlias)):
         origin = field.annotation.__origin__
         args = field.annotation.__args__
         if origin == Union:
-            if any(typ is type(None) for typ in args):
+            if len(args) == 2 and args[1] is type(None):
                 return True
-    elif isinstance(field.annotation, types.UnionType):
+    elif sys.version_info >= (3, 10) and isinstance(field.annotation, types.UnionType):
         args = field.annotation.__args__
         for typ in args:
             if typ is type(None):
                 return True
-    elif inspect.isclass(field.annotation):
-        try:
-            if issubclass(field.annotation, FixedSizeListMixin):
-                return field.annotation.nullable()
-        except TypeError:
-            return False
+    elif inspect.isclass(field.annotation) and issubclass(
+        field.annotation, FixedSizeListMixin
+    ):
+        return field.annotation.nullable()
     return False
 
 
@@ -443,6 +446,8 @@ class LanceModel(pydantic.BaseModel):
 
     @classmethod
     def safe_get_fields(cls):
+        if PYDANTIC_VERSION.major < 2:
+            return cls.__fields__
         return cls.model_fields
 
     @classmethod
@@ -485,8 +490,18 @@ def get_extras(field_info: FieldInfo, key: str) -> Any:
     return (field_info.field_info.extra or {}).get("json_schema_extra", {}).get(key)
 
 
-def model_to_dict(model: pydantic.BaseModel) -> Dict[str, Any]:
-    """
-    Convert a Pydantic model to a dictionary.
-    """
-    return model.model_dump()
+if PYDANTIC_VERSION.major < 2:
+
+    def model_to_dict(model: pydantic.BaseModel) -> Dict[str, Any]:
+        """
+        Convert a Pydantic model to a dictionary.
+        """
+        return model.dict()
+
+else:
+
+    def model_to_dict(model: pydantic.BaseModel) -> Dict[str, Any]:
+        """
+        Convert a Pydantic model to a dictionary.
+        """
+        return model.model_dump()

python/python/lancedb/table.py

@@ -2222,6 +2222,37 @@ class LanceTable(Table):
     def uri(self) -> str:
         return LOOP.run(self._table.uri())
 
+    def initial_storage_options(self) -> Optional[Dict[str, str]]:
+        """Get the initial storage options that were passed in when opening this table.
+
+        For dynamically refreshed options (e.g., credential vending), use
+        :meth:`latest_storage_options`.
+
+        Warning: This is an internal API and the return value is subject to change.
+
+        Returns
+        -------
+        Optional[Dict[str, str]]
+            The storage options, or None if no storage options were configured.
+        """
+        return LOOP.run(self._table.initial_storage_options())
+
+    def latest_storage_options(self) -> Optional[Dict[str, str]]:
+        """Get the latest storage options, refreshing from provider if configured.
+
+        This method is useful for credential vending scenarios where storage options
+        may be refreshed dynamically. If no dynamic provider is configured, this
+        returns the initial static options.
+
+        Warning: This is an internal API and the return value is subject to change.
+
+        Returns
+        -------
+        Optional[Dict[str, str]]
+            The storage options, or None if no storage options were configured.
+        """
+        return LOOP.run(self._table.latest_storage_options())
+
     def create_scalar_index(
         self,
         column: str,
@@ -3624,6 +3655,37 @@ class AsyncTable:
         """
         return await self._inner.uri()
 
+    async def initial_storage_options(self) -> Optional[Dict[str, str]]:
+        """Get the initial storage options that were passed in when opening this table.
+
+        For dynamically refreshed options (e.g., credential vending), use
+        :meth:`latest_storage_options`.
+
+        Warning: This is an internal API and the return value is subject to change.
+
+        Returns
+        -------
+        Optional[Dict[str, str]]
+            The storage options, or None if no storage options were configured.
+        """
+        return await self._inner.initial_storage_options()
+
+    async def latest_storage_options(self) -> Optional[Dict[str, str]]:
+        """Get the latest storage options, refreshing from provider if configured.
+
+        This method is useful for credential vending scenarios where storage options
+        may be refreshed dynamically. If no dynamic provider is configured, this
+        returns the initial static options.
+
+        Warning: This is an internal API and the return value is subject to change.
+
+        Returns
+        -------
+        Optional[Dict[str, str]]
+            The storage options, or None if no storage options were configured.
+        """
+        return await self._inner.latest_storage_options()
+
     async def add(
         self,
         data: DATA,

python/python/tests/test_pydantic.py

@@ -2,6 +2,7 @@
 # SPDX-FileCopyrightText: Copyright The LanceDB Authors
 
 import json
+import sys
 from datetime import date, datetime
 from typing import List, Optional, Tuple
 
@@ -19,6 +20,10 @@ from pydantic import BaseModel
 from pydantic import Field
 
 
+@pytest.mark.skipif(
+    sys.version_info < (3, 9),
+    reason="using native type alias requires python3.9 or higher",
+)
 def test_pydantic_to_arrow():
     class StructModel(pydantic.BaseModel):
         a: str
@@ -78,6 +83,10 @@ def test_pydantic_to_arrow():
     assert schema == expect_schema
 
 
+@pytest.mark.skipif(
+    sys.version_info < (3, 10),
+    reason="using | type syntax requires python3.10 or higher",
+)
 def test_optional_types_py310():
     class TestModel(pydantic.BaseModel):
         a: str | None
@@ -96,233 +105,10 @@ def test_optional_types_py310():
     assert schema == expect_schema
 
 
-def test_optional_structs():
-    class SplitInfo(pydantic.BaseModel):
-        start_frame: int
-        end_frame: int
-
-    class TestModel(pydantic.BaseModel):
-        id: str
-        split: SplitInfo | None = None
-
-    schema = pydantic_to_schema(TestModel)
-
-    expect_schema = pa.schema(
-        [
-            pa.field("id", pa.utf8(), False),
-            pa.field(
-                "split",
-                pa.struct(
-                    [
-                        pa.field("start_frame", pa.int64(), False),
-                        pa.field("end_frame", pa.int64(), False),
-                    ]
-                ),
-                True,
-            ),
-        ]
-    )
-    assert schema == expect_schema
-
-
-def test_optional_struct_list_py310():
-    class SplitInfo(pydantic.BaseModel):
-        start_frame: int
-        end_frame: int
-
-    class TestModel(pydantic.BaseModel):
-        id: str
-        splits: list[SplitInfo] | None = None
-
-    schema = pydantic_to_schema(TestModel)
-
-    expect_schema = pa.schema(
-        [
-            pa.field("id", pa.utf8(), False),
-            pa.field(
-                "splits",
-                pa.list_(
-                    pa.struct(
-                        [
-                            pa.field("start_frame", pa.int64(), False),
-                            pa.field("end_frame", pa.int64(), False),
-                        ]
-                    )
-                ),
-                True,
-            ),
-        ]
-    )
-    assert schema == expect_schema
-
-
-def test_nested_struct_list():
-    class SplitInfo(pydantic.BaseModel):
-        start_frame: int
-        end_frame: int
-
-    class TestModel(pydantic.BaseModel):
-        id: str
-        splits: list[SplitInfo]
-
-    schema = pydantic_to_schema(TestModel)
-
-    expect_schema = pa.schema(
-        [
-            pa.field("id", pa.utf8(), False),
-            pa.field(
-                "splits",
-                pa.list_(
-                    pa.struct(
-                        [
-                            pa.field("start_frame", pa.int64(), False),
-                            pa.field("end_frame", pa.int64(), False),
-                        ]
-                    )
-                ),
-                False,
-            ),
-        ]
-    )
-    assert schema == expect_schema
-
-
-def test_nested_struct_list_optional():
-    class SplitInfo(pydantic.BaseModel):
-        start_frame: int
-        end_frame: int
-
-    class TestModel(pydantic.BaseModel):
-        id: str
-        splits: Optional[list[SplitInfo]] = None
-
-    schema = pydantic_to_schema(TestModel)
-
-    expect_schema = pa.schema(
-        [
-            pa.field("id", pa.utf8(), False),
-            pa.field(
-                "splits",
-                pa.list_(
-                    pa.struct(
-                        [
-                            pa.field("start_frame", pa.int64(), False),
-                            pa.field("end_frame", pa.int64(), False),
-                        ]
-                    )
-                ),
-                True,
-            ),
-        ]
-    )
-    assert schema == expect_schema
-
-
-def test_nested_struct_list_optional_items():
-    class SplitInfo(pydantic.BaseModel):
-        start_frame: int
-        end_frame: int
-
-    class TestModel(pydantic.BaseModel):
-        id: str
-        splits: list[Optional[SplitInfo]]
-
-    schema = pydantic_to_schema(TestModel)
-
-    expect_schema = pa.schema(
-        [
-            pa.field("id", pa.utf8(), False),
-            pa.field(
-                "splits",
-                pa.list_(
-                    pa.field(
-                        "item",
-                        pa.struct(
-                            [
-                                pa.field("start_frame", pa.int64(), False),
-                                pa.field("end_frame", pa.int64(), False),
-                            ]
-                        ),
-                        True,
-                    )
-                ),
-                False,
-            ),
-        ]
-    )
-    assert schema == expect_schema
-
-
-def test_nested_struct_list_optional_container_and_items():
-    class SplitInfo(pydantic.BaseModel):
-        start_frame: int
-        end_frame: int
-
-    class TestModel(pydantic.BaseModel):
-        id: str
-        splits: Optional[list[Optional[SplitInfo]]] = None
-
-    schema = pydantic_to_schema(TestModel)
-
-    expect_schema = pa.schema(
-        [
-            pa.field("id", pa.utf8(), False),
-            pa.field(
-                "splits",
-                pa.list_(
-                    pa.field(
-                        "item",
-                        pa.struct(
-                            [
-                                pa.field("start_frame", pa.int64(), False),
-                                pa.field("end_frame", pa.int64(), False),
-                            ]
-                        ),
-                        True,
-                    )
-                ),
-                True,
-            ),
-        ]
-    )
-    assert schema == expect_schema
-
-
-def test_nested_struct_list_optional_items_pep604():
-    class SplitInfo(pydantic.BaseModel):
-        start_frame: int
-        end_frame: int
-
-    class TestModel(pydantic.BaseModel):
-        id: str
-        splits: list[SplitInfo | None]
-
-    schema = pydantic_to_schema(TestModel)
-
-    expect_schema = pa.schema(
-        [
-            pa.field("id", pa.utf8(), False),
-            pa.field(
-                "splits",
-                pa.list_(
-                    pa.field(
-                        "item",
-                        pa.struct(
-                            [
-                                pa.field("start_frame", pa.int64(), False),
-                                pa.field("end_frame", pa.int64(), False),
-                            ]
-                        ),
-                        True,
-                    )
-                ),
-                False,
-            ),
-        ]
-    )
-    assert schema == expect_schema
-
-
+@pytest.mark.skipif(
+    sys.version_info > (3, 8),
+    reason="using native type alias requires python3.9 or higher",
+)
 def test_pydantic_to_arrow_py38():
     class StructModel(pydantic.BaseModel):
         a: str

python/src/table.rs

@@ -502,6 +502,20 @@ impl Table {
         future_into_py(self_.py(), async move { inner.uri().await.infer_error() })
     }
 
+    pub fn initial_storage_options(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
+        let inner = self_.inner_ref()?.clone();
+        future_into_py(self_.py(), async move {
+            Ok(inner.initial_storage_options().await)
+        })
+    }
+
+    pub fn latest_storage_options(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
+        let inner = self_.inner_ref()?.clone();
+        future_into_py(self_.py(), async move {
+            inner.latest_storage_options().await.infer_error()
+        })
+    }
+
     pub fn __repr__(&self) -> String {
         match &self.inner {
             None => format!("ClosedTable({})", self.name),

rust/lancedb/src/dataloader/permutation/builder.rs

@@ -19,7 +19,7 @@ use crate::{
         split::{SplitStrategy, Splitter, SPLIT_ID_COLUMN},
         util::{rename_column, TemporaryDirectory},
     },
-    query::{ExecutableQuery, QueryBase, Select},
+    query::{ExecutableQuery, QueryBase},
     Error, Result, Table,
 };
 
@@ -27,8 +27,6 @@ pub const SRC_ROW_ID_COL: &str = "row_id";
 
 pub const SPLIT_NAMES_CONFIG_KEY: &str = "split_names";
 
-pub const DEFAULT_MEMORY_LIMIT: usize = 100 * 1024 * 1024;
-
 /// Where to store the permutation table
 #[derive(Debug, Clone, Default)]
 enum PermutationDestination {
@@ -169,20 +167,10 @@ impl PermutationBuilder {
         &self,
         data: SendableRecordBatchStream,
     ) -> Result<SendableRecordBatchStream> {
-        let memory_limit = std::env::var("LANCEDB_PERM_BUILDER_MEMORY_LIMIT")
-            .unwrap_or_else(|_| DEFAULT_MEMORY_LIMIT.to_string())
-            .parse::<usize>()
-            .unwrap_or_else(|_| {
-                log::error!(
-                    "Failed to parse LANCEDB_PERM_BUILDER_MEMORY_LIMIT, using default: {}",
-                    DEFAULT_MEMORY_LIMIT
-                );
-                DEFAULT_MEMORY_LIMIT
-            });
         let ctx = SessionContext::new_with_config_rt(
             SessionConfig::default(),
             RuntimeEnvBuilder::new()
-                .with_memory_limit(memory_limit, 1.0)
+                .with_memory_limit(100 * 1024 * 1024, 1.0)
                 .with_disk_manager_builder(
                     DiskManagerBuilder::default()
                         .with_mode(self.config.temp_dir.to_disk_manager_mode()),
@@ -244,7 +232,7 @@ impl PermutationBuilder {
     /// Builds the permutation table and stores it in the given database.
     pub async fn build(self) -> Result<Table> {
         // First pass, apply filter and load row ids
-        let mut rows = self.base_table.query().select(Select::columns(&[ROW_ID]));
+        let mut rows = self.base_table.query().with_row_id();
 
         if let Some(filter) = &self.config.filter {
             rows = rows.only_if(filter);
@@ -333,47 +321,6 @@ mod tests {
 
     use super::*;
 
-    #[tokio::test]
-    async fn test_permutation_table_only_stores_row_id_and_split_id() {
-        let temp_dir = tempfile::tempdir().unwrap();
-
-        let db = connect(temp_dir.path().to_str().unwrap())
-            .execute()
-            .await
-            .unwrap();
-
-        let initial_data = lance_datagen::gen_batch()
-            .col("col_a", lance_datagen::array::step::<Int32Type>())
-            .col("col_b", lance_datagen::array::step::<Int32Type>())
-            .into_ldb_stream(RowCount::from(100), BatchCount::from(10));
-        let data_table = db
-            .create_table_streaming("base_tbl", initial_data)
-            .execute()
-            .await
-            .unwrap();
-
-        let permutation_table = PermutationBuilder::new(data_table.clone())
-            .with_split_strategy(
-                SplitStrategy::Sequential {
-                    sizes: SplitSizes::Percentages(vec![0.5, 0.5]),
-                },
-                None,
-            )
-            .with_filter("col_a > 57".to_string())
-            .build()
-            .await
-            .unwrap();
-
-        let schema = permutation_table.schema().await.unwrap();
-        let field_names: Vec<&str> = schema.fields().iter().map(|f| f.name().as_str()).collect();
-        assert_eq!(
-            field_names,
-            vec!["row_id", "split_id"],
-            "Permutation table should only contain row_id and split_id columns, but found: {:?}",
-            field_names,
-        );
-    }
-
     #[tokio::test]
     async fn test_permutation_builder() {
         let temp_dir = tempfile::tempdir().unwrap();
@@ -405,6 +352,8 @@ mod tests {
             .await
             .unwrap();
 
+        println!("permutation_table: {:?}", permutation_table);
+
         // Potentially brittle seed-dependent values below
         assert_eq!(permutation_table.count_rows(None).await.unwrap(), 330);
         assert_eq!(

rust/lancedb/src/remote/table.rs

@@ -1493,6 +1493,14 @@ impl<S: HttpSend> BaseTable for RemoteTable<S> {
         None
     }
 
+    async fn initial_storage_options(&self) -> Option<HashMap<String, String>> {
+        None
+    }
+
+    async fn latest_storage_options(&self) -> Result<Option<HashMap<String, String>>> {
+        Ok(None)
+    }
+
     async fn stats(&self) -> Result<TableStatistics> {
         let request = self
             .client

rust/lancedb/src/table.rs

@@ -611,7 +611,17 @@ pub trait BaseTable: std::fmt::Display + std::fmt::Debug + Send + Sync {
     /// Get the table URI (storage location)
     async fn uri(&self) -> Result<String>;
     /// Get the storage options used when opening this table, if any.
+    #[deprecated(since = "0.25.0", note = "Use initial_storage_options() instead")]
     async fn storage_options(&self) -> Option<HashMap<String, String>>;
+    /// Get the initial storage options that were passed in when opening this table.
+    ///
+    /// For dynamically refreshed options (e.g., credential vending), use [`Self::latest_storage_options`].
+    async fn initial_storage_options(&self) -> Option<HashMap<String, String>>;
+    /// Get the latest storage options, refreshing from provider if configured.
+    ///
+    /// Returns `Ok(Some(options))` if storage options are available (static or refreshed),
+    /// `Ok(None)` if no storage options were configured, or `Err(...)` if refresh failed.
+    async fn latest_storage_options(&self) -> Result<Option<HashMap<String, String>>>;
     /// Poll until the columns are fully indexed. Will return Error::Timeout if the columns
     /// are not fully indexed within the timeout.
     async fn wait_for_index(
@@ -1328,10 +1338,32 @@ impl Table {
     /// Get the storage options used when opening this table, if any.
     ///
     /// Warning: This is an internal API and the return value is subject to change.
+    #[deprecated(since = "0.25.0", note = "Use initial_storage_options() instead")]
     pub async fn storage_options(&self) -> Option<HashMap<String, String>> {
+        #[allow(deprecated)]
         self.inner.storage_options().await
     }
 
+    /// Get the initial storage options that were passed in when opening this table.
+    ///
+    /// For dynamically refreshed options (e.g., credential vending), use [`Self::latest_storage_options`].
+    ///
+    /// Warning: This is an internal API and the return value is subject to change.
+    pub async fn initial_storage_options(&self) -> Option<HashMap<String, String>> {
+        self.inner.initial_storage_options().await
+    }
+
+    /// Get the latest storage options, refreshing from provider if configured.
+    ///
+    /// This method is useful for credential vending scenarios where storage options
+    /// may be refreshed dynamically. If no dynamic provider is configured, this
+    /// returns the initial static options.
+    ///
+    /// Warning: This is an internal API and the return value is subject to change.
+    pub async fn latest_storage_options(&self) -> Result<Option<HashMap<String, String>>> {
+        self.inner.latest_storage_options().await
+    }
+
     /// Get statistics about an index.
     /// Returns None if the index does not exist.
     pub async fn index_stats(
@@ -3231,6 +3263,10 @@ impl BaseTable for NativeTable {
     }
 
     async fn storage_options(&self) -> Option<HashMap<String, String>> {
+        self.initial_storage_options().await
+    }
+
+    async fn initial_storage_options(&self) -> Option<HashMap<String, String>> {
         self.dataset
             .get()
             .await
@@ -3238,6 +3274,11 @@ impl BaseTable for NativeTable {
             .and_then(|dataset| dataset.initial_storage_options().cloned())
     }
 
+    async fn latest_storage_options(&self) -> Result<Option<HashMap<String, String>>> {
+        let dataset = self.dataset.get().await?;
+        Ok(dataset.latest_storage_options().await?.map(|o| o.0))
+    }
+
     async fn index_stats(&self, index_name: &str) -> Result<Option<IndexStatistics>> {
         let stats = match self
             .dataset