rust/lancedb
1
0
Fork 0
mirror of https://github.com/lancedb/lancedb.git synced 2026-10-29 23:06:51 +00:00
Code Issues Packages Projects Releases Wiki Activity

chore: update Lance to v15.0.0-beta.2 (#4690)

Browse Source 
**Blocked on the corrected Lance release:**
https://github.com/lance-format/lance/pull/10370 restores the `Send`
contract required by downstream Function workers. Refresh this upgrade
with that release before merging.

Upgrade the Rust, Java, and Python Lance dependencies to
https://github.com/lance-format/lance/releases/tag/v15.0.0-beta.2. This
brings in bounded vector-index consolidation without retraining
(https://github.com/lance-format/lance/pull/10335) and deletion
filtering for index entries whose fragment coverage was pruned
(https://github.com/lance-format/lance/pull/10325).

Synchronize the vendored `lance-arrow` with the release and retain its
offset-safe validity patch. The existing `nullable_struct_merge`
regression still panics with the unpatched release, so removing that
patch would regress nullable nested queries. Refresh both lockfiles,
including the `zstd-sys` version required by the new Lance encoding
dependency.

Update the custom memtable bitmap test adapter to train on row
addresses, matching the new bitmap contract; remove its obsolete row-ID
conversion.

Preserve the Blob APIs' existing missing-row error messages when newer
Lance rejects a missing fragment earlier in the read path. Retain
upstream attribution in vendored sources while satisfying the repository
license-header check.

Validation of the compatibility fixes: all 58 Rust Blob integration
tests and 219 Python Blob tests pass (one Python test skipped). Strict
workspace Clippy, formatting, and diff checks pass.
This commit is contained in:
Xuanwo
2026-10-27 17:35:43 +08:00
committed by GitHub
parent 7fca4d666a
commit ad1212fb32
12 changed files with 957 additions and 307 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/pyproject.toml
+2 -2
View File
@@ -46,7 +46,7 @@ repository = "https://github.com/lancedb/lancedb"
[project.optional-dependencies]
pylance = [
# Keep the minimum in the bundled Lance major and compatible with tests below.
"pylance>=15.0.0b1",
"pylance>=15.0.0b2",
]
# A library only needs the OpenTelemetry API; the application supplies and
# configures the SDK (the actual exporter/reader). See
@@ -64,7 +64,7 @@ tests = [
"polars>=0.19, <=1.32.3",
"pyarrow<25",
"pyarrow-stubs>=16.0",
"pylance==15.0.0b1",
"pylance==15.0.0b2",
"requests>=2.31.0",
"datafusion>=54,<55",
"opentelemetry-sdk>=1.30.0",
python/uv.lock
Generated
+101 -101
View File
@@ -1,5 +1,5 @@
version = 1
revision = 3
revision = 5
requires-python = ">=3.10"
resolution-markers = [
"python_full_version >= '3.14' and sys_platform == 'win32'",
@@ -799,7 +799,7 @@ name = "cuda-bindings"
version = "13.3.1"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "cuda-pathfinder", marker = "(python_full_version < '3.14' and sys_platform == 'emscripten') or (python_full_version < '3.14' and sys_platform == 'win32') or (sys_platform != 'emscripten' and sys_platform != 'win32')" },
{ name = "cuda-pathfinder" },
]
wheels = [
{ url = "https://files.pythonhosted.org/packages/a9/21/8464d133752951c154feafb3b65c297e7d80f301183d220bec4c830f1441/cuda_bindings-13.3.1-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:120fcc53d57903df529c3486962c56528cba5b7d6c57c99537320ed9922c8b86", size = 6073403, upload-time = "2026-05-29T23:11:36.22Z" },
@@ -834,37 +834,37 @@ wheels = [
[package.optional-dependencies]
cublas = [
{ name = "nvidia-cublas", marker = "(python_full_version < '3.14' and sys_platform == 'win32') or sys_platform == 'linux'" },
{ name = "nvidia-cublas" },
]
cudart = [
{ name = "nvidia-cuda-runtime", marker = "(python_full_version < '3.14' and sys_platform == 'win32') or sys_platform == 'linux'" },
{ name = "nvidia-cuda-runtime" },
]
cufft = [
{ name = "nvidia-cufft", marker = "(python_full_version < '3.14' and sys_platform == 'win32') or sys_platform == 'linux'" },
{ name = "nvidia-cufft" },
]
cufile = [
{ name = "nvidia-cufile", marker = "sys_platform == 'linux'" },
{ name = "nvidia-cufile" },
]
cupti = [
{ name = "nvidia-cuda-cupti", marker = "(python_full_version < '3.14' and sys_platform == 'win32') or sys_platform == 'linux'" },
{ name = "nvidia-cuda-cupti" },
]
curand = [
{ name = "nvidia-curand", marker = "(python_full_version < '3.14' and sys_platform == 'win32') or sys_platform == 'linux'" },
{ name = "nvidia-curand" },
]
cusolver = [
{ name = "nvidia-cusolver", marker = "(python_full_version < '3.14' and sys_platform == 'win32') or sys_platform == 'linux'" },
{ name = "nvidia-cusolver" },
]
cusparse = [
{ name = "nvidia-cusparse", marker = "(python_full_version < '3.14' and sys_platform == 'win32') or sys_platform == 'linux'" },
{ name = "nvidia-cusparse" },
]
nvjitlink = [
{ name = "nvidia-nvjitlink", marker = "(python_full_version < '3.14' and sys_platform == 'win32') or sys_platform == 'linux'" },
{ name = "nvidia-nvjitlink" },
]
nvrtc = [
{ name = "nvidia-cuda-nvrtc", marker = "(python_full_version < '3.14' and sys_platform == 'win32') or sys_platform == 'linux'" },
{ name = "nvidia-cuda-nvrtc" },
]
nvtx = [
{ name = "nvidia-nvtx", marker = "(python_full_version < '3.14' and sys_platform == 'win32') or sys_platform == 'linux'" },
{ name = "nvidia-nvtx" },
]
[[package]]
@@ -1023,7 +1023,7 @@ name = "exceptiongroup"
version = "1.3.1"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "typing-extensions", marker = "python_full_version < '3.11'" },
{ name = "typing-extensions" },
]
sdist = { url = "https://files.pythonhosted.org/packages/50/79/66800aadf48771f6b62f7eb014e352e5d06856655206165d775e675a02c9/exceptiongroup-1.3.1.tar.gz", hash = "sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219", size = 30371, upload-time = "2025-11-21T23:01:54.787Z" }
wheels = [
@@ -1440,16 +1440,16 @@ resolution-markers = [
"python_full_version < '3.11'",
]
dependencies = [
{ name = "cachetools", marker = "python_full_version < '3.11'" },
{ name = "certifi", marker = "python_full_version < '3.11'" },
{ name = "httpx", marker = "python_full_version < '3.11'" },
{ name = "ibm-cos-sdk", marker = "python_full_version < '3.11'" },
{ name = "lomond", marker = "python_full_version < '3.11'" },
{ name = "packaging", marker = "python_full_version < '3.11'" },
{ name = "pandas", version = "2.2.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" },
{ name = "requests", marker = "python_full_version < '3.11'" },
{ name = "tabulate", marker = "python_full_version < '3.11'" },
{ name = "urllib3", marker = "python_full_version < '3.11'" },
{ name = "cachetools" },
{ name = "certifi" },
{ name = "httpx" },
{ name = "ibm-cos-sdk" },
{ name = "lomond" },
{ name = "packaging" },
{ name = "pandas", version = "2.2.3", source = { registry = "https://pypi.org/simple" } },
{ name = "requests" },
{ name = "tabulate" },
{ name = "urllib3" },
]
sdist = { url = "https://files.pythonhosted.org/packages/c7/56/2e3df38a1f13062095d7bde23c87a92f3898982993a15186b1bfecbd206f/ibm_watsonx_ai-1.3.42.tar.gz", hash = "sha256:ee5be59009004245d957ce97d1227355516df95a2640189749487614fef674ff", size = 688651, upload-time = "2025-10-01T13:35:41.527Z" }
wheels = [
@@ -1468,17 +1468,17 @@ resolution-markers = [
"python_full_version == '3.11.*'",
]
dependencies = [
{ name = "cachetools", marker = "python_full_version >= '3.11'" },
{ name = "certifi", marker = "python_full_version >= '3.11'" },
{ name = "httpx", marker = "python_full_version >= '3.11'" },
{ name = "ibm-cos-sdk", marker = "python_full_version >= '3.11'" },
{ name = "lomond", marker = "python_full_version >= '3.11'" },
{ name = "packaging", marker = "python_full_version >= '3.11'" },
{ name = "pandas", version = "2.3.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11' and python_full_version < '3.14'" },
{ name = "cachetools" },
{ name = "certifi" },
{ name = "httpx" },
{ name = "ibm-cos-sdk" },
{ name = "lomond" },
{ name = "packaging" },
{ name = "pandas", version = "2.3.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.14'" },
{ name = "pandas", version = "3.0.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.14'" },
{ name = "requests", marker = "python_full_version >= '3.11'" },
{ name = "tabulate", marker = "python_full_version >= '3.11'" },
{ name = "urllib3", marker = "python_full_version >= '3.11'" },
{ name = "requests" },
{ name = "tabulate" },
{ name = "urllib3" },
]
sdist = { url = "https://files.pythonhosted.org/packages/29/a3/c756b534696ab2f3f29882fdb7ca7198b7a5c94e10c0a3a327853d6d6b79/ibm_watsonx_ai-1.5.14.tar.gz", hash = "sha256:a756488bd57e87c0fc51be42dcba871143cfe0ac1e805c497c5047e1e4f13e9d", size = 735804, upload-time = "2026-06-22T12:32:43.85Z" }
wheels = [
@@ -1554,17 +1554,17 @@ resolution-markers = [
"python_full_version < '3.11'",
]
dependencies = [
{ name = "colorama", marker = "python_full_version < '3.11' and sys_platform == 'win32'" },
{ name = "decorator", marker = "python_full_version < '3.11'" },
{ name = "exceptiongroup", marker = "python_full_version < '3.11'" },
{ name = "jedi", marker = "python_full_version < '3.11'" },
{ name = "matplotlib-inline", marker = "python_full_version < '3.11'" },
{ name = "pexpect", marker = "python_full_version < '3.11' and sys_platform != 'emscripten' and sys_platform != 'win32'" },
{ name = "prompt-toolkit", marker = "python_full_version < '3.11'" },
{ name = "pygments", marker = "python_full_version < '3.11'" },
{ name = "stack-data", marker = "python_full_version < '3.11'" },
{ name = "traitlets", marker = "python_full_version < '3.11'" },
{ name = "typing-extensions", marker = "python_full_version < '3.11'" },
{ name = "colorama", marker = "sys_platform == 'win32'" },
{ name = "decorator" },
{ name = "exceptiongroup" },
{ name = "jedi" },
{ name = "matplotlib-inline" },
{ name = "pexpect", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" },
{ name = "prompt-toolkit" },
{ name = "pygments" },
{ name = "stack-data" },
{ name = "traitlets" },
{ name = "typing-extensions" },
]
sdist = { url = "https://files.pythonhosted.org/packages/40/18/f8598d287006885e7136451fdea0755af4ebcbfe342836f24deefaed1164/ipython-8.39.0.tar.gz", hash = "sha256:4110ae96012c379b8b6db898a07e186c40a2a1ef5d57a7fa83166047d9da7624", size = 5513971, upload-time = "2026-03-27T10:02:13.94Z" }
wheels = [
@@ -1583,18 +1583,18 @@ resolution-markers = [
"python_full_version == '3.11.*'",
]
dependencies = [
{ name = "colorama", marker = "python_full_version >= '3.11' and sys_platform == 'win32'" },
{ name = "decorator", marker = "python_full_version >= '3.11'" },
{ name = "ipython-pygments-lexers", marker = "python_full_version >= '3.11'" },
{ name = "jedi", marker = "python_full_version >= '3.11'" },
{ name = "matplotlib-inline", marker = "python_full_version >= '3.11'" },
{ name = "pexpect", marker = "python_full_version >= '3.11' and sys_platform != 'emscripten' and sys_platform != 'win32'" },
{ name = "prompt-toolkit", marker = "python_full_version >= '3.11'" },
{ name = "psutil", marker = "python_full_version >= '3.11' and sys_platform != 'cygwin' and sys_platform != 'emscripten'" },
{ name = "pygments", marker = "python_full_version >= '3.11'" },
{ name = "stack-data", marker = "python_full_version >= '3.11'" },
{ name = "traitlets", marker = "python_full_version >= '3.11'" },
{ name = "typing-extensions", marker = "python_full_version == '3.11.*'" },
{ name = "colorama", marker = "sys_platform == 'win32'" },
{ name = "decorator" },
{ name = "ipython-pygments-lexers" },
{ name = "jedi" },
{ name = "matplotlib-inline" },
{ name = "pexpect", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" },
{ name = "prompt-toolkit" },
{ name = "psutil", marker = "sys_platform != 'cygwin' and sys_platform != 'emscripten'" },
{ name = "pygments" },
{ name = "stack-data" },
{ name = "traitlets" },
{ name = "typing-extensions", marker = "python_full_version < '3.12'" },
]
sdist = { url = "https://files.pythonhosted.org/packages/53/59/165d3b4d75cc34add3122c4417ecb229085140ac573103c223cd01dde96f/ipython-9.15.0.tar.gz", hash = "sha256:da2819ce2aa83135257df830660b1176d986c3d2876db24df01974fa955b2756", size = 4442580, upload-time = "2026-06-26T11:03:35.913Z" }
wheels = [
@@ -1606,7 +1606,7 @@ name = "ipython-pygments-lexers"
version = "1.1.1"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "pygments", marker = "python_full_version >= '3.11'" },
{ name = "pygments" },
]
sdist = { url = "https://files.pythonhosted.org/packages/ef/4c/5dd1d8af08107f88c7f741ead7a40854b8ac24ddf9ae850afbcf698aa552/ipython_pygments_lexers-1.1.1.tar.gz", hash = "sha256:09c0138009e56b6854f9535736f4171d855c8c08a563a0dcd8022f78355c7e81", size = 8393, upload-time = "2025-01-17T11:24:34.505Z" }
wheels = [
@@ -1854,9 +1854,9 @@ source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "lance-namespace-urllib3-client" },
]
sdist = { url = "https://files.pythonhosted.org/packages/c0/e7/d5d46594678ee479c0eda830c47b2f5c46133bd100e84e9aa6e01306eca7/lance_namespace-0.13.0.tar.gz", hash = "sha256:24554a0997bdb39595c6e4cb3ac6722069f6cd3bd1a74e7acbba7bfaf774a40d", size = 11636, upload-time = "2026-09-14T02:47:03.871523Z" }
sdist = { url = "https://files.pythonhosted.org/packages/c0/e7/d5d46594678ee479c0eda830c47b2f5c46133bd100e84e9aa6e01306eca7/lance_namespace-0.13.0.tar.gz", hash = "sha256:24554a0997bdb39595c6e4cb3ac6722069f6cd3bd1a74e7acbba7bfaf774a40d", size = 11636, upload-time = "2026-09-14T02:47:03.871Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/e7/47/cfa33cca1ba7c749fd2918cfc5c8ded788f378cc6424d23e4fead5a14125/lance_namespace-0.13.0-py3-none-any.whl", hash = "sha256:438c7b17aef421c21c138196e715f2510d62f07e865372047f87c1e75e618c7a", size = 13507, upload-time = "2026-09-14T02:47:01.765655Z" },
{ url = "https://files.pythonhosted.org/packages/e7/47/cfa33cca1ba7c749fd2918cfc5c8ded788f378cc6424d23e4fead5a14125/lance_namespace-0.13.0-py3-none-any.whl", hash = "sha256:438c7b17aef421c21c138196e715f2510d62f07e865372047f87c1e75e618c7a", size = 13507, upload-time = "2026-09-14T02:47:01.765Z" },
]
[[package]]
@@ -1869,9 +1869,9 @@ dependencies = [
{ name = "typing-extensions" },
{ name = "urllib3" },
]
sdist = { url = "https://files.pythonhosted.org/packages/69/25/9aaa4a5e8999693fb0f227c2c0c4b97bd8f0539066408cdff0b89d41b2d5/lance_namespace_urllib3_client-0.13.0.tar.gz", hash = "sha256:1e8a79c6e4e6277033597fd76aa0e2f33d909ca1436c935936e9e569221f43ef", size = 238348, upload-time = "2026-09-14T02:47:04.521878Z" }
sdist = { url = "https://files.pythonhosted.org/packages/69/25/9aaa4a5e8999693fb0f227c2c0c4b97bd8f0539066408cdff0b89d41b2d5/lance_namespace_urllib3_client-0.13.0.tar.gz", hash = "sha256:1e8a79c6e4e6277033597fd76aa0e2f33d909ca1436c935936e9e569221f43ef", size = 238348, upload-time = "2026-09-14T02:47:04.521Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/60/4c/7b8f0712a7fe1b8655b711342bc4989696635a79e7027ee56ba3cae23e99/lance_namespace_urllib3_client-0.13.0-py3-none-any.whl", hash = "sha256:fb361eb4f6c7f2d1f9e92809609657b73e38241393e9c4516e12d6b6ee643a0a", size = 406848, upload-time = "2026-09-14T02:47:02.715954Z" },
{ url = "https://files.pythonhosted.org/packages/60/4c/7b8f0712a7fe1b8655b711342bc4989696635a79e7027ee56ba3cae23e99/lance_namespace_urllib3_client-0.13.0-py3-none-any.whl", hash = "sha256:fb361eb4f6c7f2d1f9e92809609657b73e38241393e9c4516e12d6b6ee643a0a", size = 406848, upload-time = "2026-09-14T02:47:02.715Z" },
]
[[package]]
@@ -2006,8 +2006,8 @@ requires-dist = [
{ name = "pyarrow", marker = "extra == 'tests'", specifier = "<25" },
{ name = "pyarrow-stubs", marker = "extra == 'tests'", specifier = ">=16.0" },
{ name = "pydantic", specifier = ">=2.7.4,<3" },
{ name = "pylance", marker = "extra == 'pylance'", specifier = ">=15.0.0b1" },
{ name = "pylance", marker = "extra == 'tests'", specifier = "==15.0.0b1" },
{ name = "pylance", marker = "extra == 'pylance'", specifier = ">=15.0.0b2" },
{ name = "pylance", marker = "extra == 'tests'", specifier = "==15.0.0b2" },
{ name = "pyright", marker = "extra == 'dev'", specifier = ">=1.1.350" },
{ name = "pytest", marker = "extra == 'tests'", specifier = ">=7.0" },
{ name = "pytest-asyncio", marker = "extra == 'tests'", specifier = ">=0.21" },
@@ -2884,7 +2884,7 @@ name = "nvidia-cudnn-cu13"
version = "9.19.0.56"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "nvidia-cublas", marker = "(python_full_version < '3.14' and sys_platform == 'emscripten') or (python_full_version < '3.14' and sys_platform == 'win32') or (sys_platform != 'emscripten' and sys_platform != 'win32')" },
{ name = "nvidia-cublas" },
]
wheels = [
{ url = "https://files.pythonhosted.org/packages/f1/84/26025437c1e6b61a707442184fa0c03d083b661adf3a3eecfd6d21677740/nvidia_cudnn_cu13-9.19.0.56-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:6ed29ffaee1176c612daf442e4dd6cfeb6a0caa43ddcbeb59da94953030b1be4", size = 433781201, upload-time = "2026-02-03T20:40:53.805Z" },
@@ -2896,7 +2896,7 @@ name = "nvidia-cufft"
version = "12.0.0.61"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "nvidia-nvjitlink", marker = "(python_full_version < '3.14' and sys_platform == 'emscripten') or (python_full_version < '3.14' and sys_platform == 'win32') or (sys_platform != 'emscripten' and sys_platform != 'win32')" },
{ name = "nvidia-nvjitlink" },
]
wheels = [
{ url = "https://files.pythonhosted.org/packages/8b/ae/f417a75c0259e85c1d2f83ca4e960289a5f814ed0cea74d18c353d3e989d/nvidia_cufft-12.0.0.61-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:2708c852ef8cd89d1d2068bdbece0aa188813a0c934db3779b9b1faa8442e5f5", size = 214053554, upload-time = "2025-09-04T08:31:38.196Z" },
@@ -2926,9 +2926,9 @@ name = "nvidia-cusolver"
version = "12.0.4.66"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "nvidia-cublas", marker = "(python_full_version < '3.14' and sys_platform == 'emscripten') or (python_full_version < '3.14' and sys_platform == 'win32') or (sys_platform != 'emscripten' and sys_platform != 'win32')" },
{ name = "nvidia-cusparse", marker = "(python_full_version < '3.14' and sys_platform == 'emscripten') or (python_full_version < '3.14' and sys_platform == 'win32') or (sys_platform != 'emscripten' and sys_platform != 'win32')" },
{ name = "nvidia-nvjitlink", marker = "(python_full_version < '3.14' and sys_platform == 'emscripten') or (python_full_version < '3.14' and sys_platform == 'win32') or (sys_platform != 'emscripten' and sys_platform != 'win32')" },
{ name = "nvidia-cublas" },
{ name = "nvidia-cusparse" },
{ name = "nvidia-nvjitlink" },
]
wheels = [
{ url = "https://files.pythonhosted.org/packages/c8/c3/b30c9e935fc01e3da443ec0116ed1b2a009bb867f5324d3f2d7e533e776b/nvidia_cusolver-12.0.4.66-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:02c2457eaa9e39de20f880f4bd8820e6a1cfb9f9a34f820eb12a155aa5bc92d2", size = 223467760, upload-time = "2025-09-04T08:33:04.222Z" },
@@ -2940,7 +2940,7 @@ name = "nvidia-cusparse"
version = "12.6.3.3"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "nvidia-nvjitlink", marker = "(python_full_version < '3.14' and sys_platform == 'emscripten') or (python_full_version < '3.14' and sys_platform == 'win32') or (sys_platform != 'emscripten' and sys_platform != 'win32')" },
{ name = "nvidia-nvjitlink" },
]
wheels = [
{ url = "https://files.pythonhosted.org/packages/f8/94/5c26f33738ae35276672f12615a64bd008ed5be6d1ebcb23579285d960a9/nvidia_cusparse-12.6.3.3-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:80bcc4662f23f1054ee334a15c72b8940402975e0eab63178fc7e670aa59472c", size = 162155568, upload-time = "2025-09-04T08:33:42.864Z" },
@@ -3117,10 +3117,10 @@ resolution-markers = [
"python_full_version < '3.11'",
]
dependencies = [
{ name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" },
{ name = "python-dateutil", marker = "python_full_version < '3.11'" },
{ name = "pytz", marker = "python_full_version < '3.11'" },
{ name = "tzdata", marker = "python_full_version < '3.11'" },
{ name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" } },
{ name = "python-dateutil" },
{ name = "pytz" },
{ name = "tzdata" },
]
sdist = { url = "https://files.pythonhosted.org/packages/9c/d6/9f8431bacc2e19dca897724cd097b1bb224a6ad5433784a44b587c7c13af/pandas-2.2.3.tar.gz", hash = "sha256:4f18ba62b61d7e192368b84517265a99b4d7ee8912f8708660fb4a366cc82667", size = 4399213, upload-time = "2024-09-20T13:10:04.827Z" }
wheels = [
@@ -3169,11 +3169,11 @@ resolution-markers = [
"python_full_version == '3.11.*'",
]
dependencies = [
{ name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*'" },
{ name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12' or python_full_version >= '3.14'" },
{ name = "numpy", version = "2.5.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12' and python_full_version < '3.14'" },
{ name = "python-dateutil", marker = "python_full_version >= '3.11' and python_full_version < '3.14'" },
{ name = "pytz", marker = "python_full_version >= '3.11' and python_full_version < '3.14'" },
{ name = "tzdata", marker = "python_full_version >= '3.11' and python_full_version < '3.14'" },
{ name = "python-dateutil" },
{ name = "pytz" },
{ name = "tzdata" },
]
sdist = { url = "https://files.pythonhosted.org/packages/33/01/d40b85317f86cf08d853a4f495195c73815fdf205eef3993821720274518/pandas-2.3.3.tar.gz", hash = "sha256:e05e1af93b977f7eafa636d043f9f94c7ee3ac81af99c13508215942e64c993b", size = 4495223, upload-time = "2025-09-29T23:34:51.853Z" }
wheels = [
@@ -3236,9 +3236,9 @@ resolution-markers = [
"python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'",
]
dependencies = [
{ name = "numpy", version = "2.5.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.14'" },
{ name = "python-dateutil", marker = "python_full_version >= '3.14'" },
{ name = "tzdata", marker = "(python_full_version >= '3.14' and sys_platform == 'emscripten') or (python_full_version >= '3.14' and sys_platform == 'win32')" },
{ name = "numpy", version = "2.5.1", source = { registry = "https://pypi.org/simple" } },
{ name = "python-dateutil" },
{ name = "tzdata", marker = "sys_platform == 'emscripten' or sys_platform == 'win32'" },
]
sdist = { url = "https://files.pythonhosted.org/packages/f8/87/4341c6252d1c47b08768c3d25ac487362bf403f0313ddae4a2a26c9b1b4c/pandas-3.0.3.tar.gz", hash = "sha256:696a4a00a2a2a35d4e5deb3fc946641b96c944f02230e4f76137fe35d806c4fc", size = 4651414, upload-time = "2026-05-11T18:54:29.21Z" }
wheels = [
@@ -3346,7 +3346,7 @@ name = "pexpect"
version = "4.9.0"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "ptyprocess", marker = "(python_full_version < '3.14' and sys_platform == 'emscripten') or (python_full_version < '3.14' and sys_platform == 'win32') or (sys_platform != 'emscripten' and sys_platform != 'win32')" },
{ name = "ptyprocess" },
]
sdist = { url = "https://files.pythonhosted.org/packages/42/92/cc564bf6381ff43ce1f4d06852fc19a2f11d180f23dc32d9588bee2f149d/pexpect-4.9.0.tar.gz", hash = "sha256:ee7d41123f3c9911050ea2c2dac107568dc43b2d3b0c7557a33212c398ead30f", size = 166450, upload-time = "2023-11-25T09:07:26.339Z" }
wheels = [
@@ -3938,7 +3938,7 @@ crypto = [
[[package]]
name = "pylance"
version = "15.0.0b1"
version = "15.0.0b2"
source = { registry = "https://pypi.fury.io/lance-format/" }
dependencies = [
{ name = "lance-namespace" },
@@ -3948,12 +3948,12 @@ dependencies = [
{ name = "pyarrow" },
]
wheels = [
{ url = "https://pypi.fury.io/lance-format/-/ver_23L1Nd/pylance-15.0.0b1-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:19605d3daca1224cc0db079054be5d4eec403dcf286bf4617430bfc259ab0a52" },
{ url = "https://pypi.fury.io/lance-format/-/ver_1Ho3RK/pylance-15.0.0b1-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:be22b0b6f1e1d250ffaa82b449e982c817db33103c483d1005020285310623cf" },
{ url = "https://pypi.fury.io/lance-format/-/ver_1QmyF0/pylance-15.0.0b1-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a5b2761d60b1b9db112058213f1f7b2de82b08cbe65888fab08b0a56e702482e" },
{ url = "https://pypi.fury.io/lance-format/-/ver_1in2Ph/pylance-15.0.0b1-cp310-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:294a0fedae312ed32d1b66cddbf7e69770ec9b4c82a873af60c619ed88450718" },
{ url = "https://pypi.fury.io/lance-format/-/ver_21xpLH/pylance-15.0.0b1-cp310-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:ff20a16703d4d3f3bc675b25ce1871ba1d54137929dc46305bffdd07f7b35a6b" },
{ url = "https://pypi.fury.io/lance-format/-/ver_AAGzF/pylance-15.0.0b1-cp310-abi3-win_amd64.whl", hash = "sha256:3b134e24a5becfc940ca134cf78aa3bae8b80ec40f8927a1b856273ab76213f2" },
{ url = "https://pypi.fury.io/lance-format/-/ver_5aLfR/pylance-15.0.0b2-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:fd6b872e348d804c3188d014f2ff469f8cfaff5bbe846617e665c1e2a2965683" },
{ url = "https://pypi.fury.io/lance-format/-/ver_1W21Yj/pylance-15.0.0b2-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b0ed217ee71bea57ef97303bd1318ae212ee005c885c16390d7efbaf1f7398e3" },
{ url = "https://pypi.fury.io/lance-format/-/ver_12xJCp/pylance-15.0.0b2-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:72580d6359db4d471c578daab46a35fd87e74e57a1cb64e333188fea1e1c2f1b" },
{ url = "https://pypi.fury.io/lance-format/-/ver_m4ypB/pylance-15.0.0b2-cp310-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:c194f95e2d2b1c5a9e4f57ed2681a9295effe5aaa165bf924f2b2090094c4645" },
{ url = "https://pypi.fury.io/lance-format/-/ver_1zYkf2/pylance-15.0.0b2-cp310-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:c14532517ed1452d9cb215f33dc4bc3166500e14145c8c6f2f3538b82ac5147b" },
{ url = "https://pypi.fury.io/lance-format/-/ver_xp2do/pylance-15.0.0b2-cp310-abi3-win_amd64.whl", hash = "sha256:ce13e573a1842eda18f9b74eec4c8d1dc60db0895581db3226c6a6b1a8e79a75" },
]
[[package]]
@@ -4709,10 +4709,10 @@ resolution-markers = [
"python_full_version < '3.11'",
]
dependencies = [
{ name = "joblib", marker = "python_full_version < '3.11'" },
{ name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" },
{ name = "scipy", version = "1.15.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" },
{ name = "threadpoolctl", marker = "python_full_version < '3.11'" },
{ name = "joblib" },
{ name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" } },
{ name = "scipy", version = "1.15.3", source = { registry = "https://pypi.org/simple" } },
{ name = "threadpoolctl" },
]
sdist = { url = "https://files.pythonhosted.org/packages/98/c2/a7855e41c9d285dfe86dc50b250978105dce513d6e459ea66a6aeb0e1e0c/scikit_learn-1.7.2.tar.gz", hash = "sha256:20e9e49ecd130598f1ca38a1d85090e1a600147b9c02fa6f15d69cb53d968fda", size = 7193136, upload-time = "2025-09-09T08:21:29.075Z" }
wheels = [
@@ -4760,13 +4760,13 @@ resolution-markers = [
"python_full_version == '3.11.*'",
]
dependencies = [
{ name = "joblib", marker = "python_full_version >= '3.11'" },
{ name = "narwhals", marker = "python_full_version >= '3.11'" },
{ name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*'" },
{ name = "joblib" },
{ name = "narwhals" },
{ name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" },
{ name = "numpy", version = "2.5.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" },
{ name = "scipy", version = "1.17.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*'" },
{ name = "scipy", version = "1.17.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" },
{ name = "scipy", version = "1.18.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" },
{ name = "threadpoolctl", marker = "python_full_version >= '3.11'" },
{ name = "threadpoolctl" },
]
sdist = { url = "https://files.pythonhosted.org/packages/fa/6f/37092bdb25f712817231799fc5674d8e704066a8a70c1d2d40517e18b4ab/scikit_learn-1.9.0.tar.gz", hash = "sha256:8833266989d3a5110178a9fae30783675460724d0e1efb13b14901d2c660c557", size = 7750767, upload-time = "2026-06-02T11:54:32.706Z" }
wheels = [
@@ -4810,7 +4810,7 @@ resolution-markers = [
"python_full_version < '3.11'",
]
dependencies = [
{ name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" },
{ name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" } },
]
sdist = { url = "https://files.pythonhosted.org/packages/0f/37/6964b830433e654ec7485e45a00fc9a27cf868d622838f6b6d9c5ec0d532/scipy-1.15.3.tar.gz", hash = "sha256:eae3cf522bc7df64b42cad3925c876e1b0b6c35c1337c93e12c0f366f55b0eaf", size = 59419214, upload-time = "2025-05-08T16:13:05.955Z" }
wheels = [
@@ -4869,7 +4869,7 @@ resolution-markers = [
"python_full_version == '3.11.*'",
]
dependencies = [
{ name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*'" },
{ name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" } },
]
sdist = { url = "https://files.pythonhosted.org/packages/7a/97/5a3609c4f8d58b039179648e62dd220f89864f56f7357f5d4f45c29eb2cc/scipy-1.17.1.tar.gz", hash = "sha256:95d8e012d8cb8816c226aef832200b1d45109ed4464303e997c5b13122b297c0", size = 30573822, upload-time = "2026-02-23T00:26:24.851Z" }
wheels = [
@@ -4946,7 +4946,7 @@ resolution-markers = [
"python_full_version >= '3.12' and python_full_version < '3.14'",
]
dependencies = [
{ name = "numpy", version = "2.5.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" },
{ name = "numpy", version = "2.5.1", source = { registry = "https://pypi.org/simple" } },
]
sdist = { url = "https://files.pythonhosted.org/packages/a7/25/c2700dfaf6442b4effaa91af24ebce5dc9d31bb4a69706313aae70d72cd0/scipy-1.18.0.tar.gz", hash = "sha256:67b2ad2ad54c72ca6d04975a9b2df8c3638c34ddd5b28738e94fc2b57929d378", size = 30774447, upload-time = "2026-06-19T15:01:43.456Z" }
wheels = [
python/vendor/lance-arrow/Cargo.toml
Vendored
+1 -4
View File
@@ -1,6 +1,6 @@
[package]
name = "lance-arrow"
version = "15.0.0-beta.1"
version = "15.0.0-beta.2"
edition = "2024"
authors = ["Lance Devs <dev@lance.org>"]
license = "Apache-2.0"
@@ -10,9 +10,6 @@ description = "Arrow Extension for Lance"
keywords = ["data-format", "data-science", "machine-learning", "apache-arrow", "data-analytics"]
categories = ["database-implementations"]
[lib]
crate-type = ["cdylib", "rlib"]
[dependencies]
arrow-array.workspace = true
arrow-buffer.workspace = true
python/vendor/lance-arrow/README.lancedb.md
Vendored
+2 -2
View File
@@ -1,5 +1,5 @@
This directory contains `rust/lance-arrow` from Lance tag `v15.0.0-beta.1`
(commit `46a7cfe1fba5efa1f90deea4c7b14d414b991409`), licensed under Apache-2.0.
This directory contains `rust/lance-arrow` from Lance tag `v15.0.0-beta.2`
(commit `f50a157269d925ad5708c7b0f7ec6c87b0fd08dd`), licensed under Apache-2.0.
LanceDB carries a local fix in `src/lib.rs` for sliced child validity bitmaps
until a Lance release includes the fix. The patch is selected in the root
python/vendor/lance-arrow/src/lib.rs
Vendored
+671 -66
View File
@@ -27,7 +27,7 @@ use arrow_array::{
use arrow_array::{
Float32Array, Float64Array, Int8Array, Int16Array, Int32Array, Int64Array, new_null_array,
};
use arrow_buffer::MutableBuffer;
use arrow_buffer::{BooleanBufferBuilder, MutableBuffer, NullBuffer};
use arrow_data::ArrayDataBuilder;
use arrow_schema::{ArrowError, DataType, Field, Fields, IntervalUnit, Schema, SortOptions};
use arrow_select::{interleave::interleave, take::take};
@@ -670,7 +670,7 @@ impl RecordBatchExt for RecordBatch {
}
let left_struct_array: StructArray = self.clone().into();
let right_struct_array: StructArray = other.clone().into();
self.try_new_from_struct_array(merge(&left_struct_array, &right_struct_array))
self.try_new_from_struct_array(merge(&left_struct_array, &right_struct_array)?)
}
fn merge_with_schema(&self, other: &RecordBatch, schema: &Schema) -> Result<RecordBatch> {
@@ -921,12 +921,44 @@ fn holds_struct(data_type: &DataType) -> bool {
}
}
/// The values of `list`, with the items of every entry that is null in `list`
/// but valid in `other` made null. Those entries become valid when the two
/// lists are merged, and their items here were never meant to be read. Merging
/// the items then pushes these nulls into their children.
fn hide_items_revealed_by<T: OffsetSizeTrait>(
list: &GenericListArray<T>,
other: Option<&NullBuffer>,
) -> ArrayRef {
let Some(nulls) = list.nulls() else {
return list.values().clone();
};
let revealed = match other {
Some(other) => other.inner() & &!nulls.inner(),
None => !nulls.inner(),
};
if revealed.count_set_bits() == 0 {
return list.values().clone();
}
let offsets = list.value_offsets();
let mut items_validity = BooleanBufferBuilder::new(list.values().len());
items_validity.append_n(list.values().len(), true);
for row in revealed.set_indices() {
for item in offsets[row].as_usize()..offsets[row + 1].as_usize() {
items_validity.set_bit(item, false);
}
}
adjust_child_validity(
list.values(),
Some(&NullBuffer::new(items_validity.finish())),
)
}
/// Merge the values of two lists: structs field by field, nested lists
/// recursively, anything else keeps the left side.
fn merge_list_values(left: &ArrayRef, right: &ArrayRef) -> ArrayRef {
match (left.data_type(), right.data_type()) {
fn merge_list_values(left: &ArrayRef, right: &ArrayRef) -> Result<ArrayRef> {
Ok(match (left.data_type(), right.data_type()) {
(DataType::Struct(_), DataType::Struct(_)) => {
Arc::new(merge(left.as_struct(), right.as_struct()))
Arc::new(merge(left.as_struct(), right.as_struct())?)
}
(DataType::List(_), DataType::List(_))
| (DataType::LargeList(_), DataType::LargeList(_))
@@ -934,27 +966,29 @@ fn merge_list_values(left: &ArrayRef, right: &ArrayRef) -> ArrayRef {
&& holds_struct(left.data_type())
&& holds_struct(right.data_type()) =>
{
merge_list_struct(left, right)
merge_list_struct(left, right, None, None)?
}
_ => left.clone(),
}
})
}
fn merge_list_struct_helper<T: OffsetSizeTrait>(
left: &dyn Array,
right: &dyn Array,
) -> Arc<dyn Array> {
null_left: bool,
null_right: bool,
) -> Result<ArrayRef> {
let left_list: &GenericListArray<T> = left.as_list();
let right_list: &GenericListArray<T> = right.as_list();
let item_name = match left.data_type() {
DataType::List(item) | DataType::LargeList(item) => item.name().clone(),
_ => unreachable!(),
let ((DataType::List(left_item), DataType::List(right_item))
| (DataType::LargeList(left_item), DataType::LargeList(right_item))) =
(left.data_type(), right.data_type())
else {
unreachable!()
};
// One side all-null (a placeholder for columns this fragment lacks) has
// no values, so stand in nulls shaped like the other side's.
let null_left = left_list.null_count() == left_list.len();
let null_right = !null_left && right_list.null_count() == right_list.len();
let (merged_items, list, items_nullable) = if null_left || null_right {
let (merged_items, list, nulls, items_nullable) = if null_left || null_right {
// One side all-null (a placeholder for columns this fragment lacks) has
// no values, so stand in nulls shaped like the other side's.
let not_null = if null_left { right_list } else { left_list };
let values_len = not_null.values().len();
let values = |list: &GenericListArray<T>, is_null: bool| {
@@ -967,39 +1001,68 @@ fn merge_list_struct_helper<T: OffsetSizeTrait>(
let merged = merge_list_values(
&values(left_list, null_left),
&values(right_list, null_right),
);
(merged, not_null, true)
)?;
(
merged,
not_null,
not_null.nulls().cloned(),
left_item.is_nullable() || right_item.is_nullable(),
)
} else {
if left_list.offsets().inner() != right_list.offsets().inner() {
panic!("Attempt to merge list struct arrays which do not have same offsets");
}
let merged = merge_list_values(left_list.values(), right_list.values());
let nullable = match left.data_type() {
DataType::List(item) | DataType::LargeList(item) => item.is_nullable(),
_ => unreachable!(),
};
(merged, left_list, nullable)
// The lists share offsets, and an entry is valid if either side has it,
// as for struct rows. Each side's items under its own null entries are
// hidden first, so the other side's validity cannot expose them.
let left_items = hide_items_revealed_by(left_list, right_list.nulls());
let right_items = hide_items_revealed_by(right_list, left_list.nulls());
let merged = merge_list_values(&left_items, &right_items)?;
(
merged,
left_list,
merge_struct_validity(left_list.nulls(), right_list.nulls()),
left_item.is_nullable() || right_item.is_nullable(),
)
};
let items_field = Arc::new(Field::new(
item_name,
left_item.name(),
merged_items.data_type().clone(),
items_nullable,
));
Arc::new(GenericListArray::<T>::new(
Ok(Arc::new(GenericListArray::<T>::new(
items_field,
list.offsets().clone(),
merged_items,
list.nulls().cloned(),
))
nulls,
)))
}
/// Merge two lists whose items hold structs. Only succeeds if both lists have
/// the same offsets or one of them is all-null; panics otherwise.
fn merge_list_struct(left: &dyn Array, right: &dyn Array) -> Arc<dyn Array> {
///
/// `left_parent` and `right_parent`, when given, null each side's entries under
/// its null parent rows before the merge.
fn merge_list_struct(
left: &ArrayRef,
right: &ArrayRef,
left_parent: Option<&NullBuffer>,
right_parent: Option<&NullBuffer>,
) -> Result<ArrayRef> {
// The parents are applied before the all-null side is picked, so a side
// whose rows all sit under null parents counts as the placeholder and the
// other side's offsets are kept.
let left = adjust_child_validity(left, left_parent);
let right = adjust_child_validity(right, right_parent);
let null_left = left.null_count() == left.len();
let null_right = !null_left && right.null_count() == right.len();
let (left, right) = (left.as_ref(), right.as_ref());
match (left.data_type(), right.data_type()) {
(DataType::List(_), DataType::List(_)) => merge_list_struct_helper::<i32>(left, right),
(DataType::List(_), DataType::List(_)) => {
merge_list_struct_helper::<i32>(left, right, null_left, null_right)
}
(DataType::LargeList(_), DataType::LargeList(_)) => {
merge_list_struct_helper::<i64>(left, right)
merge_list_struct_helper::<i64>(left, right, null_left, null_right)
}
_ => unreachable!(),
}
@@ -1187,7 +1250,46 @@ fn adjust_child_validity(
)
}
fn merge(left_struct_array: &StructArray, right_struct_array: &StructArray) -> StructArray {
/// Fails if `field`, a non-nullable child of one side's struct, sits under a
/// row that is null on that side but valid in `merged`. The merged row is valid
/// because the other side has it, and this side has no value to show there.
fn check_hideable(
field: &Field,
parent: Option<&NullBuffer>,
merged: Option<&NullBuffer>,
) -> Result<()> {
let Some(parent) = parent.filter(|_| !field.is_nullable()) else {
return Ok(());
};
let revealed = match merged {
Some(merged) => merged.inner() & &!parent.inner(),
None => !parent.inner(),
};
match revealed.set_indices().next() {
None => Ok(()),
Some(row) => Err(ArrowError::InvalidArgumentError(format!(
"Cannot merge non-nullable field '{}': its parent is null on one side but valid \
on the other at row {row}, where the field has no value",
field.name()
))),
}
}
/// `column`, a child of one side's struct, with that struct's null rows pushed
/// into it. The merged struct is valid where either side is, so the column
/// would otherwise show, at a row only the other side has, what its own null
/// parent hides.
fn hide_under_parent(
field: &Field,
column: &ArrayRef,
parent: Option<&NullBuffer>,
merged: Option<&NullBuffer>,
) -> Result<ArrayRef> {
check_hideable(field, parent, merged)?;
Ok(adjust_child_validity(column, parent))
}
fn merge(left_struct_array: &StructArray, right_struct_array: &StructArray) -> Result<StructArray> {
let mut fields: Vec<Field> = vec![];
let mut columns: Vec<ArrayRef> = vec![];
let right_fields = right_struct_array.fields();
@@ -1199,6 +1301,7 @@ fn merge(left_struct_array: &StructArray, right_struct_array: &StructArray) -> S
// Compute merged validity
let merged_validity = merge_struct_validity(left_validity, right_validity);
let merged = merged_validity.as_ref();
// iterate through the fields on the left hand side
for (left_field, left_column) in left_struct_array
@@ -1217,9 +1320,13 @@ fn merge(left_struct_array: &StructArray, right_struct_array: &StructArray) -> S
// if both fields are struct, merge them recursively
match (left_field.data_type(), right_field.data_type()) {
(DataType::Struct(_), DataType::Struct(_)) => {
let left_sub_array = left_column.as_struct();
let right_sub_array = right_column.as_struct();
let merged_sub_array = merge(left_sub_array, right_sub_array);
// Both sides feed this column, so a row one side's parent
// hides can still take its value from the other side, and
// the field's nullability is checked on the merged result.
let left_sub_array = adjust_child_validity(left_column, left_validity);
let right_sub_array = adjust_child_validity(right_column, right_validity);
let merged_sub_array =
merge(left_sub_array.as_struct(), right_sub_array.as_struct())?;
fields.push(Field::new(
left_field.name(),
merged_sub_array.data_type().clone(),
@@ -1235,12 +1342,22 @@ fn merge(left_struct_array: &StructArray, right_struct_array: &StructArray) -> S
// Identical inner types: nothing to merge, keep the left column.
if left_list.data_type() == right_list.data_type() {
fields.push(left_field.as_ref().clone());
columns.push(left_column.clone());
columns.push(hide_under_parent(
left_field,
left_column,
left_validity,
merged,
)?);
} else {
// The struct fields differ, so merge them structurally. merge_list_struct
// only succeeds when both lists share offsets or one side is all-null;
// it panics otherwise.
let merged_sub_array = merge_list_struct(&left_column, &right_column);
let merged_sub_array = merge_list_struct(
left_column,
right_column,
left_validity,
right_validity,
)?;
fields.push(Field::new(
left_field.name(),
@@ -1254,41 +1371,41 @@ fn merge(left_struct_array: &StructArray, right_struct_array: &StructArray) -> S
_ => {
// TODO handle list-of-struct and other types
fields.push(left_field.as_ref().clone());
// Adjust the column validity: if left struct was null, propagate to child
let adjusted_column = adjust_child_validity(left_column, left_validity);
columns.push(adjusted_column);
columns.push(hide_under_parent(
left_field,
left_column,
left_validity,
merged,
)?);
}
}
}
None => {
fields.push(left_field.as_ref().clone());
// Adjust the column validity: if left struct was null, propagate to child
let adjusted_column = adjust_child_validity(left_column, left_validity);
columns.push(adjusted_column);
columns.push(hide_under_parent(
left_field,
left_column,
left_validity,
merged,
)?);
}
}
}
// now iterate through the fields on the right hand side
right_fields
.iter()
.zip(right_columns.iter())
.for_each(|(field, column)| {
// add new columns on the right
if !left_struct_array
.fields()
.iter()
.any(|f| f.name() == field.name())
{
fields.push(field.as_ref().clone());
// This field doesn't exist on the left
// We use the right's column but need to adjust for struct validity
let adjusted_column = adjust_child_validity(column, right_validity);
columns.push(adjusted_column);
}
});
for (field, column) in right_fields.iter().zip(right_columns.iter()) {
// add new columns on the right
if !left_struct_array
.fields()
.iter()
.any(|f| f.name() == field.name())
{
fields.push(field.as_ref().clone());
columns.push(hide_under_parent(field, column, right_validity, merged)?);
}
}
StructArray::try_new(Fields::from(fields), columns, merged_validity).unwrap()
StructArray::try_new(Fields::from(fields), columns, merged_validity)
}
fn merge_with_schema(
@@ -1563,6 +1680,7 @@ impl BufferExt for arrow_buffer::Buffer {
#[cfg(test)]
mod tests {
use super::*;
use arrow_array::types::Int32Type;
use arrow_array::{
BinaryArray, BinaryViewArray, BooleanArray, FixedSizeBinaryArray, Float32Array, Int32Array,
LargeBinaryArray, ListArray, NullArray, StringArray, StructArray, new_empty_array,
@@ -2083,6 +2201,493 @@ mod tests {
assert_eq!(merged, left);
}
/// A list of `items`, one item per entry.
fn one_item_per_entry(large: bool, items: ArrayRef) -> ArrayRef {
let item_field = Arc::new(Field::new("item", items.data_type().clone(), true));
let lengths = vec![1; items.len()];
if large {
Arc::new(LargeListArray::new(
item_field,
OffsetBuffer::<i64>::from_lengths(lengths),
items,
None,
))
} else {
Arc::new(ListArray::new(
item_field,
OffsetBuffer::<i32>::from_lengths(lengths),
items,
None,
))
}
}
/// A batch with the single column `s`, valid per `parent_validity`.
fn batch_of_parent(parent: StructArray) -> RecordBatch {
RecordBatch::try_new(
Arc::new(Schema::new(vec![Field::new(
"s",
parent.data_type().clone(),
true,
)])),
vec![Arc::new(parent)],
)
.unwrap()
}
/// `{s: struct<l: list<struct<{child}: int32>>>}`, one row per value, each
/// list holding one item, and `s` valid per `parent_validity`.
fn list_struct_under_parent(
large: bool,
child: &str,
parent_validity: &[bool],
values: &[i32],
) -> RecordBatch {
let items = Arc::new(StructArray::new(
vec![Field::new(child, DataType::Int32, true)].into(),
vec![Arc::new(Int32Array::from(values.to_vec()))],
None,
));
let list = one_item_per_entry(large, items);
batch_of_parent(StructArray::new(
vec![Field::new("l", list.data_type().clone(), true)].into(),
vec![list],
Some(parent_validity.to_vec().into()),
))
}
/// The child `name` of the single item in row `row` of the merged `s.l`.
fn merged_item_child(merged: &RecordBatch, large: bool, row: usize, name: &str) -> ArrayRef {
let list = merged.column(0).as_struct().column(0);
let item = if large {
list.as_list::<i64>().value(row)
} else {
list.as_list::<i32>().value(row)
};
item.as_struct().column_by_name(name).unwrap().clone()
}
#[test]
fn test_merge_list_struct_identical_schema_hides_a_null_parent_row() {
for large in [false, true] {
let left = list_struct_under_parent(large, "x", &[false, true], &[11, 12]);
let right = list_struct_under_parent(large, "x", &[true, true], &[21, 22]);
let merged = left.merge(&right).unwrap();
let parent = merged.column(0).as_struct();
assert!(parent.is_valid(0), "large={large}");
assert!(
parent.column(0).is_null(0),
"large={large}: the list under the null left parent became visible"
);
assert!(parent.column(0).is_valid(1), "large={large}");
}
}
#[test]
fn test_merge_list_struct_hides_the_items_of_a_null_parent_row() {
for large in [false, true] {
// Null parent on the left: row 0 keeps the right item, not x = 11.
let left = list_struct_under_parent(large, "x", &[false, true], &[11, 12]);
let right = list_struct_under_parent(large, "y", &[true, true], &[21, 22]);
let merged = left.merge(&right).unwrap();
assert!(merged.column(0).as_struct().column(0).is_valid(0));
assert!(
merged_item_child(&merged, large, 0, "x").is_null(0),
"large={large}: x under the null left parent became visible"
);
let y = merged_item_child(&merged, large, 0, "y");
assert_eq!(y.as_primitive::<Int32Type>().value(0), 21, "large={large}");
// Null parent on the right: row 0 keeps x = 11, not y = 21.
let left = list_struct_under_parent(large, "x", &[true, true], &[11, 12]);
let right = list_struct_under_parent(large, "y", &[false, true], &[21, 22]);
let merged = left.merge(&right).unwrap();
let x = merged_item_child(&merged, large, 0, "x");
assert_eq!(x.as_primitive::<Int32Type>().value(0), 11, "large={large}");
assert!(
merged_item_child(&merged, large, 0, "y").is_null(0),
"large={large}: y under the null right parent became visible"
);
// Row 1 is valid on both sides and keeps both items.
let x = merged_item_child(&merged, large, 1, "x");
let y = merged_item_child(&merged, large, 1, "y");
assert_eq!(x.as_primitive::<Int32Type>().value(0), 12, "large={large}");
assert_eq!(y.as_primitive::<Int32Type>().value(0), 22, "large={large}");
}
}
#[test]
fn test_merge_list_struct_follows_a_sliced_parent() {
for large in [false, true] {
// After slicing, row 0 is the null parent row and row 1 is valid.
let left = list_struct_under_parent(large, "x", &[true, false, true], &[10, 11, 12]);
let left = left.slice(1, 2);
let same = list_struct_under_parent(large, "x", &[true, true, true], &[20, 21, 22]);
let merged = left.merge(&same.slice(1, 2)).unwrap();
let list = merged.column(0).as_struct().column(0);
assert!(list.is_null(0), "large={large}");
assert!(list.is_valid(1), "large={large}");
let other = list_struct_under_parent(large, "y", &[true, true, true], &[20, 21, 22]);
let merged = left.merge(&other.slice(1, 2)).unwrap();
assert!(
merged_item_child(&merged, large, 0, "x").is_null(0),
"large={large}"
);
let x = merged_item_child(&merged, large, 1, "x");
assert_eq!(x.as_primitive::<Int32Type>().value(0), 12, "large={large}");
}
}
/// A non-nullable `l` or item child cannot be nulled under a parent row that
/// only the other side has, so the merge fails there rather than show the
/// hidden value. Where both sides agree on the parent, nothing is revealed
/// and the same fields merge.
#[test]
fn test_merge_list_struct_with_non_nullable_fields_under_a_null_parent() {
#[derive(Clone, Copy)]
struct Shape {
child: &'static str,
l_nullable: bool,
item_nullable: bool,
child_nullable: bool,
null_first_entry: bool,
null_first_item: bool,
}
let shape = |child| Shape {
child,
l_nullable: true,
item_nullable: true,
child_nullable: true,
null_first_entry: false,
null_first_item: false,
};
let batch = |shape: Shape, parent_validity: [bool; 2]| {
let items = Arc::new(StructArray::new(
vec![Field::new(
shape.child,
DataType::Int32,
shape.child_nullable,
)]
.into(),
vec![Arc::new(Int32Array::from(vec![1, 2]))],
shape.null_first_item.then(|| vec![false, true].into()),
));
let list: ArrayRef = Arc::new(ListArray::new(
Arc::new(Field::new(
"item",
items.data_type().clone(),
shape.item_nullable,
)),
OffsetBuffer::<i32>::from_lengths([1, 1]),
items,
shape.null_first_entry.then(|| vec![false, true].into()),
));
batch_of_parent(StructArray::new(
vec![Field::new("l", list.data_type().clone(), shape.l_nullable)].into(),
vec![list],
Some(parent_validity.to_vec().into()),
))
};
let null_first = [false, true];
let all_valid = [true, true];
let cases = [
// Non-nullable `l`, identical item types.
(
Shape {
l_nullable: false,
..shape("x")
},
Shape {
l_nullable: false,
..shape("x")
},
Some("Cannot merge non-nullable field 'l'"),
),
// Non-nullable `l` on the left, a null entry on the right: neither
// side has an entry at the merged row.
(
Shape {
l_nullable: false,
..shape("x")
},
Shape {
null_first_entry: true,
..shape("y")
},
Some("non-nullable StructArray field \"l\""),
),
// A non-nullable child of the item struct.
(
Shape {
child_nullable: false,
..shape("x")
},
shape("y"),
Some("Cannot merge non-nullable field 'x'"),
),
// A non-nullable item can be hidden where the right item is null.
(
Shape {
item_nullable: false,
..shape("x")
},
Shape {
null_first_item: true,
..shape("y")
},
None,
),
];
for (case, (left, right, rejected)) in cases.into_iter().enumerate() {
let merged = batch(left, null_first).merge(&batch(right, all_valid));
match rejected {
Some(expected) => {
let message = merged.unwrap_err().to_string();
assert!(message.contains(expected), "case {case}: {message}");
}
None => assert!(merged.is_ok(), "case {case}: {merged:?}"),
}
let agreed = batch(left, null_first).merge(&batch(right, null_first));
assert!(
agreed.is_ok(),
"case {case} with agreeing parents: {agreed:?}"
);
}
}
/// The fragment reader merges a data batch with an all-null placeholder for
/// a nested child the file lacks. Only the placeholder's top-level field is
/// nullable, so a non-nullable field below it must still merge, in either
/// order, keeping the data and filling the new child with nulls.
#[test]
fn test_merge_with_a_placeholder_under_a_non_nullable_field() {
let inner =
|child: &str| -> Fields { vec![Field::new(child, DataType::Int32, true)].into() };
for wrap in ["struct", "list", "large_list"] {
let wrapped = |fields: Fields| match wrap {
"struct" => DataType::Struct(fields),
"list" => {
DataType::List(Arc::new(Field::new("item", DataType::Struct(fields), true)))
}
_ => DataType::LargeList(Arc::new(Field::new(
"item",
DataType::Struct(fields),
true,
))),
};
let outer = |child: &str| {
DataType::Struct(vec![Field::new("inner", wrapped(inner(child)), false)].into())
};
let items: ArrayRef = Arc::new(StructArray::new(
inner("a"),
vec![Arc::new(Int32Array::from(vec![1, 2]))],
None,
));
let data_inner = match wrap {
"struct" => items,
"list" => one_item_per_entry(false, items),
_ => one_item_per_entry(true, items),
};
let data = batch_of_parent(StructArray::new(
vec![Field::new("inner", data_inner.data_type().clone(), false)].into(),
vec![data_inner],
None,
));
let placeholder = batch_of_parent(new_null_array(&outer("b"), 2).as_struct().clone());
for (left, right) in [(&data, &placeholder), (&placeholder, &data)] {
let merged = left.merge(right).unwrap();
let inner = merged
.column(0)
.as_struct()
.column_by_name("inner")
.unwrap();
let items = match wrap {
"struct" => inner.clone(),
"list" => inner.as_list::<i32>().values().clone(),
_ => inner.as_list::<i64>().values().clone(),
};
let items = items.as_struct();
let a = items.column_by_name("a").unwrap();
assert_eq!(a.as_primitive::<Int32Type>().values(), &[1, 2], "{wrap}");
assert_eq!(a.null_count(), 0, "{wrap}");
assert_eq!(items.column_by_name("b").unwrap().null_count(), 2, "{wrap}");
}
}
}
/// The null parent row's values stay hidden below a struct nested in the
/// list item, and below a plain struct field.
#[test]
fn test_merge_hides_nested_structs_of_a_null_parent_row() {
let nested = |child: &str, values: Vec<i32>| {
let leaf = StructArray::new(
vec![Field::new(child, DataType::Int32, true)].into(),
vec![Arc::new(Int32Array::from(values))],
None,
);
StructArray::new(
vec![Field::new("payload", leaf.data_type().clone(), true)].into(),
vec![Arc::new(leaf)],
None,
)
};
let x_of = |payload: &dyn Array, row: usize| {
let x = payload.as_struct().column_by_name("x").unwrap().clone();
x.is_null(row)
};
for large in [false, true] {
let in_list = |child: &str, parent: [bool; 2]| {
let list = one_item_per_entry(large, Arc::new(nested(child, vec![11, 12])));
batch_of_parent(StructArray::new(
vec![Field::new("l", list.data_type().clone(), true)].into(),
vec![list],
Some(parent.to_vec().into()),
))
};
let merged = in_list("x", [false, true])
.merge(&in_list("y", [true, true]))
.unwrap();
let list = merged.column(0).as_struct().column(0);
let item = if large {
list.as_list::<i64>().value(0)
} else {
list.as_list::<i32>().value(0)
};
let payload = item.as_struct().column_by_name("payload").unwrap();
assert!(x_of(payload.as_ref(), 0), "large={large}");
}
let in_struct = |child: &str, parent: [bool; 2]| {
let inner = nested(child, vec![11, 12]);
batch_of_parent(StructArray::new(
vec![Field::new("c", inner.data_type().clone(), true)].into(),
vec![Arc::new(inner)],
Some(parent.to_vec().into()),
))
};
let merged = in_struct("x", [false, true])
.merge(&in_struct("y", [true, true]))
.unwrap();
let c = merged.column(0).as_struct().column(0);
let payload = c.as_struct().column_by_name("payload").unwrap();
assert!(x_of(payload.as_ref(), 0));
assert!(!x_of(payload.as_ref(), 1));
}
/// A batch whose parent rows are all null merges with an all-null
/// placeholder whose item struct holds more fields.
#[test]
fn test_merge_list_struct_under_null_parents_with_a_placeholder() {
for large in [false, true] {
let a = Field::new("a", DataType::Int32, true);
let inner = StructArray::new(
vec![a.clone()].into(),
vec![Arc::new(Int32Array::from(vec![1, 2]))],
None,
);
let items = Arc::new(StructArray::new(
vec![Field::new("x", inner.data_type().clone(), true)].into(),
vec![Arc::new(inner)],
None,
));
let list = one_item_per_entry(large, items);
let left = batch_of_parent(StructArray::new(
vec![Field::new("l", list.data_type().clone(), true)].into(),
vec![list],
Some(vec![false, false].into()),
));
let wider_x =
DataType::Struct(vec![a.clone(), Field::new("b", DataType::Int32, true)].into());
let wider_items = new_null_array(
&DataType::Struct(vec![Field::new("x", wider_x, true)].into()),
2,
);
let wider_list = one_item_per_entry(large, wider_items);
let placeholder = new_null_array(
&DataType::Struct(
vec![Field::new("l", wider_list.data_type().clone(), true)].into(),
),
2,
);
let right = batch_of_parent(placeholder.as_struct().clone());
let merged = left.merge(&right).unwrap();
assert_eq!(merged.column(0).null_count(), 2, "large={large}");
}
}
/// A side whose list entries are all hidden by null parents merges like a
/// placeholder, so the other side's lists survive even with other offsets.
#[test]
fn test_merge_list_struct_hidden_side_keeps_the_other_offsets() {
for large in [false, true] {
let list_of = |child: &str, lengths: Vec<usize>, values: Vec<i32>| -> ArrayRef {
let items: ArrayRef = Arc::new(StructArray::new(
vec![Field::new(child, DataType::Int32, true)].into(),
vec![Arc::new(Int32Array::from(values))],
None,
));
let field = Arc::new(Field::new("item", items.data_type().clone(), true));
if large {
Arc::new(LargeListArray::new(
field,
OffsetBuffer::<i64>::from_lengths(lengths),
items,
None,
))
} else {
Arc::new(ListArray::new(
field,
OffsetBuffer::<i32>::from_lengths(lengths),
items,
None,
))
}
};
let parent = |list: ArrayRef, validity: Vec<bool>| {
batch_of_parent(StructArray::new(
vec![Field::new("l", list.data_type().clone(), true)].into(),
vec![list],
Some(validity.into()),
))
};
let left = parent(list_of("x", vec![1, 1], vec![1, 2]), vec![false, false]);
let right = parent(list_of("y", vec![2, 0], vec![7, 8]), vec![true, true]);
let merged = left.merge(&right).unwrap();
let list = merged.column(0).as_struct().column(0);
let (first, second) = if large {
let list = list.as_list::<i64>();
(list.value(0), list.value(1))
} else {
let list = list.as_list::<i32>();
(list.value(0), list.value(1))
};
let first = first.as_struct();
assert_eq!(
first
.column_by_name("y")
.unwrap()
.as_primitive::<Int32Type>()
.values(),
&[7, 8],
"large={large}"
);
assert_eq!(
first.column_by_name("x").unwrap().null_count(),
2,
"large={large}"
);
assert_eq!(second.len(), 0, "large={large}");
}
}
#[test]
fn test_byte_width_opt() {
assert_eq!(DataType::Int32.byte_width_opt(), Some(4));
@@ -2315,7 +2920,7 @@ mod tests {
);
// Merge the two structs
let merged = merge(&left_struct, &right_struct);
let merged = merge(&left_struct, &right_struct).unwrap();
// Expected:
// Row 1: both non-null -> {width: 300, height: 500}
@@ -2355,7 +2960,7 @@ mod tests {
Some(vec![false, false].into()),
);
let merged = merge(&all_null_left, &all_null_right);
let merged = merge(&all_null_left, &all_null_right).unwrap();
assert_eq!(merged.null_count(), 2);
// An all-null side is the identity of the merge, so the other side decides each row.
@@ -2364,7 +2969,7 @@ mod tests {
vec![Arc::new(Int32Array::from(vec![Some(1), None])) as ArrayRef],
Some(vec![true, false].into()),
);
let merged = merge(&partial_left, &all_null_right);
let merged = merge(&partial_left, &all_null_right).unwrap();
assert!(!merged.is_null(0));
assert!(merged.is_null(1));
@@ -2374,7 +2979,7 @@ mod tests {
vec![Arc::new(Int32Array::from(vec![1, 2])) as ArrayRef],
None,
);
let merged = merge(&all_valid_left, &all_null_right);
let merged = merge(&all_valid_left, &all_null_right).unwrap();
assert_eq!(merged.null_count(), 0);
// An explicit all-valid buffer has the same semantics as a missing buffer.
@@ -2401,7 +3006,7 @@ mod tests {
);
// Previously panicked: "Arrays of type Null cannot contain a null bitmask"
let merged = merge(&left_struct, &right_struct);
let merged = merge(&left_struct, &right_struct).unwrap();
assert_eq!(merged.len(), 2);
let b_col = merged.column_by_name("b").unwrap();
// DataType::Null implies all-null by definition; no null bitmap is stored.
python/vendor/lance-arrow/src/stream.rs
Vendored
+54 -39
View File
@@ -8,16 +8,21 @@ use arrow_array::RecordBatch;
use arrow_schema::{ArrowError, SchemaRef};
use futures::stream::{self, Stream, StreamExt};
use std::pin::Pin;
use std::vec::IntoIter;
use crate::deepcopy::deep_copy_batch_sliced;
/// Rechunks a stream of [`RecordBatch`] so that each output batch has
/// approximately `target_bytes` of array data.
/// approximately `max_bytes` of array data.
///
/// Small input batches are accumulated (by concatenation) until at least
/// `min_bytes` of data has been collected. If the resulting batch exceeds
/// `max_bytes`, it is sliced into roughly equal pieces of ~`max_bytes`
/// (assuming uniform row sizes).
///
/// Slices share the input's buffers, so their array memory sizes may still
/// report the full parent allocation. Once split, all slices are delivered
/// directly, including a final slice smaller than `min_bytes`.
pub fn rechunk_stream_by_size<S, E>(
input: S,
input_schema: SchemaRef,
@@ -74,6 +79,7 @@ where
RechunkState {
input: Box::pin(input),
accumulated: Vec::new(),
slices: Vec::new().into_iter(),
acc_bytes: 0,
done: false,
input_schema,
@@ -82,6 +88,12 @@ where
deep_copy,
},
|mut state| async move {
// These slices are already sized. Measuring their shared buffers
// again would repeatedly split the tail using the parent size.
if let Some(batch) = state.slices.next() {
return Ok(Some((batch, state)));
}
if state.done && state.accumulated.is_empty() {
return Ok(None);
}
@@ -106,17 +118,6 @@ where
return Ok(None);
}
// Fast path: if the first accumulated batch already meets the
// byte threshold, deliver it directly instead of concatenating
// everything together (which would just get sliced back apart).
if state.accumulated.len() > 1
&& state.accumulated[0].get_array_memory_size() >= state.min_bytes
{
let b = state.accumulated.remove(0);
state.acc_bytes -= b.get_array_memory_size();
return Ok(Some((b, state)));
}
let batch = if state.accumulated.len() == 1 {
state.accumulated.pop().unwrap()
} else {
@@ -132,19 +133,9 @@ where
let mut slices =
slice_batch(batch, state.max_bytes, state.deep_copy).map_err(E::from)?;
if slices.len() == 1 {
Ok(Some((slices.pop().unwrap(), state)))
} else {
let first = slices.remove(0);
// Stash leftover slices for subsequent iterations.
for a in &slices {
state.acc_bytes += a.get_array_memory_size();
}
state.accumulated = slices;
Ok(Some((first, state)))
}
let first = slices.remove(0);
state.slices = slices.into_iter();
Ok(Some((first, state)))
},
)
}
@@ -206,12 +197,13 @@ fn slice_batch(
Ok(result)
}
/// Internal state for [`rechunk_stream`].
/// Internal state for [`rechunk_stream_by_size`].
///
/// Kept as a named struct so the `try_unfold` closure stays readable.
struct RechunkState<S> {
input: Pin<Box<S>>,
accumulated: Vec<RecordBatch>,
slices: IntoIter<RecordBatch>,
acc_bytes: usize,
done: bool,
input_schema: SchemaRef,
@@ -339,9 +331,9 @@ mod tests {
#[test]
fn test_sliced_leftovers_are_not_recombined() {
// Key test for the fast-path optimisation. When a large batch is
// sliced, leftover slices should be delivered one-at-a-time without
// being concatenated back together. We verify this by checking that
// When a large batch is sliced, leftover slices should be delivered
// one at a time without being split again or concatenated back
// together. We verify zero-copy behavior by checking that
// every output buffer pointer falls inside the original batch's
// allocation (i.e. they are all zero-copy slices, not fresh copies).
let batch = make_batch(1000);
@@ -354,7 +346,11 @@ mod tests {
let result = collect_rechunked(vec![batch], bytes / 8, bytes / 4);
assert_eq!(total_rows(&result), 1000);
assert!(result.len() >= 4);
assert_eq!(
result.iter().map(RecordBatch::num_rows).collect::<Vec<_>>(),
vec![250, 250, 250, 250],
"leftover slices must not be split again using the parent buffer size"
);
for (i, b) in result.iter().enumerate() {
let ptr = b.column(0).to_data().buffers()[0].as_ptr() as usize;
@@ -381,6 +377,7 @@ mod tests {
// After a large batch is fully drained, subsequent small batches
// should be accumulated normally.
let large = make_batch(1000);
let large_bytes = large.get_array_memory_size();
let small_bytes = make_batch(10).get_array_memory_size();
let batches = vec![
large,
@@ -390,12 +387,12 @@ mod tests {
make_batch(10),
make_batch(10),
];
let result = collect_rechunked(batches, small_bytes * 3, small_bytes * 100);
let result = collect_rechunked(batches, small_bytes * 3, large_bytes / 4);
assert_eq!(total_rows(&result), 1050);
// The large batch should appear (possibly sliced) followed by
// concatenated small batches, so we should have fewer output batches
// than the 6 inputs.
assert!(result.len() < 6);
assert_eq!(
result.iter().map(RecordBatch::num_rows).collect::<Vec<_>>(),
vec![250, 250, 250, 250, 30, 20]
);
}
#[test]
@@ -441,10 +438,9 @@ mod tests {
let bytes = batch.get_array_memory_size();
let result = collect_rechunked(vec![batch], 0, bytes / 4);
assert_eq!(total_rows(&result), 1000);
assert!(
result.len() >= 4,
"expected at least 4 slices, got {}",
result.len()
assert_eq!(
result.iter().map(RecordBatch::num_rows).collect::<Vec<_>>(),
vec![250, 250, 250, 250]
);
}
@@ -475,6 +471,25 @@ mod tests {
Arc::new(Schema::new(vec![Field::new("s", DataType::Utf8, false)]))
}
#[test]
fn test_variable_length_tail_is_not_resliced() {
let batch = make_variable_batch(100, 256, usize::MAX, 256);
let bytes = batch.get_array_memory_size();
let max_bytes = bytes - bytes / 4;
let input = stream::iter(vec![Ok::<_, ArrowError>(batch)]);
let rechunked = rechunk_stream_by_size(input, variable_schema(), max_bytes, max_bytes);
let result: Vec<_> = block_on(rechunked.collect::<Vec<_>>())
.into_iter()
.map(Result::unwrap)
.collect();
assert_eq!(
result.iter().map(RecordBatch::num_rows).collect::<Vec<_>>(),
vec![75, 25]
);
assert_eq!(total_rows(&result), 100);
}
fn collect_rechunked_variable(
batches: Vec<RecordBatch>,
min_bytes: usize,