rust/lancedb
1
0
Fork 0
mirror of https://github.com/lancedb/lancedb.git synced 2026-08-18 12:08:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

chore: upgrade DataFusion to 54

Browse Source 
Bumps DataFusion 53 -> 54, tracking lance-format/lance#7793 (lance's DF54
upgrade) via the `chore/upgrade-datafusion-54` branch (rev 2ebc588).

DataFusion 54 breaking changes migrated:

- `as_any` was removed from the `ExecutionPlan`, `TableProvider`, and
  `ScalarUDFImpl` traits. Dropped the impls on `MetadataEraserExec`,
  `ScannableExec`, `InsertExec`, `RemoteInsertExec`, `BaseTableAdapter`,
  and `RejectNanUdf`, and downcast via the inherent
  `dyn Trait::downcast_ref` instead of `.as_any().downcast_ref()`.
- `ExecutionPlan::partition_statistics` now returns `Arc<Statistics>`.
- `datafusion_common::hash_utils::create_hashes` now takes the DF
  `RandomState` (foldhash `FixedState`) instead of `ahash::RandomState`.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Will Jones
2026-07-14 18:36:41 -07:00
parent 8a4eaaa8b9
commit 4d6d9082e7
11 changed files with 143 additions and 175 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Cargo.lock
Generated
+112 -115
View File
@@ -2288,14 +2288,13 @@ dependencies = [
[[package]]
name = "datafusion"
version = "53.1.0"
version = "54.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "93db0e623840612f7f2cd757f7e8a8922064192363732c88692e0870016e141b"
checksum = "997a31e15872606a49478e670c58302094c97cb96abb0a7d60720f8e92170040"
dependencies = [
"arrow",
"arrow-schema",
"async-trait",
"bytes",
"chrono",
"datafusion-catalog",
"datafusion-catalog-listing",
@@ -2322,13 +2321,12 @@ dependencies = [
"datafusion-session",
"datafusion-sql",
"futures",
"indexmap 2.14.0",
"itertools 0.14.0",
"log",
"object_store",
"parking_lot",
"rand 0.9.5",
"regex",
"sqlparser 0.61.0",
"sqlparser 0.62.0",
"tempfile",
"tokio",
"url",
@@ -2337,9 +2335,9 @@ dependencies = [
[[package]]
name = "datafusion-catalog"
version = "53.1.0"
version = "54.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "37cefde60b26a7f4ff61e9d2ff2833322f91df2b568d7238afe67bde5bdffb66"
checksum = "f7dd61161508f8f5fa1107774ea687bd753c22d83a32eebf963549f89de14139"
dependencies = [
"arrow",
"async-trait",
@@ -2362,9 +2360,9 @@ dependencies = [
[[package]]
name = "datafusion-catalog-listing"
version = "53.1.0"
version = "54.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "17e112307715d6a7a331111a4c2330ff54bc237183511c319e3708a4cff431fb"
checksum = "897c70f871277f9ce99aa38347be0d679bbe3e617156c4d2a8378cec8a2a0891"
dependencies = [
"arrow",
"async-trait",
@@ -2385,32 +2383,33 @@ dependencies = [
[[package]]
name = "datafusion-common"
version = "53.1.0"
version = "54.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d72a11ca44a95e1081870d3abb80c717496e8a7acb467a1d3e932bb636af5cc2"
checksum = "121c9ded5d87d9172319e006f2afdb9928d72dbacd6a90a458d8acb1e3b43a65"
dependencies = [
"ahash",
"arrow",
"arrow-ipc",
"arrow-schema",
"chrono",
"foldhash 0.2.0",
"half",
"hashbrown 0.16.1",
"hashbrown 0.17.1",
"indexmap 2.14.0",
"itertools 0.14.0",
"libc",
"log",
"object_store",
"paste",
"sqlparser 0.61.0",
"sqlparser 0.62.0",
"tokio",
"uuid",
"web-time",
]
[[package]]
name = "datafusion-common-runtime"
version = "53.1.0"
version = "54.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "89f4afaed29670ec4fd6053643adc749fe3f4bc9d1ce1b8c5679b22c67d12def"
checksum = "981b9dae74f78ee3d9f714fb49b01919eab975461b56149510c3ba9ea11287d1"
dependencies = [
"futures",
"log",
@@ -2419,9 +2418,9 @@ dependencies = [
[[package]]
name = "datafusion-datasource"
version = "53.1.0"
version = "54.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e9fb386e1691355355a96419978a0022b7947b44d4a24a6ea99f00b6b485cbb6"
checksum = "ffd7d295b2ec7c00d8a56562f41ed41062cf0af75549ed891c12a0a09eddfefe"
dependencies = [
"arrow",
"async-trait",
@@ -2441,6 +2440,7 @@ dependencies = [
"itertools 0.14.0",
"log",
"object_store",
"parking_lot",
"rand 0.9.5",
"tokio",
"url",
@@ -2448,9 +2448,9 @@ dependencies = [
[[package]]
name = "datafusion-datasource-arrow"
version = "53.1.0"
version = "54.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ffa6c52cfed0734c5f93754d1c0175f558175248bf686c944fb05c373e5fc096"
checksum = "552b0b3f342f7ec41b3fbd70f6339dc82a30cfd0349e7f280e7852528085349f"
dependencies = [
"arrow",
"arrow-ipc",
@@ -2472,9 +2472,9 @@ dependencies = [
[[package]]
name = "datafusion-datasource-csv"
version = "53.1.0"
version = "54.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "503f29e0582c1fc189578d665ff57d9300da1f80c282777d7eb67bb79fb8cdca"
checksum = "68850aa426b897e879c8b87e512ea8124f1d0a2869a4e51808ddaaddf1bc0ada"
dependencies = [
"arrow",
"async-trait",
@@ -2495,9 +2495,9 @@ dependencies = [
[[package]]
name = "datafusion-datasource-json"
version = "53.1.0"
version = "54.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e33804749abc8d0c8cb7473228483cb8070e524c6f6086ee1b85a64debe2b3d2"
checksum = "402f93242ae08ef99139ee2c528a49d087efe88d5c7b2c3ff5480855a40ce54f"
dependencies = [
"arrow",
"async-trait",
@@ -2512,27 +2512,25 @@ dependencies = [
"datafusion-session",
"futures",
"object_store",
"serde_json",
"tokio",
"tokio-stream",
]
[[package]]
name = "datafusion-doc"
version = "53.1.0"
version = "54.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8de6ac0df1662b9148ad3c987978b32cbec7c772f199b1d53520c8fa764a87ee"
checksum = "cb9e7e5d11130c48c8bd4e80c79a9772dd28ce6dc330baca9246205d245b9e2e"
[[package]]
name = "datafusion-execution"
version = "53.1.0"
version = "54.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c03c7fbdaefcca4ef6ffe425a5fc2325763bfb426599bb0bf4536466efabe709"
checksum = "37a8643ab852eb68864e1b72ae789e8066282dce48eea6347ffb0aee33d1ccc0"
dependencies = [
"arrow",
"arrow-buffer",
"async-trait",
"chrono",
"dashmap",
"datafusion-common",
"datafusion-expr",
@@ -2548,11 +2546,12 @@ dependencies = [
[[package]]
name = "datafusion-expr"
version = "53.1.0"
version = "54.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "574b9b6977fedbd2a611cbff12e5caf90f31640ad9dc5870f152836d94bad0dd"
checksum = "6932f4d71eed9c8d9341476a2b845aadfabde5495d08dbcd8fc23881f49fa7a0"
dependencies = [
"arrow",
"arrow-schema",
"async-trait",
"chrono",
"datafusion-common",
@@ -2563,29 +2562,27 @@ dependencies = [
"datafusion-physical-expr-common",
"indexmap 2.14.0",
"itertools 0.14.0",
"paste",
"serde_json",
"sqlparser 0.61.0",
"sqlparser 0.62.0",
]
[[package]]
name = "datafusion-expr-common"
version = "53.1.0"
version = "54.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7d7c3adf3db8bf61e92eb90cb659c8e8b734593a8f7c8e12a843c7ddba24b87e"
checksum = "0225491839a31b1f7d2cb8092c2d50792e2fe1c1724e4e6d08e011f5feaf4ed2"
dependencies = [
"arrow",
"datafusion-common",
"indexmap 2.14.0",
"itertools 0.14.0",
"paste",
]
[[package]]
name = "datafusion-functions"
version = "53.1.0"
version = "54.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f28aa4e10384e782774b10e72aca4d93ef7b31aa653095d9d4536b0a3dbc51b6"
checksum = "14872c47bfc3d21e53ec82f57074e6987a15941c1e2f43cde4ac6ae2746634e3"
dependencies = [
"arrow",
"arrow-buffer",
@@ -2600,26 +2597,25 @@ dependencies = [
"datafusion-expr",
"datafusion-expr-common",
"datafusion-macros",
"datafusion-physical-expr-common",
"hex",
"itertools 0.14.0",
"log",
"md-5 0.10.6",
"md-5 0.11.0",
"memchr",
"num-traits",
"rand 0.9.5",
"regex",
"sha2 0.10.9",
"unicode-segmentation",
"sha2 0.11.0",
"uuid",
]
[[package]]
name = "datafusion-functions-aggregate"
version = "53.1.0"
version = "54.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "00aa6217e56098ba84e0a338176fe52f0a84cca398021512c6c8c5eff806d0ad"
checksum = "75a2ca14e1b609be21e657e2d3130b2f446456b08393b377bb721a33952d2e09"
dependencies = [
"ahash",
"arrow",
"datafusion-common",
"datafusion-doc",
@@ -2629,19 +2625,18 @@ dependencies = [
"datafusion-macros",
"datafusion-physical-expr",
"datafusion-physical-expr-common",
"foldhash 0.2.0",
"half",
"log",
"num-traits",
"paste",
]
[[package]]
name = "datafusion-functions-aggregate-common"
version = "53.1.0"
version = "54.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b511250349407db7c43832ab2de63f5557b19a20dfd236b39ca2c04468b50d47"
checksum = "1ece74ba09092d2ef9c9b54a38445450aea292a1f8b04faf531936b723a24b3c"
dependencies = [
"ahash",
"arrow",
"datafusion-common",
"datafusion-expr-common",
@@ -2650,9 +2645,9 @@ dependencies = [
[[package]]
name = "datafusion-functions-nested"
version = "53.1.0"
version = "54.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ef13a858e20d50f0a9bb5e96e7ac82b4e7597f247515bccca4fdd2992df0212a"
checksum = "3f3e3f9ee8ca59bf70518802107de6f1b88a9509efdc629fadc5de9d6b2d5ef5"
dependencies = [
"arrow",
"arrow-ord",
@@ -2666,34 +2661,34 @@ dependencies = [
"datafusion-functions-aggregate-common",
"datafusion-macros",
"datafusion-physical-expr-common",
"hashbrown 0.16.1",
"hashbrown 0.17.1",
"itertools 0.14.0",
"itoa",
"log",
"paste",
"memchr",
]
[[package]]
name = "datafusion-functions-table"
version = "53.1.0"
version = "54.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "72b40d3f5bbb3905f9ccb1ce9485a9595c77b69758a7c24d3ba79e334ff51e7e"
checksum = "89161dffc22cf2b50f9f4b1bee83b5221d3b4ed7c2e37fd7aa2b22a5297b3a26"
dependencies = [
"arrow",
"async-trait",
"datafusion-catalog",
"datafusion-common",
"datafusion-expr",
"datafusion-physical-expr",
"datafusion-physical-plan",
"parking_lot",
"paste",
]
[[package]]
name = "datafusion-functions-window"
version = "53.1.0"
version = "54.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d4e88ec9d57c9b685d02f58bfee7be62d72610430ddcedb82a08e5d9925dbfb6"
checksum = "d7339345b226b3874037708bf5023ba1c2de705128f8457a095aae5ae9cb9c78"
dependencies = [
"arrow",
"datafusion-common",
@@ -2704,14 +2699,13 @@ dependencies = [
"datafusion-physical-expr",
"datafusion-physical-expr-common",
"log",
"paste",
]
[[package]]
name = "datafusion-functions-window-common"
version = "53.1.0"
version = "54.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8307bb93519b1a91913723a1130cfafeee3f72200d870d88e91a6fc5470ede5c"
checksum = "fa84836dc2392df6f43d6a29d37fb56a8ebdc8b3f4e10ae8dc15861fd20278fb"
dependencies = [
"datafusion-common",
"datafusion-physical-expr-common",
@@ -2719,9 +2713,9 @@ dependencies = [
[[package]]
name = "datafusion-macros"
version = "53.1.0"
version = "54.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2e367e6a71051d0ebdd29b2f85d12059b38b1d1f172c6906e80016da662226bd"
checksum = "587164e03ad68732aa9e7bfe5686e3f25970d4c64fd4bd80790749840892dae5"
dependencies = [
"datafusion-doc",
"quote",
@@ -2730,9 +2724,9 @@ dependencies = [
[[package]]
name = "datafusion-optimizer"
version = "53.1.0"
version = "54.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e929015451a67f77d9d8b727b2bf3a40c4445fdef6cdc53281d7d97c76888ace"
checksum = "77f20e8cf9e8654d92f4c16b24c487353ee5bf153ffc12d5772cd399ab8cd281"
dependencies = [
"arrow",
"chrono",
@@ -2749,11 +2743,10 @@ dependencies = [
[[package]]
name = "datafusion-physical-expr"
version = "53.1.0"
version = "54.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4b1e68aba7a4b350401cfdf25a3d6f989ad898a7410164afe9ca52080244cb59"
checksum = "f015a4a82f6f7ff7e1d8d4bf3870a936752fa38b17705dfcc14adef95aa8922c"
dependencies = [
"ahash",
"arrow",
"datafusion-common",
"datafusion-expr",
@@ -2761,20 +2754,19 @@ dependencies = [
"datafusion-functions-aggregate-common",
"datafusion-physical-expr-common",
"half",
"hashbrown 0.16.1",
"hashbrown 0.17.1",
"indexmap 2.14.0",
"itertools 0.14.0",
"parking_lot",
"paste",
"petgraph",
"tokio",
]
[[package]]
name = "datafusion-physical-expr-adapter"
version = "53.1.0"
version = "54.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ea22315f33cf2e0adc104e8ec42e285f6ed93998d565c65e82fec6a9ee9f9db4"
checksum = "51e6ffff8acdfe54e0ea15ccf38115c4a9184433b0439f42907637928d00a235"
dependencies = [
"arrow",
"datafusion-common",
@@ -2787,26 +2779,26 @@ dependencies = [
[[package]]
name = "datafusion-physical-expr-common"
version = "53.1.0"
version = "54.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b04b45ea8ad3ac2d78f2ea2a76053e06591c9629c7a603eda16c10649ecf4362"
checksum = "7967a3e171c6a4bf09474b3f7a14f1a3db13ed1714ba12156f33fcce2bba54e8"
dependencies = [
"ahash",
"arrow",
"chrono",
"datafusion-common",
"datafusion-expr-common",
"hashbrown 0.16.1",
"hashbrown 0.17.1",
"indexmap 2.14.0",
"itertools 0.14.0",
"parking_lot",
"pin-project",
]
[[package]]
name = "datafusion-physical-optimizer"
version = "53.1.0"
version = "54.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7cb13397809a425918f608dfe8653f332015a3e330004ab191b4404187238b95"
checksum = "59ff803e2a96054cb6d83f35f9e60fd4f42eac515e1932bd1b2dbc91d5fcbf36"
dependencies = [
"arrow",
"datafusion-common",
@@ -2822,12 +2814,13 @@ dependencies = [
[[package]]
name = "datafusion-physical-plan"
version = "53.1.0"
version = "54.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5edc023675791af9d5fb4cc4c24abf5f7bd3bd4dcf9e5bd90ea1eff6976dcc79"
checksum = "776ee54d47d15bdb126452f9ca17b03761e3b004682914beaedd3f86eb507fbc"
dependencies = [
"ahash",
"arrow",
"arrow-data",
"arrow-ipc",
"arrow-ord",
"arrow-schema",
"async-trait",
@@ -2842,7 +2835,7 @@ dependencies = [
"datafusion-physical-expr-common",
"futures",
"half",
"hashbrown 0.16.1",
"hashbrown 0.17.1",
"indexmap 2.14.0",
"itertools 0.14.0",
"log",
@@ -2854,9 +2847,9 @@ dependencies = [
[[package]]
name = "datafusion-pruning"
version = "53.1.0"
version = "54.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ac8c76860e355616555081cab5968cec1af7a80701ff374510860bcd567e365a"
checksum = "d5fb9e5774660aa69c3ba93c610f175f75b65cb8c3776edb3626de8f3a4f4ee3"
dependencies = [
"arrow",
"datafusion-common",
@@ -2865,15 +2858,14 @@ dependencies = [
"datafusion-physical-expr",
"datafusion-physical-expr-common",
"datafusion-physical-plan",
"itertools 0.14.0",
"log",
]
[[package]]
name = "datafusion-session"
version = "53.1.0"
version = "54.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5412111aa48e2424ba926112e192f7a6b7e4ccb450145d25ce5ede9f19dc491e"
checksum = "15ce715fa2a61f4623cc234bcc14a3ef6a91f189128d5b14b468a6a17cdfc417"
dependencies = [
"async-trait",
"datafusion-common",
@@ -2885,9 +2877,9 @@ dependencies = [
[[package]]
name = "datafusion-sql"
version = "53.1.0"
version = "54.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fa0d133ddf8b9b3b872acac900157f783e7b879fe9a6bccf389abebbfac45ec1"
checksum = "6094ad36a3ed6d7ac87b20b479b2d0b118250f66cf997603829fdc65b44a7099"
dependencies = [
"arrow",
"bigdecimal",
@@ -2898,7 +2890,7 @@ dependencies = [
"indexmap 2.14.0",
"log",
"regex",
"sqlparser 0.61.0",
"sqlparser 0.62.0",
]
[[package]]
@@ -3430,7 +3422,7 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
[[package]]
name = "fsst"
version = "9.0.0-beta.23"
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.23#0acc51eb8f013985395bf3ac7f0ef4f8a23a377d"
source = "git+https://github.com/lance-format/lance.git?rev=2ebc588f809b2937e9dbf1bae48ffdfd844cef61#2ebc588f809b2937e9dbf1bae48ffdfd844cef61"
dependencies = [
"arrow-array",
"rand 0.9.5",
@@ -3917,6 +3909,11 @@ name = "hashbrown"
version = "0.17.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a"
dependencies = [
"allocator-api2",
"equivalent",
"foldhash 0.2.0",
]
[[package]]
name = "heapify"
@@ -4781,7 +4778,7 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a"
[[package]]
name = "lance"
version = "9.0.0-beta.23"
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.23#0acc51eb8f013985395bf3ac7f0ef4f8a23a377d"
source = "git+https://github.com/lance-format/lance.git?rev=2ebc588f809b2937e9dbf1bae48ffdfd844cef61#2ebc588f809b2937e9dbf1bae48ffdfd844cef61"
dependencies = [
"arc-swap",
"arrow",
@@ -4856,7 +4853,7 @@ dependencies = [
[[package]]
name = "lance-arrow"
version = "9.0.0-beta.23"
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.23#0acc51eb8f013985395bf3ac7f0ef4f8a23a377d"
source = "git+https://github.com/lance-format/lance.git?rev=2ebc588f809b2937e9dbf1bae48ffdfd844cef61#2ebc588f809b2937e9dbf1bae48ffdfd844cef61"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -4878,7 +4875,7 @@ dependencies = [
[[package]]
name = "lance-arrow-scalar"
version = "58.0.0"
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.23#0acc51eb8f013985395bf3ac7f0ef4f8a23a377d"
source = "git+https://github.com/lance-format/lance.git?rev=2ebc588f809b2937e9dbf1bae48ffdfd844cef61#2ebc588f809b2937e9dbf1bae48ffdfd844cef61"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -4892,7 +4889,7 @@ dependencies = [
[[package]]
name = "lance-arrow-stats"
version = "58.0.0"
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.23#0acc51eb8f013985395bf3ac7f0ef4f8a23a377d"
source = "git+https://github.com/lance-format/lance.git?rev=2ebc588f809b2937e9dbf1bae48ffdfd844cef61#2ebc588f809b2937e9dbf1bae48ffdfd844cef61"
dependencies = [
"arrow-array",
"arrow-schema",
@@ -4902,7 +4899,7 @@ dependencies = [
[[package]]
name = "lance-bitpacking"
version = "9.0.0-beta.23"
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.23#0acc51eb8f013985395bf3ac7f0ef4f8a23a377d"
source = "git+https://github.com/lance-format/lance.git?rev=2ebc588f809b2937e9dbf1bae48ffdfd844cef61#2ebc588f809b2937e9dbf1bae48ffdfd844cef61"
dependencies = [
"arrayref",
"crunchy",
@@ -4913,7 +4910,7 @@ dependencies = [
[[package]]
name = "lance-core"
version = "9.0.0-beta.23"
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.23#0acc51eb8f013985395bf3ac7f0ef4f8a23a377d"
source = "git+https://github.com/lance-format/lance.git?rev=2ebc588f809b2937e9dbf1bae48ffdfd844cef61#2ebc588f809b2937e9dbf1bae48ffdfd844cef61"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -4952,7 +4949,7 @@ dependencies = [
[[package]]
name = "lance-datafusion"
version = "9.0.0-beta.23"
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.23#0acc51eb8f013985395bf3ac7f0ef4f8a23a377d"
source = "git+https://github.com/lance-format/lance.git?rev=2ebc588f809b2937e9dbf1bae48ffdfd844cef61#2ebc588f809b2937e9dbf1bae48ffdfd844cef61"
dependencies = [
"arrow",
"arrow-array",
@@ -4983,7 +4980,7 @@ dependencies = [
[[package]]
name = "lance-datagen"
version = "9.0.0-beta.23"
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.23#0acc51eb8f013985395bf3ac7f0ef4f8a23a377d"
source = "git+https://github.com/lance-format/lance.git?rev=2ebc588f809b2937e9dbf1bae48ffdfd844cef61#2ebc588f809b2937e9dbf1bae48ffdfd844cef61"
dependencies = [
"arrow",
"arrow-array",
@@ -5001,7 +4998,7 @@ dependencies = [
[[package]]
name = "lance-derive"
version = "9.0.0-beta.23"
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.23#0acc51eb8f013985395bf3ac7f0ef4f8a23a377d"
source = "git+https://github.com/lance-format/lance.git?rev=2ebc588f809b2937e9dbf1bae48ffdfd844cef61#2ebc588f809b2937e9dbf1bae48ffdfd844cef61"
dependencies = [
"proc-macro2",
"quote",
@@ -5011,7 +5008,7 @@ dependencies = [
[[package]]
name = "lance-encoding"
version = "9.0.0-beta.23"
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.23#0acc51eb8f013985395bf3ac7f0ef4f8a23a377d"
source = "git+https://github.com/lance-format/lance.git?rev=2ebc588f809b2937e9dbf1bae48ffdfd844cef61#2ebc588f809b2937e9dbf1bae48ffdfd844cef61"
dependencies = [
"arrow-arith",
"arrow-array",
@@ -5047,7 +5044,7 @@ dependencies = [
[[package]]
name = "lance-file"
version = "9.0.0-beta.23"
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.23#0acc51eb8f013985395bf3ac7f0ef4f8a23a377d"
source = "git+https://github.com/lance-format/lance.git?rev=2ebc588f809b2937e9dbf1bae48ffdfd844cef61#2ebc588f809b2937e9dbf1bae48ffdfd844cef61"
dependencies = [
"arrow-arith",
"arrow-array",
@@ -5078,7 +5075,7 @@ dependencies = [
[[package]]
name = "lance-index"
version = "9.0.0-beta.23"
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.23#0acc51eb8f013985395bf3ac7f0ef4f8a23a377d"
source = "git+https://github.com/lance-format/lance.git?rev=2ebc588f809b2937e9dbf1bae48ffdfd844cef61#2ebc588f809b2937e9dbf1bae48ffdfd844cef61"
dependencies = [
"arc-swap",
"arrow",
@@ -5144,7 +5141,7 @@ dependencies = [
[[package]]
name = "lance-io"
version = "9.0.0-beta.23"
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.23#0acc51eb8f013985395bf3ac7f0ef4f8a23a377d"
source = "git+https://github.com/lance-format/lance.git?rev=2ebc588f809b2937e9dbf1bae48ffdfd844cef61#2ebc588f809b2937e9dbf1bae48ffdfd844cef61"
dependencies = [
"arrow",
"arrow-arith",
@@ -5187,7 +5184,7 @@ dependencies = [
[[package]]
name = "lance-linalg"
version = "9.0.0-beta.23"
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.23#0acc51eb8f013985395bf3ac7f0ef4f8a23a377d"
source = "git+https://github.com/lance-format/lance.git?rev=2ebc588f809b2937e9dbf1bae48ffdfd844cef61#2ebc588f809b2937e9dbf1bae48ffdfd844cef61"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -5204,7 +5201,7 @@ dependencies = [
[[package]]
name = "lance-namespace"
version = "9.0.0-beta.23"
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.23#0acc51eb8f013985395bf3ac7f0ef4f8a23a377d"
source = "git+https://github.com/lance-format/lance.git?rev=2ebc588f809b2937e9dbf1bae48ffdfd844cef61#2ebc588f809b2937e9dbf1bae48ffdfd844cef61"
dependencies = [
"arrow",
"async-trait",
@@ -5217,7 +5214,7 @@ dependencies = [
[[package]]
name = "lance-namespace-impls"
version = "9.0.0-beta.23"
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.23#0acc51eb8f013985395bf3ac7f0ef4f8a23a377d"
source = "git+https://github.com/lance-format/lance.git?rev=2ebc588f809b2937e9dbf1bae48ffdfd844cef61#2ebc588f809b2937e9dbf1bae48ffdfd844cef61"
dependencies = [
"arrow",
"arrow-ipc",
@@ -5272,7 +5269,7 @@ dependencies = [
[[package]]
name = "lance-select"
version = "9.0.0-beta.23"
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.23#0acc51eb8f013985395bf3ac7f0ef4f8a23a377d"
source = "git+https://github.com/lance-format/lance.git?rev=2ebc588f809b2937e9dbf1bae48ffdfd844cef61#2ebc588f809b2937e9dbf1bae48ffdfd844cef61"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -5288,7 +5285,7 @@ dependencies = [
[[package]]
name = "lance-table"
version = "9.0.0-beta.23"
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.23#0acc51eb8f013985395bf3ac7f0ef4f8a23a377d"
source = "git+https://github.com/lance-format/lance.git?rev=2ebc588f809b2937e9dbf1bae48ffdfd844cef61#2ebc588f809b2937e9dbf1bae48ffdfd844cef61"
dependencies = [
"arrow",
"arrow-array",
@@ -5328,7 +5325,7 @@ dependencies = [
[[package]]
name = "lance-testing"
version = "9.0.0-beta.23"
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.23#0acc51eb8f013985395bf3ac7f0ef4f8a23a377d"
source = "git+https://github.com/lance-format/lance.git?rev=2ebc588f809b2937e9dbf1bae48ffdfd844cef61#2ebc588f809b2937e9dbf1bae48ffdfd844cef61"
dependencies = [
"arrow-array",
"arrow-schema",
@@ -5342,7 +5339,7 @@ dependencies = [
[[package]]
name = "lance-tokenizer"
version = "9.0.0-beta.23"
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.23#0acc51eb8f013985395bf3ac7f0ef4f8a23a377d"
source = "git+https://github.com/lance-format/lance.git?rev=2ebc588f809b2937e9dbf1bae48ffdfd844cef61#2ebc588f809b2937e9dbf1bae48ffdfd844cef61"
dependencies = [
"icu_segmenter",
"jieba-rs",
@@ -9352,9 +9349,9 @@ dependencies = [
[[package]]
name = "sqlparser"
version = "0.61.0"
version = "0.62.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dbf5ea8d4d7c808e1af1cbabebca9a2abe603bcefc22294c5b95018d53200cb7"
checksum = "13c6d1b651dc4edf07eead2a0c6c78016ce971bc2c10da5266861b13f25e7cec"
dependencies = [
"log",
"sqlparser_derive",
Cargo.toml
+23 -23
View File
@@ -13,20 +13,20 @@ categories = ["database-implementations"]
rust-version = "1.91.0"
[workspace.dependencies]
lance = { "version" = "=9.0.0-beta.23", default-features = false, "tag" = "v9.0.0-beta.23", "git" = "https://github.com/lance-format/lance.git" }
lance-core = { "version" = "=9.0.0-beta.23", "tag" = "v9.0.0-beta.23", "git" = "https://github.com/lance-format/lance.git" }
lance-datagen = { "version" = "=9.0.0-beta.23", "tag" = "v9.0.0-beta.23", "git" = "https://github.com/lance-format/lance.git" }
lance-file = { "version" = "=9.0.0-beta.23", "tag" = "v9.0.0-beta.23", "git" = "https://github.com/lance-format/lance.git" }
lance-io = { "version" = "=9.0.0-beta.23", default-features = false, "tag" = "v9.0.0-beta.23", "git" = "https://github.com/lance-format/lance.git" }
lance-index = { "version" = "=9.0.0-beta.23", "tag" = "v9.0.0-beta.23", "git" = "https://github.com/lance-format/lance.git" }
lance-linalg = { "version" = "=9.0.0-beta.23", "tag" = "v9.0.0-beta.23", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace = { "version" = "=9.0.0-beta.23", "tag" = "v9.0.0-beta.23", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace-impls = { "version" = "=9.0.0-beta.23", default-features = false, "tag" = "v9.0.0-beta.23", "git" = "https://github.com/lance-format/lance.git" }
lance-table = { "version" = "=9.0.0-beta.23", "tag" = "v9.0.0-beta.23", "git" = "https://github.com/lance-format/lance.git" }
lance-testing = { "version" = "=9.0.0-beta.23", "tag" = "v9.0.0-beta.23", "git" = "https://github.com/lance-format/lance.git" }
lance-datafusion = { "version" = "=9.0.0-beta.23", "tag" = "v9.0.0-beta.23", "git" = "https://github.com/lance-format/lance.git" }
lance-encoding = { "version" = "=9.0.0-beta.23", "tag" = "v9.0.0-beta.23", "git" = "https://github.com/lance-format/lance.git" }
lance-arrow = { "version" = "=9.0.0-beta.23", "tag" = "v9.0.0-beta.23", "git" = "https://github.com/lance-format/lance.git" }
lance = { "version" = "=9.0.0-beta.23", default-features = false, "rev" = "2ebc588f809b2937e9dbf1bae48ffdfd844cef61", "git" = "https://github.com/lance-format/lance.git" }
lance-core = { "version" = "=9.0.0-beta.23", "rev" = "2ebc588f809b2937e9dbf1bae48ffdfd844cef61", "git" = "https://github.com/lance-format/lance.git" }
lance-datagen = { "version" = "=9.0.0-beta.23", "rev" = "2ebc588f809b2937e9dbf1bae48ffdfd844cef61", "git" = "https://github.com/lance-format/lance.git" }
lance-file = { "version" = "=9.0.0-beta.23", "rev" = "2ebc588f809b2937e9dbf1bae48ffdfd844cef61", "git" = "https://github.com/lance-format/lance.git" }
lance-io = { "version" = "=9.0.0-beta.23", default-features = false, "rev" = "2ebc588f809b2937e9dbf1bae48ffdfd844cef61", "git" = "https://github.com/lance-format/lance.git" }
lance-index = { "version" = "=9.0.0-beta.23", "rev" = "2ebc588f809b2937e9dbf1bae48ffdfd844cef61", "git" = "https://github.com/lance-format/lance.git" }
lance-linalg = { "version" = "=9.0.0-beta.23", "rev" = "2ebc588f809b2937e9dbf1bae48ffdfd844cef61", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace = { "version" = "=9.0.0-beta.23", "rev" = "2ebc588f809b2937e9dbf1bae48ffdfd844cef61", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace-impls = { "version" = "=9.0.0-beta.23", default-features = false, "rev" = "2ebc588f809b2937e9dbf1bae48ffdfd844cef61", "git" = "https://github.com/lance-format/lance.git" }
lance-table = { "version" = "=9.0.0-beta.23", "rev" = "2ebc588f809b2937e9dbf1bae48ffdfd844cef61", "git" = "https://github.com/lance-format/lance.git" }
lance-testing = { "version" = "=9.0.0-beta.23", "rev" = "2ebc588f809b2937e9dbf1bae48ffdfd844cef61", "git" = "https://github.com/lance-format/lance.git" }
lance-datafusion = { "version" = "=9.0.0-beta.23", "rev" = "2ebc588f809b2937e9dbf1bae48ffdfd844cef61", "git" = "https://github.com/lance-format/lance.git" }
lance-encoding = { "version" = "=9.0.0-beta.23", "rev" = "2ebc588f809b2937e9dbf1bae48ffdfd844cef61", "git" = "https://github.com/lance-format/lance.git" }
lance-arrow = { "version" = "=9.0.0-beta.23", "rev" = "2ebc588f809b2937e9dbf1bae48ffdfd844cef61", "git" = "https://github.com/lance-format/lance.git" }
ahash = "0.8"
# Note that this one does not include pyarrow
arrow = { version = "58.0.0", optional = false }
@@ -39,15 +39,15 @@ arrow-schema = "58.0.0"
arrow-select = "58.0.0"
arrow-cast = "58.0.0"
async-trait = "0"
datafusion = { version = "53.0.0", default-features = false }
datafusion-catalog = "53.0.0"
datafusion-common = { version = "53.0.0", default-features = false }
datafusion-execution = "53.0.0"
datafusion-expr = "53.0.0"
datafusion-functions = "53.0.0"
datafusion-physical-plan = "53.0.0"
datafusion-physical-expr = "53.0.0"
datafusion-sql = "53.0.0"
datafusion = { version = "54.0.0", default-features = false }
datafusion-catalog = "54.0.0"
datafusion-common = { version = "54.0.0", default-features = false }
datafusion-execution = "54.0.0"
datafusion-expr = "54.0.0"
datafusion-functions = "54.0.0"
datafusion-physical-plan = "54.0.0"
datafusion-physical-expr = "54.0.0"
datafusion-sql = "54.0.0"
env_logger = "0.11"
half = { "version" = "2.7.1", default-features = false, features = [
"num-traits",
rust/lancedb/src/dataloader/permutation/split.rs
+2 -2
View File
@@ -8,7 +8,7 @@ use std::sync::{
use arrow_array::{Array, BooleanArray, RecordBatch, UInt64Array};
use arrow_schema::{DataType, Field, Schema};
use datafusion_common::hash_utils::create_hashes;
use datafusion_common::hash_utils::{RandomState, create_hashes};
use futures::{StreamExt, TryStreamExt};
use lance_arrow::SchemaExt;
@@ -234,7 +234,7 @@ impl Splitter {
.cloned()
.collect::<Vec<_>>();
let mut hashes = vec![0; batch.num_rows()];
let random_state = ahash::RandomState::with_seeds(0, 0, 0, 0);
let random_state = RandomState::with_seed(0);
create_hashes(&arrays, &random_state, &mut hashes).unwrap();
// As an example, let's assume the weights are 1, 2. Our total weight is 3.
//
rust/lancedb/src/remote/table.rs
-1
View File
@@ -1251,7 +1251,6 @@ impl<S: HttpSend + 'static> RemoteTable<S> {
match result {
Ok(_) => {
let add_result = insert
.as_any()
.downcast_ref::<RemoteInsertExec<S>>()
.and_then(|i| i.add_result())
.unwrap_or(AddResult { version: 0 });
rust/lancedb/src/remote/table/insert.rs
-5
View File
@@ -3,7 +3,6 @@
//! DataFusion ExecutionPlan for inserting data into remote LanceDB tables.
use std::any::Any;
use std::sync::{Arc, Mutex};
use arrow_array::{ArrayRef, RecordBatch, UInt64Array};
@@ -237,10 +236,6 @@ impl<S: HttpSend + 'static> ExecutionPlan for RemoteInsertExec<S> {
Self::static_name()
}
fn as_any(&self) -> &dyn Any {
self
}
fn properties(&self) -> &Arc<PlanProperties> {
&self.properties
}
rust/lancedb/src/table/datafusion.rs
+1 -9
View File
@@ -89,10 +89,6 @@ impl ExecutionPlan for MetadataEraserExec {
"MetadataEraserExec"
}
fn as_any(&self) -> &dyn std::any::Any {
self
}
fn properties(&self) -> &Arc<PlanProperties> {
&self.properties
}
@@ -138,7 +134,7 @@ impl ExecutionPlan for MetadataEraserExec {
)
}
fn partition_statistics(&self, partition: Option<usize>) -> DataFusionResult<Statistics> {
fn partition_statistics(&self, partition: Option<usize>) -> DataFusionResult<Arc<Statistics>> {
self.input.partition_statistics(partition)
}
@@ -188,10 +184,6 @@ impl BaseTableAdapter {
#[async_trait]
impl TableProvider for BaseTableAdapter {
fn as_any(&self) -> &dyn std::any::Any {
self
}
fn schema(&self) -> Arc<ArrowSchema> {
self.schema.clone()
}
rust/lancedb/src/table/datafusion/insert.rs
-5
View File
@@ -3,7 +3,6 @@
//! DataFusion ExecutionPlan for inserting data into LanceDB tables.
use std::any::Any;
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::{Arc, LazyLock, Mutex};
@@ -146,10 +145,6 @@ impl ExecutionPlan for InsertExec {
Self::static_name()
}
fn as_any(&self) -> &dyn Any {
self
}
fn properties(&self) -> &Arc<PlanProperties> {
&self.properties
}
rust/lancedb/src/table/datafusion/reject_nan.rs
-5
View File
@@ -3,7 +3,6 @@
//! A DataFusion projection that rejects vectors containing NaN values.
use std::any::Any;
use std::sync::{Arc, LazyLock};
use arrow_array::{Array, FixedSizeListArray};
@@ -86,10 +85,6 @@ impl RejectNanUdf {
}
impl ScalarUDFImpl for RejectNanUdf {
fn as_any(&self) -> &dyn Any {
self
}
fn name(&self) -> &str {
"reject_nan"
}
rust/lancedb/src/table/datafusion/scannable_exec.rs
+3 -7
View File
@@ -66,10 +66,6 @@ impl ExecutionPlan for ScannableExec {
"ScannableExec"
}
fn as_any(&self) -> &dyn std::any::Any {
self
}
fn properties(&self) -> &Arc<PlanProperties> {
&self.properties
}
@@ -121,14 +117,14 @@ impl ExecutionPlan for ScannableExec {
)))
}
fn partition_statistics(&self, _partition: Option<usize>) -> DFResult<Statistics> {
Ok(Statistics {
fn partition_statistics(&self, _partition: Option<usize>) -> DFResult<Arc<Statistics>> {
Ok(Arc::new(Statistics {
num_rows: self
.num_rows
.map(Precision::Exact)
.unwrap_or(Precision::Absent),
total_byte_size: Precision::Absent,
column_statistics: vec![],
})
}))
}
}
rust/lancedb/src/table/datafusion/udtf/fts.rs
-1
View File
@@ -138,7 +138,6 @@ mod tests {
// Downcast to BaseTableAdapter and apply FTS query
let base_adapter = table_provider
.as_any()
.downcast_ref::<BaseTableAdapter>()
.ok_or_else(|| {
DataFusionError::Internal(
rust/lancedb/src/table/query.rs
+2 -2
View File
@@ -982,10 +982,10 @@ mod tests {
use crate::table::query::create_plan;
fn find_ann_approx_mode(plan: &dyn ExecutionPlan) -> Option<ApproxMode> {
if let Some(ann) = plan.as_any().downcast_ref::<ANNIvfSubIndexExec>() {
if let Some(ann) = plan.downcast_ref::<ANNIvfSubIndexExec>() {
return Some(ann.query().approx_mode);
}
if let Some(ann) = plan.as_any().downcast_ref::<ANNIvfPartitionExec>() {
if let Some(ann) = plan.downcast_ref::<ANNIvfPartitionExec>() {
return Some(ann.query.approx_mode);
}
plan.children()