mirror of
https://github.com/lancedb/lancedb.git
synced 2026-08-30 01:48:19 +00:00
Compare commits
3 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Xuanwo
|
1443bf703f | ||
 lancedb automation
|
9e8140ff88 | ||
|
Xuanwo
|
77c6dffc4a |
Generated
+42
-42
@@ -3455,8 +3455,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
|
||||
|
||||
[[package]]
|
||||
name = "fsst"
|
||||
version = "12.0.0-beta.5"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259"
|
||||
version = "12.0.0-beta.4"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.4#474ae88dd75c437d5d506859b627510978e502e9"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"rand 0.9.5",
|
||||
@@ -4815,8 +4815,8 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a"
|
||||
|
||||
[[package]]
|
||||
name = "lance"
|
||||
version = "12.0.0-beta.5"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259"
|
||||
version = "12.0.0-beta.4"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.4#474ae88dd75c437d5d506859b627510978e502e9"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"arrow",
|
||||
@@ -4888,8 +4888,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-arrow"
|
||||
version = "12.0.0-beta.5"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259"
|
||||
version = "12.0.0-beta.4"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.4#474ae88dd75c437d5d506859b627510978e502e9"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -4911,7 +4911,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "lance-arrow-scalar"
|
||||
version = "58.0.0"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.4#474ae88dd75c437d5d506859b627510978e502e9"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -4925,7 +4925,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "lance-arrow-stats"
|
||||
version = "58.0.0"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.4#474ae88dd75c437d5d506859b627510978e502e9"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-schema",
|
||||
@@ -4934,8 +4934,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-bitpacking"
|
||||
version = "12.0.0-beta.5"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259"
|
||||
version = "12.0.0-beta.4"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.4#474ae88dd75c437d5d506859b627510978e502e9"
|
||||
dependencies = [
|
||||
"arrayref",
|
||||
"crunchy",
|
||||
@@ -4945,8 +4945,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-core"
|
||||
version = "12.0.0-beta.5"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259"
|
||||
version = "12.0.0-beta.4"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.4#474ae88dd75c437d5d506859b627510978e502e9"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -4983,8 +4983,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-datafusion"
|
||||
version = "12.0.0-beta.5"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259"
|
||||
version = "12.0.0-beta.4"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.4#474ae88dd75c437d5d506859b627510978e502e9"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-array",
|
||||
@@ -5013,8 +5013,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-datagen"
|
||||
version = "12.0.0-beta.5"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259"
|
||||
version = "12.0.0-beta.4"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.4#474ae88dd75c437d5d506859b627510978e502e9"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-array",
|
||||
@@ -5031,8 +5031,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-derive"
|
||||
version = "12.0.0-beta.5"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259"
|
||||
version = "12.0.0-beta.4"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.4#474ae88dd75c437d5d506859b627510978e502e9"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
@@ -5041,8 +5041,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-encoding"
|
||||
version = "12.0.0-beta.5"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259"
|
||||
version = "12.0.0-beta.4"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.4#474ae88dd75c437d5d506859b627510978e502e9"
|
||||
dependencies = [
|
||||
"arrow-arith",
|
||||
"arrow-array",
|
||||
@@ -5075,8 +5075,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-file"
|
||||
version = "12.0.0-beta.5"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259"
|
||||
version = "12.0.0-beta.4"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.4#474ae88dd75c437d5d506859b627510978e502e9"
|
||||
dependencies = [
|
||||
"arrow-arith",
|
||||
"arrow-array",
|
||||
@@ -5107,8 +5107,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-index"
|
||||
version = "12.0.0-beta.5"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259"
|
||||
version = "12.0.0-beta.4"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.4#474ae88dd75c437d5d506859b627510978e502e9"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"arrow",
|
||||
@@ -5172,8 +5172,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-index-core"
|
||||
version = "12.0.0-beta.5"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259"
|
||||
version = "12.0.0-beta.4"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.4#474ae88dd75c437d5d506859b627510978e502e9"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-schema",
|
||||
@@ -5195,8 +5195,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-io"
|
||||
version = "12.0.0-beta.5"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259"
|
||||
version = "12.0.0-beta.4"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.4#474ae88dd75c437d5d506859b627510978e502e9"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-array",
|
||||
@@ -5236,8 +5236,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-linalg"
|
||||
version = "12.0.0-beta.5"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259"
|
||||
version = "12.0.0-beta.4"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.4#474ae88dd75c437d5d506859b627510978e502e9"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-schema",
|
||||
@@ -5251,8 +5251,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-namespace"
|
||||
version = "12.0.0-beta.5"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259"
|
||||
version = "12.0.0-beta.4"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.4#474ae88dd75c437d5d506859b627510978e502e9"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"async-trait",
|
||||
@@ -5264,8 +5264,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-namespace-impls"
|
||||
version = "12.0.0-beta.5"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259"
|
||||
version = "12.0.0-beta.4"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.4#474ae88dd75c437d5d506859b627510978e502e9"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-ipc",
|
||||
@@ -5318,8 +5318,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-select"
|
||||
version = "12.0.0-beta.5"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259"
|
||||
version = "12.0.0-beta.4"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.4#474ae88dd75c437d5d506859b627510978e502e9"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -5333,8 +5333,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-table"
|
||||
version = "12.0.0-beta.5"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259"
|
||||
version = "12.0.0-beta.4"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.4#474ae88dd75c437d5d506859b627510978e502e9"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-array",
|
||||
@@ -5374,8 +5374,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-testing"
|
||||
version = "12.0.0-beta.5"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259"
|
||||
version = "12.0.0-beta.4"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.4#474ae88dd75c437d5d506859b627510978e502e9"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-schema",
|
||||
@@ -5388,8 +5388,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-tokenizer"
|
||||
version = "12.0.0-beta.5"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259"
|
||||
version = "12.0.0-beta.4"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.4#474ae88dd75c437d5d506859b627510978e502e9"
|
||||
dependencies = [
|
||||
"frostem",
|
||||
"icu_segmenter",
|
||||
|
||||
+14
-14
@@ -13,20 +13,20 @@ categories = ["database-implementations"]
|
||||
rust-version = "1.91.0"
|
||||
|
||||
[workspace.dependencies]
|
||||
lance = { "version" = "=12.0.0-beta.5", default-features = false, "tag" = "v12.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-core = { "version" = "=12.0.0-beta.5", "tag" = "v12.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-datagen = { "version" = "=12.0.0-beta.5", "tag" = "v12.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-file = { "version" = "=12.0.0-beta.5", "tag" = "v12.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-io = { "version" = "=12.0.0-beta.5", default-features = false, "tag" = "v12.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-index = { "version" = "=12.0.0-beta.5", "tag" = "v12.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-linalg = { "version" = "=12.0.0-beta.5", "tag" = "v12.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-namespace = { "version" = "=12.0.0-beta.5", "tag" = "v12.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-namespace-impls = { "version" = "=12.0.0-beta.5", default-features = false, "tag" = "v12.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-table = { "version" = "=12.0.0-beta.5", "tag" = "v12.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-testing = { "version" = "=12.0.0-beta.5", "tag" = "v12.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-datafusion = { "version" = "=12.0.0-beta.5", "tag" = "v12.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-encoding = { "version" = "=12.0.0-beta.5", "tag" = "v12.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-arrow = { "version" = "=12.0.0-beta.5", "tag" = "v12.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance = { "version" = "=12.0.0-beta.4", default-features = false, "tag" = "v12.0.0-beta.4", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-core = { "version" = "=12.0.0-beta.4", "tag" = "v12.0.0-beta.4", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-datagen = { "version" = "=12.0.0-beta.4", "tag" = "v12.0.0-beta.4", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-file = { "version" = "=12.0.0-beta.4", "tag" = "v12.0.0-beta.4", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-io = { "version" = "=12.0.0-beta.4", default-features = false, "tag" = "v12.0.0-beta.4", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-index = { "version" = "=12.0.0-beta.4", "tag" = "v12.0.0-beta.4", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-linalg = { "version" = "=12.0.0-beta.4", "tag" = "v12.0.0-beta.4", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-namespace = { "version" = "=12.0.0-beta.4", "tag" = "v12.0.0-beta.4", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-namespace-impls = { "version" = "=12.0.0-beta.4", default-features = false, "tag" = "v12.0.0-beta.4", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-table = { "version" = "=12.0.0-beta.4", "tag" = "v12.0.0-beta.4", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-testing = { "version" = "=12.0.0-beta.4", "tag" = "v12.0.0-beta.4", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-datafusion = { "version" = "=12.0.0-beta.4", "tag" = "v12.0.0-beta.4", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-encoding = { "version" = "=12.0.0-beta.4", "tag" = "v12.0.0-beta.4", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-arrow = { "version" = "=12.0.0-beta.4", "tag" = "v12.0.0-beta.4", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lancedb = { path = "rust/lancedb", default-features = false }
|
||||
ahash = "0.8"
|
||||
# Note that this one does not include pyarrow
|
||||
|
||||
+1
-1
@@ -28,7 +28,7 @@
|
||||
<properties>
|
||||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||
<arrow.version>15.0.0</arrow.version>
|
||||
<lance-core.version>12.0.0-beta.5</lance-core.version>
|
||||
<lance-core.version>12.0.0-beta.4</lance-core.version>
|
||||
<spotless.skip>false</spotless.skip>
|
||||
<spotless.version>2.30.0</spotless.version>
|
||||
<spotless.java.googlejavaformat.version>1.7</spotless.java.googlejavaformat.version>
|
||||
|
||||
@@ -54,6 +54,30 @@ _UInt32 = conint(strict=True, ge=0, le=2**32 - 1)
|
||||
_UInt64 = conint(strict=True, ge=0, le=2**64 - 1)
|
||||
|
||||
|
||||
def _validate_gpu_wire_requirement(value: Any) -> str:
|
||||
if not isinstance(value, str):
|
||||
raise ValueError("runtime.gpu must be a string")
|
||||
if not value or value != value.strip():
|
||||
raise ValueError("runtime.gpu must be non-empty and trimmed")
|
||||
if value.isascii() and value.isdigit():
|
||||
count = int(value)
|
||||
if not 0 < count <= 2**32 - 1 or str(count) != value:
|
||||
raise ValueError(
|
||||
"a numeric runtime.gpu must be a canonical positive uint32"
|
||||
)
|
||||
return value
|
||||
|
||||
|
||||
def _normalize_gpu_requirement(value: Optional[int | str]) -> Optional[str]:
|
||||
if value is None:
|
||||
return None
|
||||
if isinstance(value, int) and not isinstance(value, bool):
|
||||
if not 0 < value <= 2**32 - 1:
|
||||
raise ValueError("gpu must be a whole number greater than zero")
|
||||
return str(value)
|
||||
return _validate_gpu_wire_requirement(value)
|
||||
|
||||
|
||||
class _FrozenDict(dict):
|
||||
def _immutable(self, *args, **kwargs):
|
||||
raise TypeError("remote canonical values are immutable")
|
||||
@@ -239,6 +263,23 @@ class PythonRuntimeSpec(_RemoteValue):
|
||||
python_version: Optional[str] = None
|
||||
environment: Optional[PythonEnvironmentSpec] = None
|
||||
env: Optional[Mapping[str, str]] = None
|
||||
gpu: Optional[str] = None
|
||||
|
||||
@model_validator(mode="before")
|
||||
@classmethod
|
||||
def _discard_unknown_runtime_payload(cls, value):
|
||||
if isinstance(value, Mapping):
|
||||
kind = value.get("kind")
|
||||
if isinstance(kind, str) and kind not in {"python", "python_v2"}:
|
||||
return {"kind": kind}
|
||||
return value
|
||||
|
||||
@field_validator("gpu", mode="before")
|
||||
@classmethod
|
||||
def _validate_gpu_requirement(cls, value):
|
||||
if value is None:
|
||||
return None
|
||||
return _validate_gpu_wire_requirement(value)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def _validate_runtime_kind(self):
|
||||
@@ -247,18 +288,28 @@ class PythonRuntimeSpec(_RemoteValue):
|
||||
raise ValueError("python runtime requires python_version")
|
||||
if self.environment is None:
|
||||
raise ValueError("python runtime requires environment")
|
||||
if self.gpu is not None:
|
||||
raise ValueError("python runtime with gpu requires kind='python_v2'")
|
||||
elif self.kind == "python_v2":
|
||||
if self.python_version is None:
|
||||
raise ValueError("python_v2 runtime requires python_version")
|
||||
if self.environment is None:
|
||||
raise ValueError("python_v2 runtime requires environment")
|
||||
if self.gpu is None:
|
||||
raise ValueError("python_v2 runtime requires gpu")
|
||||
else:
|
||||
object.__setattr__(self, "python_version", None)
|
||||
object.__setattr__(self, "environment", None)
|
||||
object.__setattr__(self, "env", None)
|
||||
object.__setattr__(self, "gpu", None)
|
||||
return self
|
||||
|
||||
|
||||
class FunctionVersion(_RemoteValue):
|
||||
"""An exact immutable Function version returned by Enterprise.
|
||||
|
||||
Scheduling resources, priority, concurrency, and retry policy belong to
|
||||
the submitting Job and are not part of this identity.
|
||||
The GPU requirement is part of this identity. CPU and memory sizing,
|
||||
priority, concurrency, and retry policy belong to the execution platform.
|
||||
"""
|
||||
|
||||
name: str
|
||||
@@ -910,6 +961,7 @@ class UdfDefinition:
|
||||
pip: tuple[str, ...],
|
||||
env: Mapping[str, str],
|
||||
python_version: Optional[str],
|
||||
gpu: Optional[int | str] = None,
|
||||
conda: tuple[str, ...] = (),
|
||||
conda_channels: tuple[str, ...] = (),
|
||||
):
|
||||
@@ -938,12 +990,14 @@ class UdfDefinition:
|
||||
signature = _infer_signature(function, input_schema, output_schema)
|
||||
source = _package_source(function)
|
||||
digest = f"sha256:{hashlib.sha256(source).hexdigest()}"
|
||||
gpu_requirement = _normalize_gpu_requirement(gpu)
|
||||
runtime = PythonRuntimeSpec(
|
||||
kind="python",
|
||||
kind="python_v2" if gpu_requirement is not None else "python",
|
||||
python_version=python_version
|
||||
or f"{sys.version_info.major}.{sys.version_info.minor}",
|
||||
environment=environment_spec,
|
||||
env=environment,
|
||||
gpu=gpu_requirement,
|
||||
)
|
||||
self._function = function
|
||||
self._request = FunctionRegistrationRequest(
|
||||
@@ -989,6 +1043,7 @@ def udf(
|
||||
pip: tuple[str, ...] | list[str] = (),
|
||||
env: Optional[Mapping[str, str]] = None,
|
||||
python_version: Optional[str] = None,
|
||||
gpu: Optional[int | str] = None,
|
||||
conda: tuple[str, ...] | list[str] = (),
|
||||
conda_channels: tuple[str, ...] | list[str] = (),
|
||||
) -> Callable[[Callable[..., Any]], UdfDefinition]: ...
|
||||
@@ -1003,6 +1058,7 @@ def udf(
|
||||
pip: tuple[str, ...] | list[str] = (),
|
||||
env: Optional[Mapping[str, str]] = None,
|
||||
python_version: Optional[str] = None,
|
||||
gpu: Optional[int | str] = None,
|
||||
conda: tuple[str, ...] | list[str] = (),
|
||||
conda_channels: tuple[str, ...] | list[str] = (),
|
||||
):
|
||||
@@ -1035,6 +1091,11 @@ def udf(
|
||||
Environment variables included in the Function definition.
|
||||
python_version : str, optional
|
||||
Remote Python major/minor version. Defaults to the client version.
|
||||
gpu : int or str, optional
|
||||
GPU requirement for every remote execution. A positive integer
|
||||
requests that many compatible NVIDIA GPUs. A string can select a
|
||||
platform-supported model and count, such as ``"H100:8"``. The
|
||||
requirement is part of the immutable Function version.
|
||||
|
||||
The packaged artifact is a snapshot: the function source plus exactly
|
||||
the module-level names it references (modules as imports, importable
|
||||
@@ -1059,6 +1120,11 @@ def udf(
|
||||
... return value * 2
|
||||
>>> score(1.5)
|
||||
3.0
|
||||
>>> @udf(pip=["cupy-cuda12x"], gpu="H100:8")
|
||||
... def gpu_score(value: int) -> int:
|
||||
... return value * 2
|
||||
>>> gpu_score.registration_request.runtime.gpu
|
||||
'H100:8'
|
||||
"""
|
||||
|
||||
def decorate(target: Callable[..., Any]) -> UdfDefinition:
|
||||
@@ -1070,6 +1136,7 @@ def udf(
|
||||
pip=tuple(pip),
|
||||
env={} if env is None else env,
|
||||
python_version=python_version,
|
||||
gpu=gpu,
|
||||
conda=tuple(conda),
|
||||
conda_channels=tuple(conda_channels),
|
||||
)
|
||||
|
||||
@@ -19,7 +19,7 @@ import pyarrow as pa
|
||||
import pytest
|
||||
|
||||
import lancedb
|
||||
from lancedb.functions import UdfDefinition, udf
|
||||
from lancedb.functions import PythonRuntimeSpec, UdfDefinition, udf
|
||||
|
||||
THRESHOLD = 20
|
||||
_CACHE = None
|
||||
@@ -89,6 +89,66 @@ def test_udf_conda_environment():
|
||||
udf(name="channels", conda_channels=["conda-forge"])(lambda value: value)
|
||||
|
||||
|
||||
def test_udf_gpu_requirement_uses_gpu_runtime():
|
||||
@udf(pip=["cupy-cuda12x"], gpu=1)
|
||||
def double_on_gpu(value: int) -> int:
|
||||
return value * 2
|
||||
|
||||
request = json.loads(double_on_gpu.registration_request.to_canonical_json())
|
||||
assert request["runtime"]["kind"] == "python_v2"
|
||||
assert request["runtime"]["gpu"] == "1"
|
||||
|
||||
@udf(pip=["cupy-cuda12x"], gpu="H100:8")
|
||||
def double_on_h100(value: int) -> int:
|
||||
return value * 2
|
||||
|
||||
h100_request = json.loads(double_on_h100.registration_request.to_canonical_json())
|
||||
assert h100_request["runtime"]["gpu"] == "H100:8"
|
||||
|
||||
@udf(pip=["pyarrow"])
|
||||
def cpu_function(value: int) -> int:
|
||||
return value
|
||||
|
||||
cpu_runtime = json.loads(cpu_function.registration_request.to_canonical_json())[
|
||||
"runtime"
|
||||
]
|
||||
assert cpu_runtime["kind"] == "python"
|
||||
assert "gpu" not in cpu_runtime
|
||||
|
||||
def identity(value: int) -> int:
|
||||
return value
|
||||
|
||||
for invalid in [0, -1, 1.5, True, "", "0", "01", " 1"]:
|
||||
with pytest.raises(ValueError):
|
||||
udf(name="invalid_gpu", gpu=invalid)(identity)
|
||||
|
||||
base_runtime = {
|
||||
"kind": "python_v2",
|
||||
"python_version": "3.12",
|
||||
"environment": {"kind": "pip"},
|
||||
}
|
||||
for requirement in ["H100", "H100:8"]:
|
||||
runtime = PythonRuntimeSpec.model_validate({**base_runtime, "gpu": requirement})
|
||||
assert runtime.gpu == requirement
|
||||
for invalid in [1, 0, "", "0", "01", " 1"]:
|
||||
with pytest.raises(ValueError):
|
||||
PythonRuntimeSpec.model_validate({**base_runtime, "gpu": invalid})
|
||||
|
||||
|
||||
def test_unknown_runtime_discards_payload_before_known_field_validation():
|
||||
for payload in [
|
||||
{"kind": "python_v3", "gpu": {"model": "H100"}},
|
||||
{"kind": "python_v3", "resources": []},
|
||||
{
|
||||
"kind": "python_v3",
|
||||
"environment": {"kind": []},
|
||||
"python_version": 3.15,
|
||||
},
|
||||
]:
|
||||
runtime = PythonRuntimeSpec.model_validate(payload)
|
||||
assert runtime.to_canonical_json() == '{"kind":"python_v3"}'
|
||||
|
||||
|
||||
def test_udf_packages_attribute_access_and_body_imports():
|
||||
@udf
|
||||
def word_norm(body: str) -> float:
|
||||
|
||||
+199
-27
@@ -197,6 +197,23 @@ pub struct PythonEnvironmentSpec {
|
||||
pub image: Option<String>,
|
||||
}
|
||||
|
||||
fn validate_gpu_requirement(gpu: &str) -> std::result::Result<(), String> {
|
||||
if gpu.is_empty() || gpu.trim() != gpu {
|
||||
return Err("runtime.gpu must be non-empty and trimmed".to_string());
|
||||
}
|
||||
if gpu.bytes().all(|byte| byte.is_ascii_digit()) {
|
||||
let count = gpu
|
||||
.parse::<u32>()
|
||||
.ok()
|
||||
.filter(|count| *count > 0)
|
||||
.ok_or_else(|| "a numeric runtime.gpu must be a positive uint32".to_string())?;
|
||||
if count.to_string() != gpu {
|
||||
return Err("a numeric runtime.gpu must use canonical decimal form".to_string());
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Reproducible Python runtime definition understood by Sophon.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
#[non_exhaustive]
|
||||
@@ -207,6 +224,35 @@ pub enum PythonRuntimeSpec {
|
||||
environment: PythonEnvironmentSpec,
|
||||
env: BTreeMap<String, String>,
|
||||
},
|
||||
/// The GPU-enabled Sophon-managed Python runtime.
|
||||
///
|
||||
/// # Examples
|
||||
///
|
||||
/// ```
|
||||
/// use std::collections::BTreeMap;
|
||||
/// use lancedb::function::{PythonEnvironmentSpec, PythonRuntimeSpec};
|
||||
///
|
||||
/// let runtime = PythonRuntimeSpec::PythonV2 {
|
||||
/// python_version: "3.12".to_string(),
|
||||
/// environment: PythonEnvironmentSpec {
|
||||
/// kind: "pip".to_string(),
|
||||
/// packages: vec!["cupy-cuda12x".to_string()],
|
||||
/// channels: Vec::new(),
|
||||
/// path: None,
|
||||
/// modules: Vec::new(),
|
||||
/// image: None,
|
||||
/// },
|
||||
/// env: BTreeMap::new(),
|
||||
/// gpu: "1".to_string(),
|
||||
/// };
|
||||
/// assert_eq!(runtime.gpu(), Some("1"));
|
||||
/// ```
|
||||
PythonV2 {
|
||||
python_version: String,
|
||||
environment: PythonEnvironmentSpec,
|
||||
env: BTreeMap<String, String>,
|
||||
gpu: String,
|
||||
},
|
||||
/// A runtime kind introduced by a newer server.
|
||||
///
|
||||
/// Unknown payload fields are intentionally not retained because the
|
||||
@@ -219,22 +265,27 @@ impl PythonRuntimeSpec {
|
||||
pub fn kind(&self) -> &str {
|
||||
match self {
|
||||
Self::Python { .. } => "python",
|
||||
Self::PythonV2 { .. } => "python_v2",
|
||||
Self::Unrecognized { kind } => kind,
|
||||
}
|
||||
}
|
||||
|
||||
/// The Python version for the V1 runtime, or `None` for an unknown kind.
|
||||
/// The Python version for a known Python runtime, or `None` for an unknown kind.
|
||||
pub fn python_version(&self) -> Option<&str> {
|
||||
match self {
|
||||
Self::Python { python_version, .. } => Some(python_version),
|
||||
Self::Python { python_version, .. } | Self::PythonV2 { python_version, .. } => {
|
||||
Some(python_version)
|
||||
}
|
||||
Self::Unrecognized { .. } => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// The Python environment for the V1 runtime, or `None` for an unknown kind.
|
||||
/// The Python environment for a known Python runtime, or `None` for an unknown kind.
|
||||
pub fn environment(&self) -> Option<&PythonEnvironmentSpec> {
|
||||
match self {
|
||||
Self::Python { environment, .. } => Some(environment),
|
||||
Self::Python { environment, .. } | Self::PythonV2 { environment, .. } => {
|
||||
Some(environment)
|
||||
}
|
||||
Self::Unrecognized { .. } => None,
|
||||
}
|
||||
}
|
||||
@@ -242,38 +293,75 @@ impl PythonRuntimeSpec {
|
||||
/// Environment variables, or `None` for an unknown kind.
|
||||
pub fn env(&self) -> Option<&BTreeMap<String, String>> {
|
||||
match self {
|
||||
Self::Python { env, .. } => Some(env),
|
||||
Self::Python { env, .. } | Self::PythonV2 { env, .. } => Some(env),
|
||||
Self::Unrecognized { .. } => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// GPU requirement understood by the execution platform.
|
||||
pub fn gpu(&self) -> Option<&str> {
|
||||
match self {
|
||||
Self::PythonV2 { gpu, .. } => Some(gpu),
|
||||
Self::Python { .. } | Self::Unrecognized { .. } => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct PythonRuntimeWire {
|
||||
kind: String,
|
||||
#[serde(default)]
|
||||
python_version: Option<String>,
|
||||
#[serde(default)]
|
||||
environment: Option<PythonEnvironmentSpec>,
|
||||
struct PythonRuntimeV1Wire {
|
||||
python_version: String,
|
||||
environment: PythonEnvironmentSpec,
|
||||
#[serde(default)]
|
||||
env: BTreeMap<String, String>,
|
||||
#[serde(default)]
|
||||
gpu: Option<Value>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct PythonRuntimeV2Wire {
|
||||
python_version: String,
|
||||
environment: PythonEnvironmentSpec,
|
||||
#[serde(default)]
|
||||
env: BTreeMap<String, String>,
|
||||
gpu: String,
|
||||
}
|
||||
|
||||
impl<'de> Deserialize<'de> for PythonRuntimeSpec {
|
||||
fn deserialize<D: Deserializer<'de>>(deserializer: D) -> std::result::Result<Self, D::Error> {
|
||||
let wire = PythonRuntimeWire::deserialize(deserializer)?;
|
||||
if wire.kind == "python" {
|
||||
Ok(Self::Python {
|
||||
python_version: wire
|
||||
.python_version
|
||||
.ok_or_else(|| de::Error::missing_field("python_version"))?,
|
||||
environment: wire
|
||||
.environment
|
||||
.ok_or_else(|| de::Error::missing_field("environment"))?,
|
||||
env: wire.env,
|
||||
})
|
||||
} else {
|
||||
Ok(Self::Unrecognized { kind: wire.kind })
|
||||
let value = Value::deserialize(deserializer)?;
|
||||
let kind = value
|
||||
.get("kind")
|
||||
.ok_or_else(|| de::Error::missing_field("kind"))?
|
||||
.as_str()
|
||||
.ok_or_else(|| de::Error::custom("runtime.kind must be a string"))?
|
||||
.to_string();
|
||||
match kind.as_str() {
|
||||
"python" => {
|
||||
let wire: PythonRuntimeV1Wire =
|
||||
serde_json::from_value(value).map_err(de::Error::custom)?;
|
||||
if wire.gpu.is_some() {
|
||||
return Err(de::Error::custom(
|
||||
"python runtime with gpu requires kind='python_v2'",
|
||||
));
|
||||
}
|
||||
Ok(Self::Python {
|
||||
python_version: wire.python_version,
|
||||
environment: wire.environment,
|
||||
env: wire.env,
|
||||
})
|
||||
}
|
||||
"python_v2" => {
|
||||
let wire: PythonRuntimeV2Wire =
|
||||
serde_json::from_value(value).map_err(de::Error::custom)?;
|
||||
validate_gpu_requirement(&wire.gpu).map_err(de::Error::custom)?;
|
||||
Ok(Self::PythonV2 {
|
||||
python_version: wire.python_version,
|
||||
environment: wire.environment,
|
||||
env: wire.env,
|
||||
gpu: wire.gpu,
|
||||
})
|
||||
}
|
||||
_ => Ok(Self::Unrecognized { kind }),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -287,6 +375,8 @@ impl Serialize for PythonRuntimeSpec {
|
||||
environment: &'a PythonEnvironmentSpec,
|
||||
#[serde(skip_serializing_if = "BTreeMap::is_empty")]
|
||||
env: &'a BTreeMap<String, String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
gpu: Option<&'a str>,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
@@ -304,8 +394,25 @@ impl Serialize for PythonRuntimeSpec {
|
||||
python_version,
|
||||
environment,
|
||||
env,
|
||||
gpu: None,
|
||||
}
|
||||
.serialize(serializer),
|
||||
Self::PythonV2 {
|
||||
python_version,
|
||||
environment,
|
||||
env,
|
||||
gpu,
|
||||
} => {
|
||||
validate_gpu_requirement(gpu).map_err(serde::ser::Error::custom)?;
|
||||
PythonRuntimeRef {
|
||||
kind: "python_v2",
|
||||
python_version,
|
||||
environment,
|
||||
env,
|
||||
gpu: Some(gpu),
|
||||
}
|
||||
.serialize(serializer)
|
||||
}
|
||||
Self::Unrecognized { kind } => UnrecognizedRuntimeRef { kind }.serialize(serializer),
|
||||
}
|
||||
}
|
||||
@@ -313,8 +420,8 @@ impl Serialize for PythonRuntimeSpec {
|
||||
|
||||
/// Immutable Function version returned by the Enterprise catalog.
|
||||
///
|
||||
/// Scheduling resources, priority, concurrency, and retry policy belong to
|
||||
/// the submitting Job and are not part of this identity.
|
||||
/// The GPU requirement is part of this identity. CPU and memory sizing,
|
||||
/// priority, concurrency, and retry policy belong to the execution platform.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct FunctionVersion {
|
||||
name: String,
|
||||
@@ -589,7 +696,7 @@ impl_json!(RefreshColumnResult);
|
||||
|
||||
#[cfg(test)]
|
||||
mod conda_environment_tests {
|
||||
use super::PythonEnvironmentSpec;
|
||||
use super::{PythonEnvironmentSpec, PythonRuntimeSpec};
|
||||
|
||||
#[test]
|
||||
fn conda_channels_round_trip_and_pip_stays_bare() {
|
||||
@@ -608,4 +715,69 @@ mod conda_environment_tests {
|
||||
serde_json::from_str(r#"{"kind":"pip","packages":["numpy"]}"#).unwrap();
|
||||
assert!(!serde_json::to_string(&pip).unwrap().contains("channels"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn gpu_python_runtime_round_trips_and_validates() {
|
||||
let runtime: PythonRuntimeSpec = serde_json::from_str(
|
||||
r#"{"kind":"python_v2","python_version":"3.12","environment":{"kind":"pip"},"gpu":"1"}"#,
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(runtime.kind(), "python_v2");
|
||||
assert_eq!(runtime.gpu(), Some("1"));
|
||||
assert_eq!(
|
||||
super::canonical_json(&runtime).unwrap(),
|
||||
r#"{"environment":{"kind":"pip"},"gpu":"1","kind":"python_v2","python_version":"3.12"}"#
|
||||
);
|
||||
|
||||
for requirement in ["H100", "H100:8"] {
|
||||
let encoded = format!(
|
||||
r#"{{"kind":"python_v2","python_version":"3.12","environment":{{"kind":"pip"}},"gpu":"{requirement}"}}"#
|
||||
);
|
||||
let runtime: PythonRuntimeSpec = serde_json::from_str(&encoded).unwrap();
|
||||
assert_eq!(runtime.gpu(), Some(requirement));
|
||||
}
|
||||
|
||||
for invalid in [
|
||||
r#"{"kind":"python","python_version":"3.12","environment":{"kind":"pip"},"gpu":"1"}"#,
|
||||
r#"{"kind":"python_v2","python_version":"3.12","environment":{"kind":"pip"}}"#,
|
||||
r#"{"kind":"python_v2","python_version":"3.12","environment":{"kind":"pip"},"gpu":1}"#,
|
||||
r#"{"kind":"python_v2","python_version":"3.12","environment":{"kind":"pip"},"gpu":"0"}"#,
|
||||
r#"{"kind":"python_v2","python_version":"3.12","environment":{"kind":"pip"},"gpu":"01"}"#,
|
||||
r#"{"kind":"python_v2","python_version":"3.12","environment":{"kind":"pip"},"gpu":""}"#,
|
||||
] {
|
||||
assert!(serde_json::from_str::<PythonRuntimeSpec>(invalid).is_err());
|
||||
}
|
||||
|
||||
let invalid_runtime = match runtime {
|
||||
PythonRuntimeSpec::PythonV2 {
|
||||
python_version,
|
||||
environment,
|
||||
env,
|
||||
..
|
||||
} => PythonRuntimeSpec::PythonV2 {
|
||||
python_version,
|
||||
environment,
|
||||
env,
|
||||
gpu: "0".to_string(),
|
||||
},
|
||||
_ => unreachable!(),
|
||||
};
|
||||
assert!(super::canonical_json(&invalid_runtime).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unknown_runtime_discards_payload_before_known_field_validation() {
|
||||
for encoded in [
|
||||
r#"{"kind":"python_v3","gpu":{"model":"H100"}}"#,
|
||||
r#"{"kind":"python_v3","resources":[]}"#,
|
||||
r#"{"kind":"python_v3","python_version":3.15,"environment":{"kind":[]}}"#,
|
||||
] {
|
||||
let runtime: PythonRuntimeSpec = serde_json::from_str(encoded).unwrap();
|
||||
assert_eq!(runtime.kind(), "python_v3");
|
||||
assert_eq!(
|
||||
super::canonical_json(&runtime).unwrap(),
|
||||
r#"{"kind":"python_v3"}"#
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5763,13 +5763,6 @@ mod tests {
|
||||
assert!(index_bytes > 0);
|
||||
assert_eq!(with_index, data_only + index_bytes);
|
||||
|
||||
// Release builds reject unstable overlay datasets unless explicitly opted in.
|
||||
if !lance_table::feature_flags::can_read_dataset(
|
||||
lance_table::feature_flags::FLAG_UNSTABLE_DATA_OVERLAY_FILES,
|
||||
) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Commit an overlay file supplying new `foo` values for the first three
|
||||
// rows of fragment 0. There is no high-level API that writes overlays
|
||||
// yet, so write the overlay's data file and commit the `DataOverlay`
|
||||
|
||||
@@ -1462,7 +1462,9 @@ fn plan_declarations(schema: SchemaRef, columns: &[(String, String)]) -> Result<
|
||||
|
||||
for (name, expression) in columns {
|
||||
if schema.field_with_name(name).is_ok() {
|
||||
return Err(Error::ColumnAlreadyExists { name: name.clone() });
|
||||
return Err(Error::ColumnAlreadyExists {
|
||||
name: name.to_string(),
|
||||
});
|
||||
}
|
||||
|
||||
let bound = bind(schema.clone(), name, expression)?;
|
||||
|
||||
Reference in New Issue
Block a user