mirror of
https://github.com/lancedb/lancedb.git
synced 2026-11-13 22:26:53 +00:00
Yang Cen
2dedde9b6c
Remote SQL failures currently become generic runtime errors, and HTTP table lookup can discard server error metadata. Preserve transport status, current request ID, SQL job ID and worker request ID across Rust and Python. Select typed column, table, reference and schema exceptions only when an explicit server code matches the transport status. Remote `HttpError` and `RuntimeError` handlers remain compatible. Failed SQL jobs retain `JobFailedError` with a typed SQL cause when available. Missing metadata stays absent; message text never determines the exception class. Local missing-column errors retain their type and qualified-field suggestions, and `JobFailure.source` documents structured remote Flight SQL failures. This branch integrates main at `a9ad4997`, including its released Lance 15.0.0-beta.9 dependency update and remote optimize support. The only textual conflict was two independent additions to `test_remote_db.py`; both test groups and all original assertions are retained. The feature has no dependency or lockfile delta against current main. SDK runtime validation on `03ee633a` (runtime source unchanged in `f3f2d0b8`), macOS ARM64, pinned Rust 1.97.0 and the frozen uv Python 3.14 environment: - Rebuilt the native extension from the integrated source. All 22 targeted Python tests passed: native local errors, HTTP/Flight metadata, legacy sync/async table-not-found behavior, and the three main remote-optimize regressions. - All 52 Rust SQL, metadata/source and nested-field tests passed, including the new main optimize cases. - Root and Python-binding Rust Clippy with `-D warnings`, Rust formatting, Python package Ruff and conflict-file formatting passed. - Repository-wide Ruff reports 59 existing documentation findings, and its format check identifies six documentation files. Every affected file is byte-identical to current main; these checks are not reported as passing. - Direct integration review and AST comparison confirm both conflicting test groups were preserved. Earlier exception-constructor/pickling evidence remains applicable to the unchanged Python error modules. The unlocked build exhausted disk again on a runner with the same label but only 9.4 GiB free on a 72 GiB root filesystem. Keep the existing runner and dependency caches, serialize Cargo compilation, and remove only completed test/bench executables reported by Cargo. This job never runs those binaries; normal binaries, libraries and build-script outputs remain available. Fresh resolution, the ci profile, all features, and test/bench compilation and linking are retained. Three probes using Cargo and target-path validation pass with `-D warnings`: all fixture targets link while ordinary outputs remain usable; an undefined symbol still produces Cargo exit 101 and its linker diagnostic; files outside the target directory are never removed. The [current-head build-no-lock job](https://github.com/lancedb/lancedb/actions/runs/41911613851/job/126860364065) passed on `f3f2d0b8`: all compilation/linking completed, 20 completed test executables were removed, and `target` retained about 7.3 GiB. This successful run had a 145 GiB root with about 78 GiB free afterward; it was not an identical rerun of the failed 72 GiB image. The driver tests also passed in this job. All previous-head jobs ended naturally before the push; new-head CI and review gates still apply. Typed remote input exceptions require the server metadata contract. Older servers retain generic transport errors and existing missing-table compatibility. The optimize tests use mock servers and do not establish downstream server acceptance. BREAKING CHANGE: Rust callers with exhaustive matches on `lancedb::Error` must handle the new `FieldNotFound` and `Sql` variants. Local `Error::ColumnNotFound` failures now raise `lancedb.exceptions.ColumnNotFoundError`, a `ValueError` subclass. Python callers that previously caught only `RuntimeError` for this local failure must catch `ColumnNotFoundError` or `ValueError`.
112 lines
3.7 KiB
Python
112 lines
3.7 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
|
|
|
"""Link all CI test/bench targets after the workflow removes Cargo.lock.
|
|
|
|
This job never runs the test binaries. Discard their completed
|
|
executables while retaining normal binaries, libraries and build-script outputs.
|
|
"""
|
|
|
|
import json
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
|
|
def discard_test_executable(message: dict, target_dir: Path) -> bool:
|
|
if message.get("reason") != "compiler-artifact":
|
|
return False
|
|
target = message["target"]
|
|
kinds = target["kind"]
|
|
# profile.test describes this build; target.test is only a manifest capability.
|
|
# Ordinary binaries may still be needed through CARGO_BIN_EXE by later targets.
|
|
if "custom-build" in kinds or not (
|
|
message["profile"]["test"] or "test" in kinds or "bench" in kinds
|
|
):
|
|
return False
|
|
executable = message.get("executable")
|
|
if executable is None:
|
|
return False
|
|
path = Path(executable)
|
|
if not path.resolve().is_relative_to(target_dir.resolve()):
|
|
raise ValueError(f"executable {path} is outside target directory {target_dir}")
|
|
if not path.is_file():
|
|
return False
|
|
size = path.stat().st_size
|
|
path.unlink()
|
|
print(
|
|
json.dumps(
|
|
{
|
|
"removed_test_executable": str(path),
|
|
"target": target["name"],
|
|
"logical_bytes": size,
|
|
"filesystem_free_bytes": shutil.disk_usage(path.parent).free,
|
|
}
|
|
),
|
|
flush=True,
|
|
)
|
|
return True
|
|
|
|
|
|
def main() -> int:
|
|
metadata = subprocess.run(
|
|
["cargo", "metadata", "--format-version=1", "--no-deps", "--all-features"],
|
|
stdout=subprocess.PIPE,
|
|
text=True,
|
|
)
|
|
if metadata.returncode:
|
|
return metadata.returncode
|
|
target_dir = Path(json.loads(metadata.stdout)["target_directory"])
|
|
command = [
|
|
"cargo",
|
|
"build",
|
|
"--profile",
|
|
"ci",
|
|
"--benches",
|
|
"--all-features",
|
|
"--tests",
|
|
# Bound simultaneous compiler/linker temporary outputs as well as retained files.
|
|
"--jobs",
|
|
"1",
|
|
"--message-format=json,json-diagnostic-rendered-ansi",
|
|
]
|
|
cleanup_failed = False
|
|
with subprocess.Popen(command, stdout=subprocess.PIPE, text=True) as process:
|
|
assert process.stdout is not None
|
|
for line in process.stdout:
|
|
try:
|
|
message = json.loads(line)
|
|
except json.JSONDecodeError:
|
|
print(line, end="", flush=True)
|
|
continue
|
|
if not isinstance(message, dict):
|
|
print(line, end="", flush=True)
|
|
continue
|
|
if message.get("reason") == "compiler-message":
|
|
diagnostic = message["message"]
|
|
sys.stderr.write(
|
|
diagnostic.get("rendered") or json.dumps(diagnostic) + "\n"
|
|
)
|
|
sys.stderr.flush()
|
|
elif message.get("reason") == "compiler-artifact":
|
|
try:
|
|
discard_test_executable(message, target_dir)
|
|
except (OSError, ValueError) as error:
|
|
print(
|
|
f"Cannot discard completed test executable: {error}",
|
|
file=sys.stderr,
|
|
)
|
|
cleanup_failed = True
|
|
elif message.get("reason") not in {
|
|
"build-script-executed",
|
|
"build-finished",
|
|
}:
|
|
print(line, end="", flush=True)
|
|
result = process.wait()
|
|
return result if result else int(cleanup_failed)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|