rust/lancedb
1
0
Fork 0
mirror of https://github.com/lancedb/lancedb.git synced 2026-11-13 14:16:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: add UTM-tagged docs links to SDK warnings and enterprise-only errors (#4797)

Browse Source 
Add a central documentation link registry (`lancedb::docs::DocLink`) in
the Rust core that builds `docs.lancedb.com` URLs with UTM parameters
(`utm_source=lancedb_sdk`, `utm_medium=warning|error`,
`utm_campaign=<id>`, `utm_content=<sdk>-<version>`) placed before any
`#anchor` fragment and without user-identifying data.

- Initialize `utm_content` per language binding (`python`, `nodejs`,
`cli`, defaulting to `rust`) and expose the link builder to Python so
`DBConnection` and `Job` stubs reuse the core definitions.
- Append published documentation links to Enterprise-only `NotSupported`
/ `NotImplementedError` errors across database, job, catalog, and table
operations.
- Emit rate-limited maintenance warnings (once per table per process
when `warn` logging is enabled) when opening a native table with at
least 100 small fragments (`< 100k` physical rows each) or at least
1,000,000 unindexed rows on an optimizable user index, reading only
manifest metadata without opening data or `_indices/` files, and
document the thresholds in the compaction and reindexing guides.
- Verify published/in-repo `docs/web/` targets and `#anchor` fragments
in unit tests and include extracted SDK links in the scheduled
`docs-link-check` workflow.

Linear: OSS-2772
This commit is contained in:
Xuanwo
2026-11-03 14:12:15 +08:00
committed by GitHub
parent f9e25960e3
commit 2d70bec53a
20 changed files with 1319 additions and 59 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/ci-scripts.yml
+2
View File
@@ -9,6 +9,7 @@ on:
- python/pyproject.toml
- ci/check_cargo_lock.py
- ci/check_sdist.py
- ci/extract_sdk_doc_links.py
- ci/build_cli_linux.sh
- ci/package_cli.py
- .bumpversion.toml
@@ -22,6 +23,7 @@ on:
- python/pyproject.toml
- ci/check_cargo_lock.py
- ci/check_sdist.py
- ci/extract_sdk_doc_links.py
- ci/build_cli_linux.sh
- ci/package_cli.py
- .bumpversion.toml
.github/workflows/docs-link-check.yml
+6
View File
@@ -50,6 +50,10 @@ jobs:
ref: ${{ github.event.repository.default_branch }}
persist-credentials: false
- name: Extract SDK documentation links
run: |
python3 ci/extract_sdk_doc_links.py --output ./lychee-inputs/sdk-doc-links.md
- name: Check links
id: lychee
continue-on-error: true
@@ -64,10 +68,12 @@ jobs:
args: >-
--scheme https
--scheme http
--include-fragments
--no-progress
--max-retries 3
--timeout 20
'docs/src/**/*.md'
'./lychee-inputs/sdk-doc-links.md'
format: json
output: ./lychee/out.json
jobSummary: false
ci/extract_sdk_doc_links.py
+118
View File
@@ -0,0 +1,118 @@
#!/usr/bin/env python3
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
"""Extract documentation links defined in rust/lancedb/src/docs.rs for link checking."""
from __future__ import annotations
import argparse
from pathlib import Path
import re
import sys
REPO_ROOT = Path(__file__).resolve().parents[1]
DEFAULT_DOCS_RS = REPO_ROOT / "rust" / "lancedb" / "src" / "docs.rs"
def _parse_match_arm_map(match_arms: str) -> dict[str, str | None]:
mapping: dict[str, str | None] = {}
uncommented = "\n".join(
line for line in match_arms.splitlines() if not line.strip().startswith("//")
)
for raw_arm in uncommented.split(","):
arm = " ".join(raw_arm.split())
if not arm or "=>" not in arm:
continue
lhs, rhs = [part.strip() for part in arm.split("=>", 1)]
variants = [
part.strip().removeprefix("Self::")
for part in lhs.split("|")
if part.strip()
]
if rhs == "None":
value: str | None = None
else:
str_match = re.search(r'"([^"]+)"', rhs)
if not str_match:
raise ValueError(f"unsupported match arm value: {rhs}")
value = str_match.group(1)
for variant in variants:
mapping[variant] = value
return mapping
def extract_doc_links(docs_rs_path: Path = DEFAULT_DOCS_RS) -> list[str]:
source = docs_rs_path.read_text(encoding="utf-8")
base_match = re.search(
r'pub\s+const\s+DOCS_BASE_URL:\s*&str\s*=\s*"([^"]+)";', source
)
if not base_match:
raise ValueError("DOCS_BASE_URL not found in docs.rs")
base_url = base_match.group(1)
all_match = re.search(
r"pub\s+const\s+ALL:\s*&'static\s*\[Self\]\s*=\s*&\[(.*?)\];",
source,
re.DOTALL,
)
if not all_match:
raise ValueError("DocLink::ALL not found in docs.rs")
variants = re.findall(r"Self::([A-Za-z0-9_]+)", all_match.group(1))
if not variants:
raise ValueError("no variants found in DocLink::ALL")
path_fn = re.search(
r"pub\s+const\s+fn\s+path\(self\)\s*->\s*&'static\s+str\s*\{\s*match\s+self\s*\{(.*?)\}\s*\}",
source,
re.DOTALL,
)
anchor_fn = re.search(
r"pub\s+const\s+fn\s+anchor\(self\)\s*->\s*Option<&'static\s+str>\s*\{\s*match\s+self\s*\{(.*?)\}\s*\}",
source,
re.DOTALL,
)
if not path_fn or not anchor_fn:
raise ValueError("DocLink::path or DocLink::anchor not found in docs.rs")
paths = _parse_match_arm_map(path_fn.group(1))
anchors = _parse_match_arm_map(anchor_fn.group(1))
urls: list[str] = []
for variant in variants:
path = paths.get(variant)
if not path:
raise ValueError(f"missing path mapping for DocLink::{variant}")
anchor = anchors.get(variant)
url = f"{base_url}{path}#{anchor}" if anchor else f"{base_url}{path}"
if url not in urls:
urls.append(url)
return urls
def main(argv: list[str] | None = None) -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
"--docs-rs",
type=Path,
default=DEFAULT_DOCS_RS,
help="Path to rust/lancedb/src/docs.rs",
)
parser.add_argument(
"--output",
type=Path,
required=True,
help="Markdown file to write extracted links to",
)
args = parser.parse_args(argv)
urls = extract_doc_links(args.docs_rs)
args.output.parent.mkdir(parents=True, exist_ok=True)
lines = ["# SDK documentation links", ""] + [f"- <{url}>" for url in urls] + [""]
args.output.write_text("\n".join(lines), encoding="utf-8")
return 0
if __name__ == "__main__":
sys.exit(main())
ci/tests/test_extract_sdk_doc_links.py
+47
View File
@@ -0,0 +1,47 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
from pathlib import Path
import sys
from tempfile import TemporaryDirectory
import unittest
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
from ci.extract_sdk_doc_links import extract_doc_links, main
class TestExtractSdkDocLinks(unittest.TestCase):
def test_extracts_all_links_from_repo_docs_rs(self) -> None:
urls = extract_doc_links()
self.assertIn(
"https://docs.lancedb.com/performance#compaction-and-cleanup",
urls,
)
self.assertIn(
"https://docs.lancedb.com/indexing/reindexing#incremental-reindexing",
urls,
)
self.assertIn(
"https://docs.lancedb.com/search/sql",
urls,
)
self.assertIn(
"https://docs.lancedb.com/enterprise",
urls,
)
def test_cli_writes_markdown(self) -> None:
with TemporaryDirectory() as tmp:
out = Path(tmp) / "sdk-links.md"
rc = main(["--output", str(out)])
self.assertEqual(rc, 0)
content = out.read_text(encoding="utf-8")
self.assertIn(
"<https://docs.lancedb.com/performance#compaction-and-cleanup>",
content,
)
if __name__ == "__main__":
unittest.main()
docs/web/indexing/reindexing.mdx
+3
View File
@@ -61,6 +61,9 @@ LanceDB Enterprise supports incremental reindexing through an automated backgrou
The benefit of using LanceDB Enterprise is that it automates the reindexing process
and operates continuously in the background, minimizing the impact on latency under high loads.
In LanceDB Embedded, you must manually manage the reindexing cadence based on your data growth and performance needs.
Opening a table in LanceDB Embedded logs a warning at most once per table per
process when any index has at least 1,000,000 unindexed rows. Set
`LANCEDB_LOG=warn` to surface this log if your logger hides warnings by default.
## Disk utilization {#disk-utilization}
docs/web/maintenance/compaction.mdx
+4
View File
@@ -91,6 +91,10 @@ Frequent small appends are what make it necessary, because each one commits
files. A table written in a few large batches needs it rarely; one taking
continuous small writes needs it on a schedule.
Opening a table in LanceDB Embedded logs a warning at most once per table per
process when at least 100 fragments have fewer than 100,000 rows each. Set
`LANCEDB_LOG=warn` to surface this log if your logger hides warnings by default.
<Note>
Compaction rewrites files, which conflicts with a concurrent
[computed-column refresh](/feature-engineering/add-computed-column) or
nodejs/src/lib.rs
+1
View File
@@ -115,6 +115,7 @@ pub struct ConnectNamespaceOptions {
#[napi_derive::module_init]
fn init() {
lancedb::docs::set_sdk_ident("nodejs", env!("CARGO_PKG_VERSION"));
let env = Env::new()
.filter_or("LANCEDB_LOG", "warn")
.write_style("LANCEDB_LOG_STYLE");
python/python/lancedb/_lancedb.pyi
+3
View File
@@ -27,8 +27,11 @@ from lance_namespace import (
from .remote import ClientConfig
from .types import BaseTokenizerType
__version__: str
_experimental_functions_enabled: bool
def _doc_url(campaign: str) -> str: ...
def _append_doc_url(message: str, campaign: str) -> str: ...
def _parse_field_path(path: str) -> List[str]: ...
IvfHnswPq: type[HnswPq] = HnswPq
python/python/lancedb/db.py
+126 -30
View File
@@ -57,7 +57,7 @@ from lance_namespace import (
from lance_namespace.errors import NamespaceNotEmptyError, TableNotFoundError
from . import __version__
from ._lancedb import connect as lancedb_connect # type: ignore
from ._lancedb import _append_doc_url, connect as lancedb_connect # type: ignore
from .functions import FunctionVersion, UdfDefinition
from .graph import (
AsyncPropertyGraph,
@@ -718,7 +718,10 @@ class DBConnection(EnforceOverrides):
MaterializedView
"""
raise NotImplementedError(
"materialized views are not supported on this connection type"
_append_doc_url(
"materialized views are not supported on this connection type",
"materialized_views",
)
)
def create_materialized_view_async(
@@ -743,7 +746,10 @@ class DBConnection(EnforceOverrides):
Wait for the job before opening or querying the view.
"""
raise NotImplementedError(
"materialized views are not supported on this connection type"
_append_doc_url(
"materialized views are not supported on this connection type",
"materialized_views",
)
)
def open_materialized_view(self, name: str) -> MaterializedView:
@@ -753,7 +759,10 @@ class DBConnection(EnforceOverrides):
view.
"""
raise NotImplementedError(
"materialized views are not supported on this connection type"
_append_doc_url(
"materialized views are not supported on this connection type",
"materialized_views",
)
)
def list_materialized_views(
@@ -768,7 +777,10 @@ class DBConnection(EnforceOverrides):
iteration. See [pagination state][lancedb.listing.AsyncListing].
"""
raise NotImplementedError(
"materialized views are not supported on this connection type"
_append_doc_url(
"materialized views are not supported on this connection type",
"materialized_views",
)
)
def drop_materialized_view(
@@ -781,7 +793,10 @@ class DBConnection(EnforceOverrides):
Physical cleanup continues in the background.
"""
raise NotImplementedError(
"materialized views are not supported on this connection type"
_append_doc_url(
"materialized views are not supported on this connection type",
"materialized_views",
)
)
def drop_materialized_view_async(
@@ -795,7 +810,10 @@ class DBConnection(EnforceOverrides):
catalog removal; physical cleanup continues in the background.
"""
raise NotImplementedError(
"materialized views are not supported on this connection type"
_append_doc_url(
"materialized views are not supported on this connection type",
"materialized_views",
)
)
def drop_table(self, name: str, namespace_path: Optional[List[str]] = None):
@@ -994,7 +1012,11 @@ class DBConnection(EnforceOverrides):
LanceDB Embedded connections raise ``NotImplementedError``.
"""
raise NotImplementedError(
"Function catalog operations are not supported for this connection type"
_append_doc_url(
"Function catalog operations are not supported "
"for this connection type",
"functions",
)
)
def get_function(
@@ -1006,7 +1028,11 @@ class DBConnection(EnforceOverrides):
) -> FunctionVersion:
"""Open one exact immutable Function version from the remote catalog."""
raise NotImplementedError(
"Function catalog operations are not supported for this connection type"
_append_doc_url(
"Function catalog operations are not supported "
"for this connection type",
"functions",
)
)
def list_functions(
@@ -1037,7 +1063,11 @@ class DBConnection(EnforceOverrides):
iteration. See [pagination state][lancedb.listing.AsyncListing].
"""
raise NotImplementedError(
"Function catalog operations are not supported for this connection type"
_append_doc_url(
"Function catalog operations are not supported "
"for this connection type",
"functions",
)
)
def drop_function(
@@ -1058,7 +1088,11 @@ class DBConnection(EnforceOverrides):
revision. Local connections raise NotImplementedError.
"""
raise NotImplementedError(
"Function catalog operations are not supported for this connection type"
_append_doc_url(
"Function catalog operations are not supported "
"for this connection type",
"functions",
)
)
def drop_function_async(
@@ -1078,7 +1112,11 @@ class DBConnection(EnforceOverrides):
connections raise NotImplementedError.
"""
raise NotImplementedError(
"Function catalog operations are not supported for this connection type"
_append_doc_url(
"Function catalog operations are not supported "
"for this connection type",
"functions",
)
)
def create_secret(
@@ -1092,7 +1130,10 @@ class DBConnection(EnforceOverrides):
connections raise ``NotImplementedError``.
"""
raise NotImplementedError(
"Secret operations are not supported for this connection type"
_append_doc_url(
"Secret operations are not supported for this connection type",
"secrets",
)
)
def alter_secret(
@@ -1106,7 +1147,10 @@ class DBConnection(EnforceOverrides):
before it. LanceDB Embedded connections raise ``NotImplementedError``.
"""
raise NotImplementedError(
"Secret operations are not supported for this connection type"
_append_doc_url(
"Secret operations are not supported for this connection type",
"secrets",
)
)
def list_secrets(
@@ -1127,7 +1171,10 @@ class DBConnection(EnforceOverrides):
iteration. See [pagination state][lancedb.listing.AsyncListing].
"""
raise NotImplementedError(
"Secret operations are not supported for this connection type"
_append_doc_url(
"Secret operations are not supported for this connection type",
"secrets",
)
)
def drop_secret(
@@ -1141,7 +1188,10 @@ class DBConnection(EnforceOverrides):
LanceDB Embedded connections raise ``NotImplementedError``.
"""
raise NotImplementedError(
"Secret operations are not supported for this connection type"
_append_doc_url(
"Secret operations are not supported for this connection type",
"secrets",
)
)
def describe_secret(
@@ -1153,7 +1203,10 @@ class DBConnection(EnforceOverrides):
connections raise ``NotImplementedError``.
"""
raise NotImplementedError(
"Secret operations are not supported for this connection type"
_append_doc_url(
"Secret operations are not supported for this connection type",
"secrets",
)
)
def create_view(
@@ -1178,7 +1231,10 @@ class DBConnection(EnforceOverrides):
name: string
"""
raise NotImplementedError(
"View operations are not supported for this connection type"
_append_doc_url(
"View operations are not supported for this connection type",
"views",
)
)
def describe_view(
@@ -1192,7 +1248,10 @@ class DBConnection(EnforceOverrides):
``NotImplementedError``.
"""
raise NotImplementedError(
"View operations are not supported for this connection type"
_append_doc_url(
"View operations are not supported for this connection type",
"views",
)
)
def drop_view(
@@ -1206,7 +1265,10 @@ class DBConnection(EnforceOverrides):
LanceDB Embedded connections raise ``NotImplementedError``.
"""
raise NotImplementedError(
"View operations are not supported for this connection type"
_append_doc_url(
"View operations are not supported for this connection type",
"views",
)
)
def drop_view_async(
@@ -1219,7 +1281,10 @@ class DBConnection(EnforceOverrides):
in the background. Local connections raise ``NotImplementedError``.
"""
raise NotImplementedError(
"View operations are not supported for this connection type"
_append_doc_url(
"View operations are not supported for this connection type",
"views",
)
)
def list_views(
@@ -1240,7 +1305,10 @@ class DBConnection(EnforceOverrides):
iteration. See [pagination state][lancedb.listing.AsyncListing].
"""
raise NotImplementedError(
"View operations are not supported for this connection type"
_append_doc_url(
"View operations are not supported for this connection type",
"views",
)
)
def create_property_graph(
@@ -1347,7 +1415,12 @@ class DBConnection(EnforceOverrides):
Raises `JobNotFoundError` when the server has no such job, the way
`open_table` does for a missing table.
"""
raise NotImplementedError("open_job is not supported for this connection type")
raise NotImplementedError(
_append_doc_url(
"open_job is not supported for this connection type",
"jobs",
)
)
def list_gpu_classes(self) -> List[GpuClass]:
"""List the operator-configured Function GPU classes.
@@ -1361,7 +1434,10 @@ class DBConnection(EnforceOverrides):
``GET /v1/gpu/classes``.
"""
raise NotImplementedError(
"GPU class discovery is not supported for this connection type"
_append_doc_url(
"GPU class discovery is not supported for this connection type",
"gpu_classes",
)
)
def list_jobs(
@@ -1373,7 +1449,12 @@ class DBConnection(EnforceOverrides):
``page_token`` resumes from a saved token. Requests and errors occur during
iteration. See [pagination state][lancedb.listing.AsyncListing].
"""
raise NotImplementedError("list_jobs is not supported for this connection type")
raise NotImplementedError(
_append_doc_url(
"list_jobs is not supported for this connection type",
"jobs",
)
)
def cancel_job(self, job_id: str) -> bool:
"""Request cancellation of a server-side job by id.
@@ -1383,7 +1464,10 @@ class DBConnection(EnforceOverrides):
success.
"""
raise NotImplementedError(
"cancel_job is not supported for this connection type"
_append_doc_url(
"cancel_job is not supported for this connection type",
"jobs",
)
)
def pause_job(self, job_id: str) -> str:
@@ -1393,7 +1477,12 @@ class DBConnection(EnforceOverrides):
"pausing", "already_paused", or "committing" -- a job finalizing its
results cannot be parked; retry shortly.
"""
raise NotImplementedError("pause_job is not supported for this connection type")
raise NotImplementedError(
_append_doc_url(
"pause_job is not supported for this connection type",
"jobs",
)
)
def resume_job(self, job_id: str) -> str:
"""Resume a paused server-side job by id.
@@ -1403,7 +1492,10 @@ class DBConnection(EnforceOverrides):
confirmed yet; retry shortly -- or "not_paused".
"""
raise NotImplementedError(
"resume_job is not supported for this connection type"
_append_doc_url(
"resume_job is not supported for this connection type",
"jobs",
)
)
def execute_query(
@@ -1459,11 +1551,15 @@ class DBConnection(EnforceOverrides):
LanceDB Embedded connections do not support SQL.
"""
raise NotImplementedError("SQL is not supported for this connection type")
raise NotImplementedError(
_append_doc_url("SQL is not supported for this connection type", "sql")
)
def describe_query(self, query_id: Union[UUID, str]) -> QueryDescription:
"""Describe a submitted SQL query by its connection-scoped id."""
raise NotImplementedError("SQL is not supported for this connection type")
raise NotImplementedError(
_append_doc_url("SQL is not supported for this connection type", "sql")
)
class LanceDBConnection(DBConnection):
python/python/lancedb/job.py
+9 -3
View File
@@ -13,7 +13,7 @@ import pyarrow as pa
from lancedb.background_loop import LOOP
from . import _lancedb
from ._lancedb import JobDescription, JobFailureInfo, JobInfo
from ._lancedb import JobDescription, JobFailureInfo, JobInfo, _append_doc_url
from .remote.errors import HttpError, RetryError
T = TypeVar("T")
@@ -266,7 +266,10 @@ class AsyncJob(Generic[T]):
"""
if self._inner is None:
raise NotImplementedError(
"job event history is only available for server-side jobs"
_append_doc_url(
"job event history is only available for server-side jobs",
"jobs",
)
)
return await self._inner.events(limit=limit, filter=filter)
@@ -431,7 +434,10 @@ class Job(Generic[T]):
"""
if self._inner is None:
raise NotImplementedError(
"job event history is only available for server-side jobs"
_append_doc_url(
"job event history is only available for server-side jobs",
"jobs",
)
)
return LOOP.run(self._inner.events(limit=limit, filter=filter))
python/python/tests/test_doc_links.py
+54
View File
@@ -0,0 +1,54 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
import lancedb
from lancedb._lancedb import __version__ as _core_version, _append_doc_url, _doc_url
from lancedb.db import DBConnection
from lancedb.job import AsyncJob, Job
import pytest
def test_python_sdk_utm_content_and_doc_links(tmp_path):
compaction_url = _doc_url("compaction")
expected_content = f"utm_content=python-{_core_version}"
assert (
compaction_url
== f"https://docs.lancedb.com/performance?utm_source=lancedb_sdk&utm_medium=warning&utm_campaign=compaction&{expected_content}#compaction-and-cleanup"
)
db = lancedb.connect(tmp_path)
with pytest.raises(NotImplementedError) as sql_exc:
db.execute_query("SELECT 1")
assert _doc_url("sql") in str(sql_exc.value)
with pytest.raises(NotImplementedError) as gpu_exc:
db.list_gpu_classes()
assert _doc_url("gpu_classes") in str(gpu_exc.value)
with pytest.raises(NotImplementedError) as job_exc:
db.open_job("job-1")
assert _doc_url("jobs") in str(job_exc.value)
with pytest.raises(NotImplementedError) as secret_exc:
db.create_secret("s", "v")
assert _doc_url("secrets") in str(secret_exc.value)
with pytest.raises(NotImplementedError) as view_exc:
db.create_view("v", "SELECT 1")
assert _doc_url("views") in str(view_exc.value)
with pytest.raises(NotImplementedError) as mv_exc:
DBConnection.open_materialized_view(db, "mv")
assert _doc_url("materialized_views") in str(mv_exc.value)
with pytest.raises(NotImplementedError) as create_mv_exc:
DBConnection.create_materialized_view(db, "mv", "SELECT 1")
assert _doc_url("materialized_views") in str(create_mv_exc.value)
local_job = Job(AsyncJob(None))
with pytest.raises(NotImplementedError) as events_exc:
local_job.events()
assert _doc_url("jobs") in str(events_exc.value)
appended = _append_doc_url("Something failed.", "sql")
assert appended.endswith(f"See {_doc_url('sql')}")
python/src/lib.rs
+19
View File
@@ -58,8 +58,25 @@ fn shutdown_runtime(py: Python<'_>) {
py.detach(|| runtime::shutdown(std::time::Duration::from_secs(5)));
}
#[pyfunction]
fn _doc_url(campaign: &str) -> PyResult<String> {
let link = lancedb::docs::DocLink::from_campaign(campaign).ok_or_else(|| {
pyo3::exceptions::PyValueError::new_err(format!("unknown doc link campaign: {campaign}"))
})?;
Ok(link.url())
}
#[pyfunction]
fn _append_doc_url(message: &str, campaign: &str) -> PyResult<String> {
let link = lancedb::docs::DocLink::from_campaign(campaign).ok_or_else(|| {
pyo3::exceptions::PyValueError::new_err(format!("unknown doc link campaign: {campaign}"))
})?;
Ok(link.append_to(message))
}
#[pymodule]
pub fn _lancedb(py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> {
lancedb::docs::set_sdk_ident("python", env!("CARGO_PKG_VERSION"));
let env = Env::new()
.filter_or("LANCEDB_LOG", "warn")
.write_style("LANCEDB_LOG_STYLE");
@@ -133,6 +150,8 @@ pub fn _lancedb(py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> {
m.add_function(wrap_pyfunction!(expr_col, m)?)?;
m.add_function(wrap_pyfunction!(expr_lit, m)?)?;
m.add_function(wrap_pyfunction!(expr_func, m)?)?;
m.add_function(wrap_pyfunction!(_doc_url, m)?)?;
m.add_function(wrap_pyfunction!(_append_doc_url, m)?)?;
m.add("__version__", env!("CARGO_PKG_VERSION"))?;
m.add(
"_experimental_functions_enabled",
rust/lancedb/cli/src/main.rs
+1
View File
@@ -5,6 +5,7 @@ use clap::Parser;
#[tokio::main]
async fn main() -> anyhow::Result<()> {
lancedb::docs::set_sdk_ident("cli", env!("CARGO_PKG_VERSION"));
let _ = rustls::crypto::ring::default_provider().install_default();
lancedb_cli::run(lancedb_cli::Args::parse()).await
}
rust/lancedb/src/catalog.rs
+2 -1
View File
@@ -206,7 +206,8 @@ pub trait Catalog: Send + Sync + std::fmt::Debug + 'static {
/// Obtain deployment-wide authorization, if supported by this backend.
fn authz(&self) -> Result<Arc<dyn crate::authz::Authorization>> {
Err(crate::Error::NotSupported {
message: "Authorization administration is not supported by this catalog".into(),
message: crate::docs::DocLink::Authorization
.append_to("Authorization administration is not supported by this catalog"),
})
}
rust/lancedb/src/database.rs
+26 -16
View File
@@ -28,6 +28,7 @@ use lance_namespace::models::{
};
use crate::data::scannable::Scannable;
use crate::docs::DocLink;
use crate::error::Result;
use crate::function::GpuClass;
use crate::graph::{PropertyGraphDefinition, PropertyGraphDescription};
@@ -296,31 +297,40 @@ pub enum ResumeJobStatus {
fn job_op_not_supported<T>(what: &str) -> Result<T> {
Err(crate::error::Error::NotSupported {
message: format!("{} is not supported by this database", what),
message: DocLink::Jobs.append_to(format!("{what} is not supported by this database")),
})
}
fn materialized_view_op_not_supported<T>(what: &str) -> Result<T> {
Err(crate::error::Error::NotSupported {
message: DocLink::MaterializedViews
.append_to(format!("{what} is not supported by this database")),
})
}
fn function_catalog_not_supported<T>() -> Result<T> {
Err(crate::error::Error::NotSupported {
message: "Function catalog operations are not supported by this database".to_string(),
message: DocLink::Functions
.append_to("Function catalog operations are not supported by this database"),
})
}
fn gpu_class_catalog_not_supported<T>() -> Result<T> {
Err(crate::error::Error::NotSupported {
message: "GPU class discovery is supported only by LanceDB Enterprise".to_string(),
message: DocLink::GpuClasses
.append_to("GPU class discovery is supported only by LanceDB Enterprise"),
})
}
fn secret_catalog_not_supported<T>() -> Result<T> {
Err(crate::error::Error::NotSupported {
message: "Secret operations are not supported by this database".to_string(),
message: DocLink::Secrets.append_to("Secret operations are not supported by this database"),
})
}
fn view_ops_not_supported<T>() -> Result<T> {
Err(crate::error::Error::NotSupported {
message: "View operations are not supported by this database".to_string(),
message: DocLink::Views.append_to("View operations are not supported by this database"),
})
}
@@ -330,6 +340,12 @@ fn property_graph_ops_not_supported<T>() -> Result<T> {
})
}
fn sql_not_supported<T>() -> Result<T> {
Err(crate::error::Error::NotSupported {
message: DocLink::Sql.append_to("SQL is not supported by this database"),
})
}
/// The `Database` trait defines the interface for database implementations.
///
/// A database is responsible for managing tables and their metadata.
@@ -396,7 +412,7 @@ pub trait Database:
&self,
_request: CreateMaterializedViewRequest,
) -> Result<Job> {
job_op_not_supported("remote materialized-view creation")
materialized_view_op_not_supported("remote materialized-view creation")
}
/// Drop a materialized view through its resource endpoint and return its
/// cleanup job. Local connections validate the view and use table drop.
@@ -406,7 +422,7 @@ pub trait Database:
_name: &str,
_namespace_path: &[String],
) -> Result<Job> {
job_op_not_supported("remote materialized-view drop")
materialized_view_op_not_supported("remote materialized-view drop")
}
/// Fetch one page of materialized-view names in a namespace.
/// Local backends filter a page of table names, so an empty result may
@@ -676,24 +692,18 @@ pub trait Database:
}
/// Establish the SQL transport without executing a statement.
async fn check_sql_connection(&self) -> Result<()> {
Err(crate::Error::NotSupported {
message: "SQL is not supported by this database".into(),
})
sql_not_supported()
}
/// Start executing a SQL statement on a LanceDB Enterprise database.
async fn execute_query_async(
&self,
_request: ExecuteQueryRequest,
) -> Result<crate::sql::Query> {
Err(crate::error::Error::NotSupported {
message: "SQL is not supported by this database".to_string(),
})
sql_not_supported()
}
/// Describe a submitted SQL query by its connection-scoped id.
async fn describe_query(&self, _query_id: uuid::Uuid) -> Result<crate::sql::QueryDescription> {
Err(crate::error::Error::NotSupported {
message: "SQL is not supported by this database".to_string(),
})
sql_not_supported()
}
/// Open a table in the database
async fn open_table(&self, request: OpenTableRequest) -> Result<Arc<dyn BaseTable>>;
rust/lancedb/src/docs.rs
+877
View File
@@ -0,0 +1,877 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
//! Central registry of documentation links emitted in SDK warnings and
//! Enterprise-only error messages.
//!
//! Every link appends standard UTM query parameters (`utm_source=lancedb_sdk`,
//! `utm_medium=warning|error`, `utm_campaign=<link id>`,
//! `utm_content=<sdk>-<version>`) before any `#anchor` fragment, and never
//! includes table names, URIs, hostnames, machine IDs, or other user data.
use std::collections::HashSet;
use std::sync::{LazyLock, Mutex, OnceLock};
use lance::Dataset;
use roaring::RoaringBitmap;
/// Base URL for the LanceDB documentation site.
pub const DOCS_BASE_URL: &str = "https://docs.lancedb.com";
/// Fragment size threshold (in physical rows) below which a fragment is counted
/// as a small fragment for compaction warnings and `Table::stats()`.
pub const SMALL_FRAGMENT_ROW_THRESHOLD: usize = 100_000;
/// Minimum number of small fragments (`< 100k` physical rows each) on an opened
/// table before emitting a one-time compaction warning.
pub const SMALL_FRAGMENT_WARN_COUNT_THRESHOLD: usize = 100;
/// Minimum number of unindexed rows on any user index of an opened table before
/// emitting a one-time index-maintenance warning.
pub const UNINDEXED_ROWS_WARN_THRESHOLD: u64 = 1_000_000;
static SDK_UTM_CONTENT: OnceLock<String> = OnceLock::new();
static SMALL_FRAGMENTS_WARNED: LazyLock<Mutex<HashSet<String>>> =
LazyLock::new(|| Mutex::new(HashSet::new()));
static UNINDEXED_ROWS_WARNED: LazyLock<Mutex<HashSet<String>>> =
LazyLock::new(|| Mutex::new(HashSet::new()));
/// Record which SDK and version are running in this process for `utm_content`.
///
/// Language bindings (`python`, `nodejs`, `cli`) call this once during module
/// initialization. If never called, `utm_content` defaults to
/// `rust-<CARGO_PKG_VERSION>`.
pub fn set_sdk_ident(sdk: &str, version: &str) {
let _ = SDK_UTM_CONTENT.set(format!("{sdk}-{version}"));
}
/// The `<sdk>-<version>` value used for `utm_content`.
pub fn sdk_utm_content() -> &'static str {
SDK_UTM_CONTENT
.get_or_init(|| concat!("rust-", env!("CARGO_PKG_VERSION")).to_string())
.as_str()
}
/// The `utm_medium` parameter value for a documentation link.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum UtmMedium {
Warning,
Error,
}
impl UtmMedium {
pub const fn as_str(self) -> &'static str {
match self {
Self::Warning => "warning",
Self::Error => "error",
}
}
}
/// Every documentation link emitted by the SDK.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum DocLink {
/// Compaction maintenance warning for tables with many small fragments.
Compaction,
/// Incremental reindexing maintenance warning for tables with many unindexed rows.
Reindexing,
/// SQL execution (Enterprise only).
Sql,
/// Server-side job management and event history (Enterprise only).
Jobs,
/// Registered Python Functions and Function-backed columns (Enterprise only).
Functions,
/// GPU class discovery for Function workers (Enterprise only).
GpuClasses,
/// Database Secrets and storage integrations (Enterprise only).
Secrets,
/// Per-row Function error store (Enterprise only).
FunctionErrors,
/// SQL views (Enterprise only).
Views,
/// Materialized views (Enterprise only).
MaterializedViews,
/// Branch diff against `main` (Enterprise only).
BranchDiff,
/// Branch cherry-pick onto `main` (Enterprise only).
BranchCherryPick,
/// Data cache prewarming (Enterprise only).
PrewarmData,
/// Deployment-wide authorization administration (Enterprise only).
Authorization,
}
impl DocLink {
/// Every defined [`DocLink`] variant, enumerated by CI and unit tests.
pub const ALL: &'static [Self] = &[
Self::Compaction,
Self::Reindexing,
Self::Sql,
Self::Jobs,
Self::Functions,
Self::GpuClasses,
Self::Secrets,
Self::FunctionErrors,
Self::Views,
Self::MaterializedViews,
Self::BranchDiff,
Self::BranchCherryPick,
Self::PrewarmData,
Self::Authorization,
];
/// Look up a [`DocLink`] by its `utm_campaign` identifier.
pub fn from_campaign(campaign: &str) -> Option<Self> {
Self::ALL
.iter()
.copied()
.find(|link| link.campaign() == campaign)
}
/// The `utm_campaign` identifier for this link.
pub const fn campaign(self) -> &'static str {
match self {
Self::Compaction => "compaction",
Self::Reindexing => "reindexing",
Self::Sql => "sql",
Self::Jobs => "jobs",
Self::Functions => "functions",
Self::GpuClasses => "gpu_classes",
Self::Secrets => "secrets",
Self::FunctionErrors => "function_errors",
Self::Views => "views",
Self::MaterializedViews => "materialized_views",
Self::BranchDiff => "branch_diff",
Self::BranchCherryPick => "branch_cherry_pick",
Self::PrewarmData => "prewarm_data",
Self::Authorization => "authorization",
}
}
/// The documentation page path (starting with `/`).
///
/// Every path here must return HTTP 200 on the currently published
/// `https://docs.lancedb.com` edition and also resolve (directly or via a
/// redirect entry in `docs/web/docs.json`) in the upcoming `docs/web`
/// edition.
pub const fn path(self) -> &'static str {
match self {
Self::Compaction => "/performance",
Self::Reindexing => "/indexing/reindexing",
Self::Sql => "/search/sql",
Self::BranchDiff | Self::BranchCherryPick => "/tables/branching",
Self::Jobs
| Self::Functions
| Self::GpuClasses
| Self::Secrets
| Self::FunctionErrors
| Self::Views
| Self::MaterializedViews
| Self::PrewarmData
| Self::Authorization => "/enterprise",
}
}
/// Optional `#anchor` fragment on the target documentation page.
pub const fn anchor(self) -> Option<&'static str> {
match self {
Self::Compaction => Some("compaction-and-cleanup"),
Self::Reindexing => Some("incremental-reindexing"),
Self::BranchDiff => Some("diff-a-branch"),
Self::BranchCherryPick => Some("merge-a-branch"),
Self::Sql
| Self::Jobs
| Self::Functions
| Self::GpuClasses
| Self::Secrets
| Self::FunctionErrors
| Self::Views
| Self::MaterializedViews
| Self::PrewarmData
| Self::Authorization => None,
}
}
/// The `utm_medium` associated with this link.
pub const fn medium(self) -> UtmMedium {
match self {
Self::Compaction | Self::Reindexing => UtmMedium::Warning,
Self::Sql
| Self::Jobs
| Self::Functions
| Self::GpuClasses
| Self::Secrets
| Self::FunctionErrors
| Self::Views
| Self::MaterializedViews
| Self::BranchDiff
| Self::BranchCherryPick
| Self::PrewarmData
| Self::Authorization => UtmMedium::Error,
}
}
/// Build the full URL with UTM parameters placed before any `#anchor`.
pub fn url(self) -> String {
self.url_with_content(sdk_utm_content())
}
pub(crate) fn url_with_content(self, utm_content: &str) -> String {
let path = self.path();
let medium = self.medium().as_str();
let campaign = self.campaign();
match self.anchor() {
Some(anchor) => format!(
"{DOCS_BASE_URL}{path}?utm_source=lancedb_sdk&utm_medium={medium}&utm_campaign={campaign}&utm_content={utm_content}#{anchor}"
),
None => format!(
"{DOCS_BASE_URL}{path}?utm_source=lancedb_sdk&utm_medium={medium}&utm_campaign={campaign}&utm_content={utm_content}"
),
}
}
/// Append `. See <url>` (or ` See <url>` if `message` already ends with `.`)
/// to an error or warning message.
pub fn append_to(self, message: impl AsRef<str>) -> String {
let message = message.as_ref().trim_end();
let url = self.url();
if message.ends_with('.') {
format!("{message} See {url}")
} else {
format!("{message}. See {url}")
}
}
}
fn is_warned(set: &Mutex<HashSet<String>>, table_key: &str) -> bool {
set.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.contains(table_key)
}
fn mark_warned(set: &Mutex<HashSet<String>>, table_key: &str) -> bool {
set.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.insert(table_key.to_string())
}
/// Check already-loaded dataset metadata when a table is opened and emit
/// rate-limited maintenance warnings (at most once per table per process).
pub(crate) async fn check_table_maintenance_warnings(dataset: &Dataset, table_key: &str) {
if !log::log_enabled!(log::Level::Warn) {
return;
}
if let Some(msg) = check_small_fragments_warning(dataset, table_key) {
log::warn!("{msg}");
}
if let Some(msg) = check_unindexed_rows_warning(dataset, table_key).await {
log::warn!("{msg}");
}
}
pub(crate) fn check_small_fragments_warning(dataset: &Dataset, table_key: &str) -> Option<String> {
if is_warned(&SMALL_FRAGMENTS_WARNED, table_key) {
return None;
}
dataset.manifest().writer_version.as_ref()?;
let num_small_fragments = dataset
.manifest()
.fragments
.iter()
.filter(|frag| {
frag.physical_rows
.is_some_and(|rows| rows < SMALL_FRAGMENT_ROW_THRESHOLD)
})
.count();
if num_small_fragments >= SMALL_FRAGMENT_WARN_COUNT_THRESHOLD
&& mark_warned(&SMALL_FRAGMENTS_WARNED, table_key)
{
Some(format!(
"Table has {num_small_fragments} small fragments (<100k rows each), \
which can degrade query performance. Run optimize() to compact fragments. \
See {}",
DocLink::Compaction.url()
))
} else {
None
}
}
fn segment_supports_optimize_without_io(segment: &lance_table::format::IndexMetadata) -> bool {
let Some(raw_details) = segment.index_details.as_ref() else {
return false;
};
let details = lance::index::scalar::IndexDetails(raw_details.clone());
if details.is_vector() {
return true;
}
details
.get_plugin()
.is_ok_and(|plugin| plugin.supports_optimization())
}
pub(crate) async fn check_unindexed_rows_warning(
dataset: &Dataset,
table_key: &str,
) -> Option<String> {
if is_warned(&UNINDEXED_ROWS_WARNED, table_key) {
return None;
}
dataset.manifest().index_section?;
let total_rows: u64 = dataset
.manifest()
.fragments
.iter()
.filter_map(|frag| frag.num_rows().map(|rows| rows as u64))
.sum();
if total_rows < UNINDEXED_ROWS_WARN_THRESHOLD {
return None;
}
let store = dataset.object_store(None).await.ok()?;
let raw_indices = lance_table::io::manifest::read_manifest_indexes(
store.as_ref(),
dataset.manifest_location(),
dataset.manifest(),
)
.await
.ok()?;
if raw_indices
.iter()
.any(|idx| idx.name == lance_index::frag_reuse::FRAG_REUSE_INDEX_NAME)
{
// Remapping fragment coverage after fragment reuse requires opening the
// fragment-reuse index artifact under `_indices/`. Skip rather than
// read index files during `open_table()`.
return None;
}
let mut covered_by_index: std::collections::BTreeMap<&str, Option<RoaringBitmap>> =
std::collections::BTreeMap::new();
for index in &raw_indices {
if lance_index::is_system_index(index) {
continue;
}
let Some(bitmap) = index
.fragment_bitmap
.as_ref()
.filter(|_| segment_supports_optimize_without_io(index))
else {
covered_by_index.insert(index.name.as_str(), None);
continue;
};
match covered_by_index.get_mut(index.name.as_str()) {
Some(Some(acc)) => *acc |= bitmap,
Some(None) => {}
None => {
covered_by_index.insert(index.name.as_str(), Some(bitmap.clone()));
}
}
}
for covered in covered_by_index.values().flatten() {
let unindexed_rows: u64 = dataset
.manifest()
.fragments
.iter()
.filter(|frag| !covered.contains(frag.id as u32))
.filter_map(|frag| frag.num_rows().map(|rows| rows as u64))
.sum();
if unindexed_rows >= UNINDEXED_ROWS_WARN_THRESHOLD {
if mark_warned(&UNINDEXED_ROWS_WARNED, table_key) {
return Some(format!(
"Table has {unindexed_rows} unindexed rows, \
which can degrade search performance. Run optimize() to update indexes. \
See {}",
DocLink::Reindexing.url()
));
}
return None;
}
}
None
}
#[cfg(test)]
mod tests {
use std::path::PathBuf;
use std::sync::Arc;
use arrow_array::{FixedSizeListArray, Float32Array, Int32Array, RecordBatch};
use arrow_schema::{DataType, Field, Schema};
use lance::dataset::WriteDestination;
use lance::dataset::builder::DatasetBuilder;
use lance::dataset::transaction::Operation;
use lance_table::format::Fragment;
use lance_table::io::commit::write_manifest_file_to_path;
use lance_table::io::manifest::read_manifest_indexes;
use tempfile::tempdir;
use crate::connect;
use crate::database::ExecuteQueryRequest;
use crate::function::FunctionErrorsRequest;
use crate::index::Index;
use crate::index::vector::IvfFlatIndexBuilder;
use super::*;
fn strip_mdx_comments(content: &str) -> String {
let mut out = String::with_capacity(content.len());
let mut rest = content;
while let Some(start) = rest.find("{/*") {
out.push_str(&rest[..start]);
match rest[start + 3..].find("*/}") {
Some(end) => {
rest = &rest[start + 3 + end + 3..];
}
None => return out,
}
}
out.push_str(rest);
out
}
#[test]
fn test_doc_link_url_format_and_campaign_lookup() {
for &link in DocLink::ALL {
assert_eq!(DocLink::from_campaign(link.campaign()), Some(link));
let url = link.url_with_content("python-0.42.0");
let (before_hash, fragment) = match url.split_once('#') {
Some((before, frag)) => (before, Some(frag)),
None => (url.as_str(), None),
};
assert!(
before_hash.starts_with(&format!(
"{DOCS_BASE_URL}{}?utm_source=lancedb_sdk&",
link.path()
)),
"unexpected URL prefix: {url}"
);
assert!(
before_hash.contains(&format!("utm_medium={}", link.medium().as_str())),
"missing utm_medium in {url}"
);
assert!(
before_hash.contains(&format!("utm_campaign={}", link.campaign())),
"missing utm_campaign in {url}"
);
assert!(
before_hash.contains("utm_content=python-0.42.0"),
"missing utm_content before fragment in {url}"
);
assert_eq!(fragment, link.anchor(), "mismatched #anchor in {url}");
}
}
#[test]
fn test_all_doc_links_resolve_in_repo_docs() {
let manifest_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
let docs_web = manifest_dir.join("../../docs/web");
let docs_json = std::fs::read_to_string(docs_web.join("docs.json"))
.expect("docs/web/docs.json must exist");
let parsed_docs_json: serde_json::Value =
serde_json::from_str(&docs_json).expect("docs/web/docs.json must be valid JSON");
let redirect_map: std::collections::HashMap<String, String> = parsed_docs_json
.get("redirects")
.and_then(|v| v.as_array())
.into_iter()
.flatten()
.filter_map(|entry| {
Some((
entry.get("source")?.as_str()?.to_string(),
entry.get("destination")?.as_str()?.to_string(),
))
})
.collect();
let mintignore = std::fs::read_to_string(docs_web.join(".mintignore")).unwrap_or_default();
let ignored: HashSet<&str> = mintignore
.lines()
.map(str::trim)
.filter(|line| !line.is_empty() && !line.starts_with('#'))
.collect();
for &link in DocLink::ALL {
if let Some(destination) = redirect_map.get(link.path()) {
let dest_page = destination
.split_once('#')
.map_or(destination.as_str(), |(p, _)| p)
.trim_start_matches('/');
let dest_mdx = if dest_page.is_empty() || dest_page == "index" {
docs_web.join("index.mdx")
} else if docs_web.join(format!("{dest_page}.mdx")).is_file() {
docs_web.join(format!("{dest_page}.mdx"))
} else {
docs_web.join(format!("{dest_page}/index.mdx"))
};
assert!(
dest_mdx.is_file(),
"DocLink::{link:?} redirect destination '{destination}' does not resolve under docs/web"
);
continue;
}
let rel_page = link.path().trim_start_matches('/');
let direct_mdx = format!("{rel_page}.mdx");
let index_mdx = format!("{rel_page}/index.mdx");
assert!(
!ignored.contains(direct_mdx.as_str()) && !ignored.contains(index_mdx.as_str()),
"DocLink::{link:?} points to a .mintignore'd page: {rel_page}"
);
let mdx_path = if docs_web.join(&direct_mdx).is_file() {
docs_web.join(&direct_mdx)
} else if docs_web.join(&index_mdx).is_file() {
docs_web.join(&index_mdx)
} else {
panic!("DocLink::{link:?} target page does not exist under docs/web: {rel_page}");
};
assert!(
docs_json.contains(&format!("\"{rel_page}\""))
|| docs_json.contains(&format!("\"{rel_page}/index\"")),
"DocLink::{link:?} target page is not routed in docs/web/docs.json: {rel_page}"
);
if let Some(anchor) = link.anchor() {
let raw_mdx = std::fs::read_to_string(&mdx_path)
.unwrap_or_else(|e| panic!("failed to read {}: {e}", mdx_path.display()));
let active_mdx = strip_mdx_comments(&raw_mdx);
let heading_anchor = format!("{{#{anchor}}}");
let span_anchor_double = format!("id=\"{anchor}\"");
let span_anchor_single = format!("id='{anchor}'");
assert!(
active_mdx.contains(&heading_anchor)
|| active_mdx.contains(&span_anchor_double)
|| active_mdx.contains(&span_anchor_single),
"DocLink::{link:?} anchor '#{anchor}' not found in {}",
mdx_path.display()
);
}
}
}
fn make_test_batch() -> RecordBatch {
let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int32, false)]));
RecordBatch::try_new(schema, vec![Arc::new(Int32Array::from_iter_values(0..10))]).unwrap()
}
#[tokio::test]
async fn test_small_fragments_warning_and_rate_limit() {
let tmp_dir = tempdir().unwrap();
let conn = connect(tmp_dir.path().to_str().unwrap())
.execute()
.await
.unwrap();
let table = conn
.create_table("small_frags_tbl", make_test_batch())
.execute()
.await
.unwrap();
let dataset = table.as_native().unwrap().dataset.get().await.unwrap();
assert!(
check_small_fragments_warning(&dataset, "test://small_frags_below").is_none(),
"1 small fragment must not trigger the compaction warning"
);
let existing_frag = dataset.manifest().fragments[0].clone();
let extra_fragments: Vec<Fragment> = (0..SMALL_FRAGMENT_WARN_COUNT_THRESHOLD)
.map(|i| {
let mut frag = existing_frag.clone();
frag.id = (i + 1) as u64;
frag.physical_rows = Some(100);
frag
})
.collect();
let read_version = dataset.version().version;
let updated = Dataset::commit(
WriteDestination::Dataset(dataset),
Operation::Append {
fragments: extra_fragments,
},
Some(read_version),
None,
None,
Arc::new(Default::default()),
false,
)
.await
.unwrap();
let warning = check_small_fragments_warning(&updated, "test://small_frags_table_a")
.expect("101 small fragments should trigger compaction warning");
assert!(warning.contains(&DocLink::Compaction.url()));
assert!(!warning.contains("small_frags_table_a"));
assert!(
check_small_fragments_warning(&updated, "test://small_frags_table_a").is_none(),
"same table must be rate-limited to once per process"
);
assert!(
check_small_fragments_warning(&updated, "test://small_frags_table_b").is_some(),
"a different table must still emit its own warning once"
);
}
#[tokio::test]
async fn test_unindexed_rows_warning_and_rate_limit() {
let tmp_dir = tempdir().unwrap();
let conn = connect(tmp_dir.path().to_str().unwrap())
.execute()
.await
.unwrap();
let table = conn
.create_table("unindexed_rows_tbl", make_test_batch())
.execute()
.await
.unwrap();
table
.create_index(&["id"], Index::Auto)
.execute()
.await
.unwrap();
let dataset = table.as_native().unwrap().dataset.get().await.unwrap();
assert!(
check_unindexed_rows_warning(&dataset, "test://unindexed_below")
.await
.is_none(),
"fully indexed table must not trigger the reindexing warning"
);
let mut unindexed_frag = dataset.manifest().fragments[0].clone();
unindexed_frag.id = 99;
unindexed_frag.physical_rows = Some(UNINDEXED_ROWS_WARN_THRESHOLD as usize);
let read_version = dataset.version().version;
let updated = Dataset::commit(
WriteDestination::Dataset(dataset),
Operation::Append {
fragments: vec![unindexed_frag],
},
Some(read_version),
None,
None,
Arc::new(Default::default()),
false,
)
.await
.unwrap();
let warning = check_unindexed_rows_warning(&updated, "test://unindexed_table_a")
.await
.expect("1,000,000 unindexed rows should trigger reindexing warning");
assert!(warning.contains(&DocLink::Reindexing.url()));
assert!(!warning.contains("unindexed_table_a"));
assert!(
check_unindexed_rows_warning(&updated, "test://unindexed_table_a")
.await
.is_none(),
"same table must be rate-limited to once per process"
);
assert!(
check_unindexed_rows_warning(&updated, "test://unindexed_table_b")
.await
.is_some(),
"a different table must still emit its own warning once"
);
}
#[tokio::test]
async fn test_unindexed_rows_warning_never_reads_index_or_data_files() {
let tmp_dir = tempdir().unwrap();
let conn = connect(tmp_dir.path().to_str().unwrap())
.execute()
.await
.unwrap();
let dim = 4_i32;
let num_rows = 32_usize;
let values =
Float32Array::from_iter_values((0..(num_rows * dim as usize)).map(|v| v as f32));
let vec_field = Arc::new(Field::new("item", DataType::Float32, true));
let vec_col = FixedSizeListArray::try_new(vec_field, dim, Arc::new(values), None).unwrap();
let schema = Arc::new(Schema::new(vec![
Field::new("id", DataType::Int32, false),
Field::new(
"vec",
DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), dim),
false,
),
]));
let batch = RecordBatch::try_new(
schema,
vec![
Arc::new(Int32Array::from_iter_values(0..num_rows as i32)),
Arc::new(vec_col),
],
)
.unwrap();
let table = conn
.create_table("vec_no_io_tbl", batch)
.execute()
.await
.unwrap();
table
.create_index(
&["vec"],
Index::IvfFlat(IvfFlatIndexBuilder::default().num_partitions(1)),
)
.execute()
.await
.unwrap();
let dataset = table.as_native().unwrap().dataset.get().await.unwrap();
let mut unindexed_frag = dataset.manifest().fragments[0].clone();
unindexed_frag.id = 99;
unindexed_frag.physical_rows = Some(UNINDEXED_ROWS_WARN_THRESHOLD as usize);
let read_version = dataset.version().version;
let updated = Dataset::commit(
WriteDestination::Dataset(dataset),
Operation::Append {
fragments: vec![unindexed_frag],
},
Some(read_version),
None,
None,
Arc::new(Default::default()),
false,
)
.await
.unwrap();
let table_uri = tmp_dir
.path()
.join("vec_no_io_tbl.lance")
.to_string_lossy()
.to_string();
// 1. Modern vector index: warns using only manifest metadata, with zero
// `_indices/` or `data/` file reads.
let reopened = DatasetBuilder::from_uri(&table_uri).load().await.unwrap();
let store = reopened.object_store(None).await.unwrap();
let _ = store.io_tracker().incremental_stats();
let warning = check_unindexed_rows_warning(&reopened, "test://vec_modern_no_io").await;
assert!(
warning.is_some(),
"modern vector index with 1M unindexed rows should trigger warning"
);
let stats = store.io_tracker().incremental_stats();
for req in &stats.requests {
let path = req.path.as_ref();
assert!(
!path.contains("_indices/") && !path.contains("/data/"),
"unexpected index or data I/O during warning check: {path}"
);
}
// 2. Legacy vector index without `index_details`: skipped cleanly without
// opening `_indices/<uuid>/index.idx` to infer details.
let mut indices = read_manifest_indexes(
store.as_ref(),
updated.manifest_location(),
updated.manifest(),
)
.await
.unwrap();
for idx in &mut indices {
idx.index_details = None;
}
let mut legacy_manifest = (*updated.manifest()).clone();
write_manifest_file_to_path(
store.as_ref(),
&mut legacy_manifest,
Some(indices),
&reopened.manifest_location().path,
None,
)
.await
.unwrap();
let reopened_legacy = DatasetBuilder::from_uri(&table_uri).load().await.unwrap();
let legacy_store = reopened_legacy.object_store(None).await.unwrap();
let _ = legacy_store.io_tracker().incremental_stats();
let legacy_warning =
check_unindexed_rows_warning(&reopened_legacy, "test://vec_legacy_no_io").await;
assert!(
legacy_warning.is_none(),
"legacy index missing index_details must be skipped without disk inference"
);
let legacy_stats = legacy_store.io_tracker().incremental_stats();
for req in &legacy_stats.requests {
let path = req.path.as_ref();
assert!(
!path.contains("_indices/") && !path.contains("/data/"),
"unexpected index or data I/O for legacy index: {path}"
);
}
}
#[tokio::test]
async fn test_enterprise_only_errors_contain_utm_doc_links() {
let tmp_dir = tempdir().unwrap();
let conn = connect(tmp_dir.path().to_str().unwrap())
.execute()
.await
.unwrap();
let table = conn
.create_table("err_tbl", make_test_batch())
.execute()
.await
.unwrap();
let sql_err = conn
.database()
.execute_query_async(ExecuteQueryRequest::new("SELECT 1"))
.await
.unwrap_err()
.to_string();
assert!(sql_err.contains(&DocLink::Sql.url()), "{sql_err}");
let job_err = conn.open_job("job-1").await.unwrap_err().to_string();
assert!(job_err.contains(&DocLink::Jobs.url()), "{job_err}");
let gpu_err = conn.list_gpu_classes().await.unwrap_err().to_string();
assert!(gpu_err.contains(&DocLink::GpuClasses.url()), "{gpu_err}");
let secret_err = conn
.create_secret("my_secret", "val", &[])
.await
.unwrap_err()
.to_string();
assert!(secret_err.contains(&DocLink::Secrets.url()), "{secret_err}");
let view_err = conn
.create_view("v", "SELECT 1", &[])
.await
.unwrap_err()
.to_string();
assert!(view_err.contains(&DocLink::Views.url()), "{view_err}");
let diff_err = table.diff_branch("feature").await.unwrap_err().to_string();
assert!(diff_err.contains(&DocLink::BranchDiff.url()), "{diff_err}");
let cp_err = table
.cherry_pick("feature", false)
.await
.unwrap_err()
.to_string();
assert!(
cp_err.contains(&DocLink::BranchCherryPick.url()),
"{cp_err}"
);
let prewarm_err = table.prewarm_data(None).await.unwrap_err().to_string();
assert!(
prewarm_err.contains(&DocLink::PrewarmData.url()),
"{prewarm_err}"
);
let fn_err = table
.function_errors(FunctionErrorsRequest::new())
.await
.unwrap_err()
.to_string();
assert!(fn_err.contains(&DocLink::FunctionErrors.url()), "{fn_err}");
}
}
rust/lancedb/src/job.rs
+2 -1
View File
@@ -16,6 +16,7 @@ use tokio::task::{AbortHandle, JoinHandle};
use uuid::Uuid;
use crate::database::JobDescription;
use crate::docs::DocLink;
use crate::error::{Error, JobFailure, Result};
/// Which of a job's events [`Job::events`] returns.
@@ -49,7 +50,7 @@ impl JobEventsRequest {
fn job_detail_not_supported<T>(what: &str) -> Result<T> {
Err(Error::NotSupported {
message: format!("{what} is only available for server-side jobs"),
message: DocLink::Jobs.append_to(format!("{what} is only available for server-side jobs")),
})
}
rust/lancedb/src/lib.rs
+1
View File
@@ -195,6 +195,7 @@ pub mod connection;
pub mod data;
pub mod database;
pub mod dataloader;
pub mod docs;
pub mod embeddings;
pub mod error;
pub mod expr;
rust/lancedb/src/table.rs
+16 -7
View File
@@ -55,6 +55,7 @@ use crate::blob::BlobRangeRequest;
use crate::data::scannable::{PeekedScannable, Scannable, estimate_write_partitions};
use crate::database::Database;
use crate::database::read_freshness::TableFreshness;
use crate::docs::DocLink;
use crate::embeddings::{EmbeddingDefinition, EmbeddingRegistry, MemoryRegistry};
use crate::error::{Error, Result};
use crate::function::FunctionErrorsRequest;
@@ -589,7 +590,8 @@ pub trait BaseTable: std::fmt::Display + std::fmt::Debug + Send + Sync {
_full: bool,
) -> Result<Job<crate::materialized_view::RefreshMaterializedViewResult>> {
Err(Error::NotSupported {
message: "remote materialized-view refresh is not supported on this table type".into(),
message: DocLink::MaterializedViews
.append_to("remote materialized-view refresh is not supported on this table type"),
})
}
/// Create a read-only handle pinned to the table's current active revision.
@@ -867,7 +869,8 @@ pub trait BaseTable: std::fmt::Display + std::fmt::Debug + Send + Sync {
_options: computed_columns::ComputedColumnOptions,
) -> Result<AddColumnsResult> {
Err(Error::NotSupported {
message: "Function columns are supported only in LanceDB Enterprise".into(),
message: DocLink::Functions
.append_to("Function columns are supported only in LanceDB Enterprise"),
})
}
/// Fill a computed column's unfilled rows and recompute those whose
@@ -915,7 +918,8 @@ pub trait BaseTable: std::fmt::Display + std::fmt::Debug + Send + Sync {
_request: &crate::function::FunctionErrorsRequest,
) -> Result<crate::function::FunctionErrors> {
Err(Error::NotSupported {
message: "per-row Function errors are recorded only in LanceDB Enterprise".into(),
message: DocLink::FunctionErrors
.append_to("per-row Function errors are recorded only in LanceDB Enterprise"),
})
}
/// Alter columns in the table.
@@ -988,14 +992,16 @@ pub trait BaseTable: std::fmt::Display + std::fmt::Debug + Send + Sync {
/// Diff a branch against main. Remote only.
async fn diff_branch(&self, _from_branch: &str) -> Result<BranchDiff> {
Err(Error::NotSupported {
message: "Branch diffs are only supported on Enterprise tables.".into(),
message: DocLink::BranchDiff
.append_to("Branch diffs are only supported on Enterprise tables."),
})
}
/// Cherry-pick a branch onto main, or dry-run. Remote only.
/// HTTP 409 still returns [`Ok`] with [`CherryPickStatus::Failed`].
async fn cherry_pick(&self, _from_branch: &str, _dry_run: bool) -> Result<CherryPickResult> {
Err(Error::NotSupported {
message: "Cherry-picking branches is only supported on Enterprise tables.".into(),
message: DocLink::BranchCherryPick
.append_to("Cherry-picking branches is only supported on Enterprise tables."),
})
}
/// The branch this handle is scoped to, or `None` for `main`.
@@ -2968,6 +2974,8 @@ impl NativeTable {
Err(e) => return Err(e.into()),
};
crate::docs::check_table_maintenance_warnings(&dataset, uri).await;
let dataset = DatasetConsistencyWrapper::new_latest(dataset, read_consistency_interval);
let id = Self::build_id(&namespace, name);
@@ -3846,7 +3854,8 @@ impl BaseTable for NativeTable {
async fn prewarm_data(&self, _columns: Option<Vec<String>>) -> Result<()> {
Err(Error::NotSupported {
message: "prewarm_data is currently only supported in LanceDB Enterprise.".into(),
message: DocLink::PrewarmData
.append_to("prewarm_data is currently only supported in LanceDB Enterprise."),
})
}
@@ -4272,7 +4281,7 @@ impl BaseTable for NativeTable {
.await;
sorted_sizes.sort();
let small_frag_threshold = 100000;
let small_frag_threshold = crate::docs::SMALL_FRAGMENT_ROW_THRESHOLD;
let num_fragments = sorted_sizes.len();
let num_small_fragments = sorted_sizes
.iter()
rust/lancedb/tests/first_class_function_slice2.rs
+2 -1
View File
@@ -113,7 +113,8 @@ async fn local_function_catalog_operations_return_stable_not_supported() {
assert!(matches!(
error,
Error::NotSupported { message }
if message == "Function catalog operations are not supported by this database"
if message.starts_with("Function catalog operations are not supported by this database")
&& message.contains("utm_campaign=functions")
));
}
}