mirror of
https://github.com/lancedb/lancedb.git
synced 2026-10-12 14:52:26 +00:00
Jack YeandBrendan Clement
47b7babc08
Move the public website pages into `docs/web` beside the SDK, with
shared navigation and executable example sources in `docs/web-tests`.
The combined website is assembled by
[lancedb/docs#355](https://github.com/lancedb/docs/pull/355), with six
Enterprise replacements from
[lancedb/sophon#7681](https://github.com/lancedb/sophon/pull/7681).
This includes Function guidance, runnable Quickstarts, regenerated
snippets, production contact/demo fixes, the legacy SQL route, and the
LeRobot redirect. LeRobot remains attributed to Geneva; all 34 Geneva
pages remain in the docs repository until Function officially launches.
Materialized-view guidance follows the full rebuild refresh introduced
by #4373: local refresh writes the complete result, has no `full`
argument, supports `source_version`, and no longer requires stable row
IDs. Deployment SQL refresh options and stable-ID setup are qualified by
version/configuration. Function input guidance distinguishes ordinary
stored columns from computed inputs supported by newer deployments.
**Validation:** independently accepted local checks include 95 Python
tests (real Enterprise excluded), eight TypeScript tests, Rust
Quickstart, snippet parity, typos, deterministic assembly, Mint
validation and links/anchors/redirects. The combined baseline retains
all 181 production routes and 17 legacy redirects. Examples ran locally
against source-built Python SDK 0.40.0b13; existing GitHub workflows do
not run `docs/web-tests`. New main at `069b2f7d` is 0.41.0b0 and adds
computed Function inputs; that SDK was not exercised by the accepted
tests. Latest prose corrections receive focused site validation.
**Merge/rollout:** merge this and #7681 before #355's combined CI, which
reads producer `main`. This is migration infrastructure: section
overlays, source-owned skills and automatic versioned paired releases
remain follow-ups. The current manual publisher is transitional. Hosted
validation, a supported SDK/Enterprise release pair and a real
Enterprise walkthrough remain pending.
Production Mintlify watches `lancedb/docs:deploy-freeze`, directory
`docs`; preserve that source through review. The separate SDK GitHub
Pages reference workflow rebuilds on main merges.
**Updated-head CI (9a50f8db):**
[Typos](https://github.com/lancedb/lancedb/actions/runs/37654185443/job/112904990558),
[PR
title](https://github.com/lancedb/lancedb/actions/runs/37654179507/job/112904974595)
and
[Aikido](https://app.aikido.dev/featurebranch/scan/215386145?groupId=74229)
pass. Critical findings in the example lockfiles were patched: all Sharp
copies use 0.35.5, protobufjs 7.6.6, tar 7.5.22, quinn-proto 0.11.19,
zerovec 0.11.8 and zerovec-derive 0.11.6. The SDK's product dependencies
are unchanged. The revised lockfiles pass eight TypeScript examples,
HuggingFace decode/resize/crop/rescale/normalize plus AVIF roundtrip,
and Rust Quickstart. Aikido passing does not mean every advisory is
cleared; npm still reports 29 high and five moderate advisories in the
example dependency graph. No scanner bypass or suppression was added.
The Function prose correction passes combined/standalone Mint,
deterministic assembly, routes and assets. The revised 325-file site
checksum is
`175bec06864abb400dd292e667ba93448c0479b8359fd7bf6146afabf9040647`. This
PR remains open for independent review; further merges are on hold.
---------
Co-authored-by: Brendan Clement <brendan.clement@lancedb.com>
85 lines
2.9 KiB
Python
85 lines
2.9 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
|
|
|
import lancedb
|
|
import numpy as np
|
|
import pyarrow as pa
|
|
import pytest
|
|
|
|
|
|
def test_rebuild_vector_index(tmp_db):
|
|
rng = np.random.default_rng(7)
|
|
table = tmp_db.create_table(
|
|
"rebuild_index",
|
|
[{"id": i, "vector": v.tolist()} for i, v in enumerate(rng.random((512, 4)))],
|
|
)
|
|
table.create_index(
|
|
"vector", config=lancedb.index.IvfFlat(distance_type="l2", num_partitions=4)
|
|
)
|
|
original_index = table.list_indices()[0]
|
|
table.add(
|
|
[
|
|
{"id": 512 + i, "vector": v.tolist()}
|
|
for i, v in enumerate(rng.random((64, 4)) + 5)
|
|
]
|
|
)
|
|
assert table.index_stats("vector_idx").num_unindexed_rows == 64
|
|
|
|
# --8<-- [start:storage_rebuild_vector_index]
|
|
from lancedb.index import IvfFlat
|
|
|
|
table.create_index(
|
|
"vector",
|
|
config=IvfFlat(distance_type="l2", num_partitions=4),
|
|
replace=True,
|
|
)
|
|
# --8<-- [end:storage_rebuild_vector_index]
|
|
|
|
indices = table.list_indices()
|
|
assert len(indices) == 1
|
|
assert indices[0].name == original_index.name
|
|
assert indices[0].index_uuid != original_index.index_uuid
|
|
stats = table.index_stats("vector_idx")
|
|
assert stats.num_indexed_rows == 576
|
|
assert stats.num_unindexed_rows == 0
|
|
|
|
|
|
def test_local_shallow_clone_pins_source_version(tmp_path):
|
|
db = lancedb.connect(str(tmp_path))
|
|
source = db.create_table("source", [{"id": 1}])
|
|
clone = db.clone_table("snapshot", source_uri=str(tmp_path / "source.lance"))
|
|
source.add([{"id": 2}])
|
|
assert source.count_rows() == 2
|
|
assert clone.to_arrow().to_pylist() == [{"id": 1}]
|
|
|
|
|
|
def _reader():
|
|
return pa.RecordBatchReader.from_batches(
|
|
pa.schema([("id", pa.int64())]),
|
|
[pa.record_batch([[1, 2]], names=["id"]), pa.record_batch([[3]], names=["id"])],
|
|
)
|
|
|
|
|
|
@pytest.mark.parametrize("parallelism", [1, 2])
|
|
def test_streaming_write_parallelism(tmp_db, parallelism):
|
|
table = tmp_db.create_table("streaming", [{"id": 0}])
|
|
progress = []
|
|
table.add(_reader(), write_parallelism=parallelism, progress=progress.append)
|
|
assert sorted(table.to_arrow()["id"].to_pylist()) == [0, 1, 2, 3]
|
|
assert progress[-1]["done"]
|
|
assert progress[-1]["output_rows"] == 3
|
|
assert {p["total_tasks"] for p in progress} == {parallelism}
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.parametrize("parallelism", [1, 2])
|
|
async def test_async_streaming_write_parallelism(mem_db_async, parallelism):
|
|
table = await mem_db_async.create_table("streaming", [{"id": 0}])
|
|
progress = []
|
|
await table.add(_reader(), write_parallelism=parallelism, progress=progress.append)
|
|
data = await table.to_arrow()
|
|
assert sorted(data["id"].to_pylist()) == [0, 1, 2, 3]
|
|
assert progress[-1]["done"]
|
|
assert progress[-1]["output_rows"] == 3
|
|
assert {p["total_tasks"] for p in progress} == {parallelism}
|