From b87c4de53bbf40e936d245933b74fffbb02e1f22 Mon Sep 17 00:00:00 2001 From: Brendan Clement Date: Mon, 5 Oct 2026 14:07:14 -0700 Subject: [PATCH] docs: regenerate the vector-index snippets the page imports Porting the upstream changes replaced snippets/indexing.mdx with a hand copy from lancedb/docs, which dropped six exports the vector-index page still imports (`mint validate` warns on each) and added PyQuantizationCustomParams, which had no source here. The quantization test region is ported from lancedb/docs and the module is regenerated, so every export comes from a test again. The regenerated PyQuantizationCustomParams matches the copy byte for byte. Also fixes "convetion" and "entires" on the page. --- docs/web-tests/py/test_indexing.py | 19 +++++++++++++++++++ docs/web/indexing/vector-index.mdx | 4 ++-- docs/web/snippets/indexing.mdx | 14 +++++++++++++- 3 files changed, 34 insertions(+), 3 deletions(-) diff --git a/docs/web-tests/py/test_indexing.py b/docs/web-tests/py/test_indexing.py index 7f63e8f04..80010343c 100644 --- a/docs/web-tests/py/test_indexing.py +++ b/docs/web-tests/py/test_indexing.py @@ -336,6 +336,25 @@ def test_vector_index_binary(tmp_db): assert not df.empty +def test_quantization_custom_params(tmp_db): + table = tmp_db.create_table( + "quantization-custom-params", + _make_vector_rows(256, 64), + mode="overwrite", + ) + + # --8<-- [start:quantization_custom_params] + table.create_index( + index_type="IVF_RQ", + num_bits=2, + max_iterations=100, + sample_rate=512, + ) + # --8<-- [end:quantization_custom_params] + + assert table.list_indices() + + def test_vector_index_check_status(tmp_db): table = tmp_db.create_table( "vector_index_check_status", diff --git a/docs/web/indexing/vector-index.mdx b/docs/web/indexing/vector-index.mdx index 21fa3c053..3cd3a970a 100644 --- a/docs/web/indexing/vector-index.mdx +++ b/docs/web/indexing/vector-index.mdx @@ -322,7 +322,7 @@ These fields are populated for local and embedded tables. On LanceDB Enterprise #### Custom Index Names -The `{column}_idx` suffix is the default naming convetion. +The `{column}_idx` suffix is the default naming convention. Pass `name=...` to `create_index()` to override it. Once set, the custom name will be reflected in `list_indices()`, `index_stats(name)`, and `wait_for_index([name])`. @@ -388,7 +388,7 @@ See this [blog post](https://lancedb.com/blog/feature-rabitq-quantization/) for Scalar quantization quantizes each entry of a vector independently, by simply replacing it with the closest of a pre-defined set of values. In practice, it often uses $8$ bits per dimension of a vector, and supports very fast encoding and decoding. -For example, suppose we know all vector entires across our dataset lie in the range $[-128 \times 10^5, 127 \times 10^5]$. +For example, suppose we know all vector entries across our dataset lie in the range $[-128 \times 10^5, 127 \times 10^5]$. In this case, we could quantize a given value $v$ as an $8$-bit representation of the integer $j$, where $j \times 10^5$ is the closest value to $v$ among all integral multiples $\{j \times 10^5: -128 \leq j \leq 127 \}$. diff --git a/docs/web/snippets/indexing.mdx b/docs/web/snippets/indexing.mdx index a26cb1c0c..34e9b72f7 100644 --- a/docs/web/snippets/indexing.mdx +++ b/docs/web/snippets/indexing.mdx @@ -40,14 +40,24 @@ export const PyScalarIndexWait = "index_name = \"label_idx\"\ntable.wait_for_ind export const PyVectorIndexAsyncConfig = "import lancedb\nimport numpy as np\nfrom lancedb.index import IvfPq\n\nasync def main():\n data = [\n {\"id\": i, \"vector\": np.random.random(8).astype(np.float32).tolist()}\n for i in range(512)\n ]\n\n db = await lancedb.connect_async(\"ex_lancedb\")\n table = await db.create_table(\n \"vector_index_async\", data=data, mode=\"overwrite\"\n )\n\n await table.create_index(\n \"vector\",\n config=IvfPq(\n distance_type=\"cosine\",\n num_partitions=16,\n num_sub_vectors=4,\n ),\n )\n return await table.list_indices()\n"; +export const PyVectorIndexBinaryAddData = "table.add(data)\n"; + +export const PyVectorIndexBinaryBuildIndex = "table.create_index(\n metric=\"hamming\",\n vector_column_name=\"vector\",\n index_type=\"IVF_FLAT\",\n)\n"; + +export const PyVectorIndexBinarySchema = "table = tmp_db.create_table(table_name, schema=schema, mode=\"overwrite\")\n"; + +export const PyVectorIndexBinarySearch = "query = np.random.randint(0, 2, size=ndim)\nquery = np.packbits(query)\ndf = table.search(query).metric(\"hamming\").limit(10).to_pandas()\ndf.vector = df.vector.apply(np.unpackbits)\n"; + export const PyVectorIndexBuildHnsw = "table.create_index(index_type=\"IVF_HNSW_SQ\")\n"; -export const PyVectorIndexBuildIvf = "table_name = \"vector-index-build-ivf\"\ntable = db.open_table(table_name)\ntable.create_index(\n metric=\"cosine\",\n vector_column_name=\"keywords_embeddings\",\n index_type=\"IVF_PQ\",\n)\n"; +export const PyVectorIndexBuildIvf = "table_name = \"vector-index-build-ivf\"\ntable = db.open_table(table_name)\ntable.create_index(\n metric=\"cosine\",\n vector_column_name=\"keywords_embeddings\",\n)\n"; export const PyVectorIndexBypassRecall = "query = np.random.random(128)\nk = 10\n\n# Ground truth: flat (exhaustive) scan, ignoring the ANN index.\ntruth = set(table.search(query).bypass_vector_index().limit(k).to_pandas()[\"id\"])\n\n# ANN results with the current nprobes setting.\nann = set(table.search(query).nprobes(20).limit(k).to_pandas()[\"id\"])\n\nrecall_at_k = len(truth & ann) / k\n"; export const PyVectorIndexCheckStatus = "index_name = \"keywords_embeddings_idx\"\ntable.wait_for_index([index_name])\nprint(table.index_stats(index_name))\n"; +export const PyVectorIndexConfigureIvf = "table.create_index(metric=\"l2\", num_partitions=16, num_sub_vectors=4)\n"; + export const PyVectorIndexCustomName = "# Override the default `{column}_idx` convention by passing `name=...`.\ntable.create_index(\n metric=\"cosine\",\n vector_column_name=\"keywords_embeddings\",\n name=\"my_custom_index\",\n)\ntable.wait_for_index([\"my_custom_index\"])\nprint(table.index_stats(\"my_custom_index\"))\n"; export const PyVectorIndexDistanceRange = "# Only return results whose distance falls within [0.0, 0.5).\n# Useful for near-duplicate detection or thresholded similarity search.\n(\n table.search(np.random.random(128))\n .distance_range(lower_bound=0.0, upper_bound=0.5)\n .limit(10)\n .to_pandas()\n)\n"; @@ -60,3 +70,5 @@ export const PyVectorIndexQueryHnsw = "tbl = table\ntbl.search(np.random.random( export const PyVectorIndexQueryIvf = "tbl = table\ntbl.search(np.random.random((1536))).limit(2).nprobes(20).refine_factor(\n 10\n).to_pandas()\n"; +export const PyVectorIndexSetup = "table_name = \"vector-index-tbl\"\ntable = db.open_table(table_name)\n"; +