mirror of
https://github.com/lancedb/lancedb.git
synced 2026-10-16 08:42:29 +00:00
Jack Ye
70c10e4aa4
lancedb now holds a page at every path the site publishes, including the twelve Enterprise ones. Those are real open-source pages -- what the capability is, and what the embedded form does instead -- rather than blank placeholders, so the open-source site is complete on its own. sophon becomes an overlay whose page replaces the one at the same path, which means navigation lives in one place and the two repositories only have to agree on a path. That removes the whole class of problem where a page could only be reached from one of the two trees. The open-source site went from twenty-four dangling Enterprise links to one, and that one points at the dataset pages the build repository owns. Ports four changes that landed on the docs repository after this branch left it: #344 indexing and quantization rewritten #354 LeRobot native reader #361 training/index renamed to training/data-loading and rewritten #362 distance metric mismatch falls back to a brute-force scan None applied as patches, because these files were merged or split here -- quantization into the vector index page, multi-vector into vector search, performance into four pages. Each was ported by taking the upstream page and re-applying the restructure on top, which also picked up a correction that would otherwise have been lost: the stricter brute-force threshold for multi-vector columns is a Cloud limit, not an Enterprise one. The upstream LeRobot page linked to a Geneva page that no longer exists; it now points at the feature-engineering pages that replaced it.
63 lines
8.9 KiB
Plaintext
63 lines
8.9 KiB
Plaintext
{/* Auto-generated by scripts/mdx_snippets_gen.py. Do not edit manually. */}
|
|
|
|
export const PyFtsIndexAsync = "import asyncio\n\nimport lancedb\nimport polars as pl\nfrom lancedb.index import FTS\n\ndata = pl.DataFrame(\n {\n \"id\": [1, 2],\n \"text\": [\n \"His first language is spanish\",\n \"Her first language is english\",\n ],\n }\n)\n\nasync def main(data: pl.DataFrame):\n uri = \"ex_lancedb\"\n db = await lancedb.connect_async(uri)\n tbl = await db.create_table(\"my_text\", data=data, mode=\"overwrite\")\n\n await tbl.create_index(\"text\", config=FTS(language=\"English\"))\n\n response = await tbl.search(\"spanish\", query_type=\"fts\")\n result = await response.limit(1).to_polars()\n print(result)\n return result\n\nif __name__ == \"__main__\":\n asyncio.run(main(data))\n";
|
|
|
|
export const PyFtsIndexCreate = "table_name = \"fts-index-create\"\ntable = db.open_table(table_name)\ntable.create_fts_index(\"text\")\n";
|
|
|
|
export const PyFtsIndexNested = "from lancedb.query import MatchQuery, PhraseQuery\n\ntable = db.open_table(\"fts-index-nested\")\n\n# Index a text leaf inside a struct column using a dotted path.\ntable.create_fts_index(\"payload.text\", with_position=True)\n\n# The same dotted path works in MatchQuery and PhraseQuery.\nmatches = (\n table.search(MatchQuery(\"puppy\", \"payload.text\")).limit(5).to_list()\n)\nphrases = (\n table.search(PhraseQuery(\"puppy runs\", \"payload.text\"))\n .limit(5)\n .to_list()\n)\n";
|
|
|
|
export const PyFtsIndexWait = "table_name = \"fts-index-wait\"\n\ntable = db.open_table(table_name)\ntable.create_fts_index(\"text\")\n\nindex_name = \"text_idx\"\ntable.wait_for_index([index_name])\n";
|
|
|
|
export const PyGpuIndexCuda = "table.create_index(\n num_partitions=256,\n num_sub_vectors=96,\n accelerator=\"cuda\",\n)\n";
|
|
|
|
export const PyGpuIndexMps = "table.create_index(\n num_partitions=256,\n num_sub_vectors=96,\n accelerator=\"mps\",\n)\n";
|
|
|
|
export const PyQuantizationCustomParams = "table.create_index(\n index_type=\"IVF_RQ\",\n num_bits=2,\n max_iterations=100,\n sample_rate=512,\n)\n";
|
|
|
|
export const PyReindexingIncremental = "table = db.open_table(\"reindexing_incremental\")\ntable.add([{\"vector\": [3.1, 4.1], \"text\": \"Frodo was a happy puppy\"}])\ntable.optimize()\n";
|
|
|
|
export const PyScalarIndexBuild = "tbl = db.open_table(\"scalar_index_build\")\ntbl.create_scalar_index(\"book_id\")\ntbl.create_scalar_index(\"publisher\", index_type=\"BITMAP\")\n";
|
|
|
|
export const PyScalarIndexFilter = "table = db.open_table(\"books\")\nresult = table.search().where(\"book_id = 2\").limit(10).to_pandas()\n";
|
|
|
|
export const PyScalarIndexNestedFields = "import pyarrow as pa\nfrom lancedb.index import BTree\n\nmetadata_type = pa.struct(\n [\n pa.field(\"user_id\", pa.int32()),\n pa.field(\"user.id\", pa.int32()),\n ]\n)\ndata = pa.Table.from_arrays(\n [\n pa.array([1, 2, 3], type=pa.int32()),\n pa.array(\n [\n {\"user_id\": 10, \"user.id\": 100},\n {\"user_id\": 20, \"user.id\": 200},\n {\"user_id\": 30, \"user.id\": 300},\n ],\n type=metadata_type,\n ),\n ],\n names=[\"user_id\", \"metadata\"],\n)\ntable = await db.create_table(\"nested_scalar_index\", data)\n\n# Index a nested struct field.\nawait table.create_index(\n \"metadata.user_id\", config=BTree(), name=\"nested_user_id_idx\"\n)\n\n# Escape literal dots inside a segment with backticks.\nawait table.create_index(\n \"metadata.`user.id`\", config=BTree(), name=\"escaped_user_id_idx\"\n)\n\n# `columns` is returned as the canonical path you passed in.\nfor index in await table.list_indices():\n print(index.name, index.columns)\n# nested_user_id_idx ['metadata.user_id']\n# escaped_user_id_idx ['metadata.`user.id`']\n";
|
|
|
|
export const PyScalarIndexOptimize = "table.add([{\"vector\": [7, 8], \"book_id\": 4}])\ntable.optimize()\n";
|
|
|
|
export const PyScalarIndexPrefilter = "table = db.open_table(\"book_with_embeddings\")\ntable.search([1.2] * 2).where(\"book_id != 3\").limit(10).to_pandas()\n";
|
|
|
|
export const PyScalarIndexUuidData = "def generate_random_names():\n base_names = [\"Alice\", \"Bob\", \"Carla\", \"David\", \"Eve\", \"Frank\", \"Grace\"]\n letter = random.choice(string.ascii_uppercase)\n return f\"{random.choice(base_names)} {letter}.\"\n\ndef generate_uuids(num_items):\n return [uuid.uuid4().bytes for _ in range(num_items)]\n\n# Generate some UUIDs and random names\nn = 7\nuuids = generate_uuids(n)\nnames = [generate_random_names() for _ in range(n)]\n";
|
|
|
|
export const PyScalarIndexUuidTable = "table_name = \"index-on-uuid\"\n\nuuid_array = pa.array(uuids, pa.uuid())\nname_array = pa.array(names, pa.string())\nschema = pa.schema(\n [\n pa.field(\"id\", pa.uuid()),\n pa.field(\"name\", pa.string()),\n ]\n)\ndata_table = pa.Table.from_arrays([uuid_array, name_array], schema=schema)\ntable = db.create_table(table_name, data=data_table, mode=\"overwrite\")\n";
|
|
|
|
export const PyScalarIndexUuidType = "import pyarrow as pa\n";
|
|
|
|
export const PyScalarIndexUuidUpsert = "new_users = [\n {\"id\": uuid.uuid4().bytes, \"name\": \"Hannah D.\"},\n {\"id\": uuid.uuid4().bytes, \"name\": \"Ian B.\"},\n]\n# Insert or update using the UUID index\ntable.merge_insert(\n \"id\"\n).when_matched_update_all().when_not_matched_insert_all().execute(new_users)\n";
|
|
|
|
export const PyScalarIndexUuidWait = "index_name = \"id_idx\"\ntable.create_scalar_index(\"id\")\ntable.wait_for_index([index_name])\n";
|
|
|
|
export const PyScalarIndexWait = "index_name = \"label_idx\"\ntable.wait_for_index([index_name])\n";
|
|
|
|
export const PyVectorIndexAsyncConfig = "import lancedb\nimport numpy as np\nfrom lancedb.index import IvfPq\n\nasync def main():\n data = [\n {\"id\": i, \"vector\": np.random.random(8).astype(np.float32).tolist()}\n for i in range(512)\n ]\n\n db = await lancedb.connect_async(\"ex_lancedb\")\n table = await db.create_table(\n \"vector_index_async\", data=data, mode=\"overwrite\"\n )\n\n await table.create_index(\n \"vector\",\n config=IvfPq(\n distance_type=\"cosine\",\n num_partitions=16,\n num_sub_vectors=4,\n ),\n )\n return await table.list_indices()\n";
|
|
|
|
export const PyVectorIndexBuildHnsw = "table.create_index(index_type=\"IVF_HNSW_SQ\")\n";
|
|
|
|
export const PyVectorIndexBuildIvf = "table_name = \"vector-index-build-ivf\"\ntable = db.open_table(table_name)\ntable.create_index(\n metric=\"cosine\",\n vector_column_name=\"keywords_embeddings\",\n index_type=\"IVF_PQ\",\n)\n";
|
|
|
|
export const PyVectorIndexBypassRecall = "query = np.random.random(128)\nk = 10\n\n# Ground truth: flat (exhaustive) scan, ignoring the ANN index.\ntruth = set(table.search(query).bypass_vector_index().limit(k).to_pandas()[\"id\"])\n\n# ANN results with the current nprobes setting.\nann = set(table.search(query).nprobes(20).limit(k).to_pandas()[\"id\"])\n\nrecall_at_k = len(truth & ann) / k\n";
|
|
|
|
export const PyVectorIndexCheckStatus = "index_name = \"keywords_embeddings_idx\"\ntable.wait_for_index([index_name])\nprint(table.index_stats(index_name))\n";
|
|
|
|
export const PyVectorIndexCustomName = "# Override the default `{column}_idx` convention by passing `name=...`.\ntable.create_index(\n metric=\"cosine\",\n vector_column_name=\"keywords_embeddings\",\n name=\"my_custom_index\",\n)\ntable.wait_for_index([\"my_custom_index\"])\nprint(table.index_stats(\"my_custom_index\"))\n";
|
|
|
|
export const PyVectorIndexDistanceRange = "# Only return results whose distance falls within [0.0, 0.5).\n# Useful for near-duplicate detection or thresholded similarity search.\n(\n table.search(np.random.random(128))\n .distance_range(lower_bound=0.0, upper_bound=0.5)\n .limit(10)\n .to_pandas()\n)\n";
|
|
|
|
export const PyVectorIndexNestedField = "# The vector column `embedding` is nested inside the `image` struct.\n# Pass its full dotted path as `vector_column_name`; the same path is used\n# at query time and is what `list_indices()` reports under `columns`.\ntable.create_index(\n vector_column_name=\"image.embedding\",\n num_partitions=1,\n num_sub_vectors=1,\n name=\"image_embedding_idx\",\n)\n\nresults = (\n table.search([0.0, 1.0], vector_column_name=\"image.embedding\")\n .limit(1)\n .to_list()\n)\n";
|
|
|
|
export const PyVectorIndexNprobes = "# Always scan 10 partitions; scan up to 50 only if the initial pass\n# returns fewer than `limit` results (common with narrow filters).\n(\n table.search(np.random.random(128))\n .minimum_nprobes(10)\n .maximum_nprobes(50)\n .where(\"id > 100\")\n .limit(5)\n .to_pandas()\n)\n";
|
|
|
|
export const PyVectorIndexQueryHnsw = "tbl = table\ntbl.search(np.random.random((16))).limit(2).to_pandas()\n";
|
|
|
|
export const PyVectorIndexQueryIvf = "tbl = table\ntbl.search(np.random.random((1536))).limit(2).nprobes(20).refine_factor(\n 10\n).to_pandas()\n";
|
|
|