docs: bring the open-source documentation into this repository

The pages describing this code lived in lancedb/docs, a repository with ten
stars that neither contributing guide mentioned. A user who hit a wrong page
while reading the source had to find a separate, near-invisible repository to
fix it. These 102 pages now sit beside the code, and a documentation fix is a
pull request in the repository the reader already had open.

`docs/web` is a complete Mintlify site: `cd docs/web && mint dev` serves it with
no other checkout. Enterprise pages stay private and are merged in when the
published site is assembled, so links into them do not resolve here — every
open-source page does.

`docs/web-tests` holds the tests the code examples are extracted from. They ran
against a published wheel before; here they can run against this repository's
own SDK, so an example that stops compiling fails the pull request that broke
it rather than being noticed later.

Both directories sit beside `docs/src` and are invisible to it: mkdocs reads
`docs_dir: src`, so the SDK reference builds and deploys exactly as before.

docs/web-tests/camelot.json

@@ -0,0 +1,66 @@
+[
+  {
+    "id": 1,
+    "name": "King Arthur",
+    "role": "King of Camelot",
+    "description": "The legendary ruler of Camelot, wielder of Excalibur, and leader of the Knights of the Round Table.",
+    "vector": [0.72, -0.28, 0.60, 0.86],
+    "stats": { "strength": 2, "courage": 5, "magic": 1, "wisdom": 4 }
+  },
+  {
+    "id": 2,
+    "name": "Merlin",
+    "role": "Wizard and Advisor",
+    "description": "A powerful wizard and prophet who mentors Arthur and shapes the destiny of Camelot through magic and foresight.",
+    "vector": [0.05, 0.88, 0.62, 0.85],
+    "stats": { "strength": 2, "courage": 4, "magic": 5, "wisdom": 5 }
+  },
+  {
+    "id": 3,
+    "name": "Queen Guinevere",
+    "role": "Queen of Camelot",
+    "description": "Arthur's queen, admired for her grace and diplomacy, whose romances and loyalties influence Camelot's fate.",
+    "vector": [0.22, -0.22, 0.42, 0.82],
+    "stats": { "strength": 1, "courage": 3, "magic": 1, "wisdom": 4 }
+  },
+  {
+    "id": 4,
+    "name": "Sir Lancelot",
+    "role": "Knight of the Round Table",
+    "description": "Arthur's most skilled knight, famed for unmatched combat prowess and his tragic love for Queen Guinevere.",
+    "vector": [0.86, -0.35, 0.38, 0.55],
+    "stats": { "strength": 5, "courage": 5, "magic": 1, "wisdom": 3 }
+  },
+  {
+    "id": 5,
+    "name": "Sir Gawain",
+    "role": "Knight of the Round Table",
+    "description": "A noble and honorable knight known for his courtesy and his encounter with the Green Knight.",
+    "vector": [0.82, -0.32, 0.52, 0.60],
+    "stats": { "strength": 4, "courage": 5, "magic": 1, "wisdom": 4 }
+  },
+  {
+    "id": 6,
+    "name": "Sir Galahad",
+    "role": "Knight of the Round Table",
+    "description": "The purest and most virtuous knight, chosen to achieve the Holy Grail due to his unwavering spiritual purity.",
+    "vector": [0.80, -0.20, 0.70, 0.78],
+    "stats": { "strength": 4, "courage": 5, "magic": 2, "wisdom": 5 }
+  },
+  {
+    "id": 7,
+    "name": "Sir Percival",
+    "role": "Knight of the Round Table",
+    "description": "A loyal and innocent knight whose bravery and sincerity make him one of the key seekers of the Holy Grail.",
+    "vector": [0.78, -0.36, 0.48, 0.52],
+    "stats": { "strength": 4, "courage": 4, "magic": 1, "wisdom": 3 }
+  },
+  {
+    "id": 8,
+    "name": "Mordred",
+    "role": "Traitor Knight",
+    "description": "Arthur's treacherous son or nephew who ultimately rebels against him, leading to Camelot's downfall.",
+    "vector": [0.68, -0.30, -0.65, 0.20],
+    "stats": { "strength": 4, "courage": 2, "magic": 1, "wisdom": 2 }
+  }
+]

docs/web-tests/mdx_snippets_gen.py

@@ -0,0 +1,362 @@
+"""
+Generate MDX-ready snippet modules that export raw string constants.
+
+The generator groups snippets by the originating test file name rather than by
+language. For a source file like `tests/py/test_basic_usage.py`, the snippets are
+written under `docs/snippets/basic.mdx`. Each exported constant is prefixed with
+a language identifier (Py|Ts|Rs) followed by the snippet name converted to
+TitleCase so that docs authors can selectively assemble `<CodeGroup>` blocks.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import re
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Dict, Iterable, Iterator, List, Mapping, MutableMapping, Tuple
+
+# Supported languages and relevant metadata.
+LANG_ORDER = ("py", "ts", "rs")
+LANG_PREFIX = {"py": "Py", "ts": "Ts", "rs": "Rs"}
+LANG_EXTENSIONS = {
+    ".py": "py",
+    ".ts": "ts",
+    ".tsx": "ts",
+    ".rs": "rs",
+}
+LANG_MARKERS = {"py": ["#"], "ts": ["//"], "rs": ["//"]}
+# Match each export block; we rely on the blank line we emit after each export to
+# avoid accidentally truncating on semicolons inside the JSON string literal.
+EXPORT_RE = re.compile(r"export const (\w+)\s*=\s*(.+?);\s*\n(?:\s*\n|$)", re.DOTALL)
+
+DEFAULT_SOURCE_DIRS = (
+    Path("tests") / "py",
+    Path("tests") / "ts",
+    Path("tests") / "rs",
+)
+
+
+@dataclass(frozen=True)
+class SnippetRecord:
+    lang: str
+    snippet_name: str
+    export_name: str
+    text: str
+    source_rel: str
+
+
+def build_markers(
+    comment_markers: Iterable[str],
+) -> Tuple[re.Pattern[str], re.Pattern[str]]:
+    escaped = "|".join(re.escape(c) for c in comment_markers)
+    start_re = re.compile(rf"^\s*(?:{escaped})\s*--8<--\s*\[start:([^\]]+)\]")
+    end_re = re.compile(rf"^\s*(?:{escaped})\s*--8<--\s*\[end:([^\]]+)\]")
+    return start_re, end_re
+
+
+def parse_snippets(
+    lines: List[str], start_re: re.Pattern[str], end_re: re.Pattern[str]
+) -> Dict[str, List[str]]:
+    snippets: Dict[str, List[str]] = {}
+    stack: List[Tuple[str, int]] = []
+
+    for idx, line in enumerate(lines):
+        m_start = start_re.match(line)
+        if m_start:
+            name = m_start.group(1).strip()
+            stack.append((name, idx))
+            continue
+
+        m_end = end_re.match(line)
+        if m_end:
+            name = m_end.group(1).strip()
+            if not stack:
+                raise ValueError(
+                    f"End marker for '{name}' found at line {idx + 1} without matching start"
+                )
+            start_name, start_idx = stack.pop()
+            if start_name != name:
+                raise ValueError(
+                    f"Mismatched markers: start '{start_name}' at line {start_idx + 1}, "
+                    f"end '{name}' at line {idx + 1}"
+                )
+            snippets[name] = lines[start_idx + 1 : idx]
+            continue
+
+    if stack:
+        open_names = ", ".join(f"{n}@{i + 1}" for n, i in stack)
+        raise ValueError(f"Unclosed snippet markers for: {open_names}")
+
+    return snippets
+
+
+def dedent(lines: List[str]) -> List[str]:
+    min_indent = None
+    for ln in lines:
+        if ln.strip() == "":
+            continue
+        expanded = ln.expandtabs(4)
+        indent = len(expanded) - len(expanded.lstrip(" "))
+        if min_indent is None or indent < min_indent:
+            min_indent = indent
+
+    if not lines:
+        return []
+    if min_indent in (None, 0):
+        return [ln.rstrip("\n") for ln in lines]
+
+    dedented: List[str] = []
+    for ln in lines:
+        expanded = ln.expandtabs(4)
+        if expanded.strip() == "":
+            dedented.append(expanded.rstrip("\n"))
+        else:
+            dedented.append(expanded[min_indent:].rstrip("\n"))
+    return dedented
+
+
+def ensure_dir(path: Path) -> None:
+    path.mkdir(parents=True, exist_ok=True)
+
+
+def detect_language(file_path: Path) -> str:
+    lang = LANG_EXTENSIONS.get(file_path.suffix.lower())
+    if not lang:
+        raise ValueError(f"Unsupported file extension for {file_path}")
+    return lang
+
+
+def markers_for_lang(lang: str) -> Tuple[re.Pattern[str], re.Pattern[str]]:
+    markers = LANG_MARKERS.get(lang)
+    if not markers:
+        raise ValueError(f"Unsupported language for markers: {lang}")
+    return build_markers(markers)
+
+
+def normalize_target_name(file_path: Path) -> str:
+    stem = file_path.stem
+    if stem.startswith("test_"):
+        stem = stem[len("test_") :]
+    if stem.endswith("_test"):
+        stem = stem[: -len("_test")]
+    if stem.endswith(".test"):
+        stem = stem[: -len(".test")]
+    if stem.endswith(".spec"):
+        stem = stem[: -len(".spec")]
+    if stem == "":
+        stem = file_path.stem
+    return stem.replace(" ", "_")
+
+
+def to_title_case(name: str) -> str:
+    parts = re.split(r"[^0-9a-zA-Z]+", name)
+    filtered = [p for p in parts if p]
+    return "".join(p[:1].upper() + p[1:].lower() for p in filtered) or "Snippet"
+
+
+def format_export_name(lang: str, snippet_name: str) -> str:
+    prefix = LANG_PREFIX[lang]
+    return f"{prefix}{to_title_case(snippet_name)}"
+
+
+def snippet_text(lines: List[str]) -> str:
+    if not lines:
+        return ""
+    joined = "\n".join(lines).rstrip("\n")
+    return (joined + "\n") if joined else ""
+
+
+def to_js_literal(text: str) -> str:
+    return json.dumps(text, ensure_ascii=False)
+
+
+def split_export_prefix(export_name: str) -> Tuple[str | None, str]:
+    for lang, prefix in LANG_PREFIX.items():
+        if export_name.startswith(prefix):
+            suffix = export_name[len(prefix) :]
+            return lang, suffix or export_name
+    return None, export_name
+
+
+def parse_existing_module(path: Path) -> Mapping[str, Mapping[str, SnippetRecord]]:
+    try:
+        text = path.read_text(encoding="utf-8")
+    except FileNotFoundError:
+        return {}
+
+    existing: Dict[str, Dict[str, SnippetRecord]] = {}
+    for match in EXPORT_RE.finditer(text):
+        export_name, literal = match.group(1), match.group(2).strip()
+        lang, snippet_suffix = split_export_prefix(export_name)
+        if not lang:
+            continue
+        try:
+            decoded = json.loads(literal)
+        except Exception:
+            decoded = literal.strip().strip(";")
+
+        record = SnippetRecord(
+            lang=lang,
+            snippet_name=snippet_suffix or export_name,
+            export_name=export_name,
+            text=decoded,
+            source_rel="(existing module)",
+        )
+        existing.setdefault(lang, {})[record.snippet_name] = record
+
+    return existing
+
+
+def iter_source_files(source_dirs: Iterable[Path]) -> Iterator[Path]:
+    for root in source_dirs:
+        for path in root.rglob("*"):
+            if path.is_file() and path.suffix.lower() in LANG_EXTENSIONS:
+                yield path
+
+
+def write_if_changed(path: Path, content: str) -> bool:
+    if path.exists():
+        try:
+            current = path.read_text(encoding="utf-8")
+            if current == content:
+                return False
+        except Exception:
+            pass
+    ensure_dir(path.parent)
+    path.write_text(content, encoding="utf-8")
+    return True
+
+
+def collect_snippets(
+    source_dirs: Iterable[Path],
+) -> Mapping[str, Mapping[str, Mapping[str, SnippetRecord]]]:
+    result: Dict[str, Dict[str, Dict[str, SnippetRecord]]] = {}
+    for file_path in iter_source_files(source_dirs):
+        lang = detect_language(file_path)
+        start_re, end_re = markers_for_lang(lang)
+        try:
+            text = file_path.read_text(encoding="utf-8")
+        except Exception as exc:  # pragma: no cover
+            raise RuntimeError(f"Failed to read {file_path}: {exc}") from exc
+        lines = text.splitlines(keepends=True)
+        snippets = parse_snippets(lines, start_re, end_re)
+        if not snippets:
+            continue
+
+        target = normalize_target_name(file_path)
+        lang_map = result.setdefault(target, {}).setdefault(lang, {})
+        for snippet_name, content_lines in snippets.items():
+            if snippet_name in lang_map:
+                raise ValueError(
+                    f"Duplicate snippet '{snippet_name}' in {file_path} and {lang_map[snippet_name].source_rel}"
+                )
+            dedented = dedent(content_lines)
+            export_name = format_export_name(lang, snippet_name)
+            source_rel = os.path.relpath(file_path, Path.cwd())
+            lang_map[snippet_name] = SnippetRecord(
+                lang=lang,
+                snippet_name=snippet_name,
+                export_name=export_name,
+                text=snippet_text(dedented),
+                source_rel=source_rel,
+            )
+    return result
+
+
+def render_module(
+    target: str, lang_map: Mapping[str, Mapping[str, SnippetRecord]]
+) -> str:
+    parts: List[str] = []
+    parts.append(
+        "{/* Auto-generated by scripts/mdx_snippets_gen.py. Do not edit manually. */}\n\n"
+    )
+    for lang in LANG_ORDER:
+        snippets = lang_map.get(lang)
+        if not snippets:
+            continue
+        for snippet_name in sorted(snippets.keys()):
+            record = snippets[snippet_name]
+            literal = to_js_literal(record.text)
+            parts.append(f"export const {record.export_name} = {literal};\n\n")
+    return "".join(parts)
+
+
+def generate_modules(
+    snippets_by_target: Mapping[str, Mapping[str, Mapping[str, SnippetRecord]]],
+    output_root: Path,
+) -> None:
+    modules_written = 0
+    total_exports = 0
+
+    for target in sorted(snippets_by_target.keys()):
+        module_path = output_root / f"{target}.mdx"
+        lang_map = snippets_by_target[target]
+        existing_langs = parse_existing_module(module_path)
+        for lang, snippets in existing_langs.items():
+            if lang in lang_map:
+                continue
+            print(module_path, " -- ", lang)
+            merged_lang = lang_map.setdefault(lang, {})
+            existing_export_names = {rec.export_name for rec in merged_lang.values()}
+            for snippet_name, record in snippets.items():
+                # If an export with the same name already exists (e.g., regenerated),
+                # prefer the newly generated snippet and skip the old one to avoid duplicates.
+                if record.export_name in existing_export_names:
+                    continue
+                merged_lang.setdefault(snippet_name, record)
+
+        total_exports += sum(len(snippets) for snippets in lang_map.values())
+        module_content = render_module(target, lang_map)
+        if write_if_changed(module_path, module_content):
+            modules_written += 1
+    print(
+        f"\nGenerated {total_exports} snippets\n---"
+    )
+
+
+def resolve_source_dirs(args_dirs: List[str] | None) -> List[Path]:
+    if args_dirs:
+        dirs = [Path(p) for p in args_dirs]
+    else:
+        dirs = list(DEFAULT_SOURCE_DIRS)
+
+    missing = [str(p) for p in dirs if not p.exists()]
+    if missing:
+        raise FileNotFoundError(f"Source directories not found: {', '.join(missing)}")
+    return dirs
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(
+        description="Generate MDX snippet modules grouped by test filename."
+    )
+    parser.add_argument(
+        "-s",
+        "--source-dir",
+        action="append",
+        help="Path(s) to language test directories (default: tests/py, ts, rs)",
+    )
+    parser.add_argument(
+        "-o",
+        "--output-dir",
+        default=str(Path("docs") / "snippets"),
+        help="Directory where snippet modules are written (default: docs/snippets)",
+    )
+    args = parser.parse_args()
+
+    source_dirs = resolve_source_dirs(args.source_dir)
+    snippets = collect_snippets(source_dirs)
+    if not snippets:
+        print("No snippets found.")
+        return 0
+
+    output_root = Path(args.output_dir)
+    generate_modules(snippets, output_root)
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

docs/web-tests/py/conftest.py

@@ -0,0 +1,56 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+import shutil
+from pathlib import Path
+
+import lancedb
+import pytest
+import pytest_asyncio
+from lancedb.db import AsyncConnection, DBConnection
+
+
+class DatabasePathFactory:
+    """Create per-test database directories and ensure they get removed."""
+
+    def __init__(self, base_dir: Path) -> None:
+        self._base_dir = base_dir
+        self._created: list[Path] = []
+
+    def __call__(self, name: str) -> Path:
+        safe_name = name.replace("/", "_")
+        path = self._base_dir / safe_name
+        if path.exists():
+            shutil.rmtree(path, ignore_errors=True)
+        path.mkdir(parents=True, exist_ok=True)
+        self._created.append(path)
+        return path
+
+    def cleanup(self) -> None:
+        for path in reversed(self._created):
+            shutil.rmtree(path, ignore_errors=True)
+        self._created.clear()
+
+
+@pytest.fixture
+def db_path_factory(tmp_path: Path):
+    factory = DatabasePathFactory(tmp_path)
+    yield factory
+    factory.cleanup()
+
+
+@pytest.fixture
+def mem_db() -> DBConnection:
+    return lancedb.connect("memory://")
+
+
+@pytest.fixture
+def tmp_db(db_path_factory) -> DBConnection:
+    """Create a temporary database connection for testing."""
+    db_path = db_path_factory("tmp_db")
+    return lancedb.connect(str(db_path))
+
+
+@pytest_asyncio.fixture
+async def mem_db_async() -> AsyncConnection:
+    return await lancedb.connect_async("memory://")

docs/web-tests/py/test_basic_usage.py

@@ -0,0 +1,205 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+# --8<-- [start:basic_imports]
+import json
+
+import lancedb
+import pandas as pd
+import polars as pl
+import pyarrow as pa
+# --8<-- [end:basic_imports]
+import pytest
+
+data_path = "tests/camelot.json"
+
+def test_basic_usage(db_path_factory):
+    uri = "./basic_usage_db"
+    uri = db_path_factory("basic_usage_db")
+    db = lancedb.connect(uri)
+
+    # --8<-- [start:data_load]
+    with open(data_path, "r") as f:
+        data = json.load(f)
+    # --8<-- [end:data_load]
+
+    # --8<-- [start:basic_create_table]
+    table = db.create_table("camelot", data=data, mode="overwrite")
+    # --8<-- [end:basic_create_table]
+    assert len(table) == 8
+
+    # --8<-- [start:basic_open_table]
+    table = db.open_table("camelot")
+    # --8<-- [end:basic_open_table]
+
+    # --8<-- [start:basic_create_table_pandas]
+    pandas_df = pd.DataFrame(data)
+    table_pd = db.create_table("camelot_pd", data=pandas_df, mode="overwrite")
+    # --8<-- [end:basic_create_table_pandas]
+    assert len(table_pd) == 8
+    db.drop_table("camelot_pd")
+
+    # --8<-- [start:basic_create_table_polars]
+    polars_df = pl.DataFrame(data)
+    table_pl = db.create_table("camelot_pl", data=polars_df, mode="overwrite")
+    # --8<-- [end:basic_create_table_polars]
+    assert len(table_pl) == 8
+    db.drop_table("camelot_pl")
+
+    # --8<-- [start:basic_create_empty_table]
+    schema = pa.schema(
+        [
+            pa.field("id", pa.uint16()),
+            pa.field("name", pa.string()),
+            pa.field("role", pa.string()),
+            pa.field("description", pa.string()),
+            pa.field("vector", pa.list_(pa.float32(), 4)),
+            pa.field(
+                "stats",
+                pa.struct(
+                    [
+                        pa.field("strength", pa.int8()),
+                        pa.field("courage", pa.int8()),
+                        pa.field("magic", pa.int8()),
+                        pa.field("wisdom", pa.int8()),
+                    ]
+                ),
+            ),
+        ]
+    )
+    db.create_table("camelot_pa", schema=schema, mode="overwrite")
+    # --8<-- [end:basic_create_empty_table]
+    assert "camelot_pa" in db.list_tables().tables
+    db.drop_table("camelot_pa")
+
+    # --8<-- [start:basic_add_data]
+    magical_characters = [
+        {
+            "id": 9,
+            "name": "Morgan le Fay",
+            "role": "Sorceress",
+            "description": "A powerful enchantress, Arthur's half-sister, and a complex figure who oscillates between aiding and opposing Camelot.",
+            "vector": [0.10, 0.84, 0.25, 0.70],
+            "stats": { "strength": 2, "courage": 3, "magic": 5, "wisdom": 4 }
+        },
+        {
+            "id": 10,
+            "name": "The Lady of the Lake",
+            "role": "Mystical Guardian",
+            "description": "A mysterious supernatural figure associated with Avalon, known for giving Arthur the sword Excalibur.",
+            "vector": [0.00, 0.90, 0.58, 0.88],
+            "stats": { "strength": 2, "courage": 3, "magic": 5, "wisdom": 5 }
+        }
+    ]
+    table.add(magical_characters)
+    # --8<-- [end:basic_add_data]
+    assert len(table) == 10
+
+    # --8<-- [start:basic_vector_search]
+    query_vector = [0.03, 0.85, 0.61, 0.90]
+    result = table.search(query_vector).limit(5).to_polars()
+    print(result)
+    # --8<-- [end:basic_vector_search]
+
+    # --8<-- [start:basic_add_columns]
+    table.add_columns(
+        {
+            "power": "cast(((stats.strength + stats.courage + stats.magic + stats.wisdom) / 4.0) as float)"
+        }
+    )
+    # --8<-- [end:basic_add_columns]
+    assert "power" in table.schema.names
+
+    # Run examples to illustrate search
+    # --8<-- [start:basic_vector_search_q1]
+    # Who are the characters similar to  "wizard"?
+    query_vector_1 = [0.03, 0.85, 0.61, 0.90]
+    r1 = (
+        table.search(query_vector_1)
+        .limit(5)
+        .select(["name", "role", "description"])
+        .to_polars()
+    )
+    print(r1)
+    # --8<-- [end:basic_vector_search_q1]
+
+    # --8<-- [start:basic_vector_search_q2]
+    # Who are the characters with high magic stats?
+    query_vector_2 = [0.03, 0.85, 0.61, 0.90]
+    r2 = (
+        table.search(query_vector_2)
+        .where("stats.magic > 3")
+        .select(["name", "role", "description"])
+        .limit(5)
+        .to_polars()
+    )
+    print(r2)
+    # --8<-- [end:basic_vector_search_q2]
+
+    # --8<-- [start:basic_vector_search_q3]
+    # Who are the strongest characters?
+    r3 = (
+        table.search()
+        .where("stats.strength > 3")
+        .select(["name", "role", "description"])
+        .limit(5)
+        .to_polars()
+    )
+    print(r3)
+    # --8<-- [end:basic_vector_search_q3]
+
+    # --8<-- [start:basic_vector_search_q4]
+    # Who are the strongest characters?
+    r4 = (
+        table.search()
+        .select(["name", "role", "description", "power"])
+        .to_polars()
+    )
+    print(r4)
+    # --8<-- [end:basic_vector_search_q4]
+
+    # --8<-- [start:basic_sort_polars]
+    # Sort Polars DataFrame by power in descending order
+    print(r4.sort("power", descending=True).limit(5))
+    # --8<-- [end:basic_sort_polars]
+    sorted_power = r4.sort("power", descending=True)["power"].to_list()
+    assert sorted_power == sorted(sorted_power, reverse=True)
+    assert sorted_power[0] == 4.0
+
+    # --8<-- [start:basic_drop_columns]
+    table.drop_columns(["power"])
+    # --8<-- [end:basic_drop_columns]
+    # --8<-- [start:basic_delete_rows]
+    table.delete('role = "Traitor Knight"')
+    # --8<-- [end:basic_delete_rows]
+    assert len(table) == 9
+    # --8<-- [start:basic_drop_table]
+    db.drop_table("camelot")
+    # --8<-- [end:basic_drop_table]
+    assert "camelot" not in db.list_tables().tables
+
+
+@pytest.mark.asyncio
+async def test_basic_usage_async_api(db_path_factory):
+    uri = db_path_factory("basic_usage_async_db")
+    with open(data_path, "r") as f:
+        data = json.load(f)
+
+    # --8<-- [start:basic_async_api]
+    import lancedb
+
+    async_db = await lancedb.connect_async(uri)
+    async_table = await async_db.create_table(
+        "camelot_async",
+        data=data,
+        mode="overwrite",
+    )
+
+    query_vector = [0.03, 0.85, 0.61, 0.90]
+    async_results = await (
+        await async_table.search(query_vector)
+    ).limit(5).select(["name", "role", "description"]).to_polars()
+    print(async_results)
+    # --8<-- [end:basic_async_api]
+
+    assert async_results.height >= 1

docs/web-tests/py/test_build_with_ai_agents.py

@@ -0,0 +1,109 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+# --8<-- [start:camelot_schema]
+from lancedb.pydantic import LanceModel
+from pydantic import ConfigDict
+
+
+class Stats(LanceModel):
+    model_config = ConfigDict(strict=True, extra="forbid")
+
+    strength: int
+    courage: int
+    magic: int
+    wisdom: int
+
+
+class Character(LanceModel):
+    model_config = ConfigDict(strict=True, extra="forbid")
+
+    id: int
+    name: str
+    role: str
+    description: str
+    stats: Stats
+    image_filename: str
+    image: bytes
+# --8<-- [end:camelot_schema]
+
+
+# --8<-- [start:camelot_batches]
+import json
+from collections.abc import Iterator
+from pathlib import Path
+
+
+def validated_batches(
+    input_path: Path, batch_size: int
+) -> Iterator[list[dict]]:
+    raw_records = json.loads(input_path.read_text())
+    asset_root = input_path.parent.parent
+    batch: list[dict] = []
+
+    for raw in raw_records:
+        payload = dict(raw)
+        image_path = asset_root / payload.pop("img")
+        payload["image_filename"] = image_path.name
+        payload["image"] = image_path.read_bytes()
+
+        character = Character.model_validate(payload)
+        batch.append(character.model_dump(mode="python"))
+
+        if len(batch) == batch_size:
+            yield batch
+            batch = []
+
+    if batch:
+        yield batch
+# --8<-- [end:camelot_batches]
+
+
+# --8<-- [start:camelot_oss_ingestion]
+import lancedb
+
+
+def ingest_oss(
+    input_path: Path,
+    uri: str = "data/camelot.lancedb",
+    table_name: str = "camelot_multimodal",
+    batch_size: int = 64,
+):
+    db = lancedb.connect(uri)
+    if table_name in db.list_tables():
+        raise ValueError(
+            f"Table {table_name!r} already exists. Choose a fresh table name."
+        )
+
+    table = db.create_table(table_name, schema=Character)
+    for batch in validated_batches(input_path, batch_size):
+        table.add(batch)
+
+    table.optimize()
+    return table
+
+
+if __name__ == "__main__":
+    ingest_oss(Path("data/camelot.json"))
+# --8<-- [end:camelot_oss_ingestion]
+
+
+def test_camelot_oss_ingestion(tmp_path):
+    source = Path(
+        "docs/static/assets/tutorials/build-with-ai-agents/camelot/data/camelot.json"
+    )
+    table = ingest_oss(
+        source,
+        uri=str(tmp_path / "camelot.lancedb"),
+        table_name="camelot",
+    )
+
+    rows = (
+        table.search()
+        .select(["id", "name", "image", "image_filename"])
+        .limit(8)
+        .to_list()
+    )
+    assert len(rows) == 8
+    assert all(row["image"] for row in rows)
+    assert all(row["image_filename"].endswith(".jpg") for row in rows)

docs/web-tests/py/test_connection.py

@@ -0,0 +1,148 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+from pathlib import Path
+import shutil
+
+
+def test_connection():
+    # --8<-- [start:connect]
+    import lancedb
+
+    uri = "ex_lancedb"
+    db = lancedb.connect(uri)
+    # --8<-- [end:connect]
+    assert db is not None
+    shutil.rmtree(uri, ignore_errors=True)
+    assert not Path(uri).exists()
+
+
+async def connect_async_example():
+    # --8<-- [start:connect_async]
+    import lancedb
+
+    uri = "ex_lancedb"
+    async_db = await lancedb.connect_async(uri)
+    # --8<-- [end:connect_async]
+
+    return async_db
+
+
+def connect_enterprise_quickstart_config():
+    import lancedb
+
+    # --8<-- [start:connect_enterprise_quickstart]
+    uri = "db://your-database-uri"
+    api_key = "your-api-key"
+    region = "us-east-1"
+    host_override = "https://your-enterprise-endpoint.com"
+
+    db = lancedb.connect(
+        uri=uri,
+        api_key=api_key,
+        region=region,
+        host_override=host_override,
+    )
+    # --8<-- [end:connect_enterprise_quickstart]
+    return db
+
+
+def test_connect_enterprise_quickstart(monkeypatch):
+    import lancedb
+
+    captured = {}
+
+    def fake_connect(**kwargs):
+        captured.update(kwargs)
+        return object()
+
+    monkeypatch.setattr(lancedb, "connect", fake_connect)
+
+    db = connect_enterprise_quickstart_config()
+    assert db is not None
+    assert captured == {
+        "uri": "db://your-database-uri",
+        "api_key": "your-api-key",
+        "region": "us-east-1",
+        "host_override": "https://your-enterprise-endpoint.com",
+    }
+
+
+def connect_object_storage_config():
+    # --8<-- [start:connect_object_storage]
+    import lancedb
+
+    uri = "s3://your-bucket/path"
+    # You can also use "gs://your-bucket/path" or "az://your-container/path".
+    db = lancedb.connect(uri)
+    # --8<-- [end:connect_object_storage]
+
+    return db
+
+
+def namespace_table_ops_example():
+    # --8<-- [start:namespace_table_ops]
+    import lancedb
+
+    db = lancedb.connect_namespace("dir", {"root": "./local_lancedb"})
+
+    # Create namespace tree: prod/search
+    db.create_namespace(["prod"], mode="exist_ok")
+    db.create_namespace(["prod", "search"], mode="exist_ok")
+    db.create_namespace(["prod", "recommendations"], mode="exist_ok")
+
+    db.create_table(
+        "user",
+        data=[{"id": 1, "vector": [0.1, 0.2], "name": "alice"}],
+        namespace_path=["prod", "search"],
+        mode="create",  # use "overwrite" only if you want to replace existing table
+    )
+
+    db.create_table(
+        "user",
+        data=[{"id": 2, "vector": [0.3, 0.4], "name": "bob"}],
+        namespace_path=["prod", "recommendations"],
+        mode="create",  # use "overwrite" only if you want to replace existing table
+    )
+
+    # Verify
+    print(db.list_namespaces())  # ['prod']
+    print(db.list_namespaces(namespace_path=["prod"]))  # ['recommendations', 'search']
+    print(db.list_tables(namespace_path=["prod", "search"]))  # ['user']
+    print(db.list_tables(namespace_path=["prod", "recommendations"]))  # ['user']
+    # --8<-- [end:namespace_table_ops]
+
+
+def namespace_admin_ops_example():
+    # --8<-- [start:namespace_admin_ops]
+    import lancedb
+
+    db = lancedb.connect_namespace("dir", {"root": "./local_lancedb"})
+    namespace = ["prod", "search"]
+
+    db.create_namespace(["prod"])
+    db.create_namespace(["prod", "search"])
+
+    child_namespaces = db.list_namespaces(namespace_path=["prod"]).namespaces
+    print(f"Child namespaces under {namespace}: {child_namespaces}")
+    # Child namespaces under ['prod', 'search']: ['search']
+
+    metadata = db.describe_namespace(["prod", "search"])
+    print(f"Metadata for namespace {namespace}: {metadata}")
+    # Metadata for namespace ['prod', 'search']: properties=None
+
+    db.drop_namespace(["prod", "search"], mode="skip")
+    db.drop_namespace(["prod"], mode="skip")
+    # --8<-- [end:namespace_admin_ops]
+    return child_namespaces, metadata
+
+async def connect_object_storage_config_async():
+    # --8<-- [start:connect_object_storage_async]
+    import lancedb
+
+    uri = "s3://your-bucket/path"
+    # You can also use "gs://your-bucket/path" or "az://your-container/path".
+    async_db = await lancedb.connect_async(uri)
+    # --8<-- [end:connect_object_storage_async]
+
+    return async_db

docs/web-tests/py/test_embedding.py

@@ -0,0 +1,141 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+import lancedb
+
+# --8<-- [start:imports]
+from lancedb.pydantic import LanceModel, Vector
+from lancedb.embeddings import get_registry
+
+# --8<-- [end:imports]
+import pytest
+
+
+@pytest.mark.skip(reason="OpenAI is not available in the test environment")
+def test_create_embedding_function():
+    # --8<-- [start:create_embedding_function]
+    func = get_registry().get("openai").create(
+        name="text-embedding-3-small",
+        max_retries=7,
+    )
+    # --8<-- [end:create_embedding_function]
+    assert func is not None
+
+
+@pytest.mark.skip(reason="OpenAI is not available in the test environment")
+def test_embeddings_openai():
+    # --8<-- [start:openai_embeddings]
+    db = lancedb.connect("/tmp/db")
+    func = get_registry().get("openai").create(name="text-embedding-ada-002")
+
+    class Words(LanceModel):
+        text: str = func.SourceField()
+        vector: Vector(func.ndims()) = func.VectorField()
+
+    table = db.create_table("words", schema=Words, mode="overwrite")
+    table.add([{"text": "hello world"}, {"text": "goodbye world"}])
+
+    query = "greetings"
+    actual = table.search(query).limit(1).to_pydantic(Words)[0]
+    print(actual.text)
+    # --8<-- [end:openai_embeddings]
+
+
+@pytest.mark.skip(reason="OpenAI is not available in the test environment")
+@pytest.mark.asyncio
+async def test_embeddings_openai_async():
+    uri = "memory://"
+    # --8<-- [start:async_openai_embeddings]
+    db = await lancedb.connect_async(uri)
+    func = get_registry().get("openai").create(name="text-embedding-ada-002")
+
+    class Words(LanceModel):
+        text: str = func.SourceField()
+        vector: Vector(func.ndims()) = func.VectorField()
+
+    table = await db.create_table("words", schema=Words, mode="overwrite")
+    await table.add([{"text": "hello world"}, {"text": "goodbye world"}])
+
+    query = "greetings"
+    actual = await (await table.search(query)).limit(1).to_pydantic(Words)[0]
+    print(actual.text)
+    # --8<-- [end:async_openai_embeddings]
+
+
+@pytest.mark.skip(reason="OpenAI is not available in the test environment")
+def test_embeddings_manual_query():
+    # --8<-- [start:manual_query_embeddings]
+    db = lancedb.connect("/tmp/db")
+    func = get_registry().get("openai").create(name="text-embedding-ada-002")
+
+    class Words(LanceModel):
+        text: str = func.SourceField()
+        vector: Vector(func.ndims()) = func.VectorField()
+
+    table = db.create_table("words", schema=Words, mode="overwrite")
+    table.add([{"text": "hello world"}, {"text": "goodbye world"}])
+
+    query_vector = func.generate_embeddings(["greetings"])[0]
+    # --8<-- [start:manual_query_search]
+    # query_vector is assumed to already be generated by your embedding function
+    actual = table.search(query_vector).limit(1).to_pydantic(Words)[0]
+    print(actual.text)
+    # --8<-- [end:manual_query_search]
+    # --8<-- [end:manual_query_embeddings]
+
+
+def test_custom_embedding_function():
+    # --8<-- [start:embedding_function]
+    from functools import cached_property
+
+    from lancedb.embeddings import TextEmbeddingFunction, register
+
+    class MyEmbeddingModel:
+        def __init__(self, model_name: str):
+            self.model_name = model_name
+
+        def encode(self, texts: list[str]) -> list[list[float]]:
+            return [[1.0, 2.0, 3.0] for _ in texts]
+
+    @register("my-embedder")
+    class MyTextEmbedder(TextEmbeddingFunction):
+        model_name: str = "my-model"
+
+        def generate_embeddings(self, texts: list[str]) -> list[list[float]]:
+            # Your embedding logic here
+            return self._model.encode(texts)
+
+        def ndims(self) -> int:
+            # Return the dimensionality of the embeddings
+            return len(self.generate_embeddings(["test"])[0])
+
+        @cached_property
+        def _model(self) -> MyEmbeddingModel:
+            # Initialize your model once
+            return MyEmbeddingModel(self.model_name)
+    # --8<-- [end:embedding_function]
+
+
+def test_embeddings_secret():
+    # --8<-- [start:register_secret]
+    registry = get_registry()
+    registry.set_var("api_key", "sk-...")
+
+    func = registry.get("openai").create(api_key="$var:api_key")
+    # --8<-- [end:register_secret]
+
+    try:
+        import torch
+    except ImportError:
+        pytest.skip("torch not installed")
+
+    # --8<-- [start:register_device]
+    import torch
+
+    registry = get_registry()
+    if torch.cuda.is_available():
+        registry.set_var("device", "cuda")
+
+    func = registry.get("huggingface").create(device="$var:device:cpu")
+    # --8<-- [end:register_device]
+    assert func.device == "cuda" if torch.cuda.is_available() else "cpu"

docs/web-tests/py/test_geneva_defaults.py

@@ -0,0 +1,53 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+# These tests assert the default resource values for Geneva KubeRay cluster nodes.
+# If any of these values change, update the tables in:
+#   docs/geneva/jobs/performance.mdx (the "Geneva defaults" section)
+
+import pytest
+
+
+def test_head_node_defaults():
+    # If these change, update the Head Node table in performance.mdx
+    from geneva.cluster.builder import KubeRayClusterBuilder
+
+    builder = KubeRayClusterBuilder()
+    assert builder._head_cpus == 4
+    assert builder._head_memory == "8Gi"
+    assert builder._head_gpus == 0
+    assert builder._head_node_selector == {"geneva.lancedb.com/ray-head": "true"}
+    assert builder._head_service_account == "geneva-service-account"
+
+
+def test_cpu_worker_defaults():
+    # If these change, update the CPU Workers table in performance.mdx
+    from geneva.cluster.builder import CpuWorkerBuilder
+
+    worker = CpuWorkerBuilder()
+    assert worker._num_cpus == 4
+    assert worker._memory == "8Gi"
+    assert worker._node_selector == {"geneva.lancedb.com/ray-worker-cpu": "true"}
+    assert worker._replicas == 1
+    assert worker._min_replicas == 0
+    assert worker._max_replicas == 100
+    assert worker._idle_timeout_seconds == 60
+
+    # Confirm build produces 0 GPUs
+    config = worker.build()
+    assert config.num_gpus == 0
+
+
+def test_gpu_worker_defaults():
+    # If these change, update the GPU Workers table in performance.mdx
+    from geneva.cluster.builder import GpuWorkerBuilder
+
+    worker = GpuWorkerBuilder()
+    assert worker._num_cpus == 8
+    assert worker._memory == "16Gi"
+    assert worker._num_gpus == 1
+    assert worker._node_selector == {"geneva.lancedb.com/ray-worker-gpu": "true"}
+    assert worker._replicas == 1
+    assert worker._min_replicas == 0
+    assert worker._max_replicas == 100
+    assert worker._idle_timeout_seconds == 60

docs/web-tests/py/test_geneva_dependency_verification.py

@@ -0,0 +1,107 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+import os
+
+import pytest
+
+
+def test_quick_fix_manifest():
+    # --8<-- [start:quick_fix_manifest]
+    from geneva.manifest.builder import PipManifestBuilder
+
+    manifest = PipManifestBuilder.create("fix").pip(["numpy==1.26.4"]).build()
+    # --8<-- [end:quick_fix_manifest]
+    assert manifest.pip == ["numpy==1.26.4"]
+
+
+def test_env_vars_via_cluster(monkeypatch):
+    monkeypatch.setenv("AWS_ACCESS_KEY_ID", "test-key-id")
+    monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "test-secret")
+
+    # --8<-- [start:env_vars_via_cluster]
+    from geneva.cluster.builder import KubeRayClusterBuilder
+    import os
+
+    cluster = (
+        KubeRayClusterBuilder.create("my-cluster")
+        .ray_init_kwargs({
+            "runtime_env": {
+                "env_vars": {
+                    "AWS_ACCESS_KEY_ID": os.environ["AWS_ACCESS_KEY_ID"],
+                    "AWS_SECRET_ACCESS_KEY": os.environ["AWS_SECRET_ACCESS_KEY"],
+                }
+            }
+        })
+        .build()
+    )
+    # --8<-- [end:env_vars_via_cluster]
+    assert cluster.kuberay.ray_init_kwargs["runtime_env"]["env_vars"]["AWS_ACCESS_KEY_ID"] == "test-key-id"
+
+
+def test_pip_manifest(monkeypatch):
+    import geneva
+    from unittest.mock import MagicMock, create_autospec
+    mock_conn = create_autospec(geneva.db.Connection, instance=True)
+    monkeypatch.setattr("geneva.connect", MagicMock(return_value=mock_conn))
+
+    # --8<-- [start:pip_manifest]
+    import geneva
+    from geneva.manifest.builder import PipManifestBuilder
+
+    manifest = (
+        PipManifestBuilder.create("my-manifest")
+        .pip([
+            "numpy==1.26.4",
+            "torch==2.0.1",
+            "attrs==23.2.0",
+        ])
+        .build()
+    )
+
+    conn = geneva.connect("s3://my-bucket/my-db")
+    conn.define_manifest("my-manifest", manifest)
+    with conn.context(cluster="my-cluster", manifest="my-manifest"):
+        conn.open_table("my-table").backfill("my-column")
+    # --8<-- [end:pip_manifest]
+    assert "numpy==1.26.4" in manifest.pip
+
+
+def test_conda_cluster_path():
+    # --8<-- [start:conda_cluster_path]
+    from geneva.cluster.builder import KubeRayClusterBuilder
+
+    cluster = (
+        KubeRayClusterBuilder.create("my-cluster")
+        .ray_init_kwargs({
+            "runtime_env": {"conda": "environment.yml"}
+        })
+        .build()
+    )
+    # --8<-- [end:conda_cluster_path]
+    assert cluster.kuberay.ray_init_kwargs["runtime_env"]["conda"] == "environment.yml"
+
+
+def test_conda_cluster_inline():
+    # --8<-- [start:conda_cluster_inline]
+    from geneva.cluster.builder import KubeRayClusterBuilder
+
+    cluster = (
+        KubeRayClusterBuilder.create("my-cluster")
+        .ray_init_kwargs({
+            "runtime_env": {
+                "conda": {
+                    "channels": ["conda-forge"],
+                    "dependencies": [
+                        "python=3.10",
+                        "ffmpeg<8",
+                        "torchvision=0.22.1",
+                    ],
+                },
+                "config": {"eager_install": True},
+            }
+        })
+        .build()
+    )
+    # --8<-- [end:conda_cluster_inline]
+    assert cluster.kuberay.ray_init_kwargs["runtime_env"]["conda"]["channels"] == ["conda-forge"]

docs/web-tests/py/test_geneva_profiling_memory.py

@@ -0,0 +1,248 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+"""Snippets for docs/geneva/udfs/profiling-memory.mdx."""
+
+from unittest.mock import MagicMock
+
+
+def test_stateful_udf_class():
+    import geneva
+    import pyarrow as pa
+
+    load_model = MagicMock(return_value=MagicMock(embed=MagicMock(return_value=[0.1] * 512)))
+
+    # --8<-- [start:stateful_udf_class]
+    @geneva.udf(data_type=pa.list_(pa.float32(), 512))
+    class MyEmbedding:
+        def __init__(self):
+            self.model = None
+
+        def setup(self):
+            self.model = load_model()  # allocated once per actor
+
+        def __call__(self, text: str) -> list[float]:
+            if self.model is None:
+                self.setup()
+            return self.model.embed(text)
+    # --8<-- [end:stateful_udf_class]
+
+    assert MyEmbedding is not None
+
+
+def test_memray_tracker_udf(monkeypatch):
+    import sys
+
+    load_model = MagicMock(return_value=MagicMock(embed=MagicMock(return_value=[0.1] * 512)))
+    monkeypatch.setitem(sys.modules, "memray", MagicMock())
+
+    # --8<-- [start:memray_tracker_udf]
+    import os, pathlib, uuid
+    from typing import Any
+    import memray
+    import geneva
+    import pyarrow as pa
+
+    _MEMRAY_OUT_DIR_ENV = "MY_UDF_MEMRAY_OUT_DIR"
+
+
+    @geneva.udf(data_type=pa.list_(pa.float32(), 512))
+    class MyEmbedding:
+        def __init__(self):
+            self.model = None
+            self._tracker: Any = None  # memray.Tracker, when profiling is on
+
+        def setup(self):
+            # Open a memray tracker per worker process, if requested. Each
+            # worker writes its own .bin file so traces don't collide.
+            out_dir = os.environ.get(_MEMRAY_OUT_DIR_ENV)
+            if out_dir:
+                pathlib.Path(out_dir).mkdir(parents=True, exist_ok=True)
+                bin_path = pathlib.Path(out_dir) / (
+                    f"memray-{os.getpid()}-{uuid.uuid4().hex}.bin"
+                )
+                self._tracker = memray.Tracker(
+                    str(bin_path), native_traces=False, follow_fork=False
+                )
+                self._tracker.__enter__()
+            self.model = load_model()
+
+        def __call__(self, text: str) -> list[float]:
+            if self.model is None:
+                self.setup()
+            return self.model.embed(text)
+    # --8<-- [end:memray_tracker_udf]
+
+    assert MyEmbedding is not None
+
+
+def test_ray_cluster_profile(monkeypatch):
+    from contextlib import contextmanager
+
+    table = MagicMock()
+
+    @contextmanager
+    def _mock_cluster(*args, **kwargs):
+        yield
+
+    monkeypatch.setattr("geneva.runners.ray._mgr.ray_cluster", _mock_cluster)
+
+    # --8<-- [start:ray_cluster_profile]
+    from geneva.runners.ray._mgr import ray_cluster
+
+    with ray_cluster(
+        local=True,
+        extra_env={"MY_UDF_MEMRAY_OUT_DIR": "/tmp/my-udf-profile"},
+    ):
+        table.backfill("embedding", concurrency=1)
+    # --8<-- [end:ray_cluster_profile]
+
+    table.backfill.assert_called_once_with("embedding", concurrency=1)
+
+
+def test_log_memory(capsys):
+    # --8<-- [start:log_memory]
+    import resource, pyarrow as pa
+
+    def log_memory(seq: int) -> None:
+        rss_bytes = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
+        # ru_maxrss is bytes on macOS, KiB on Linux:
+        import sys
+        if sys.platform != "darwin":
+            rss_bytes *= 1024
+        arrow_live = pa.total_allocated_bytes()
+        print(
+            f"seq={seq} "
+            f"rss_mb={rss_bytes // 1024**2} "
+            f"arrow_live_mb={arrow_live // 1024**2} "
+            f"gap_mb={(rss_bytes - arrow_live) // 1024**2}",
+            flush=True,
+        )
+    # --8<-- [end:log_memory]
+
+    log_memory(0)
+    captured = capsys.readouterr()
+    assert "seq=0" in captured.out
+    assert "rss_mb=" in captured.out
+
+
+def test_leaky_cache():
+    # --8<-- [start:leaky_cache]
+    class BadEmbedding:
+        def __init__(self):
+            self.cache: dict[str, list[float]] = {}
+
+        def __call__(self, text: str) -> list[float]:
+            if text not in self.cache:
+                self.cache[text] = self.model.embed(text)
+            return self.cache[text]
+    # --8<-- [end:leaky_cache]
+
+    obj = BadEmbedding()
+    obj.model = MagicMock(embed=MagicMock(return_value=[0.1, 0.2]))
+    obj("hello")
+    obj("hello")
+    assert obj.model.embed.call_count == 1  # second call served from cache
+    assert len(obj.cache) == 1
+
+
+def test_bounded_cache():
+    load_model = MagicMock(return_value=MagicMock(embed=MagicMock(return_value=[0.1, 0.2])))
+
+    # --8<-- [start:bounded_cache]
+    from functools import lru_cache
+
+    class GoodEmbedding:
+        def __init__(self):
+            self._embed = None
+
+        def setup(self):
+            model = load_model()
+            self._embed = lru_cache(maxsize=1024)(model.embed)
+
+        def __call__(self, text: str) -> list[float]:
+            if self._embed is None:
+                self.setup()
+            return self._embed(text)
+    # --8<-- [end:bounded_cache]
+
+    obj = GoodEmbedding()
+    assert obj("hello") == [0.1, 0.2]
+    load_model.assert_called_once()
+
+
+def test_leaky_aggregator():
+    import pyarrow as pa
+
+    # --8<-- [start:leaky_aggregator]
+    class BadAggregator:
+        def __init__(self):
+            self.history = []
+
+        def __call__(self, batch: pa.RecordBatch) -> pa.Array:
+            self.history.append(batch)  # holds every batch ever processed
+            ...
+    # --8<-- [end:leaky_aggregator]
+
+    obj = BadAggregator()
+    assert isinstance(obj.history, list)
+
+
+def test_leaky_closure():
+    import pyarrow as pa
+
+    def expensive(x):
+        return x
+
+    # --8<-- [start:leaky_closure]
+    class BadDeferred:
+        def __init__(self):
+            self.work_queue = []
+
+        def __call__(self, x: pa.Array) -> pa.Array:
+            # Lambda captures `x` by reference — the whole Array stays alive
+            self.work_queue.append(lambda: expensive(x))
+            ...
+    # --8<-- [end:leaky_closure]
+
+    obj = BadDeferred()
+    assert isinstance(obj.work_queue, list)
+
+
+def test_torch_inference_mode(monkeypatch):
+    import sys
+
+    mock_torch = MagicMock()
+    monkeypatch.setitem(sys.modules, "torch", mock_torch)
+
+    class TorchUDF:
+        def __init__(self):
+            self.model = MagicMock(encode=MagicMock(return_value=[0.1]))
+
+        # --8<-- [start:torch_inference_mode]
+        def __call__(self, text: str) -> list[float]:
+            with torch.inference_mode():            # <-- prevents autograd graph retention
+                return self.model.encode(text)
+        # --8<-- [end:torch_inference_mode]
+
+    import torch  # resolves to monkeypatched mock above
+    obj = TorchUDF()
+    assert obj("hello") == [0.1]
+
+
+def test_confidence_check():
+    class DebuggingUDF:
+        def __init__(self):
+            self._scratches = []
+
+        # --8<-- [start:confidence_check]
+        def __call__(self, x):
+            scratch = bytearray(8 * 1024 * 1024)  # 8 MiB
+            self._scratches.append(scratch)        # <-- deliberate leak
+            return ...
+        # --8<-- [end:confidence_check]
+
+    obj = DebuggingUDF()
+    obj(None)
+    assert len(obj._scratches) == 1
+    assert len(obj._scratches[0]) == 8 * 1024 * 1024

docs/web-tests/py/test_geneva_scalar_udtfs.py

@@ -0,0 +1,311 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+import pytest
+from unittest.mock import MagicMock
+
+
+def test_scalar_udtf_iterator():
+    def extract_video_segment(path, start, end):
+        return bytes(f"clip:{start}-{end}", "utf-8")
+
+    # --8<-- [start:scalar_udtf_iterator]
+    from geneva import chunker
+    from typing import Iterator, NamedTuple
+
+    class Clip(NamedTuple):
+        clip_start: float
+        clip_end: float
+        clip_bytes: bytes
+
+    @chunker
+    def extract_clips(video_path: str, duration: float) -> Iterator[Clip]:
+        """Yields multiple clips per video."""
+        clip_length = 10.0
+        for start in range(0, int(duration), int(clip_length)):
+            end = min(start + clip_length, duration)
+            clip_data = extract_video_segment(video_path, start, end)
+            yield Clip(clip_start=start, clip_end=end, clip_bytes=clip_data)
+    # --8<-- [end:scalar_udtf_iterator]
+
+    from geneva.transformer import Chunker
+    assert isinstance(extract_clips, Chunker)
+    assert extract_clips.input_columns == ["video_path", "duration"]
+    assert set(extract_clips.output_schema.names) == {"clip_start", "clip_end", "clip_bytes"}
+
+    clips = list(extract_clips.func("/v/a.mp4", 30.0))
+    assert len(clips) == 3
+    assert clips[0].clip_start == 0.0
+    assert clips[2].clip_end == 30.0
+
+
+def test_scalar_udtf_list_return():
+    from geneva import chunker
+    from typing import NamedTuple
+
+    class Clip(NamedTuple):
+        clip_start: float
+        clip_end: float
+        clip_bytes: bytes
+
+    # --8<-- [start:scalar_udtf_list]
+    @chunker
+    def extract_clips(video_path: str, duration: float) -> list[Clip]:
+        clips = []
+        for start in range(0, int(duration), 10):
+            end = min(start + 10, duration)
+            clips.append(Clip(clip_start=start, clip_end=end, clip_bytes=b"..."))
+        return clips
+    # --8<-- [end:scalar_udtf_list]
+
+    from geneva.transformer import Chunker
+    assert isinstance(extract_clips, Chunker)
+    clips = extract_clips.func("/v/a.mp4", 30.0)
+    assert len(clips) == 3
+    assert all(c.clip_bytes == b"..." for c in clips)
+
+
+def test_scalar_udtf_batch():
+    import pyarrow as pa
+    from geneva import chunker
+
+    clip_schema = pa.schema([
+        ("clip_start", pa.float64()),
+        ("clip_end", pa.float64()),
+    ])
+
+    # --8<-- [start:scalar_udtf_batch]
+    @chunker(batch=True, output_schema=clip_schema)
+    def extract_clips(batch: pa.RecordBatch) -> pa.RecordBatch:
+        """Process rows in batches. Same 1:N semantic per row."""
+        ...
+    # --8<-- [end:scalar_udtf_batch]
+
+    from geneva.transformer import Chunker
+    assert isinstance(extract_clips, Chunker)
+    assert extract_clips.batch is True
+
+
+def test_create_scalar_udtf_view(monkeypatch):
+    from typing import Iterator, NamedTuple
+    from geneva.transformer import chunker
+
+    class Clip(NamedTuple):
+        clip_start: float
+        clip_end: float
+        clip_bytes: bytes
+
+    @chunker
+    def extract_clips(video_path: str, duration: float) -> Iterator[Clip]:
+        for start in range(0, int(duration), 10):
+            yield Clip(clip_start=start, clip_end=min(start + 10.0, duration), clip_bytes=b"")
+
+    import geneva
+    from unittest.mock import create_autospec
+    mock_clips = MagicMock()
+    mock_db = create_autospec(geneva.db.Connection, instance=True)
+    mock_db.create_udtf_view.return_value = mock_clips
+    monkeypatch.setattr("geneva.connect", MagicMock(return_value=mock_db))
+
+    # --8<-- [start:create_scalar_udtf_view]
+    import geneva
+
+    db = geneva.connect("/data/mydb")
+    videos = db.open_table("videos")
+
+    # Create the 1:N materialized view
+    clips = db.create_udtf_view(
+        "clips",
+        source=videos.search(None).select(["video_path", "metadata"]),
+        udtf=extract_clips,
+    )
+
+    # Populate — runs the UDTF on every source row
+    clips.refresh()
+    # --8<-- [end:create_scalar_udtf_view]
+
+    call_kwargs = mock_db.create_udtf_view.call_args
+    assert call_kwargs.args[0] == "clips"
+    assert call_kwargs.kwargs["udtf"] is extract_clips
+    mock_clips.refresh.assert_called_once()
+
+
+def test_add_columns_scalar_udtf():
+    import pyarrow as pa
+    from geneva.transformer import udf
+
+    clips = MagicMock()
+    embed_model = MagicMock()
+    embed_model.encode.return_value = [0.1] * 512
+
+    # --8<-- [start:add_columns_scalar_udtf]
+    @udf(data_type=pa.list_(pa.float32(), 512))
+    def clip_embedding(clip_bytes: bytes) -> list[float]:
+        return embed_model.encode(clip_bytes)
+
+    # Add an embedding column to the clips table
+    clips.add_columns({"embedding": clip_embedding})
+
+    # Backfill computes embeddings for all existing clips
+    clips.backfill("embedding")
+    # --8<-- [end:add_columns_scalar_udtf]
+
+    clips.add_columns.assert_called_once_with({"embedding": clip_embedding})
+    clips.backfill.assert_called_once_with("embedding")
+
+
+def test_incremental_refresh():
+    videos = MagicMock()
+    clips = MagicMock()
+    new_video_data = [{"video_path": "/v/c.mp4", "duration": 45.0}]
+
+    # --8<-- [start:incremental_refresh]
+    # Add new videos to the source table
+    videos.add(new_video_data)
+
+    # Incremental refresh — only processes the new videos
+    clips.refresh()
+    # --8<-- [end:incremental_refresh]
+
+    videos.add.assert_called_once_with(new_video_data)
+    clips.refresh.assert_called_once()
+
+
+def test_chaining_udtf_views(monkeypatch):
+    from typing import Iterator, NamedTuple
+    from geneva.transformer import chunker
+
+    class Clip(NamedTuple):
+        clip_start: float
+        clip_end: float
+
+    class Frame(NamedTuple):
+        frame_index: int
+        frame_bytes: bytes
+
+    @chunker
+    def extract_clips(video_path: str, duration: float) -> Iterator[Clip]:
+        for start in range(0, int(duration), 10):
+            yield Clip(clip_start=start, clip_end=min(start + 10.0, duration))
+
+    @chunker
+    def extract_frames(clip_start: float, clip_end: float) -> Iterator[Frame]:
+        yield Frame(frame_index=0, frame_bytes=b"")
+
+    import geneva
+    from unittest.mock import create_autospec
+    mock_db = create_autospec(geneva.db.Connection, instance=True)
+    monkeypatch.setattr("geneva.connect", MagicMock(return_value=mock_db))
+
+    db = geneva.connect("/data/mydb")
+    videos = db.open_table("videos")
+
+    # --8<-- [start:chaining_udtf_views]
+    # videos → clips (1:N)
+    clips = db.create_udtf_view(
+        "clips", source=videos.search(None), udtf=extract_clips
+    )
+
+    # clips → frames (1:N)
+    frames = db.create_udtf_view(
+        "frames", source=clips.search(None), udtf=extract_frames
+    )
+    # --8<-- [end:chaining_udtf_views]
+
+    assert mock_db.create_udtf_view.call_count == 2
+    first_call = mock_db.create_udtf_view.call_args_list[0]
+    second_call = mock_db.create_udtf_view.call_args_list[1]
+    assert first_call.kwargs["udtf"] is extract_clips
+    assert second_call.kwargs["udtf"] is extract_frames
+
+
+def test_document_chunking_udtf():
+    # --8<-- [start:document_chunking_udtf]
+    from geneva import chunker
+    from typing import Iterator, NamedTuple
+
+    class Chunk(NamedTuple):
+        chunk_index: int
+        chunk_text: str
+
+    @chunker
+    def chunk_document(text: str) -> Iterator[Chunk]:
+        """Split a document into overlapping chunks."""
+        words = text.split()
+        chunk_size = 500
+        overlap = 50
+        for i, start in enumerate(range(0, len(words), chunk_size - overlap)):
+            chunk_words = words[start:start + chunk_size]
+            yield Chunk(chunk_index=i, chunk_text=" ".join(chunk_words))
+    # --8<-- [end:document_chunking_udtf]
+
+    sample_text = " ".join(["word"] * 600)
+    chunks = list(chunk_document.func(sample_text))
+    assert len(chunks) == 2
+    assert chunks[0].chunk_index == 0
+    assert chunks[1].chunk_index == 1
+    assert len(chunks[0].chunk_text.split()) == 500
+
+
+def test_document_chunking_full(monkeypatch):
+    import pyarrow as pa
+    from unittest.mock import MagicMock
+
+    import geneva
+    from unittest.mock import create_autospec
+    mock_chunks_table = MagicMock()
+    mock_db = create_autospec(geneva.db.Connection, instance=True)
+    mock_db.create_udtf_view.return_value = mock_chunks_table
+    monkeypatch.setattr("geneva.connect", MagicMock(return_value=mock_db))
+
+    embedding_model = MagicMock()
+    embedding_model.encode.return_value = [0.1] * 1536
+
+    # --8<-- [start:document_chunking_full]
+    from geneva import connect, chunker, udf
+    from typing import Iterator, NamedTuple
+    import pyarrow as pa
+
+    class Chunk(NamedTuple):
+        chunk_index: int
+        chunk_text: str
+
+    @chunker
+    def chunk_document(text: str) -> Iterator[Chunk]:
+        """Split a document into overlapping chunks."""
+        words = text.split()
+        chunk_size = 500
+        overlap = 50
+        for i, start in enumerate(range(0, len(words), chunk_size - overlap)):
+            chunk_words = words[start:start + chunk_size]
+            yield Chunk(chunk_index=i, chunk_text=" ".join(chunk_words))
+
+    db = connect("/data/mydb")
+    docs = db.open_table("documents")
+
+    # Create chunked view — inherits doc_id, title, etc. from source
+    chunks = db.create_udtf_view(
+        "doc_chunks",
+        source=docs.search(None).select(["doc_id", "title", "text"]),
+        udtf=chunk_document,
+    )
+    chunks.refresh()
+
+    # Add embeddings to chunks for semantic search
+    @udf(data_type=pa.list_(pa.float32(), 1536))
+    def embed_text(chunk_text: str) -> list[float]:
+        return embedding_model.encode(chunk_text)
+
+    chunks.add_columns({"embedding": embed_text})
+    chunks.backfill("embedding")  # Backfills embeddings on all existing chunks
+
+    # Query — parent columns available alongside chunk columns
+    chunks.search(None).select(["doc_id", "title", "chunk_text", "embedding"]).to_pandas()
+    # --8<-- [end:document_chunking_full]
+
+    call_kwargs = mock_db.create_udtf_view.call_args
+    assert call_kwargs.args[0] == "doc_chunks"
+    assert call_kwargs.kwargs["udtf"] is chunk_document
+    mock_chunks_table.refresh.assert_called_once()
+    mock_chunks_table.add_columns.assert_called_once()
+    mock_chunks_table.backfill.assert_called_once_with("embedding")

docs/web-tests/py/test_geneva_udfs_index.py

@@ -0,0 +1,52 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+from unittest.mock import MagicMock, create_autospec
+
+
+def test_registration_udf(monkeypatch):
+    # Verifies that Table.add_columns exists with the expected signature.
+    # If this fails, update the Registration row in docs/geneva/udfs/index.mdx.
+    import geneva
+    mock_table = create_autospec(geneva.table.Table, instance=True)
+    my_udf = MagicMock()
+
+    # --8<-- [start:registration_udf]
+    mock_table.add_columns({"col": my_udf})
+    # --8<-- [end:registration_udf]
+
+    mock_table.add_columns.assert_called_once()
+
+
+def test_registration_scalar_udtf(monkeypatch):
+    # Verifies that Connection.create_udtf_view accepts a chunker.
+    # If this fails, update the Registration row in docs/geneva/udfs/index.mdx.
+    import geneva
+    mock_db = create_autospec(geneva.db.Connection, instance=True)
+    monkeypatch.setattr("geneva.connect", MagicMock(return_value=mock_db))
+    my_source = MagicMock()
+    my_chunker = MagicMock()
+
+    # --8<-- [start:registration_scalar_udtf]
+    db = geneva.connect("/data/mydb")
+    db.create_udtf_view("my_view", source=my_source, udtf=my_chunker)
+    # --8<-- [end:registration_scalar_udtf]
+
+    mock_db.create_udtf_view.assert_called_once()
+
+
+def test_registration_udtf(monkeypatch):
+    # Verifies that Connection.create_udtf_view exists with the expected signature.
+    # If this fails, update the Registration row in docs/geneva/udfs/index.mdx.
+    import geneva
+    mock_db = create_autospec(geneva.db.Connection, instance=True)
+    monkeypatch.setattr("geneva.connect", MagicMock(return_value=mock_db))
+    my_source = MagicMock()
+    my_udtf = MagicMock()
+
+    # --8<-- [start:registration_udtf]
+    db = geneva.connect("/data/mydb")
+    db.create_udtf_view("my_view", source=my_source, udtf=my_udtf)
+    # --8<-- [end:registration_udtf]
+
+    mock_db.create_udtf_view.assert_called_once()

docs/web-tests/py/test_indexing.py

@@ -0,0 +1,738 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+import random
+import string
+import uuid
+
+import numpy as np
+import pyarrow as pa
+import pytest
+
+
+def _make_vector_rows(count: int, dim: int, column: str = "vector"):
+    rows = [
+        {column: np.random.random(dim).astype(np.float32).tolist(), "id": i}
+        for i in range(count)
+    ]
+    assert len(rows) == count
+    assert all(len(row[column]) == dim for row in rows)
+    return rows
+
+
+def test_vector_index_configure_ivf(tmp_db):
+    table = tmp_db.create_table(
+        "vector_index_configure_ivf",
+        _make_vector_rows(512, 4),
+        mode="overwrite",
+    )
+
+    # --8<-- [start:vector_index_configure_ivf]
+    table.create_index(metric="l2", num_partitions=16, num_sub_vectors=4)
+    # --8<-- [end:vector_index_configure_ivf]
+
+    assert table.list_indices()
+
+
+def test_vector_index_setup(tmp_db):
+    tmp_db.create_table(
+        "vector-index-tbl",
+        _make_vector_rows(8, 4),
+        mode="overwrite",
+    )
+
+    db = tmp_db
+    # --8<-- [start:vector_index_setup]
+    table_name = "vector-index-tbl"
+    table = db.open_table(table_name)
+    # --8<-- [end:vector_index_setup]
+
+    assert table.name == table_name
+
+
+def test_vector_index_build_ivf(tmp_db):
+    table = tmp_db.create_table(
+        "vector-index-build-ivf",
+        _make_vector_rows(512, 4, column="keywords_embeddings"),
+        mode="overwrite",
+    )
+    db = tmp_db
+    # --8<-- [start:vector_index_build_ivf]
+    table_name = "vector-index-build-ivf"
+    table = db.open_table(table_name)
+    table.create_index(
+        metric="cosine",
+        vector_column_name="keywords_embeddings",
+    )
+    # --8<-- [end:vector_index_build_ivf]
+
+    assert table.list_indices()
+
+
+def test_vector_index_nested_field(tmp_db):
+    dim = 2
+    schema = pa.schema(
+        [
+            pa.field("id", pa.int32()),
+            pa.field(
+                "image",
+                pa.struct([pa.field("embedding", pa.list_(pa.float32(), dim))]),
+            ),
+        ]
+    )
+    data = [
+        {
+            "id": i,
+            "image": {"embedding": np.random.random(dim).astype(np.float32).tolist()},
+        }
+        for i in range(512)
+    ]
+    table = tmp_db.create_table(
+        "vector_index_nested_field", data=data, schema=schema, mode="overwrite"
+    )
+
+    # --8<-- [start:vector_index_nested_field]
+    # The vector column `embedding` is nested inside the `image` struct.
+    # Pass its full dotted path as `vector_column_name`; the same path is used
+    # at query time and is what `list_indices()` reports under `columns`.
+    table.create_index(
+        vector_column_name="image.embedding",
+        num_partitions=1,
+        num_sub_vectors=1,
+        name="image_embedding_idx",
+    )
+
+    results = (
+        table.search([0.0, 1.0], vector_column_name="image.embedding")
+        .limit(1)
+        .to_list()
+    )
+    # --8<-- [end:vector_index_nested_field]
+
+    assert table.index_stats("image_embedding_idx")
+    assert len(results) == 1
+
+
+@pytest.mark.asyncio
+async def test_vector_index_async_config(tmp_path, monkeypatch):
+    monkeypatch.chdir(tmp_path)
+
+    # --8<-- [start:vector_index_async_config]
+    import lancedb
+    import numpy as np
+    from lancedb.index import IvfPq
+
+    async def main():
+        data = [
+            {"id": i, "vector": np.random.random(8).astype(np.float32).tolist()}
+            for i in range(512)
+        ]
+
+        db = await lancedb.connect_async("ex_lancedb")
+        table = await db.create_table(
+            "vector_index_async", data=data, mode="overwrite"
+        )
+
+        await table.create_index(
+            "vector",
+            config=IvfPq(
+                distance_type="cosine",
+                num_partitions=16,
+                num_sub_vectors=4,
+            ),
+        )
+        return await table.list_indices()
+    # --8<-- [end:vector_index_async_config]
+
+    assert await main()
+
+
+def test_vector_index_query_ivf(tmp_db):
+    dim = 1536
+    data = [
+        {"id": i, "keywords_embeddings": np.random.random(dim).tolist()}
+        for i in range(512)
+    ]
+    table = tmp_db.create_table("vector_index_query_ivf", data, mode="overwrite")
+    table.create_index(
+        metric="cosine",
+        vector_column_name="keywords_embeddings",
+    )
+
+    # --8<-- [start:vector_index_query_ivf]
+    tbl = table
+    tbl.search(np.random.random((1536))).limit(2).nprobes(20).refine_factor(
+        10
+    ).to_pandas()
+    # --8<-- [end:vector_index_query_ivf]
+
+    df = (
+        table.search(np.random.random((1536)))
+        .limit(2)
+        .nprobes(20)
+        .refine_factor(10)
+        .to_pandas()
+    )
+    assert len(df) == 2
+
+
+def test_vector_index_nprobes(tmp_db):
+    dim = 128
+    data = [
+        {"id": i, "keywords_embeddings": np.random.random(dim).tolist()}
+        for i in range(512)
+    ]
+    table = tmp_db.create_table("vector_index_nprobes", data, mode="overwrite")
+    table.create_index(
+        metric="cosine",
+        vector_column_name="keywords_embeddings",
+    )
+
+    # --8<-- [start:vector_index_nprobes]
+    # Always scan 10 partitions; scan up to 50 only if the initial pass
+    # returns fewer than `limit` results (common with narrow filters).
+    (
+        table.search(np.random.random(128))
+        .minimum_nprobes(10)
+        .maximum_nprobes(50)
+        .where("id > 100")
+        .limit(5)
+        .to_pandas()
+    )
+    # --8<-- [end:vector_index_nprobes]
+
+
+def test_vector_index_distance_range(tmp_db):
+    dim = 128
+    data = [
+        {"id": i, "keywords_embeddings": np.random.random(dim).tolist()}
+        for i in range(256)
+    ]
+    table = tmp_db.create_table("vector_index_distance_range", data, mode="overwrite")
+    table.create_index(
+        metric="cosine",
+        vector_column_name="keywords_embeddings",
+    )
+
+    # --8<-- [start:vector_index_distance_range]
+    # Only return results whose distance falls within [0.0, 0.5).
+    # Useful for near-duplicate detection or thresholded similarity search.
+    (
+        table.search(np.random.random(128))
+        .distance_range(lower_bound=0.0, upper_bound=0.5)
+        .limit(10)
+        .to_pandas()
+    )
+    # --8<-- [end:vector_index_distance_range]
+
+
+def test_vector_index_bypass_recall(tmp_db):
+    dim = 128
+    data = [
+        {"id": i, "keywords_embeddings": np.random.random(dim).tolist()}
+        for i in range(256)
+    ]
+    table = tmp_db.create_table("vector_index_bypass_recall", data, mode="overwrite")
+    table.create_index(
+        metric="cosine",
+        vector_column_name="keywords_embeddings",
+    )
+
+    # --8<-- [start:vector_index_bypass_recall]
+    query = np.random.random(128)
+    k = 10
+
+    # Ground truth: flat (exhaustive) scan, ignoring the ANN index.
+    truth = set(table.search(query).bypass_vector_index().limit(k).to_pandas()["id"])
+
+    # ANN results with the current nprobes setting.
+    ann = set(table.search(query).nprobes(20).limit(k).to_pandas()["id"])
+
+    recall_at_k = len(truth & ann) / k
+    # --8<-- [end:vector_index_bypass_recall]
+    assert 0.0 <= recall_at_k <= 1.0
+
+
+def test_vector_index_custom_name(tmp_db):
+    table = tmp_db.create_table(
+        "vector_index_custom_name",
+        _make_vector_rows(512, 8, column="keywords_embeddings"),
+        mode="overwrite",
+    )
+
+    # --8<-- [start:vector_index_custom_name]
+    # Override the default `{column}_idx` convention by passing `name=...`.
+    table.create_index(
+        metric="cosine",
+        vector_column_name="keywords_embeddings",
+        name="my_custom_index",
+    )
+    table.wait_for_index(["my_custom_index"])
+    print(table.index_stats("my_custom_index"))
+    # --8<-- [end:vector_index_custom_name]
+
+    assert table.index_stats("my_custom_index")
+
+
+def test_vector_index_hnsw(tmp_db):
+    table = tmp_db.create_table(
+        "vector_index_hnsw",
+        _make_vector_rows(64, 16),
+        mode="overwrite",
+    )
+
+    # --8<-- [start:vector_index_build_hnsw]
+    table.create_index(index_type="IVF_HNSW_SQ")
+    # --8<-- [end:vector_index_build_hnsw]
+
+    # --8<-- [start:vector_index_query_hnsw]
+    tbl = table
+    tbl.search(np.random.random((16))).limit(2).to_pandas()
+    # --8<-- [end:vector_index_query_hnsw]
+
+    df = table.search(np.random.random((16))).limit(2).to_pandas()
+    assert len(df) == 2
+
+
+def test_vector_index_binary(tmp_db):
+    table_name = "hamming-index-tbl"
+    ndim = 256
+    schema = pa.schema(
+        [
+            pa.field("id", pa.int64()),
+            pa.field("vector", pa.list_(pa.uint8(), ndim // 8)),
+        ]
+    )
+
+    # --8<-- [start:vector_index_binary_schema]
+    table = tmp_db.create_table(table_name, schema=schema, mode="overwrite")
+    # --8<-- [end:vector_index_binary_schema]
+
+    data = []
+    for i in range(64):
+        vector = np.random.randint(0, 2, size=ndim)
+        vector = np.packbits(vector)
+        data.append({"id": i, "vector": vector})
+
+    # --8<-- [start:vector_index_binary_add_data]
+    table.add(data)
+    # --8<-- [end:vector_index_binary_add_data]
+
+    # --8<-- [start:vector_index_binary_build_index]
+    table.create_index(
+        metric="hamming",
+        vector_column_name="vector",
+        index_type="IVF_FLAT",
+    )
+    # --8<-- [end:vector_index_binary_build_index]
+
+    # --8<-- [start:vector_index_binary_search]
+    query = np.random.randint(0, 2, size=ndim)
+    query = np.packbits(query)
+    df = table.search(query).metric("hamming").limit(10).to_pandas()
+    df.vector = df.vector.apply(np.unpackbits)
+    # --8<-- [end:vector_index_binary_search]
+
+    assert not df.empty
+
+
+def test_vector_index_check_status(tmp_db):
+    table = tmp_db.create_table(
+        "vector_index_check_status",
+        _make_vector_rows(512, 8, column="keywords_embeddings"),
+        mode="overwrite",
+    )
+    table.create_index(
+        metric="cosine",
+        vector_column_name="keywords_embeddings",
+    )
+
+    # --8<-- [start:vector_index_check_status]
+    index_name = "keywords_embeddings_idx"
+    table.wait_for_index([index_name])
+    print(table.index_stats(index_name))
+    # --8<-- [end:vector_index_check_status]
+
+    assert table.index_stats(index_name)
+
+
+def test_scalar_index_build(tmp_db):
+    table = tmp_db.create_table(
+        "scalar_index_build",
+        [
+            {"book_id": 1, "publisher": "A", "vector": [0.1, 0.2]},
+            {"book_id": 2, "publisher": "B", "vector": [0.2, 0.3]},
+        ],
+        mode="overwrite",
+    )
+    db = tmp_db
+    # --8<-- [start:scalar_index_build]
+    tbl = db.open_table("scalar_index_build")
+    tbl.create_scalar_index("book_id")
+    tbl.create_scalar_index("publisher", index_type="BITMAP")
+    # --8<-- [end:scalar_index_build]
+
+    assert tbl.list_indices()
+
+
+def test_scalar_index_wait(tmp_db):
+    table = tmp_db.create_table(
+        "scalar_index_wait",
+        [{"label": "fiction"}],
+        mode="overwrite",
+    )
+    table.create_scalar_index("label")
+
+    # --8<-- [start:scalar_index_wait]
+    index_name = "label_idx"
+    table.wait_for_index([index_name])
+    # --8<-- [end:scalar_index_wait]
+
+    assert table.list_indices()
+
+
+def test_scalar_index_optimize(tmp_db):
+    table = tmp_db.create_table(
+        "scalar_index_optimize",
+        [{"vector": [7.0, 8.0], "book_id": 3}],
+        mode="overwrite",
+    )
+
+    # --8<-- [start:scalar_index_optimize]
+    table.add([{"vector": [7, 8], "book_id": 4}])
+    table.optimize()
+    # --8<-- [end:scalar_index_optimize]
+
+    result = table.search().where("book_id = 4").limit(10).to_pandas()
+    assert len(result) == 1
+
+
+def test_scalar_index_filter(tmp_db):
+    table = tmp_db.create_table(
+        "books",
+        [
+            {"vector": [1.1, 1.2], "book_id": 1},
+            {"vector": [2.1, 2.2], "book_id": 2},
+        ],
+        mode="overwrite",
+    )
+    db = tmp_db
+    # --8<-- [start:scalar_index_filter]
+    table = db.open_table("books")
+    result = table.search().where("book_id = 2").limit(10).to_pandas()
+    # --8<-- [end:scalar_index_filter]
+
+    assert len(result) == 1
+
+
+def test_scalar_index_prefilter(tmp_db):
+    table = tmp_db.create_table(
+        "book_with_embeddings",
+        [
+            {"vector": [1.2, 1.3], "book_id": 1},
+            {"vector": [4.2, 4.3], "book_id": 2},
+        ],
+        mode="overwrite",
+    )
+    db = tmp_db
+    # --8<-- [start:scalar_index_prefilter]
+    table = db.open_table("book_with_embeddings")
+    table.search([1.2] * 2).where("book_id != 3").limit(10).to_pandas()
+    # --8<-- [end:scalar_index_prefilter]
+
+    result = table.search([1.2] * 2).where("book_id != 3").limit(10).to_pandas()
+    assert len(result) == 2
+
+
+def test_scalar_index_uuid(tmp_db):
+    # --8<-- [start:scalar_index_uuid_type]
+    import pyarrow as pa
+    # --8<-- [end:scalar_index_uuid_type]
+
+    # --8<-- [start:scalar_index_uuid_data]
+    def generate_random_names():
+        base_names = ["Alice", "Bob", "Carla", "David", "Eve", "Frank", "Grace"]
+        letter = random.choice(string.ascii_uppercase)
+        return f"{random.choice(base_names)} {letter}."
+
+    def generate_uuids(num_items):
+        return [uuid.uuid4().bytes for _ in range(num_items)]
+
+    # Generate some UUIDs and random names
+    n = 7
+    uuids = generate_uuids(n)
+    names = [generate_random_names() for _ in range(n)]
+    # --8<-- [end:scalar_index_uuid_data]
+
+    db = tmp_db
+    # --8<-- [start:scalar_index_uuid_table]
+    table_name = "index-on-uuid"
+
+    uuid_array = pa.array(uuids, pa.uuid())
+    name_array = pa.array(names, pa.string())
+    schema = pa.schema(
+        [
+            pa.field("id", pa.uuid()),
+            pa.field("name", pa.string()),
+        ]
+    )
+    data_table = pa.Table.from_arrays([uuid_array, name_array], schema=schema)
+    table = db.create_table(table_name, data=data_table, mode="overwrite")
+    # --8<-- [end:scalar_index_uuid_table]
+
+    # --8<-- [start:scalar_index_uuid_wait]
+    index_name = "id_idx"
+    table.create_scalar_index("id")
+    table.wait_for_index([index_name])
+    # --8<-- [end:scalar_index_uuid_wait]
+
+    # --8<-- [start:scalar_index_uuid_upsert]
+    new_users = [
+        {"id": uuid.uuid4().bytes, "name": "Hannah D."},
+        {"id": uuid.uuid4().bytes, "name": "Ian B."},
+    ]
+    # Insert or update using the UUID index
+    table.merge_insert(
+        "id"
+    ).when_matched_update_all().when_not_matched_insert_all().execute(new_users)
+    # --8<-- [end:scalar_index_uuid_upsert]
+
+    assert table.list_indices()
+    result = table.search().limit(100).to_pandas()
+    assert len(result) == n + len(new_users)
+
+
+@pytest.mark.asyncio
+async def test_scalar_index_nested_fields(mem_db_async):
+    db = mem_db_async
+
+    # --8<-- [start:scalar_index_nested_fields]
+    import pyarrow as pa
+    from lancedb.index import BTree
+
+    metadata_type = pa.struct(
+        [
+            pa.field("user_id", pa.int32()),
+            pa.field("user.id", pa.int32()),
+        ]
+    )
+    data = pa.Table.from_arrays(
+        [
+            pa.array([1, 2, 3], type=pa.int32()),
+            pa.array(
+                [
+                    {"user_id": 10, "user.id": 100},
+                    {"user_id": 20, "user.id": 200},
+                    {"user_id": 30, "user.id": 300},
+                ],
+                type=metadata_type,
+            ),
+        ],
+        names=["user_id", "metadata"],
+    )
+    table = await db.create_table("nested_scalar_index", data)
+
+    # Index a nested struct field.
+    await table.create_index(
+        "metadata.user_id", config=BTree(), name="nested_user_id_idx"
+    )
+
+    # Escape literal dots inside a segment with backticks.
+    await table.create_index(
+        "metadata.`user.id`", config=BTree(), name="escaped_user_id_idx"
+    )
+
+    # `columns` is returned as the canonical path you passed in.
+    for index in await table.list_indices():
+        print(index.name, index.columns)
+    # nested_user_id_idx  ['metadata.user_id']
+    # escaped_user_id_idx ['metadata.`user.id`']
+    # --8<-- [end:scalar_index_nested_fields]
+
+    index_columns = {index.name: index.columns for index in await table.list_indices()}
+    assert index_columns["nested_user_id_idx"] == ["metadata.user_id"]
+    assert index_columns["escaped_user_id_idx"] == ["metadata.`user.id`"]
+
+
+def test_fts_index_create(tmp_db):
+    table = tmp_db.create_table(
+        "fts-index-create",
+        [{"text": "hello world", "vector": [0.1, 0.2]}],
+        mode="overwrite",
+    )
+
+    db = tmp_db
+    # --8<-- [start:fts_index_create]
+    table_name = "fts-index-create"
+    table = db.open_table(table_name)
+    table.create_fts_index("text")
+    # --8<-- [end:fts_index_create]
+
+    assert table.list_indices()
+
+
+def test_fts_index_wait(tmp_db):
+    table = tmp_db.create_table(
+        "fts-index-wait",
+        [{"text": "full text search"}],
+        mode="overwrite",
+    )
+    
+    db = tmp_db
+    # --8<-- [start:fts_index_wait]
+    table_name = "fts-index-wait"
+
+    table = db.open_table(table_name)
+    table.create_fts_index("text")
+
+    index_name = "text_idx"
+    table.wait_for_index([index_name])
+    # --8<-- [end:fts_index_wait]
+
+    assert table.list_indices()
+
+
+def test_fts_index_nested_field(tmp_db):
+    nested_schema = pa.struct([
+        pa.field("text", pa.string()),
+        pa.field("count", pa.int32()),
+    ])
+    schema = pa.schema([
+        pa.field("id", pa.int64()),
+        pa.field("payload", nested_schema),
+    ])
+    tmp_db.create_table(
+        "fts-index-nested",
+        pa.table(
+            {
+                "id": pa.array([1, 2], pa.int64()),
+                "payload": pa.array(
+                    [
+                        {"text": "Frodo was a happy puppy", "count": 1},
+                        {"text": "puppy runs through the meadow", "count": 2},
+                    ],
+                    type=nested_schema,
+                ),
+            },
+            schema=schema,
+        ),
+        mode="overwrite",
+    )
+
+    db = tmp_db
+    # --8<-- [start:fts_index_nested]
+    from lancedb.query import MatchQuery, PhraseQuery
+
+    table = db.open_table("fts-index-nested")
+
+    # Index a text leaf inside a struct column using a dotted path.
+    table.create_fts_index("payload.text", with_position=True)
+
+    # The same dotted path works in MatchQuery and PhraseQuery.
+    matches = (
+        table.search(MatchQuery("puppy", "payload.text")).limit(5).to_list()
+    )
+    phrases = (
+        table.search(PhraseQuery("puppy runs", "payload.text"))
+        .limit(5)
+        .to_list()
+    )
+    # --8<-- [end:fts_index_nested]
+
+    assert len(matches) > 0
+    assert all("puppy" in row["payload"]["text"] for row in matches)
+    assert len(phrases) > 0
+    assert all("puppy runs" in row["payload"]["text"] for row in phrases)
+
+
+@pytest.mark.asyncio
+async def test_fts_index_async(tmp_path, monkeypatch):
+    monkeypatch.chdir(tmp_path)
+
+    # --8<-- [start:fts_index_async]
+    import asyncio
+
+    import lancedb
+    import polars as pl
+    from lancedb.index import FTS
+
+    data = pl.DataFrame(
+        {
+            "id": [1, 2],
+            "text": [
+                "His first language is spanish",
+                "Her first language is english",
+            ],
+        }
+    )
+
+    async def main(data: pl.DataFrame):
+        uri = "ex_lancedb"
+        db = await lancedb.connect_async(uri)
+        tbl = await db.create_table("my_text", data=data, mode="overwrite")
+
+        await tbl.create_index("text", config=FTS(language="English"))
+
+        response = await tbl.search("spanish", query_type="fts")
+        result = await response.limit(1).to_polars()
+        print(result)
+        return result
+
+    if __name__ == "__main__":
+        asyncio.run(main(data))
+    # --8<-- [end:fts_index_async]
+
+    result = await main(data)
+    assert result.height == 1
+
+
+def test_gpu_index_snippets(tmp_db, monkeypatch):
+    table = tmp_db.create_table(
+        "gpu_index",
+        _make_vector_rows(32, 8),
+        mode="overwrite",
+    )
+
+    calls = []
+
+    def fake_create_index(*args, **kwargs):
+        calls.append(kwargs)
+        return None
+
+    monkeypatch.setattr(table, "create_index", fake_create_index)
+
+    # --8<-- [start:gpu_index_cuda]
+    table.create_index(
+        num_partitions=256,
+        num_sub_vectors=96,
+        accelerator="cuda",
+    )
+    # --8<-- [end:gpu_index_cuda]
+
+    # --8<-- [start:gpu_index_mps]
+    table.create_index(
+        num_partitions=256,
+        num_sub_vectors=96,
+        accelerator="mps",
+    )
+    # --8<-- [end:gpu_index_mps]
+
+    assert calls[0]["accelerator"] == "cuda"
+    assert calls[1]["accelerator"] == "mps"
+
+
+def test_reindexing_incremental(tmp_db):
+    table = tmp_db.create_table(
+        "reindexing_incremental",
+        [{"vector": [3.1, 4.1], "text": "Frodo was a happy puppy"}],
+        mode="overwrite",
+    )
+    db = tmp_db
+    # --8<-- [start:reindexing_incremental]
+    table = db.open_table("reindexing_incremental")
+    table.add([{"vector": [3.1, 4.1], "text": "Frodo was a happy puppy"}])
+    table.optimize()
+    # --8<-- [end:reindexing_incremental]
+
+    result = table.search().limit(10).to_pandas()
+    assert len(result) == 2

docs/web-tests/py/test_multimodal.py

@@ -0,0 +1,232 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+import pytest
+try:
+    import lancedb
+    import numpy as np
+    import pyarrow as pa
+    import io
+    from PIL import Image
+except ImportError:
+    pass
+
+# --8<-- [start:multimodal_imports]
+import lancedb
+import pyarrow as pa
+import pandas as pd
+import numpy as np
+import io
+from PIL import Image
+# --8<-- [end:multimodal_imports]
+
+def test_multimodal_ingestion(db_path_factory):
+    # Ensure dependencies are available
+    pytest.importorskip("PIL")
+    pytest.importorskip("lancedb")
+    pytest.importorskip("numpy")
+
+    # --8<-- [start:create_dummy_data]
+    # Create some dummy images
+    def create_dummy_image(color):
+        img = Image.new('RGB', (100, 100), color=color)
+        buf = io.BytesIO()
+        img.save(buf, format='PNG')
+        return buf.getvalue()
+
+    # Create dataset with metadata, vectors, and image blobs
+    data = [
+        {
+            "id": 1,
+            "filename": "red_square.png",
+            "vector": np.random.rand(128).astype(np.float32),
+            "image_blob": create_dummy_image('red'),
+            "label": "red"
+        },
+        {
+            "id": 2,
+            "filename": "blue_square.png",
+            "vector": np.random.rand(128).astype(np.float32),
+            "image_blob": create_dummy_image('blue'),
+            "label": "blue"
+        }
+    ]
+    # --8<-- [end:create_dummy_data]
+
+    # --8<-- [start:define_schema]
+    # Define schema explictly to ensure image_blob is treated as binary
+    schema = pa.schema([
+        pa.field("id", pa.int32()),
+        pa.field("filename", pa.string()),
+        pa.field("vector", pa.list_(pa.float32(), 128)),
+        pa.field("image_blob", pa.binary()), # Important: Use pa.binary() for blobs
+        pa.field("label", pa.string())
+    ])
+    # --8<-- [end:define_schema]
+
+    db_uri = db_path_factory("multimodal_db")
+    db = lancedb.connect(db_uri)
+
+    # --8<-- [start:ingest_data]
+    tbl = db.create_table("images", data=data, schema=schema, mode="overwrite")
+    # --8<-- [end:ingest_data]
+   
+    assert len(tbl) == 2
+
+    # --8<-- [start:search_data]
+    # Search for similar images
+    query_vector = np.random.rand(128).astype(np.float32)
+    results = tbl.search(query_vector).limit(1).to_pandas()
+    # --8<-- [end:search_data]
+
+    # --8<-- [start:process_results]
+    # Convert back to PIL Image
+    for _, row in results.iterrows():
+        image_bytes = row['image_blob']
+        image = Image.open(io.BytesIO(image_bytes))
+        print(f"Retrieved image: {row['filename']}, Size: {image.size}")
+        # You can now use 'image' with other libraries or display it
+    # --8<-- [end:process_results]
+   
+    assert len(results) == 1
+
+def test_blob_api_definition(db_path_factory):
+    # --8<-- [start:blob_api_schema]
+    import pyarrow as pa
+
+    # Define schema with Blob API metadata for lazy loading
+    schema = pa.schema([
+        pa.field("id", pa.int64()),
+        pa.field(
+            "video", 
+            pa.large_binary(), 
+            metadata={"lance-encoding:blob": "true"} # Enable Blob API
+        ),
+    ])
+    # --8<-- [end:blob_api_schema]
+
+    # --8<-- [start:blob_api_ingest]
+    import lancedb
+
+    db = lancedb.connect(db_path_factory("blob_db"))
+    
+    # Create sample data
+    data = [
+        {"id": 1, "video": b"fake_video_bytes_1"},
+        {"id": 2, "video": b"fake_video_bytes_2"}
+    ]
+    
+    # Create the table
+    tbl = db.create_table("videos", data=data, schema=schema)
+    # --8<-- [end:blob_api_ingest]
+    assert len(tbl) == 2
+
+
+def test_blob_api_to_pandas(db_path_factory):
+    db = lancedb.connect(db_path_factory("blob_to_pandas_db"))
+    schema = pa.schema([
+        pa.field("id", pa.int64()),
+        pa.field(
+            "video",
+            pa.large_binary(),
+            metadata={"lance-encoding:blob": "true"},
+        ),
+    ])
+    tbl = db.create_table(
+        "videos",
+        data=[
+            {"id": 1, "video": b"fake_video_bytes_1"},
+            {"id": 2, "video": b"fake_video_bytes_2"},
+        ],
+        schema=schema,
+        mode="overwrite",
+    )
+
+    # --8<-- [start:blob_api_to_pandas]
+    # Default: blob columns come back lazily
+    df_lazy = tbl.to_pandas()
+
+    # Materialize blob bytes eagerly
+    df_bytes = tbl.to_pandas(blob_mode="bytes")
+
+    # Return descriptors instead of payloads
+    df_desc = tbl.to_pandas(blob_mode="descriptions")
+
+    # Forward extra kwargs to PyArrow's to_pandas
+    df_typed = tbl.to_pandas(split_blocks=True, self_destruct=True)
+    # --8<-- [end:blob_api_to_pandas]
+
+    assert len(df_lazy) == 2
+    assert isinstance(df_bytes["video"].iloc[0], bytes)
+    assert df_bytes["video"].tolist() == [
+        b"fake_video_bytes_1",
+        b"fake_video_bytes_2",
+    ]
+    assert len(df_desc) == 2
+    assert len(df_typed) == 2
+
+
+@pytest.mark.asyncio
+async def test_query_to_pandas_kwargs(db_path_factory):
+    schema = pa.schema([
+        pa.field("id", pa.int64()),
+        pa.field("vector", pa.list_(pa.float32(), 128)),
+        pa.field(
+            "video",
+            pa.large_binary(),
+            metadata={"lance-encoding:blob": "true"},
+        ),
+    ])
+    data = [
+        {
+            "id": i,
+            "vector": np.random.rand(128).astype(np.float32),
+            "video": f"fake_video_bytes_{i}".encode(),
+        }
+        for i in range(10)
+    ]
+
+    db = lancedb.connect(db_path_factory("query_to_pandas_db"))
+    tbl = db.create_table("search_demo", data=data, schema=schema, mode="overwrite")
+
+    async_db = await lancedb.connect_async(
+        str(db_path_factory("query_to_pandas_async_db"))
+    )
+    tbl_async = await async_db.create_table(
+        "search_demo", data=data, schema=schema, mode="overwrite"
+    )
+
+    query_vector = np.random.rand(128).astype(np.float32)
+
+    # --8<-- [start:query_to_pandas_kwargs]
+    # Plain scan query: blob_mode is supported end to end
+    df_lazy = (
+        tbl.search()
+        .where("id = 1")
+        .select(["id", "video"])
+        .to_pandas(blob_mode="lazy")
+    )
+
+    # Same call shape works on async query builders
+    df_bytes = await (
+        tbl_async.query()
+        .where("id = 1")
+        .select(["id", "video"])
+        .to_pandas(blob_mode="bytes")
+    )
+
+    # Vector / FTS / hybrid queries can't materialize blob columns,
+    # so omit them from the projection
+    df_vec = (
+        tbl.search(query_vector)
+        .limit(10)
+        .select(["id", "vector"])
+        .to_pandas(split_blocks=True, self_destruct=True)
+    )
+    # --8<-- [end:query_to_pandas_kwargs]
+
+    assert len(df_lazy) == 1
+    assert len(df_bytes) == 1
+    assert df_bytes["video"].iloc[0] == b"fake_video_bytes_1"
+    assert len(df_vec) == 10
+    assert "video" not in df_vec.columns

docs/web-tests/py/test_quickstart.py

@@ -0,0 +1,208 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+import lancedb
+import pytest
+
+
+def test_quickstart(db_path_factory):
+    uri = db_path_factory("quickstart_db")
+    db = lancedb.connect(uri)
+
+    # --8<-- [start:quickstart_data]
+    data = [
+        {
+            "id": "1",
+            "name": "King Arthur",
+            "role": "King",
+            "description": "Leader of Camelot and wielder of Excalibur.",
+            "stats": {"strength": 4, "magic": 1, "leadership": 5, "wisdom": 4},
+            "vector": [0.7, 0.1, 0.9, 0.7],
+        },
+        {
+            "id": "2",
+            "name": "Merlin",
+            "role": "Wizard",
+            "description": "Advisor and prophet with deep magical knowledge.",
+            "stats": {"strength": 2, "magic": 5, "leadership": 4, "wisdom": 5},
+            "vector": [0.2, 0.9, 0.4, 0.9],
+        },
+        {
+            "id": "3",
+            "name": "Sir Lancelot",
+            "role": "Knight",
+            "description": "Legendary knight known for courage and combat skill.",
+            "stats": {"strength": 5, "magic": 1, "leadership": 3, "wisdom": 3},
+            "vector": [0.9, 0.1, 0.5, 0.4],
+        },
+    ]
+    # --8<-- [end:quickstart_data]
+
+    # --8<-- [start:quickstart_create_table]
+    table = db.create_table("characters", data=data, mode="overwrite")
+    # --8<-- [end:quickstart_create_table]
+    assert len(table) == 3
+    # Drop the table to test create without overwrite
+    db.drop_table("characters")
+
+    # --8<-- [start:quickstart_create_table_no_overwrite]
+    table = db.create_table("characters", data=data)
+    # --8<-- [end:quickstart_create_table_no_overwrite]
+    assert len(table) == 3
+    # --8<-- [start:quickstart_vector_search_1]
+    # Search for examples similar to a "wise magical advisor"
+    query_vector = [0.2, 0.8, 0.4, 0.9]
+
+    # Ensure you run `pip install polars` beforehand
+    result = (
+        table.search(query_vector)
+        .select(["name", "role", "description", "_distance"])
+        .limit(2)
+        .to_polars()
+    )
+    print(result)
+    # --8<-- [end:quickstart_vector_search_1]
+    assert result.head(1)["name"][0] == "Merlin"
+
+    # --8<-- [start:quickstart_curate_with_metadata]
+    curated = (
+        table.search(query_vector)
+        .where("stats.magic >= 4")
+        .select(["name", "role", "description", "_distance"])
+        .limit(2)
+        .to_polars()
+    )
+    print(curated)
+    # --8<-- [end:quickstart_curate_with_metadata]
+    assert curated.head(1)["name"][0] == "Merlin"
+
+    # --8<-- [start:quickstart_output_pandas]
+    # Ensure you run `pip install pandas` beforehand
+    result = table.search(query_vector).limit(2).to_pandas()
+    print(result)
+    # --8<-- [end:quickstart_output_pandas]
+    assert result.iloc[0]["name"] == "Merlin"
+
+    # --8<-- [start:quickstart_add_feature]
+    table.add_columns(
+        {
+            "power_score": "cast(((stats.strength + stats.magic + stats.leadership + stats.wisdom) / 4.0) as float)"
+        }
+    )
+    # --8<-- [end:quickstart_add_feature]
+    assert "power_score" in table.schema.names
+
+    # --8<-- [start:quickstart_query_feature]
+    features = table.search().select(["name", "role", "power_score"]).to_polars()
+    print(features)
+    # --8<-- [end:quickstart_query_feature]
+    assert "power_score" in features.columns
+
+    # --8<-- [start:quickstart_multimodal_bytes]
+    from pathlib import Path
+
+    image_path = Path("docs/static/assets/images/quickstart/sir-lancelot.jpg")
+    image_bytes = image_path.read_bytes()
+
+    multimodal_table = db.create_table(
+        "character_images",
+        data=[
+            {
+                "id": "lancelot",
+                "description": "Portrait of Sir Lancelot",
+                "image": image_bytes,
+                "vector": [0.9, 0.1, 0.5, 0.4],
+            }
+        ],
+        mode="overwrite",
+    )
+    # --8<-- [end:quickstart_multimodal_bytes]
+    assert len(multimodal_table) == 1
+
+    # --8<-- [start:quickstart_open_table]
+    table = db.open_table("characters")
+    # --8<-- [end:quickstart_open_table]
+
+    # --8<-- [start:quickstart_add_data]
+    more_data = [
+        {
+            "id": "4",
+            "name": "Morgana",
+            "role": "Sorceress",
+            "description": "Powerful sorceress of Avalon.",
+            "stats": {"strength": 2, "magic": 5, "leadership": 4, "wisdom": 4},
+            "vector": [0.3, 0.9, 0.6, 0.8],
+            "power_score": 3.75,
+        },
+    ]
+
+    # Add data to table
+    table.add(more_data)
+    # --8<-- [end:quickstart_add_data]
+    assert len(table) == 4
+
+    # --8<-- [start:quickstart_vector_search_2]
+    # Search for examples similar to a "powerful sorceress"
+    query_vector = [0.3, 0.9, 0.6, 0.8]
+
+    results = table.search(query_vector).limit(2).to_polars()
+    print(results)
+    # --8<-- [end:quickstart_vector_search_2]
+    assert results.head(1)["name"][0] == "Morgana"
+
+
+@pytest.mark.asyncio
+async def test_quickstart_async_api(db_path_factory):
+    db_uri = db_path_factory("quickstart_async_db")
+    import lancedb
+    async_db = await lancedb.connect_async(db_uri)
+
+    # --8<-- [start:quickstart_data_async]
+    data = [
+        {
+            "id": "1",
+            "name": "King Arthur",
+            "role": "King",
+            "description": "Leader of Camelot and wielder of Excalibur.",
+            "stats": {"strength": 4, "magic": 1, "leadership": 5, "wisdom": 4},
+            "vector": [0.7, 0.1, 0.9, 0.7],
+        },
+        {
+            "id": "2",
+            "name": "Merlin",
+            "role": "Wizard",
+            "description": "Advisor and prophet with deep magical knowledge.",
+            "stats": {"strength": 2, "magic": 5, "leadership": 4, "wisdom": 5},
+            "vector": [0.2, 0.9, 0.4, 0.9],
+        },
+        {
+            "id": "3",
+            "name": "Sir Lancelot",
+            "role": "Knight",
+            "description": "Legendary knight known for courage and combat skill.",
+            "stats": {"strength": 5, "magic": 1, "leadership": 3, "wisdom": 3},
+            "vector": [0.9, 0.1, 0.5, 0.4],
+        },
+    ]
+    # --8<-- [end:quickstart_data_async]
+
+    # --8<-- [start:quickstart_create_table_async]
+    async_table = await async_db.create_table(
+        "characters",
+        data=data,
+        mode="overwrite",
+    )
+    # --8<-- [end:quickstart_create_table_async]
+
+    # --8<-- [start:quickstart_vector_search_1_async]
+    # Search for examples similar to a "wise magical advisor"
+    query_vector = [0.2, 0.8, 0.4, 0.9]
+
+    # Ensure you run `pip install polars` beforehand
+    async_result = await (
+        await async_table.search(query_vector)
+    ).select(["name", "role", "description", "_distance"]).limit(2).to_polars()
+    print(async_result)
+    # --8<-- [end:quickstart_vector_search_1_async]
+
+    assert async_result.head(1)["name"][0] == "Merlin"

docs/web-tests/py/test_search.py

@@ -0,0 +1,730 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+# --8<-- [start:import-lancedb]
+import lancedb
+
+# --8<-- [end:import-lancedb]
+# --8<-- [start:import-numpy]
+from lancedb.query import BoostQuery, MatchQuery
+import numpy as np
+import pyarrow as pa
+
+# --8<-- [end:import-numpy]
+# --8<-- [start:import-datetime]
+from datetime import datetime
+
+# --8<-- [end:import-datetime]
+# --8<-- [start:import-lancedb-pydantic]
+from lancedb.pydantic import Vector, LanceModel
+
+# --8<-- [end:import-lancedb-pydantic]
+# --8<-- [start:import-pydantic-base-model]
+from pydantic import BaseModel
+
+# --8<-- [end:import-pydantic-base-model]
+# --8<-- [start:import-lancedb-fts]
+from lancedb.index import FTS
+
+# --8<-- [end:import-lancedb-fts]
+# --8<-- [start:import-os]
+import os
+
+# --8<-- [end:import-os]
+# --8<-- [start:import-embeddings]
+from lancedb.embeddings import get_registry
+
+# --8<-- [end:import-embeddings]
+import pytest
+
+
+# --8<-- [start:class-definition]
+class Metadata(BaseModel):
+    source: str
+    timestamp: datetime
+
+
+class Document(BaseModel):
+    content: str
+    meta: Metadata
+
+
+class LanceSchema(LanceModel):
+    id: str
+    vector: Vector(1536)
+    payload: Document
+
+
+# --8<-- [end:class-definition]
+
+
+def test_vector_search():
+    # --8<-- [start:exhaustive_search]
+    uri = "data/sample-lancedb"
+    db = lancedb.connect(uri)
+    data = [
+        {"vector": row, "item": f"item {i}"}
+        for i, row in enumerate(np.random.random((10_000, 1536)).astype("float32"))
+    ]
+    tbl = db.create_table("vector_search", data=data, mode="overwrite")
+    tbl.search(np.random.random((1536))).limit(10).to_list()
+    # --8<-- [end:exhaustive_search]
+    # --8<-- [start:exhaustive_search_cosine]
+    tbl.search(np.random.random((1536))).distance_type("cosine").limit(10).to_list()
+    # --8<-- [end:exhaustive_search_cosine]
+    # --8<-- [start:create_table_with_nested_schema]
+    # Let's add 100 sample rows to our dataset
+    data = [
+        LanceSchema(
+            id=f"id{i}",
+            vector=np.random.randn(1536),
+            payload=Document(
+                content=f"document{i}",
+                meta=Metadata(source=f"source{i % 10}", timestamp=datetime.now()),
+            ),
+        )
+        for i in range(100)
+    ]
+
+    # Synchronous client
+    tbl = db.create_table("documents", data=data, mode="overwrite")
+    # --8<-- [end:create_table_with_nested_schema]
+    # --8<-- [start:search_result_as_pyarrow]
+    tbl.search(np.random.randn(1536)).to_arrow()
+    # --8<-- [end:search_result_as_pyarrow]
+    # --8<-- [start:search_result_as_pandas]
+    tbl.search(np.random.randn(1536)).to_pandas()
+    # --8<-- [end:search_result_as_pandas]
+    # --8<-- [start:search_result_as_pandas_flatten_true]
+    tbl.search(np.random.randn(1536)).to_pandas(flatten=True)
+    # --8<-- [end:search_result_as_pandas_flatten_true]
+    # --8<-- [start:search_result_as_pandas_flatten_1]
+    tbl.search(np.random.randn(1536)).to_pandas(flatten=1)
+    # --8<-- [end:search_result_as_pandas_flatten_1]
+    # --8<-- [start:search_result_as_list]
+    tbl.search(np.random.randn(1536)).to_list()
+    # --8<-- [end:search_result_as_list]
+    # --8<-- [start:search_result_as_pydantic]
+    tbl.search(np.random.randn(1536)).to_pydantic(LanceSchema)
+    # --8<-- [end:search_result_as_pydantic]
+
+
+@pytest.mark.asyncio
+async def test_vector_search_async():
+    # --8<-- [start:exhaustive_search_async]
+    uri = "data/sample-lancedb"
+    async_db = await lancedb.connect_async(uri)
+    data = [
+        {"vector": row, "item": f"item {i}"}
+        for i, row in enumerate(np.random.random((10_000, 1536)).astype("float32"))
+    ]
+    async_tbl = await async_db.create_table(
+        "vector_search_async", data=data, mode="overwrite"
+    )
+    (await (await async_tbl.search(np.random.random((1536)))).limit(10).to_list())
+    # --8<-- [end:exhaustive_search_async]
+    # --8<-- [start:exhaustive_search_async_cosine]
+    (
+        await (await async_tbl.search(np.random.random((1536))))
+        .distance_type("cosine")
+        .limit(10)
+        .to_list()
+    )
+    # --8<-- [end:exhaustive_search_async_cosine]
+    # --8<-- [start:create_table_async_with_nested_schema]
+    # Let's add 100 sample rows to our dataset
+    data = [
+        LanceSchema(
+            id=f"id{i}",
+            vector=np.random.randn(1536),
+            payload=Document(
+                content=f"document{i}",
+                meta=Metadata(source=f"source{i % 10}", timestamp=datetime.now()),
+            ),
+        )
+        for i in range(100)
+    ]
+
+    async_tbl = await async_db.create_table(
+        "documents_async", data=data, mode="overwrite"
+    )
+    # --8<-- [end:create_table_async_with_nested_schema]
+    # --8<-- [start:search_result_async_as_pyarrow]
+    await (await async_tbl.search(np.random.randn(1536))).to_arrow()
+    # --8<-- [end:search_result_async_as_pyarrow]
+    # --8<-- [start:search_result_async_as_pandas]
+    await (await async_tbl.search(np.random.randn(1536))).to_pandas()
+    # --8<-- [end:search_result_async_as_pandas]
+    # --8<-- [start:search_result_async_as_list]
+    await (await async_tbl.search(np.random.randn(1536))).to_list()
+    # --8<-- [end:search_result_async_as_list]
+
+
+def test_fts_fuzzy_query():
+    uri = "data/fuzzy-example"
+    db = lancedb.connect(uri)
+
+    table = db.create_table(
+        "my_table_fts_fuzzy",
+        data=pa.table(
+            {
+                "text": [
+                    "fa",
+                    "fo",  # spellchecker:disable-line
+                    "fob",
+                    "focus",
+                    "foo",
+                    "food",
+                    "foul",
+                ]
+            }
+        ),
+        mode="overwrite",
+    )
+    table.create_fts_index("text", replace=True)
+
+    results = table.search(MatchQuery("foo", "text", fuzziness=1)).to_pandas()
+    assert len(results) == 4
+    assert set(results["text"].to_list()) == {
+        "foo",
+        "fo",  # 1 deletion # spellchecker:disable-line
+        "fob",  # 1 substitution
+        "food",  # 1 insertion
+    }
+
+
+def test_fts_boost_query():
+    uri = "data/boost-example"
+    db = lancedb.connect(uri)
+
+    table = db.create_table(
+        "my_table_fts_boost",
+        data=pa.table(
+            {
+                "title": [
+                    "The Hidden Gems of Travel",
+                    "Exploring Nature's Wonders",
+                    "Cultural Treasures Unveiled",
+                    "The Nightlife Chronicles",
+                    "Scenic Escapes and Challenges",
+                ],
+                "desc": [
+                    "A vibrant city with occasional traffic jams.",
+                    "Beautiful landscapes but overpriced tourist spots.",
+                    "Rich cultural heritage but humid summers.",
+                    "Bustling nightlife but noisy streets.",
+                    "Scenic views but limited public transport options.",
+                ],
+            }
+        ),
+        mode="overwrite",
+    )
+    table.create_fts_index("desc", replace=True)
+
+    results = table.search(
+        BoostQuery(
+            MatchQuery("beautiful, cultural, nightlife", "desc"),
+            MatchQuery("bad traffic jams, overpriced", "desc"),
+        ),
+    ).to_pandas()
+
+    # we will hit 3 results because the positive query has 3 hits
+    assert len(results) == 3
+    # the one containing "overpriced" will be negatively boosted,
+    # so it will be the last one
+    assert (
+        results["desc"].to_list()[2]
+        == "Beautiful landscapes but overpriced tourist spots."
+    )
+
+
+def test_fts_native():
+    # --8<-- [start:basic_fts]
+    uri = "data/sample-lancedb"
+    db = lancedb.connect(uri)
+
+    table = db.create_table(
+        "my_table_fts",
+        data=[
+            {"vector": [3.1, 4.1], "text": "Frodo was a happy puppy"},
+            {"vector": [5.9, 26.5], "text": "There are several kittens playing"},
+        ],
+        mode="overwrite",
+    )
+
+    table.create_fts_index("text")
+    table.search("puppy").limit(10).select(["text"]).to_list()
+    # [{'text': 'Frodo was a happy puppy', '_score': 0.6931471824645996}]
+    # ...
+    # --8<-- [end:basic_fts]
+    # --8<-- [start:fts_config_stem]
+    table.create_fts_index("text", tokenizer_name="en_stem", replace=True)
+    # --8<-- [end:fts_config_stem]
+    # --8<-- [start:fts_config_folding]
+    table.create_fts_index(
+        "text",
+        language="French",
+        stem=True,
+        ascii_folding=True,
+        replace=True,
+    )
+    # --8<-- [end:fts_config_folding]
+    # --8<-- [start:fts_prefiltering]
+    table.search("puppy").limit(10).where("text='foo'", prefilter=True).to_list()
+    # --8<-- [end:fts_prefiltering]
+    # --8<-- [start:fts_postfiltering]
+    table.search("puppy").limit(10).where("text='foo'", prefilter=False).to_list()
+    # --8<-- [end:fts_postfiltering]
+    # --8<-- [start:fts_with_position]
+    table.create_fts_index("text", with_position=True, replace=True)
+    # --8<-- [end:fts_with_position]
+    # --8<-- [start:fts_incremental_index]
+    table.add([{"vector": [3.1, 4.1], "text": "Frodo was a happy puppy"}])
+    table.optimize()
+    # --8<-- [end:fts_incremental_index]
+
+
+@pytest.mark.asyncio
+async def test_fts_native_async():
+    # --8<-- [start:basic_fts_async]
+    uri = "data/sample-lancedb"
+    async_db = await lancedb.connect_async(uri)
+
+    async_tbl = await async_db.create_table(
+        "my_table_fts_async",
+        data=[
+            {"vector": [3.1, 4.1], "text": "Frodo was a happy puppy"},
+            {"vector": [5.9, 26.5], "text": "There are several kittens playing"},
+        ],
+        mode="overwrite",
+    )
+
+    # async API uses our native FTS algorithm
+    await async_tbl.create_index("text", config=FTS())
+    await (await async_tbl.search("puppy")).select(["text"]).limit(10).to_list()
+    # [{'text': 'Frodo was a happy puppy', '_score': 0.6931471824645996}]
+    # ...
+    # --8<-- [end:basic_fts_async]
+    # --8<-- [start:fts_config_stem_async]
+    await async_tbl.create_index(
+        "text", config=FTS(language="English", stem=True, remove_stop_words=True)
+    )
+    # --8<-- [end:fts_config_stem_async]
+    # --8<-- [start:fts_config_folding_async]
+    await async_tbl.create_index(
+        "text", config=FTS(language="French", stem=True, ascii_folding=True)
+    )
+    # --8<-- [end:fts_config_folding_async]
+    # --8<-- [start:fts_prefiltering_async]
+    await (await async_tbl.search("puppy")).limit(10).where("text='foo'").to_list()
+    # --8<-- [end:fts_prefiltering_async]
+    # --8<-- [start:fts_postfiltering_async]
+    await (
+        (await async_tbl.search("puppy"))
+        .limit(10)
+        .where("text='foo'")
+        .postfilter()
+        .to_list()
+    )
+    # --8<-- [end:fts_postfiltering_async]
+    # --8<-- [start:fts_with_position_async]
+    await async_tbl.create_index("text", config=FTS(with_position=True))
+    # --8<-- [end:fts_with_position_async]
+    # --8<-- [start:fts_incremental_index_async]
+    await async_tbl.add([{"vector": [3.1, 4.1], "text": "Frodo was a happy puppy"}])
+    await async_tbl.optimize()
+    # --8<-- [end:fts_incremental_index_async]
+
+
+def _vectors(n, dim, seed=0, column="vector"):
+    rng = np.random.default_rng(seed)
+    return [
+        {column: rng.random(dim).astype("float32"), "id": i} for i in range(n)
+    ]
+
+
+def _build_indexed_table(db, name, dim=128, n=512):
+    tbl = db.create_table(name, _vectors(n, dim), mode="overwrite")
+    tbl.create_index(metric="cosine", num_partitions=4, num_sub_vectors=8)
+    return tbl
+
+
+def test_vs_distance_metric_and_brute_force(tmp_db):
+    tbl = tmp_db.create_table("vs_metric", _vectors(64, 1536), mode="overwrite")
+
+    # --8<-- [start:configure_distance_metric]
+    tbl.search(np.random.random((1536))).distance_type("cosine").limit(10).to_list()
+    # --8<-- [end:configure_distance_metric]
+
+    # --8<-- [start:brute_force_search]
+    tbl.search(np.random.random((1536))).limit(3).to_list()
+    # --8<-- [end:brute_force_search]
+
+
+def test_vs_select_vector_column(tmp_db):
+    db = tmp_db
+
+    # --8<-- [start:select_vector_column]
+    import pyarrow as pa
+
+    schema = pa.schema([
+        pa.field("id", pa.int32()),
+        pa.field(
+            "image",
+            pa.struct([pa.field("embedding", pa.list_(pa.float32(), 2))]),
+        ),
+    ])
+    table = db.create_table(
+        "nested",
+        data=[{"id": 0, "image": {"embedding": [0.0, 1.0]}}],
+        schema=schema,
+    )
+
+    # Inferred: the only vector leaf is `image.embedding`.
+    table.search([0.0, 1.0]).limit(1).to_list()
+
+    # Explicit: required when more than one vector column matches.
+    table.search([0.0, 1.0], vector_column_name="image.embedding").limit(1).to_list()
+    # --8<-- [end:select_vector_column]
+
+
+def test_vs_index_nested_column(tmp_db):
+    dim = 16
+    schema = pa.schema(
+        [
+            pa.field("id", pa.int32()),
+            pa.field(
+                "image",
+                pa.struct([pa.field("embedding", pa.list_(pa.float32(), dim))]),
+            ),
+        ]
+    )
+    rng = np.random.default_rng(0)
+    data = [
+        {"id": i, "image": {"embedding": rng.random(dim).astype("float32").tolist()}}
+        for i in range(512)
+    ]
+    table = tmp_db.create_table(
+        "nested_index", data=data, schema=schema, mode="overwrite"
+    )
+
+    # --8<-- [start:index_nested_column]
+    table.create_index(vector_column_name="image.embedding")
+    # --8<-- [end:index_nested_column]
+
+    assert table.list_indices()
+
+
+def test_vs_indexed_queries(tmp_db):
+    table = _build_indexed_table(tmp_db, "vs_indexed", dim=128, n=512)
+    embedding = np.random.random(128)
+
+    # --8<-- [start:exact_vs_approximate_distances]
+    # Indexed ANN search without refinement (fast, approximate `_distance`)
+    fast_results = (
+        table.search(embedding)
+        .limit(10)
+        .to_pandas()
+    )
+
+    # Recompute distances on full vectors for reranked candidates
+    exact_distance_results = (
+        table.search(embedding)
+        .limit(10)
+        .refine_factor(1)
+        .to_pandas()
+    )
+
+    # Rerank a larger candidate set for better recall (higher latency)
+    higher_recall_results = (
+        table.search(embedding)
+        .limit(10)
+        .refine_factor(20)
+        .to_pandas()
+    )
+    # --8<-- [end:exact_vs_approximate_distances]
+
+    # --8<-- [start:bypass_vector_index]
+    table.search(embedding).bypass_vector_index().limit(5).to_pandas()
+    # --8<-- [end:bypass_vector_index]
+
+    assert len(fast_results) == 10
+    assert len(exact_distance_results) == 10
+    assert len(higher_recall_results) == 10
+
+
+def test_vs_fast_search(tmp_db, monkeypatch):
+    table = _build_indexed_table(tmp_db, "vs_fast", dim=128, n=512)
+    embedding = np.random.random(128)
+
+    # `fast_search` is an Enterprise/async query flag; strip it so the snippet
+    # runs against a local OSS table without changing what readers see.
+    real_search = table.search
+    monkeypatch.setattr(
+        table,
+        "search",
+        lambda *a, **k: real_search(
+            *a, **{key: val for key, val in k.items() if key != "fast_search"}
+        ),
+    )
+
+    # --8<-- [start:fast_search]
+    table.search(embedding, fast_search=True).limit(5).to_pandas()
+    # --8<-- [end:fast_search]
+
+
+def test_vs_distance_range(tmp_db):
+    tbl = tmp_db.create_table("vs_distance_range", _vectors(256, 256), mode="overwrite")
+
+    # --8<-- [start:search_distance_range]
+    query = np.random.random(256)
+
+    # Search for the vectors within the range of [0.1, 0.5)
+    tbl.search(query).distance_range(0.1, 0.5).to_arrow()
+
+    # Search for the vectors with the distance less than 0.5
+    tbl.search(query).distance_range(upper_bound=0.5).to_arrow()
+
+    # Search for the vectors with the distance greater or equal to 0.1
+    tbl.search(query).distance_range(lower_bound=0.1).to_arrow()
+    # --8<-- [end:search_distance_range]
+
+
+def test_vs_multivector_search(tmp_db):
+    schema = pa.schema(
+        [
+            pa.field("id", pa.int64()),
+            pa.field("vector", pa.list_(pa.list_(pa.float32(), 256))),
+        ]
+    )
+    rng = np.random.default_rng(0)
+    data = [{"id": i, "vector": rng.random(size=(2, 256)).tolist()} for i in range(64)]
+    tbl = tmp_db.create_table(
+        "vs_multivector", data=data, schema=schema, mode="overwrite"
+    )
+
+    # --8<-- [start:multivector_search]
+    query_multi = np.random.random(size=(2, 256))
+    results_multi = tbl.search(query_multi).limit(5).to_pandas()
+    # --8<-- [end:multivector_search]
+
+    assert len(results_multi) <= 5
+
+
+def test_vs_binary_search(tmp_db):
+    db = tmp_db
+
+    # --8<-- [start:search_binary_vectors]
+    import numpy as np
+    import pyarrow as pa
+
+    schema = pa.schema(
+        [
+            pa.field("id", pa.int64()),
+            # for dim=256, lance stores every 8 bits in a byte
+            # so the vector field should be a list of 256 / 8 = 32 bytes
+            pa.field("vector", pa.list_(pa.uint8(), 32)),
+        ]
+    )
+    tbl = db.create_table("my_binary_vectors", schema=schema)
+
+    data = []
+    for i in range(1024):
+        vector = np.random.randint(0, 2, size=256)
+        # pack the binary vector into bytes to save space
+        packed_vector = np.packbits(vector)
+        data.append(
+            {
+                "id": i,
+                "vector": packed_vector,
+            }
+        )
+    tbl.add(data)
+
+    query = np.random.randint(0, 2, size=256)
+    packed_query = np.packbits(query)
+    tbl.search(packed_query).distance_type("hamming").to_arrow()
+    # --8<-- [end:search_binary_vectors]
+
+
+def test_vs_enterprise_filtering(tmp_db, monkeypatch):
+    import sys
+    import types
+
+    dim = 384
+    rng = np.random.default_rng(0)
+    rows = [
+        {
+            "vector": rng.random(dim).astype("float32"),
+            "text": f"story {i}",
+            "keywords": f"kw{i}",
+            "label": i % 4,
+        }
+        for i in range(50)
+    ]
+    tmp_db.create_table("lancedb-enterprise-quickstart", data=rows, mode="overwrite")
+    db = tmp_db
+
+    # Mock the Hugging Face dataset loader to avoid network downloads.
+    class _FakeDataset:
+        def __init__(self, n):
+            self._emb = [rng.random(dim).astype("float32") for _ in range(n)]
+            self._kw = [f"kw{i}" for i in range(n)]
+
+        def __getitem__(self, key):
+            if isinstance(key, str):
+                return {"keywords_embeddings": self._emb, "keywords": self._kw}[key]
+            return {"keywords": self._kw[key], "keywords_embeddings": self._emb[key]}
+
+    fake_datasets = types.ModuleType("datasets")
+    fake_datasets.load_dataset = lambda *a, **k: _FakeDataset(10)
+    monkeypatch.setitem(sys.modules, "datasets", fake_datasets)
+
+    # --8<-- [start:vector_search_prefilter]
+    from datasets import load_dataset
+
+    # Load query vector from dataset
+    query_dataset = load_dataset("sunhaozhepy/ag_news_sbert_keywords_embeddings", split="test[5000:5001]")
+    print(f"Query keywords: {query_dataset[0]['keywords']}")
+    query_embed = query_dataset["keywords_embeddings"][0]
+
+    # Open table and perform search
+    table_name = "lancedb-enterprise-quickstart"
+    table = db.open_table(table_name)
+
+    # Vector search with filters (pre-filtering is the default)
+    search_results = (
+        table.search(query_embed)
+        .where("label > 2")
+        .select(["text", "keywords", "label"])
+        .limit(5)
+        .to_pandas()
+    )
+
+    print("Search results (with pre-filtering):")
+    print(search_results)
+    # --8<-- [end:vector_search_prefilter]
+
+    # --8<-- [start:vector_search_postfilter]
+    results_post_filtered = (
+        table.search(query_embed)
+        .where("label > 1", prefilter=False)
+        .select(["text", "keywords", "label"])
+        .limit(5)
+        .to_pandas()
+    )
+
+    print("Vector search results with post-filter:")
+    print(results_post_filtered)
+    # --8<-- [end:vector_search_postfilter]
+
+    # --8<-- [start:batch_search]
+    # Load a batch of query embeddings
+    query_dataset = load_dataset(
+        "sunhaozhepy/ag_news_sbert_keywords_embeddings", split="test[5000:5005]"
+    )
+    query_embeds = query_dataset["keywords_embeddings"]
+    batch_results = table.search(query_embeds).limit(5).to_pandas()
+    print(batch_results)
+    # --8<-- [end:batch_search]
+
+
+@pytest.mark.skip()
+def test_hybrid_search():
+    # --8<-- [start:import-openai]
+    import openai
+
+    # --8<-- [end:import-openai]
+    # --8<-- [start:openai-embeddings]
+    # Ingest embedding function in LanceDB table
+    # Configuring the environment variable OPENAI_API_KEY
+    if "OPENAI_API_KEY" not in os.environ:
+        # OR set the key here as a variable
+        openai.api_key = "sk-..."
+    embeddings = get_registry().get("openai").create()
+
+    # --8<-- [end:openai-embeddings]
+    # --8<-- [start:class-Documents]
+    class Documents(LanceModel):
+        vector: Vector(embeddings.ndims()) = embeddings.VectorField()
+        text: str = embeddings.SourceField()
+
+    # --8<-- [end:class-Documents]
+    # --8<-- [start:basic_hybrid_search]
+    data = [
+        {"text": "rebel spaceships striking from a hidden base"},
+        {"text": "have won their first victory against the evil Galactic Empire"},
+        {"text": "during the battle rebel spies managed to steal secret plans"},
+        {"text": "to the Empire's ultimate weapon the Death Star"},
+    ]
+    uri = "data/sample-lancedb"
+    db = lancedb.connect(uri)
+    table = db.create_table("documents", schema=Documents)
+    # ingest docs with auto-vectorization
+    table.add(data)
+    # Create a fts index before the hybrid search
+    table.create_fts_index("text")
+    # hybrid search with default re-ranker
+    table.search("flower moon", query_type="hybrid").to_pandas()
+    # --8<-- [end:basic_hybrid_search]
+    # --8<-- [start:hybrid_search_pass_vector_text]
+    vector_query = [0.1, 0.2, 0.3, 0.4, 0.5]
+    text_query = "flower moon"
+    (
+        table.search(query_type="hybrid")
+        .vector(vector_query)
+        .text(text_query)
+        .limit(5)
+        .to_pandas()
+    )
+    # --8<-- [end:hybrid_search_pass_vector_text]
+
+
+@pytest.mark.skip
+async def test_hybrid_search_async():
+    import openai
+
+    # --8<-- [start:openai-embeddings]
+    # Ingest embedding function in LanceDB table
+    # Configuring the environment variable OPENAI_API_KEY
+    if "OPENAI_API_KEY" not in os.environ:
+        # OR set the key here as a variable
+        openai.api_key = "sk-..."
+    embeddings = get_registry().get("openai").create()
+
+    # --8<-- [end:openai-embeddings]
+    # --8<-- [start:class-Documents]
+    class Documents(LanceModel):
+        vector: Vector(embeddings.ndims()) = embeddings.VectorField()
+        text: str = embeddings.SourceField()
+
+    # --8<-- [end:class-Documents]
+    # --8<-- [start:basic_hybrid_search_async]
+    uri = "data/sample-lancedb"
+    async_db = await lancedb.connect_async(uri)
+    data = [
+        {"text": "rebel spaceships striking from a hidden base"},
+        {"text": "have won their first victory against the evil Galactic Empire"},
+        {"text": "during the battle rebel spies managed to steal secret plans"},
+        {"text": "to the Empire's ultimate weapon the Death Star"},
+    ]
+    async_tbl = await async_db.create_table("documents_async", schema=Documents)
+    # ingest docs with auto-vectorization
+    await async_tbl.add(data)
+    # Create a fts index before the hybrid search
+    await async_tbl.create_index("text", config=FTS())
+    text_query = "flower moon"
+    # hybrid search with default re-ranker
+    await (await async_tbl.search("flower moon", query_type="hybrid")).to_pandas()
+    # --8<-- [end:basic_hybrid_search_async]
+    # --8<-- [start:hybrid_search_pass_vector_text_async]
+    vector_query = [0.1, 0.2, 0.3, 0.4, 0.5]
+    text_query = "flower moon"
+    await (
+        async_tbl.query()
+        .nearest_to(vector_query)
+        .nearest_to_text(text_query)
+        .limit(5)
+        .to_pandas()
+    )
+    # --8<-- [end:hybrid_search_pass_vector_text_async]

docs/web-tests/py/test_storage.py

@@ -0,0 +1,162 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+import lancedb
+import pytest
+
+
+class DummyTable:
+    def __init__(self, name: str, storage_options: dict | None = None):
+        self.name = name
+        self.storage_options = storage_options
+
+
+class DummyConnection:
+    def __init__(self, uri: str, options: dict):
+        self.uri = uri
+        self.options = options
+        self.created_tables: list[DummyTable] = []
+
+    def create_table(
+        self,
+        name: str,
+        data,
+        storage_options: dict | None = None,
+    ):
+        table = DummyTable(name, storage_options=storage_options)
+        self.created_tables.append(table)
+        return table
+
+
+@pytest.fixture
+def fake_connect(monkeypatch):
+    calls: list[DummyConnection] = []
+
+    def _fake_connect(uri: str, **kwargs):
+        conn = DummyConnection(uri, kwargs)
+        calls.append(conn)
+        return conn
+
+    monkeypatch.setattr(lancedb, "connect", _fake_connect)
+    return calls
+
+
+def test_storage_snippets(fake_connect):
+    # --8<-- [start:storage_connect_s3]
+    db = lancedb.connect("s3://bucket/path")
+    # --8<-- [end:storage_connect_s3]
+
+    # --8<-- [start:storage_connect_gcs]
+    db = lancedb.connect("gs://bucket/path")
+    # --8<-- [end:storage_connect_gcs]
+
+    # --8<-- [start:storage_connect_azure]
+    db = lancedb.connect("az://bucket/path")
+    # --8<-- [end:storage_connect_azure]
+
+    # --8<-- [start:storage_connect_timeout]
+    db = lancedb.connect(
+        "s3://bucket/path",
+        storage_options={"timeout": "60s"},
+    )
+    # --8<-- [end:storage_connect_timeout]
+
+    # --8<-- [start:storage_table_timeout]
+    table = db.create_table(
+        "table",
+        [{"a": 1, "b": 2}],
+        storage_options={"timeout": "60s"},
+    )
+    # --8<-- [end:storage_table_timeout]
+
+    # --8<-- [start:storage_s3_ddb]
+    db = lancedb.connect(
+        "s3+ddb://bucket/path?ddbTableName=my-dynamodb-table",
+    )
+    # --8<-- [end:storage_s3_ddb]
+
+    # --8<-- [start:storage_s3_ddb_local]
+    db = lancedb.connect(
+        "s3+ddb://bucket/path?ddbTableName=my-dynamodb-table",
+        storage_options={
+            "endpoint": "http://localhost:4566",
+            "dynamodb_endpoint": "http://localhost:4566",
+            "allow_http": "true",
+        },
+    )
+    # --8<-- [end:storage_s3_ddb_local]
+
+    # --8<-- [start:storage_s3_sse_kms]
+    db = lancedb.connect(
+        "s3://bucket/path",
+        storage_options={
+            "aws_server_side_encryption": "aws:kms",
+            "aws_sse_kms_key_id": "<your-kms-key-id-or-arn>",
+        },
+    )
+    # --8<-- [end:storage_s3_sse_kms]
+
+    # --8<-- [start:storage_azure_sas]
+    db = lancedb.connect(
+        "az://my-container/my-database",
+        storage_options={
+            "azure_storage_account_name": "some-account",
+            "azure_storage_sas_token": "<sas-token>",
+        },
+    )
+    # --8<-- [end:storage_azure_sas]
+
+    # --8<-- [start:storage_s3_minio]
+    db = lancedb.connect(
+        "s3://bucket/path",
+        storage_options={
+            "region": "us-east-1",
+            "endpoint": "http://minio:9000",
+        },
+    )
+    # --8<-- [end:storage_s3_minio]
+
+    # --8<-- [start:storage_s3_express]
+    db = lancedb.connect(
+        "s3://my-bucket--use1-az4--x-s3/path",
+        storage_options={
+            "region": "us-east-1",
+            "s3_express": "true",
+        },
+    )
+    # --8<-- [end:storage_s3_express]
+
+    # --8<-- [start:storage_gcs_service_account]
+    db = lancedb.connect(
+        "gs://my-bucket/my-database",
+        storage_options={
+            "service_account": "path/to/service-account.json",
+        },
+    )
+    # --8<-- [end:storage_gcs_service_account]
+
+    # --8<-- [start:storage_azure_account]
+    db = lancedb.connect(
+        "az://my-container/my-database",
+        storage_options={
+            "account_name": "some-account",
+            "account_key": "some-key",
+        },
+    )
+    # --8<-- [end:storage_azure_account]
+
+    # --8<-- [start:storage_tigris_connect]
+    db = lancedb.connect(
+        "s3://your-bucket/path",
+        storage_options={
+            "endpoint": "https://t3.storage.dev",
+            "region": "auto",
+        },
+    )
+    # --8<-- [end:storage_tigris_connect]
+
+    assert len(fake_connect) == 13
+    assert all(
+        conn.uri.startswith(("s3://", "gs://", "az://", "s3+ddb://"))
+        for conn in fake_connect
+    )

docs/web-tests/rs/Cargo.toml

@@ -0,0 +1,43 @@
+[package]
+name = "rs"
+version = "0.1.0"
+edition = "2024"
+
+[dependencies]
+arrow-array = "58.3.0"
+arrow-json = "58.3.0"
+arrow-schema = "58.3.0"
+futures = "0.3.31"
+futures-util = "0.3.31"
+lance-namespace = "7.0.0"
+lancedb = { version = "0.31.0", features = ["polars"] }
+polars = ">=0.37,<0.40.0"
+polars-arrow = ">=0.37,<0.40.0"
+serde = { version = "1", features = ["derive"] }
+serde_json = "1.0.145"
+tempfile = "3.23.0"
+tokio = { version = "1", features = ["macros", "rt-multi-thread"] }
+
+[[example]]
+name = "basic_usage"
+path = "basic_usage.rs"
+
+[[example]]
+name = "connection"
+path = "connection.rs"
+
+[[example]]
+name = "quickstart"
+path = "quickstart.rs"
+
+[[example]]
+name = "tables"
+path = "tables.rs"
+
+[[example]]
+name = "multimodal"
+path = "multimodal.rs"
+
+[[example]]
+name = "search"
+path = "search.rs"

docs/web-tests/rs/basic_usage.rs

@@ -0,0 +1,346 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+use std::fs;
+use std::path::PathBuf;
+use std::sync::Arc;
+
+use serde::Deserialize;
+
+type BatchIter = Box<dyn arrow_array::RecordBatchReader + Send>;
+
+// --8<-- [start:basic_imports]
+use arrow_array::types::Float32Type;
+use arrow_array::{
+    FixedSizeListArray, Int8Array, Int16Array, RecordBatch, RecordBatchIterator, StringArray,
+    StructArray,
+};
+use arrow_schema::{DataType, Field, FieldRef, Schema};
+use futures_util::TryStreamExt;
+use lancedb::database::CreateTableMode;
+use lancedb::query::{ExecutableQuery, QueryBase, Select};
+use lancedb::{connect, table::NewColumnTransform};
+// --8<-- [end:basic_imports]
+
+#[derive(Debug, Clone, Deserialize)]
+struct Stats {
+    strength: i8,
+    courage: i8,
+    magic: i8,
+    wisdom: i8,
+}
+
+#[derive(Debug, Clone, Deserialize)]
+struct Character {
+    id: i16,
+    name: String,
+    role: String,
+    description: String,
+    vector: [f32; 4],
+    stats: Stats,
+}
+
+fn camelot_schema() -> Arc<Schema> {
+    Arc::new(Schema::new(vec![
+        Field::new("id", DataType::Int16, false),
+        Field::new("name", DataType::Utf8, false),
+        Field::new("role", DataType::Utf8, false),
+        Field::new("description", DataType::Utf8, false),
+        Field::new(
+            "vector",
+            DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), 4),
+            false,
+        ),
+        Field::new(
+            "stats",
+            DataType::Struct(arrow_schema::Fields::from(vec![
+                Arc::new(Field::new("strength", DataType::Int8, false)),
+                Arc::new(Field::new("courage", DataType::Int8, false)),
+                Arc::new(Field::new("magic", DataType::Int8, false)),
+                Arc::new(Field::new("wisdom", DataType::Int8, false)),
+            ])),
+            false,
+        ),
+    ]))
+}
+
+fn characters_to_record_batch(schema: Arc<Schema>, characters: &[Character]) -> RecordBatch {
+    let ids = Int16Array::from_iter_values(characters.iter().map(|c| c.id));
+    let names = StringArray::from_iter_values(characters.iter().map(|c| c.name.as_str()));
+    let roles = StringArray::from_iter_values(characters.iter().map(|c| c.role.as_str()));
+    let descriptions =
+        StringArray::from_iter_values(characters.iter().map(|c| c.description.as_str()));
+
+    let vectors = FixedSizeListArray::from_iter_primitive::<Float32Type, _, _>(
+        characters
+            .iter()
+            .map(|c| Some(c.vector.iter().copied().map(Some).collect::<Vec<_>>())),
+        4,
+    );
+
+    let strength = Int8Array::from_iter_values(characters.iter().map(|c| c.stats.strength));
+    let courage = Int8Array::from_iter_values(characters.iter().map(|c| c.stats.courage));
+    let magic = Int8Array::from_iter_values(characters.iter().map(|c| c.stats.magic));
+    let wisdom = Int8Array::from_iter_values(characters.iter().map(|c| c.stats.wisdom));
+
+    let stats_fields: Vec<FieldRef> = vec![
+        Arc::new(Field::new("strength", DataType::Int8, false)),
+        Arc::new(Field::new("courage", DataType::Int8, false)),
+        Arc::new(Field::new("magic", DataType::Int8, false)),
+        Arc::new(Field::new("wisdom", DataType::Int8, false)),
+    ];
+    let stats = StructArray::new(
+        stats_fields.into(),
+        vec![
+            Arc::new(strength),
+            Arc::new(courage),
+            Arc::new(magic),
+            Arc::new(wisdom),
+        ],
+        None,
+    );
+
+    RecordBatch::try_new(
+        schema,
+        vec![
+            Arc::new(ids),
+            Arc::new(names),
+            Arc::new(roles),
+            Arc::new(descriptions),
+            Arc::new(vectors),
+            Arc::new(stats),
+        ],
+    )
+    .unwrap()
+}
+
+fn characters_to_reader(schema: Arc<Schema>, characters: &[Character]) -> BatchIter {
+    let batch = characters_to_record_batch(schema.clone(), characters);
+    Box::new(RecordBatchIterator::new(vec![Ok(batch)].into_iter(), schema))
+}
+
+fn camelot_json_path() -> PathBuf {
+    PathBuf::from(env!("CARGO_MANIFEST_DIR"))
+        .join("..")
+        .join("camelot.json")
+}
+
+#[tokio::main]
+async fn main() {
+    let temp_dir = tempfile::tempdir().unwrap();
+    let uri = temp_dir.path().to_str().unwrap();
+    let db = connect(uri).execute().await.unwrap();
+
+    // --8<-- [start:data_load]
+    let data: Vec<Character> =
+        serde_json::from_str(&fs::read_to_string(camelot_json_path()).unwrap()).unwrap();
+    // --8<-- [end:data_load]
+
+    let schema = camelot_schema();
+
+    // --8<-- [start:basic_create_table]
+    let mut table = db
+        .create_table("camelot", characters_to_reader(schema.clone(), &data))
+        .mode(CreateTableMode::Overwrite)
+        .execute()
+        .await
+        .unwrap();
+    // --8<-- [end:basic_create_table]
+    assert_eq!(table.count_rows(None).await.unwrap(), 8);
+
+    // --8<-- [start:basic_open_table]
+    table = db.open_table("camelot").execute().await.unwrap();
+    // --8<-- [end:basic_open_table]
+
+    // --8<-- [start:basic_create_empty_table]
+    let schema = Arc::new(Schema::new(vec![
+        Field::new("id", DataType::Int16, false),
+        Field::new("name", DataType::Utf8, false),
+        Field::new("role", DataType::Utf8, false),
+        Field::new("description", DataType::Utf8, false),
+        Field::new(
+            "vector",
+            DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), 4),
+            false,
+        ),
+        Field::new(
+            "stats",
+            DataType::Struct(arrow_schema::Fields::from(vec![
+                Arc::new(Field::new("strength", DataType::Int8, false)),
+                Arc::new(Field::new("courage", DataType::Int8, false)),
+                Arc::new(Field::new("magic", DataType::Int8, false)),
+                Arc::new(Field::new("wisdom", DataType::Int8, false)),
+            ])),
+            false,
+        ),
+    ]));
+    db.create_empty_table("camelot_empty", schema)
+        .mode(CreateTableMode::Overwrite)
+        .execute()
+        .await
+        .unwrap();
+    // --8<-- [end:basic_create_empty_table]
+    db.drop_table("camelot_empty", &[]).await.unwrap();
+
+    // --8<-- [start:basic_add_data]
+    let magical_characters = vec![
+        Character {
+            id: 9,
+            name: "Morgan le Fay".to_string(),
+            role: "Sorceress".to_string(),
+            description: "A powerful enchantress, Arthur's half-sister, and a complex figure who oscillates between aiding and opposing Camelot.".to_string(),
+            vector: [0.10, 0.84, 0.25, 0.70],
+            stats: Stats {
+                strength: 2,
+                courage: 3,
+                magic: 5,
+                wisdom: 4,
+            },
+        },
+        Character {
+            id: 10,
+            name: "The Lady of the Lake".to_string(),
+            role: "Mystical Guardian".to_string(),
+            description: "A mysterious supernatural figure associated with Avalon, known for giving Arthur the sword Excalibur.".to_string(),
+            vector: [0.00, 0.90, 0.58, 0.88],
+            stats: Stats {
+                strength: 2,
+                courage: 3,
+                magic: 5,
+                wisdom: 5,
+            },
+        },
+    ];
+    table
+        .add(characters_to_reader(camelot_schema(), &magical_characters))
+        .execute()
+        .await
+        .unwrap();
+    // --8<-- [end:basic_add_data]
+
+    // --8<-- [start:basic_vector_search]
+    let query_vector = [0.03, 0.85, 0.61, 0.90];
+    let result = table
+        .query()
+        .nearest_to(&query_vector)
+        .unwrap()
+        .limit(5)
+        .execute()
+        .await
+        .unwrap()
+        .try_collect::<Vec<_>>()
+        .await
+        .unwrap();
+    println!("{result:?}");
+    // --8<-- [end:basic_vector_search]
+
+    // --8<-- [start:basic_add_columns]
+    table
+        .add_columns(
+            NewColumnTransform::SqlExpressions(vec![(
+                "power".to_string(),
+                "cast(((stats.strength + stats.courage + stats.magic + stats.wisdom) / 4.0) as float)"
+                    .to_string(),
+            )]),
+            None,
+        )
+        .await
+        .unwrap();
+    // --8<-- [end:basic_add_columns]
+
+    // --8<-- [start:basic_vector_search_q1]
+    // Who are the characters similar to  "wizard"?
+    let query_vector_1 = [0.03, 0.85, 0.61, 0.90];
+    let r1 = table
+        .query()
+        .nearest_to(&query_vector_1)
+        .unwrap()
+        .limit(5)
+        .select(Select::Columns(vec![
+            "name".to_string(),
+            "role".to_string(),
+            "description".to_string(),
+        ]))
+        .execute()
+        .await
+        .unwrap()
+        .try_collect::<Vec<_>>()
+        .await
+        .unwrap();
+    println!("{r1:?}");
+    // --8<-- [end:basic_vector_search_q1]
+
+    // --8<-- [start:basic_vector_search_q2]
+    // Who are the characters similar to "wizard" with high magic stats?
+    let query_vector_2 = [0.03, 0.85, 0.61, 0.90];
+    let r2 = table
+        .query()
+        .nearest_to(&query_vector_2)
+        .unwrap()
+        .only_if("stats.magic > 3")
+        .select(Select::Columns(vec![
+            "name".to_string(),
+            "role".to_string(),
+            "description".to_string(),
+        ]))
+        .limit(5)
+        .execute()
+        .await
+        .unwrap()
+        .try_collect::<Vec<_>>()
+        .await
+        .unwrap();
+    println!("{r2:?}");
+    // --8<-- [end:basic_vector_search_q2]
+
+    // --8<-- [start:basic_vector_search_q3]
+    // Who are the strongest characters?
+    let r3 = table
+        .query()
+        .only_if("stats.strength > 3")
+        .select(Select::Columns(vec![
+            "name".to_string(),
+            "role".to_string(),
+            "description".to_string(),
+        ]))
+        .limit(5)
+        .execute()
+        .await
+        .unwrap()
+        .try_collect::<Vec<_>>()
+        .await
+        .unwrap();
+    println!("{r3:?}");
+    // --8<-- [end:basic_vector_search_q3]
+
+    // --8<-- [start:basic_vector_search_q4]
+    // Who are the strongest characters?
+    let r4 = table
+        .query()
+        .select(Select::Columns(vec![
+            "name".to_string(),
+            "role".to_string(),
+            "description".to_string(),
+            "power".to_string(),
+        ]))
+        .execute()
+        .await
+        .unwrap()
+        .try_collect::<Vec<_>>()
+        .await
+        .unwrap();
+    println!("{r4:?}");
+    // --8<-- [end:basic_vector_search_q4]
+
+    // --8<-- [start:basic_drop_columns]
+    table.drop_columns(&["power"]).await.unwrap();
+    // --8<-- [end:basic_drop_columns]
+
+    // --8<-- [start:basic_delete_rows]
+    table.delete("role = 'Traitor Knight'").await.unwrap();
+    // --8<-- [end:basic_delete_rows]
+
+    // --8<-- [start:basic_drop_table]
+    db.drop_table("camelot", &[]).await.unwrap();
+    // --8<-- [end:basic_drop_table]
+}

docs/web-tests/rs/bedrock.rs

@@ -0,0 +1,92 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+use std::{iter::once, sync::Arc};
+
+use arrow_array::{Float64Array, Int32Array, RecordBatch, RecordBatchIterator, StringArray};
+use arrow_schema::{DataType, Field, Schema};
+use aws_config::Region;
+use aws_sdk_bedrockruntime::Client;
+use futures::StreamExt;
+use lancedb::{
+    arrow::IntoArrow,
+    connect,
+    embeddings::{bedrock::BedrockEmbeddingFunction, EmbeddingDefinition, EmbeddingFunction},
+    query::{ExecutableQuery, QueryBase},
+    Result,
+};
+
+#[tokio::main]
+async fn main() -> Result<()> {
+    let tempdir = tempfile::tempdir().unwrap();
+    let tempdir = tempdir.path().to_str().unwrap();
+
+    // create Bedrock embedding function
+    let region: String = "us-east-1".to_string();
+    let config = aws_config::defaults(aws_config::BehaviorVersion::latest())
+        .region(Region::new(region))
+        .load()
+        .await;
+
+    let embedding = Arc::new(BedrockEmbeddingFunction::new(
+        Client::new(&config), // AWS Region
+    ));
+
+    let db = connect(tempdir).execute().await?;
+    db.embedding_registry()
+        .register("bedrock", embedding.clone())?;
+
+    let table = db
+        .create_table("vectors", make_data())
+        .add_embedding(EmbeddingDefinition::new(
+            "text",
+            "bedrock",
+            Some("embeddings"),
+        ))?
+        .execute()
+        .await?;
+
+    // execute vector search
+    let query = Arc::new(StringArray::from_iter_values(once("something warm")));
+    let query_vector = embedding.compute_query_embeddings(query)?;
+    let mut results = table
+        .vector_search(query_vector)?
+        .limit(1)
+        .execute()
+        .await?;
+
+    let rb = results.next().await.unwrap()?;
+    let out = rb
+        .column_by_name("text")
+        .unwrap()
+        .as_any()
+        .downcast_ref::<StringArray>()
+        .unwrap();
+    let text = out.iter().next().unwrap().unwrap();
+    println!("Closest match: {}", text);
+    Ok(())
+}
+
+fn make_data() -> impl IntoArrow {
+    let schema = Schema::new(vec![
+        Field::new("id", DataType::Int32, true),
+        Field::new("text", DataType::Utf8, false),
+        Field::new("price", DataType::Float64, false),
+    ]);
+
+    let id = Int32Array::from(vec![1, 2, 3, 4]);
+    let text = StringArray::from_iter_values(vec![
+        "Black T-Shirt",
+        "Leather Jacket",
+        "Winter Parka",
+        "Hooded Sweatshirt",
+    ]);
+    let price = Float64Array::from(vec![10.0, 50.0, 100.0, 30.0]);
+    let schema = Arc::new(schema);
+    let rb = RecordBatch::try_new(
+        schema.clone(),
+        vec![Arc::new(id), Arc::new(text), Arc::new(price)],
+    )
+    .unwrap();
+    Box::new(RecordBatchIterator::new(vec![Ok(rb)], schema))
+}

docs/web-tests/rs/connection.rs

@@ -0,0 +1,134 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+use std::path::PathBuf;
+
+use lancedb::connect;
+
+// --8<-- [start:connect]
+async fn connect_example(uri: &str) {
+    let db = connect(uri).execute().await.unwrap();
+    let _ = db;
+}
+// --8<-- [end:connect]
+
+#[tokio::main]
+async fn main() {
+    let temp_dir = tempfile::tempdir().unwrap();
+    let uri = temp_dir.path().join("ex_lancedb");
+    connect_example(uri.to_str().unwrap()).await;
+
+    // Keep the enterprise snippet in this file, but don't run it in CI.
+    let _ = connect_enterprise_quickstart_config();
+    let _ = connect_object_storage_config();
+}
+
+fn connect_enterprise_quickstart_config() -> (String, String, String, String) {
+    // --8<-- [start:connect_enterprise_quickstart]
+    let uri = "db://your-database-uri";
+    let api_key = "your-api-key";
+    let region = "us-east-1";
+    let host_override = "https://your-enterprise-endpoint.com";
+    // --8<-- [end:connect_enterprise_quickstart]
+    (
+        uri.to_string(),
+        api_key.to_string(),
+        region.to_string(),
+        host_override.to_string(),
+    )
+}
+
+fn connect_object_storage_config() -> &'static str {
+    // --8<-- [start:connect_object_storage]
+    let uri = "s3://your-bucket/path";
+    // You can also use "gs://your-bucket/path" or "az://your-container/path".
+    // --8<-- [end:connect_object_storage]
+
+    uri
+}
+
+async fn namespace_table_ops_example(uri: &str) -> lancedb::Result<()> {
+    // --8<-- [start:namespace_table_ops]
+    let conn = connect(uri).execute().await?;
+    let search_namespace = vec!["prod".to_string(), "search".to_string()];
+    let recommendations_namespace = vec!["prod".to_string(), "recommendations".to_string()];
+
+    let schema = std::sync::Arc::new(arrow_schema::Schema::new(vec![
+        arrow_schema::Field::new("id", arrow_schema::DataType::Int64, false),
+    ]));
+
+    conn.create_empty_table("user", schema.clone())
+        .namespace(search_namespace.clone())
+        .execute()
+        .await?;
+
+    conn.create_empty_table("user", schema)
+        .namespace(recommendations_namespace.clone())
+        .execute()
+        .await?;
+
+    let search_table_names = conn
+        .table_names()
+        .namespace(search_namespace)
+        .execute()
+        .await?;
+    let recommendation_table_names = conn
+        .table_names()
+        .namespace(recommendations_namespace)
+        .execute()
+        .await?;
+
+    println!("{search_table_names:?}"); // ["user"]
+    println!("{recommendation_table_names:?}"); // ["user"]
+    // --8<-- [end:namespace_table_ops]
+    Ok(())
+}
+
+async fn namespace_admin_ops_example() -> lancedb::Result<()> {
+    // --8<-- [start:namespace_admin_ops]
+    let mut properties = std::collections::HashMap::new();
+    properties.insert("root".to_string(), "./local_lancedb".to_string());
+    let db = lancedb::connect_namespace("dir", properties).execute().await?;
+    let namespace = vec!["prod".to_string(), "search".to_string()];
+
+    db.create_namespace(lance_namespace::models::CreateNamespaceRequest {
+        id: Some(vec!["prod".to_string()]),
+        ..Default::default()
+    })
+    .await?;
+    db.create_namespace(lance_namespace::models::CreateNamespaceRequest {
+        id: Some(namespace.clone()),
+        ..Default::default()
+    })
+    .await?;
+
+    let child_namespaces = db
+        .list_namespaces(lance_namespace::models::ListNamespacesRequest {
+            id: Some(vec!["prod".to_string()]),
+            ..Default::default()
+        })
+        .await?;
+    println!(
+        "Child namespaces under {:?}: {:?}",
+        namespace, child_namespaces
+    );
+    // Child namespaces under ["prod", "search"]: ["search"]
+
+    db.drop_namespace(lance_namespace::models::DropNamespaceRequest {
+        id: Some(namespace.clone()),
+        ..Default::default()
+    })
+    .await?;
+    db.drop_namespace(lance_namespace::models::DropNamespaceRequest {
+        id: Some(vec!["prod".to_string()]),
+        ..Default::default()
+    })
+    .await?;
+    // --8<-- [end:namespace_admin_ops]
+    Ok(())
+}
+
+#[allow(dead_code)]
+fn repo_root() -> PathBuf {
+    PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("..")
+}

docs/web-tests/rs/embedding.rs

@@ -0,0 +1,182 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+// --8<-- [start:openai_embeddings]
+use std::{iter::once, sync::Arc};
+
+use arrow_array::{record_batch, StringArray};
+use arrow_schema::{DataType, Field, Schema};
+use futures::StreamExt;
+use lancedb::{
+    connect,
+    embeddings::{openai::OpenAIEmbeddingFunction, EmbeddingDefinition, EmbeddingFunction},
+    query::{ExecutableQuery, QueryBase},
+    Result,
+};
+
+#[tokio::main]
+async fn main() -> Result<()> {
+    let db = connect("./mydb").execute().await?;
+    let api_key = std::env::var("OPENAI_API_KEY").expect("OPENAI_API_KEY is not set");
+    let embedding = Arc::new(OpenAIEmbeddingFunction::new_with_model(
+        api_key,
+        "text-embedding-3-large",
+    )?);
+
+    db.embedding_registry().register("openai", embedding.clone())?;
+
+    let schema = Arc::new(Schema::new(vec![Field::new("text", DataType::Utf8, false)]));
+    let table = db
+        .create_empty_table("mytable", schema)
+        .add_embedding(EmbeddingDefinition::new("text", "openai", Some("vector")))?
+        .execute()
+        .await?;
+
+    table
+        .add(record_batch!(("text", Utf8, ["This is a test.", "Another example."]))?)
+        .execute()
+        .await?;
+
+    let query = Arc::new(StringArray::from_iter_values(once("test example")));
+    let query_vector = embedding.compute_query_embeddings(query)?;
+    let mut results = table.vector_search(query_vector)?.limit(5).execute().await?;
+
+    while let Some(batch) = results.next().await {
+        println!("{:?}", batch?);
+    }
+
+    Ok(())
+}
+// --8<-- [end:openai_embeddings]
+
+// --8<-- [start:create_embedding_function]
+use std::sync::Arc;
+
+use lancedb::embeddings::openai::OpenAIEmbeddingFunction;
+
+let api_key = std::env::var("OPENAI_API_KEY").expect("OPENAI_API_KEY is not set");
+let embedding = Arc::new(
+    OpenAIEmbeddingFunction::new_with_model(api_key, "text-embedding-3-small")
+        .expect("failed to create OpenAI embedding function"),
+);
+// --8<-- [end:create_embedding_function]
+
+// --8<-- [start:manual_query_embeddings]
+use std::{iter::once, sync::Arc};
+
+use arrow_array::{record_batch, StringArray};
+use arrow_schema::{DataType, Field, Schema};
+use futures::StreamExt;
+use lancedb::{
+    connect,
+    embeddings::{openai::OpenAIEmbeddingFunction, EmbeddingDefinition, EmbeddingFunction},
+    query::{ExecutableQuery, QueryBase},
+    Result,
+};
+
+#[tokio::main]
+async fn main() -> Result<()> {
+    let db = connect("./mydb").execute().await?;
+    let api_key = std::env::var("OPENAI_API_KEY").expect("OPENAI_API_KEY is not set");
+    let embedding = Arc::new(OpenAIEmbeddingFunction::new_with_model(
+        api_key,
+        "text-embedding-3-large",
+    )?);
+    db.embedding_registry().register("openai", embedding.clone())?;
+
+    let schema = Arc::new(Schema::new(vec![Field::new("text", DataType::Utf8, false)]));
+    let table = db
+        .create_empty_table("mytable", schema)
+        .add_embedding(EmbeddingDefinition::new("text", "openai", Some("vector")))?
+        .execute()
+        .await?;
+
+    table
+        .add(record_batch!(("text", Utf8, ["This is a test.", "Another example."]))?)
+        .execute()
+        .await?;
+
+    // Manually generate embeddings for the query (Enterprise path)
+    let query = Arc::new(StringArray::from_iter_values(once("test example")));
+    let query_vector = embedding.compute_query_embeddings(query)?;
+    // --8<-- [start:manual_query_search]
+    // query_vector is assumed to already be generated by your embedding function
+    let mut results = table.vector_search(query_vector)?.limit(5).execute().await?;
+
+    while let Some(batch) = results.next().await {
+        println!("{:?}", batch?);
+    }
+    // --8<-- [end:manual_query_search]
+
+    Ok(())
+}
+// --8<-- [end:manual_query_embeddings]
+
+// --8<-- [start:embedding_function]
+use std::{borrow::Cow, sync::Arc};
+
+use arrow_array::{Array, FixedSizeListArray, Float32Array};
+use arrow_schema::{DataType, Field, Schema};
+use lancedb::{
+    connect,
+    embeddings::{EmbeddingDefinition, EmbeddingFunction},
+    Result,
+};
+
+#[derive(Debug, Clone)]
+struct MyTextEmbedder {
+    dim: usize,
+}
+
+impl EmbeddingFunction for MyTextEmbedder {
+    fn name(&self) -> &str {
+        "my-embedder"
+    }
+
+    fn source_type(&self) -> Result<Cow<'_, DataType>> {
+        Ok(Cow::Owned(DataType::Utf8))
+    }
+
+    fn dest_type(&self) -> Result<Cow<'_, DataType>> {
+        Ok(Cow::Owned(DataType::new_fixed_size_list(
+            DataType::Float32,
+            self.dim as i32,
+            true,
+        )))
+    }
+
+    fn compute_source_embeddings(&self, source: Arc<dyn Array>) -> Result<Arc<dyn Array>> {
+        let values = Arc::new(Float32Array::from(vec![1.0f32; source.len() * self.dim]));
+        let field = Arc::new(Field::new("item", DataType::Float32, true));
+        Ok(Arc::new(FixedSizeListArray::new(
+            field,
+            self.dim as i32,
+            values,
+            None,
+        )))
+    }
+
+    fn compute_query_embeddings(&self, _input: Arc<dyn Array>) -> Result<Arc<dyn Array>> {
+        unimplemented!()
+    }
+}
+
+#[tokio::main]
+async fn main() -> Result<()> {
+    let db = connect("./mydb").execute().await?;
+    db.embedding_registry()
+        .register("my-embedder", Arc::new(MyTextEmbedder { dim: 3 }))?;
+
+    let schema = Arc::new(Schema::new(vec![Field::new("text", DataType::Utf8, false)]));
+    db.create_empty_table("mytable", schema)
+        .add_embedding(EmbeddingDefinition::new(
+            "text",
+            "my-embedder",
+            Some("vector"),
+        ))?
+        .execute()
+        .await?;
+
+    Ok(())
+}
+// --8<-- [end:embedding_function]

docs/web-tests/rs/full_text_search.rs

@@ -0,0 +1,103 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+use std::sync::Arc;
+
+use arrow_array::{Int32Array, RecordBatch, RecordBatchIterator, RecordBatchReader, StringArray};
+use arrow_schema::{DataType, Field, Schema};
+
+use futures::TryStreamExt;
+use lance_index::scalar::FullTextSearchQuery;
+use lancedb::connection::Connection;
+use lancedb::index::scalar::FtsIndexBuilder;
+use lancedb::index::Index;
+use lancedb::query::{ExecutableQuery, QueryBase};
+use lancedb::{connect, Result, Table};
+use rand::random;
+
+#[tokio::main]
+async fn main() -> Result<()> {
+    if std::path::Path::new("data").exists() {
+        std::fs::remove_dir_all("data").unwrap();
+    }
+    let uri = "data/sample-lancedb";
+    let db = connect(uri).execute().await?;
+    let tbl = create_table(&db).await?;
+
+    create_index(&tbl).await?;
+    search_index(&tbl).await?;
+    Ok(())
+}
+
+fn create_some_records() -> Result<Box<dyn RecordBatchReader + Send>> {
+    const TOTAL: usize = 1000;
+
+    let schema = Arc::new(Schema::new(vec![
+        Field::new("id", DataType::Int32, false),
+        Field::new("doc", DataType::Utf8, true),
+    ]));
+
+    let words = random_word::all(random_word::Lang::En)
+        .iter()
+        .step_by(1024)
+        .take(500)
+        .copied()
+        .collect::<Vec<_>>();
+    let n_terms = 3;
+    let batches = RecordBatchIterator::new(
+        vec![RecordBatch::try_new(
+            schema.clone(),
+            vec![
+                Arc::new(Int32Array::from_iter_values(0..TOTAL as i32)),
+                Arc::new(StringArray::from_iter_values((0..TOTAL).map(|_| {
+                    (0..n_terms)
+                        .map(|_| words[random::<usize>() % words.len()])
+                        .collect::<Vec<_>>()
+                        .join(" ")
+                }))),
+            ],
+        )
+        .unwrap()]
+        .into_iter()
+        .map(Ok),
+        schema.clone(),
+    );
+    Ok(Box::new(batches))
+}
+
+async fn create_table(db: &Connection) -> Result<Table> {
+    let initial_data: Box<dyn RecordBatchReader + Send> = create_some_records()?;
+    let tbl = db.create_table("my_table", initial_data).execute().await?;
+    Ok(tbl)
+}
+
+async fn create_index(table: &Table) -> Result<()> {
+    table
+        .create_index(&["doc"], Index::FTS(FtsIndexBuilder::default()))
+        .execute()
+        .await?;
+    Ok(())
+}
+
+async fn search_index(table: &Table) -> Result<()> {
+    let words = random_word::all(random_word::Lang::En)
+        .iter()
+        .step_by(1024)
+        .take(500)
+        .copied()
+        .collect::<Vec<_>>();
+    let query = words[0].to_owned();
+    println!("Searching for: {}", query);
+
+    let mut results = table
+        .query()
+        .full_text_search(FullTextSearchQuery::new(words[0].to_owned()))
+        .select(lancedb::query::Select::Columns(vec!["doc".to_owned()]))
+        .limit(10)
+        .execute()
+        .await?;
+    while let Some(batch) = results.try_next().await? {
+        println!("{:?}", batch);
+    }
+    Ok(())
+}

docs/web-tests/rs/ivf_pq.rs

@@ -0,0 +1,154 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+//! This example demonstrates setting advanced parameters when building an IVF PQ index
+//!
+//! Snippets from this example are used in the documentation on ANN indices.
+
+use std::sync::Arc;
+
+use arrow_array::types::Float32Type;
+use arrow_array::{
+    FixedSizeListArray, Int32Array, RecordBatch, RecordBatchIterator, RecordBatchReader,
+};
+use arrow_schema::{DataType, Field, Schema};
+
+use futures::TryStreamExt;
+use lancedb::connection::Connection;
+use lancedb::index::vector::IvfPqIndexBuilder;
+use lancedb::index::Index;
+use lancedb::query::{ExecutableQuery, QueryBase};
+use lancedb::{connect, DistanceType, Result, Table};
+
+#[tokio::main]
+async fn main() -> Result<()> {
+    if std::path::Path::new("data").exists() {
+        std::fs::remove_dir_all("data").unwrap();
+    }
+    let uri = "data/sample-lancedb";
+    let db = connect(uri).execute().await?;
+    let tbl = create_table(&db).await?;
+
+    create_index(&tbl).await?;
+    search_index(&tbl).await?;
+    Ok(())
+}
+
+fn create_some_records() -> Result<Box<dyn RecordBatchReader + Send>> {
+    const TOTAL: usize = 1000;
+    const DIM: usize = 128;
+
+    let schema = Arc::new(Schema::new(vec![
+        Field::new("id", DataType::Int32, false),
+        Field::new(
+            "vector",
+            DataType::FixedSizeList(
+                Arc::new(Field::new("item", DataType::Float32, true)),
+                DIM as i32,
+            ),
+            true,
+        ),
+    ]));
+
+    // Create a RecordBatch stream.
+    let batches = RecordBatchIterator::new(
+        vec![RecordBatch::try_new(
+            schema.clone(),
+            vec![
+                Arc::new(Int32Array::from_iter_values(0..TOTAL as i32)),
+                Arc::new(
+                    FixedSizeListArray::from_iter_primitive::<Float32Type, _, _>(
+                        (0..TOTAL).map(|_| Some(vec![Some(1.0); DIM])),
+                        DIM as i32,
+                    ),
+                ),
+            ],
+        )
+        .unwrap()]
+        .into_iter()
+        .map(Ok),
+        schema.clone(),
+    );
+    Ok(Box::new(batches))
+}
+
+async fn create_table(db: &Connection) -> Result<Table> {
+    let initial_data: Box<dyn RecordBatchReader + Send> = create_some_records()?;
+    let tbl = db
+        .create_table("my_table", initial_data)
+        .execute()
+        .await
+        .unwrap();
+    Ok(tbl)
+}
+
+async fn create_index(table: &Table) -> Result<()> {
+    // --8<-- [start:create_index]
+    // For this example, `table` is a lancedb::Table with a column named
+    // "vector" that is a vector column with dimension 128.
+
+    // By default, if the column "vector" appears to be a vector column,
+    // then an IVF_PQ index with reasonable defaults is created.
+    table
+        .create_index(&["vector"], Index::Auto)
+        .execute()
+        .await?;
+    // For advanced cases, it is also possible to specifically request an
+    // IVF_PQ index and provide custom parameters.
+    table
+        .create_index(
+            &["vector"],
+            Index::IvfPq(
+                // Here we specify advanced indexing parameters.  In this case
+                // we are creating an index that my have better recall than the
+                // default but is also larger and slower.
+                IvfPqIndexBuilder::default()
+                    // This overrides the default distance type of l2
+                    .distance_type(DistanceType::Cosine)
+                    // With 1000 rows this have been ~31 by default
+                    .num_partitions(50)
+                    // With dimension 128 this would have been 8 by default
+                    .num_sub_vectors(16),
+            ),
+        )
+        .execute()
+        .await?;
+    // --8<-- [end:create_index]
+    Ok(())
+}
+
+async fn search_index(table: &Table) -> Result<()> {
+    // --8<-- [start:search1]
+    let query_vector = [1.0; 128];
+    // By default the index will find the 10 closest results using default
+    // search parameters that give a reasonable tradeoff between accuracy
+    // and search latency
+    let mut results = table
+        .vector_search(&query_vector)?
+        // Note: you should always set the distance_type to match the value used
+        // to train the index
+        .distance_type(DistanceType::Cosine)
+        .execute()
+        .await?;
+    while let Some(batch) = results.try_next().await? {
+        println!("{:?}", batch);
+    }
+    // We can also provide custom search parameters.  Here we perform a
+    // slower but more accurate search
+    let mut results = table
+        .vector_search(&query_vector)?
+        .distance_type(DistanceType::Cosine)
+        // Override the default of 10 to get more rows
+        .limit(15)
+        // Override the default of 20 to search more partitions
+        .nprobes(30)
+        // Override the default of None to apply a refine step
+        .refine_factor(1)
+        .execute()
+        .await?;
+    while let Some(batch) = results.try_next().await? {
+        println!("{:?}", batch);
+    }
+    Ok(())
+    // --8<-- [end:search1]
+}

docs/web-tests/rs/multimodal.rs

@@ -0,0 +1,178 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+// --8<-- [start:multimodal_imports]
+use std::collections::HashMap;
+use std::sync::Arc;
+
+use arrow_array::types::Float32Type;
+use arrow_array::{
+    BinaryArray, FixedSizeListArray, Int32Array, Int64Array, LargeBinaryArray, RecordBatch,
+    RecordBatchIterator, StringArray,
+};
+use arrow_schema::{DataType, Field, Schema};
+use futures_util::TryStreamExt;
+use lancedb::connect;
+use lancedb::database::CreateTableMode;
+use lancedb::query::{ExecutableQuery, QueryBase};
+// --8<-- [end:multimodal_imports]
+
+#[tokio::main]
+async fn main() {
+    let temp_dir = tempfile::tempdir().unwrap();
+    let db_uri = temp_dir.path().to_str().unwrap().to_string();
+    let db = connect(&db_uri).execute().await.unwrap();
+
+    // --8<-- [start:create_dummy_data]
+    let create_dummy_image = |color: u8| -> Vec<u8> {
+        let mut png_like = vec![137, 80, 78, 71, 13, 10, 26, 10];
+        png_like.push(color);
+        png_like
+    };
+
+    let data = vec![
+        (
+            1_i32,
+            "red_square.png",
+            vec![0.1_f32; 128],
+            create_dummy_image(1),
+            "red",
+        ),
+        (
+            2_i32,
+            "blue_square.png",
+            vec![0.2_f32; 128],
+            create_dummy_image(2),
+            "blue",
+        ),
+    ];
+    // --8<-- [end:create_dummy_data]
+
+    // --8<-- [start:define_schema]
+    let schema = Schema::new(vec![
+        Field::new("id", DataType::Int32, false),
+        Field::new("filename", DataType::Utf8, false),
+        Field::new(
+            "vector",
+            DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), 128),
+            false,
+        ),
+        Field::new("image_blob", DataType::Binary, false),
+        Field::new("label", DataType::Utf8, false),
+    ]);
+    // --8<-- [end:define_schema]
+
+    // --8<-- [start:ingest_data]
+    let schema = Arc::new(schema);
+    let image_batch = RecordBatch::try_new(
+        schema.clone(),
+        vec![
+            Arc::new(Int32Array::from_iter_values(data.iter().map(|row| row.0))),
+            Arc::new(StringArray::from_iter_values(data.iter().map(|row| row.1))),
+            Arc::new(
+                FixedSizeListArray::from_iter_primitive::<Float32Type, _, _>(
+                    data.iter()
+                        .map(|row| Some(row.2.iter().copied().map(Some).collect::<Vec<_>>())),
+                    128,
+                ),
+            ),
+            Arc::new(BinaryArray::from_iter_values(
+                data.iter().map(|row| row.3.as_slice()),
+            )),
+            Arc::new(StringArray::from_iter_values(data.iter().map(|row| row.4))),
+        ],
+    )
+    .unwrap();
+    let image_reader: Box<dyn arrow_array::RecordBatchReader + Send> =
+        Box::new(RecordBatchIterator::new(vec![Ok(image_batch)].into_iter(), schema.clone()));
+    let table = db
+        .create_table("images", image_reader)
+        .mode(CreateTableMode::Overwrite)
+        .execute()
+        .await
+        .unwrap();
+    // --8<-- [end:ingest_data]
+    assert_eq!(table.count_rows(None).await.unwrap(), 2);
+
+    // --8<-- [start:search_data]
+    let query_vector = vec![0.1_f32; 128];
+    let results = table
+        .query()
+        .nearest_to(query_vector)
+        .unwrap()
+        .limit(1)
+        .execute()
+        .await
+        .unwrap()
+        .try_collect::<Vec<_>>()
+        .await
+        .unwrap();
+    // --8<-- [end:search_data]
+
+    // --8<-- [start:process_results]
+    for batch in &results {
+        let filenames = batch
+            .column_by_name("filename")
+            .unwrap()
+            .as_any()
+            .downcast_ref::<StringArray>()
+            .unwrap();
+        let images = batch
+            .column_by_name("image_blob")
+            .unwrap()
+            .as_any()
+            .downcast_ref::<BinaryArray>()
+            .unwrap();
+
+        for row in 0..batch.num_rows() {
+            let image_bytes = images.value(row);
+            println!(
+                "Retrieved image: {}, Byte length: {}",
+                filenames.value(row),
+                image_bytes.len()
+            );
+        }
+    }
+    // --8<-- [end:process_results]
+    let search_rows: usize = results.iter().map(|batch| batch.num_rows()).sum();
+    assert_eq!(search_rows, 1);
+
+    // --8<-- [start:blob_api_schema]
+    let blob_metadata = HashMap::from([(
+        "lance-encoding:blob".to_string(),
+        "true".to_string(),
+    )]);
+    let blob_schema = Schema::new(vec![
+        Field::new("id", DataType::Int64, false),
+        Field::new("video", DataType::LargeBinary, true).with_metadata(blob_metadata),
+    ]);
+    // --8<-- [end:blob_api_schema]
+
+    // --8<-- [start:blob_api_ingest]
+    let blob_rows = vec![
+        (1_i64, b"fake_video_bytes_1".to_vec()),
+        (2_i64, b"fake_video_bytes_2".to_vec()),
+    ];
+
+    let blob_schema = Arc::new(blob_schema);
+    let blob_batch = RecordBatch::try_new(
+        blob_schema.clone(),
+        vec![
+            Arc::new(Int64Array::from_iter_values(blob_rows.iter().map(|row| row.0))),
+            Arc::new(LargeBinaryArray::from_iter_values(
+                blob_rows.iter().map(|row| row.1.as_slice()),
+            )),
+        ],
+    )
+    .unwrap();
+    let blob_reader: Box<dyn arrow_array::RecordBatchReader + Send> =
+        Box::new(RecordBatchIterator::new(vec![Ok(blob_batch)].into_iter(), blob_schema));
+    let blob_table = db
+        .create_table("videos", blob_reader)
+        .mode(CreateTableMode::Overwrite)
+        .execute()
+        .await
+        .unwrap();
+    // --8<-- [end:blob_api_ingest]
+    assert_eq!(blob_table.count_rows(None).await.unwrap(), 2);
+}

docs/web-tests/rs/openai.rs

@@ -0,0 +1,89 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+// --8<-- [start:imports]
+
+use std::{iter::once, sync::Arc};
+
+use arrow_array::{Float64Array, Int32Array, RecordBatch, RecordBatchIterator, StringArray};
+use arrow_schema::{DataType, Field, Schema};
+use futures::StreamExt;
+use lancedb::{
+    arrow::IntoArrow,
+    connect,
+    embeddings::{openai::OpenAIEmbeddingFunction, EmbeddingDefinition, EmbeddingFunction},
+    query::{ExecutableQuery, QueryBase},
+    Result,
+};
+
+// --8<-- [end:imports]
+
+// --8<-- [start:openai_embeddings]
+#[tokio::main]
+async fn main() -> Result<()> {
+    let tempdir = tempfile::tempdir().unwrap();
+    let tempdir = tempdir.path().to_str().unwrap();
+    let api_key = std::env::var("OPENAI_API_KEY").expect("OPENAI_API_KEY is not set");
+    let embedding = Arc::new(OpenAIEmbeddingFunction::new_with_model(
+        api_key,
+        "text-embedding-3-large",
+    )?);
+
+    let db = connect(tempdir).execute().await?;
+    db.embedding_registry()
+        .register("openai", embedding.clone())?;
+
+    let table = db
+        .create_table("vectors", make_data())
+        .add_embedding(EmbeddingDefinition::new(
+            "text",
+            "openai",
+            Some("embeddings"),
+        ))?
+        .execute()
+        .await?;
+
+    let query = Arc::new(StringArray::from_iter_values(once("something warm")));
+    let query_vector = embedding.compute_query_embeddings(query)?;
+    let mut results = table
+        .vector_search(query_vector)?
+        .limit(1)
+        .execute()
+        .await?;
+
+    let rb = results.next().await.unwrap()?;
+    let out = rb
+        .column_by_name("text")
+        .unwrap()
+        .as_any()
+        .downcast_ref::<StringArray>()
+        .unwrap();
+    let text = out.iter().next().unwrap().unwrap();
+    println!("Closest match: {}", text);
+    Ok(())
+}
+// --8<-- [end:openai_embeddings]
+
+fn make_data() -> impl IntoArrow {
+    let schema = Schema::new(vec![
+        Field::new("id", DataType::Int32, true),
+        Field::new("text", DataType::Utf8, false),
+        Field::new("price", DataType::Float64, false),
+    ]);
+
+    let id = Int32Array::from(vec![1, 2, 3, 4]);
+    let text = StringArray::from_iter_values(vec![
+        "Black T-Shirt",
+        "Leather Jacket",
+        "Winter Parka",
+        "Hooded Sweatshirt",
+    ]);
+    let price = Float64Array::from(vec![10.0, 50.0, 100.0, 30.0]);
+    let schema = Arc::new(schema);
+    let rb = RecordBatch::try_new(
+        schema.clone(),
+        vec![Arc::new(id), Arc::new(text), Arc::new(price)],
+    )
+    .unwrap();
+    Box::new(RecordBatchIterator::new(vec![Ok(rb)], schema))
+}

docs/web-tests/rs/quickstart.rs

@@ -0,0 +1,347 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+use std::sync::Arc;
+
+use arrow_array::types::Float32Type;
+use arrow_array::{
+    FixedSizeListArray, Int8Array, LargeStringArray, RecordBatch, RecordBatchIterator, StructArray,
+};
+use arrow_schema::{DataType, Field, FieldRef, Schema};
+use lancedb::arrow::IntoPolars;
+use lancedb::database::CreateTableMode;
+use lancedb::query::{ExecutableQuery, QueryBase, Select};
+use lancedb::{connect, table::NewColumnTransform};
+use polars::prelude::DataFrame;
+use serde::{Deserialize, Serialize};
+
+// --8<-- [start:quickstart_define_struct]
+// Define structs representing the data schema
+#[derive(Debug, Clone, Serialize, Deserialize)]
+struct Stats {
+    strength: i8,
+    magic: i8,
+    leadership: i8,
+    wisdom: i8,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+struct Character {
+    id: String,
+    name: String,
+    role: String,
+    description: String,
+    stats: Stats,
+    vector: [f32; 4],
+}
+
+fn characters_schema() -> Arc<Schema> {
+    Arc::new(Schema::new(vec![
+        Field::new("id", DataType::LargeUtf8, false),
+        Field::new("name", DataType::LargeUtf8, false),
+        Field::new("role", DataType::LargeUtf8, false),
+        Field::new("description", DataType::LargeUtf8, false),
+        Field::new(
+            "stats",
+            DataType::Struct(arrow_schema::Fields::from(vec![
+                Arc::new(Field::new("strength", DataType::Int8, false)),
+                Arc::new(Field::new("magic", DataType::Int8, false)),
+                Arc::new(Field::new("leadership", DataType::Int8, false)),
+                Arc::new(Field::new("wisdom", DataType::Int8, false)),
+            ])),
+            false,
+        ),
+        Field::new(
+            "vector",
+            DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), 4),
+            false,
+        ),
+    ]))
+}
+// --8<-- [end:quickstart_define_struct]
+
+type BatchIter = Box<dyn arrow_array::RecordBatchReader + Send>;
+
+fn characters_to_reader(schema: Arc<Schema>, rows: &[Character]) -> BatchIter {
+    let ids = LargeStringArray::from_iter_values(rows.iter().map(|row| row.id.as_str()));
+    let names = LargeStringArray::from_iter_values(rows.iter().map(|row| row.name.as_str()));
+    let roles = LargeStringArray::from_iter_values(rows.iter().map(|row| row.role.as_str()));
+    let descriptions =
+        LargeStringArray::from_iter_values(rows.iter().map(|row| row.description.as_str()));
+
+    let strength = Int8Array::from_iter_values(rows.iter().map(|row| row.stats.strength));
+    let magic = Int8Array::from_iter_values(rows.iter().map(|row| row.stats.magic));
+    let leadership = Int8Array::from_iter_values(rows.iter().map(|row| row.stats.leadership));
+    let wisdom = Int8Array::from_iter_values(rows.iter().map(|row| row.stats.wisdom));
+    let stats_fields: Vec<FieldRef> = vec![
+        Arc::new(Field::new("strength", DataType::Int8, false)),
+        Arc::new(Field::new("magic", DataType::Int8, false)),
+        Arc::new(Field::new("leadership", DataType::Int8, false)),
+        Arc::new(Field::new("wisdom", DataType::Int8, false)),
+    ];
+    let stats = StructArray::new(
+        stats_fields.into(),
+        vec![
+            Arc::new(strength),
+            Arc::new(magic),
+            Arc::new(leadership),
+            Arc::new(wisdom),
+        ],
+        None,
+    );
+
+    let vectors = FixedSizeListArray::from_iter_primitive::<Float32Type, _, _>(
+        rows.iter()
+            .map(|row| Some(row.vector.iter().copied().map(Some).collect::<Vec<_>>())),
+        4,
+    );
+
+    let batch = RecordBatch::try_new(
+        schema.clone(),
+        vec![
+            Arc::new(ids),
+            Arc::new(names),
+            Arc::new(roles),
+            Arc::new(descriptions),
+            Arc::new(stats),
+            Arc::new(vectors),
+        ],
+    )
+    .unwrap();
+
+    Box::new(RecordBatchIterator::new(
+        vec![Ok(batch)].into_iter(),
+        schema,
+    ))
+}
+
+#[tokio::main]
+async fn main() {
+    let temp_dir = tempfile::tempdir().unwrap();
+    let uri = temp_dir.path().to_str().unwrap();
+    let db = connect(uri).execute().await.unwrap();
+
+    // --8<-- [start:quickstart_data]
+    let data = vec![
+        Character {
+            id: "1".to_string(),
+            name: "King Arthur".to_string(),
+            role: "King".to_string(),
+            description: "Leader of Camelot and wielder of Excalibur.".to_string(),
+            stats: Stats {
+                strength: 4,
+                magic: 1,
+                leadership: 5,
+                wisdom: 4,
+            },
+            vector: [0.7, 0.1, 0.9, 0.7],
+        },
+        Character {
+            id: "2".to_string(),
+            name: "Merlin".to_string(),
+            role: "Wizard".to_string(),
+            description: "Advisor and prophet with deep magical knowledge.".to_string(),
+            stats: Stats {
+                strength: 2,
+                magic: 5,
+                leadership: 4,
+                wisdom: 5,
+            },
+            vector: [0.2, 0.9, 0.4, 0.9],
+        },
+        Character {
+            id: "3".to_string(),
+            name: "Sir Lancelot".to_string(),
+            role: "Knight".to_string(),
+            description: "Legendary knight known for courage and combat skill.".to_string(),
+            stats: Stats {
+                strength: 5,
+                magic: 1,
+                leadership: 3,
+                wisdom: 3,
+            },
+            vector: [0.9, 0.1, 0.5, 0.4],
+        },
+    ];
+    // --8<-- [end:quickstart_data]
+
+    // --8<-- [start:quickstart_create_table]
+    let schema = characters_schema();
+    let table = db
+        .create_table("characters", characters_to_reader(schema.clone(), &data))
+        .mode(CreateTableMode::Overwrite)
+        .execute()
+        .await
+        .unwrap();
+    // --8<-- [end:quickstart_create_table]
+    assert_eq!(table.count_rows(None).await.unwrap(), 3);
+    db.drop_table("characters", &[]).await.unwrap();
+
+    // --8<-- [start:quickstart_create_table_no_overwrite]
+    let table = db
+        .create_table("characters", characters_to_reader(schema.clone(), &data))
+        .execute()
+        .await
+        .unwrap();
+    // --8<-- [end:quickstart_create_table_no_overwrite]
+    assert_eq!(table.count_rows(None).await.unwrap(), 3);
+
+    // --8<-- [start:quickstart_vector_search_1]
+    // Search for examples similar to a "wise magical advisor"
+    let query_vector = [0.2, 0.8, 0.4, 0.9];
+
+    let result: DataFrame = table
+        .query()
+        .nearest_to(&query_vector)
+        .unwrap()
+        .select(Select::Columns(vec![
+            "name".to_string(),
+            "role".to_string(),
+            "description".to_string(),
+            "_distance".to_string(),
+        ]))
+        .limit(2)
+        .execute()
+        .await
+        .unwrap()
+        .into_polars()
+        .await
+        .unwrap();
+    println!("{result:?}");
+    // --8<-- [end:quickstart_vector_search_1]
+    let name_col = result.column("name").unwrap().str().unwrap();
+    assert_eq!(name_col.get(0).unwrap(), "Merlin");
+
+    // --8<-- [start:quickstart_curate_with_metadata]
+    let curated: DataFrame = table
+        .query()
+        .nearest_to(&query_vector)
+        .unwrap()
+        .only_if("stats.magic >= 4")
+        .select(Select::Columns(vec![
+            "name".to_string(),
+            "role".to_string(),
+            "description".to_string(),
+            "_distance".to_string(),
+        ]))
+        .limit(2)
+        .execute()
+        .await
+        .unwrap()
+        .into_polars()
+        .await
+        .unwrap();
+    println!("{curated:?}");
+    // --8<-- [end:quickstart_curate_with_metadata]
+    let curated_name_col = curated.column("name").unwrap().str().unwrap();
+    assert_eq!(curated_name_col.get(0).unwrap(), "Merlin");
+
+    // --8<-- [start:quickstart_output_array]
+    let result: DataFrame = table
+        .query()
+        .nearest_to(&query_vector)
+        .unwrap()
+        .select(Select::Columns(vec![
+            "name".to_string(),
+            "role".to_string(),
+            "description".to_string(),
+            "_distance".to_string(),
+        ]))
+        .limit(2)
+        .execute()
+        .await
+        .unwrap()
+        .into_polars()
+        .await
+        .unwrap();
+    println!("{result:?}");
+    // --8<-- [end:quickstart_output_array]
+    let name_col = result.column("name").unwrap().str().unwrap();
+    assert_eq!(name_col.get(0).unwrap(), "Merlin");
+
+    // --8<-- [start:quickstart_add_feature]
+    table
+        .add_columns(
+            NewColumnTransform::SqlExpressions(vec![(
+                "power_score".to_string(),
+                "cast(((stats.strength + stats.magic + stats.leadership + stats.wisdom) / 4.0) as float)"
+                    .to_string(),
+            )]),
+            None,
+        )
+        .await
+        .unwrap();
+    // --8<-- [end:quickstart_add_feature]
+
+    // --8<-- [start:quickstart_query_feature]
+    let features: DataFrame = table
+        .query()
+        .select(Select::Columns(vec![
+            "name".to_string(),
+            "role".to_string(),
+            "power_score".to_string(),
+        ]))
+        .execute()
+        .await
+        .unwrap()
+        .into_polars()
+        .await
+        .unwrap();
+    println!("{features:?}");
+    // --8<-- [end:quickstart_query_feature]
+    assert!(features.column("power_score").is_ok());
+
+    // --8<-- [start:quickstart_multimodal_bytes]
+    use std::sync::Arc;
+
+    use arrow_array::{
+        BinaryArray, FixedSizeListArray, LargeStringArray, RecordBatch, RecordBatchIterator,
+    };
+    use arrow_schema::{DataType, Field, Schema};
+
+    let image_path = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
+        .join("../../docs/static/assets/images/quickstart/sir-lancelot.jpg");
+    let image_bytes = std::fs::read(image_path).unwrap();
+
+    let image_schema = Arc::new(Schema::new(vec![
+        Field::new("id", DataType::LargeUtf8, false),
+        Field::new("description", DataType::LargeUtf8, false),
+        Field::new("image", DataType::Binary, false),
+        Field::new(
+            "vector",
+            DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), 4),
+            false,
+        ),
+    ]));
+    let image_vectors = [[0.9_f32, 0.1, 0.5, 0.4]];
+    let image_batch = RecordBatch::try_new(
+        image_schema.clone(),
+        vec![
+            Arc::new(LargeStringArray::from_iter_values(["lancelot"])),
+            Arc::new(LargeStringArray::from_iter_values([
+                "Portrait of Sir Lancelot",
+            ])),
+            Arc::new(BinaryArray::from_iter_values([image_bytes.as_slice()])),
+            Arc::new(
+                FixedSizeListArray::from_iter_primitive::<Float32Type, _, _>(
+                    image_vectors
+                        .iter()
+                        .map(|vector| Some(vector.iter().copied().map(Some).collect::<Vec<_>>())),
+                    4,
+                ),
+            ),
+        ],
+    )
+    .unwrap();
+    let image_reader: Box<dyn arrow_array::RecordBatchReader + Send> = Box::new(
+        RecordBatchIterator::new(vec![Ok(image_batch)].into_iter(), image_schema),
+    );
+    let multimodal_table = db
+        .create_table("character_images", image_reader)
+        .mode(CreateTableMode::Overwrite)
+        .execute()
+        .await
+        .unwrap();
+    // --8<-- [end:quickstart_multimodal_bytes]
+    assert_eq!(multimodal_table.count_rows(None).await.unwrap(), 1);
+}

docs/web-tests/rs/search.rs

@@ -0,0 +1,311 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+//! This example demonstrates vector search query options.
+//!
+//! Snippets from this example are used in the documentation on vector search.
+
+use std::sync::Arc;
+
+use arrow_array::types::Float32Type;
+use arrow_array::{
+    Array, FixedSizeListArray, Int32Array, Int64Array, RecordBatch, RecordBatchIterator,
+    RecordBatchReader, UInt8Array,
+};
+use arrow_schema::{DataType, Field, Schema};
+
+use futures::TryStreamExt;
+use lancedb::connection::Connection;
+use lancedb::index::vector::IvfFlatIndexBuilder;
+use lancedb::index::Index;
+use lancedb::query::{ExecutableQuery, QueryBase, Select};
+use lancedb::{connect, DistanceType, Result, Table};
+
+const DIM: usize = 128;
+
+#[tokio::main]
+async fn main() -> Result<()> {
+    if std::path::Path::new("data").exists() {
+        std::fs::remove_dir_all("data").unwrap();
+    }
+    let uri = "data/sample-lancedb";
+    let db = connect(uri).execute().await?;
+    let tbl = create_table(&db).await?;
+    tbl.create_index(&["vector"], Index::Auto).execute().await?;
+
+    configure_distance_metric(&tbl).await?;
+    exact_vs_approximate(&tbl).await?;
+    search_distance_range(&tbl).await?;
+    vector_search_prefilter(&tbl).await?;
+    vector_search_postfilter(&tbl).await?;
+    fast_search(&tbl).await?;
+    brute_force_search(&tbl).await?;
+    bypass_vector_index(&tbl).await?;
+    batch_search(&tbl).await?;
+    binary_search(&db).await?;
+    Ok(())
+}
+
+fn create_some_records() -> Result<Box<dyn RecordBatchReader + Send>> {
+    const TOTAL: usize = 1000;
+
+    let schema = Arc::new(Schema::new(vec![
+        Field::new("id", DataType::Int32, false),
+        Field::new(
+            "vector",
+            DataType::FixedSizeList(
+                Arc::new(Field::new("item", DataType::Float32, true)),
+                DIM as i32,
+            ),
+            true,
+        ),
+    ]));
+
+    let batches = RecordBatchIterator::new(
+        vec![RecordBatch::try_new(
+            schema.clone(),
+            vec![
+                Arc::new(Int32Array::from_iter_values(0..TOTAL as i32)),
+                Arc::new(
+                    FixedSizeListArray::from_iter_primitive::<Float32Type, _, _>(
+                        (0..TOTAL).map(|_| Some(vec![Some(1.0); DIM])),
+                        DIM as i32,
+                    ),
+                ),
+            ],
+        )
+        .unwrap()]
+        .into_iter()
+        .map(Ok),
+        schema.clone(),
+    );
+    Ok(Box::new(batches))
+}
+
+async fn create_table(db: &Connection) -> Result<Table> {
+    let initial_data: Box<dyn RecordBatchReader + Send> = create_some_records()?;
+    let tbl = db
+        .create_table("my_vectors", initial_data)
+        .execute()
+        .await
+        .unwrap();
+    Ok(tbl)
+}
+
+async fn configure_distance_metric(table: &Table) -> Result<()> {
+    let query_vector = [1.0; DIM];
+    // --8<-- [start:configure_distance_metric]
+    // Use the same distance metric the index was trained with.
+    let mut results = table
+        .vector_search(&query_vector)?
+        .distance_type(DistanceType::Cosine)
+        .limit(10)
+        .execute()
+        .await?;
+    while let Some(batch) = results.try_next().await? {
+        println!("{:?}", batch);
+    }
+    // --8<-- [end:configure_distance_metric]
+    Ok(())
+}
+
+async fn exact_vs_approximate(table: &Table) -> Result<()> {
+    let query_vector = [1.0; DIM];
+    // --8<-- [start:exact_vs_approximate]
+    // Approximate ANN search (fast, distances may come from the index representation)
+    let mut fast_results = table.vector_search(&query_vector)?.limit(10).execute().await?;
+    while let Some(batch) = fast_results.try_next().await? {
+        println!("{:?}", batch);
+    }
+
+    // Rerank a larger candidate set on full vectors for better recall
+    let mut refined_results = table
+        .vector_search(&query_vector)?
+        .limit(10)
+        .refine_factor(20)
+        .execute()
+        .await?;
+    while let Some(batch) = refined_results.try_next().await? {
+        println!("{:?}", batch);
+    }
+    // --8<-- [end:exact_vs_approximate]
+    Ok(())
+}
+
+async fn search_distance_range(table: &Table) -> Result<()> {
+    let query_vector = [1.0; DIM];
+    // --8<-- [start:search_distance_range]
+    // Only return rows whose distance falls within [0.1, 0.5).
+    let mut results = table
+        .vector_search(&query_vector)?
+        .distance_range(Some(0.1), Some(0.5))
+        .execute()
+        .await?;
+    while let Some(batch) = results.try_next().await? {
+        println!("{:?}", batch);
+    }
+    // --8<-- [end:search_distance_range]
+    Ok(())
+}
+
+async fn vector_search_prefilter(table: &Table) -> Result<()> {
+    let query_vector = [1.0; DIM];
+    // --8<-- [start:vector_search_prefilter]
+    // Prefiltering is the default: the filter is applied before vector search.
+    let mut results = table
+        .vector_search(&query_vector)?
+        .only_if("id > 100")
+        .select(Select::columns(&["id"]))
+        .limit(5)
+        .execute()
+        .await?;
+    while let Some(batch) = results.try_next().await? {
+        println!("{:?}", batch);
+    }
+    // --8<-- [end:vector_search_prefilter]
+    Ok(())
+}
+
+async fn vector_search_postfilter(table: &Table) -> Result<()> {
+    let query_vector = [1.0; DIM];
+    // --8<-- [start:vector_search_postfilter]
+    // Apply the filter after vector search by calling postfilter().
+    let mut results = table
+        .vector_search(&query_vector)?
+        .only_if("id > 100")
+        .postfilter()
+        .select(Select::columns(&["id"]))
+        .limit(5)
+        .execute()
+        .await?;
+    while let Some(batch) = results.try_next().await? {
+        println!("{:?}", batch);
+    }
+    // --8<-- [end:vector_search_postfilter]
+    Ok(())
+}
+
+async fn fast_search(table: &Table) -> Result<()> {
+    let query_vector = [1.0; DIM];
+    // --8<-- [start:fast_search]
+    // Skip unindexed data for lower latency.
+    let mut results = table
+        .vector_search(&query_vector)?
+        .fast_search()
+        .limit(5)
+        .execute()
+        .await?;
+    while let Some(batch) = results.try_next().await? {
+        println!("{:?}", batch);
+    }
+    // --8<-- [end:fast_search]
+    Ok(())
+}
+
+async fn brute_force_search(table: &Table) -> Result<()> {
+    let query_vector = [1.0; DIM];
+    // --8<-- [start:brute_force_search]
+    // A plain vector search returns the top-k closest rows.
+    let mut results = table.vector_search(&query_vector)?.limit(3).execute().await?;
+    while let Some(batch) = results.try_next().await? {
+        println!("{:?}", batch);
+    }
+    // --8<-- [end:brute_force_search]
+    Ok(())
+}
+
+async fn bypass_vector_index(table: &Table) -> Result<()> {
+    let query_vector = [1.0; DIM];
+    // --8<-- [start:bypass_vector_index]
+    // Force an exhaustive (flat) scan for exact, ground-truth results.
+    let mut results = table
+        .vector_search(&query_vector)?
+        .bypass_vector_index()
+        .limit(5)
+        .execute()
+        .await?;
+    while let Some(batch) = results.try_next().await? {
+        println!("{:?}", batch);
+    }
+    // --8<-- [end:bypass_vector_index]
+    Ok(())
+}
+
+async fn batch_search(table: &Table) -> Result<()> {
+    let query_1 = [1.0; DIM];
+    let query_2 = [0.5; DIM];
+    // --8<-- [start:batch_search]
+    // Search multiple query vectors in one call. Each result row carries a
+    // `query_index` mapping it back to the query it matched.
+    let mut results = table
+        .vector_search(&query_1)?
+        .add_query_vector(&query_2)?
+        .limit(5)
+        .execute()
+        .await?;
+    while let Some(batch) = results.try_next().await? {
+        println!("{:?}", batch);
+    }
+    // --8<-- [end:batch_search]
+    Ok(())
+}
+
+async fn binary_search(db: &Connection) -> Result<()> {
+    // A 256-bit binary vector is stored as 256 / 8 = 32 packed uint8 bytes.
+    const NUM_BYTES: i32 = 32;
+    const TOTAL: usize = 1024;
+
+    let schema = Arc::new(Schema::new(vec![
+        Field::new("id", DataType::Int64, false),
+        Field::new(
+            "vector",
+            DataType::FixedSizeList(
+                Arc::new(Field::new("item", DataType::UInt8, true)),
+                NUM_BYTES,
+            ),
+            true,
+        ),
+    ]));
+
+    let values =
+        UInt8Array::from_iter_values((0..TOTAL * NUM_BYTES as usize).map(|i| (i % 256) as u8));
+    let vectors = FixedSizeListArray::try_new(
+        Arc::new(Field::new("item", DataType::UInt8, true)),
+        NUM_BYTES,
+        Arc::new(values),
+        None,
+    )
+    .unwrap();
+    let batch = RecordBatch::try_new(
+        schema.clone(),
+        vec![
+            Arc::new(Int64Array::from_iter_values(0..TOTAL as i64)),
+            Arc::new(vectors),
+        ],
+    )
+    .unwrap();
+    let reader: Box<dyn RecordBatchReader + Send> =
+        Box::new(RecordBatchIterator::new(vec![Ok(batch)].into_iter(), schema.clone()));
+    let tbl = db.create_table("binary_vectors", reader).execute().await?;
+    tbl.create_index(
+        &["vector"],
+        Index::IvfFlat(IvfFlatIndexBuilder::default().distance_type(DistanceType::Hamming)),
+    )
+    .execute()
+    .await?;
+
+    // --8<-- [start:binary_search]
+    // Binary vectors use `hamming` distance over the packed uint8 bytes.
+    let query: Arc<dyn Array> = Arc::new(UInt8Array::from(vec![1u8; NUM_BYTES as usize]));
+    let mut results = tbl
+        .vector_search(query)?
+        .distance_type(DistanceType::Hamming)
+        .limit(10)
+        .execute()
+        .await?;
+    while let Some(batch) = results.try_next().await? {
+        println!("{:?}", batch);
+    }
+    // --8<-- [end:binary_search]
+    Ok(())
+}

docs/web-tests/rs/sentence_transformers.rs

@@ -0,0 +1,95 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+use std::{iter::once, sync::Arc};
+
+use arrow_array::{RecordBatch, RecordBatchIterator, StringArray};
+use arrow_schema::{DataType, Field, Schema};
+use futures::StreamExt;
+use lancedb::{
+    arrow::IntoArrow,
+    connect,
+    embeddings::{
+        sentence_transformers::SentenceTransformersEmbeddings, EmbeddingDefinition,
+        EmbeddingFunction,
+    },
+    query::{ExecutableQuery, QueryBase},
+    Result,
+};
+
+#[tokio::main]
+async fn main() -> Result<()> {
+    let tempdir = tempfile::tempdir().unwrap();
+    let tempdir = tempdir.path().to_str().unwrap();
+    let embedding = SentenceTransformersEmbeddings::builder().build()?;
+    let embedding = Arc::new(embedding);
+    let db = connect(tempdir).execute().await?;
+    db.embedding_registry()
+        .register("sentence-transformers", embedding.clone())?;
+
+    let table = db
+        .create_table("vectors", make_data())
+        .add_embedding(EmbeddingDefinition::new(
+            "facts",
+            "sentence-transformers",
+            Some("embeddings"),
+        ))?
+        .execute()
+        .await?;
+
+    let query = Arc::new(StringArray::from_iter_values(once(
+        "How many bones are in the human body?",
+    )));
+    let query_vector = embedding.compute_query_embeddings(query)?;
+    let mut results = table
+        .vector_search(query_vector)?
+        .limit(1)
+        .execute()
+        .await?;
+
+    let rb = results.next().await.unwrap()?;
+    let out = rb
+        .column_by_name("facts")
+        .unwrap()
+        .as_any()
+        .downcast_ref::<StringArray>()
+        .unwrap();
+    let text = out.iter().next().unwrap().unwrap();
+    println!("Answer: {}", text);
+    Ok(())
+}
+
+fn make_data() -> impl IntoArrow {
+    let schema = Schema::new(vec![Field::new("facts", DataType::Utf8, false)]);
+
+    let facts = StringArray::from_iter_values(vec![
+        "Albert Einstein was a theoretical physicist.",
+        "The capital of France is Paris.",
+        "The Great Wall of China is one of the Seven Wonders of the World.",
+        "Python is a popular programming language.",
+        "Mount Everest is the highest mountain in the world.",
+        "Leonardo da Vinci painted the Mona Lisa.",
+        "Shakespeare wrote Hamlet.",
+        "The human body has 206 bones.",
+        "The speed of light is approximately 299,792 kilometers per second.",
+        "Water boils at 100 degrees Celsius.",
+        "The Earth orbits the Sun.",
+        "The Pyramids of Giza are located in Egypt.",
+        "Coffee is one of the most popular beverages in the world.",
+        "Tokyo is the capital city of Japan.",
+        "Photosynthesis is the process by which plants make their food.",
+        "The Pacific Ocean is the largest ocean on Earth.",
+        "Mozart was a prolific composer of classical music.",
+        "The Internet is a global network of computers.",
+        "Basketball is a sport played with a ball and a hoop.",
+        "The first computer virus was created in 1983.",
+        "Artificial neural networks are inspired by the human brain.",
+        "Deep learning is a subset of machine learning.",
+        "IBM's Watson won Jeopardy! in 2011.",
+        "The first computer programmer was Ada Lovelace.",
+        "The first chatbot was ELIZA, created in the 1960s.",
+    ]);
+    let schema = Arc::new(schema);
+    let rb = RecordBatch::try_new(schema.clone(), vec![Arc::new(facts)]).unwrap();
+    Box::new(RecordBatchIterator::new(vec![Ok(rb)], schema))
+}

docs/web-tests/rs/simple.rs

@@ -0,0 +1,147 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+//! This example demonstrates basic usage of LanceDb.
+//!
+//! Snippets from this example are used in the quickstart documentation.
+
+use std::sync::Arc;
+
+use arrow_array::types::Float32Type;
+use arrow_array::{FixedSizeListArray, Int32Array, RecordBatch, RecordBatchIterator};
+use arrow_schema::{DataType, Field, Schema};
+use futures::TryStreamExt;
+
+use lancedb::arrow::IntoArrow;
+use lancedb::connection::Connection;
+use lancedb::index::Index;
+use lancedb::query::{ExecutableQuery, QueryBase};
+use lancedb::{connect, Result, Table as LanceDbTable};
+
+// --8<-- [start:connect]
+#[tokio::main]
+async fn main() -> Result<()> {
+    if std::path::Path::new("data").exists() {
+        std::fs::remove_dir_all("data").unwrap();
+    }
+    // --8<-- [start:connect_uri]
+    let uri = "data/sample-lancedb";
+    let db = connect(uri).execute().await?;
+    // --8<-- [end:connect_uri]
+
+    // --8<-- [start:list_names]
+    println!("{:?}", db.table_names().execute().await?);
+    // --8<-- [end:list_names]
+    let tbl = create_table(&db).await?;
+    create_index(&tbl).await?;
+    let batches = search(&tbl).await?;
+    println!("{:?}", batches);
+
+    create_empty_table(&db).await.unwrap();
+
+    // --8<-- [start:delete]
+    tbl.delete("id > 24").await.unwrap();
+    // --8<-- [end:delete]
+
+    // --8<-- [start:drop_table]
+    db.drop_table("my_table").await.unwrap();
+    // --8<-- [end:drop_table]
+    Ok(())
+}
+// --8<-- [end:connect]
+
+#[allow(dead_code)]
+async fn open_with_existing_tbl() -> Result<()> {
+    let uri = "data/sample-lancedb";
+    let db = connect(uri).execute().await?;
+    #[allow(unused_variables)]
+    // --8<-- [start:open_existing_tbl]
+    let table = db.open_table("my_table").execute().await.unwrap();
+    // --8<-- [end:open_existing_tbl]
+    Ok(())
+}
+
+fn create_some_records() -> Result<impl IntoArrow> {
+    const TOTAL: usize = 1000;
+    const DIM: usize = 128;
+
+    let schema = Arc::new(Schema::new(vec![
+        Field::new("id", DataType::Int32, false),
+        Field::new(
+            "vector",
+            DataType::FixedSizeList(
+                Arc::new(Field::new("item", DataType::Float32, true)),
+                DIM as i32,
+            ),
+            true,
+        ),
+    ]));
+
+    // Create a RecordBatch stream.
+    let batches = RecordBatchIterator::new(
+        vec![RecordBatch::try_new(
+            schema.clone(),
+            vec![
+                Arc::new(Int32Array::from_iter_values(0..TOTAL as i32)),
+                Arc::new(
+                    FixedSizeListArray::from_iter_primitive::<Float32Type, _, _>(
+                        (0..TOTAL).map(|_| Some(vec![Some(1.0); DIM])),
+                        DIM as i32,
+                    ),
+                ),
+            ],
+        )
+        .unwrap()]
+        .into_iter()
+        .map(Ok),
+        schema.clone(),
+    );
+    Ok(Box::new(batches))
+}
+
+async fn create_table(db: &Connection) -> Result<LanceDbTable> {
+    // --8<-- [start:create_table]
+    let initial_data = create_some_records()?;
+    let tbl = db
+        .create_table("my_table", initial_data)
+        .execute()
+        .await
+        .unwrap();
+    // --8<-- [end:create_table]
+
+    // --8<-- [start:add]
+    let new_data = create_some_records()?;
+    tbl.add(new_data).execute().await.unwrap();
+    // --8<-- [end:add]
+
+    Ok(tbl)
+}
+
+async fn create_empty_table(db: &Connection) -> Result<LanceDbTable> {
+    // --8<-- [start:create_empty_table]
+    let schema = Arc::new(Schema::new(vec![
+        Field::new("id", DataType::Int32, false),
+        Field::new("item", DataType::Utf8, true),
+    ]));
+    db.create_empty_table("empty_table", schema).execute().await
+    // --8<-- [end:create_empty_table]
+}
+
+async fn create_index(table: &LanceDbTable) -> Result<()> {
+    // --8<-- [start:create_index]
+    table.create_index(&["vector"], Index::Auto).execute().await
+    // --8<-- [end:create_index]
+}
+
+async fn search(table: &LanceDbTable) -> Result<Vec<RecordBatch>> {
+    // --8<-- [start:search]
+    table
+        .query()
+        .limit(2)
+        .nearest_to(&[1.0; 128])?
+        .execute()
+        .await?
+        .try_collect::<Vec<_>>()
+        .await
+    // --8<-- [end:search]
+}

docs/web-tests/rs/src/main.rs

@@ -0,0 +1,3 @@
+fn main() {
+    println!("Hello, world!");
+}

docs/web-tests/ts/ann_indexes.test.ts

@@ -0,0 +1,57 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+import { expect, test } from "@jest/globals";
+// --8<-- [start:import]
+import * as lancedb from "@lancedb/lancedb";
+import type { VectorQuery } from "@lancedb/lancedb";
+// --8<-- [end:import]
+import { withTempDirectory } from "./util.ts";
+
+test("ann index examples", async () => {
+  await withTempDirectory(async (databaseDir) => {
+    // --8<-- [start:ingest]
+    const db = await lancedb.connect(databaseDir);
+
+    const data = Array.from({ length: 5_000 }, (_, i) => ({
+      vector: Array(128).fill(i),
+      id: `${i}`,
+      content: "",
+      longId: `${i}`,
+    }));
+
+    const table = await db.createTable("my_vectors", data, {
+      mode: "overwrite",
+    });
+    await table.createIndex("vector", {
+      config: lancedb.Index.ivfPq({
+        numPartitions: 10,
+        numSubVectors: 16,
+      }),
+    });
+    // --8<-- [end:ingest]
+
+    // --8<-- [start:search1]
+    const search = table.search(Array(128).fill(1.2)).limit(2) as VectorQuery;
+    const results1 = await search.nprobes(20).refineFactor(10).toArray();
+    // --8<-- [end:search1]
+    expect(results1.length).toBe(2);
+
+    // --8<-- [start:search2]
+    const results2 = await table
+      .search(Array(128).fill(1.2))
+      .where("id != '1141'")
+      .limit(2)
+      .toArray();
+    // --8<-- [end:search2]
+    expect(results2.length).toBe(2);
+
+    // --8<-- [start:search3]
+    const results3 = await table
+      .search(Array(128).fill(1.2))
+      .select(["id"])
+      .limit(2)
+      .toArray();
+    // --8<-- [end:search3]
+    expect(results3.length).toBe(2);
+  });
+}, 100_000);

docs/web-tests/ts/basic_usage.test.ts

@@ -0,0 +1,161 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+import { expect, test } from "@jest/globals";
+import * as fs from "node:fs";
+import * as path from "node:path";
+// --8<-- [start:basic_imports]
+import * as lancedb from "@lancedb/lancedb";
+import * as arrow from "apache-arrow";
+// --8<-- [end:basic_imports]
+import { withTempDirectory } from "./util.ts";
+
+const dataPath = path.join(__dirname, "..", "camelot.json");
+
+test("basic usage examples (async)", async () => {
+  await withTempDirectory(async (databaseDir) => {
+    const db = await lancedb.connect(databaseDir);
+
+    // --8<-- [start:data_load]
+    const data = JSON.parse(fs.readFileSync(dataPath, "utf-8"));
+    // --8<-- [end:data_load]
+
+    // --8<-- [start:basic_create_table]
+    let table = await db.createTable("camelot", data, {
+       mode: "overwrite",
+    });
+    // --8<-- [end:basic_create_table]
+    expect(await table.countRows()).toBe(8);
+
+    // --8<-- [start:basic_open_table]
+    table = await db.openTable("camelot");
+    // --8<-- [end:basic_open_table]
+
+    // --8<-- [start:basic_create_empty_table]
+    const schema = new arrow.Schema([
+      new arrow.Field("id", new arrow.Int16()),
+      new arrow.Field("name", new arrow.Utf8()),
+      new arrow.Field("role", new arrow.Utf8()),
+      new arrow.Field("description", new arrow.Utf8()),
+      new arrow.Field(
+        "vector",
+        new arrow.FixedSizeList(
+          4,
+          new arrow.Field("item", new arrow.Float32(), true),
+        ),
+      ),
+      new arrow.Field(
+        "stats",
+        new arrow.Struct([
+          new arrow.Field("strength", new arrow.Int8()),
+          new arrow.Field("courage", new arrow.Int8()),
+          new arrow.Field("magic", new arrow.Int8()),
+          new arrow.Field("wisdom", new arrow.Int8()),
+        ]),
+      ),
+    ]);
+    await db.createEmptyTable("camelot_empty", schema, { mode: "overwrite" });
+    // --8<-- [end:basic_create_empty_table]
+    expect(await db.tableNames()).toContain("camelot_empty");
+    await db.dropTable("camelot_empty");
+
+    // --8<-- [start:basic_add_data]
+    const magicalCharacters = [
+      {
+        id: 9,
+        name: "Morgan le Fay",
+        role: "Sorceress",
+        description:
+          "A powerful enchantress, Arthur's half-sister, and a complex figure who oscillates between aiding and opposing Camelot.",
+        vector: [0.1, 0.84, 0.25, 0.7],
+        stats: { strength: 2, courage: 3, magic: 5, wisdom: 4 },
+      },
+      {
+        id: 10,
+        name: "The Lady of the Lake",
+        role: "Mystical Guardian",
+        description:
+          "A mysterious supernatural figure associated with Avalon, known for giving Arthur the sword Excalibur.",
+        vector: [0.0, 0.9, 0.58, 0.88],
+        stats: { strength: 2, courage: 3, magic: 5, wisdom: 5 },
+      },
+    ];
+    await table.add(magicalCharacters);
+    // --8<-- [end:basic_add_data]
+    expect(await table.countRows()).toBe(10);
+
+    // --8<-- [start:basic_vector_search]
+    const queryVector = [0.03, 0.85, 0.61, 0.9];
+    const result = await table.search(queryVector).limit(5).toArray();
+    console.log(result);
+    // --8<-- [end:basic_vector_search]
+
+    // --8<-- [start:basic_add_columns]
+    await table.addColumns([
+      {
+        name: "power",
+        valueSql:
+          "cast(((stats.strength + stats.courage + stats.magic + stats.wisdom) / 4.0) as float)",
+      },
+    ]);
+    // --8<-- [end:basic_add_columns]
+    const schemaWithPower = await table.schema();
+    expect(schemaWithPower.fields.some((f) => f.name === "power")).toBe(true);
+
+    // --8<-- [start:basic_vector_search_q1]
+    // Who are the characters similar to  "wizard"?
+    const queryVector1 = [0.03, 0.85, 0.61, 0.9];
+    const r1 = await table
+      .search(queryVector1)
+      .limit(5)
+      .select(["name", "role", "description"])
+      .toArray();
+    console.log(r1);
+    // --8<-- [end:basic_vector_search_q1]
+
+    // --8<-- [start:basic_vector_search_q2]
+    // Who are the characters similar to "wizard" with high magic stats?
+    const queryVector2 = [0.03, 0.85, 0.61, 0.9];
+    const r2 = await table
+      .search(queryVector2)
+      .where("stats.magic > 3")
+      .select(["name", "role", "description"])
+      .limit(5)
+      .toArray();
+    console.log(r2);
+    // --8<-- [end:basic_vector_search_q2]
+
+    // --8<-- [start:basic_vector_search_q3]
+    // Who are the strongest characters?
+    const r3 = await table
+      .query()
+      .where("stats.strength > 3")
+      .select(["name", "role", "description"])
+      .limit(5)
+      .toArray();
+    console.log(r3);
+    // --8<-- [end:basic_vector_search_q3]
+
+    // --8<-- [start:basic_vector_search_q4]
+    // Who are the strongest characters?
+    const r4 = await table
+      .query()
+      .select(["name", "role", "description", "power"])
+      .toArray();
+    console.log(r4);
+    // --8<-- [end:basic_vector_search_q4]
+
+    // --8<-- [start:basic_drop_columns]
+    await table.dropColumns(["power"]);
+    // --8<-- [end:basic_drop_columns]
+
+    // --8<-- [start:basic_delete_rows]
+    await table.delete('role = "Traitor Knight"');
+    // --8<-- [end:basic_delete_rows]
+    expect(await table.countRows()).toBe(9);
+
+    // --8<-- [start:basic_drop_table]
+    await db.dropTable("camelot");
+    // --8<-- [end:basic_drop_table]
+    expect(await db.tableNames()).not.toContain("camelot");
+  });
+});

docs/web-tests/ts/biome.json

@@ -0,0 +1,52 @@
+{
+  "$schema": "https://biomejs.dev/schemas/1.9.4/schema.json",
+  "vcs": {
+    "enabled": false,
+    "clientKind": "git",
+    "useIgnoreFile": false
+  },
+  "files": {
+    "ignoreUnknown": false,
+    "ignore": []
+  },
+  "formatter": {
+    "enabled": true,
+    "indentStyle": "space"
+  },
+  "organizeImports": {
+    "enabled": true
+  },
+  "linter": {
+    "enabled": true,
+    "rules": {
+      "recommended": true
+    }
+  },
+  "javascript": {
+    "formatter": {
+      "quoteStyle": "double"
+    }
+  },
+  "overrides": [
+    {
+      "include": ["*"],
+      "linter": {
+        "rules": {
+          "style": {
+            "noNonNullAssertion": "off"
+          }
+        }
+      }
+    },
+    {
+      "include": ["merge_insert.test.ts"],
+      "linter": {
+        "rules": {
+          "style": {
+            "useNamingConvention": "off"
+          }
+        }
+      }
+    }
+  ]
+}

docs/web-tests/ts/connection.test.ts

@@ -0,0 +1,121 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+import { expect, jest, test } from "@jest/globals";
+import * as path from "node:path";
+import { withTempDirectory } from "./util.ts";
+// --8<-- [start:connect]
+import * as lancedb from "@lancedb/lancedb";
+
+async function connectExample(uri: string) {
+  const db = await lancedb.connect(uri);
+  return db;
+}
+// --8<-- [end:connect]
+
+test("connect to a local database", async () => {
+  await withTempDirectory(async (tempDir) => {
+    const uri = path.join(tempDir, "ex_lancedb");
+    const db = await connectExample(uri);
+    expect(db).toBeDefined();
+  });
+});
+
+async function connectEnterpriseQuickstart() {
+  // --8<-- [start:connect_enterprise_quickstart]
+  const uri = "db://your-database-uri";
+  const apiKey = "your-api-key";
+  const region = "us-east-1";
+  const hostOverride = "https://your-enterprise-endpoint.com";
+
+  const db = await lancedb.connect(uri, {
+    apiKey,
+    region,
+    hostOverride,
+  });
+  // --8<-- [end:connect_enterprise_quickstart]
+  return db;
+}
+
+test("enterprise quickstart connect uses placeholder config", async () => {
+  const mockDb = { __mock: true } as unknown as Awaited<
+    ReturnType<typeof lancedb.connect>
+  >;
+  const spy = jest.spyOn(lancedb, "connect").mockResolvedValue(mockDb);
+
+  const db = await connectEnterpriseQuickstart();
+  expect(db).toBe(mockDb);
+  expect(spy).toHaveBeenCalledWith("db://your-database-uri", {
+    apiKey: "your-api-key",
+    region: "us-east-1",
+    hostOverride: "https://your-enterprise-endpoint.com",
+  });
+  spy.mockRestore();
+});
+
+// --8<-- [start:connect_object_storage]
+async function connectObjectStorageExample() {
+  const uri = "s3://your-bucket/path";
+  // You can also use "gs://your-bucket/path" or "az://your-container/path".
+  const db = await lancedb.connect(uri);
+  return db;
+}
+// --8<-- [end:connect_object_storage]
+
+async function namespaceTableOpsExample() {
+  // --8<-- [start:namespace_table_ops]
+  const db = await lancedb.connectNamespace("dir", { root: "./local_lancedb" });
+
+  // Create namespace tree: prod/search and prod/recommendations
+  await db.createNamespace(["prod"], { mode: "exist_ok" });
+  await db.createNamespace(["prod", "search"], { mode: "exist_ok" });
+  await db.createNamespace(["prod", "recommendations"], { mode: "exist_ok" });
+
+  await db.createTable(
+    "user",
+    [{ id: 1, vector: [0.1, 0.2], name: "alice" }],
+    ["prod", "search"],
+    { mode: "create" }, // use "overwrite" only if you want to replace existing table
+  );
+
+  await db.createTable(
+    "user",
+    [{ id: 2, vector: [0.3, 0.4], name: "bob" }],
+    ["prod", "recommendations"],
+    { mode: "create" },
+  );
+
+  // Verify
+  console.log((await db.listNamespaces()).namespaces); // ["prod"]
+  console.log((await db.listNamespaces(["prod"])).namespaces); // ["recommendations", "search"]
+  console.log(await db.tableNames(["prod", "search"])); // ["user"]
+  console.log(await db.tableNames(["prod", "recommendations"])); // ["user"]
+  // --8<-- [end:namespace_table_ops]
+}
+
+async function namespaceAdminOpsExample() {
+  // --8<-- [start:namespace_admin_ops]
+  const db = await lancedb.connectNamespace("dir", { root: "./local_lancedb" });
+  const namespace = ["prod", "search"];
+
+  await db.createNamespace(["prod"]);
+  await db.createNamespace(["prod", "search"]);
+
+  const childNamespaces = (await db.listNamespaces(["prod"])).namespaces;
+  console.log(`Child namespaces under ${JSON.stringify(namespace)}:`, childNamespaces);
+  // Child namespaces under ["prod","search"]: [ 'search' ]
+
+  const metadata = await db.describeNamespace(["prod", "search"]);
+  console.log(`Metadata for namespace ${JSON.stringify(namespace)}:`, metadata);
+
+  await db.dropNamespace(["prod", "search"], { mode: "skip" });
+  await db.dropNamespace(["prod"], { mode: "skip" });
+  // --8<-- [end:namespace_admin_ops]
+  return { childNamespaces, metadata };
+}
+
+void [
+  connectObjectStorageExample,
+  connectEnterpriseQuickstart,
+  namespaceTableOpsExample,
+  namespaceAdminOpsExample,
+];

docs/web-tests/ts/custom_embedding_function.test.ts

@@ -0,0 +1,79 @@
+import {
+  type FeatureExtractionPipeline,
+  pipeline,
+} from "@huggingface/transformers";
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+import { expect, test } from "@jest/globals";
+// --8<-- [start:imports]
+import * as lancedb from "@lancedb/lancedb";
+import {
+  LanceSchema,
+  TextEmbeddingFunction,
+  getRegistry,
+  register,
+} from "@lancedb/lancedb/embedding";
+// --8<-- [end:imports]
+import { withTempDirectory } from "./util.ts";
+
+// --8<-- [start:embedding_impl]
+@register("sentence-transformers")
+class SentenceTransformersEmbeddings extends TextEmbeddingFunction {
+  name = "Xenova/all-miniLM-L6-v2";
+  #ndims!: number;
+  extractor!: FeatureExtractionPipeline;
+
+  async init() {
+    this.extractor = await pipeline("feature-extraction", this.name, {
+      dtype: "fp32",
+    });
+    this.#ndims = await this.generateEmbeddings(["hello"]).then(
+      (e) => e[0].length,
+    );
+  }
+
+  ndims() {
+    return this.#ndims;
+  }
+
+  toJSON() {
+    return {
+      name: this.name,
+    };
+  }
+  async generateEmbeddings(texts: string[]) {
+    const output = await this.extractor(texts, {
+      pooling: "mean",
+      normalize: true,
+    });
+    return output.tolist();
+  }
+}
+// --8<-- [end:embedding_impl]
+
+test("Registry examples", async () => {
+  await withTempDirectory(async (databaseDir) => {
+    // --8<-- [start:call_custom_function]
+    const registry = getRegistry();
+
+    const sentenceTransformer = await registry
+      .get<SentenceTransformersEmbeddings>("sentence-transformers")!
+      .create();
+
+    const schema = LanceSchema({
+      vector: sentenceTransformer.vectorField(),
+      text: sentenceTransformer.sourceField(),
+    });
+
+    const db = await lancedb.connect(databaseDir);
+    const table = await db.createEmptyTable("table", schema, {
+      mode: "overwrite",
+    });
+
+    await table.add([{ text: "hello" }, { text: "world" }]);
+
+    const results = await table.search("greeting").limit(1).toArray();
+    // --8<-- [end:call_custom_function]
+    expect(results.length).toBe(1);
+  });
+}, 100_000);

docs/web-tests/ts/embedding.test.ts

@@ -0,0 +1,160 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+import { expect, test } from "@jest/globals";
+// --8<-- [start:imports]
+import * as lancedb from "@lancedb/lancedb";
+import "@lancedb/lancedb/embedding/openai";
+import { LanceSchema, getRegistry, register } from "@lancedb/lancedb/embedding";
+import { EmbeddingFunction } from "@lancedb/lancedb/embedding";
+import { type Float, Float32, Utf8 } from "apache-arrow";
+// --8<-- [end:imports]
+import { withTempDirectory } from "./util.ts";
+
+const openAiTest = process.env.OPENAI_API_KEY == null ? test.skip : test;
+
+openAiTest("create embedding function", () => {
+  // --8<-- [start:create_embedding_function]
+  const func = getRegistry().get("openai")!.create({
+    model: "text-embedding-3-small",
+  });
+  // --8<-- [end:create_embedding_function]
+  expect(func).toBeDefined();
+});
+
+openAiTest("openai embeddings", async () => {
+  await withTempDirectory(async (databaseDir) => {
+    // --8<-- [start:openai_embeddings]
+    const db = await lancedb.connect(databaseDir);
+    const func = getRegistry()
+      .get("openai")
+      ?.create({ model: "text-embedding-ada-002" }) as EmbeddingFunction;
+
+    const wordsSchema = LanceSchema({
+      text: func.sourceField(new Utf8()),
+      vector: func.vectorField(),
+    });
+    const tbl = await db.createEmptyTable("words", wordsSchema, {
+      mode: "overwrite",
+    });
+    await tbl.add([{ text: "hello world" }, { text: "goodbye world" }]);
+
+    const query = "greetings";
+    const actual = (await tbl.search(query).limit(1).toArray())[0];
+    // --8<-- [end:openai_embeddings]
+    expect(actual).toHaveProperty("text");
+  });
+});
+
+openAiTest("manual query embeddings", async () => {
+  await withTempDirectory(async (databaseDir) => {
+    // --8<-- [start:manual_query_embeddings]
+    const db = await lancedb.connect(databaseDir);
+    const func = getRegistry()
+      .get("openai")
+      ?.create({ model: "text-embedding-ada-002" }) as EmbeddingFunction;
+
+    const wordsSchema = LanceSchema({
+      text: func.sourceField(new Utf8()),
+      vector: func.vectorField(),
+    });
+    const tbl = await db.createEmptyTable("words", wordsSchema, {
+      mode: "overwrite",
+    });
+    await tbl.add([{ text: "hello world" }, { text: "goodbye world" }]);
+
+    const queryVector = await func.computeQueryEmbeddings("greetings");
+    // --8<-- [start:manual_query_search]
+    // queryVector is assumed to already be generated by your embedding function
+    const actual = (await tbl.search(queryVector).limit(1).toArray())[0];
+    // --8<-- [end:manual_query_search]
+    // --8<-- [end:manual_query_embeddings]
+    expect(actual).toHaveProperty("text");
+  });
+});
+
+test("custom embedding function", async () => {
+  await withTempDirectory(async (databaseDir) => {
+    // --8<-- [start:embedding_function]
+    const db = await lancedb.connect(databaseDir);
+
+    @register("my_embedding")
+    class MyEmbeddingFunction extends EmbeddingFunction<string> {
+      constructor(optionsRaw = {}) {
+        super();
+        const options = this.resolveVariables(optionsRaw);
+        // Initialize using options
+      }
+      ndims() {
+        return 3;
+      }
+      protected getSensitiveKeys(): string[] {
+        return [];
+      }
+      embeddingDataType(): Float {
+        return new Float32();
+      }
+      async computeQueryEmbeddings(_data: string) {
+        // This is a placeholder for a real embedding function
+        return [1, 2, 3];
+      }
+      async computeSourceEmbeddings(data: string[]) {
+        // This is a placeholder for a real embedding function
+        return Array.from({ length: data.length }).fill([
+          1, 2, 3,
+        ]) as number[][];
+      }
+    }
+
+    const func = new MyEmbeddingFunction();
+
+    const data = [{ text: "pepperoni" }, { text: "pineapple" }];
+
+    // Option 1: manually specify the embedding function
+    const table = await db.createTable("vectors", data, {
+      embeddingFunction: {
+        function: func,
+        sourceColumn: "text",
+        vectorColumn: "vector",
+      },
+      mode: "overwrite",
+    });
+
+    // Option 2: provide the embedding function through a schema
+
+    const schema = LanceSchema({
+      text: func.sourceField(new Utf8()),
+      vector: func.vectorField(),
+    });
+
+    const table2 = await db.createTable("vectors2", data, {
+      schema,
+      mode: "overwrite",
+    });
+    // --8<-- [end:embedding_function]
+    expect(await table.countRows()).toBe(2);
+    expect(await table2.countRows()).toBe(2);
+  });
+});
+
+test("embedding function api_key", async () => {
+  // --8<-- [start:register_secret]
+  const registry = getRegistry();
+  registry.setVar("api_key", "sk-...");
+
+  const func = registry.get("openai")!.create({
+    apiKey: "$var:api_key",
+  });
+  // --8<-- [end:register_secret]
+});
+
+openAiTest("embedding function variable fallback", () => {
+  // --8<-- [start:register_model_fallback]
+  const registry = getRegistry();
+  registry.setVar("openai_model", "text-embedding-3-large");
+
+  const func = registry.get("openai")!.create({
+    model: "$var:openai_model:text-embedding-3-small",
+  });
+  // --8<-- [end:register_model_fallback]
+  expect(func).toBeDefined();
+});

docs/web-tests/ts/filtering.test.ts

@@ -0,0 +1,42 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+import { expect, test } from "@jest/globals";
+import * as lancedb from "@lancedb/lancedb";
+import { withTempDirectory } from "./util.ts";
+
+test("filtering examples", async () => {
+  await withTempDirectory(async (databaseDir) => {
+    const db = await lancedb.connect(databaseDir);
+
+    const data = Array.from({ length: 10_000 }, (_, i) => ({
+      vector: Array(1536).fill(i),
+      id: i,
+      item: `item ${i}`,
+      strId: `${i}`,
+    }));
+
+    const tbl = await db.createTable("myVectors", data, { mode: "overwrite" });
+
+    // --8<-- [start:search]
+    const _result = await tbl
+      .search(Array(1536).fill(0.5))
+      .limit(1)
+      .where("id = 10")
+      .toArray();
+    // --8<-- [end:search]
+
+    // --8<-- [start:vec_search]
+    const result = await (
+      tbl.search(Array(1536).fill(0)) as lancedb.VectorQuery
+    )
+      .where("(item IN ('item 0', 'item 2')) AND (id > 10)")
+      .postfilter()
+      .toArray();
+    // --8<-- [end:vec_search]
+    expect(result.length).toBe(0);
+
+    // --8<-- [start:sql_search]
+    await tbl.query().where("id = 10").limit(10).toArray();
+    // --8<-- [end:sql_search]
+  });
+});

docs/web-tests/ts/full_text_search.test.ts

@@ -0,0 +1,45 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+import { expect, test } from "@jest/globals";
+import * as lancedb from "@lancedb/lancedb";
+import { withTempDirectory } from "./util.ts";
+
+test("full text search", async () => {
+  await withTempDirectory(async (databaseDir) => {
+    const db = await lancedb.connect(databaseDir);
+
+    const words = [
+      "apple",
+      "banana",
+      "cherry",
+      "date",
+      "elderberry",
+      "fig",
+      "grape",
+    ];
+
+    const data = Array.from({ length: 10_000 }, (_, i) => ({
+      vector: Array(1536).fill(i),
+      id: i,
+      item: `item ${i}`,
+      strId: `${i}`,
+      doc: words[i % words.length],
+    }));
+
+    const tbl = await db.createTable("myVectors", data, { mode: "overwrite" });
+
+    await tbl.createIndex("doc", {
+      config: lancedb.Index.fts(),
+    });
+
+    // --8<-- [start:full_text_search]
+    const result = await tbl
+      .query()
+      .nearestToText("apple")
+      .select(["id", "doc"])
+      .limit(10)
+      .toArray();
+    expect(result.length).toBe(10);
+    // --8<-- [end:full_text_search]
+  });
+}, 10_000);

docs/web-tests/ts/integrations.ts

@@ -0,0 +1,134 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+// --8<-- [start:frameworks_genkit_usage]
+import { lancedbIndexerRef, lancedb, lancedbRetrieverRef, WriteMode } from "genkitx-lancedb";
+import { textEmbedding004, vertexAI } from "@genkit-ai/vertexai";
+import { gemini } from "@genkit-ai/vertexai";
+import { z, genkit } from "genkit";
+import { Document } from "genkit/retriever";
+import { chunk } from "llm-chunk";
+import { readFile } from "fs/promises";
+import path from "path";
+import pdf from "pdf-parse/lib/pdf-parse";
+
+const ai = genkit({
+  plugins: [
+    // vertexAI provides the textEmbedding004 embedder
+    vertexAI(),
+
+    // the local vector store requires an embedder to translate from text to vector
+    lancedb([
+      {
+        dbUri: ".db", // optional lancedb uri, default to .db
+        tableName: "table", // optional table name, default to table
+        embedder: textEmbedding004,
+      },
+    ]),
+  ],
+});
+// --8<-- [end:frameworks_genkit_usage]
+
+// --8<-- [start:frameworks_genkit_custom_indexer]
+export const menuPdfIndexer = lancedbIndexerRef({
+  // Using all defaults, for dbUri, tableName, and embedder, etc
+});
+
+const chunkingConfig = {
+  minLength: 1000,
+  maxLength: 2000,
+  splitter: "sentence",
+  overlap: 100,
+  delimiters: "",
+} as any;
+
+async function extractTextFromPdf(filePath: string) {
+  const pdfFile = path.resolve(filePath);
+  const dataBuffer = await readFile(pdfFile);
+  const data = await pdf(dataBuffer);
+  return data.text;
+}
+
+export const indexMenu = ai.defineFlow(
+  {
+    name: "indexMenu",
+    inputSchema: z.string().describe("PDF file path"),
+    outputSchema: z.void(),
+  },
+  async (filePath: string) => {
+    filePath = path.resolve(filePath);
+
+    // Read the pdf.
+    const pdfTxt = await ai.run("extract-text", () => extractTextFromPdf(filePath));
+
+    // Divide the pdf text into segments.
+    const chunks = await ai.run("chunk-it", async () => chunk(pdfTxt, chunkingConfig));
+
+    // Convert chunks of text into documents to store in the index.
+    const documents = chunks.map((text) => {
+      return Document.fromText(text, { filePath });
+    });
+
+    // Add documents to the index.
+    await ai.index({
+      indexer: menuPdfIndexer,
+      documents,
+      options: {
+        writeMode: WriteMode.Overwrite,
+      } as any,
+    });
+  },
+);
+// --8<-- [end:frameworks_genkit_custom_indexer]
+
+// --8<-- [start:frameworks_genkit_custom_retriever]
+export const menuRetriever = lancedbRetrieverRef({
+  tableName: "table", // Use the same table name as the indexer.
+  displayName: "Menu", // Use a custom display name.
+});
+
+export const menuQAFlow = ai.defineFlow(
+  { name: "Menu", inputSchema: z.string(), outputSchema: z.string() },
+  async (input: string) => {
+    // retrieve relevant documents
+    const docs = await ai.retrieve({
+      retriever: menuRetriever,
+      query: input,
+      options: {
+        k: 3,
+      },
+    });
+
+    const extractedContent = docs.map((doc) => {
+      if (doc.content && Array.isArray(doc.content) && doc.content.length > 0) {
+        if (doc.content[0].media && doc.content[0].media.url) {
+          return doc.content[0].media.url;
+        }
+      }
+      return "No content found";
+    });
+
+    console.log("Extracted content:", extractedContent);
+
+    const { text } = await ai.generate({
+      model: gemini("gemini-2.0-flash"),
+      prompt: `
+You are acting as a helpful AI assistant that can answer 
+questions about the food available on the menu at Genkit Grub Pub.
+
+Use only the context provided to answer the question.
+If you don't know, do not make up an answer.
+Do not add or change items on the menu.
+
+Context:
+${extractedContent.join("\n\n")}
+
+Question: ${input}`,
+      docs,
+    });
+
+    return text;
+  },
+);
+// --8<-- [end:frameworks_genkit_custom_retriever]
+

docs/web-tests/ts/jest.config.cjs

@@ -0,0 +1,6 @@
+/** @type {import('ts-jest').JestConfigWithTsJest} */
+module.exports = {
+  preset: "ts-jest",
+  testEnvironment: "node",
+  testPathIgnorePatterns: ["./dist"],
+};

docs/web-tests/ts/merge_insert.test.ts

@@ -0,0 +1,68 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+import { expect, test } from "@jest/globals";
+import * as lancedb from "@lancedb/lancedb";
+
+test("basic upsert", async () => {
+  const db = await lancedb.connect("memory://");
+
+  // --8<-- [start:upsert_basic]
+  const table = await db.createTable("users", [
+    { id: 0, name: "Alice" },
+    { id: 1, name: "Bob" },
+  ]);
+
+  const newUsers = [
+    { id: 1, name: "Bobby" },
+    { id: 2, name: "Charlie" },
+  ];
+  await table
+    .mergeInsert("id")
+    .whenMatchedUpdateAll()
+    .whenNotMatchedInsertAll()
+    .execute(newUsers);
+
+  await table.countRows(); // 3
+  // --8<-- [end:upsert_basic]
+  expect(await table.countRows()).toBe(3);
+
+  // --8<-- [start:insert_if_not_exists]
+  const table2 = await db.createTable("domains", [
+    { domain: "google.com", name: "Google" },
+    { domain: "github.com", name: "GitHub" },
+  ]);
+
+  const newDomains = [
+    { domain: "google.com", name: "Google" },
+    { domain: "facebook.com", name: "Facebook" },
+  ];
+  await table2
+    .mergeInsert("domain")
+    .whenNotMatchedInsertAll()
+    .execute(newDomains);
+  await table2.countRows(); // 3
+  // --8<-- [end:insert_if_not_exists]
+  expect(await table2.countRows()).toBe(3);
+
+  // --8<-- [start:replace_range]
+  const table3 = await db.createTable("chunks", [
+    { doc_id: 0, chunk_id: 0, text: "Hello" },
+    { doc_id: 0, chunk_id: 1, text: "World" },
+    { doc_id: 1, chunk_id: 0, text: "Foo" },
+    { doc_id: 1, chunk_id: 1, text: "Bar" },
+  ]);
+
+  const newChunks = [{ doc_id: 1, chunk_id: 0, text: "Baz" }];
+
+  await table3
+    .mergeInsert(["doc_id", "chunk_id"])
+    .whenMatchedUpdateAll()
+    .whenNotMatchedInsertAll()
+    .whenNotMatchedBySourceDelete({ where: "doc_id = 1" })
+    .execute(newChunks);
+
+  await table3.countRows("doc_id = 1"); // 1
+  // --8<-- [end:replace_range]
+  expect(await table3.countRows("doc_id = 1")).toBe(1);
+});

docs/web-tests/ts/multimodal.test.ts

@@ -0,0 +1,104 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+import { expect, test } from "@jest/globals";
+// --8<-- [start:multimodal_imports]
+import * as arrow from "apache-arrow";
+import { Buffer } from "node:buffer";
+import * as lancedb from "@lancedb/lancedb";
+// --8<-- [end:multimodal_imports]
+import { withTempDirectory } from "./util.ts";
+
+test("multimodal snippets (async)", async () => {
+  await withTempDirectory(async (databaseDir) => {
+    const db = await lancedb.connect(databaseDir);
+
+    // --8<-- [start:create_dummy_data]
+    const createDummyImage = (color: string): Uint8Array => {
+      const pngHeader = Uint8Array.from([137, 80, 78, 71, 13, 10, 26, 10]);
+      return Buffer.concat([Buffer.from(pngHeader), Buffer.from(color, "utf8")]);
+    };
+
+    const data = [
+      {
+        id: 1,
+        filename: "red_square.png",
+        vector: Array.from({ length: 128 }, (_, i) => (i % 16) / 16),
+        image_blob: createDummyImage("red"),
+        label: "red",
+      },
+      {
+        id: 2,
+        filename: "blue_square.png",
+        vector: Array.from({ length: 128 }, (_, i) => ((i + 8) % 16) / 16),
+        image_blob: createDummyImage("blue"),
+        label: "blue",
+      },
+    ];
+    // --8<-- [end:create_dummy_data]
+
+    // --8<-- [start:define_schema]
+    const schema = new arrow.Schema([
+      new arrow.Field("id", new arrow.Int32()),
+      new arrow.Field("filename", new arrow.Utf8()),
+      new arrow.Field(
+        "vector",
+        new arrow.FixedSizeList(
+          128,
+          new arrow.Field("item", new arrow.Float32(), true),
+        ),
+      ),
+      new arrow.Field("image_blob", new arrow.Binary()),
+      new arrow.Field("label", new arrow.Utf8()),
+    ]);
+    // --8<-- [end:define_schema]
+
+    // --8<-- [start:ingest_data]
+    const multimodalData = lancedb.makeArrowTable(data, { schema });
+    const tbl = await db.createTable("images", multimodalData, {
+      mode: "overwrite",
+    });
+    // --8<-- [end:ingest_data]
+    expect(await tbl.countRows()).toBe(2);
+
+    // --8<-- [start:search_data]
+    const queryVector = Array.from({ length: 128 }, (_, i) => (i % 16) / 16);
+    const results = await tbl.search(queryVector).limit(1).toArray();
+    // --8<-- [end:search_data]
+
+    // --8<-- [start:process_results]
+    for (const row of results) {
+      const imageBytes = row.image_blob as Uint8Array;
+      console.log(
+        `Retrieved image: ${row.filename}, Byte length: ${imageBytes.length}`,
+      );
+    }
+    // --8<-- [end:process_results]
+    expect(results).toHaveLength(1);
+
+    // --8<-- [start:blob_api_schema]
+    const blobSchema = new arrow.Schema([
+      new arrow.Field("id", new arrow.Int64()),
+      new arrow.Field(
+        "video",
+        new arrow.LargeBinary(),
+        true,
+        new Map([["lance-encoding:blob", "true"]]),
+      ),
+    ]);
+    // --8<-- [end:blob_api_schema]
+
+    // --8<-- [start:blob_api_ingest]
+    const blobData = lancedb.makeArrowTable(
+      [
+        { id: 1, video: Buffer.from("fake_video_bytes_1") },
+        { id: 2, video: Buffer.from("fake_video_bytes_2") },
+      ],
+      { schema: blobSchema },
+    );
+    const blobTable = await db.createTable("videos", blobData, {
+      mode: "overwrite",
+    });
+    // --8<-- [end:blob_api_ingest]
+    expect(await blobTable.countRows()).toBe(2);
+  });
+});

docs/web-tests/ts/package.json

@@ -0,0 +1,33 @@
+{
+  "name": "examples",
+  "version": "1.0.0",
+  "description": "Examples for LanceDB",
+  "main": "index.js",
+  "type": "module",
+  "scripts": {
+    "//1": "--experimental-vm-modules is needed to run jest with sentence-transformers",
+    "//2": "--testEnvironment is needed to run jest with sentence-transformers",
+    "//3": "See: https://github.com/huggingface/transformers.js/issues/57",
+    "test": "node --experimental-vm-modules node_modules/.bin/jest --testEnvironment jest-environment-node-single-context --verbose",
+    "lint": "biome check *.ts && biome format *.ts",
+    "lint-ci": "biome ci .",
+    "lint-fix": "biome check --write *.ts && npm run format",
+    "format": "biome format --write *.ts"
+  },
+  "author": "Lance Devs",
+  "license": "Apache-2.0",
+  "dependencies": {
+    "@huggingface/transformers": "^3.0.2",
+    "@lancedb/lancedb": "^0.31.0",
+    "openai": "^4.29.2",
+    "sharp": "^0.33.5"
+  },
+  "devDependencies": {
+    "@biomejs/biome": "^1.7.3",
+    "@jest/globals": "^29.7.0",
+    "jest": "^29.7.0",
+    "jest-environment-node-single-context": "^29.4.0",
+    "ts-jest": "^29.2.5",
+    "typescript": "^5.5.4"
+  }
+}

docs/web-tests/ts/quickstart.test.ts

@@ -0,0 +1,175 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+import { expect, test } from "@jest/globals";
+import * as lancedb from "@lancedb/lancedb";
+import { withTempDirectory } from "./util.ts";
+
+test("quickstart example (async)", async () => {
+  await withTempDirectory(async (databaseDir) => {
+    const db = await lancedb.connect(databaseDir);
+
+    // --8<-- [start:quickstart_data]
+    const data = [
+      {
+        id: "1",
+        name: "King Arthur",
+        role: "King",
+        description: "Leader of Camelot and wielder of Excalibur.",
+        stats: { strength: 4, magic: 1, leadership: 5, wisdom: 4 },
+        vector: [0.7, 0.1, 0.9, 0.7],
+      },
+      {
+        id: "2",
+        name: "Merlin",
+        role: "Wizard",
+        description: "Advisor and prophet with deep magical knowledge.",
+        stats: { strength: 2, magic: 5, leadership: 4, wisdom: 5 },
+        vector: [0.2, 0.9, 0.4, 0.9],
+      },
+      {
+        id: "3",
+        name: "Sir Lancelot",
+        role: "Knight",
+        description: "Legendary knight known for courage and combat skill.",
+        stats: { strength: 5, magic: 1, leadership: 3, wisdom: 3 },
+        vector: [0.9, 0.1, 0.5, 0.4],
+      },
+    ];
+    // --8<-- [end:quickstart_data]
+
+    // --8<-- [start:quickstart_create_table]
+    let table = await db.createTable("characters", data, { mode: "overwrite" });
+    // --8<-- [end:quickstart_create_table]
+    expect(await table.countRows()).toBe(3);
+    await db.dropTable("characters");
+
+    // --8<-- [start:quickstart_create_table_no_overwrite]
+    table = await db.createTable("characters", data);
+    // --8<-- [end:quickstart_create_table_no_overwrite]
+    expect(await table.countRows()).toBe(3);
+
+    // --8<-- [start:quickstart_vector_search_1]
+    // Search for examples similar to a "wise magical advisor"
+    let queryVector = [0.2, 0.8, 0.4, 0.9];
+
+    let result = await table
+      .search(queryVector)
+      .select(["name", "role", "description", "_distance"])
+      .limit(2)
+      .toArray();
+    console.table(result);
+    // --8<-- [end:quickstart_vector_search_1]
+    expect(result[0].name).toBe("Merlin");
+
+    // --8<-- [start:quickstart_curate_with_metadata]
+    const curated = await table
+      .search(queryVector)
+      .where("stats.magic >= 4")
+      .select(["name", "role", "description", "_distance"])
+      .limit(2)
+      .toArray();
+    console.table(curated);
+    // --8<-- [end:quickstart_curate_with_metadata]
+    expect(curated[0].name).toBe("Merlin");
+
+    // --8<-- [start:quickstart_output_array]
+    result = await table.search(queryVector).limit(2).toArray();
+    console.table(result);
+    // --8<-- [end:quickstart_output_array]
+    expect(result[0].name).toBe("Merlin");
+
+    // --8<-- [start:quickstart_add_feature]
+    await table.addColumns([
+      {
+        name: "power_score",
+        valueSql:
+          "cast(((stats.strength + stats.magic + stats.leadership + stats.wisdom) / 4.0) as float)",
+      },
+    ]);
+    // --8<-- [end:quickstart_add_feature]
+    const schemaWithFeature = await table.schema();
+    expect(schemaWithFeature.fields.some((f) => f.name === "power_score")).toBe(
+      true,
+    );
+
+    // --8<-- [start:quickstart_query_feature]
+    const features = await table
+      .query()
+      .select(["name", "role", "power_score"])
+      .toArray();
+    console.table(features);
+    // --8<-- [end:quickstart_query_feature]
+    expect(features[0]).toHaveProperty("power_score");
+
+    // --8<-- [start:quickstart_multimodal_bytes]
+    const arrow = await import("apache-arrow");
+    const path = await import("node:path");
+    const { readFile } = await import("node:fs/promises");
+
+    const imagePath = path.resolve(
+      "../../docs/static/assets/images/quickstart/sir-lancelot.jpg",
+    );
+    const imageBytes = await readFile(imagePath);
+    const imageSchema = new arrow.Schema([
+      new arrow.Field("id", new arrow.Utf8()),
+      new arrow.Field("description", new arrow.Utf8()),
+      new arrow.Field("image", new arrow.Binary()),
+      new arrow.Field(
+        "vector",
+        new arrow.FixedSizeList(
+          4,
+          new arrow.Field("item", new arrow.Float32(), true),
+        ),
+      ),
+    ]);
+    const imageData = lancedb.makeArrowTable(
+      [
+        {
+          id: "lancelot",
+          description: "Portrait of Sir Lancelot",
+          image: imageBytes,
+          vector: [0.9, 0.1, 0.5, 0.4],
+        },
+      ],
+      { schema: imageSchema },
+    );
+    const multimodalTable = await db.createTable(
+      "character_images",
+      imageData,
+      { mode: "overwrite" },
+    );
+    // --8<-- [end:quickstart_multimodal_bytes]
+    expect(await multimodalTable.countRows()).toBe(1);
+
+    // --8<-- [start:quickstart_open_table]
+    table = await db.openTable("characters");
+    // --8<-- [end:quickstart_open_table]
+
+    // --8<-- [start:quickstart_add_data]
+    const moreData = [
+      {
+        id: "4",
+        name: "Morgana",
+        role: "Sorceress",
+        description: "Powerful sorceress of Avalon.",
+        stats: { strength: 2, magic: 5, leadership: 4, wisdom: 4 },
+        vector: [0.3, 0.9, 0.6, 0.8],
+        power_score: 3.75,
+      },
+    ];
+
+    // Add data to table
+    await table.add(moreData);
+    // --8<-- [end:quickstart_add_data]
+    expect(await table.countRows()).toBe(4);
+
+    // --8<-- [start:quickstart_vector_search_2]
+    // Search for examples similar to a "powerful sorceress"
+    queryVector = [0.3, 0.9, 0.6, 0.8];
+
+    const results = await table.search(queryVector).limit(2).toArray();
+    console.table(results);
+    // --8<-- [end:quickstart_vector_search_2]
+    expect(results[0].name).toBe("Morgana");
+  });
+});

docs/web-tests/ts/search.test.ts

@@ -0,0 +1,355 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+import { expect, test } from "@jest/globals";
+// --8<-- [start:import]
+import * as lancedb from "@lancedb/lancedb";
+// --8<-- [end:import]
+// --8<-- [start:import_bin_util]
+import { Field, FixedSizeList, Int32, Schema, Uint8 } from "apache-arrow";
+// --8<-- [end:import_bin_util]
+import { Float32, Struct } from "apache-arrow";
+import { withTempDirectory } from "./util.ts";
+
+async function buildIndexedTable(db: lancedb.Connection, name: string) {
+  const data = Array.from({ length: 512 }, (_, i) => ({
+    vector: Array.from({ length: 128 }, () => Math.random()),
+    id: i,
+  }));
+  const tbl = await db.createTable(name, data, { mode: "overwrite" });
+  await tbl.createIndex("vector", {
+    config: lancedb.Index.ivfPq({ numPartitions: 4, numSubVectors: 8 }),
+  });
+  return tbl;
+}
+
+test("vector search", async () => {
+  await withTempDirectory(async (databaseDir) => {
+    {
+      const db = await lancedb.connect(databaseDir);
+
+      const data = Array.from({ length: 10_000 }, (_, i) => ({
+        vector: Array(128).fill(i),
+        id: `${i}`,
+      }));
+
+      await db.createTable("my_vectors", data);
+    }
+
+    // --8<-- [start:search1]
+    const db = await lancedb.connect(databaseDir);
+    const tbl = await db.openTable("my_vectors");
+
+    const results1 = await tbl.search(Array(128).fill(1.2)).limit(10).toArray();
+    // --8<-- [end:search1]
+    expect(results1.length).toBe(10);
+
+    // --8<-- [start:search2]
+    const results2 = await (
+      tbl.search(Array(128).fill(1.2)) as lancedb.VectorQuery
+    )
+      .distanceType("cosine")
+      .limit(10)
+      .toArray();
+    // --8<-- [end:search2]
+    expect(results2.length).toBe(10);
+
+    // --8<-- [start:distance_range]
+    const results3 = await (
+      tbl.search(Array(128).fill(1.2)) as lancedb.VectorQuery
+    )
+      .distanceType("cosine")
+      .distanceRange(0.1, 0.2)
+      .limit(10)
+      .toArray();
+    // --8<-- [end:distance_range]
+    for (const r of results3) {
+      expect(r.distance).toBeGreaterThanOrEqual(0.1);
+      expect(r.distance).toBeLessThan(0.2);
+    }
+
+    {
+      // --8<-- [start:ingest_binary_data]
+      const schema = new Schema([
+        new Field("id", new Int32(), true),
+        new Field("vec", new FixedSizeList(32, new Field("item", new Uint8()))),
+      ]);
+      const data = lancedb.makeArrowTable(
+        Array(1_000)
+          .fill(0)
+          .map((_, i) => ({
+            // the 256 bits would be store in 32 bytes,
+            // if your data is already in this format, you can skip the packBits step
+            id: i,
+            vec: lancedb.packBits(Array(256).fill(i % 2)),
+          })),
+        { schema: schema },
+      );
+
+      const tbl = await db.createTable("binary_table", data);
+      await tbl.createIndex("vec", {
+        config: lancedb.Index.ivfFlat({
+          numPartitions: 10,
+          distanceType: "hamming",
+        }),
+      });
+      // --8<-- [end:ingest_binary_data]
+
+      // --8<-- [start:search_binary_data]
+      const query = Array(32)
+        .fill(1)
+        .map(() => Math.floor(Math.random() * 255));
+      const results = await tbl.query().nearestTo(query).limit(10).toArrow();
+      // --8<-- [end:search_binary_data]
+      expect(results.numRows).toBe(10);
+    }
+  });
+});
+
+test("vector search docs snippets", async () => {
+  await withTempDirectory(async (databaseDir) => {
+    const db = await lancedb.connect(databaseDir);
+
+    // ---- Nested vector column: inference and explicit selection ----
+    {
+      const nestedSchema = new Schema([
+        new Field("id", new Int32()),
+        new Field(
+          "image",
+          new Struct([
+            new Field(
+              "embedding",
+              new FixedSizeList(2, new Field("item", new Float32(), true)),
+            ),
+          ]),
+        ),
+      ]);
+      const arrowTbl = lancedb.makeArrowTable(
+        [{ id: 0, image: { embedding: [0.0, 1.0] } }],
+        { schema: nestedSchema },
+      );
+      await db.createTable("nested", arrowTbl, { mode: "overwrite" });
+
+      // --8<-- [start:select_vector_column]
+      const table = await db.openTable("nested");
+
+      // Inferred: LanceDB finds the single nested vector leaf automatically.
+      await table.query().nearestTo([0.0, 1.0]).limit(1).toArray();
+
+      // Explicit: required when more than one vector column matches.
+      await table
+        .query()
+        .nearestTo([0.0, 1.0])
+        .column("image.embedding")
+        .limit(1)
+        .toArray();
+      // --8<-- [end:select_vector_column]
+    }
+
+    // ---- Index a nested vector column ----
+    {
+      const dim = 16;
+      const nestedSchema = new Schema([
+        new Field("id", new Int32()),
+        new Field(
+          "image",
+          new Struct([
+            new Field(
+              "embedding",
+              new FixedSizeList(dim, new Field("item", new Float32(), true)),
+            ),
+          ]),
+        ),
+      ]);
+      const rows = Array.from({ length: 512 }, (_, i) => ({
+        id: i,
+        image: { embedding: Array.from({ length: dim }, () => Math.random()) },
+      }));
+      const arrowTbl = lancedb.makeArrowTable(rows, { schema: nestedSchema });
+      const table = await db.createTable("nested_index", arrowTbl, {
+        mode: "overwrite",
+      });
+
+      // --8<-- [start:index_nested_column]
+      await table.createIndex("image.embedding");
+      // --8<-- [end:index_nested_column]
+    }
+
+    // ---- Indexed queries: refine, fast search, bypass index ----
+    {
+      const table = await buildIndexedTable(db, "ann_table");
+      const embedding = Array.from({ length: 128 }, () => Math.random());
+
+      // --8<-- [start:exact_vs_approximate]
+      // Indexed ANN search without refinement (fast, approximate `_distance`)
+      const fastResults = await (table.search(embedding) as lancedb.VectorQuery)
+        .limit(10)
+        .toArray();
+
+      // Recompute distances on full vectors for reranked candidates
+      const exactDistanceResults = await (
+        table.search(embedding) as lancedb.VectorQuery
+      )
+        .limit(10)
+        .refineFactor(1)
+        .toArray();
+
+      // Rerank a larger candidate set for better recall (higher latency)
+      const higherRecallResults = await (
+        table.search(embedding) as lancedb.VectorQuery
+      )
+        .limit(10)
+        .refineFactor(20)
+        .toArray();
+      // --8<-- [end:exact_vs_approximate]
+      expect(fastResults.length).toBe(10);
+      expect(exactDistanceResults.length).toBe(10);
+      expect(higherRecallResults.length).toBe(10);
+
+      // --8<-- [start:fast_search]
+      await table
+        .query()
+        .nearestTo(embedding)
+        .fastSearch()
+        .limit(5)
+        .toArray();
+      // --8<-- [end:fast_search]
+
+      // --8<-- [start:bypass_vector_index]
+      await table
+        .query()
+        .nearestTo(embedding)
+        .bypassVectorIndex()
+        .limit(5)
+        .toArray();
+      // --8<-- [end:bypass_vector_index]
+    }
+
+    // ---- Brute force search (no index) ----
+    {
+      const data = Array.from({ length: 64 }, (_, i) => ({
+        vector: Array(128).fill(i),
+        id: `${i}`,
+      }));
+      await db.createTable("my_vectors", data, { mode: "overwrite" });
+
+      // --8<-- [start:brute_force_search]
+      const tbl = await db.openTable("my_vectors");
+
+      const results1 = await tbl.search(Array(128).fill(1.2)).limit(3).toArray();
+      // --8<-- [end:brute_force_search]
+      expect(results1.length).toBe(3);
+    }
+
+    // ---- Binary (hamming) vector search ----
+    {
+      const schema = new Schema([
+        new Field("id", new Int32(), true),
+        new Field("vector", new FixedSizeList(32, new Field("item", new Uint8()))),
+      ]);
+      const data = lancedb.makeArrowTable(
+        Array(1000)
+          .fill(0)
+          .map((_, i) => ({
+            // the 256 bits are stored in 32 bytes; if your data is already in
+            // this format, you can skip the packBits step
+            id: i,
+            vector: lancedb.packBits(Array(256).fill(i % 2)),
+          })),
+        { schema },
+      );
+
+      // --8<-- [start:binary_search]
+      const tbl = await db.createTable("binary_vectors", data, {
+        mode: "overwrite",
+      });
+      await tbl.createIndex("vector", {
+        config: lancedb.Index.ivfFlat({
+          numPartitions: 10,
+          distanceType: "hamming",
+        }),
+      });
+
+      const query = Array(32)
+        .fill(1)
+        .map(() => Math.floor(Math.random() * 255));
+      const results = await tbl.query().nearestTo(query).limit(10).toArray();
+      // --8<-- [end:binary_search]
+      expect(results.length).toBeLessThanOrEqual(10);
+    }
+
+    // ---- Enterprise-style prefilter / postfilter / batch search ----
+    {
+      const dimensions = 768;
+      const rows = Array.from({ length: 50 }, (_, i) => ({
+        vector: Array.from({ length: dimensions }, () => Math.random() * 2 - 1),
+        text: `story ${i}`,
+        keywords: `kw${i}`,
+        label: i % 4,
+      }));
+      await db.createTable("lancedb-enterprise-quickstart", rows, {
+        mode: "overwrite",
+      });
+
+      {
+        // --8<-- [start:vector_search_prefilter]
+        // Generate a sample 768-dimension embedding vector (typical for BERT-based models)
+        // In real applications, you would get this from an embedding model
+        const dimensions = 768;
+        const queryEmbed = Array.from(
+          { length: dimensions },
+          () => Math.random() * 2 - 1,
+        );
+
+        // Open table and perform search
+        const tableName = "lancedb-enterprise-quickstart";
+        const table = await db.openTable(tableName);
+
+        // Vector search with filters (pre-filtering is the default)
+        const vectorResults = await table
+          .search(queryEmbed)
+          .where("label > 2")
+          .select(["text", "keywords", "label"])
+          .limit(5)
+          .toArray();
+
+        console.log("Search results (with pre-filtering):");
+        console.log(vectorResults);
+        // --8<-- [end:vector_search_prefilter]
+
+        // --8<-- [start:vector_search_postfilter]
+        const vectorResultsWithPostFilter = await (
+          table.search(queryEmbed) as lancedb.VectorQuery
+        )
+          .where("label > 2")
+          .postfilter()
+          .select(["text", "keywords", "label"])
+          .limit(5)
+          .toArray();
+
+        console.log("Vector search results with post-filter:");
+        console.log(vectorResultsWithPostFilter);
+        // --8<-- [end:vector_search_postfilter]
+
+        // --8<-- [start:batch_search]
+        // Batch query
+        console.log("Performing batch vector search...");
+        const batchSize = 5;
+        const queryVectors = Array.from({ length: batchSize }, () =>
+          Array.from({ length: dimensions }, () => Math.random() * 2 - 1),
+        );
+        let batchQuery = table.search(queryVectors[0]) as lancedb.VectorQuery;
+        for (let i = 1; i < batchSize; i++) {
+          batchQuery = batchQuery.addQueryVector(queryVectors[i]);
+        }
+        const batchResults = await batchQuery
+          .select(["text", "keywords", "label"])
+          .limit(5)
+          .toArray();
+        console.log("Batch vector search results:");
+        console.log(batchResults);
+        // --8<-- [end:batch_search]
+        expect(batchResults.length).toBeGreaterThan(0);
+      }
+    }
+  });
+});

docs/web-tests/ts/sentence-transformers.test.ts

@@ -0,0 +1,96 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+import { expect, test } from "@jest/globals";
+import { withTempDirectory } from "./util.ts";
+
+// --8<-- [start:quickstart_imports]
+import * as lancedb from "@lancedb/lancedb";
+import "@lancedb/lancedb/embedding/transformers";
+import { Utf8 } from "apache-arrow";
+// --8<-- [end:quickstart_imports]
+
+test("full text search", async () => {
+  await withTempDirectory(async (databaseDir) => {
+    const db = await lancedb.connect(databaseDir);
+    const func = (await lancedb.embedding
+      .getRegistry()
+      .get("huggingface")
+      ?.create()) as lancedb.embedding.EmbeddingFunction;
+
+    const facts = [
+      "Albert Einstein was a theoretical physicist.",
+      "The capital of France is Paris.",
+      "The Great Wall of China is one of the Seven Wonders of the World.",
+      "Python is a popular programming language.",
+      "Mount Everest is the highest mountain in the world.",
+      "Leonardo da Vinci painted the Mona Lisa.",
+      "Shakespeare wrote Hamlet.",
+      "The human body has 206 bones.",
+      "The speed of light is approximately 299,792 kilometers per second.",
+      "Water boils at 100 degrees Celsius.",
+      "The Earth orbits the Sun.",
+      "The Pyramids of Giza are located in Egypt.",
+      "Coffee is one of the most popular beverages in the world.",
+      "Tokyo is the capital city of Japan.",
+      "Photosynthesis is the process by which plants make their food.",
+      "The Pacific Ocean is the largest ocean on Earth.",
+      "Mozart was a prolific composer of classical music.",
+      "The Internet is a global network of computers.",
+      "Basketball is a sport played with a ball and a hoop.",
+      "The first computer virus was created in 1983.",
+      "Artificial neural networks are inspired by the human brain.",
+      "Deep learning is a subset of machine learning.",
+      "IBM's Watson won Jeopardy! in 2011.",
+      "The first computer programmer was Ada Lovelace.",
+      "The first chatbot was ELIZA, created in the 1960s.",
+    ].map((text) => ({ text }));
+
+    const factsSchema = lancedb.embedding.LanceSchema({
+      text: func.sourceField(new Utf8()),
+      vector: func.vectorField(),
+    });
+
+    const tbl = await db.createTable("facts", facts, {
+      mode: "overwrite",
+      schema: factsSchema,
+    });
+
+    const query = "How many bones are in the human body?";
+    const actual = await tbl.search(query).limit(1).toArray();
+
+    expect(actual[0].text).toBe("The human body has 206 bones.");
+  });
+}, 100_000);
+
+test.skip("embedding quickstart snippets", async () => {
+  // --8<-- [start:quickstart_connect]
+  const db = await lancedb.connect("data/sample-lancedb");
+  // --8<-- [end:quickstart_connect]
+
+  // --8<-- [start:quickstart_init_model]
+  const model = (await lancedb.embedding
+    .getRegistry()
+    .get("huggingface")
+    ?.create()) as lancedb.embedding.EmbeddingFunction;
+  // --8<-- [end:quickstart_init_model]
+
+  // --8<-- [start:quickstart_schema]
+  const wordsSchema = lancedb.embedding.LanceSchema({
+    text: model.sourceField(new Utf8()),
+    vector: model.vectorField(),
+  });
+  // --8<-- [end:quickstart_schema]
+
+  // --8<-- [start:quickstart_create_table]
+  const table = await db.createEmptyTable("words", wordsSchema, {
+    mode: "overwrite",
+  });
+  await table.add([{ text: "hello world" }, { text: "goodbye world" }]);
+  // --8<-- [end:quickstart_create_table]
+
+  // --8<-- [start:quickstart_query]
+  const query = "greetings";
+  const actual = (await table.search(query).limit(1).toArray())[0];
+  console.log(actual.text);
+  // --8<-- [end:quickstart_query]
+});

docs/web-tests/ts/storage.test.ts

@@ -0,0 +1,187 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+import { expect, test } from "@jest/globals";
+import * as lancedb from "@lancedb/lancedb";
+
+// --8<-- [start:storage_connect_s3]
+async function storageConnectS3() {
+  const db = await lancedb.connect("s3://bucket/path");
+  return db;
+}
+// --8<-- [end:storage_connect_s3]
+
+// --8<-- [start:storage_connect_gcs]
+async function storageConnectGcs() {
+  const db = await lancedb.connect("gs://bucket/path");
+  return db;
+}
+// --8<-- [end:storage_connect_gcs]
+
+// --8<-- [start:storage_connect_azure]
+async function storageConnectAzure() {
+  const db = await lancedb.connect("az://bucket/path");
+  return db;
+}
+// --8<-- [end:storage_connect_azure]
+
+// --8<-- [start:storage_connect_timeout]
+async function storageConnectTimeout() {
+  const db = await lancedb.connect("s3://bucket/path", {
+    storageOptions: { timeout: "60s" },
+  });
+  return db;
+}
+// --8<-- [end:storage_connect_timeout]
+
+// --8<-- [start:storage_table_timeout]
+async function storageTableTimeout() {
+  const db = await lancedb.connect("s3://bucket/path");
+  const table = await db.createTable(
+    "table",
+    [{ a: 1, b: 2 }],
+    { storageOptions: { timeout: "60s" } },
+  );
+  return table;
+}
+// --8<-- [end:storage_table_timeout]
+
+// --8<-- [start:storage_s3_ddb]
+async function storageS3Ddb() {
+  const db = await lancedb.connect(
+    "s3+ddb://bucket/path?ddbTableName=my-dynamodb-table",
+  );
+  return db;
+}
+// --8<-- [end:storage_s3_ddb]
+
+// --8<-- [start:storage_s3_ddb_local]
+async function storageS3DdbLocal() {
+  const db = await lancedb.connect(
+    "s3+ddb://bucket/path?ddbTableName=my-dynamodb-table",
+    {
+      storageOptions: {
+        endpoint: "http://localhost:4566",
+        dynamodbEndpoint: "http://localhost:4566",
+        allowHttp: "true",
+      },
+    },
+  );
+  return db;
+}
+// --8<-- [end:storage_s3_ddb_local]
+
+// --8<-- [start:storage_s3_sse_kms]
+async function storageS3SseKms() {
+  const db = await lancedb.connect("s3://bucket/path", {
+    storageOptions: {
+      awsServerSideEncryption: "aws:kms",
+      awsSseKmsKeyId: "<your-kms-key-id-or-arn>",
+    },
+  });
+  return db;
+}
+// --8<-- [end:storage_s3_sse_kms]
+
+// --8<-- [start:storage_azure_sas]
+async function storageAzureSas() {
+  const db = await lancedb.connect(
+    "az://my-container/my-database",
+    {
+      storageOptions: {
+        azureStorageAccountName: "some-account",
+        azureStorageSasToken: "<sas-token>",
+      },
+    },
+  );
+  return db;
+}
+// --8<-- [end:storage_azure_sas]
+
+// --8<-- [start:storage_s3_minio]
+async function storageS3Minio() {
+  const db = await lancedb.connect("s3://bucket/path", {
+    storageOptions: {
+      region: "us-east-1",
+      endpoint: "http://minio:9000",
+    },
+  });
+  return db;
+}
+// --8<-- [end:storage_s3_minio]
+
+// --8<-- [start:storage_s3_express]
+async function storageS3Express() {
+  const db = await lancedb.connect(
+    "s3://my-bucket--use1-az4--x-s3/path",
+    {
+      storageOptions: {
+        region: "us-east-1",
+        s3Express: "true",
+      },
+    },
+  );
+  return db;
+}
+// --8<-- [end:storage_s3_express]
+
+// --8<-- [start:storage_gcs_service_account]
+async function storageGcsServiceAccount() {
+  const db = await lancedb.connect(
+    "gs://my-bucket/my-database",
+    {
+      storageOptions: {
+        serviceAccount: "path/to/service-account.json",
+      },
+    },
+  );
+  return db;
+}
+// --8<-- [end:storage_gcs_service_account]
+
+// --8<-- [start:storage_azure_account]
+async function storageAzureAccount() {
+  const db = await lancedb.connect(
+    "az://my-container/my-database",
+    {
+      storageOptions: {
+        accountName: "some-account",
+        accountKey: "some-key",
+      },
+    },
+  );
+  return db;
+}
+// --8<-- [end:storage_azure_account]
+
+// --8<-- [start:storage_tigris_connect]
+async function storageTigrisConnect() {
+  const db = await lancedb.connect(
+    "s3://your-bucket/path",
+    {
+      storageOptions: {
+        endpoint: "https://t3.storage.dev",
+        region: "auto",
+      },
+    },
+  );
+  return db;
+}
+// --8<-- [end:storage_tigris_connect]
+
+test("storage ts snippets compile", async () => {
+  expect(storageConnectS3).toBeDefined();
+  expect(storageConnectGcs).toBeDefined();
+  expect(storageConnectAzure).toBeDefined();
+  expect(storageConnectTimeout).toBeDefined();
+  expect(storageTableTimeout).toBeDefined();
+  expect(storageS3Ddb).toBeDefined();
+  expect(storageS3DdbLocal).toBeDefined();
+  expect(storageS3Minio).toBeDefined();
+  expect(storageS3Express).toBeDefined();
+  expect(storageS3SseKms).toBeDefined();
+  expect(storageGcsServiceAccount).toBeDefined();
+  expect(storageAzureAccount).toBeDefined();
+  expect(storageAzureSas).toBeDefined();
+  expect(storageTigrisConnect).toBeDefined();
+});
+

docs/web-tests/ts/tsconfig.json

@@ -0,0 +1,17 @@
+{
+  "include": ["*.test.ts"],
+  "compilerOptions": {
+    "target": "es2022",
+    "module": "NodeNext",
+    "declaration": true,
+    "outDir": "./dist",
+    "strict": true,
+    "allowJs": true,
+    "resolveJsonModule": true,
+    "emitDecoratorMetadata": true,
+    "experimentalDecorators": true,
+    "moduleResolution": "NodeNext",
+    "allowImportingTsExtensions": true,
+    "emitDeclarationOnly": true
+  }
+}

docs/web-tests/ts/util.ts

@@ -0,0 +1,16 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+import * as fs from "node:fs";
+import { tmpdir } from "node:os";
+import * as path from "node:path";
+
+export async function withTempDirectory(
+  fn: (tempDir: string) => Promise<void>,
+) {
+  const tmpDirPath = fs.mkdtempSync(path.join(tmpdir(), "temp-dir-"));
+  try {
+    await fn(tmpDirPath);
+  } finally {
+    fs.rmSync(tmpDirPath, { recursive: true });
+  }
+}

docs/web/README.md

@@ -0,0 +1,35 @@
+# LanceDB documentation
+
+The open-source pages of [docs.lancedb.com](https://docs.lancedb.com), beside the
+code they describe.
+
+## Preview locally
+
+```bash
+npm i -g mint      # https://mintlify.com/docs/installation
+cd docs/web && mint dev
+```
+
+That serves this directory on its own. Enterprise pages live in a private
+repository and are merged in when the published site is assembled, so links to
+`/enterprise/...` and the SQL and Geneva sections will not resolve here. Every
+open-source page does.
+
+## Code examples
+
+Examples are not written into the pages. They live in real tests under
+`docs/web-tests/{py,ts,rs}`, are extracted into `docs/web/snippets/`, and are
+imported by the pages. Edit the test, then regenerate:
+
+```bash
+uv run docs/web-tests/mdx_snippets_gen.py -s docs/web-tests/py
+```
+
+An example that does not compile fails the pull request that broke it.
+
+## Section anchors
+
+Every heading carries an explicit `{#anchor}`. Those anchors are how the
+Enterprise pages attach their additions to the right section, so they are
+assigned once and not regenerated — leave an existing one alone even if the
+heading it sits on is reworded.

docs/web/agent-branch-experiments.mdx

@@ -0,0 +1,100 @@
+---
+title: "Run experiments on branches"
+sidebarTitle: "Run experiments on branches"
+description: "Use LanceDB branches to isolate agent-driven experiments from main, evaluate them on a fixed test set, and promote only the winner."
+icon: "flask"
+keywords: ["AI agents", "branches", "experiments", "embeddings", "evaluation"]
+---
+
+LanceDB supports [branching](/tables/branching) to isolate experiments from the main table when working with agents.
+Branches are useful when you instruct an agent to try several approaches without
+affecting the production data on the `main` branch.
+Each experiment gets its own writable table history.
+
+For example, you can generate and compare embeddings from two text embedding
+models on the given table:
+
+| Branch | Model | Added column |
+| --- | --- | --- |
+| `embed-minilm` | `sentence-transformers/all-MiniLM-L6-v2` | `vector_minilm` |
+| `embed-nomic` | `nomic-embed-text` from Ollama | `vector_nomic` |
+
+The branches share a logical table, but each has its own schema and history.
+The original table handle still points to `main`. LanceDB does not have a
+process-wide current branch.
+
+Install the model packages, then download the `nomic-embed-text` model from Ollama
+(or any other model you prefer):
+
+```bash
+uv add sentence-transformers ollama
+ollama pull nomic-embed-text
+```
+
+It's always recommended to measure the results of an experiment on a fixed evaluation set.
+Here's an example of how you could ask the agent to run both experiments and compare the results:
+
+```text Agent prompt
+Use the lancedb plugin and the LanceDB branching documentation to compare two
+embedding experiments on the `camelot_multimodal` table.
+
+1. Fork `embed-minilm` from `main`. Build normalized embeddings from
+   `"{role}. {description}"` with
+   `sentence-transformers/all-MiniLM-L6-v2` and add them as `vector_minilm`.
+2. Fork `embed-nomic` from `main`. Embed the identical text with Ollama
+   `nomic-embed-text` and add the results as `vector_nomic`.
+3. Process rows in batches and write only through each branch-scoped table
+   handle. Verify that neither new column appears on `main`.
+4. Evaluate both branches with the queries "wise magical advisor",
+   "treacherous rebel", and "virtuous Grail knight". Report the top three names,
+   latency, and whether the expected character ranks first.
+5. Do not modify `main` and do not delete either branch. Recommend a winner
+   based on the results.
+```
+
+<Note>
+The vector dimensions may differ because the columns live on separate branches.
+Use a different column name for each model. Within an experiment, use the same
+model for the stored text and the search queries.
+</Note>
+
+## Apply the winning experiment to `main` {#apply-the-winning-experiment-to-main}
+
+The branch experiments leave you with the results side by side, and they
+deliberately never touch `main`. Once you've picked a winner, apply it to
+`main` yourself by rerunning that experiment's validated transformation
+directly against the `main` table. Rerunning the reviewed operation is the
+reliable path on both OSS and Enterprise: it replays exactly the transformation
+you validated on the branch, and it works the same regardless of how the branch
+evolved.
+
+Suppose `nomic-embed-text` wins. Rerun the same Nomic embedding transformation
+you validated on `embed-nomic` against `main` in batches, then call
+`table.optimize()` on OSS. Verify a bounded vector search on `main`, and keep
+the losing branch around until you're confident in the result.
+
+See [Branches](/tables/branching) for more on how branches, versions, and tags
+relate.
+
+## More experiments you can run {#more-experiments-you-can-run}
+
+Swapping embedding models on a branch while working with agents is only one example
+of what you can do with the LanceDB plugin. The table below shows other experiments
+you can run with the plugin.
+
+| Hypothesis | Change on the branch | What to evaluate |
+| --- | --- | --- |
+| A different embedding model improves retrieval | Add a new vector column | Recall, ranking quality, latency, and cost |
+| A new parser or OCR model improves source data | Reprocess files and backfill metadata | Validation failures and a reviewed sample |
+| A new search setup works better | Build an index or change hybrid-search and reranking settings | Relevance and latency on a fixed query set |
+| A curation rule improves the corpus | Deduplicate, classify, or filter records | False positives, false negatives, and row counts |
+| A migration is safe to deploy | Add columns or run a backfill | Schema compatibility and application checks |
+
+For each experiment, ask the agent to state the hypothesis, use a fixed
+evaluation set, report the comparison, and, if you want, work on an alternate
+branch that's isolated from `main` until you decide you want the new derived
+column in `main`.
+
+Once you've chosen a winner, rerun that validated transformation to ingest it into `main`
+yourself, on either LanceDB OSS or Enterprise. See our documentation on [branches](/tables/branching)
+for more information.

docs/web/api-reference/index.mdx

@@ -0,0 +1,48 @@
+---
+title: SDKs and REST API Reference
+sidebarTitle: "SDKs"
+description: "SDK and REST API reference for LanceDB Enterprise and OSS."
+
+---
+
+For detailed information of the available functions and methods in your preferred language's SDKs,
+refer to the API documentation linked below.
+
+If you're looking for a REST API reference, visit the [REST API](/api-reference/rest) page.
+
+If you're looking for conceptual and practical namespace guidance before diving into method signatures, see
+[Namespaces and Catalog Model](/namespaces) and [Using Namespaces in SDKs](/namespaces/usage).
+
+## Supported SDKs {#supported-sdks}
+
+Python, Typescript and Rust SDKs are officially supported by LanceDB. You can use these SDKs to interact with both LanceDB OSS and Enterprise deployments.
+
+| Reference | Description |
+|:--------------|-------------------|
+| <Icon icon="python" /> [Python](https://lancedb.github.io/lancedb/python/python/) | Full-featured Python client with pandas & numpy integration |
+| <Icon icon="square-js" /> [Typescript](https://lancedb.github.io/lancedb/js/) | A TypeScript wrapper around the Rust library, built with `napi-rs`
+| <Icon icon="rust" /> [Rust](https://docs.rs/lancedb/latest/lancedb/index.html) | Native Rust library with persistent-storage and high performance |
+
+## REST API SDKs {#rest-api-sdks}
+
+<Badge color="red">Enterprise</Badge>
+
+REST API-based SDKs provide a convenient way to interact with LanceDB Enterprise deployments using the Lance REST Namespace API.
+
+| Reference | Description |
+|:--------------|-------------------|
+| <Icon icon="java" /> [Java](https://lancedb.github.io/lancedb/java/java/)| REST API Enterprise SDK in Java |
+
+## Community-driven SDKs {#community-driven-sdks}
+
+In addition to the officially supported SDKs, the LanceDB community may contribute SDKs in other languages.
+These SDKs may not have the same level of support or feature parity as the official ones supported by LanceDB, but they can be an option
+for users working in languages other than those listed above.
+
+| Reference | Description |
+|:--------------|-------------------|
+| <Icon icon="golang" /> [Go](https://pkg.go.dev/github.com/lancedb/lancedb-go/pkg/lancedb) | Community-contributed Go SDK for LanceDB |
+| <Icon icon="gem" /> [Ruby](https://github.com/scientist-labs/lancelot) | Community-contributed Ruby bindings for LanceDB |
+| <Icon icon="swift" /> [Swift](https://github.com/RyanLisse/LanceDbSwiftKit) | Community-contributed Swift SDK for LanceDB |
+| <Icon icon="R" /> [R](https://github.com/CathalByrneGit/lancedb) | Community-contributed R package for LanceDB |
+| <Icon icon="flutter" /> [Flutter](https://github.com/Alexcn/flutter_lancedb) | Community-contributed Flutter bindings for LanceDB |

docs/web/api-reference/rest/index.mdx

@@ -0,0 +1,84 @@
+---
+title: REST API Reference
+sidebarTitle: "Overview"
+description: "API reference for LanceDB"
+
+---
+
+[Lance REST Namespace](https://lance.org/format/namespace/) spec
+is an OpenAPI protocol that enables reading, writing and managing Lance tables by connecting
+those metadata services or building a custom metadata server in a standardized way.
+
+LanceDB OSS allows you to interface with Lance tables via the REST Namespace.
+LanceDB Enterprise provides an extended REST API with
+additional endpoints for managing tables and data.
+If you have specific needs or questions about the Enterprise REST API Namespace,
+please [contact us](mailto:support@lancedb.com).
+
+## Authentication {#authentication}
+
+<Badge color="red">Enterprise</Badge>
+
+All HTTP requests to LanceDB APIs must contain an <u>x-api-key</u> header that specifies a valid API key and
+must be encoded as JSON or Arrow RPC.
+
+To authenticate to the Enterprise REST API, you need the endpoint for your deployment and a valid API key for that deployment.
+
+### Get your Enterprise credentials {#get-your-enterprise-credentials}
+
+1. Obtain the following values from your LanceDB administrator or the LanceDB team that provisioned your Enterprise deployment:
+   - an API key
+   - your Enterprise REST endpoint
+   - your database name, if your deployment uses a private endpoint or `host_override`
+
+2. Export those values in your terminal:
+
+```bash
+export LANCEDB_API_KEY="<your-api-key>"
+export LANCEDB_URI="https://your-enterprise-endpoint.com"
+export LANCEDB_DATABASE="your-database-name"
+```
+
+3. If your Enterprise deployment is private, connect through the private network endpoint provided for your deployment. For example, Azure Private Link deployments commonly use a private IP or an internal DNS name as the endpoint.
+
+### Verify authentication {#verify-authentication}
+
+4. Check that you can reach the deployment and list tables:
+
+```bash
+curl -X GET "$LANCEDB_URI/v1/tables" \
+   -H "Content-Type: application/json" \
+   -H "x-api-key: $LANCEDB_API_KEY" \
+   -H "x-lancedb-database: $LANCEDB_DATABASE"
+```
+
+If your deployment endpoint already includes the database host name, you can omit the `x-lancedb-database` header.
+
+5. Create a table to confirm write access. Let's call it `words`.
+
+```bash
+curl -X POST "$LANCEDB_URI/v1/tables/words" \
+   -H "Content-Type: application/vnd.apache.arrow.stream" \
+   -H "x-api-key: $LANCEDB_API_KEY" \
+   -H "x-lancedb-database: $LANCEDB_DATABASE"
+```
+
+6. Check that the table has been created:
+
+```bash
+curl -X GET "$LANCEDB_URI/v1/tables" \
+   -H "Content-Type: application/json" \
+   -H "x-api-key: $LANCEDB_API_KEY" \
+   -H "x-lancedb-database: $LANCEDB_DATABASE"
+```
+
+That's it -- you're connected! Now, you can start adding data and querying it.
+You can visit the tutorial section to build your own applications with LanceDB.
+
+<Card
+   title="Tutorials"
+   icon="book"
+   href="/tutorials/"
+>
+   Check out our tutorials on building various applications with LanceDB.
+</Card>

docs/web/build-with-ai-agents.mdx

@@ -0,0 +1,270 @@
+---
+title: "Tutorial: Use the LanceDB agent plugin"
+sidebarTitle: "LanceDB agent plugin"
+description: "Install the LanceDB plugin and use an AI coding agent to quickly build a multimodal ingestion pipeline."
+icon: "robot"
+keywords: ["AI agents", "agent plugins", "multimodal", "Pydantic", "branches", "embeddings"]
+---
+
+import {
+  PyCamelotSchema,
+  PyCamelotBatches,
+  PyCamelotOssIngestion,
+} from '/snippets/build_with_ai_agents.mdx';
+
+The LanceDB agent plugin gives coding agents a maintained reference for the
+Python and TypeScript APIs. It also covers portable OSS and Enterprise code,
+ingestion performance, and branch operations. It ships as the `lancedb` plugin in
+the [lancedb-agent-plugins](https://github.com/lancedb/lancedb-agent-plugins)
+repository, which is also a plugin marketplace. Install it with your agent's
+plugin manager:
+
+<CodeGroup>
+```text Claude Code
+/plugin marketplace add lancedb/lancedb-agent-plugins
+/plugin install lancedb@lancedb
+```
+
+```bash Codex icon="terminal"
+codex plugin marketplace add lancedb/lancedb-agent-plugins
+codex plugin add lancedb@lancedb
+```
+
+```bash Other agents icon="terminal"
+npx plugins add lancedb/lancedb-agent-plugins
+```
+</CodeGroup>
+
+The [`plugins`](https://www.npmjs.com/package/plugins) installer shown under
+**Other agents** is a cross-tool option. It detects which agent CLIs are on your
+`PATH` — Cursor, GitHub Copilot CLI, VS Code, Grok Build, and Kimi Code, as well
+as Claude Code and Codex — and installs through each one's native plugin system.
+
+The plugin supplements the agent's training data with current LanceDB
+instructions. To pick up later revisions, refresh the marketplace:
+
+<CodeGroup>
+```text Claude Code
+/plugin marketplace update lancedb
+```
+
+```bash Codex icon="terminal"
+codex plugin marketplace upgrade
+```
+
+```bash Other agents icon="terminal"
+npx plugins add lancedb/lancedb-agent-plugins
+```
+</CodeGroup>
+
+<Accordion title="Where the plugin gets installed">
+Each plugin manager clones the repository, reads its marketplace manifest to find
+the `lancedb` plugin, and hands the plugin to the agent's own plugin store rather
+than copying files into your project. Claude Code and Codex keep it under
+`~/.claude/plugins` and `~/.codex/plugins` respectively, so the plugin is available
+in every project on the machine.
+
+The `npx plugins` installer defaults to the same user-wide scope. Pass
+`--scope project` to record the plugin in the current repository instead, so that
+everyone working in it gets the same plugin, or `-t <target>` to install for a
+single agent rather than every one it detects:
+
+```bash
+npx plugins add lancedb/lancedb-agent-plugins --scope project -t cursor
+```
+
+Not every agent supports project scope; `npx plugins targets` lists what it
+found and what each target supports.
+</Accordion>
+
+## Get started with the LanceDB agent plugin {#get-started-with-the-lancedb-agent-plugin}
+
+This tutorial uses the Camelot dataset from the [quickstart](/quickstart), with
+a portrait added for each character. Each LanceDB row contains validated
+metadata and raw JPEG bytes. Text, images, and any embeddings you add later
+remain in the same table.
+
+### 1\. Download the multimodal dataset {#1-download-the-multimodal-dataset}
+
+From a new project directory, download the JSON file and portraits:
+
+```bash
+mkdir -p data/img
+
+BASE_URL="https://docs.lancedb.com/static/assets/tutorials/build-with-ai-agents/camelot/data"
+curl -fsSL "$BASE_URL/camelot.json" -o data/camelot.json
+
+for image in \
+  arthur.jpg \
+  guinevere.jpg \
+  merlin.jpg \
+  mordred.jpg \
+  sir_galahad.jpg \
+  sir_gawain.jpg \
+  sir_lancelot.jpg \
+  sir_percival.jpg
+do
+  curl -fsSL "$BASE_URL/img/$image" -o "data/img/$image"
+done
+```
+
+Each JSON record has this shape:
+
+```json
+{
+  "id": 2,
+  "name": "Merlin",
+  "role": "Wizard and Advisor",
+  "description": "A powerful wizard and prophet who mentors Arthur.",
+  "stats": {
+    "strength": 2,
+    "courage": 4,
+    "magic": 5,
+    "wisdom": 5
+  },
+  "img": "data/img/merlin.jpg"
+}
+```
+
+JSON input may have missing fields, unexpected fields, or values of the wrong
+type. The LanceDB plugin tells the agent to validate each record with strict
+Pydantic models before writing it.
+
+After the agent writes the pipeline, inspect the schema, batching, and write
+path rather than assuming it followed the plugin's guidance correctly.
+
+### 2\. Prompt your agent to build the pipeline {#2-prompt-your-agent-to-build-the-pipeline}
+
+Install the Python packages used by the example:
+
+```bash icon="terminal"
+uv init
+uv add lancedb pyarrow pydantic
+```
+
+If you're using LanceDB Enterprise, ask the agent to ingest into an Enterprise table,
+provide the relevant environment variables for connecting to your Enterprise deployment
+in a local `.env` file, and point the agent to it.
+
+```text .env
+LANCEDB_URI=db://your_project_name
+LANCEDB_API_KEY=your_api_key_here
+LANCEDB_REGION=us-east-1
+LANCEDB_HOST_OVERRIDE=https://your-enterprise-endpoint.com
+```
+
+If you're using LanceDB OSS, no connection settings are required, as it runs as an
+embedded retrieval library. A simple prompt like this should work:
+
+```text Agent prompt
+# If using OSS
+Use the lancedb plugin to ingest the dataset in `data/` into a LanceDB
+OSS table.
+
+# If using enterprise
+Use the lancedb plugin to ingest the dataset in `data/` into a LanceDB
+Enterprise table using the connection information in `.env`.
+```
+
+Because the plugin is registered with the agent's own plugin system, the agent
+should pick it up on its own once you restart the session. If it does not,
+simply ask it to use the `lancedb` plugin in the prompt, as shown above.
+
+That should be enough! The agent will create `ingest_multimodal.py`, or similar.
+The following sections inspect the script to verify that it follows the plugin's guidance.
+
+#### Data validation {#data-validation}
+
+The plugin encourages the agent to validate each record with Pydantic before writing it.
+The agent should ideally define a schema for the table and a nested schema for the
+`stats` field.
+
+<CodeBlock filename="ingest_multimodal.py" language="Python" icon="python">
+  {PyCamelotSchema}
+</CodeBlock>
+
+In this case, our agent correctly defined `Character` and `Stats` Pydantic models
+and validated the JSON before adding it to the table.
+
+#### Batched ingestion {#batched-ingestion}
+
+Naively calling `table.add()` once per row is slow, and is considered an anti-pattern
+in LanceDB. The plugin encourages the agent to collect incoming rows into batches and
+write them with a single `table.add()` call. When you use the plugin, the agent should
+produce something like this:
+
+<CodeBlock filename="ingest_multimodal.py" language="Python" icon="python">
+  {PyCamelotBatches}
+</CodeBlock>
+
+The script calls `Character.model_validate(...)` before adding a record to the
+batch. If validation fails, that batch is never written. The function yields up
+to `batch_size` rows at a time, providing an iterable of batches for the ingestion step,
+shown next.
+
+#### Table maintenance {#table-maintenance}
+For LanceDB OSS, the plugin instructs the agent to call
+`table.optimize()` after the ingestion loop. This compacts small fragments, cleans up
+old versions according to the retention policy, and incorporates new data into indexes.
+
+<CodeBlock filename="ingest_multimodal.py" language="Python" icon="python">
+  {PyCamelotOssIngestion}
+</CodeBlock>
+
+If you're using LanceDB Enterprise, the plugin mentions that this step is not needed
+because LanceDB Enterprise handles maintenance automatically.
+
+
+<Note>
+This example dataset has only eight rows, so the default batch size writes it in
+one call. Larger inputs should still avoid single-row write commits.
+</Note>
+
+### 3\. Run the OSS pipeline {#3-run-the-oss-pipeline}
+
+```bash
+uv run python ingest_multimodal.py
+```
+
+The table now contains the validated character records and their JPEG bytes.
+Here are the first three rows:
+
+| Image | Character | Role |
+| --- | --- | --- |
+| <img src="/static/assets/tutorials/build-with-ai-agents/camelot/data/img/arthur.jpg" alt="King Arthur" width="128" height="128" /> | King Arthur | King of Camelot |
+| <img src="/static/assets/tutorials/build-with-ai-agents/camelot/data/img/merlin.jpg" alt="Merlin" width="128" height="128" /> | Merlin | Wizard and Advisor |
+| <img src="/static/assets/tutorials/build-with-ai-agents/camelot/data/img/guinevere.jpg" alt="Queen Guinevere" width="128" height="128" /> | Queen Guinevere | Queen of Camelot |
+
+Once your pipeline works, you can [run experiments on branches](/agent-branch-experiments)
+to try new embedding models, parsers, or search settings without touching `main`.
+
+## Takeaways {#takeaways}
+
+The example in this tutorial was small, but similar ideas apply to other workflows, too.
+Give the agent the data source, the constraints it must respect, and the
+artifacts it should return.
+
+The plugin supplies LanceDB-specific guidance, but it's the user's responsibility to
+ensure the output makes sense for the application.
+
+### Try the plugin with your own dataset {#try-the-plugin-with-your-own-dataset}
+
+The plugin shown in this tutorial should generalize reasonably well to other use cases.
+If you find any issues, open [an issue](https://github.com/lancedb/lancedb-agent-plugins/issues)
+on GitHub, clearly describing the intended behavior.
+
+You can choose OSS or Enterprise based on how the work will run:
+
+Start with [LanceDB OSS](/quickstart) during the early stages of a project
+when an agent is helping you prototype,
+explore a dataset, or run small workflows on a subset of the data. The application owns
+the storage and lifecycle work, so ask the agent to validate inputs, write in
+batches, and it will include table maintenance operations such as `optimize()`
+where appropriate.
+
+Choose [LanceDB Enterprise](/enterprise) when the resulting table becomes
+shared production infrastructure, and the workload needs distributed capacity,
+private deployment, or platform-managed operations. The underlying data format and table
+API stay the same, so the pipeline does not need to be redesigned. The agent
+instead connects to a remote `db://` table and lets the cluster handle
+maintenance and background work.

docs/web/demos/index.mdx

@@ -0,0 +1,78 @@
+---
+title: "Demo Application Gallery"
+sidebarTitle: "App Gallery"
+description: "Demo apps showcasing end-to-end applications built with LanceDB for production use cases."
+---
+
+Explore the demo applications built with LanceDB below.
+
+App | Description
+--- | ---
+[Semantic.Art](#semantic-art) | A multimodal art discovery platform using feelings, phrases, and images.
+[Wikipedia 41M Hybrid Search](#wikipedia-41m-hybrid-search) | An interactive hybrid search demo combining full-text search and vector search.
+[Video Search](#video-search) | A video search application that allows searching through a library of videos using natural language queries.
+
+
+## Semantic.art {#semantic-art}
+
+<Badge color="green">multimodal</Badge>
+<Badge color="red">hybrid-search</Badge>
+<Badge color="purple">vector-search</Badge>
+<Badge color="surface-destructive">semantic-routing</Badge>
+
+<Card
+  title="Semantic.art"
+  img="/static/assets/images/demos/sda_hero.jpg"
+  href="https://www.semantic.art/"
+>
+Semantic.art turns real, human-made art discovery into a multimodal search experience using
+feelings, phrases, and images. It's built with LanceDB hybrid search and semantic routing.
+
+<Card
+    title="Learn how it's built"
+    href="https://lancedb.com/blog/semanticdotart/"
+>
+Read in detail about how Semantic.art is built in this blog post.
+</Card>
+
+</Card>
+
+## Wikipedia 41M Hybrid Search {#wikipedia-41m-hybrid-search}
+
+<Badge color="green">multimodal</Badge>
+<Badge color="red">hybrid-search</Badge>
+<Badge color="purple">vector-search</Badge>
+<Badge color="gray">fts</Badge>
+
+<Card
+  title="Wikipedia 41M Hybrid Search"
+  img="/static/assets/images/demos/wiki_hero.png"
+  href="https://lancedb-demos.vercel.app/demo/wikipedia-search"
+>
+Interactive hybrid search, full-text search (FTS) and vector search demo with 41M+ Wikipedia entries.
+Explore the power of combining FTS with vector search for more relevant results.
+
+<Card
+    title="Learn how it's built"
+    href="https://lancedb.com/blog/feature-full-text-search/"
+>
+Read in detail about how the Wikipedia 41M hybrid search demo is built in this blog post.
+</Card>
+
+</Card>
+
+## Video Search {#video-search}
+
+<Badge color="green">multimodal</Badge>
+<Badge color="orange">video-search</Badge>
+<Badge color="purple">vector-search</Badge>
+
+<Card
+  title="Video Search"
+  img="/static/assets/images/demos/video_demo_hero.png"
+  href="https://lancedb-demos.vercel.app/demo/video-search"
+>
+Search through a library of videos using natural language queries. This demo showcases how to use LanceDB
+to perform semantic search on video content.
+
+</Card>

docs/web/docs.json

@@ -0,0 +1,338 @@
+{
+  "$schema": "https://mintlify.com/docs.json",
+  "appearance": {
+    "strict": false
+  },
+  "theme": "mint",
+  "name": "LanceDB",
+  "banner": {
+    "content": "[Why Multimodal Data Needs a Better Lakehouse? — Download the Research Study](https://lancedb.com/download/)",
+    "dismissible": true
+  },
+  "colors": {
+    "primary": "#FF6B35",
+    "light": "#FF8A5C",
+    "dark": "#E55A2B"
+  },
+  "fonts": {
+    "family": "Inter"
+  },
+  "styling": {
+    "codeblocks": {
+      "theme": {
+        "light": "vitesse-light",
+        "dark": "catppuccin-mocha"
+      }
+    }
+  },
+  "favicon": "/static/favicon.ico",
+  "navigation": {
+    "tabs": [
+      {
+        "tab": "Documentation",
+        "groups": [
+          {
+            "group": "Get started",
+            "pages": [
+              "quickstart",
+              {
+                "group": "What is LanceDB?",
+                "pages": [
+                  "index",
+                  "lance",
+                  "tables-and-namespaces"
+                ]
+              },
+              "performance"
+            ]
+          },
+          {
+            "group": "Model training",
+            "pages": [
+              "training/why-lancedb",
+              "training/index",
+              "training/torch",
+              "training/object-detection",
+              "training/vlm-finetuning"
+            ]
+          },
+          {
+            "group": "Guides",
+            "pages": [
+              {
+                "group": "Table operations",
+                "pages": [
+                  "tables/index",
+                  "tables/create",
+                  "tables/multimodal",
+                  "tables/schema",
+                  "tables/update",
+                  "tables/versioning",
+                  "tables/branching",
+                  "tables/consistency"
+                ]
+              },
+              {
+                "group": "Namespaces",
+                "pages": [
+                  "namespaces/index",
+                  "namespaces/usage"
+                ]
+              },
+              {
+                "group": "Embeddings",
+                "pages": [
+                  "embedding/index",
+                  "embedding/quickstart"
+                ]
+              },
+              {
+                "group": "Indexing",
+                "pages": [
+                  "indexing/index",
+                  "indexing/vector-index",
+                  "indexing/fts-index",
+                  "indexing/scalar-index",
+                  "indexing/quantization",
+                  "indexing/reindexing"
+                ]
+              },
+              {
+                "group": "Search",
+                "pages": [
+                  "search/index",
+                  "search/vector-search",
+                  "search/multivector-search",
+                  "search/full-text-search",
+                  "search/fts-examples",
+                  "search/hybrid-search",
+                  "search/filtering",
+                  "search/optimize-queries"
+                ]
+              },
+              {
+                "group": "Reranking",
+                "pages": [
+                  "reranking/index",
+                  "reranking/rrf",
+                  "reranking/linear_combination",
+                  "reranking/mrr",
+                  "reranking/cross_encoder",
+                  "reranking/custom-reranker",
+                  "reranking/eval"
+                ]
+              },
+              {
+                "group": "Storage",
+                "pages": [
+                  "storage/index",
+                  "storage/configuration",
+                  "storage/monitoring"
+                ]
+              },
+              {
+                "group": "Build with AI agents",
+                "expanded": true,
+                "pages": [
+                  "build-with-ai-agents",
+                  "agent-branch-experiments"
+                ]
+              }
+            ]
+          },
+          {
+            "group": "Support",
+            "pages": [
+              "troubleshooting",
+              {
+                "group": "FAQ",
+                "pages": [
+                  "faq/index",
+                  "faq/faq-oss"
+                ]
+              }
+            ]
+          }
+        ]
+      },
+      {
+        "tab": "Integrations",
+        "groups": [
+          {
+            "group": "Integrations",
+            "pages": [
+              "integrations/index",
+              {
+                "group": "Embedding Providers",
+                "pages": [
+                  "integrations/embedding/huggingface",
+                  "integrations/embedding/aws",
+                  "integrations/embedding/cohere",
+                  "integrations/embedding/colpali",
+                  "integrations/embedding/gemini",
+                  "integrations/embedding/ibm",
+                  "integrations/embedding/imagebind",
+                  "integrations/embedding/instructor",
+                  "integrations/embedding/jina",
+                  "integrations/embedding/ollama",
+                  "integrations/embedding/openai",
+                  "integrations/embedding/openclip",
+                  "integrations/embedding/sentence-transformers",
+                  "integrations/embedding/voyageai",
+                  "integrations/embedding/superlinked"
+                ]
+              },
+              {
+                "group": "Rerankers",
+                "pages": [
+                  "integrations/reranking/answerdotai",
+                  "integrations/reranking/cohere",
+                  "integrations/reranking/colbert",
+                  "integrations/reranking/jina",
+                  "integrations/reranking/openai",
+                  "integrations/reranking/voyageai",
+                  "integrations/reranking/watsonx"
+                ]
+              },
+              {
+                "group": "Data Platforms & Frameworks",
+                "expanded": true,
+                "pages": [
+                  "integrations/data/pydantic",
+                  "integrations/data/duckdb",
+                  "integrations/data/pandas_and_pyarrow",
+                  "integrations/data/polars_arrow",
+                  "integrations/data/dlt",
+                  "integrations/data/voxel51"
+                ]
+              },
+              {
+                "group": "AI Platforms & Frameworks",
+                "expanded": true,
+                "pages": [
+                  "integrations/ai/agno",
+                  "integrations/ai/hermes-agent",
+                  "integrations/ai/huggingface",
+                  "integrations/ai/langchain",
+                  "integrations/ai/llamaIndex",
+                  "integrations/ai/genkit",
+                  "integrations/ai/kiln",
+                  "integrations/ai/prompttools",
+                  "integrations/ai/synthetic-data-kit"
+                ]
+              }
+            ]
+          }
+        ]
+      },
+      {
+        "tab": "Tutorials",
+        "groups": [
+          {
+            "group": "Tutorials",
+            "pages": [
+              "tutorials/index",
+              {
+                "group": "Search",
+                "expanded": true,
+                "pages": [
+                  "tutorials/search/index",
+                  "tutorials/search/multivector-needle-in-a-haystack"
+                ]
+              },
+              {
+                "group": "RAG & Agents",
+                "expanded": true,
+                "pages": [
+                  "tutorials/agents/index",
+                  "tutorials/agents/nvidia-rag-blueprint/index",
+                  "tutorials/agents/time-travel-rag/index",
+                  "tutorials/agents/multimodal-agent/index"
+                ]
+              },
+              "tutorials/feature-engineering/index"
+            ]
+          }
+        ]
+      },
+      {
+        "tab": "Demos",
+        "groups": [
+          {
+            "group": "Demos",
+            "pages": [
+              "demos/index"
+            ]
+          }
+        ]
+      },
+      {
+        "tab": "Use Cases",
+        "groups": [
+          {
+            "group": "Robotics",
+            "pages": [
+              "integrations/lerobotdataset"
+            ]
+          },
+          {
+            "group": "World Models",
+            "pages": [
+              "integrations/stable-worldmodel"
+            ]
+          }
+        ]
+      },
+      {
+        "tab": "API Reference",
+        "groups": [
+          {
+            "group": "API Reference",
+            "pages": [
+              "api-reference/index",
+              {
+                "group": "REST API",
+                "expanded": true,
+                "pages": [
+                  "api-reference/rest/index"
+                ]
+              }
+            ]
+          }
+        ]
+      }
+    ]
+  },
+  "logo": {
+    "light": "/static/assets/logo/dark-lancedb-logo.svg",
+    "dark": "/static/assets/logo/light-lancedb-logo.svg",
+    "href": "https://lancedb.com"
+  },
+  "background": {
+    "color": {
+      "light": "#FAF5F0",
+      "dark": "#000000"
+    }
+  },
+  "navbar": {
+    "links": [
+      {
+        "label": "Support",
+        "href": "mailto:contact@lancedb.com"
+      }
+    ]
+  },
+  "footer": {
+    "socials": {
+      "github": "https://github.com/lancedb",
+      "linkedin": "https://www.linkedin.com/company/lancedb",
+      "x": "https://x.com/lancedb",
+      "discord": "https://discord.gg/AUEWnJ7Txb"
+    }
+  },
+  "integrations": {
+    "ga4": {
+      "measurementId": "G-1TMC2PR69E"
+    }
+  }
+}

docs/web/embedding/index.mdx

@@ -0,0 +1,239 @@
+---
+title: "Managing Embeddings"
+sidebarTitle: Overview
+description: "Use the embedding API in LanceDB -- registry, functions, schemas, and multi-language SDK support."
+icon: "bars"
+---
+
+import {
+    PyOpenaiEmbeddings,
+    PyCreateEmbeddingFunction,
+    PyManualQuerySearch,
+    PyEmbeddingFunction,
+    PyRegisterDevice,
+    PyRegisterSecret,
+    TsOpenaiEmbeddings,
+    TsCreateEmbeddingFunction,
+    TsManualQuerySearch,
+    TsEmbeddingFunction,
+    TsRegisterModelFallback,
+    TsRegisterSecret,
+    RsOpenaiEmbeddings,
+    RsCreateEmbeddingFunction,
+    RsManualQuerySearch,
+    RsEmbeddingFunction,
+} from '/snippets/embedding.mdx';
+
+Modern machine learning models can be trained to convert raw data into embeddings, which are vectors
+of floating point numbers. The position of an embedding in vector space captures the semantics of
+the data, so vectors that are close to each other are considered similar.
+
+LanceDB provides an embedding function registry in OSS as well as its Enterprise versions
+([see below](#embeddings-in-lancedb-enterprise))
+that automatically generates vector embeddings during data ingestion. Automatic query-time embedding
+generation is available in LanceDB OSS, with SDK-specific query ergonomics. The API abstracts
+embedding generation, allowing you to focus on your application logic.
+
+## Embedding Registry {#embedding-registry}
+
+You can get a supported embedding function from the registry, and then use it in your table schema. 
+Once configured, the embedding function will automatically generate embeddings when you insert data
+into the table. Query-time behavior depends on SDK: Python/TypeScript can query with text directly,
+while Rust examples typically compute query embeddings explicitly before vector search.
+
+<CodeGroup>
+    <CodeBlock filename="Python" language="python" icon="python">
+    {PyOpenaiEmbeddings}
+    </CodeBlock>
+
+    <CodeBlock filename="TypeScript" language="typescript" icon="square-js">
+    {TsOpenaiEmbeddings}
+    </CodeBlock>
+
+    <CodeBlock filename="Rust" language="rust" icon="rust">
+    {RsOpenaiEmbeddings}
+    </CodeBlock>
+</CodeGroup>
+
+### Using an embedding function {#using-an-embedding-function}
+
+Create an embedding function before you attach it to table or schema metadata. Python and TypeScript fetch
+provider implementations from the embedding registry, while Rust constructs the provider embedding function
+directly and registers it on the connection before using it in an `EmbeddingDefinition`.
+
+<CodeGroup>
+    <CodeBlock filename="Python" language="python" icon="python">
+    {PyCreateEmbeddingFunction}
+    </CodeBlock>
+
+    <CodeBlock filename="TypeScript" language="typescript" icon="square-js">
+    {TsCreateEmbeddingFunction}
+    </CodeBlock>
+
+    <CodeBlock filename="Rust" language="rust" icon="rust">
+    {RsCreateEmbeddingFunction}
+    </CodeBlock>
+</CodeGroup>
+
+Provider configuration is SDK-specific, so copy the option names from the provider page for the SDK you use.
+For example, the OpenAI model is selected with `name` in Python, `model` in TypeScript, and the model argument
+to `OpenAIEmbeddingFunction::new_with_model` in Rust.
+
+| Concept | Python | TypeScript | Rust |
+| --- | --- | --- | --- |
+| Model | `name="text-embedding-3-small"` | `{ model: "text-embedding-3-small" }` | `new_with_model(api_key, "text-embedding-3-small")` |
+| Retry count | `max_retries=7` | Provider/client-specific | Provider/client-specific |
+| API key | `api_key="..."`, environment variables, or `$var:` | `apiKey: "..."`, environment variables, or `$var:` | Constructor argument or environment variable |
+| Device | Provider-specific, for example `device="cuda"` | Provider-specific | Provider-specific |
+
+When ingesting data with an embedding definition, LanceDB only computes the vector column if that
+column is missing from the incoming batch or present but entirely null. If you provide any non-null
+values in the vector column, LanceDB treats the column as user-supplied and does not backfill the
+remaining rows in that batch.
+
+For reusable runtime configuration, the registry also supports `$var:` placeholders in embedding-function config.
+This is useful for provider secrets and environment-specific settings in Python and TypeScript.
+
+- Python uses `registry.set_var(...)`.
+- TypeScript uses `registry.setVar(...)`.
+- You can provide a fallback with `$var:name:default`.
+- Sensitive values such as API keys should be passed through registry variables instead of hardcoding them in config.
+
+<CodeGroup>
+    <CodeBlock filename="Python" language="python" icon="python">
+    {PyRegisterSecret}
+    </CodeBlock>
+
+    <CodeBlock filename="TypeScript" language="typescript" icon="square-js">
+    {TsRegisterSecret}
+    </CodeBlock>
+</CodeGroup>
+
+For non-sensitive settings such as inference device selection, you can also use a default fallback:
+
+<CodeGroup>
+    <CodeBlock filename="Python" language="python" icon="python">
+    {PyRegisterDevice}
+    </CodeBlock>
+
+    <CodeBlock filename="TypeScript" language="typescript" icon="square-js">
+    {TsRegisterModelFallback}
+    </CodeBlock>
+</CodeGroup>
+
+Find the full list of arguments for each provider in the [integrations](/integrations/embedding) section.
+
+## Multiple embedding columns {#multiple-embedding-columns}
+
+A single table can include more than one embedding definition when you want to store multiple semantic views
+of the same data, or generate embeddings from different source columns. In practice, each embedding definition
+maps one source column to one vector column, and the table schema can contain multiple such pairs.
+
+The exact setup differs by SDK, but the underlying pattern is the same: define a distinct source/vector pair
+for each embedding function you want applied during ingest.
+
+In TypeScript, automatic query embedding currently uses the first embedding function stored in the
+table metadata. If a table has multiple embedding definitions and you need to query a specific vector
+column, compute the query embedding explicitly and pass the vector to the search builder.
+
+## Embedding model providers {#embedding-model-providers}
+
+LanceDB supports most popular embedding providers.
+
+### Text embeddings {#text-embeddings}
+
+| Provider | Model ID | Default Model |
+|----------|----------|---------------|
+| OpenAI | `openai` | `text-embedding-ada-002` |
+| Sentence Transformers | `sentence-transformers` | `all-MiniLM-L6-v2` |
+| Hugging Face | `huggingface` | `colbert-ir/colbertv2.0` |
+| Cohere | `cohere` | `embed-english-v3.0` |
+| ... | ... | ... |
+
+### Multimodal embedding {#multimodal-embedding}
+
+| Provider | Model ID | Supported Inputs |
+|----------|----------|------------------|
+| OpenCLIP | `open-clip` | Text, Images |
+| ImageBind | `imagebind` | Text, Images, Audio, Video |
+| ... | ... | ... |
+
+You can find all supported embedding models in the [integrations](/integrations/embedding) section.
+
+## Embeddings in LanceDB Enterprise {#embeddings-in-lancedb-enterprise}
+<Badge color="red">Enterprise</Badge>
+
+In LanceDB Enterprise, embedding generation during data ingestion is client-side and the resulting vectors are
+stored on the remote table.
+
+<Info>
+The Enterprise server does not currently generate embeddings from query text on its own. Any automatic
+query-time embedding happens on the client side.
+</Info>
+
+### How string queries are interpreted {#how-string-queries-are-interpreted}
+
+For the Python remote client, `table.search("hello")` can take two different paths:
+
+- If the selected vector column has embedding metadata
+  (i.e., the table schema stores the source-column, vector-column, and
+  embedding-function mapping created from fields like `SourceField()` and
+  `VectorField()` during table creation), then the embeddings are computed in the Python client process.
+  The client uses the same local LanceDB embedding registry used by OSS tables to
+  reconstruct the embedding function from schema metadata, compute the query vector in
+  the client process, and send that vector to Enterprise for search.
+  ```python
+  result = table.search("hello").limit(1).to_list()
+  # The Python client computes the query embedding locally, then sends a vector search.
+  ```
+- If the table does not have embedding metadata for that search, `table.search("hello")` in `auto` mode is
+  treated as an FTS query instead.
+  ```python
+  result = table.search("hello").limit(5).to_list()
+  # In auto mode this is treated as an FTS query, not a vector query.
+  ```
+
+If you want explicit vector or hybrid behavior and the client cannot resolve an embedding function from the
+table metadata, generate the query embedding yourself and pass the vector directly.
+
+TypeScript has a similar string-query distinction in `auto` mode: if no embedding providers have been
+registered in the process, `search("text")` falls back to FTS. Once an embedding provider has been
+imported and registered, the client expects table embedding metadata for automatic vector search and
+raises an error if the table has none.
+
+The manual query-embedding flow below works across Enterprise SDKs and is an explicit path you can use when you
+want full control over query-time behavior.
+
+<CodeGroup>
+    <CodeBlock filename="Python" language="python" icon="python">
+    {PyManualQuerySearch}
+    </CodeBlock>
+
+    <CodeBlock filename="TypeScript" language="typescript" icon="square-js">
+    {TsManualQuerySearch}
+    </CodeBlock>
+
+    <CodeBlock filename="Rust" language="rust" icon="rust">
+    {RsManualQuerySearch}
+    </CodeBlock>
+</CodeGroup>
+
+## Custom Embedding Functions {#custom-embedding-functions}
+
+You can always implement your own embedding function:
+- Python/TypeScript: subclass `TextEmbeddingFunction` (text) or `EmbeddingFunction` (multimodal).
+- Rust: implement the `EmbeddingFunction` trait.
+
+<CodeGroup>
+    <CodeBlock filename="Python" language="python" icon="python">
+    {PyEmbeddingFunction}
+    </CodeBlock>
+
+    <CodeBlock filename="TypeScript" language="typescript" icon="square-js">
+    {TsEmbeddingFunction}
+    </CodeBlock>
+
+    <CodeBlock filename="Rust" language="rust" icon="rust">
+    {RsEmbeddingFunction}
+    </CodeBlock>
+</CodeGroup>

docs/web/embedding/quickstart.mdx

@@ -0,0 +1,155 @@
+---
+title: "Embeddings: Quickstart"
+sidebarTitle: "Quickstart"
+description: "Quickstart guide for generating and working with embeddings."
+icon: "rocket"
+---
+
+import {
+    TsQuickstartConnect,
+    TsQuickstartCreateTable,
+    TsQuickstartImports,
+    TsQuickstartInitModel,
+    TsQuickstartQuery,
+    TsQuickstartSchema,
+} from '/snippets/sentence-transformers.mdx';
+
+LanceDB will automatically vectorize the data both at ingestion and query time. All you need to do is specify which model to use.
+Popular embedding models like OpenAI, Hugging Face, Sentence Transformers, CLIP, and more, are supported.
+
+## Step 1: Import Required Libraries {#step-1-import-required-libraries}
+
+First, import the necessary LanceDB components:
+
+<CodeGroup>
+```python Python icon="python"
+import lancedb
+from lancedb.pydantic import LanceModel, Vector
+from lancedb.embeddings import get_registry
+```
+
+    <CodeBlock filename="TypeScript" language="typescript" icon="square-js">
+    {TsQuickstartImports}
+    </CodeBlock>
+</CodeGroup>
+
+- `lancedb`: The main database connection and operations
+- `LanceModel`: Pydantic model for defining table schemas
+- `Vector`: Field type for storing vector embeddings
+- `get_registry()`: Access to the embedding function registry. It has all the supported as well as custom embedding functions registered by the user
+- TypeScript uses `lancedb.embedding.getRegistry()` and `lancedb.embedding.LanceSchema()` for the same registry/schema workflow
+- In TypeScript, import the provider module before calling `getRegistry().get(...)`; the provider import is what registers names such as `"huggingface"` or `"openai"`
+
+## Step 2: Connect to LanceDB {#step-2-connect-to-lancedb}
+
+Establish a connection to your LanceDB OSS directory or Enterprise cluster:
+
+<CodeGroup>
+```python Python icon="python"
+# Enter your LanceDB connection URI for OSS or Enterprise here
+db = lancedb.connect(...)
+```
+
+    <CodeBlock filename="TypeScript" language="typescript" icon="square-js">
+    {TsQuickstartConnect}
+    </CodeBlock>
+</CodeGroup>
+
+## Step 3: Initialize the Embedding Function {#step-3-initialize-the-embedding-function}
+
+Choose and configure your embedding model:
+
+<CodeGroup>
+```python Python icon="python"
+model = get_registry().get("sentence-transformers").create(name="BAAI/bge-small-en-v1.5", )
+```
+
+    <CodeBlock filename="TypeScript" language="typescript" icon="square-js">
+    {TsQuickstartInitModel}
+    </CodeBlock>
+</CodeGroup>
+
+This creates an embedding function from the local embedding registry. The Python snippet uses the
+`sentence-transformers` provider with the BGE model; the TypeScript snippet uses the Transformers-backed
+`huggingface` provider. You can:
+- Change `"sentence-transformers"` to other providers like `"openai"`, `"cohere"`, etc.
+- Modify the model name for different embedding models
+- Set `device="cuda"` for GPU acceleration if available
+
+## Step 4: Define Your Schema {#step-4-define-your-schema}
+
+Create a Pydantic model that defines your table structure:
+
+<CodeGroup>
+```python Python icon="python"
+class Words(LanceModel):
+    text: str = model.SourceField()  
+    vector: Vector(model.ndims()) = model.VectorField()
+```
+
+    <CodeBlock filename="TypeScript" language="typescript" icon="square-js">
+    {TsQuickstartSchema}
+    </CodeBlock>
+</CodeGroup>
+
+- `SourceField()`: This field will be embedded
+- `VectorField()`: This stores the embeddings
+- `model.ndims()`: Sets vector dimensions for your model
+- In TypeScript, use `model.sourceField(...)` and `model.vectorField()` inside `LanceSchema(...)`
+
+## Step 5: Create Table and Ingest Data {#step-5-create-table-and-ingest-data}
+
+Create a table with your schema and add data:
+
+<CodeGroup>
+```python Python icon="python"
+table = db.create_table("words", schema=Words)
+table.add([
+    {"text": "hello world"},
+    {"text": "goodbye world"}
+])
+```
+
+    <CodeBlock filename="TypeScript" language="typescript" icon="square-js">
+    {TsQuickstartCreateTable}
+    </CodeBlock>
+</CodeGroup>
+
+The `table.add()` call automatically:
+- Takes the text from each document
+- Generates embeddings using your chosen model
+- Stores both the original text and the vector embeddings
+
+If your input already includes the vector column, automatic embedding only runs when that column is
+absent or entirely null for the batch. Partially supplied vectors are treated as manual data, so
+LanceDB preserves them instead of filling only the missing rows.
+
+## Step 6: Query with Automatic Embedding {#step-6-query-with-automatic-embedding}
+
+Note: On LanceDB Enterprise, the server does not generate embeddings from query text. In the Python remote
+client, `table.search("greetings")` can still work when the table schema includes embedding metadata, because
+the client computes the query embedding before sending the vector search. If there is no embedding metadata for
+that search, `search("greetings")` in `auto` mode is treated as FTS instead.
+
+Search your data using natural language queries:
+
+<CodeGroup>
+```python Python icon="python"
+query = "greetings"
+actual = table.search(query).limit(1).to_pydantic(Words)[0]
+print(actual.text)
+```
+
+    <CodeBlock filename="TypeScript" language="typescript" icon="square-js">
+    {TsQuickstartQuery}
+    </CodeBlock>
+</CodeGroup>
+
+The search process:
+1. Automatically converts your query text to embeddings
+2. Finds the most similar vectors in your table
+3. Returns the matching documents
+
+Automatic text search depends on the table's embedding metadata. If the client cannot reconstruct an
+embedding function from that metadata, compute the query embedding yourself and search with the
+vector directly.

docs/web/faq/faq-oss.mdx

@@ -0,0 +1,90 @@
+---
+title: "LanceDB: Frequently Asked Questions"
+sidebarTitle: "LanceDB OSS"
+description: "Commonly asked questions about LanceDB OSS."
+icon: "code"
+mode: wide
+---
+
+This section covers some common questions and issues that you may encounter when using LanceDB.
+
+### Is LanceDB open source? {#is-lancedb-open-source}
+
+LanceDB OSS is a permissively licensed embedded retrieval library available under an Apache 2.0 license. We also have a LanceDB Enterprise, a commercial product that can be deployed on a private cloud or a bring-your-own-cloud (BYOC) solution. LanceDB Enterprise transforms your data lake into a high-performance multimodal lakehouse.
+
+### What is the difference between Lance and LanceDB? {#what-is-the-difference-between-lance-and-lancedb}
+
+[Lance](https://github.com/lancedb/lance) is a modern lakehouse format for multimodal AI. It's perfect for building search engines, feature stores and being the foundation of large-scale ML training jobs requiring high performance IO and shuffles. It also has native support for storing, querying, and inspecting deeply nested data for robotics or large blobs like images, point clouds, and more.
+
+LanceDB is the multimodal lakehouse that's built on top of Lance, and utilizes the underlying optimized storage format to build efficient disk-based indexes that power semantic search & retrieval applications, from RAGs to QA bots to recommender systems.
+
+### Why invent another data format instead of using Parquet? {#why-invent-another-data-format-instead-of-using-parquet}
+
+As we mention in our talk titled "[Lance, a modern columnar data format](https://www.youtube.com/watch?v=ixpbVyrsuL8)", Parquet and other tabular formats that derive from it are rather dated (Parquet is over 10 years old), especially when it comes to random access on vectors. We needed a format that's able to handle the complex trade-offs involved in shuffling, scanning, OLAP and filtering large datasets involving vectors, and our extensive experiments with Parquet didn't yield sufficient levels of performance for modern ML. [Our benchmarks](https://lancedb.com/blog/benchmarking-random-access-in-lance/) show that Lance is up to 1000x faster than Parquet for random access, which we believe justifies our decision to create a new data format for AI.
+
+### Why build in Rust? {#why-build-in-rust}
+
+We believe that the Rust ecosystem has attained mainstream maturity and that Rust will form the underpinnings of large parts of the data and ML landscape in a few years. Performance, latency and reliability are paramount to a vector DB, and building in Rust allows us to iterate and release updates more rapidly due to Rust's safety guarantees. Both Lance (the data format) and LanceDB (the database) are written entirely in Rust. We also provide Python, JavaScript, and Rust client libraries to interact with the database.
+
+### What makes LanceDB different? {#what-makes-lancedb-different}
+
+LanceDB is among the few embedded vector DBs out there that we believe can unlock a whole new class of LLM-powered applications in the browser or via edge functions. Lance's multimodal nature allows you to store the raw data, metadata and the embeddings all at once, unlike other solutions that typically store just the embeddings and metadata.
+
+The Lance data format that powers our storage system also provides true zero-copy access and seamless interoperability with numerous other data formats (like Pandas, Polars, Pydantic) via Apache Arrow, as well as automatic data versioning and data management without needing extra infrastructure.
+
+### How large of a dataset can LanceDB handle? {#how-large-of-a-dataset-can-lancedb-handle}
+
+LanceDB and its underlying data format, Lance, are built to scale to really large amounts of data. LanceDB OSS can comfortably handle millions of vectors on a single node, making it a great fit for most applications. Its disk-based indexes keep performance strong without requiring expensive infrastructure.
+
+If you need to scale to hundreds of millions of vectors or work with terabytes of data, we recommend [LanceDB Enterprise](/enterprise). Enterprise customers regularly operate on billions of rows, backed by distributed infrastructure designed for large-scale production workloads.
+
+### Do I need to build a vector index to run vector search? {#do-i-need-to-build-a-vector-index-to-run-vector-search}
+
+No. LanceDB is blazing fast (due to its disk-based index) for even brute force kNN search, within reason. In our benchmarks, computing 100K pairs of 1000-dimension vectors takes less than 20ms. For small datasets of ~100K records or applications that can accept ~100ms latency, a vector index is usually not necessary.
+
+For large-scale (>1M) or higher dimension vectors, it is beneficial to create a vector index. See the [Vector Indexes](/indexing/vector-index/) section for more details.
+
+### How can I speed up data inserts? {#how-can-i-speed-up-data-inserts}
+
+LanceDB auto-parallelizes large writes when you call `table.add()` with materialized
+data such as `pa.Table`, `pd.DataFrame`, or `pa.dataset()`. No extra configuration
+is needed — writes are automatically split into partitions of ~1M rows or 2GB.
+
+For best results:
+
+- **Create an empty table first**, then call `table.add()`. The `add()` path enables
+  automatic write parallelism, while passing data directly to `create_table()` does not.
+- **For file-based data**, use `pyarrow.dataset.dataset("path/to/data/", format="parquet")`
+  so LanceDB can stream from disk without loading everything into memory.
+- **Avoid inserting one row at a time.** Each insert creates a new data fragment on
+  disk. Batch your data into Arrow tables, DataFrames, or use iterators.
+
+See [Loading Large Datasets](/tables/create#loading-large-datasets) for full examples.
+
+### Do I need to set a refine factor when using an index? {#do-i-need-to-set-a-refine-factor-when-using-an-index}
+
+Yes. LanceDB uses PQ, or Product Quantization, to compress vectors and speed up search when using an ANN index. However, because PQ is a lossy compression algorithm, it tends to reduce recall while also reducing the index size. To address this trade-off, we introduce a process called **refinement**. The normal process computes distances by operating on the compressed PQ vectors. The refinement factor (*rf*) is a multiplier that takes the top-k similar PQ vectors to a given query, fetches `rf * k` *full* vectors and computes the raw vector distances between them and the query vector, reordering the top-k results based on these scores instead.
+
+For example, if you're retrieving the top 10 results and set `refine_factor` to 25, LanceDB will fetch the 250 most similar vectors (according to PQ), compute the distances again based on the full vectors for those 250 and then re-rank based on their scores. This can significantly improve recall, with a small added latency cost (typically a few milliseconds), so it's recommended you set a `refine_factor` of anywhere between 5-50 and measure its impact on latency prior to deploying your solution.
+
+### How can I improve IVF-PQ recall while keeping latency low? {#how-can-i-improve-ivf-pq-recall-while-keeping-latency-low}
+
+When using an IVF-PQ index, there's a trade-off between recall and latency at query time. You can improve recall by increasing the number of probes and the `refine_factor`. In our benchmark on the GIST-1M dataset, we show that it's possible to achieve >0.95 recall with a latency of under 10 ms on most systems, using ~50 probes and a `refine_factor` of 50. This is, of course, subject to the dataset at hand and a quick sensitivity study can be performed on your own data. You can find more details on the benchmark in a past [blog post](https://medium.com/etoai/benchmarking-lancedb-92b01032874a).
+
+![](/static/assets/images/faq/recall-vs-latency.webp)
+
+### How much data can LanceDB practically manage without affecting performance? {#how-much-data-can-lancedb-practically-manage-without-affecting-performance}
+
+We target good performance on ~10-50 billion rows and ~10-30 TB of data. For the best performance and
+scalability guarantees, check out [LanceDB Enterprise](/enterprise).
+
+### Does LanceDB support concurrent operations? {#does-lancedb-support-concurrent-operations}
+
+LanceDB can handle concurrent reads very well, and can scale horizontally. The main constraint is how well the storage layer you've chosen, scales. For writes, we support concurrent writing, though too many concurrent writers can lead to failing writes as there is a limited number of times a writer retries a commit.
+
+<Warning>
+If you use Python's multiprocessing, you should probably not use `fork` as Lance is multi-threaded
+internally and `fork` and multi-threaded Python do not work well together.
+[Refer to this discussion](https://discuss.python.org/t/concerns-regarding-deprecation-of-fork-with-alive-threads/33555)
+for more information.
+</Warning>

docs/web/faq/index.mdx

@@ -0,0 +1,17 @@
+---
+title: Frequently Asked Questions
+sidebarTitle: FAQ
+description: "Common questions about LanceDB"
+icon: "question-circle"
+---
+
+Find answers to common questions about LanceDB across different deployment options and use cases.
+
+<Card title="Can't find what you're looking for?" icon="question">
+Reach out on [Discord](https://discord.gg/AUEWnJ7Txb) for community support or [contact us](mailto:support@lancedb.com) for Enterprise assistance.
+</Card>
+
+| Category | Description |
+|:---------|:------------|
+| [LanceDB OSS](/faq/faq-oss) | Questions about LanceDB open source deployment, installation, and community support |
+| [LanceDB Enterprise](/faq/faq-enterprise) | Questions about LanceDB Enterprise features, security, compliance, and support |

docs/web/index.mdx

@@ -0,0 +1,124 @@
+---
+title: LanceDB
+sidebarTitle: "LanceDB"
+description: "Multimodal lakehouse for AI."
+icon: "/static/assets/logo/lancedb-icon-gray.svg"
+keywords: ["multimodal lakehouse", "training", "feature engineering", "search", "open source", "oss"]
+---
+
+**LanceDB** is a [multimodal lakehouse](https://lancedb.com/blog/multimodal-lakehouse/) for AI teams that need
+one data layer for curation, feature engineering, search and retrieval, and model training.
+It is built on top of [Lance](/lance), an open-source lakehouse format designed for multimodal AI data.
+
+Move from data exploration to model training on one, unified platform without needing to manage a
+fragmented stack of storage, feature, retrieval, and training systems.
+
+## Build better models, faster {#build-better-models-faster}
+
+Training data and experimentation slow down when raw data, metadata, embeddings, features, and governance
+artifacts live in separate systems. LanceDB keeps them together in one versioned multimodal table, so AI teams spend less
+time stitching infrastructure together and more time improving datasets, testing features, and keeping GPUs fed.
+
+![Training data lifecycle: Curation, Feature Engineering, Search and Retrieval, Training](/static/assets/images/overview/training-data-lifecycle.svg)
+
+Use the same table to curate training data, add derived features, retrieve examples, and feed training jobs that rely on expensive GPUs.
+Training workloads can sample, shuffle, and scan projected columns from local storage or object storage, then assemble
+GPU-ready batches from a tagged dataset version.
+
+For a deeper look at how this works in training pipelines, start with [Why LanceDB for training](/training/why-lancedb).
+
+## LanceDB suite {#lancedb-suite}
+
+The LanceDB suite includes LanceDB OSS, an open-source embedded retrieval library, and LanceDB Enterprise,
+a multimodal lakehouse platform for the full AI data lifecycle.
+OSS is easy to set up on a local machine for search and regular-scale workflows. LanceDB Enterprise is built
+for teams that need scale without building bespoke infrastructure for curation,
+feature engineering, search and retrieval, and efficient training data access.
+
+![LanceDB suite: OSS search and Enterprise multimodal lakehouse on Lance format](/static/assets/images/overview/lancedb-suite.svg)
+
+## Why teams use LanceDB {#why-teams-use-lancedb}
+
+<Steps>
+  <Step title="One table for the whole AI data loop">
+    Store images, video, audio, text, annotations, embeddings, and model-generated features together in one schema-enforced table.
+    The same table can support dataset curation, feature backfills, experiment splits, retrieval, and training.
+  </Step>
+  <Step title="High-throughput data access for training">
+    Training workloads mix fast random access with high-throughput sequential scans. LanceDB is designed for both, so
+    teams can shuffle data into GPU-ready batches more efficiently, improve input throughput, and iterate on experiments faster.
+  </Step>
+  <Step title="Fast, versatile search and retrieval">
+    Whether the end user is a human or an agent, LanceDB powers production retrieval workloads such as semantic search,
+    hybrid search, RAG, agent memory, and recommendation systems. Retrieval runs against the same LanceDB tables used
+    for curation, feature engineering, and training workflows.
+  </Step>
+</Steps>
+
+## Start with your workload {#start-with-your-workload}
+
+<CardGroup cols={2}>
+  <Card title="Train and fine-tune models" icon="fire" href="/training/why-lancedb">
+    Learn why LanceDB works well as the data layer for training workloads.
+  </Card>
+  <Card title="Load data into PyTorch" icon="boxes-stacked" href="/training/">
+    Use LanceDB tables and permutations for projected, shuffled, random-access training reads.
+  </Card>
+  <Card title="Browse ready-to-use datasets" icon="database" href="/datasets">
+    Explore Lance-formatted multimodal datasets with raw bytes, metadata, embeddings, and indices.
+  </Card>
+  <Card title="Build search and retrieval" icon="search" href="/search/">
+    Use vector search, full-text search, hybrid search, reranking, filtering, and SQL.
+  </Card>
+</CardGroup>
+
+## From local development to production scale {#from-local-development-to-production-scale}
+
+LanceDB OSS and LanceDB Enterprise share the same Lance format and table model. Start locally with the embedded OSS
+library, then move to Enterprise when your team needs distributed scale, managed infrastructure, private deployment,
+or higher-throughput curation, feature engineering, search and retrieval, and training workflows.
+
+### 1\. LanceDB OSS {#1-lancedb-oss}
+The fastest way to get started is the open-source embedded library, with client SDKs in Python, TypeScript
+and Rust. Run it locally in just a few steps, which lets you explore datasets, curate data, and run search and retrieval workloads
+for agents. Start here:
+
+<Columns cols={2}>
+  <Card
+    title="Quickstart"
+    icon="rocket"
+    href="/quickstart"
+  >
+    Get started with LanceDB in minutes.
+  </Card>
+  <Card
+    title="Basic Table Operations"
+    icon="table"
+    href="/tables/"
+  >
+    Create tables, evolve schemas, version data, and modify rows in LanceDB.
+  </Card>
+</Columns>
+
+### 2\. LanceDB Enterprise {#2-lancedb-enterprise}
+
+[LanceDB Enterprise](/enterprise) is a petabyte-scale (and beyond), distributed **multimodal lakehouse** platform built for
+search, curation, feature engineering, and high-throughput training data access workflows on top of the same core table
+abstraction. This eliminates the need for teams to build bespoke infrastructure to manage large multimodal datasets.
+To set up LanceDB Enterprise in your organization, reach out to us at
+[contact@lancedb.com](mailto:contact@lancedb.com).
+
+<Info>
+**Built with scale, performance, and security in mind.**
+
+LanceDB Enterprise is designed for very large-scale, high-performance, distributed workloads in
+private deployments, and can operate under strict [security requirements](/enterprise/security).
+</Info>
+
+<Card
+  title="Quickstart"
+  icon="rocket"
+  href="/quickstart"
+>
+  Get started with LanceDB in minutes, including Enterprise `db://` connections.
+</Card>

docs/web/indexing/fts-index.mdx

@@ -0,0 +1,165 @@
+---
+title: "Full-Text Search (FTS) Index"
+sidebarTitle: "FTS index"
+description: "Create and tune BM25-based full-text search indexes in LanceDB."
+icon: "book"
+---
+import { PyFtsIndexAsync as FtsIndexAsync, PyFtsIndexCreate as FtsIndexCreate, PyFtsIndexNested as FtsIndexNested, PyFtsIndexWait as FtsIndexWait } from '/snippets/indexing.mdx';
+
+LanceDB provides performant full-text search based on BM25, allowing you to incorporate keyword-based search in your retrieval solutions. This page shows
+examples on how to create and configure FTS indexes in LanceDB OSS and Enterprise, using the synchronous and asynchronous APIs.
+
+<Note>
+In LanceDB Enterprise, `create_fts_index` API returns immediately, but index building happens asynchronously.
+</Note>
+
+## Creating FTS Indexes {#creating-fts-indexes}
+
+### Synchronous API {#synchronous-api}
+
+Use `create_fts_index` with synchronous LanceDB connections:
+
+<CodeGroup>
+    <CodeBlock filename="Python" language="Python" icon="python">
+    {FtsIndexCreate}
+    </CodeBlock>
+</CodeGroup>
+
+Check FTS index status using the API:
+
+<CodeGroup>
+    <CodeBlock filename="Python" language="Python" icon="python">
+    {FtsIndexWait}
+    </CodeBlock>
+</CodeGroup>
+
+`wait_for_index(...)` waits until the named FTS index exists and `index_stats(...)` reports `num_unindexed_rows == 0`. It can time out if writes keep adding rows faster than the index catches up. If a table has multiple FTS indexes, specify the target text column when querying instead of relying on implicit selection.
+
+### Asynchronous API {#asynchronous-api}
+
+When using async connections (`connect_async`), use `create_index` with the `FTS` configuration:
+
+<CodeGroup>
+    <CodeBlock filename="Python" language="Python" icon="python">
+    {FtsIndexAsync}
+    </CodeBlock>
+</CodeGroup>
+
+<Note>
+The `create_fts_index` method is not available on `AsyncTable`. Use `create_index` with `FTS` config instead.
+</Note>
+
+The current FTS implementation is Lance-native. Legacy Tantivy-only options, including
+`use_tantivy`, are no longer accepted by the index creation APIs.
+
+## Nested field paths {#nested-field-paths}
+
+FTS indexes can target text leaves inside struct columns by passing a dotted path (for example, `payload.text`). The same path works for [`MatchQuery`](/search/full-text-search) and [`PhraseQuery`](/search/full-text-search), and for the `columns` argument on async `nearest_to_text` queries.
+
+You can point an index at any string leaf nested in a struct, regardless of depth. The struct container itself isn't indexable: you have to name a specific text field.
+
+<CodeGroup>
+    <CodeBlock filename="Python" language="Python" icon="python">
+    {FtsIndexNested}
+    </CodeBlock>
+</CodeGroup>
+
+LanceDB rejects paths that don't resolve to a text leaf:
+
+- A struct container (for example, `payload`): raises `ValueError: FTS index cannot be created ...`.
+- A non-text leaf such as an integer or float (for example, `payload.count`): raises the same error.
+- A path that doesn't exist in the schema (for example, `payload.missing`): raises `ValueError: Field path ... not found`.
+
+The async API accepts the same dotted paths through `create_index`:
+
+```python Python icon="python"
+from lancedb.index import FTS
+
+await async_table.create_index("payload.text", config=FTS(with_position=True))
+```
+
+## Configuration Options {#configuration-options}
+
+### FTS Parameters {#fts-parameters}
+
+| Parameter | Type | Default | Description |
+|:----------|:-----|:--------|:------------|
+| `with_position` | bool | `False` | Store token positions (required for phrase queries) |
+| `base_tokenizer` | str | `"simple"` | Text splitting method (`simple`, `whitespace`, `raw`, `ngram`, `icu`, `jieba/*`, or `lindera/*`) |
+| `language` | str | `"English"` | Language for stemming and stop-word filters. Choose CJK and mixed-language segmentation with `base_tokenizer`. |
+| `max_token_length` | int | `40` | Maximum token size; longer tokens are omitted |
+| `lower_case` | bool | `True` | Lowercase tokens |
+| `stem` | bool | `True` | Apply stemming (`running` → `run`) |
+| `remove_stop_words` | bool | `True` | Drop common stop words |
+| `ascii_folding` | bool | `True` | Normalize accented characters |
+| `custom_stop_words` | list[str] | `None` | Extra stop words to drop in addition to the language defaults. Requires `remove_stop_words=True`. |
+| `ngram_min_length` | int | `3` | Minimum n-gram length. Applies only when `base_tokenizer="ngram"`. |
+| `ngram_max_length` | int | `3` | Maximum n-gram length. Applies only when `base_tokenizer="ngram"`. |
+| `prefix_only` | bool | `False` | Index only prefix n-grams rather than all substrings. Applies only when `base_tokenizer="ngram"`. |
+| `block_size` | int | `128` | Number of documents per compressed posting block. Supported values are `128` and `256`. Setting this to `256` opts in to the experimental FTS V3 layout. |
+
+<Note title="Key parameters">
+- `max_token_length` can filter out base64 blobs or long URLs.
+- Disabling `with_position` reduces index size but disables phrase queries.
+- `ascii_folding` helps with international text (e.g., “café” → “cafe”).
+</Note>
+
+### Tokenizer choices {#tokenizer-choices}
+
+`base_tokenizer` controls segmentation before token filters run:
+
+- `simple`, `whitespace`, and `raw` cover common tokenization strategies for space-delimited text.
+- `ngram` indexes overlapping character spans for substring-style matching.
+- `icu` uses bundled ICU4X word segmentation for mixed-language text and scripts where whitespace splitting is not enough. ICU stands for [International Components for Unicode](https://icu.unicode.org/), and this tokenizer does not need external model files.
+- `jieba/*` is for Chinese word segmentation with Jieba.
+- `lindera/*` loads a compiled Lindera dictionary, such as `lindera/ipadic` for Japanese or `lindera/ko-dic` for Korean.
+
+Model-backed tokenizers such as `jieba/default`, `lindera/ipadic`, and `lindera/ko-dic` require tokenizer model files in Lance's language model home. Lance looks under the default platform data directory for `lance/language_models`, or you can set `LANCE_LANGUAGE_MODEL_HOME` to point to another model root. For example, `jieba/default` is resolved under `<model-home>/jieba/default/...`.
+
+`language` is used by token filters, not by the base tokenizer. Stemming supports Arabic, Danish, Dutch, English, Finnish, French, German, Greek, Hungarian, Italian, Norwegian, Portuguese, Romanian, Russian, Spanish, Swedish, Tamil, and Turkish. Built-in stop-word removal supports Danish, Dutch, English, Finnish, French, German, Hungarian, Italian, Norwegian, Portuguese, Russian, Spanish, and Swedish. For other stemming languages, set `remove_stop_words=False` or pass `custom_stop_words`.
+
+### Posting block size {#posting-block-size}
+
+`block_size` controls the number of documents packed into each compressed posting block on disk. The default of `128` matches the current FTS layout and is the right choice for most workloads. Setting it to `256` opts in to the experimental FTS V3 format, which changes how postings are encoded and may introduce breaking changes in future releases. Any other value is rejected at index creation time.
+
+You can set the option through either the synchronous or asynchronous API. In the async Python API, pass it on the `FTS` config, and in the TypeScript API use the camelCase `blockSize` field on `FtsOptions`:
+
+```python Python icon="python"
+from lancedb.index import FTS
+
+await async_table.create_index("text", config=FTS(block_size=256))
+```
+
+```typescript TypeScript icon="square-js"
+await table.createIndex("text", {
+  config: lancedb.Index.fts({ blockSize: 256 }),
+});
+```
+
+### Phrase Query Configuration {#phrase-query-configuration}
+
+Enable phrase queries by setting:
+
+| Parameter | Required Value | Purpose |
+|:----------|:---------------|:--------|
+| `with_position` | `True` | Track token positions for phrase matching |
+| `remove_stop_words` | `False` | Preserve stop words for exact phrase matching |
+
+## Indexing nested string fields {#indexing-nested-string-fields}
+
+You can build an FTS index on a string field inside a struct by passing its full dotted path, like `nested.text`. The same path is used when you query the index through `fts_columns`, and the indexed column is reported back as the full path from `list_indices()`.
+
+```python
+# Schema: pa.struct([pa.field("text", pa.string())]) stored under the `nested` column.
+table.create_fts_index("nested.text")
+
+results = (
+    table.search("puppy", query_type="fts", fts_columns="nested.text")
+    .limit(5)
+    .to_list()
+)
+```
+
+<Note>
+Use the canonical Lance path: dot-separate each struct field from root to leaf (for example, `metadata.author.name`). The same convention applies to scalar and vector indexes.
+</Note>

docs/web/indexing/index.mdx

@@ -0,0 +1,161 @@
+---
+title: "Indexing Data"
+sidebarTitle: "Overview"
+description: "Optimize search performance in LanceDB using vector indexes, full-text search, and scalar indexes. Understand IVF-PQ indexing for efficient vector similarity search."
+icon: "list"
+---
+
+An **index** is a data structure that facilitates efficient scans and lookups on the embeddings of a given dataset. LanceDB provides a comprehensive suite of indexes to optimize query performance across diverse workloads:
+
+- **Vector Index**: Optimized for searching high-dimensional data (like images, audio, or text embeddings) by efficiently finding the most similar vectors
+- **Full-Text Search Index**: Enables fast keyword-based searches by indexing words and phrases
+- **Scalar Index**: Accelerates filtering and sorting of structured numeric or categorical data (e.g., timestamps, prices)
+
+<Tip title="Working with scalar indices">
+Scalar indices serve as a foundational optimization layer, accelerating filtering across diverse search workloads. They can be combined with:
+
+- Vector search (prefilter or post-filter results using metadata)
+- Full-text search (combining keyword matching with structured filters) 
+- SQL scans (optimizing WHERE clauses on scalar columns)
+- Key-value lookups (enabling rapid primary key-based retrievals)
+</Tip>
+
+## Supported Index Types {#supported-index-types}
+
+LanceDB provides a comprehensive suite of indexing strategies for different data types and use cases:
+
+| Index | Use Case | Description |
+| :--------- | :------- | :---------- |
+| `IVF` (Vector) | Large-scale vector search with configurable accuracy/speed trade-offs. Supports binary vectors with hamming distance. | Inverted File Index—a partition-based approximate nearest neighbor algorithm that groups similar vectors into partitions for efficient search.<br />Distance metrics: `l2` `cosine` `dot` `hamming`<br />Quantizations: `None/Flat` `PQ` `SQ` `RQ`|
+| `IVF_HNSW` (Vector) | Large-scale vector search requiring both high recall and efficient partitioning. Combines the scalability of IVF with the search quality of HNSW. | Hybrid index combining IVF partitioning with HNSW graphs built within each partition. Provides improved search quality over pure IVF while maintaining scalability.<br />Distance metrics: `l2` `cosine` `dot`<br />Quantizations: `None/Flat` `SQ` `PQ`|
+| `FTS` (Full-text search) | String columns (e.g., title, description, content) requiring keyword-based search with BM25 ranking. | Full-text search index using BM25 ranking algorithm. Tokenizes text with configurable tokenization, stemming, stop word removal, and language-specific processing. |
+| `BTree` (Scalar) | Numeric, temporal, and string columns with mostly distinct values. Best for selective equality, inequality, and range predicates. | Sorted index storing sorted copies of scalar columns with block headers in a btree cache. Header entries map to blocks of rows (4096 rows per block) for efficient disk reads. |
+| `Bitmap` (Scalar) | Low-cardinality columns with few thousand or fewer distinct values. Accelerates equality and range filters. | Stores a bitmap for each distinct value in the column, with one bit per row indicating presence. Memory-efficient for low-cardinality data. |
+| `LabelList` (Scalar) | List columns (e.g., tags, categories, keywords) requiring `array_contains_all` or `array_contains_any` filters. | Scalar index for `List<T>` and `LargeList<T>` columns of primitive values, using an underlying bitmap index structure to enable fast array membership lookups. |
+| `FM` (Scalar) | String or binary columns that need raw substring search. | FM-Index over `Utf8`, `LargeUtf8`, `Binary`, or `LargeBinary` data for filters such as `contains(path, 'needle')`. Use FTS instead for tokenized word search and BM25 ranking. |
+
+<Note>
+TypeScript currently doesn't support `IvfSq` (IVF with Scalar Quantization).
+</Note>
+
+<Info>
+**Operational checks**
+
+For vector indexes, use the same distance metric when creating the index and searching it. After appends or other writes, use `optimize()` to fold new rows into existing indexes, then check `index_stats(...)` or `wait_for_index(...)` if you need to confirm the index has caught up. `wait_for_index(...)` waits until the named indexes exist and report `num_unindexed_rows == 0`; it can time out if writes keep adding unindexed rows.
+
+By default, automatic vector indexing creates `IVF_PQ`, and scalar index creation defaults to
+`BTree` unless you pass another scalar index config. `BTree` and `Bitmap` indexes target scalar
+columns, not list columns; use `LabelList` for list containment filters.
+</Info>
+
+### Quantization Types {#quantization-types}
+
+Vector indexes can use different quantization methods to compress vectors and improve search performance:
+
+| Quantization | Use Case | Description |
+| :----------- | :------- | :---------- |
+| `PQ` (Product Quantization) | Default choice for most vector search scenarios. Use when you need to balance index size and recall. | Divides vectors into subvectors and quantizes each subvector independently. Provides a good balance between compression ratio and search accuracy. |
+| `SQ` (Scalar Quantization) | Use when you need faster indexing or when vector dimensions have consistent value ranges. | Quantizes each dimension independently. Simpler than PQ but typically provides less compression. |
+| `RQ` (RabitQ Quantization) | Use when you need maximum compression or have specific per-dimension requirements. | Per-dimension quantization using a RabitQ codebook. Provides fine-grained control over compression per dimension. For `IVF_RQ`, vector dimensions must be divisible by `8`. |
+| `None/Flat` | Use for binary vectors (with `hamming` distance) or when you need maximum recall and have sufficient storage. | No quantization—stores raw vectors. Provides the highest accuracy but requires more storage and memory. |
+
+## Understanding the IVF-PQ Index {#understanding-the-ivf-pq-index}
+
+An ANN (Approximate Nearest Neighbors) index is a data structure that quickly produces an approximate solution to the **k-nearest neighbors (kNN)** problem. 
+It greatly improves upon the runtime of a brute-force kNN search, while admitting a slight decrease in accuracy. LanceDB uses the disk-based indexing technique IVF-PQ, discussed below.
+
+LanceDB differs from other vector databases in that it is built on top of [Lance](https://github.com/lancedb/lance), an open-source columnar data format designed for performant ML workloads and fast random access. Due to the design of Lance, LanceDB's indexing philosophy adopts a primarily *disk-based* indexing philosophy.
+
+## IVF-PQ {#ivf-pq}
+
+LanceDB uses **IVF-PQ** indexing, which combines the clustering-based **Inverted File Index (IVF)** with **Product Quantization (PQ)** to efficiently compress embeddings. 
+The implementation provides several parameters to fine-tune the index's size, query throughput, latency, and recall.
+
+### Product Quantization {#product-quantization}
+
+Quantization is a compression technique used to reduce the dimensionality of an embedding to speed up search.
+
+Product quantization (PQ) works by dividing a large, high-dimensional vector of size into equally sized subvectors. Each subvector is assigned a "reproduction value" that maps to the nearest centroid of points for that subvector. The reproduction values are then assigned to a codebook using unique IDs, which can be used to reconstruct the original vector.
+
+![](/static/assets/images/indexing/ivfpq_pq_desc.png)
+
+It's important to remember that quantization is a *lossy process*, i.e., the reconstructed vector is not identical to the original vector. This results in a trade-off between the size of the index and the accuracy of the search results.
+
+As an example, consider starting with 128-dimensional vector consisting of 32-bit floats. Quantizing it to an 8-bit integer vector with 4 dimensions as in the image above, we can significantly reduce memory requirements.
+
+<Note title="Effect of quantization">
+Original: `128 × 32 = 4096` bits
+Quantized: `4 × 8 = 32` bits
+
+Quantization results in a **128x** reduction in memory requirements for each vector in the index, which is substantial.
+</Note>
+
+### Inverted File Index (IVF) Implementation {#inverted-file-index-ivf-implementation}
+
+While PQ helps with reducing the size of the index, IVF primarily addresses search performance. The primary purpose of an inverted file index is to facilitate rapid and effective nearest neighbor search by narrowing down the search space.
+
+In IVF, the PQ vector space is divided into *Voronoi cells*, which are essentially partitions that consist of all the points in the space that are within a threshold distance of the given region's seed point. These seed points are initialized by running K-means over the stored vectors. The centroids of K-means turn into the seed points which then each define a region. These regions are then are used to create an inverted index that correlates each centroid with a list of vectors in the space, allowing a search to be restricted to just a subset of vectors in the index.
+
+![](/static/assets/images/indexing/ivfpq_ivf_desc.webp)
+
+During query time, depending on where the query lands in vector space, it may be close to the border of multiple Voronoi cells, which could make the top-k results ambiguous and span across multiple cells. To address this, the IVF-PQ introduces the `nprobe` parameter, which controls the number of Voronoi cells to search during a query. The higher the `nprobe`, the more accurate the results, but the slower the query.
+![](/static/assets/images/indexing/ivfpq_query_vector.webp)
+
+## HNSW Index Implementation {#hnsw-index-implementation}
+
+Approximate Nearest Neighbor (ANN) search is a method for finding data points near a given point in a dataset, though not always the exact nearest one. HNSW is one of the most accurate and fastest Approximate Nearest Neighbour search algorithms, It's beneficial in high-dimensional spaces where finding the same nearest neighbor would be too slow and costly.
+
+### Types of ANN Search Algorithms {#types-of-ann-search-algorithms}
+
+Approximate Nearest Neighbor (ANN) search is a method for finding data points near a given point in a dataset, though not always the exact nearest one. 
+For example, HNSW is an ANN index that performs well in high-dimensional spaces where other techniques prove too slow and costly.
+
+There are three main types of ANN search algorithms:
+
+* **Tree-based search algorithms**: Use a tree structure to organize and store data points.
+* **Hash-based search algorithms**: Use a specialized geometric hash table to store and manage data points. These algorithms typically focus on theoretical guarantees, and don't usually perform as well as the other approaches in practice.
+* **Graph-based search algorithms**: Use a graph structure to store data points, which can be a bit complex. 
+
+HNSW is a graph-based algorithm. All graph-based search algorithms rely on the idea of a k-nearest neighbor (or k-approximate nearest neighbor) graph, which we outline below.  
+HNSW also combines this with the ideas behind a classic 1-dimensional search data structure: the skip list.
+
+### Understanding k-Nearest Neighbor Graphs {#understanding-k-nearest-neighbor-graphs}
+
+The k-nearest neighbor graph actually predates its use for ANN search. Its construction is quite simple:
+
+* Each vector in the dataset is given an associated vertex.
+* Each vertex has outgoing edges to its k nearest neighbors. That is, the k closest other vertices by Euclidean distance between the two corresponding vectors. This can be thought of as a "friend list" for the vertex.
+* For some applications (including nearest-neighbor search), the incoming edges are also added.
+
+Eventually, it was realized that the following greedy search method over such a graph typically results in good approximate nearest neighbors:
+
+* Given a query vector, start at some fixed "entry point" vertex (e.g. the approximate center node).
+* Look at that vertex's neighbors. If any of them are closer to the query vector than the current vertex, then move to that vertex.
+* Repeat until a local optimum is found.
+
+The above algorithm also generalizes to e.g. top 10 approximate nearest neighbors.
+
+Computing a k-nearest neighbor graph is actually quite slow, taking quadratic time in the dataset size. It was quickly realized that near-identical performance can be achieved using a k-approximate nearest neighbor graph. That is, instead of obtaining the k-nearest neighbors for each vertex, an approximate nearest neighbor search data structure is used to build much faster.  
+In fact, another data structure is not needed: This can be done "incrementally".
+That is, if you start with a k-ANN graph for n-1 vertices, you can extend it to a k-ANN graph for n vertices as well by using the graph to obtain the k-ANN for the new vertex.
+
+One downside of k-NN and k-ANN graphs alone is that one must typically build them with a large value of k to get decent results, resulting in a large index.
+
+### Hierarchical Navigable Small Worlds (HNSW) {#hierarchical-navigable-small-worlds-hnsw}
+
+HNSW builds on k-ANN in two main ways:
+
+* Instead of getting the k-approximate nearest neighbors for a large value of k, it sparsifies the k-ANN graph using a carefully chosen "edge pruning" heuristic, allowing for the number of edges per vertex to be limited to a relatively small constant.
+* The "entry point" vertex is chosen dynamically using a recursively constructed data structure on a subset of the data, similarly to a skip list.
+
+This recursive structure can be thought of as separating into layers:
+
+* At the bottom-most layer, a k-ANN graph on the whole dataset is present.
+* At the second layer, a k-ANN graph on a fraction of the dataset (e.g. 10%) is present.
+* At the Lth layer, a k-ANN graph is present. It is over a (constant) fraction (e.g. 10%) of the vectors/vertices present in the L-1th layer.
+
+Then the greedy search routine operates as follows:
+
+* At the top layer (using an arbitrary vertex as an entry point), use the greedy local search routine on the k-ANN graph to get an approximate nearest neighbor at that layer.
+* Using the approximate nearest neighbor found in the previous layer as an entry point, find an approximate nearest neighbor in the next layer with the same method.
+* Repeat until the bottom-most layer is reached. Then use the entry point to find multiple nearest neighbors (e.g. top 10).

docs/web/indexing/quantization.mdx

@@ -0,0 +1,79 @@
+---
+title: "Quantization"
+sidebarTitle: "Quantization"
+description: "Learn about quantization when creating an index in LanceDB."
+icon: "compress"
+keywords: ["quantization", "quantize", "rabitq"]
+---
+
+Quantization compresses high-dimensional float vectors into a smaller, approximate representation, where instead of storing every vector as a float32 or float64, it's stored in compressed form, without too much of a compromise in search quality.
+
+Use quantization when:
+
+- You have a large dataset with relatively high-dimensional vectors (512, 768, 1024+)
+- Index build time and query latency matter
+
+LanceDB currently exposes multiple quantized vector index types, including:
+- `IVF_PQ` -- Inverted File index with Product Quantization (default). See the [vector indexing guide](/indexing/vector-index) for `IVF_PQ` examples.
+- `IVF_SQ` -- Inverted File index with Scalar Quantization. This is available in Python and Rust; TypeScript does not currently expose `IvfSq`.
+- `IVF_RQ` -- Inverted File index with **RaBitQ** quantization (binary, 1 bit per dimension). Requires vector dimensions divisible by `8`. See [below](#rabitq-quantization) for details.
+- `IVF_HNSW_SQ` -- IVF partitions with an **HNSW graph per partition** plus **Scalar Quantization**. Strong recall/latency/size trade-off for most workloads.
+- `IVF_HNSW_PQ` -- IVF partitions with an **HNSW graph per partition** plus **Product Quantization**. Prefer when PQ-level compression matters and you still want HNSW-style in-partition search.
+
+Two axes are being combined here: whether partitions are searched flatly or via an HNSW graph (`IVF_*` vs. `IVF_HNSW_*`), and which quantizer compresses the vectors (`PQ`, `RQ`, or `SQ`). `IVF_PQ` is the default and works well in many cases. For more drastic compression, RaBitQ (`IVF_RQ`) is a reasonable option. For higher recall at low latency, the HNSW-backed variants are usually the right pick. The ["Choose the Right Index"](/indexing/vector-index#choose-the-right-index) table on the vector indexing page is the canonical decision tool.
+
+Use the same distance metric when training the index and running queries against it. For IVF-based indexes, `num_partitions` controls the number of groups and `sample_rate` controls how many training vectors are sampled per partition, so the training sample is roughly `sample_rate * num_partitions`.
+
+## RaBitQ quantization {#rabitq-quantization}
+
+RaBitQ is a binary quantization method that represents each normalized embedding using **1 bit per dimension**, plus a couple of small corrective scalars. In practice, a 1,024-dimensional `float32` vector that would normally take 4 KB can be compressed to roughly a few hundred bytes with RaBitQ, while still maintaining reasonable recall.
+
+### How RaBitQ works {#how-rabitq-works}
+
+- Embeddings are grouped around centroids (as in other IVF indexes).
+- Each residual vector is normalized and mapped to the nearest vertex of a randomly rotated hypercube on the unit sphere.
+- The sign pattern of that vector is stored as bits (1 bit per dimension).
+- Two small corrective factors are stored:
+  1. The distance from the original vector to its centroid
+  2. The dot product between the normalized vector and its quantized version
+
+Compared to `IVF_PQ`, RaBitQ:
+- Avoids training expensive PQ codebooks
+- Builds indexes faster and handles updates more easily
+- Maintains or improves recall at high dimensionality under the same storage budget
+
+For a deeper dive into the theory and some benchmark results, see the blog post: [LanceDB's RaBitQ Quantization for Blazing Fast Vector Search](https://lancedb.com/blog/feature-rabitq-quantization/).
+
+### Using RaBitQ {#using-rabitq}
+
+You can create an RaBitQ-backed vector index by setting `index_type="IVF_RQ"` when calling `create_index`. 
+
+<Note title="Dimension requirement">
+When using `IVF_RQ`, vector dimensions must be divisible by `8`.
+</Note>
+
+`num_bits` controls how many bits per dimension are used:
+
+1 bit is the classic RaBitQ setting. You can set it to 2, 4, or 8 bits to improve fidelity for better precision or recall — the main trade-off is additional storage for the extra bits per dimension, with only a modest increase in query-time compute.
+It's also possible to tune the number of IVF partitions in `IVF_RQ`, similar to how you would do in `IVF_PQ`.
+
+<Warning title="Reading multi-bit indexes across versions">
+Indexes built with `num_bits >= 2` use an updated on-disk layout. Older LanceDB versions cannot read them and will fail with a clear missing-column error rather than returning incorrect results. Existing indexes keep working and upgrade automatically when they are rewritten (for example, during compaction, optimize, or remap). `num_bits=1` indexes are unaffected in both directions.
+</Warning>
+
+## API Reference {#api-reference}
+
+The full list of parameters to the algorithm are listed below.
+
+- `distance_type`: Literal["l2", "cosine", "dot"], defaults to "l2"  
+  The distance metric to use for similarity comparison. Choose "l2" for Euclidean, "cosine" for cosine similarity, or "dot" for dot product.
+- `num_partitions`: Optional[int], defaults to None  
+  Number of IVF partitions (affects index build time and query accuracy). More partitions can improve recall but may increase build time. When unset, LanceDB chooses roughly the square root of the row count.
+- `num_bits`: int, defaults to 1  
+  Bits per dimension for quantization (1 is standard RaBitQ). Higher values improve fidelity, mainly at the cost of additional storage.
+- `max_iterations`: int, defaults to 50  
+  Maximum number of iterations for training the quantizer. Increase for larger datasets or to improve quantization quality.
+- `sample_rate`: int, defaults to 256  
+  Number of samples per partition during training. Higher values may improve accuracy but increase training time.
+- `target_partition_size`: Optional[int], defaults to None  
+  Target number of vectors per partition. Adjust to control partition granularity and memory usage. If `num_partitions` is also set, `num_partitions` takes precedence.

docs/web/indexing/reindexing.mdx

@@ -0,0 +1,59 @@
+---
+title: "Keeping Indexes Up-to-Date with Reindexing"
+sidebarTitle: "Reindexing"
+description: "Learn how to keep your indexes up-to-date in LanceDB using incremental indexing, including best practices for adding new records without full reindexing."
+icon: "refresh"
+keywords: ["reindexing", "re-index", "incremental indexing", "reindex"]
+---
+import { PyReindexingIncremental as ReindexingIncremental } from '/snippets/indexing.mdx';
+
+As you add new data to your LanceDB tables, your indexes may become outdated.
+Reindexing is the process of updating the index to account for new data -- this applies to either a full-text search (FTS) index or a vector index. Reindexing is an important operation to run periodically as your data grows, as it has performance implications.
+
+As data is being added and a reindex operation is running, LanceDB will combine results from the existing index with exhaustive/flat search on the new data. This is done to ensure that you're still retrieving results over all your data, but it does come at a performance cost. The more data that you add without reindexing, the impact on latency (due to exhaustive search) can be noticeable.
+
+Rather than dropping an existing index entirely and reindexing from scratch, LanceDB supports **incremental indexing**.
+
+## Incremental Reindexing {#incremental-reindexing}
+
+You can manually trigger an incremental indexing operation on updated data
+using the `optimize()` method on a table.
+
+Table optimization performs three maintenance operations:
+
+1. **Compaction**: merges small fragments into larger ones to improve read performance
+2. **Pruning/Cleanup**: removes files from versions older than a retention window (7 days by default)
+3. **Index update**: adds newly-ingested data to existing vector, scalar, and FTS indexes
+
+<CodeGroup>
+    <CodeBlock filename="Python" language="Python" icon="python">
+    {ReindexingIncremental}
+    </CodeBlock>
+</CodeGroup>
+
+<Badge color="red">Enterprise</Badge>
+
+LanceDB Enterprise support incremental reindexing through an automated background process. When new data is added to a table, the system automatically triggers a new index build. As the dataset grows, indexes are asynchronously updated in the background.
+
+- While indexes are being rebuilt, queries use brute force methods on unindexed rows, which may temporarily increase latency. To avoid this, set `fast_search=True` to search only indexed data.
+- Use `index_stats()` to view the number of unindexed rows. This will be zero when indexes are fully up-to-date. If you call `wait_for_index(...)`, it polls the same status and can time out while continuous writes keep adding unindexed rows.
+
+The benefit of using LanceDB Enterprise is that it automates the reindexing process
+and operates continuously in the background, minimizing the impact on latency under high loads.
+In OSS, you must manually manage the reindexing cadence based on your data growth and performance needs.
+
+## Disk utilization {#disk-utilization}
+
+Compaction by itself does not immediately free disk space, and can temporarily increase it because new
+compacted files are written before old-version files are deleted. Disk space is reclaimed when old versions
+are pruned during cleanup. Set retention only as low as your rollback and time-travel requirements allow.
+
+If you need to reclaim space more aggressively in OSS, use a shorter retention window:
+<CodeGroup>
+    ```python Python icon=Python
+    from datetime import timedelta
+
+    table.optimize(cleanup_older_than=timedelta(days=1))
+    ```
+</CodeGroup>
+

docs/web/indexing/scalar-index.mdx

@@ -0,0 +1,243 @@
+---
+title: "Scalar Indexes"
+sidebarTitle: "Scalar Index"
+description: "Learn how to use scalar indexes in LanceDB for efficient metadata filtering and query optimization."
+icon: "tree"
+---
+import {
+    PyScalarIndexBuild as ScalarIndexBuild,
+    PyScalarIndexWait as ScalarIndexWait,
+    PyScalarIndexOptimize as ScalarIndexOptimize,
+    PyScalarIndexFilter as ScalarIndexFilter,
+    PyScalarIndexPrefilter as ScalarIndexPrefilter,
+    PyScalarIndexUuidType as ScalarIndexUuidType,
+    PyScalarIndexUuidData as ScalarIndexUuidData,
+    PyScalarIndexUuidTable as ScalarIndexUuidTable,
+    PyScalarIndexUuidWait as ScalarIndexUuidWait,
+    PyScalarIndexUuidUpsert as ScalarIndexUuidUpsert,
+    PyScalarIndexNestedFields as ScalarIndexNestedFields,
+} from '/snippets/indexing.mdx';
+
+Scalar indexes organize data by scalar attributes (e.g., numbers, categories) and enable fast filtering of vector data. They accelerate retrieval of scalar data associated with vectors, thus enhancing query performance.
+
+LanceDB supports four types of scalar indexes:
+
+- `BTREE`: Stores column data in sorted order for binary search. Best for columns with many unique values.
+- `BITMAP`: Uses bitmaps to track value presence. Ideal for columns with few unique values (e.g., categories, tags).
+- `LABEL_LIST`: Special index for `List<T>` and `LargeList<T>` columns of primitive values supporting `array_contains_all` and `array_contains_any` queries.
+- `FM`: FM-Index over string or binary columns that accelerates substring search via `contains(col, 'needle')`.
+
+## Choosing the Right Index Type {#choosing-the-right-index-type}
+
+| Data Type                                                       | Filter                                    | Index Type   |
+|:----------------------------------------------------------------|:------------------------------------------|:-------------|
+| Numeric, String, Temporal                                       | `<`, `=`, `>`, `in`, `between`, `is null` | `BTREE`      |
+| Boolean, numbers or strings with fewer than 1,000 unique values | `<`, `=`, `>`, `in`, `between`, `is null` | `BITMAP`     |
+| List of low cardinality of numbers or strings                   | `array_has_any`, `array_has_all`          | `LABEL_LIST` |
+| String or binary (`Utf8`, `LargeUtf8`, `Binary`, `LargeBinary`) | `contains(col, 'needle')`                 | `FM`         |
+
+## Scalar Index Operations {#scalar-index-operations}
+
+### 1\. Build the Index {#1-build-the-index}
+
+You can create multiple scalar indexes within a table. By default, the index will be `BTREE`, but you can always configure another type like `BITMAP`
+
+<CodeGroup>
+    <CodeBlock filename="Python" language="Python" icon="python">
+    {ScalarIndexBuild}
+    </CodeBlock>
+</CodeGroup>
+
+<Note title="LanceDB Enterprise">
+If you are using LanceDB Enterprise, the `create_scalar_index` API returns immediately, but the building of the scalar index is asynchronous. To wait until all data is fully indexed, you can specify the `wait_timeout` parameter on `create_scalar_index()` or call `wait_for_index()` on the table.
+</Note>
+
+### 2\. Check Index Status {#2-check-index-status}
+
+<CodeGroup>
+    <CodeBlock filename="Python" language="Python" icon="python">
+    {ScalarIndexWait}
+    </CodeBlock>
+</CodeGroup>
+
+`wait_for_index(...)` waits until the named scalar indexes exist and `index_stats(...)` reports `num_unindexed_rows == 0`. If a table is receiving steady writes, that fully indexed state may not stabilize before the timeout.
+
+### 3\. Update the Index {#3-update-the-index}
+
+Updating the table data (adding, deleting, or modifying records) requires that you also update the scalar index. This can be done by calling `optimize`, which will trigger an update to the existing scalar index.
+
+<CodeGroup>
+    <CodeBlock filename="Python" language="Python" icon="python">
+    {ScalarIndexOptimize}
+    </CodeBlock>
+</CodeGroup>
+
+<Note title="LanceDB Enterprise">
+New data added after creating the scalar index will still appear in search results if optimize is not used, but with increased latency due to a flat search on the unindexed portion. LanceDB Enterprise automates the optimize process, minimizing the impact on search speed.
+</Note>
+
+### 4\. Run Indexed Searches {#4-run-indexed-searches}
+
+The following scan will be faster if the column `book_id` has a scalar index:
+
+<CodeGroup>
+    <CodeBlock filename="Python" language="Python" icon="python">
+    {ScalarIndexFilter}
+    </CodeBlock>
+</CodeGroup>
+
+Scalar indexes can also speed up scans containing a vector search or full text search, and a prefilter:
+
+<CodeGroup>
+    <CodeBlock filename="Python" language="Python" icon="python">
+    {ScalarIndexPrefilter}
+    </CodeBlock>
+</CodeGroup>
+
+## Indexing nested fields {#indexing-nested-fields}
+
+Scalar indexes can target a scalar field inside a struct by passing its full dotted path. The path is preserved end to end: it's the value you pass to `create_scalar_index`, it's what `list_indices()` reports under `columns`, and it's the column reference you use in filter predicates.
+
+```python
+# Schema: pa.struct([pa.field("user_id", pa.int32())]) stored under the `metadata` column.
+table.create_scalar_index("metadata.user_id", name="metadata_user_id_idx")
+
+# The same dotted path works in WHERE clauses.
+table.search().where("metadata.user_id = 42").limit(1).to_list()
+```
+
+<Note>
+Nested paths follow Lance field-path semantics: dot-separate each struct field from root to leaf (for example, `metadata.author.name`). The same convention applies to FTS and vector indexes.
+</Note>
+
+## FM-Index for substring search {#fm-index-for-substring-search}
+
+The `FM` index is a scalar index built over string or binary columns that
+accelerates substring lookups expressed as `contains(col, 'needle')`. Unlike the
+tokenized [FTS index](/indexing/fts-index), which matches whole words after
+tokenization, the FM-Index matches arbitrary substrings of the raw bytes — so it
+works well for URLs, file paths, identifiers, log lines, or any column where you
+search for a fragment rather than a word.
+
+Use the FM-Index when:
+
+- Filters use `contains(col, 'needle')` (substring), not equality or word search.
+- The column is `Utf8`, `LargeUtf8`, `Binary`, or `LargeBinary`.
+- You want substring matches without paying for tokenization, language analysis,
+  or BM25 scoring.
+
+Pick `FTS` instead when you need word-level relevance ranking, phrase queries,
+or language-aware tokenization.
+
+### Create an FM-Index {#create-an-fm-index}
+
+Build an FM-Index with the async `create_index` API by passing the `Fm` config in
+Python or `Index.fm()` in TypeScript. In Rust, use `Index::Fm(FmIndexBuilder::default())`.
+
+<CodeGroup>
+```python Python icon="python"
+from lancedb.index import Fm
+
+await tbl.create_index("text", config=Fm())
+```
+
+```typescript TypeScript icon="square-js"
+import * as lancedb from "@lancedb/lancedb";
+import { Index } from "@lancedb/lancedb";
+
+await tbl.createIndex("text", { config: Index.fm() });
+```
+
+```rust Rust icon="rust"
+use lancedb::index::Index;
+use lancedb::index::scalar::FmIndexBuilder;
+
+tbl.create_index(&["text"], Index::Fm(FmIndexBuilder::default()))
+    .execute()
+    .await?;
+```
+</CodeGroup>
+
+After the index is built, substring filters use it automatically:
+
+```python
+table.search().where("contains(text, 'needle')").limit(10).to_pandas()
+```
+
+`list_indices()` reports the index type as `"Fm"`.
+
+## Index UUID Columns {#index-uuid-columns}
+
+LanceDB supports scalar indexes on UUID columns (stored as `FixedSizeBinary(16)`), enabling efficient lookups and filtering on UUID-based primary keys.
+
+<Note>
+**To use `FixedSizeBinary`, ensure you have:**
+
+- Python SDK version `0.22.0` or later
+- TypeScript SDK version `0.19.0` or later
+</Note>
+
+### 1\. Define UUID Type {#1-define-uuid-type}
+
+<CodeGroup>
+    <CodeBlock filename="Python" language="Python" icon="python">
+    {ScalarIndexUuidType}
+    </CodeBlock>
+</CodeGroup>
+
+### 2\. Generate UUID Data {#2-generate-uuid-data}
+
+<CodeGroup>
+    <CodeBlock filename="Python" language="Python" icon="python">
+    {ScalarIndexUuidData}
+    </CodeBlock>
+</CodeGroup>
+
+### 3\. Create Table with UUID Column {#3-create-table-with-uuid-column}
+
+<CodeGroup>
+    <CodeBlock filename="Python" language="Python" icon="python">
+    {ScalarIndexUuidTable}
+    </CodeBlock>
+</CodeGroup>
+
+### 4\. Create and Wait for the Index {#4-create-and-wait-for-the-index}
+
+<CodeGroup>
+    <CodeBlock filename="Python" language="Python" icon="python">
+    {ScalarIndexUuidWait}
+    </CodeBlock>
+</CodeGroup>
+
+### 5\. Perform Operations with the UUID Index {#5-perform-operations-with-the-uuid-index}
+
+<CodeGroup>
+    <CodeBlock filename="Python" language="Python" icon="python">
+    {ScalarIndexUuidUpsert}
+    </CodeBlock>
+</CodeGroup>
+
+## Index nested fields {#index-nested-fields}
+
+You can build a scalar index on a field inside a struct column by passing the
+canonical dot-separated path to `create_index`. This is useful when filters
+target attributes nested under a `metadata`-style column, for example
+`metadata.user_id` or `metadata.event.type`.
+
+If a literal segment of the path itself contains a dot (for example a column
+named `user.id` nested inside `metadata`), wrap that segment in backticks so
+LanceDB can tell the dot apart from the path separator: `` metadata.`user.id` ``.
+
+`list_indices()` echoes the same canonical path back, so the column you pass in
+round-trips through index metadata regardless of nesting depth or escaping.
+
+<CodeGroup>
+    <CodeBlock filename="Python" language="Python" icon="python">
+    {ScalarIndexNestedFields}
+    </CodeBlock>
+</CodeGroup>
+
+<Note>
+Composite indexes that cover multiple columns aren't supported yet. Each
+`create_index` call must target a single (possibly nested) field path.
+</Note>

docs/web/indexing/vector-index.mdx

@@ -0,0 +1,379 @@
+---
+title: "Vector Indexes"
+sidebarTitle: "Vector Index"
+description: "Build and optimize LanceDB vector indexes, including IVF, HNSW and binary quantized indexes."
+icon: "arrow-up-right-dots"
+---
+import {
+    PyVectorIndexConfigureIvf as VectorIndexConfigureIvf,
+    PyVectorIndexSetup as VectorIndexSetup,
+    PyVectorIndexBuildIvf as VectorIndexBuildIvf,
+    PyVectorIndexNestedField as VectorIndexNestedField,
+    PyVectorIndexAsyncConfig as VectorIndexAsyncConfig,
+    PyVectorIndexQueryIvf as VectorIndexQueryIvf,
+    PyVectorIndexBuildHnsw as VectorIndexBuildHnsw,
+    PyVectorIndexQueryHnsw as VectorIndexQueryHnsw,
+    PyVectorIndexBinarySchema as VectorIndexBinarySchema,
+    PyVectorIndexBinaryAddData as VectorIndexBinaryAddData,
+    PyVectorIndexBinaryBuildIndex as VectorIndexBinaryBuildIndex,
+    PyVectorIndexBinarySearch as VectorIndexBinarySearch,
+    PyVectorIndexCheckStatus as VectorIndexCheckStatus,
+    PyVectorIndexNprobes as VectorIndexNprobes,
+    PyVectorIndexDistanceRange as VectorIndexDistanceRange,
+    PyVectorIndexBypassRecall as VectorIndexBypassRecall,
+    PyVectorIndexCustomName as VectorIndexCustomName,
+} from '/snippets/indexing.mdx';
+
+You can create and manage multiple vector indexes on any Lance dataset. LanceDB offers two kinds of vector indexing algorithms: **Inverted File (IVF)** and **Hierarchical Navigable Small World (HNSW)**.
+
+<Info>
+**IVF + HNSW**
+
+In LanceDB, HNSW is not exposed as a top-level vector index. Instead, it's available as a sub-index inside IVF partitions. What this means in practice is that vectors are first partitioned by IVF, then each selected partition is searched using an HNSW graph. LanceDB supports the unquantized variant `IVF_HNSW_FLAT`, along with quantized variants such as `IVF_HNSW_PQ` and `IVF_HNSW_SQ`. This combines IVF's scalability with HNSW's higher-recall ANN search within partitions.
+</Info>
+
+### Manual Indexing {#manual-indexing}
+
+If using LanceDB OSS, you will have to create the vector index manually, by calling `table.create_index()`, and updating the index as new data arrives and tuning its parameters is also a manual process.
+
+### Automatic Indexing {#automatic-indexing}
+
+ <Badge color="red">Enterprise-only</Badge> 
+Vector indexing is managed **automatically** in LanceDB Enterprise. As soon as data is updated, the system updates the index and optimizates it. *This is done asynchronously as a background process*.
+
+When you create a table in LanceDB Enterprise, LanceDB automatically:
+
+- Infers the vector columns from the schema
+- Create an optimized `IVF_PQ` index without manual configuration
+- Automatically configure indexing parameters
+
+The default distance is `l2` (Euclidean).
+
+<Note>
+You can call `create_index()` with different parameters to create a new index -- this replaces any existing index.
+Although the `create_index` API returns immediately, the building of the vector index is asynchronous. To wait until all data is fully indexed, you can specify the `wait_timeout` parameter.
+</Note>
+
+Use the same distance metric for index creation and search. Once a vector index exists, queries use the metric stored with that index. If you need to confirm an async build or refresh is finished, `wait_for_index(...)` waits for the named index to exist and for `index_stats(...)` to report `num_unindexed_rows == 0`; it can time out if new writes keep arriving.
+
+Rows appended after an index build remain outside that index until optimization refreshes it. Normal
+search still checks those unindexed rows with a slower fallback path; `fast_search()` skips that
+fallback and searches only indexed rows.
+
+## Choose the Right Index {#choose-the-right-index}
+
+Use this table as a quick starting point for choosing the right index type and quantization method for your use case:
+
+| If your top priority is... | Use this index | Why | Typical compressed size vs. raw vectors |
+| :--- | :--- | :--- | :--- |
+| Highest recall / no quantization | `IVF_HNSW_FLAT` | Uses raw vectors inside the IVF+HNSW structure, avoiding quantization loss. | Around raw vector size plus HNSW graph overhead |
+| Best recall/latency trade-off | `IVF_HNSW_SQ` | Combines IVF partitioning with HNSW graph search for strong quality at low latency. | Typically a little larger than `1/4` of raw size |
+| Maximum compression | `IVF_RQ` | RaBitQ-style quantization with very strong compression. | Around `1/32` of raw size |
+| Higher accuracy at small dimensions (`dimension <= 256`) | `IVF_PQ` | On small-dimensional vectors, `IVF_PQ` often provides higher accuracy with similar performance compared to `IVF_RQ`. | Usually `1/64` to `1/16` of raw size (depends on `num_sub_vectors`) |
+
+<Warning>
+If your vector search frequently includes metadata filters (`where(...)`), prefer `IVF_RQ` or `IVF_PQ`. In filtered workloads, HNSW-backed IVF indexes such as `IVF_HNSW_FLAT` and `IVF_HNSW_SQ` can show higher latency variance.
+</Warning>
+
+Compression ratios are practical rules of thumb and can vary with vector distribution, metric, and configuration.
+For small dimensions, choose `IVF_PQ` for accuracy, not for guaranteed higher compression than `IVF_RQ`.
+
+### Index Tuning {#index-tuning}
+
+Start with these values, then tune for your workload:
+
+- HNSW-backed IVF indexes (`IVF_HNSW_FLAT`, `IVF_HNSW_SQ`, `IVF_HNSW_PQ`)
+  - `num_partitions`: start at `num_rows // 1,048,576` (rounded to an integer)
+  - Lower `num_partitions` can reduce search latency, but index build may become slower because partitions are larger.
+  - `ef_construction`: start at `150`; increase for better recall, decrease for faster indexing.
+- `IVF_RQ`
+  - `num_partitions`: start at `num_rows // 4096` (rounded to an integer). This is a strong default for most datasets.
+- `IVF_PQ`
+  - `num_partitions`: start at `num_rows // 4096` (rounded to an integer).
+  - `num_sub_vectors`: start at `dimension // 8`. Increase for better recall, decrease for faster search and smaller indexes.
+  - For small dimensions (`dimension <= 256`), `IVF_PQ` is often preferred over `IVF_RQ` for better accuracy at similar query performance.
+
+## Example: Construct an IVF Index {#example-construct-an-ivf-index}
+
+In this example, we will create an index for a table containing 1536-dimensional vectors. The index will use IVF_PQ with L2 distance, which is well-suited for high-dimensional vector search. 
+
+Make sure you have enough data in your table (at least a few thousand rows) for effective index training.
+
+### Index Configuration {#index-configuration}
+
+Sometimes you need to configure the index beyond default parameters:
+
+- Index Types:
+    - `IVF_HNSW_FLAT`: highest recall, with no vector quantization
+    - `IVF_HNSW_SQ`: best recall/latency trade-off
+    - `IVF_RQ`: best compression for large, high-dimensional datasets
+    - `IVF_PQ`: often higher accuracy than `IVF_RQ` for small dimensions (`<= 256`) at similar query performance
+- `metrics`: default is `l2`, other available are `cosine` or `dot`
+    - When using `cosine` similarity, distances range from 0 (identical vectors) to 2 (maximally dissimilar)
+- `num_partitions`: use index-specific starting points from the section above:
+    - HNSW-backed IVF indexes (`IVF_HNSW_FLAT`, `IVF_HNSW_SQ`, `IVF_HNSW_PQ`): `num_rows // 1,048,576`
+    - `IVF_RQ` and `IVF_PQ`: `num_rows // 4096`
+- `target_partition_size`: alternative IVF sizing knob that asks LanceDB to derive the partition
+  count from a target number of rows per partition. If you set both `num_partitions` and
+  `target_partition_size`, `num_partitions` takes precedence.
+- `num_sub_vectors`: applies to `IVF_PQ`; start with `dimension // 8`. Larger values often improve recall but can slow search.
+
+Let's take a look at a sample request for an IVF index:
+
+
+<CodeGroup>
+    <CodeBlock filename="Python" language="Python" icon="python">
+    {VectorIndexConfigureIvf}
+    </CodeBlock>
+</CodeGroup>
+
+### 1\. Setup {#1-setup}
+
+Connect to LanceDB and open the table you want to index.
+
+<CodeGroup>
+    <CodeBlock filename="Python" language="Python" icon="python">
+    {VectorIndexSetup}
+    </CodeBlock>
+</CodeGroup>
+
+### 2\. Construct an IVF Index {#2-construct-an-ivf-index}
+
+Create an `IVF_PQ` index with `cosine` similarity. Specify `vector_column_name` if you use multiple vector columns or non-default names. For a vector field nested inside a struct, use dot notation (e.g. `image.embedding`); see [Selecting the vector column](/search/vector-search#selecting-the-vector-column) for the full syntax. You can switch `index_type` to `IVF_RQ`, `IVF_HNSW_SQ`, or `IVF_HNSW_FLAT` depending on your recall/latency/compression target.
+
+<CodeGroup>
+    <CodeBlock filename="Python" language="Python" icon="python">
+    {VectorIndexBuildIvf}
+    </CodeBlock>
+</CodeGroup>
+
+#### Indexing nested vector fields {#indexing-nested-vector-fields}
+
+If your vector column lives inside a struct, pass its full dotted path as `vector_column_name`. The same path is used at query time and is what `list_indices()` reports under `columns`:
+
+<CodeGroup>
+    <CodeBlock filename="Python" language="Python" icon="python">
+    {VectorIndexNestedField}
+    </CodeBlock>
+</CodeGroup>
+
+<Note>
+Nested paths follow Lance field-path semantics: dot-separate each struct field from root to leaf (for example, `image.thumbnail.embedding`). The same convention applies to FTS and scalar indexes.
+</Note>
+
+### Async API and Config Objects {#async-api-and-config-objects}
+
+With asynchronous Python connections, create vector indexes with `await table.create_index("vector", config=...)`. The `config` object carries the same index choices you configure in the synchronous API, such as distance metric, partition count, and quantization settings:
+
+<CodeGroup>
+    <CodeBlock filename="Python" language="Python" icon="python">
+    {VectorIndexAsyncConfig}
+    </CodeBlock>
+</CodeGroup>
+
+Use these Python config classes for the index types shown on this page:
+
+| Index type | Python config class |
+| :--- | :--- |
+| `IVF_FLAT` | `IvfFlat` |
+| `IVF_PQ` | `IvfPq` |
+| `IVF_RQ` | `IvfRq` |
+| `IVF_SQ` | `IvfSq` |
+| `IVF_HNSW_FLAT` | `IvfHnswFlat` |
+| `IVF_HNSW_PQ` | `IvfHnswPq` |
+| `IVF_HNSW_SQ` | `IvfHnswSq` |
+
+### 3\. Query the IVF Index {#3-query-the-ivf-index}
+
+Search using a random 1,536-dimensional embedding.
+
+<CodeGroup>
+    <CodeBlock filename="Python" language="Python" icon="python">
+    {VectorIndexQueryIvf}
+    </CodeBlock>
+</CodeGroup>
+
+#### Search Configuration {#search-configuration}
+
+Core knobs available on a vector search call:
+
+| Parameter | Description |
+| :--- | :--- |
+| `limit` | Number of results to return (`k`). |
+| `nprobes` | Shorthand that sets both `minimum_nprobes` and `maximum_nprobes` to the same value. LanceDB auto-tunes this by default. |
+| `minimum_nprobes` | Partitions that are *always* scanned. Higher values raise recall at the cost of latency. |
+| `maximum_nprobes` | Upper bound on partitions scanned. The partitions above `minimum_nprobes` are only searched if the initial pass does not return enough results — useful for narrow filters. Set to `0` to remove the cap. |
+| `ef` | HNSW search-time exploration factor. Relevant for `IVF_HNSW_FLAT` and `IVF_HNSW_SQ`; start around `1.5 * k` and increase up to `10 * k` for higher recall. |
+| `refine_factor` | Reads additional candidates and reranks them in memory to recover recall lost to quantization. |
+
+<Note>
+**Filtered queries and adaptive nprobes.** When a `where(...)` filter is active, LanceDB starts by scanning `minimum_nprobes` partitions and only extends toward `maximum_nprobes` if fewer than `limit` rows survive the filter. Setting `minimum_nprobes == maximum_nprobes` (or calling `nprobes(n)`) disables this adaptive behavior and fixes the partition count.
+</Note>
+
+<CodeGroup>
+    <CodeBlock filename="Python" language="Python" icon="python">
+    {VectorIndexNprobes}
+    </CodeBlock>
+</CodeGroup>
+
+Recommended `nprobes` behavior by index type:
+
+| Index type | Guidance |
+| :--- | :--- |
+| `IVF_HNSW_FLAT`, `IVF_HNSW_SQ` | Keep the auto-tuned `nprobes`, then tune `ef` first. Expect higher latency variance under filtered search. |
+| `IVF_RQ` | Keep auto-tuned `nprobes`; raise only when recall is insufficient. |
+| `IVF_PQ` | Keep auto-tuned `nprobes`; raise when recall is insufficient. Often preferred over `IVF_RQ` when `dimension <= 256`. |
+
+#### Advanced Search Controls {#advanced-search-controls}
+
+These controls are useful for thresholded retrieval, recall measurement, and working around index-level metric constraints.
+
+| Method | Description |
+| :--- | :--- |
+| `distance_range(lower_bound, upper_bound)` | Return only rows whose distance falls within `[lower_bound, upper_bound)`. Either bound is optional. Useful for near-duplicate detection or "close-enough" matching. |
+| `bypass_vector_index()` | Skip the ANN index and perform an exhaustive (flat) scan. Primary uses: (1) compute ground-truth results to measure ANN recall@k, and (2) query with a metric the index was not built for (e.g., a non-cosine query on a multivector column). |
+
+**Thresholding with `distance_range`:**
+
+<CodeGroup>
+    <CodeBlock filename="Python" language="Python" icon="python">
+    {VectorIndexDistanceRange}
+    </CodeBlock>
+</CodeGroup>
+
+**Measuring recall with `bypass_vector_index`:**
+
+Compare ANN results against a flat-scan ground truth to compute recall@k. This is the standard way to pick `nprobes` for your workload.
+
+<CodeGroup>
+    <CodeBlock filename="Python" language="Python" icon="python">
+    {VectorIndexBypassRecall}
+    </CodeBlock>
+</CodeGroup>
+
+<Warning>
+Flat search is $O(n)$ — reserve `bypass_vector_index()` for sampled recall measurements or small tables, not production queries.
+</Warning>
+
+<Note title="Multivector distance constraint">
+Multivector indexing currently requires `distance_type="cosine"` — `l2` is rejected at index-creation time. That restriction is why `bypass_vector_index()` is the escape hatch for non-cosine queries on a multivector column: the metric you want at query time cannot be served by the index, so you fall back to a flat scan. See [Multivector Search](/search/multivector-search) for the full rules.
+</Note>
+
+## Example: Construct an HNSW Index {#example-construct-an-hnsw-index}
+
+### Index Configuration {#index-configuration-2}
+
+There are four key parameters to set when constructing an HNSW index:
+
+- `index_type`: choose `IVF_HNSW_SQ` for a strong recall/latency/size trade-off, or `IVF_HNSW_FLAT` when you want the IVF+HNSW structure without vector quantization.
+- `metric`: The default is `l2` euclidean distance metric. Other available are `dot` and `cosine`.
+- `m`: The number of neighbors to select for each vector in the HNSW graph.
+- `ef_construction`: The number of candidates to evaluate during the construction of the HNSW graph.
+
+### 1\. Construct an HNSW Index {#1-construct-an-hnsw-index}
+
+The snippet below uses `IVF_HNSW_SQ`. If you want the unquantized variant, change `index_type` to `IVF_HNSW_FLAT`.
+
+<CodeGroup>
+    <CodeBlock filename="Python" language="Python" icon="python">
+    {VectorIndexBuildHnsw}
+    </CodeBlock>
+</CodeGroup>
+
+### 2\. Query the HNSW Index {#2-query-the-hnsw-index}
+
+<CodeGroup>
+    <CodeBlock filename="Python" language="Python" icon="python">
+    {VectorIndexQueryHnsw}
+    </CodeBlock>
+</CodeGroup>
+
+## Example: Construct a Binary Vector Index {#example-construct-a-binary-vector-index}
+
+Binary vectors are useful for hash-based retrieval, fingerprinting, or any scenario where data can be represented as bits.
+
+### Index Configuration {#index-configuration-3}
+
+- Store binary vectors as fixed-size binary data (uint8 arrays, with 8 bits per byte). For storage, pack binary vectors into bytes to save space. 
+- Index Type: `IVF_FLAT` is used for indexing binary vectors
+- `metric`: the `hamming` distance is used for similarity search
+- The dimension of binary vectors must be a multiple of 8. For example, a 128-dimensional vector is stored as a uint8 array of size 16.
+
+<Warning>
+**`IVF_FLAT` + `hamming` is the only supported path for binary vectors.**
+
+- `hamming` distance is only valid on packed binary (uint8) data; it is rejected on float vector columns.
+- Quantized index types (`IVF_PQ`, `IVF_RQ`, `IVF_SQ`, `IVF_HNSW_PQ`, `IVF_HNSW_SQ`) do not accept binary inputs — their `distance_type` is restricted to `l2`, `cosine`, or `dot`.
+</Warning>
+
+### 1\. Create Table and Schema {#1-create-table-and-schema}
+
+<CodeGroup>
+    <CodeBlock filename="Python" language="Python" icon="python">
+    {VectorIndexBinarySchema}
+    </CodeBlock>
+</CodeGroup>
+
+### 2\. Generate and Add Data {#2-generate-and-add-data}
+
+<CodeGroup>
+    <CodeBlock filename="Python" language="Python" icon="python">
+    {VectorIndexBinaryAddData}
+    </CodeBlock>
+</CodeGroup>
+
+### 3\. Construct the Binary Index {#3-construct-the-binary-index}
+
+<CodeGroup>
+    <CodeBlock filename="Python" language="Python" icon="python">
+    {VectorIndexBinaryBuildIndex}
+    </CodeBlock>
+</CodeGroup>
+
+### 4\. Vector Search {#4-vector-search}
+
+<CodeGroup>
+    <CodeBlock filename="Python" language="Python" icon="python">
+    {VectorIndexBinarySearch}
+    </CodeBlock>
+</CodeGroup>
+
+## Check Index Status {#check-index-status}
+
+Vector index creation runs in the background and may take some time to complete. While it is ongoing, you can check its status either programmatically through the API or from the **LanceDB Enterprise UI**.
+
+In the LanceDB Enterprise UI, navigate to your table page - the "Index" column reflects each column's index status: it is blank when no index exists, shows an "in progress" label while the index is being built, and shows the index type once the build completes.
+
+Programmatically, use `list_indices()` and `index_stats()`. **By default**, the index name is formed by appending `_idx` to the column name (e.g., a `keywords_embeddings` column produces `keywords_embeddings_idx`). Note that `list_indices()` only returns information after the index is fully built.
+To wait until all data is fully indexed, you can specify the `wait_timeout` parameter on `create_index()` or call `wait_for_index()` on the table.
+
+Each entry returned by `list_indices()` also carries detailed per-index metadata, so you can inspect an index without a follow-up `index_stats()` call. Node.js exposes the same fields in camelCase (`num_indexed_rows` → `numIndexedRows`):
+
+| Field | What it tells you |
+| :--- | :--- |
+| `num_indexed_rows`, `num_unindexed_rows` | Index coverage over the table |
+| `size_bytes` | Total size of the index files on disk |
+| `num_segments`, `index_version` | On-disk layout and format version |
+| `created_at` | Creation time (ms since the Unix epoch in Node.js) |
+| `index_uuid`, `type_url` | Internal identifiers for the index segment |
+| `index_details` | Type-specific details (e.g. IVF partition counts or quantization settings) |
+
+<Note>
+These fields are populated for local and embedded tables. On LanceDB Enterprise remote tables they are returned as `None` / `undefined` until the server response surfaces them.
+</Note>
+
+<CodeGroup>
+    <CodeBlock filename="Python" language="Python" icon="python">
+    {VectorIndexCheckStatus}
+    </CodeBlock>
+</CodeGroup>
+
+## Custom Index Names {#custom-index-names}
+
+The `{column}_idx` suffix is a default convention, not the only supported naming path. Pass `name=...` to `create_index()` to override it — useful when you want to manage multiple indexes on the same column (for example, side-by-side `IVF_PQ` and `IVF_HNSW_SQ` builds) or when you script index replacement by name. Once set, `list_indices()`, `index_stats(name)`, and `wait_for_index([name])` all reference the custom name.
+
+<CodeGroup>
+    <CodeBlock filename="Python" language="Python" icon="python">
+    {VectorIndexCustomName}
+    </CodeBlock>
+</CodeGroup>

docs/web/integrations/ai/agno.mdx

@@ -0,0 +1,151 @@
+---
+title: "Agno"
+sidebarTitle: "Agno"
+description: "Build a search assistant using the Agno agent framework with LanceDB as the knowledge backend."
+---
+
+import {
+  PyFrameworksAgnoAgent,
+  PyFrameworksAgnoCliChat,
+  PyFrameworksAgnoIngestYoutube,
+  PyFrameworksAgnoSetup,
+} from '/snippets/integrations.mdx';
+
+[Agno](https://docs.agno.com/introduction) is a framework for building agentic AI applications.
+It supports LanceDB as a knowledge backend, allowing you to easily ingest and retrieve external content for your agents.
+
+When you pair Agno's `Knowledge` system with LanceDB, you get a clean Agentic RAG setup.
+We'll walk through the steps below to build a YouTube transcript-aware Agno assistant that can:
+- Ingest a transcript from a YouTube video via the YouTube API
+- Store embeddings and metadata in LanceDB
+- Retrieve context during responses with hybrid search
+- Ask questions about the video content in a CLI chat loop
+
+## Prerequisites {#prerequisites}
+
+Install dependencies:
+
+<CodeGroup>
+```bash pip icon="terminal"
+pip install -U agno openai lancedb youtube-transcript-api beautifulsoup4
+```
+
+```bash uv icon="terminal"
+uv add agno openai lancedb youtube-transcript-api beautifulsoup4
+```
+</CodeGroup>
+
+## Step 1: Configure LanceDB-backed knowledge {#step-1-configure-lancedb-backed-knowledge}
+
+First, you can initialize the core `Knowledge` object that your agent will use for retrieval.
+It configures LanceDB as the vector store, enables hybrid search with native LanceDB FTS, and sets the embedding model.
+
+<CodeBlock filename="Python" language="Python" icon="python">
+  {PyFrameworksAgnoSetup}
+</CodeBlock>
+
+## Step 2: Fetch and ingest the YouTube transcript {#step-2-fetch-and-ingest-the-youtube-transcript}
+
+Next, extract a YouTube video ID, fetch the full transcript, and flatten it into text for indexing.
+The snippet shown below then inserts that transcript text into the Agno knowledge base, which writes vectors and metadata to LanceDB.
+
+<CodeBlock filename="Python" language="Python" icon="python">
+  {PyFrameworksAgnoIngestYoutube}
+</CodeBlock>
+
+<Info>
+This path explicitly fetches the transcript first, then inserts transcript text into LanceDB through Agno.
+</Info>
+
+## Step 3: Create an Agno agent with knowledge search {#step-3-create-an-agno-agent-with-knowledge-search}
+
+The next step is to construct an Agno `Agent` and attach the knowledge base you just populated.
+With `search_knowledge=True`, the agent performs retrieval before answering, so responses stay grounded in transcript context.
+
+In Agno, retrieval is exposed as a tool call that the model can invoke at runtime.
+When `search_knowledge=True`, Agno makes a knowledge-search tool (shown in output as `search_knowledge_base(...)`) available to the model; the model decides when to call it, Agno executes the tool, and the returned context is fed back into the final answer.
+
+<CodeBlock filename="Python" language="Python" icon="python">
+  {PyFrameworksAgnoAgent}
+</CodeBlock>
+
+## Step 4: Start a CLI chat loop {#step-4-start-a-cli-chat-loop}
+
+You can now ask an initial question and then start an interactive loop for follow-up queries.
+Each prompt runs through the same retrieval pipeline, so you can iteratively inspect what the transcript contains.
+
+<CodeBlock filename="Python" language="Python" icon="python">
+  {PyFrameworksAgnoCliChat}
+</CodeBlock>
+
+<Info>
+Want local-first inference? Replace OpenAI model/embedder classes with Agno's Ollama providers. See Agno's Ollama knowledge examples: [docs.agno.com/examples/models/ollama/chat/knowledge](https://docs.agno.com/examples/models/ollama/chat/knowledge).
+</Info>
+
+### Question 1 {#question-1}
+
+The following question is asked in the CLI chat loop:
+```
+┏━ Message ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
+┃                                                                                  ┃
+┃ Q: What kinds of data can LanceDB handle?                                        ┃
+┃                                                                                  ┃
+┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
+┏━ Tool Calls ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
+┃                                                                                  ┃
+┃ • search_knowledge_base(query=What kinds of data can LanceDB handle?)            ┃
+┃ • search_knowledge_base(query=LanceDB images audio video handle kinds of data    ┃
+┃ can handle 'LanceDB can handle' 'kinds of data' 'images audio video' transcript) ┃
+┃                                                                                  ┃
+┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
+┏━ Response (19.1s) ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
+┃                                                                                  ┃
+┃                                                                                  ┃
+┃  • Images, audio, video — i.e., multimodal AI data and “all manners of things    ┃
+┃    you don't put into traditional databases” (per the transcript).               ┃
+┃                                                                                  ┃
+┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
+```
+
+We get the response based on the transcript's contents as expected.
+
+### Question 2 {#question-2}
+
+Let's ask a more specific question about the CEO of LanceDB, which is also in the transcript:
+
+```
+You: What is the name of the CEO of LanceDB? 
+INFO Found 10 documents                                                             
+┏━ Message ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
+┃                                                                                  ┃
+┃ What is the name of the CEO of LanceDB?                                          ┃
+┃                                                                                  ┃
+┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
+┏━ Tool Calls ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
+┃                                                                                  ┃
+┃ • search_knowledge_base(query=CEO of LanceDB)                                    ┃
+┃                                                                                  ┃
+┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
+┏━ Response (16.7s) ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
+┃                                                                                  ┃
+┃                                                                                  ┃
+┃  • According to the retrieved YouTube transcript/title, the CEO of LanceDB is    ┃
+┃    Chang She.                                                                    ┃
+┃                                                                                  ┃
+┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
+```
+
+We get the response based on the transcript's contents and title as expected.
+
+## Why this works well {#why-this-works-well}
+
+To start, LanceDB OSS can run from a local directory, so transcript data can stay on your machine when you are using the OSS stack.
+
+- You do not need to maintain a separate transcript parser in your application code.
+- You do not need to hand-roll chunking and retrieval orchestration across multiple modules.
+- One explicit Agno `Knowledge` object, backed by LanceDB, defines both ingestion and search behavior in one place.
+- Fewer moving parts means the tutorial stays readable and the same pattern is easier to carry into production code.
+
+As your application needs grow, you can migrate to LanceDB [Enterprise](/enterprise) for
+convenience features like automatic compaction and reindexing and the ability to scale to
+really large datasets.

docs/web/integrations/ai/genkit.mdx

@@ -0,0 +1,73 @@
+---
+title: "GenKit"
+sidebarTitle: "GenKit"
+
+---
+
+import {
+  TsFrameworksGenkitCustomIndexer,
+  TsFrameworksGenkitCustomRetriever,
+  TsFrameworksGenkitUsage,
+} from '/snippets/integrations.mdx';
+
+### genkitx-lancedb {#genkitx-lancedb}
+Genkit is an open-source framework for building end-to-end AI and RAG pipelines with a clean, TypeScript-first
+developer experience. The genkitx-lancedb plugin lets you use LanceDB as a high-performance vector store
+inside your Genkit flows, so you can index, search, and retrieve data efficiently as part of your AI
+applications.
+
+### Installation {#installation}
+```bash
+pnpm install genkitx-lancedb
+```
+
+### Usage {#usage}
+
+Adding LanceDB plugin to your genkit instance.
+
+<CodeBlock filename="TypeScript" language="TypeScript" icon="square-js">
+  {TsFrameworksGenkitUsage}
+</CodeBlock>
+
+You can run this app with the following command:
+```bash
+genkit start -- tsx --watch src/index.ts
+```
+
+This'll add LanceDB as a retriever and indexer to the genkit instance. You can see it in the GUI view
+<img width="1710" alt="Screenshot 2025-05-11 at 7 21 05 PM" src="https://github.com/user-attachments/assets/e752f7f4-785b-4797-a11e-72ab06a531b7" />
+
+**Testing retrieval on a sample table**
+Let's see the raw retrieval results
+
+<img width="1710" alt="Screenshot 2025-05-11 at 7 21 05 PM" src="https://github.com/user-attachments/assets/b8d356ed-8421-4790-8fc0-d6af563b9657" />
+On running this query, you'll get 5 results fetched from the lancedb table, where each result looks something like this:
+<img width="1417" alt="Screenshot 2025-05-11 at 7 21 18 PM" src="https://github.com/user-attachments/assets/77429525-36e2-4da6-a694-e58c1cf9eb83" />
+
+
+
+## Creating a custom RAG flow {#creating-a-custom-rag-flow}
+
+Now that we've seen how you can use LanceDB in a Genkit pipeline, let's refine the flow and create a RAG. A RAG flow will consist of an index and a retriever with its outputs postprocessed and fed into an LLM for final response
+
+### Creating custom indexer flows {#creating-custom-indexer-flows}
+You can also create custom indexer flows, utilizing more options and features provided by LanceDB.
+
+<CodeBlock filename="TypeScript" language="TypeScript" icon="square-js">
+  {TsFrameworksGenkitCustomIndexer}
+</CodeBlock>
+
+<img width="1316" alt="Screenshot 2025-05-11 at 8 35 56 PM" src="https://github.com/user-attachments/assets/e2a20ce4-d1d0-4fa2-9a84-f2cc26e3a29f" />
+
+In your console, you can see the logs
+
+<img width="511" alt="Screenshot 2025-05-11 at 7 19 14 PM" src="https://github.com/user-attachments/assets/243f26c5-ed38-40b6-b661-002f40f0423a" />
+
+### Creating custom retriever flows {#creating-custom-retriever-flows}
+You can also create custom retriever flows, utilizing more options and features provided by LanceDB.
+<CodeBlock filename="TypeScript" language="TypeScript" icon="square-js">
+  {TsFrameworksGenkitCustomRetriever}
+</CodeBlock>
+Now using our retrieval flow, we can ask a question about the ingested PDF
+<img width="1306" alt="Screenshot 2025-05-11 at 7 18 45 PM" src="https://github.com/user-attachments/assets/86c66b13-7c12-4d5f-9d81-ae36bfb1c346" />
+

docs/web/integrations/ai/hermes-agent.mdx

@@ -0,0 +1,327 @@
+---
+title: "Hermes Agent"
+sidebarTitle: "Hermes Agent"
+description: "Use LanceDB as a persistent, semantic memory backend for Hermes Agent. Get durable recall across sessions with vector and hybrid search."
+---
+
+[Hermes Agent](https://github.com/NousResearch/hermes-agent) is a self-hosted, open-source
+personal agent from [Nous Research](https://nousresearch.com). You can talk to it from a
+terminal UI or reach the same agent from Telegram, Discord, and Slack, and it exposes a
+dedicated slot for external *memory providers* that run alongside its built-in notes.
+
+The [LanceDB memory plugin](https://github.com/lancedb/hermes-agent-memory) fills that slot.
+It gives Hermes durable, semantic recall across sessions: state a preference or a project
+convention once, and the agent can retrieve it weeks later in a brand-new session — even when
+you ask for it in completely different words. Everything runs inside Hermes' own Python
+process, storing a single LanceDB table on local disk. There's no memory server to operate.
+
+<Info>
+**The mental model is clean**
+
+- Hermes owns the agent loop
+- LanceDB manages the durable long-term memory and offers semantic recall.
+</Info>
+
+## Why LanceDB fits agent memory {#why-lancedb-fits-agent-memory}
+
+Out of the box, Hermes remembers with a small curated notes file frozen into the system
+prompt, plus lexical (keyword) search over past sessions. Both are useful, but keyword search
+misses paraphrases of what you originally typed — the exact thing you need when recalling a
+fact you phrased differently months ago.
+
+LanceDB is an embedded retrieval library, which makes it a natural fit here:
+
+- **No server to stand up** — it reads and writes a table on local disk, so the plugin ships
+  as a dependency rather than a service to operate.
+- **One table holds everything** — content, metadata, and embeddings live together. A memory
+  becomes a structured row with a category, tags, timestamps, and provenance, not just a text
+  blob.
+- **Query it any way you need** — vector similarity for meaning, BM25 full-text for exact
+  names and jargon, a hybrid of the two, or plain metadata filters to keep recall scoped to
+  the right workspace.
+- **It scales up** — the same table abstraction carries over to larger LanceDB deployments
+  later, so the local setup is never a dead end.
+
+## Install and activate {#install-and-activate}
+
+<Tip>
+Want to try this without touching your existing Hermes setup? Run everything in an isolated
+profile: `hermes profile create demo`, then add `-p demo` to the commands below. When you're
+done, `rm -rf ~/.hermes/profiles/demo` removes all trace.
+</Tip>
+
+<Steps>
+<Step title="Install Hermes Agent">
+Skip this if you already have Hermes installed.
+
+```bash
+curl -fsSL https://raw.githubusercontent.com/NousResearch/hermes-agent/main/scripts/install.sh | bash
+```
+</Step>
+
+<Step title="Install the plugin">
+This shallow-clones the plugin into `~/.hermes/plugins/lancedb/`.
+
+```bash
+hermes plugins install lancedb/hermes-agent-memory
+```
+</Step>
+
+<Step title="Install runtime dependencies into Hermes' environment">
+Hermes loads plugins inside its own Python interpreter, so the dependencies go *there* — not
+into a separate virtualenv. (This interpreter is shared across profiles, so you only install
+once.)
+
+```bash
+uv pip install --python ~/.hermes/hermes-agent/venv/bin/python3 lancedb openai pyyaml
+```
+</Step>
+
+<Step title="Set your embeddings API key">
+The plugin turns conversations into embeddings, so it needs an embeddings key. By default that
+is OpenAI, so set `OPENAI_API_KEY` in your environment or in `~/.hermes/.env`.
+
+<Info>
+Prefer a local or non-OpenAI model? The plugin uses an OpenAI-compatible client, so you can
+point it at any compatible endpoint (OpenRouter, Ollama, vLLM, …) in your config — no code
+change needed. See [Configuration](#configuration) below.
+</Info>
+</Step>
+
+<Step title="Activate and verify">
+Switch memory on and pick this plugin:
+
+```bash
+hermes memory setup     # choose "lancedb"
+```
+
+Then confirm it's actually active before you start chatting — this is the one step worth not
+skipping, because Hermes quietly falls back to its built-in notes if the provider isn't set:
+
+```bash
+hermes memory status
+```
+
+```text
+Memory status
+────────────────────────────────────────
+  Built-in:  always active
+  Provider:  lancedb
+
+  Plugin:    installed ✓
+  Status:    available ✓
+```
+
+You want to see `Provider: lancedb` with both `installed ✓` and `available ✓`.
+</Step>
+</Steps>
+
+## The memory tools {#the-memory-tools}
+
+Once activated, the agent has four tools for working with long-term memory:
+
+| Tool | What it does |
+|:--|:--|
+| `lancedb_recall` | Semantic (vector, the default) or hybrid search over your workspace memory. Returns matching facts with scores and provenance. |
+| `lancedb_remember` | Stores a durable fact when you explicitly ask. Deduplicated by content hash, so remembering the same thing twice doesn't pile up rows. |
+| `lancedb_read` | Fetches a single memory by ID, optionally with the original conversation messages it was distilled from. |
+| `lancedb_forget` | Deletes safely: previews candidates first, then deletes by exact ID, so nothing disappears by accident. |
+
+Beyond these tools, the plugin also captures durable facts from your conversations
+automatically — an auxiliary model distills them before context is compressed and again when a
+session ends, so insights survive even when the raw messages are summarized away.
+
+## Walkthrough {#walkthrough}
+
+"_Teach it your project preferences_"
+
+Let's make this concrete with the pain we opened on: re-explaining your setup to the agent every session.
+We'll save a convention once and then prove a brand-new session can recall it. This example will touch all four
+tools along the way.
+
+### Remember {#remember}
+
+Ask Hermes to commit a convention to long-term memory. Saying "remember in long-term memory"
+makes sure it lands in the LanceDB store, which shows up as the `⚡ lancedb_r` (`lancedb_remember`)
+line below:
+
+```text
+● Remember in long-term memory: for this project I only use uv, never pip, and I always add type hints to Python functions.
+
+  ┊ 🧠 memory    +memory: "For this project, the user only uses uv for Python package management, never pip, and always adds type hints to Python functions."
+  ┊ ⚡ lancedb_r   0.0s
+ ─  ⚕ Hermes  ────────────────────────────────────────────────────────────────
+
+     Remembered. I've stored that project convention: use uv only, never pip, and always add type hints to Python functions.
+```
+
+### Recall {#recall}
+
+First, take Hermes' built-in notes out of the picture so recall can *only* come from LanceDB —
+the two layers run side by side otherwise, and either could answer:
+
+```bash
+hermes config set memory.memory_enabled false
+```
+
+Now start a fresh session (`/new`) and ask for the convention back, worded completely
+differently from how you saved it:
+
+```text
+● /new
+● Look through your long-term memory for my Python project preferences.
+
+ ─  ⚕ Hermes  ────────────────────────────────────────────────────────────────
+
+     I found one durable memory about Python project preferences:
+
+     - Project Python tooling preference is uv; pip is not used; type hints are required.
+```
+
+This is the whole point in one exchange. You asked about your *preferences* using none of the
+words you stored the fact with — no *uv*, no *pip*, no *type hints* — yet a fresh session, with
+none of the earlier conversation in context, pulls it straight back. Semantic search matches on
+meaning rather than exact wording, which is precisely where keyword search falls short.
+
+<Tip>
+Turn the built-in layer back on for everyday use with `hermes config set memory.memory_enabled true`
+— the two memory layers happily coexist. We only disabled it here to prove the recall came from
+LanceDB.
+</Tip>
+
+### Read {#read}
+
+You can also ask where a fact came from. Hermes attributes the answer to its stored memory
+rather than guessing from a file in the repo (under the hood, `lancedb_read` can also return
+the exact source messages a fact was distilled from):
+
+```text
+● Where did that uv preference come from?
+
+ ─  ⚕ Hermes  ────────────────────────────────────────────────────────────────
+
+     It came from the project's persistent memory, not from a codebase file I inspected.
+
+     Specifically, the recalled memory says:
+     - "For this project, the user only uses uv for Python package management, never pip, and always adds type hints to Python functions."
+```
+
+### Forget {#forget}
+
+When a preference changes, ask Hermes to drop the old fact. The tool calls tell the whole
+story: the two `⚡ lancedb_f` (`lancedb_forget`) lines are it previewing matches and then
+deleting, and the trailing `⚡ lancedb_r` is it saving the replacement in the same breath:
+
+```text
+● I've switched from uv to pixi. Forget the uv preference.
+
+  ┊ ⚡ lancedb_f uv preference python package management never pip type hints  0.3s
+  ┊ ⚡ lancedb_f   0.0s
+  ┊ ⚡ lancedb_r   0.2s
+ ─  ⚕ Hermes  ────────────────────────────────────────────────────────────────
+
+     Done — I removed the uv preference from memory and replaced it with:
+
+     - This project uses pixi for Python package management
+     - Type hints on Python functions are still preferred
+```
+
+Remember, recall, read, forget: four small operations that between them cover the entire
+lifecycle of a durable memory.
+
+## Retrieval modes {#retrieval-modes}
+
+Recall ships in `vector` mode by default — pure semantic search, which is what survives the
+paraphrasing you saw above. If you also need exact name or jargon matching, switch to `hybrid`
+(vector + BM25) and choose how the two legs are fused: RRF, a vector-biased linear blend, or a
+cross-encoder reranker. Mode is set per call; fusion is a config setting.
+
+```yaml
+# ~/.hermes/config.yaml
+plugins:
+  lancedb:
+    retrieval:
+      mode: hybrid          # vector (default) | hybrid
+      reranker:
+        type: rrf           # how the vector + BM25 legs are fused
+        # Swap RRF for a reranking pass (pulls in sentence-transformers + torch):
+        # type: cross-encoder
+        # model: cross-encoder/ettin-reranker-17m-v1
+        # rerank_top_n: 50
+```
+
+The cross-encoder is the one path that pulls in a local ML stack, so it stays opt-in. It
+defaults to the compact 17M-parameter [ettin reranker](https://huggingface.co/cross-encoder/ettin-reranker-17m-v1).
+
+## Inspect the store {#inspect-the-store}
+
+Everything lives in one table named `memories` at `~/.hermes/lancedb/memories.lance`. Because
+it's a plain LanceDB table, you can open it directly and see exactly what the agent has stored
+— a `kind` column separates extracted `fact` rows from the raw `turn` rows they were drawn
+from:
+
+```python
+import lancedb
+
+db = lancedb.connect("~/.hermes/lancedb")
+tbl = db.open_table("memories")
+print(tbl.to_pandas()[["kind", "category", "content"]].head())
+```
+
+## Configuration {#configuration}
+
+The plugin runs on sensible defaults once activated — you don't have to configure anything.
+`~/.hermes/config.yaml` is purely for overrides. Two common ones:
+
+Use a cheaper model for the auxiliary fact-extraction calls:
+
+```yaml
+# ~/.hermes/config.yaml
+auxiliary:
+  lancedb_extraction:
+    provider: openrouter
+    model: google/gemini-3-flash
+```
+
+Point embeddings at a fully local endpoint (for example, Ollama) so nothing leaves your
+machine:
+
+```yaml
+# ~/.hermes/config.yaml
+plugins:
+  lancedb:
+    embedding:
+      model: nomic-embed-text
+      base_url: http://localhost:11434/v1
+      api_key_env: OLLAMA_API_KEY      # any value works for local Ollama
+```
+
+<Info>
+Changing the embedding model (or its dimension) against an existing store requires recreating
+the table — the plugin fails loudly on a dimension mismatch rather than silently returning
+nothing. Every option is documented in the plugin's [`default_config.yaml`](https://github.com/lancedb/hermes-agent-memory/blob/main/src/default_config.yaml).
+</Info>
+
+## Benchmark {#benchmark}
+
+On [LongMemEval-S](https://huggingface.co/datasets/xiaowu0162/longmemeval-cleaned), a
+long-conversation QA benchmark, LanceDB's semantic recall clearly beat Hermes' built-in lexical
+search (0.66 vs. 0.53 answer accuracy) by finding the right messages even when the question was
+worded differently from the original conversation. For the full methodology, the
+per-question-type breakdown, and a reproducible harness, see the
+[blog post](https://www.lancedb.com/blog/semantic-memory-for-hermes-agent-with-lancedb) and the
+[benchmark harness](https://github.com/lancedb/hermes-agent-memory/tree/main/benchmarks).
+
+## Why this works well {#why-this-works-well}
+
+- **It's local-first and embedded.** The LanceDB memory table lives on your disk with no server to run;
+  the plugin installs as a dependency of Hermes' own environment.
+- **Recall survives paraphrasing.** Semantic search matches meaning, not spelling, which is the
+  failure mode that sinks keyword-only session search.
+- **Memories are structured and traceable.** Each fact is a row with metadata and a link back
+  to the messages it came from, and `forget` always previews before it deletes.
+- **Nothing about it is a dead end.** As your needs grow, the same table abstraction carries
+  over to LanceDB [Enterprise](/enterprise) for automatic compaction, reindexing, and scale.
+
+To try it, install the plugin, enable it with `hermes memory setup`, and run the kind of
+workflow we walked through above.

docs/web/integrations/ai/huggingface.mdx

@@ -0,0 +1,341 @@
+---
+title: "Hugging Face Hub"
+sidebarTitle: "Hugging Face"
+description: "Use LanceDB directly on Lance datasets hosted on the Hugging Face Hub for multimodal search and retrieval."
+---
+
+[Hugging Face Hub](https://huggingface.co/datasets?format=format:lance&sort=trending) is a popular platform for sharing machine learning datasets, models, and other resources.
+
+LanceDB can directly scan Lance datasets hosted on the [Hugging Face Hub](https://huggingface.co/datasets?format=format:lance) with `hf://` URIs.
+This is enabled under the hood by the [lance-huggingface](https://lance.org/integrations/huggingface/)
+integration that allows users to stream Lance datasets directly from Hugging Face without needing to
+download them first.
+
+For ML and AI engineers working in LanceDB, this capability is incredibly useful for quickly exploring
+multimodal datasets and reusing Lance datasets shared by others, without writing custom data loaders
+or preprocessing pipelines.
+
+The snippets below use the [`lance-format/laion-1m`](https://huggingface.co/datasets/lance-format/laion-1m)
+dataset published in Lance format. The dataset includes a million image-caption pairs, and the
+Lance dataset can package image embeddings alongside the metadata. This makes it useful for
+demonstrating LanceDB's multimodal search capabilities in combination with easy sharing via the
+Hugging Face Hub.
+
+The LAION table includes multimodal columns such as:
+
+- `image` (inline JPEG bytes)
+- `caption` (text)
+- `img_emb` (image embedding vector)
+- metadata fields such as `url` and `similarity`
+
+## Install dependencies {#install-dependencies}
+
+```bash
+pip install lancedb pillow
+```
+
+## Open the dataset with LanceDB {#open-the-dataset-with-lancedb}
+
+LanceDB can open the dataset directly from the Hub, without needing to download it first.
+Note that in LanceDB, you need to specify the table name when opening a Lance dataset,
+and the Hugging Face convention is to use `train` and `test` splits for datasets.
+The LAION dataset is uploaded as a single split named `train`, so we specify the table name
+that contains the `*.lance` files when opening the dataset.
+
+```python
+import lancedb
+
+db = lancedb.connect("hf://datasets/lance-format/laion-1m/data")
+table = db.open_table("train")
+
+print(f"Opened table: {table.name}")
+print(f"Rows: {len(table)}")
+```
+
+## Inspect schema and available indexes {#inspect-schema-and-available-indexes}
+
+```python
+print(table.schema)
+```
+This prints the schema of the LAION table. Note that there's an image embedding column that's
+a fixed-size list of 768-dimensional floats, and a binary column containing the raw JPEG bytes of the image.
+```
+image_path: string
+caption: string
+NSFW: string
+similarity: double
+LICENSE: string
+url: string
+key: string
+status: string
+error_message: null
+width: int64
+height: int64
+original_width: int64
+original_height: int64
+exif: string
+md5: string
+img_emb: fixed_size_list<item: float>[768]
+  child 0, item: float
+image: binary
+```
+
+When inspecting Lance datasets from Hugging Face, it's also a good idea to check whether the dataset author included
+any pre-built indexes that you can use for search. You can check the available indexes with:
+
+```python
+print(table.list_indices())
+```
+```
+[
+    Index(IvfPq, columns=["img_emb"], name="img_emb_idx"),
+    Index(FTS, columns=["caption"], name="caption_idx")
+]
+```
+
+In this case, we see that we have an IVF_PQ vector index on the `img_emb` column, and an FTS index on the `caption`
+column, which means we can directly do vector search on the image embeddings and keyword search on the captions
+without needing to build the indexes ourselves!
+
+<Info>
+If you see an empty list, it may be because the dataset author did not include the index files when uploading
+to Hugging Face. You can download the dataset locally, and build the indexes yourself. See the [indexing guide](/indexing/)
+for instructions on building different types of indexes with LanceDB.
+</Info>
+
+## Projection scan {#projection-scan}
+
+Run a simple scan by projecting relevant columns to get a feel for the dataset. For example, we
+can run a search without any filters or input parameters to get a small subset of the data:
+
+```python
+rows = (
+    table.search()
+    .select(["caption", "url", "similarity"])
+    .limit(3)
+    .to_list()
+)
+
+for i, row in enumerate(rows, start=1):
+    print(f"{i}. {row['caption']}")
+    print(f"   url={row['url']}")
+    print(f"   similarity={row['similarity']}")
+```
+
+We get the first three rows and their metadata printed out, which look like this:
+```
+1. Cordelia and Dudley on their wedding  day last year
+   url=https://i.dailymail.co.uk/i/pix/2012/01/05/article-2082728-0EF8956600000578-53_233x315.jpg
+   similarity=0.2926466464996338
+2. Statistics on challenges for automation in 2021
+   url=https://verloop.io/wp-content/uploads/2021/02/Challenges.jpg
+   similarity=0.30174341797828674
+3. Teacher Gifts / Great gifts for your child's teacher.  Don't know what to get?  Take a look at these gifts that the teacher in your life will love!
+   url=https://i.pinimg.com/custom_covers/216x146/550494823141083777_1487893945.jpg
+   similarity=0.3362061381340027
+```
+
+## Scan and filter data {#scan-and-filter-data}
+
+Filtered search is a common pattern to narrow down interesting subsets of the data during early
+exploration. Here's an example:
+
+```python
+filtered = (
+    table.search()
+    .where("height > 600")
+    .select(["caption", "url", "width", "height"])
+    .limit(3)
+    .to_list()
+)
+
+for row in filtered:
+    print(row["caption"], row["url"], row["width"], row["height"])
+```
+
+This prints out the metadata for large images with height greater than 600 pixels:
+```
+Luca Trousers, mustard stripe https://cdn.shopify.com/s/files/1/0151/5333/products/IMG_0791_1024x1024.jpg?v=1585142190 384 766
+Baby Blue Fitted Short Sleeve T Shirt 3 https://cdn-img.prettylittlething.com/a/d/d/1/add198cab3ec30a61102437275573f4963642528_cmf6022_3.jpg 384 612
+pattern cutting made easy pdf https://i.pinimg.com/736x/7c/6c/a7/7c6ca7361815a8929b3dd6ad34a03ab9.jpg 384 1045
+```
+
+## Export image bytes to local files {#export-image-bytes-to-local-files}
+
+To work with a subset of the data locally, you can export the image bytes from the table and save them as JPEG files.
+```python
+from pathlib import Path
+
+sample = (
+    table.search()
+    .select(["image", "caption"])
+    .limit(3)
+    .to_list()
+)
+
+out_dir = Path("samples")
+out_dir.mkdir(exist_ok=True)
+
+for i, row in enumerate(sample):
+    out_path = out_dir / f"laion_{i}.jpg"
+    with open(out_path, "wb") as f:
+        f.write(row["image"])
+    print(f"Saved {out_path} | caption={row['caption']}")
+```
+
+You can now preview the images you just exported on your local machine to get a better sense of the data.
+
+## Vector search {#vector-search}
+
+You can use LanceDB to run vector search directly on the data on the Hub, **without needing to download the dataset
+or build your own vector index**. This makes it incredibly easy to explore the dataset and iterate on your search queries
+before you decide to download a local copy for further experimentation on your end.
+
+```python
+# Pick an arbitrary image embedding from the dataset
+query_embedding = (
+    table.search()
+    .select(["img_emb"])
+    .limit(1)
+    .to_list()[0]["img_emb"]
+)
+
+results = (
+    table.search(query_embedding, vector_column_name="img_emb")
+    .select(["caption", "url", "_distance"])
+    .limit(3)
+    .to_list()
+)
+
+for row in results:
+    print(row["_distance"], row["caption"])
+```
+
+| distance | caption |
+| --- | --- |
+| 0.17765313386917114 | Cordelia and Dudley on their wedding  day last year |
+| 0.17765313386917114 | Cordelia and Dudley on their wedding  day last year |
+| 0.17765313386917114 | Cordelia and Dudley on their wedding  day last year |
+
+<Warning>
+Note that the LAION dataset is known to contain a lot of duplicate images, so you may see the same image
+showing up multiple times in the search results.
+</Warning>
+
+## Full-text search {#full-text-search}
+
+Run an FTS search query that uses BM25 ranking on the `caption` column (on which we already have an FTS index):
+
+```python
+fts_results = (
+    table.search("dog running on beach", query_type="fts")
+    .select(["caption", "url", "_score"])
+    .limit(3)
+    .to_list()
+)
+```
+
+| caption | url | _score |
+| --- | --- | --- |
+| running with dog | https://www.doggytastic.com/wp… | 15.73168 |
+| Dog Running in Water | https://static.wixstatic.com/m… | 14.756516 |
+| Dogs on the run by heidiannemo… | http://ih2.redbubble.net/image… | 14.756516 |
+
+## Download the full dataset {#download-the-full-dataset}
+
+<Warning>
+You may hit Hugging Face rate limits when streaming large samples from `hf://`, despite using a Hugging Face token.
+For repeated queries or queries that operate on the full dataset, it's recommended to
+download the dataset locally and query from disk.
+</Warning>
+
+Here's how to download the entire dataset via the [Hugging Face CLI](https://huggingface.co/docs/huggingface_hub/en/guides/cli):
+
+```bash
+huggingface-cli download lance-format/laion-1m --repo-type dataset --local-dir ./laion-1m
+```
+
+## Upload your own datasets to Hugging Face in Lance format {#upload-your-own-datasets-to-hugging-face-in-lance-format}
+
+This section shows how you can upload your own Lance datasets to the Hugging Face Hub to share with the community.
+
+First, install the [Hugging Face CLI](https://huggingface.co/docs/huggingface_hub/en/guides/cli) and export both `OPENAI_API_KEY` and `HF_TOKEN`.
+Then, create a Lance dataset using LanceDB on a local machine, and then proceed to upload it to the Hub via a CLI command.
+
+```bash
+export OPENAI_API_KEY=...
+export HF_TOKEN=hf_...
+hf auth login --token "$HF_TOKEN"
+```
+
+A typical sequence of steps is given below.
+
+### 1\. Upload your local directory to the Hub {#1-upload-your-local-directory-to-the-hub}
+
+Upload the full local directory to a specified repository on the Hugging Face Hub. The command below uploads the contents of your local LanceDB directory at `/path/to/your_local_dir` to a new repository named `your_hf_org/repo_name` under your Hugging Face account.
+
+```bash bash icon="code"
+hf upload-large-folder /path/to/your_local_dir your_hf_org/repo_name \
+  --repo-type dataset \
+  --revision main
+```
+
+<Info>
+The `upload-large-folder` command is designed for [uploading large datasets](https://huggingface.co/docs/huggingface_hub/en/guides/upload) (potentially terabytes in size) and will handle multipart uploads, retries, and resuming interrupted uploads.
+</Info>
+
+### 2\. Inspect dataset versions {#2-inspect-dataset-versions}
+
+Because you can query your remote dataset directly from Hugging Face with `hf://` URIs in LanceDB, you can easily inspect the dataset versions and updates on the Hub without needing to download the data locally. This is very useful to keep track of changes to the dataset and iterate on your data collection and curation process.
+
+```python Python icon="python"
+import lancedb
+
+db = lancedb.connect("hf://datasets/your_hf_org/repo_name")
+table = db.open_table("table_name")
+
+versions = table.list_versions()
+print(versions)
+```
+This will print out the list of versions available for the dataset on the Hub, along with their metadata such as creation date and description.
+
+### 3\. Add a dataset card {#3-add-a-dataset-card}
+
+The Hub dataset card allows you to communicate the schema and usage of the dataset to other developers. It sits at the repo's root in a file named `README.md` on the Hub.
+This project keeps the source card text in `HF_DATASET_CARD.md`, so you can publish updates
+to the dataset there and upload it as `README.md` using the following command on the HF CLI:
+this requires a regular `hf upload` because it is a single-file upload to a specific target path (a custom commit message can be added if you wish).
+
+```bash
+hf upload lancedb/magical_kingdom HF_DATASET_CARD.md README.md \
+  --repo-type dataset \
+  --commit-message "Update dataset card"
+```
+
+### 4\. Update the dataset {#4-update-the-dataset}
+
+Over time, you may want to add new rows (append) or columns (backfill) to your dataset as your needs evolve. You can make the necessary updates to your local dataset using LanceDB, and then upload the updated version back to the Hub with the same `hf upload-large-folder` command.
+
+```bash bash icon="code"
+hf upload-large-folder /path/to/your_local_dir your_hf_org/repo_name \
+  --repo-type dataset \
+  --revision main
+```
+The CLI will only upload the new data that has changed since the last upload, avoiding wasted I/O while making it easy to keep your dataset up-to-date on the Hub.
+
+That's it! Your dataset is now updated on the Hub with the new data and schema changes, and other users can query the latest version of the dataset directly from Hugging Face with `hf://` URIs in LanceDB.
+
+## Explore more Lance datasets on Hugging Face {#explore-more-lance-datasets-on-hugging-face}
+
+The LanceDB team is actively uploading useful and interesting datasets in Lance format to the Hugging Face Hub
+under the [lance-format](https://huggingface.co/lance-format) organization. We actively encourage the Hugging Face
+and LanceDB communities to upload their own Lance datasets to the Hub to share with others!
+
+In the meantime, feel free to check out the Hugging Face Hub to discover more Lance datasets uploaded by the community.
+
+<Card
+  icon="/static/assets/logo/huggingface-logo.svg"
+  href="https://huggingface.co/datasets?format=format:lance&sort=trending"
+>
+Click here to explore the latest trending Lance datasets on 🤗 Hugging Face!
+</Card>

docs/web/integrations/ai/kiln.mdx

@@ -0,0 +1,48 @@
+---
+title: "Kiln AI"
+sidebarTitle: "Kiln"
+
+---
+
+[**Kiln**](https://kiln.tech) is a free tool for building production-ready AI systems, combining an intuitive desktop application and an open-source Python library. It supports RAG pipelines, evaluations, agents, MCP tool-calling, synthetic data generation, and fine-tuning. Kiln provides deep integration with LanceDB for vector search, full-text search (BM25), and hybrid search. 
+
+## Quick Start: Build a RAG Pipeline in 5 Minutes with Kiln and LanceDB {#quick-start-build-a-rag-pipeline-in-5-minutes-with-kiln-and-lancedb}
+
+Watch the [quick start overview on Vimeo](https://vimeo.com/1119945690).
+
+Kiln's [app](https://kiln.tech/download) makes it easy to:
+
+ - Build a RAG pipeline with a simple drag-and-drop interface
+ - [Compare](#find-the-best-rag-pipeline-for-your-use-case) search index options (powered by LanceDB), document extractors, embedding models, and chunking strategies
+ - Create end-to-end [evaluations](https://docs.kiln.tech/docs/evaluations) to determine which search configuration works best for your use case
+ - Load your data from Kiln into [LanceDB Enterprise](/enterprise) for production use
+ - Iterate with confidence by evaluating new content, prompts, models, and embeddings in minutes instead of weeks
+
+## Find the Best RAG Pipeline for Your Use Case {#find-the-best-rag-pipeline-for-your-use-case}
+
+There is no universal best RAG solution—only the best solution for your specific use case. Kiln makes it easy to compare state-of-the-art configurations and find which works best for you.
+
+Start with pre-configured templates for state-of-the-art RAG at various performance/quality/cost levels, or experiment with any combination of options:
+
+|Area|Technologies|Description|
+|:----|:----|:----|
+|Search Index|LanceDB|Compare LanceDB's vector search, full-text search (BM25), and hybrid search to find the best approach for your use case.|
+|Content|Kiln Document Library|Collaborate on a document library with your team to find the best content for your RAG. Track every revision and tag document sets.|
+|Document Extraction|Gemini, OpenAI GPT, Qwen VL, and more|Find the most accurate document extraction models for converting PDFs, images, audio, video, and other formats into textual data for RAG.|
+|Embeddings|Embedding models from Gemini, OpenAI, Nomic, Qwen, and more|Find the embedding model best suited to your use case.|
+|Chunking|LlamaIndex|Find the ideal chunk size and method.|
+
+## Get Started {#get-started}
+
+To get started, download the [Kiln App](https://kiln.tech/download), create a project, and navigate to "Docs & Search".
+
+See the [Kiln documentation for creating a RAG system](https://docs.kiln.tech/docs/documents-and-search-rag) for details on each step of the process.
+
+## More Information {#more-information}
+
+ - [Kiln Homepage](https://kiln.tech)
+ - [Download the Kiln App](https://kiln.tech/download)
+ - [Kiln GitHub Repository](https://github.com/Kiln-AI/Kiln)
+ - [Building RAG Systems - Kiln Documentation](https://docs.kiln.tech/docs/documents-and-search-rag)
+ - [Python Library](https://pypi.org/project/kiln-ai/) or `pip install kiln_ai`
+

docs/web/integrations/ai/langchain.mdx

@@ -0,0 +1,226 @@
+---
+title: "LangChain"
+sidebarTitle: "LangChain"
+
+---
+
+import {
+  PyFrameworksLangchainAddImages,
+  PyFrameworksLangchainAddTexts,
+  PyFrameworksLangchainCreateIndex,
+  PyFrameworksLangchainMaxMarginalRelevance,
+  PyFrameworksLangchainQuickStart,
+  PyFrameworksLangchainSimilaritySearch,
+  PyFrameworksLangchainSimilaritySearchByVector,
+  PyFrameworksLangchainSimilaritySearchByVectorWithScores,
+  PyFrameworksLangchainSimilaritySearchWithScores,
+  PyFrameworksLangchainVectorStoreConfig,
+} from '/snippets/integrations.mdx';
+
+**LangChain** is a framework designed for building applications with large language models (LLMs) by chaining together various components. It supports a range of functionalities including memory, agents, and chat models, enabling developers to create context-aware applications.
+
+![Illustration](https://raw.githubusercontent.com/lancedb/assets/refs/heads/main/docs/assets/integration/langchain_rag.png)
+
+LangChain streamlines these stages (in figure above) by providing pre-built components and tools for integration, memory management, and deployment, allowing developers to focus on application logic rather than underlying complexities.
+
+Integration of **Langchain** with **LanceDB** enables applications to retrieve the most relevant data by comparing query vectors against stored vectors, facilitating effective information retrieval. It results in better and context aware replies and actions by the LLMs.
+
+## Quick Start {#quick-start}
+You can load your document data using langchain's loaders, for this example we are using `TextLoader` and `OpenAIEmbeddings` as the embedding model.
+
+<CodeBlock filename="Python" language="Python" icon="python">
+  {PyFrameworksLangchainQuickStart}
+</CodeBlock>
+
+## Documentation {#documentation}
+In the above example `LanceDB` vector store class object is created using `from_documents()` method  which is a `classmethod` and returns the initialized class object. 
+
+You can also use `LanceDB.from_texts(texts: List[str],embedding: Embeddings)` class method.  
+
+The exhaustive list of parameters for `LanceDB` vector store are : 
+
+|Name|type|Purpose|default|
+|:----|:----|:----|:----|
+|`connection`| (Optional) `Any` |`lancedb.db.LanceDBConnection` connection object to use.  If not provided, a new connection will be created.|`None`|
+|`embedding`| (Optional) `Embeddings` | Langchain embedding model.|Provided by user.|
+|`uri`| (Optional) `str` |It specifies the directory location of **LanceDB database** and establishes a connection that can be used to interact with the database. |`/tmp/lancedb`|
+|`vector_key` |(Optional) `str`| Column name to use for vector's in the table.|`'vector'`|
+|`id_key` |(Optional) `str`| Column name to use for id's in the table.|`'id'`|
+|`text_key` |(Optional) `str` |Column name to use for text in the table.|`'text'`|
+|`table_name` |(Optional) `str`| Name of your table in the database.|`'vectorstore'`|
+|`api_key` |(Optional `str`) |API key to use for LanceDB Enterprise deployment.|`None`|
+|`region` |(Optional) `str`| Region to use for LanceDB Enterprise deployment.| Only for LanceDB Enterprise : `None`.|
+|`mode` |(Optional) `str` | Mode to use for adding data to the table. Valid values are "append" and "overwrite".|`'overwrite'`|
+|`table`| (Optional) `Any`| You can connect to an existing table of LanceDB, created outside of langchain, and utilize it.|`None`|
+|`distance`|(Optional) `str`| The choice of distance metric used to calculate the similarity between vectors.|`'l2'`|
+|`reranker` |(Optional) `Any`| The reranker to use for LanceDB.|`None`|
+|`relevance_score_fn` |(Optional) `Callable[[float], float]` | Langchain relevance score function to be used.|`None`|
+|`limit`|`int`|Set the maximum number of results to return.| `DEFAULT_K` (it is 4)|
+
+<CodeBlock filename="Python" language="Python" icon="python">
+  {PyFrameworksLangchainVectorStoreConfig}
+</CodeBlock>
+
+### Methods {#methods}
+
+##### `add_texts()`
+
+This method turn texts into embedding and add it to the database.
+
+|Name|Purpose|defaults|
+|:---|:---|:---|
+|`texts`|`Iterable` of strings to add to the vectorstore.|Provided by user|
+|`metadatas`|Optional `list[dict()]` of metadatas associated with the texts.|`None`|
+|`ids`|Optional `list` of ids to associate with the texts.|`None`|
+|`kwargs`| Other keyworded arguments provided by the user. |-|
+
+It returns list of ids of the added texts.
+
+<CodeBlock filename="Python" language="Python" icon="python">
+  {PyFrameworksLangchainAddTexts}
+</CodeBlock>
+
+------
+
+
+##### create_index() 
+
+This method creates a scalar(for non-vector cols) or a vector index on a table.
+
+|Name|type|Purpose|defaults|
+|:---|:---|:---|:---|
+|`vector_col`|`Optional[str]`| Provide if you want to create index on a vector column. |`None`|
+|`col_name`|`Optional[str]`| Provide if you want to create index on a non-vector column. |`None`|
+|`metric`|`Optional[str]` |Provide the metric to use for vector index. choice of metrics: 'l2', 'dot', 'cosine'. |`l2`|
+|`num_partitions`|`Optional[int]`|Number of partitions to use for the index.|`256`|
+|`num_sub_vectors`|`Optional[int]` |Number of sub-vectors to use for the index.|`96`|
+|`index_cache_size`|`Optional[int]` |Size of the index cache.|`None`|
+|`name`|`Optional[str]` |Name of the table to create index on.|`None`|
+
+For index creation make sure your table has enough data in it. An ANN index is usually not needed for datasets ~100K vectors. For large-scale (>1M) or higher dimension vectors, it is beneficial to create an ANN index.
+
+<CodeBlock filename="Python" language="Python" icon="python">
+  {PyFrameworksLangchainCreateIndex}
+</CodeBlock>
+
+------
+
+##### similarity_search()
+
+This method performs similarity search based on **text query**.
+
+| Name    | Type                 | Purpose | Default |
+|---------|----------------------|---------|---------|
+| `query` | `str`                |  A `str` representing the text query that you want to search for in the vector store.         | N/A     |
+| `k`         | `Optional[int]`           | It specifies the number of documents to return.        | `None`    |
+| `filter`    | `Optional[Dict[str, str]]`| It is used to filter the search results by specific metadata criteria.        | `None`    |
+| `fts`   | `Optional[bool]`     |   It indicates whether to perform a full-text search (FTS).      | `False`   |
+| `name`      | `Optional[str]`           | It is used for specifying the name of the table to query. If not provided, it uses the default table set during the initialization of the LanceDB instance.        | `None`    |
+| `kwargs`    | `Any`                     | Other keyworded arguments provided by the user.        | N/A     |
+
+Return documents most similar to the query **without relevance scores**.
+
+<CodeBlock filename="Python" language="Python" icon="python">
+  {PyFrameworksLangchainSimilaritySearch}
+</CodeBlock>
+
+------
+
+##### similarity_search_by_vector()
+
+The method returns documents that are most similar to the specified **embedding (query) vector**. 
+
+| Name        | Type                      | Purpose | Default |
+|-------------|---------------------------|---------|---------|
+| `embedding` | `List[float]`             | The embedding vector you want to use to search for similar documents in the vector store.         | N/A     |
+| `k`         | `Optional[int]`           | It specifies the number of documents to return.        | `None`    |
+| `filter`    | `Optional[Dict[str, str]]`| It is used to filter the search results by specific metadata criteria.        | `None`    |
+| `name`      | `Optional[str]`           | It is used for specifying the name of the table to query. If not provided, it uses the default table set during the initialization of the LanceDB instance.        | `None`    |
+| `kwargs`    | `Any`                     | Other keyworded arguments provided by the user.        | N/A     |
+
+**It does not provide relevance scores.**
+
+<CodeBlock filename="Python" language="Python" icon="python">
+  {PyFrameworksLangchainSimilaritySearchByVector}
+</CodeBlock>
+
+------
+
+##### similarity_search_with_score()
+
+Returns documents most similar to the **query string** along with their relevance scores.
+
+| Name     | Type                      | Purpose | Default |
+|----------|---------------------------|---------|---------|
+| `query`  | `str`                     |A `str` representing the text query you want to search for in the vector store. This query will be converted into an embedding using the specified embedding function.         | N/A     |
+| `k`      | `Optional[int]`           | It specifies the number of documents to return.  | `None`    |
+| `filter` | `Optional[Dict[str, str]]`|  It is used to filter the search results by specific metadata criteria. This allows you to narrow down the search results based on certain metadata attributes associated with the documents.       | `None`    |
+| `kwargs` | `Any`                     |  Other keyworded arguments provided by the user.       | N/A     |
+
+It gets called by base class's `similarity_search_with_relevance_scores` which selects relevance score based on our `_select_relevance_score_fn`.
+
+<CodeBlock filename="Python" language="Python" icon="python">
+  {PyFrameworksLangchainSimilaritySearchWithScores}
+</CodeBlock>
+
+------
+
+##### similarity_search_by_vector_with_relevance_scores()
+
+Similarity search using **query vector**.
+
+| Name        | Type                      | Purpose | Default |
+|-------------|---------------------------|---------|---------|
+| `embedding` | `List[float]`             | The embedding vector you want to use to search for similar documents in the vector store.        | N/A     |
+| `k`         | `Optional[int]`           |   It specifies the number of documents to return.       | `None`    |
+| `filter`    | `Optional[Dict[str, str]]`|  It is used to filter the search results by specific metadata criteria.       | `None`    |
+| `name`      | `Optional[str]`           |   It is used for specifying the name of the table to query.       | `None`    |
+| `kwargs`    | `Any`                     |   Other keyworded arguments provided by the user.      | N/A     |
+
+The method returns documents most similar to the specified embedding (query) vector, along with their relevance scores.
+
+<CodeBlock filename="Python" language="Python" icon="python">
+  {PyFrameworksLangchainSimilaritySearchByVectorWithScores}
+</CodeBlock>
+
+------
+
+##### max_marginal_relevance_search()
+
+This method returns docs selected using the maximal marginal relevance(MMR).
+Maximal marginal relevance optimizes for similarity to query AND diversity among selected documents.
+
+| Name | Type | Purpose | Default |
+|---------------|-----------------|-----------|---------|
+| `query`       | `str` | Text to look up documents similar to. | N/A     |
+| `k`   | `Optional[int]` | Number of Documents to return.| `4`       |
+| `fetch_k`| `Optional[int]`| Number of Documents to fetch to pass to MMR algorithm.| `None`    |
+| `lambda_mult` | `float`                   | Number between 0 and 1 that determines the degree of diversity among the results with 0 corresponding to maximum diversity and 1 to minimum diversity. | `0.5`     |
+| `filter`| `Optional[Dict[str, str]]`| Filter by metadata. | `None`    |
+|`kwargs`| Other keyworded arguments provided by the user. |-|
+
+Similarly, `max_marginal_relevance_search_by_vector()` function returns docs most similar to the embedding passed to the function using MMR. instead of a string query you need to pass the embedding to be searched for. 
+
+<CodeBlock filename="Python" language="Python" icon="python">
+  {PyFrameworksLangchainMaxMarginalRelevance}
+</CodeBlock>
+
+------
+
+##### add_images()
+
+This method adds images by automatically creating their embeddings and adds them to the vectorstore.
+
+| Name       | Type                          | Purpose                        | Default |
+|------------|-------------------------------|--------------------------------|---------|
+| `uris`     | `List[str]`                   | File path to the image         | N/A     |
+| `metadatas`| `Optional[List[dict]]`        | Optional list of metadatas     | `None`    |
+| `ids`      | `Optional[List[str]]`         | Optional list of IDs           | `None`    |
+
+It returns list of IDs of the added images.
+
+<CodeBlock filename="Python" language="Python" icon="python">
+  {PyFrameworksLangchainAddImages}
+</CodeBlock>
+
+

docs/web/integrations/ai/llamaIndex.mdx

@@ -0,0 +1,55 @@
+---
+title: "LlamaIndex"
+sidebarTitle: "LlamaIndex"
+
+---
+
+import {
+  PyFrameworksLlamaindexAddReranker,
+  PyFrameworksLlamaindexFiltering,
+  PyFrameworksLlamaindexHybridSearch,
+  PyFrameworksLlamaindexQuickStart,
+} from '/snippets/integrations.mdx';
+
+## Quickstart {#quickstart}
+
+LlamaIndex is a well-known framework for building LLM-powered agents over your data with LLMs and workflows.
+You can build your LlamaIndex pipeline and persist your metadata and embeddings in LanceDB via the `LanceDBVectorStore` class.
+
+First, install the LlamaIndex-LanceDB integration.
+
+<CodeBlock filename="bash" language="bash" icon="terminal">
+pip install llama-index-vector-stores-LanceDB
+</CodeBlock>
+
+Run the below script as an example.
+
+<CodeBlock filename="Python" language="Python" icon="python">
+  {PyFrameworksLlamaindexQuickStart}
+</CodeBlock>
+
+The vector store connector will open an existing LanceDB directory or create the directory if it does not exist.
+
+### Filtering {#filtering}
+For metadata filtering, you can use a Lance SQL-like string filter as demonstrated in the example above. Additionally, you can also filter using the `MetadataFilters` class from LlamaIndex:
+<CodeBlock filename="Python" language="Python" icon="python">
+  {PyFrameworksLlamaindexFiltering}
+</CodeBlock>
+
+### Hybrid Search {#hybrid-search}
+For complete documentation, refer [here](https://docs.lancedb.com/search/hybrid-search). This example uses the `colbert` reranker. Make sure to install necessary dependencies for the reranker you choose.
+<CodeBlock filename="Python" language="Python" icon="python">
+  {PyFrameworksLlamaindexHybridSearch}
+</CodeBlock>
+
+In the snippet above, you can change/specify `query_type` when creating the engine/retriever
+to use different search strategies, such as vector search or FTS.
+
+## API reference {#api-reference}
+
+<Card
+  title="LlamaIndex Vector Stores API reference"
+  href="https://developers.llamaindex.ai/python/framework-api-reference/storage/vector_store/lancedb/"
+>
+See the official LlamaIndex Vector Stores API reference for more details.
+</Card>

docs/web/integrations/ai/prompttools.mdx

@@ -0,0 +1,15 @@
+---
+title: "PromptTools"
+sidebarTitle: "PromptTools"
+
+---
+
+[PromptTools](https://github.com/hegelai/prompttools) offers a set of free, open-source tools for testing and experimenting with models, prompts, and configurations. The core idea is to enable developers to evaluate prompts using familiar interfaces like code and notebooks. You can use it to experiment with different configurations of LanceDB, and test how LanceDB integrates with the LLM of your choice.
+{/*
+[Evaluating Prompts with PromptTools](./examples/prompttools-eval-prompts/) | <a href="https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/prompttools-eval-prompts/main.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"></a>
+*/}
+
+<a href="https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/prompttools-eval-prompts/main.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab" /></a>
+
+![Alt text](https://prompttools.readthedocs.io/en/latest/_images/demo.gif "a title")
+

docs/web/integrations/ai/synthetic-data-kit.mdx

@@ -0,0 +1,81 @@
+---
+title: "Meta Llama Synthetic Data Kit"
+sidebarTitle: "Synthetic Data Kit"
+description: "Use Meta Llama's Synthetic Data Kit with LanceDB to generate high-quality synthetic datasets for LLM fine-tuning and training."
+
+---
+
+
+
+[Synthetic Data Kit](https://github.com/meta-llama/synthetic-data-kit) is a tool from Meta LLAMA that helps you generate high-quality synthetic datasets for fine-tuning large language models (LLMs). It simplifies the process of preparing data for fine-tuning by providing a command-line interface (CLI) with a modular four-command flow.
+
+One of the key features of the `synthetic-data-kit` is its use of the Lance format for storing and ingesting datasets. This allows for efficient storage and retrieval of data, which is crucial when working with large datasets.
+
+### Key Features: {#key-features}
+
+*   **Data Ingestion:** The toolkit can ingest various file formats, including PDF, HTML, YouTube transcripts, DOCX, PPT, and TXT.
+*   **Fine-tuning Format Creation:** It can create different fine-tuning formats, such as question-answer (QA) pairs, QA pairs with Chain-of-Thought (CoT), and summarization formats.
+*   **Data Curation:** The tool uses Llama as a judge to curate high-quality examples, ensuring the quality of the generated dataset.
+*   **Flexible Saving Options:** You can save the generated datasets in various formats compatible with your fine-tuning workflow, including Hugging Face, JSONL, and JSON.
+
+### How it Works: {#how-it-works}
+
+The synthetic-data-kit follows a simple four-step process:
+
+1.  **Ingest:** Import your input files into the toolkit. The data is stored in the Lance format for efficient processing.
+2.  **Create:** Generate diverse fine-tuning datasets, such as reasoning, summarization, and QA pairs, from the ingested documents.
+3.  **Curate:** Use Llama to filter and select high-quality examples from the generated dataset.
+4.  **Save-as:** Export the curated dataset in your preferred format.
+
+### Usage {#usage}
+
+The `synthetic-data-kit` uses Lance format to store and manage the data that you ingest. The workflow is a series of commands that build on each other, starting with the `ingest` command.
+
+Here is an example of the end-to-end workflow:
+
+1.  **Ingest Data into a LanceDB dataset**
+
+    This command takes a directory of source files and creates a LanceDB dataset from them.
+
+    ```bash
+    synthetic-data-kit ingest docs/report.pdf --multimodal
+    # This will create a Lance dataset at data/parsed/report.lance
+    # with 'text' and 'image' columns.
+    
+    #Generate multimodal-qa pairs from the ingested data
+    synthetic-data-kit create data/parsed/report.lance --type multimodal-qa
+    ```
+
+2.  **Create fine-tuning data**
+
+    This command uses the LanceDB dataset created in the previous step to generate synthetic data in the desired format.
+
+    ```bash
+    synthetic_data create data/parsed/report.lance
+    ```
+
+3.  **Curate the data**
+
+    This step uses a language model to curate the generated data and ensure its quality.
+
+    ```bash
+    synthetic_data curate report.json
+    ```
+
+4.  **Save the final dataset**
+
+    Finally, save the curated data to a file in the desired format.
+
+    ```bash
+    synthetic_data save-as  report.json --save_path ./my_finetuning_data.jsonl
+    ```
+
+This workflow allows you to go from a collection of documents to a high-quality, fine-tuning dataset with just a few commands. The use of LanceDB in the background makes the process efficient and scalable.
+
+### Getting Started: {#getting-started}
+
+To get started with the synthetic-data-kit, you can clone the [GitHub Repository](https://github.com/meta-llama/synthetic-data-kit) and install the necessary dependencies. 
+
+> **Note:** You will also need access to a Llama model, either running locally or via a hosted API.
+
+

docs/web/integrations/data/dlt.mdx

@@ -0,0 +1,97 @@
+---
+title: "dlt"
+sidebarTitle: "DLT"
+
+---
+
+import {
+  PyPlatformsDltAdapterImport,
+  PyPlatformsDltAdapterUsage,
+  PyPlatformsDltPipeline,
+} from '/snippets/integrations.mdx';
+
+[dlt](https://dlthub.com/docs/intro) is an open-source library that you can add to your Python scripts to load data from various and often messy data sources into well-structured, live datasets. dlt's [integration with LanceDB](https://dlthub.com/docs/dlt-ecosystem/destinations/lancedb) lets you ingest data from any source (databases, APIs, CSVs, dataframes, JSONs, and more) into LanceDB with a few lines of simple python code. The integration enables automatic normalization of nested data, schema inference, incremental loading and embedding the data. dlt also has integrations with several other tools like dbt, airflow, dagster etc. that can be inserted into your LanceDB workflow.
+
+## How to ingest data into LanceDB {#how-to-ingest-data-into-lancedb}
+
+In this example, we will be fetching movie information from the [Open Movie Database (OMDb) API](https://www.omdbapi.com/) and loading it into a local LanceDB instance. To implement it, you will need an API key for the OMDb API (which can be created freely [here](https://www.omdbapi.com/apikey.aspx)).
+  
+1. **Install `dlt` with LanceDB extras:**  
+    ```sh 
+    pip install dlt[lancedb]
+    ```
+
+2. **Inside an empty directory, initialize a `dlt` project with:**  
+    ```sh
+    dlt init rest_api lancedb
+    ```
+    This will add all the files necessary to create a `dlt` pipeline that can ingest data from any REST API (ex: OMDb API) and load into LanceDB.
+    ```text
+    ├── .dlt
+    │   ├── config.toml
+    │   └── secrets.toml
+    ├── rest_api
+    ├── rest_api_pipeline.py
+    └── requirements.txt
+    ```
+  
+    dlt has a list of pre-built [sources](https://dlthub.com/docs/dlt-ecosystem/verified-sources/) like [SQL databases](https://dlthub.com/docs/dlt-ecosystem/verified-sources/sql_database), [REST APIs](https://dlthub.com/docs/dlt-ecosystem/verified-sources/rest_api), [Google Sheets](https://dlthub.com/docs/dlt-ecosystem/verified-sources/google_sheets), [Notion](https://dlthub.com/docs/dlt-ecosystem/verified-sources/notion) etc., that can be used out-of-the-box by running `dlt init <source_name> lancedb`. Since dlt is a python library, it is also very easy to modify these pre-built sources or to write your own custom source from scratch.
+
+
+3. **Specify necessary credentials and/or embedding model details:**  
+    
+    In order to fetch data from the OMDb API, you will need to pass a valid API key into your pipeline. Depending on whether you're using LanceDB OSS or LanceDB Enterprise, you also may need to provide the necessary credentials to connect to the LanceDB instance. These can be pasted inside `.dlt/secrets.toml`. 
+
+    dlt's LanceDB integration also allows you to automatically embed the data during ingestion. Depending on the embedding model chosen, you may need to paste the necessary credentials inside `.dlt/secrets.toml`:
+    ```toml
+    [sources.rest_api]
+    api_key = "api_key" # Enter the API key for the OMDb API
+
+    [destination.lancedb]
+    embedding_model_provider = "sentence-transformers"
+    embedding_model = "all-MiniLM-L6-v2"
+    [destination.lancedb.credentials]
+    uri = ".lancedb"
+    api_key = "api_key" # API key to connect to LanceDB Enterprise. Leave out if you are using LanceDB OSS.
+    embedding_model_provider_api_key = "embedding_model_provider_api_key" # Not needed for providers that don't need authentication (ollama, sentence-transformers).
+    ```
+    See [here](https://dlthub.com/docs/dlt-ecosystem/destinations/lancedb#configure-the-destination) for more information and for a list of available models and model providers.  
+
+
+4. **Write the pipeline code inside `rest_api_pipeline.py`:**  
+
+    The following code shows how you can configure dlt's REST API source to connect to the [OMDb API](https://www.omdbapi.com/), fetch all movies with the word "godzilla" in the title, and load it into a LanceDB table. The REST API source allows you to pull data from any API with minimal code, to learn more read the [dlt docs](https://dlthub.com/docs/dlt-ecosystem/verified-sources/rest_api).
+
+    <CodeBlock filename="Python" language="Python" icon="python">
+      {PyPlatformsDltPipeline}
+    </CodeBlock>
+
+    The script above will ingest the data into LanceDB as it is, i.e. without creating any embeddings. If we want to embed one of the fields (for example, `"Title"` that contains the movie titles), then we will use dlt's `lancedb_adapter` and modify the script as follows:  
+    
+    - Add the following import statement:
+        <CodeBlock filename="Python" language="Python" icon="python">
+          {PyPlatformsDltAdapterImport}
+        </CodeBlock>
+    - Modify the pipeline run like this:
+        <CodeBlock filename="Python" language="Python" icon="python">
+          {PyPlatformsDltAdapterUsage}
+        </CodeBlock>
+    This will use the embedding model specified inside `.dlt/secrets.toml` to embed the field `"Title"`.
+
+5. **Install necessary dependencies:**  
+    ```sh
+    pip install -r requirements.txt
+    ```  
+
+    Note: You may need to install the dependencies for your embedding models separately.
+    ```sh
+    pip install sentence-transformers
+    ```
+
+6. **Run the pipeline:**
+    Finally, running the following command will ingest the data into your LanceDB instance.
+    ```sh
+    python custom_source.py
+    ```
+
+For more information and advanced usage of dlt's LanceDB integration, read [the dlt documentation](https://dlthub.com/docs/dlt-ecosystem/destinations/lancedb).

docs/web/integrations/data/duckdb.mdx

@@ -0,0 +1,119 @@
+---
+title: "DuckDB"
+sidebarTitle: "DuckDB"
+description: "Learn how to use the DuckDB-Lance extension to query Lance tables with SQL."
+---
+
+LanceDB integrates with [DuckDB](https://duckdb.org/) through the [Lance extension](https://github.com/lance-format/lance-duckdb) for DuckDB. In this page, we'll show how LanceDB manages table lifecycle, and DuckDB provides SQL analytics (including joins) and search over those tables.
+
+Note that earlier versions of LanceDB used to recommend converting Lance tables to Arrow tables via `table.to_arrow()`. Although this method is still available (because DuckDB [natively scans Arrow tables](https://duckdb.org/2021/12/03/duck-arrow)), it is no longer the recommended workflow for working with Lance tables in DuckDB. This page shows how to use the Lance extension with namespace-attached LanceDB tables, allowing you to pushdown SQL queries directly to the Lance layer.
+
+
+## Install {#install}
+
+Install the DuckDB CLI as per [their docs](https://duckdb.org/install) and alternatively, their Python package with `pip install duckdb`.
+
+Then, open the DuckDB CLI and install and load the Lance extension as follows:
+
+```sql SQL icon="database"
+INSTALL lance;
+LOAD lance;
+```
+
+## Attach the directory namespace in DuckDB {#attach-the-directory-namespace-in-duckdb}
+
+Attach the LanceDB root directory as a Lance namespace:
+
+```sql SQL icon="database"
+ATTACH './local_lancedb' AS lance_ns (TYPE LANCE);
+```
+
+In this page, tables are referenced using `lance_ns.main.<table_name>`, so the table path is `lance_ns.main.lance_duck`.
+
+## Write Lance table {#write-lance-table}
+
+Create the `lance_duck` table using SQL and populate it with sample data:
+
+```sql SQL icon="database"
+CREATE OR REPLACE TABLE lance_ns.main.lance_duck AS
+SELECT *
+FROM (
+  VALUES
+    ('duck', 'quack', [0.9, 0.7, 0.1]::FLOAT[]),
+    ('horse', 'neigh', [0.3, 0.1, 0.5]::FLOAT[]),
+    ('dragon', 'roar', [0.5, 0.2, 0.7]::FLOAT[])
+) AS t(animal, noise, vector);
+```
+
+This table is the source of truth for all DuckDB queries below.
+
+<Info>
+The examples below show SQL entered in the DuckDB CLI. You can run the same SQL from
+Python as well, using LanceDB and DuckDB's Python clients in your application code.
+</Info>
+
+## Query the table with SQL {#query-the-table-with-sql}
+
+```sql SQL icon="database"
+SELECT *
+  FROM lance_ns.main.lance_duck
+  LIMIT 5;
+```
+
+## Vector search {#vector-search}
+
+```sql SQL icon="database"
+SELECT animal, noise, vector, _distance
+  FROM lance_vector_search(
+    'lance_ns.main.lance_duck',
+    'vector',
+    [0.8, 0.7, 0.2]::FLOAT[],
+    k = 1,
+    prefilter = true
+  )
+  ORDER BY _distance ASC;
+```
+
+## Full-text search {#full-text-search}
+
+```sql SQL icon="database"
+SELECT animal, noise, vector, _score
+  FROM lance_fts(
+    'lance_ns.main.lance_duck',
+    'animal',
+    'the brave knight faced the dragon',
+    k = 1,
+    prefilter = true
+  )
+  ORDER BY _score DESC;
+```
+
+## Hybrid search {#hybrid-search}
+
+```sql SQL icon="database"
+SELECT animal, noise, vector, _hybrid_score, _distance, _score
+  FROM lance_hybrid_search(
+    'lance_ns.main.lance_duck',
+    'vector',
+    [0.8, 0.7, 0.2]::FLOAT[],
+    'animal',
+    'the duck surprised the dragon',
+    k = 2,
+    prefilter = false,
+    alpha = 0.5,
+    oversample_factor = 4
+  )
+  ORDER BY _hybrid_score DESC;
+```
+
+## Directory namespace model {#directory-namespace-model}
+
+A directory namespace maps a LanceDB catalog root to namespace-qualified table identifiers in DuckDB. This keeps table discovery and table naming stable as your project grows.
+
+To learn more about the catalog and namespace model, see [Namespaces and the Catalog Model](/namespaces).
+
+
+## Advanced usage {#advanced-usage}
+
+See the [docs](https://github.com/lance-format/lance-duckdb) directory in the Lance-DuckDB extension repo
+for more advanced usage on SQL and REST API clients.

docs/web/integrations/data/pandas_and_pyarrow.mdx

@@ -0,0 +1,46 @@
+---
+title: "Pandas and PyArrow"
+sidebarTitle: "Pandas & PyArrow"
+
+---
+
+import {
+  PyPlatformsPandasAsyncExample,
+  PyPlatformsPandasCreateTable,
+  PyPlatformsPandasImports,
+  PyPlatformsPandasVectorSearch,
+} from '/snippets/integrations.mdx';
+
+Because Lance is built on top of [Apache Arrow](https://arrow.apache.org/),
+LanceDB fits naturally into Pandas-first workflows. You can ingest a `DataFrame`,
+query it with LanceDB's vector operators, and keep working in Pandas without any glue code.
+
+## Create a dataset {#create-a-dataset}
+
+Start by importing LanceDB alongside your usual Pandas utilities and connect to a temporary database.
+
+<CodeBlock filename="Python" language="Python" icon="python">
+  {PyPlatformsPandasImports}
+</CodeBlock>
+
+Use the familiar `pd.DataFrame` API to prepare your rows, then pass the entire frame to `db.create_table`.
+
+<CodeBlock filename="Python" language="Python" icon="python">
+  {PyPlatformsPandasCreateTable}
+</CodeBlock>
+
+## Vector search {#vector-search}
+
+Queries can return Pandas frames as well, so you can immediately inspect the results or pipe them into downstream analytics.
+
+<CodeBlock filename="Python" language="Python" icon="python">
+  {PyPlatformsPandasVectorSearch}
+</CodeBlock>
+
+## Async API {#async-api}
+
+For web services or background jobs that already rely on `asyncio`, use the asynchronous helpers to keep everything non-blocking.
+
+<CodeBlock filename="Python" language="Python" icon="python">
+  {PyPlatformsPandasAsyncExample}
+</CodeBlock>

docs/web/integrations/data/polars_arrow.mdx

@@ -0,0 +1,51 @@
+---
+title: "Polars"
+sidebarTitle: "Polars & Arrow"
+
+---
+
+import {
+  PyPlatformsPolarsCreateTable,
+  PyPlatformsPolarsImports,
+  PyPlatformsPolarsLazyframe,
+  PyPlatformsPolarsPydantic,
+  PyPlatformsPolarsVectorSearch,
+} from '/snippets/integrations.mdx';
+
+LanceDB supports [Polars](https://github.com/pola-rs/polars), a blazingly fast DataFrame library for Python written in Rust. Under the hood, both Lance and Polars speak Arrow, so passing data back and forth stays zero-copy and ergonomic.
+
+## Create and Query a Table {#create-and-query-a-table}
+
+Import the required libraries, including the optional Pydantic helpers if you plan to define schemas.
+
+<CodeBlock filename="Python" language="Python" icon="python">
+  {PyPlatformsPolarsImports}
+</CodeBlock>
+
+Build a Polars `DataFrame`, convert it to Arrow, and use it directly when creating a LanceDB table.
+
+<CodeBlock filename="Python" language="Python" icon="python">
+  {PyPlatformsPolarsCreateTable}
+</CodeBlock>
+
+Run vector search and keep the results as a Polars `DataFrame` for further processing or visualization.
+
+<CodeBlock filename="Python" language="Python" icon="python">
+  {PyPlatformsPolarsVectorSearch}
+</CodeBlock>
+
+## Work with LazyFrames {#work-with-lazyframes}
+
+When you want to operate on the entire table (potentially larger than RAM), convert to a Polars `LazyFrame` so you can chain transformations without loading everything at once.
+
+<CodeBlock filename="Python" language="Python" icon="python">
+  {PyPlatformsPolarsLazyframe}
+</CodeBlock>
+
+## Define Schemas with Pydantic {#define-schemas-with-pydantic}
+
+You can also describe your table via `LanceModel` and continue ingesting data from Polars. This is useful when multiple teams share a schema or when you want validation.
+
+<CodeBlock filename="Python" language="Python" icon="python">
+  {PyPlatformsPolarsPydantic}
+</CodeBlock>

docs/web/integrations/data/pydantic.mdx

@@ -0,0 +1,90 @@
+---
+title: "Pydantic"
+sidebarTitle: "Pydantic"
+
+---
+
+import {
+  PyFrameworksPydanticBaseExample,
+  PyFrameworksPydanticBaseModel,
+  PyFrameworksPydanticImports,
+  PyFrameworksPydanticSetUrl,
+  PyFrameworksPydanticTypeConversion,
+  PyFrameworksPydanticVectorField,
+} from '/snippets/integrations.mdx';
+
+[Pydantic](https://docs.pydantic.dev/latest/) is a data validation library in Python.
+LanceDB integrates with Pydantic for schema inference, data ingestion, and query result casting.
+Using `lancedb.pydantic.LanceModel`, users can seamlessly
+integrate Pydantic with the rest of the LanceDB APIs.
+
+First, import the necessary LanceDB and Pydantic modules:
+
+<CodeBlock filename="Python" language="Python" icon="python">
+  {PyFrameworksPydanticImports}
+</CodeBlock>
+
+Next, define your Pydantic model by inheriting from `LanceModel` and specifying your fields including a vector field:
+
+<CodeBlock filename="Python" language="Python" icon="python">
+  {PyFrameworksPydanticBaseModel}
+</CodeBlock>
+
+Set the database connection URL:
+
+<CodeBlock filename="Python" language="Python" icon="python">
+  {PyFrameworksPydanticSetUrl}
+</CodeBlock>
+
+Now you can create a table, add data, and perform vector search operations:
+
+<CodeBlock filename="Python" language="Python" icon="python">
+  {PyFrameworksPydanticBaseExample}
+</CodeBlock>
+
+
+## Vector Field {#vector-field}
+
+LanceDB provides a `lancedb.pydantic.Vector` method to define a
+vector Field in a Pydantic Model.
+
+<CodeBlock filename="Python" language="Python" icon="python">
+  {PyFrameworksPydanticVectorField}
+</CodeBlock>
+
+This example demonstrates how LanceDB automatically converts Pydantic field types to their corresponding Apache Arrow data types. The `pydantic_to_schema()` function takes a Pydantic model and generates an Arrow schema where:
+- `int` fields become `pa.int64()` (64-bit integers)
+- `str` fields become `pa.utf8()` (UTF-8 encoded strings)  
+- `Vector(768)` becomes `pa.list_(pa.float32(), 768)` (fixed-size list of 768 float32 values)
+- The `False` parameter indicates that the fields are not nullable
+
+## Type Conversion {#type-conversion}
+
+LanceDB automatically convert Pydantic fields to
+[Apache Arrow DataType](https://arrow.apache.org/docs/python/generated/pyarrow.DataType.html#pyarrow.DataType).
+
+Current supported type conversions:
+
+| Pydantic Field Type | PyArrow Data Type |
+| ------------------- | ----------------- |
+| `int`               | `pyarrow.int64`   |
+| `float`              | `pyarrow.float64`  |
+| `bool`              | `pyarrow.bool`    |
+| `str`               | `pyarrow.utf8()`    |
+| `list`              | `pyarrow.List`    |
+| `BaseModel`         | `pyarrow.Struct`    |
+| `Vector(n)`         | `pyarrow.FixedSizeList(float32, n)` |
+
+LanceDB supports to create Apache Arrow Schema from a
+`pydantic.BaseModel`
+via `lancedb.pydantic.pydantic_to_schema` method.
+
+<CodeBlock filename="Python" language="Python" icon="python">
+  {PyFrameworksPydanticTypeConversion}
+</CodeBlock>
+
+This example shows a more complex Pydantic model with various field types and demonstrates how LanceDB handles:
+- Basic types: `int` and `str` fields
+- Vector fields: `Vector(1536)` creates a fixed-size list of 1536 float32 values
+- List fields: `List[int]` becomes a variable-length list of int64 values
+- Schema generation: The `pydantic_to_schema()` function automatically converts all these types to their Arrow equivalents

docs/web/integrations/data/voxel51.mdx

@@ -0,0 +1,212 @@
+---
+title: "Voxel51"
+sidebarTitle: "Voxel51"
+---
+
+import {
+  PyPlatformsVoxel51BackendFlag,
+  PyPlatformsVoxel51BackendParams,
+  PyPlatformsVoxel51BrainConfig,
+  PyPlatformsVoxel51Cleanup,
+  PyPlatformsVoxel51ComputeSimilarity,
+  PyPlatformsVoxel51LoadDataset,
+  PyPlatformsVoxel51SortBySimilarity,
+} from '/snippets/integrations.mdx';
+
+# FiftyOne
+
+[FiftyOne](https://docs.voxel51.com/) is an open source toolkit that enables users to curate better data and build better models. It includes tools for data exploration, visualization, and management, as well as features for collaboration and sharing. 
+
+Any developers, data scientists, and researchers who work with computer vision and machine learning can use FiftyOne to improve the quality of their datasets and deliver insights about their models.
+
+![example](/static/assets/images/integrations/voxel.gif)
+
+**FiftyOne** provides an API to create LanceDB tables and run similarity queries, both **programmatically in Python** and via **point-and-click in the App**.
+
+Let's get started and see how to use **LanceDB** to create a **similarity index** on your FiftyOne datasets.
+
+## Overview {#overview}
+
+[Embeddings](/embedding/) are foundational to all of the **vector search** features. In FiftyOne, embeddings are managed by the [**FiftyOne Brain**](https://docs.voxel51.com/user_guide/brain.html) that provides powerful machine learning techniques designed to transform how you curate your data from an art into a measurable science. 
+
+> _Have you ever wanted to find the images most similar to an image in your dataset?_
+
+The **FiftyOne Brain** makes computing **visual similarity** really easy. You can compute the similarity of samples in your dataset using an embedding model and store the results in the **brain key**. 
+You can then sort your samples by similarity or use this information to find potential duplicate images.
+
+
+We'll be doing the following : 
+
+1. **Create Index** - In order to run similarity queries against our media, we need to **index** the data. We can do this via the `compute_similarity()` function. 
+  -  In the function, specify the **model** you want to use to generate the embedding vectors, and what **vector search engine** you want to use on the **backend** (here LanceDB). 
+  <Tip>
+  You can also give the similarity index a name(`brain_key`), which is useful if you want to run vector searches against multiple indexes.
+  </Tip>
+
+2. **Query** - Once you have generated your similarity index, you can query your dataset with `sort_by_similarity()`. The query can be any of the following:
+
+  - An ID (sample or patch)
+  - A query vector of same dimension as the index
+  - A list of IDs (samples or patches)
+  - A text prompt (search semantically)
+
+## Prerequisites: install necessary dependencies {#prerequisites-install-necessary-dependencies}
+
+1. **Create and activate a virtual environment**
+
+Install virtualenv package and run the following command in your project directory.
+<CodeBlock filename="bash" language="bash" icon="terminal">
+python -m venv fiftyone_
+</CodeBlock>
+From inside the project directory run the following to activate the virtual environment.
+
+<CodeGroup>
+    <CodeBlock filename="macOS/Linux" language="bash" icon="linux">
+    source fiftyone_/Scripts/activate
+    </CodeBlock>
+    <CodeBlock filename="Windows" language="bash" icon="windows">
+    fiftyone_/Scripts/activate
+    </CodeBlock>
+</CodeGroup>
+
+2. **Install the following packages in the virtual environment**
+
+    To install FiftyOne, ensure you have activated any virtual environment that you are using, then run
+<CodeBlock filename="bash" language="bash" icon="terminal">
+pip install fiftyone
+</CodeBlock>
+
+
+## Understand basic workflow {#understand-basic-workflow}
+
+The basic workflow shown below uses LanceDB to create a similarity index on your FiftyOne datasets:
+
+1. Load a dataset into FiftyOne.
+
+2. Compute embedding vectors for samples or patches in your dataset, or select a model to use to generate embeddings.
+
+3. Use the `compute_similarity()` method to generate a LanceDB table for the samples or object patches embeddings in a dataset by setting the parameter `backend="lancedb"` and specifying a `brain_key` of your choice.
+
+4. Use this LanceDB table to query your data with `sort_by_similarity()`.
+
+5. If desired, delete the table.
+
+## Quick Example {#quick-example}
+
+Let's jump on a quick example that demonstrates this workflow.
+
+
+<CodeBlock filename="Python" language="Python" icon="python">
+  {PyPlatformsVoxel51LoadDataset}
+</CodeBlock>
+Make sure you install torch ([guide here](https://pytorch.org/get-started/locally/)) before proceeding. 
+
+<CodeBlock filename="Python" language="Python" icon="python">
+  {PyPlatformsVoxel51ComputeSimilarity}
+</CodeBlock>
+
+!!! note
+    Running the code above will download the clip model (2.6Gb)
+
+Once the similarity index has been generated, we can query our data in FiftyOne by specifying the `brain_key`:
+
+<CodeBlock filename="Python" language="Python" icon="python">
+  {PyPlatformsVoxel51SortBySimilarity}
+</CodeBlock>
+
+The returned result are of type - `DatasetView`.
+
+<Note>
+`DatasetView` does not hold its contents in-memory. Views simply store the rule(s) that are applied to extract the content of interest from the underlying Dataset when the view is iterated/aggregated on.
+
+This means, for example, that the contents of a `DatasetView` may change as the underlying Dataset is modified.
+</Note>
+
+> _Can you query a view instead of dataset?_
+
+Yes, you can also query a view.
+
+Performing a similarity search on a `DatasetView` will only return results from the view; if the view contains samples that were not included in the index, they will never be included in the result.
+
+This means that you can index an entire Dataset once and then perform searches on subsets of the dataset by constructing views that contain the images of interest.
+
+<CodeBlock filename="Python" language="Python" icon="python">
+  {PyPlatformsVoxel51Cleanup}
+</CodeBlock>
+
+
+## Using LanceDB backend {#using-lancedb-backend}
+By default, calling `compute_similarity()` or `sort_by_similarity()` will use an sklearn backend.
+
+To use the LanceDB backend, simply set the optional `backend` parameter of `compute_similarity()` to `"lancedb"`:
+
+<CodeBlock filename="Python" language="Python" icon="python">
+  {PyPlatformsVoxel51BackendFlag}
+</CodeBlock>
+
+Alternatively, you can configure FiftyOne to use the LanceDB backend by setting the following environment variable.
+
+In your terminal, set the environment variable using:
+
+<CodeGroup>
+    <CodeBlock filename="macOS/Linux" language="bash" icon="linux">
+    export FIFTYONE_BRAIN_DEFAULT_SIMILARITY_BACKEND=lancedb
+    </CodeBlock>
+
+    <CodeBlock filename="Windows" language="powershell" icon="windows">
+    $Env:FIFTYONE_BRAIN_DEFAULT_SIMILARITY_BACKEND="lancedb" //powershell
+
+    set FIFTYONE_BRAIN_DEFAULT_SIMILARITY_BACKEND=lancedb //cmd
+    </CodeBlock>
+</CodeGroup>
+
+<Warning>
+This will only run during the terminal session. Once terminal is closed, environment variable is deleted.
+</Warning>
+
+Alternatively, you can **permanently** configure FiftyOne to use the LanceDB backend creating a `brain_config.json` at `~/.fiftyone/brain_config.json`. The JSON file may contain any desired subset of config fields that you wish to customize.
+
+```json
+{
+  "default_similarity_backend": "lancedb"
+}
+```
+This will override the default `brain_config` and will set it according to your customization. You can check the configuration by running the following code : 
+
+<CodeBlock filename="Python" language="Python" icon="python">
+  {PyPlatformsVoxel51BrainConfig}
+</CodeBlock>
+
+## LanceDB config parameters {#lancedb-config-parameters}
+
+The LanceDB backend supports query parameters that can be used to customize your similarity queries. These parameters include:
+
+| Name           | Purpose                                                                                                          | Default          |
+| :------------- | :--------------------------------------------------------------------------------------------------------------- | :--------------- |
+| **table_name** | The name of the LanceDB table to use. If none is provided, a new table will be created                           | `None`           |
+| **metric**     | The embedding distance metric to use when creating a new table. The supported values are ("cosine", "euclidean") | `"cosine"`       |
+| **uri**        | The database URI to use. In this Database URI, tables will be created.                                           | `"/tmp/lancedb"` |
+
+There are two ways to specify/customize the parameters:
+
+1. **Using `brain_config.json` file** 
+
+```json
+{
+    "similarity_backends": {
+        "lancedb": {
+            "table_name": "your-table",
+            "metric": "euclidean",
+            "uri": "/tmp/lancedb"
+        }
+    }
+}
+```
+
+2. **Directly passing to `compute_similarity()` to configure a specific new index** : 
+
+<CodeBlock filename="Python" language="Python" icon="python">
+    {PyPlatformsVoxel51BackendParams}
+</CodeBlock>
+
+For a much more in depth walkthrough of the integration, visit the LanceDB x Voxel51 [docs page](https://docs.voxel51.com/integrations/lancedb.html).

docs/web/integrations/embedding/aws.mdx

@@ -0,0 +1,42 @@
+---
+title: AWS Bedrock
+sidebarTitle: AWS
+---
+
+import { PyEmbeddingAwsUsage } from '/snippets/integrations.mdx';
+
+AWS Bedrock supports multiple base models for generating text embeddings. You need to setup the AWS credentials to use this embedding function.
+You can do so by using `awscli` and also add your session_token:
+```shell
+aws configure
+aws configure set aws_session_token "<your_session_token>"
+```
+to ensure that the credentials are set up correctly, you can run the following command:
+```shell
+aws sts get-caller-identity
+```
+
+Supported Embedding modelIDs are:
+* `amazon.titan-embed-text-v1`
+* `cohere.embed-english-v3`
+* `cohere.embed-multilingual-v3`
+
+Supported parameters (to be passed in `create` method) are:
+
+| Parameter | Type | Default Value | Description |
+|---|---|---|---|
+| **name** | str | "amazon.titan-embed-text-v1" | The model ID of the bedrock model to use. Supported base models for Text Embeddings: amazon.titan-embed-text-v1, cohere.embed-english-v3, cohere.embed-multilingual-v3 |
+| **region** | str | "us-east-1" | Optional name of the AWS Region in which the service should be called (e.g., "us-east-1"). |
+| **profile_name** | str | None | Optional name of the AWS profile to use for calling the Bedrock service. If not specified, the default profile will be used. |
+| **assumed_role** | str | None | Optional ARN of an AWS IAM role to assume for calling the Bedrock service. If not specified, the current active credentials will be used. |
+| **role_session_name** | str | "lancedb-embeddings" | Optional name of the AWS IAM role session to use for calling the Bedrock service. If not specified, a "lancedb-embeddings" name will be used. |
+| **runtime** | bool | True | Optional choice of getting different client to perform operations with the Amazon Bedrock service. |
+| **max_retries** | int | 7 | Optional number of retries to perform when a request fails. |
+
+Usage Example:
+
+<CodeGroup>
+  <CodeBlock filename="Python" language="Python" icon="python">
+    {PyEmbeddingAwsUsage}
+  </CodeBlock>
+</CodeGroup>

docs/web/integrations/embedding/cohere.mdx

@@ -0,0 +1,51 @@
+---
+title: Cohere
+sidebarTitle: Cohere
+---
+
+import { PyEmbeddingCohereUsage } from '/snippets/integrations.mdx';
+
+Using cohere API requires cohere package, which can be installed using `pip install cohere`. Cohere embeddings are used to generate embeddings for text data. The embeddings can be used for various tasks like semantic search, clustering, and classification.
+You also need to set the `COHERE_API_KEY` environment variable to use the Cohere API.
+
+Supported models are:
+
+- embed-english-v3.0
+- embed-multilingual-v3.0
+- embed-english-light-v3.0
+- embed-multilingual-light-v3.0
+- embed-english-v2.0
+- embed-english-light-v2.0
+- embed-multilingual-v2.0
+
+
+Supported parameters (to be passed in `create` method) are:
+
+| Parameter | Type | Default Value | Description |
+|---|---|--------|---------|
+| `name` | `str` | `"embed-english-v2.0"` | The model ID of the cohere model to use. Supported base models for Text Embeddings: embed-english-v3.0, embed-multilingual-v3.0, embed-english-light-v3.0, embed-multilingual-light-v3.0, embed-english-v2.0, embed-english-light-v2.0, embed-multilingual-v2.0 |
+| `source_input_type` | `str` | `"search_document"` | The type of input data to be used for the source column. |
+| `query_input_type` | `str` | `"search_query"` | The type of input data to be used for the query. |
+
+Cohere supports following input types:
+
+| Input Type               | Description                          |
+|-------------------------|---------------------------------------|
+| "`search_document`"     | Used for embeddings stored in a vector|
+|                         | database for search use-cases.        |
+| "`search_query`"        | Used for embeddings of search queries |
+|                         | run against a vector DB               |
+| "`semantic_similarity`" | Specifies the given text will be used |
+|                         | for Semantic Textual Similarity (STS) |
+| "`classification`"      | Used for embeddings passed through a  |
+|                         | text classifier.                      |
+| "`clustering`"          | Used for the embeddings run through a |
+|                         | clustering algorithm                  |
+
+Usage Example:
+
+<CodeGroup>
+  <CodeBlock filename="Python" language="Python" icon="python">
+    {PyEmbeddingCohereUsage}
+  </CodeBlock>
+</CodeGroup>

docs/web/integrations/embedding/colpali.mdx

@@ -0,0 +1,53 @@
+---
+title: ColPali
+sidebarTitle: ColPali
+---
+
+import {
+  PyEmbeddingColpaliSetup,
+  PyEmbeddingColpaliTextSearch,
+} from '/snippets/integrations.mdx';
+
+We support [ColPali](https://github.com/illuin-tech/colpali) model embeddings for multimodal multi-vector retrieval. ColPali produces multiple embedding vectors per input (multi-vector), enabling more nuanced similarity matching between text queries and image documents.
+
+Using ColPali requires the colpali-engine package, which can be installed using `pip install colpali-engine`.
+
+<Info>
+ColPali produces **multi-vector** embeddings, meaning each input generates multiple embedding vectors rather than a single vector. Use `MultiVector(func.ndims())` instead of `Vector(func.ndims())` when defining your schema.
+</Info>
+
+Supported models are:
+
+- Metric-AI/ColQwen2.5-3b-multilingual-v1.0 (default)
+- vidore/colpali-v1.3
+- vidore/colqwen2-v1.0
+- vidore/colSmol-256M
+
+Supported parameters (to be passed in `create` method) are:
+
+| Parameter | Type | Default Value | Description |
+|---|---|---|---|
+| `model_name` | `str` | `"Metric-AI/ColQwen2.5-3b-multilingual-v1.0"` | The name of the model to use. |
+| `device` | `str` | `"auto"` | The device for inference. Can be `"auto"`, `"cpu"`, `"cuda"`, or `"mps"`. |
+| `dtype` | `str` | `"bfloat16"` | Data type for model weights (bfloat16, float16, float32, float64). |
+| `pooling_strategy` | `str` | `"hierarchical"` | Token pooling strategy: `"hierarchical"`, `"lambda"`, or `None`. |
+| `pool_factor` | `int` | `2` | Factor to reduce sequence length when pooling is enabled. |
+| `batch_size` | `int` | `2` | Batch size for processing inputs. |
+| `quantization_config` | `Optional[BitsAndBytesConfig]` | `None` | Quantization configuration for the model (requires bitsandbytes). |
+
+This embedding function supports ingesting images as both bytes and URLs. You can query them using text.
+
+<CodeGroup>
+  <CodeBlock filename="Python" language="Python" icon="python">
+    {PyEmbeddingColpaliSetup}
+  </CodeBlock>
+</CodeGroup>
+
+Now we can search using text queries:
+
+<CodeGroup>
+  <CodeBlock filename="Python" language="Python" icon="python">
+    {PyEmbeddingColpaliTextSearch}
+  </CodeBlock>
+</CodeGroup>
+

docs/web/integrations/embedding/gemini.mdx

@@ -0,0 +1,26 @@
+---
+title: Gemini
+sidebarTitle: Gemini
+---
+
+import { PyEmbeddingGeminiUsage } from '/snippets/integrations.mdx';
+
+With Google's Gemini, you can represent text (words, sentences, and blocks of text) in a vectorized form, making it easier to compare and contrast embeddings. For example, two texts that share a similar subject matter or sentiment should have similar embeddings, which can be identified through mathematical comparison techniques such as cosine similarity. For more on how and why you should use embeddings, refer to the Embeddings guide.
+The Gemini Embedding Model API supports various task types:
+
+| Task Type               | Description                                                                                                                                                |
+|-------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| "`retrieval_query`"     | Specifies the given text is a query in a search/retrieval setting.                                                                                         |
+| "`retrieval_document`"  | Specifies the given text is a document in a search/retrieval setting. Using this task type requires a title but is automatically provided by Embeddings API |
+| "`semantic_similarity`" | Specifies the given text will be used for Semantic Textual Similarity (STS).                                                                               |
+| "`classification`"      | Specifies that the embeddings will be used for classification.                                                                                             |
+| "`clustering`"           | Specifies that the embeddings will be used for clustering.                                                                                                 |
+
+
+Usage Example:
+
+<CodeGroup>
+  <CodeBlock filename="Python" language="Python" icon="python">
+    {PyEmbeddingGeminiUsage}
+  </CodeBlock>
+</CodeGroup>

docs/web/integrations/embedding/huggingface.mdx

@@ -0,0 +1,16 @@
+---
+title: Hugging Face
+sidebarTitle: Hugging Face
+---
+
+import { PyEmbeddingHuggingfaceUsage } from '/snippets/integrations.mdx';
+
+We offer support for all Hugging Face models (which can be loaded via [transformers](https://huggingface.co/docs/transformers/en/index) library). The default model is `colbert-ir/colbertv2.0` which also has its own special callout - `registry.get("colbert")`. Some Hugging Face models might require custom models defined on the HuggingFace Hub in their own modeling files. You may enable this by setting `trust_remote_code=True`. This option should only be set to True for repositories you trust and in which you have read the code, as it will execute code present on the Hub on your local machine. 
+
+Example usage:
+
+<CodeGroup>
+  <CodeBlock filename="Python" language="Python" icon="python">
+    {PyEmbeddingHuggingfaceUsage}
+  </CodeBlock>
+</CodeGroup>

docs/web/integrations/embedding/ibm.mdx

@@ -0,0 +1,64 @@
+---
+title: IBM watsonx
+sidebarTitle: IBM WatsonX
+---
+
+import { PyEmbeddingIbmUsage } from '/snippets/integrations.mdx';
+
+Generate text embeddings using IBM's watsonx.ai platform.
+
+## Supported Models {#supported-models}
+
+You can find a list of supported models at [IBM watsonx.ai Documentation](https://dataplatform.cloud.ibm.com/docs/content/wsj/analyze-data/fm-models-embed.html?context=wx). The currently supported model names are:
+
+| Model ID                                     | Dimensions |
+|----------------------------------------------|------------|
+| `ibm/granite-embedding-278m-multilingual`    | 768        |
+| `ibm/slate-125m-english-rtrvr-v2`            | 768        |
+| `ibm/slate-30m-english-rtrvr-v2`             | 384        |
+| `intfloat/multilingual-e5-large`             | 1024       |
+| `sentence-transformers/all-minilm-l6-v2`     | 384        |
+
+<Info>
+For new tables, `ibm/granite-embedding-278m-multilingual` is the recommended default. Older model IDs (such as `ibm/slate-125m-english-rtrvr` and `sentence-transformers/all-minilm-l12-v2`) remain resolvable for tables whose stored metadata references them, but they are no longer advertised for new use.
+</Info>
+
+## Parameters {#parameters}
+
+The following parameters can be passed to the `create` method:
+
+| Parameter  | Type     | Default Value                    | Description                                               |
+|------------|----------|----------------------------------|-----------------------------------------------------------|
+| name       | str      | `"ibm/slate-125m-english-rtrvr"` | The model ID of the watsonx.ai model to use. Pass one of the current supported IDs above (e.g. `"ibm/granite-embedding-278m-multilingual"`) when creating new tables. |
+| api_key    | str      | None                             | Optional IBM Cloud API key (or set `WATSONX_API_KEY`)     |
+| project_id | str      | None                             | Optional watsonx project ID (or set `WATSONX_PROJECT_ID`). Mutually exclusive with `space_id`. |
+| space_id   | str      | None                             | Optional watsonx deployment space ID (or set `WATSONX_SPACE_ID`). Mutually exclusive with `project_id`. |
+| url        | str      | None                             | Optional custom URL for the watsonx.ai instance           |
+| params     | dict     | None                             | Optional additional parameters for the embedding model (e.g. `{"truncate_input_tokens": 512}`) |
+
+<Info>
+You must supply exactly one of `project_id` or `space_id` (either as an argument or via its environment variable). Setting both, or neither, raises a `ValueError`.
+</Info>
+
+## Usage Example {#usage-example}
+
+First, the watsonx.ai library is an optional dependency, so must be installed separately:
+
+```
+pip install ibm-watsonx-ai
+```
+
+Optionally set environment variables (if not passing credentials to `create` directly):
+
+```sh
+export WATSONX_API_KEY="YOUR_WATSONX_API_KEY"
+# Provide exactly one of the following:
+export WATSONX_PROJECT_ID="YOUR_WATSONX_PROJECT_ID"
+export WATSONX_SPACE_ID="YOUR_WATSONX_SPACE_ID"
+```
+
+<CodeGroup>
+  <CodeBlock filename="Python" language="Python" icon="python">
+    {PyEmbeddingIbmUsage}
+  </CodeBlock>
+</CodeGroup>

docs/web/integrations/embedding/imagebind.mdx

@@ -0,0 +1,54 @@
+---
+title: ImageBind
+sidebarTitle: ImageBind
+---
+
+import {
+  PyEmbeddingImagebindSetup,
+  PyEmbeddingImagebindImageSearch,
+  PyEmbeddingImagebindAudioSearch,
+  PyEmbeddingImagebindTextSearch,
+} from '/snippets/integrations.mdx';
+
+We have support for [imagebind](https://github.com/facebookresearch/ImageBind) model embeddings. You can download our version of the packaged model via - `pip install imagebind-packaged==0.1.2`.
+
+This function is registered as `imagebind` and supports Audio, Video and Text modalities(extending to Thermal,Depth,IMU data):
+
+| Parameter | Type | Default Value | Description |
+|---|---|---|---|
+| `name` | `str` | `"imagebind_huge"` | Name of the model. |
+| `device` | `str` | `"cpu"` | The device to run the model on. Can be `"cpu"` or `"gpu"`. |
+| `normalize` | `bool` | `False` | set to `True` to normalize your inputs before model ingestion. |
+
+Below is an example demonstrating how the API works:
+
+<CodeGroup>
+  <CodeBlock filename="Python" language="Python" icon="python">
+    {PyEmbeddingImagebindSetup}
+  </CodeBlock>
+</CodeGroup>
+
+Now, we can search using any modality:
+
+#### image search {#image-search}
+<CodeGroup>
+  <CodeBlock filename="Python" language="Python" icon="python">
+    {PyEmbeddingImagebindImageSearch}
+  </CodeBlock>
+</CodeGroup>
+#### audio search {#audio-search}
+
+<CodeGroup>
+  <CodeBlock filename="Python" language="Python" icon="python">
+    {PyEmbeddingImagebindAudioSearch}
+  </CodeBlock>
+</CodeGroup>
+#### Text search {#text-search}
+You can add any input query and fetch the result as follows:
+<CodeGroup>
+  <CodeBlock filename="Python" language="Python" icon="python">
+    {PyEmbeddingImagebindTextSearch}
+  </CodeBlock>
+</CodeGroup>
+
+If you have any questions about the embeddings API, supported models, or see a relevant model missing, please raise an issue [on GitHub](https://github.com/lancedb/lancedb/issues).

docs/web/integrations/embedding/instructor.mdx

@@ -0,0 +1,39 @@
+---
+title: Instructor
+sidebarTitle: Instructor
+---
+
+import { PyEmbeddingInstructorUsage } from '/snippets/integrations.mdx';
+
+[Instructor](https://instructor-embedding.github.io/) is an instruction-finetuned text embedding model that can generate text embeddings tailored to any task (e.g. classification, retrieval, clustering, text evaluation, etc.) and domains (e.g. science, finance, etc.) by simply providing the task instruction, without any finetuning.
+
+If you want to calculate customized embeddings for specific sentences, you can follow the unified template to write instructions.
+
+<Info>
+Represent the `domain` `text_type` for `task_objective`:
+
+- `domain` is optional, and it specifies the domain of the text, e.g. science, finance, medicine, etc.
+- `text_type` is required, and it specifies the encoding unit, e.g. sentence, document, paragraph, etc.
+- `task_objective` is optional, and it specifies the objective of embedding, e.g. retrieve a document, classify the sentence, etc.
+</Info>
+
+More information about the model can be found at the [source URL](https://github.com/xlang-ai/instructor-embedding).
+
+| Argument | Type | Default | Description |
+|---|---|---|---|
+| `name` | `str` | "hkunlp/instructor-base" | The name of the model to use |
+| `batch_size` | `int` | `32` | The batch size to use when generating embeddings |
+| `device` | `str` | `"cpu"` | The device to use when generating embeddings |
+| `show_progress_bar` | `bool` | `True` | Whether to show a progress bar when generating embeddings |
+| `normalize_embeddings` | `bool` | `True` | Whether to normalize the embeddings |
+| `quantize` | `bool` | `False` | Whether to quantize the model |
+| `source_instruction` | `str` | `"represent the document for retrieval"` | The instruction for the source column |
+| `query_instruction` | `str` | `"represent the document for retrieving the most similar documents"` | The instruction for the query |
+
+
+
+<CodeGroup>
+  <CodeBlock filename="Python" language="Python" icon="python">
+    {PyEmbeddingInstructorUsage}
+  </CodeBlock>
+</CodeGroup>

docs/web/integrations/embedding/jina.mdx

@@ -0,0 +1,49 @@
+---
+title: Jina
+sidebarTitle: Jina
+---
+
+import {
+  PyEmbeddingJinaText,
+  PyEmbeddingJinaMultimodal,
+} from '/snippets/integrations.mdx';
+
+## Text Embedding Models {#text-embedding-models}
+
+Jina embeddings are used to generate embeddings for text and image data.
+You also need to set the `JINA_API_KEY` environment variable to use the Jina API.
+
+You can find a list of supported models under [https://jina.ai/embeddings/](https://jina.ai/embeddings/)
+
+Supported parameters (to be passed in `create` method) are:
+
+| Parameter | Type | Default Value | Description |
+|---|---|---|---|
+| `name` | `str` | `"jina-clip-v1"` | The model ID of the jina model to use |
+
+Usage Example:
+
+<CodeGroup>
+  <CodeBlock filename="Python" language="Python" icon="python">
+    {PyEmbeddingJinaText}
+  </CodeBlock>
+</CodeGroup>
+
+## Multimodal Embedding Models {#multimodal-embedding-models}
+
+Jina embeddings can also be used to embed both text and image data, only some of the models support image data and you can check the list
+under [https://jina.ai/embeddings/](https://jina.ai/embeddings/)
+
+Supported parameters (to be passed in `create` method) are:
+
+| Parameter | Type | Default Value | Description |
+|---|---|---|---|
+| `name` | `str` | `"jina-clip-v1"` | The model ID of the jina model to use |
+
+Usage Example:
+
+<CodeGroup>
+  <CodeBlock filename="Python" language="Python" icon="python">
+    {PyEmbeddingJinaMultimodal}
+  </CodeBlock>
+</CodeGroup>

docs/web/integrations/embedding/ollama.mdx

@@ -0,0 +1,25 @@
+---
+title: Ollama
+sidebarTitle: Ollama
+---
+
+import { PyEmbeddingOllamaUsage } from '/snippets/integrations.mdx';
+
+Generate embeddings via the [ollama](https://github.com/ollama/ollama-python) python library. More details:
+
+- [Ollama docs on embeddings](https://github.com/ollama/ollama/blob/main/docs/api.md#generate-embeddings)
+- [Ollama blog on embeddings](https://ollama.com/blog/embedding-models)
+
+| Parameter              | Type                       | Default Value            | Description                                                                                                                                    |
+|------------------------|----------------------------|--------------------------|------------------------------------------------------------------------------------------------------------------------------------------------|
+| `name`                 | `str`                      | `nomic-embed-text`       | The name of the model.                                                                                                                         |
+| `host`                 | `str`                      | `http://localhost:11434` | The Ollama host to connect to.                                                                                                                 |
+| `options`              | `ollama.Options` or `dict` | `None`                   | Additional model parameters listed in the documentation for the Modelfile such as `temperature`. |
+| `keep_alive`           | `float` or `str`           | `"5m"`                   | Controls how long the model will stay loaded into memory following the request.                                                                |
+| `ollama_client_kwargs` | `dict`                     | `{}`                     | kwargs that can be past to the `ollama.Client`.                                                                                                |
+
+<CodeGroup>
+  <CodeBlock filename="Python" language="Python" icon="python">
+    {PyEmbeddingOllamaUsage}
+  </CodeBlock>
+</CodeGroup>

docs/web/integrations/embedding/openai.mdx

@@ -0,0 +1,20 @@
+---
+title: OpenAI
+sidebarTitle: OpenAI
+---
+
+import { PyEmbeddingOpenaiBasic } from '/snippets/integrations.mdx';
+
+LanceDB registers the OpenAI embeddings function in the registry by default, as `openai`. Below are the parameters that you can customize when creating the instances:
+
+| Parameter | Type | Default Value | Description |
+|---|---|---|---|
+| `name` | `str` | `"text-embedding-ada-002"` | The name of the model. |
+| `dim` | `int` |  Model default   | For OpenAI's newer text-embedding-3 model, we can specify a dimensionality that is smaller than the 1536 size. This feature supports it |
+| `use_azure` | bool | `False` | Set true to use Azure OpenAI SDK |
+
+<CodeGroup>
+  <CodeBlock filename="Python" language="Python" icon="python">
+    {PyEmbeddingOpenaiBasic}
+  </CodeBlock>
+</CodeGroup>