diff --git a/docs/web/docs.json b/docs/web/docs.json index e15887823..984ff58e7 100644 --- a/docs/web/docs.json +++ b/docs/web/docs.json @@ -142,6 +142,10 @@ { "source": "/build-with-ai-agents", "destination": "/basics/agentic-skills" + }, + { + "source": "/core/update", + "destination": "/ingestion/update" } ], "navigation": { @@ -477,7 +481,17 @@ "linkedin": "https://www.linkedin.com/company/lancedb", "x": "https://x.com/lancedb", "discord": "https://discord.gg/AUEWnJ7Txb" - } + }, + "links": [ + { + "items": [ + { + "label": "Cookie settings", + "href": "#cookie-settings" + } + ] + } + ] }, "integrations": { "ga4": { diff --git a/docs/web/enterprise/deployment/azure.mdx b/docs/web/enterprise/deployment/azure.mdx index a045e6204..349393249 100644 --- a/docs/web/enterprise/deployment/azure.mdx +++ b/docs/web/enterprise/deployment/azure.mdx @@ -14,4 +14,4 @@ of LanceDB has no server to provide it -- see The Azure deployment procedure is published with LanceDB Enterprise. See [Deployment](/enterprise/deployment) for the general shape, and -[contact us](mailto:contact@lancedb.com) to get started. +[contact us](https://www.lancedb.com/contact) to get started. diff --git a/docs/web/enterprise/deployment/index.mdx b/docs/web/enterprise/deployment/index.mdx index 3002cc844..5fe98eba5 100644 --- a/docs/web/enterprise/deployment/index.mdx +++ b/docs/web/enterprise/deployment/index.mdx @@ -13,7 +13,7 @@ of LanceDB has no server to provide it -- see [Architecture](/architecture) for what differs between the two. Deployment is specific to your cloud and account, so the procedure is published -with LanceDB Enterprise. [Contact us](mailto:contact@lancedb.com) to get +with LanceDB Enterprise. [Contact us](https://www.lancedb.com/contact) to get started. Embedded LanceDB has nothing to deploy -- it is a library your process links diff --git a/docs/web/index.mdx b/docs/web/index.mdx index 86bbf9e5c..b61d542ef 100644 --- a/docs/web/index.mdx +++ b/docs/web/index.mdx @@ -106,7 +106,7 @@ for agents. Start here: search, curation, feature engineering, and high-throughput training data access workflows on top of the same core table abstraction. This eliminates the need for teams to build bespoke infrastructure to manage large multimodal datasets. To set up LanceDB Enterprise in your organization, reach out to us at -[contact@lancedb.com](mailto:contact@lancedb.com). +[contact@lancedb.com](https://www.lancedb.com/contact). **Built with scale, performance, and security in mind.** diff --git a/docs/web/indexing/reindexing.mdx b/docs/web/indexing/reindexing.mdx index e427a328e..5ebf94e5c 100644 --- a/docs/web/indexing/reindexing.mdx +++ b/docs/web/indexing/reindexing.mdx @@ -73,7 +73,7 @@ is capable of indexing billions of rows in under four hours on a 1-8 GPU cluster Enterprise-only Automatic GPU Indexing is currently only available in [LanceDB Enterprise](/enterprise/). -Please [contact us](mailto:contact@lancedb.com) to enable this feature for your deployment. +Please [contact us](https://www.lancedb.com/contact) to enable this feature for your deployment. The vector index is created when you call `create_index`. The backend will use GPU resources to build either the IVF or HNSW indexes. The system automatically selects the optimal GPU diff --git a/docs/web/integrations/lerobotdataset.mdx b/docs/web/integrations/lerobotdataset.mdx index 70db08d3f..e1a88a30e 100644 --- a/docs/web/integrations/lerobotdataset.mdx +++ b/docs/web/integrations/lerobotdataset.mdx @@ -18,16 +18,18 @@ What you get: - **Train from object storage while keeping your GPUs fed.** Lance reads random frames straight from S3, GCS, the HF Hub or HF Storage Buckets, fetching only the bytes each batch needs, so every batch is a global shuffle: any frame, from anywhere in the dataset. In the table below (samples/s, 8 workers, batch 64), Lance and upstream local both draw every batch from the whole dataset. Upstream streaming does a reservoir shuffle, reading the dataset in order and drawing each batch from a pool of recently seen frames, so those two columns are not the same job: -| dataset | frames | cameras | Lance, S3 (global shuffle) | upstream, local (global shuffle) | same job | upstream, streaming (reservoir shuffle) | vs streaming | -|---|---|---|---|---|---|---|---| -| DROID | 27,630,375 | 3 | 2,258.5 | 346.7 | 6.5x | 898.6 | 1.7x | -| berkeley_rpt | 392,578 | 1 | 1,740.6 | 1,209.9 | 1.4x | 274.5 | 6.3x | -| toto | 325,699 | 1 | 1,097.0 | 936.3 | 1.2x | 121.3 | 9.0x | -| roboturk | 187,507 | 1 | 1,276.8 | 1,082.3 | 1.2x | 128.2 | 10.0x | -| aloha_mobile_cabinet | 127,500 | 3 | 815.7 | 294.2 | 2.8x | 77.7 | 10.5x | -| koch_pick_place | 37,972 | 2 | 854.7 | 274.9 | 3.1x | 58.1 | 14.7x | +| dataset | window shuffle, Lance from object store | window shuffle, upstream streaming from the Hub | Lance (object store) vs upstream streaming | RAM, Lance | RAM, streaming | global shuffle, Lance from object store | global shuffle, upstream from local NVMe | Lance (object store) vs upstream (local NVMe) | +|---|---|---|---|---|---|---|---|---| +| DROID | 904 | 478 | 1.9x | 8 GB | 122 GB | 257 | 224 | 1.1x | +| roboturk | 778 | 152 | 5.1x | 6 GB | 9 GB | 1,074 | 983 | 1.1x | +| aloha_mobile_cabinet | 772 | 51 | 15.2x | 9 GB | 30 GB | 693 | 573 | 1.2x | +| koch_pick_place_5_lego | 701 | 54 | 13.0x | 8 GB | 15 GB | 876 | 526 | 1.7x | +| pusht | 5,340 | 648 | 8.2x | 3 GB | 7 GB | 5,508 | 4,224 | 1.3x | +| aloha_static_cups_open | 597 | 45 | 13.2x | 10 GB | 38 GB | 575 | 483 | 1.2x | -The gap grows with how much video each sample carries: about 1.2x with one camera, 2.8x with three, 6.5x on DROID. +The shuffle window is 0.05% of the dataset, at least 5,000 frames: 5,000 on the small sets, 13,815 on DROID. Streaming DROID at that window in upstream default needs 122 GB, so those two runs used a 256 GB machine with the same decode speed. The DROID pair is noisy: Lance 684 and 1,124, streaming 303 and 653. + +Same batches: Lance from an object store is 1.9x faster than upstream streaming on DROID and 5x to 15x on the other datasets, with 8 GB of RAM against 122 GB on DROID. Global shuffle: Lance with nothing downloaded is 1.1x to 1.7x faster than upstream reading local NVMe after the download. berkeley_rpt, toto and soarm101 are left out because their metadata lists frames the videos do not have; `lerobot-lance-doctor` reports that before you train. - **Faster iteration, end to end.** 10,000 steps of SmolVLA on DROID, 8xH100, lerobot's default `num_workers=4`, same seed: diff --git a/docs/web/quickstart.mdx b/docs/web/quickstart.mdx index aff71bc71..43ba8c18c 100644 --- a/docs/web/quickstart.mdx +++ b/docs/web/quickstart.mdx @@ -205,7 +205,7 @@ requests to your deployment. `host_override` is the full URL of your cluster endpoint, including the scheme (`https://`) and a port if your deployment listens on a non-default one (e.g. `https://your-enterprise-endpoint.com:443`). If you don't have the -endpoint, [contact the LanceDB team](mailto:contact@lancedb.com). +endpoint, [contact the LanceDB team](https://www.lancedb.com/contact). To learn more about `RemoteTable` semantics and how Enterprise differs operationally from diff --git a/docs/web/static/scripts/cookie-consent.js b/docs/web/static/scripts/cookie-consent.js new file mode 100644 index 000000000..75c5c6639 --- /dev/null +++ b/docs/web/static/scripts/cookie-consent.js @@ -0,0 +1,213 @@ +/** + * Cookie consent banner + consent-gated RB2B loader for docs.lancedb.com. + * + * Mintlify loads every .js file in the content directory on every page, so this + * file runs site-wide. RB2B is opt-in: its script is never requested until the + * visitor clicks "Accept". The choice is stored in localStorage and can be + * changed later from the "Cookie settings" footer link (any link to + * #cookie-settings) or window.lancedbCookieConsent.open(). + * + * GA4 and Mintlify's own analytics are NOT gated by this banner. + */ +(function () { + "use strict"; + + if (typeof window === "undefined" || window.lancedbCookieConsent) return; + + // RB2B script ID: app.rb2b.com > Script, the value inside reb2b.load("..."). + var RB2B_KEY = "9NMMZHRRWYNW"; + + var STORAGE_KEY = "lancedb-cookie-consent"; // "accepted" | "rejected" + var POLICY_URL = + "https://242023405.fs1.hubspotusercontent-na2.net/hubfs/242023405/Website%20Documentation/privacy.pdf"; + var BANNER_ID = "lancedb-cookie-banner"; + var STYLE_ID = "lancedb-cookie-banner-style"; + var SCRIPT_ID = "rb2b-script"; + var SETTINGS_HASH = "#cookie-settings"; + + function readConsent() { + try { + return window.localStorage.getItem(STORAGE_KEY); + } catch (e) { + return null; + } + } + + function writeConsent(value) { + try { + window.localStorage.setItem(STORAGE_KEY, value); + } catch (e) { + /* storage blocked: banner will simply reappear next visit */ + } + } + + function hasKey() { + return RB2B_KEY && RB2B_KEY.indexOf("REPLACE_") !== 0; + } + + // ---- RB2B ----------------------------------------------------------------- + // Mirrors RB2B's official SPA guidance: remove and re-append the script on + // every route change so each client-side navigation is recorded. + function loadRb2b() { + if (!hasKey() || readConsent() !== "accepted") return; + var existing = document.getElementById(SCRIPT_ID); + if (existing) existing.remove(); + var s = document.createElement("script"); + s.id = SCRIPT_ID; + s.async = true; + s.src = + "https://ddwl4m2hdecbv.cloudfront.net/b/" + RB2B_KEY + "/" + RB2B_KEY + ".js.gz"; + document.body.appendChild(s); + } + + var lastPath = null; + function onRouteChange() { + var path = window.location.pathname + window.location.search; + if (path === lastPath) return; + lastPath = path; + loadRb2b(); + } + + function watchRoutes() { + ["pushState", "replaceState"].forEach(function (method) { + var original = window.history[method]; + window.history[method] = function () { + var result = original.apply(this, arguments); + setTimeout(onRouteChange, 0); + return result; + }; + }); + window.addEventListener("popstate", onRouteChange); + } + + // ---- Banner --------------------------------------------------------------- + var CSS = [ + "#" + BANNER_ID + "{", + " --lcb-bg:#ffffff;--lcb-fg:#1f2328;--lcb-muted:#57606a;--lcb-border:rgba(0,0,0,.12);", + " --lcb-accent:#FF6B35;--lcb-accent-fg:#1b0f09;--lcb-link:#B8431A;--lcb-btn2:#f3f4f6;--lcb-btn2-fg:#1f2328;", + " position:fixed;z-index:2147483000;left:16px;bottom:16px;max-width:400px;", + " box-sizing:border-box;padding:16px 18px;border-radius:12px;", + " background:var(--lcb-bg);color:var(--lcb-fg);border:1px solid var(--lcb-border);", + " box-shadow:0 8px 30px rgba(0,0,0,.12);", + " font:14px/1.5 Inter,ui-sans-serif,system-ui,-apple-system,'Segoe UI',sans-serif;", + "}", + "html.dark #" + BANNER_ID + "{", + " --lcb-bg:#161616;--lcb-fg:#e6e6e6;--lcb-muted:#a0a0a0;--lcb-border:rgba(255,255,255,.12);", + " --lcb-btn2:#262626;--lcb-btn2-fg:#e6e6e6;--lcb-link:#FF8A5C;box-shadow:0 8px 30px rgba(0,0,0,.5);", + "}", + "#" + BANNER_ID + " p{margin:0 0 12px;color:var(--lcb-muted);}", + "#" + BANNER_ID + " strong{display:block;margin-bottom:4px;color:var(--lcb-fg);font-weight:600;}", + "#" + BANNER_ID + " a{color:var(--lcb-link);text-decoration:underline;}", + "#" + BANNER_ID + " .lcb-actions{display:flex;gap:8px;}", + "#" + BANNER_ID + " button{flex:1;cursor:pointer;border:0;border-radius:8px;padding:8px 12px;", + " font:inherit;font-weight:600;}", + "#" + BANNER_ID + " .lcb-reject{background:var(--lcb-btn2);color:var(--lcb-btn2-fg);}", + "#" + BANNER_ID + " .lcb-accept{background:var(--lcb-accent);color:var(--lcb-accent-fg);}", + "#" + BANNER_ID + " button:focus-visible{outline:2px solid var(--lcb-accent);outline-offset:2px;}", + "@media (max-width:480px){#" + BANNER_ID + "{left:16px;right:16px;max-width:none;}}", + "@media print{#" + BANNER_ID + "{display:none;}}", + ].join("\n"); + + function injectStyles() { + if (document.getElementById(STYLE_ID)) return; + var style = document.createElement("style"); + style.id = STYLE_ID; + style.textContent = CSS; + document.head.appendChild(style); + } + + function closeBanner() { + var el = document.getElementById(BANNER_ID); + if (el) el.remove(); + } + + function choose(value) { + var previous = readConsent(); + writeConsent(value); + closeBanner(); + if (value === "accepted") { + lastPath = window.location.pathname + window.location.search; + loadRb2b(); + } else if (previous === "accepted") { + // RB2B may already be running in this tab; reload so it stops. + window.location.reload(); + } + } + + function openBanner() { + if (document.getElementById(BANNER_ID)) return; + injectStyles(); + + var el = document.createElement("div"); + el.id = BANNER_ID; + el.setAttribute("role", "dialog"); + el.setAttribute("aria-live", "polite"); + el.setAttribute("aria-labelledby", BANNER_ID + "-title"); + el.innerHTML = + '

' + + "With your permission, we use a third-party service (RB2B) that sets cookies " + + "to help us understand which companies and people visit our docs. " + + "Nothing loads unless you accept. " + + 'Privacy policy

' + + '
' + + '' + + '' + + "
"; + + el.querySelector(".lcb-reject").addEventListener("click", function () { + choose("rejected"); + }); + el.querySelector(".lcb-accept").addEventListener("click", function () { + choose("accepted"); + }); + + document.body.appendChild(el); + } + + // Any link to #cookie-settings (e.g. the footer link) reopens the banner. + function watchSettingsLinks() { + document.addEventListener( + "click", + function (event) { + var link = event.target && event.target.closest && event.target.closest("a[href]"); + if (!link) return; + var href = link.getAttribute("href") || ""; + if (href.slice(-SETTINGS_HASH.length) !== SETTINGS_HASH) return; + event.preventDefault(); + event.stopPropagation(); + openBanner(); + }, + true + ); + } + + // ---- Init ----------------------------------------------------------------- + window.lancedbCookieConsent = { + open: openBanner, + get: readConsent, + reset: function () { + try { + window.localStorage.removeItem(STORAGE_KEY); + } catch (e) {} + openBanner(); + }, + }; + + function init() { + watchRoutes(); + watchSettingsLinks(); + var consent = readConsent(); + if (consent === "accepted") { + onRouteChange(); + } else if (consent !== "rejected") { + openBanner(); + } + if (window.location.hash === SETTINGS_HASH) openBanner(); + } + + if (document.readyState === "loading") { + document.addEventListener("DOMContentLoaded", init); + } else { + init(); + } +})();