diff --git a/docs/web-tests/ts/vscode_extension.ts b/docs/web-tests/ts/vscode_extension.ts new file mode 100644 index 000000000..abd5c98ad --- /dev/null +++ b/docs/web-tests/ts/vscode_extension.ts @@ -0,0 +1,117 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The LanceDB Authors + +// The `src/extension.ts` of the sample extension in the VS Code integration guide. +// It needs the VS Code extension host, so jest doesn't run it: the LanceDB code it +// calls is tested in vscode_search.test.ts, and the extension itself was tested +// with @vscode/test-cli. + +// --8<-- [start:vscode_extension] +import * as vscode from "vscode"; +import { + type Chunk, + chunkMarkdown, + indexChunks, + openDatabase, + searchChunks, +} from "./search"; + +export async function activate(context: vscode.ExtensionContext) { + // One index per workspace. `storageUri` is undefined when no folder is open. + const storage = context.storageUri ?? context.globalStorageUri; + + // Transformers.js downloads the embedding model on first use. Keep it in + // global storage so every workspace shares one copy and it survives + // extension updates. Use `import()`: LanceDB loads the ES module build, and + // `require()` would return a separate copy whose settings LanceDB never sees. + const { env } = await import("@huggingface/transformers"); + env.cacheDir = vscode.Uri.joinPath(context.globalStorageUri, "models").fsPath; + + context.subscriptions.push( + vscode.commands.registerCommand("lancedbSearch.index", async () => { + const files = await vscode.workspace.findFiles( + "**/*.md", + "**/node_modules/**", + ); + const chunks: Chunk[] = []; + for (const file of files) { + const bytes = await vscode.workspace.fs.readFile(file); + // Store the file's URI, not a path relative to its folder: in a + // multi-root workspace `findFiles` searches every folder, and only the + // URI says which one a result came from. + chunks.push( + ...chunkMarkdown(file.toString(), new TextDecoder().decode(bytes)), + ); + } + if (chunks.length === 0) { + vscode.window.showWarningMessage("No Markdown files to index."); + return 0; + } + + await vscode.window.withProgress( + { + location: vscode.ProgressLocation.Notification, + title: `Indexing ${files.length} Markdown files`, + }, + async () => { + await vscode.workspace.fs.createDirectory(storage); + await indexChunks(await openDatabase(storage.fsPath), chunks); + }, + ); + vscode.window.showInformationMessage( + `Indexed ${chunks.length} paragraphs from ${files.length} files.`, + ); + return chunks.length; + }), + + // Called with a query (for example from a test or another extension), the + // command returns the matching rows instead of showing a picker. + vscode.commands.registerCommand( + "lancedbSearch.search", + async (query?: string) => { + const text = + query ?? + (await vscode.window.showInputBox({ + prompt: "Search Markdown by meaning", + })); + if (!text) { + return []; + } + const results = await searchChunks( + await openDatabase(storage.fsPath), + text, + ); + if (query !== undefined) { + return results; + } + + const picked = await vscode.window.showQuickPick( + results.map((row) => { + const uri = vscode.Uri.parse(row.path); + return { + label: `${vscode.workspace.asRelativePath(uri)}:${row.line + 1}`, + description: `distance ${row._distance.toFixed(3)}`, + detail: row.text, + uri, + line: row.line, + }; + }), + { placeHolder: text, matchOnDetail: true }, + ); + if (picked) { + const editor = await vscode.window.showTextDocument(picked.uri); + const start = new vscode.Position(picked.line, 0); + editor.selection = new vscode.Selection(start, start); + editor.revealRange( + new vscode.Range(start, start), + vscode.TextEditorRevealType.InCenter, + ); + } + return results; + }, + ), + ); +} + +export function deactivate() {} +// --8<-- [end:vscode_extension] diff --git a/docs/web-tests/ts/vscode_search.test.ts b/docs/web-tests/ts/vscode_search.test.ts new file mode 100644 index 000000000..21dd4d816 --- /dev/null +++ b/docs/web-tests/ts/vscode_search.test.ts @@ -0,0 +1,67 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The LanceDB Authors +import { expect, test } from "@jest/globals"; +import { withTempDirectory } from "./util.ts"; +import { + chunkMarkdown, + indexChunks, + openDatabase, + searchChunks, +} from "./vscode_search.ts"; + +// The same files make up the sample workspace used in the docs page. +const workspace: Record = { + "README.md": + "# Acme CLI\n\nAcme CLI syncs your local notes to the Acme cloud.\n\nInstall it with `npm install -g acme-cli`, then run `acme login`.\n", + "docs/auth.md": + "# Authentication\n\nAcme uses personal access tokens. Create one under Settings > Tokens.\n\nIf a token leaks, revoke it on the Tokens page and generate a replacement.\nRevoked tokens stop working immediately.\n", + "docs/sync.md": + "# Syncing\n\nRun `acme sync` to upload changed notes. Only files modified since the last sync are sent.\n\nConflicts are resolved by keeping both copies and adding `.conflict` to the older one.\n", + "docs/config.md": + "# Configuration\n\nSettings live in `~/.acme/config.toml`.\n\nSet `sync.interval` to control how often background sync runs, in minutes.\n", + "docs/troubleshooting.md": + "# Troubleshooting\n\nIf `acme sync` hangs, check that your proxy allows connections to api.acme.dev on port 443.\n\nRun `acme doctor` to print diagnostics you can attach to a bug report.\n", + "CONTRIBUTING.md": + "# Contributing\n\nRun `npm test` before opening a pull request. New commands need a page in docs/.\n", +}; + +test("chunkMarkdown splits paragraphs and records their start line", () => { + expect(chunkMarkdown("docs/auth.md", workspace["docs/auth.md"])).toEqual([ + { path: "docs/auth.md", line: 0, text: "# Authentication" }, + { + path: "docs/auth.md", + line: 2, + text: "Acme uses personal access tokens. Create one under Settings > Tokens.", + }, + { + path: "docs/auth.md", + line: 4, + text: "If a token leaks, revoke it on the Tokens page and generate a replacement.\nRevoked tokens stop working immediately.", + }, + ]); +}); + +test("index workspace Markdown and search it by meaning", async () => { + await withTempDirectory(async (storageDir) => { + const chunks = Object.entries(workspace).flatMap(([filePath, content]) => + chunkMarkdown(filePath, content), + ); + await indexChunks(await openDatabase(storageDir), chunks); + + // Each command in the extension opens its own connection, so search + // through a fresh one to check the table is usable after a reload. + const db = await openDatabase(storageDir); + const leaked = await searchChunks( + db, + "my credentials were exposed, what should I do?", + ); + expect(leaked[0].path).toBe("docs/auth.md"); + expect(leaked[0].line).toBe(4); + + const firewall = await searchChunks( + db, + "uploads are stuck behind the company firewall", + ); + expect(firewall[0].path).toBe("docs/troubleshooting.md"); + }); +}, 100_000); diff --git a/docs/web-tests/ts/vscode_search.ts b/docs/web-tests/ts/vscode_search.ts new file mode 100644 index 000000000..d090084a1 --- /dev/null +++ b/docs/web-tests/ts/vscode_search.ts @@ -0,0 +1,77 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The LanceDB Authors + +// --8<-- [start:vscode_imports] +import * as path from "node:path"; +import * as lancedb from "@lancedb/lancedb"; +import "@lancedb/lancedb/embedding/transformers"; +import { Int32, Utf8 } from "apache-arrow"; +// --8<-- [end:vscode_imports] + +// --8<-- [start:vscode_chunk] +export type Chunk = { path: string; line: number; text: string }; + +// Split a Markdown file into paragraphs, keeping the line each one starts on +// so a search result can jump straight to it. +export function chunkMarkdown(filePath: string, content: string): Chunk[] { + const lines = content.split(/\r?\n/); + const chunks: Chunk[] = []; + let start = 0; + for (let i = 0; i <= lines.length; i++) { + if (i === lines.length || lines[i].trim() === "") { + const text = lines.slice(start, i).join("\n").trim(); + if (text) { + chunks.push({ path: filePath, line: start, text }); + } + start = i + 1; + } + } + return chunks; +} +// --8<-- [end:vscode_chunk] + +// --8<-- [start:vscode_connect] +// Pass an absolute directory, such as `context.storageUri.fsPath`. A relative +// path resolves against the extension host's working directory, which is not +// your workspace and may be read-only (on macOS it is usually `/`). +export async function openDatabase(storageDir: string) { + return lancedb.connect(path.join(storageDir, "lancedb")); +} +// --8<-- [end:vscode_connect] + +// --8<-- [start:vscode_index] +export async function indexChunks(db: lancedb.Connection, chunks: Chunk[]) { + // Runs all-MiniLM-L6-v2 locally through Transformers.js: no API key needed. + const embedder = (await lancedb.embedding + .getRegistry() + .get("huggingface") + ?.create()) as lancedb.embedding.EmbeddingFunction; + + const schema = lancedb.embedding.LanceSchema({ + path: new Utf8(), + line: new Int32(), + text: embedder.sourceField(new Utf8()), + vector: embedder.vectorField(), + }); + + // LanceDB fills in the `vector` column from `text` as the rows are written. + return db.createTable("chunks", chunks, { schema, mode: "overwrite" }); +} +// --8<-- [end:vscode_index] + +// --8<-- [start:vscode_search] +export async function searchChunks( + db: lancedb.Connection, + query: string, + limit = 5, +) { + // The table stores its embedding function in its metadata, so a text query + // is embedded with the same model that embedded the rows. + const table = await db.openTable("chunks"); + return table + .search(query) + .select(["path", "line", "text", "_distance"]) + .limit(limit) + .toArray(); +} +// --8<-- [end:vscode_search] diff --git a/docs/web/docs.json b/docs/web/docs.json index 54203f1a0..66ce3acd3 100644 --- a/docs/web/docs.json +++ b/docs/web/docs.json @@ -490,7 +490,8 @@ "integrations/ai/genkit", "integrations/ai/kiln", "integrations/ai/prompttools", - "integrations/ai/synthetic-data-kit" + "integrations/ai/synthetic-data-kit", + "integrations/ai/vscode" ] } ] diff --git a/docs/web/integrations/ai/vscode.mdx b/docs/web/integrations/ai/vscode.mdx new file mode 100644 index 000000000..4c9319dd5 --- /dev/null +++ b/docs/web/integrations/ai/vscode.mdx @@ -0,0 +1,297 @@ +--- +title: "VS Code extensions" +sidebarTitle: "VS Code" +description: "Embed LanceDB in a VS Code extension to search a workspace by meaning, with no server, and bundle and package its native binaries correctly." +keywords: ["vscode", "visual studio code", "extension", "electron", "esbuild", "vsix", "semantic search"] +--- + +import { + TsVscodeChunk, + TsVscodeConnect, + TsVscodeImports, + TsVscodeIndex, + TsVscodeSearch, +} from '/snippets/vscode_search.mdx'; +import { TsVscodeExtension } from '/snippets/vscode_extension.mdx'; + +A VS Code extension runs in the *extension host*, a Node.js process, so LanceDB's TypeScript SDK +(`@lancedb/lancedb`) runs inside it like any other npm package. There's no database server to +start: the extension opens a table on local disk and queries it in-process. That makes LanceDB +a good fit for features such as semantic search over a workspace, retrieval for an AI assistant, +or a cache of embeddings that survives restarts. + +The extension host differs from a plain Node.js script in three ways that trip people up: + +- **Where you can write.** Relative paths resolve against the extension host's working + directory, not the workspace, and that directory may be read-only. +- **Bundling.** The default extension template bundles everything with esbuild, which can't + bundle LanceDB's native `.node` binary. +- **Packaging.** LanceDB installs a native binary for the current platform only, so a VSIX + built on one OS doesn't run on another. + +This guide builds a small extension that handles all three. It indexes the Markdown files in a +workspace and searches them by meaning, using a local embedding model, so no API key is needed. + + +**What you'll build** + +- **LanceDB: Index Markdown in Workspace** splits every `*.md` file into paragraphs and stores + them in a LanceDB table, with embeddings from `Xenova/all-MiniLM-L6-v2` run locally through + Transformers.js. +- **LanceDB: Search Markdown** embeds your query, shows the five closest paragraphs, and opens + the one you pick. + + +## Create the extension {#create-the-extension} + + + +Use the official generator. Choose **New Extension (TypeScript)** and **esbuild** as the bundler. + +```bash +npx --package yo --package generator-code -- yo code +``` + + + +```bash +npm install @lancedb/lancedb apache-arrow@18.1.0 +``` + +Your code imports Arrow types directly, so add `apache-arrow` as a dependency, pinned inside +LanceDB's peer dependency range (`>=15.0.0 <=18.1.0`). If the project already depends on a +newer `apache-arrow`, installing LanceDB fails with `ERESOLVE`. + +LanceDB also needs `@types/node` 22 or later. The generator's template already uses a recent +version; an old one causes `Cannot find module 'node:stream'` errors. + +You don't need to install `@huggingface/transformers` yourself. LanceDB pins it as an optional +dependency, and installing a different version gives your code a copy that LanceDB doesn't use. + + + +The type declarations of `apache-arrow` and the ONNX runtime used by Transformers.js refer to +browser types such as `ReadableStreamReadResult` and `HTMLCanvasElement`. The template's +`tsconfig.json` doesn't include the DOM library, so `tsc` reports errors inside `node_modules`. +Turn on `skipLibCheck`: + +```json tsconfig.json icon="brackets-curly" +{ + "compilerOptions": { + "skipLibCheck": true + } +} +``` + + + +## Store the database in extension storage {#store-the-database-in-extension-storage} + +Give LanceDB an absolute path inside the storage VS Code reserves for your extension: + +- `context.storageUri`: a folder for the current workspace. It's `undefined` when no folder is + open. Use it for anything derived from the workspace, such as this index. +- `context.globalStorageUri`: a folder shared by all workspaces. Use it for things like + downloaded models. + +Neither folder is guaranteed to exist, so create it before connecting. Both live outside the +user's project, so the index never shows up in their files or in version control. + +Don't pass a relative path such as `lancedb.connect("data")`. It resolves against the extension +host's working directory, which isn't the workspace and may be read-only: this causes the +`Read-only file system (os error 30)` error on macOS. + +Put the LanceDB code in its own module, `src/search.ts`: + + + {TsVscodeImports} + + + + {TsVscodeConnect} + + +## Index and search {#index-and-search} + +Split each file into paragraphs, and keep the line each one starts on so a result can open the +file at the right place: + + + {TsVscodeChunk} + + +Create the table with LanceDB's built-in `huggingface` [embedding function](/embedding/quickstart). +Because the `text` column is declared as the function's source field, LanceDB computes the +`vector` column for you as the rows are written: + + + {TsVscodeIndex} + + +The table remembers its embedding function, so you can search it with plain text after VS Code +restarts: + + + {TsVscodeSearch} + + +`mode: "overwrite"` rebuilds the index each time, which is fine for a few thousand paragraphs. +For larger workspaces, update only the files that changed with +[`mergeInsert`](/ingestion/merge-insert). + +## Wire up the commands {#wire-up-the-commands} + +Declare the two commands in `package.json`: + +```json package.json icon="brackets-curly" +"contributes": { + "commands": [ + { + "command": "lancedbSearch.index", + "title": "Index Markdown in Workspace", + "category": "LanceDB" + }, + { + "command": "lancedbSearch.search", + "title": "Search Markdown", + "category": "LanceDB" + } + ] +} +``` + +Then register them in `src/extension.ts`: + + + {TsVscodeExtension} + + +A few details in this file are specific to running inside VS Code: + +- The index goes in `storageUri`, so each workspace gets its own table. +- The embedding model (about 90 MB) downloads on first use. By default Transformers.js caches it + inside `node_modules` in the installed extension, which is replaced on every update. Setting + `env.cacheDir` keeps it in global storage. Load the module with `import()`, not `require()`: + LanceDB loads the ES module build of `@huggingface/transformers`, and `require()` returns a + separate copy whose settings LanceDB never sees. +- Each paragraph is stored with its file's URI, not a path relative to a workspace folder. In a + multi-root workspace, `findFiles` searches every folder and two folders can hold the same + relative path, so only the URI opens the right file. `asRelativePath` turns it back into a + short label, which names the folder when there's more than one. +- When another extension or a test passes a query to `lancedbSearch.search`, it returns the + rows instead of opening a picker. That's how the results below were produced, with each URI + shown through `asRelativePath`. + +## Sample results {#sample-results} + +Here are the results of running the extension in VS Code against a small workspace of six +Markdown files that document a fictional CLI. The columns are the vector distance (lower is +closer), the file and line, and the paragraph. + +```text +query: my credentials were exposed, what should I do? + 33.100 docs/auth.md:5 If a token leaks, revoke it on the Tokens page and generate a replacement. Revoked tokens stop working immediately. + 40.091 docs/troubleshooting.md:5 Run `acme doctor` to print diagnostics you can attach to a bug report. + 42.334 CONTRIBUTING.md:3 Run `npm test` before opening a pull request. New commands need a page in docs/. + 43.105 docs/auth.md:3 Acme uses personal access tokens. Create one under Settings > Tokens. + 45.664 README.md:5 Install it with `npm install -g acme-cli`, then run `acme login`. + +query: uploads are stuck behind the company firewall + 38.420 docs/troubleshooting.md:3 If `acme sync` hangs, check that your proxy allows connections to api.acme.dev on port 443. + 39.212 docs/sync.md:3 Run `acme sync` to upload changed notes. Only files modified since the last sync are sent. + 40.859 docs/auth.md:5 If a token leaks, revoke it on the Tokens page and generate a replacement. Revoked tokens stop working immediately. + 42.537 docs/troubleshooting.md:5 Run `acme doctor` to print diagnostics you can attach to a bug report. + 45.853 CONTRIBUTING.md:3 Run `npm test` before opening a pull request. New commands need a page in docs/. +``` + +Neither top result shares a keyword with its query: "credentials were exposed" matches "a +token leaks", and "stuck behind the company firewall" matches "hangs … proxy". Picking a result +in the search box opens the file at that paragraph. + +## Bundle with esbuild {#bundle-with-esbuild} + +The template bundles your code and its dependencies into `dist/extension.js`. esbuild can't +bundle native binaries, so building with LanceDB fails with +`No loader is configured for ".node" files`. Mark the packages that load native code as +external so they're loaded from `node_modules` at runtime: + +```js esbuild.js icon="square-js" +external: [ + "vscode", + "@lancedb/lancedb", + "@huggingface/transformers", + // LanceDB loads apache-arrow from node_modules anyway; bundling it ships a second copy. + "apache-arrow", +], +``` + +External packages must be in the VSIX, but the template's `.vscodeignore` excludes +`node_modules/**`. Delete that line. `vsce` then includes your production dependencies and +leaves out the dev dependencies. If you skip this step, the packaged extension fails to +activate with `Cannot find module '@lancedb/lancedb'`, and its commands report +`command 'lancedbSearch.index' not found`. + +## Package a VSIX for each platform {#package-a-vsix-for-each-platform} + +npm installs LanceDB's native binary only for the machine it runs on, such as +`@lancedb/lancedb-win32-x64-msvc` on 64-bit Windows. Build one VSIX per platform, each on a +matching machine or CI runner, and publish them all. The Marketplace serves each user the right one. + +```bash +npx @vscode/vsce package --target win32-x64 +``` + +| `vsce` target | LanceDB native package | +|:--|:--| +| `win32-x64` | `@lancedb/lancedb-win32-x64-msvc` | +| `win32-arm64` | `@lancedb/lancedb-win32-arm64-msvc` | +| `linux-x64` | `@lancedb/lancedb-linux-x64-gnu` | +| `linux-arm64` | `@lancedb/lancedb-linux-arm64-gnu` | +| `alpine-x64` | `@lancedb/lancedb-linux-x64-musl` | +| `alpine-arm64` | `@lancedb/lancedb-linux-arm64-musl` | +| `darwin-arm64` | `@lancedb/lancedb-darwin-arm64` | + +LanceDB doesn't publish a binary for Intel Macs (`darwin-x64`). + +The table covers LanceDB's own binaries. This sample's local model also needs +`onnxruntime-node`, which Transformers.js loads when the extension activates. Its Linux binaries +link against glibc and it publishes no musl build, so the sample as written doesn't run on +Alpine, whatever `--target` you pass. To ship the `alpine-*` targets, embed with a hosted API or +use [full-text search](/search/full-text-search) instead, or build ONNX Runtime for musl yourself. + + +Native binaries make the VSIX large: the `win32-x64` build of this extension is about 200 MB. +`onnxruntime-node`, which Transformers.js uses, ships binaries for every platform. Excluding +the ones a target doesn't need brings the `win32-x64` build to about 150 MB. Keep one ignore +file per target and pass it with `--ignoreFile`: + +```text .vscodeignore.win32-x64 +node_modules/onnxruntime-node/bin/napi-v3/darwin/** +node_modules/onnxruntime-node/bin/napi-v3/linux/** +node_modules/onnxruntime-node/bin/napi-v3/win32/arm64/** +``` + +If you don't need a local model, embed text with a hosted API instead, or use +[full-text search](/search/full-text-search), which needs no model at all. + + +## Keep the extension host responsive {#keep-the-extension-host-responsive} + +All extensions share one extension host, and loading the model and computing embeddings in it +can stall the others. While the sample indexes, VS Code may briefly report that the extension +host is unresponsive. That's fine for a handful of files. For a large workspace, move indexing into a +[worker thread](https://nodejs.org/api/worker_threads.html) or a child process, and index in +batches as files change instead of all at once. + +## Troubleshooting {#troubleshooting} + +| Error | Cause | Fix | +|:--|:--|:--| +| `Read-only file system (os error 30)` | A relative database path resolved against the extension host's working directory | Connect to an absolute path under `context.storageUri` or `context.globalStorageUri` | +| `No loader is configured for ".node" files` | esbuild tried to bundle a native binary | Mark `@lancedb/lancedb` and `@huggingface/transformers` as `external` | +| `Cannot find module '@lancedb/lancedb'`, or `command '…' not found` | The VSIX doesn't contain `node_modules` | Remove `node_modules/**` from `.vscodeignore` | +| A native module isn't found on another OS | The VSIX was built on a different platform | Package one VSIX per `--target` on a matching machine | +| `Cannot find module 'node:stream' or its corresponding type declarations` | `@types/node` is too old | Use `@types/node` 22 or later | +| `Cannot find name 'HTMLCanvasElement'` and similar errors in `node_modules` | Dependency typings refer to DOM types | Set `"skipLibCheck": true` | +| `ERESOLVE` with `peer apache-arrow@">=15.0.0 <=18.1.0"` | The project depends on a newer `apache-arrow` | Install `apache-arrow@18.1.0` | +| The model downloads again after an update | Transformers.js caches inside the extension's `node_modules` | Set `env.cacheDir` through `import("@huggingface/transformers")` | diff --git a/docs/web/snippets/vscode_extension.mdx b/docs/web/snippets/vscode_extension.mdx new file mode 100644 index 000000000..df5618c38 --- /dev/null +++ b/docs/web/snippets/vscode_extension.mdx @@ -0,0 +1,4 @@ +{/* Auto-generated by scripts/mdx_snippets_gen.py. Do not edit manually. */} + +export const TsVscodeExtension = "import * as vscode from \"vscode\";\nimport {\n type Chunk,\n chunkMarkdown,\n indexChunks,\n openDatabase,\n searchChunks,\n} from \"./search\";\n\nexport async function activate(context: vscode.ExtensionContext) {\n // One index per workspace. `storageUri` is undefined when no folder is open.\n const storage = context.storageUri ?? context.globalStorageUri;\n\n // Transformers.js downloads the embedding model on first use. Keep it in\n // global storage so every workspace shares one copy and it survives\n // extension updates. Use `import()`: LanceDB loads the ES module build, and\n // `require()` would return a separate copy whose settings LanceDB never sees.\n const { env } = await import(\"@huggingface/transformers\");\n env.cacheDir = vscode.Uri.joinPath(context.globalStorageUri, \"models\").fsPath;\n\n context.subscriptions.push(\n vscode.commands.registerCommand(\"lancedbSearch.index\", async () => {\n const files = await vscode.workspace.findFiles(\n \"**/*.md\",\n \"**/node_modules/**\",\n );\n const chunks: Chunk[] = [];\n for (const file of files) {\n const bytes = await vscode.workspace.fs.readFile(file);\n // Store the file's URI, not a path relative to its folder: in a\n // multi-root workspace `findFiles` searches every folder, and only the\n // URI says which one a result came from.\n chunks.push(\n ...chunkMarkdown(file.toString(), new TextDecoder().decode(bytes)),\n );\n }\n if (chunks.length === 0) {\n vscode.window.showWarningMessage(\"No Markdown files to index.\");\n return 0;\n }\n\n await vscode.window.withProgress(\n {\n location: vscode.ProgressLocation.Notification,\n title: `Indexing ${files.length} Markdown files`,\n },\n async () => {\n await vscode.workspace.fs.createDirectory(storage);\n await indexChunks(await openDatabase(storage.fsPath), chunks);\n },\n );\n vscode.window.showInformationMessage(\n `Indexed ${chunks.length} paragraphs from ${files.length} files.`,\n );\n return chunks.length;\n }),\n\n // Called with a query (for example from a test or another extension), the\n // command returns the matching rows instead of showing a picker.\n vscode.commands.registerCommand(\n \"lancedbSearch.search\",\n async (query?: string) => {\n const text =\n query ??\n (await vscode.window.showInputBox({\n prompt: \"Search Markdown by meaning\",\n }));\n if (!text) {\n return [];\n }\n const results = await searchChunks(\n await openDatabase(storage.fsPath),\n text,\n );\n if (query !== undefined) {\n return results;\n }\n\n const picked = await vscode.window.showQuickPick(\n results.map((row) => {\n const uri = vscode.Uri.parse(row.path);\n return {\n label: `${vscode.workspace.asRelativePath(uri)}:${row.line + 1}`,\n description: `distance ${row._distance.toFixed(3)}`,\n detail: row.text,\n uri,\n line: row.line,\n };\n }),\n { placeHolder: text, matchOnDetail: true },\n );\n if (picked) {\n const editor = await vscode.window.showTextDocument(picked.uri);\n const start = new vscode.Position(picked.line, 0);\n editor.selection = new vscode.Selection(start, start);\n editor.revealRange(\n new vscode.Range(start, start),\n vscode.TextEditorRevealType.InCenter,\n );\n }\n return results;\n },\n ),\n );\n}\n\nexport function deactivate() {}\n"; + diff --git a/docs/web/snippets/vscode_search.mdx b/docs/web/snippets/vscode_search.mdx new file mode 100644 index 000000000..4d2423e7f --- /dev/null +++ b/docs/web/snippets/vscode_search.mdx @@ -0,0 +1,12 @@ +{/* Auto-generated by scripts/mdx_snippets_gen.py. Do not edit manually. */} + +export const TsVscodeChunk = "export type Chunk = { path: string; line: number; text: string };\n\n// Split a Markdown file into paragraphs, keeping the line each one starts on\n// so a search result can jump straight to it.\nexport function chunkMarkdown(filePath: string, content: string): Chunk[] {\n const lines = content.split(/\\r?\\n/);\n const chunks: Chunk[] = [];\n let start = 0;\n for (let i = 0; i <= lines.length; i++) {\n if (i === lines.length || lines[i].trim() === \"\") {\n const text = lines.slice(start, i).join(\"\\n\").trim();\n if (text) {\n chunks.push({ path: filePath, line: start, text });\n }\n start = i + 1;\n }\n }\n return chunks;\n}\n"; + +export const TsVscodeConnect = "// Pass an absolute directory, such as `context.storageUri.fsPath`. A relative\n// path resolves against the extension host's working directory, which is not\n// your workspace and may be read-only (on macOS it is usually `/`).\nexport async function openDatabase(storageDir: string) {\n return lancedb.connect(path.join(storageDir, \"lancedb\"));\n}\n"; + +export const TsVscodeImports = "import * as path from \"node:path\";\nimport * as lancedb from \"@lancedb/lancedb\";\nimport \"@lancedb/lancedb/embedding/transformers\";\nimport { Int32, Utf8 } from \"apache-arrow\";\n"; + +export const TsVscodeIndex = "export async function indexChunks(db: lancedb.Connection, chunks: Chunk[]) {\n // Runs all-MiniLM-L6-v2 locally through Transformers.js: no API key needed.\n const embedder = (await lancedb.embedding\n .getRegistry()\n .get(\"huggingface\")\n ?.create()) as lancedb.embedding.EmbeddingFunction;\n\n const schema = lancedb.embedding.LanceSchema({\n path: new Utf8(),\n line: new Int32(),\n text: embedder.sourceField(new Utf8()),\n vector: embedder.vectorField(),\n });\n\n // LanceDB fills in the `vector` column from `text` as the rows are written.\n return db.createTable(\"chunks\", chunks, { schema, mode: \"overwrite\" });\n}\n"; + +export const TsVscodeSearch = "export async function searchChunks(\n db: lancedb.Connection,\n query: string,\n limit = 5,\n) {\n // The table stores its embedding function in its metadata, so a text query\n // is embedded with the same model that embedded the rows.\n const table = await db.openTable(\"chunks\");\n return table\n .search(query)\n .select([\"path\", \"line\", \"text\", \"_distance\"])\n .limit(limit)\n .toArray();\n}\n"; +