Add docs code testing & documentation syntax changes (#196)

- Creates testing files `md_testing.py` and `md_testing.js` for testing python and nodejs code in markdown files in the documentation This listens for HTML tags as well: `` will create a set-up file to create some mock tables or to fulfill some assumptions in the documentation. - Creates a github action workflow that triggers every push/pr to `docs/**` - Modifies documentation so tests run (mostly indentation, some small syntax errors and some missing imports) A list of excluded files that we need to take a closer look at later on: ```javascript const excludedFiles = [ "../src/fts.md", "../src/embedding.md", "../src/examples/serverless_lancedb_with_s3_and_lambda.md", "../src/examples/serverless_qa_bot_with_modal_and_langchain.md", "../src/examples/youtube_transcript_bot_with_nodejs.md", ]; ``` Many of them can't be done because we need the OpenAI API key :(. `fts.md` has some issues with the library, I believe this is still experimental? Closes #170 --------- Co-authored-by: Will Jones <willjones127@gmail.com>
2025-12-22 21:09:58 +00:00 · 2023-06-28 11:07:26 -07:00
parent 0f58bd7af2
commit b731a6aed9
16 changed files with 1173 additions and 49 deletions
--- a/.github/workflows/docs_test.yml
+++ b/.github/workflows/docs_test.yml
@@ -0,0 +1,96 @@
+name: Documentation Code Testing
+
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - docs/**
+      - .github/workflows/docs_test.yml
+  pull_request:
+    paths:
+      - docs/**
+      - .github/workflows/docs_test.yml
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+env:
+  # Disable full debug symbol generation to speed up CI build and keep memory down
+  # "1" means line tables only, which is useful for panic tracebacks.
+  RUSTFLAGS: "-C debuginfo=1"
+  RUST_BACKTRACE: "1"
+
+jobs:
+  test-python:
+    name: Test doc python code
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        python-minor-version: [ "11" ]
+        os: ["ubuntu-22.04"]
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v3
+    - name: Set up Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: 3.${{ matrix.python-minor-version }}
+        cache: "pip"
+        cache-dependency-path: "docs/test/requirements.txt"
+    - name: Build Python
+      working-directory: docs/test
+      run:
+        python -m pip install -r requirements.txt
+    - name: Create test files
+      run: |
+        cd docs/test
+        python md_testing.py
+    - name: Test
+      run: |
+        cd docs/test/python
+        for d in *; do cd "$d"; echo "$d".py; python "$d".py; cd ..; done
+  test-node:
+    name: Test doc nodejs code
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        node-version: [ "18" ]
+        os: ["ubuntu-22.04"]
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v3
+      with:
+        fetch-depth: 0
+        lfs: true
+    - name: Set up Node
+      uses: actions/setup-node@v3
+      with:
+        node-version: ${{ matrix.node-version }}
+        cache: 'npm'
+        cache-dependency-path: docs/test/package-lock.json
+    - name: Install dependecies needed for ubuntu
+      if: ${{ matrix.os == 'ubuntu-22.04' }}
+      run: |
+        sudo apt install -y protobuf-compiler libssl-dev
+    - name: Install node dependencies
+      run: |
+        cd docs/test
+        npm cache clean --force
+        npm ci
+    - name: Rust cache
+      uses: swatinem/rust-cache@v2
+    - name: Install LanceDB
+      run: |
+        cd docs/test/node_modules/vectordb 
+        npm ci
+        npm run build
+        npm run tsc
+    - name: Create test files
+      run: |
+        cd docs/test
+        node md_testing.js
+    - name: Test
+      run: |
+        cd docs/test/node
+        for d in *; do cd "$d"; echo "$d".js; node "$d".js; cd ..; done
--- a/README.md
+++ b/README.md
@@ -65,7 +65,7 @@ pip install lancedb
 ```python
 import lancedb

-uri = "/tmp/lancedb"
+uri = "data/sample-lancedb"
 db = lancedb.connect(uri)
 table = db.create_table("my_table",
                         data=[{"vector": [3.1, 4.1], "item": "foo", "price": 10.0},
--- a/docs/src/ann_indexes.md
+++ b/docs/src/ann_indexes.md
@@ -23,7 +23,7 @@ In the future we will look to automatically create and configure the ANN index.

     # Create 10,000 sample vectors
     data = [{"vector": row, "item": f"item {i}"}
-        for i, row in enumerate(np.random.random((10_000, 768)).astype('float32'))]
+        for i, row in enumerate(np.random.random((10_000, 1536)).astype('float32'))]

     # Add the vectors to a table
     tbl = db.create_table("my_vectors", data=data)
@@ -41,8 +41,8 @@ In the future we will look to automatically create and configure the ANN index.
     for (let i = 0; i < 10_000; i++) {
         data.push({vector: Array(1536).fill(i), id: `${i}`, content: "", longId: `${i}`},)
     }
-     const table = await db.createTable('vectors', data)
-     await table.create_index({ type: 'ivf_pq', column: 'vector', num_partitions: 256, num_sub_vectors: 96 })
+     const table = await db.createTable('my_vectors', data)
+     await table.createIndex({ type: 'ivf_pq', column: 'vector', num_partitions: 256, num_sub_vectors: 96 })
     ```

 Since `create_index` has a training step, it can take a few minutes to finish for large tables. You can control the index
@@ -73,12 +73,13 @@ There are a couple of parameters that can be used to fine-tune the search:

 === "Python"
     ```python
-     tbl.search(np.random.random((768))) \
+     tbl.search(np.random.random((1536))) \
         .limit(2) \
         .nprobes(20) \
         .refine_factor(10) \
         .to_df()
-
+     ```
+     ```
                                              vector       item       score
     0  [0.44949695, 0.8444449, 0.06281311, 0.23338133...  item 1141  103.575333
     1  [0.48587373, 0.269207, 0.15095535, 0.65531915,...  item 3953  108.393867
@@ -86,8 +87,8 @@ There are a couple of parameters that can be used to fine-tune the search:

 === "Javascript"
     ```javascript
-     const results = await table
-         .search(Array(768).fill(1.2))
+     const results_1 = await table
+         .search(Array(1536).fill(1.2))
         .limit(2)
         .nprobes(20)
         .refineFactor(10)
@@ -104,14 +105,14 @@ You can further filter the elements returned by a search using a where clause.

 === "Python"
     ```python
-     tbl.search(np.random.random((768))).where("item != 'item 1141'").to_df()
+     tbl.search(np.random.random((1536))).where("item != 'item 1141'").to_df()
     ```

 === "Javascript"
     ```javascript
-     const results = await table
+     const results_2 = await table
         .search(Array(1536).fill(1.2))
-         .where("item != 'item 1141'")
+         .where("id != '1141'")
         .execute()
     ```

@@ -121,7 +122,9 @@ You can select the columns returned by the query using a select clause.

 === "Python"
     ```python
-     tbl.search(np.random.random((768))).select(["vector"]).to_df()
+     tbl.search(np.random.random((1536))).select(["vector"]).to_df()
+     ```
+     ```
     vector      score
     0  [0.30928212, 0.022668175, 0.1756372, 0.4911822...  93.971092
     1  [0.2525465, 0.01723831, 0.261568, 0.002007689,...  95.173485
@@ -130,7 +133,7 @@ You can select the columns returned by the query using a select clause.

 === "Javascript"
     ```javascript
-     const results = await table
+     const results_3 = await table
         .search(Array(1536).fill(1.2))
         .select(["id"])
         .execute()
--- a/docs/src/basic.md
+++ b/docs/src/basic.md
@@ -23,7 +23,7 @@ We'll cover the basics of using LanceDB on your local machine in this section.
 === "Python"
      ```python
      import lancedb
-      uri = "~/.lancedb"
+      uri = "data/sample-lancedb"
      db = lancedb.connect(uri)
      ```

@@ -35,7 +35,7 @@ We'll cover the basics of using LanceDB on your local machine in this section.
      ```javascript
      const lancedb = require("vectordb");

-      const uri = "~./lancedb";
+      const uri = "data/sample-lancedb";
      const db = await lancedb.connect(uri);
      ```
      
@@ -102,7 +102,7 @@ Once created, you can open a table using the following code:
      If you forget the name of your table, you can always get a listing of all table names:

      ```javascript
-      console.log(db.tableNames());
+      console.log(await db.tableNames());
      ```

 ## How to add data to a table
@@ -118,7 +118,7 @@ After a table has been created, you can always add more data to it using

 === "Javascript"
      ```javascript
-      await tbl.add([vector: [1.3, 1.4], item: "fizz", price: 100.0},
+      await tbl.add([{vector: [1.3, 1.4], item: "fizz", price: 100.0},
              {vector: [9.5, 56.2], item: "buzz", price: 200.0}])
      ```

--- a/docs/src/embedding.md
+++ b/docs/src/embedding.md
@@ -98,7 +98,7 @@ You can also use an external API like OpenAI to generate embeddings
     embededings for your data.

      ```javascript
-      const db = await lancedb.connect("/tmp/lancedb");
+      const db = await lancedb.connect("data/sample-lancedb");
      const data = [
        { text: 'pepperoni'  },
        { text: 'pineapple' }
--- a/docs/src/fts.md
+++ b/docs/src/fts.md
@@ -18,6 +18,20 @@ Assume:
 1. `table` is a LanceDB Table
 2. `text` is the name of the Table column that we want to index

+For example,
+
+```python
+import lancedb
+
+uri = "data/sample-lancedb"
+db = lancedb.connect(uri)
+
+table = db.create_table("my_table",
+            data=[{"vector": [3.1, 4.1], "text": "Frodo was a happy puppy"},
+                  {"vector": [5.9, 26.5], "text": "There are several kittens playing"}])
+
+```
+
 To create the index:

 ```python
--- a/docs/src/index.md
+++ b/docs/src/index.md
@@ -28,7 +28,7 @@ LanceDB's core is written in Rust 🦀 and is built using <a href="https://githu
      ```python
      import lancedb

-      uri = "/tmp/lancedb"
+      uri = "data/sample-lancedb"
      db = lancedb.connect(uri)
      table = db.create_table("my_table",
                              data=[{"vector": [3.1, 4.1], "item": "foo", "price": 10.0},
@@ -44,7 +44,7 @@ LanceDB's core is written in Rust 🦀 and is built using <a href="https://githu
      ```javascript
      const lancedb = require("vectordb");

-      const uri = "/tmp/lancedb";
+      const uri = "data/sample-lancedb";
      const db = await lancedb.connect(uri);
      const table = await db.createTable("my_table", 
            [{ id: 1, vector: [3.1, 4.1], item: "foo", price: 10.0 },
--- a/docs/src/integrations.md
+++ b/docs/src/integrations.md
@@ -6,11 +6,11 @@ Built on top of Apache Arrow, `LanceDB` is easy to integrate with the Python eco

 First, we need to connect to a `LanceDB` database.

-``` py
+```py

 import lancedb

-db = lancedb.connect("/tmp/lancedb")
+db = lancedb.connect("data/sample-lancedb")
 ```

 And write a `Pandas DataFrame` to LanceDB directly.
@@ -79,7 +79,7 @@ We will re-use the dataset created previously
 ```python
 import lancedb

-db = lancedb.connect("/tmp/lancedb")
+db = lancedb.connect("data/sample-lancedb")
 table = db.open_table("pd_table")
 arrow_table = table.to_arrow()
 ```
@@ -87,8 +87,12 @@ arrow_table = table.to_arrow()
 `DuckDB` can directly query the `arrow_table`:

 ```python
-In [15]: duckdb.query("SELECT * FROM t")
-Out[15]:
+import duckdb 
+
+duckdb.query("SELECT * FROM arrow_table")
+```
+
+```
 ┌─────────────┬─────────┬────────┐
 │   vector    │  item   │ price  │
 │   float[]   │ varchar │ double │
@@ -96,8 +100,12 @@ Out[15]:
 │ [3.1, 4.1]  │ foo     │   10.0 │
 │ [5.9, 26.5] │ bar     │   20.0 │
 └─────────────┴─────────┴────────┘
+```
+```python
+duckdb.query("SELECT mean(price) FROM arrow_table")
+```

-In [16]: duckdb.query("SELECT mean(price) FROM t")
+```
 Out[16]:
 ┌─────────────┐
 │ mean(price) │
--- a/docs/src/javascript/README.md
+++ b/docs/src/javascript/README.md
@@ -16,9 +16,11 @@ npm install vectordb

 ```javascript
 const lancedb = require('vectordb');
-const db = lancedb.connect('<PATH_TO_LANCEDB_DATASET>');
-const table = await db.openTable('my_table');
-const query = await table.search([0.1, 0.3]).limit(20).execute();
+const db = await lancedb.connect('data/sample-lancedb');
+const table = await db.createTable("my_table", 
+      [{ id: 1, vector: [0.1, 1.0], item: "foo", price: 10.0 },
+      { id: 2, vector: [3.9, 0.5], item: "bar", price: 20.0 }])
+const results = await table.search([0.1, 0.3]).limit(20).execute();
 console.log(results);
 ```

--- a/docs/src/search.md
+++ b/docs/src/search.md
@@ -17,7 +17,7 @@ Currently, we support the following metrics:
 | Metric      | Description                          |
 | ----------- | ------------------------------------ |
 | `L2`        | [Euclidean / L2 distance](https://en.wikipedia.org/wiki/Euclidean_distance) |
-| `Cosine`    | [Cosine Similarity](https://en.wikipedia.org/wiki/Cosine_similarity)|
+<!-- | `Cosine`    | [Cosine Similarity](https://en.wikipedia.org/wiki/Cosine_similarity)| -->


 ## Search
@@ -28,16 +28,44 @@ Currently, we support the following metrics:
 If there is no [vector index is created](ann_indexes.md), LanceDB will just brute-force scan
 the vector column and compute the distance.

+<!-- Setup Code
+```python 
+import lancedb
+import numpy as np
+uri = "data/sample-lancedb"
+db = lancedb.connect(uri)
+
+data = [{"vector": row, "item": f"item {i}"}
+     for i, row in enumerate(np.random.random((10_000, 1536)).astype('float32'))]
+
+db.create_table("my_vectors", data=data)
+```
+-->
+<!-- Setup Code
+```javascript 
+const vectordb_setup = require('vectordb')
+const db_setup = await vectordb_setup.connect('data/sample-lancedb')
+
+let data = []
+for (let i = 0; i < 10_000; i++) {
+     data.push({vector: Array(1536).fill(i), id: `${i}`, content: "", longId: `${i}`},)
+}
+await db_setup.createTable('my_vectors', data)
+```
+-->
 === "Python"

+
    ```python
    import lancedb
+    import numpy as np
+    
    db = lancedb.connect("data/sample-lancedb")

    tbl = db.open_table("my_vectors")

-    df = tbl.search(np.random.random((768)))
-        .limit(10)
+    df = tbl.search(np.random.random((1536))) \
+        .limit(10) \
        .to_df()
    ```

@@ -47,38 +75,41 @@ the vector column and compute the distance.
    const vectordb = require('vectordb')
    const db = await vectordb.connect('data/sample-lancedb')

-    tbl = db.open_table("my_vectors")
+    const tbl = await db.openTable("my_vectors")

-    const results = await tbl.search(Array(768))
+    const results_1 = await tbl.search(Array(1536).fill(1.2))
        .limit(20)
        .execute()
    ```

+
+<!-- Commenting out for now since metricType fails for JS on Ubuntu 22.04.
+
 By default, `l2` will be used as `Metric` type. You can customize the metric type
 as well.
+-->

+<!--
 === "Python"
-
-    ```python
-    df = tbl.search(np.random.random((768)))
-        .metric("cosine")
-        .limit(10)
+-->
+<!--    ```python
+    df = tbl.search(np.random.random((1536))) \
+        .metric("cosine") \
+        .limit(10) \
        .to_df()
    ```
-
+-->
+<!--
 === "JavaScript"
+-->

-    ```javascript
-    const vectordb = require('vectordb')
-    const db = await vectordb.connect('data/sample-lancedb')
-
-    tbl = db.open_table("my_vectors")
-
-    const results = await tbl.search(Array(768))
-        .metric("cosine")
+<!--   ```javascript
+    const results_2 = await tbl.search(Array(1536).fill(1.2))
+        .metricType("cosine")
        .limit(20)
        .execute()
    ```
+-->

 ### Search with Vector Index.

--- a/docs/src/sql.md
+++ b/docs/src/sql.md
@@ -15,11 +15,35 @@ Currently, Lance supports a growing list of expressions.
 * ``regexp_match(column, pattern)``

 For example, the following filter string is acceptable:
+<!-- Setup Code
+```python 
+import lancedb
+import numpy as np
+uri = "data/sample-lancedb"
+db = lancedb.connect(uri)

+data = [{"vector": row, "item": f"item {i}"}
+     for i, row in enumerate(np.random.random((10_000, 2)).astype('int'))]
+
+tbl = db.create_table("my_vectors", data=data)
+```
+-->
+<!-- Setup Code
+```javascript 
+const vectordb = require('vectordb')
+const db = await vectordb.connect('data/sample-lancedb')
+
+let data = []
+for (let i = 0; i < 10_000; i++) {
+     data.push({vector: Array(1536).fill(i), id: `${i}`, content: "", longId: `${i}`},)
+}
+const tbl = await db.createTable('my_vectors', data)
+```
+-->
 === "Python"

    ```python
-    tbl.search([100, 102])
+    tbl.search([100, 102]) \
       .where("""(
        (label IN [10, 20])
        AND
--- a/docs/test/md_testing.js
+++ b/docs/test/md_testing.js
@@ -0,0 +1,51 @@
+const glob = require("glob");
+const fs = require("fs");
+const path = require("path");
+
+const excludedFiles = [
+  "../src/fts.md",
+  "../src/embedding.md",
+  "../src/examples/serverless_lancedb_with_s3_and_lambda.md",
+  "../src/examples/serverless_qa_bot_with_modal_and_langchain.md",
+  "../src/examples/youtube_transcript_bot_with_nodejs.md",
+];
+const nodePrefix = "javascript";
+const nodeFile = ".js";
+const nodeFolder = "node";
+const globString = "../src/**/*.md";
+const asyncPrefix = "(async () => {\n";
+const asyncSuffix = "})();";
+
+function* yieldLines(lines, prefix, suffix) {
+  let inCodeBlock = false;
+  for (const line of lines) {
+    if (line.trim().startsWith(prefix + nodePrefix)) {
+      inCodeBlock = true;
+    } else if (inCodeBlock && line.trim().startsWith(suffix)) {
+      inCodeBlock = false;
+      yield "\n";
+    } else if (inCodeBlock) {
+      yield line;
+    }
+  }
+}
+
+const files = glob.sync(globString, { recursive: true });
+
+for (const file of files.filter((file) => !excludedFiles.includes(file))) {
+  const lines = [];
+  const data = fs.readFileSync(file, "utf-8");
+  const fileLines = data.split("\n");
+
+  for (const line of yieldLines(fileLines, "```", "```")) {
+    lines.push(line);
+  }
+
+  if (lines.length > 0) {
+    const fileName = path.basename(file, ".md");
+    const outPath = path.join(nodeFolder, fileName, `${fileName}${nodeFile}`);
+    console.log(outPath)
+    fs.mkdirSync(path.dirname(outPath), { recursive: true });
+    fs.writeFileSync(outPath, asyncPrefix + "\n" + lines.join("\n") + asyncSuffix);
+  }
+}
--- a/docs/test/md_testing.py
+++ b/docs/test/md_testing.py
@@ -0,0 +1,41 @@
+import glob
+from typing import Iterator
+from pathlib import Path
+
+excluded_files = [
+    "../src/fts.md",
+    "../src/embedding.md",
+    "../src/examples/serverless_lancedb_with_s3_and_lambda.md",
+    "../src/examples/serverless_qa_bot_with_modal_and_langchain.md",
+    "../src/examples/youtube_transcript_bot_with_nodejs.md"
+]
+
+python_prefix = "py"
+python_file = ".py"
+python_folder = "python"
+glob_string = "../src/**/*.md"
+
+def yield_lines(lines: Iterator[str], prefix: str, suffix: str):
+    in_code_block = False
+    # Python code has strict indentation
+    strip_length = 0
+    for line in lines:
+        if line.strip().startswith(prefix + python_prefix):
+            in_code_block = True
+            strip_length = len(line) - len(line.lstrip())
+        elif in_code_block and line.strip().startswith(suffix):
+            in_code_block = False
+            yield "\n"
+        elif in_code_block:
+            yield line[strip_length:]
+
+for file in filter(lambda file: file not in excluded_files, glob.glob(glob_string, recursive=True)):
+    with open(file, "r") as f:
+        lines = list(yield_lines(iter(f), "```", "```"))
+
+    if len(lines) > 0:
+        out_path = Path(python_folder) / Path(file).name.strip(".md") / (Path(file).name.strip(".md") + python_file)
+        print(out_path)
+        out_path.parent.mkdir(exist_ok=True, parents=True)
+        with open(out_path, "w") as out:
+            out.writelines(lines)
--- a/docs/test/package-lock.json
+++ b/docs/test/package-lock.json
@@ -0,0 +1,834 @@
+{
+  "name": "lancedb-docs-test",
+  "version": "1.0.0",
+  "lockfileVersion": 3,
+  "requires": true,
+  "packages": {
+    "": {
+      "name": "lancedb-docs-test",
+      "version": "1.0.0",
+      "license": "ISC",
+      "dependencies": {
+        "fs": "^0.0.1-security",
+        "glob": "^10.2.7",
+        "path": "^0.12.7"
+      },
+      "devDependencies": {
+        "vectordb": "https://gitpkg.now.sh/lancedb/lancedb/node?main"
+      }
+    },
+    "node_modules/@apache-arrow/ts": {
+      "version": "12.0.1",
+      "resolved": "https://registry.npmjs.org/@apache-arrow/ts/-/ts-12.0.1.tgz",
+      "integrity": "sha512-oLL/la62/eff4/nTtU4mZzx/SuIIAjrFx6sTSRDZ65a1Ej84IIMptVb/i5rnp2kY14GFnyc928MyJexQmQc4zA==",
+      "dev": true,
+      "dependencies": {
+        "@types/command-line-args": "5.2.0",
+        "@types/command-line-usage": "5.0.2",
+        "@types/node": "18.14.5",
+        "@types/pad-left": "2.1.1",
+        "command-line-args": "5.2.1",
+        "command-line-usage": "6.1.3",
+        "flatbuffers": "23.3.3",
+        "json-bignum": "^0.0.3",
+        "pad-left": "^2.1.0",
+        "tslib": "^2.5.0"
+      }
+    },
+    "node_modules/@isaacs/cliui": {
+      "version": "8.0.2",
+      "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz",
+      "integrity": "sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==",
+      "dependencies": {
+        "string-width": "^5.1.2",
+        "string-width-cjs": "npm:string-width@^4.2.0",
+        "strip-ansi": "^7.0.1",
+        "strip-ansi-cjs": "npm:strip-ansi@^6.0.1",
+        "wrap-ansi": "^8.1.0",
+        "wrap-ansi-cjs": "npm:wrap-ansi@^7.0.0"
+      },
+      "engines": {
+        "node": ">=12"
+      }
+    },
+    "node_modules/@pkgjs/parseargs": {
+      "version": "0.11.0",
+      "resolved": "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz",
+      "integrity": "sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==",
+      "optional": true,
+      "engines": {
+        "node": ">=14"
+      }
+    },
+    "node_modules/@types/command-line-args": {
+      "version": "5.2.0",
+      "resolved": "https://registry.npmjs.org/@types/command-line-args/-/command-line-args-5.2.0.tgz",
+      "integrity": "sha512-UuKzKpJJ/Ief6ufIaIzr3A/0XnluX7RvFgwkV89Yzvm77wCh1kFaFmqN8XEnGcN62EuHdedQjEMb8mYxFLGPyA==",
+      "dev": true
+    },
+    "node_modules/@types/command-line-usage": {
+      "version": "5.0.2",
+      "resolved": "https://registry.npmjs.org/@types/command-line-usage/-/command-line-usage-5.0.2.tgz",
+      "integrity": "sha512-n7RlEEJ+4x4TS7ZQddTmNSxP+zziEG0TNsMfiRIxcIVXt71ENJ9ojeXmGO3wPoTdn7pJcU2xc3CJYMktNT6DPg==",
+      "dev": true
+    },
+    "node_modules/@types/node": {
+      "version": "18.14.5",
+      "resolved": "https://registry.npmjs.org/@types/node/-/node-18.14.5.tgz",
+      "integrity": "sha512-CRT4tMK/DHYhw1fcCEBwME9CSaZNclxfzVMe7GsO6ULSwsttbj70wSiX6rZdIjGblu93sTJxLdhNIT85KKI7Qw==",
+      "dev": true
+    },
+    "node_modules/@types/pad-left": {
+      "version": "2.1.1",
+      "resolved": "https://registry.npmjs.org/@types/pad-left/-/pad-left-2.1.1.tgz",
+      "integrity": "sha512-Xd22WCRBydkGSApl5Bw0PhAOHKSVjNL3E3AwzKaps96IMraPqy5BvZIsBVK6JLwdybUzjHnuWVwpDd0JjTfHXA==",
+      "dev": true
+    },
+    "node_modules/ansi-regex": {
+      "version": "6.0.1",
+      "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.0.1.tgz",
+      "integrity": "sha512-n5M855fKb2SsfMIiFFoVrABHJC8QtHwVx+mHWP3QcEqBHYienj5dHSgjbxtC0WEZXYt4wcD6zrQElDPhFuZgfA==",
+      "engines": {
+        "node": ">=12"
+      },
+      "funding": {
+        "url": "https://github.com/chalk/ansi-regex?sponsor=1"
+      }
+    },
+    "node_modules/ansi-styles": {
+      "version": "3.2.1",
+      "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz",
+      "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==",
+      "dev": true,
+      "dependencies": {
+        "color-convert": "^1.9.0"
+      },
+      "engines": {
+        "node": ">=4"
+      }
+    },
+    "node_modules/apache-arrow": {
+      "version": "12.0.1",
+      "resolved": "https://registry.npmjs.org/apache-arrow/-/apache-arrow-12.0.1.tgz",
+      "integrity": "sha512-g17ARsc/KEAzViy8PEFsDBlL4ZLx3BesgQCplDLgUWtY0aFWNdEmfaZsbbXVRDfQ21D7vbUKtu0ZWNgcbxDrig==",
+      "dev": true,
+      "dependencies": {
+        "@types/command-line-args": "5.2.0",
+        "@types/command-line-usage": "5.0.2",
+        "@types/node": "18.14.5",
+        "@types/pad-left": "2.1.1",
+        "command-line-args": "5.2.1",
+        "command-line-usage": "6.1.3",
+        "flatbuffers": "23.3.3",
+        "json-bignum": "^0.0.3",
+        "pad-left": "^2.1.0",
+        "tslib": "^2.5.0"
+      },
+      "bin": {
+        "arrow2csv": "bin/arrow2csv.js"
+      }
+    },
+    "node_modules/array-back": {
+      "version": "3.1.0",
+      "resolved": "https://registry.npmjs.org/array-back/-/array-back-3.1.0.tgz",
+      "integrity": "sha512-TkuxA4UCOvxuDK6NZYXCalszEzj+TLszyASooky+i742l9TqsOdYCMJJupxRic61hwquNtppB3hgcuq9SVSH1Q==",
+      "dev": true,
+      "engines": {
+        "node": ">=6"
+      }
+    },
+    "node_modules/balanced-match": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz",
+      "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw=="
+    },
+    "node_modules/brace-expansion": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz",
+      "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==",
+      "dependencies": {
+        "balanced-match": "^1.0.0"
+      }
+    },
+    "node_modules/chalk": {
+      "version": "2.4.2",
+      "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz",
+      "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==",
+      "dev": true,
+      "dependencies": {
+        "ansi-styles": "^3.2.1",
+        "escape-string-regexp": "^1.0.5",
+        "supports-color": "^5.3.0"
+      },
+      "engines": {
+        "node": ">=4"
+      }
+    },
+    "node_modules/color-convert": {
+      "version": "1.9.3",
+      "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz",
+      "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==",
+      "dev": true,
+      "dependencies": {
+        "color-name": "1.1.3"
+      }
+    },
+    "node_modules/color-name": {
+      "version": "1.1.3",
+      "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz",
+      "integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==",
+      "dev": true
+    },
+    "node_modules/command-line-args": {
+      "version": "5.2.1",
+      "resolved": "https://registry.npmjs.org/command-line-args/-/command-line-args-5.2.1.tgz",
+      "integrity": "sha512-H4UfQhZyakIjC74I9d34fGYDwk3XpSr17QhEd0Q3I9Xq1CETHo4Hcuo87WyWHpAF1aSLjLRf5lD9ZGX2qStUvg==",
+      "dev": true,
+      "dependencies": {
+        "array-back": "^3.1.0",
+        "find-replace": "^3.0.0",
+        "lodash.camelcase": "^4.3.0",
+        "typical": "^4.0.0"
+      },
+      "engines": {
+        "node": ">=4.0.0"
+      }
+    },
+    "node_modules/command-line-usage": {
+      "version": "6.1.3",
+      "resolved": "https://registry.npmjs.org/command-line-usage/-/command-line-usage-6.1.3.tgz",
+      "integrity": "sha512-sH5ZSPr+7UStsloltmDh7Ce5fb8XPlHyoPzTpyyMuYCtervL65+ubVZ6Q61cFtFl62UyJlc8/JwERRbAFPUqgw==",
+      "dev": true,
+      "dependencies": {
+        "array-back": "^4.0.2",
+        "chalk": "^2.4.2",
+        "table-layout": "^1.0.2",
+        "typical": "^5.2.0"
+      },
+      "engines": {
+        "node": ">=8.0.0"
+      }
+    },
+    "node_modules/command-line-usage/node_modules/array-back": {
+      "version": "4.0.2",
+      "resolved": "https://registry.npmjs.org/array-back/-/array-back-4.0.2.tgz",
+      "integrity": "sha512-NbdMezxqf94cnNfWLL7V/im0Ub+Anbb0IoZhvzie8+4HJ4nMQuzHuy49FkGYCJK2yAloZ3meiB6AVMClbrI1vg==",
+      "dev": true,
+      "engines": {
+        "node": ">=8"
+      }
+    },
+    "node_modules/command-line-usage/node_modules/typical": {
+      "version": "5.2.0",
+      "resolved": "https://registry.npmjs.org/typical/-/typical-5.2.0.tgz",
+      "integrity": "sha512-dvdQgNDNJo+8B2uBQoqdb11eUCE1JQXhvjC/CZtgvZseVd5TYMXnq0+vuUemXbd/Se29cTaUuPX3YIc2xgbvIg==",
+      "dev": true,
+      "engines": {
+        "node": ">=8"
+      }
+    },
+    "node_modules/cross-spawn": {
+      "version": "7.0.3",
+      "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz",
+      "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==",
+      "dependencies": {
+        "path-key": "^3.1.0",
+        "shebang-command": "^2.0.0",
+        "which": "^2.0.1"
+      },
+      "engines": {
+        "node": ">= 8"
+      }
+    },
+    "node_modules/deep-extend": {
+      "version": "0.6.0",
+      "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz",
+      "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==",
+      "dev": true,
+      "engines": {
+        "node": ">=4.0.0"
+      }
+    },
+    "node_modules/eastasianwidth": {
+      "version": "0.2.0",
+      "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz",
+      "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA=="
+    },
+    "node_modules/emoji-regex": {
+      "version": "9.2.2",
+      "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz",
+      "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg=="
+    },
+    "node_modules/escape-string-regexp": {
+      "version": "1.0.5",
+      "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz",
+      "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==",
+      "dev": true,
+      "engines": {
+        "node": ">=0.8.0"
+      }
+    },
+    "node_modules/find-replace": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/find-replace/-/find-replace-3.0.0.tgz",
+      "integrity": "sha512-6Tb2myMioCAgv5kfvP5/PkZZ/ntTpVK39fHY7WkWBgvbeE+VHd/tZuZ4mrC+bxh4cfOZeYKVPaJIZtZXV7GNCQ==",
+      "dev": true,
+      "dependencies": {
+        "array-back": "^3.0.1"
+      },
+      "engines": {
+        "node": ">=4.0.0"
+      }
+    },
+    "node_modules/flatbuffers": {
+      "version": "23.3.3",
+      "resolved": "https://registry.npmjs.org/flatbuffers/-/flatbuffers-23.3.3.tgz",
+      "integrity": "sha512-jmreOaAT1t55keaf+Z259Tvh8tR/Srry9K8dgCgvizhKSEr6gLGgaOJI2WFL5fkOpGOGRZwxUrlFn0GCmXUy6g==",
+      "dev": true
+    },
+    "node_modules/foreground-child": {
+      "version": "3.1.1",
+      "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.1.1.tgz",
+      "integrity": "sha512-TMKDUnIte6bfb5nWv7V/caI169OHgvwjb7V4WkeUvbQQdjr5rWKqHFiKWb/fcOwB+CzBT+qbWjvj+DVwRskpIg==",
+      "dependencies": {
+        "cross-spawn": "^7.0.0",
+        "signal-exit": "^4.0.1"
+      },
+      "engines": {
+        "node": ">=14"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/isaacs"
+      }
+    },
+    "node_modules/fs": {
+      "version": "0.0.1-security",
+      "resolved": "https://registry.npmjs.org/fs/-/fs-0.0.1-security.tgz",
+      "integrity": "sha512-3XY9e1pP0CVEUCdj5BmfIZxRBTSDycnbqhIOGec9QYtmVH2fbLpj86CFWkrNOkt/Fvty4KZG5lTglL9j/gJ87w=="
+    },
+    "node_modules/glob": {
+      "version": "10.2.7",
+      "resolved": "https://registry.npmjs.org/glob/-/glob-10.2.7.tgz",
+      "integrity": "sha512-jTKehsravOJo8IJxUGfZILnkvVJM/MOfHRs8QcXolVef2zNI9Tqyy5+SeuOAZd3upViEZQLyFpQhYiHLrMUNmA==",
+      "dependencies": {
+        "foreground-child": "^3.1.0",
+        "jackspeak": "^2.0.3",
+        "minimatch": "^9.0.1",
+        "minipass": "^5.0.0 || ^6.0.2",
+        "path-scurry": "^1.7.0"
+      },
+      "bin": {
+        "glob": "dist/cjs/src/bin.js"
+      },
+      "engines": {
+        "node": ">=16 || 14 >=14.17"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/isaacs"
+      }
+    },
+    "node_modules/has-flag": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz",
+      "integrity": "sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==",
+      "dev": true,
+      "engines": {
+        "node": ">=4"
+      }
+    },
+    "node_modules/inherits": {
+      "version": "2.0.3",
+      "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz",
+      "integrity": "sha512-x00IRNXNy63jwGkJmzPigoySHbaqpNuzKbBOmzK+g2OdZpQ9w+sxCN+VSB3ja7IAge2OP2qpfxTjeNcyjmW1uw=="
+    },
+    "node_modules/is-fullwidth-code-point": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz",
+      "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==",
+      "engines": {
+        "node": ">=8"
+      }
+    },
+    "node_modules/isexe": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz",
+      "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw=="
+    },
+    "node_modules/jackspeak": {
+      "version": "2.2.1",
+      "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-2.2.1.tgz",
+      "integrity": "sha512-MXbxovZ/Pm42f6cDIDkl3xpwv1AGwObKwfmjs2nQePiy85tP3fatofl3FC1aBsOtP/6fq5SbtgHwWcMsLP+bDw==",
+      "dependencies": {
+        "@isaacs/cliui": "^8.0.2"
+      },
+      "engines": {
+        "node": ">=14"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/isaacs"
+      },
+      "optionalDependencies": {
+        "@pkgjs/parseargs": "^0.11.0"
+      }
+    },
+    "node_modules/json-bignum": {
+      "version": "0.0.3",
+      "resolved": "https://registry.npmjs.org/json-bignum/-/json-bignum-0.0.3.tgz",
+      "integrity": "sha512-2WHyXj3OfHSgNyuzDbSxI1w2jgw5gkWSWhS7Qg4bWXx1nLk3jnbwfUeS0PSba3IzpTUWdHxBieELUzXRjQB2zg==",
+      "dev": true,
+      "engines": {
+        "node": ">=0.8"
+      }
+    },
+    "node_modules/lodash.camelcase": {
+      "version": "4.3.0",
+      "resolved": "https://registry.npmjs.org/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz",
+      "integrity": "sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA==",
+      "dev": true
+    },
+    "node_modules/lru-cache": {
+      "version": "9.1.2",
+      "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-9.1.2.tgz",
+      "integrity": "sha512-ERJq3FOzJTxBbFjZ7iDs+NiK4VI9Wz+RdrrAB8dio1oV+YvdPzUEE4QNiT2VD51DkIbCYRUUzCRkssXCHqSnKQ==",
+      "engines": {
+        "node": "14 || >=16.14"
+      }
+    },
+    "node_modules/minimatch": {
+      "version": "9.0.1",
+      "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.1.tgz",
+      "integrity": "sha512-0jWhJpD/MdhPXwPuiRkCbfYfSKp2qnn2eOc279qI7f+osl/l+prKSrvhg157zSYvx/1nmgn2NqdT6k2Z7zSH9w==",
+      "dependencies": {
+        "brace-expansion": "^2.0.1"
+      },
+      "engines": {
+        "node": ">=16 || 14 >=14.17"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/isaacs"
+      }
+    },
+    "node_modules/minipass": {
+      "version": "6.0.2",
+      "resolved": "https://registry.npmjs.org/minipass/-/minipass-6.0.2.tgz",
+      "integrity": "sha512-MzWSV5nYVT7mVyWCwn2o7JH13w2TBRmmSqSRCKzTw+lmft9X4z+3wjvs06Tzijo5z4W/kahUCDpRXTF+ZrmF/w==",
+      "engines": {
+        "node": ">=16 || 14 >=14.17"
+      }
+    },
+    "node_modules/pad-left": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/pad-left/-/pad-left-2.1.0.tgz",
+      "integrity": "sha512-HJxs9K9AztdIQIAIa/OIazRAUW/L6B9hbQDxO4X07roW3eo9XqZc2ur9bn1StH9CnbbI9EgvejHQX7CBpCF1QA==",
+      "dev": true,
+      "dependencies": {
+        "repeat-string": "^1.5.4"
+      },
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
+    "node_modules/path": {
+      "version": "0.12.7",
+      "resolved": "https://registry.npmjs.org/path/-/path-0.12.7.tgz",
+      "integrity": "sha512-aXXC6s+1w7otVF9UletFkFcDsJeO7lSZBPUQhtb5O0xJe8LtYhj/GxldoL09bBj9+ZmE2hNoHqQSFMN5fikh4Q==",
+      "dependencies": {
+        "process": "^0.11.1",
+        "util": "^0.10.3"
+      }
+    },
+    "node_modules/path-key": {
+      "version": "3.1.1",
+      "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz",
+      "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==",
+      "engines": {
+        "node": ">=8"
+      }
+    },
+    "node_modules/path-scurry": {
+      "version": "1.9.2",
+      "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-1.9.2.tgz",
+      "integrity": "sha512-qSDLy2aGFPm8i4rsbHd4MNyTcrzHFsLQykrtbuGRknZZCBBVXSv2tSCDN2Cg6Rt/GFRw8GoW9y9Ecw5rIPG1sg==",
+      "dependencies": {
+        "lru-cache": "^9.1.1",
+        "minipass": "^5.0.0 || ^6.0.2"
+      },
+      "engines": {
+        "node": ">=16 || 14 >=14.17"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/isaacs"
+      }
+    },
+    "node_modules/process": {
+      "version": "0.11.10",
+      "resolved": "https://registry.npmjs.org/process/-/process-0.11.10.tgz",
+      "integrity": "sha512-cdGef/drWFoydD1JsMzuFf8100nZl+GT+yacc2bEced5f9Rjk4z+WtFUTBu9PhOi9j/jfmBPu0mMEY4wIdAF8A==",
+      "engines": {
+        "node": ">= 0.6.0"
+      }
+    },
+    "node_modules/reduce-flatten": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/reduce-flatten/-/reduce-flatten-2.0.0.tgz",
+      "integrity": "sha512-EJ4UNY/U1t2P/2k6oqotuX2Cc3T6nxJwsM0N0asT7dhrtH1ltUxDn4NalSYmPE2rCkVpcf/X6R0wDwcFpzhd4w==",
+      "dev": true,
+      "engines": {
+        "node": ">=6"
+      }
+    },
+    "node_modules/repeat-string": {
+      "version": "1.6.1",
+      "resolved": "https://registry.npmjs.org/repeat-string/-/repeat-string-1.6.1.tgz",
+      "integrity": "sha512-PV0dzCYDNfRi1jCDbJzpW7jNNDRuCOG/jI5ctQcGKt/clZD+YcPS3yIlWuTJMmESC8aevCFmWJy5wjAFgNqN6w==",
+      "dev": true,
+      "engines": {
+        "node": ">=0.10"
+      }
+    },
+    "node_modules/shebang-command": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz",
+      "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==",
+      "dependencies": {
+        "shebang-regex": "^3.0.0"
+      },
+      "engines": {
+        "node": ">=8"
+      }
+    },
+    "node_modules/shebang-regex": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz",
+      "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==",
+      "engines": {
+        "node": ">=8"
+      }
+    },
+    "node_modules/signal-exit": {
+      "version": "4.0.2",
+      "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.0.2.tgz",
+      "integrity": "sha512-MY2/qGx4enyjprQnFaZsHib3Yadh3IXyV2C321GY0pjGfVBu4un0uDJkwgdxqO+Rdx8JMT8IfJIRwbYVz3Ob3Q==",
+      "engines": {
+        "node": ">=14"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/isaacs"
+      }
+    },
+    "node_modules/string-width": {
+      "version": "5.1.2",
+      "resolved": "https://registry.npmjs.org/string-width/-/string-width-5.1.2.tgz",
+      "integrity": "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==",
+      "dependencies": {
+        "eastasianwidth": "^0.2.0",
+        "emoji-regex": "^9.2.2",
+        "strip-ansi": "^7.0.1"
+      },
+      "engines": {
+        "node": ">=12"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
+      }
+    },
+    "node_modules/string-width-cjs": {
+      "name": "string-width",
+      "version": "4.2.3",
+      "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz",
+      "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==",
+      "dependencies": {
+        "emoji-regex": "^8.0.0",
+        "is-fullwidth-code-point": "^3.0.0",
+        "strip-ansi": "^6.0.1"
+      },
+      "engines": {
+        "node": ">=8"
+      }
+    },
+    "node_modules/string-width-cjs/node_modules/ansi-regex": {
+      "version": "5.0.1",
+      "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz",
+      "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==",
+      "engines": {
+        "node": ">=8"
+      }
+    },
+    "node_modules/string-width-cjs/node_modules/emoji-regex": {
+      "version": "8.0.0",
+      "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz",
+      "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A=="
+    },
+    "node_modules/string-width-cjs/node_modules/strip-ansi": {
+      "version": "6.0.1",
+      "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz",
+      "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==",
+      "dependencies": {
+        "ansi-regex": "^5.0.1"
+      },
+      "engines": {
+        "node": ">=8"
+      }
+    },
+    "node_modules/strip-ansi": {
+      "version": "7.1.0",
+      "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.1.0.tgz",
+      "integrity": "sha512-iq6eVVI64nQQTRYq2KtEg2d2uU7LElhTJwsH4YzIHZshxlgZms/wIc4VoDQTlG/IvVIrBKG06CrZnp0qv7hkcQ==",
+      "dependencies": {
+        "ansi-regex": "^6.0.1"
+      },
+      "engines": {
+        "node": ">=12"
+      },
+      "funding": {
+        "url": "https://github.com/chalk/strip-ansi?sponsor=1"
+      }
+    },
+    "node_modules/strip-ansi-cjs": {
+      "name": "strip-ansi",
+      "version": "6.0.1",
+      "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz",
+      "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==",
+      "dependencies": {
+        "ansi-regex": "^5.0.1"
+      },
+      "engines": {
+        "node": ">=8"
+      }
+    },
+    "node_modules/strip-ansi-cjs/node_modules/ansi-regex": {
+      "version": "5.0.1",
+      "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz",
+      "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==",
+      "engines": {
+        "node": ">=8"
+      }
+    },
+    "node_modules/supports-color": {
+      "version": "5.5.0",
+      "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz",
+      "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==",
+      "dev": true,
+      "dependencies": {
+        "has-flag": "^3.0.0"
+      },
+      "engines": {
+        "node": ">=4"
+      }
+    },
+    "node_modules/table-layout": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/table-layout/-/table-layout-1.0.2.tgz",
+      "integrity": "sha512-qd/R7n5rQTRFi+Zf2sk5XVVd9UQl6ZkduPFC3S7WEGJAmetDTjY3qPN50eSKzwuzEyQKy5TN2TiZdkIjos2L6A==",
+      "dev": true,
+      "dependencies": {
+        "array-back": "^4.0.1",
+        "deep-extend": "~0.6.0",
+        "typical": "^5.2.0",
+        "wordwrapjs": "^4.0.0"
+      },
+      "engines": {
+        "node": ">=8.0.0"
+      }
+    },
+    "node_modules/table-layout/node_modules/array-back": {
+      "version": "4.0.2",
+      "resolved": "https://registry.npmjs.org/array-back/-/array-back-4.0.2.tgz",
+      "integrity": "sha512-NbdMezxqf94cnNfWLL7V/im0Ub+Anbb0IoZhvzie8+4HJ4nMQuzHuy49FkGYCJK2yAloZ3meiB6AVMClbrI1vg==",
+      "dev": true,
+      "engines": {
+        "node": ">=8"
+      }
+    },
+    "node_modules/table-layout/node_modules/typical": {
+      "version": "5.2.0",
+      "resolved": "https://registry.npmjs.org/typical/-/typical-5.2.0.tgz",
+      "integrity": "sha512-dvdQgNDNJo+8B2uBQoqdb11eUCE1JQXhvjC/CZtgvZseVd5TYMXnq0+vuUemXbd/Se29cTaUuPX3YIc2xgbvIg==",
+      "dev": true,
+      "engines": {
+        "node": ">=8"
+      }
+    },
+    "node_modules/tslib": {
+      "version": "2.5.3",
+      "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.3.tgz",
+      "integrity": "sha512-mSxlJJwl3BMEQCUNnxXBU9jP4JBktcEGhURcPR6VQVlnP0FdDEsIaz0C35dXNGLyRfrATNofF0F5p2KPxQgB+w==",
+      "dev": true
+    },
+    "node_modules/typical": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/typical/-/typical-4.0.0.tgz",
+      "integrity": "sha512-VAH4IvQ7BDFYglMd7BPRDfLgxZZX4O4TFcRDA6EN5X7erNJJq+McIEp8np9aVtxrCJ6qx4GTYVfOWNjcqwZgRw==",
+      "dev": true,
+      "engines": {
+        "node": ">=8"
+      }
+    },
+    "node_modules/util": {
+      "version": "0.10.4",
+      "resolved": "https://registry.npmjs.org/util/-/util-0.10.4.tgz",
+      "integrity": "sha512-0Pm9hTQ3se5ll1XihRic3FDIku70C+iHUdT/W926rSgHV5QgXsYbKZN8MSC3tJtSkhuROzvsQjAaFENRXr+19A==",
+      "dependencies": {
+        "inherits": "2.0.3"
+      }
+    },
+    "node_modules/vectordb": {
+      "version": "0.1.9",
+      "resolved": "https://gitpkg.now.sh/lancedb/lancedb/node?main",
+      "integrity": "sha512-c5GLOueasILhiymXdRmr9qeQOp0u1Jihqb2ztxU+NWrR0zgaWD5L03Hjh45kcJRQSLi7ZpmJ4Waxqi/+CWtQlw==",
+      "dev": true,
+      "license": "Apache-2.0",
+      "dependencies": {
+        "@apache-arrow/ts": "^12.0.0",
+        "apache-arrow": "^12.0.0"
+      }
+    },
+    "node_modules/which": {
+      "version": "2.0.2",
+      "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz",
+      "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==",
+      "dependencies": {
+        "isexe": "^2.0.0"
+      },
+      "bin": {
+        "node-which": "bin/node-which"
+      },
+      "engines": {
+        "node": ">= 8"
+      }
+    },
+    "node_modules/wordwrapjs": {
+      "version": "4.0.1",
+      "resolved": "https://registry.npmjs.org/wordwrapjs/-/wordwrapjs-4.0.1.tgz",
+      "integrity": "sha512-kKlNACbvHrkpIw6oPeYDSmdCTu2hdMHoyXLTcUKala++lx5Y+wjJ/e474Jqv5abnVmwxw08DiTuHmw69lJGksA==",
+      "dev": true,
+      "dependencies": {
+        "reduce-flatten": "^2.0.0",
+        "typical": "^5.2.0"
+      },
+      "engines": {
+        "node": ">=8.0.0"
+      }
+    },
+    "node_modules/wordwrapjs/node_modules/typical": {
+      "version": "5.2.0",
+      "resolved": "https://registry.npmjs.org/typical/-/typical-5.2.0.tgz",
+      "integrity": "sha512-dvdQgNDNJo+8B2uBQoqdb11eUCE1JQXhvjC/CZtgvZseVd5TYMXnq0+vuUemXbd/Se29cTaUuPX3YIc2xgbvIg==",
+      "dev": true,
+      "engines": {
+        "node": ">=8"
+      }
+    },
+    "node_modules/wrap-ansi": {
+      "version": "8.1.0",
+      "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-8.1.0.tgz",
+      "integrity": "sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==",
+      "dependencies": {
+        "ansi-styles": "^6.1.0",
+        "string-width": "^5.0.1",
+        "strip-ansi": "^7.0.1"
+      },
+      "engines": {
+        "node": ">=12"
+      },
+      "funding": {
+        "url": "https://github.com/chalk/wrap-ansi?sponsor=1"
+      }
+    },
+    "node_modules/wrap-ansi-cjs": {
+      "name": "wrap-ansi",
+      "version": "7.0.0",
+      "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz",
+      "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==",
+      "dependencies": {
+        "ansi-styles": "^4.0.0",
+        "string-width": "^4.1.0",
+        "strip-ansi": "^6.0.0"
+      },
+      "engines": {
+        "node": ">=10"
+      },
+      "funding": {
+        "url": "https://github.com/chalk/wrap-ansi?sponsor=1"
+      }
+    },
+    "node_modules/wrap-ansi-cjs/node_modules/ansi-regex": {
+      "version": "5.0.1",
+      "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz",
+      "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==",
+      "engines": {
+        "node": ">=8"
+      }
+    },
+    "node_modules/wrap-ansi-cjs/node_modules/ansi-styles": {
+      "version": "4.3.0",
+      "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz",
+      "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==",
+      "dependencies": {
+        "color-convert": "^2.0.1"
+      },
+      "engines": {
+        "node": ">=8"
+      },
+      "funding": {
+        "url": "https://github.com/chalk/ansi-styles?sponsor=1"
+      }
+    },
+    "node_modules/wrap-ansi-cjs/node_modules/color-convert": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
+      "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
+      "dependencies": {
+        "color-name": "~1.1.4"
+      },
+      "engines": {
+        "node": ">=7.0.0"
+      }
+    },
+    "node_modules/wrap-ansi-cjs/node_modules/color-name": {
+      "version": "1.1.4",
+      "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
+      "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA=="
+    },
+    "node_modules/wrap-ansi-cjs/node_modules/emoji-regex": {
+      "version": "8.0.0",
+      "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz",
+      "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A=="
+    },
+    "node_modules/wrap-ansi-cjs/node_modules/string-width": {
+      "version": "4.2.3",
+      "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz",
+      "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==",
+      "dependencies": {
+        "emoji-regex": "^8.0.0",
+        "is-fullwidth-code-point": "^3.0.0",
+        "strip-ansi": "^6.0.1"
+      },
+      "engines": {
+        "node": ">=8"
+      }
+    },
+    "node_modules/wrap-ansi-cjs/node_modules/strip-ansi": {
+      "version": "6.0.1",
+      "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz",
+      "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==",
+      "dependencies": {
+        "ansi-regex": "^5.0.1"
+      },
+      "engines": {
+        "node": ">=8"
+      }
+    },
+    "node_modules/wrap-ansi/node_modules/ansi-styles": {
+      "version": "6.2.1",
+      "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.1.tgz",
+      "integrity": "sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug==",
+      "engines": {
+        "node": ">=12"
+      },
+      "funding": {
+        "url": "https://github.com/chalk/ansi-styles?sponsor=1"
+      }
+    }
+  }
+}
--- a/docs/test/package.json
+++ b/docs/test/package.json
@@ -0,0 +1,15 @@
+{
+  "name": "lancedb-docs-test",
+  "version": "1.0.0",
+  "description": "",
+  "author": "",
+  "license": "ISC",
+  "devDependencies": {
+    "vectordb": "https://gitpkg.now.sh/lancedb/lancedb/node?main"
+  },
+  "dependencies": {
+    "fs": "^0.0.1-security",
+    "glob": "^10.2.7",
+    "path": "^0.12.7"
+  }
+}
--- a/docs/test/requirements.txt
+++ b/docs/test/requirements.txt
@@ -0,0 +1,5 @@
+lancedb @ git+https://github.com/lancedb/lancedb.git#egg=subdir&subdirectory=python
+numpy
+pandas
+pylance
+duckdb