feat: dynamodb commit store support (#1410)

This allows users to specify URIs like:

```
s3+ddb://my_bucket/path?ddbTableName=myCommitTable
```

and it will support concurrent writes in S3.

* [x] Add dynamodb integration tests
* [x] Add modifications to get it working in Python sync API
* [x] Added section in documentation describing how to configure.

Closes #534

---------

Co-authored-by: universalmind303 <cory.grinstead@gmail.com>

.pre-commit-config.yaml

@@ -14,7 +14,7 @@ repos:
   hooks:
     - id: local-biome-check
       name: biome check
-      entry: npx @biomejs/biome@1.7.3 check --config-path nodejs/biome.json nodejs/
+      entry: npx @biomejs/biome@1.8.3 check --config-path nodejs/biome.json nodejs/
       language: system
       types: [text]
       files: "nodejs/.*"

docs/src/guides/storage.md

@@ -265,6 +265,108 @@ For **read-only access**, LanceDB will need a policy such as:
 }
 ```
 
+#### DynamoDB Commit Store for concurrent writes
+
+By default, S3 does not support concurrent writes. Having two or more processes
+writing to the same table at the same time can lead to data corruption. This is
+because S3, unlike other object stores, does not have any atomic put or copy
+operation.
+
+To enable concurrent writes, you can configure LanceDB to use a DynamoDB table
+as a commit store. This table will be used to coordinate writes between
+different processes. To enable this feature, you must modify your connection
+URI to use the `s3+ddb` scheme and add a query parameter `ddbTableName` with the
+name of the table to use.
+
+=== "Python"
+
+    ```python
+    import lancedb
+    db = await lancedb.connect_async(
+        "s3+ddb://bucket/path?ddbTableName=my-dynamodb-table",
+    )
+    ```
+
+=== "JavaScript"
+
+    ```javascript
+    const lancedb = require("lancedb");
+
+    const db = await lancedb.connect(
+        "s3+ddb://bucket/path?ddbTableName=my-dynamodb-table",
+    );
+    ```
+
+The DynamoDB table must be created with the following schema:
+
+- Hash key: `base_uri` (string)
+- Range key: `version` (number)
+
+You can create this programmatically with:
+
+=== "Python"
+
+    <!-- skip-test -->
+    ```python
+    import boto3
+
+    dynamodb = boto3.client("dynamodb")
+    table = dynamodb.create_table(
+        TableName=table_name,
+        KeySchema=[
+            {"AttributeName": "base_uri", "KeyType": "HASH"},
+            {"AttributeName": "version", "KeyType": "RANGE"},
+        ],
+        AttributeDefinitions=[
+            {"AttributeName": "base_uri", "AttributeType": "S"},
+            {"AttributeName": "version", "AttributeType": "N"},
+        ],
+        ProvisionedThroughput={"ReadCapacityUnits": 1, "WriteCapacityUnits": 1},
+    )
+    ```
+
+=== "JavaScript"
+
+    <!-- skip-test -->
+    ```javascript
+    import {
+      CreateTableCommand,
+      DynamoDBClient,
+    } from "@aws-sdk/client-dynamodb";
+
+    const dynamodb = new DynamoDBClient({
+      region: CONFIG.awsRegion,
+      credentials: {
+        accessKeyId: CONFIG.awsAccessKeyId,
+        secretAccessKey: CONFIG.awsSecretAccessKey,
+      },
+      endpoint: CONFIG.awsEndpoint,
+    });
+    const command = new CreateTableCommand({
+      TableName: table_name,
+      AttributeDefinitions: [
+        {
+          AttributeName: "base_uri",
+          AttributeType: "S",
+        },
+        {
+          AttributeName: "version",
+          AttributeType: "N",
+        },
+      ],
+      KeySchema: [
+        { AttributeName: "base_uri", KeyType: "HASH" },
+        { AttributeName: "version", KeyType: "RANGE" },
+      ],
+      ProvisionedThroughput: {
+        ReadCapacityUnits: 1,
+        WriteCapacityUnits: 1,
+      },
+    });
+    await client.send(command);
+    ```
+
+
 #### S3-compatible stores
 
 LanceDB can also connect to S3-compatible stores, such as MinIO. To do so, you must specify both region and endpoint:

nodejs/__test__/s3_integration.test.ts

@@ -14,6 +14,11 @@
 
 /* eslint-disable @typescript-eslint/naming-convention */
 
+import {
+  CreateTableCommand,
+  DeleteTableCommand,
+  DynamoDBClient,
+} from "@aws-sdk/client-dynamodb";
 import {
   CreateKeyCommand,
   KMSClient,
@@ -38,6 +43,7 @@ const CONFIG = {
   awsAccessKeyId: "ACCESSKEY",
   awsSecretAccessKey: "SECRETKEY",
   awsEndpoint: "http://127.0.0.1:4566",
+  dynamodbEndpoint: "http://127.0.0.1:4566",
   awsRegion: "us-east-1",
 };
 
@@ -66,7 +72,6 @@ class S3Bucket {
     } catch {
       // It's fine if the bucket doesn't exist
     }
-    // biome-ignore lint/style/useNamingConvention: we dont control s3's api
     await client.send(new CreateBucketCommand({ Bucket: name }));
     return new S3Bucket(name);
   }
@@ -79,32 +84,27 @@ class S3Bucket {
   static async deleteBucket(client: S3Client, name: string) {
     // Must delete all objects before we can delete the bucket
     const objects = await client.send(
-      // biome-ignore lint/style/useNamingConvention: we dont control s3's api
       new ListObjectsV2Command({ Bucket: name }),
     );
     if (objects.Contents) {
       for (const object of objects.Contents) {
         await client.send(
-          // biome-ignore lint/style/useNamingConvention: we dont control s3's api
           new DeleteObjectCommand({ Bucket: name, Key: object.Key }),
         );
       }
     }
 
-    // biome-ignore lint/style/useNamingConvention: we dont control s3's api
     await client.send(new DeleteBucketCommand({ Bucket: name }));
   }
 
   public async assertAllEncrypted(path: string, keyId: string) {
     const client = S3Bucket.s3Client();
     const objects = await client.send(
-      // biome-ignore lint/style/useNamingConvention: we dont control s3's api
       new ListObjectsV2Command({ Bucket: this.name, Prefix: path }),
     );
     if (objects.Contents) {
       for (const object of objects.Contents) {
         const metadata = await client.send(
-          // biome-ignore lint/style/useNamingConvention: we dont control s3's api
           new HeadObjectCommand({ Bucket: this.name, Key: object.Key }),
         );
         expect(metadata.ServerSideEncryption).toBe("aws:kms");
@@ -143,7 +143,6 @@ class KmsKey {
 
   public async delete() {
     const client = KmsKey.kmsClient();
-    // biome-ignore lint/style/useNamingConvention: we dont control s3's api
     await client.send(new ScheduleKeyDeletionCommand({ KeyId: this.keyId }));
   }
 }
@@ -224,3 +223,91 @@ maybeDescribe("storage_options", () => {
     await bucket.assertAllEncrypted("test/table2.lance", kmsKey.keyId);
   });
 });
+
+class DynamoDBCommitTable {
+  name: string;
+  constructor(name: string) {
+    this.name = name;
+  }
+
+  static dynamoClient() {
+    return new DynamoDBClient({
+      region: CONFIG.awsRegion,
+      credentials: {
+        accessKeyId: CONFIG.awsAccessKeyId,
+        secretAccessKey: CONFIG.awsSecretAccessKey,
+      },
+      endpoint: CONFIG.awsEndpoint,
+    });
+  }
+
+  public static async create(name: string): Promise<DynamoDBCommitTable> {
+    const client = DynamoDBCommitTable.dynamoClient();
+    const command = new CreateTableCommand({
+      TableName: name,
+      AttributeDefinitions: [
+        {
+          AttributeName: "base_uri",
+          AttributeType: "S",
+        },
+        {
+          AttributeName: "version",
+          AttributeType: "N",
+        },
+      ],
+      KeySchema: [
+        { AttributeName: "base_uri", KeyType: "HASH" },
+        { AttributeName: "version", KeyType: "RANGE" },
+      ],
+      ProvisionedThroughput: {
+        ReadCapacityUnits: 1,
+        WriteCapacityUnits: 1,
+      },
+    });
+    await client.send(command);
+    return new DynamoDBCommitTable(name);
+  }
+
+  public async delete() {
+    const client = DynamoDBCommitTable.dynamoClient();
+    await client.send(new DeleteTableCommand({ TableName: this.name }));
+  }
+}
+
+maybeDescribe("DynamoDB Lock", () => {
+  let bucket: S3Bucket;
+  let commitTable: DynamoDBCommitTable;
+
+  beforeAll(async () => {
+    bucket = await S3Bucket.create("lancedb2");
+    commitTable = await DynamoDBCommitTable.create("commitTable");
+  });
+
+  afterAll(async () => {
+    await commitTable.delete();
+    await bucket.delete();
+  });
+
+  it("can be used to configure a DynamoDB table for commit log", async () => {
+    const uri = `s3+ddb://${bucket.name}/test?ddbTableName=${commitTable.name}`;
+    const db = await connect(uri, {
+      storageOptions: CONFIG,
+      readConsistencyInterval: 0,
+    });
+
+    const table = await db.createTable("test", [{ a: 1, b: 2 }]);
+
+    // 5 concurrent appends
+    const futs = Array.from({ length: 5 }, async () => {
+      // Open a table so each append has a separate table reference. Otherwise
+      // they will share the same table reference and the internal ReadWriteLock
+      // will prevent any real concurrency.
+      const table = await db.openTable("test");
+      await table.add([{ a: 2, b: 3 }]);
+    });
+    await Promise.all(futs);
+
+    const rowCount = await table.countRows();
+    expect(rowCount).toBe(6);
+  });
+});

nodejs/biome.json

@@ -1,5 +1,5 @@
 {
-  "$schema": "https://biomejs.dev/schemas/1.7.3/schema.json",
+  "$schema": "https://biomejs.dev/schemas/1.8.3/schema.json",
   "organizeImports": {
     "enabled": true
   },
@@ -100,6 +100,16 @@
     "globals": []
   },
   "overrides": [
+    {
+      "include": ["__test__/s3_integration.test.ts"],
+      "linter": {
+        "rules": {
+          "style": {
+            "useNamingConvention": "off"
+          }
+        }
+      }
+    },
     {
       "include": [
         "**/*.ts",

nodejs/lancedb/remote/client.ts

@@ -55,7 +55,7 @@ export class RestfulLanceDBClient {
       return axios.create({
         baseURL: this.url,
         headers: {
-          // biome-ignore lint/style/useNamingConvention: external api
+          // biome-ignore lint: external API
           Authorization: `Bearer ${this.#apiKey}`,
         },
         transformResponse: decodeErrorData,

nodejs/package.json

@@ -34,6 +34,7 @@
   "devDependencies": {
     "@aws-sdk/client-kms": "^3.33.0",
     "@aws-sdk/client-s3": "^3.33.0",
+    "@aws-sdk/client-dynamodb": "^3.33.0",
     "@biomejs/biome": "^1.7.3",
     "@jest/globals": "^29.7.0",
     "@napi-rs/cli": "^2.18.0",
@@ -68,7 +69,7 @@
     "lint-ci": "biome ci .",
     "docs": "typedoc --plugin typedoc-plugin-markdown --out ../docs/src/js lancedb/index.ts",
     "lint": "biome check . && biome format .",
-    "lint-fix": "biome check --apply-unsafe  . && biome format --write .",
+    "lint-fix": "biome check --write . && biome format --write .",
     "prepublishOnly": "napi prepublish -t npm",
     "test": "jest --verbose",
     "integration": "S3_TEST=1 npm run test",

python/python/lancedb/db.py

@@ -28,12 +28,11 @@ from lancedb.common import data_to_reader, validate_schema
 
 from ._lancedb import connect as lancedb_connect
 from .pydantic import LanceModel
-from .table import AsyncTable, LanceTable, Table, _sanitize_data
+from .table import AsyncTable, LanceTable, Table, _sanitize_data, _table_path
 from .util import (
     fs_from_uri,
     get_uri_location,
     get_uri_scheme,
-    join_uri,
     validate_table_name,
 )
 
@@ -457,16 +456,18 @@ class LanceDBConnection(DBConnection):
             If True, ignore if the table does not exist.
         """
         try:
-            filesystem, path = fs_from_uri(self.uri)
-            table_path = join_uri(path, name + ".lance")
-            filesystem.delete_dir(table_path)
+            table_uri = _table_path(self.uri, name)
+            filesystem, path = fs_from_uri(table_uri)
+            filesystem.delete_dir(path)
         except FileNotFoundError:
             if not ignore_missing:
                 raise
 
     @override
     def drop_database(self):
-        filesystem, path = fs_from_uri(self.uri)
+        dummy_table_uri = _table_path(self.uri, "dummy")
+        uri = dummy_table_uri.removesuffix("dummy.lance")
+        filesystem, path = fs_from_uri(uri)
         filesystem.delete_dir(path)
 
 

python/python/lancedb/table.py

@@ -30,6 +30,7 @@ from typing import (
     Tuple,
     Union,
 )
+from urllib.parse import urlparse
 
 import lance
 import numpy as np
@@ -47,6 +48,7 @@ from .pydantic import LanceModel, model_to_dict
 from .query import AsyncQuery, AsyncVectorQuery, LanceQueryBuilder, Query
 from .util import (
     fs_from_uri,
+    get_uri_scheme,
     inf_vector_column_query,
     join_uri,
     safe_import_pandas,
@@ -208,6 +210,26 @@ def _to_record_batch_generator(
             yield b
 
 
+def _table_path(base: str, table_name: str) -> str:
+    """
+    Get a table path that can be used in PyArrow FS.
+
+    Removes any weird schemes (such as "s3+ddb") and drops any query params.
+    """
+    uri = _table_uri(base, table_name)
+    # Parse as URL
+    parsed = urlparse(uri)
+    # If scheme is s3+ddb, convert to s3
+    if parsed.scheme == "s3+ddb":
+        parsed = parsed._replace(scheme="s3")
+    # Remove query parameters
+    return parsed._replace(query=None).geturl()
+
+
+def _table_uri(base: str, table_name: str) -> str:
+    return join_uri(base, f"{table_name}.lance")
+
+
 class Table(ABC):
     """
     A Table is a collection of Records in a LanceDB Database.
@@ -908,7 +930,7 @@ class LanceTable(Table):
     @classmethod
     def open(cls, db, name, **kwargs):
         tbl = cls(db, name, **kwargs)
-        fs, path = fs_from_uri(tbl._dataset_uri)
+        fs, path = fs_from_uri(tbl._dataset_path)
         file_info = fs.get_file_info(path)
         if file_info.type != pa.fs.FileType.Directory:
             raise FileNotFoundError(
@@ -918,9 +940,14 @@ class LanceTable(Table):
 
         return tbl
 
-    @property
+    @cached_property
+    def _dataset_path(self) -> str:
+        # Cacheable since it's deterministic
+        return _table_path(self._conn.uri, self.name)
+
+    @cached_property
     def _dataset_uri(self) -> str:
-        return join_uri(self._conn.uri, f"{self.name}.lance")
+        return _table_uri(self._conn.uri, self.name)
 
     @property
     def _dataset(self) -> LanceDataset:
@@ -1230,6 +1257,10 @@ class LanceTable(Table):
         )
 
     def _get_fts_index_path(self):
+        if get_uri_scheme(self._dataset_uri) != "file":
+            raise NotImplementedError(
+                "Full-text search is not supported on object stores."
+            )
         return join_uri(self._dataset_uri, "_indices", "tantivy")
 
     def add(

python/python/lancedb/util.py

@@ -139,8 +139,11 @@ def join_uri(base: Union[str, pathlib.Path], *parts: str) -> str:
         # using pathlib for local paths make this windows compatible
         # `get_uri_scheme` returns `file` for windows drive names (e.g. `c:\path`)
         return str(pathlib.Path(base, *parts))
-    # for remote paths, just use os.path.join
-    return "/".join([p.rstrip("/") for p in [base, *parts]])
+    else:
+        # there might be query parameters in the base URI
+        url = urlparse(base)
+        new_path = "/".join([p.rstrip("/") for p in [url.path, *parts]])
+        return url._replace(path=new_path).geturl()
 
 
 def attempt_import_or_raise(module: str, mitigation=None):

python/python/tests/test_s3.py

@@ -13,6 +13,8 @@
 
 import asyncio
 import copy
+from datetime import timedelta
+import threading
 
 import pytest
 import pyarrow as pa
@@ -25,6 +27,7 @@ CONFIG = {
     "aws_access_key_id": "ACCESSKEY",
     "aws_secret_access_key": "SECRETKEY",
     "aws_endpoint": "http://localhost:4566",
+    "dynamodb_endpoint": "http://localhost:4566",
     "aws_region": "us-east-1",
 }
 
@@ -156,3 +159,104 @@ def test_s3_sse(s3_bucket: str, kms_key: str):
         validate_objects_encrypted(s3_bucket, path, kms_key)
 
     asyncio.run(test())
+
+
+@pytest.fixture(scope="module")
+def commit_table():
+    ddb = get_boto3_client("dynamodb", endpoint_url=CONFIG["dynamodb_endpoint"])
+    table_name = "lance-integtest"
+    try:
+        ddb.delete_table(TableName=table_name)
+    except ddb.exceptions.ResourceNotFoundException:
+        pass
+    ddb.create_table(
+        TableName=table_name,
+        KeySchema=[
+            {"AttributeName": "base_uri", "KeyType": "HASH"},
+            {"AttributeName": "version", "KeyType": "RANGE"},
+        ],
+        AttributeDefinitions=[
+            {"AttributeName": "base_uri", "AttributeType": "S"},
+            {"AttributeName": "version", "AttributeType": "N"},
+        ],
+        ProvisionedThroughput={"ReadCapacityUnits": 1, "WriteCapacityUnits": 1},
+    )
+    yield table_name
+    ddb.delete_table(TableName=table_name)
+
+
+@pytest.mark.s3_test
+def test_s3_dynamodb(s3_bucket: str, commit_table: str):
+    storage_options = copy.copy(CONFIG)
+
+    uri = f"s3+ddb://{s3_bucket}/test?ddbTableName={commit_table}"
+    data = pa.table({"x": [1, 2, 3]})
+
+    async def test():
+        db = await lancedb.connect_async(
+            uri,
+            storage_options=storage_options,
+            read_consistency_interval=timedelta(0),
+        )
+
+        table = await db.create_table("test", data)
+
+        # Five concurrent writers
+        async def insert():
+            # independent table refs for true concurrent writes.
+            table = await db.open_table("test")
+            await table.add(data, mode="append")
+
+        tasks = [insert() for _ in range(5)]
+        await asyncio.gather(*tasks)
+
+        row_count = await table.count_rows()
+        assert row_count == 3 * 6
+
+    asyncio.run(test())
+
+
+@pytest.mark.s3_test
+def test_s3_dynamodb_sync(s3_bucket: str, commit_table: str, monkeypatch):
+    # Sync API doesn't support storage_options, so we have to provide as env vars
+    for key, value in CONFIG.items():
+        monkeypatch.setenv(key.upper(), value)
+
+    uri = f"s3+ddb://{s3_bucket}/test2?ddbTableName={commit_table}"
+    data = pa.table({"x": ["a", "b", "c"]})
+
+    db = lancedb.connect(
+        uri,
+        read_consistency_interval=timedelta(0),
+    )
+
+    table = db.create_table("test_ddb_sync", data)
+
+    # Five concurrent writers
+    def insert():
+        table = db.open_table("test_ddb_sync")
+        table.add(data, mode="append")
+
+    threads = []
+    for _ in range(5):
+        thread = threading.Thread(target=insert)
+        threads.append(thread)
+        thread.start()
+
+    for thread in threads:
+        thread.join()
+
+    row_count = table.count_rows()
+    assert row_count == 3 * 6
+
+    # FTS indices should error since they are not supported yet.
+    with pytest.raises(
+        NotImplementedError, match="Full-text search is not supported on object stores."
+    ):
+        table.create_fts_index("x")
+
+    # make sure list tables still works
+    assert db.table_names() == ["test_ddb_sync"]
+    db.drop_table("test_ddb_sync")
+    assert db.table_names() == []
+    db.drop_database()

rust/lancedb/Cargo.toml

@@ -55,10 +55,11 @@ walkdir = "2"
 # For s3 integration tests (dev deps aren't allowed to be optional atm)
 # We pin these because the content-length check breaks with localstack
 # https://github.com/smithy-lang/smithy-rs/releases/tag/release-2024-05-21
+aws-sdk-dynamodb = { version = "=1.23.0" }
 aws-sdk-s3 = { version = "=1.23.0" }
 aws-sdk-kms = { version = "=1.21.0" }
 aws-config = { version = "1.0" }
-aws-smithy-runtime = { version = "=1.3.0" }
+aws-smithy-runtime = { version = "=1.3.1" }
 
 [features]
 default = []

rust/lancedb/tests/object_store_test.rs

@@ -25,7 +25,9 @@ const CONFIG: &[(&str, &str)] = &[
     ("access_key_id", "ACCESS_KEY"),
     ("secret_access_key", "SECRET_KEY"),
     ("endpoint", "http://127.0.0.1:4566"),
+    ("dynamodb_endpoint", "http://127.0.0.1:4566"),
     ("allow_http", "true"),
+    ("region", "us-east-1"),
 ];
 
 async fn aws_config() -> SdkConfig {
@@ -288,3 +290,126 @@ async fn test_encryption() -> Result<()> {
 
     Ok(())
 }
+
+struct DynamoDBCommitTable(String);
+
+impl DynamoDBCommitTable {
+    async fn new(name: &str) -> Self {
+        let config = aws_config().await;
+        let client = aws_sdk_dynamodb::Client::new(&config);
+
+        // In case it wasn't deleted earlier
+        Self::delete_table(client.clone(), name).await;
+        tokio::time::sleep(std::time::Duration::from_millis(200)).await;
+
+        use aws_sdk_dynamodb::types::*;
+
+        client
+            .create_table()
+            .table_name(name)
+            .attribute_definitions(
+                AttributeDefinition::builder()
+                    .attribute_name("base_uri")
+                    .attribute_type(ScalarAttributeType::S)
+                    .build()
+                    .unwrap(),
+            )
+            .attribute_definitions(
+                AttributeDefinition::builder()
+                    .attribute_name("version")
+                    .attribute_type(ScalarAttributeType::N)
+                    .build()
+                    .unwrap(),
+            )
+            .key_schema(
+                KeySchemaElement::builder()
+                    .attribute_name("base_uri")
+                    .key_type(KeyType::Hash)
+                    .build()
+                    .unwrap(),
+            )
+            .key_schema(
+                KeySchemaElement::builder()
+                    .attribute_name("version")
+                    .key_type(KeyType::Range)
+                    .build()
+                    .unwrap(),
+            )
+            .provisioned_throughput(
+                ProvisionedThroughput::builder()
+                    .read_capacity_units(1)
+                    .write_capacity_units(1)
+                    .build()
+                    .unwrap(),
+            )
+            .send()
+            .await
+            .unwrap();
+
+        Self(name.to_string())
+    }
+
+    async fn delete_table(client: aws_sdk_dynamodb::Client, name: &str) {
+        match client
+            .delete_table()
+            .table_name(name)
+            .send()
+            .await
+            .map_err(|err| err.into_service_error())
+        {
+            Ok(_) => {}
+            Err(e) if e.is_resource_not_found_exception() => {}
+            Err(e) => panic!("Failed to delete table: {}", e),
+        };
+    }
+}
+
+impl Drop for DynamoDBCommitTable {
+    fn drop(&mut self) {
+        let table_name = self.0.clone();
+        tokio::task::spawn(async move {
+            let config = aws_config().await;
+            let client = aws_sdk_dynamodb::Client::new(&config);
+            Self::delete_table(client, &table_name).await;
+        });
+    }
+}
+
+#[tokio::test]
+async fn test_concurrent_dynamodb_commit() {
+    // test concurrent commit on dynamodb
+    let bucket = S3Bucket::new("test-dynamodb").await;
+    let table = DynamoDBCommitTable::new("test_table").await;
+
+    let uri = format!("s3+ddb://{}?ddbTableName={}", bucket.0, table.0);
+    let db = lancedb::connect(&uri)
+        .storage_options(CONFIG.iter().cloned())
+        .execute()
+        .await
+        .unwrap();
+
+    let data = test_data();
+    let data = RecordBatchIterator::new(vec![Ok(data.clone())], data.schema());
+
+    let table = db.create_table("test_table", data).execute().await.unwrap();
+
+    let data = test_data();
+
+    let mut tasks = vec![];
+    for _ in 0..5 {
+        let table = db.open_table("test_table").execute().await.unwrap();
+        let data = data.clone();
+        tasks.push(tokio::spawn(async move {
+            let data = RecordBatchIterator::new(vec![Ok(data.clone())], data.schema());
+            table.add(data).execute().await.unwrap();
+        }));
+    }
+
+    for task in tasks {
+        task.await.unwrap();
+    }
+
+    table.checkout_latest().await.unwrap();
+    let row_count = table.count_rows(None).await.unwrap();
+    assert_eq!(row_count, 18);
+}