diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0849b4c7..bbe6aa53 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -14,7 +14,7 @@ repos: hooks: - id: local-biome-check name: biome check - entry: npx @biomejs/biome@1.7.3 check --config-path nodejs/biome.json nodejs/ + entry: npx @biomejs/biome@1.8.3 check --config-path nodejs/biome.json nodejs/ language: system types: [text] files: "nodejs/.*" diff --git a/docs/src/guides/storage.md b/docs/src/guides/storage.md index 078ff1b6..b165a536 100644 --- a/docs/src/guides/storage.md +++ b/docs/src/guides/storage.md @@ -265,6 +265,108 @@ For **read-only access**, LanceDB will need a policy such as: } ``` +#### DynamoDB Commit Store for concurrent writes + +By default, S3 does not support concurrent writes. Having two or more processes +writing to the same table at the same time can lead to data corruption. This is +because S3, unlike other object stores, does not have any atomic put or copy +operation. + +To enable concurrent writes, you can configure LanceDB to use a DynamoDB table +as a commit store. This table will be used to coordinate writes between +different processes. To enable this feature, you must modify your connection +URI to use the `s3+ddb` scheme and add a query parameter `ddbTableName` with the +name of the table to use. + +=== "Python" + + ```python + import lancedb + db = await lancedb.connect_async( + "s3+ddb://bucket/path?ddbTableName=my-dynamodb-table", + ) + ``` + +=== "JavaScript" + + ```javascript + const lancedb = require("lancedb"); + + const db = await lancedb.connect( + "s3+ddb://bucket/path?ddbTableName=my-dynamodb-table", + ); + ``` + +The DynamoDB table must be created with the following schema: + +- Hash key: `base_uri` (string) +- Range key: `version` (number) + +You can create this programmatically with: + +=== "Python" + + + ```python + import boto3 + + dynamodb = boto3.client("dynamodb") + table = dynamodb.create_table( + TableName=table_name, + KeySchema=[ + {"AttributeName": "base_uri", "KeyType": "HASH"}, + {"AttributeName": "version", "KeyType": "RANGE"}, + ], + AttributeDefinitions=[ + {"AttributeName": "base_uri", "AttributeType": "S"}, + {"AttributeName": "version", "AttributeType": "N"}, + ], + ProvisionedThroughput={"ReadCapacityUnits": 1, "WriteCapacityUnits": 1}, + ) + ``` + +=== "JavaScript" + + + ```javascript + import { + CreateTableCommand, + DynamoDBClient, + } from "@aws-sdk/client-dynamodb"; + + const dynamodb = new DynamoDBClient({ + region: CONFIG.awsRegion, + credentials: { + accessKeyId: CONFIG.awsAccessKeyId, + secretAccessKey: CONFIG.awsSecretAccessKey, + }, + endpoint: CONFIG.awsEndpoint, + }); + const command = new CreateTableCommand({ + TableName: table_name, + AttributeDefinitions: [ + { + AttributeName: "base_uri", + AttributeType: "S", + }, + { + AttributeName: "version", + AttributeType: "N", + }, + ], + KeySchema: [ + { AttributeName: "base_uri", KeyType: "HASH" }, + { AttributeName: "version", KeyType: "RANGE" }, + ], + ProvisionedThroughput: { + ReadCapacityUnits: 1, + WriteCapacityUnits: 1, + }, + }); + await client.send(command); + ``` + + #### S3-compatible stores LanceDB can also connect to S3-compatible stores, such as MinIO. To do so, you must specify both region and endpoint: diff --git a/nodejs/__test__/s3_integration.test.ts b/nodejs/__test__/s3_integration.test.ts index 1b1d48f6..9fd33f3f 100644 --- a/nodejs/__test__/s3_integration.test.ts +++ b/nodejs/__test__/s3_integration.test.ts @@ -14,6 +14,11 @@ /* eslint-disable @typescript-eslint/naming-convention */ +import { + CreateTableCommand, + DeleteTableCommand, + DynamoDBClient, +} from "@aws-sdk/client-dynamodb"; import { CreateKeyCommand, KMSClient, @@ -38,6 +43,7 @@ const CONFIG = { awsAccessKeyId: "ACCESSKEY", awsSecretAccessKey: "SECRETKEY", awsEndpoint: "http://127.0.0.1:4566", + dynamodbEndpoint: "http://127.0.0.1:4566", awsRegion: "us-east-1", }; @@ -66,7 +72,6 @@ class S3Bucket { } catch { // It's fine if the bucket doesn't exist } - // biome-ignore lint/style/useNamingConvention: we dont control s3's api await client.send(new CreateBucketCommand({ Bucket: name })); return new S3Bucket(name); } @@ -79,32 +84,27 @@ class S3Bucket { static async deleteBucket(client: S3Client, name: string) { // Must delete all objects before we can delete the bucket const objects = await client.send( - // biome-ignore lint/style/useNamingConvention: we dont control s3's api new ListObjectsV2Command({ Bucket: name }), ); if (objects.Contents) { for (const object of objects.Contents) { await client.send( - // biome-ignore lint/style/useNamingConvention: we dont control s3's api new DeleteObjectCommand({ Bucket: name, Key: object.Key }), ); } } - // biome-ignore lint/style/useNamingConvention: we dont control s3's api await client.send(new DeleteBucketCommand({ Bucket: name })); } public async assertAllEncrypted(path: string, keyId: string) { const client = S3Bucket.s3Client(); const objects = await client.send( - // biome-ignore lint/style/useNamingConvention: we dont control s3's api new ListObjectsV2Command({ Bucket: this.name, Prefix: path }), ); if (objects.Contents) { for (const object of objects.Contents) { const metadata = await client.send( - // biome-ignore lint/style/useNamingConvention: we dont control s3's api new HeadObjectCommand({ Bucket: this.name, Key: object.Key }), ); expect(metadata.ServerSideEncryption).toBe("aws:kms"); @@ -143,7 +143,6 @@ class KmsKey { public async delete() { const client = KmsKey.kmsClient(); - // biome-ignore lint/style/useNamingConvention: we dont control s3's api await client.send(new ScheduleKeyDeletionCommand({ KeyId: this.keyId })); } } @@ -224,3 +223,91 @@ maybeDescribe("storage_options", () => { await bucket.assertAllEncrypted("test/table2.lance", kmsKey.keyId); }); }); + +class DynamoDBCommitTable { + name: string; + constructor(name: string) { + this.name = name; + } + + static dynamoClient() { + return new DynamoDBClient({ + region: CONFIG.awsRegion, + credentials: { + accessKeyId: CONFIG.awsAccessKeyId, + secretAccessKey: CONFIG.awsSecretAccessKey, + }, + endpoint: CONFIG.awsEndpoint, + }); + } + + public static async create(name: string): Promise { + const client = DynamoDBCommitTable.dynamoClient(); + const command = new CreateTableCommand({ + TableName: name, + AttributeDefinitions: [ + { + AttributeName: "base_uri", + AttributeType: "S", + }, + { + AttributeName: "version", + AttributeType: "N", + }, + ], + KeySchema: [ + { AttributeName: "base_uri", KeyType: "HASH" }, + { AttributeName: "version", KeyType: "RANGE" }, + ], + ProvisionedThroughput: { + ReadCapacityUnits: 1, + WriteCapacityUnits: 1, + }, + }); + await client.send(command); + return new DynamoDBCommitTable(name); + } + + public async delete() { + const client = DynamoDBCommitTable.dynamoClient(); + await client.send(new DeleteTableCommand({ TableName: this.name })); + } +} + +maybeDescribe("DynamoDB Lock", () => { + let bucket: S3Bucket; + let commitTable: DynamoDBCommitTable; + + beforeAll(async () => { + bucket = await S3Bucket.create("lancedb2"); + commitTable = await DynamoDBCommitTable.create("commitTable"); + }); + + afterAll(async () => { + await commitTable.delete(); + await bucket.delete(); + }); + + it("can be used to configure a DynamoDB table for commit log", async () => { + const uri = `s3+ddb://${bucket.name}/test?ddbTableName=${commitTable.name}`; + const db = await connect(uri, { + storageOptions: CONFIG, + readConsistencyInterval: 0, + }); + + const table = await db.createTable("test", [{ a: 1, b: 2 }]); + + // 5 concurrent appends + const futs = Array.from({ length: 5 }, async () => { + // Open a table so each append has a separate table reference. Otherwise + // they will share the same table reference and the internal ReadWriteLock + // will prevent any real concurrency. + const table = await db.openTable("test"); + await table.add([{ a: 2, b: 3 }]); + }); + await Promise.all(futs); + + const rowCount = await table.countRows(); + expect(rowCount).toBe(6); + }); +}); diff --git a/nodejs/biome.json b/nodejs/biome.json index 61ed1209..1bd8738b 100644 --- a/nodejs/biome.json +++ b/nodejs/biome.json @@ -1,5 +1,5 @@ { - "$schema": "https://biomejs.dev/schemas/1.7.3/schema.json", + "$schema": "https://biomejs.dev/schemas/1.8.3/schema.json", "organizeImports": { "enabled": true }, @@ -100,6 +100,16 @@ "globals": [] }, "overrides": [ + { + "include": ["__test__/s3_integration.test.ts"], + "linter": { + "rules": { + "style": { + "useNamingConvention": "off" + } + } + } + }, { "include": [ "**/*.ts", diff --git a/nodejs/lancedb/remote/client.ts b/nodejs/lancedb/remote/client.ts index d7cf9d3e..a8ccbf5c 100644 --- a/nodejs/lancedb/remote/client.ts +++ b/nodejs/lancedb/remote/client.ts @@ -55,7 +55,7 @@ export class RestfulLanceDBClient { return axios.create({ baseURL: this.url, headers: { - // biome-ignore lint/style/useNamingConvention: external api + // biome-ignore lint: external API Authorization: `Bearer ${this.#apiKey}`, }, transformResponse: decodeErrorData, diff --git a/nodejs/package-lock.json b/nodejs/package-lock.json index 6b197fca..ac8b4d32 100644 --- a/nodejs/package-lock.json +++ b/nodejs/package-lock.json @@ -24,6 +24,7 @@ "reflect-metadata": "^0.2.2" }, "devDependencies": { + "@aws-sdk/client-dynamodb": "^3.33.0", "@aws-sdk/client-kms": "^3.33.0", "@aws-sdk/client-s3": "^3.33.0", "@biomejs/biome": "^1.7.3", @@ -230,6 +231,1202 @@ "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==", "dev": true }, + "node_modules/@aws-sdk/client-dynamodb": { + "version": "3.602.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-dynamodb/-/client-dynamodb-3.602.0.tgz", + "integrity": "sha512-q7lH7YD9KvHLF3tyAG1UqaPv4a6KiHLunqKYh8vt3d1WJK7t4wzE97Vf19MfNpza1MuZ0OF/SK8Kl69vEMrtOA==", + "dev": true, + "dependencies": { + "@aws-crypto/sha256-browser": "5.2.0", + "@aws-crypto/sha256-js": "5.2.0", + "@aws-sdk/client-sso-oidc": "3.600.0", + "@aws-sdk/client-sts": "3.600.0", + "@aws-sdk/core": "3.598.0", + "@aws-sdk/credential-provider-node": "3.600.0", + "@aws-sdk/middleware-endpoint-discovery": "3.598.0", + "@aws-sdk/middleware-host-header": "3.598.0", + "@aws-sdk/middleware-logger": "3.598.0", + "@aws-sdk/middleware-recursion-detection": "3.598.0", + "@aws-sdk/middleware-user-agent": "3.598.0", + "@aws-sdk/region-config-resolver": "3.598.0", + "@aws-sdk/types": "3.598.0", + "@aws-sdk/util-endpoints": "3.598.0", + "@aws-sdk/util-user-agent-browser": "3.598.0", + "@aws-sdk/util-user-agent-node": "3.598.0", + "@smithy/config-resolver": "^3.0.2", + "@smithy/core": "^2.2.1", + "@smithy/fetch-http-handler": "^3.0.2", + "@smithy/hash-node": "^3.0.1", + "@smithy/invalid-dependency": "^3.0.1", + "@smithy/middleware-content-length": "^3.0.1", + "@smithy/middleware-endpoint": "^3.0.2", + "@smithy/middleware-retry": "^3.0.4", + "@smithy/middleware-serde": "^3.0.1", + "@smithy/middleware-stack": "^3.0.1", + "@smithy/node-config-provider": "^3.1.1", + "@smithy/node-http-handler": "^3.0.1", + "@smithy/protocol-http": "^4.0.1", + "@smithy/smithy-client": "^3.1.2", + "@smithy/types": "^3.1.0", + "@smithy/url-parser": "^3.0.1", + "@smithy/util-base64": "^3.0.0", + "@smithy/util-body-length-browser": "^3.0.0", + "@smithy/util-body-length-node": "^3.0.0", + "@smithy/util-defaults-mode-browser": "^3.0.4", + "@smithy/util-defaults-mode-node": "^3.0.4", + "@smithy/util-endpoints": "^2.0.2", + "@smithy/util-middleware": "^3.0.1", + "@smithy/util-retry": "^3.0.1", + "@smithy/util-utf8": "^3.0.0", + "@smithy/util-waiter": "^3.0.1", + "tslib": "^2.6.2", + "uuid": "^9.0.1" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@aws-crypto/sha256-browser": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@aws-crypto/sha256-browser/-/sha256-browser-5.2.0.tgz", + "integrity": "sha512-AXfN/lGotSQwu6HNcEsIASo7kWXZ5HYWvfOmSNKDsEqC4OashTp8alTmaz+F7TC2L083SFv5RdB+qU3Vs1kZqw==", + "dev": true, + "dependencies": { + "@aws-crypto/sha256-js": "^5.2.0", + "@aws-crypto/supports-web-crypto": "^5.2.0", + "@aws-crypto/util": "^5.2.0", + "@aws-sdk/types": "^3.222.0", + "@aws-sdk/util-locate-window": "^3.0.0", + "@smithy/util-utf8": "^2.0.0", + "tslib": "^2.6.2" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@aws-crypto/sha256-browser/node_modules/@smithy/util-utf8": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-2.3.0.tgz", + "integrity": "sha512-R8Rdn8Hy72KKcebgLiv8jQcQkXoLMOGGv5uI1/k0l+snqkOzQ1R0ChUBCxWMlBsFMekWjq0wRudIweFs7sKT5A==", + "dev": true, + "dependencies": { + "@smithy/util-buffer-from": "^2.2.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@aws-crypto/sha256-js": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@aws-crypto/sha256-js/-/sha256-js-5.2.0.tgz", + "integrity": "sha512-FFQQyu7edu4ufvIZ+OadFpHHOt+eSTBaYaki44c+akjg7qZg9oOQeLlk77F6tSYqjDAFClrHJk9tMf0HdVyOvA==", + "dev": true, + "dependencies": { + "@aws-crypto/util": "^5.2.0", + "@aws-sdk/types": "^3.222.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@aws-crypto/supports-web-crypto": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@aws-crypto/supports-web-crypto/-/supports-web-crypto-5.2.0.tgz", + "integrity": "sha512-iAvUotm021kM33eCdNfwIN//F77/IADDSs58i+MDaOqFrVjZo9bAal0NK7HurRuWLLpF1iLX7gbWrjHjeo+YFg==", + "dev": true, + "dependencies": { + "tslib": "^2.6.2" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@aws-crypto/util": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@aws-crypto/util/-/util-5.2.0.tgz", + "integrity": "sha512-4RkU9EsI6ZpBve5fseQlGNUWKMa1RLPQ1dnjnQoe07ldfIzcsGb5hC5W0Dm7u423KWzawlrpbjXBrXCEv9zazQ==", + "dev": true, + "dependencies": { + "@aws-sdk/types": "^3.222.0", + "@smithy/util-utf8": "^2.0.0", + "tslib": "^2.6.2" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@aws-crypto/util/node_modules/@smithy/util-utf8": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-2.3.0.tgz", + "integrity": "sha512-R8Rdn8Hy72KKcebgLiv8jQcQkXoLMOGGv5uI1/k0l+snqkOzQ1R0ChUBCxWMlBsFMekWjq0wRudIweFs7sKT5A==", + "dev": true, + "dependencies": { + "@smithy/util-buffer-from": "^2.2.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@aws-sdk/client-sso": { + "version": "3.598.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-sso/-/client-sso-3.598.0.tgz", + "integrity": "sha512-nOI5lqPYa+YZlrrzwAJywJSw3MKVjvu6Ge2fCqQUNYMfxFB0NAaDFnl0EPjXi+sEbtCuz/uWE77poHbqiZ+7Iw==", + "dev": true, + "dependencies": { + "@aws-crypto/sha256-browser": "5.2.0", + "@aws-crypto/sha256-js": "5.2.0", + "@aws-sdk/core": "3.598.0", + "@aws-sdk/middleware-host-header": "3.598.0", + "@aws-sdk/middleware-logger": "3.598.0", + "@aws-sdk/middleware-recursion-detection": "3.598.0", + "@aws-sdk/middleware-user-agent": "3.598.0", + "@aws-sdk/region-config-resolver": "3.598.0", + "@aws-sdk/types": "3.598.0", + "@aws-sdk/util-endpoints": "3.598.0", + "@aws-sdk/util-user-agent-browser": "3.598.0", + "@aws-sdk/util-user-agent-node": "3.598.0", + "@smithy/config-resolver": "^3.0.2", + "@smithy/core": "^2.2.1", + "@smithy/fetch-http-handler": "^3.0.2", + "@smithy/hash-node": "^3.0.1", + "@smithy/invalid-dependency": "^3.0.1", + "@smithy/middleware-content-length": "^3.0.1", + "@smithy/middleware-endpoint": "^3.0.2", + "@smithy/middleware-retry": "^3.0.4", + "@smithy/middleware-serde": "^3.0.1", + "@smithy/middleware-stack": "^3.0.1", + "@smithy/node-config-provider": "^3.1.1", + "@smithy/node-http-handler": "^3.0.1", + "@smithy/protocol-http": "^4.0.1", + "@smithy/smithy-client": "^3.1.2", + "@smithy/types": "^3.1.0", + "@smithy/url-parser": "^3.0.1", + "@smithy/util-base64": "^3.0.0", + "@smithy/util-body-length-browser": "^3.0.0", + "@smithy/util-body-length-node": "^3.0.0", + "@smithy/util-defaults-mode-browser": "^3.0.4", + "@smithy/util-defaults-mode-node": "^3.0.4", + "@smithy/util-endpoints": "^2.0.2", + "@smithy/util-middleware": "^3.0.1", + "@smithy/util-retry": "^3.0.1", + "@smithy/util-utf8": "^3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@aws-sdk/client-sso-oidc": { + "version": "3.600.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-sso-oidc/-/client-sso-oidc-3.600.0.tgz", + "integrity": "sha512-7+I8RWURGfzvChyNQSyj5/tKrqRbzRl7H+BnTOf/4Vsw1nFOi5ROhlhD4X/Y0QCTacxnaoNcIrqnY7uGGvVRzw==", + "dev": true, + "dependencies": { + "@aws-crypto/sha256-browser": "5.2.0", + "@aws-crypto/sha256-js": "5.2.0", + "@aws-sdk/client-sts": "3.600.0", + "@aws-sdk/core": "3.598.0", + "@aws-sdk/credential-provider-node": "3.600.0", + "@aws-sdk/middleware-host-header": "3.598.0", + "@aws-sdk/middleware-logger": "3.598.0", + "@aws-sdk/middleware-recursion-detection": "3.598.0", + "@aws-sdk/middleware-user-agent": "3.598.0", + "@aws-sdk/region-config-resolver": "3.598.0", + "@aws-sdk/types": "3.598.0", + "@aws-sdk/util-endpoints": "3.598.0", + "@aws-sdk/util-user-agent-browser": "3.598.0", + "@aws-sdk/util-user-agent-node": "3.598.0", + "@smithy/config-resolver": "^3.0.2", + "@smithy/core": "^2.2.1", + "@smithy/fetch-http-handler": "^3.0.2", + "@smithy/hash-node": "^3.0.1", + "@smithy/invalid-dependency": "^3.0.1", + "@smithy/middleware-content-length": "^3.0.1", + "@smithy/middleware-endpoint": "^3.0.2", + "@smithy/middleware-retry": "^3.0.4", + "@smithy/middleware-serde": "^3.0.1", + "@smithy/middleware-stack": "^3.0.1", + "@smithy/node-config-provider": "^3.1.1", + "@smithy/node-http-handler": "^3.0.1", + "@smithy/protocol-http": "^4.0.1", + "@smithy/smithy-client": "^3.1.2", + "@smithy/types": "^3.1.0", + "@smithy/url-parser": "^3.0.1", + "@smithy/util-base64": "^3.0.0", + "@smithy/util-body-length-browser": "^3.0.0", + "@smithy/util-body-length-node": "^3.0.0", + "@smithy/util-defaults-mode-browser": "^3.0.4", + "@smithy/util-defaults-mode-node": "^3.0.4", + "@smithy/util-endpoints": "^2.0.2", + "@smithy/util-middleware": "^3.0.1", + "@smithy/util-retry": "^3.0.1", + "@smithy/util-utf8": "^3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@aws-sdk/client-sts": { + "version": "3.600.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-sts/-/client-sts-3.600.0.tgz", + "integrity": "sha512-KQG97B7LvTtTiGmjlrG1LRAY8wUvCQzrmZVV5bjrJ/1oXAU7DITYwVbSJeX9NWg6hDuSk0VE3MFwIXS2SvfLIA==", + "dev": true, + "dependencies": { + "@aws-crypto/sha256-browser": "5.2.0", + "@aws-crypto/sha256-js": "5.2.0", + "@aws-sdk/client-sso-oidc": "3.600.0", + "@aws-sdk/core": "3.598.0", + "@aws-sdk/credential-provider-node": "3.600.0", + "@aws-sdk/middleware-host-header": "3.598.0", + "@aws-sdk/middleware-logger": "3.598.0", + "@aws-sdk/middleware-recursion-detection": "3.598.0", + "@aws-sdk/middleware-user-agent": "3.598.0", + "@aws-sdk/region-config-resolver": "3.598.0", + "@aws-sdk/types": "3.598.0", + "@aws-sdk/util-endpoints": "3.598.0", + "@aws-sdk/util-user-agent-browser": "3.598.0", + "@aws-sdk/util-user-agent-node": "3.598.0", + "@smithy/config-resolver": "^3.0.2", + "@smithy/core": "^2.2.1", + "@smithy/fetch-http-handler": "^3.0.2", + "@smithy/hash-node": "^3.0.1", + "@smithy/invalid-dependency": "^3.0.1", + "@smithy/middleware-content-length": "^3.0.1", + "@smithy/middleware-endpoint": "^3.0.2", + "@smithy/middleware-retry": "^3.0.4", + "@smithy/middleware-serde": "^3.0.1", + "@smithy/middleware-stack": "^3.0.1", + "@smithy/node-config-provider": "^3.1.1", + "@smithy/node-http-handler": "^3.0.1", + "@smithy/protocol-http": "^4.0.1", + "@smithy/smithy-client": "^3.1.2", + "@smithy/types": "^3.1.0", + "@smithy/url-parser": "^3.0.1", + "@smithy/util-base64": "^3.0.0", + "@smithy/util-body-length-browser": "^3.0.0", + "@smithy/util-body-length-node": "^3.0.0", + "@smithy/util-defaults-mode-browser": "^3.0.4", + "@smithy/util-defaults-mode-node": "^3.0.4", + "@smithy/util-endpoints": "^2.0.2", + "@smithy/util-middleware": "^3.0.1", + "@smithy/util-retry": "^3.0.1", + "@smithy/util-utf8": "^3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@aws-sdk/core": { + "version": "3.598.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/core/-/core-3.598.0.tgz", + "integrity": "sha512-HaSjt7puO5Cc7cOlrXFCW0rtA0BM9lvzjl56x0A20Pt+0wxXGeTOZZOkXQIepbrFkV2e/HYukuT9e99vXDm59g==", + "dev": true, + "dependencies": { + "@smithy/core": "^2.2.1", + "@smithy/protocol-http": "^4.0.1", + "@smithy/signature-v4": "^3.1.0", + "@smithy/smithy-client": "^3.1.2", + "@smithy/types": "^3.1.0", + "fast-xml-parser": "4.2.5", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@aws-sdk/credential-provider-env": { + "version": "3.598.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-env/-/credential-provider-env-3.598.0.tgz", + "integrity": "sha512-vi1khgn7yXzLCcgSIzQrrtd2ilUM0dWodxj3PQ6BLfP0O+q1imO3hG1nq7DVyJtq7rFHs6+9N8G4mYvTkxby2w==", + "dev": true, + "dependencies": { + "@aws-sdk/types": "3.598.0", + "@smithy/property-provider": "^3.1.1", + "@smithy/types": "^3.1.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@aws-sdk/credential-provider-http": { + "version": "3.598.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-http/-/credential-provider-http-3.598.0.tgz", + "integrity": "sha512-N7cIafi4HVlQvEgvZSo1G4T9qb/JMLGMdBsDCT5XkeJrF0aptQWzTFH0jIdZcLrMYvzPcuEyO3yCBe6cy/ba0g==", + "dev": true, + "dependencies": { + "@aws-sdk/types": "3.598.0", + "@smithy/fetch-http-handler": "^3.0.2", + "@smithy/node-http-handler": "^3.0.1", + "@smithy/property-provider": "^3.1.1", + "@smithy/protocol-http": "^4.0.1", + "@smithy/smithy-client": "^3.1.2", + "@smithy/types": "^3.1.0", + "@smithy/util-stream": "^3.0.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@aws-sdk/credential-provider-ini": { + "version": "3.598.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-ini/-/credential-provider-ini-3.598.0.tgz", + "integrity": "sha512-/ppcIVUbRwDIwJDoYfp90X3+AuJo2mvE52Y1t2VSrvUovYn6N4v95/vXj6LS8CNDhz2jvEJYmu+0cTMHdhI6eA==", + "dev": true, + "dependencies": { + "@aws-sdk/credential-provider-env": "3.598.0", + "@aws-sdk/credential-provider-http": "3.598.0", + "@aws-sdk/credential-provider-process": "3.598.0", + "@aws-sdk/credential-provider-sso": "3.598.0", + "@aws-sdk/credential-provider-web-identity": "3.598.0", + "@aws-sdk/types": "3.598.0", + "@smithy/credential-provider-imds": "^3.1.1", + "@smithy/property-provider": "^3.1.1", + "@smithy/shared-ini-file-loader": "^3.1.1", + "@smithy/types": "^3.1.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + }, + "peerDependencies": { + "@aws-sdk/client-sts": "^3.598.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@aws-sdk/credential-provider-node": { + "version": "3.600.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-node/-/credential-provider-node-3.600.0.tgz", + "integrity": "sha512-1pC7MPMYD45J7yFjA90SxpR0yaSvy+yZiq23aXhAPZLYgJBAxHLu0s0mDCk/piWGPh8+UGur5K0bVdx4B1D5hw==", + "dev": true, + "dependencies": { + "@aws-sdk/credential-provider-env": "3.598.0", + "@aws-sdk/credential-provider-http": "3.598.0", + "@aws-sdk/credential-provider-ini": "3.598.0", + "@aws-sdk/credential-provider-process": "3.598.0", + "@aws-sdk/credential-provider-sso": "3.598.0", + "@aws-sdk/credential-provider-web-identity": "3.598.0", + "@aws-sdk/types": "3.598.0", + "@smithy/credential-provider-imds": "^3.1.1", + "@smithy/property-provider": "^3.1.1", + "@smithy/shared-ini-file-loader": "^3.1.1", + "@smithy/types": "^3.1.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@aws-sdk/credential-provider-process": { + "version": "3.598.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-process/-/credential-provider-process-3.598.0.tgz", + "integrity": "sha512-rM707XbLW8huMk722AgjVyxu2tMZee++fNA8TJVNgs1Ma02Wx6bBrfIvlyK0rCcIRb0WdQYP6fe3Xhiu4e8IBA==", + "dev": true, + "dependencies": { + "@aws-sdk/types": "3.598.0", + "@smithy/property-provider": "^3.1.1", + "@smithy/shared-ini-file-loader": "^3.1.1", + "@smithy/types": "^3.1.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@aws-sdk/credential-provider-sso": { + "version": "3.598.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-sso/-/credential-provider-sso-3.598.0.tgz", + "integrity": "sha512-5InwUmrAuqQdOOgxTccRayMMkSmekdLk6s+az9tmikq0QFAHUCtofI+/fllMXSR9iL6JbGYi1940+EUmS4pHJA==", + "dev": true, + "dependencies": { + "@aws-sdk/client-sso": "3.598.0", + "@aws-sdk/token-providers": "3.598.0", + "@aws-sdk/types": "3.598.0", + "@smithy/property-provider": "^3.1.1", + "@smithy/shared-ini-file-loader": "^3.1.1", + "@smithy/types": "^3.1.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@aws-sdk/credential-provider-web-identity": { + "version": "3.598.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-web-identity/-/credential-provider-web-identity-3.598.0.tgz", + "integrity": "sha512-GV5GdiMbz5Tz9JO4NJtRoFXjW0GPEujA0j+5J/B723rTN+REHthJu48HdBKouHGhdzkDWkkh1bu52V02Wprw8w==", + "dev": true, + "dependencies": { + "@aws-sdk/types": "3.598.0", + "@smithy/property-provider": "^3.1.1", + "@smithy/types": "^3.1.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + }, + "peerDependencies": { + "@aws-sdk/client-sts": "^3.598.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@aws-sdk/middleware-host-header": { + "version": "3.598.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-host-header/-/middleware-host-header-3.598.0.tgz", + "integrity": "sha512-WiaG059YBQwQraNejLIi0gMNkX7dfPZ8hDIhvMr5aVPRbaHH8AYF3iNSsXYCHvA2Cfa1O9haYXsuMF9flXnCmA==", + "dev": true, + "dependencies": { + "@aws-sdk/types": "3.598.0", + "@smithy/protocol-http": "^4.0.1", + "@smithy/types": "^3.1.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@aws-sdk/middleware-logger": { + "version": "3.598.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-logger/-/middleware-logger-3.598.0.tgz", + "integrity": "sha512-bxBjf/VYiu3zfu8SYM2S9dQQc3tz5uBAOcPz/Bt8DyyK3GgOpjhschH/2XuUErsoUO1gDJqZSdGOmuHGZQn00Q==", + "dev": true, + "dependencies": { + "@aws-sdk/types": "3.598.0", + "@smithy/types": "^3.1.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@aws-sdk/middleware-recursion-detection": { + "version": "3.598.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-recursion-detection/-/middleware-recursion-detection-3.598.0.tgz", + "integrity": "sha512-vjT9BeFY9FeN0f8hm2l6F53tI0N5bUq6RcDkQXKNabXBnQxKptJRad6oP2X5y3FoVfBLOuDkQgiC2940GIPxtQ==", + "dev": true, + "dependencies": { + "@aws-sdk/types": "3.598.0", + "@smithy/protocol-http": "^4.0.1", + "@smithy/types": "^3.1.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@aws-sdk/middleware-user-agent": { + "version": "3.598.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-user-agent/-/middleware-user-agent-3.598.0.tgz", + "integrity": "sha512-4tjESlHG5B5MdjUaLK7tQs/miUtHbb6deauQx8ryqSBYOhfHVgb1ZnzvQR0bTrhpqUg0WlybSkDaZAICf9xctg==", + "dev": true, + "dependencies": { + "@aws-sdk/types": "3.598.0", + "@aws-sdk/util-endpoints": "3.598.0", + "@smithy/protocol-http": "^4.0.1", + "@smithy/types": "^3.1.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@aws-sdk/region-config-resolver": { + "version": "3.598.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/region-config-resolver/-/region-config-resolver-3.598.0.tgz", + "integrity": "sha512-oYXhmTokSav4ytmWleCr3rs/1nyvZW/S0tdi6X7u+dLNL5Jee+uMxWGzgOrWK6wrQOzucLVjS4E/wA11Kv2GTw==", + "dev": true, + "dependencies": { + "@aws-sdk/types": "3.598.0", + "@smithy/node-config-provider": "^3.1.1", + "@smithy/types": "^3.1.0", + "@smithy/util-config-provider": "^3.0.0", + "@smithy/util-middleware": "^3.0.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@aws-sdk/token-providers": { + "version": "3.598.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/token-providers/-/token-providers-3.598.0.tgz", + "integrity": "sha512-TKY1EVdHVBnZqpyxyTHdpZpa1tUpb6nxVeRNn1zWG8QB5MvH4ALLd/jR+gtmWDNQbIG4cVuBOZFVL8hIYicKTA==", + "dev": true, + "dependencies": { + "@aws-sdk/types": "3.598.0", + "@smithy/property-provider": "^3.1.1", + "@smithy/shared-ini-file-loader": "^3.1.1", + "@smithy/types": "^3.1.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + }, + "peerDependencies": { + "@aws-sdk/client-sso-oidc": "^3.598.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@aws-sdk/types": { + "version": "3.598.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.598.0.tgz", + "integrity": "sha512-742uRl6z7u0LFmZwDrFP6r1wlZcgVPw+/TilluDJmCAR8BgRw3IR+743kUXKBGd8QZDRW2n6v/PYsi/AWCDDMQ==", + "dev": true, + "dependencies": { + "@smithy/types": "^3.1.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@aws-sdk/util-endpoints": { + "version": "3.598.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-endpoints/-/util-endpoints-3.598.0.tgz", + "integrity": "sha512-Qo9UoiVVZxcOEdiOMZg3xb1mzkTxrhd4qSlg5QQrfWPJVx/QOg+Iy0NtGxPtHtVZNHZxohYwDwV/tfsnDSE2gQ==", + "dev": true, + "dependencies": { + "@aws-sdk/types": "3.598.0", + "@smithy/types": "^3.1.0", + "@smithy/util-endpoints": "^2.0.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@aws-sdk/util-user-agent-browser": { + "version": "3.598.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-user-agent-browser/-/util-user-agent-browser-3.598.0.tgz", + "integrity": "sha512-36Sxo6F+ykElaL1mWzWjlg+1epMpSe8obwhCN1yGE7Js9ywy5U6k6l+A3q3YM9YRbm740sNxncbwLklMvuhTKw==", + "dev": true, + "dependencies": { + "@aws-sdk/types": "3.598.0", + "@smithy/types": "^3.1.0", + "bowser": "^2.11.0", + "tslib": "^2.6.2" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@aws-sdk/util-user-agent-node": { + "version": "3.598.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-user-agent-node/-/util-user-agent-node-3.598.0.tgz", + "integrity": "sha512-oyWGcOlfTdzkC6SVplyr0AGh54IMrDxbhg5RxJ5P+V4BKfcDoDcZV9xenUk9NsOi9MuUjxMumb9UJGkDhM1m0A==", + "dev": true, + "dependencies": { + "@aws-sdk/types": "3.598.0", + "@smithy/node-config-provider": "^3.1.1", + "@smithy/types": "^3.1.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + }, + "peerDependencies": { + "aws-crt": ">=1.0.0" + }, + "peerDependenciesMeta": { + "aws-crt": { + "optional": true + } + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@smithy/abort-controller": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/@smithy/abort-controller/-/abort-controller-3.1.0.tgz", + "integrity": "sha512-XOm4LkuC0PsK1sf2bBJLIlskn5ghmVxiEBVlo/jg0R8hxASBKYYgOoJEhKWgOr4vWGkN+5rC+oyBAqHYtxjnwQ==", + "dev": true, + "dependencies": { + "@smithy/types": "^3.2.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@smithy/config-resolver": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@smithy/config-resolver/-/config-resolver-3.0.3.tgz", + "integrity": "sha512-4wHqCMkdfVDP4qmr4fVPYOFOH+vKhOv3X4e6KEU9wIC8xXUQ24tnF4CW+sddGDX1zU86GGyQ7A+rg2xmUD6jpQ==", + "dev": true, + "dependencies": { + "@smithy/node-config-provider": "^3.1.2", + "@smithy/types": "^3.2.0", + "@smithy/util-config-provider": "^3.0.0", + "@smithy/util-middleware": "^3.0.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@smithy/core": { + "version": "2.2.3", + "resolved": "https://registry.npmjs.org/@smithy/core/-/core-2.2.3.tgz", + "integrity": "sha512-SpyLOL2vgE6sUYM6nQfu82OirCPkCDKctyG3aMgjMlDPTJpUlmlNH0ttu9ZWwzEjrzzr8uABmPjJTRI7gk1HFQ==", + "dev": true, + "dependencies": { + "@smithy/middleware-endpoint": "^3.0.3", + "@smithy/middleware-retry": "^3.0.6", + "@smithy/middleware-serde": "^3.0.2", + "@smithy/protocol-http": "^4.0.2", + "@smithy/smithy-client": "^3.1.4", + "@smithy/types": "^3.2.0", + "@smithy/util-middleware": "^3.0.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@smithy/credential-provider-imds": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/@smithy/credential-provider-imds/-/credential-provider-imds-3.1.2.tgz", + "integrity": "sha512-gqVmUaNoeqyrOAjgZg+rTmFLsphh/vS59LCMdFfVpthVS0jbfBzvBmEPktBd+y9ME4DYMGHFAMSYJDK8q0noOQ==", + "dev": true, + "dependencies": { + "@smithy/node-config-provider": "^3.1.2", + "@smithy/property-provider": "^3.1.2", + "@smithy/types": "^3.2.0", + "@smithy/url-parser": "^3.0.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@smithy/fetch-http-handler": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/@smithy/fetch-http-handler/-/fetch-http-handler-3.1.0.tgz", + "integrity": "sha512-s7oQjEOUH9TYjctpITtWF4qxOdg7pBrP9eigEQ8SBsxF3dRFV0S28pGMllC83DUr7ECmErhO/BUwnULfoNhKgQ==", + "dev": true, + "dependencies": { + "@smithy/protocol-http": "^4.0.2", + "@smithy/querystring-builder": "^3.0.2", + "@smithy/types": "^3.2.0", + "@smithy/util-base64": "^3.0.0", + "tslib": "^2.6.2" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@smithy/hash-node": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@smithy/hash-node/-/hash-node-3.0.2.tgz", + "integrity": "sha512-43uGA6o6QJQdXwAogybdTDHDd3SCdKyoiHIHb8PpdE2rKmVicjG9b1UgVwdgO8QPytmVqHFaUw27M3LZKwu8Yg==", + "dev": true, + "dependencies": { + "@smithy/types": "^3.2.0", + "@smithy/util-buffer-from": "^3.0.0", + "@smithy/util-utf8": "^3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@smithy/hash-node/node_modules/@smithy/util-buffer-from": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-3.0.0.tgz", + "integrity": "sha512-aEOHCgq5RWFbP+UDPvPot26EJHjOC+bRgse5A8V3FSShqd5E5UN4qc7zkwsvJPPAVsf73QwYcHN1/gt/rtLwQA==", + "dev": true, + "dependencies": { + "@smithy/is-array-buffer": "^3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@smithy/invalid-dependency": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@smithy/invalid-dependency/-/invalid-dependency-3.0.2.tgz", + "integrity": "sha512-+BAY3fMhomtq470tswXyrdVBSUhiLuhBVT+rOmpbz5e04YX+s1dX4NxTLzZGwBjCpeWZNtTxP8zbIvvFk81gUg==", + "dev": true, + "dependencies": { + "@smithy/types": "^3.2.0", + "tslib": "^2.6.2" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@smithy/is-array-buffer": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-3.0.0.tgz", + "integrity": "sha512-+Fsu6Q6C4RSJiy81Y8eApjEB5gVtM+oFKTffg+jSuwtvomJJrhUJBu2zS8wjXSgH/g1MKEWrzyChTBe6clb5FQ==", + "dev": true, + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@smithy/middleware-content-length": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@smithy/middleware-content-length/-/middleware-content-length-3.0.2.tgz", + "integrity": "sha512-/Havz3PkYIEmwpqkyRTR21yJsWnFbD1ec4H1pUL+TkDnE7RCQkAVUQepLL/UeCaZeCBXvfdoKbOjSbV01xIinQ==", + "dev": true, + "dependencies": { + "@smithy/protocol-http": "^4.0.2", + "@smithy/types": "^3.2.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@smithy/middleware-endpoint": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@smithy/middleware-endpoint/-/middleware-endpoint-3.0.3.tgz", + "integrity": "sha512-ARAXHodhj4tttKa9y75zvENdSoHq6VGsSi7XS3+yLutrnxttJs6N10UMInCC1yi3/bopT8xug3iOP/y9R6sKJQ==", + "dev": true, + "dependencies": { + "@smithy/middleware-serde": "^3.0.2", + "@smithy/node-config-provider": "^3.1.2", + "@smithy/shared-ini-file-loader": "^3.1.2", + "@smithy/types": "^3.2.0", + "@smithy/url-parser": "^3.0.2", + "@smithy/util-middleware": "^3.0.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@smithy/middleware-retry": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/@smithy/middleware-retry/-/middleware-retry-3.0.6.tgz", + "integrity": "sha512-ICsFKp8eAyIMmxN5UT3IU37S6886L879TKtgxPsn/VD/laYNwqTLmJaCAn5//+2fRIrV0dnHp6LFlMwdXlWoUQ==", + "dev": true, + "dependencies": { + "@smithy/node-config-provider": "^3.1.2", + "@smithy/protocol-http": "^4.0.2", + "@smithy/service-error-classification": "^3.0.2", + "@smithy/smithy-client": "^3.1.4", + "@smithy/types": "^3.2.0", + "@smithy/util-middleware": "^3.0.2", + "@smithy/util-retry": "^3.0.2", + "tslib": "^2.6.2", + "uuid": "^9.0.1" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@smithy/middleware-serde": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@smithy/middleware-serde/-/middleware-serde-3.0.2.tgz", + "integrity": "sha512-oT2abV5zLhBucJe1LIIFEcRgIBDbZpziuMPswTMbBQNcaEUycLFvX63zsFmqfwG+/ZQKsNx+BSE8W51CMuK7Yw==", + "dev": true, + "dependencies": { + "@smithy/types": "^3.2.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@smithy/middleware-stack": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@smithy/middleware-stack/-/middleware-stack-3.0.2.tgz", + "integrity": "sha512-6fRcxomlNKBPIy/YjcnC7YHpMAjRvGUYlYVJAfELqZjkW0vQegNcImjY7T1HgYA6u3pAcCxKVBLYnkTw8z/l0A==", + "dev": true, + "dependencies": { + "@smithy/types": "^3.2.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@smithy/node-config-provider": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/@smithy/node-config-provider/-/node-config-provider-3.1.2.tgz", + "integrity": "sha512-388fEAa7+6ORj/BDC70peg3fyFBTTXJyXfXJ0Bwd6FYsRltePr2oGzIcm5AuC1WUSLtZ/dF+hYOnfTMs04rLvA==", + "dev": true, + "dependencies": { + "@smithy/property-provider": "^3.1.2", + "@smithy/shared-ini-file-loader": "^3.1.2", + "@smithy/types": "^3.2.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@smithy/node-http-handler": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-3.1.0.tgz", + "integrity": "sha512-pOpgB6B+VLXLwAyyvRz+ZAVXABlbAsJ2xvn3WZvrppAPImxwQOPFbeSUzWYMhpC8Tr7yQ3R8fG990QDhskkf1Q==", + "dev": true, + "dependencies": { + "@smithy/abort-controller": "^3.1.0", + "@smithy/protocol-http": "^4.0.2", + "@smithy/querystring-builder": "^3.0.2", + "@smithy/types": "^3.2.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@smithy/property-provider": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/@smithy/property-provider/-/property-provider-3.1.2.tgz", + "integrity": "sha512-Hzp32BpeFFexBpO1z+ts8okbq/VLzJBadxanJAo/Wf2CmvXMBp6Q/TLWr7Js6IbMEcr0pDZ02V3u1XZkuQUJaA==", + "dev": true, + "dependencies": { + "@smithy/types": "^3.2.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@smithy/protocol-http": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/@smithy/protocol-http/-/protocol-http-4.0.2.tgz", + "integrity": "sha512-X/90xNWIOqSR2tLUyWxVIBdatpm35DrL44rI/xoeBWUuanE0iyCXJpTcnqlOpnEzgcu0xCKE06+g70TTu2j7RQ==", + "dev": true, + "dependencies": { + "@smithy/types": "^3.2.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@smithy/querystring-builder": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@smithy/querystring-builder/-/querystring-builder-3.0.2.tgz", + "integrity": "sha512-xhv1+HacDYsOLdNt7zW+8Fe779KYAzmWvzs9bC5NlKM8QGYCwwuFwDBynhlU4D5twgi2pZ14Lm4h6RiAazCtmA==", + "dev": true, + "dependencies": { + "@smithy/types": "^3.2.0", + "@smithy/util-uri-escape": "^3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@smithy/querystring-parser": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@smithy/querystring-parser/-/querystring-parser-3.0.2.tgz", + "integrity": "sha512-C5hyRKgrZGPNh5QqIWzXnW+LXVrPmVQO0iJKjHeb5v3C61ZkP9QhrKmbfchcTyg/VnaE0tMNf/nmLpQlWuiqpg==", + "dev": true, + "dependencies": { + "@smithy/types": "^3.2.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@smithy/service-error-classification": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@smithy/service-error-classification/-/service-error-classification-3.0.2.tgz", + "integrity": "sha512-cu0WV2XRttItsuXlcM0kq5MKdphbMMmSd2CXF122dJ75NrFE0o7rruXFGfxAp3BKzgF/DMxX+PllIA/cj4FHMw==", + "dev": true, + "dependencies": { + "@smithy/types": "^3.2.0" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@smithy/shared-ini-file-loader": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/@smithy/shared-ini-file-loader/-/shared-ini-file-loader-3.1.2.tgz", + "integrity": "sha512-tgnXrXbLMO8vo6VeuqabMw/eTzQHlLmZx0TC0TjtjJghnD0Xl4pEnJtBjTJr6XF5fHMNrt5BcczDXHJT9yNQnA==", + "dev": true, + "dependencies": { + "@smithy/types": "^3.2.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@smithy/signature-v4": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/@smithy/signature-v4/-/signature-v4-3.1.1.tgz", + "integrity": "sha512-2/vlG86Sr489XX8TA/F+VDA+P04ESef04pSz0wRtlQBExcSPjqO08rvrkcas2zLnJ51i+7ukOURCkgqixBYjSQ==", + "dev": true, + "dependencies": { + "@smithy/is-array-buffer": "^3.0.0", + "@smithy/types": "^3.2.0", + "@smithy/util-hex-encoding": "^3.0.0", + "@smithy/util-middleware": "^3.0.2", + "@smithy/util-uri-escape": "^3.0.0", + "@smithy/util-utf8": "^3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@smithy/smithy-client": { + "version": "3.1.4", + "resolved": "https://registry.npmjs.org/@smithy/smithy-client/-/smithy-client-3.1.4.tgz", + "integrity": "sha512-y6xJROGrIoitjpwXLY7P9luDHvuT9jWpAluliuSFdBymFxcl6iyQjo9U/JhYfRHFNTruqsvKOrOESVuPGEcRmQ==", + "dev": true, + "dependencies": { + "@smithy/middleware-endpoint": "^3.0.3", + "@smithy/middleware-stack": "^3.0.2", + "@smithy/protocol-http": "^4.0.2", + "@smithy/types": "^3.2.0", + "@smithy/util-stream": "^3.0.4", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@smithy/types": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/@smithy/types/-/types-3.2.0.tgz", + "integrity": "sha512-cKyeKAPazZRVqm7QPvcPD2jEIt2wqDPAL1KJKb0f/5I7uhollvsWZuZKLclmyP6a+Jwmr3OV3t+X0pZUUHS9BA==", + "dev": true, + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@smithy/url-parser": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@smithy/url-parser/-/url-parser-3.0.2.tgz", + "integrity": "sha512-pRiPHrgibeAr4avtXDoBHmTLtthwA4l8jKYRfZjNgp+bBPyxDMPRg2TMJaYxqbKemvrOkHu9MIBTv2RkdNfD6w==", + "dev": true, + "dependencies": { + "@smithy/querystring-parser": "^3.0.2", + "@smithy/types": "^3.2.0", + "tslib": "^2.6.2" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@smithy/util-base64": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/util-base64/-/util-base64-3.0.0.tgz", + "integrity": "sha512-Kxvoh5Qtt0CDsfajiZOCpJxgtPHXOKwmM+Zy4waD43UoEMA+qPxxa98aE/7ZhdnBFZFXMOiBR5xbcaMhLtznQQ==", + "dev": true, + "dependencies": { + "@smithy/util-buffer-from": "^3.0.0", + "@smithy/util-utf8": "^3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@smithy/util-base64/node_modules/@smithy/util-buffer-from": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-3.0.0.tgz", + "integrity": "sha512-aEOHCgq5RWFbP+UDPvPot26EJHjOC+bRgse5A8V3FSShqd5E5UN4qc7zkwsvJPPAVsf73QwYcHN1/gt/rtLwQA==", + "dev": true, + "dependencies": { + "@smithy/is-array-buffer": "^3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@smithy/util-body-length-browser": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/util-body-length-browser/-/util-body-length-browser-3.0.0.tgz", + "integrity": "sha512-cbjJs2A1mLYmqmyVl80uoLTJhAcfzMOyPgjwAYusWKMdLeNtzmMz9YxNl3/jRLoxSS3wkqkf0jwNdtXWtyEBaQ==", + "dev": true, + "dependencies": { + "tslib": "^2.6.2" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@smithy/util-body-length-node": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/util-body-length-node/-/util-body-length-node-3.0.0.tgz", + "integrity": "sha512-Tj7pZ4bUloNUP6PzwhN7K386tmSmEET9QtQg0TgdNOnxhZvCssHji+oZTUIuzxECRfG8rdm2PMw2WCFs6eIYkA==", + "dev": true, + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@smithy/util-config-provider": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/util-config-provider/-/util-config-provider-3.0.0.tgz", + "integrity": "sha512-pbjk4s0fwq3Di/ANL+rCvJMKM5bzAQdE5S/6RL5NXgMExFAi6UgQMPOm5yPaIWPpr+EOXKXRonJ3FoxKf4mCJQ==", + "dev": true, + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@smithy/util-defaults-mode-browser": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/@smithy/util-defaults-mode-browser/-/util-defaults-mode-browser-3.0.6.tgz", + "integrity": "sha512-tAgoc++Eq+KL7g55+k108pn7nAob3GLWNEMbXhZIQyBcBNaE/o3+r4AEbae0A8bWvLRvArVsjeiuhMykGa04/A==", + "dev": true, + "dependencies": { + "@smithy/property-provider": "^3.1.2", + "@smithy/smithy-client": "^3.1.4", + "@smithy/types": "^3.2.0", + "bowser": "^2.11.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">= 10.0.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@smithy/util-defaults-mode-node": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/@smithy/util-defaults-mode-node/-/util-defaults-mode-node-3.0.6.tgz", + "integrity": "sha512-UNerul6/E8aiCyFTBHk+RSIZCo7m96d/N5K3FeO/wFeZP6oy5HAicLzxqa85Wjv7MkXSxSySX29L/LwTV/QMag==", + "dev": true, + "dependencies": { + "@smithy/config-resolver": "^3.0.3", + "@smithy/credential-provider-imds": "^3.1.2", + "@smithy/node-config-provider": "^3.1.2", + "@smithy/property-provider": "^3.1.2", + "@smithy/smithy-client": "^3.1.4", + "@smithy/types": "^3.2.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">= 10.0.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@smithy/util-endpoints": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/@smithy/util-endpoints/-/util-endpoints-2.0.3.tgz", + "integrity": "sha512-Dyi+pfLglDHSGsKSYunuUUSFM5V0tz7UDgv1Ex97yg+Xkn0Eb0rH0rcvl1n0MaJ11fac3HKDOH0DkALyQYCQag==", + "dev": true, + "dependencies": { + "@smithy/node-config-provider": "^3.1.2", + "@smithy/types": "^3.2.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@smithy/util-hex-encoding": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/util-hex-encoding/-/util-hex-encoding-3.0.0.tgz", + "integrity": "sha512-eFndh1WEK5YMUYvy3lPlVmYY/fZcQE1D8oSf41Id2vCeIkKJXPcYDCZD+4+xViI6b1XSd7tE+s5AmXzz5ilabQ==", + "dev": true, + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@smithy/util-middleware": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@smithy/util-middleware/-/util-middleware-3.0.2.tgz", + "integrity": "sha512-7WW5SD0XVrpfqljBYzS5rLR+EiDzl7wCVJZ9Lo6ChNFV4VYDk37Z1QI5w/LnYtU/QKnSawYoHRd7VjSyC8QRQQ==", + "dev": true, + "dependencies": { + "@smithy/types": "^3.2.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@smithy/util-retry": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@smithy/util-retry/-/util-retry-3.0.2.tgz", + "integrity": "sha512-HUVOb1k8p/IH6WFUjsLa+L9H1Zi/FAAB2CDOpWuffI1b2Txi6sknau8kNfC46Xrt39P1j2KDzCE1UlLa2eW5+A==", + "dev": true, + "dependencies": { + "@smithy/service-error-classification": "^3.0.2", + "@smithy/types": "^3.2.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@smithy/util-stream": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@smithy/util-stream/-/util-stream-3.0.4.tgz", + "integrity": "sha512-CcMioiaOOsEVdb09pS7ux1ij7QcQ2jE/cE1+iin1DXMeRgAEQN/47m7Xztu7KFQuQsj0A5YwB2UN45q97CqKCg==", + "dev": true, + "dependencies": { + "@smithy/fetch-http-handler": "^3.1.0", + "@smithy/node-http-handler": "^3.1.0", + "@smithy/types": "^3.2.0", + "@smithy/util-base64": "^3.0.0", + "@smithy/util-buffer-from": "^3.0.0", + "@smithy/util-hex-encoding": "^3.0.0", + "@smithy/util-utf8": "^3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@smithy/util-stream/node_modules/@smithy/util-buffer-from": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-3.0.0.tgz", + "integrity": "sha512-aEOHCgq5RWFbP+UDPvPot26EJHjOC+bRgse5A8V3FSShqd5E5UN4qc7zkwsvJPPAVsf73QwYcHN1/gt/rtLwQA==", + "dev": true, + "dependencies": { + "@smithy/is-array-buffer": "^3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@smithy/util-uri-escape": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/util-uri-escape/-/util-uri-escape-3.0.0.tgz", + "integrity": "sha512-LqR7qYLgZTD7nWLBecUi4aqolw8Mhza9ArpNEQ881MJJIU2sE5iHCK6TdyqqzcDLy0OPe10IY4T8ctVdtynubg==", + "dev": true, + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@smithy/util-utf8": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-3.0.0.tgz", + "integrity": "sha512-rUeT12bxFnplYDe815GXbq/oixEGHfRFFtcTF3YdDi/JaENIM6aSYYLJydG83UNzLXeRI5K8abYd/8Sp/QM0kA==", + "dev": true, + "dependencies": { + "@smithy/util-buffer-from": "^3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@smithy/util-utf8/node_modules/@smithy/util-buffer-from": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-3.0.0.tgz", + "integrity": "sha512-aEOHCgq5RWFbP+UDPvPot26EJHjOC+bRgse5A8V3FSShqd5E5UN4qc7zkwsvJPPAVsf73QwYcHN1/gt/rtLwQA==", + "dev": true, + "dependencies": { + "@smithy/is-array-buffer": "^3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@smithy/util-waiter": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/@smithy/util-waiter/-/util-waiter-3.1.1.tgz", + "integrity": "sha512-xT+Bbpe5sSrC7cCWSElOreDdWzqovR1V+7xrp+fmwGAA+TPYBb78iasaXjO1pa+65sY6JjW5GtGeIoJwCK9B1g==", + "dev": true, + "dependencies": { + "@smithy/abort-controller": "^3.1.0", + "@smithy/types": "^3.2.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, "node_modules/@aws-sdk/client-kms": { "version": "3.549.0", "resolved": "https://registry.npmjs.org/@aws-sdk/client-kms/-/client-kms-3.549.0.tgz", @@ -651,6 +1848,19 @@ "node": ">=14.0.0" } }, + "node_modules/@aws-sdk/endpoint-cache": { + "version": "3.572.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/endpoint-cache/-/endpoint-cache-3.572.0.tgz", + "integrity": "sha512-CzuRWMj/xtN9p9eP915nlPmlyniTzke732Ow/M60++gGgB3W+RtZyFftw3TEx+NzNhd1tH54dEcGiWdiNaBz3Q==", + "dev": true, + "dependencies": { + "mnemonist": "0.38.3", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, "node_modules/@aws-sdk/middleware-bucket-endpoint": { "version": "3.535.0", "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-bucket-endpoint/-/middleware-bucket-endpoint-3.535.0.tgz", @@ -669,6 +1879,102 @@ "node": ">=14.0.0" } }, + "node_modules/@aws-sdk/middleware-endpoint-discovery": { + "version": "3.598.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-endpoint-discovery/-/middleware-endpoint-discovery-3.598.0.tgz", + "integrity": "sha512-TaFo3rfapVP0FiddH2zDyA5R5XNk2M+zMeUZaBRveYamSQ11F+fMGcedBgbOsv7yNESvaZvjlcw2K+cx3jOchA==", + "dev": true, + "dependencies": { + "@aws-sdk/endpoint-cache": "3.572.0", + "@aws-sdk/types": "3.598.0", + "@smithy/node-config-provider": "^3.1.1", + "@smithy/protocol-http": "^4.0.1", + "@smithy/types": "^3.1.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/middleware-endpoint-discovery/node_modules/@aws-sdk/types": { + "version": "3.598.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.598.0.tgz", + "integrity": "sha512-742uRl6z7u0LFmZwDrFP6r1wlZcgVPw+/TilluDJmCAR8BgRw3IR+743kUXKBGd8QZDRW2n6v/PYsi/AWCDDMQ==", + "dev": true, + "dependencies": { + "@smithy/types": "^3.1.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/middleware-endpoint-discovery/node_modules/@smithy/node-config-provider": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/@smithy/node-config-provider/-/node-config-provider-3.1.2.tgz", + "integrity": "sha512-388fEAa7+6ORj/BDC70peg3fyFBTTXJyXfXJ0Bwd6FYsRltePr2oGzIcm5AuC1WUSLtZ/dF+hYOnfTMs04rLvA==", + "dev": true, + "dependencies": { + "@smithy/property-provider": "^3.1.2", + "@smithy/shared-ini-file-loader": "^3.1.2", + "@smithy/types": "^3.2.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/middleware-endpoint-discovery/node_modules/@smithy/property-provider": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/@smithy/property-provider/-/property-provider-3.1.2.tgz", + "integrity": "sha512-Hzp32BpeFFexBpO1z+ts8okbq/VLzJBadxanJAo/Wf2CmvXMBp6Q/TLWr7Js6IbMEcr0pDZ02V3u1XZkuQUJaA==", + "dev": true, + "dependencies": { + "@smithy/types": "^3.2.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/middleware-endpoint-discovery/node_modules/@smithy/protocol-http": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/@smithy/protocol-http/-/protocol-http-4.0.2.tgz", + "integrity": "sha512-X/90xNWIOqSR2tLUyWxVIBdatpm35DrL44rI/xoeBWUuanE0iyCXJpTcnqlOpnEzgcu0xCKE06+g70TTu2j7RQ==", + "dev": true, + "dependencies": { + "@smithy/types": "^3.2.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/middleware-endpoint-discovery/node_modules/@smithy/shared-ini-file-loader": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/@smithy/shared-ini-file-loader/-/shared-ini-file-loader-3.1.2.tgz", + "integrity": "sha512-tgnXrXbLMO8vo6VeuqabMw/eTzQHlLmZx0TC0TjtjJghnD0Xl4pEnJtBjTJr6XF5fHMNrt5BcczDXHJT9yNQnA==", + "dev": true, + "dependencies": { + "@smithy/types": "^3.2.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/middleware-endpoint-discovery/node_modules/@smithy/types": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/@smithy/types/-/types-3.2.0.tgz", + "integrity": "sha512-cKyeKAPazZRVqm7QPvcPD2jEIt2wqDPAL1KJKb0f/5I7uhollvsWZuZKLclmyP6a+Jwmr3OV3t+X0pZUUHS9BA==", + "dev": true, + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, "node_modules/@aws-sdk/middleware-expect-continue": { "version": "3.535.0", "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-expect-continue/-/middleware-expect-continue-3.535.0.tgz", @@ -1654,9 +2960,9 @@ "dev": true }, "node_modules/@biomejs/biome": { - "version": "1.7.3", - "resolved": "https://registry.npmjs.org/@biomejs/biome/-/biome-1.7.3.tgz", - "integrity": "sha512-ogFQI+fpXftr+tiahA6bIXwZ7CSikygASdqMtH07J2cUzrpjyTMVc9Y97v23c7/tL1xCZhM+W9k4hYIBm7Q6cQ==", + "version": "1.8.3", + "resolved": "https://registry.npmjs.org/@biomejs/biome/-/biome-1.8.3.tgz", + "integrity": "sha512-/uUV3MV+vyAczO+vKrPdOW0Iaet7UnJMU4bNMinggGJTAnBPjCoLEYcyYtYHNnUNYlv4xZMH6hVIQCAozq8d5w==", "dev": true, "hasInstallScript": true, "bin": { @@ -1670,20 +2976,20 @@ "url": "https://opencollective.com/biome" }, "optionalDependencies": { - "@biomejs/cli-darwin-arm64": "1.7.3", - "@biomejs/cli-darwin-x64": "1.7.3", - "@biomejs/cli-linux-arm64": "1.7.3", - "@biomejs/cli-linux-arm64-musl": "1.7.3", - "@biomejs/cli-linux-x64": "1.7.3", - "@biomejs/cli-linux-x64-musl": "1.7.3", - "@biomejs/cli-win32-arm64": "1.7.3", - "@biomejs/cli-win32-x64": "1.7.3" + "@biomejs/cli-darwin-arm64": "1.8.3", + "@biomejs/cli-darwin-x64": "1.8.3", + "@biomejs/cli-linux-arm64": "1.8.3", + "@biomejs/cli-linux-arm64-musl": "1.8.3", + "@biomejs/cli-linux-x64": "1.8.3", + "@biomejs/cli-linux-x64-musl": "1.8.3", + "@biomejs/cli-win32-arm64": "1.8.3", + "@biomejs/cli-win32-x64": "1.8.3" } }, "node_modules/@biomejs/cli-darwin-arm64": { - "version": "1.7.3", - "resolved": "https://registry.npmjs.org/@biomejs/cli-darwin-arm64/-/cli-darwin-arm64-1.7.3.tgz", - "integrity": "sha512-eDvLQWmGRqrPIRY7AIrkPHkQ3visEItJKkPYSHCscSDdGvKzYjmBJwG1Gu8+QC5ed6R7eiU63LEC0APFBobmfQ==", + "version": "1.8.3", + "resolved": "https://registry.npmjs.org/@biomejs/cli-darwin-arm64/-/cli-darwin-arm64-1.8.3.tgz", + "integrity": "sha512-9DYOjclFpKrH/m1Oz75SSExR8VKvNSSsLnVIqdnKexj6NwmiMlKk94Wa1kZEdv6MCOHGHgyyoV57Cw8WzL5n3A==", "cpu": [ "arm64" ], @@ -1697,9 +3003,9 @@ } }, "node_modules/@biomejs/cli-darwin-x64": { - "version": "1.7.3", - "resolved": "https://registry.npmjs.org/@biomejs/cli-darwin-x64/-/cli-darwin-x64-1.7.3.tgz", - "integrity": "sha512-JXCaIseKRER7dIURsVlAJacnm8SG5I0RpxZ4ya3dudASYUc68WGl4+FEN03ABY3KMIq7hcK1tzsJiWlmXyosZg==", + "version": "1.8.3", + "resolved": "https://registry.npmjs.org/@biomejs/cli-darwin-x64/-/cli-darwin-x64-1.8.3.tgz", + "integrity": "sha512-UeW44L/AtbmOF7KXLCoM+9PSgPo0IDcyEUfIoOXYeANaNXXf9mLUwV1GeF2OWjyic5zj6CnAJ9uzk2LT3v/wAw==", "cpu": [ "x64" ], @@ -1713,9 +3019,9 @@ } }, "node_modules/@biomejs/cli-linux-arm64": { - "version": "1.7.3", - "resolved": "https://registry.npmjs.org/@biomejs/cli-linux-arm64/-/cli-linux-arm64-1.7.3.tgz", - "integrity": "sha512-phNTBpo7joDFastnmZsFjYcDYobLTx4qR4oPvc9tJ486Bd1SfEVPHEvJdNJrMwUQK56T+TRClOQd/8X1nnjA9w==", + "version": "1.8.3", + "resolved": "https://registry.npmjs.org/@biomejs/cli-linux-arm64/-/cli-linux-arm64-1.8.3.tgz", + "integrity": "sha512-fed2ji8s+I/m8upWpTJGanqiJ0rnlHOK3DdxsyVLZQ8ClY6qLuPc9uehCREBifRJLl/iJyQpHIRufLDeotsPtw==", "cpu": [ "arm64" ], @@ -1729,9 +3035,9 @@ } }, "node_modules/@biomejs/cli-linux-arm64-musl": { - "version": "1.7.3", - "resolved": "https://registry.npmjs.org/@biomejs/cli-linux-arm64-musl/-/cli-linux-arm64-musl-1.7.3.tgz", - "integrity": "sha512-c8AlO45PNFZ1BYcwaKzdt46kYbuP6xPGuGQ6h4j3XiEDpyseRRUy/h+6gxj07XovmyxKnSX9GSZ6nVbZvcVUAw==", + "version": "1.8.3", + "resolved": "https://registry.npmjs.org/@biomejs/cli-linux-arm64-musl/-/cli-linux-arm64-musl-1.8.3.tgz", + "integrity": "sha512-9yjUfOFN7wrYsXt/T/gEWfvVxKlnh3yBpnScw98IF+oOeCYb5/b/+K7YNqKROV2i1DlMjg9g/EcN9wvj+NkMuQ==", "cpu": [ "arm64" ], @@ -1745,9 +3051,9 @@ } }, "node_modules/@biomejs/cli-linux-x64": { - "version": "1.7.3", - "resolved": "https://registry.npmjs.org/@biomejs/cli-linux-x64/-/cli-linux-x64-1.7.3.tgz", - "integrity": "sha512-vnedYcd5p4keT3iD48oSKjOIRPYcjSNNbd8MO1bKo9ajg3GwQXZLAH+0Cvlr+eMsO67/HddWmscSQwTFrC/uPA==", + "version": "1.8.3", + "resolved": "https://registry.npmjs.org/@biomejs/cli-linux-x64/-/cli-linux-x64-1.8.3.tgz", + "integrity": "sha512-I8G2QmuE1teISyT8ie1HXsjFRz9L1m5n83U1O6m30Kw+kPMPSKjag6QGUn+sXT8V+XWIZxFFBoTDEDZW2KPDDw==", "cpu": [ "x64" ], @@ -1761,9 +3067,9 @@ } }, "node_modules/@biomejs/cli-linux-x64-musl": { - "version": "1.7.3", - "resolved": "https://registry.npmjs.org/@biomejs/cli-linux-x64-musl/-/cli-linux-x64-musl-1.7.3.tgz", - "integrity": "sha512-UdEHKtYGWEX3eDmVWvQeT+z05T9/Sdt2+F/7zmMOFQ7boANeX8pcO6EkJPK3wxMudrApsNEKT26rzqK6sZRTRA==", + "version": "1.8.3", + "resolved": "https://registry.npmjs.org/@biomejs/cli-linux-x64-musl/-/cli-linux-x64-musl-1.8.3.tgz", + "integrity": "sha512-UHrGJX7PrKMKzPGoEsooKC9jXJMa28TUSMjcIlbDnIO4EAavCoVmNQaIuUSH0Ls2mpGMwUIf+aZJv657zfWWjA==", "cpu": [ "x64" ], @@ -1777,9 +3083,9 @@ } }, "node_modules/@biomejs/cli-win32-arm64": { - "version": "1.7.3", - "resolved": "https://registry.npmjs.org/@biomejs/cli-win32-arm64/-/cli-win32-arm64-1.7.3.tgz", - "integrity": "sha512-unNCDqUKjujYkkSxs7gFIfdasttbDC4+z0kYmcqzRk6yWVoQBL4dNLcCbdnJS+qvVDNdI9rHp2NwpQ0WAdla4Q==", + "version": "1.8.3", + "resolved": "https://registry.npmjs.org/@biomejs/cli-win32-arm64/-/cli-win32-arm64-1.8.3.tgz", + "integrity": "sha512-J+Hu9WvrBevfy06eU1Na0lpc7uR9tibm9maHynLIoAjLZpQU3IW+OKHUtyL8p6/3pT2Ju5t5emReeIS2SAxhkQ==", "cpu": [ "arm64" ], @@ -1793,9 +3099,9 @@ } }, "node_modules/@biomejs/cli-win32-x64": { - "version": "1.7.3", - "resolved": "https://registry.npmjs.org/@biomejs/cli-win32-x64/-/cli-win32-x64-1.7.3.tgz", - "integrity": "sha512-ZmByhbrnmz/UUFYB622CECwhKIPjJLLPr5zr3edhu04LzbfcOrz16VYeNq5dpO1ADG70FORhAJkaIGdaVBG00w==", + "version": "1.8.3", + "resolved": "https://registry.npmjs.org/@biomejs/cli-win32-x64/-/cli-win32-x64-1.8.3.tgz", + "integrity": "sha512-/PJ59vA1pnQeKahemaQf4Nyj7IKUvGQSc3Ze1uIGi+Wvr1xF7rGobSrAAG01T/gUDG21vkDsZYM03NAmPiVkqg==", "cpu": [ "x64" ], @@ -6010,6 +7316,15 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/mnemonist": { + "version": "0.38.3", + "resolved": "https://registry.npmjs.org/mnemonist/-/mnemonist-0.38.3.tgz", + "integrity": "sha512-2K9QYubXx/NAjv4VLq1d1Ly8pWNC5L3BrixtdkyTegXWJIqY+zLNDhhX/A+ZwWt70tB1S8H4BE8FLYEFyNoOBw==", + "dev": true, + "dependencies": { + "obliterator": "^1.6.1" + } + }, "node_modules/ms": { "version": "2.1.3", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", @@ -6085,6 +7400,12 @@ "node": ">=0.10.0" } }, + "node_modules/obliterator": { + "version": "1.6.1", + "resolved": "https://registry.npmjs.org/obliterator/-/obliterator-1.6.1.tgz", + "integrity": "sha512-9WXswnqINnnhOG/5SLimUlzuU1hFJUc8zkwyD59Sd+dPOMf05PmnYG/d6Q7HZ+KmgkZJa1PxRso6QdM3sTNHig==", + "dev": true + }, "node_modules/once": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", diff --git a/nodejs/package.json b/nodejs/package.json index 1db9810b..75ed1537 100644 --- a/nodejs/package.json +++ b/nodejs/package.json @@ -34,6 +34,7 @@ "devDependencies": { "@aws-sdk/client-kms": "^3.33.0", "@aws-sdk/client-s3": "^3.33.0", + "@aws-sdk/client-dynamodb": "^3.33.0", "@biomejs/biome": "^1.7.3", "@jest/globals": "^29.7.0", "@napi-rs/cli": "^2.18.0", @@ -68,7 +69,7 @@ "lint-ci": "biome ci .", "docs": "typedoc --plugin typedoc-plugin-markdown --out ../docs/src/js lancedb/index.ts", "lint": "biome check . && biome format .", - "lint-fix": "biome check --apply-unsafe . && biome format --write .", + "lint-fix": "biome check --write . && biome format --write .", "prepublishOnly": "napi prepublish -t npm", "test": "jest --verbose", "integration": "S3_TEST=1 npm run test", diff --git a/python/python/lancedb/db.py b/python/python/lancedb/db.py index dc7ccd8d..71073a28 100644 --- a/python/python/lancedb/db.py +++ b/python/python/lancedb/db.py @@ -28,12 +28,11 @@ from lancedb.common import data_to_reader, validate_schema from ._lancedb import connect as lancedb_connect from .pydantic import LanceModel -from .table import AsyncTable, LanceTable, Table, _sanitize_data +from .table import AsyncTable, LanceTable, Table, _sanitize_data, _table_path from .util import ( fs_from_uri, get_uri_location, get_uri_scheme, - join_uri, validate_table_name, ) @@ -457,16 +456,18 @@ class LanceDBConnection(DBConnection): If True, ignore if the table does not exist. """ try: - filesystem, path = fs_from_uri(self.uri) - table_path = join_uri(path, name + ".lance") - filesystem.delete_dir(table_path) + table_uri = _table_path(self.uri, name) + filesystem, path = fs_from_uri(table_uri) + filesystem.delete_dir(path) except FileNotFoundError: if not ignore_missing: raise @override def drop_database(self): - filesystem, path = fs_from_uri(self.uri) + dummy_table_uri = _table_path(self.uri, "dummy") + uri = dummy_table_uri.removesuffix("dummy.lance") + filesystem, path = fs_from_uri(uri) filesystem.delete_dir(path) diff --git a/python/python/lancedb/table.py b/python/python/lancedb/table.py index ca8fd881..b08c9de5 100644 --- a/python/python/lancedb/table.py +++ b/python/python/lancedb/table.py @@ -30,6 +30,7 @@ from typing import ( Tuple, Union, ) +from urllib.parse import urlparse import lance import numpy as np @@ -47,6 +48,7 @@ from .pydantic import LanceModel, model_to_dict from .query import AsyncQuery, AsyncVectorQuery, LanceQueryBuilder, Query from .util import ( fs_from_uri, + get_uri_scheme, inf_vector_column_query, join_uri, safe_import_pandas, @@ -208,6 +210,26 @@ def _to_record_batch_generator( yield b +def _table_path(base: str, table_name: str) -> str: + """ + Get a table path that can be used in PyArrow FS. + + Removes any weird schemes (such as "s3+ddb") and drops any query params. + """ + uri = _table_uri(base, table_name) + # Parse as URL + parsed = urlparse(uri) + # If scheme is s3+ddb, convert to s3 + if parsed.scheme == "s3+ddb": + parsed = parsed._replace(scheme="s3") + # Remove query parameters + return parsed._replace(query=None).geturl() + + +def _table_uri(base: str, table_name: str) -> str: + return join_uri(base, f"{table_name}.lance") + + class Table(ABC): """ A Table is a collection of Records in a LanceDB Database. @@ -908,7 +930,7 @@ class LanceTable(Table): @classmethod def open(cls, db, name, **kwargs): tbl = cls(db, name, **kwargs) - fs, path = fs_from_uri(tbl._dataset_uri) + fs, path = fs_from_uri(tbl._dataset_path) file_info = fs.get_file_info(path) if file_info.type != pa.fs.FileType.Directory: raise FileNotFoundError( @@ -918,9 +940,14 @@ class LanceTable(Table): return tbl - @property + @cached_property + def _dataset_path(self) -> str: + # Cacheable since it's deterministic + return _table_path(self._conn.uri, self.name) + + @cached_property def _dataset_uri(self) -> str: - return join_uri(self._conn.uri, f"{self.name}.lance") + return _table_uri(self._conn.uri, self.name) @property def _dataset(self) -> LanceDataset: @@ -1230,6 +1257,10 @@ class LanceTable(Table): ) def _get_fts_index_path(self): + if get_uri_scheme(self._dataset_uri) != "file": + raise NotImplementedError( + "Full-text search is not supported on object stores." + ) return join_uri(self._dataset_uri, "_indices", "tantivy") def add( diff --git a/python/python/lancedb/util.py b/python/python/lancedb/util.py index 4470754d..f3c933e5 100644 --- a/python/python/lancedb/util.py +++ b/python/python/lancedb/util.py @@ -139,8 +139,11 @@ def join_uri(base: Union[str, pathlib.Path], *parts: str) -> str: # using pathlib for local paths make this windows compatible # `get_uri_scheme` returns `file` for windows drive names (e.g. `c:\path`) return str(pathlib.Path(base, *parts)) - # for remote paths, just use os.path.join - return "/".join([p.rstrip("/") for p in [base, *parts]]) + else: + # there might be query parameters in the base URI + url = urlparse(base) + new_path = "/".join([p.rstrip("/") for p in [url.path, *parts]]) + return url._replace(path=new_path).geturl() def attempt_import_or_raise(module: str, mitigation=None): diff --git a/python/python/tests/test_s3.py b/python/python/tests/test_s3.py index 9766fc24..2b6ed38a 100644 --- a/python/python/tests/test_s3.py +++ b/python/python/tests/test_s3.py @@ -13,6 +13,8 @@ import asyncio import copy +from datetime import timedelta +import threading import pytest import pyarrow as pa @@ -25,6 +27,7 @@ CONFIG = { "aws_access_key_id": "ACCESSKEY", "aws_secret_access_key": "SECRETKEY", "aws_endpoint": "http://localhost:4566", + "dynamodb_endpoint": "http://localhost:4566", "aws_region": "us-east-1", } @@ -156,3 +159,104 @@ def test_s3_sse(s3_bucket: str, kms_key: str): validate_objects_encrypted(s3_bucket, path, kms_key) asyncio.run(test()) + + +@pytest.fixture(scope="module") +def commit_table(): + ddb = get_boto3_client("dynamodb", endpoint_url=CONFIG["dynamodb_endpoint"]) + table_name = "lance-integtest" + try: + ddb.delete_table(TableName=table_name) + except ddb.exceptions.ResourceNotFoundException: + pass + ddb.create_table( + TableName=table_name, + KeySchema=[ + {"AttributeName": "base_uri", "KeyType": "HASH"}, + {"AttributeName": "version", "KeyType": "RANGE"}, + ], + AttributeDefinitions=[ + {"AttributeName": "base_uri", "AttributeType": "S"}, + {"AttributeName": "version", "AttributeType": "N"}, + ], + ProvisionedThroughput={"ReadCapacityUnits": 1, "WriteCapacityUnits": 1}, + ) + yield table_name + ddb.delete_table(TableName=table_name) + + +@pytest.mark.s3_test +def test_s3_dynamodb(s3_bucket: str, commit_table: str): + storage_options = copy.copy(CONFIG) + + uri = f"s3+ddb://{s3_bucket}/test?ddbTableName={commit_table}" + data = pa.table({"x": [1, 2, 3]}) + + async def test(): + db = await lancedb.connect_async( + uri, + storage_options=storage_options, + read_consistency_interval=timedelta(0), + ) + + table = await db.create_table("test", data) + + # Five concurrent writers + async def insert(): + # independent table refs for true concurrent writes. + table = await db.open_table("test") + await table.add(data, mode="append") + + tasks = [insert() for _ in range(5)] + await asyncio.gather(*tasks) + + row_count = await table.count_rows() + assert row_count == 3 * 6 + + asyncio.run(test()) + + +@pytest.mark.s3_test +def test_s3_dynamodb_sync(s3_bucket: str, commit_table: str, monkeypatch): + # Sync API doesn't support storage_options, so we have to provide as env vars + for key, value in CONFIG.items(): + monkeypatch.setenv(key.upper(), value) + + uri = f"s3+ddb://{s3_bucket}/test2?ddbTableName={commit_table}" + data = pa.table({"x": ["a", "b", "c"]}) + + db = lancedb.connect( + uri, + read_consistency_interval=timedelta(0), + ) + + table = db.create_table("test_ddb_sync", data) + + # Five concurrent writers + def insert(): + table = db.open_table("test_ddb_sync") + table.add(data, mode="append") + + threads = [] + for _ in range(5): + thread = threading.Thread(target=insert) + threads.append(thread) + thread.start() + + for thread in threads: + thread.join() + + row_count = table.count_rows() + assert row_count == 3 * 6 + + # FTS indices should error since they are not supported yet. + with pytest.raises( + NotImplementedError, match="Full-text search is not supported on object stores." + ): + table.create_fts_index("x") + + # make sure list tables still works + assert db.table_names() == ["test_ddb_sync"] + db.drop_table("test_ddb_sync") + assert db.table_names() == [] + db.drop_database() diff --git a/rust/lancedb/Cargo.toml b/rust/lancedb/Cargo.toml index e80d1b09..290b16c7 100644 --- a/rust/lancedb/Cargo.toml +++ b/rust/lancedb/Cargo.toml @@ -55,10 +55,11 @@ walkdir = "2" # For s3 integration tests (dev deps aren't allowed to be optional atm) # We pin these because the content-length check breaks with localstack # https://github.com/smithy-lang/smithy-rs/releases/tag/release-2024-05-21 +aws-sdk-dynamodb = { version = "=1.23.0" } aws-sdk-s3 = { version = "=1.23.0" } aws-sdk-kms = { version = "=1.21.0" } aws-config = { version = "1.0" } -aws-smithy-runtime = { version = "=1.3.0" } +aws-smithy-runtime = { version = "=1.3.1" } [features] default = [] diff --git a/rust/lancedb/tests/object_store_test.rs b/rust/lancedb/tests/object_store_test.rs index 623b2484..ba9d8e1c 100644 --- a/rust/lancedb/tests/object_store_test.rs +++ b/rust/lancedb/tests/object_store_test.rs @@ -25,7 +25,9 @@ const CONFIG: &[(&str, &str)] = &[ ("access_key_id", "ACCESS_KEY"), ("secret_access_key", "SECRET_KEY"), ("endpoint", "http://127.0.0.1:4566"), + ("dynamodb_endpoint", "http://127.0.0.1:4566"), ("allow_http", "true"), + ("region", "us-east-1"), ]; async fn aws_config() -> SdkConfig { @@ -288,3 +290,126 @@ async fn test_encryption() -> Result<()> { Ok(()) } + +struct DynamoDBCommitTable(String); + +impl DynamoDBCommitTable { + async fn new(name: &str) -> Self { + let config = aws_config().await; + let client = aws_sdk_dynamodb::Client::new(&config); + + // In case it wasn't deleted earlier + Self::delete_table(client.clone(), name).await; + tokio::time::sleep(std::time::Duration::from_millis(200)).await; + + use aws_sdk_dynamodb::types::*; + + client + .create_table() + .table_name(name) + .attribute_definitions( + AttributeDefinition::builder() + .attribute_name("base_uri") + .attribute_type(ScalarAttributeType::S) + .build() + .unwrap(), + ) + .attribute_definitions( + AttributeDefinition::builder() + .attribute_name("version") + .attribute_type(ScalarAttributeType::N) + .build() + .unwrap(), + ) + .key_schema( + KeySchemaElement::builder() + .attribute_name("base_uri") + .key_type(KeyType::Hash) + .build() + .unwrap(), + ) + .key_schema( + KeySchemaElement::builder() + .attribute_name("version") + .key_type(KeyType::Range) + .build() + .unwrap(), + ) + .provisioned_throughput( + ProvisionedThroughput::builder() + .read_capacity_units(1) + .write_capacity_units(1) + .build() + .unwrap(), + ) + .send() + .await + .unwrap(); + + Self(name.to_string()) + } + + async fn delete_table(client: aws_sdk_dynamodb::Client, name: &str) { + match client + .delete_table() + .table_name(name) + .send() + .await + .map_err(|err| err.into_service_error()) + { + Ok(_) => {} + Err(e) if e.is_resource_not_found_exception() => {} + Err(e) => panic!("Failed to delete table: {}", e), + }; + } +} + +impl Drop for DynamoDBCommitTable { + fn drop(&mut self) { + let table_name = self.0.clone(); + tokio::task::spawn(async move { + let config = aws_config().await; + let client = aws_sdk_dynamodb::Client::new(&config); + Self::delete_table(client, &table_name).await; + }); + } +} + +#[tokio::test] +async fn test_concurrent_dynamodb_commit() { + // test concurrent commit on dynamodb + let bucket = S3Bucket::new("test-dynamodb").await; + let table = DynamoDBCommitTable::new("test_table").await; + + let uri = format!("s3+ddb://{}?ddbTableName={}", bucket.0, table.0); + let db = lancedb::connect(&uri) + .storage_options(CONFIG.iter().cloned()) + .execute() + .await + .unwrap(); + + let data = test_data(); + let data = RecordBatchIterator::new(vec![Ok(data.clone())], data.schema()); + + let table = db.create_table("test_table", data).execute().await.unwrap(); + + let data = test_data(); + + let mut tasks = vec![]; + for _ in 0..5 { + let table = db.open_table("test_table").execute().await.unwrap(); + let data = data.clone(); + tasks.push(tokio::spawn(async move { + let data = RecordBatchIterator::new(vec![Ok(data.clone())], data.schema()); + table.add(data).execute().await.unwrap(); + })); + } + + for task in tasks { + task.await.unwrap(); + } + + table.checkout_latest().await.unwrap(); + let row_count = table.count_rows(None).await.unwrap(); + assert_eq!(row_count, 18); +}