From 81af350d854f03d8800176279fe84abfdbf08fe9 Mon Sep 17 00:00:00 2001 From: Chang She <759245+changhiskhan@users.noreply.github.com> Date: Wed, 10 Jan 2024 16:44:00 -0800 Subject: [PATCH] feat(node): align incoming data to table schema (#802) --- node/package-lock.json | 520 +++++++++++++++++------------------------ node/package.json | 4 +- node/src/arrow.ts | 53 ++++- node/src/index.ts | 7 +- node/src/test/test.ts | 39 ++++ 5 files changed, 307 insertions(+), 316 deletions(-) diff --git a/node/package-lock.json b/node/package-lock.json index 541662fb..1373fb99 100644 --- a/node/package-lock.json +++ b/node/package-lock.json @@ -18,9 +18,9 @@ "win32" ], "dependencies": { - "@apache-arrow/ts": "^12.0.0", + "@apache-arrow/ts": "^14.0.2", "@neon-rs/load": "^0.0.74", - "apache-arrow": "^12.0.0", + "apache-arrow": "^14.0.2", "axios": "^1.4.0" }, "devDependencies": { @@ -60,32 +60,52 @@ "@lancedb/vectordb-win32-x64-msvc": "0.4.2" } }, + "node_modules/@75lb/deep-merge": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@75lb/deep-merge/-/deep-merge-1.1.1.tgz", + "integrity": "sha512-xvgv6pkMGBA6GwdyJbNAnDmfAIR/DfWhrj9jgWh3TY7gRm3KO46x/GPjRg6wJ0nOepwqrNxFfojebh0Df4h4Tw==", + "dependencies": { + "lodash.assignwith": "^4.2.0", + "typical": "^7.1.1" + }, + "engines": { + "node": ">=12.17" + } + }, + "node_modules/@75lb/deep-merge/node_modules/typical": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/typical/-/typical-7.1.1.tgz", + "integrity": "sha512-T+tKVNs6Wu7IWiAce5BgMd7OZfNYUndHwc5MknN+UHOudi7sGZzuHdCadllRuqJ3fPtgFtIH9+lt9qRv6lmpfA==", + "engines": { + "node": ">=12.17" + } + }, "node_modules/@apache-arrow/ts": { - "version": "12.0.0", - "resolved": "https://registry.npmjs.org/@apache-arrow/ts/-/ts-12.0.0.tgz", - "integrity": "sha512-ArJ3Fw5W9RAeNWuyCU2CdjL/nEAZSVDG1p3jz/ZtLo/q3NTz2w7HUCOJeszejH/5alGX+QirYrJ5c6BW++/P7g==", + "version": "14.0.2", + "resolved": "https://registry.npmjs.org/@apache-arrow/ts/-/ts-14.0.2.tgz", + "integrity": "sha512-CtwAvLkK0CZv7xsYeCo91ml6PvlfzAmAJZkRYuz2GNBwfYufj5SVi0iuSMwIMkcU/szVwvLdzORSLa5PlF/2ug==", "dependencies": { "@types/command-line-args": "5.2.0", "@types/command-line-usage": "5.0.2", - "@types/node": "18.14.5", + "@types/node": "20.3.0", "@types/pad-left": "2.1.1", "command-line-args": "5.2.1", - "command-line-usage": "6.1.3", - "flatbuffers": "23.3.3", + "command-line-usage": "7.0.1", + "flatbuffers": "23.5.26", "json-bignum": "^0.0.3", "pad-left": "^2.1.0", - "tslib": "^2.5.0" + "tslib": "^2.5.3" } }, "node_modules/@apache-arrow/ts/node_modules/@types/node": { - "version": "18.14.5", - "resolved": "https://registry.npmjs.org/@types/node/-/node-18.14.5.tgz", - "integrity": "sha512-CRT4tMK/DHYhw1fcCEBwME9CSaZNclxfzVMe7GsO6ULSwsttbj70wSiX6rZdIjGblu93sTJxLdhNIT85KKI7Qw==" + "version": "20.3.0", + "resolved": "https://registry.npmjs.org/@types/node/-/node-20.3.0.tgz", + "integrity": "sha512-cumHmIAf6On83X7yP+LrsEyUOf/YlociZelmpRYaGFydoaPdxdt80MAbu6vWerQT2COCp2nPvHdsbD7tHn/YlQ==" }, "node_modules/@apache-arrow/ts/node_modules/tslib": { - "version": "2.5.0", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.0.tgz", - "integrity": "sha512-336iVw3rtn2BUK7ORdIAHTyxHGRIHVReokCR3XjbckJMK7ms8FysBfhLR8IXnAgy7T0PTPNBWKiH514FOW/WSg==" + "version": "2.6.2", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.2.tgz", + "integrity": "sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q==" }, "node_modules/@cargo-messages/android-arm-eabi": { "version": "0.0.160", @@ -866,7 +886,6 @@ "version": "4.3.0", "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", - "dev": true, "dependencies": { "color-convert": "^2.0.1" }, @@ -891,34 +910,34 @@ } }, "node_modules/apache-arrow": { - "version": "12.0.0", - "resolved": "https://registry.npmjs.org/apache-arrow/-/apache-arrow-12.0.0.tgz", - "integrity": "sha512-uI+hnZZsGfNJiR/wG8j5yPQuDjmOHx4hZpkA743G4x3TlFrCpA3MMX7KUkIOIw0e/CwZ8NYuaMzaQsblA47qVA==", + "version": "14.0.2", + "resolved": "https://registry.npmjs.org/apache-arrow/-/apache-arrow-14.0.2.tgz", + "integrity": "sha512-EBO2xJN36/XoY81nhLcwCJgFwkboDZeyNQ+OPsG7bCoQjc2BT0aTyH/MR6SrL+LirSNz+cYqjGRlupMMlP1aEg==", "dependencies": { "@types/command-line-args": "5.2.0", "@types/command-line-usage": "5.0.2", - "@types/node": "18.14.5", + "@types/node": "20.3.0", "@types/pad-left": "2.1.1", "command-line-args": "5.2.1", - "command-line-usage": "6.1.3", - "flatbuffers": "23.3.3", + "command-line-usage": "7.0.1", + "flatbuffers": "23.5.26", "json-bignum": "^0.0.3", "pad-left": "^2.1.0", - "tslib": "^2.5.0" + "tslib": "^2.5.3" }, "bin": { "arrow2csv": "bin/arrow2csv.js" } }, "node_modules/apache-arrow/node_modules/@types/node": { - "version": "18.14.5", - "resolved": "https://registry.npmjs.org/@types/node/-/node-18.14.5.tgz", - "integrity": "sha512-CRT4tMK/DHYhw1fcCEBwME9CSaZNclxfzVMe7GsO6ULSwsttbj70wSiX6rZdIjGblu93sTJxLdhNIT85KKI7Qw==" + "version": "20.3.0", + "resolved": "https://registry.npmjs.org/@types/node/-/node-20.3.0.tgz", + "integrity": "sha512-cumHmIAf6On83X7yP+LrsEyUOf/YlociZelmpRYaGFydoaPdxdt80MAbu6vWerQT2COCp2nPvHdsbD7tHn/YlQ==" }, "node_modules/apache-arrow/node_modules/tslib": { - "version": "2.5.0", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.0.tgz", - "integrity": "sha512-336iVw3rtn2BUK7ORdIAHTyxHGRIHVReokCR3XjbckJMK7ms8FysBfhLR8IXnAgy7T0PTPNBWKiH514FOW/WSg==" + "version": "2.6.2", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.2.tgz", + "integrity": "sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q==" }, "node_modules/arg": { "version": "4.1.3", @@ -1170,7 +1189,6 @@ "version": "4.1.2", "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", - "dev": true, "dependencies": { "ansi-styles": "^4.1.0", "supports-color": "^7.1.0" @@ -1182,11 +1200,24 @@ "url": "https://github.com/chalk/chalk?sponsor=1" } }, + "node_modules/chalk-template": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/chalk-template/-/chalk-template-0.4.0.tgz", + "integrity": "sha512-/ghrgmhfY8RaSdeo43hNXxpoHAtxdbskUHjPpfqUWGttFgycUhYPGx3YZBCnUCvOa7Doivn1IZec3DEGFoMgLg==", + "dependencies": { + "chalk": "^4.1.2" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/chalk-template?sponsor=1" + } + }, "node_modules/chalk/node_modules/supports-color": { "version": "7.2.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", - "dev": true, "dependencies": { "has-flag": "^4.0.0" }, @@ -1245,7 +1276,6 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", - "dev": true, "dependencies": { "color-name": "~1.1.4" }, @@ -1256,8 +1286,7 @@ "node_modules/color-name": { "version": "1.1.4", "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", - "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", - "dev": true + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" }, "node_modules/combined-stream": { "version": "1.0.8", @@ -1285,97 +1314,33 @@ } }, "node_modules/command-line-usage": { - "version": "6.1.3", - "resolved": "https://registry.npmjs.org/command-line-usage/-/command-line-usage-6.1.3.tgz", - "integrity": "sha512-sH5ZSPr+7UStsloltmDh7Ce5fb8XPlHyoPzTpyyMuYCtervL65+ubVZ6Q61cFtFl62UyJlc8/JwERRbAFPUqgw==", + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/command-line-usage/-/command-line-usage-7.0.1.tgz", + "integrity": "sha512-NCyznE//MuTjwi3y84QVUGEOT+P5oto1e1Pk/jFPVdPPfsG03qpTIl3yw6etR+v73d0lXsoojRpvbru2sqePxQ==", "dependencies": { - "array-back": "^4.0.2", - "chalk": "^2.4.2", - "table-layout": "^1.0.2", - "typical": "^5.2.0" + "array-back": "^6.2.2", + "chalk-template": "^0.4.0", + "table-layout": "^3.0.0", + "typical": "^7.1.1" }, "engines": { - "node": ">=8.0.0" - } - }, - "node_modules/command-line-usage/node_modules/ansi-styles": { - "version": "3.2.1", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", - "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", - "dependencies": { - "color-convert": "^1.9.0" - }, - "engines": { - "node": ">=4" + "node": ">=12.20.0" } }, "node_modules/command-line-usage/node_modules/array-back": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/array-back/-/array-back-4.0.2.tgz", - "integrity": "sha512-NbdMezxqf94cnNfWLL7V/im0Ub+Anbb0IoZhvzie8+4HJ4nMQuzHuy49FkGYCJK2yAloZ3meiB6AVMClbrI1vg==", + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/array-back/-/array-back-6.2.2.tgz", + "integrity": "sha512-gUAZ7HPyb4SJczXAMUXMGAvI976JoK3qEx9v1FTmeYuJj0IBiaKttG1ydtGKdkfqWkIkouke7nG8ufGy77+Cvw==", "engines": { - "node": ">=8" - } - }, - "node_modules/command-line-usage/node_modules/chalk": { - "version": "2.4.2", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", - "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", - "dependencies": { - "ansi-styles": "^3.2.1", - "escape-string-regexp": "^1.0.5", - "supports-color": "^5.3.0" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/command-line-usage/node_modules/color-convert": { - "version": "1.9.3", - "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", - "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", - "dependencies": { - "color-name": "1.1.3" - } - }, - "node_modules/command-line-usage/node_modules/color-name": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", - "integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==" - }, - "node_modules/command-line-usage/node_modules/escape-string-regexp": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", - "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==", - "engines": { - "node": ">=0.8.0" - } - }, - "node_modules/command-line-usage/node_modules/has-flag": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", - "integrity": "sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==", - "engines": { - "node": ">=4" - } - }, - "node_modules/command-line-usage/node_modules/supports-color": { - "version": "5.5.0", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", - "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", - "dependencies": { - "has-flag": "^3.0.0" - }, - "engines": { - "node": ">=4" + "node": ">=12.17" } }, "node_modules/command-line-usage/node_modules/typical": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/typical/-/typical-5.2.0.tgz", - "integrity": "sha512-dvdQgNDNJo+8B2uBQoqdb11eUCE1JQXhvjC/CZtgvZseVd5TYMXnq0+vuUemXbd/Se29cTaUuPX3YIc2xgbvIg==", + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/typical/-/typical-7.1.1.tgz", + "integrity": "sha512-T+tKVNs6Wu7IWiAce5BgMd7OZfNYUndHwc5MknN+UHOudi7sGZzuHdCadllRuqJ3fPtgFtIH9+lt9qRv6lmpfA==", "engines": { - "node": ">=8" + "node": ">=12.17" } }, "node_modules/concat-map": { @@ -1451,14 +1416,6 @@ "node": ">=6" } }, - "node_modules/deep-extend": { - "version": "0.6.0", - "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz", - "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==", - "engines": { - "node": ">=4.0.0" - } - }, "node_modules/deep-is": { "version": "0.1.4", "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz", @@ -2237,9 +2194,9 @@ } }, "node_modules/flatbuffers": { - "version": "23.3.3", - "resolved": "https://registry.npmjs.org/flatbuffers/-/flatbuffers-23.3.3.tgz", - "integrity": "sha512-jmreOaAT1t55keaf+Z259Tvh8tR/Srry9K8dgCgvizhKSEr6gLGgaOJI2WFL5fkOpGOGRZwxUrlFn0GCmXUy6g==" + "version": "23.5.26", + "resolved": "https://registry.npmjs.org/flatbuffers/-/flatbuffers-23.5.26.tgz", + "integrity": "sha512-vE+SI9vrJDwi1oETtTIFldC/o9GsVKRM+s6EL0nQgxXlYV1Vc4Tk30hj4xGICftInKQKj1F3up2n8UbIVobISQ==" }, "node_modules/flatted": { "version": "3.2.7", @@ -2535,7 +2492,6 @@ "version": "4.0.0", "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", - "dev": true, "engines": { "node": ">=8" } @@ -3048,6 +3004,11 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/lodash.assignwith": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/lodash.assignwith/-/lodash.assignwith-4.2.0.tgz", + "integrity": "sha512-ZznplvbvtjK2gMvnQ1BR/zqPFZmS6jbK4p+6Up4xcRYA7yMIwxHCfbTcrYxXKzzqLsQ05eJPVznEW3tuwV7k1g==" + }, "node_modules/lodash.camelcase": { "version": "4.3.0", "resolved": "https://registry.npmjs.org/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz", @@ -3668,14 +3629,6 @@ "node": ">=8.10.0" } }, - "node_modules/reduce-flatten": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/reduce-flatten/-/reduce-flatten-2.0.0.tgz", - "integrity": "sha512-EJ4UNY/U1t2P/2k6oqotuX2Cc3T6nxJwsM0N0asT7dhrtH1ltUxDn4NalSYmPE2rCkVpcf/X6R0wDwcFpzhd4w==", - "engines": { - "node": ">=6" - } - }, "node_modules/regexp.prototype.flags": { "version": "1.5.0", "resolved": "https://registry.npmjs.org/regexp.prototype.flags/-/regexp.prototype.flags-1.5.0.tgz", @@ -3965,6 +3918,14 @@ "source-map": "^0.6.0" } }, + "node_modules/stream-read-all": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/stream-read-all/-/stream-read-all-3.0.1.tgz", + "integrity": "sha512-EWZT9XOceBPlVJRrYcykW8jyRSZYbkb/0ZK36uLEmoWVO5gxBOnntNTseNzfREsqxqdfEGQrD8SXQ3QWbBmq8A==", + "engines": { + "node": ">=10" + } + }, "node_modules/string-width": { "version": "4.2.3", "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", @@ -4082,33 +4043,39 @@ } }, "node_modules/table-layout": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/table-layout/-/table-layout-1.0.2.tgz", - "integrity": "sha512-qd/R7n5rQTRFi+Zf2sk5XVVd9UQl6ZkduPFC3S7WEGJAmetDTjY3qPN50eSKzwuzEyQKy5TN2TiZdkIjos2L6A==", + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/table-layout/-/table-layout-3.0.2.tgz", + "integrity": "sha512-rpyNZYRw+/C+dYkcQ3Pr+rLxW4CfHpXjPDnG7lYhdRoUcZTUt+KEsX+94RGp/aVp/MQU35JCITv2T/beY4m+hw==", "dependencies": { - "array-back": "^4.0.1", - "deep-extend": "~0.6.0", - "typical": "^5.2.0", - "wordwrapjs": "^4.0.0" + "@75lb/deep-merge": "^1.1.1", + "array-back": "^6.2.2", + "command-line-args": "^5.2.1", + "command-line-usage": "^7.0.0", + "stream-read-all": "^3.0.1", + "typical": "^7.1.1", + "wordwrapjs": "^5.1.0" + }, + "bin": { + "table-layout": "bin/cli.js" }, "engines": { - "node": ">=8.0.0" + "node": ">=12.17" } }, "node_modules/table-layout/node_modules/array-back": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/array-back/-/array-back-4.0.2.tgz", - "integrity": "sha512-NbdMezxqf94cnNfWLL7V/im0Ub+Anbb0IoZhvzie8+4HJ4nMQuzHuy49FkGYCJK2yAloZ3meiB6AVMClbrI1vg==", + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/array-back/-/array-back-6.2.2.tgz", + "integrity": "sha512-gUAZ7HPyb4SJczXAMUXMGAvI976JoK3qEx9v1FTmeYuJj0IBiaKttG1ydtGKdkfqWkIkouke7nG8ufGy77+Cvw==", "engines": { - "node": ">=8" + "node": ">=12.17" } }, "node_modules/table-layout/node_modules/typical": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/typical/-/typical-5.2.0.tgz", - "integrity": "sha512-dvdQgNDNJo+8B2uBQoqdb11eUCE1JQXhvjC/CZtgvZseVd5TYMXnq0+vuUemXbd/Se29cTaUuPX3YIc2xgbvIg==", + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/typical/-/typical-7.1.1.tgz", + "integrity": "sha512-T+tKVNs6Wu7IWiAce5BgMd7OZfNYUndHwc5MknN+UHOudi7sGZzuHdCadllRuqJ3fPtgFtIH9+lt9qRv6lmpfA==", "engines": { - "node": ">=8" + "node": ">=12.17" } }, "node_modules/temp": { @@ -4553,23 +4520,11 @@ "dev": true }, "node_modules/wordwrapjs": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/wordwrapjs/-/wordwrapjs-4.0.1.tgz", - "integrity": "sha512-kKlNACbvHrkpIw6oPeYDSmdCTu2hdMHoyXLTcUKala++lx5Y+wjJ/e474Jqv5abnVmwxw08DiTuHmw69lJGksA==", - "dependencies": { - "reduce-flatten": "^2.0.0", - "typical": "^5.2.0" - }, + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/wordwrapjs/-/wordwrapjs-5.1.0.tgz", + "integrity": "sha512-JNjcULU2e4KJwUNv6CHgI46UvDGitb6dGryHajXTDiLgg1/RiGoPSDw4kZfYnwGtEXf2ZMeIewDQgFGzkCB2Sg==", "engines": { - "node": ">=8.0.0" - } - }, - "node_modules/wordwrapjs/node_modules/typical": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/typical/-/typical-5.2.0.tgz", - "integrity": "sha512-dvdQgNDNJo+8B2uBQoqdb11eUCE1JQXhvjC/CZtgvZseVd5TYMXnq0+vuUemXbd/Se29cTaUuPX3YIc2xgbvIg==", - "engines": { - "node": ">=8" + "node": ">=12.17" } }, "node_modules/workerpool": { @@ -4690,32 +4645,48 @@ } }, "dependencies": { + "@75lb/deep-merge": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@75lb/deep-merge/-/deep-merge-1.1.1.tgz", + "integrity": "sha512-xvgv6pkMGBA6GwdyJbNAnDmfAIR/DfWhrj9jgWh3TY7gRm3KO46x/GPjRg6wJ0nOepwqrNxFfojebh0Df4h4Tw==", + "requires": { + "lodash.assignwith": "^4.2.0", + "typical": "^7.1.1" + }, + "dependencies": { + "typical": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/typical/-/typical-7.1.1.tgz", + "integrity": "sha512-T+tKVNs6Wu7IWiAce5BgMd7OZfNYUndHwc5MknN+UHOudi7sGZzuHdCadllRuqJ3fPtgFtIH9+lt9qRv6lmpfA==" + } + } + }, "@apache-arrow/ts": { - "version": "12.0.0", - "resolved": "https://registry.npmjs.org/@apache-arrow/ts/-/ts-12.0.0.tgz", - "integrity": "sha512-ArJ3Fw5W9RAeNWuyCU2CdjL/nEAZSVDG1p3jz/ZtLo/q3NTz2w7HUCOJeszejH/5alGX+QirYrJ5c6BW++/P7g==", + "version": "14.0.2", + "resolved": "https://registry.npmjs.org/@apache-arrow/ts/-/ts-14.0.2.tgz", + "integrity": "sha512-CtwAvLkK0CZv7xsYeCo91ml6PvlfzAmAJZkRYuz2GNBwfYufj5SVi0iuSMwIMkcU/szVwvLdzORSLa5PlF/2ug==", "requires": { "@types/command-line-args": "5.2.0", "@types/command-line-usage": "5.0.2", - "@types/node": "18.14.5", + "@types/node": "20.3.0", "@types/pad-left": "2.1.1", "command-line-args": "5.2.1", - "command-line-usage": "6.1.3", - "flatbuffers": "23.3.3", + "command-line-usage": "7.0.1", + "flatbuffers": "23.5.26", "json-bignum": "^0.0.3", "pad-left": "^2.1.0", - "tslib": "^2.5.0" + "tslib": "^2.5.3" }, "dependencies": { "@types/node": { - "version": "18.14.5", - "resolved": "https://registry.npmjs.org/@types/node/-/node-18.14.5.tgz", - "integrity": "sha512-CRT4tMK/DHYhw1fcCEBwME9CSaZNclxfzVMe7GsO6ULSwsttbj70wSiX6rZdIjGblu93sTJxLdhNIT85KKI7Qw==" + "version": "20.3.0", + "resolved": "https://registry.npmjs.org/@types/node/-/node-20.3.0.tgz", + "integrity": "sha512-cumHmIAf6On83X7yP+LrsEyUOf/YlociZelmpRYaGFydoaPdxdt80MAbu6vWerQT2COCp2nPvHdsbD7tHn/YlQ==" }, "tslib": { - "version": "2.5.0", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.0.tgz", - "integrity": "sha512-336iVw3rtn2BUK7ORdIAHTyxHGRIHVReokCR3XjbckJMK7ms8FysBfhLR8IXnAgy7T0PTPNBWKiH514FOW/WSg==" + "version": "2.6.2", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.2.tgz", + "integrity": "sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q==" } } }, @@ -5268,7 +5239,6 @@ "version": "4.3.0", "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", - "dev": true, "requires": { "color-convert": "^2.0.1" } @@ -5284,31 +5254,31 @@ } }, "apache-arrow": { - "version": "12.0.0", - "resolved": "https://registry.npmjs.org/apache-arrow/-/apache-arrow-12.0.0.tgz", - "integrity": "sha512-uI+hnZZsGfNJiR/wG8j5yPQuDjmOHx4hZpkA743G4x3TlFrCpA3MMX7KUkIOIw0e/CwZ8NYuaMzaQsblA47qVA==", + "version": "14.0.2", + "resolved": "https://registry.npmjs.org/apache-arrow/-/apache-arrow-14.0.2.tgz", + "integrity": "sha512-EBO2xJN36/XoY81nhLcwCJgFwkboDZeyNQ+OPsG7bCoQjc2BT0aTyH/MR6SrL+LirSNz+cYqjGRlupMMlP1aEg==", "requires": { "@types/command-line-args": "5.2.0", "@types/command-line-usage": "5.0.2", - "@types/node": "18.14.5", + "@types/node": "20.3.0", "@types/pad-left": "2.1.1", "command-line-args": "5.2.1", - "command-line-usage": "6.1.3", - "flatbuffers": "23.3.3", + "command-line-usage": "7.0.1", + "flatbuffers": "23.5.26", "json-bignum": "^0.0.3", "pad-left": "^2.1.0", - "tslib": "^2.5.0" + "tslib": "^2.5.3" }, "dependencies": { "@types/node": { - "version": "18.14.5", - "resolved": "https://registry.npmjs.org/@types/node/-/node-18.14.5.tgz", - "integrity": "sha512-CRT4tMK/DHYhw1fcCEBwME9CSaZNclxfzVMe7GsO6ULSwsttbj70wSiX6rZdIjGblu93sTJxLdhNIT85KKI7Qw==" + "version": "20.3.0", + "resolved": "https://registry.npmjs.org/@types/node/-/node-20.3.0.tgz", + "integrity": "sha512-cumHmIAf6On83X7yP+LrsEyUOf/YlociZelmpRYaGFydoaPdxdt80MAbu6vWerQT2COCp2nPvHdsbD7tHn/YlQ==" }, "tslib": { - "version": "2.5.0", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.0.tgz", - "integrity": "sha512-336iVw3rtn2BUK7ORdIAHTyxHGRIHVReokCR3XjbckJMK7ms8FysBfhLR8IXnAgy7T0PTPNBWKiH514FOW/WSg==" + "version": "2.6.2", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.2.tgz", + "integrity": "sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q==" } } }, @@ -5505,7 +5475,6 @@ "version": "4.1.2", "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", - "dev": true, "requires": { "ansi-styles": "^4.1.0", "supports-color": "^7.1.0" @@ -5515,13 +5484,20 @@ "version": "7.2.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", - "dev": true, "requires": { "has-flag": "^4.0.0" } } } }, + "chalk-template": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/chalk-template/-/chalk-template-0.4.0.tgz", + "integrity": "sha512-/ghrgmhfY8RaSdeo43hNXxpoHAtxdbskUHjPpfqUWGttFgycUhYPGx3YZBCnUCvOa7Doivn1IZec3DEGFoMgLg==", + "requires": { + "chalk": "^4.1.2" + } + }, "check-error": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/check-error/-/check-error-1.0.2.tgz", @@ -5559,7 +5535,6 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", - "dev": true, "requires": { "color-name": "~1.1.4" } @@ -5567,8 +5542,7 @@ "color-name": { "version": "1.1.4", "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", - "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", - "dev": true + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" }, "combined-stream": { "version": "1.0.8", @@ -5590,74 +5564,25 @@ } }, "command-line-usage": { - "version": "6.1.3", - "resolved": "https://registry.npmjs.org/command-line-usage/-/command-line-usage-6.1.3.tgz", - "integrity": "sha512-sH5ZSPr+7UStsloltmDh7Ce5fb8XPlHyoPzTpyyMuYCtervL65+ubVZ6Q61cFtFl62UyJlc8/JwERRbAFPUqgw==", + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/command-line-usage/-/command-line-usage-7.0.1.tgz", + "integrity": "sha512-NCyznE//MuTjwi3y84QVUGEOT+P5oto1e1Pk/jFPVdPPfsG03qpTIl3yw6etR+v73d0lXsoojRpvbru2sqePxQ==", "requires": { - "array-back": "^4.0.2", - "chalk": "^2.4.2", - "table-layout": "^1.0.2", - "typical": "^5.2.0" + "array-back": "^6.2.2", + "chalk-template": "^0.4.0", + "table-layout": "^3.0.0", + "typical": "^7.1.1" }, "dependencies": { - "ansi-styles": { - "version": "3.2.1", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", - "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", - "requires": { - "color-convert": "^1.9.0" - } - }, "array-back": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/array-back/-/array-back-4.0.2.tgz", - "integrity": "sha512-NbdMezxqf94cnNfWLL7V/im0Ub+Anbb0IoZhvzie8+4HJ4nMQuzHuy49FkGYCJK2yAloZ3meiB6AVMClbrI1vg==" - }, - "chalk": { - "version": "2.4.2", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", - "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", - "requires": { - "ansi-styles": "^3.2.1", - "escape-string-regexp": "^1.0.5", - "supports-color": "^5.3.0" - } - }, - "color-convert": { - "version": "1.9.3", - "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", - "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", - "requires": { - "color-name": "1.1.3" - } - }, - "color-name": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", - "integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==" - }, - "escape-string-regexp": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", - "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==" - }, - "has-flag": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", - "integrity": "sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==" - }, - "supports-color": { - "version": "5.5.0", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", - "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", - "requires": { - "has-flag": "^3.0.0" - } + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/array-back/-/array-back-6.2.2.tgz", + "integrity": "sha512-gUAZ7HPyb4SJczXAMUXMGAvI976JoK3qEx9v1FTmeYuJj0IBiaKttG1ydtGKdkfqWkIkouke7nG8ufGy77+Cvw==" }, "typical": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/typical/-/typical-5.2.0.tgz", - "integrity": "sha512-dvdQgNDNJo+8B2uBQoqdb11eUCE1JQXhvjC/CZtgvZseVd5TYMXnq0+vuUemXbd/Se29cTaUuPX3YIc2xgbvIg==" + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/typical/-/typical-7.1.1.tgz", + "integrity": "sha512-T+tKVNs6Wu7IWiAce5BgMd7OZfNYUndHwc5MknN+UHOudi7sGZzuHdCadllRuqJ3fPtgFtIH9+lt9qRv6lmpfA==" } } }, @@ -5716,11 +5641,6 @@ "type-detect": "^4.0.0" } }, - "deep-extend": { - "version": "0.6.0", - "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz", - "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==" - }, "deep-is": { "version": "0.1.4", "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz", @@ -6297,9 +6217,9 @@ } }, "flatbuffers": { - "version": "23.3.3", - "resolved": "https://registry.npmjs.org/flatbuffers/-/flatbuffers-23.3.3.tgz", - "integrity": "sha512-jmreOaAT1t55keaf+Z259Tvh8tR/Srry9K8dgCgvizhKSEr6gLGgaOJI2WFL5fkOpGOGRZwxUrlFn0GCmXUy6g==" + "version": "23.5.26", + "resolved": "https://registry.npmjs.org/flatbuffers/-/flatbuffers-23.5.26.tgz", + "integrity": "sha512-vE+SI9vrJDwi1oETtTIFldC/o9GsVKRM+s6EL0nQgxXlYV1Vc4Tk30hj4xGICftInKQKj1F3up2n8UbIVobISQ==" }, "flatted": { "version": "3.2.7", @@ -6502,8 +6422,7 @@ "has-flag": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", - "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", - "dev": true + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==" }, "has-property-descriptors": { "version": "1.0.0", @@ -6856,6 +6775,11 @@ "p-locate": "^5.0.0" } }, + "lodash.assignwith": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/lodash.assignwith/-/lodash.assignwith-4.2.0.tgz", + "integrity": "sha512-ZznplvbvtjK2gMvnQ1BR/zqPFZmS6jbK4p+6Up4xcRYA7yMIwxHCfbTcrYxXKzzqLsQ05eJPVznEW3tuwV7k1g==" + }, "lodash.camelcase": { "version": "4.3.0", "resolved": "https://registry.npmjs.org/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz", @@ -7323,11 +7247,6 @@ "picomatch": "^2.2.1" } }, - "reduce-flatten": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/reduce-flatten/-/reduce-flatten-2.0.0.tgz", - "integrity": "sha512-EJ4UNY/U1t2P/2k6oqotuX2Cc3T6nxJwsM0N0asT7dhrtH1ltUxDn4NalSYmPE2rCkVpcf/X6R0wDwcFpzhd4w==" - }, "regexp.prototype.flags": { "version": "1.5.0", "resolved": "https://registry.npmjs.org/regexp.prototype.flags/-/regexp.prototype.flags-1.5.0.tgz", @@ -7523,6 +7442,11 @@ "source-map": "^0.6.0" } }, + "stream-read-all": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/stream-read-all/-/stream-read-all-3.0.1.tgz", + "integrity": "sha512-EWZT9XOceBPlVJRrYcykW8jyRSZYbkb/0ZK36uLEmoWVO5gxBOnntNTseNzfREsqxqdfEGQrD8SXQ3QWbBmq8A==" + }, "string-width": { "version": "4.2.3", "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", @@ -7604,25 +7528,28 @@ "dev": true }, "table-layout": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/table-layout/-/table-layout-1.0.2.tgz", - "integrity": "sha512-qd/R7n5rQTRFi+Zf2sk5XVVd9UQl6ZkduPFC3S7WEGJAmetDTjY3qPN50eSKzwuzEyQKy5TN2TiZdkIjos2L6A==", + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/table-layout/-/table-layout-3.0.2.tgz", + "integrity": "sha512-rpyNZYRw+/C+dYkcQ3Pr+rLxW4CfHpXjPDnG7lYhdRoUcZTUt+KEsX+94RGp/aVp/MQU35JCITv2T/beY4m+hw==", "requires": { - "array-back": "^4.0.1", - "deep-extend": "~0.6.0", - "typical": "^5.2.0", - "wordwrapjs": "^4.0.0" + "@75lb/deep-merge": "^1.1.1", + "array-back": "^6.2.2", + "command-line-args": "^5.2.1", + "command-line-usage": "^7.0.0", + "stream-read-all": "^3.0.1", + "typical": "^7.1.1", + "wordwrapjs": "^5.1.0" }, "dependencies": { "array-back": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/array-back/-/array-back-4.0.2.tgz", - "integrity": "sha512-NbdMezxqf94cnNfWLL7V/im0Ub+Anbb0IoZhvzie8+4HJ4nMQuzHuy49FkGYCJK2yAloZ3meiB6AVMClbrI1vg==" + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/array-back/-/array-back-6.2.2.tgz", + "integrity": "sha512-gUAZ7HPyb4SJczXAMUXMGAvI976JoK3qEx9v1FTmeYuJj0IBiaKttG1ydtGKdkfqWkIkouke7nG8ufGy77+Cvw==" }, "typical": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/typical/-/typical-5.2.0.tgz", - "integrity": "sha512-dvdQgNDNJo+8B2uBQoqdb11eUCE1JQXhvjC/CZtgvZseVd5TYMXnq0+vuUemXbd/Se29cTaUuPX3YIc2xgbvIg==" + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/typical/-/typical-7.1.1.tgz", + "integrity": "sha512-T+tKVNs6Wu7IWiAce5BgMd7OZfNYUndHwc5MknN+UHOudi7sGZzuHdCadllRuqJ3fPtgFtIH9+lt9qRv6lmpfA==" } } }, @@ -7940,20 +7867,9 @@ "dev": true }, "wordwrapjs": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/wordwrapjs/-/wordwrapjs-4.0.1.tgz", - "integrity": "sha512-kKlNACbvHrkpIw6oPeYDSmdCTu2hdMHoyXLTcUKala++lx5Y+wjJ/e474Jqv5abnVmwxw08DiTuHmw69lJGksA==", - "requires": { - "reduce-flatten": "^2.0.0", - "typical": "^5.2.0" - }, - "dependencies": { - "typical": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/typical/-/typical-5.2.0.tgz", - "integrity": "sha512-dvdQgNDNJo+8B2uBQoqdb11eUCE1JQXhvjC/CZtgvZseVd5TYMXnq0+vuUemXbd/Se29cTaUuPX3YIc2xgbvIg==" - } - } + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/wordwrapjs/-/wordwrapjs-5.1.0.tgz", + "integrity": "sha512-JNjcULU2e4KJwUNv6CHgI46UvDGitb6dGryHajXTDiLgg1/RiGoPSDw4kZfYnwGtEXf2ZMeIewDQgFGzkCB2Sg==" }, "workerpool": { "version": "6.2.1", diff --git a/node/package.json b/node/package.json index fb15797d..0b59a2aa 100644 --- a/node/package.json +++ b/node/package.json @@ -57,9 +57,9 @@ "uuid": "^9.0.0" }, "dependencies": { - "@apache-arrow/ts": "^12.0.0", + "@apache-arrow/ts": "^14.0.2", "@neon-rs/load": "^0.0.74", - "apache-arrow": "^12.0.0", + "apache-arrow": "^14.0.2", "axios": "^1.4.0" }, "os": [ diff --git a/node/src/arrow.ts b/node/src/arrow.ts index 90e59d31..2010c220 100644 --- a/node/src/arrow.ts +++ b/node/src/arrow.ts @@ -17,10 +17,9 @@ import { Float32, makeBuilder, RecordBatchFileWriter, - Utf8, - type Vector, + Utf8, type Vector, FixedSizeList, - vectorFromArray, type Schema, Table as ArrowTable, RecordBatchStreamWriter, List, Float64 + vectorFromArray, type Schema, Table as ArrowTable, RecordBatchStreamWriter, List, Float64, RecordBatch, makeData, Struct } from 'apache-arrow' import { type EmbeddingFunction } from './index' @@ -78,6 +77,7 @@ export async function convertToTable (data: Array>, e } records[columnsKey] = listBuilder.finish().toVector() } else { + // TODO if this is a struct field then recursively align the subfields records[columnsKey] = vectorFromArray(values) } } @@ -110,21 +110,27 @@ function newVectorType (dim: number): FixedSizeList { } // Converts an Array of records into Arrow IPC format -export async function fromRecordsToBuffer (data: Array>, embeddings?: EmbeddingFunction): Promise { - const table = await convertToTable(data, embeddings) +export async function fromRecordsToBuffer (data: Array>, embeddings?: EmbeddingFunction, schema?: Schema): Promise { + let table = await convertToTable(data, embeddings) + if (schema !== undefined) { + table = alignTable(table, schema) + } const writer = RecordBatchFileWriter.writeAll(table) return Buffer.from(await writer.toUint8Array()) } // Converts an Array of records into Arrow IPC stream format -export async function fromRecordsToStreamBuffer (data: Array>, embeddings?: EmbeddingFunction): Promise { - const table = await convertToTable(data, embeddings) +export async function fromRecordsToStreamBuffer (data: Array>, embeddings?: EmbeddingFunction, schema?: Schema): Promise { + let table = await convertToTable(data, embeddings) + if (schema !== undefined) { + table = alignTable(table, schema) + } const writer = RecordBatchStreamWriter.writeAll(table) return Buffer.from(await writer.toUint8Array()) } // Converts an Arrow Table into Arrow IPC format -export async function fromTableToBuffer (table: ArrowTable, embeddings?: EmbeddingFunction): Promise { +export async function fromTableToBuffer (table: ArrowTable, embeddings?: EmbeddingFunction, schema?: Schema): Promise { if (embeddings !== undefined) { const source = table.getChild(embeddings.sourceColumn) @@ -136,12 +142,15 @@ export async function fromTableToBuffer (table: ArrowTable, embeddings?: Embe const column = vectorFromArray(vectors, newVectorType(vectors[0].length)) table = table.assign(new ArrowTable({ vector: column })) } + if (schema !== undefined) { + table = alignTable(table, schema) + } const writer = RecordBatchFileWriter.writeAll(table) return Buffer.from(await writer.toUint8Array()) } // Converts an Arrow Table into Arrow IPC stream format -export async function fromTableToStreamBuffer (table: ArrowTable, embeddings?: EmbeddingFunction): Promise { +export async function fromTableToStreamBuffer (table: ArrowTable, embeddings?: EmbeddingFunction, schema?: Schema): Promise { if (embeddings !== undefined) { const source = table.getChild(embeddings.sourceColumn) @@ -153,10 +162,36 @@ export async function fromTableToStreamBuffer (table: ArrowTable, embeddings? const column = vectorFromArray(vectors, newVectorType(vectors[0].length)) table = table.assign(new ArrowTable({ vector: column })) } + if (schema !== undefined) { + table = alignTable(table, schema) + } const writer = RecordBatchStreamWriter.writeAll(table) return Buffer.from(await writer.toUint8Array()) } +function alignBatch (batch: RecordBatch, schema: Schema): RecordBatch { + const alignedChildren = [] + for (const field of schema.fields) { + const indexInBatch = batch.schema.fields?.findIndex((f) => f.name === field.name) + if (indexInBatch < 0) { + throw new Error(`The column ${field.name} was not found in the Arrow Table`) + } + alignedChildren.push(batch.data.children[indexInBatch]) + } + const newData = makeData({ + type: new Struct(schema.fields), + length: batch.numRows, + nullCount: batch.nullCount, + children: alignedChildren + }) + return new RecordBatch(schema, newData) +} + +function alignTable (table: ArrowTable, schema: Schema): ArrowTable { + const alignedBatches = table.batches.map(batch => alignBatch(batch, schema)) + return new ArrowTable(schema, alignedBatches) +} + // Creates an empty Arrow Table export function createEmptyTable (schema: Schema): ArrowTable { return new ArrowTable(schema) diff --git a/node/src/index.ts b/node/src/index.ts index a5de03eb..e11659e5 100644 --- a/node/src/index.ts +++ b/node/src/index.ts @@ -485,10 +485,10 @@ export class LocalConnection implements Connection { } buffer = await fromTableToBuffer(createEmptyTable(schema)) } else if (data instanceof ArrowTable) { - buffer = await fromTableToBuffer(data, embeddingFunction) + buffer = await fromTableToBuffer(data, embeddingFunction, schema) } else { // data is Array> - buffer = await fromRecordsToBuffer(data, embeddingFunction) + buffer = await fromRecordsToBuffer(data, embeddingFunction, schema) } const tbl = await tableCreate.call(this._db, name, buffer, writeOptions?.writeMode?.toString(), ...getAwsArgs(this._options())) @@ -560,9 +560,10 @@ export class LocalTable implements Table { * @return The number of rows added to the table */ async add (data: Array>): Promise { + const schema = await this.schema return tableAdd.call( this._tbl, - await fromRecordsToBuffer(data, this._embeddings), + await fromRecordsToBuffer(data, this._embeddings, schema), WriteMode.Append.toString(), ...getAwsArgs(this._options()) ).then((newTable: any) => { this._tbl = newTable }) diff --git a/node/src/test/test.ts b/node/src/test/test.ts index 68a0f331..d1b44eb9 100644 --- a/node/src/test/test.ts +++ b/node/src/test/test.ts @@ -176,6 +176,26 @@ describe('LanceDB client', function () { assert.deepEqual(await con.tableNames(), ['vectors']) }) + it('create a table with a schema and records', async function () { + const dir = await track().mkdir('lancejs') + const con = await lancedb.connect(dir) + + const schema = new Schema( + [new Field('id', new Int32()), + new Field('name', new Utf8()), + new Field('vector', new FixedSizeList(2, new Field('item', new Float32(), true)), false) + ] + ) + const data = [ + { vector: [0.5, 0.2], name: 'foo', id: 0 }, + { vector: [0.3, 0.1], name: 'bar', id: 1 } + ] + // even thought the keys in data is out of order it should still work + const table = await con.createTable({ name: 'vectors', data, schema }) + assert.equal(table.name, 'vectors') + assert.deepEqual(await con.tableNames(), ['vectors']) + }) + it('create a table with a empty data array', async function () { const dir = await track().mkdir('lancejs') const con = await lancedb.connect(dir) @@ -294,6 +314,25 @@ describe('LanceDB client', function () { assert.equal(await table.countRows(), 4) }) + it('appends records with fields in a different order', async function () { + const dir = await track().mkdir('lancejs') + const con = await lancedb.connect(dir) + + const data = [ + { id: 1, vector: [0.1, 0.2], price: 10, name: 'a' }, + { id: 2, vector: [1.1, 1.2], price: 50, name: 'b' } + ] + + const table = await con.createTable('vectors', data) + + const dataAdd = [ + { id: 3, vector: [2.1, 2.2], name: 'c', price: 10 }, + { id: 4, vector: [3.1, 3.2], name: 'd', price: 50 } + ] + await table.add(dataAdd) + assert.equal(await table.countRows(), 4) + }) + it('overwrite all records in a table', async function () { const uri = await createTestDB() const con = await lancedb.connect(uri)