mirror of
https://github.com/lancedb/lancedb.git
synced 2025-12-24 05:49:57 +00:00
Compare commits
9 Commits
rmeng/migr
...
python-v0.
| Author | SHA1 | Date | |
|---|---|---|---|
|
0a16e29b93 | ||
|
cf7d7a19f5 | ||
|
fe2fb91a8b | ||
|
81af350d85 | ||
|
99adfe065a | ||
|
|
277406509e | ||
|
|
63411b4d8b | ||
|
|
d998f80b04 | ||
|
|
629379a532 |
4
.github/workflows/python.yml
vendored
4
.github/workflows/python.yml
vendored
@@ -49,7 +49,7 @@ jobs:
|
||||
timeout-minutes: 30
|
||||
strategy:
|
||||
matrix:
|
||||
config:
|
||||
config:
|
||||
- name: x86 Mac
|
||||
runner: macos-13
|
||||
- name: Arm Mac
|
||||
@@ -74,7 +74,7 @@ jobs:
|
||||
run: |
|
||||
pip install -e .[tests]
|
||||
pip install tantivy@git+https://github.com/quickwit-oss/tantivy-py#164adc87e1a033117001cf70e38c82a53014d985
|
||||
pip install pytest pytest-mock black
|
||||
pip install pytest pytest-mock
|
||||
- name: Run tests
|
||||
run: pytest -m "not slow" -x -v --durations=30 tests
|
||||
pydantic1x:
|
||||
|
||||
@@ -5,10 +5,10 @@ exclude = ["python"]
|
||||
resolver = "2"
|
||||
|
||||
[workspace.dependencies]
|
||||
lance = { "version" = "=0.9.5", "features" = ["dynamodb"] }
|
||||
lance-index = { "version" = "=0.9.5" }
|
||||
lance-linalg = { "version" = "=0.9.5" }
|
||||
lance-testing = { "version" = "=0.9.5" }
|
||||
lance = { "version" = "=0.9.6", "features" = ["dynamodb"] }
|
||||
lance-index = { "version" = "=0.9.6" }
|
||||
lance-linalg = { "version" = "=0.9.6" }
|
||||
lance-testing = { "version" = "=0.9.6" }
|
||||
# Note that this one does not include pyarrow
|
||||
arrow = { version = "49.0.0", optional = false }
|
||||
arrow-array = "49.0"
|
||||
|
||||
520
node/package-lock.json
generated
520
node/package-lock.json
generated
@@ -18,9 +18,9 @@
|
||||
"win32"
|
||||
],
|
||||
"dependencies": {
|
||||
"@apache-arrow/ts": "^12.0.0",
|
||||
"@apache-arrow/ts": "^14.0.2",
|
||||
"@neon-rs/load": "^0.0.74",
|
||||
"apache-arrow": "^12.0.0",
|
||||
"apache-arrow": "^14.0.2",
|
||||
"axios": "^1.4.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
@@ -60,32 +60,52 @@
|
||||
"@lancedb/vectordb-win32-x64-msvc": "0.4.2"
|
||||
}
|
||||
},
|
||||
"node_modules/@75lb/deep-merge": {
|
||||
"version": "1.1.1",
|
||||
"resolved": "https://registry.npmjs.org/@75lb/deep-merge/-/deep-merge-1.1.1.tgz",
|
||||
"integrity": "sha512-xvgv6pkMGBA6GwdyJbNAnDmfAIR/DfWhrj9jgWh3TY7gRm3KO46x/GPjRg6wJ0nOepwqrNxFfojebh0Df4h4Tw==",
|
||||
"dependencies": {
|
||||
"lodash.assignwith": "^4.2.0",
|
||||
"typical": "^7.1.1"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=12.17"
|
||||
}
|
||||
},
|
||||
"node_modules/@75lb/deep-merge/node_modules/typical": {
|
||||
"version": "7.1.1",
|
||||
"resolved": "https://registry.npmjs.org/typical/-/typical-7.1.1.tgz",
|
||||
"integrity": "sha512-T+tKVNs6Wu7IWiAce5BgMd7OZfNYUndHwc5MknN+UHOudi7sGZzuHdCadllRuqJ3fPtgFtIH9+lt9qRv6lmpfA==",
|
||||
"engines": {
|
||||
"node": ">=12.17"
|
||||
}
|
||||
},
|
||||
"node_modules/@apache-arrow/ts": {
|
||||
"version": "12.0.0",
|
||||
"resolved": "https://registry.npmjs.org/@apache-arrow/ts/-/ts-12.0.0.tgz",
|
||||
"integrity": "sha512-ArJ3Fw5W9RAeNWuyCU2CdjL/nEAZSVDG1p3jz/ZtLo/q3NTz2w7HUCOJeszejH/5alGX+QirYrJ5c6BW++/P7g==",
|
||||
"version": "14.0.2",
|
||||
"resolved": "https://registry.npmjs.org/@apache-arrow/ts/-/ts-14.0.2.tgz",
|
||||
"integrity": "sha512-CtwAvLkK0CZv7xsYeCo91ml6PvlfzAmAJZkRYuz2GNBwfYufj5SVi0iuSMwIMkcU/szVwvLdzORSLa5PlF/2ug==",
|
||||
"dependencies": {
|
||||
"@types/command-line-args": "5.2.0",
|
||||
"@types/command-line-usage": "5.0.2",
|
||||
"@types/node": "18.14.5",
|
||||
"@types/node": "20.3.0",
|
||||
"@types/pad-left": "2.1.1",
|
||||
"command-line-args": "5.2.1",
|
||||
"command-line-usage": "6.1.3",
|
||||
"flatbuffers": "23.3.3",
|
||||
"command-line-usage": "7.0.1",
|
||||
"flatbuffers": "23.5.26",
|
||||
"json-bignum": "^0.0.3",
|
||||
"pad-left": "^2.1.0",
|
||||
"tslib": "^2.5.0"
|
||||
"tslib": "^2.5.3"
|
||||
}
|
||||
},
|
||||
"node_modules/@apache-arrow/ts/node_modules/@types/node": {
|
||||
"version": "18.14.5",
|
||||
"resolved": "https://registry.npmjs.org/@types/node/-/node-18.14.5.tgz",
|
||||
"integrity": "sha512-CRT4tMK/DHYhw1fcCEBwME9CSaZNclxfzVMe7GsO6ULSwsttbj70wSiX6rZdIjGblu93sTJxLdhNIT85KKI7Qw=="
|
||||
"version": "20.3.0",
|
||||
"resolved": "https://registry.npmjs.org/@types/node/-/node-20.3.0.tgz",
|
||||
"integrity": "sha512-cumHmIAf6On83X7yP+LrsEyUOf/YlociZelmpRYaGFydoaPdxdt80MAbu6vWerQT2COCp2nPvHdsbD7tHn/YlQ=="
|
||||
},
|
||||
"node_modules/@apache-arrow/ts/node_modules/tslib": {
|
||||
"version": "2.5.0",
|
||||
"resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.0.tgz",
|
||||
"integrity": "sha512-336iVw3rtn2BUK7ORdIAHTyxHGRIHVReokCR3XjbckJMK7ms8FysBfhLR8IXnAgy7T0PTPNBWKiH514FOW/WSg=="
|
||||
"version": "2.6.2",
|
||||
"resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.2.tgz",
|
||||
"integrity": "sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q=="
|
||||
},
|
||||
"node_modules/@cargo-messages/android-arm-eabi": {
|
||||
"version": "0.0.160",
|
||||
@@ -866,7 +886,6 @@
|
||||
"version": "4.3.0",
|
||||
"resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz",
|
||||
"integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==",
|
||||
"dev": true,
|
||||
"dependencies": {
|
||||
"color-convert": "^2.0.1"
|
||||
},
|
||||
@@ -891,34 +910,34 @@
|
||||
}
|
||||
},
|
||||
"node_modules/apache-arrow": {
|
||||
"version": "12.0.0",
|
||||
"resolved": "https://registry.npmjs.org/apache-arrow/-/apache-arrow-12.0.0.tgz",
|
||||
"integrity": "sha512-uI+hnZZsGfNJiR/wG8j5yPQuDjmOHx4hZpkA743G4x3TlFrCpA3MMX7KUkIOIw0e/CwZ8NYuaMzaQsblA47qVA==",
|
||||
"version": "14.0.2",
|
||||
"resolved": "https://registry.npmjs.org/apache-arrow/-/apache-arrow-14.0.2.tgz",
|
||||
"integrity": "sha512-EBO2xJN36/XoY81nhLcwCJgFwkboDZeyNQ+OPsG7bCoQjc2BT0aTyH/MR6SrL+LirSNz+cYqjGRlupMMlP1aEg==",
|
||||
"dependencies": {
|
||||
"@types/command-line-args": "5.2.0",
|
||||
"@types/command-line-usage": "5.0.2",
|
||||
"@types/node": "18.14.5",
|
||||
"@types/node": "20.3.0",
|
||||
"@types/pad-left": "2.1.1",
|
||||
"command-line-args": "5.2.1",
|
||||
"command-line-usage": "6.1.3",
|
||||
"flatbuffers": "23.3.3",
|
||||
"command-line-usage": "7.0.1",
|
||||
"flatbuffers": "23.5.26",
|
||||
"json-bignum": "^0.0.3",
|
||||
"pad-left": "^2.1.0",
|
||||
"tslib": "^2.5.0"
|
||||
"tslib": "^2.5.3"
|
||||
},
|
||||
"bin": {
|
||||
"arrow2csv": "bin/arrow2csv.js"
|
||||
}
|
||||
},
|
||||
"node_modules/apache-arrow/node_modules/@types/node": {
|
||||
"version": "18.14.5",
|
||||
"resolved": "https://registry.npmjs.org/@types/node/-/node-18.14.5.tgz",
|
||||
"integrity": "sha512-CRT4tMK/DHYhw1fcCEBwME9CSaZNclxfzVMe7GsO6ULSwsttbj70wSiX6rZdIjGblu93sTJxLdhNIT85KKI7Qw=="
|
||||
"version": "20.3.0",
|
||||
"resolved": "https://registry.npmjs.org/@types/node/-/node-20.3.0.tgz",
|
||||
"integrity": "sha512-cumHmIAf6On83X7yP+LrsEyUOf/YlociZelmpRYaGFydoaPdxdt80MAbu6vWerQT2COCp2nPvHdsbD7tHn/YlQ=="
|
||||
},
|
||||
"node_modules/apache-arrow/node_modules/tslib": {
|
||||
"version": "2.5.0",
|
||||
"resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.0.tgz",
|
||||
"integrity": "sha512-336iVw3rtn2BUK7ORdIAHTyxHGRIHVReokCR3XjbckJMK7ms8FysBfhLR8IXnAgy7T0PTPNBWKiH514FOW/WSg=="
|
||||
"version": "2.6.2",
|
||||
"resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.2.tgz",
|
||||
"integrity": "sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q=="
|
||||
},
|
||||
"node_modules/arg": {
|
||||
"version": "4.1.3",
|
||||
@@ -1170,7 +1189,6 @@
|
||||
"version": "4.1.2",
|
||||
"resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz",
|
||||
"integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==",
|
||||
"dev": true,
|
||||
"dependencies": {
|
||||
"ansi-styles": "^4.1.0",
|
||||
"supports-color": "^7.1.0"
|
||||
@@ -1182,11 +1200,24 @@
|
||||
"url": "https://github.com/chalk/chalk?sponsor=1"
|
||||
}
|
||||
},
|
||||
"node_modules/chalk-template": {
|
||||
"version": "0.4.0",
|
||||
"resolved": "https://registry.npmjs.org/chalk-template/-/chalk-template-0.4.0.tgz",
|
||||
"integrity": "sha512-/ghrgmhfY8RaSdeo43hNXxpoHAtxdbskUHjPpfqUWGttFgycUhYPGx3YZBCnUCvOa7Doivn1IZec3DEGFoMgLg==",
|
||||
"dependencies": {
|
||||
"chalk": "^4.1.2"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=12"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/chalk/chalk-template?sponsor=1"
|
||||
}
|
||||
},
|
||||
"node_modules/chalk/node_modules/supports-color": {
|
||||
"version": "7.2.0",
|
||||
"resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
|
||||
"integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
|
||||
"dev": true,
|
||||
"dependencies": {
|
||||
"has-flag": "^4.0.0"
|
||||
},
|
||||
@@ -1245,7 +1276,6 @@
|
||||
"version": "2.0.1",
|
||||
"resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
|
||||
"integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
|
||||
"dev": true,
|
||||
"dependencies": {
|
||||
"color-name": "~1.1.4"
|
||||
},
|
||||
@@ -1256,8 +1286,7 @@
|
||||
"node_modules/color-name": {
|
||||
"version": "1.1.4",
|
||||
"resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
|
||||
"integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
|
||||
"dev": true
|
||||
"integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA=="
|
||||
},
|
||||
"node_modules/combined-stream": {
|
||||
"version": "1.0.8",
|
||||
@@ -1285,97 +1314,33 @@
|
||||
}
|
||||
},
|
||||
"node_modules/command-line-usage": {
|
||||
"version": "6.1.3",
|
||||
"resolved": "https://registry.npmjs.org/command-line-usage/-/command-line-usage-6.1.3.tgz",
|
||||
"integrity": "sha512-sH5ZSPr+7UStsloltmDh7Ce5fb8XPlHyoPzTpyyMuYCtervL65+ubVZ6Q61cFtFl62UyJlc8/JwERRbAFPUqgw==",
|
||||
"version": "7.0.1",
|
||||
"resolved": "https://registry.npmjs.org/command-line-usage/-/command-line-usage-7.0.1.tgz",
|
||||
"integrity": "sha512-NCyznE//MuTjwi3y84QVUGEOT+P5oto1e1Pk/jFPVdPPfsG03qpTIl3yw6etR+v73d0lXsoojRpvbru2sqePxQ==",
|
||||
"dependencies": {
|
||||
"array-back": "^4.0.2",
|
||||
"chalk": "^2.4.2",
|
||||
"table-layout": "^1.0.2",
|
||||
"typical": "^5.2.0"
|
||||
"array-back": "^6.2.2",
|
||||
"chalk-template": "^0.4.0",
|
||||
"table-layout": "^3.0.0",
|
||||
"typical": "^7.1.1"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=8.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/command-line-usage/node_modules/ansi-styles": {
|
||||
"version": "3.2.1",
|
||||
"resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz",
|
||||
"integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==",
|
||||
"dependencies": {
|
||||
"color-convert": "^1.9.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=4"
|
||||
"node": ">=12.20.0"
|
||||
}
|
||||
},
|
||||
"node_modules/command-line-usage/node_modules/array-back": {
|
||||
"version": "4.0.2",
|
||||
"resolved": "https://registry.npmjs.org/array-back/-/array-back-4.0.2.tgz",
|
||||
"integrity": "sha512-NbdMezxqf94cnNfWLL7V/im0Ub+Anbb0IoZhvzie8+4HJ4nMQuzHuy49FkGYCJK2yAloZ3meiB6AVMClbrI1vg==",
|
||||
"version": "6.2.2",
|
||||
"resolved": "https://registry.npmjs.org/array-back/-/array-back-6.2.2.tgz",
|
||||
"integrity": "sha512-gUAZ7HPyb4SJczXAMUXMGAvI976JoK3qEx9v1FTmeYuJj0IBiaKttG1ydtGKdkfqWkIkouke7nG8ufGy77+Cvw==",
|
||||
"engines": {
|
||||
"node": ">=8"
|
||||
}
|
||||
},
|
||||
"node_modules/command-line-usage/node_modules/chalk": {
|
||||
"version": "2.4.2",
|
||||
"resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz",
|
||||
"integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==",
|
||||
"dependencies": {
|
||||
"ansi-styles": "^3.2.1",
|
||||
"escape-string-regexp": "^1.0.5",
|
||||
"supports-color": "^5.3.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=4"
|
||||
}
|
||||
},
|
||||
"node_modules/command-line-usage/node_modules/color-convert": {
|
||||
"version": "1.9.3",
|
||||
"resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz",
|
||||
"integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==",
|
||||
"dependencies": {
|
||||
"color-name": "1.1.3"
|
||||
}
|
||||
},
|
||||
"node_modules/command-line-usage/node_modules/color-name": {
|
||||
"version": "1.1.3",
|
||||
"resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz",
|
||||
"integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw=="
|
||||
},
|
||||
"node_modules/command-line-usage/node_modules/escape-string-regexp": {
|
||||
"version": "1.0.5",
|
||||
"resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz",
|
||||
"integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==",
|
||||
"engines": {
|
||||
"node": ">=0.8.0"
|
||||
}
|
||||
},
|
||||
"node_modules/command-line-usage/node_modules/has-flag": {
|
||||
"version": "3.0.0",
|
||||
"resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz",
|
||||
"integrity": "sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==",
|
||||
"engines": {
|
||||
"node": ">=4"
|
||||
}
|
||||
},
|
||||
"node_modules/command-line-usage/node_modules/supports-color": {
|
||||
"version": "5.5.0",
|
||||
"resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz",
|
||||
"integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==",
|
||||
"dependencies": {
|
||||
"has-flag": "^3.0.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=4"
|
||||
"node": ">=12.17"
|
||||
}
|
||||
},
|
||||
"node_modules/command-line-usage/node_modules/typical": {
|
||||
"version": "5.2.0",
|
||||
"resolved": "https://registry.npmjs.org/typical/-/typical-5.2.0.tgz",
|
||||
"integrity": "sha512-dvdQgNDNJo+8B2uBQoqdb11eUCE1JQXhvjC/CZtgvZseVd5TYMXnq0+vuUemXbd/Se29cTaUuPX3YIc2xgbvIg==",
|
||||
"version": "7.1.1",
|
||||
"resolved": "https://registry.npmjs.org/typical/-/typical-7.1.1.tgz",
|
||||
"integrity": "sha512-T+tKVNs6Wu7IWiAce5BgMd7OZfNYUndHwc5MknN+UHOudi7sGZzuHdCadllRuqJ3fPtgFtIH9+lt9qRv6lmpfA==",
|
||||
"engines": {
|
||||
"node": ">=8"
|
||||
"node": ">=12.17"
|
||||
}
|
||||
},
|
||||
"node_modules/concat-map": {
|
||||
@@ -1451,14 +1416,6 @@
|
||||
"node": ">=6"
|
||||
}
|
||||
},
|
||||
"node_modules/deep-extend": {
|
||||
"version": "0.6.0",
|
||||
"resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz",
|
||||
"integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==",
|
||||
"engines": {
|
||||
"node": ">=4.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/deep-is": {
|
||||
"version": "0.1.4",
|
||||
"resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz",
|
||||
@@ -2237,9 +2194,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/flatbuffers": {
|
||||
"version": "23.3.3",
|
||||
"resolved": "https://registry.npmjs.org/flatbuffers/-/flatbuffers-23.3.3.tgz",
|
||||
"integrity": "sha512-jmreOaAT1t55keaf+Z259Tvh8tR/Srry9K8dgCgvizhKSEr6gLGgaOJI2WFL5fkOpGOGRZwxUrlFn0GCmXUy6g=="
|
||||
"version": "23.5.26",
|
||||
"resolved": "https://registry.npmjs.org/flatbuffers/-/flatbuffers-23.5.26.tgz",
|
||||
"integrity": "sha512-vE+SI9vrJDwi1oETtTIFldC/o9GsVKRM+s6EL0nQgxXlYV1Vc4Tk30hj4xGICftInKQKj1F3up2n8UbIVobISQ=="
|
||||
},
|
||||
"node_modules/flatted": {
|
||||
"version": "3.2.7",
|
||||
@@ -2535,7 +2492,6 @@
|
||||
"version": "4.0.0",
|
||||
"resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
|
||||
"integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==",
|
||||
"dev": true,
|
||||
"engines": {
|
||||
"node": ">=8"
|
||||
}
|
||||
@@ -3048,6 +3004,11 @@
|
||||
"url": "https://github.com/sponsors/sindresorhus"
|
||||
}
|
||||
},
|
||||
"node_modules/lodash.assignwith": {
|
||||
"version": "4.2.0",
|
||||
"resolved": "https://registry.npmjs.org/lodash.assignwith/-/lodash.assignwith-4.2.0.tgz",
|
||||
"integrity": "sha512-ZznplvbvtjK2gMvnQ1BR/zqPFZmS6jbK4p+6Up4xcRYA7yMIwxHCfbTcrYxXKzzqLsQ05eJPVznEW3tuwV7k1g=="
|
||||
},
|
||||
"node_modules/lodash.camelcase": {
|
||||
"version": "4.3.0",
|
||||
"resolved": "https://registry.npmjs.org/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz",
|
||||
@@ -3668,14 +3629,6 @@
|
||||
"node": ">=8.10.0"
|
||||
}
|
||||
},
|
||||
"node_modules/reduce-flatten": {
|
||||
"version": "2.0.0",
|
||||
"resolved": "https://registry.npmjs.org/reduce-flatten/-/reduce-flatten-2.0.0.tgz",
|
||||
"integrity": "sha512-EJ4UNY/U1t2P/2k6oqotuX2Cc3T6nxJwsM0N0asT7dhrtH1ltUxDn4NalSYmPE2rCkVpcf/X6R0wDwcFpzhd4w==",
|
||||
"engines": {
|
||||
"node": ">=6"
|
||||
}
|
||||
},
|
||||
"node_modules/regexp.prototype.flags": {
|
||||
"version": "1.5.0",
|
||||
"resolved": "https://registry.npmjs.org/regexp.prototype.flags/-/regexp.prototype.flags-1.5.0.tgz",
|
||||
@@ -3965,6 +3918,14 @@
|
||||
"source-map": "^0.6.0"
|
||||
}
|
||||
},
|
||||
"node_modules/stream-read-all": {
|
||||
"version": "3.0.1",
|
||||
"resolved": "https://registry.npmjs.org/stream-read-all/-/stream-read-all-3.0.1.tgz",
|
||||
"integrity": "sha512-EWZT9XOceBPlVJRrYcykW8jyRSZYbkb/0ZK36uLEmoWVO5gxBOnntNTseNzfREsqxqdfEGQrD8SXQ3QWbBmq8A==",
|
||||
"engines": {
|
||||
"node": ">=10"
|
||||
}
|
||||
},
|
||||
"node_modules/string-width": {
|
||||
"version": "4.2.3",
|
||||
"resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz",
|
||||
@@ -4082,33 +4043,39 @@
|
||||
}
|
||||
},
|
||||
"node_modules/table-layout": {
|
||||
"version": "1.0.2",
|
||||
"resolved": "https://registry.npmjs.org/table-layout/-/table-layout-1.0.2.tgz",
|
||||
"integrity": "sha512-qd/R7n5rQTRFi+Zf2sk5XVVd9UQl6ZkduPFC3S7WEGJAmetDTjY3qPN50eSKzwuzEyQKy5TN2TiZdkIjos2L6A==",
|
||||
"version": "3.0.2",
|
||||
"resolved": "https://registry.npmjs.org/table-layout/-/table-layout-3.0.2.tgz",
|
||||
"integrity": "sha512-rpyNZYRw+/C+dYkcQ3Pr+rLxW4CfHpXjPDnG7lYhdRoUcZTUt+KEsX+94RGp/aVp/MQU35JCITv2T/beY4m+hw==",
|
||||
"dependencies": {
|
||||
"array-back": "^4.0.1",
|
||||
"deep-extend": "~0.6.0",
|
||||
"typical": "^5.2.0",
|
||||
"wordwrapjs": "^4.0.0"
|
||||
"@75lb/deep-merge": "^1.1.1",
|
||||
"array-back": "^6.2.2",
|
||||
"command-line-args": "^5.2.1",
|
||||
"command-line-usage": "^7.0.0",
|
||||
"stream-read-all": "^3.0.1",
|
||||
"typical": "^7.1.1",
|
||||
"wordwrapjs": "^5.1.0"
|
||||
},
|
||||
"bin": {
|
||||
"table-layout": "bin/cli.js"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=8.0.0"
|
||||
"node": ">=12.17"
|
||||
}
|
||||
},
|
||||
"node_modules/table-layout/node_modules/array-back": {
|
||||
"version": "4.0.2",
|
||||
"resolved": "https://registry.npmjs.org/array-back/-/array-back-4.0.2.tgz",
|
||||
"integrity": "sha512-NbdMezxqf94cnNfWLL7V/im0Ub+Anbb0IoZhvzie8+4HJ4nMQuzHuy49FkGYCJK2yAloZ3meiB6AVMClbrI1vg==",
|
||||
"version": "6.2.2",
|
||||
"resolved": "https://registry.npmjs.org/array-back/-/array-back-6.2.2.tgz",
|
||||
"integrity": "sha512-gUAZ7HPyb4SJczXAMUXMGAvI976JoK3qEx9v1FTmeYuJj0IBiaKttG1ydtGKdkfqWkIkouke7nG8ufGy77+Cvw==",
|
||||
"engines": {
|
||||
"node": ">=8"
|
||||
"node": ">=12.17"
|
||||
}
|
||||
},
|
||||
"node_modules/table-layout/node_modules/typical": {
|
||||
"version": "5.2.0",
|
||||
"resolved": "https://registry.npmjs.org/typical/-/typical-5.2.0.tgz",
|
||||
"integrity": "sha512-dvdQgNDNJo+8B2uBQoqdb11eUCE1JQXhvjC/CZtgvZseVd5TYMXnq0+vuUemXbd/Se29cTaUuPX3YIc2xgbvIg==",
|
||||
"version": "7.1.1",
|
||||
"resolved": "https://registry.npmjs.org/typical/-/typical-7.1.1.tgz",
|
||||
"integrity": "sha512-T+tKVNs6Wu7IWiAce5BgMd7OZfNYUndHwc5MknN+UHOudi7sGZzuHdCadllRuqJ3fPtgFtIH9+lt9qRv6lmpfA==",
|
||||
"engines": {
|
||||
"node": ">=8"
|
||||
"node": ">=12.17"
|
||||
}
|
||||
},
|
||||
"node_modules/temp": {
|
||||
@@ -4553,23 +4520,11 @@
|
||||
"dev": true
|
||||
},
|
||||
"node_modules/wordwrapjs": {
|
||||
"version": "4.0.1",
|
||||
"resolved": "https://registry.npmjs.org/wordwrapjs/-/wordwrapjs-4.0.1.tgz",
|
||||
"integrity": "sha512-kKlNACbvHrkpIw6oPeYDSmdCTu2hdMHoyXLTcUKala++lx5Y+wjJ/e474Jqv5abnVmwxw08DiTuHmw69lJGksA==",
|
||||
"dependencies": {
|
||||
"reduce-flatten": "^2.0.0",
|
||||
"typical": "^5.2.0"
|
||||
},
|
||||
"version": "5.1.0",
|
||||
"resolved": "https://registry.npmjs.org/wordwrapjs/-/wordwrapjs-5.1.0.tgz",
|
||||
"integrity": "sha512-JNjcULU2e4KJwUNv6CHgI46UvDGitb6dGryHajXTDiLgg1/RiGoPSDw4kZfYnwGtEXf2ZMeIewDQgFGzkCB2Sg==",
|
||||
"engines": {
|
||||
"node": ">=8.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/wordwrapjs/node_modules/typical": {
|
||||
"version": "5.2.0",
|
||||
"resolved": "https://registry.npmjs.org/typical/-/typical-5.2.0.tgz",
|
||||
"integrity": "sha512-dvdQgNDNJo+8B2uBQoqdb11eUCE1JQXhvjC/CZtgvZseVd5TYMXnq0+vuUemXbd/Se29cTaUuPX3YIc2xgbvIg==",
|
||||
"engines": {
|
||||
"node": ">=8"
|
||||
"node": ">=12.17"
|
||||
}
|
||||
},
|
||||
"node_modules/workerpool": {
|
||||
@@ -4690,32 +4645,48 @@
|
||||
}
|
||||
},
|
||||
"dependencies": {
|
||||
"@75lb/deep-merge": {
|
||||
"version": "1.1.1",
|
||||
"resolved": "https://registry.npmjs.org/@75lb/deep-merge/-/deep-merge-1.1.1.tgz",
|
||||
"integrity": "sha512-xvgv6pkMGBA6GwdyJbNAnDmfAIR/DfWhrj9jgWh3TY7gRm3KO46x/GPjRg6wJ0nOepwqrNxFfojebh0Df4h4Tw==",
|
||||
"requires": {
|
||||
"lodash.assignwith": "^4.2.0",
|
||||
"typical": "^7.1.1"
|
||||
},
|
||||
"dependencies": {
|
||||
"typical": {
|
||||
"version": "7.1.1",
|
||||
"resolved": "https://registry.npmjs.org/typical/-/typical-7.1.1.tgz",
|
||||
"integrity": "sha512-T+tKVNs6Wu7IWiAce5BgMd7OZfNYUndHwc5MknN+UHOudi7sGZzuHdCadllRuqJ3fPtgFtIH9+lt9qRv6lmpfA=="
|
||||
}
|
||||
}
|
||||
},
|
||||
"@apache-arrow/ts": {
|
||||
"version": "12.0.0",
|
||||
"resolved": "https://registry.npmjs.org/@apache-arrow/ts/-/ts-12.0.0.tgz",
|
||||
"integrity": "sha512-ArJ3Fw5W9RAeNWuyCU2CdjL/nEAZSVDG1p3jz/ZtLo/q3NTz2w7HUCOJeszejH/5alGX+QirYrJ5c6BW++/P7g==",
|
||||
"version": "14.0.2",
|
||||
"resolved": "https://registry.npmjs.org/@apache-arrow/ts/-/ts-14.0.2.tgz",
|
||||
"integrity": "sha512-CtwAvLkK0CZv7xsYeCo91ml6PvlfzAmAJZkRYuz2GNBwfYufj5SVi0iuSMwIMkcU/szVwvLdzORSLa5PlF/2ug==",
|
||||
"requires": {
|
||||
"@types/command-line-args": "5.2.0",
|
||||
"@types/command-line-usage": "5.0.2",
|
||||
"@types/node": "18.14.5",
|
||||
"@types/node": "20.3.0",
|
||||
"@types/pad-left": "2.1.1",
|
||||
"command-line-args": "5.2.1",
|
||||
"command-line-usage": "6.1.3",
|
||||
"flatbuffers": "23.3.3",
|
||||
"command-line-usage": "7.0.1",
|
||||
"flatbuffers": "23.5.26",
|
||||
"json-bignum": "^0.0.3",
|
||||
"pad-left": "^2.1.0",
|
||||
"tslib": "^2.5.0"
|
||||
"tslib": "^2.5.3"
|
||||
},
|
||||
"dependencies": {
|
||||
"@types/node": {
|
||||
"version": "18.14.5",
|
||||
"resolved": "https://registry.npmjs.org/@types/node/-/node-18.14.5.tgz",
|
||||
"integrity": "sha512-CRT4tMK/DHYhw1fcCEBwME9CSaZNclxfzVMe7GsO6ULSwsttbj70wSiX6rZdIjGblu93sTJxLdhNIT85KKI7Qw=="
|
||||
"version": "20.3.0",
|
||||
"resolved": "https://registry.npmjs.org/@types/node/-/node-20.3.0.tgz",
|
||||
"integrity": "sha512-cumHmIAf6On83X7yP+LrsEyUOf/YlociZelmpRYaGFydoaPdxdt80MAbu6vWerQT2COCp2nPvHdsbD7tHn/YlQ=="
|
||||
},
|
||||
"tslib": {
|
||||
"version": "2.5.0",
|
||||
"resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.0.tgz",
|
||||
"integrity": "sha512-336iVw3rtn2BUK7ORdIAHTyxHGRIHVReokCR3XjbckJMK7ms8FysBfhLR8IXnAgy7T0PTPNBWKiH514FOW/WSg=="
|
||||
"version": "2.6.2",
|
||||
"resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.2.tgz",
|
||||
"integrity": "sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q=="
|
||||
}
|
||||
}
|
||||
},
|
||||
@@ -5268,7 +5239,6 @@
|
||||
"version": "4.3.0",
|
||||
"resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz",
|
||||
"integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"color-convert": "^2.0.1"
|
||||
}
|
||||
@@ -5284,31 +5254,31 @@
|
||||
}
|
||||
},
|
||||
"apache-arrow": {
|
||||
"version": "12.0.0",
|
||||
"resolved": "https://registry.npmjs.org/apache-arrow/-/apache-arrow-12.0.0.tgz",
|
||||
"integrity": "sha512-uI+hnZZsGfNJiR/wG8j5yPQuDjmOHx4hZpkA743G4x3TlFrCpA3MMX7KUkIOIw0e/CwZ8NYuaMzaQsblA47qVA==",
|
||||
"version": "14.0.2",
|
||||
"resolved": "https://registry.npmjs.org/apache-arrow/-/apache-arrow-14.0.2.tgz",
|
||||
"integrity": "sha512-EBO2xJN36/XoY81nhLcwCJgFwkboDZeyNQ+OPsG7bCoQjc2BT0aTyH/MR6SrL+LirSNz+cYqjGRlupMMlP1aEg==",
|
||||
"requires": {
|
||||
"@types/command-line-args": "5.2.0",
|
||||
"@types/command-line-usage": "5.0.2",
|
||||
"@types/node": "18.14.5",
|
||||
"@types/node": "20.3.0",
|
||||
"@types/pad-left": "2.1.1",
|
||||
"command-line-args": "5.2.1",
|
||||
"command-line-usage": "6.1.3",
|
||||
"flatbuffers": "23.3.3",
|
||||
"command-line-usage": "7.0.1",
|
||||
"flatbuffers": "23.5.26",
|
||||
"json-bignum": "^0.0.3",
|
||||
"pad-left": "^2.1.0",
|
||||
"tslib": "^2.5.0"
|
||||
"tslib": "^2.5.3"
|
||||
},
|
||||
"dependencies": {
|
||||
"@types/node": {
|
||||
"version": "18.14.5",
|
||||
"resolved": "https://registry.npmjs.org/@types/node/-/node-18.14.5.tgz",
|
||||
"integrity": "sha512-CRT4tMK/DHYhw1fcCEBwME9CSaZNclxfzVMe7GsO6ULSwsttbj70wSiX6rZdIjGblu93sTJxLdhNIT85KKI7Qw=="
|
||||
"version": "20.3.0",
|
||||
"resolved": "https://registry.npmjs.org/@types/node/-/node-20.3.0.tgz",
|
||||
"integrity": "sha512-cumHmIAf6On83X7yP+LrsEyUOf/YlociZelmpRYaGFydoaPdxdt80MAbu6vWerQT2COCp2nPvHdsbD7tHn/YlQ=="
|
||||
},
|
||||
"tslib": {
|
||||
"version": "2.5.0",
|
||||
"resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.0.tgz",
|
||||
"integrity": "sha512-336iVw3rtn2BUK7ORdIAHTyxHGRIHVReokCR3XjbckJMK7ms8FysBfhLR8IXnAgy7T0PTPNBWKiH514FOW/WSg=="
|
||||
"version": "2.6.2",
|
||||
"resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.2.tgz",
|
||||
"integrity": "sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q=="
|
||||
}
|
||||
}
|
||||
},
|
||||
@@ -5505,7 +5475,6 @@
|
||||
"version": "4.1.2",
|
||||
"resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz",
|
||||
"integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"ansi-styles": "^4.1.0",
|
||||
"supports-color": "^7.1.0"
|
||||
@@ -5515,13 +5484,20 @@
|
||||
"version": "7.2.0",
|
||||
"resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
|
||||
"integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"has-flag": "^4.0.0"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"chalk-template": {
|
||||
"version": "0.4.0",
|
||||
"resolved": "https://registry.npmjs.org/chalk-template/-/chalk-template-0.4.0.tgz",
|
||||
"integrity": "sha512-/ghrgmhfY8RaSdeo43hNXxpoHAtxdbskUHjPpfqUWGttFgycUhYPGx3YZBCnUCvOa7Doivn1IZec3DEGFoMgLg==",
|
||||
"requires": {
|
||||
"chalk": "^4.1.2"
|
||||
}
|
||||
},
|
||||
"check-error": {
|
||||
"version": "1.0.2",
|
||||
"resolved": "https://registry.npmjs.org/check-error/-/check-error-1.0.2.tgz",
|
||||
@@ -5559,7 +5535,6 @@
|
||||
"version": "2.0.1",
|
||||
"resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
|
||||
"integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"color-name": "~1.1.4"
|
||||
}
|
||||
@@ -5567,8 +5542,7 @@
|
||||
"color-name": {
|
||||
"version": "1.1.4",
|
||||
"resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
|
||||
"integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
|
||||
"dev": true
|
||||
"integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA=="
|
||||
},
|
||||
"combined-stream": {
|
||||
"version": "1.0.8",
|
||||
@@ -5590,74 +5564,25 @@
|
||||
}
|
||||
},
|
||||
"command-line-usage": {
|
||||
"version": "6.1.3",
|
||||
"resolved": "https://registry.npmjs.org/command-line-usage/-/command-line-usage-6.1.3.tgz",
|
||||
"integrity": "sha512-sH5ZSPr+7UStsloltmDh7Ce5fb8XPlHyoPzTpyyMuYCtervL65+ubVZ6Q61cFtFl62UyJlc8/JwERRbAFPUqgw==",
|
||||
"version": "7.0.1",
|
||||
"resolved": "https://registry.npmjs.org/command-line-usage/-/command-line-usage-7.0.1.tgz",
|
||||
"integrity": "sha512-NCyznE//MuTjwi3y84QVUGEOT+P5oto1e1Pk/jFPVdPPfsG03qpTIl3yw6etR+v73d0lXsoojRpvbru2sqePxQ==",
|
||||
"requires": {
|
||||
"array-back": "^4.0.2",
|
||||
"chalk": "^2.4.2",
|
||||
"table-layout": "^1.0.2",
|
||||
"typical": "^5.2.0"
|
||||
"array-back": "^6.2.2",
|
||||
"chalk-template": "^0.4.0",
|
||||
"table-layout": "^3.0.0",
|
||||
"typical": "^7.1.1"
|
||||
},
|
||||
"dependencies": {
|
||||
"ansi-styles": {
|
||||
"version": "3.2.1",
|
||||
"resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz",
|
||||
"integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==",
|
||||
"requires": {
|
||||
"color-convert": "^1.9.0"
|
||||
}
|
||||
},
|
||||
"array-back": {
|
||||
"version": "4.0.2",
|
||||
"resolved": "https://registry.npmjs.org/array-back/-/array-back-4.0.2.tgz",
|
||||
"integrity": "sha512-NbdMezxqf94cnNfWLL7V/im0Ub+Anbb0IoZhvzie8+4HJ4nMQuzHuy49FkGYCJK2yAloZ3meiB6AVMClbrI1vg=="
|
||||
},
|
||||
"chalk": {
|
||||
"version": "2.4.2",
|
||||
"resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz",
|
||||
"integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==",
|
||||
"requires": {
|
||||
"ansi-styles": "^3.2.1",
|
||||
"escape-string-regexp": "^1.0.5",
|
||||
"supports-color": "^5.3.0"
|
||||
}
|
||||
},
|
||||
"color-convert": {
|
||||
"version": "1.9.3",
|
||||
"resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz",
|
||||
"integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==",
|
||||
"requires": {
|
||||
"color-name": "1.1.3"
|
||||
}
|
||||
},
|
||||
"color-name": {
|
||||
"version": "1.1.3",
|
||||
"resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz",
|
||||
"integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw=="
|
||||
},
|
||||
"escape-string-regexp": {
|
||||
"version": "1.0.5",
|
||||
"resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz",
|
||||
"integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg=="
|
||||
},
|
||||
"has-flag": {
|
||||
"version": "3.0.0",
|
||||
"resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz",
|
||||
"integrity": "sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw=="
|
||||
},
|
||||
"supports-color": {
|
||||
"version": "5.5.0",
|
||||
"resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz",
|
||||
"integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==",
|
||||
"requires": {
|
||||
"has-flag": "^3.0.0"
|
||||
}
|
||||
"version": "6.2.2",
|
||||
"resolved": "https://registry.npmjs.org/array-back/-/array-back-6.2.2.tgz",
|
||||
"integrity": "sha512-gUAZ7HPyb4SJczXAMUXMGAvI976JoK3qEx9v1FTmeYuJj0IBiaKttG1ydtGKdkfqWkIkouke7nG8ufGy77+Cvw=="
|
||||
},
|
||||
"typical": {
|
||||
"version": "5.2.0",
|
||||
"resolved": "https://registry.npmjs.org/typical/-/typical-5.2.0.tgz",
|
||||
"integrity": "sha512-dvdQgNDNJo+8B2uBQoqdb11eUCE1JQXhvjC/CZtgvZseVd5TYMXnq0+vuUemXbd/Se29cTaUuPX3YIc2xgbvIg=="
|
||||
"version": "7.1.1",
|
||||
"resolved": "https://registry.npmjs.org/typical/-/typical-7.1.1.tgz",
|
||||
"integrity": "sha512-T+tKVNs6Wu7IWiAce5BgMd7OZfNYUndHwc5MknN+UHOudi7sGZzuHdCadllRuqJ3fPtgFtIH9+lt9qRv6lmpfA=="
|
||||
}
|
||||
}
|
||||
},
|
||||
@@ -5716,11 +5641,6 @@
|
||||
"type-detect": "^4.0.0"
|
||||
}
|
||||
},
|
||||
"deep-extend": {
|
||||
"version": "0.6.0",
|
||||
"resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz",
|
||||
"integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA=="
|
||||
},
|
||||
"deep-is": {
|
||||
"version": "0.1.4",
|
||||
"resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz",
|
||||
@@ -6297,9 +6217,9 @@
|
||||
}
|
||||
},
|
||||
"flatbuffers": {
|
||||
"version": "23.3.3",
|
||||
"resolved": "https://registry.npmjs.org/flatbuffers/-/flatbuffers-23.3.3.tgz",
|
||||
"integrity": "sha512-jmreOaAT1t55keaf+Z259Tvh8tR/Srry9K8dgCgvizhKSEr6gLGgaOJI2WFL5fkOpGOGRZwxUrlFn0GCmXUy6g=="
|
||||
"version": "23.5.26",
|
||||
"resolved": "https://registry.npmjs.org/flatbuffers/-/flatbuffers-23.5.26.tgz",
|
||||
"integrity": "sha512-vE+SI9vrJDwi1oETtTIFldC/o9GsVKRM+s6EL0nQgxXlYV1Vc4Tk30hj4xGICftInKQKj1F3up2n8UbIVobISQ=="
|
||||
},
|
||||
"flatted": {
|
||||
"version": "3.2.7",
|
||||
@@ -6502,8 +6422,7 @@
|
||||
"has-flag": {
|
||||
"version": "4.0.0",
|
||||
"resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
|
||||
"integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==",
|
||||
"dev": true
|
||||
"integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ=="
|
||||
},
|
||||
"has-property-descriptors": {
|
||||
"version": "1.0.0",
|
||||
@@ -6856,6 +6775,11 @@
|
||||
"p-locate": "^5.0.0"
|
||||
}
|
||||
},
|
||||
"lodash.assignwith": {
|
||||
"version": "4.2.0",
|
||||
"resolved": "https://registry.npmjs.org/lodash.assignwith/-/lodash.assignwith-4.2.0.tgz",
|
||||
"integrity": "sha512-ZznplvbvtjK2gMvnQ1BR/zqPFZmS6jbK4p+6Up4xcRYA7yMIwxHCfbTcrYxXKzzqLsQ05eJPVznEW3tuwV7k1g=="
|
||||
},
|
||||
"lodash.camelcase": {
|
||||
"version": "4.3.0",
|
||||
"resolved": "https://registry.npmjs.org/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz",
|
||||
@@ -7323,11 +7247,6 @@
|
||||
"picomatch": "^2.2.1"
|
||||
}
|
||||
},
|
||||
"reduce-flatten": {
|
||||
"version": "2.0.0",
|
||||
"resolved": "https://registry.npmjs.org/reduce-flatten/-/reduce-flatten-2.0.0.tgz",
|
||||
"integrity": "sha512-EJ4UNY/U1t2P/2k6oqotuX2Cc3T6nxJwsM0N0asT7dhrtH1ltUxDn4NalSYmPE2rCkVpcf/X6R0wDwcFpzhd4w=="
|
||||
},
|
||||
"regexp.prototype.flags": {
|
||||
"version": "1.5.0",
|
||||
"resolved": "https://registry.npmjs.org/regexp.prototype.flags/-/regexp.prototype.flags-1.5.0.tgz",
|
||||
@@ -7523,6 +7442,11 @@
|
||||
"source-map": "^0.6.0"
|
||||
}
|
||||
},
|
||||
"stream-read-all": {
|
||||
"version": "3.0.1",
|
||||
"resolved": "https://registry.npmjs.org/stream-read-all/-/stream-read-all-3.0.1.tgz",
|
||||
"integrity": "sha512-EWZT9XOceBPlVJRrYcykW8jyRSZYbkb/0ZK36uLEmoWVO5gxBOnntNTseNzfREsqxqdfEGQrD8SXQ3QWbBmq8A=="
|
||||
},
|
||||
"string-width": {
|
||||
"version": "4.2.3",
|
||||
"resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz",
|
||||
@@ -7604,25 +7528,28 @@
|
||||
"dev": true
|
||||
},
|
||||
"table-layout": {
|
||||
"version": "1.0.2",
|
||||
"resolved": "https://registry.npmjs.org/table-layout/-/table-layout-1.0.2.tgz",
|
||||
"integrity": "sha512-qd/R7n5rQTRFi+Zf2sk5XVVd9UQl6ZkduPFC3S7WEGJAmetDTjY3qPN50eSKzwuzEyQKy5TN2TiZdkIjos2L6A==",
|
||||
"version": "3.0.2",
|
||||
"resolved": "https://registry.npmjs.org/table-layout/-/table-layout-3.0.2.tgz",
|
||||
"integrity": "sha512-rpyNZYRw+/C+dYkcQ3Pr+rLxW4CfHpXjPDnG7lYhdRoUcZTUt+KEsX+94RGp/aVp/MQU35JCITv2T/beY4m+hw==",
|
||||
"requires": {
|
||||
"array-back": "^4.0.1",
|
||||
"deep-extend": "~0.6.0",
|
||||
"typical": "^5.2.0",
|
||||
"wordwrapjs": "^4.0.0"
|
||||
"@75lb/deep-merge": "^1.1.1",
|
||||
"array-back": "^6.2.2",
|
||||
"command-line-args": "^5.2.1",
|
||||
"command-line-usage": "^7.0.0",
|
||||
"stream-read-all": "^3.0.1",
|
||||
"typical": "^7.1.1",
|
||||
"wordwrapjs": "^5.1.0"
|
||||
},
|
||||
"dependencies": {
|
||||
"array-back": {
|
||||
"version": "4.0.2",
|
||||
"resolved": "https://registry.npmjs.org/array-back/-/array-back-4.0.2.tgz",
|
||||
"integrity": "sha512-NbdMezxqf94cnNfWLL7V/im0Ub+Anbb0IoZhvzie8+4HJ4nMQuzHuy49FkGYCJK2yAloZ3meiB6AVMClbrI1vg=="
|
||||
"version": "6.2.2",
|
||||
"resolved": "https://registry.npmjs.org/array-back/-/array-back-6.2.2.tgz",
|
||||
"integrity": "sha512-gUAZ7HPyb4SJczXAMUXMGAvI976JoK3qEx9v1FTmeYuJj0IBiaKttG1ydtGKdkfqWkIkouke7nG8ufGy77+Cvw=="
|
||||
},
|
||||
"typical": {
|
||||
"version": "5.2.0",
|
||||
"resolved": "https://registry.npmjs.org/typical/-/typical-5.2.0.tgz",
|
||||
"integrity": "sha512-dvdQgNDNJo+8B2uBQoqdb11eUCE1JQXhvjC/CZtgvZseVd5TYMXnq0+vuUemXbd/Se29cTaUuPX3YIc2xgbvIg=="
|
||||
"version": "7.1.1",
|
||||
"resolved": "https://registry.npmjs.org/typical/-/typical-7.1.1.tgz",
|
||||
"integrity": "sha512-T+tKVNs6Wu7IWiAce5BgMd7OZfNYUndHwc5MknN+UHOudi7sGZzuHdCadllRuqJ3fPtgFtIH9+lt9qRv6lmpfA=="
|
||||
}
|
||||
}
|
||||
},
|
||||
@@ -7940,20 +7867,9 @@
|
||||
"dev": true
|
||||
},
|
||||
"wordwrapjs": {
|
||||
"version": "4.0.1",
|
||||
"resolved": "https://registry.npmjs.org/wordwrapjs/-/wordwrapjs-4.0.1.tgz",
|
||||
"integrity": "sha512-kKlNACbvHrkpIw6oPeYDSmdCTu2hdMHoyXLTcUKala++lx5Y+wjJ/e474Jqv5abnVmwxw08DiTuHmw69lJGksA==",
|
||||
"requires": {
|
||||
"reduce-flatten": "^2.0.0",
|
||||
"typical": "^5.2.0"
|
||||
},
|
||||
"dependencies": {
|
||||
"typical": {
|
||||
"version": "5.2.0",
|
||||
"resolved": "https://registry.npmjs.org/typical/-/typical-5.2.0.tgz",
|
||||
"integrity": "sha512-dvdQgNDNJo+8B2uBQoqdb11eUCE1JQXhvjC/CZtgvZseVd5TYMXnq0+vuUemXbd/Se29cTaUuPX3YIc2xgbvIg=="
|
||||
}
|
||||
}
|
||||
"version": "5.1.0",
|
||||
"resolved": "https://registry.npmjs.org/wordwrapjs/-/wordwrapjs-5.1.0.tgz",
|
||||
"integrity": "sha512-JNjcULU2e4KJwUNv6CHgI46UvDGitb6dGryHajXTDiLgg1/RiGoPSDw4kZfYnwGtEXf2ZMeIewDQgFGzkCB2Sg=="
|
||||
},
|
||||
"workerpool": {
|
||||
"version": "6.2.1",
|
||||
|
||||
@@ -57,9 +57,9 @@
|
||||
"uuid": "^9.0.0"
|
||||
},
|
||||
"dependencies": {
|
||||
"@apache-arrow/ts": "^12.0.0",
|
||||
"@apache-arrow/ts": "^14.0.2",
|
||||
"@neon-rs/load": "^0.0.74",
|
||||
"apache-arrow": "^12.0.0",
|
||||
"apache-arrow": "^14.0.2",
|
||||
"axios": "^1.4.0"
|
||||
},
|
||||
"os": [
|
||||
|
||||
@@ -17,10 +17,9 @@ import {
|
||||
Float32,
|
||||
makeBuilder,
|
||||
RecordBatchFileWriter,
|
||||
Utf8,
|
||||
type Vector,
|
||||
Utf8, type Vector,
|
||||
FixedSizeList,
|
||||
vectorFromArray, type Schema, Table as ArrowTable, RecordBatchStreamWriter, List, Float64
|
||||
vectorFromArray, type Schema, Table as ArrowTable, RecordBatchStreamWriter, List, Float64, RecordBatch, makeData, Struct
|
||||
} from 'apache-arrow'
|
||||
import { type EmbeddingFunction } from './index'
|
||||
|
||||
@@ -78,6 +77,7 @@ export async function convertToTable<T> (data: Array<Record<string, unknown>>, e
|
||||
}
|
||||
records[columnsKey] = listBuilder.finish().toVector()
|
||||
} else {
|
||||
// TODO if this is a struct field then recursively align the subfields
|
||||
records[columnsKey] = vectorFromArray(values)
|
||||
}
|
||||
}
|
||||
@@ -110,21 +110,27 @@ function newVectorType (dim: number): FixedSizeList<Float32> {
|
||||
}
|
||||
|
||||
// Converts an Array of records into Arrow IPC format
|
||||
export async function fromRecordsToBuffer<T> (data: Array<Record<string, unknown>>, embeddings?: EmbeddingFunction<T>): Promise<Buffer> {
|
||||
const table = await convertToTable(data, embeddings)
|
||||
export async function fromRecordsToBuffer<T> (data: Array<Record<string, unknown>>, embeddings?: EmbeddingFunction<T>, schema?: Schema): Promise<Buffer> {
|
||||
let table = await convertToTable(data, embeddings)
|
||||
if (schema !== undefined) {
|
||||
table = alignTable(table, schema)
|
||||
}
|
||||
const writer = RecordBatchFileWriter.writeAll(table)
|
||||
return Buffer.from(await writer.toUint8Array())
|
||||
}
|
||||
|
||||
// Converts an Array of records into Arrow IPC stream format
|
||||
export async function fromRecordsToStreamBuffer<T> (data: Array<Record<string, unknown>>, embeddings?: EmbeddingFunction<T>): Promise<Buffer> {
|
||||
const table = await convertToTable(data, embeddings)
|
||||
export async function fromRecordsToStreamBuffer<T> (data: Array<Record<string, unknown>>, embeddings?: EmbeddingFunction<T>, schema?: Schema): Promise<Buffer> {
|
||||
let table = await convertToTable(data, embeddings)
|
||||
if (schema !== undefined) {
|
||||
table = alignTable(table, schema)
|
||||
}
|
||||
const writer = RecordBatchStreamWriter.writeAll(table)
|
||||
return Buffer.from(await writer.toUint8Array())
|
||||
}
|
||||
|
||||
// Converts an Arrow Table into Arrow IPC format
|
||||
export async function fromTableToBuffer<T> (table: ArrowTable, embeddings?: EmbeddingFunction<T>): Promise<Buffer> {
|
||||
export async function fromTableToBuffer<T> (table: ArrowTable, embeddings?: EmbeddingFunction<T>, schema?: Schema): Promise<Buffer> {
|
||||
if (embeddings !== undefined) {
|
||||
const source = table.getChild(embeddings.sourceColumn)
|
||||
|
||||
@@ -136,12 +142,15 @@ export async function fromTableToBuffer<T> (table: ArrowTable, embeddings?: Embe
|
||||
const column = vectorFromArray(vectors, newVectorType(vectors[0].length))
|
||||
table = table.assign(new ArrowTable({ vector: column }))
|
||||
}
|
||||
if (schema !== undefined) {
|
||||
table = alignTable(table, schema)
|
||||
}
|
||||
const writer = RecordBatchFileWriter.writeAll(table)
|
||||
return Buffer.from(await writer.toUint8Array())
|
||||
}
|
||||
|
||||
// Converts an Arrow Table into Arrow IPC stream format
|
||||
export async function fromTableToStreamBuffer<T> (table: ArrowTable, embeddings?: EmbeddingFunction<T>): Promise<Buffer> {
|
||||
export async function fromTableToStreamBuffer<T> (table: ArrowTable, embeddings?: EmbeddingFunction<T>, schema?: Schema): Promise<Buffer> {
|
||||
if (embeddings !== undefined) {
|
||||
const source = table.getChild(embeddings.sourceColumn)
|
||||
|
||||
@@ -153,10 +162,36 @@ export async function fromTableToStreamBuffer<T> (table: ArrowTable, embeddings?
|
||||
const column = vectorFromArray(vectors, newVectorType(vectors[0].length))
|
||||
table = table.assign(new ArrowTable({ vector: column }))
|
||||
}
|
||||
if (schema !== undefined) {
|
||||
table = alignTable(table, schema)
|
||||
}
|
||||
const writer = RecordBatchStreamWriter.writeAll(table)
|
||||
return Buffer.from(await writer.toUint8Array())
|
||||
}
|
||||
|
||||
function alignBatch (batch: RecordBatch, schema: Schema): RecordBatch {
|
||||
const alignedChildren = []
|
||||
for (const field of schema.fields) {
|
||||
const indexInBatch = batch.schema.fields?.findIndex((f) => f.name === field.name)
|
||||
if (indexInBatch < 0) {
|
||||
throw new Error(`The column ${field.name} was not found in the Arrow Table`)
|
||||
}
|
||||
alignedChildren.push(batch.data.children[indexInBatch])
|
||||
}
|
||||
const newData = makeData({
|
||||
type: new Struct(schema.fields),
|
||||
length: batch.numRows,
|
||||
nullCount: batch.nullCount,
|
||||
children: alignedChildren
|
||||
})
|
||||
return new RecordBatch(schema, newData)
|
||||
}
|
||||
|
||||
function alignTable (table: ArrowTable, schema: Schema): ArrowTable {
|
||||
const alignedBatches = table.batches.map(batch => alignBatch(batch, schema))
|
||||
return new ArrowTable(schema, alignedBatches)
|
||||
}
|
||||
|
||||
// Creates an empty Arrow Table
|
||||
export function createEmptyTable (schema: Schema): ArrowTable {
|
||||
return new ArrowTable(schema)
|
||||
|
||||
@@ -485,10 +485,10 @@ export class LocalConnection implements Connection {
|
||||
}
|
||||
buffer = await fromTableToBuffer(createEmptyTable(schema))
|
||||
} else if (data instanceof ArrowTable) {
|
||||
buffer = await fromTableToBuffer(data, embeddingFunction)
|
||||
buffer = await fromTableToBuffer(data, embeddingFunction, schema)
|
||||
} else {
|
||||
// data is Array<Record<...>>
|
||||
buffer = await fromRecordsToBuffer(data, embeddingFunction)
|
||||
buffer = await fromRecordsToBuffer(data, embeddingFunction, schema)
|
||||
}
|
||||
|
||||
const tbl = await tableCreate.call(this._db, name, buffer, writeOptions?.writeMode?.toString(), ...getAwsArgs(this._options()))
|
||||
@@ -560,9 +560,10 @@ export class LocalTable<T = number[]> implements Table<T> {
|
||||
* @return The number of rows added to the table
|
||||
*/
|
||||
async add (data: Array<Record<string, unknown>>): Promise<number> {
|
||||
const schema = await this.schema
|
||||
return tableAdd.call(
|
||||
this._tbl,
|
||||
await fromRecordsToBuffer(data, this._embeddings),
|
||||
await fromRecordsToBuffer(data, this._embeddings, schema),
|
||||
WriteMode.Append.toString(),
|
||||
...getAwsArgs(this._options())
|
||||
).then((newTable: any) => { this._tbl = newTable })
|
||||
|
||||
@@ -176,6 +176,26 @@ describe('LanceDB client', function () {
|
||||
assert.deepEqual(await con.tableNames(), ['vectors'])
|
||||
})
|
||||
|
||||
it('create a table with a schema and records', async function () {
|
||||
const dir = await track().mkdir('lancejs')
|
||||
const con = await lancedb.connect(dir)
|
||||
|
||||
const schema = new Schema(
|
||||
[new Field('id', new Int32()),
|
||||
new Field('name', new Utf8()),
|
||||
new Field('vector', new FixedSizeList(2, new Field('item', new Float32(), true)), false)
|
||||
]
|
||||
)
|
||||
const data = [
|
||||
{ vector: [0.5, 0.2], name: 'foo', id: 0 },
|
||||
{ vector: [0.3, 0.1], name: 'bar', id: 1 }
|
||||
]
|
||||
// even thought the keys in data is out of order it should still work
|
||||
const table = await con.createTable({ name: 'vectors', data, schema })
|
||||
assert.equal(table.name, 'vectors')
|
||||
assert.deepEqual(await con.tableNames(), ['vectors'])
|
||||
})
|
||||
|
||||
it('create a table with a empty data array', async function () {
|
||||
const dir = await track().mkdir('lancejs')
|
||||
const con = await lancedb.connect(dir)
|
||||
@@ -294,6 +314,25 @@ describe('LanceDB client', function () {
|
||||
assert.equal(await table.countRows(), 4)
|
||||
})
|
||||
|
||||
it('appends records with fields in a different order', async function () {
|
||||
const dir = await track().mkdir('lancejs')
|
||||
const con = await lancedb.connect(dir)
|
||||
|
||||
const data = [
|
||||
{ id: 1, vector: [0.1, 0.2], price: 10, name: 'a' },
|
||||
{ id: 2, vector: [1.1, 1.2], price: 50, name: 'b' }
|
||||
]
|
||||
|
||||
const table = await con.createTable('vectors', data)
|
||||
|
||||
const dataAdd = [
|
||||
{ id: 3, vector: [2.1, 2.2], name: 'c', price: 10 },
|
||||
{ id: 4, vector: [3.1, 3.2], name: 'd', price: 50 }
|
||||
]
|
||||
await table.add(dataAdd)
|
||||
assert.equal(await table.countRows(), 4)
|
||||
})
|
||||
|
||||
it('overwrite all records in a table', async function () {
|
||||
const uri = await createTestDB()
|
||||
const con = await lancedb.connect(uri)
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
[bumpversion]
|
||||
current_version = 0.4.3
|
||||
current_version = 0.4.4
|
||||
commit = True
|
||||
message = [python] Bump version: {current_version} → {new_version}
|
||||
tag = True
|
||||
|
||||
@@ -45,8 +45,8 @@ pytest
|
||||
To run linter and automatically fix all errors:
|
||||
|
||||
```bash
|
||||
black .
|
||||
isort .
|
||||
ruff format python
|
||||
ruff --fix python
|
||||
```
|
||||
|
||||
If any packages are missing, install them with:
|
||||
@@ -82,4 +82,4 @@ pip install tantivy
|
||||
To run the unit tests:
|
||||
```bash
|
||||
pytest
|
||||
```
|
||||
```
|
||||
|
||||
@@ -260,20 +260,30 @@ class LanceQueryBuilder(ABC):
|
||||
for row in self.to_arrow().to_pylist()
|
||||
]
|
||||
|
||||
def limit(self, limit: int) -> LanceQueryBuilder:
|
||||
def limit(self, limit: Union[int, None]) -> LanceQueryBuilder:
|
||||
"""Set the maximum number of results to return.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
limit: int
|
||||
The maximum number of results to return.
|
||||
By default the query is limited to the first 10.
|
||||
Call this method and pass 0, a negative value,
|
||||
or None to remove the limit.
|
||||
*WARNING* if you have a large dataset, removing
|
||||
the limit can potentially result in reading a
|
||||
large amount of data into memory and cause
|
||||
out of memory issues.
|
||||
|
||||
Returns
|
||||
-------
|
||||
LanceQueryBuilder
|
||||
The LanceQueryBuilder object.
|
||||
"""
|
||||
self._limit = limit
|
||||
if limit is None or limit <= 0:
|
||||
self._limit = None
|
||||
else:
|
||||
self._limit = limit
|
||||
return self
|
||||
|
||||
def select(self, columns: list) -> LanceQueryBuilder:
|
||||
@@ -468,6 +478,24 @@ class LanceFtsQueryBuilder(LanceQueryBuilder):
|
||||
def __init__(self, table: "lancedb.table.Table", query: str):
|
||||
super().__init__(table)
|
||||
self._query = query
|
||||
self._phrase_query = False
|
||||
|
||||
def phrase_query(self, phrase_query: bool = True) -> LanceFtsQueryBuilder:
|
||||
"""Set whether to use phrase query.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
phrase_query: bool, default True
|
||||
If True, then the query will be wrapped in quotes and
|
||||
double quotes replaced by single quotes.
|
||||
|
||||
Returns
|
||||
-------
|
||||
LanceFtsQueryBuilder
|
||||
The LanceFtsQueryBuilder object.
|
||||
"""
|
||||
self._phrase_query = phrase_query
|
||||
return self
|
||||
|
||||
def to_arrow(self) -> pa.Table:
|
||||
try:
|
||||
@@ -490,7 +518,11 @@ class LanceFtsQueryBuilder(LanceQueryBuilder):
|
||||
# open the index
|
||||
index = tantivy.Index.open(index_path)
|
||||
# get the scores and doc ids
|
||||
row_ids, scores = search_index(index, self._query, self._limit)
|
||||
query = self._query
|
||||
if self._phrase_query:
|
||||
query = query.replace('"', "'")
|
||||
query = f'"{query}"'
|
||||
row_ids, scores = search_index(index, query, self._limit)
|
||||
if len(row_ids) == 0:
|
||||
empty_schema = pa.schema([pa.field("score", pa.float32())])
|
||||
return pa.Table.from_pylist([], schema=empty_schema)
|
||||
|
||||
@@ -13,10 +13,10 @@
|
||||
|
||||
|
||||
import functools
|
||||
from typing import Any, Callable, Dict, Iterable, Optional, Union
|
||||
from typing import Any, Callable, Dict, Iterable, List, Optional, Union
|
||||
from urllib.parse import urljoin
|
||||
|
||||
import requests
|
||||
import urllib.parse
|
||||
import attrs
|
||||
import pyarrow as pa
|
||||
from pydantic import BaseModel
|
||||
@@ -39,7 +39,7 @@ def _check_not_closed(f):
|
||||
|
||||
|
||||
def _read_ipc(resp: requests.Response) -> pa.Table:
|
||||
resp_body = resp.raw.read()
|
||||
resp_body = resp.content
|
||||
with pa.ipc.open_file(pa.BufferReader(resp_body)) as reader:
|
||||
return reader.read_all()
|
||||
|
||||
@@ -55,21 +55,15 @@ class RestfulLanceDBClient:
|
||||
|
||||
@functools.cached_property
|
||||
def session(self) -> requests.Session:
|
||||
session = requests.session()
|
||||
session.stream = True
|
||||
return requests.Session()
|
||||
|
||||
return session
|
||||
|
||||
@functools.cached_property
|
||||
@property
|
||||
def url(self) -> str:
|
||||
return (
|
||||
self.host_override
|
||||
or f"https://{self.db_name}.{self.region}.api.lancedb.com"
|
||||
)
|
||||
|
||||
def _get_request_url(self, uri: str) -> str:
|
||||
return urllib.parse.urljoin(self.url, uri)
|
||||
|
||||
def close(self):
|
||||
self.session.close()
|
||||
self.closed = True
|
||||
@@ -85,22 +79,36 @@ class RestfulLanceDBClient:
|
||||
headers["x-lancedb-database"] = self.db_name
|
||||
return headers
|
||||
|
||||
@staticmethod
|
||||
def _check_status(resp: requests.Response):
|
||||
if resp.status_code == 404:
|
||||
raise LanceDBClientError(f"Not found: {resp.text}")
|
||||
elif 400 <= resp.status_code < 500:
|
||||
raise LanceDBClientError(
|
||||
f"Bad Request: {resp.status_code}, error: {resp.text}"
|
||||
)
|
||||
elif 500 <= resp.status_code < 600:
|
||||
raise LanceDBClientError(
|
||||
f"Internal Server Error: {resp.status_code}, error: {resp.text}"
|
||||
)
|
||||
elif resp.status_code != 200:
|
||||
raise LanceDBClientError(
|
||||
f"Unknown Error: {resp.status_code}, error: {resp.text}"
|
||||
)
|
||||
|
||||
@_check_not_closed
|
||||
def get(self, uri: str, params: Union[Dict[str, Any], BaseModel] = None):
|
||||
"""Send a GET request and returns the deserialized response payload."""
|
||||
if isinstance(params, BaseModel):
|
||||
params: Dict[str, Any] = params.dict(exclude_none=True)
|
||||
|
||||
resp = self.session.get(
|
||||
self._get_request_url(uri),
|
||||
with self.session.get(
|
||||
urljoin(self.url, uri),
|
||||
params=params,
|
||||
headers=self.headers,
|
||||
# 5s connect timeout, 30s read timeout
|
||||
timeout=(5.0, 30.0),
|
||||
)
|
||||
|
||||
resp.raise_for_status()
|
||||
return resp.json()
|
||||
) as resp:
|
||||
self._check_status(resp)
|
||||
return resp.json()
|
||||
|
||||
@_check_not_closed
|
||||
def post(
|
||||
@@ -135,23 +143,18 @@ class RestfulLanceDBClient:
|
||||
headers["content-type"] = content_type
|
||||
if request_id is not None:
|
||||
headers["x-request-id"] = request_id
|
||||
|
||||
resp = self.session.post(
|
||||
self._get_request_url(uri),
|
||||
with self.session.post(
|
||||
urljoin(self.url, uri),
|
||||
headers=headers,
|
||||
params=params,
|
||||
headers=self.headers,
|
||||
# 5s connect timeout, 30s read timeout
|
||||
timeout=(5.0, 30.0),
|
||||
**req_kwargs,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
|
||||
return deserialize(resp)
|
||||
) as resp:
|
||||
self._check_status(resp)
|
||||
return deserialize(resp)
|
||||
|
||||
@_check_not_closed
|
||||
def list_tables(
|
||||
self, limit: int, page_token: Optional[str] = None
|
||||
) -> Iterable[str]:
|
||||
def list_tables(self, limit: int, page_token: Optional[str] = None) -> List[str]:
|
||||
"""List all tables in the database."""
|
||||
if page_token is None:
|
||||
page_token = ""
|
||||
|
||||
@@ -73,12 +73,13 @@ class RemoteDBConnection(DBConnection):
|
||||
"""
|
||||
while True:
|
||||
result = self._client.list_tables(limit, page_token)
|
||||
|
||||
if len(result) > 0:
|
||||
page_token = result[len(result) - 1]
|
||||
else:
|
||||
break
|
||||
for item in result:
|
||||
yield item
|
||||
if len(result) < limit:
|
||||
break
|
||||
else:
|
||||
page_token = result[len(result) - 1]
|
||||
|
||||
@override
|
||||
def open_table(self, name: str) -> Table:
|
||||
@@ -246,6 +247,7 @@ class RemoteDBConnection(DBConnection):
|
||||
request_id=request_id,
|
||||
content_type=ARROW_STREAM_CONTENT_TYPE,
|
||||
)
|
||||
|
||||
return RemoteTable(self, name)
|
||||
|
||||
@override
|
||||
@@ -257,6 +259,7 @@ class RemoteDBConnection(DBConnection):
|
||||
name: str
|
||||
The name of the table.
|
||||
"""
|
||||
|
||||
self._client.post(
|
||||
f"/v1/table/{name}/drop/",
|
||||
)
|
||||
|
||||
@@ -11,6 +11,7 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import asyncio
|
||||
import uuid
|
||||
from functools import cached_property
|
||||
from typing import Dict, Optional, Union
|
||||
@@ -114,6 +115,7 @@ class RemoteTable(Table):
|
||||
resp = self._conn._client.post(
|
||||
f"/v1/table/{self._name}/create_index/", data=data
|
||||
)
|
||||
|
||||
return resp
|
||||
|
||||
def add(
|
||||
@@ -226,17 +228,19 @@ class RemoteTable(Table):
|
||||
and len(query.vector) > 0
|
||||
and not isinstance(query.vector[0], float)
|
||||
):
|
||||
result = []
|
||||
results = []
|
||||
for v in query.vector:
|
||||
v = list(v)
|
||||
q = query.copy()
|
||||
q.vector = v
|
||||
result.append(self._conn._client.query(self._name, q))
|
||||
results.append(self._conn._client.query(self._name, q))
|
||||
|
||||
return pa.concat_tables(
|
||||
[add_index(r.to_arrow(), i) for i, r in enumerate(result)]
|
||||
[add_index(r.to_arrow(), i) for i, r in enumerate(results)]
|
||||
)
|
||||
else:
|
||||
return self._conn._client.query(self._name, query).to_arrow()
|
||||
result = self._conn._client.query(self._name, query)
|
||||
return result.to_arrow()
|
||||
|
||||
def delete(self, predicate: str):
|
||||
"""Delete rows from the table.
|
||||
|
||||
@@ -647,8 +647,19 @@ class LanceTable(Table):
|
||||
self._dataset.restore()
|
||||
self._reset_dataset()
|
||||
|
||||
def count_rows(self, filter: Optional[str] = None) -> int:
|
||||
"""
|
||||
Count the number of rows in the table.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
filter: str, optional
|
||||
A SQL where clause to filter the rows to count.
|
||||
"""
|
||||
return self._dataset.count_rows(filter)
|
||||
|
||||
def __len__(self):
|
||||
return self._dataset.count_rows()
|
||||
return self.count_rows()
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return f"LanceTable({self.name})"
|
||||
|
||||
@@ -1,13 +1,12 @@
|
||||
[project]
|
||||
name = "lancedb"
|
||||
version = "0.4.3"
|
||||
version = "0.4.4"
|
||||
dependencies = [
|
||||
"deprecation",
|
||||
"pylance==0.9.5",
|
||||
"pylance==0.9.6",
|
||||
"ratelimiter~=1.0",
|
||||
"retry>=0.9.2",
|
||||
"tqdm>=4.27.0",
|
||||
"requests>=2.31,<3",
|
||||
"pydantic>=1.10",
|
||||
"attrs>=21.3.0",
|
||||
"semver>=3.0",
|
||||
@@ -49,8 +48,8 @@ classifiers = [
|
||||
repository = "https://github.com/lancedb/lancedb"
|
||||
|
||||
[project.optional-dependencies]
|
||||
tests = ["pandas>=1.4", "pytest", "pytest-mock", "pytest-asyncio", "requests", "duckdb", "pytz"]
|
||||
dev = ["ruff", "pre-commit", "black"]
|
||||
tests = ["aiohttp", "pandas>=1.4", "pytest", "pytest-mock", "pytest-asyncio", "duckdb", "pytz"]
|
||||
dev = ["ruff", "pre-commit"]
|
||||
docs = ["mkdocs", "mkdocs-jupyter", "mkdocs-material", "mkdocstrings[python]"]
|
||||
clip = ["torch", "pillow", "open-clip"]
|
||||
embeddings = ["openai>=1.6.1", "sentence-transformers", "torch", "pillow", "open-clip-torch", "cohere", "InstructorEmbedding"]
|
||||
@@ -62,9 +61,6 @@ lancedb = "lancedb.cli.cli:cli"
|
||||
requires = ["setuptools", "wheel"]
|
||||
build-backend = "setuptools.build_meta"
|
||||
|
||||
[tool.isort]
|
||||
profile = "black"
|
||||
|
||||
[tool.ruff]
|
||||
select = ["F", "E", "W", "I", "G", "TCH", "PERF"]
|
||||
|
||||
|
||||
27
python/tests/test_e2e_remote_db.py
Normal file
27
python/tests/test_e2e_remote_db.py
Normal file
@@ -0,0 +1,27 @@
|
||||
# Copyright 2023 LanceDB Developers
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
from lancedb import LanceDBConnection
|
||||
|
||||
# TODO: setup integ test mark and script
|
||||
|
||||
|
||||
@pytest.mark.skip(reason="Need to set up a local server")
|
||||
def test_against_local_server():
|
||||
conn = LanceDBConnection("lancedb+http://localhost:10024")
|
||||
table = conn.open_table("sift1m_ivf1024_pq16")
|
||||
df = table.search(np.random.rand(128)).to_pandas()
|
||||
assert len(df) == 10
|
||||
@@ -169,13 +169,16 @@ def test_syntax(table):
|
||||
table.create_fts_index("text")
|
||||
with pytest.raises(ValueError, match="Syntax Error"):
|
||||
table.search("they could have been dogs OR cats").limit(10).to_list()
|
||||
table.search("they could have been dogs OR cats").phrase_query().limit(10).to_list()
|
||||
# this should work
|
||||
table.search('"they could have been dogs OR cats"').limit(10).to_list()
|
||||
# this should work too
|
||||
table.search('''"the cats OR dogs were not really 'pets' at all"''').limit(
|
||||
10
|
||||
).to_list()
|
||||
with pytest.raises(ValueError, match="Syntax Error"):
|
||||
table.search('''"the cats OR dogs were not really "pets" at all"''').limit(
|
||||
10
|
||||
).to_list()
|
||||
table.search('the cats OR dogs were not really "pets" at all').phrase_query().limit(
|
||||
10
|
||||
).to_list()
|
||||
table.search('the cats OR dogs were not really "pets" at all').phrase_query().limit(
|
||||
10
|
||||
).to_list()
|
||||
|
||||
95
python/tests/test_remote_client.py
Normal file
95
python/tests/test_remote_client.py
Normal file
@@ -0,0 +1,95 @@
|
||||
# Copyright 2023 LanceDB Developers
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import attrs
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import pyarrow as pa
|
||||
import pytest
|
||||
from aiohttp import web
|
||||
|
||||
from lancedb.remote.client import RestfulLanceDBClient, VectorQuery
|
||||
|
||||
|
||||
@attrs.define
|
||||
class MockLanceDBServer:
|
||||
runner: web.AppRunner = attrs.field(init=False)
|
||||
site: web.TCPSite = attrs.field(init=False)
|
||||
|
||||
async def query_handler(self, request: web.Request) -> web.Response:
|
||||
table_name = request.match_info["table_name"]
|
||||
assert table_name == "test_table"
|
||||
|
||||
await request.json()
|
||||
# TODO: do some matching
|
||||
|
||||
vecs = pd.Series([np.random.rand(128) for x in range(10)], name="vector")
|
||||
ids = pd.Series(range(10), name="id")
|
||||
df = pd.DataFrame([vecs, ids]).T
|
||||
|
||||
batch = pa.RecordBatch.from_pandas(
|
||||
df,
|
||||
schema=pa.schema(
|
||||
[
|
||||
pa.field("vector", pa.list_(pa.float32(), 128)),
|
||||
pa.field("id", pa.int64()),
|
||||
]
|
||||
),
|
||||
)
|
||||
|
||||
sink = pa.BufferOutputStream()
|
||||
with pa.ipc.new_file(sink, batch.schema) as writer:
|
||||
writer.write_batch(batch)
|
||||
|
||||
return web.Response(body=sink.getvalue().to_pybytes())
|
||||
|
||||
async def setup(self):
|
||||
app = web.Application()
|
||||
app.add_routes([web.post("/table/{table_name}", self.query_handler)])
|
||||
self.runner = web.AppRunner(app)
|
||||
await self.runner.setup()
|
||||
self.site = web.TCPSite(self.runner, "localhost", 8111)
|
||||
|
||||
async def start(self):
|
||||
await self.site.start()
|
||||
|
||||
async def stop(self):
|
||||
await self.runner.cleanup()
|
||||
|
||||
|
||||
@pytest.mark.skip(reason="flaky somehow, fix later")
|
||||
@pytest.mark.asyncio
|
||||
async def test_e2e_with_mock_server():
|
||||
mock_server = MockLanceDBServer()
|
||||
await mock_server.setup()
|
||||
await mock_server.start()
|
||||
|
||||
try:
|
||||
client = RestfulLanceDBClient("lancedb+http://localhost:8111")
|
||||
df = (
|
||||
await client.query(
|
||||
"test_table",
|
||||
VectorQuery(
|
||||
vector=np.random.rand(128).tolist(),
|
||||
k=10,
|
||||
_metric="L2",
|
||||
columns=["id", "vector"],
|
||||
),
|
||||
)
|
||||
).to_pandas()
|
||||
|
||||
assert "vector" in df.columns
|
||||
assert "id" in df.columns
|
||||
finally:
|
||||
# make sure we don't leak resources
|
||||
await mock_server.stop()
|
||||
@@ -18,15 +18,15 @@ from lancedb.remote.client import VectorQuery, VectorQueryResult
|
||||
|
||||
|
||||
class FakeLanceDBClient:
|
||||
async def close(self):
|
||||
def close(self):
|
||||
pass
|
||||
|
||||
async def query(self, table_name: str, query: VectorQuery) -> VectorQueryResult:
|
||||
def query(self, table_name: str, query: VectorQuery) -> VectorQueryResult:
|
||||
assert table_name == "test"
|
||||
t = pa.schema([]).empty_table()
|
||||
return VectorQueryResult(t)
|
||||
|
||||
async def post(self, path: str):
|
||||
def post(self, path: str):
|
||||
pass
|
||||
|
||||
|
||||
|
||||
@@ -569,6 +569,14 @@ def test_empty_query(db):
|
||||
val = df.id.iloc[0]
|
||||
assert val == 1
|
||||
|
||||
table = LanceTable.create(db, "my_table2", data=[{"id": i} for i in range(100)])
|
||||
df = table.search().select(["id"]).to_pandas()
|
||||
assert len(df) == 10
|
||||
df = table.search().select(["id"]).limit(None).to_pandas()
|
||||
assert len(df) == 100
|
||||
df = table.search().select(["id"]).limit(-1).to_pandas()
|
||||
assert len(df) == 100
|
||||
|
||||
|
||||
def test_compact_cleanup(db):
|
||||
table = LanceTable.create(
|
||||
@@ -597,3 +605,14 @@ def test_compact_cleanup(db):
|
||||
|
||||
with pytest.raises(Exception, match="Version 3 no longer exists"):
|
||||
table.checkout(3)
|
||||
|
||||
|
||||
def test_count_rows(db):
|
||||
table = LanceTable.create(
|
||||
db,
|
||||
"my_table",
|
||||
data=[{"text": "foo", "id": 0}, {"text": "bar", "id": 1}],
|
||||
)
|
||||
assert len(table) == 2
|
||||
assert table.count_rows() == 2
|
||||
assert table.count_rows(filter="text='bar'") == 1
|
||||
|
||||
@@ -50,7 +50,7 @@ pub(crate) fn record_batch_to_buffer(batches: Vec<RecordBatch>) -> Result<Vec<u8
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
|
||||
let schema = batches.get(0).unwrap().schema();
|
||||
let schema = batches.first().unwrap().schema();
|
||||
let mut fr = FileWriter::try_new(Vec::new(), schema.deref())?;
|
||||
for batch in batches.iter() {
|
||||
fr.write(batch)?
|
||||
|
||||
Reference in New Issue
Block a user