From e8a8dd1b2a240646b39ecfc848b1c3dd4a0701fc Mon Sep 17 00:00:00 2001 From: Jack Ye Date: Sat, 14 Nov 2026 16:55:42 -0800 Subject: [PATCH] feat(sql): use server-issued catalog listing tokens (#5026) Migrates SQL-backed catalog listing APIs to `LIMIT ... FROM ''` and server-issued Arrow continuation metadata. Namespace-specific APIs send an explicit `IN` schema clause, including `IN "public"` for root, so raw SQL can default to all namespaces. ```sql SHOW VIEWS LIMIT 100; SHOW VIEWS IN team$ml LIMIT 100; SHOW VIEWS IN team$ml LIMIT 100 FROM ''; SHOW VIEWS IN other_db.team$ml LIMIT 100; ``` SDK clients forward opaque tokens unchanged, preserve empty continuations, and decode identifier columns by name. Removes legacy cursor reconstruction and updates the SQL reference and Python Flight fixture. Public docs cover optional direct schema paths, all-namespace defaults, and the shared pagination syntax across every catalog listing. --- docs/web/docs.json | 1 + docs/web/management/function.mdx | 4 +- docs/web/management/index.mdx | 2 +- docs/web/sql/ddl/database.mdx | 20 +- docs/web/sql/ddl/function.mdx | 37 +- docs/web/sql/ddl/materialized-view.mdx | 25 +- docs/web/sql/ddl/namespace.mdx | 40 +- docs/web/sql/ddl/secret.mdx | 30 +- docs/web/sql/ddl/storage-integration.mdx | 25 +- docs/web/sql/ddl/table.mdx | 20 + docs/web/sql/ddl/view.mdx | 33 +- docs/web/sql/pagination.mdx | 68 +++ .../python/tests/test_storage_integration.py | 30 +- rust/lancedb/src/listing.rs | 8 +- rust/lancedb/src/remote/db.rs | 27 +- rust/lancedb/src/remote/sql/function.rs | 85 +-- .../src/remote/sql/materialized_view.rs | 58 +- rust/lancedb/src/remote/sql/objects.rs | 144 ++++- rust/lancedb/src/remote/sql/secret.rs | 62 +- rust/lancedb/src/remote/sql/view.rs | 62 +- rust/lancedb/src/remote/sql_test.rs | 554 +++++++++--------- .../lancedb/src/remote/storage_integration.rs | 23 - 22 files changed, 663 insertions(+), 695 deletions(-) create mode 100644 docs/web/sql/pagination.mdx diff --git a/docs/web/docs.json b/docs/web/docs.json index c586d4f62..0b26a7fc0 100644 --- a/docs/web/docs.json +++ b/docs/web/docs.json @@ -543,6 +543,7 @@ "group": "SQL Reference", "pages": [ "sql/index", + "sql/pagination", "sql/data-types", { "group": "DDL", diff --git a/docs/web/management/function.mdx b/docs/web/management/function.mdx index 41e47dee1..562de05c6 100644 --- a/docs/web/management/function.mdx +++ b/docs/web/management/function.mdx @@ -280,7 +280,9 @@ let latest = connection.get_function_latest("double", &[]).await?; ```sql SQL icon="database" SHOW FUNCTIONS -SHOW FUNCTIONS IN "analytics" WITH (name = 'double', limit = 100) +SHOW FUNCTIONS IN "analytics" LIMIT 100 +SHOW FUNCTIONS HISTORY IN "analytics" LIMIT 100 +SHOW FUNCTIONS HISTORY IN "analytics" LIMIT 100 FROM '' DESCRIBE FUNCTION double WITH (version = '1') DESCRIBE FUNCTION double diff --git a/docs/web/management/index.mdx b/docs/web/management/index.mdx index fbef65756..50f6e7507 100644 --- a/docs/web/management/index.mdx +++ b/docs/web/management/index.mdx @@ -85,7 +85,7 @@ enables them; the SDK methods send the view's SQL query to the deployment. See | --- | --- | --- | --- | | create | `create_function`, `create_function_async` | -- | `CREATE FUNCTION` | | get | `get_function` | -- | `DESCRIBE FUNCTION` | -| list | `list_functions` | -- | `SHOW FUNCTIONS` | +| list | `list_functions` | -- | `SHOW FUNCTIONS HISTORY` | | drop | `drop_function` | -- | `DROP FUNCTION` | Enterprise A function runs on the server, and only on diff --git a/docs/web/sql/ddl/database.mdx b/docs/web/sql/ddl/database.mdx index e8cdb55bc..f84d4cfb9 100644 --- a/docs/web/sql/ddl/database.mdx +++ b/docs/web/sql/ddl/database.mdx @@ -57,15 +57,21 @@ properties and requires system `USAGE`. ## SHOW DATABASES {#show} - -```sql SQL icon="database" -SHOW DATABASES +```sql +SHOW DATABASES LIMIT 100; +SHOW DATABASES LIMIT 100 FROM ''; ``` - -Returns a sorted `name` column containing the databases the caller has -`USAGE` on, including `system` when permitted. `LIKE` and `WHERE` filters -are unsupported. +Returns one non-null `name` column, sorted by database name and filtered to +databases the caller has `USAGE` on, including `system` when permitted. +Databases are above namespaces, so this command does not accept `IN` or +return a `namespace_path` column. + +Page through results using `LIMIT n [FROM '']`. Read the opaque +`lancedb.next_page_token` from the Arrow result schema metadata and pass it back +unchanged with the same object type and scope. Stop when no token is returned; +a short or empty page can still carry a continuation. See +[catalog listing pagination](/sql/pagination). ## ATTACH DATABASE {#attach} diff --git a/docs/web/sql/ddl/function.mdx b/docs/web/sql/ddl/function.mdx index 55e2d6fe7..691118026 100644 --- a/docs/web/sql/ddl/function.mdx +++ b/docs/web/sql/ddl/function.mdx @@ -135,20 +135,33 @@ from it. ## SHOW FUNCTIONS {#show} - -```sql SQL icon="database" -SHOW FUNCTIONS -SHOW FUNCTIONS HISTORY IN "analytics" WITH (name = 'embed', limit = 100) -SHOW FUNCTIONS HISTORY WITH (after_name = 'embed', after_version = '1', limit = 100) +```sql +SHOW FUNCTIONS IN analytics.`team$ml` LIMIT 100; +SHOW FUNCTIONS IN analytics.`team$ml` LIMIT 100 FROM ''; ``` - -`SHOW FUNCTIONS` returns the highest published numeric revision for each name. -Add `HISTORY` to list every published revision, as Python's `list_functions()` -does. Rows are ordered by name and numeric revision, so `2` comes before `10`. -`after_name` and `after_version` must be supplied together for keyset pagination. -`limit` bounds -the response; `include_definition = false` omits the optional definition JSON. +Returns two non-null string columns: `namespace_path`, a compact JSON array +of parent namespace components (`[]` for root), and `name`, the unqualified +object name. Omitting `IN` lists all authorized namespaces in the current database; +`IN team$ml` selects that exact path without a database qualifier; `IN public` selects +the current database's root. Listings require the relevant database and +namespace `USAGE` privileges. + +`SHOW FUNCTIONS` lists each function once. Add `HISTORY` to list every +published revision with its metadata and definition, as `list_functions()` +does. History uses the same pagination suffix; numeric revision order is +encoded in the token, so revision `2` precedes `10`. + +```sql +SHOW FUNCTIONS HISTORY IN analytics.`team$ml` LIMIT 100; +SHOW FUNCTIONS HISTORY IN analytics.`team$ml` LIMIT 100 FROM ''; +``` + +Page through results using `LIMIT n [FROM '']`. Read the opaque +`lancedb.next_page_token` from the Arrow result schema metadata and pass it back +unchanged with the same object type and scope. Stop when no token is returned; +a short or empty page can still carry a continuation. See +[catalog listing pagination](/sql/pagination). ## DROP FUNCTION {#drop} diff --git a/docs/web/sql/ddl/materialized-view.mdx b/docs/web/sql/ddl/materialized-view.mdx index a2453ba01..a2a5024f6 100644 --- a/docs/web/sql/ddl/materialized-view.mdx +++ b/docs/web/sql/ddl/materialized-view.mdx @@ -70,16 +70,25 @@ automatic execution needs the same server-side switch enabled. Omitting ## SHOW MATERIALIZED VIEWS {#show} - -```sql SQL icon="database" -SHOW MATERIALIZED VIEWS +```sql +SHOW MATERIALIZED VIEWS IN analytics.`team$ml` LIMIT 100; +SHOW MATERIALIZED VIEWS IN analytics.`team$ml` LIMIT 100 FROM ''; ``` - - -This asks the server for its views, as the SDK's `list_materialized_views` does -on a `db://` connection. - +Returns two non-null string columns: `namespace_path`, a compact JSON array +of parent namespace components (`[]` for root), and `name`, the unqualified +object name. Omitting `IN` lists all authorized namespaces in the current database; +`IN team$ml` selects that exact path without a database qualifier; `IN public` selects +the current database's root. Listings require the relevant database and +namespace `USAGE` privileges. + +The listing contains identifiers; use `DESCRIBE MATERIALIZED VIEW` for its defining query and refresh metadata. + +Page through results using `LIMIT n [FROM '']`. Read the opaque +`lancedb.next_page_token` from the Arrow result schema metadata and pass it back +unchanged with the same object type and scope. Stop when no token is returned; +a short or empty page can still carry a continuation. See +[catalog listing pagination](/sql/pagination). ## Refreshing one {#refresh} diff --git a/docs/web/sql/ddl/namespace.mdx b/docs/web/sql/ddl/namespace.mdx index 8f446f1e7..14391ac18 100644 --- a/docs/web/sql/ddl/namespace.mdx +++ b/docs/web/sql/ddl/namespace.mdx @@ -17,7 +17,7 @@ backtick-quoted identifier with `$` between components, such as one path component, relative to an explicit parent schema or the session's current schema -- it cannot contain `$` or be qualified with a database. -`SCHEMA` never accepts `IN`. `NAMESPACE` accepts `IN` followed by a complete +For single-object commands, `SCHEMA` never accepts `IN`. `NAMESPACE` accepts `IN` followed by a complete schema identifier. Names are lowercased unless quoted. ## CREATE SCHEMA and CREATE NAMESPACE {#create} @@ -126,27 +126,30 @@ Requires `USAGE` on the database and the namespace. ## SHOW SCHEMAS and SHOW NAMESPACES {#show} - -```sql SQL icon="database" -SHOW SCHEMAS -SHOW NAMESPACES -SHOW NAMESPACES IN analytics.team +```sql +SHOW SCHEMAS LIMIT 100; +SHOW SCHEMAS LIMIT 100 FROM ''; +SHOW NAMESPACES IN analytics.team LIMIT 100; +SHOW NAMESPACES IN analytics.team LIMIT 100 FROM ''; ``` - -`SHOW SCHEMAS` returns complete paths in the current database, recursively, -including `public` when the caller can use it. It does not accept `IN`. +Both commands return two non-null string columns: `namespace_path`, the +parent path as a compact JSON array, and `name`, the unqualified child name. +Results are ordered by parent path components, then name, and omit namespaces +the caller cannot use. -`SHOW NAMESPACES` lists only immediate child names under the session's -current schema. `IN` selects a different parent schema, optionally in -another database -- use `IN public` for the current database's root, or -`IN analytics.public` for another database's root. A missing parent is an -error. +`SHOW SCHEMAS` and `SHOW NAMESPACES` are equivalent listings. Without `IN`, +both list recursively in the current database, include the implicit `public` +root, and ignore the session namespace. With `IN`, both list immediate children +of the selected parent. A schema path can appear directly: `IN team$features`; +a database qualifier is optional. `IN public` selects the current database root, +and `IN analytics.public` selects another database's root. -Both forms return a sorted `name` column and omit namespaces the caller -cannot use. Large listings that exceed 10,000 namespaces, or recursive -listings deeper than 32 levels, return an error suggesting a narrower -listing. +Page through results using `LIMIT n [FROM '']`. Read the opaque +`lancedb.next_page_token` from the Arrow result schema metadata and pass it back +unchanged with the same object type and scope. Stop when no token is returned; +a short or empty page can still carry a continuation. See +[catalog listing pagination](/sql/pagination). ## DROP SCHEMA and DROP NAMESPACE {#drop} @@ -187,7 +190,6 @@ with `IF EXISTS`. Inspect a cascade drop's child jobs with | `CREATE SCHEMA child IN parent` | `CREATE NAMESPACE child IN parent` or `` CREATE SCHEMA `parent$child` `` | | `` DESCRIBE NAMESPACE `parent$child` `` | `` DESCRIBE SCHEMA `parent$child` `` or `DESCRIBE NAMESPACE child IN parent` | | `DROP NAMESPACE analytics.child` | `DROP NAMESPACE child IN analytics.public` | -| `SHOW SCHEMAS IN parent` | `SHOW NAMESPACES IN parent` | ## What to read next {#next} diff --git a/docs/web/sql/ddl/secret.mdx b/docs/web/sql/ddl/secret.mdx index 922dec247..15039c6c7 100644 --- a/docs/web/sql/ddl/secret.mdx +++ b/docs/web/sql/ddl/secret.mdx @@ -98,21 +98,25 @@ picked up by every Function still bound to that name. ## SHOW SECRETS {#show} - -```sql SQL icon="database" -SHOW SECRETS -SHOW SECRETS IN platform -SHOW SECRETS IN analytics.`ml$vision` -SHOW SECRETS IN public -SHOW SECRETS IN platform WHERE name > 'model-api-key' LIMIT 50 +```sql +SHOW SECRETS IN analytics.`team$ml` LIMIT 100; +SHOW SECRETS IN analytics.`team$ml` LIMIT 100 FROM ''; ``` - -Lists the Secret names in one namespace as a sorted `name` column. Without -`IN`, it lists the session's namespace; `IN public` lists the root. `WHERE -name > ''` pages past the last name already seen, and `LIMIT` -bounds the page size; `WHERE` must compare `name` with `>`. Requires -`USAGE` on the database and the namespace. +Returns two non-null string columns: `namespace_path`, a compact JSON array +of parent namespace components (`[]` for root), and `name`, the unqualified +object name. Omitting `IN` lists all authorized namespaces in the current database; +`IN team$ml` selects that exact path without a database qualifier; `IN public` selects +the current database's root. Listings require the relevant database and +namespace `USAGE` privileges. + +Secret listings contain no values or Secret metadata. + +Page through results using `LIMIT n [FROM '']`. Read the opaque +`lancedb.next_page_token` from the Arrow result schema metadata and pass it back +unchanged with the same object type and scope. Stop when no token is returned; +a short or empty page can still carry a continuation. See +[catalog listing pagination](/sql/pagination). ## DESCRIBE SECRET {#describe} diff --git a/docs/web/sql/ddl/storage-integration.mdx b/docs/web/sql/ddl/storage-integration.mdx index 9444824ac..de64ba4e5 100644 --- a/docs/web/sql/ddl/storage-integration.mdx +++ b/docs/web/sql/ddl/storage-integration.mdx @@ -194,16 +194,25 @@ only the reference naming it. ## SHOW STORAGE INTEGRATIONS {#show} - -```sql SQL icon="database" -SHOW STORAGE INTEGRATIONS -SHOW STORAGE INTEGRATIONS IN platform -LIST STORAGE INTEGRATIONS +```sql +SHOW STORAGE INTEGRATIONS IN analytics.`team$ml` LIMIT 100; +SHOW STORAGE INTEGRATIONS IN analytics.`team$ml` LIMIT 100 FROM ''; ``` - -`LIST` is an alias for `SHOW` here. Returns one `name` column; `IN` -narrows the listing to one namespace. +Returns two non-null string columns: `namespace_path`, a compact JSON array +of parent namespace components (`[]` for root), and `name`, the unqualified +object name. Omitting `IN` lists all authorized namespaces in the current database; +`IN team$ml` selects that exact path without a database qualifier; `IN public` selects +the current database's root. Listings require the relevant database and +namespace `USAGE` privileges. + +`LIST STORAGE INTEGRATIONS` is an exact alias, including pagination. + +Page through results using `LIMIT n [FROM '']`. Read the opaque +`lancedb.next_page_token` from the Arrow result schema metadata and pass it back +unchanged with the same object type and scope. Stop when no token is returned; +a short or empty page can still carry a continuation. See +[catalog listing pagination](/sql/pagination). ## DROP STORAGE INTEGRATION {#drop} diff --git a/docs/web/sql/ddl/table.mdx b/docs/web/sql/ddl/table.mdx index c8c42d4e0..d2196b93a 100644 --- a/docs/web/sql/ddl/table.mdx +++ b/docs/web/sql/ddl/table.mdx @@ -268,3 +268,23 @@ deployment stores its catalog. See Renaming, retyping and dropping columns from the SDK. + +## SHOW TABLES {#show} + +```sql +SHOW TABLES IN analytics.`team$ml` LIMIT 100; +SHOW TABLES IN analytics.`team$ml` LIMIT 100 FROM ''; +``` + +Returns two non-null string columns: `namespace_path`, a compact JSON array +of parent namespace components (`[]` for root), and `name`, the unqualified +object name. Omitting `IN` lists all authorized namespaces in the current database; +`IN team$ml` selects that exact path without a database qualifier; `IN public` selects +the current database's root. Listings require the relevant database and +namespace `USAGE` privileges. + +Page through results using `LIMIT n [FROM '']`. Read the opaque +`lancedb.next_page_token` from the Arrow result schema metadata and pass it back +unchanged with the same object type and scope. Stop when no token is returned; +a short or empty page can still carry a continuation. See +[catalog listing pagination](/sql/pagination). diff --git a/docs/web/sql/ddl/view.mdx b/docs/web/sql/ddl/view.mdx index 3d94292ab..38fd37e30 100644 --- a/docs/web/sql/ddl/view.mdx +++ b/docs/web/sql/ddl/view.mdx @@ -81,29 +81,22 @@ was none to begin with. ## Listing {#listing} ```sql -SHOW VIEWS -SHOW VIEWS IN "analytics" WHERE name > 'previous_view' LIMIT 100 +SHOW VIEWS IN analytics.`team$ml` LIMIT 100; +SHOW VIEWS IN analytics.`team$ml` LIMIT 100 FROM ''; ``` -`SHOW VIEWS` returns ascending `name` values. The `name` cursor is scoped to -the selected namespace. +Returns two non-null string columns: `namespace_path`, a compact JSON array +of parent namespace components (`[]` for root), and `name`, the unqualified +object name. Omitting `IN` lists all authorized namespaces in the current database; +`IN team$ml` selects that exact path without a database qualifier; `IN public` selects +the current database's root. Listings require the relevant database and +namespace `USAGE` privileges. -Object commands retain their small results in shared storage even when general -query-result copying is disabled, so asynchronous polling can use any query node. - -Views and tables share a namespace, and both are things you can query. The -catalog distinguishes them: - -| | | -| --- | --- | -| view names | the views in the database | -| table names | the tables | -| relation names | everything queryable (both) | - -Use the relation listing when the question is "what can I select from," and the -separate listings when the distinction matters: before dropping, for -instance, since dropping a view removes only the saved query while dropping a -table removes data. +Page through results using `LIMIT n [FROM '']`. Read the opaque +`lancedb.next_page_token` from the Arrow result schema metadata and pass it back +unchanged with the same object type and scope. Stop when no token is returned; +a short or empty page can still carry a continuation. See +[catalog listing pagination](/sql/pagination). ## Choosing between a view and a materialized view {#choosing} diff --git a/docs/web/sql/pagination.mdx b/docs/web/sql/pagination.mdx new file mode 100644 index 000000000..d6649ebfe --- /dev/null +++ b/docs/web/sql/pagination.mdx @@ -0,0 +1,68 @@ +--- +title: "Catalog listing pagination" +description: "Use one continuation-token syntax for SQL catalog listings." +--- + +Catalog `SHOW` statements share one pagination suffix: + +```sql +SHOW [IN ] [LIMIT [FROM '']]; +``` + +This applies to tables, views, materialized views, functions and function +history, secrets, storage integrations, namespaces/schemas, and databases. + +```sql +SHOW TABLES LIMIT 100; +SHOW TABLES IN team$ml LIMIT 100; +SHOW TABLES IN team$ml LIMIT 100 FROM ''; +SHOW TABLES IN analytics.team$ml LIMIT 100; +SHOW DATABASES LIMIT 100 FROM ''; +``` + +## Namespace scope + +`IN` is optional. Without it, listings include root and every authorized +namespace in the current database, regardless of the session's default namespace. +With `IN`, objects are listed only in that exact namespace. The scope accepts a +schema directly: `IN team$ml` means the absolute namespace path `["team", "ml"]` +in the current database. Use `IN public` for root, or `IN other_db.team$ml` to +select a different database. Quote schema identifiers containing punctuation or +mixed case, for example `IN "Team$ML"`. + +`SHOW SCHEMAS` and `SHOW NAMESPACES` use the same rules: without `IN` they +include all namespace paths and the public root; with `IN` they list the +selected parent's immediate children. `SHOW DATABASES` remains global and +does not accept `IN`. + +SDK APIs that take a namespace send an explicit `IN` clause and continue to +list that one namespace. + +## Continuation metadata + +The Arrow result schema carries `lancedb.next_page_token` when more results +remain. Pass that opaque value unchanged in `FROM` on the next request. +The token identifies the object type, database, namespace scope, and complete +position, including namespace path and any numeric revision. It cannot be +reused with another object type or scope. + +The token is metadata, not an additional result column or a synthetic row. +Namespace-scoped object listings contain `namespace_path` and `name`; +databases contain only `name`. Function history retains revision metadata. +SDK listing APIs expose this server-issued value as the page's `page_token`. + +Continue until no token is returned. A short or empty page may still have a +token. Pages observe the live catalog; continuation does not create a snapshot. + +## Page size and ordering + +The default page limit is 1,000, with a maximum of 10,000. `LIMIT 0` returns +no rows. Object catalogs order identifiers by namespace path components and +unqualified name; database names are ordered by name. Table page boundaries +follow native catalog order, which may differ from logical name order for +directory-backed tables; table names are sorted within each returned page. Function history uses +numeric revision order within each name. The token preserves that complete +position, including when names repeat in different namespaces. + +The old name-comparison `WHERE` and Function `WITH` pagination forms are no +longer supported. Use the same `LIMIT ... FROM ...` suffix for every listing. diff --git a/python/python/tests/test_storage_integration.py b/python/python/tests/test_storage_integration.py index ff7995ab3..84052e54c 100644 --- a/python/python/tests/test_storage_integration.py +++ b/python/python/tests/test_storage_integration.py @@ -3,6 +3,7 @@ import contextlib import json +import re from pathlib import Path import pytest @@ -39,7 +40,30 @@ def remote_server(text_type=pa.string()): command = ticket.ticket calls.append(command) if b"SHOW STORAGE INTEGRATIONS" in command: - result = pa.table({"name": pa.array(["zeta", NAME], type=text_type)}) + match = re.search( + rb'SHOW STORAGE INTEGRATIONS IN "public" ' + rb"LIMIT (\d+)(?: FROM '([^']+)')?", + command, + ) + assert match is not None, command + limit = int(match.group(1)) + names = [NAME, "zeta"] + if match.group(2) is not None: + assert match.group(2) == b"server-storage-page", command + names = names[1:] + page = names[:limit] + result = pa.table( + { + "namespace_path": pa.array( + ["[]"] * len(page), type=pa.string() + ), + "name": pa.array(page, type=text_type), + } + ) + if len(names) > limit: + result = result.replace_schema_metadata( + {b"lancedb.next_page_token": b"server-storage-page"} + ) elif b"DROP STORAGE INTEGRATION" in command: result = pa.table({"dropped": [True]}) else: @@ -195,6 +219,10 @@ def test_sync_storage_integration_lifecycle_and_aws_information(text_type): assert b'CREATE STORAGE INTEGRATION "raw"' in calls[0] assert b"STORAGE_AWS_ROLE_ARN = '" + ROLE.encode() + b"'" in calls[0] assert b'DROP STORAGE INTEGRATION "raw"' in calls[-1] + listings = [call for call in calls if b"SHOW STORAGE INTEGRATIONS" in call] + assert len(listings) == 2 + assert b'SHOW STORAGE INTEGRATIONS IN "public" LIMIT 1' in listings[0] + assert b"FROM 'server-storage-page'" in listings[1] @pytest.mark.asyncio diff --git a/rust/lancedb/src/listing.rs b/rust/lancedb/src/listing.rs index a0f6e5d5e..a5541b629 100644 --- a/rust/lancedb/src/listing.rs +++ b/rust/lancedb/src/listing.rs @@ -19,7 +19,10 @@ pub struct ListingOptions { /// Starting continuation token; `None` starts at the beginning. pub page_token: Option, /// Maximum results per request, not a limit on the entire listing. - /// `None` uses the backend default. Must be between 1 and `i32::MAX`. + /// `None` uses the backend default. SQL-backed catalog listings default to + /// 1,000 rows and accept 1 through 10,000. The public listing API rejects + /// zero; SQL statements can separately use `LIMIT 0`. Other backends may + /// impose different limits. pub page_limit: Option, } @@ -31,6 +34,9 @@ impl ListingOptions { } /// Set the maximum number of results requested per page. + /// + /// SQL-backed catalog listings accept 1 through 10,000. A server can return + /// fewer rows, including none, with a continuation token. pub fn page_limit(mut self, limit: u32) -> Self { self.page_limit = Some(limit); self diff --git a/rust/lancedb/src/remote/db.rs b/rust/lancedb/src/remote/db.rs index 7100e1b04..1a825bebc 100644 --- a/rust/lancedb/src/remote/db.rs +++ b/rust/lancedb/src/remote/db.rs @@ -1008,30 +1008,9 @@ impl Database for RemoteDatabase { namespace_path: &[String], options: ListingOptions, ) -> Result> { - validate_namespace(namespace_path)?; - let result = self - .execute_query_async(ExecuteQueryRequest { - query: "SHOW STORAGE INTEGRATIONS".into(), - default_namespace_path: namespace_path.to_vec(), - parameters: None, - }) - .await?; - let mut names = - integration_sql::decode_names(&result.reader().await?.try_collect::>().await?)?; - if let Some(token) = options.page_token { - names.retain(|name| name > &token); - } - let mut page_token = None; - if let Some(limit) = options.page_limit - && names.len() > limit as usize - { - names.truncate(limit as usize); - page_token = names.last().cloned(); - } - Ok(ListingPage { - items: names, - page_token, - }) + self.sql_client()? + .list_names("SHOW STORAGE INTEGRATIONS", namespace_path, options) + .await } async fn drop_storage_integration(&self, name: &str, namespace_path: &[String]) -> Result<()> { diff --git a/rust/lancedb/src/remote/sql/function.rs b/rust/lancedb/src/remote/sql/function.rs index 19a1a6cbc..f0ebf1894 100644 --- a/rust/lancedb/src/remote/sql/function.rs +++ b/rust/lancedb/src/remote/sql/function.rs @@ -14,7 +14,6 @@ use crate::{ listing::{ListingOptions, ListingPage}, utils::{validate_function_name, validate_namespace}, }; -use base64::{Engine, engine::general_purpose::URL_SAFE_NO_PAD}; /// The SQL result of one Function drop, before the public boolean wrapper /// discards the advisory details. @@ -89,81 +88,17 @@ impl SqlClient { namespace: &[String], options: ListingOptions, ) -> Result> { - validate_namespace(namespace)?; - let limit = options.page_limit.unwrap_or(100); - if limit == 0 || limit > i32::MAX as u32 { - return Err(Error::InvalidInput { - message: "page limit must be between 1 and i32::MAX".into(), - }); - } - let invalid = || Error::InvalidInput { - message: "invalid function page token for this database or namespace".into(), - }; - let after = if let Some(token) = options.page_token.filter(|token| !token.is_empty()) { - let bytes = URL_SAFE_NO_PAD.decode(token).map_err(|_| invalid())?; - let (database, prefix, path, name, version): ( - String, - Option, - Vec, - String, - String, - ) = serde_json::from_slice(&bytes).map_err(|_| invalid())?; - if database != self.inner.database - || prefix != self.inner.database_prefix - || path != namespace - { - return Err(invalid()); - } - Some((name, version)) - } else { - None - }; - let mut sql_options = vec![ - format!("limit = {}", u64::from(limit) + 1), - "include_definition = true".to_owned(), - ]; - if let Some((name, version)) = &after { - sql_options.push(format!("after_name = {}", literal(name))); - sql_options.push(format!("after_version = {}", literal(version))); - } - let mut items: Vec = self - .object_job( - ExecuteQueryRequest { - query: format!("SHOW FUNCTIONS HISTORY WITH ({})", sql_options.join(", ")), - default_namespace_path: namespace.to_vec(), - parameters: None, - }, - |batches| { - strings(&batches, "definition")? - .into_iter() - .map(|value| { - serde_json::from_str(&value).map_err(|error| Error::Runtime { - message: error.to_string(), - }) - }) - .collect::>>() - }, - ) - .await? - .wait() + let (batches, page_token) = self + .listing_page("SHOW FUNCTIONS HISTORY", namespace, options) .await?; - let page_token = if items.len() > limit as usize { - items.truncate(limit as usize); - let last = items.last().unwrap(); - let bytes = serde_json::to_vec(&( - &self.inner.database, - &self.inner.database_prefix, - namespace, - last.name(), - last.version(), - )) - .map_err(|error| Error::Runtime { - message: error.to_string(), - })?; - Some(URL_SAFE_NO_PAD.encode(bytes)) - } else { - None - }; + let items = strings(&batches, "definition")? + .into_iter() + .map(|value| { + serde_json::from_str(&value).map_err(|error| Error::Runtime { + message: error.to_string(), + }) + }) + .collect::>>()?; Ok(ListingPage { items, page_token }) } pub(in crate::remote) async fn drop_function_outcome( diff --git a/rust/lancedb/src/remote/sql/materialized_view.rs b/rust/lancedb/src/remote/sql/materialized_view.rs index e982afac7..c8e1aa5c3 100644 --- a/rust/lancedb/src/remote/sql/materialized_view.rs +++ b/rust/lancedb/src/remote/sql/materialized_view.rs @@ -4,7 +4,6 @@ use std::sync::Arc; use arrow_array::{Array, RecordBatch, StringArray}; -use base64::{Engine, engine::general_purpose::URL_SAFE_NO_PAD}; use futures::TryStreamExt; use super::SqlClient; @@ -157,60 +156,7 @@ impl SqlClient { namespace: &[String], options: ListingOptions, ) -> Result> { - validate_namespace(namespace)?; - let limit = options.page_limit.unwrap_or(100); - if limit == 0 || limit > i32::MAX as u32 { - return Err(Error::InvalidInput { - message: "page limit must be between 1 and i32::MAX".into(), - }); - } - let invalid_token = || Error::InvalidInput { - message: "invalid materialized-view page token for this database or namespace".into(), - }; - let after = if let Some(token) = options.page_token.filter(|token| !token.is_empty()) { - let decoded = URL_SAFE_NO_PAD.decode(token).map_err(|_| invalid_token())?; - let (database, prefix, path, last): (String, Option, Vec, String) = - serde_json::from_slice(&decoded).map_err(|_| invalid_token())?; - if database != self.inner.database - || prefix != self.inner.database_prefix - || path != namespace - { - return Err(invalid_token()); - } - format!(" WHERE name > '{}'", last.replace('\'', "''")) - } else { - String::new() - }; - let query = self - .execute(ExecuteQueryRequest { - query: format!( - "SHOW MATERIALIZED VIEWS{after} LIMIT {}", - u64::from(limit) + 1 - ), - default_namespace_path: namespace.to_vec(), - parameters: None, - }) - .await?; - let mut reader = query.reader().await?; - let mut items = Vec::new(); - while let Some(batch) = reader.try_next().await? { - items.extend(strings(&batch, "name")?); - } - let page_token = if items.len() > limit as usize { - items.truncate(limit as usize); - let cursor = serde_json::to_vec(&( - &self.inner.database, - &self.inner.database_prefix, - namespace, - items.last().unwrap(), - )) - .map_err(|error| Error::Runtime { - message: error.to_string(), - })?; - Some(URL_SAFE_NO_PAD.encode(cursor)) - } else { - None - }; - Ok(ListingPage { items, page_token }) + self.list_names("SHOW MATERIALIZED VIEWS", namespace, options) + .await } } diff --git a/rust/lancedb/src/remote/sql/objects.rs b/rust/lancedb/src/remote/sql/objects.rs index ff87e0000..2ffb08936 100644 --- a/rust/lancedb/src/remote/sql/objects.rs +++ b/rust/lancedb/src/remote/sql/objects.rs @@ -1,42 +1,118 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The LanceDB Authors -use super::SqlClient; +use super::{SqlClient, materialized_view::quote_identifier}; use crate::{ database::ExecuteQueryRequest, error::{Error, Result}, job::Job, + listing::{ListingOptions, ListingPage}, + utils::validate_namespace, }; -use arrow_array::{Array, ListArray, RecordBatch, StringArray}; +use arrow_array::{Array, LargeStringArray, ListArray, RecordBatch, StringArray, StringViewArray}; use futures::TryStreamExt; use serde::{Serialize, de::DeserializeOwned}; use std::sync::Arc; +const DEFAULT_SQL_LISTING_LIMIT: u32 = 1_000; +const MAX_SQL_LISTING_LIMIT: u32 = 10_000; +const NEXT_PAGE_TOKEN_METADATA_KEY: &str = "lancedb.next_page_token"; + +impl SqlClient { + pub(in crate::remote) async fn listing_page( + &self, + statement: &str, + namespace: &[String], + options: ListingOptions, + ) -> Result<(Vec, Option)> { + validate_namespace(namespace)?; + let limit = options.page_limit.unwrap_or(DEFAULT_SQL_LISTING_LIMIT); + if limit > MAX_SQL_LISTING_LIMIT { + return Err(Error::InvalidInput { + message: format!("page limit must be between 0 and {MAX_SQL_LISTING_LIMIT}"), + }); + } + if options.page_token.as_ref().is_some_and(String::is_empty) { + return Err(Error::InvalidInput { + message: "page token must not be empty".into(), + }); + } + if limit == 0 { + return Ok((Vec::new(), None)); + } + let schema = if namespace.is_empty() { + "public".to_owned() + } else { + namespace.join("$") + }; + let mut statement = format!("{statement} IN {} LIMIT {limit}", quote_identifier(&schema)); + if let Some(token) = options.page_token { + statement.push_str(&format!(" FROM {}", literal(&token))); + } + let query = self + .execute(ExecuteQueryRequest { + query: statement, + default_namespace_path: namespace.to_vec(), + parameters: None, + }) + .await?; + let reader = query.reader().await?; + let token = reader + .schema() + .metadata() + .get(NEXT_PAGE_TOKEN_METADATA_KEY) + .cloned() + .filter(|token| !token.is_empty()); + let batches = reader.try_collect::>().await?; + if batches.iter().map(RecordBatch::num_rows).sum::() > limit as usize { + return Err(Error::Runtime { + message: "SQL listing exceeded its requested page size".into(), + }); + } + Ok((batches, token)) + } + + pub(in crate::remote) async fn list_names( + &self, + statement: &str, + namespace: &[String], + options: ListingOptions, + ) -> Result> { + let (batches, page_token) = self.listing_page(statement, namespace, options).await?; + Ok(ListingPage { + items: strings(&batches, "name")?, + page_token, + }) + } +} + pub(super) fn literal(value: &str) -> String { format!("'{}'", value.replace('\'', "''")) } pub(super) fn strings(batches: &[RecordBatch], name: &str) -> Result> { - batches - .iter() - .map(|batch| { - let values = batch - .column_by_name(name) - .and_then(|column| column.as_any().downcast_ref::()) - .ok_or_else(|| Error::Runtime { - message: format!("SQL result is missing UTF-8 column '{name}'"), - })?; - if values.null_count() != 0 { - return Err(Error::Runtime { - message: format!("SQL result contains null '{name}'"), - }); - } - Ok(values - .iter() - .map(|value| value.unwrap().to_owned()) - .collect::>()) - }) - .collect::>>() - .map(|rows| rows.into_iter().flatten().collect()) + let mut result = Vec::new(); + for batch in batches { + let column = batch.column_by_name(name).ok_or_else(|| Error::Runtime { + message: format!("SQL result is missing string column '{name}'"), + })?; + if column.null_count() != 0 { + return Err(Error::Runtime { + message: format!("SQL result contains null '{name}'"), + }); + } + if let Some(values) = column.as_any().downcast_ref::() { + result.extend(values.iter().map(|value| value.unwrap().to_owned())); + } else if let Some(values) = column.as_any().downcast_ref::() { + result.extend(values.iter().map(|value| value.unwrap().to_owned())); + } else if let Some(values) = column.as_any().downcast_ref::() { + result.extend(values.iter().map(|value| value.unwrap().to_owned())); + } else { + return Err(Error::Runtime { + message: format!("SQL result column '{name}' is not a string"), + }); + } + } + Ok(result) } /// Read an optional `List` column. Older servers do not return the @@ -112,9 +188,9 @@ impl SqlClient { #[cfg(test)] mod tests { - use super::string_lists; + use super::{string_lists, strings}; use arrow_array::builder::{ListBuilder, StringBuilder}; - use arrow_array::{ArrayRef, RecordBatch}; + use arrow_array::{ArrayRef, LargeStringArray, RecordBatch, StringArray, StringViewArray}; use arrow_schema::{DataType, Field, Schema}; use std::sync::Arc; @@ -126,6 +202,24 @@ mod tests { )])) } + #[test] + fn listing_strings_accept_all_arrow_string_representations() { + for values in [ + Arc::new(StringArray::from(vec!["a", "z"])) as ArrayRef, + Arc::new(LargeStringArray::from(vec!["a", "z"])) as ArrayRef, + Arc::new(StringViewArray::from(vec!["a", "z"])) as ArrayRef, + ] { + let batch = RecordBatch::try_from_iter(vec![("name", values)]).unwrap(); + assert_eq!(strings(&[batch], "name").unwrap(), ["a", "z"]); + } + let batch = RecordBatch::try_from_iter(vec![( + "name", + Arc::new(StringViewArray::from(vec![Some("a"), None])) as ArrayRef, + )]) + .unwrap(); + assert!(strings(&[batch], "name").is_err()); + } + #[test] fn reads_warnings_after_an_empty_first_batch() { let schema = warning_schema(); diff --git a/rust/lancedb/src/remote/sql/secret.rs b/rust/lancedb/src/remote/sql/secret.rs index 6ddc78645..93c2a6c6e 100644 --- a/rust/lancedb/src/remote/sql/secret.rs +++ b/rust/lancedb/src/remote/sql/secret.rs @@ -10,8 +10,6 @@ use crate::{ utils::{validate_namespace, validate_secret_name}, }; use arrow_array::{Array, Int64Array, RecordBatch, StringArray}; -use base64::{Engine, engine::general_purpose::URL_SAFE_NO_PAD}; -use futures::TryStreamExt; use std::sync::Arc; pub(in crate::remote) enum SecretWriteOperation { @@ -135,65 +133,7 @@ impl SqlClient { namespace: &[String], options: ListingOptions, ) -> Result> { - validate_namespace(namespace)?; - let limit = options.page_limit.unwrap_or(100); - if limit == 0 || limit > i32::MAX as u32 { - return Err(Error::InvalidInput { - message: "page limit must be between 1 and i32::MAX".into(), - }); - } - let invalid_token = || Error::InvalidInput { - message: "invalid secret page token for this database or namespace".into(), - }; - let after = if let Some(token) = options.page_token.filter(|token| !token.is_empty()) { - let decoded = URL_SAFE_NO_PAD.decode(token).map_err(|_| invalid_token())?; - let (kind, database, prefix, path, last): ( - String, - String, - Option, - Vec, - String, - ) = serde_json::from_slice(&decoded).map_err(|_| invalid_token())?; - if kind != "secret" - || database != self.inner.database - || prefix != self.inner.database_prefix - || path != namespace - { - return Err(invalid_token()); - } - format!(" WHERE name > '{}'", last.replace('\'', "''")) - } else { - String::new() - }; - let query = self - .execute(ExecuteQueryRequest { - query: format!("SHOW SECRETS{after} LIMIT {}", u64::from(limit) + 1), - default_namespace_path: namespace.to_vec(), - parameters: None, - }) - .await?; - let mut reader = query.reader().await?; - let mut items = Vec::new(); - while let Some(batch) = reader.try_next().await? { - items.extend(strings(&[batch], "name")?); - } - let page_token = if items.len() > limit as usize { - items.truncate(limit as usize); - let cursor = serde_json::to_vec(&( - "secret", - &self.inner.database, - &self.inner.database_prefix, - namespace, - items.last().unwrap(), - )) - .map_err(|error| Error::Runtime { - message: error.to_string(), - })?; - Some(URL_SAFE_NO_PAD.encode(cursor)) - } else { - None - }; - Ok(ListingPage { items, page_token }) + self.list_names("SHOW SECRETS", namespace, options).await } } diff --git a/rust/lancedb/src/remote/sql/view.rs b/rust/lancedb/src/remote/sql/view.rs index 513113187..b058269a0 100644 --- a/rust/lancedb/src/remote/sql/view.rs +++ b/rust/lancedb/src/remote/sql/view.rs @@ -9,8 +9,6 @@ use crate::{ utils::{validate_namespace, validate_view_name}, view::ViewDescription, }; -use base64::{Engine, engine::general_purpose::URL_SAFE_NO_PAD}; -use futures::TryStreamExt; use lance_namespace::models::JsonArrowSchema; use std::sync::Arc; #[derive(Clone, serde::Serialize, serde::Deserialize)] @@ -147,64 +145,6 @@ impl SqlClient { namespace: &[String], options: ListingOptions, ) -> Result> { - validate_namespace(namespace)?; - let limit = options.page_limit.unwrap_or(100); - if limit == 0 || limit > i32::MAX as u32 { - return Err(Error::InvalidInput { - message: "page limit must be between 1 and i32::MAX".into(), - }); - } - let invalid_token = || Error::InvalidInput { - message: "invalid view page token for this database or namespace".into(), - }; - let after = if let Some(token) = options.page_token.filter(|token| !token.is_empty()) { - let decoded = URL_SAFE_NO_PAD.decode(token).map_err(|_| invalid_token())?; - let (kind, database, prefix, path, last): ( - String, - String, - Option, - Vec, - String, - ) = serde_json::from_slice(&decoded).map_err(|_| invalid_token())?; - if kind != "view" - || database != self.inner.database - || prefix != self.inner.database_prefix - || path != namespace - { - return Err(invalid_token()); - } - format!(" WHERE name > '{}'", last.replace('\'', "''")) - } else { - String::new() - }; - let query = self - .execute(ExecuteQueryRequest { - query: format!("SHOW VIEWS{after} LIMIT {}", u64::from(limit) + 1), - default_namespace_path: namespace.to_vec(), - parameters: None, - }) - .await?; - let mut reader = query.reader().await?; - let mut items = Vec::new(); - while let Some(batch) = reader.try_next().await? { - items.extend(strings(&[batch], "name")?); - } - let page_token = if items.len() > limit as usize { - items.truncate(limit as usize); - let cursor = serde_json::to_vec(&( - "view", - &self.inner.database, - &self.inner.database_prefix, - namespace, - items.last().unwrap(), - )) - .map_err(|error| Error::Runtime { - message: error.to_string(), - })?; - Some(URL_SAFE_NO_PAD.encode(cursor)) - } else { - None - }; - Ok(ListingPage { items, page_token }) + self.list_names("SHOW VIEWS", namespace, options).await } } diff --git a/rust/lancedb/src/remote/sql_test.rs b/rust/lancedb/src/remote/sql_test.rs index 5c7ae7673..d7249da9b 100644 --- a/rust/lancedb/src/remote/sql_test.rs +++ b/rust/lancedb/src/remote/sql_test.rs @@ -14,7 +14,6 @@ use arrow_flight::{ FlightInfo, HandshakeRequest, HandshakeResponse, PollInfo, PutResult, SchemaResult, Ticket, }; use arrow_schema::{DataType, Field, Schema}; -use base64::{Engine, engine::general_purpose::URL_SAFE_NO_PAD}; use futures::stream::BoxStream; use futures::{StreamExt, TryStreamExt}; use tonic::metadata::MetadataMap; @@ -2919,25 +2918,54 @@ async fn remote_optimize_full_rejects_explicit_pruning_options() { assert_rejected_before_submission(None, None, Some(true)).await; } +fn sql_listing_batch(column: &str, values: &[&str], token: Option<&str>) -> RecordBatch { + let mut columns: Vec<(&str, ArrayRef)> = Vec::new(); + if column == "name" { + columns.push(( + "namespace_path", + Arc::new(StringArray::from(vec![ + r#"["team","reports"]"#; + values.len() + ])), + )); + } + columns.push((column, Arc::new(StringArray::from(values.to_vec())))); + let batch = RecordBatch::try_from_iter(columns).unwrap(); + let metadata = token + .map(|token| HashMap::from([("lancedb.next_page_token".to_owned(), token.to_owned())])) + .unwrap_or_default(); + let schema = Arc::new(batch.schema().as_ref().clone().with_metadata(metadata)); + RecordBatch::try_new(schema, batch.columns().to_vec()).unwrap() +} + #[tokio::test] -async fn remote_mv_listing_uses_scoped_sql_keyset_pagination() { +async fn remote_mv_listing_uses_server_continuation_tokens() { for defer in [false, true] { - let result = RecordBatch::try_from_iter(vec![( - "name", - Arc::new(StringArray::from(vec!["a", "b'c", "z"])) as ArrayRef, - )]) - .unwrap(); + let token = "server'opaque:token"; + let next_sql = + "SHOW MATERIALIZED VIEWS IN \"team$reports\" LIMIT 2 FROM 'server''opaque:token'"; let service = TestSqlService { adaptive: true, defer, - result, + result_overrides: Arc::new(std::sync::Mutex::new(HashMap::from([ + ( + "SHOW MATERIALIZED VIEWS IN \"team$reports\" LIMIT 2".to_owned(), + sql_listing_batch("name", &["a", "b"], Some(token)), + ), + (next_sql.to_owned(), sql_listing_batch("name", &["z"], None)), + ]))), ..Default::default() }; let statements = service.statements.clone(); let headers = service.headers.clone(); - let (client, shutdown) = start_sql_service(service, ClientConfig::default()).await; - let db = RemoteDatabase::new_mock(|request| -> http::Response { - panic!("unexpected REST request: {}", request.url().path()); + let (client, shutdown) = start_sql_service_with_prefix( + service, + ClientConfig::default(), + Some("tenant/production"), + ) + .await; + let db = RemoteDatabase::new_mock(|_| -> http::Response { + panic!("unexpected REST request") }) .with_sql_client(client); let namespace = vec!["team".to_owned(), "reports".to_owned()]; @@ -2948,41 +2976,31 @@ async fn remote_mv_listing_uses_scoped_sql_keyset_pagination() { ) .await .unwrap(); - assert_eq!(first.items, ["a", "b'c"]); - let token = first.page_token.unwrap(); - db.list_materialized_views( - &namespace, - crate::listing::ListingOptions::default() - .page_limit(2) - .page_token(&token), - ) - .await - .unwrap(); + assert_eq!(first.items, ["a", "b"]); + assert_eq!(first.page_token.as_deref(), Some(token)); + let second = db + .list_materialized_views( + &namespace, + crate::listing::ListingOptions::default() + .page_limit(2) + .page_token(first.page_token.unwrap()), + ) + .await + .unwrap(); + assert_eq!(second.items, ["z"]); + assert!(second.page_token.is_none()); assert_eq!( *statements.lock().unwrap(), [ - "SHOW MATERIALIZED VIEWS LIMIT 3", - "SHOW MATERIALIZED VIEWS WHERE name > 'b''c' LIMIT 3" + "SHOW MATERIALIZED VIEWS IN \"team$reports\" LIMIT 2", + next_sql ] ); - assert!( - headers - .lock() - .unwrap() - .iter() - .all(|header| header.namespace_path == "team$reports") - ); - for (path, token) in [(vec![], token), (namespace, "invalid".into())] { - assert!(matches!( - db.list_materialized_views( - &path, - crate::listing::ListingOptions::default().page_token(token) - ) - .await, - Err(Error::InvalidInput { .. }) - )); + for header in headers.lock().unwrap().iter() { + assert_eq!(header.database, "analytics"); + assert_eq!(header.database_prefix, "tenant/production"); + assert_eq!(header.namespace_path, "team$reports"); } - assert_eq!(statements.lock().unwrap().len(), 2); shutdown.send(()).unwrap(); } } @@ -3123,7 +3141,10 @@ async fn storage_integration_methods_use_sql_with_namespace_and_typed_metadata() let statements = statements.lock().unwrap(); assert!(statements[0].contains("STORAGE_AWS_REGION = 'us-east-1'")); assert_eq!(statements[1], "DESCRIBE STORAGE INTEGRATION \"raw\""); - assert_eq!(statements[2], "SHOW STORAGE INTEGRATIONS"); + assert_eq!( + statements[2], + "SHOW STORAGE INTEGRATIONS IN \"platform$aws\" LIMIT 1000" + ); assert_eq!(statements[3], "DROP STORAGE INTEGRATION \"raw\""); assert!(statements[4].contains( "STORAGE_AWS_ACCESS_KEY_ID_SECRET = \"platform/raw\".\"credentials\".\"key\"" @@ -3670,29 +3691,20 @@ async fn secret_description_uses_the_sql_metadata_projection() { } #[tokio::test] -async fn secret_listing_uses_bounded_sql_and_scoped_cursors() { +async fn secret_listing_uses_server_continuation_tokens() { for defer in [false, true] { - let first_result = RecordBatch::try_from_iter(vec![( - "name", - Arc::new(StringArray::from(vec!["api-key", "hf'prod", "openai-prod"])) as ArrayRef, - )]) - .unwrap(); - let second_result = RecordBatch::try_from_iter(vec![( - "name", - Arc::new(StringArray::from(vec!["openai-prod"])) as ArrayRef, - )]) - .unwrap(); - let result_overrides = Arc::new(std::sync::Mutex::new(HashMap::from([ - ("SHOW SECRETS LIMIT 3".to_string(), first_result), - ( - "SHOW SECRETS WHERE name > 'hf''prod' LIMIT 3".to_string(), - second_result, - ), - ]))); + let token = "server'opaque:token"; + let next_sql = "SHOW SECRETS IN \"team$reports\" LIMIT 2 FROM 'server''opaque:token'"; let service = TestSqlService { adaptive: true, defer, - result_overrides, + result_overrides: Arc::new(std::sync::Mutex::new(HashMap::from([ + ( + "SHOW SECRETS IN \"team$reports\" LIMIT 2".to_owned(), + sql_listing_batch("name", &["a", "b"], Some(token)), + ), + (next_sql.to_owned(), sql_listing_batch("name", &["z"], None)), + ]))), ..Default::default() }; let statements = service.statements.clone(); @@ -3704,11 +3716,10 @@ async fn secret_listing_uses_bounded_sql_and_scoped_cursors() { ) .await; let db = RemoteDatabase::new_mock(|_| -> http::Response { - panic!("unexpected REST request"); + panic!("unexpected REST request") }) .with_sql_client(client); let namespace = vec!["team".to_owned(), "reports".to_owned()]; - let first = db .list_secrets( &namespace, @@ -3716,90 +3727,144 @@ async fn secret_listing_uses_bounded_sql_and_scoped_cursors() { ) .await .unwrap(); - assert_eq!(first.items, ["api-key", "hf'prod"]); - let token = first.page_token.expect("a third row must produce a token"); - + assert_eq!(first.items, ["a", "b"]); + assert_eq!(first.page_token.as_deref(), Some(token)); let second = db .list_secrets( &namespace, crate::listing::ListingOptions::default() .page_limit(2) - .page_token(&token), + .page_token(first.page_token.unwrap()), ) .await .unwrap(); - assert_eq!(second.items, ["openai-prod"]); + assert_eq!(second.items, ["z"]); assert!(second.page_token.is_none()); - assert_eq!( *statements.lock().unwrap(), - [ - "SHOW SECRETS LIMIT 3", - "SHOW SECRETS WHERE name > 'hf''prod' LIMIT 3" - ] + ["SHOW SECRETS IN \"team$reports\" LIMIT 2", next_sql] ); - { - let headers = headers.lock().unwrap(); - assert!(!headers.is_empty()); - for header in headers.iter() { - assert_eq!(header.database, "analytics"); - assert_eq!(header.database_prefix, "tenant/production"); - assert_eq!(header.namespace_path, "team$reports"); - } + for header in headers.lock().unwrap().iter() { + assert_eq!(header.database, "analytics"); + assert_eq!(header.database_prefix, "tenant/production"); + assert_eq!(header.namespace_path, "team$reports"); } - - for (path, token) in [ - (vec!["other".to_owned()], token), - (namespace.clone(), "invalid".to_owned()), - ] { - assert!(matches!( - db.list_secrets( - &path, - crate::listing::ListingOptions::default().page_token(token) - ) - .await, - Err(Error::InvalidInput { .. }) - )); - } - assert_eq!(statements.lock().unwrap().len(), 2); shutdown.send(()).unwrap(); } } #[tokio::test] -async fn secret_listing_handles_empty_pages_and_rejects_invalid_limits() { - let empty = RecordBatch::try_from_iter(vec![( - "name", - Arc::new(StringArray::from(Vec::::new())) as ArrayRef, - )]) - .unwrap(); +async fn storage_integration_listing_uses_server_continuation_tokens() { + for defer in [false, true] { + let token = "server'opaque:token"; + let next_sql = + "SHOW STORAGE INTEGRATIONS IN \"team$reports\" LIMIT 2 FROM 'server''opaque:token'"; + let service = TestSqlService { + adaptive: true, + defer, + result_overrides: Arc::new(std::sync::Mutex::new(HashMap::from([ + ( + "SHOW STORAGE INTEGRATIONS IN \"team$reports\" LIMIT 2".to_owned(), + sql_listing_batch("name", &["a", "b"], Some(token)), + ), + (next_sql.to_owned(), sql_listing_batch("name", &["z"], None)), + ]))), + ..Default::default() + }; + let statements = service.statements.clone(); + let headers = service.headers.clone(); + let (client, shutdown) = start_sql_service_with_prefix( + service, + ClientConfig::default(), + Some("tenant/production"), + ) + .await; + let db = RemoteDatabase::new_mock(|_| -> http::Response { + panic!("unexpected REST request") + }) + .with_sql_client(client); + let namespace = vec!["team".to_owned(), "reports".to_owned()]; + let first = db + .list_storage_integrations( + &namespace, + crate::listing::ListingOptions::default().page_limit(2), + ) + .await + .unwrap(); + assert_eq!(first.items, ["a", "b"]); + assert_eq!(first.page_token.as_deref(), Some(token)); + let second = db + .list_storage_integrations( + &namespace, + crate::listing::ListingOptions::default() + .page_limit(2) + .page_token(first.page_token.unwrap()), + ) + .await + .unwrap(); + assert_eq!(second.items, ["z"]); + assert!(second.page_token.is_none()); + assert_eq!( + *statements.lock().unwrap(), + [ + "SHOW STORAGE INTEGRATIONS IN \"team$reports\" LIMIT 2", + next_sql + ] + ); + for header in headers.lock().unwrap().iter() { + assert_eq!(header.database, "analytics"); + assert_eq!(header.database_prefix, "tenant/production"); + assert_eq!(header.namespace_path, "team$reports"); + } + shutdown.send(()).unwrap(); + } +} + +#[tokio::test] +async fn secret_listing_preserves_empty_continuations_and_validates_page_options() { let service = TestSqlService { adaptive: true, defer: true, - result: empty, + result_overrides: Arc::new(std::sync::Mutex::new(HashMap::from([ + ( + "SHOW SECRETS IN \"team$reports\" LIMIT 2".to_owned(), + sql_listing_batch("name", &[], Some("continue-empty")), + ), + ( + "SHOW SECRETS IN \"team$reports\" LIMIT 2 FROM 'continue-empty'".to_owned(), + sql_listing_batch("name", &["last"], None), + ), + ]))), ..Default::default() }; let statements = service.statements.clone(); let (client, shutdown) = start_sql_service(service, ClientConfig::default()).await; let db = RemoteDatabase::new_mock(|_| -> http::Response { - panic!("unexpected REST request"); + panic!("unexpected REST request") }) .with_sql_client(client); let namespace = vec!["team".to_owned(), "reports".to_owned()]; - let page = db .list_secrets( &namespace, - crate::listing::ListingOptions::default() - .page_limit(2) - .page_token(""), + crate::listing::ListingOptions::default().page_limit(2), ) .await .unwrap(); assert!(page.items.is_empty()); - assert!(page.page_token.is_none()); - - for limit in [0, i32::MAX as u32 + 1] { + assert_eq!(page.page_token.as_deref(), Some("continue-empty")); + let last = db + .list_secrets( + &namespace, + crate::listing::ListingOptions::default() + .page_limit(2) + .page_token(page.page_token.unwrap()), + ) + .await + .unwrap(); + assert_eq!(last.items, ["last"]); + assert!(last.page_token.is_none()); + for limit in [10_001, u32::MAX] { assert!(matches!( db.list_secrets( &namespace, @@ -3809,12 +3874,29 @@ async fn secret_listing_handles_empty_pages_and_rejects_invalid_limits() { Err(Error::InvalidInput { .. }) )); } - assert_eq!(*statements.lock().unwrap(), ["SHOW SECRETS LIMIT 3"]); + assert!( + db.list_secrets( + &namespace, + crate::listing::ListingOptions::default().page_token("") + ) + .await + .is_err() + ); + let zero = db + .list_secrets( + &namespace, + crate::listing::ListingOptions::default().page_limit(0), + ) + .await + .unwrap(); + assert!(zero.items.is_empty()); + assert!(zero.page_token.is_none()); + assert_eq!(statements.lock().unwrap().len(), 2); shutdown.send(()).unwrap(); } #[tokio::test] -async fn function_listing_uses_bounded_sql_and_scoped_version_cursors() { +async fn function_listing_uses_server_tokens_for_numeric_revision_pages() { const VERSION: &str = include_str!( "../../tests/fixtures/first_class_functions/v1/remote_function_version.canonical.json" ); @@ -3824,74 +3906,65 @@ async fn function_listing_uses_bounded_sql_and_scoped_version_cursors() { let mut tenth: serde_json::Value = serde_json::from_str(VERSION).unwrap(); tenth["version"] = serde_json::json!("10"); let tenth = tenth.to_string(); - let result = RecordBatch::try_from_iter(vec![( - "definition", - Arc::new(StringArray::from(vec![ - VERSION, - second.as_str(), - tenth.as_str(), - ])) as ArrayRef, - )]) - .unwrap(); - let service = TestSqlService { - adaptive: true, - defer: true, - result, - ..Default::default() - }; - service.result_overrides.lock().unwrap().insert( - "SHOW FUNCTIONS HISTORY WITH (limit = 3, include_definition = true, after_name = 'embed', after_version = '2')".into(), - RecordBatch::try_from_iter(vec![("definition", Arc::new(StringArray::from(vec![tenth.as_str()])) as ArrayRef)]).unwrap(), - ); - let statements = service.statements.clone(); - let (client, shutdown) = start_sql_service(service, ClientConfig::default()).await; - let db = RemoteDatabase::new_mock(|_| -> http::Response { - panic!("unexpected REST request"); - }) - .with_sql_client(client); - let first = db - .list_functions( - &["team".into()], - crate::listing::ListingOptions::default().page_limit(2), - ) - .await - .unwrap(); - assert_eq!( - first - .items - .iter() - .map(|item| item.version()) - .collect::>(), - ["1", "2"] - ); - let token = first.page_token.unwrap(); - let second = db - .list_functions( - &["team".into()], - crate::listing::ListingOptions::default() - .page_limit(2) - .page_token(token.clone()), - ) - .await - .unwrap(); - assert_eq!(second.items.len(), 1); - assert_eq!(second.items[0].version(), "10"); - assert!(second.page_token.is_none()); - assert!( - db.list_functions( - &["other".into()], - crate::listing::ListingOptions::default() - .page_limit(2) - .page_token(token) - ) - .await - .is_err() - ); - let seen = statements.lock().unwrap(); - assert_eq!(seen.len(), 2); - assert!(seen[0].starts_with("SHOW FUNCTIONS HISTORY WITH (limit = 3,")); - assert!(seen[1].contains("after_version = '2'")); - shutdown.send(()).unwrap(); + for defer in [false, true] { + let service = TestSqlService { + adaptive: true, + defer, + result_overrides: Arc::new(std::sync::Mutex::new(HashMap::from([ + ( + "SHOW FUNCTIONS HISTORY IN \"team\" LIMIT 2".to_owned(), + sql_listing_batch("definition", &[VERSION, &second], Some("revision-cursor")), + ), + ( + "SHOW FUNCTIONS HISTORY IN \"team\" LIMIT 2 FROM 'revision-cursor'".to_owned(), + sql_listing_batch("definition", &[&tenth], None), + ), + ]))), + ..Default::default() + }; + let statements = service.statements.clone(); + let (client, shutdown) = start_sql_service(service, ClientConfig::default()).await; + let db = RemoteDatabase::new_mock(|_| -> http::Response { + panic!("unexpected REST request") + }) + .with_sql_client(client); + let first = db + .list_functions( + &["team".into()], + crate::listing::ListingOptions::default().page_limit(2), + ) + .await + .unwrap(); + assert_eq!( + first + .items + .iter() + .map(|item| item.version()) + .collect::>(), + ["1", "2"] + ); + assert_eq!(first.page_token.as_deref(), Some("revision-cursor")); + let last = db + .list_functions( + &["team".into()], + crate::listing::ListingOptions::default() + .page_limit(2) + .page_token(first.page_token.unwrap()), + ) + .await + .unwrap(); + assert_eq!(last.items.len(), 1); + assert_eq!(last.items[0].version(), "10"); + assert!(last.page_token.is_none()); + assert_eq!( + *statements.lock().unwrap(), + [ + "SHOW FUNCTIONS HISTORY IN \"team\" LIMIT 2", + "SHOW FUNCTIONS HISTORY IN \"team\" LIMIT 2 FROM 'revision-cursor'" + ] + ); + shutdown.send(()).unwrap(); + } } #[tokio::test] @@ -3999,29 +4072,20 @@ async fn view_drop_uses_a_repeatable_query_job() { } #[tokio::test] -async fn view_listing_uses_bounded_sql_and_scoped_cursors() { +async fn view_listing_uses_server_continuation_tokens() { for defer in [false, true] { - let first_result = RecordBatch::try_from_iter(vec![( - "name", - Arc::new(StringArray::from(vec!["adults", "hf'prod", "openai-prod"])) as ArrayRef, - )]) - .unwrap(); - let second_result = RecordBatch::try_from_iter(vec![( - "name", - Arc::new(StringArray::from(vec!["openai-prod"])) as ArrayRef, - )]) - .unwrap(); - let result_overrides = Arc::new(std::sync::Mutex::new(HashMap::from([ - ("SHOW VIEWS LIMIT 3".to_string(), first_result), - ( - "SHOW VIEWS WHERE name > 'hf''prod' LIMIT 3".to_string(), - second_result, - ), - ]))); + let token = "server'opaque:token"; + let next_sql = "SHOW VIEWS IN \"team$reports\" LIMIT 2 FROM 'server''opaque:token'"; let service = TestSqlService { adaptive: true, defer, - result_overrides, + result_overrides: Arc::new(std::sync::Mutex::new(HashMap::from([ + ( + "SHOW VIEWS IN \"team$reports\" LIMIT 2".to_owned(), + sql_listing_batch("name", &["a", "b"], Some(token)), + ), + (next_sql.to_owned(), sql_listing_batch("name", &["z"], None)), + ]))), ..Default::default() }; let statements = service.statements.clone(); @@ -4033,107 +4097,39 @@ async fn view_listing_uses_bounded_sql_and_scoped_cursors() { ) .await; let db = RemoteDatabase::new_mock(|_| -> http::Response { - panic!("unexpected REST request"); + panic!("unexpected REST request") }) .with_sql_client(client); - let connection = Connection::new( - Arc::new(db), - Arc::new(crate::embeddings::MemoryRegistry::new()), - ); let namespace = vec!["team".to_owned(), "reports".to_owned()]; - - let first = connection + let first = db .list_views( &namespace, crate::listing::ListingOptions::default().page_limit(2), ) - .next_page() .await - .unwrap() .unwrap(); - assert_eq!(first.items, ["adults", "hf'prod"]); - let token = first.page_token.expect("a third row must produce a token"); - - let second = connection + assert_eq!(first.items, ["a", "b"]); + assert_eq!(first.page_token.as_deref(), Some(token)); + let second = db .list_views( &namespace, crate::listing::ListingOptions::default() .page_limit(2) - .page_token(&token), + .page_token(first.page_token.unwrap()), ) - .next_page() .await - .unwrap() .unwrap(); - assert_eq!(second.items, ["openai-prod"]); + assert_eq!(second.items, ["z"]); assert!(second.page_token.is_none()); - assert_eq!( *statements.lock().unwrap(), - [ - "SHOW VIEWS LIMIT 3", - "SHOW VIEWS WHERE name > 'hf''prod' LIMIT 3" - ] + ["SHOW VIEWS IN \"team$reports\" LIMIT 2", next_sql] ); - { - let headers = headers.lock().unwrap(); - assert!(!headers.is_empty()); - for header in headers.iter() { - assert_eq!(header.database, "analytics"); - assert_eq!(header.database_prefix, "tenant/production"); - assert_eq!(header.namespace_path, "team$reports"); - } + for header in headers.lock().unwrap().iter() { + assert_eq!(header.database, "analytics"); + assert_eq!(header.database_prefix, "tenant/production"); + assert_eq!(header.namespace_path, "team$reports"); } - - let encode_token = - |kind: &str, database: &str, prefix: Option<&str>, path: &[String], last: &str| { - URL_SAFE_NO_PAD.encode( - serde_json::to_vec(&(kind, database, prefix, path.to_vec(), last)).unwrap(), - ) - }; - for forged in [ - encode_token( - "secret", - "analytics", - Some("tenant/production"), - &namespace, - "hf'prod", - ), - encode_token( - "view", - "billing", - Some("tenant/production"), - &namespace, - "hf'prod", - ), - encode_token("view", "analytics", Some("other"), &namespace, "hf'prod"), - encode_token( - "view", - "analytics", - Some("tenant/production"), - &["other".to_owned()], - "hf'prod", - ), - "not-a-json-token".to_string(), - ] { - let mut listing = connection.list_views( - &namespace, - crate::listing::ListingOptions::default().page_token(forged), - ); - assert!(matches!( - listing.next_page().await, - Err(Error::InvalidInput { .. }) - )); - } - let mut listing = connection.list_views( - &["other".to_owned()], - crate::listing::ListingOptions::default().page_token(&token), - ); - assert!(matches!( - listing.next_page().await, - Err(Error::InvalidInput { .. }) - )); - assert_eq!(statements.lock().unwrap().len(), 2); shutdown.send(()).unwrap(); } } diff --git a/rust/lancedb/src/remote/storage_integration.rs b/rust/lancedb/src/remote/storage_integration.rs index 48dab4c6a..6c132ab4b 100644 --- a/rust/lancedb/src/remote/storage_integration.rs +++ b/rust/lancedb/src/remote/storage_integration.rs @@ -259,29 +259,6 @@ pub(super) fn decode_info(batches: &[RecordBatch]) -> Result Result> { - let mut names = Vec::new(); - for batch in batches { - let column = batch - .column_by_name("name") - .ok_or_else(|| invalid("SQL storage integration listing is missing names"))?; - for row in 0..batch.num_rows() { - if column.is_null(row) { - return Err(invalid("SQL storage integration name cannot be null")); - } - names.push( - string_value(column.as_ref(), row) - .ok_or_else(|| { - invalid("SQL storage integration listing requires string names") - })? - .to_string(), - ); - } - } - names.sort(); - Ok(names) -} - #[cfg(test)] mod tests { use std::sync::Arc;