diff --git a/docs/web/docs.json b/docs/web/docs.json index c586d4f62..0b26a7fc0 100644 --- a/docs/web/docs.json +++ b/docs/web/docs.json @@ -543,6 +543,7 @@ "group": "SQL Reference", "pages": [ "sql/index", + "sql/pagination", "sql/data-types", { "group": "DDL", diff --git a/docs/web/management/function.mdx b/docs/web/management/function.mdx index 41e47dee1..562de05c6 100644 --- a/docs/web/management/function.mdx +++ b/docs/web/management/function.mdx @@ -280,7 +280,9 @@ let latest = connection.get_function_latest("double", &[]).await?; ```sql SQL icon="database" SHOW FUNCTIONS -SHOW FUNCTIONS IN "analytics" WITH (name = 'double', limit = 100) +SHOW FUNCTIONS IN "analytics" LIMIT 100 +SHOW FUNCTIONS HISTORY IN "analytics" LIMIT 100 +SHOW FUNCTIONS HISTORY IN "analytics" LIMIT 100 FROM '' DESCRIBE FUNCTION double WITH (version = '1') DESCRIBE FUNCTION double diff --git a/docs/web/management/index.mdx b/docs/web/management/index.mdx index fbef65756..50f6e7507 100644 --- a/docs/web/management/index.mdx +++ b/docs/web/management/index.mdx @@ -85,7 +85,7 @@ enables them; the SDK methods send the view's SQL query to the deployment. See | --- | --- | --- | --- | | create | `create_function`, `create_function_async` | -- | `CREATE FUNCTION` | | get | `get_function` | -- | `DESCRIBE FUNCTION` | -| list | `list_functions` | -- | `SHOW FUNCTIONS` | +| list | `list_functions` | -- | `SHOW FUNCTIONS HISTORY` | | drop | `drop_function` | -- | `DROP FUNCTION` | Enterprise A function runs on the server, and only on diff --git a/docs/web/sql/ddl/database.mdx b/docs/web/sql/ddl/database.mdx index e8cdb55bc..f84d4cfb9 100644 --- a/docs/web/sql/ddl/database.mdx +++ b/docs/web/sql/ddl/database.mdx @@ -57,15 +57,21 @@ properties and requires system `USAGE`. ## SHOW DATABASES {#show} - -```sql SQL icon="database" -SHOW DATABASES +```sql +SHOW DATABASES LIMIT 100; +SHOW DATABASES LIMIT 100 FROM ''; ``` - -Returns a sorted `name` column containing the databases the caller has -`USAGE` on, including `system` when permitted. `LIKE` and `WHERE` filters -are unsupported. +Returns one non-null `name` column, sorted by database name and filtered to +databases the caller has `USAGE` on, including `system` when permitted. +Databases are above namespaces, so this command does not accept `IN` or +return a `namespace_path` column. + +Page through results using `LIMIT n [FROM '']`. Read the opaque +`lancedb.next_page_token` from the Arrow result schema metadata and pass it back +unchanged with the same object type and scope. Stop when no token is returned; +a short or empty page can still carry a continuation. See +[catalog listing pagination](/sql/pagination). ## ATTACH DATABASE {#attach} diff --git a/docs/web/sql/ddl/function.mdx b/docs/web/sql/ddl/function.mdx index 55e2d6fe7..691118026 100644 --- a/docs/web/sql/ddl/function.mdx +++ b/docs/web/sql/ddl/function.mdx @@ -135,20 +135,33 @@ from it. ## SHOW FUNCTIONS {#show} - -```sql SQL icon="database" -SHOW FUNCTIONS -SHOW FUNCTIONS HISTORY IN "analytics" WITH (name = 'embed', limit = 100) -SHOW FUNCTIONS HISTORY WITH (after_name = 'embed', after_version = '1', limit = 100) +```sql +SHOW FUNCTIONS IN analytics.`team$ml` LIMIT 100; +SHOW FUNCTIONS IN analytics.`team$ml` LIMIT 100 FROM ''; ``` - -`SHOW FUNCTIONS` returns the highest published numeric revision for each name. -Add `HISTORY` to list every published revision, as Python's `list_functions()` -does. Rows are ordered by name and numeric revision, so `2` comes before `10`. -`after_name` and `after_version` must be supplied together for keyset pagination. -`limit` bounds -the response; `include_definition = false` omits the optional definition JSON. +Returns two non-null string columns: `namespace_path`, a compact JSON array +of parent namespace components (`[]` for root), and `name`, the unqualified +object name. Omitting `IN` lists all authorized namespaces in the current database; +`IN team$ml` selects that exact path without a database qualifier; `IN public` selects +the current database's root. Listings require the relevant database and +namespace `USAGE` privileges. + +`SHOW FUNCTIONS` lists each function once. Add `HISTORY` to list every +published revision with its metadata and definition, as `list_functions()` +does. History uses the same pagination suffix; numeric revision order is +encoded in the token, so revision `2` precedes `10`. + +```sql +SHOW FUNCTIONS HISTORY IN analytics.`team$ml` LIMIT 100; +SHOW FUNCTIONS HISTORY IN analytics.`team$ml` LIMIT 100 FROM ''; +``` + +Page through results using `LIMIT n [FROM '']`. Read the opaque +`lancedb.next_page_token` from the Arrow result schema metadata and pass it back +unchanged with the same object type and scope. Stop when no token is returned; +a short or empty page can still carry a continuation. See +[catalog listing pagination](/sql/pagination). ## DROP FUNCTION {#drop} diff --git a/docs/web/sql/ddl/materialized-view.mdx b/docs/web/sql/ddl/materialized-view.mdx index a2453ba01..a2a5024f6 100644 --- a/docs/web/sql/ddl/materialized-view.mdx +++ b/docs/web/sql/ddl/materialized-view.mdx @@ -70,16 +70,25 @@ automatic execution needs the same server-side switch enabled. Omitting ## SHOW MATERIALIZED VIEWS {#show} - -```sql SQL icon="database" -SHOW MATERIALIZED VIEWS +```sql +SHOW MATERIALIZED VIEWS IN analytics.`team$ml` LIMIT 100; +SHOW MATERIALIZED VIEWS IN analytics.`team$ml` LIMIT 100 FROM ''; ``` - - -This asks the server for its views, as the SDK's `list_materialized_views` does -on a `db://` connection. - +Returns two non-null string columns: `namespace_path`, a compact JSON array +of parent namespace components (`[]` for root), and `name`, the unqualified +object name. Omitting `IN` lists all authorized namespaces in the current database; +`IN team$ml` selects that exact path without a database qualifier; `IN public` selects +the current database's root. Listings require the relevant database and +namespace `USAGE` privileges. + +The listing contains identifiers; use `DESCRIBE MATERIALIZED VIEW` for its defining query and refresh metadata. + +Page through results using `LIMIT n [FROM '']`. Read the opaque +`lancedb.next_page_token` from the Arrow result schema metadata and pass it back +unchanged with the same object type and scope. Stop when no token is returned; +a short or empty page can still carry a continuation. See +[catalog listing pagination](/sql/pagination). ## Refreshing one {#refresh} diff --git a/docs/web/sql/ddl/namespace.mdx b/docs/web/sql/ddl/namespace.mdx index 8f446f1e7..14391ac18 100644 --- a/docs/web/sql/ddl/namespace.mdx +++ b/docs/web/sql/ddl/namespace.mdx @@ -17,7 +17,7 @@ backtick-quoted identifier with `$` between components, such as one path component, relative to an explicit parent schema or the session's current schema -- it cannot contain `$` or be qualified with a database. -`SCHEMA` never accepts `IN`. `NAMESPACE` accepts `IN` followed by a complete +For single-object commands, `SCHEMA` never accepts `IN`. `NAMESPACE` accepts `IN` followed by a complete schema identifier. Names are lowercased unless quoted. ## CREATE SCHEMA and CREATE NAMESPACE {#create} @@ -126,27 +126,30 @@ Requires `USAGE` on the database and the namespace. ## SHOW SCHEMAS and SHOW NAMESPACES {#show} - -```sql SQL icon="database" -SHOW SCHEMAS -SHOW NAMESPACES -SHOW NAMESPACES IN analytics.team +```sql +SHOW SCHEMAS LIMIT 100; +SHOW SCHEMAS LIMIT 100 FROM ''; +SHOW NAMESPACES IN analytics.team LIMIT 100; +SHOW NAMESPACES IN analytics.team LIMIT 100 FROM ''; ``` - -`SHOW SCHEMAS` returns complete paths in the current database, recursively, -including `public` when the caller can use it. It does not accept `IN`. +Both commands return two non-null string columns: `namespace_path`, the +parent path as a compact JSON array, and `name`, the unqualified child name. +Results are ordered by parent path components, then name, and omit namespaces +the caller cannot use. -`SHOW NAMESPACES` lists only immediate child names under the session's -current schema. `IN` selects a different parent schema, optionally in -another database -- use `IN public` for the current database's root, or -`IN analytics.public` for another database's root. A missing parent is an -error. +`SHOW SCHEMAS` and `SHOW NAMESPACES` are equivalent listings. Without `IN`, +both list recursively in the current database, include the implicit `public` +root, and ignore the session namespace. With `IN`, both list immediate children +of the selected parent. A schema path can appear directly: `IN team$features`; +a database qualifier is optional. `IN public` selects the current database root, +and `IN analytics.public` selects another database's root. -Both forms return a sorted `name` column and omit namespaces the caller -cannot use. Large listings that exceed 10,000 namespaces, or recursive -listings deeper than 32 levels, return an error suggesting a narrower -listing. +Page through results using `LIMIT n [FROM '']`. Read the opaque +`lancedb.next_page_token` from the Arrow result schema metadata and pass it back +unchanged with the same object type and scope. Stop when no token is returned; +a short or empty page can still carry a continuation. See +[catalog listing pagination](/sql/pagination). ## DROP SCHEMA and DROP NAMESPACE {#drop} @@ -187,7 +190,6 @@ with `IF EXISTS`. Inspect a cascade drop's child jobs with | `CREATE SCHEMA child IN parent` | `CREATE NAMESPACE child IN parent` or `` CREATE SCHEMA `parent$child` `` | | `` DESCRIBE NAMESPACE `parent$child` `` | `` DESCRIBE SCHEMA `parent$child` `` or `DESCRIBE NAMESPACE child IN parent` | | `DROP NAMESPACE analytics.child` | `DROP NAMESPACE child IN analytics.public` | -| `SHOW SCHEMAS IN parent` | `SHOW NAMESPACES IN parent` | ## What to read next {#next} diff --git a/docs/web/sql/ddl/secret.mdx b/docs/web/sql/ddl/secret.mdx index 922dec247..15039c6c7 100644 --- a/docs/web/sql/ddl/secret.mdx +++ b/docs/web/sql/ddl/secret.mdx @@ -98,21 +98,25 @@ picked up by every Function still bound to that name. ## SHOW SECRETS {#show} - -```sql SQL icon="database" -SHOW SECRETS -SHOW SECRETS IN platform -SHOW SECRETS IN analytics.`ml$vision` -SHOW SECRETS IN public -SHOW SECRETS IN platform WHERE name > 'model-api-key' LIMIT 50 +```sql +SHOW SECRETS IN analytics.`team$ml` LIMIT 100; +SHOW SECRETS IN analytics.`team$ml` LIMIT 100 FROM ''; ``` - -Lists the Secret names in one namespace as a sorted `name` column. Without -`IN`, it lists the session's namespace; `IN public` lists the root. `WHERE -name > ''` pages past the last name already seen, and `LIMIT` -bounds the page size; `WHERE` must compare `name` with `>`. Requires -`USAGE` on the database and the namespace. +Returns two non-null string columns: `namespace_path`, a compact JSON array +of parent namespace components (`[]` for root), and `name`, the unqualified +object name. Omitting `IN` lists all authorized namespaces in the current database; +`IN team$ml` selects that exact path without a database qualifier; `IN public` selects +the current database's root. Listings require the relevant database and +namespace `USAGE` privileges. + +Secret listings contain no values or Secret metadata. + +Page through results using `LIMIT n [FROM '']`. Read the opaque +`lancedb.next_page_token` from the Arrow result schema metadata and pass it back +unchanged with the same object type and scope. Stop when no token is returned; +a short or empty page can still carry a continuation. See +[catalog listing pagination](/sql/pagination). ## DESCRIBE SECRET {#describe} diff --git a/docs/web/sql/ddl/storage-integration.mdx b/docs/web/sql/ddl/storage-integration.mdx index 9444824ac..de64ba4e5 100644 --- a/docs/web/sql/ddl/storage-integration.mdx +++ b/docs/web/sql/ddl/storage-integration.mdx @@ -194,16 +194,25 @@ only the reference naming it. ## SHOW STORAGE INTEGRATIONS {#show} - -```sql SQL icon="database" -SHOW STORAGE INTEGRATIONS -SHOW STORAGE INTEGRATIONS IN platform -LIST STORAGE INTEGRATIONS +```sql +SHOW STORAGE INTEGRATIONS IN analytics.`team$ml` LIMIT 100; +SHOW STORAGE INTEGRATIONS IN analytics.`team$ml` LIMIT 100 FROM ''; ``` - -`LIST` is an alias for `SHOW` here. Returns one `name` column; `IN` -narrows the listing to one namespace. +Returns two non-null string columns: `namespace_path`, a compact JSON array +of parent namespace components (`[]` for root), and `name`, the unqualified +object name. Omitting `IN` lists all authorized namespaces in the current database; +`IN team$ml` selects that exact path without a database qualifier; `IN public` selects +the current database's root. Listings require the relevant database and +namespace `USAGE` privileges. + +`LIST STORAGE INTEGRATIONS` is an exact alias, including pagination. + +Page through results using `LIMIT n [FROM '']`. Read the opaque +`lancedb.next_page_token` from the Arrow result schema metadata and pass it back +unchanged with the same object type and scope. Stop when no token is returned; +a short or empty page can still carry a continuation. See +[catalog listing pagination](/sql/pagination). ## DROP STORAGE INTEGRATION {#drop} diff --git a/docs/web/sql/ddl/table.mdx b/docs/web/sql/ddl/table.mdx index c8c42d4e0..d2196b93a 100644 --- a/docs/web/sql/ddl/table.mdx +++ b/docs/web/sql/ddl/table.mdx @@ -268,3 +268,23 @@ deployment stores its catalog. See Renaming, retyping and dropping columns from the SDK. + +## SHOW TABLES {#show} + +```sql +SHOW TABLES IN analytics.`team$ml` LIMIT 100; +SHOW TABLES IN analytics.`team$ml` LIMIT 100 FROM ''; +``` + +Returns two non-null string columns: `namespace_path`, a compact JSON array +of parent namespace components (`[]` for root), and `name`, the unqualified +object name. Omitting `IN` lists all authorized namespaces in the current database; +`IN team$ml` selects that exact path without a database qualifier; `IN public` selects +the current database's root. Listings require the relevant database and +namespace `USAGE` privileges. + +Page through results using `LIMIT n [FROM '']`. Read the opaque +`lancedb.next_page_token` from the Arrow result schema metadata and pass it back +unchanged with the same object type and scope. Stop when no token is returned; +a short or empty page can still carry a continuation. See +[catalog listing pagination](/sql/pagination). diff --git a/docs/web/sql/ddl/view.mdx b/docs/web/sql/ddl/view.mdx index 3d94292ab..38fd37e30 100644 --- a/docs/web/sql/ddl/view.mdx +++ b/docs/web/sql/ddl/view.mdx @@ -81,29 +81,22 @@ was none to begin with. ## Listing {#listing} ```sql -SHOW VIEWS -SHOW VIEWS IN "analytics" WHERE name > 'previous_view' LIMIT 100 +SHOW VIEWS IN analytics.`team$ml` LIMIT 100; +SHOW VIEWS IN analytics.`team$ml` LIMIT 100 FROM ''; ``` -`SHOW VIEWS` returns ascending `name` values. The `name` cursor is scoped to -the selected namespace. +Returns two non-null string columns: `namespace_path`, a compact JSON array +of parent namespace components (`[]` for root), and `name`, the unqualified +object name. Omitting `IN` lists all authorized namespaces in the current database; +`IN team$ml` selects that exact path without a database qualifier; `IN public` selects +the current database's root. Listings require the relevant database and +namespace `USAGE` privileges. -Object commands retain their small results in shared storage even when general -query-result copying is disabled, so asynchronous polling can use any query node. - -Views and tables share a namespace, and both are things you can query. The -catalog distinguishes them: - -| | | -| --- | --- | -| view names | the views in the database | -| table names | the tables | -| relation names | everything queryable (both) | - -Use the relation listing when the question is "what can I select from," and the -separate listings when the distinction matters: before dropping, for -instance, since dropping a view removes only the saved query while dropping a -table removes data. +Page through results using `LIMIT n [FROM '']`. Read the opaque +`lancedb.next_page_token` from the Arrow result schema metadata and pass it back +unchanged with the same object type and scope. Stop when no token is returned; +a short or empty page can still carry a continuation. See +[catalog listing pagination](/sql/pagination). ## Choosing between a view and a materialized view {#choosing} diff --git a/docs/web/sql/pagination.mdx b/docs/web/sql/pagination.mdx new file mode 100644 index 000000000..d6649ebfe --- /dev/null +++ b/docs/web/sql/pagination.mdx @@ -0,0 +1,68 @@ +--- +title: "Catalog listing pagination" +description: "Use one continuation-token syntax for SQL catalog listings." +--- + +Catalog `SHOW` statements share one pagination suffix: + +```sql +SHOW [IN ] [LIMIT [FROM '']]; +``` + +This applies to tables, views, materialized views, functions and function +history, secrets, storage integrations, namespaces/schemas, and databases. + +```sql +SHOW TABLES LIMIT 100; +SHOW TABLES IN team$ml LIMIT 100; +SHOW TABLES IN team$ml LIMIT 100 FROM ''; +SHOW TABLES IN analytics.team$ml LIMIT 100; +SHOW DATABASES LIMIT 100 FROM ''; +``` + +## Namespace scope + +`IN` is optional. Without it, listings include root and every authorized +namespace in the current database, regardless of the session's default namespace. +With `IN`, objects are listed only in that exact namespace. The scope accepts a +schema directly: `IN team$ml` means the absolute namespace path `["team", "ml"]` +in the current database. Use `IN public` for root, or `IN other_db.team$ml` to +select a different database. Quote schema identifiers containing punctuation or +mixed case, for example `IN "Team$ML"`. + +`SHOW SCHEMAS` and `SHOW NAMESPACES` use the same rules: without `IN` they +include all namespace paths and the public root; with `IN` they list the +selected parent's immediate children. `SHOW DATABASES` remains global and +does not accept `IN`. + +SDK APIs that take a namespace send an explicit `IN` clause and continue to +list that one namespace. + +## Continuation metadata + +The Arrow result schema carries `lancedb.next_page_token` when more results +remain. Pass that opaque value unchanged in `FROM` on the next request. +The token identifies the object type, database, namespace scope, and complete +position, including namespace path and any numeric revision. It cannot be +reused with another object type or scope. + +The token is metadata, not an additional result column or a synthetic row. +Namespace-scoped object listings contain `namespace_path` and `name`; +databases contain only `name`. Function history retains revision metadata. +SDK listing APIs expose this server-issued value as the page's `page_token`. + +Continue until no token is returned. A short or empty page may still have a +token. Pages observe the live catalog; continuation does not create a snapshot. + +## Page size and ordering + +The default page limit is 1,000, with a maximum of 10,000. `LIMIT 0` returns +no rows. Object catalogs order identifiers by namespace path components and +unqualified name; database names are ordered by name. Table page boundaries +follow native catalog order, which may differ from logical name order for +directory-backed tables; table names are sorted within each returned page. Function history uses +numeric revision order within each name. The token preserves that complete +position, including when names repeat in different namespaces. + +The old name-comparison `WHERE` and Function `WITH` pagination forms are no +longer supported. Use the same `LIMIT ... FROM ...` suffix for every listing. diff --git a/python/python/tests/test_storage_integration.py b/python/python/tests/test_storage_integration.py index ff7995ab3..84052e54c 100644 --- a/python/python/tests/test_storage_integration.py +++ b/python/python/tests/test_storage_integration.py @@ -3,6 +3,7 @@ import contextlib import json +import re from pathlib import Path import pytest @@ -39,7 +40,30 @@ def remote_server(text_type=pa.string()): command = ticket.ticket calls.append(command) if b"SHOW STORAGE INTEGRATIONS" in command: - result = pa.table({"name": pa.array(["zeta", NAME], type=text_type)}) + match = re.search( + rb'SHOW STORAGE INTEGRATIONS IN "public" ' + rb"LIMIT (\d+)(?: FROM '([^']+)')?", + command, + ) + assert match is not None, command + limit = int(match.group(1)) + names = [NAME, "zeta"] + if match.group(2) is not None: + assert match.group(2) == b"server-storage-page", command + names = names[1:] + page = names[:limit] + result = pa.table( + { + "namespace_path": pa.array( + ["[]"] * len(page), type=pa.string() + ), + "name": pa.array(page, type=text_type), + } + ) + if len(names) > limit: + result = result.replace_schema_metadata( + {b"lancedb.next_page_token": b"server-storage-page"} + ) elif b"DROP STORAGE INTEGRATION" in command: result = pa.table({"dropped": [True]}) else: @@ -195,6 +219,10 @@ def test_sync_storage_integration_lifecycle_and_aws_information(text_type): assert b'CREATE STORAGE INTEGRATION "raw"' in calls[0] assert b"STORAGE_AWS_ROLE_ARN = '" + ROLE.encode() + b"'" in calls[0] assert b'DROP STORAGE INTEGRATION "raw"' in calls[-1] + listings = [call for call in calls if b"SHOW STORAGE INTEGRATIONS" in call] + assert len(listings) == 2 + assert b'SHOW STORAGE INTEGRATIONS IN "public" LIMIT 1' in listings[0] + assert b"FROM 'server-storage-page'" in listings[1] @pytest.mark.asyncio diff --git a/rust/lancedb/src/listing.rs b/rust/lancedb/src/listing.rs index a0f6e5d5e..a5541b629 100644 --- a/rust/lancedb/src/listing.rs +++ b/rust/lancedb/src/listing.rs @@ -19,7 +19,10 @@ pub struct ListingOptions { /// Starting continuation token; `None` starts at the beginning. pub page_token: Option, /// Maximum results per request, not a limit on the entire listing. - /// `None` uses the backend default. Must be between 1 and `i32::MAX`. + /// `None` uses the backend default. SQL-backed catalog listings default to + /// 1,000 rows and accept 1 through 10,000. The public listing API rejects + /// zero; SQL statements can separately use `LIMIT 0`. Other backends may + /// impose different limits. pub page_limit: Option, } @@ -31,6 +34,9 @@ impl ListingOptions { } /// Set the maximum number of results requested per page. + /// + /// SQL-backed catalog listings accept 1 through 10,000. A server can return + /// fewer rows, including none, with a continuation token. pub fn page_limit(mut self, limit: u32) -> Self { self.page_limit = Some(limit); self diff --git a/rust/lancedb/src/remote/db.rs b/rust/lancedb/src/remote/db.rs index 7100e1b04..1a825bebc 100644 --- a/rust/lancedb/src/remote/db.rs +++ b/rust/lancedb/src/remote/db.rs @@ -1008,30 +1008,9 @@ impl Database for RemoteDatabase { namespace_path: &[String], options: ListingOptions, ) -> Result> { - validate_namespace(namespace_path)?; - let result = self - .execute_query_async(ExecuteQueryRequest { - query: "SHOW STORAGE INTEGRATIONS".into(), - default_namespace_path: namespace_path.to_vec(), - parameters: None, - }) - .await?; - let mut names = - integration_sql::decode_names(&result.reader().await?.try_collect::>().await?)?; - if let Some(token) = options.page_token { - names.retain(|name| name > &token); - } - let mut page_token = None; - if let Some(limit) = options.page_limit - && names.len() > limit as usize - { - names.truncate(limit as usize); - page_token = names.last().cloned(); - } - Ok(ListingPage { - items: names, - page_token, - }) + self.sql_client()? + .list_names("SHOW STORAGE INTEGRATIONS", namespace_path, options) + .await } async fn drop_storage_integration(&self, name: &str, namespace_path: &[String]) -> Result<()> { diff --git a/rust/lancedb/src/remote/sql/function.rs b/rust/lancedb/src/remote/sql/function.rs index 19a1a6cbc..f0ebf1894 100644 --- a/rust/lancedb/src/remote/sql/function.rs +++ b/rust/lancedb/src/remote/sql/function.rs @@ -14,7 +14,6 @@ use crate::{ listing::{ListingOptions, ListingPage}, utils::{validate_function_name, validate_namespace}, }; -use base64::{Engine, engine::general_purpose::URL_SAFE_NO_PAD}; /// The SQL result of one Function drop, before the public boolean wrapper /// discards the advisory details. @@ -89,81 +88,17 @@ impl SqlClient { namespace: &[String], options: ListingOptions, ) -> Result> { - validate_namespace(namespace)?; - let limit = options.page_limit.unwrap_or(100); - if limit == 0 || limit > i32::MAX as u32 { - return Err(Error::InvalidInput { - message: "page limit must be between 1 and i32::MAX".into(), - }); - } - let invalid = || Error::InvalidInput { - message: "invalid function page token for this database or namespace".into(), - }; - let after = if let Some(token) = options.page_token.filter(|token| !token.is_empty()) { - let bytes = URL_SAFE_NO_PAD.decode(token).map_err(|_| invalid())?; - let (database, prefix, path, name, version): ( - String, - Option, - Vec, - String, - String, - ) = serde_json::from_slice(&bytes).map_err(|_| invalid())?; - if database != self.inner.database - || prefix != self.inner.database_prefix - || path != namespace - { - return Err(invalid()); - } - Some((name, version)) - } else { - None - }; - let mut sql_options = vec![ - format!("limit = {}", u64::from(limit) + 1), - "include_definition = true".to_owned(), - ]; - if let Some((name, version)) = &after { - sql_options.push(format!("after_name = {}", literal(name))); - sql_options.push(format!("after_version = {}", literal(version))); - } - let mut items: Vec = self - .object_job( - ExecuteQueryRequest { - query: format!("SHOW FUNCTIONS HISTORY WITH ({})", sql_options.join(", ")), - default_namespace_path: namespace.to_vec(), - parameters: None, - }, - |batches| { - strings(&batches, "definition")? - .into_iter() - .map(|value| { - serde_json::from_str(&value).map_err(|error| Error::Runtime { - message: error.to_string(), - }) - }) - .collect::>>() - }, - ) - .await? - .wait() + let (batches, page_token) = self + .listing_page("SHOW FUNCTIONS HISTORY", namespace, options) .await?; - let page_token = if items.len() > limit as usize { - items.truncate(limit as usize); - let last = items.last().unwrap(); - let bytes = serde_json::to_vec(&( - &self.inner.database, - &self.inner.database_prefix, - namespace, - last.name(), - last.version(), - )) - .map_err(|error| Error::Runtime { - message: error.to_string(), - })?; - Some(URL_SAFE_NO_PAD.encode(bytes)) - } else { - None - }; + let items = strings(&batches, "definition")? + .into_iter() + .map(|value| { + serde_json::from_str(&value).map_err(|error| Error::Runtime { + message: error.to_string(), + }) + }) + .collect::>>()?; Ok(ListingPage { items, page_token }) } pub(in crate::remote) async fn drop_function_outcome( diff --git a/rust/lancedb/src/remote/sql/materialized_view.rs b/rust/lancedb/src/remote/sql/materialized_view.rs index e982afac7..c8e1aa5c3 100644 --- a/rust/lancedb/src/remote/sql/materialized_view.rs +++ b/rust/lancedb/src/remote/sql/materialized_view.rs @@ -4,7 +4,6 @@ use std::sync::Arc; use arrow_array::{Array, RecordBatch, StringArray}; -use base64::{Engine, engine::general_purpose::URL_SAFE_NO_PAD}; use futures::TryStreamExt; use super::SqlClient; @@ -157,60 +156,7 @@ impl SqlClient { namespace: &[String], options: ListingOptions, ) -> Result> { - validate_namespace(namespace)?; - let limit = options.page_limit.unwrap_or(100); - if limit == 0 || limit > i32::MAX as u32 { - return Err(Error::InvalidInput { - message: "page limit must be between 1 and i32::MAX".into(), - }); - } - let invalid_token = || Error::InvalidInput { - message: "invalid materialized-view page token for this database or namespace".into(), - }; - let after = if let Some(token) = options.page_token.filter(|token| !token.is_empty()) { - let decoded = URL_SAFE_NO_PAD.decode(token).map_err(|_| invalid_token())?; - let (database, prefix, path, last): (String, Option, Vec, String) = - serde_json::from_slice(&decoded).map_err(|_| invalid_token())?; - if database != self.inner.database - || prefix != self.inner.database_prefix - || path != namespace - { - return Err(invalid_token()); - } - format!(" WHERE name > '{}'", last.replace('\'', "''")) - } else { - String::new() - }; - let query = self - .execute(ExecuteQueryRequest { - query: format!( - "SHOW MATERIALIZED VIEWS{after} LIMIT {}", - u64::from(limit) + 1 - ), - default_namespace_path: namespace.to_vec(), - parameters: None, - }) - .await?; - let mut reader = query.reader().await?; - let mut items = Vec::new(); - while let Some(batch) = reader.try_next().await? { - items.extend(strings(&batch, "name")?); - } - let page_token = if items.len() > limit as usize { - items.truncate(limit as usize); - let cursor = serde_json::to_vec(&( - &self.inner.database, - &self.inner.database_prefix, - namespace, - items.last().unwrap(), - )) - .map_err(|error| Error::Runtime { - message: error.to_string(), - })?; - Some(URL_SAFE_NO_PAD.encode(cursor)) - } else { - None - }; - Ok(ListingPage { items, page_token }) + self.list_names("SHOW MATERIALIZED VIEWS", namespace, options) + .await } } diff --git a/rust/lancedb/src/remote/sql/objects.rs b/rust/lancedb/src/remote/sql/objects.rs index ff87e0000..2ffb08936 100644 --- a/rust/lancedb/src/remote/sql/objects.rs +++ b/rust/lancedb/src/remote/sql/objects.rs @@ -1,42 +1,118 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The LanceDB Authors -use super::SqlClient; +use super::{SqlClient, materialized_view::quote_identifier}; use crate::{ database::ExecuteQueryRequest, error::{Error, Result}, job::Job, + listing::{ListingOptions, ListingPage}, + utils::validate_namespace, }; -use arrow_array::{Array, ListArray, RecordBatch, StringArray}; +use arrow_array::{Array, LargeStringArray, ListArray, RecordBatch, StringArray, StringViewArray}; use futures::TryStreamExt; use serde::{Serialize, de::DeserializeOwned}; use std::sync::Arc; +const DEFAULT_SQL_LISTING_LIMIT: u32 = 1_000; +const MAX_SQL_LISTING_LIMIT: u32 = 10_000; +const NEXT_PAGE_TOKEN_METADATA_KEY: &str = "lancedb.next_page_token"; + +impl SqlClient { + pub(in crate::remote) async fn listing_page( + &self, + statement: &str, + namespace: &[String], + options: ListingOptions, + ) -> Result<(Vec, Option)> { + validate_namespace(namespace)?; + let limit = options.page_limit.unwrap_or(DEFAULT_SQL_LISTING_LIMIT); + if limit > MAX_SQL_LISTING_LIMIT { + return Err(Error::InvalidInput { + message: format!("page limit must be between 0 and {MAX_SQL_LISTING_LIMIT}"), + }); + } + if options.page_token.as_ref().is_some_and(String::is_empty) { + return Err(Error::InvalidInput { + message: "page token must not be empty".into(), + }); + } + if limit == 0 { + return Ok((Vec::new(), None)); + } + let schema = if namespace.is_empty() { + "public".to_owned() + } else { + namespace.join("$") + }; + let mut statement = format!("{statement} IN {} LIMIT {limit}", quote_identifier(&schema)); + if let Some(token) = options.page_token { + statement.push_str(&format!(" FROM {}", literal(&token))); + } + let query = self + .execute(ExecuteQueryRequest { + query: statement, + default_namespace_path: namespace.to_vec(), + parameters: None, + }) + .await?; + let reader = query.reader().await?; + let token = reader + .schema() + .metadata() + .get(NEXT_PAGE_TOKEN_METADATA_KEY) + .cloned() + .filter(|token| !token.is_empty()); + let batches = reader.try_collect::>().await?; + if batches.iter().map(RecordBatch::num_rows).sum::() > limit as usize { + return Err(Error::Runtime { + message: "SQL listing exceeded its requested page size".into(), + }); + } + Ok((batches, token)) + } + + pub(in crate::remote) async fn list_names( + &self, + statement: &str, + namespace: &[String], + options: ListingOptions, + ) -> Result> { + let (batches, page_token) = self.listing_page(statement, namespace, options).await?; + Ok(ListingPage { + items: strings(&batches, "name")?, + page_token, + }) + } +} + pub(super) fn literal(value: &str) -> String { format!("'{}'", value.replace('\'', "''")) } pub(super) fn strings(batches: &[RecordBatch], name: &str) -> Result> { - batches - .iter() - .map(|batch| { - let values = batch - .column_by_name(name) - .and_then(|column| column.as_any().downcast_ref::()) - .ok_or_else(|| Error::Runtime { - message: format!("SQL result is missing UTF-8 column '{name}'"), - })?; - if values.null_count() != 0 { - return Err(Error::Runtime { - message: format!("SQL result contains null '{name}'"), - }); - } - Ok(values - .iter() - .map(|value| value.unwrap().to_owned()) - .collect::>()) - }) - .collect::>>() - .map(|rows| rows.into_iter().flatten().collect()) + let mut result = Vec::new(); + for batch in batches { + let column = batch.column_by_name(name).ok_or_else(|| Error::Runtime { + message: format!("SQL result is missing string column '{name}'"), + })?; + if column.null_count() != 0 { + return Err(Error::Runtime { + message: format!("SQL result contains null '{name}'"), + }); + } + if let Some(values) = column.as_any().downcast_ref::() { + result.extend(values.iter().map(|value| value.unwrap().to_owned())); + } else if let Some(values) = column.as_any().downcast_ref::() { + result.extend(values.iter().map(|value| value.unwrap().to_owned())); + } else if let Some(values) = column.as_any().downcast_ref::() { + result.extend(values.iter().map(|value| value.unwrap().to_owned())); + } else { + return Err(Error::Runtime { + message: format!("SQL result column '{name}' is not a string"), + }); + } + } + Ok(result) } /// Read an optional `List` column. Older servers do not return the @@ -112,9 +188,9 @@ impl SqlClient { #[cfg(test)] mod tests { - use super::string_lists; + use super::{string_lists, strings}; use arrow_array::builder::{ListBuilder, StringBuilder}; - use arrow_array::{ArrayRef, RecordBatch}; + use arrow_array::{ArrayRef, LargeStringArray, RecordBatch, StringArray, StringViewArray}; use arrow_schema::{DataType, Field, Schema}; use std::sync::Arc; @@ -126,6 +202,24 @@ mod tests { )])) } + #[test] + fn listing_strings_accept_all_arrow_string_representations() { + for values in [ + Arc::new(StringArray::from(vec!["a", "z"])) as ArrayRef, + Arc::new(LargeStringArray::from(vec!["a", "z"])) as ArrayRef, + Arc::new(StringViewArray::from(vec!["a", "z"])) as ArrayRef, + ] { + let batch = RecordBatch::try_from_iter(vec![("name", values)]).unwrap(); + assert_eq!(strings(&[batch], "name").unwrap(), ["a", "z"]); + } + let batch = RecordBatch::try_from_iter(vec![( + "name", + Arc::new(StringViewArray::from(vec![Some("a"), None])) as ArrayRef, + )]) + .unwrap(); + assert!(strings(&[batch], "name").is_err()); + } + #[test] fn reads_warnings_after_an_empty_first_batch() { let schema = warning_schema(); diff --git a/rust/lancedb/src/remote/sql/secret.rs b/rust/lancedb/src/remote/sql/secret.rs index 6ddc78645..93c2a6c6e 100644 --- a/rust/lancedb/src/remote/sql/secret.rs +++ b/rust/lancedb/src/remote/sql/secret.rs @@ -10,8 +10,6 @@ use crate::{ utils::{validate_namespace, validate_secret_name}, }; use arrow_array::{Array, Int64Array, RecordBatch, StringArray}; -use base64::{Engine, engine::general_purpose::URL_SAFE_NO_PAD}; -use futures::TryStreamExt; use std::sync::Arc; pub(in crate::remote) enum SecretWriteOperation { @@ -135,65 +133,7 @@ impl SqlClient { namespace: &[String], options: ListingOptions, ) -> Result> { - validate_namespace(namespace)?; - let limit = options.page_limit.unwrap_or(100); - if limit == 0 || limit > i32::MAX as u32 { - return Err(Error::InvalidInput { - message: "page limit must be between 1 and i32::MAX".into(), - }); - } - let invalid_token = || Error::InvalidInput { - message: "invalid secret page token for this database or namespace".into(), - }; - let after = if let Some(token) = options.page_token.filter(|token| !token.is_empty()) { - let decoded = URL_SAFE_NO_PAD.decode(token).map_err(|_| invalid_token())?; - let (kind, database, prefix, path, last): ( - String, - String, - Option, - Vec, - String, - ) = serde_json::from_slice(&decoded).map_err(|_| invalid_token())?; - if kind != "secret" - || database != self.inner.database - || prefix != self.inner.database_prefix - || path != namespace - { - return Err(invalid_token()); - } - format!(" WHERE name > '{}'", last.replace('\'', "''")) - } else { - String::new() - }; - let query = self - .execute(ExecuteQueryRequest { - query: format!("SHOW SECRETS{after} LIMIT {}", u64::from(limit) + 1), - default_namespace_path: namespace.to_vec(), - parameters: None, - }) - .await?; - let mut reader = query.reader().await?; - let mut items = Vec::new(); - while let Some(batch) = reader.try_next().await? { - items.extend(strings(&[batch], "name")?); - } - let page_token = if items.len() > limit as usize { - items.truncate(limit as usize); - let cursor = serde_json::to_vec(&( - "secret", - &self.inner.database, - &self.inner.database_prefix, - namespace, - items.last().unwrap(), - )) - .map_err(|error| Error::Runtime { - message: error.to_string(), - })?; - Some(URL_SAFE_NO_PAD.encode(cursor)) - } else { - None - }; - Ok(ListingPage { items, page_token }) + self.list_names("SHOW SECRETS", namespace, options).await } } diff --git a/rust/lancedb/src/remote/sql/view.rs b/rust/lancedb/src/remote/sql/view.rs index 513113187..b058269a0 100644 --- a/rust/lancedb/src/remote/sql/view.rs +++ b/rust/lancedb/src/remote/sql/view.rs @@ -9,8 +9,6 @@ use crate::{ utils::{validate_namespace, validate_view_name}, view::ViewDescription, }; -use base64::{Engine, engine::general_purpose::URL_SAFE_NO_PAD}; -use futures::TryStreamExt; use lance_namespace::models::JsonArrowSchema; use std::sync::Arc; #[derive(Clone, serde::Serialize, serde::Deserialize)] @@ -147,64 +145,6 @@ impl SqlClient { namespace: &[String], options: ListingOptions, ) -> Result> { - validate_namespace(namespace)?; - let limit = options.page_limit.unwrap_or(100); - if limit == 0 || limit > i32::MAX as u32 { - return Err(Error::InvalidInput { - message: "page limit must be between 1 and i32::MAX".into(), - }); - } - let invalid_token = || Error::InvalidInput { - message: "invalid view page token for this database or namespace".into(), - }; - let after = if let Some(token) = options.page_token.filter(|token| !token.is_empty()) { - let decoded = URL_SAFE_NO_PAD.decode(token).map_err(|_| invalid_token())?; - let (kind, database, prefix, path, last): ( - String, - String, - Option, - Vec, - String, - ) = serde_json::from_slice(&decoded).map_err(|_| invalid_token())?; - if kind != "view" - || database != self.inner.database - || prefix != self.inner.database_prefix - || path != namespace - { - return Err(invalid_token()); - } - format!(" WHERE name > '{}'", last.replace('\'', "''")) - } else { - String::new() - }; - let query = self - .execute(ExecuteQueryRequest { - query: format!("SHOW VIEWS{after} LIMIT {}", u64::from(limit) + 1), - default_namespace_path: namespace.to_vec(), - parameters: None, - }) - .await?; - let mut reader = query.reader().await?; - let mut items = Vec::new(); - while let Some(batch) = reader.try_next().await? { - items.extend(strings(&[batch], "name")?); - } - let page_token = if items.len() > limit as usize { - items.truncate(limit as usize); - let cursor = serde_json::to_vec(&( - "view", - &self.inner.database, - &self.inner.database_prefix, - namespace, - items.last().unwrap(), - )) - .map_err(|error| Error::Runtime { - message: error.to_string(), - })?; - Some(URL_SAFE_NO_PAD.encode(cursor)) - } else { - None - }; - Ok(ListingPage { items, page_token }) + self.list_names("SHOW VIEWS", namespace, options).await } } diff --git a/rust/lancedb/src/remote/sql_test.rs b/rust/lancedb/src/remote/sql_test.rs index 5c7ae7673..d7249da9b 100644 --- a/rust/lancedb/src/remote/sql_test.rs +++ b/rust/lancedb/src/remote/sql_test.rs @@ -14,7 +14,6 @@ use arrow_flight::{ FlightInfo, HandshakeRequest, HandshakeResponse, PollInfo, PutResult, SchemaResult, Ticket, }; use arrow_schema::{DataType, Field, Schema}; -use base64::{Engine, engine::general_purpose::URL_SAFE_NO_PAD}; use futures::stream::BoxStream; use futures::{StreamExt, TryStreamExt}; use tonic::metadata::MetadataMap; @@ -2919,25 +2918,54 @@ async fn remote_optimize_full_rejects_explicit_pruning_options() { assert_rejected_before_submission(None, None, Some(true)).await; } +fn sql_listing_batch(column: &str, values: &[&str], token: Option<&str>) -> RecordBatch { + let mut columns: Vec<(&str, ArrayRef)> = Vec::new(); + if column == "name" { + columns.push(( + "namespace_path", + Arc::new(StringArray::from(vec![ + r#"["team","reports"]"#; + values.len() + ])), + )); + } + columns.push((column, Arc::new(StringArray::from(values.to_vec())))); + let batch = RecordBatch::try_from_iter(columns).unwrap(); + let metadata = token + .map(|token| HashMap::from([("lancedb.next_page_token".to_owned(), token.to_owned())])) + .unwrap_or_default(); + let schema = Arc::new(batch.schema().as_ref().clone().with_metadata(metadata)); + RecordBatch::try_new(schema, batch.columns().to_vec()).unwrap() +} + #[tokio::test] -async fn remote_mv_listing_uses_scoped_sql_keyset_pagination() { +async fn remote_mv_listing_uses_server_continuation_tokens() { for defer in [false, true] { - let result = RecordBatch::try_from_iter(vec![( - "name", - Arc::new(StringArray::from(vec!["a", "b'c", "z"])) as ArrayRef, - )]) - .unwrap(); + let token = "server'opaque:token"; + let next_sql = + "SHOW MATERIALIZED VIEWS IN \"team$reports\" LIMIT 2 FROM 'server''opaque:token'"; let service = TestSqlService { adaptive: true, defer, - result, + result_overrides: Arc::new(std::sync::Mutex::new(HashMap::from([ + ( + "SHOW MATERIALIZED VIEWS IN \"team$reports\" LIMIT 2".to_owned(), + sql_listing_batch("name", &["a", "b"], Some(token)), + ), + (next_sql.to_owned(), sql_listing_batch("name", &["z"], None)), + ]))), ..Default::default() }; let statements = service.statements.clone(); let headers = service.headers.clone(); - let (client, shutdown) = start_sql_service(service, ClientConfig::default()).await; - let db = RemoteDatabase::new_mock(|request| -> http::Response { - panic!("unexpected REST request: {}", request.url().path()); + let (client, shutdown) = start_sql_service_with_prefix( + service, + ClientConfig::default(), + Some("tenant/production"), + ) + .await; + let db = RemoteDatabase::new_mock(|_| -> http::Response { + panic!("unexpected REST request") }) .with_sql_client(client); let namespace = vec!["team".to_owned(), "reports".to_owned()]; @@ -2948,41 +2976,31 @@ async fn remote_mv_listing_uses_scoped_sql_keyset_pagination() { ) .await .unwrap(); - assert_eq!(first.items, ["a", "b'c"]); - let token = first.page_token.unwrap(); - db.list_materialized_views( - &namespace, - crate::listing::ListingOptions::default() - .page_limit(2) - .page_token(&token), - ) - .await - .unwrap(); + assert_eq!(first.items, ["a", "b"]); + assert_eq!(first.page_token.as_deref(), Some(token)); + let second = db + .list_materialized_views( + &namespace, + crate::listing::ListingOptions::default() + .page_limit(2) + .page_token(first.page_token.unwrap()), + ) + .await + .unwrap(); + assert_eq!(second.items, ["z"]); + assert!(second.page_token.is_none()); assert_eq!( *statements.lock().unwrap(), [ - "SHOW MATERIALIZED VIEWS LIMIT 3", - "SHOW MATERIALIZED VIEWS WHERE name > 'b''c' LIMIT 3" + "SHOW MATERIALIZED VIEWS IN \"team$reports\" LIMIT 2", + next_sql ] ); - assert!( - headers - .lock() - .unwrap() - .iter() - .all(|header| header.namespace_path == "team$reports") - ); - for (path, token) in [(vec![], token), (namespace, "invalid".into())] { - assert!(matches!( - db.list_materialized_views( - &path, - crate::listing::ListingOptions::default().page_token(token) - ) - .await, - Err(Error::InvalidInput { .. }) - )); + for header in headers.lock().unwrap().iter() { + assert_eq!(header.database, "analytics"); + assert_eq!(header.database_prefix, "tenant/production"); + assert_eq!(header.namespace_path, "team$reports"); } - assert_eq!(statements.lock().unwrap().len(), 2); shutdown.send(()).unwrap(); } } @@ -3123,7 +3141,10 @@ async fn storage_integration_methods_use_sql_with_namespace_and_typed_metadata() let statements = statements.lock().unwrap(); assert!(statements[0].contains("STORAGE_AWS_REGION = 'us-east-1'")); assert_eq!(statements[1], "DESCRIBE STORAGE INTEGRATION \"raw\""); - assert_eq!(statements[2], "SHOW STORAGE INTEGRATIONS"); + assert_eq!( + statements[2], + "SHOW STORAGE INTEGRATIONS IN \"platform$aws\" LIMIT 1000" + ); assert_eq!(statements[3], "DROP STORAGE INTEGRATION \"raw\""); assert!(statements[4].contains( "STORAGE_AWS_ACCESS_KEY_ID_SECRET = \"platform/raw\".\"credentials\".\"key\"" @@ -3670,29 +3691,20 @@ async fn secret_description_uses_the_sql_metadata_projection() { } #[tokio::test] -async fn secret_listing_uses_bounded_sql_and_scoped_cursors() { +async fn secret_listing_uses_server_continuation_tokens() { for defer in [false, true] { - let first_result = RecordBatch::try_from_iter(vec![( - "name", - Arc::new(StringArray::from(vec!["api-key", "hf'prod", "openai-prod"])) as ArrayRef, - )]) - .unwrap(); - let second_result = RecordBatch::try_from_iter(vec![( - "name", - Arc::new(StringArray::from(vec!["openai-prod"])) as ArrayRef, - )]) - .unwrap(); - let result_overrides = Arc::new(std::sync::Mutex::new(HashMap::from([ - ("SHOW SECRETS LIMIT 3".to_string(), first_result), - ( - "SHOW SECRETS WHERE name > 'hf''prod' LIMIT 3".to_string(), - second_result, - ), - ]))); + let token = "server'opaque:token"; + let next_sql = "SHOW SECRETS IN \"team$reports\" LIMIT 2 FROM 'server''opaque:token'"; let service = TestSqlService { adaptive: true, defer, - result_overrides, + result_overrides: Arc::new(std::sync::Mutex::new(HashMap::from([ + ( + "SHOW SECRETS IN \"team$reports\" LIMIT 2".to_owned(), + sql_listing_batch("name", &["a", "b"], Some(token)), + ), + (next_sql.to_owned(), sql_listing_batch("name", &["z"], None)), + ]))), ..Default::default() }; let statements = service.statements.clone(); @@ -3704,11 +3716,10 @@ async fn secret_listing_uses_bounded_sql_and_scoped_cursors() { ) .await; let db = RemoteDatabase::new_mock(|_| -> http::Response { - panic!("unexpected REST request"); + panic!("unexpected REST request") }) .with_sql_client(client); let namespace = vec!["team".to_owned(), "reports".to_owned()]; - let first = db .list_secrets( &namespace, @@ -3716,90 +3727,144 @@ async fn secret_listing_uses_bounded_sql_and_scoped_cursors() { ) .await .unwrap(); - assert_eq!(first.items, ["api-key", "hf'prod"]); - let token = first.page_token.expect("a third row must produce a token"); - + assert_eq!(first.items, ["a", "b"]); + assert_eq!(first.page_token.as_deref(), Some(token)); let second = db .list_secrets( &namespace, crate::listing::ListingOptions::default() .page_limit(2) - .page_token(&token), + .page_token(first.page_token.unwrap()), ) .await .unwrap(); - assert_eq!(second.items, ["openai-prod"]); + assert_eq!(second.items, ["z"]); assert!(second.page_token.is_none()); - assert_eq!( *statements.lock().unwrap(), - [ - "SHOW SECRETS LIMIT 3", - "SHOW SECRETS WHERE name > 'hf''prod' LIMIT 3" - ] + ["SHOW SECRETS IN \"team$reports\" LIMIT 2", next_sql] ); - { - let headers = headers.lock().unwrap(); - assert!(!headers.is_empty()); - for header in headers.iter() { - assert_eq!(header.database, "analytics"); - assert_eq!(header.database_prefix, "tenant/production"); - assert_eq!(header.namespace_path, "team$reports"); - } + for header in headers.lock().unwrap().iter() { + assert_eq!(header.database, "analytics"); + assert_eq!(header.database_prefix, "tenant/production"); + assert_eq!(header.namespace_path, "team$reports"); } - - for (path, token) in [ - (vec!["other".to_owned()], token), - (namespace.clone(), "invalid".to_owned()), - ] { - assert!(matches!( - db.list_secrets( - &path, - crate::listing::ListingOptions::default().page_token(token) - ) - .await, - Err(Error::InvalidInput { .. }) - )); - } - assert_eq!(statements.lock().unwrap().len(), 2); shutdown.send(()).unwrap(); } } #[tokio::test] -async fn secret_listing_handles_empty_pages_and_rejects_invalid_limits() { - let empty = RecordBatch::try_from_iter(vec![( - "name", - Arc::new(StringArray::from(Vec::::new())) as ArrayRef, - )]) - .unwrap(); +async fn storage_integration_listing_uses_server_continuation_tokens() { + for defer in [false, true] { + let token = "server'opaque:token"; + let next_sql = + "SHOW STORAGE INTEGRATIONS IN \"team$reports\" LIMIT 2 FROM 'server''opaque:token'"; + let service = TestSqlService { + adaptive: true, + defer, + result_overrides: Arc::new(std::sync::Mutex::new(HashMap::from([ + ( + "SHOW STORAGE INTEGRATIONS IN \"team$reports\" LIMIT 2".to_owned(), + sql_listing_batch("name", &["a", "b"], Some(token)), + ), + (next_sql.to_owned(), sql_listing_batch("name", &["z"], None)), + ]))), + ..Default::default() + }; + let statements = service.statements.clone(); + let headers = service.headers.clone(); + let (client, shutdown) = start_sql_service_with_prefix( + service, + ClientConfig::default(), + Some("tenant/production"), + ) + .await; + let db = RemoteDatabase::new_mock(|_| -> http::Response { + panic!("unexpected REST request") + }) + .with_sql_client(client); + let namespace = vec!["team".to_owned(), "reports".to_owned()]; + let first = db + .list_storage_integrations( + &namespace, + crate::listing::ListingOptions::default().page_limit(2), + ) + .await + .unwrap(); + assert_eq!(first.items, ["a", "b"]); + assert_eq!(first.page_token.as_deref(), Some(token)); + let second = db + .list_storage_integrations( + &namespace, + crate::listing::ListingOptions::default() + .page_limit(2) + .page_token(first.page_token.unwrap()), + ) + .await + .unwrap(); + assert_eq!(second.items, ["z"]); + assert!(second.page_token.is_none()); + assert_eq!( + *statements.lock().unwrap(), + [ + "SHOW STORAGE INTEGRATIONS IN \"team$reports\" LIMIT 2", + next_sql + ] + ); + for header in headers.lock().unwrap().iter() { + assert_eq!(header.database, "analytics"); + assert_eq!(header.database_prefix, "tenant/production"); + assert_eq!(header.namespace_path, "team$reports"); + } + shutdown.send(()).unwrap(); + } +} + +#[tokio::test] +async fn secret_listing_preserves_empty_continuations_and_validates_page_options() { let service = TestSqlService { adaptive: true, defer: true, - result: empty, + result_overrides: Arc::new(std::sync::Mutex::new(HashMap::from([ + ( + "SHOW SECRETS IN \"team$reports\" LIMIT 2".to_owned(), + sql_listing_batch("name", &[], Some("continue-empty")), + ), + ( + "SHOW SECRETS IN \"team$reports\" LIMIT 2 FROM 'continue-empty'".to_owned(), + sql_listing_batch("name", &["last"], None), + ), + ]))), ..Default::default() }; let statements = service.statements.clone(); let (client, shutdown) = start_sql_service(service, ClientConfig::default()).await; let db = RemoteDatabase::new_mock(|_| -> http::Response { - panic!("unexpected REST request"); + panic!("unexpected REST request") }) .with_sql_client(client); let namespace = vec!["team".to_owned(), "reports".to_owned()]; - let page = db .list_secrets( &namespace, - crate::listing::ListingOptions::default() - .page_limit(2) - .page_token(""), + crate::listing::ListingOptions::default().page_limit(2), ) .await .unwrap(); assert!(page.items.is_empty()); - assert!(page.page_token.is_none()); - - for limit in [0, i32::MAX as u32 + 1] { + assert_eq!(page.page_token.as_deref(), Some("continue-empty")); + let last = db + .list_secrets( + &namespace, + crate::listing::ListingOptions::default() + .page_limit(2) + .page_token(page.page_token.unwrap()), + ) + .await + .unwrap(); + assert_eq!(last.items, ["last"]); + assert!(last.page_token.is_none()); + for limit in [10_001, u32::MAX] { assert!(matches!( db.list_secrets( &namespace, @@ -3809,12 +3874,29 @@ async fn secret_listing_handles_empty_pages_and_rejects_invalid_limits() { Err(Error::InvalidInput { .. }) )); } - assert_eq!(*statements.lock().unwrap(), ["SHOW SECRETS LIMIT 3"]); + assert!( + db.list_secrets( + &namespace, + crate::listing::ListingOptions::default().page_token("") + ) + .await + .is_err() + ); + let zero = db + .list_secrets( + &namespace, + crate::listing::ListingOptions::default().page_limit(0), + ) + .await + .unwrap(); + assert!(zero.items.is_empty()); + assert!(zero.page_token.is_none()); + assert_eq!(statements.lock().unwrap().len(), 2); shutdown.send(()).unwrap(); } #[tokio::test] -async fn function_listing_uses_bounded_sql_and_scoped_version_cursors() { +async fn function_listing_uses_server_tokens_for_numeric_revision_pages() { const VERSION: &str = include_str!( "../../tests/fixtures/first_class_functions/v1/remote_function_version.canonical.json" ); @@ -3824,74 +3906,65 @@ async fn function_listing_uses_bounded_sql_and_scoped_version_cursors() { let mut tenth: serde_json::Value = serde_json::from_str(VERSION).unwrap(); tenth["version"] = serde_json::json!("10"); let tenth = tenth.to_string(); - let result = RecordBatch::try_from_iter(vec![( - "definition", - Arc::new(StringArray::from(vec![ - VERSION, - second.as_str(), - tenth.as_str(), - ])) as ArrayRef, - )]) - .unwrap(); - let service = TestSqlService { - adaptive: true, - defer: true, - result, - ..Default::default() - }; - service.result_overrides.lock().unwrap().insert( - "SHOW FUNCTIONS HISTORY WITH (limit = 3, include_definition = true, after_name = 'embed', after_version = '2')".into(), - RecordBatch::try_from_iter(vec![("definition", Arc::new(StringArray::from(vec![tenth.as_str()])) as ArrayRef)]).unwrap(), - ); - let statements = service.statements.clone(); - let (client, shutdown) = start_sql_service(service, ClientConfig::default()).await; - let db = RemoteDatabase::new_mock(|_| -> http::Response { - panic!("unexpected REST request"); - }) - .with_sql_client(client); - let first = db - .list_functions( - &["team".into()], - crate::listing::ListingOptions::default().page_limit(2), - ) - .await - .unwrap(); - assert_eq!( - first - .items - .iter() - .map(|item| item.version()) - .collect::>(), - ["1", "2"] - ); - let token = first.page_token.unwrap(); - let second = db - .list_functions( - &["team".into()], - crate::listing::ListingOptions::default() - .page_limit(2) - .page_token(token.clone()), - ) - .await - .unwrap(); - assert_eq!(second.items.len(), 1); - assert_eq!(second.items[0].version(), "10"); - assert!(second.page_token.is_none()); - assert!( - db.list_functions( - &["other".into()], - crate::listing::ListingOptions::default() - .page_limit(2) - .page_token(token) - ) - .await - .is_err() - ); - let seen = statements.lock().unwrap(); - assert_eq!(seen.len(), 2); - assert!(seen[0].starts_with("SHOW FUNCTIONS HISTORY WITH (limit = 3,")); - assert!(seen[1].contains("after_version = '2'")); - shutdown.send(()).unwrap(); + for defer in [false, true] { + let service = TestSqlService { + adaptive: true, + defer, + result_overrides: Arc::new(std::sync::Mutex::new(HashMap::from([ + ( + "SHOW FUNCTIONS HISTORY IN \"team\" LIMIT 2".to_owned(), + sql_listing_batch("definition", &[VERSION, &second], Some("revision-cursor")), + ), + ( + "SHOW FUNCTIONS HISTORY IN \"team\" LIMIT 2 FROM 'revision-cursor'".to_owned(), + sql_listing_batch("definition", &[&tenth], None), + ), + ]))), + ..Default::default() + }; + let statements = service.statements.clone(); + let (client, shutdown) = start_sql_service(service, ClientConfig::default()).await; + let db = RemoteDatabase::new_mock(|_| -> http::Response { + panic!("unexpected REST request") + }) + .with_sql_client(client); + let first = db + .list_functions( + &["team".into()], + crate::listing::ListingOptions::default().page_limit(2), + ) + .await + .unwrap(); + assert_eq!( + first + .items + .iter() + .map(|item| item.version()) + .collect::>(), + ["1", "2"] + ); + assert_eq!(first.page_token.as_deref(), Some("revision-cursor")); + let last = db + .list_functions( + &["team".into()], + crate::listing::ListingOptions::default() + .page_limit(2) + .page_token(first.page_token.unwrap()), + ) + .await + .unwrap(); + assert_eq!(last.items.len(), 1); + assert_eq!(last.items[0].version(), "10"); + assert!(last.page_token.is_none()); + assert_eq!( + *statements.lock().unwrap(), + [ + "SHOW FUNCTIONS HISTORY IN \"team\" LIMIT 2", + "SHOW FUNCTIONS HISTORY IN \"team\" LIMIT 2 FROM 'revision-cursor'" + ] + ); + shutdown.send(()).unwrap(); + } } #[tokio::test] @@ -3999,29 +4072,20 @@ async fn view_drop_uses_a_repeatable_query_job() { } #[tokio::test] -async fn view_listing_uses_bounded_sql_and_scoped_cursors() { +async fn view_listing_uses_server_continuation_tokens() { for defer in [false, true] { - let first_result = RecordBatch::try_from_iter(vec![( - "name", - Arc::new(StringArray::from(vec!["adults", "hf'prod", "openai-prod"])) as ArrayRef, - )]) - .unwrap(); - let second_result = RecordBatch::try_from_iter(vec![( - "name", - Arc::new(StringArray::from(vec!["openai-prod"])) as ArrayRef, - )]) - .unwrap(); - let result_overrides = Arc::new(std::sync::Mutex::new(HashMap::from([ - ("SHOW VIEWS LIMIT 3".to_string(), first_result), - ( - "SHOW VIEWS WHERE name > 'hf''prod' LIMIT 3".to_string(), - second_result, - ), - ]))); + let token = "server'opaque:token"; + let next_sql = "SHOW VIEWS IN \"team$reports\" LIMIT 2 FROM 'server''opaque:token'"; let service = TestSqlService { adaptive: true, defer, - result_overrides, + result_overrides: Arc::new(std::sync::Mutex::new(HashMap::from([ + ( + "SHOW VIEWS IN \"team$reports\" LIMIT 2".to_owned(), + sql_listing_batch("name", &["a", "b"], Some(token)), + ), + (next_sql.to_owned(), sql_listing_batch("name", &["z"], None)), + ]))), ..Default::default() }; let statements = service.statements.clone(); @@ -4033,107 +4097,39 @@ async fn view_listing_uses_bounded_sql_and_scoped_cursors() { ) .await; let db = RemoteDatabase::new_mock(|_| -> http::Response { - panic!("unexpected REST request"); + panic!("unexpected REST request") }) .with_sql_client(client); - let connection = Connection::new( - Arc::new(db), - Arc::new(crate::embeddings::MemoryRegistry::new()), - ); let namespace = vec!["team".to_owned(), "reports".to_owned()]; - - let first = connection + let first = db .list_views( &namespace, crate::listing::ListingOptions::default().page_limit(2), ) - .next_page() .await - .unwrap() .unwrap(); - assert_eq!(first.items, ["adults", "hf'prod"]); - let token = first.page_token.expect("a third row must produce a token"); - - let second = connection + assert_eq!(first.items, ["a", "b"]); + assert_eq!(first.page_token.as_deref(), Some(token)); + let second = db .list_views( &namespace, crate::listing::ListingOptions::default() .page_limit(2) - .page_token(&token), + .page_token(first.page_token.unwrap()), ) - .next_page() .await - .unwrap() .unwrap(); - assert_eq!(second.items, ["openai-prod"]); + assert_eq!(second.items, ["z"]); assert!(second.page_token.is_none()); - assert_eq!( *statements.lock().unwrap(), - [ - "SHOW VIEWS LIMIT 3", - "SHOW VIEWS WHERE name > 'hf''prod' LIMIT 3" - ] + ["SHOW VIEWS IN \"team$reports\" LIMIT 2", next_sql] ); - { - let headers = headers.lock().unwrap(); - assert!(!headers.is_empty()); - for header in headers.iter() { - assert_eq!(header.database, "analytics"); - assert_eq!(header.database_prefix, "tenant/production"); - assert_eq!(header.namespace_path, "team$reports"); - } + for header in headers.lock().unwrap().iter() { + assert_eq!(header.database, "analytics"); + assert_eq!(header.database_prefix, "tenant/production"); + assert_eq!(header.namespace_path, "team$reports"); } - - let encode_token = - |kind: &str, database: &str, prefix: Option<&str>, path: &[String], last: &str| { - URL_SAFE_NO_PAD.encode( - serde_json::to_vec(&(kind, database, prefix, path.to_vec(), last)).unwrap(), - ) - }; - for forged in [ - encode_token( - "secret", - "analytics", - Some("tenant/production"), - &namespace, - "hf'prod", - ), - encode_token( - "view", - "billing", - Some("tenant/production"), - &namespace, - "hf'prod", - ), - encode_token("view", "analytics", Some("other"), &namespace, "hf'prod"), - encode_token( - "view", - "analytics", - Some("tenant/production"), - &["other".to_owned()], - "hf'prod", - ), - "not-a-json-token".to_string(), - ] { - let mut listing = connection.list_views( - &namespace, - crate::listing::ListingOptions::default().page_token(forged), - ); - assert!(matches!( - listing.next_page().await, - Err(Error::InvalidInput { .. }) - )); - } - let mut listing = connection.list_views( - &["other".to_owned()], - crate::listing::ListingOptions::default().page_token(&token), - ); - assert!(matches!( - listing.next_page().await, - Err(Error::InvalidInput { .. }) - )); - assert_eq!(statements.lock().unwrap().len(), 2); shutdown.send(()).unwrap(); } } diff --git a/rust/lancedb/src/remote/storage_integration.rs b/rust/lancedb/src/remote/storage_integration.rs index 48dab4c6a..6c132ab4b 100644 --- a/rust/lancedb/src/remote/storage_integration.rs +++ b/rust/lancedb/src/remote/storage_integration.rs @@ -259,29 +259,6 @@ pub(super) fn decode_info(batches: &[RecordBatch]) -> Result Result> { - let mut names = Vec::new(); - for batch in batches { - let column = batch - .column_by_name("name") - .ok_or_else(|| invalid("SQL storage integration listing is missing names"))?; - for row in 0..batch.num_rows() { - if column.is_null(row) { - return Err(invalid("SQL storage integration name cannot be null")); - } - names.push( - string_value(column.as_ref(), row) - .ok_or_else(|| { - invalid("SQL storage integration listing requires string names") - })? - .to_string(), - ); - } - } - names.sort(); - Ok(names) -} - #[cfg(test)] mod tests { use std::sync::Arc;