From f36aa69fc31de06fe32912c00a65855b8ce2fc0f Mon Sep 17 00:00:00 2001 From: Diego Imbert Date: Wed, 16 Sep 2026 15:23:20 +0200 Subject: [PATCH] feat(datatables): data table roles in the DB manager and raw apps Co-Authored-By: Claude Opus 5 (1M context) --- backend/ee-repo-ref.txt | 2 +- .../tests/datatable_roles.rs | 64 + .../src/datatable_permissions_oss.rs | 26 +- .../windmill-api-workspaces/src/workspaces.rs | 148 ++- backend/windmill-api/openapi.yaml | 52 +- backend/windmill-api/src/jobs.rs | 7 +- .../src/datatable_roles_oss.rs | 3 +- backend/windmill-common/src/query_builders.rs | 37 +- cli/src/commands/app/raw_apps.ts | 2 + cli/src/guidance/skills.gen.ts | 4 + frontend/src/lib/components/DBManager.svelte | 1154 +++++++++++------ .../lib/components/DBManagerContent.svelte | 123 +- .../src/lib/components/DBManagerDrawer.svelte | 254 +++- .../src/lib/components/DBTableEditor.svelte | 6 +- .../lib/components/DatatableRoleBadge.svelte | 66 + .../lib/components/DdlMigrationGuard.svelte | 19 +- frontend/src/lib/components/SqlRepl.svelte | 7 +- frontend/src/lib/components/Star.svelte | 9 +- .../apps/components/display/dbtable/utils.ts | 3 +- .../ConfirmationModal.svelte | 2 +- .../copilot/chat/AIChatManager.svelte.ts | 6 +- .../chat/DatatableCreationPolicy.svelte | 1 + .../lib/components/copilot/chat/app/core.ts | 24 +- .../components/copilot/chat/datatableTools.ts | 84 +- .../components/copilot/chat/global/core.ts | 7 +- .../datatableAcl/AclTargetPicker.svelte | 50 - .../datatableAcl/PgAclEditor.svelte | 12 +- .../components/datatableMigrationRole.test.ts | 60 + .../lib/components/datatableMigrationRole.ts | 70 + .../lib/components/datatableUsableRoles.ts | 34 + .../components/dbManagerDrawerModel.svelte.ts | 50 +- .../src/lib/components/dbManagerRole.test.ts | 60 + frontend/src/lib/components/dbOps.ts | 46 +- frontend/src/lib/components/dbSchemaCache.ts | 21 + frontend/src/lib/components/dbTypes.ts | 25 + .../raw_apps/DefaultDatabaseSelector.svelte | 21 +- .../raw_apps/RawAppDataTableDrawer.svelte | 329 +++-- .../raw_apps/RawAppDataTableList.svelte | 12 + .../components/raw_apps/RawAppEditor.svelte | 86 +- .../components/raw_apps/RawAppSidebar.svelte | 29 +- .../raw_apps/RawAppTemplatePicker.svelte | 262 +++- .../raw_apps/dataTableRefUtils.test.ts | 28 +- .../components/raw_apps/dataTableRefUtils.ts | 37 + .../raw_apps/datatableUtils.svelte.ts | 178 ++- .../DataTableMigrationsButton.svelte | 10 +- .../DataTablePermissionsButton.svelte | 477 ++++--- .../DataTableRolesSection.svelte | 14 +- .../DataTableSettings.svelte | 56 +- .../ForkDatatableSection.svelte | 18 +- .../InstanceRolesButton.svelte | 37 +- .../NewDataTableMigrationModal.svelte | 185 ++- .../apps_raw/edit/[...path]/+page.svelte | 3 +- .../auto-generated/skills/raw-app/SKILL.md | 4 + system_prompts/base/raw-app-cli.md | 4 + 54 files changed, 3275 insertions(+), 1053 deletions(-) create mode 100644 frontend/src/lib/components/DatatableRoleBadge.svelte delete mode 100644 frontend/src/lib/components/datatableAcl/AclTargetPicker.svelte create mode 100644 frontend/src/lib/components/datatableMigrationRole.test.ts create mode 100644 frontend/src/lib/components/datatableMigrationRole.ts create mode 100644 frontend/src/lib/components/datatableUsableRoles.ts create mode 100644 frontend/src/lib/components/dbManagerRole.test.ts create mode 100644 frontend/src/lib/components/dbSchemaCache.ts diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index b878c8de1f..bbb7ee4259 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -6f26308c67acf9fcc45773b373aa30a2593b665c +6ced9da74422ae8feee10da70419ccedf313506c diff --git a/backend/windmill-api-integration-tests/tests/datatable_roles.rs b/backend/windmill-api-integration-tests/tests/datatable_roles.rs index a15439455f..d79dd5a3e3 100644 --- a/backend/windmill-api-integration-tests/tests/datatable_roles.rs +++ b/backend/windmill-api-integration-tests/tests/datatable_roles.rs @@ -1009,6 +1009,70 @@ async fn an_entry_without_roles_cannot_newly_reach_a_database_under_roles( Ok(()) } +/// Browsing names the role it connects as, and a role the caller may not use is refused rather +/// than quietly listed as the default. The refusal is decided before connecting, so the fixture's +/// database never has to exist. +#[cfg(all(feature = "private", feature = "enterprise"))] +#[sqlx::test(migrations = "../migrations", fixtures("base", "datatable_roles"))] +async fn browsing_as_a_role_the_caller_may_not_use_is_refused( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let base = format!("http://localhost:{port}/api/w/test-workspace/workspaces"); + + // `test-user-2` is a tenant of `analytics` only. + let resp = authed( + client().get(format!( + "{base}/list_datatable_tables?role_for=main&role=admin" + )), + "SECRET_TOKEN_2", + ) + .send() + .await?; + assert_eq!(resp.status(), 200); + let body: Value = resp.json().await?; + let entry = body + .as_array() + .and_then(|a| a.iter().find(|e| e["datatable_name"] == "main")) + .expect("main is listed"); + assert_eq!(entry["usable_roles"], json!(["analytics"]), "{entry}"); + assert_eq!(entry["default_role"], "analytics", "{entry}"); + assert_eq!(entry["permissioned"], true, "{entry}"); + assert_eq!(entry["instance"], true, "{entry}"); + let error = entry["error"].as_str().unwrap_or_default(); + assert!( + error.contains("Not allowed to use role 'admin'"), + "listed as another role than the one asked for: {entry}" + ); + + let resp = authed( + client().get(format!( + "{base}/get_datatable_table_schema?datatable_name=main&schema_name=public&table_name=t&role=admin" + )), + "SECRET_TOKEN_2", + ) + .send() + .await?; + let status = resp.status(); + let text = resp.text().await?; + assert!( + text.contains("Not allowed to use role 'admin'"), + "{status}: {text}" + ); + + // A role means nothing without the data table it belongs to. + let resp = authed( + client().get(format!("{base}/list_datatable_tables?role=analytics")), + "SECRET_TOKEN_2", + ) + .send() + .await?; + assert_eq!(resp.status(), 400, "{}", resp.text().await?); + Ok(()) +} + #[cfg(not(all(feature = "private", feature = "enterprise")))] const ENTERPRISE_REFUSAL: &str = "Data table roles are a Windmill Enterprise Edition feature"; diff --git a/backend/windmill-api-workspaces/src/datatable_permissions_oss.rs b/backend/windmill-api-workspaces/src/datatable_permissions_oss.rs index f4d8c6a7ad..b94d72afa9 100644 --- a/backend/windmill-api-workspaces/src/datatable_permissions_oss.rs +++ b/backend/windmill-api-workspaces/src/datatable_permissions_oss.rs @@ -13,7 +13,7 @@ #[cfg(all(feature = "private", feature = "enterprise"))] pub(crate) use crate::datatable_permissions_ee::{ ensure_governs_datatable, ensure_reaches_datatable, get_datatable_permissions, - list_usable_datatable_roles, set_datatable_permissions, + list_usable_datatable_roles, set_datatable_permissions, usable_datatable_roles, }; #[cfg(not(all(feature = "private", feature = "enterprise")))] @@ -70,4 +70,28 @@ mod ce { pub(crate) async fn list_usable_datatable_roles(_authed: ApiAuthed) -> Result { Err(unavailable()) } + + pub(crate) struct UsableDatatableRoles { + pub(crate) permissioned: bool, + pub(crate) roles: Vec, + pub(crate) default_role: String, + } + + /// A data table not under roles is used as `admin`, as before roles existed. One under roles + /// is refused: no role of it can be connected as. + pub(crate) async fn usable_datatable_roles( + _db: &DB, + _authed: &ApiAuthed, + _w_id: &str, + governing: &GoverningDatatable, + ) -> Result { + if governing.datatable.permissions.is_some() { + return Err(unavailable()); + } + Ok(UsableDatatableRoles { + permissioned: false, + roles: vec![], + default_role: windmill_common::datatable_roles::ADMIN_DATATABLE_ROLE.to_string(), + }) + } } diff --git a/backend/windmill-api-workspaces/src/workspaces.rs b/backend/windmill-api-workspaces/src/workspaces.rs index 110bd39bec..0a2e343065 100644 --- a/backend/windmill-api-workspaces/src/workspaces.rs +++ b/backend/windmill-api-workspaces/src/workspaces.rs @@ -2269,6 +2269,25 @@ struct DataTableTables { schemas: TableListMap, #[serde(skip_serializing_if = "Option::is_none")] error: Option, + /// On the instance database: the only kind that can be under roles or have its access edited. + instance: bool, + permissioned: bool, + /// The roles this caller may connect as, by name; empty when not under roles. + usable_roles: Vec, + default_role: String, + /// What the role the listing connected as may create. + can_create_schema: bool, + creatable_schemas: Vec, +} + +#[derive(Deserialize)] +struct ListDataTableTablesQuery { + /// List only this data table: each entry opens a connection to its database. + datatable_name: Option, + /// The data table `role` applies to. Every other one is listed as its default role, since a + /// role name means nothing outside the data table it belongs to. + role_for: Option, + role: Option, } #[derive(Deserialize)] @@ -2276,6 +2295,7 @@ struct GetDataTableSchemaQuery { datatable_name: String, schema_name: String, table_name: String, + role: Option, } #[derive(Serialize, Debug)] @@ -2443,25 +2463,89 @@ async fn list_datatable_tables( authed: ApiAuthed, Extension(db): Extension, Path(w_id): Path, + Query(query): Query, ) -> JsonResult> { - let datatable_names = list_datatable_names(&db, &w_id).await?; + if query.role.is_some() && query.role_for.is_none() { + return Err(Error::BadRequest( + "`role` needs `role_for`, the data table it is a role of".to_string(), + )); + } + if let (Some(only), Some(role_for)) = + (query.datatable_name.as_deref(), query.role_for.as_deref()) + { + if only != role_for { + return Err(Error::BadRequest(format!( + "`role_for` names '{role_for}', which `datatable_name` leaves out of the listing" + ))); + } + } + let mut datatable_names = list_datatable_names(&db, &w_id).await?; + for named in [query.role_for.as_deref(), query.datatable_name.as_deref()] + .into_iter() + .flatten() + { + if !datatable_names.iter().any(|n| n == named) { + return Err(Error::NotFound(format!( + "No data table named '{named}' in this workspace" + ))); + } + } + if let Some(only) = query.datatable_name.as_deref() { + datatable_names.retain(|n| n == only); + } let mut results = Vec::new(); for datatable_name in datatable_names { - let tables = match get_datatable_tables(&db, &authed, &w_id, &datatable_name).await { - Ok(schemas) => DataTableTables { datatable_name, schemas, error: None }, - Err(e) => DataTableTables { - datatable_name, - schemas: HashMap::new(), - error: Some(e.to_string()), - }, - }; - results.push(tables); + let role = query + .role + .as_deref() + .filter(|_| query.role_for.as_deref() == Some(datatable_name.as_str())); + results.push(list_one_datatable_tables(&db, &authed, &w_id, datatable_name, role).await); } Ok(Json(results)) } +async fn list_one_datatable_tables( + db: &DB, + authed: &ApiAuthed, + w_id: &str, + datatable_name: String, + role: Option<&str>, +) -> DataTableTables { + let mut entry = DataTableTables { + datatable_name, + schemas: HashMap::new(), + error: None, + instance: false, + permissioned: false, + usable_roles: vec![], + default_role: windmill_common::datatable_roles::ADMIN_DATATABLE_ROLE.to_string(), + can_create_schema: false, + creatable_schemas: vec![], + }; + let result: Result<()> = async { + let governing = resolve_governing_datatable(db, w_id, &entry.datatable_name).await?; + entry.instance = governing.is_instance(); + let usable = + crate::datatable_permissions_oss::usable_datatable_roles(db, authed, w_id, &governing) + .await?; + entry.permissioned = usable.permissioned; + entry.usable_roles = usable.roles; + entry.default_role = usable.default_role; + let listing = get_datatable_tables(db, authed, w_id, &entry.datatable_name, role).await?; + entry.schemas = listing.schemas; + entry.can_create_schema = listing.can_create_schema; + entry.creatable_schemas = listing.creatable_schemas; + Ok(()) + } + .await; + if let Err(e) = result { + entry.error = Some(e.to_string()); + } + entry +} + async fn get_datatable_table_schema( authed: ApiAuthed, Extension(db): Extension, @@ -2475,6 +2559,7 @@ async fn get_datatable_table_schema( &query.datatable_name, &query.schema_name, &query.table_name, + query.role.as_deref(), ) .await?; @@ -2513,14 +2598,13 @@ async fn resolve_datatable_pg_as_caller( authed: &ApiAuthed, w_id: &str, datatable_name: &str, + role: Option<&str>, ) -> Result { let db_resource = get_datatable_resource_from_db( db, w_id, datatable_name, - // The data table's default role. Browsing has no way to name another one yet; when the - // database manager grows a role picker it passes the pick through here. - None, + role, DatatableAccess::Authed(authed.to_authed_ref()), ) .await?; @@ -2534,7 +2618,7 @@ async fn get_datatable_schema( w_id: &str, datatable_name: &str, ) -> Result { - let pg_db = resolve_datatable_pg_as_caller(db, authed, w_id, datatable_name).await?; + let pg_db = resolve_datatable_pg_as_caller(db, authed, w_id, datatable_name, None).await?; // Connect to the datatable database let (client, connection) = pg_db.connect(Some(db)).await?; @@ -2622,13 +2706,20 @@ async fn get_datatable_schema( Ok(schema_map) } +struct DatatableTableListing { + schemas: TableListMap, + can_create_schema: bool, + creatable_schemas: Vec, +} + async fn get_datatable_tables( db: &DB, authed: &ApiAuthed, w_id: &str, datatable_name: &str, -) -> Result { - let pg_db = resolve_datatable_pg_as_caller(db, authed, w_id, datatable_name).await?; + role: Option<&str>, +) -> Result { + let pg_db = resolve_datatable_pg_as_caller(db, authed, w_id, datatable_name, role).await?; let (client, connection) = pg_db.connect(Some(db)).await?; tokio::spawn(async move { @@ -2640,7 +2731,7 @@ async fn get_datatable_tables( let schema_rows = client .query( r#" - SELECT nspname::text AS schema_name + SELECT nspname::text AS schema_name, has_schema_privilege(oid, 'CREATE') AS can_create FROM pg_namespace WHERE nspname NOT IN ('information_schema', 'pg_toast', 'pg_catalog') AND nspname NOT LIKE 'pg_%' @@ -2654,11 +2745,29 @@ async fn get_datatable_tables( Error::internal_err(format!("Failed to query schemas: {}", pg_error_message(&e))) })?; + let can_create_schema: bool = client + .query_one( + "SELECT has_database_privilege(current_database(), 'CREATE')", + &[], + ) + .await + .map_err(|e| { + Error::internal_err(format!( + "Failed to read database privileges: {}", + pg_error_message(&e) + )) + })? + .get(0); + let mut table_map: TableListMap = HashMap::new(); + let mut creatable_schemas = Vec::new(); let schema_names: Vec = schema_rows .iter() .map(|row| { let name: String = row.get(0); + if row.get::<_, bool>(1) { + creatable_schemas.push(name.clone()); + } table_map.entry(name.clone()).or_default(); name }) @@ -2688,7 +2797,7 @@ async fn get_datatable_tables( table_map.entry(table_schema).or_default().push(table_name); } - Ok(table_map) + Ok(DatatableTableListing { schemas: table_map, can_create_schema, creatable_schemas }) } async fn get_datatable_table_columns( @@ -2698,6 +2807,7 @@ async fn get_datatable_table_columns( datatable_name: &str, schema_name: &str, table_name: &str, + role: Option<&str>, ) -> Result { if is_system_pg_schema(schema_name) { return Err(Error::BadRequest(format!( @@ -2706,7 +2816,7 @@ async fn get_datatable_table_columns( ))); } - let pg_db = resolve_datatable_pg_as_caller(db, authed, w_id, datatable_name).await?; + let pg_db = resolve_datatable_pg_as_caller(db, authed, w_id, datatable_name, role).await?; let (client, connection) = pg_db.connect(Some(db)).await?; tokio::spawn(async move { diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index fe067a536b..a3292ecfbc 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -5488,6 +5488,21 @@ paths: - workspace parameters: - $ref: "#/components/parameters/WorkspaceId" + - name: datatable_name + in: query + description: list only this data table; each listed data table opens a connection to its database + schema: + type: string + - name: role_for + in: query + description: the data table `role` applies to; every other one is listed as its default role + schema: + type: string + - name: role + in: query + description: the role to list `role_for` as; refused, in that entry's `error`, if the caller may not use it + schema: + type: string responses: "200": description: table metadata of all datatables @@ -5521,6 +5536,11 @@ paths: required: true schema: type: string + - name: role + in: query + description: the data table role to read the table as; defaults to the data table's default role + schema: + type: string responses: "200": description: schema of one datatable table @@ -36053,7 +36073,17 @@ components: DataTableTables: type: object - required: [datatable_name, schemas] + required: + [ + datatable_name, + schemas, + instance, + permissioned, + usable_roles, + default_role, + can_create_schema, + creatable_schemas, + ] properties: datatable_name: type: string @@ -36066,6 +36096,26 @@ components: type: string error: type: string + instance: + type: boolean + description: on the instance database, the only kind that can be under roles or have its access edited + permissioned: + type: boolean + usable_roles: + type: array + description: the roles the caller may connect as, by name; empty when not under roles + items: + type: string + default_role: + type: string + can_create_schema: + type: boolean + description: whether the role the listing connected as may create schemas + creatable_schemas: + type: array + description: the schemas the role the listing connected as may create in + items: + type: string DataTableTableSchema: type: object diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index e1d6db595a..600aa30487 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -8701,7 +8701,12 @@ fn register_potential_assets_on_inline_execution( .as_ref() .and_then(|args| args.get("database")) .map(|v| v.get().trim_matches('"')) - .and_then(|dt| dt.strip_prefix("datatable://")); + .and_then(|dt| dt.strip_prefix("datatable://")) + // `?role=` picks the connection, not the data table. Anything else after a `?` may be + // part of a name stored before names were restricted, so it stays. + .map(|dt| { + windmill_common::workspaces::parse_datatable_ref(dt).map_or(dt, |(name, _)| name) + }); if let Some(datatable) = datatable { let re = regex::Regex::new(r#"SET search_path TO "([^"]+)";"#).unwrap(); let (schema, content) = if let Some(captures) = re.captures(&preview.content) { diff --git a/backend/windmill-common/src/datatable_roles_oss.rs b/backend/windmill-common/src/datatable_roles_oss.rs index 98fef4260f..569aa7ff1f 100644 --- a/backend/windmill-common/src/datatable_roles_oss.rs +++ b/backend/windmill-common/src/datatable_roles_oss.rs @@ -15,7 +15,8 @@ use crate::error::Error; -/// What every roles path answers without the Enterprise Edition. +/// What every roles path answers without the Enterprise Edition. The frontend matches this exact +/// sentence (`datatableUsableRoles.ts`) to read the refusal as "not under roles": reword both. pub fn datatable_roles_unavailable() -> Error { Error::BadRequest("Data table roles are a Windmill Enterprise Edition feature".to_string()) } diff --git a/backend/windmill-common/src/query_builders.rs b/backend/windmill-common/src/query_builders.rs index 60d74f189d..9d85278404 100644 --- a/backend/windmill-common/src/query_builders.rs +++ b/backend/windmill-common/src/query_builders.rs @@ -329,6 +329,7 @@ pub fn try_expand_internal_db_query( "ALTER_TABLE" => expand_alter_table(json_str, db_type).map(ExpandedQuery::sql), "CREATE_SCHEMA" => expand_create_schema(json_str, db_type).map(ExpandedQuery::sql), "DROP_SCHEMA" => expand_drop_schema(json_str, db_type).map(ExpandedQuery::sql), + "RENAME_SCHEMA" => expand_rename_schema(json_str, db_type).map(ExpandedQuery::sql), // Metadata queries "LOAD_TABLE_METADATA" => expand_load_table_metadata(json_str, db_type), "FOREIGN_KEYS" => expand_foreign_keys(json_str, db_type).map(ExpandedQuery::sql), @@ -1716,6 +1717,13 @@ struct DropSchemaPayload { ducklake: Option, } +#[derive(Deserialize)] +struct RenameSchemaPayload { + schema: String, + new_schema: String, + ducklake: Option, +} + #[derive(Debug, Clone, Deserialize)] struct TableEditorColumn { name: String, @@ -2004,6 +2012,23 @@ fn expand_drop_schema(json_str: &str, db_type: DbType) -> Result Ok(maybe_wrap_ducklake(query, p.ducklake.as_deref())) } +fn expand_rename_schema(json_str: &str, db_type: DbType) -> Result { + let p: RenameSchemaPayload = serde_json::from_str(json_str) + .map_err(|e| format!("Invalid RENAME_SCHEMA payload: {}", e))?; + if !matches!(db_type, DbType::Postgresql | DbType::Snowflake) || p.ducklake.is_some() { + return Err(format!( + "Renaming a schema is not supported on {:?}", + db_type + )); + } + let query = format!( + "ALTER SCHEMA {} RENAME TO {};", + qi(&p.schema, db_type), + qi(&p.new_schema, db_type) + ); + Ok(query) +} + fn expand_create_table(json_str: &str, db_type: DbType) -> Result { let p: CreateTablePayload = serde_json::from_str(json_str) .map_err(|e| format!("Invalid CREATE_TABLE payload: {}", e))?; @@ -2598,7 +2623,9 @@ WHERE table_catalog = current_database()", ) } else { ( - "\nWHERE c.relkind = 'r' AND a.attnum > 0 AND NOT a.attisdropped\n AND ns.nspname != 'pg_catalog' AND ns.nspname != 'information_schema'".to_string(), + // pg_catalog is readable by everyone: without the privilege check this lists + // tables of schemas the connection's role cannot even enter. + "\nWHERE c.relkind = 'r' AND a.attnum > 0 AND NOT a.attisdropped\n AND ns.nspname != 'pg_catalog' AND ns.nspname != 'information_schema'\n AND has_schema_privilege(ns.oid, 'USAGE')".to_string(), ",\n ns.nspname AS schema_name,\n c.relname AS table_name".to_string(), "\nJOIN pg_catalog.pg_class c ON a.attrelid = c.oid\nJOIN pg_catalog.pg_namespace ns ON c.relnamespace = ns.oid".to_string(), "ns.nspname, c.relname, a.attnum".to_string(), @@ -4101,6 +4128,13 @@ mod tests { assert_eq!(sql, "DROP SCHEMA \"old_schema\" CASCADE;"); } + #[test] + fn test_expand_rename_schema() { + let marker = r#"-- WM_INTERNAL_DB_RENAME_SCHEMA {"schema":"old","new_schema":"new"}"#; + let sql = expand_code(marker, &ScriptLang::Postgresql); + assert_eq!(sql, "ALTER SCHEMA \"old\" RENAME TO \"new\";"); + } + #[test] fn test_expand_create_schema_with_ducklake() { let marker = r#"-- WM_INTERNAL_DB_CREATE_SCHEMA {"schema":"s","ducklake":"lake"}"#; @@ -4468,6 +4502,7 @@ mod tests { assert!(sql.contains("schema_name")); assert!(sql.contains("table_name")); assert!(sql.contains("c.relkind = 'r'")); + assert!(sql.contains("has_schema_privilege(ns.oid, 'USAGE')")); } #[test] diff --git a/cli/src/commands/app/raw_apps.ts b/cli/src/commands/app/raw_apps.ts index 8d82edb9df..36c8260e25 100644 --- a/cli/src/commands/app/raw_apps.ts +++ b/cli/src/commands/app/raw_apps.ts @@ -55,6 +55,8 @@ export interface AppFile { tables?: string[]; datatable?: string; schema?: string; + /** The role the app uses each data table through, by data table name. */ + roles?: Record; }; // Mirrors granular ACLs on the raw_app path. Synced via /acls/* by // applyExtraPermsDiff — never through update_app_raw — so a perm-only diff --git a/cli/src/guidance/skills.gen.ts b/cli/src/guidance/skills.gen.ts index d3ba30ce47..004c3d728f 100644 --- a/cli/src/guidance/skills.gen.ts +++ b/cli/src/guidance/skills.gen.ts @@ -5775,6 +5775,8 @@ data: tables: - main/users # Table in public schema - main/app_schema:items # Table in specific schema + roles: # Optional: the role the app uses each datatable through + main: analyst \`\`\` **Table reference formats:** @@ -5782,6 +5784,8 @@ data: - \`/\` — Specific table in public schema - \`/:
\` — Table in specific schema +**Roles:** when a datatable is under roles, its queries run as a role, which only reaches what it was granted. \`roles\` records the role the app uses each datatable through; the app's code must pass the same role: \`wmill.datatable('main', { role: 'analyst' })\` in TypeScript, \`wmill.datatable('main', role='analyst')\` in Python. A datatable without an entry is used as its default role. + ## SQL Migrations (sql_to_apply/) The \`sql_to_apply/\` folder is for creating/modifying database tables during development. diff --git a/frontend/src/lib/components/DBManager.svelte b/frontend/src/lib/components/DBManager.svelte index 7520cc5013..d0638f3670 100644 --- a/frontend/src/lib/components/DBManager.svelte +++ b/frontend/src/lib/components/DBManager.svelte @@ -1,4 +1,6 @@ - +
- {#if dbSelector} - {@render dbSelector()} - {/if} - {#if dbSupportsSchemas && !multiSelectMode} - e.stopPropagation()} - onchange={() => toggleSchemaSelection(schemaKey)} - /> - - {/if} - {schemaKey} - - {schemaTables.length} - - - -
- - {#each schemaTables as tableKey} - {@const isDisabled = isTableDisabled(schemaKey, tableKey)} - {@const isChecked = isTableSelected(schemaKey, tableKey) || isDisabled} - {@const isCurrentPreview = - selected.schemaKey === schemaKey && selected.tableKey === tableKey} -
{ - selectTable(schemaKey, tableKey) - toggleTableSelection(schemaKey, tableKey) - }} - onkeydown={(e) => { - if (e.key === 'Enter' || e.key === ' ') { - selectTable(schemaKey, tableKey) - toggleTableSelection(schemaKey, tableKey) - } - }} - > - - e.stopPropagation()} - onchange={() => toggleTableSelection(schemaKey, tableKey)} - /> - - -

{tableKey}

- + {#if dtOpen} + {#if root.error} +

{root.error}

+ {/if} + {#each root.schemas as sc (sc.schemaKey)} + {@const schemaOpen = isExpanded(root.datatable, sc.schemaKey)} + {@const indent = root.datatable !== undefined ? 'pl-7' : 'pl-3'} + {#if dbSupportsSchemas} -
- {/each} - - - {/each} - {:else} - - {#each filteredTableKeys as tableKey} - - - {/each} - {/if} + ]} + btnId={'db-manager-schema-actions-' + onlyAlphaNumAndUnderscore(sc.schemaKey)} + /> + {/if} + + + {/if} + + + {#if schemaOpen || !dbSupportsSchemas} + {@const tableIndent = dbSupportsSchemas + ? root.datatable !== undefined + ? 'pl-11' + : 'pl-7' + : root.datatable !== undefined + ? 'pl-7' + : 'pl-3'} + {#each sc.tables as tableKey (tableKey)} + {@const entry = { + datatable: root.datatable, + schema: sc.schemaKey, + table: tableKey + }} + {@const hasMenu = !multiSelectMode} + {@const isSelected = + root.datatable === currentDatatable && + selected.schemaKey === sc.schemaKey && + selected.tableKey === tableKey} + + {/each} + {#if canCreateTableIn(root.datatable, sc.schemaKey)} + + {/if} + {/if} + + {/each} + {#if dbSupportsSchemas && search.trim() === '' && canCreateSchemaIn(root.datatable)} + + {/if} + {/if} + {/each} - {#if !multiSelectMode} - - {/if} - {#if tableKey && colDefs?.[tableKey]?.length} + {#if mainPane} + {@render mainPane()} + {:else if tableKey && colDefs?.[tableKey]?.length} {@const dbTableOps = dbTableOpsFactory({ colDefs: colDefs[tableKey], tableKey, whereClause })} + (aclDrawer = undefined)}> + (aclDrawer = undefined)} + tooltip="Who owns this, and what each role may do with it." + > + {#if aclDrawer && workspace} + {@const dt = aclDrawer.datatable ?? currentDatatable} + {#if dt} + {#key `${dt}~${JSON.stringify(aclDrawer.target)}`} + + {/key} + {/if} + {/if} + + + (askingForConfirmation = undefined)} @@ -750,20 +1160,12 @@ - { - newSchemaDialogOpen = false - newSchemaName = '' - }} -> + { - newSchemaDialogOpen = false - newSchemaName = '' - }} - title="Create a new schema" + on:close={closeSchemaDialog} + title={schemaDialog?.mode === 'rename' + ? `Rename ${schemaDialog.schema}` + : 'Create a new schema'} >
@@ -775,27 +1177,7 @@ placeholder="Enter schema name..." autofocus on:keydown={(e) => { - if (e.key === 'Enter' && sanitizedNewSchemaName && !schemaAlreadyExists) { - askingForConfirmation = { - confirmationText: `Create ${sanitizedNewSchemaName}`, - type: 'reload', - title: `This will run 'CREATE SCHEMA ${sanitizedNewSchemaName}' on your database. Are you sure?`, - open: true, - id: 'db-create-schema-confirmation-modal', - onConfirm: async () => { - askingForConfirmation && (askingForConfirmation.loading = true) - try { - await dbSchemaOps.onCreateSchema({ schema: sanitizedNewSchemaName }) - refresh?.() - selected.schemaKey = sanitizedNewSchemaName - newSchemaDialogOpen = false - newSchemaName = '' - } finally { - askingForConfirmation = undefined - } - } - } - } + if (e.key === 'Enter') submitSchemaName() }} /> {#if schemaAlreadyExists} @@ -810,32 +1192,8 @@
{#snippet actions()} - {/snippet} diff --git a/frontend/src/lib/components/DBManagerContent.svelte b/frontend/src/lib/components/DBManagerContent.svelte index b0ef2859df..a5368dac18 100644 --- a/frontend/src/lib/components/DBManagerContent.svelte +++ b/frontend/src/lib/components/DBManagerContent.svelte @@ -1,16 +1,21 @@ + + (open = false) }} +> + { + // The row underneath folds on click, and picking a role is not that. + e.stopPropagation() + open = !open + }} + > + + {role} + + + anchorEl!.getBoundingClientRect())} + onSelectValue={(item) => { + open = false + if (item.value !== role) onSelect(item.value) + }} + /> + diff --git a/frontend/src/lib/components/DdlMigrationGuard.svelte b/frontend/src/lib/components/DdlMigrationGuard.svelte index 3bda6aa216..0dab262fd0 100644 --- a/frontend/src/lib/components/DdlMigrationGuard.svelte +++ b/frontend/src/lib/components/DdlMigrationGuard.svelte @@ -6,8 +6,18 @@ import { joinSqlStatements, splitSqlRuns } from './sqlDdl' import { logDdlGuardChoice } from './workspaceSettings/datatableTelemetry' import { CornerDownLeft } from 'lucide-svelte' + import { withMigrationRole } from './datatableMigrationRole' - let { workspace, datatable }: { workspace: string; datatable: string } = $props() + let { + workspace, + datatable, + role + }: { + workspace: string + datatable: string + /** The role the editor runs as. The migration declares it, or it would run as admin. */ + role?: string + } = $props() type Choice = 'run' | 'migrate' | 'cancel' @@ -73,7 +83,7 @@ function openMigrationModal(sql: string): Promise { return new Promise((resolve) => { resolveMigrationClosed = (created: boolean) => resolve(created) - newMigrationModal?.open({ codeUp: sql }) + newMigrationModal?.open({ codeUp: withMigrationRole(sql, role) }) }) } @@ -145,6 +155,11 @@ migrations rather than run ad-hoc. Create a migration for it instead? {/if}

+ {#if role} +

+ It will run as role {role}. +

+ {/if} 
{promptSql}
{#if datatableName && ws} - + {/if} diff --git a/frontend/src/lib/components/Star.svelte b/frontend/src/lib/components/Star.svelte index dd91f671c8..4f0f169109 100644 --- a/frontend/src/lib/components/Star.svelte +++ b/frontend/src/lib/components/Star.svelte @@ -9,9 +9,10 @@ kind: FavoriteKind summary?: string workspaceId?: string + size?: number } - let { path, kind, workspaceId, summary }: Props = $props() + let { path, kind, workspaceId, summary, size = 16 }: Props = $props() let buttonHover = $state(false) let starred = $derived(favoriteManager.isStarred(path, kind)) @@ -31,14 +32,14 @@ > {#if starred} {#if buttonHover} - + {:else} - + {/if} {:else} {/if} diff --git a/frontend/src/lib/components/apps/components/display/dbtable/utils.ts b/frontend/src/lib/components/apps/components/display/dbtable/utils.ts index c0477e8408..f2337d699b 100644 --- a/frontend/src/lib/components/apps/components/display/dbtable/utils.ts +++ b/frontend/src/lib/components/apps/components/display/dbtable/utils.ts @@ -282,7 +282,8 @@ const scriptsV2: typeof legacyScripts = { ...legacyScripts.postgresql, code: ` SELECT table_name, column_name, udt_name, column_default, is_nullable, nsp.nspname AS table_schema FROM information_schema.columns -RIGHT JOIN pg_namespace nsp ON table_schema = nsp.nspname WHERE nsp.nspname NOT IN ('information_schema', 'pg_toast', 'pg_catalog')` +RIGHT JOIN pg_namespace nsp ON table_schema = nsp.nspname WHERE nsp.nspname NOT IN ('information_schema', 'pg_toast', 'pg_catalog') +AND NOT starts_with(nsp.nspname, 'pg_') AND has_schema_privilege(nsp.oid, 'USAGE')` } } diff --git a/frontend/src/lib/components/common/confirmationModal/ConfirmationModal.svelte b/frontend/src/lib/components/common/confirmationModal/ConfirmationModal.svelte index c858f9f08f..f85d34f225 100644 --- a/frontend/src/lib/components/common/confirmationModal/ConfirmationModal.svelte +++ b/frontend/src/lib/components/common/confirmationModal/ConfirmationModal.svelte @@ -186,7 +186,7 @@ {/if} -
+

{title}

diff --git a/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts b/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts index aafb6a2ed1..85d70b1981 100644 --- a/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts +++ b/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts @@ -713,12 +713,14 @@ export class AIChatManager { /** Every mounted flow editor. */ #flowEditors = new Set() appAiChatHelpers = $state(undefined) - /** Datatable creation policy: enabled flag, datatable name, and optional schema */ + /** Datatable creation policy: enabled flag, datatable name, optional schema, and the role the + * app uses each data table through */ datatableCreationPolicy = $state<{ enabled: boolean datatable: string | undefined schema: string | undefined - }>({ enabled: false, datatable: undefined, schema: undefined }) + roles?: Record + }>({ enabled: false, datatable: undefined, schema: undefined, roles: undefined }) pendingNewCode = $state(undefined) apiTools = $state[]>([]) aiChatInput = $state(null) diff --git a/frontend/src/lib/components/copilot/chat/DatatableCreationPolicy.svelte b/frontend/src/lib/components/copilot/chat/DatatableCreationPolicy.svelte index 65abb0e1c5..43bd4f578e 100644 --- a/frontend/src/lib/components/copilot/chat/DatatableCreationPolicy.svelte +++ b/frontend/src/lib/components/copilot/chat/DatatableCreationPolicy.svelte @@ -67,6 +67,7 @@ diff --git a/frontend/src/lib/components/copilot/chat/app/core.ts b/frontend/src/lib/components/copilot/chat/app/core.ts index 5ab271cc6f..634341b966 100644 --- a/frontend/src/lib/components/copilot/chat/app/core.ts +++ b/frontend/src/lib/components/copilot/chat/app/core.ts @@ -18,6 +18,7 @@ import { type AppCodeSelectionElement, type AppDatatableElement } from '../context' +import { appDatatableRole, sdkDatatableCall } from '$lib/components/raw_apps/dataTableRefUtils' // Backend runnable types export type BackendRunnableType = 'script' | 'flow' | 'hubscript' | 'inline' @@ -921,9 +922,20 @@ export function prepareAppSystemMessage(customPrompt?: string): ChatCompletionSy const policy = aiChatManager.datatableCreationPolicy const datatableName = policy.datatable ?? 'main' const schemaPrefix = policy.schema ? `${policy.schema}.` : '' - // Use wmill.datatable() for 'main' (default), otherwise wmill.datatable('name') - const datatableCall = - datatableName === 'main' ? 'wmill.datatable()' : `wmill.datatable('${datatableName}')` + // A role names the privileges the app's queries run with, so it has to be in the code the + // model writes. + const datatableRole = appDatatableRole(policy.roles, datatableName) + const tsDatatableCall = sdkDatatableCall(datatableName, datatableRole, 'typescript') + const pyDatatableCall = sdkDatatableCall(datatableName, datatableRole, 'python') + const roleEntries = Object.entries(policy.roles ?? {}) + const rolesNote = + roleEntries.length > 0 + ? `\n\nThis app uses these data tables through a role: ${roleEntries + .map(([dt, role]) => `\`${dt}\` as \`${role}\``) + .join( + ', ' + )}. Always pass that role when calling \`wmill.datatable\` on them, as in the examples. The role only reaches what it was granted, so a query on a table it lacks privileges on fails with \`permission denied\`.` + : '' let content = `You are a helpful assistant that creates and edits apps on the Windmill platform. Apps are defined as a collection of files that contains both the frontend and the backend. @@ -1024,7 +1036,7 @@ Backend runnables should only perform **data operations** (SELECT, INSERT, UPDAT import * as wmill from 'windmill-client'; export async function main(user_id: string) { - const sql = ${datatableCall}; + const sql = ${tsDatatableCall}; const user = await sql\`SELECT * FROM ${schemaPrefix}users WHERE id = \${user_id}\`.fetchOne(); return user; } @@ -1035,12 +1047,12 @@ export async function main(user_id: string) { import wmill def main(user_id: str): - db = ${datatableCall} + db = ${pyDatatableCall} user = db.query('SELECT * FROM ${schemaPrefix}users WHERE id = $1', user_id).fetch_one() return user \`\`\` -Use these examples for normal datatable access. +Use these examples for normal datatable access.${rolesNote} ### Schema Modifications (DDL) - Use exec_datatable_sql tool ONLY diff --git a/frontend/src/lib/components/copilot/chat/datatableTools.ts b/frontend/src/lib/components/copilot/chat/datatableTools.ts index 7232500898..bb7976d396 100644 --- a/frontend/src/lib/components/copilot/chat/datatableTools.ts +++ b/frontend/src/lib/components/copilot/chat/datatableTools.ts @@ -2,6 +2,7 @@ import { z } from 'zod' import { WorkspaceService, type CompletedJob } from '$lib/gen' import type { DataTableTables } from '$lib/gen/types.gen' import { runScript } from '$lib/components/jobs/utils' +import { datatableReference } from '$lib/components/dbTypes' import { createToolDef, executeTestRun, @@ -15,9 +16,9 @@ import { * * Datatables are workspace-level managed PostgreSQL databases. The backend * endpoints used here (`list_datatable_tables`, `get_datatable_table_schema`) - * and SQL execution (`datatable://`) are gated only by workspace - * membership, so these tools need no app context and operate directly on the - * workspace. This is the unrestricted counterpart to the app-mode datatable + * and SQL execution (`datatable://`) need no app context: the server + * decides what the caller reaches, as the datatable role they name or its + * default. This is the unrestricted counterpart to the app-mode datatable * tools in `app/core.ts`, which additionally filter by the app's whitelist. */ @@ -31,9 +32,19 @@ const memo = (factory: () => T): (() => T) => { // ============= Pure workspace-scoped operations ============= -/** List all datatables configured in the workspace, with their schema/table names. */ -export async function listDatatables(workspace: string): Promise { - return await WorkspaceService.listDataTableTables({ workspace }) +/** List the datatables configured in the workspace, with their schema/table names: all of them as + * their default role, or only `datatableName`, as `role` when one is given. */ +export async function listDatatables( + workspace: string, + datatableName?: string, + role?: string +): Promise { + if (datatableName === undefined) return await WorkspaceService.listDataTableTables({ workspace }) + return await WorkspaceService.listDataTableTables({ + workspace, + datatableName, + ...(role !== undefined && { roleFor: datatableName, role }) + }) } /** Get the columns (column_name -> compact_type) of one datatable table. */ @@ -41,13 +52,15 @@ export async function getDatatableColumns( workspace: string, datatableName: string, schemaName: string, - tableName: string + tableName: string, + role?: string ): Promise> { const schema = await WorkspaceService.getDataTableTableSchema({ workspace, datatableName, schemaName, - tableName + tableName, + role }) return schema.columns } @@ -81,7 +94,26 @@ const NO_DATATABLES_CONFIGURED_MESSAGE = // ============= Tool definitions ============= -const getListDatatablesSchema = memo(() => z.object({})) +// The same rule the server applies to `-- role `; a name it would refuse fails here instead. +const getRoleSchema = memo(() => + z + .string() + .regex(/^[A-Za-z0-9_-]{1,63}$/) + .optional() + .describe( + "The datatable role to connect as, when the code you are working on uses one (an app's `data.roles` entry, or the `role` it passes to wmill.datatable). Omit for the datatable's default role." + ) +) + +const getListDatatablesSchema = memo(() => + z.object({ + datatable_name: z + .string() + .optional() + .describe('List only this datatable. Required with `role`.'), + role: getRoleSchema() + }) +) const getListDatatablesToolDef = memo(() => createToolDef( getListDatatablesSchema(), @@ -94,7 +126,8 @@ const getGetDatatableTableSchemaSchema = memo(() => z.object({ datatable_name: z.string().describe('The datatable name to inspect, e.g. "main".'), schema_name: z.string().describe('The schema name, e.g. "public".'), - table_name: z.string().describe('The table name to inspect.') + table_name: z.string().describe('The table name to inspect.'), + role: getRoleSchema() }) ) const getGetDatatableTableSchemaToolDef = memo(() => @@ -117,6 +150,7 @@ const getExecDatatableSqlSchema = memo(() => .describe( 'The SQL query to execute. Supports SELECT, INSERT, UPDATE, DELETE, CREATE TABLE, ALTER TABLE, DROP TABLE, etc. For SELECT queries, results are returned as an array of objects. A newly created table will appear in list_datatables automatically.' ), + role: getRoleSchema(), background: z .boolean() .optional() @@ -217,10 +251,18 @@ export function getDatatableTools(): Tool<{}>[] { { def: getListDatatablesToolDef(), planModeSafe: true, - fn: async ({ workspace, toolId, toolCallbacks }) => { + fn: async ({ args, workspace, toolId, toolCallbacks }) => { toolCallbacks.setToolStatus(toolId, { content: 'Listing datatables...' }) try { - const metadata = await listDatatables(workspace) + const parsedArgs = getListDatatablesSchema().parse(args ?? {}) + if (parsedArgs.role !== undefined && parsedArgs.datatable_name === undefined) { + throw new Error('`role` needs `datatable_name`, the datatable it is a role of') + } + const metadata = await listDatatables( + workspace, + parsedArgs.datatable_name, + parsedArgs.role + ) if (metadata.length === 0) { toolCallbacks.setToolStatus(toolId, { content: 'No datatables configured — set one up in workspace settings' @@ -236,7 +278,18 @@ export function getDatatableTools(): Tool<{}>[] { toolCallbacks.setToolStatus(toolId, { content: `Listed ${metadata.length} datatable(s) with ${totalTables} table(s)` }) - return JSON.stringify(metadata, null, 2) + // Only what the model acts on: the roles it may pass, not the creation privileges + // the manager's UI gates on. + return JSON.stringify( + metadata.map((d) => ({ + datatable_name: d.datatable_name, + schemas: d.schemas, + ...(d.error && { error: d.error }), + ...(d.permissioned && { usable_roles: d.usable_roles, default_role: d.default_role }) + })), + null, + 2 + ) } catch (e) { const errorMsg = `Error listing datatables: ${e instanceof Error ? e.message : String(e)}` toolCallbacks.setToolStatus(toolId, { content: errorMsg, error: errorMsg }) @@ -257,7 +310,8 @@ export function getDatatableTools(): Tool<{}>[] { workspace, parsedArgs.datatable_name, parsedArgs.schema_name, - parsedArgs.table_name + parsedArgs.table_name, + parsedArgs.role ) toolCallbacks.setToolStatus(toolId, { content: `Retrieved schema for ${parsedArgs.schema_name}.${parsedArgs.table_name}` @@ -300,7 +354,7 @@ export function getDatatableTools(): Tool<{}>[] { requestBody: { language: 'postgresql', content: parsedArgs.sql, - args: { database: `datatable://${name}` } + args: { database: datatableReference(name, parsedArgs.role) } } }), workspace, diff --git a/frontend/src/lib/components/copilot/chat/global/core.ts b/frontend/src/lib/components/copilot/chat/global/core.ts index 52f4ee39ef..d34085f317 100644 --- a/frontend/src/lib/components/copilot/chat/global/core.ts +++ b/frontend/src/lib/components/copilot/chat/global/core.ts @@ -1411,7 +1411,12 @@ Data Tables: - Datatables are workspace-scoped managed PostgreSQL databases, shared across the workspace (not owned by any single app). They must be configured by the user in their workspace settings (Workspace settings → Data Tables); they cannot be created via SQL. - Use list_datatables to discover the available datatables and their tables. Reuse an existing table rather than creating a duplicate. If list_datatables reports none, this is a blocking prerequisite — tell the user to set up a datatable in their workspace settings and stop; do not assume a "main" datatable exists or call exec_datatable_sql. - Use get_datatable_table_schema only when you need a table's column names/types; list_datatables is enough for table-list or availability summaries. -- Use exec_datatable_sql to explore data, run queries, mutate rows, or change schema (CREATE/ALTER/DROP). Creating a table is a normal CREATE TABLE statement — it appears in list_datatables afterward, with no registration step. +- Use exec_datatable_sql to explore data, run queries, mutate rows, or change schema (CREATE/ALTER/DROP). Creating a table is a normal CREATE TABLE statement — it appears in list_datatables afterward, with no registration step.${ + isCloudHosted() + ? '' + : ` +- A raw app may use a datatable through a role (\`data.roles\` in its raw_app.yaml). When working on such an app, pass that role to the datatable tools, and to wmill.datatable in its runnables, so you see and change only what the app itself can.` + } - When writing runnable code (inline app runnables, scripts, flow modules) that reads or writes datatable data at runtime, it accesses a datatable via wmill.datatable(). Default to TypeScript (bun) unless the user asked for another language. Call get_instructions with subject "datatable" and language "bun" for the TypeScript SQL SDK reference (or language "python3" for Python) — it returns only that language so you get just what you need.${ skills.length > 0 ? ` diff --git a/frontend/src/lib/components/datatableAcl/AclTargetPicker.svelte b/frontend/src/lib/components/datatableAcl/AclTargetPicker.svelte deleted file mode 100644 index cbfb43ad39..0000000000 --- a/frontend/src/lib/components/datatableAcl/AclTargetPicker.svelte +++ /dev/null @@ -1,50 +0,0 @@ - - -
- ({ value: t, label: t }))} - bind:value={table} - placeholder="The whole schema" - clearable - size="sm" - class="w-56" - /> - {/if} -
diff --git a/frontend/src/lib/components/datatableAcl/PgAclEditor.svelte b/frontend/src/lib/components/datatableAcl/PgAclEditor.svelte index e0ae596a3d..801ec13ffb 100644 --- a/frontend/src/lib/components/datatableAcl/PgAclEditor.svelte +++ b/frontend/src/lib/components/datatableAcl/PgAclEditor.svelte @@ -25,30 +25,24 @@ let { workspace, datatable, - target, - onLoaded + target }: { workspace: string datatable: string /** What owner and grants are read and written for. */ target: AclTarget - /** Each read, with the target it was made for: it also lists what the target holds. */ - onLoaded?: (target: AclTarget, info: DatatableAclInfo) => void } = $props() const acl = resource( () => [workspace, datatable, target] as const, - async ([ws, dt, t]) => { - const loaded = await WorkspaceService.getDatatableAcl({ + async ([ws, dt, t]) => + await WorkspaceService.getDatatableAcl({ workspace: ws, datatableName: dt, kind: t.kind, schema: t.kind === 'database' ? undefined : t.schema, table: t.kind === 'table' ? t.table : undefined }) - onLoaded?.(t, loaded) - return loaded - } ) // Nothing is written before its SQL has been shown, and the apply runs exactly that SQL: the diff --git a/frontend/src/lib/components/datatableMigrationRole.test.ts b/frontend/src/lib/components/datatableMigrationRole.test.ts new file mode 100644 index 0000000000..6d136b8371 --- /dev/null +++ b/frontend/src/lib/components/datatableMigrationRole.test.ts @@ -0,0 +1,60 @@ +import { describe, test, expect } from 'vitest' +import { parseMigrationRole, withMigrationRole } from './datatableMigrationRole' + +describe('parseMigrationRole', () => { + test('reads every spelling the server accepts from the leading comment block', () => { + for (const line of [ + '-- role analyst', + '-- Role: analyst', + '-- role=analyst', + '-- role analyst;' + ]) { + expect(parseMigrationRole(`\n${line}\nBEGIN;\nEND;`)).toEqual({ + kind: 'role', + role: 'analyst' + }) + } + }) + + test('an annotation below BEGIN is not one', () => { + expect(parseMigrationRole('BEGIN;\n-- role analyst\nEND;')).toEqual({ kind: 'none' }) + }) + + test('a malformed attempt is an error, not the default', () => { + for (const line of [ + '-- role based access below', + '-- role', + '-- role:', + '-- role an;alytics' + ]) { + expect(parseMigrationRole(`${line}\nBEGIN;`)).toEqual({ kind: 'malformed', line }) + } + }) + + test('comments that do not start with the word role are ignored', () => { + expect(parseMigrationRole('-- roles analyst\n-- rolex\nBEGIN;')).toEqual({ kind: 'none' }) + }) +}) + +describe('withMigrationRole', () => { + test('leads above BEGIN, so the server reads it', () => { + const out = withMigrationRole('BEGIN;\nSELECT 1;\nEND;', 'analyst') + expect(out).toBe('-- role analyst\nBEGIN;\nSELECT 1;\nEND;') + }) + + test('replaces any attempt rather than stacking, malformed ones included', () => { + const out = withMigrationRole( + '-- Role: auditor\n-- role oops no\n-- keep me\nBEGIN;', + 'analyst' + ) + expect(out).toBe('-- role analyst\n-- keep me\nBEGIN;') + }) + + test('undefined strips the annotation, so it runs as admin', () => { + expect(withMigrationRole('-- role analyst\n\nBEGIN;\nEND;', undefined)).toBe('BEGIN;\nEND;') + }) + + test('refuses a name the server would refuse', () => { + expect(() => withMigrationRole('BEGIN;', 'bad;name')).toThrow() + }) +}) diff --git a/frontend/src/lib/components/datatableMigrationRole.ts b/frontend/src/lib/components/datatableMigrationRole.ts new file mode 100644 index 0000000000..5475bf2fd7 --- /dev/null +++ b/frontend/src/lib/components/datatableMigrationRole.ts @@ -0,0 +1,70 @@ +import { isDatatableRoleName } from './dbTypes' + +/** + * A migration carries the data table role it runs as in its own SQL, as a `-- role ` + * annotation. There is no separate field: the annotation is what the server reads, and keeping + * it in the SQL is what lets it survive a `wmill sync` round-trip. + * + * Mirrors `SqlAnnotations::datatable_role` on the backend. It is only read from the leading + * comment block, so an annotation below `BEGIN;` is ignored and the migration runs as admin. A + * leading comment whose first word is `role` is an annotation attempt, and a malformed one is an + * error there, so it is one here too. + */ + +export type MigrationRole = + | { kind: 'none' } + | { kind: 'role'; role: string } + | { kind: 'malformed'; line: string } + +/** The body of a leading comment line that attempts a role annotation, or undefined. */ +function roleAttempt(line: string): string | undefined { + if (!line.startsWith('--')) return undefined + const body = line.slice(2).trimStart() + if (body.slice(0, 4).toLowerCase() !== 'role') return undefined + const after = body.slice(4) + if (after !== '' && !/^[\s:=]/.test(after)) return undefined + return after +} + +function parseAttempt(after: string): string | undefined { + let rest = after.trimStart() + if (rest.startsWith(':') || rest.startsWith('=')) rest = rest.slice(1) + const tokens = rest.split(/\s+/).filter((t) => t !== '') + if (tokens.length !== 1) return undefined + const role = tokens[0].endsWith(';') ? tokens[0].slice(0, -1) : tokens[0] + return isDatatableRoleName(role) ? role : undefined +} + +export function parseMigrationRole(sql: string): MigrationRole { + for (const raw of sql.split('\n')) { + const line = raw.trim() + if (line === '') continue + if (!line.startsWith('--')) break + const after = roleAttempt(line) + if (after === undefined) continue + const role = parseAttempt(after) + return role === undefined ? { kind: 'malformed', line } : { kind: 'role', role } + } + return { kind: 'none' } +} + +/** + * `sql` declaring `role`: any role annotation attempt in the leading comment block is removed, + * and `-- role ` is prepended above everything, or nothing when `role` is undefined. + */ +export function withMigrationRole(sql: string, role: string | undefined): string { + if (role !== undefined && !isDatatableRoleName(role)) { + throw new Error(`Invalid data table role '${role}'`) + } + const lines = sql.split('\n') + const kept: string[] = [] + let i = 0 + for (; i < lines.length; i++) { + const line = lines[i].trim() + if (line !== '' && !line.startsWith('--')) break + if (roleAttempt(line) === undefined) kept.push(lines[i]) + } + const rest = [...kept, ...lines.slice(i)] + while (rest.length > 0 && rest[0].trim() === '') rest.shift() + return role === undefined ? rest.join('\n') : [`-- role ${role}`, ...rest].join('\n') +} diff --git a/frontend/src/lib/components/datatableUsableRoles.ts b/frontend/src/lib/components/datatableUsableRoles.ts new file mode 100644 index 0000000000..0f53a540af --- /dev/null +++ b/frontend/src/lib/components/datatableUsableRoles.ts @@ -0,0 +1,34 @@ +import { WorkspaceService, type ListUsableDatatableRolesResponse } from '$lib/gen' +import { isCloudHosted } from '$lib/cloud' +import { ADMIN_DATATABLE_ROLE } from './dbTypes' + +// `datatable_roles_unavailable` on the server, which is a plain 400: rewording it there without +// here makes every role picker on a non-Enterprise build fail instead of reading "not under roles". +const ROLES_UNAVAILABLE = 'Data table roles are a Windmill Enterprise Edition feature' + +const NOT_UNDER_ROLES: ListUsableDatatableRolesResponse = { + permissioned: false, + roles: [], + default_role: ADMIN_DATATABLE_ROLE +} + +/** + * The roles the caller may connect as on a data table. Cloud has no instance database, so no data + * table there is under roles, and none of the role pickers show. Without the Enterprise Edition + * every roles route refuses, which reads the same way: the data table is then used the way it was + * before roles, and one that is under roles is refused when something connects to it. + */ +export async function listUsableDatatableRoles( + workspace: string, + datatableName: string +): Promise { + if (isCloudHosted()) return NOT_UNDER_ROLES + try { + return await WorkspaceService.listUsableDatatableRoles({ workspace, datatableName }) + } catch (e) { + const body = (e as { body?: unknown })?.body + const detail = `${typeof body === 'string' ? body : JSON.stringify(body ?? '')} ${(e as Error)?.message ?? e}` + if (detail.includes(ROLES_UNAVAILABLE)) return NOT_UNDER_ROLES + throw e + } +} diff --git a/frontend/src/lib/components/dbManagerDrawerModel.svelte.ts b/frontend/src/lib/components/dbManagerDrawerModel.svelte.ts index 59345c6ae7..890635dcf8 100644 --- a/frontend/src/lib/components/dbManagerDrawerModel.svelte.ts +++ b/frontend/src/lib/components/dbManagerDrawerModel.svelte.ts @@ -5,7 +5,7 @@ import { isDbType } from './dbTypes' /** * Single URL param `dbm` encodes the full DB manager state: - * firstSegment~path~schema.table + * firstSegment~path~schema.table~role=name * * firstSegment: * datatable – database with datatable:// resource (resourceType always postgresql) @@ -26,28 +26,46 @@ import { isDbType } from './dbTypes' * datatable~main~.customers (schema "public" implied) * ducklake~main~.orders (schema "main" implied) * postgresql~$res:u/user/my_pg~public.customers + * datatable~main~.customers~role=analyst + * datatable~main~role=analyst (no schema/table selected) + * + * role=name (last segment, optional, data tables only): the data table role to connect as. + * Omitted means the data table's default role. A trailing segment starting with `role=` is always + * the role, whatever follows. The name is kept as written, even when invalid (a `.` included), so + * the connection refuses it visibly instead of falling back to the default. */ const dbManagerSchema = z.object({ dbm: z.string().nullable() }) -interface ParsedDbm { +export interface ParsedDbm { type: 'database' | 'datatable' | 'ducklake' path: string resType?: string schema?: string table?: string + role?: string } -function parseDbm(raw: unknown): ParsedDbm | null { +const ROLE_SEGMENT_PREFIX = 'role=' + +function isRoleSegment(segment: string | undefined): segment is string { + return !!segment && segment.startsWith(ROLE_SEGMENT_PREFIX) +} + +export function parseDbm(raw: unknown): ParsedDbm | null { if (!raw || typeof raw !== 'string') return null const parts = raw.split('~') if (parts.length < 2 || !parts[1]) return null const firstSeg = parts[0] const path = parts[1] - const schemaTable = parts[2] ?? '' + const rest = parts.slice(2) + const role = isRoleSegment(rest.at(-1)) + ? rest.pop()!.slice(ROLE_SEGMENT_PREFIX.length) + : undefined + const schemaTable = rest[0] ?? '' let type: ParsedDbm['type'] let resType: string | undefined @@ -81,12 +99,12 @@ function parseDbm(raw: unknown): ParsedDbm | null { schema = defaultSchemas[type] } - return { type, path, resType, schema, table } + return { type, path, resType, schema, table, role: type === 'datatable' ? role : undefined } } const defaultSchemas: Record = { datatable: 'public', ducklake: 'main' } -function buildDbm(p: ParsedDbm): string { +export function buildDbm(p: ParsedDbm): string { const firstSeg = p.type === 'database' ? p.resType! : p.type const schema = p.schema === defaultSchemas[p.type] ? undefined : p.schema let schemaTable = '' @@ -97,7 +115,12 @@ function buildDbm(p: ParsedDbm): string { } else if (schema) { schemaTable = `${schema}.` } - return schemaTable ? `${firstSeg}~${p.path}~${schemaTable}` : `${firstSeg}~${p.path}` + const segments = [firstSeg, p.path] + if (schemaTable) segments.push(schemaTable) + if (p.type === 'datatable' && p.role !== undefined) { + segments.push(`${ROLE_SEGMENT_PREFIX}${p.role}`) + } + return segments.join('~') } export interface DbManagerUriState { @@ -105,6 +128,8 @@ export interface DbManagerUriState { readonly effectiveInput: DbInput | undefined readonly isDatatableInput: boolean selectedDatatable: string | undefined + /** The data table role the drawer connects as; undefined means its default. */ + selectedRole: string | undefined selectedSchema: string | undefined selectedTable: string | undefined readonly open: boolean @@ -137,6 +162,7 @@ export function useDbManagerUriState(): DbManagerUriState { type: 'database' as const, resourceType: resType as DbType, resourcePath: parsed.type === 'datatable' ? `datatable://${parsed.path}` : parsed.path, + role: parsed.role, specificSchema: parsed.schema, specificTable: parsed.table } @@ -163,6 +189,7 @@ export function useDbManagerUriState(): DbManagerUriState { type: isDatatable ? 'datatable' : 'database', path: isDatatable ? nInput.resourcePath.slice('datatable://'.length) : nInput.resourcePath, resType: isDatatable ? undefined : nInput.resourceType, + role: isDatatable ? nInput.role : undefined, schema: nInput.specificSchema, table: nInput.specificTable }) @@ -194,7 +221,14 @@ export function useDbManagerUriState(): DbManagerUriState { return parsed?.type === 'datatable' ? parsed.path : undefined }, set selectedDatatable(v: string | undefined) { - if (v) updateField({ path: v }) + // A role belongs to one data table, so it cannot carry over to another. + if (v) updateField({ path: v, role: undefined }) + }, + get selectedRole() { + return parsed?.role + }, + set selectedRole(v: string | undefined) { + updateField({ role: v }) }, get selectedSchema() { return parsed?.schema diff --git a/frontend/src/lib/components/dbManagerRole.test.ts b/frontend/src/lib/components/dbManagerRole.test.ts new file mode 100644 index 0000000000..305ac18edd --- /dev/null +++ b/frontend/src/lib/components/dbManagerRole.test.ts @@ -0,0 +1,60 @@ +import { describe, expect, it } from 'vitest' +import { buildDbm, parseDbm } from './dbManagerDrawerModel.svelte' +import { schemaCacheKey } from './dbSchemaCache' +import { datatableReference, type DbInput } from './dbTypes' + +describe('dbm role segment', () => { + it('round-trips a role, with and without a table', () => { + for (const dbm of ['datatable~main~.orders~role=p4_analytics', 'datatable~main~role=p4-op']) { + expect(buildDbm(parseDbm(dbm)!)).toBe(dbm) + } + expect(parseDbm('datatable~main~sales.orders~role=analyst')).toMatchObject({ + path: 'main', + schema: 'sales', + table: 'orders', + role: 'analyst' + }) + }) + + it('reads a link without a role as the default role', () => { + const parsed = parseDbm('datatable~main~.orders')! + expect(parsed.role).toBeUndefined() + expect(parsed).toMatchObject({ schema: 'public', table: 'orders' }) + expect(buildDbm(parsed)).toBe('datatable~main~.orders') + }) + + it('keeps an invalid role as written, so the connection refuses it', () => { + expect(parseDbm('datatable~main~role=a;b')?.role).toBe('a;b') + // A dot does not turn it into a schema.table selection read as the default role. + expect(parseDbm('datatable~main~role=bad.name')).toMatchObject({ + role: 'bad.name', + schema: undefined, + table: undefined + }) + }) +}) + +describe('connecting as a role', () => { + const input = (role?: string): DbInput => ({ + type: 'database', + resourceType: 'postgresql', + resourcePath: 'datatable://main', + role + }) + + // What `getDatabaseArg` builds every DB manager connection from. + it('appends the role to the data table reference', () => { + expect(datatableReference('main', 'p4_analytics')).toBe('datatable://main?role=p4_analytics') + expect(datatableReference('main', undefined)).toBe('datatable://main') + }) + + it('refuses a role name the server would not accept', () => { + expect(() => datatableReference('main', 'a&role=admin')).toThrow(/Invalid data table role/) + expect(() => datatableReference('main', '')).toThrow(/Invalid data table role/) + }) + + it('keys the schema cache by role', () => { + expect(schemaCacheKey('ws', input('a'))).not.toBe(schemaCacheKey('ws', input('b'))) + expect(schemaCacheKey('ws', input('a'))).not.toBe(schemaCacheKey('ws', input())) + }) +}) diff --git a/frontend/src/lib/components/dbOps.ts b/frontend/src/lib/components/dbOps.ts index 5005cc931c..02c26f034f 100644 --- a/frontend/src/lib/components/dbOps.ts +++ b/frontend/src/lib/components/dbOps.ts @@ -8,7 +8,8 @@ import { runScriptAndPollResult } from './jobs/utils' import { writingJobOptions } from './jobs/writingJob' import type { DBSchema, SQLSchema } from '$lib/stores' import { stringifySchema } from './copilot/lib' -import type { DbInput, DbType } from './dbTypes' +import { datatableReference, type DbInput, type DbType } from './dbTypes' +import { withMigrationRole } from './datatableMigrationRole' import { assert } from '$lib/utils' import { WorkspaceService } from '$lib/gen' import { pendingMigrations } from './workspaceSettings/datatableMigrationUtils' @@ -70,7 +71,9 @@ export function dbTableOpsWithPreviewScripts({ }): IDbTableOps { const dbType = getDbType(input) const language = getLanguageByResourceType(dbType) - const dbArg = getDatabaseArg(input) + // Built per call: an invalid role throws there, as that operation's error, rather than while + // the manager renders. + const dbArg = () => getDatabaseArg(input) const ducklake = input.type === 'ducklake' ? input.ducklake : undefined function makeMarker(op: string, payload: Record): string { @@ -91,7 +94,7 @@ export function dbTableOpsWithPreviewScripts({ }) const result = await runScriptAndPollResult({ workspace, - requestBody: { args: { ...dbArg, quicksearch }, language, content, tag } + requestBody: { args: { ...dbArg(), quicksearch }, language, content, tag } }) const count = result?.[0].count as number return count @@ -106,7 +109,7 @@ export function dbTableOpsWithPreviewScripts({ }) let items = (await runScriptAndPollResult({ workspace, - requestBody: { args: { ...dbArg, ...params }, language, content, tag } + requestBody: { args: { ...dbArg(), ...params }, language, content, tag } })) as unknown[] if (!items || !Array.isArray(items)) { throw 'items is not an array' @@ -123,7 +126,7 @@ export function dbTableOpsWithPreviewScripts({ { workspace, requestBody: { - args: { ...dbArg, value_to_update: newValue, ...values }, + args: { ...dbArg(), value_to_update: newValue, ...values }, language, content, tag @@ -135,14 +138,14 @@ export function dbTableOpsWithPreviewScripts({ onDelete: async ({ values }) => { const content = makeMarker('DELETE', { table: tableKey, columns: colDefs }) await runScriptAndPollResult( - { workspace, requestBody: { args: { ...dbArg, ...values }, language, content, tag } }, + { workspace, requestBody: { args: { ...dbArg(), ...values }, language, content, tag } }, writingJobOptions ) }, onInsert: async ({ values }) => { const content = makeMarker('INSERT', { table: tableKey, columns: colDefs }) await runScriptAndPollResult( - { workspace, requestBody: { args: { ...dbArg, ...values }, language, content, tag } }, + { workspace, requestBody: { args: { ...dbArg(), ...values }, language, content, tag } }, writingJobOptions ) } @@ -246,6 +249,7 @@ export type IDbSchemaOps = { previewAlterSql: (params: { values: AlterTableValues; schema?: string }) => Promise onCreateSchema: (params: { schema: string }) => Promise onDeleteSchema: (params: { schema: string }) => Promise + onRenameSchema: (params: { schema: string; newSchema: string }) => Promise onFetchTableEditorDefinition: (params: { table: string schema?: string @@ -283,7 +287,8 @@ export function dbSchemaOpsWithPreviewScripts({ tag?: string }): IDbSchemaOps { const dbType = getDbType(input) - const dbArg = getDatabaseArg(input) + // Built per call, for the same reason as in the table ops above. + const dbArg = () => getDatabaseArg(input) const language = getLanguageByResourceType(dbType) const ducklake = input.type === 'ducklake' ? input.ducklake : undefined @@ -293,6 +298,8 @@ export function dbSchemaOpsWithPreviewScripts({ input.type === 'database' && input.resourcePath.startsWith('datatable://') ? input.resourcePath.slice('datatable://'.length) : undefined + // A migration declaring no role runs as admin, whatever role the manager connects as. + const migrationRole = input.type === 'database' ? input.role : undefined function makeMarker(op: string, payload: Record): string { if (ducklake) payload.ducklake = ducklake @@ -359,7 +366,7 @@ export function dbSchemaOpsWithPreviewScripts({ : undefined if (!datatableName || !status?.enabled) { await runScriptAndPollResult( - { workspace, requestBody: { args: dbArg, content, language, tag } }, + { workspace, requestBody: { args: dbArg(), content, language, tag } }, writingJobOptions ) return @@ -373,12 +380,16 @@ export function dbSchemaOpsWithPreviewScripts({ throw new MigrationRunCancelled() } } - const codeUp = wrapMigration(await expandMarker(workspace, language, content)) + // Wrapped before annotating: the annotation must lead, above `BEGIN;`. + const codeUp = withMigrationRole( + wrapMigration(await expandMarker(workspace, language, content)), + migrationRole + ) // Down migrations are only generated for Postgres for now. let codeDown: string | undefined if (downContent && dbType === 'postgresql') { const downSql = (await expandMarker(workspace, language, downContent)).trim() - if (downSql) codeDown = wrapMigration(downSql) + if (downSql) codeDown = withMigrationRole(wrapMigration(downSql), migrationRole) } const created = await WorkspaceService.createDatatableMigration({ workspace, @@ -415,7 +426,7 @@ export function dbSchemaOpsWithPreviewScripts({ const fkContent = makeMarker('FOREIGN_KEYS', { table, schema }) const fkResult = await runScriptAndPollResult({ workspace, - requestBody: { args: dbArg, content: fkContent, language, tag } + requestBody: { args: dbArg(), content: fkContent, language, tag } }) let rawForeignKeys: RawForeignKey[] @@ -501,6 +512,11 @@ export function dbSchemaOpsWithPreviewScripts({ const downContent = makeMarker('CREATE_SCHEMA', { schema }) await applyDdl(migrationName('drop_schema', schema), content, downContent) }, + onRenameSchema: async ({ schema, newSchema }) => { + const content = makeMarker('RENAME_SCHEMA', { schema, new_schema: newSchema }) + const downContent = makeMarker('RENAME_SCHEMA', { schema: newSchema, new_schema: schema }) + await applyDdl(migrationName('rename_schema', schema), content, downContent) + }, onFetchForeignKeys: fetchForeignKeys, onFetchTableEditorDefinition: async ({ table, schema, colDefs }) => { const foreignKeys = await fetchForeignKeys({ table, schema }) @@ -512,7 +528,7 @@ export function dbSchemaOpsWithPreviewScripts({ const pkContent = makeMarker('PRIMARY_KEY_CONSTRAINT', { table, schema }) const pkResult = (await runScriptAndPollResult({ workspace, - requestBody: { args: dbArg, content: pkContent, language, tag } + requestBody: { args: dbArg(), content: pkContent, language, tag } })) as { constraint_name?: string; CONSTRAINT_NAME?: string }[] if (pkResult && Array.isArray(pkResult) && pkResult.length > 0) { @@ -611,7 +627,9 @@ export function getDefaultDbTag(input: DbInput): string { export function getDatabaseArg(input: DbInput | undefined) { if (input?.type === 'database') { if (input.resourcePath.startsWith('datatable://')) { - return { database: input.resourcePath } + return { + database: datatableReference(input.resourcePath.slice('datatable://'.length), input.role) + } } else { return { database: '$res:' + input.resourcePath } } diff --git a/frontend/src/lib/components/dbSchemaCache.ts b/frontend/src/lib/components/dbSchemaCache.ts new file mode 100644 index 0000000000..0421c0c033 --- /dev/null +++ b/frontend/src/lib/components/dbSchemaCache.ts @@ -0,0 +1,21 @@ +import type { DbInput } from './dbTypes' + +/** What identifies a database's schema, role included: two roles on one data table may reach + * different schemas, so they cannot share a cache entry. Never throws, since it keys derived + * state; the connection itself is what refuses an invalid role. */ +export function getDbSchemasPath(input: DbInput): string { + switch (input.type) { + case 'database': + return input.role !== undefined && input.resourcePath.startsWith('datatable://') + ? `${input.resourcePath}?role=${input.role}` + : input.resourcePath + case 'ducklake': + return 'ducklake://' + input.ducklake + } +} + +/** Scoped by the acting workspace: a data table of the same name can exist in both the nav and + * the acting workspace, and one's schema must not be reused for the other. */ +export function schemaCacheKey(workspace: string | undefined, input: DbInput): string { + return `${workspace}:${getDbSchemasPath(input)}` +} diff --git a/frontend/src/lib/components/dbTypes.ts b/frontend/src/lib/components/dbTypes.ts index 6a85617110..a673a11c2e 100644 --- a/frontend/src/lib/components/dbTypes.ts +++ b/frontend/src/lib/components/dbTypes.ts @@ -3,6 +3,9 @@ export type DbInput = type: 'database' resourceType: DbType resourcePath: string + /** The data table role to connect as; the data table's default when unset. Only + * meaningful for a `datatable://` path. */ + role?: string specificSchema?: string specificTable?: string } @@ -23,3 +26,25 @@ export const dbTypes = [ 'duckdb' ] as const export const isDbType = (str?: string): str is DbType => !!str && dbTypes.includes(str as DbType) + +/** The role every data table has: the one it connects as when it is not under roles. */ +export const ADMIN_DATATABLE_ROLE = 'admin' + +/** What the server accepts in `-- role ` and `?role=`. */ +export function isDatatableRoleName(name: string): boolean { + return /^[A-Za-z0-9_-]{1,63}$/.test(name) +} + +/** `datatable://`, with `?role=` when a role is named. Throws rather than build a + * reference the executor would refuse, or one that would silently mean another role. */ +export function datatableReference(name: string, role: string | undefined): string { + if (role === undefined) return `datatable://${name}` + if (!isDatatableRoleName(role)) { + throw new Error( + `Invalid data table role '${role}': only letters, digits, '_' and '-' are allowed` + ) + } + return `datatable://${name}?role=${role}` +} + +export type DatatableRowAction = 'migrations' | 'roles' | 'export' | 'import' diff --git a/frontend/src/lib/components/raw_apps/DefaultDatabaseSelector.svelte b/frontend/src/lib/components/raw_apps/DefaultDatabaseSelector.svelte index 66e3ab0355..f5323955dc 100644 --- a/frontend/src/lib/components/raw_apps/DefaultDatabaseSelector.svelte +++ b/frontend/src/lib/components/raw_apps/DefaultDatabaseSelector.svelte @@ -4,13 +4,14 @@ import Select from '$lib/components/select/Select.svelte' import { workspaceStore } from '$lib/stores' import { + createDatatableAccessResource, createDatatablesResource, - createSchemasResource, toDatatableItems, toSchemaItems } from './datatableUtils.svelte' import { Button } from '../common' import { getRawAppOperatingWorkspace } from './rawAppWorkspace' + import { appDatatableRole } from './dataTableRefUtils' const getOpWs = getRawAppOperatingWorkspace() let opWs = $derived(getOpWs?.() ?? $workspaceStore) @@ -20,6 +21,8 @@ datatable: string | undefined /** Currently selected schema */ schema: string | undefined + /** The role the app uses each data table through: schemas are listed as that role. */ + roles?: Record /** Callback when either value changes */ onChange?: (datatable: string | undefined, schema: string | undefined) => void /** Description text to show in the popover */ @@ -29,19 +32,28 @@ let { datatable, schema, + roles, onChange, description = 'Set the default datatable and schema for new tables. This is where AI will create new tables when needed.' }: Props = $props() + const role = $derived(datatable ? appDatatableRole(roles, datatable) : undefined) + // Load available datatables and schemas using shared utilities const datatables = createDatatablesResource(() => opWs) - const schemas = createSchemasResource( + const access = createDatatableAccessResource( () => datatable, + () => role, () => opWs ) const datatableItems = $derived(toDatatableItems(datatables.current)) - const schemaItems = $derived(toSchemaItems(schemas.current)) + // Until the answer is for this data table and role, the schemas in hand belong to another. + const schemaItems = $derived( + access.current.datatable === datatable && access.current.role === role + ? toSchemaItems(access.current.schemas) + : [] + ) // Track datatable changes to reset schema let previousDatatable = $state(undefined) @@ -82,6 +94,9 @@ placeholder="Select database" size="sm" /> + {#if role} + Used as role {role} + {/if}
diff --git a/frontend/src/lib/components/raw_apps/RawAppDataTableDrawer.svelte b/frontend/src/lib/components/raw_apps/RawAppDataTableDrawer.svelte index 5f0488638d..f267ab7a78 100644 --- a/frontend/src/lib/components/raw_apps/RawAppDataTableDrawer.svelte +++ b/frontend/src/lib/components/raw_apps/RawAppDataTableDrawer.svelte @@ -1,35 +1,46 @@ - - {/if} -
- {#if editable && row.id !== ADMIN_ROLE} - removeRole(row.id)} /> - {/if} -
- -
- {/each} - - - {#if editable && unusedRoles.length > 0} -
- -
+ + Role + + admin is the connection the data table used before roles, so it owns every + existing object and cannot be removed. Every other role is a login defined for + the whole instance, with only the privileges granted to it under Access. + + + + Tenants + + Users, groups and folders allowed to connect as this role. Workspace admins can + use every role. + + + + Default + + The role a job gets when it names none — no `-- role` annotation, no `?role=` in + the reference. Callers still have to be one of its tenants. + + + + + + + {#each roles as role (roleKey(role))} + {@const isAdmin = role.id === ADMIN_DATATABLE_ROLE} + + +
+ {role.name ?? role.id} + {#if !role.name} + + no longer defined on this instance + + {:else if role.id === undefined} + + {#if $superadmin} +
+ Create it on the instance to use it here. + +
+ {:else} + Only a superadmin can create it on the instance. + {/if} + + {/if} +
+ + + item.group} + disabled={!editable} + placeholder="Nobody — add users, groups or folders" + /> + + +
+ { + if (role.id !== undefined) defaultRoleId = role.id + }} + /> +
+ + + {#if editable && !isAdmin} + removeRole(role)} /> + {/if} + + + {/each} + {#if editable} + + +
+
+ {/if} + + {#if info?.supported && !hasUnsavedChanges} +
+ +
+ {/if} {/if} + + {#snippet actions()} + {#if editable} + + {/if} + {/snippet} + +{#if $superadmin} + +{/if} diff --git a/frontend/src/lib/components/workspaceSettings/DataTableRolesSection.svelte b/frontend/src/lib/components/workspaceSettings/DataTableRolesSection.svelte index d100b1681a..b863dac06b 100644 --- a/frontend/src/lib/components/workspaceSettings/DataTableRolesSection.svelte +++ b/frontend/src/lib/components/workspaceSettings/DataTableRolesSection.svelte @@ -13,11 +13,22 @@ import { SettingService, type InstanceDatatableRole } from '$lib/gen' import { sendUserToast } from '$lib/toast' + let { + initialName = '', + onChanged + }: { + /** Prefills the name of the role to add. */ + initialName?: string + /** Called after every change to the catalog, whether or not it went through. */ + onChanged?: () => void + } = $props() + let roles = $state([]) let loading = $state(true) let loadError = $state(undefined) let busy = $state(false) - let newName = $state('') + // svelte-ignore state_referenced_locally + let newName = $state(initialName) /** Which role's name is being edited, and to what. */ let renaming = $state<{ id: string; name: string } | undefined>(undefined) @@ -48,6 +59,7 @@ // holds, so a failed flip has to snap back rather than sit there claiming it landed. await load() busy = false + onChanged?.() } } diff --git a/frontend/src/lib/components/workspaceSettings/DataTableSettings.svelte b/frontend/src/lib/components/workspaceSettings/DataTableSettings.svelte index 53b4819689..ebc4bd8a7f 100644 --- a/frontend/src/lib/components/workspaceSettings/DataTableSettings.svelte +++ b/frontend/src/lib/components/workspaceSettings/DataTableSettings.svelte @@ -64,12 +64,11 @@ - +{#if !hideTrigger} + +{/if} Beta {/snippet} - + {#key openCount} + + {/key} diff --git a/frontend/src/lib/components/workspaceSettings/NewDataTableMigrationModal.svelte b/frontend/src/lib/components/workspaceSettings/NewDataTableMigrationModal.svelte index d16f948b88..04cf841b28 100644 --- a/frontend/src/lib/components/workspaceSettings/NewDataTableMigrationModal.svelte +++ b/frontend/src/lib/components/workspaceSettings/NewDataTableMigrationModal.svelte @@ -8,12 +8,17 @@ import TextInput from '../text_input/TextInput.svelte' import SimpleEditor from '../SimpleEditor.svelte' import { WorkspaceService, type DatatableMigration } from '$lib/gen' + import { listUsableDatatableRoles } from '../datatableUsableRoles' import { sendUserToast } from '$lib/toast' import { tick } from 'svelte' import ConfirmationModal from '../common/confirmationModal/ConfirmationModal.svelte' import { createAsyncConfirmationModal } from '../common/confirmationModal/asyncConfirmationModal.svelte' import Portal from '$lib/components/Portal.svelte' import { fetchPendingMigrations, outOfOrderRunMessage } from './datatableMigrationUtils' + import Select from '../select/Select.svelte' + import { parseMigrationRole, withMigrationRole } from '../datatableMigrationRole' + import { ADMIN_DATATABLE_ROLE } from '../dbTypes' + import { resource } from 'runed' let { workspace, @@ -50,6 +55,8 @@ let tab = $state('up') let name = $state('') let nameInput = $state() + let upEditor = $state() + let downEditor = $state() // A valid migration name is non-empty and limited to letters, digits, '_' and '-'. const MIGRATION_NAME_RE = /^[a-zA-Z0-9_-]+$/ let nameInvalid = $derived(!MIGRATION_NAME_RE.test(name.trim())) @@ -60,6 +67,96 @@ const confirmationModal = createAsyncConfirmationModal() + // The role lives in the SQL as its `-- role ` annotation, so the code is the single + // source of truth and the Select is a view onto it: reading parses, writing rewrites the + // annotation. + const usableRoles = resource( + () => [workspace, datatable] as const, + async ([ws, dt]) => { + try { + return await listUsableDatatableRoles(ws, dt) + } catch (e) { + console.error('Failed to load data table roles:', e) + return null + } + } + ) + // Not a valid role name, so it cannot collide with one. + const NO_ROLE = '(no role)' + let declaredUp = $derived(parseMigrationRole(codeUp)) + let declaredDown = $derived(enableDown ? parseMigrationRole(codeDown) : undefined) + let malformedLine = $derived( + declaredUp.kind === 'malformed' + ? declaredUp.line + : declaredDown?.kind === 'malformed' + ? declaredDown.line + : undefined + ) + const roleOf = (d: typeof declaredUp) => (d.kind === 'role' ? d.role : undefined) + // A rollback runs as the role its own SQL names: under another role than the up migration it + // typically cannot touch what the up created. + let sqlProblem = $derived( + malformedLine !== undefined + ? malformedMessage(malformedLine) + : declaredDown !== undefined && roleOf(declaredDown) !== roleOf(declaredUp) + ? `The down migration runs as ${roleOf(declaredDown) ?? 'admin (no role)'} but the up migration as ${roleOf(declaredUp) ?? 'admin (no role)'}: make their role annotations match` + : undefined + ) + let selectedRole = $derived(declaredUp.kind === 'role' ? declaredUp.role : NO_ROLE) + let permissioned = $derived(!!usableRoles.current?.permissioned) + // No annotation runs as admin, which the server allows exactly to those who may use `admin`. + let adminUsable = $derived(!!usableRoles.current?.roles.includes(ADMIN_DATATABLE_ROLE)) + let roleItems = $derived.by(() => { + const usable = usableRoles.current + if (!usable?.permissioned) return [] + const names = usable.roles.filter((r) => r !== ADMIN_DATATABLE_ROLE) + // A role the SQL names but the caller cannot use is still shown, or the picker would + // misreport what the migration runs as. + if (declaredUp.kind === 'role' && !names.includes(declaredUp.role)) { + names.push(declaredUp.role) + } + const items = names.map((r) => ({ + value: r, + label: r === usable.default_role ? `${r} (default)` : r + })) + if (adminUsable || declaredUp.kind === 'none') { + items.push({ + value: NO_ROLE, + label: 'No role — runs as admin with full access' + }) + } + return items + }) + + function setRole(value: string | undefined) { + const role = value === NO_ROLE ? undefined : value + codeUp = withMigrationRole(codeUp, role) + // Up and down agree: a rollback run as another role could fail on objects it does not own. + if (enableDown) codeDown = withMigrationRole(codeDown, role) + // Assigning the bound value does not repaint the editor, and its next keystroke would write + // the stale text back. + upEditor?.setCode(codeUp) + if (enableDown) downEditor?.setCode(codeDown) + } + + // Set by `open` when the SQL names no role yet: the data table's default is written once its + // roles are known. + let applyDefaultRole = $state(false) + $effect(() => { + // `undefined` until the first answer lands; `null` when it failed. + const usable = usableRoles.current + if (!applyDefaultRole || !isOpen || usableRoles.loading || usable === undefined) return + applyDefaultRole = false + if (!usable?.permissioned || declaredUp.kind !== 'none') return + const role = + usable.default_role !== ADMIN_DATATABLE_ROLE && usable.roles.includes(usable.default_role) + ? usable.default_role + : usable.default_role === ADMIN_DATATABLE_ROLE && adminUsable + ? undefined + : usable.roles.find((r) => r !== ADMIN_DATATABLE_ROLE) + if (role !== undefined) setRole(role) + }) + // Frame the migration body in an explicit transaction so it applies atomically. function wrapInTransaction(body: string): string { return `BEGIN;\n\n${body}\n\nEND;` @@ -72,15 +169,35 @@ } const PLACEHOLDER = wrapInTransaction('-- Add your migration here') + function malformedMessage(line: string): string { + return `Malformed role annotation \`${line}\`: write it as \`-- role \`, or pick the role above` + } + export function open(prefill?: { name?: string; codeUp?: string; codeDown?: string }) { + // Roles and the default can have changed since the last open (the roles drawer sits next + // to this modal), and the default role is written from this answer. + usableRoles.refetch() name = prefill?.name ?? '' // Start from the transaction template; when prefilled from detected DDL, - // wrap that DDL in the same BEGIN; ... END; frame. - codeUp = prefill?.codeUp - ? wrapInTransaction(ensureTrailingSemicolon(prefill.codeUp)) - : PLACEHOLDER + // wrap that DDL in the same BEGIN; ... END; frame. A role the prefill declares is taken + // out first and put back on top: below `BEGIN;` it would not be read. + const prefillRole = prefill?.codeUp ? parseMigrationRole(prefill.codeUp) : undefined + if (prefill?.codeUp) { + const wrapped = wrapInTransaction( + ensureTrailingSemicolon(withMigrationRole(prefill.codeUp, undefined)) + ) + codeUp = + prefillRole?.kind === 'role' + ? withMigrationRole(wrapped, prefillRole.role) + : prefillRole?.kind === 'malformed' + ? `${prefillRole.line}\n${wrapped}` + : wrapped + } else { + codeUp = PLACEHOLDER + } codeDown = prefill?.codeDown ?? PLACEHOLDER enableDown = (prefill?.codeDown ?? '') !== '' + applyDefaultRole = prefillRole === undefined || prefillRole.kind === 'none' tab = 'up' isOpen = true // Focus the name field once the modal content has rendered. @@ -96,6 +213,10 @@ sendUserToast("Invalid migration name: use only letters, digits, '_' and '-'", true) return } + if (sqlProblem !== undefined) { + sendUserToast(sqlProblem, true) + return + } if (run) { // A new migration gets the highest timestamp, so any still-pending // migration is earlier: running only this one applies it out of order. @@ -176,29 +297,65 @@ closeOnOutsideClick={false} >
- +
+ + {#if permissioned} +
` — Specific table in public schema - `/:
` — Table in specific schema +**Roles:** when a datatable is under roles, its queries run as a role, which only reaches what it was granted. `roles` records the role the app uses each datatable through; the app's code must pass the same role: `wmill.datatable('main', { role: 'analyst' })` in TypeScript, `wmill.datatable('main', role='analyst')` in Python. A datatable without an entry is used as its default role. + ## SQL Migrations (sql_to_apply/) The `sql_to_apply/` folder is for creating/modifying database tables during development. diff --git a/system_prompts/base/raw-app-cli.md b/system_prompts/base/raw-app-cli.md index cd98e06ff5..030b958379 100644 --- a/system_prompts/base/raw-app-cli.md +++ b/system_prompts/base/raw-app-cli.md @@ -167,6 +167,8 @@ data: tables: - main/users # Table in public schema - main/app_schema:items # Table in specific schema + roles: # Optional: the role the app uses each datatable through + main: analyst ``` **Table reference formats:** @@ -174,6 +176,8 @@ data: - `/
` — Specific table in public schema - `/:
` — Table in specific schema +**Roles:** when a datatable is under roles, its queries run as a role, which only reaches what it was granted. `roles` records the role the app uses each datatable through; the app's code must pass the same role: `wmill.datatable('main', { role: 'analyst' })` in TypeScript, `wmill.datatable('main', role='analyst')` in Python. A datatable without an entry is used as its default role. + ## SQL Migrations (sql_to_apply/) The `sql_to_apply/` folder is for creating/modifying database tables during development.