feat(datatables): data table roles in the DB manager and raw apps

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

backend/ee-repo-ref.txt

@@ -1 +1 @@
-6f26308c67acf9fcc45773b373aa30a2593b665c
+6ced9da74422ae8feee10da70419ccedf313506c

backend/windmill-api-integration-tests/tests/datatable_roles.rs

@@ -1009,6 +1009,70 @@ async fn an_entry_without_roles_cannot_newly_reach_a_database_under_roles(
     Ok(())
 }
 
+/// Browsing names the role it connects as, and a role the caller may not use is refused rather
+/// than quietly listed as the default. The refusal is decided before connecting, so the fixture's
+/// database never has to exist.
+#[cfg(all(feature = "private", feature = "enterprise"))]
+#[sqlx::test(migrations = "../migrations", fixtures("base", "datatable_roles"))]
+async fn browsing_as_a_role_the_caller_may_not_use_is_refused(
+    db: Pool<Postgres>,
+) -> anyhow::Result<()> {
+    initialize_tracing().await;
+    let server = ApiServer::start(db.clone()).await?;
+    let port = server.addr.port();
+    let base = format!("http://localhost:{port}/api/w/test-workspace/workspaces");
+
+    // `test-user-2` is a tenant of `analytics` only.
+    let resp = authed(
+        client().get(format!(
+            "{base}/list_datatable_tables?role_for=main&role=admin"
+        )),
+        "SECRET_TOKEN_2",
+    )
+    .send()
+    .await?;
+    assert_eq!(resp.status(), 200);
+    let body: Value = resp.json().await?;
+    let entry = body
+        .as_array()
+        .and_then(|a| a.iter().find(|e| e["datatable_name"] == "main"))
+        .expect("main is listed");
+    assert_eq!(entry["usable_roles"], json!(["analytics"]), "{entry}");
+    assert_eq!(entry["default_role"], "analytics", "{entry}");
+    assert_eq!(entry["permissioned"], true, "{entry}");
+    assert_eq!(entry["instance"], true, "{entry}");
+    let error = entry["error"].as_str().unwrap_or_default();
+    assert!(
+        error.contains("Not allowed to use role 'admin'"),
+        "listed as another role than the one asked for: {entry}"
+    );
+
+    let resp = authed(
+        client().get(format!(
+            "{base}/get_datatable_table_schema?datatable_name=main&schema_name=public&table_name=t&role=admin"
+        )),
+        "SECRET_TOKEN_2",
+    )
+    .send()
+    .await?;
+    let status = resp.status();
+    let text = resp.text().await?;
+    assert!(
+        text.contains("Not allowed to use role 'admin'"),
+        "{status}: {text}"
+    );
+
+    // A role means nothing without the data table it belongs to.
+    let resp = authed(
+        client().get(format!("{base}/list_datatable_tables?role=analytics")),
+        "SECRET_TOKEN_2",
+    )
+    .send()
+    .await?;
+    assert_eq!(resp.status(), 400, "{}", resp.text().await?);
+    Ok(())
+}
+
 #[cfg(not(all(feature = "private", feature = "enterprise")))]
 const ENTERPRISE_REFUSAL: &str = "Data table roles are a Windmill Enterprise Edition feature";
 

backend/windmill-api-workspaces/src/datatable_permissions_oss.rs

@@ -13,7 +13,7 @@
 #[cfg(all(feature = "private", feature = "enterprise"))]
 pub(crate) use crate::datatable_permissions_ee::{
     ensure_governs_datatable, ensure_reaches_datatable, get_datatable_permissions,
-    list_usable_datatable_roles, set_datatable_permissions,
+    list_usable_datatable_roles, set_datatable_permissions, usable_datatable_roles,
 };
 
 #[cfg(not(all(feature = "private", feature = "enterprise")))]
@@ -70,4 +70,28 @@ mod ce {
     pub(crate) async fn list_usable_datatable_roles(_authed: ApiAuthed) -> Result<String> {
         Err(unavailable())
     }
+
+    pub(crate) struct UsableDatatableRoles {
+        pub(crate) permissioned: bool,
+        pub(crate) roles: Vec<String>,
+        pub(crate) default_role: String,
+    }
+
+    /// A data table not under roles is used as `admin`, as before roles existed. One under roles
+    /// is refused: no role of it can be connected as.
+    pub(crate) async fn usable_datatable_roles(
+        _db: &DB,
+        _authed: &ApiAuthed,
+        _w_id: &str,
+        governing: &GoverningDatatable,
+    ) -> Result<UsableDatatableRoles> {
+        if governing.datatable.permissions.is_some() {
+            return Err(unavailable());
+        }
+        Ok(UsableDatatableRoles {
+            permissioned: false,
+            roles: vec![],
+            default_role: windmill_common::datatable_roles::ADMIN_DATATABLE_ROLE.to_string(),
+        })
+    }
 }

backend/windmill-api-workspaces/src/workspaces.rs

@@ -2269,6 +2269,25 @@ struct DataTableTables {
     schemas: TableListMap,
     #[serde(skip_serializing_if = "Option::is_none")]
     error: Option<String>,
+    /// On the instance database: the only kind that can be under roles or have its access edited.
+    instance: bool,
+    permissioned: bool,
+    /// The roles this caller may connect as, by name; empty when not under roles.
+    usable_roles: Vec<String>,
+    default_role: String,
+    /// What the role the listing connected as may create.
+    can_create_schema: bool,
+    creatable_schemas: Vec<String>,
+}
+
+#[derive(Deserialize)]
+struct ListDataTableTablesQuery {
+    /// List only this data table: each entry opens a connection to its database.
+    datatable_name: Option<String>,
+    /// The data table `role` applies to. Every other one is listed as its default role, since a
+    /// role name means nothing outside the data table it belongs to.
+    role_for: Option<String>,
+    role: Option<String>,
 }
 
 #[derive(Deserialize)]
@@ -2276,6 +2295,7 @@ struct GetDataTableSchemaQuery {
     datatable_name: String,
     schema_name: String,
     table_name: String,
+    role: Option<String>,
 }
 
 #[derive(Serialize, Debug)]
@@ -2443,25 +2463,89 @@ async fn list_datatable_tables(
     authed: ApiAuthed,
     Extension(db): Extension<DB>,
     Path(w_id): Path<String>,
+    Query(query): Query<ListDataTableTablesQuery>,
 ) -> JsonResult<Vec<DataTableTables>> {
-    let datatable_names = list_datatable_names(&db, &w_id).await?;
+    if query.role.is_some() && query.role_for.is_none() {
+        return Err(Error::BadRequest(
+            "`role` needs `role_for`, the data table it is a role of".to_string(),
+        ));
+    }
+    if let (Some(only), Some(role_for)) =
+        (query.datatable_name.as_deref(), query.role_for.as_deref())
+    {
+        if only != role_for {
+            return Err(Error::BadRequest(format!(
+                "`role_for` names '{role_for}', which `datatable_name` leaves out of the listing"
+            )));
+        }
+    }
+    let mut datatable_names = list_datatable_names(&db, &w_id).await?;
+    for named in [query.role_for.as_deref(), query.datatable_name.as_deref()]
+        .into_iter()
+        .flatten()
+    {
+        if !datatable_names.iter().any(|n| n == named) {
+            return Err(Error::NotFound(format!(
+                "No data table named '{named}' in this workspace"
+            )));
+        }
+    }
+    if let Some(only) = query.datatable_name.as_deref() {
+        datatable_names.retain(|n| n == only);
+    }
     let mut results = Vec::new();
 
     for datatable_name in datatable_names {
-        let tables = match get_datatable_tables(&db, &authed, &w_id, &datatable_name).await {
-            Ok(schemas) => DataTableTables { datatable_name, schemas, error: None },
-            Err(e) => DataTableTables {
-                datatable_name,
-                schemas: HashMap::new(),
-                error: Some(e.to_string()),
-            },
-        };
-        results.push(tables);
+        let role = query
+            .role
+            .as_deref()
+            .filter(|_| query.role_for.as_deref() == Some(datatable_name.as_str()));
+        results.push(list_one_datatable_tables(&db, &authed, &w_id, datatable_name, role).await);
     }
 
     Ok(Json(results))
 }
 
+async fn list_one_datatable_tables(
+    db: &DB,
+    authed: &ApiAuthed,
+    w_id: &str,
+    datatable_name: String,
+    role: Option<&str>,
+) -> DataTableTables {
+    let mut entry = DataTableTables {
+        datatable_name,
+        schemas: HashMap::new(),
+        error: None,
+        instance: false,
+        permissioned: false,
+        usable_roles: vec![],
+        default_role: windmill_common::datatable_roles::ADMIN_DATATABLE_ROLE.to_string(),
+        can_create_schema: false,
+        creatable_schemas: vec![],
+    };
+    let result: Result<()> = async {
+        let governing = resolve_governing_datatable(db, w_id, &entry.datatable_name).await?;
+        entry.instance = governing.is_instance();
+        let usable =
+            crate::datatable_permissions_oss::usable_datatable_roles(db, authed, w_id, &governing)
+                .await?;
+        entry.permissioned = usable.permissioned;
+        entry.usable_roles = usable.roles;
+        entry.default_role = usable.default_role;
+        let listing = get_datatable_tables(db, authed, w_id, &entry.datatable_name, role).await?;
+        entry.schemas = listing.schemas;
+        entry.can_create_schema = listing.can_create_schema;
+        entry.creatable_schemas = listing.creatable_schemas;
+        Ok(())
+    }
+    .await;
+    if let Err(e) = result {
+        entry.error = Some(e.to_string());
+    }
+    entry
+}
+
 async fn get_datatable_table_schema(
     authed: ApiAuthed,
     Extension(db): Extension<DB>,
@@ -2475,6 +2559,7 @@ async fn get_datatable_table_schema(
         &query.datatable_name,
         &query.schema_name,
         &query.table_name,
+        query.role.as_deref(),
     )
     .await?;
 
@@ -2513,14 +2598,13 @@ async fn resolve_datatable_pg_as_caller(
     authed: &ApiAuthed,
     w_id: &str,
     datatable_name: &str,
+    role: Option<&str>,
 ) -> Result<PgDatabase> {
     let db_resource = get_datatable_resource_from_db(
         db,
         w_id,
         datatable_name,
-        // The data table's default role. Browsing has no way to name another one yet; when the
-        // database manager grows a role picker it passes the pick through here.
-        None,
+        role,
         DatatableAccess::Authed(authed.to_authed_ref()),
     )
     .await?;
@@ -2534,7 +2618,7 @@ async fn get_datatable_schema(
     w_id: &str,
     datatable_name: &str,
 ) -> Result<SchemaMap> {
-    let pg_db = resolve_datatable_pg_as_caller(db, authed, w_id, datatable_name).await?;
+    let pg_db = resolve_datatable_pg_as_caller(db, authed, w_id, datatable_name, None).await?;
 
     // Connect to the datatable database
     let (client, connection) = pg_db.connect(Some(db)).await?;
@@ -2622,13 +2706,20 @@ async fn get_datatable_schema(
     Ok(schema_map)
 }
 
+struct DatatableTableListing {
+    schemas: TableListMap,
+    can_create_schema: bool,
+    creatable_schemas: Vec<String>,
+}
+
 async fn get_datatable_tables(
     db: &DB,
     authed: &ApiAuthed,
     w_id: &str,
     datatable_name: &str,
-) -> Result<TableListMap> {
-    let pg_db = resolve_datatable_pg_as_caller(db, authed, w_id, datatable_name).await?;
+    role: Option<&str>,
+) -> Result<DatatableTableListing> {
+    let pg_db = resolve_datatable_pg_as_caller(db, authed, w_id, datatable_name, role).await?;
     let (client, connection) = pg_db.connect(Some(db)).await?;
 
     tokio::spawn(async move {
@@ -2640,7 +2731,7 @@ async fn get_datatable_tables(
     let schema_rows = client
         .query(
             r#"
-            SELECT nspname::text AS schema_name
+            SELECT nspname::text AS schema_name, has_schema_privilege(oid, 'CREATE') AS can_create
             FROM pg_namespace
             WHERE nspname NOT IN ('information_schema', 'pg_toast', 'pg_catalog')
               AND nspname NOT LIKE 'pg_%'
@@ -2654,11 +2745,29 @@ async fn get_datatable_tables(
             Error::internal_err(format!("Failed to query schemas: {}", pg_error_message(&e)))
         })?;
 
+    let can_create_schema: bool = client
+        .query_one(
+            "SELECT has_database_privilege(current_database(), 'CREATE')",
+            &[],
+        )
+        .await
+        .map_err(|e| {
+            Error::internal_err(format!(
+                "Failed to read database privileges: {}",
+                pg_error_message(&e)
+            ))
+        })?
+        .get(0);
+
     let mut table_map: TableListMap = HashMap::new();
+    let mut creatable_schemas = Vec::new();
     let schema_names: Vec<String> = schema_rows
         .iter()
         .map(|row| {
             let name: String = row.get(0);
+            if row.get::<_, bool>(1) {
+                creatable_schemas.push(name.clone());
+            }
             table_map.entry(name.clone()).or_default();
             name
         })
@@ -2688,7 +2797,7 @@ async fn get_datatable_tables(
         table_map.entry(table_schema).or_default().push(table_name);
     }
 
-    Ok(table_map)
+    Ok(DatatableTableListing { schemas: table_map, can_create_schema, creatable_schemas })
 }
 
 async fn get_datatable_table_columns(
@@ -2698,6 +2807,7 @@ async fn get_datatable_table_columns(
     datatable_name: &str,
     schema_name: &str,
     table_name: &str,
+    role: Option<&str>,
 ) -> Result<ColumnMap> {
     if is_system_pg_schema(schema_name) {
         return Err(Error::BadRequest(format!(
@@ -2706,7 +2816,7 @@ async fn get_datatable_table_columns(
         )));
     }
 
-    let pg_db = resolve_datatable_pg_as_caller(db, authed, w_id, datatable_name).await?;
+    let pg_db = resolve_datatable_pg_as_caller(db, authed, w_id, datatable_name, role).await?;
     let (client, connection) = pg_db.connect(Some(db)).await?;
 
     tokio::spawn(async move {

backend/windmill-api/openapi.yaml

@@ -5488,6 +5488,21 @@ paths:
         - workspace
       parameters:
         - $ref: "#/components/parameters/WorkspaceId"
+        - name: datatable_name
+          in: query
+          description: list only this data table; each listed data table opens a connection to its database
+          schema:
+            type: string
+        - name: role_for
+          in: query
+          description: the data table `role` applies to; every other one is listed as its default role
+          schema:
+            type: string
+        - name: role
+          in: query
+          description: the role to list `role_for` as; refused, in that entry's `error`, if the caller may not use it
+          schema:
+            type: string
       responses:
         "200":
           description: table metadata of all datatables
@@ -5521,6 +5536,11 @@ paths:
           required: true
           schema:
             type: string
+        - name: role
+          in: query
+          description: the data table role to read the table as; defaults to the data table's default role
+          schema:
+            type: string
       responses:
         "200":
           description: schema of one datatable table
@@ -36053,7 +36073,17 @@ components:
 
     DataTableTables:
       type: object
-      required: [datatable_name, schemas]
+      required:
+        [
+          datatable_name,
+          schemas,
+          instance,
+          permissioned,
+          usable_roles,
+          default_role,
+          can_create_schema,
+          creatable_schemas,
+        ]
       properties:
         datatable_name:
           type: string
@@ -36066,6 +36096,26 @@ components:
               type: string
         error:
           type: string
+        instance:
+          type: boolean
+          description: on the instance database, the only kind that can be under roles or have its access edited
+        permissioned:
+          type: boolean
+        usable_roles:
+          type: array
+          description: the roles the caller may connect as, by name; empty when not under roles
+          items:
+            type: string
+        default_role:
+          type: string
+        can_create_schema:
+          type: boolean
+          description: whether the role the listing connected as may create schemas
+        creatable_schemas:
+          type: array
+          description: the schemas the role the listing connected as may create in
+          items:
+            type: string
 
     DataTableTableSchema:
       type: object

backend/windmill-api/src/jobs.rs

@@ -8701,7 +8701,12 @@ fn register_potential_assets_on_inline_execution(
             .as_ref()
             .and_then(|args| args.get("database"))
             .map(|v| v.get().trim_matches('"'))
-            .and_then(|dt| dt.strip_prefix("datatable://"));
+            .and_then(|dt| dt.strip_prefix("datatable://"))
+            // `?role=` picks the connection, not the data table. Anything else after a `?` may be
+            // part of a name stored before names were restricted, so it stays.
+            .map(|dt| {
+                windmill_common::workspaces::parse_datatable_ref(dt).map_or(dt, |(name, _)| name)
+            });
         if let Some(datatable) = datatable {
             let re = regex::Regex::new(r#"SET search_path TO "([^"]+)";"#).unwrap();
             let (schema, content) = if let Some(captures) = re.captures(&preview.content) {

backend/windmill-common/src/datatable_roles_oss.rs

@@ -15,7 +15,8 @@
 
 use crate::error::Error;
 
-/// What every roles path answers without the Enterprise Edition.
+/// What every roles path answers without the Enterprise Edition. The frontend matches this exact
+/// sentence (`datatableUsableRoles.ts`) to read the refusal as "not under roles": reword both.
 pub fn datatable_roles_unavailable() -> Error {
     Error::BadRequest("Data table roles are a Windmill Enterprise Edition feature".to_string())
 }

backend/windmill-common/src/query_builders.rs

@@ -329,6 +329,7 @@ pub fn try_expand_internal_db_query(
         "ALTER_TABLE" => expand_alter_table(json_str, db_type).map(ExpandedQuery::sql),
         "CREATE_SCHEMA" => expand_create_schema(json_str, db_type).map(ExpandedQuery::sql),
         "DROP_SCHEMA" => expand_drop_schema(json_str, db_type).map(ExpandedQuery::sql),
+        "RENAME_SCHEMA" => expand_rename_schema(json_str, db_type).map(ExpandedQuery::sql),
         // Metadata queries
         "LOAD_TABLE_METADATA" => expand_load_table_metadata(json_str, db_type),
         "FOREIGN_KEYS" => expand_foreign_keys(json_str, db_type).map(ExpandedQuery::sql),
@@ -1716,6 +1717,13 @@ struct DropSchemaPayload {
     ducklake: Option<String>,
 }
 
+#[derive(Deserialize)]
+struct RenameSchemaPayload {
+    schema: String,
+    new_schema: String,
+    ducklake: Option<String>,
+}
+
 #[derive(Debug, Clone, Deserialize)]
 struct TableEditorColumn {
     name: String,
@@ -2004,6 +2012,23 @@ fn expand_drop_schema(json_str: &str, db_type: DbType) -> Result<String, String>
     Ok(maybe_wrap_ducklake(query, p.ducklake.as_deref()))
 }
 
+fn expand_rename_schema(json_str: &str, db_type: DbType) -> Result<String, String> {
+    let p: RenameSchemaPayload = serde_json::from_str(json_str)
+        .map_err(|e| format!("Invalid RENAME_SCHEMA payload: {}", e))?;
+    if !matches!(db_type, DbType::Postgresql | DbType::Snowflake) || p.ducklake.is_some() {
+        return Err(format!(
+            "Renaming a schema is not supported on {:?}",
+            db_type
+        ));
+    }
+    let query = format!(
+        "ALTER SCHEMA {} RENAME TO {};",
+        qi(&p.schema, db_type),
+        qi(&p.new_schema, db_type)
+    );
+    Ok(query)
+}
+
 fn expand_create_table(json_str: &str, db_type: DbType) -> Result<String, String> {
     let p: CreateTablePayload = serde_json::from_str(json_str)
         .map_err(|e| format!("Invalid CREATE_TABLE payload: {}", e))?;
@@ -2598,7 +2623,9 @@ WHERE table_catalog = current_database()",
                 )
             } else {
                 (
-                    "\nWHERE c.relkind = 'r' AND a.attnum > 0 AND NOT a.attisdropped\n    AND ns.nspname != 'pg_catalog' AND ns.nspname != 'information_schema'".to_string(),
+                    // pg_catalog is readable by everyone: without the privilege check this lists
+                    // tables of schemas the connection's role cannot even enter.
+                    "\nWHERE c.relkind = 'r' AND a.attnum > 0 AND NOT a.attisdropped\n    AND ns.nspname != 'pg_catalog' AND ns.nspname != 'information_schema'\n    AND has_schema_privilege(ns.oid, 'USAGE')".to_string(),
                     ",\n    ns.nspname AS schema_name,\n    c.relname AS table_name".to_string(),
                     "\nJOIN pg_catalog.pg_class c ON a.attrelid = c.oid\nJOIN pg_catalog.pg_namespace ns ON c.relnamespace = ns.oid".to_string(),
                     "ns.nspname, c.relname, a.attnum".to_string(),
@@ -4101,6 +4128,13 @@ mod tests {
         assert_eq!(sql, "DROP SCHEMA \"old_schema\" CASCADE;");
     }
 
+    #[test]
+    fn test_expand_rename_schema() {
+        let marker = r#"-- WM_INTERNAL_DB_RENAME_SCHEMA {"schema":"old","new_schema":"new"}"#;
+        let sql = expand_code(marker, &ScriptLang::Postgresql);
+        assert_eq!(sql, "ALTER SCHEMA \"old\" RENAME TO \"new\";");
+    }
+
     #[test]
     fn test_expand_create_schema_with_ducklake() {
         let marker = r#"-- WM_INTERNAL_DB_CREATE_SCHEMA {"schema":"s","ducklake":"lake"}"#;
@@ -4468,6 +4502,7 @@ mod tests {
         assert!(sql.contains("schema_name"));
         assert!(sql.contains("table_name"));
         assert!(sql.contains("c.relkind = 'r'"));
+        assert!(sql.contains("has_schema_privilege(ns.oid, 'USAGE')"));
     }
 
     #[test]