feat: instrument sandbox isolation, data tables and in-flow script edits

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01GLnp4v49BozkDd3KeWn5Q3
This commit is contained in:
hugocasa
2026-09-04 18:19:35 +02:00
co-authored by Claude Opus 5
parent f037c73d10
commit d3f8b0e56f
16 changed files with 323 additions and 12 deletions
@@ -0,0 +1,26 @@
{
"db_name": "PostgreSQL",
"query": "SELECT\n COALESCE(dt.value->'database'->>'resource_type', 'unknown') AS \"kind!\",\n COUNT(*)::BIGINT AS \"count!\"\n FROM workspace_settings ws,\n LATERAL jsonb_each(ws.datatable->'datatables') dt\n WHERE jsonb_typeof(ws.datatable->'datatables') = 'object'\n GROUP BY 1",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "kind!",
"type_info": "Text"
},
{
"ordinal": 1,
"name": "count!",
"type_info": "Int8"
}
],
"parameters": {
"Left": []
},
"nullable": [
null,
null
]
},
"hash": "0411a67eb9d88244fa654eda51123e16b5931c08c1073b09ab01dad205f161ed"
}
@@ -0,0 +1,32 @@
{
"db_name": "PostgreSQL",
"query": "SELECT\n COUNT(*) FILTER (WHERE dt.value->>'migrations_enabled' = 'true')::BIGINT AS \"enabled!\",\n COUNT(*) FILTER (WHERE dt.value->>'migrations_enabled' = 'false')::BIGINT AS \"disabled!\",\n COUNT(*) FILTER (WHERE dt.value->>'migrations_enabled' IS NULL)::BIGINT AS \"unset!\"\n FROM workspace_settings ws,\n LATERAL jsonb_each(ws.datatable->'datatables') dt\n WHERE jsonb_typeof(ws.datatable->'datatables') = 'object'",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "enabled!",
"type_info": "Int8"
},
{
"ordinal": 1,
"name": "disabled!",
"type_info": "Int8"
},
{
"ordinal": 2,
"name": "unset!",
"type_info": "Int8"
}
],
"parameters": {
"Left": []
},
"nullable": [
null,
null,
null
]
},
"hash": "209c96d522f9683b39f053707568f9943111e0bac7d4fb478300f0cd8b799f23"
}
@@ -0,0 +1,26 @@
{
"db_name": "PostgreSQL",
"query": "SELECT\n COUNT(*) FILTER (WHERE av.raw_app = false)::BIGINT AS \"low_code!\",\n COUNT(*) FILTER (WHERE av.raw_app = true)::BIGINT AS \"raw!\"\n FROM app a\n JOIN app_version av ON av.id = a.versions[array_upper(a.versions, 1)]\n WHERE a.policy->>'sandbox' = 'true'",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "low_code!",
"type_info": "Int8"
},
{
"ordinal": 1,
"name": "raw!",
"type_info": "Int8"
}
],
"parameters": {
"Left": []
},
"nullable": [
null,
null
]
},
"hash": "9b727f03e74ea4a35146c9a339cda82104a9ee39bfbe5d932340a3c2c209c5d0"
}
@@ -0,0 +1,32 @@
{
"db_name": "PostgreSQL",
"query": "SELECT\n COUNT(*)::BIGINT AS \"total!\",\n COUNT(*) FILTER (WHERE code_down IS NOT NULL)::BIGINT AS \"with_down!\",\n COUNT(DISTINCT (workspace_id, datatable))::BIGINT AS \"datatables!\"\n FROM datatable_migrations",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "total!",
"type_info": "Int8"
},
{
"ordinal": 1,
"name": "with_down!",
"type_info": "Int8"
},
{
"ordinal": 2,
"name": "datatables!",
"type_info": "Int8"
}
],
"parameters": {
"Left": []
},
"nullable": [
null,
null,
null
]
},
"hash": "bf7528712e452e72611e35fb0803cb0bb29879ec010e1f9f36f19a2b3b5a721c"
}
@@ -0,0 +1,26 @@
{
"db_name": "PostgreSQL",
"query": "SELECT usage_kind::text AS \"kind!\", COUNT(*)::BIGINT AS \"count!\"\n FROM asset\n WHERE kind = 'datatable'\n AND (usage_kind <> 'job' OR created_at > now() - interval '30 days')\n GROUP BY 1",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "kind!",
"type_info": "Text"
},
{
"ordinal": 1,
"name": "count!",
"type_info": "Int8"
}
],
"parameters": {
"Left": []
},
"nullable": [
null,
null
]
},
"hash": "c2eca1ac257d9daeefaadb4ae853d0db3f649844ba3851fe29fc3eb9e5cad4ec"
}
@@ -455,6 +455,23 @@ async fn run_datatable_migrations(
applied.push(AppliedMigration { version: m.timestamp, name: m.name });
}
// One event per run that moved the data table forward, keyed by how the user
// scoped it. A run with nothing pending is not counted: it is the common
// outcome of opening the list and would drown out the runs that did something.
if !applied.is_empty() {
windmill_common::feature_usage::log_feature_usage(
"datatable",
"migration_run",
if query.only.is_some() {
"only"
} else if query.up_to.is_some() {
"up_to"
} else {
"all"
},
);
}
Ok(Json(RunDatatableMigrationsResult { applied }))
}
@@ -594,6 +611,12 @@ async fn rollback_datatable_migrations(
))
})?;
windmill_common::feature_usage::log_feature_usage(
"datatable",
"migration_rollback",
if query.only.is_some() { "only" } else { "last" },
);
Ok(Json(RollbackDatatableMigrationsResult {
rolled_back: vec![RolledBackMigration { version, name: definition.name }],
}))
@@ -824,6 +847,8 @@ async fn enable_datatable_migrations(
)
.await?;
windmill_common::feature_usage::log_feature_usage("datatable", "migrations_toggled", "on");
Ok(format!(
"Enabled migrations for data table {datatable_name}"
))
@@ -892,6 +917,8 @@ async fn disable_datatable_migrations(
.await?;
}
windmill_common::feature_usage::log_feature_usage("datatable", "migrations_toggled", "off");
Ok(format!(
"Disabled migrations for data table {datatable_name} and deleted its migrations"
))
@@ -1134,6 +1161,8 @@ async fn create_datatable_migration(
)
.await?;
windmill_common::feature_usage::log_feature_usage("datatable", "migration_created", "manual");
Ok(Json(DatatableMigration {
datatable: datatable_name,
timestamp,
@@ -1371,6 +1400,20 @@ async fn upsert_datatable_migration(
)
.await?;
// An unchanged re-push is not counted: `wmill sync push` sends every migration
// on every sync, so counting those would swamp the definitions people write.
if !unchanged {
windmill_common::feature_usage::log_feature_usage(
"datatable",
"migration_created",
if existing.is_none() {
"synced"
} else {
"edited"
},
);
}
Ok(format!(
"Upserted migration {} in {}",
payload.timestamp, datatable_name
@@ -1477,6 +1520,12 @@ async fn generate_initial_datatable_migration(
)
.await?;
windmill_common::feature_usage::log_feature_usage(
"datatable",
"migration_created",
"initial_snapshot",
);
Ok(Json(DatatableMigration {
datatable: datatable_name,
timestamp,
@@ -3491,6 +3491,9 @@ async fn edit_datatable_config(
// Migrations opt-in is owned by the enable/disable endpoints, not this config
// form: preserve each existing data table's flag, and default brand-new data
// tables to enabled.
// Counted here rather than after the write because this is where a rename is
// still distinguishable from a creation; emitted once the commit lands.
let mut created_substrates: Vec<&'static str> = Vec::new();
for (name, dt) in new_config.settings.datatables.iter_mut() {
let lookup = rename_src
.get(name.as_str())
@@ -3498,7 +3501,15 @@ async fn edit_datatable_config(
.unwrap_or(name.as_str());
dt.migrations_enabled = match old_datatables.get(lookup) {
Some(old) => old.migrations_enabled,
None => Some(true),
None => {
// Keyed by how the substrate is serialized into `workspace_settings`,
// so these line up with the `datatable_configured` adoption counts.
created_substrates.push(match dt.database.resource_type {
DataTableCatalogResourceType::Instance => "instance",
DataTableCatalogResourceType::Postgresql => "postgresql",
});
Some(true)
}
};
}
@@ -3556,6 +3567,10 @@ async fn edit_datatable_config(
tx.commit().await?;
for substrate in created_substrates {
windmill_common::feature_usage::log_feature_usage("datatable", "created", substrate);
}
crate::datatable_migrations::record_datatable_cascade_deployments(
&authed,
&db,
+4 -3
View File
@@ -4,9 +4,10 @@
anonymous usage-stats payload. It answers "does anyone use this, and which variant do they pick"
without any identifying data leaving the instance.
It currently carries 32 registered actions across fifteen features (`ai_session`, `ai_chat`,
`ai_fix`, `ai_agent`, `ai_agent_eval`, `flow_editor`, `flow_run`, `flow_step`, `run_form`,
`debugger`, `trigger`, `command_script`, `hub_script`, `usage_meter`, `sso_groups_claim`). Nearly all of the
It currently carries 41 registered actions across seventeen features (`ai_session`, `ai_chat`,
`ai_fix`, `ai_agent`, `ai_agent_eval`, `app_sandbox`, `datatable`, `flow_editor`, `flow_run`,
`flow_step`, `run_form`, `debugger`, `trigger`, `command_script`, `hub_script`, `usage_meter`,
`sso_groups_claim`). Nearly all of the
product is uninstrumented, so new user-facing work is the opportunity to change that.
## When to instrument
@@ -4,6 +4,7 @@
import NewDataTableMigrationModal from './workspaceSettings/NewDataTableMigrationModal.svelte'
import DataTableMigrationsButton from './workspaceSettings/DataTableMigrationsButton.svelte'
import { splitSqlStatements, isDdlStatement } from './sqlDdl'
import { logDdlGuardChoice } from './workspaceSettings/datatableTelemetry'
import { CornerDownLeft } from 'lucide-svelte'
let { workspace, datatable }: { workspace: string; datatable: string } = $props()
@@ -97,9 +98,11 @@
for (;;) {
const choice = await promptDdl(statement)
if (choice === 'cancel') {
logDdlGuardChoice('cancelled')
return { proceed: false, code, ranMigration: migrationRan }
}
if (choice === 'run') {
logDdlGuardChoice('run_anyway')
kept.push(statement)
break
}
@@ -107,6 +110,7 @@
// created; if the modal was cancelled, loop back to the prompt.
const created = await openMigrationModal(statement)
if (created) {
logDdlGuardChoice('migrated')
break
}
}
@@ -1072,12 +1072,18 @@
model identifiers, the names of public hub scripts used, the languages debug sessions
are started for, whether AI chat skills are turned on or off and how often one is
loaded, whether SSO logins evaluate an IdP groups claim (SAML or OIDC) and change a
membership, and the plan tier and quota shown when the execution meter is opened, last
30 days)</li
membership, the plan tier and quota shown when the execution meter is opened, whether
sandbox isolation is turned on or off for low-code and raw apps, whether a step's
workspace script is opened and saved from inside the flow editor, and how data tables
are set up and used (which database kind is picked, how far the setup wizard gets,
which choice the schema-change prompt is answered with, and whether migrations are
opted into, written, run or rolled back), last 30 days)</li
>
<li
>feature adoption (counts of which flow, script, trigger and worker features your
deployed items use)</li
>feature adoption (counts of which flow, script, trigger, worker and data table
features your deployed items use, including how many apps run sandboxed, how many data
tables are configured per database kind, how many migrations they carry, and how many
scripts, flows and jobs read or write a data table)</li
>
<li
>resource counts (workspaces, scripts per language, flows, workflows as code, low-code
@@ -1127,12 +1133,18 @@
model identifiers, the names of public hub scripts used, the languages debug sessions
are started for, whether AI chat skills are turned on or off and how often one is
loaded, whether SSO logins evaluate an IdP groups claim (SAML or OIDC) and change a
membership, and the plan tier and quota shown when the execution meter is opened, last
30 days)</li
membership, the plan tier and quota shown when the execution meter is opened, whether
sandbox isolation is turned on or off for low-code and raw apps, whether a step's
workspace script is opened and saved from inside the flow editor, and how data tables
are set up and used (which database kind is picked, how far the setup wizard gets,
which choice the schema-change prompt is answered with, and whether migrations are
opted into, written, run or rolled back), last 30 days)</li
>
<li
>feature adoption (counts of which flow, script, trigger and worker features your
deployed items use)</li
>feature adoption (counts of which flow, script, trigger, worker and data table
features your deployed items use, including how many apps run sandboxed, how many data
tables are configured per database kind, how many migrations they carry, and how many
scripts, flows and jobs read or write a data table)</li
>
<li
>resource counts (workspaces, scripts per language, flows, workflows as code, low-code
@@ -22,6 +22,7 @@
} from '$lib/components/OnBehalfOfSelector.svelte'
import { canUserBypassRuleKind, protectionRulesState } from '$lib/workspaceProtectionRules.svelte'
import { FRONTEND_SDK_SCOPES } from '$lib/components/raw_apps/sdkScopes'
import { logAppSandboxToggle } from './appSandboxTelemetry'
const WM_DEPLOYERS_GROUP = 'wm_deployers'
@@ -300,6 +301,7 @@
checked={policy.sandbox == true}
on:change={(e) => {
policy.sandbox = e.detail || undefined
logAppSandboxToggle(rawApp ? 'raw' : 'low_code', e.detail)
// Frontend API access exists only for a sandboxed app, so turning
// isolation off drops the declared scopes with it rather than leaving
// them set but inert.
@@ -0,0 +1,17 @@
import { logFeatureUsage } from '$lib/utils/featureUsage'
// Anonymous counters for the take rate of the alpha sandbox-isolation toggle. Same rules as
// every other `logFeatureUsage` caller: aggregated counts only, and the four keys below are
// the whole vocabulary — no app path, policy or scope ever reaches here.
/** Which editor the toggle was flipped in, since the two app kinds adopt it independently. */
export type AppSandboxKind = 'low_code' | 'raw'
/**
* Counted where the user flips the toggle, not where the policy is persisted: a not-yet-deployed
* app only mutates its policy locally, and dropping those would count the toggle as unused in
* exactly the case where it is picked up front.
*/
export function logAppSandboxToggle(kind: AppSandboxKind, enabled: boolean): void {
logFeatureUsage('app_sandbox', 'toggled', { key: `${kind}:${enabled ? 'on' : 'off'}` })
}
@@ -25,6 +25,7 @@
import { sendUserToast, type Item } from '$lib/utils'
import { twMerge } from 'tailwind-merge'
import { getToolNameError } from '$lib/components/flows/agentToolUtils'
import { logStepScriptEdit } from '$lib/components/flows/stepScriptEditTelemetry'
import autosize from '$lib/autosize'
interface Props {
@@ -104,7 +105,11 @@
if (flowModuleValue?.type !== 'script') return
const hash =
flowModuleValue.hash ?? (await getLatestHashForScript(flowModuleValue.path, opWs))
logStepScriptEdit('opened')
// The drawer only runs this callback once a new version is deployed, so it is
// what separates opening the editor from actually editing the script here.
$scriptEditorDrawer?.openDrawer(hash, () => {
logStepScriptEdit('saved')
dispatch('reload')
sendUserToast('Script has been updated')
})
@@ -0,0 +1,15 @@
import { logFeatureUsage } from '$lib/utils/featureUsage'
// Anonymous counters for editing a workspace script's code without leaving the flow editor.
// Same rules as every other `logFeatureUsage` caller: aggregated counts only, and the two keys
// below are the whole vocabulary — no script path, hash, language or code ever reaches here.
export type StepScriptEditEvent =
/** The drawer was opened on the step's script. */
| 'opened'
/** A new version of that script was deployed from the drawer. */
| 'saved'
export function logStepScriptEdit(event: StepScriptEditEvent): void {
logFeatureUsage('flow_step', 'script_edit', { key: event })
}
@@ -34,6 +34,7 @@
import DataTableConnectionReport from './DataTableConnectionReport.svelte'
import { useSupabaseOauth } from './supabaseOauth.svelte'
import { probeDatatableConnection } from './datatableProbe'
import { logDatatableWizard } from './datatableTelemetry'
import {
anythingClaimed,
claimOf,
@@ -526,11 +527,13 @@
await loadTargetUser()
reset(parked ?? resume)
opened = true
logDatatableWizard({ step: 'opened' })
}
function selectProvider(key: Provider) {
if (key === wiz.provider) return
wiz.provider = key
logDatatableWizard({ step: 'picked', provider: key })
invalidate()
if (key === 'instance') wiz.instance.dbName ??= defaultInstanceDbName()
}
@@ -833,6 +836,11 @@
createdProjects
}
}
// The setup's own verdict, so a data table that exists counts as done even when the
// caller's appended `onFinishAlso` step failed after it.
if (wiz.provider) {
logDatatableWizard({ step: result?.ok ? 'done' : 'failed', provider: wiz.provider })
}
onDone()
}
}
@@ -0,0 +1,41 @@
import { logFeatureUsage } from '$lib/utils/featureUsage'
// Anonymous counters for the data table surfaces the backend cannot see: which substrate the
// add-wizard is pointed at and how far a run gets, and what the DDL guard talks people into.
// Same rules as every other `logFeatureUsage` caller: aggregated counts only, and the keys
// below are the whole vocabulary — no data table name, connection string, resource path or SQL
// ever reaches here.
/** The substrate a wizard run is pointed at. Mirrors the wizard's own `Provider`. */
export type DatatableWizardProvider = 'supabase' | 'instance' | 'resource'
export type DatatableWizardEvent =
/** The wizard was opened, including a run resumed from the Supabase redirect. */
| { step: 'opened' }
/** A substrate was picked. Re-picking a different one counts again, by design: the
* abandoned branch is the interesting half of a funnel. */
| { step: 'picked'; provider: DatatableWizardProvider }
/** A run finished, with the verdict the checklist reported. */
| { step: 'done' | 'failed'; provider: DatatableWizardProvider }
export function logDatatableWizard(event: DatatableWizardEvent): void {
const key = event.step === 'opened' ? 'opened' : `${event.step}_${event.provider}`
logFeatureUsage('datatable', 'wizard', { key })
}
export type DdlGuardChoice =
/** The DDL was run ad-hoc, against the guard's advice. */
| 'run_anyway'
/** The DDL became a migration definition. */
| 'migrated'
/** The statement was abandoned, so nothing ran. */
| 'cancelled'
/**
* Counted once per prompt that reaches a terminal choice. Picking "create a migration" and then
* dismissing the modal loops back to the prompt instead, and is deliberately not counted: it is
* the same statement still undecided, not a fourth outcome.
*/
export function logDdlGuardChoice(choice: DdlGuardChoice): void {
logFeatureUsage('datatable', 'ddl_guard', { key: choice })
}