feat: add session recording to wmill app dev (#10457)

* feat: add session recording to wmill app dev

* fix: harden dev recorder shell, bundle staleness guard and save route

* fix: keep dev-server recordings out of the raw app sync diff

* style: drop em dashes from new cli comments

* fix: tighten dev recorder save route origin, naming and io

* test: pin that only the root recordings folder is skipped

* fix: survive an oversized recording upload and match paths on windows

* fix: keep the app at the root and settle runnables stranded by a reload

* fix: make the recorder bundle hash stable on a crlf checkout

* docs: state the preflight-free content type the origin check guards

* test: build the sync-skip fixture with the platform separator

* docs: align the origin-guard test comment with the code

* chore: mark generated .gen.ts files as generated
This commit is contained in:
Ruben Fiszel
2026-08-02 22:54:37 +02:00
committed by GitHub
parent eca24bdfb5
commit 2105540cca
18 changed files with 822 additions and 6 deletions
+3
View File
@@ -51,6 +51,9 @@ import { pollJobWithQueueLogging } from "../../utils/job_polling.ts";
const TOP_HASH = "__app_hash";
export const APP_BACKEND_FOLDER = "backend";
/** Where `wmill app dev --recording` writes finished session recordings. Local
* artifacts, so the app source push skips them. */
export const RECORDINGS_FOLDER = "recordings";
// Union type for app files that can be either raw or normal apps
type AppFile = RawAppFile | NormalAppFile;
+188 -2
View File
@@ -48,6 +48,17 @@ import {
hasFolderSuffix,
loadNonDottedPathsSetting,
} from "../../utils/resource_folders.ts";
import {
createRecorderShellHTML,
DEV_RECORDER_BUNDLE,
isOwnOrigin,
isRecordingFileName,
RECORDER_BUNDLE_PATH,
RECORDER_SAVE_PATH,
RECORDER_SHELL_PATH,
recordingFileName,
RECORDINGS_FOLDER,
} from "./devRecorder.ts";
// Resolved once per `wmill app dev` run from wmill.yaml; a bare `.ts` under
// backend/ denotes this runtime, so readers must agree with the path assigner.
@@ -333,6 +344,7 @@ interface DevOptions extends GlobalOptions {
host?: string;
entry?: string;
open?: boolean;
recording?: boolean;
}
async function dev(opts: DevOptions, appFolder?: string) {
@@ -537,6 +549,23 @@ async function dev(opts: DevOptions, appFolder?: string) {
fs.mkdirSync(distDir);
}
// Session recording: the app moves into an iframe of a shell page holding the
// recorder toolbar, and finished recordings land in the app folder.
const recordingEnabled = opts.recording ?? false;
const recordingsDir = path.join(process.cwd(), RECORDINGS_FOLDER);
// The player is a page of the instance this app is developed against, so the
// recording it fetches from here crosses origins.
const playerBaseUrl = workspace.remote.endsWith("/")
? workspace.remote
: `${workspace.remote}/`;
const playerOrigin = (() => {
try {
return new URL(playerBaseUrl).origin;
} catch {
return undefined;
}
})();
// SSE clients for live reload
const clients: http.ServerResponse[] = [];
@@ -721,10 +750,152 @@ async function dev(opts: DevOptions, appFolder?: string) {
);
}
// Same ceiling the player refuses to load past, so nothing is written here
// that could not be replayed.
const MAX_RECORDING_BYTES = 100 * 1024 * 1024;
function sendJson(res: http.ServerResponse, status: number, body: unknown) {
res.writeHead(status, { "Content-Type": "application/json" });
res.end(JSON.stringify(body));
}
/** Write the recording under a name nothing else holds. `wx` is what makes
* this safe: two tabs stopping in the same millisecond both create, and the
* loser retries rather than overwriting the winner. */
async function writeRecording(body: string): Promise<string> {
const now = new Date();
for (let attempt = 0; attempt < 100; attempt++) {
const file = recordingFileName(now, attempt);
try {
await fs.promises.writeFile(path.join(recordingsDir, file), body, {
flag: "wx",
});
return file;
} catch (error: any) {
if (error?.code !== "EEXIST") throw error;
}
}
throw new Error("Could not find a free recording file name");
}
function saveRecording(req: http.IncomingMessage, res: http.ServerResponse) {
if (!isOwnOrigin(req.headers.origin, req.headers.host)) {
sendJson(res, 403, { error: "Cross-origin recording upload refused" });
return;
}
const chunks: Buffer[] = [];
let size = 0;
let refused = false;
// An upload cut short (by the refusal below, or by the browser) raises
// 'error' on the request, which unhandled takes the dev server down.
req.on("error", (error: Error) => {
if (!refused) {
log.warn(colors.yellow(`Recording upload failed: ${error.message}`));
}
});
req.on("data", (chunk: Buffer) => {
if (refused) return;
size += chunk.length;
if (size > MAX_RECORDING_BYTES) {
refused = true;
// Torn down only once the 413 is on the wire: destroying the socket
// first loses the response the browser is waiting to read.
res.on("finish", () => req.destroy());
sendJson(res, 413, {
error: `Recording exceeds ${MAX_RECORDING_BYTES} bytes`,
});
return;
}
chunks.push(chunk);
});
req.on("end", async () => {
if (refused) return;
const body = Buffer.concat(chunks).toString("utf-8");
try {
JSON.parse(body);
} catch {
sendJson(res, 400, { error: "Body is not valid JSON" });
return;
}
let file: string;
try {
await fs.promises.mkdir(recordingsDir, { recursive: true });
file = await writeRecording(body);
} catch (error: any) {
log.error(colors.red(`Failed to save recording: ${error.message}`));
sendJson(res, 500, { error: error.message });
return;
}
log.info(
colors.green(
`🎬 Recording saved to ${path.join(RECORDINGS_FOLDER, file)}`,
),
);
log.info(
colors.gray(
` Replay it at ${playerBaseUrl}replay (open the file, or use ?src=)`,
),
);
sendJson(res, 200, { file });
});
}
function serveRecording(url: string, res: http.ServerResponse) {
const file = url.slice(RECORDER_SAVE_PATH.length + 1);
if (!isRecordingFileName(file)) {
sendJson(res, 400, { error: "Invalid recording name" });
return;
}
const filePath = path.join(recordingsDir, file);
if (!fs.existsSync(filePath)) {
sendJson(res, 404, { error: "Recording not found" });
return;
}
res.writeHead(200, {
"Content-Type": "application/json",
// The player runs on the instance, not on the dev server: without this it
// can fetch the recording but never read it.
...(playerOrigin ? { "Access-Control-Allow-Origin": playerOrigin } : {}),
});
// Streamed: a recording is megabytes, and this server also carries the app,
// its live reload and every runnable call.
fs.createReadStream(filePath).on("error", () => res.end()).pipe(res);
}
// Create HTTP server
const server = http.createServer((req, res) => {
const url = req.url || "/";
if (recordingEnabled) {
const pathname = url.split("?")[0];
if (pathname === RECORDER_BUNDLE_PATH) {
res.writeHead(200, { "Content-Type": "application/javascript" });
res.end(DEV_RECORDER_BUNDLE);
return;
}
if (pathname === RECORDER_SAVE_PATH && req.method === "POST") {
saveRecording(req, res);
return;
}
if (
pathname.startsWith(`${RECORDER_SAVE_PATH}/`) && req.method === "GET"
) {
serveRecording(pathname, res);
return;
}
if (pathname === RECORDER_SHELL_PATH) {
res.writeHead(200, { "Content-Type": "text/html" });
res.end(
createRecorderShellHTML({
appPath,
workspace: workspaceId,
playerBaseUrl,
}),
);
return;
}
}
// SSE endpoint for live reload
if (url === "/__events") {
res.writeHead(200, {
@@ -1359,18 +1530,29 @@ async function dev(opts: DevOptions, appFolder?: string) {
server.listen(port, host, () => {
const url = `http://${host}:${port}`;
// The toolbar lives beside the app, not in front of it, so recording mode
// is what the browser should land on.
const openUrl = recordingEnabled ? `${url}${RECORDER_SHELL_PATH}` : url;
log.info(colors.bold.green(`🚀 Dev server running at ${url}`));
log.info(
colors.cyan(`🔌 WebSocket server running at ws://${host}:${port}`),
);
log.info(colors.gray(`📦 Serving files from: ${process.cwd()}`));
log.info(colors.gray(`🔄 Live reload enabled\n`));
log.info(colors.gray(`🔄 Live reload enabled`));
if (recordingEnabled) {
log.info(
colors.magenta(
`🎬 Session recording at ${openUrl} : press Record in the toolbar. Recordings are saved to ${RECORDINGS_FOLDER}/`,
),
);
}
log.info("");
// Open browser if requested
if (shouldOpen) {
try {
open
.openApp(open.apps.browser, { arguments: [url] })
.openApp(open.apps.browser, { arguments: [openUrl] })
.catch((error: any) => {
log.error(
colors.yellow(
@@ -1423,6 +1605,10 @@ const command = new Command()
"Entry point file (default: index.ts for Svelte/Vue, index.tsx otherwise)",
)
.option("--no-open", "Don't automatically open the browser")
.option(
"--recording",
"Frame the app in a shell with a Record button, to capture a replayable session recording of the app under development",
)
.action(dev as any);
export default command;
+270
View File
@@ -0,0 +1,270 @@
/**
* `wmill app dev --recording`: a shell page that frames the app under
* development and records the session with the same recorder the Windmill UI
* uses, so a locally-built app can be demoed without deploying it first.
*
* The shell holds the toolbar rather than the app page: the recorder snapshots
* the framed document on every mutation, and a toolbar living in that document
* would record itself. It also lives at its own path, so the app keeps the root
* URL its router and its own links expect.
*/
import { DEV_RECORDER_BUNDLE } from "./devRecorderBundle.gen.ts";
/** Path the bundled recorder is served from. */
export const RECORDER_BUNDLE_PATH = "/__wm_recorder.js";
/** Path the toolbar shell is served from. Not the root: the app stays there, so
* a link or a router push to `/` inside it reloads the app rather than nesting
* another shell in the frame. */
export const RECORDER_SHELL_PATH = "/__record";
/** Recordings are POSTed here and served back from `<path>/<file>`. */
export const RECORDER_SAVE_PATH = "/__recordings";
export { DEV_RECORDER_BUNDLE };
export { RECORDINGS_FOLDER } from "./app_metadata.ts";
/** Whether a request to the save route came from the shell this server serves.
* A cross-site POST carrying JSON under a simple content type (`text/plain`)
* needs no preflight, so without this any page open in the developer's browser
* could write files into the app folder. Compared against the `Host` the browser
* actually reached us on, since the shell is equally `localhost`, `127.0.0.1` or
* whatever `--host` binds. */
export function isOwnOrigin(
origin: string | undefined,
host: string | undefined,
): boolean {
// Absent on a same-origin request in some browsers, and on non-browser
// clients (curl, a script); a forged one is no worse than no header at all.
if (origin === undefined) return true;
if (host === undefined) return false;
try {
return new URL(origin).host === host;
} catch {
return false;
}
}
/** Name a recording after the moment it was saved. Millisecond resolution plus
* the caller's collision retry: two tabs stopping together must not have one
* silently overwrite the other. */
export function recordingFileName(now: Date, attempt = 0): string {
const stamp = now.toISOString().slice(0, 23).replace(/[:T.]/g, "-");
return `recording-${stamp}${attempt > 0 ? `-${attempt}` : ""}.json`;
}
/** Names the read route accepts. Rejects every separator and escape, so the
* name can only ever resolve inside the recordings folder. */
export function isRecordingFileName(file: string): boolean {
return /^[A-Za-z0-9._-]+\.json$/.test(file) && !file.includes("..");
}
export function createRecorderShellHTML(opts: {
appPath: string;
workspace: string;
/** Base URL of the Windmill instance, for the "Open in player" link. */
playerBaseUrl?: string;
}): string {
// `<` escaped: this lands inside an inline <script>, where a `</script>` in
// any value (the app path comes from raw_app.yaml) would end the block.
const config = JSON.stringify({
appPath: opts.appPath,
workspace: opts.workspace,
playerBaseUrl: opts.playerBaseUrl ?? null,
savePath: RECORDER_SAVE_PATH,
}).replace(/</g, "\\u003c");
return `
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Windmill App Dev Recording</title>
<style>
* { margin: 0; padding: 0; box-sizing: border-box; }
html, body { height: 100%; }
body {
display: flex;
flex-direction: column;
font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', 'Roboto', sans-serif;
background: #18181b;
}
#wm-rec-bar {
display: flex;
align-items: center;
gap: 12px;
padding: 8px 12px;
color: #e4e4e7;
font-size: 13px;
border-bottom: 1px solid #27272a;
flex-wrap: wrap;
}
#wm-rec-bar button, #wm-rec-bar a.wm-rec-action {
font: inherit;
display: inline-flex;
align-items: center;
gap: 6px;
padding: 5px 12px;
border-radius: 6px;
border: 1px solid #3f3f46;
background: #27272a;
color: #e4e4e7;
cursor: pointer;
text-decoration: none;
}
#wm-rec-bar button:hover, #wm-rec-bar a.wm-rec-action:hover { background: #3f3f46; }
#wm-rec-bar button:disabled { opacity: 0.5; cursor: not-allowed; }
#wm-rec-toggle.recording { background: #dc2626; border-color: #dc2626; color: white; }
#wm-rec-toggle.recording:hover { background: #b91c1c; }
.wm-rec-dot { width: 8px; height: 8px; border-radius: 50%; background: #dc2626; }
#wm-rec-toggle.recording .wm-rec-dot { background: white; }
#wm-rec-status { color: #a1a1aa; }
#wm-rec-hint { margin-left: auto; color: #71717a; font-size: 12px; }
#wm-rec-hint code { background: #27272a; padding: 1px 5px; border-radius: 4px; }
#wm-rec-frame { flex: 1; width: 100%; border: 0; background: white; }
[hidden] { display: none !important; }
</style>
</head>
<body>
<div id="wm-rec-bar">
<button id="wm-rec-toggle"><span class="wm-rec-dot"></span><span id="wm-rec-toggle-label">Record</span></button>
<span id="wm-rec-status">Not recording</span>
<a id="wm-rec-open" class="wm-rec-action" target="_blank" rel="noopener" hidden>Open in player</a>
<button id="wm-rec-download" hidden>Download JSON</button>
<span id="wm-rec-hint">Passwords are masked. Mark sensitive elements with <code>data-wm-no-record</code></span>
</div>
<iframe id="wm-rec-frame" src="/"></iframe>
<script src="${RECORDER_BUNDLE_PATH}"></script>
<script>
(function () {
var config = ${config};
var iframe = document.getElementById('wm-rec-frame');
var toggle = document.getElementById('wm-rec-toggle');
var toggleLabel = document.getElementById('wm-rec-toggle-label');
var status = document.getElementById('wm-rec-status');
var openLink = document.getElementById('wm-rec-open');
var downloadBtn = document.getElementById('wm-rec-download');
var recorder = window.__wmillRecorder.createRawAppRecording();
var recording = null;
var ticker = null;
// The dev-server shim posts both the request and its answer up to the
// shell; the recorder reads the answer off the framed window, the way the
// deployed runner delivers it. Relaying is what makes a step wait for the
// job it launched instead of recording the spinner.
var pending = [];
var stranded = [];
window.addEventListener('message', function (e) {
var frameWindow = iframe.contentWindow;
if (!frameWindow || e.source !== frameWindow) return;
var data = e.data;
if (!data || typeof data.type !== 'string') return;
// A fresh document announced itself, so every call the previous one had
// in flight died with its WebSocket.
if (data.type === 'wmillDevReady') {
stranded = stranded.concat(pending);
pending = [];
return;
}
if (data.reqId === undefined) return;
if (!/Res$/.test(data.type)) {
pending.push(data.reqId);
return;
}
pending = pending.filter(function (id) { return id !== data.reqId });
frameWindow.postMessage(data, window.location.origin);
});
// The recorder keeps a reload's outstanding request ids on purpose (a
// deployed app is answered by its parent, which survives). Here the answer
// died with the frame's WebSocket, so Stop would wait out the whole job
// budget unless the shell settles them. Deferred past the load handlers:
// the recorder rebinds its listener onto the new document in one of them.
iframe.addEventListener('load', function () {
if (stranded.length === 0) return;
var ids = stranded;
stranded = [];
setTimeout(function () {
var frameWindow = iframe.contentWindow;
if (!frameWindow) return;
ids.forEach(function (reqId) {
frameWindow.postMessage({ type: 'unloadRes', reqId: reqId }, window.location.origin);
});
}, 0);
});
function setStatus(text) { status.textContent = text; }
function steps(n) { return n + (n === 1 ? ' step' : ' steps'); }
function tick() {
// Stop can wait a minute on a runnable the last step launched, so the
// toolbar has to say what it is waiting for rather than look wedged.
if (recorder.stopping) setStatus('Waiting for the last job to finish…');
else if (recorder.active) {
setStatus('Recording: ' + steps(recorder.stepCount));
}
}
function start() {
recording = null;
openLink.hidden = true;
downloadBtn.hidden = true;
if (!recorder.start(iframe, { appPath: config.appPath, workspace: config.workspace })) {
setStatus('Cannot record: the app document is unreachable');
return;
}
toggle.classList.add('recording');
toggleLabel.textContent = 'Stop';
tick();
ticker = setInterval(tick, 250);
}
async function stop() {
toggle.disabled = true;
setStatus('Finishing…');
// The ticker outlives the click: it is what reports the drain below.
recording = await recorder.stop();
clearInterval(ticker);
ticker = null;
toggle.classList.remove('recording');
toggleLabel.textContent = 'Record';
downloadBtn.hidden = false;
setStatus(steps(recording.steps.length) + ' recorded');
// Only now: starting a fresh recording drops the one being uploaded.
await save();
toggle.disabled = false;
}
async function save() {
try {
var res = await fetch(config.savePath, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(recording)
});
var body = await res.json();
if (!res.ok) throw new Error(body.error || res.statusText);
setStatus(steps(recording.steps.length) + ' saved to ' + body.file);
if (config.playerBaseUrl) {
var src = window.location.origin + config.savePath + '/' + body.file;
openLink.href = config.playerBaseUrl + 'replay?src=' + encodeURIComponent(src);
openLink.hidden = false;
}
} catch (e) {
setStatus('Recorded, but saving failed: ' + (e && e.message ? e.message : e));
}
}
toggle.addEventListener('click', function () {
if (recorder.active) stop();
else start();
});
downloadBtn.addEventListener('click', function () {
if (recording) recorder.download(recording);
});
})();
</script>
</body>
</html>
`;
}
File diff suppressed because one or more lines are too long
+7 -1
View File
@@ -17,7 +17,7 @@ import { deepEqual, readTextFile } from "../../utils/utils.ts";
import { replaceInlineScripts, repopulateFields } from "./app.ts";
import { createBundle, detectFrameworks } from "./bundle.ts";
import { APP_BACKEND_FOLDER } from "./app_metadata.ts";
import { APP_BACKEND_FOLDER, RECORDINGS_FOLDER } from "./app_metadata.ts";
import { writeIfChanged } from "../../utils/utils.ts";
import { yamlOptions } from "../sync/sync.ts";
import { applyExtraPermsDiff } from "../../core/extra_perms.ts";
@@ -319,6 +319,12 @@ async function collectAppFiles(
) {
continue;
}
// Session recordings, which the dev server only ever writes at the app
// root. Matched there alone, so an app of its own with a `recordings/`
// component folder still ships it.
if (basePath === "/" && entry.name === RECORDINGS_FOLDER) {
continue;
}
await readDirRecursive(fullPath + SEP, relativePath + "/");
} else if (entry.isFile()) {
// Skip generated/metadata files that shouldn't be part of the app
+36 -2
View File
@@ -68,11 +68,45 @@ function initWebSocket() {
initWebSocket()
/** A runnable call leaves this page over the WebSocket without touching the DOM,
* so the session recorder of \`wmill app dev --recording\` (which frames the app)
* has nothing else to tell it a step is still waiting on the backend. Announcing
* the request and its answer to the shell mirrors what the deployed runner posts
* across the same boundary. */
const framed = typeof window !== 'undefined' && window.parent !== window
function notifyRecorder(type: string, reqId: string) {
if (framed) window.parent.postMessage({ type, reqId }, window.location.origin)
}
// A reload takes the previous context and its WebSocket with it, so whatever it
// had in flight can never answer. Announcing a fresh module is how the shell
// learns those calls are dead: a message posted from the unloading document
// would be dropped with the realm that sent it, and this runs before any app
// code can issue a call of its own.
if (framed) {
window.parent.postMessage({ type: 'wmillDevReady' }, window.location.origin)
}
function tracked(type: string, reqId: string, resolve: (v: any) => void, reject: (e: any) => void) {
notifyRecorder(type, reqId)
let settled = false
const done = () => {
if (settled) return
settled = true
notifyRecorder(type + 'Res', reqId)
}
return {
resolve: (v: any) => { done(); resolve(v) },
reject: (e: any) => { done(); reject(e) }
}
}
async function doRequest(type: string, o: object) {
await wsReady
return new Promise((resolve, reject) => {
const reqId = Math.random().toString(36)
reqs[reqId] = { resolve, reject }
reqs[reqId] = tracked(type, reqId, resolve, reject)
ws?.send(JSON.stringify({ ...o, type, reqId }))
})
}
@@ -119,7 +153,7 @@ export function streamJob(
return new Promise(async (resolve, reject) => {
await wsReady
const reqId = Math.random().toString(36)
reqs[reqId] = { resolve, reject, onUpdate }
reqs[reqId] = { ...tracked('streamJob', reqId, resolve, reject), onUpdate }
ws?.send(JSON.stringify({ jobId, type: 'streamJob', reqId }))
})
}
+9 -1
View File
@@ -141,6 +141,7 @@ import { isExecutionModeAnonymous } from "../app/app.ts";
import {
APP_BACKEND_FOLDER,
generateAppLocksInternal,
RECORDINGS_FOLDER,
} from "../app/app_metadata.ts";
import {
isFlowPath,
@@ -1867,9 +1868,16 @@ export async function elementsToMap(
}
if (isRawAppFile(path)) {
const suffix = path.split(getFolderSuffix("raw_app") + SEP).pop();
// FSFSElement builds paths with the platform separator, while the checks
// below are written with "/": without normalizing, none of them match on
// Windows and the push collector's own exclusions become perpetual diffs.
const suffix = path
.split(getFolderSuffix("raw_app") + SEP)
.pop()
?.replaceAll(SEP, "/");
if (
suffix?.startsWith("dist/") ||
suffix?.startsWith(RECORDINGS_FOLDER + "/") ||
suffix == "wmill.d.ts" ||
suffix == "package-lock.json" ||
suffix == "DATATABLES.md"
+1
View File
@@ -6734,6 +6734,7 @@ app related commands
- \`--host <host:string>\` - Host to bind the dev server to
- \`--entry <entry:string>\` - Entry point file (default: index.ts for Svelte/Vue, index.tsx otherwise)
- \`--no-open\` - Don't automatically open the browser
- \`--recording\` - Frame the app in a shell with a Record button, to capture a replayable session recording of the app under development
- \`app lint [app_folder:string]\` - Lint a raw app folder to validate structure and buildability
- \`--fix\` - Attempt to fix common issues (not implemented yet)
- \`app new\` - create a new raw app from a template