diff --git a/.claude/hooks/allow-fileops-in-tmp.sh b/.claude/hooks/allow-fileops-in-tmp.sh index ce2ce81831..87ce6541aa 100755 --- a/.claude/hooks/allow-fileops-in-tmp.sh +++ b/.claude/hooks/allow-fileops-in-tmp.sh @@ -1,15 +1,31 @@ #!/usr/bin/env bash -# PreToolUse allowance for scratch file ops: auto-allow a single, plain, single-line -# `mkdir` / `cp` / `mv` / `touch` / `chmod` / `tar` / `unzip` whose every path operand -# resolves under /tmp. Anything else makes no decision (exit 0) and falls back to the normal -# permission flow — where `Bash(mv:*)` and `Bash(chmod:*)` in the `ask` list prompt. A -# PreToolUse `allow` overrides those ask rules, which is why this is a hook and not an allow -# rule: permission rules match a command prefix, so they can only constrain the FIRST operand. -# `cp /tmp/x ~/.zshrc` matches a `cp /tmp/` prefix, and requiring every operand is the point. +# PreToolUse allowance for scratch file ops: auto-allow `mkdir` / `cp` / `mv` / `touch` / +# `chmod` whose every path operand resolves inside one of the roots `path_class` recognizes — +# under /tmp, or inside a git working tree under $HOME — and `tar` / `unzip` confined to /tmp. +# Anything else makes no decision (exit 0) and falls back to the normal permission flow, except +# for `mv` and `chmod`: those get an explicit `ask`, the only prompt they get (see +# lib-guarded-verb.sh). # -# Requiring the sources under /tmp too (not just the destination) keeps this from becoming a -# read-exfiltration path around the `Read(**/.env)` / `Read(**/secrets/**)` deny rules: a copy -# out of the project into /tmp would land the content somewhere `Read(/tmp/**)` allows. +# The command is read one segment at a time, so chaining and line breaks carry no weight of +# their own: `cd /tmp/scratch && mv /tmp/a /tmp/b` is proved on the operands of the `mv`. A +# decision covers the whole command line, so `allow` is emitted only when every segment is one +# of these verbs proved here or a `cd` that resolved, AND exactly one of them writes (see the +# gate at the foot of this file — an earlier write can change what a later operand means). A +# line that mixes a proven op with some other command makes no decision instead and leaves that +# line to the normal permission flow, rather than waving an unexamined command through with it. +# +# This is a hook rather than an allow rule because permission rules match a command prefix, so +# they can only constrain the FIRST operand. `cp /tmp/x ~/.zshrc` matches a `cp /tmp/` prefix, +# and requiring every operand is the point. +# +# One operation may not straddle two roots, sources included, and a sibling checkout is a +# different root — `path_class` names the git tree, not just its kind. A copy out of a checkout +# into /tmp would be a read-exfiltration path around the `Read(**/secrets/**)` / `Read(**/*.pem)` +# deny rules, since the content lands where `Read(/tmp/**)` allows it to be read back, and one +# out of a repo the Read tool is not confined to would do the same for that repo. Keeping every +# operand of one operation inside a single root closes both without restating those rules here. +# The checkout root itself is what makes an in-repo `mv` or `chmod` auto-allowable: deleting a +# file there has never prompted, and moving or chmod-ing one is not the graver act. # # Deny-by-default tokenizing, in the same spirit as guard-rm-outside-tmp.sh: every path token # must consist only of alphanumerics and `. _ / -`. That set contains none of the characters @@ -17,12 +33,19 @@ # any glob character, so all of those forms fail by construction. `realpath -m` then resolves # `..` and existing symlinks, so `/tmp/link` pointing at /etc/passwd is caught. # +# `tar` and `unzip` keep the stricter rule — /tmp only, and absolute operands only — because +# their positional grammar makes a bare word ambiguous: `tar P -xf ...` is --absolute-names, +# not a file named P, and resolving it as a path would put an option in a root and allow it. +# The other five take relative operands, resolved against the working directory that `cd` +# tracking maintains, since for those a bare word really is a path (a GNU option starts with +# `-`, and the option allowlist below rejects the ones that would change symlink handling). +# # `tar` and `unzip` get their own parser: their write destination arrives as a flag VALUE # (`-C`, `-d`) rather than a positional, and a bundle like `-xzf` consumes the token after it. # Flags are an allowlist, not a denylist, so `-P` / `--absolute-names` — which turn off tar's # refusal to extract `..` and absolute member paths — defer rather than needing enumeration. -# Extraction additionally requires an explicit destination under /tmp, or a cwd already under -# /tmp, since otherwise members land in the project checkout. +# Extraction additionally requires an explicit destination under /tmp, or a working directory +# already under /tmp, since otherwise members land in the project checkout. # # Residual risk accepted: an archive whose members include a symlink pointing out of /tmp # followed by a write through it can still escape, because tar applies member symlinks as it @@ -31,6 +54,7 @@ # # Assumes GNU `realpath` (-m) and `jq`, both present in this repo's Linux dev env. set -uo pipefail +. "${BASH_SOURCE[0]%/*}/lib-guarded-verb.sh" input=$(cat) command -v jq >/dev/null 2>&1 || exit 0 @@ -38,25 +62,62 @@ cmd=$(printf '%s' "$input" | jq -r '.tool_input.command // empty' 2>/dev/null) [ -z "$cmd" ] && exit 0 cwd=$(printf '%s' "$input" | jq -r '.cwd // empty' 2>/dev/null) -# A newline separates commands, and the tokenizer below only reads the first line — defer. -case "$cmd" in *$'\n'*) exit 0 ;; esac +# Every bail-out below goes through `defer`: `mv` and `chmod` prompt from here, since no rule +# covers them, while the other verbs stay silent and leave the decision to the normal flow. +guarded=0 +for verb in mv chmod; do + runs_verb "$verb" "$cmd" && { guarded=1; break; } +done +defer() { + [ "$guarded" = 1 ] && decide ask "$1" + exit 0 +} -read -r -a toks <<< "$cmd" +has_substitution "$cmd" && defer "command substitution in the command line" -# 0 iff the token is charset-safe and resolves to a path strictly inside /tmp. +# 0 iff the token is a literal path this hook may reason about. A glob never auto-allows: bash +# expands it only after the hook has decided, so realpath sees the unexpanded pattern — +# `/tmp/link*` canonicalizes to itself and passes, then expands onto a symlink whose target is +# outside, and `cp` and `chmod` follow a command-line symlink, so that is a write to the target. +# (guard-rm-outside-tmp.sh can allow globs because `rm` unlinks the symlink rather than following +# it.) The charset holds none of the characters bash uses for quoting, expansion or separation. +literal_path() { + case "$1" in *[*?[]*) return 1 ;; esac + [ -z "$(printf '%s' "$1" | tr -d 'A-Za-z0-9._/-')" ] +} + +# Prints the root class of a path token, then the path it resolved to on a second line, +# resolving a relative one against the tracked working directory. Fails, printing nothing, +# when the token is unsafe to reason about or lands outside every root. +operand_class() { + local t="$1" canon alt cls alt_cls="" + literal_path "$t" || return 1 + case "$t" in + /*) canon=$(realpath -m -- "$t" 2>/dev/null) ;; + *) # A `cd` may fail at runtime and leave the command where it started, so a relative + # operand has to land in the same root either way. + [ -n "$seg_cwd" ] || return 1 + canon=$(realpath -m -- "$seg_cwd/$t" 2>/dev/null) + if [ -n "$alt_cwd" ]; then + alt=$(realpath -m -- "$alt_cwd/$t" 2>/dev/null) + [ -n "$alt" ] || return 1 + alt_cls=$(path_class "$alt") || return 1 + fi + ;; + esac + [ -n "$canon" ] || return 1 + cls=$(path_class "$canon") || return 1 + [ -n "$alt_cls" ] && [ "$alt_cls" != "$cls" ] && return 1 + # Class and resolved path together: a caller runs this in a command substitution, so a global + # set here would be set in that subshell and lost. + printf '%s\n%s' "$cls" "$canon" +} + +# 0 iff the token is charset-safe and resolves to a path strictly inside /tmp. The archive +# parser's stricter check; everything else goes through operand_class. under_tmp() { local t="$1" canon - # Globs never auto-allow. Bash expands them only after this hook has decided, so realpath - # sees the unexpanded pattern: `/tmp/link*` canonicalizes to itself and passes, then - # expands onto a symlink whose target is outside /tmp. chmod and cp follow command-line - # symlinks, so that is a write to the target. guard-rm-outside-tmp.sh can allow globs - # because `rm` unlinks the symlink itself rather than following it. - case "$t" in *[*?[]*) return 1 ;; esac - [ -n "$(printf '%s' "$t" | tr -d 'A-Za-z0-9._/-')" ] && return 1 - # Absolute only. Resolving a relative operand against the cwd makes any bare word look like - # a safe path whenever the cwd is under /tmp, while the tool itself reads it as an option: - # `tar P -xf ...` is --absolute-names, not ./P, and `cp /tmp/t -RL /tmp/o` is a - # dereferencing recursive copy, not a file named -RL. + literal_path "$t" || return 1 case "$t" in /*) ;; *) return 1 ;; esac canon=$(realpath -m -- "$t" 2>/dev/null) [ -n "$canon" ] || return 1 @@ -65,34 +126,16 @@ under_tmp() { return 1 } -allow() { - jq -nc --arg r "$1" '{hookSpecificOutput:{hookEventName:"PreToolUse",permissionDecision:"allow",permissionDecisionReason:$r}}' - exit 0 -} - -# Bare command word only; wrappers (`timeout cp`), env prefixes, and `/bin/cp` defer. -# Options are an allowlist per command, so anything that changes how symlinks are followed -# defers instead of needing enumeration. `cp -L` / `-H` matter most: they dereference while -# recursing, which copies the CONTENT of a symlink target from outside /tmp into a scratch -# dir that `Read(/tmp/**)` then exposes. Plain `-r` and `-a` (which implies `-d`) recreate -# such a symlink as a symlink instead, so no outside content is materialized. -case "${toks[0]:-}" in - mkdir) takes_mode=0; ok_opts='pv' ;; - cp) takes_mode=0; ok_opts='rRvfnpa' ;; - mv) takes_mode=0; ok_opts='vfn' ;; - touch) takes_mode=0; ok_opts='acmv' ;; - chmod) takes_mode=1; ok_opts='Rvfc' ;; # chmod's first operand is a mode, not a path - tar) ok_flags='xctzjJavfC'; val_flags='fC' ;; - unzip) ok_flags='oqnljvd'; val_flags='d' ;; - *) exit 0 ;; -esac - -# ---------------------------------------------------------------- tar / unzip -if [ -n "${ok_flags:-}" ]; then - saw_archive=0 saw_dest=0 extracting=0 listing=0 end_opts=0 - i=1 - while [ "$i" -lt "${#toks[@]}" ]; do - t="${toks[$i]}" +# Proves one `tar` / `unzip` segment ($1 = the verb), whose tokens are in SEG_TOKS. +check_archive_segment() { + local verb="$1" ok_flags val_flags t flags val + local saw_archive=0 saw_dest=0 extracting=0 listing=0 end_opts=0 i=1 + case "$verb" in + tar) ok_flags='xctzjJavfC'; val_flags='fC' ;; + unzip) ok_flags='oqnljvd'; val_flags='d' ;; + esac + while [ "$i" -lt "${#SEG_TOKS[@]}" ]; do + t="${SEG_TOKS[$i]}" i=$((i + 1)) if [ "$end_opts" = 0 ]; then [ "$t" = "--" ] && { end_opts=1; continue; } @@ -101,18 +144,18 @@ if [ -n "${ok_flags:-}" ]; then flags="${t#-}" # Allowlist: a long option, -P/--absolute-names, --transform, -I and friends all # leave a residue here and defer rather than being enumerated as denials. - [ -n "$(printf '%s' "$flags" | tr -d "$ok_flags")" ] && exit 0 + [ -n "$(printf '%s' "$flags" | tr -d "$ok_flags")" ] && defer "unrecognized option \`$t\`" case "$flags" in *x*) extracting=1 ;; esac - case "${toks[0]}$flags" in unzip*[lv]*) listing=1 ;; esac + case "$verb$flags" in unzip*[lv]*) listing=1 ;; esac # A flag consuming the next token must be alone in its bundle's final position # (`-xzf a.tar`), else the token it eats is ambiguous. - case "${flags%?}" in *[$val_flags]*) exit 0 ;; esac + case "${flags%?}" in *[$val_flags]*) defer "ambiguous option bundle \`$t\`" ;; esac case "${flags: -1}" in [$val_flags]) - val="${toks[$i]:-}" + val="${SEG_TOKS[$i]:-}" i=$((i + 1)) - [ -n "$val" ] || exit 0 - under_tmp "$val" || exit 0 + [ -n "$val" ] || defer "option \`$t\` has no value" + under_tmp "$val" || defer "\`$val\` is outside /tmp" case "${flags: -1}" in f) saw_archive=1 ;; C | d) saw_dest=1 ;; @@ -126,54 +169,153 @@ if [ -n "${ok_flags:-}" ]; then # Positional. For tar these are sources (create) or member names (extract); for unzip the # first is the archive. Requiring every one under /tmp is conservative for member names, # which are not filesystem paths — those defer rather than being wrongly allowed. - under_tmp "$t" || exit 0 - [ "${toks[0]}" = "unzip" ] && saw_archive=1 + under_tmp "$t" || defer "\`$t\` is outside /tmp" + [ "$verb" = "unzip" ] && saw_archive=1 done - [ "$saw_archive" = 1 ] || exit 0 # tar without -f reads a tape/stdin; unzip needs an archive + # tar without -f reads a tape/stdin; unzip needs an archive + [ "$saw_archive" = 1 ] || defer "no archive operand" # Writes land relative to the working directory unless a destination was given. `unzip -l` # and `-v` only list, so they need no destination. - if [ "$extracting" = 1 ] || { [ "${toks[0]}" = "unzip" ] && [ "$listing" = 0 ]; }; then - [ "$saw_dest" = 1 ] || under_tmp "${cwd:-$PWD}" || exit 0 + if [ "$extracting" = 1 ] || { [ "$verb" = "unzip" ] && [ "$listing" = 0 ]; }; then + # An extraction with no destination lands in the working directory. Word splitting cannot + # tell a `cd` inside a quoted string from one the shell runs, and believing a false one + # would put an archive's members in the checkout, so once any `cd` is in the line only an + # explicit destination will do. + [ "$saw_dest" = 1 ] \ + || { [ "$saw_cd" = 0 ] && [ -n "$seg_cwd" ] && under_tmp "$seg_cwd"; } \ + || defer "extraction target is outside /tmp" fi - allow "archive paths and extraction target are under /tmp" -fi +} -# ------------------------------------------- mkdir / cp / mv / touch / chmod -path_operand=0 -seen_mode=0 -end_opts=0 -i=1 -while [ "$i" -lt "${#toks[@]}" ]; do - t="${toks[$i]}" - i=$((i + 1)) +# Proves one `mkdir` / `cp` / `mv` / `touch` / `chmod` segment ($1 = the verb), whose tokens +# are in SEG_TOKS. +check_fileops_segment() { + local verb="$1" takes_mode ok_opts t cls resolved seen_class="" + local path_operand=0 seen_mode=0 end_opts=0 i=1 rel_operand=0 + local -a ops=() + # Options are an allowlist per command, so anything that changes how symlinks are followed + # defers instead of needing enumeration. `cp -L` / `-H` matter most: they dereference while + # recursing, which copies the CONTENT of a symlink target from outside /tmp into a scratch + # dir that `Read(/tmp/**)` then exposes. Plain `-r` and `-a` (which implies `-d`) recreate + # such a symlink as a symlink instead, so no outside content is materialized. + case "$verb" in + mkdir) takes_mode=0; ok_opts='pv' ;; + cp) takes_mode=0; ok_opts='rRvfnpa' ;; + mv) takes_mode=0; ok_opts='vfn' ;; + touch) takes_mode=0; ok_opts='acmv' ;; + chmod) takes_mode=1; ok_opts='Rvfc' ;; # chmod's first operand is a mode, not a path + esac + while [ "$i" -lt "${#SEG_TOKS[@]}" ]; do + t="${SEG_TOKS[$i]}" + i=$((i + 1)) - if [ "$end_opts" = 0 ]; then - [ "$t" = "--" ] && { end_opts=1; continue; } - # Checked at any position, not just before the first operand: GNU utils permute, so - # `cp /tmp/tree -RL /tmp/out` still enables dereferencing recursion. - case "$t" in - -?*) - # Allowlist: long options and the dereferencing flags leave a residue and defer. - [ -n "$(printf '%s' "${t#-}" | tr -d "$ok_opts")" ] && exit 0 - continue - ;; - esac - fi + if [ "$end_opts" = 0 ]; then + [ "$t" = "--" ] && { end_opts=1; continue; } + # Checked at any position, not just before the first operand: GNU utils permute, so + # `cp /tmp/tree -RL /tmp/out` still enables dereferencing recursion. + case "$t" in + -?*) + # Allowlist: long options and the dereferencing flags leave a residue and defer. + [ -n "$(printf '%s' "${t#-}" | tr -d "$ok_opts")" ] && defer "unrecognized option \`$t\`" + continue + ;; + esac + fi - # chmod: consume the mode operand without a path check. Octal, or symbolic clauses. - if [ "$takes_mode" = 1 ] && [ "$seen_mode" = 0 ]; then - case "$t" in - [0-7] | [0-7][0-7] | [0-7][0-7][0-7] | [0-7][0-7][0-7][0-7]) ;; - *) printf '%s' "$t" | grep -Eq '^[ugoa]*[+=-][rwxXst]*(,[ugoa]*[+=-][rwxXst]*)*$' || exit 0 ;; - esac - seen_mode=1 - continue - fi + # chmod: consume the mode operand without a path check. Octal, or symbolic clauses. + if [ "$takes_mode" = 1 ] && [ "$seen_mode" = 0 ]; then + case "$t" in + [0-7] | [0-7][0-7] | [0-7][0-7][0-7] | [0-7][0-7][0-7][0-7]) ;; + *) printf '%s' "$t" | grep -Eq '^[ugoa]*[+=-][rwxXst]*(,[ugoa]*[+=-][rwxXst]*)*$' || defer "unrecognized mode \`$t\`" ;; + esac + seen_mode=1 + continue + fi - under_tmp "$t" || exit 0 - path_operand=1 + resolved=$(operand_class "$t") || defer "\`$t\` is outside /tmp and not inside a git checkout in \$HOME" + cls="${resolved%%$'\n'*}" + # Every operand of one operation stays in one root: see the exfiltration note above. + [ -n "$seen_class" ] && [ "$cls" != "$seen_class" ] && defer "\`$t\` puts this $verb across two roots" + seen_class="$cls" + ops+=("${resolved#*$'\n'}") + case "$t" in /*) ;; *) rel_operand=1 ;; esac + path_operand=1 + done + + [ "$path_operand" = 1 ] || defer "no path operand" + + # In directory form the command writes a path it does not name: `cp x dir` writes `dir/x`, + # and `cp` follows that child when it is a symlink — this checkout is full of them, every + # `*_ee.rs` pointing into the sibling EE repo. Deriving that child would mean reproducing + # which name the tool picks (the operand as written, not as resolved — a symlinked source + # keeps its own name) and how deep `-r` recurses. The form is left unproved instead. + case "$verb" in + cp | mv) + [ "${#ops[@]}" -ge 2 ] || return 0 + # Whether the destination is an existing directory is itself a question about which of + # the two candidate working directories the command ran in, and only one of them is in + # `ops`. A `cd` that fails at runtime would otherwise let the form through: the + # destination resolved against the directory the command never reached is some path that + # does not exist, while the one it actually ran in is a directory full of symlinks. + [ -n "$alt_cwd" ] && [ "$rel_operand" = 1 ] \ + && defer "a relative operand after a \`cd\` lands in one of two directories" + [ -d "${ops[-1]}" ] \ + && defer "\`${ops[-1]}\` already exists as a directory, so this $verb writes a path it does not name" + ;; + esac +} + +split_segments "$cmd" +seg_cwd="${cwd:-$PWD}" +alt_cwd="" # where a `cd` that failed would have left the command +saw_cd=0 # a `cd` moved the working directory somewhere +proved=0 # how many ops came out inside a single root +only_ours=1 # ... and nothing else shares the command line + +for seg in "${SEGMENTS[@]}"; do + segment_tokens "$seg" + case "${SEG_TOKS[0]:-}" in + "") continue ;; + mkdir | cp | mv | touch | chmod) + check_fileops_segment "${SEG_TOKS[0]}" + proved=$((proved + 1)) + continue + ;; + tar | unzip) + check_archive_segment "${SEG_TOKS[0]}" + proved=$((proved + 1)) + continue + ;; + cd) + # A `cd` writes nothing, so it never blocks an allow; it only moves where a later relative + # operand points, to one of the two candidates `apply_cd` describes. + if [ "$saw_cd" = 0 ] && new_cwd=$(apply_cd "$seg_cwd" "${SEG_TOKS[@]:1}"); then + alt_cwd="$seg_cwd" + seg_cwd="$new_cwd" + else + # Not the harmless segment an allow assumes: whatever this guard could not account for + # may be a redirect, and a redirect writes. Leave the line to the normal flow. + seg_cwd="" alt_cwd="" + only_ours=0 + fi + saw_cd=1 + continue + ;; + esac + # Some other command shares the line. If an `mv` or `chmod` runs inside it after all — behind + # a wrapper, an env prefix or a path — this hook cannot say what it writes to. + for verb in mv chmod; do + segment_runs_verb "$verb" "$seg" && defer "$verb is not the leading command word in \`$seg\`" + done + only_ours=0 done -[ "$path_operand" = 1 ] || exit 0 -allow "every path operand is under /tmp" +# Exactly one write per line. Each segment is proved against the filesystem as it stands now, +# and an earlier write can change what a later operand means: `cp -r /tmp/tree /tmp/live` that +# recreates a symlink out of /tmp turns `/tmp/live/link` — a path under /tmp when this ran — +# into a write through that symlink. Deletes compose safely and guard-rm-outside-tmp.sh allows +# several, because `rm` unlinks a symlink rather than following it. +[ "$proved" -ge 1 ] || exit 0 +[ "$only_ours" = 1 ] && [ "$proved" = 1 ] && decide allow "every path operand is inside a single root" +exit 0 diff --git a/.claude/hooks/guard-rm-outside-tmp.sh b/.claude/hooks/guard-rm-outside-tmp.sh index 66d4dd27b5..4d253ffc62 100755 --- a/.claude/hooks/guard-rm-outside-tmp.sh +++ b/.claude/hooks/guard-rm-outside-tmp.sh @@ -1,14 +1,17 @@ #!/usr/bin/env bash -# PreToolUse guard for `rm`: auto-allow ONLY a single, plain, single-line `rm` whose every -# operand is a whitelisted target — under /tmp, or inside a git working tree located in $HOME -# (a version-controlled project dir). Anything else makes no decision (exit 0) and falls back -# to the normal permission flow, where the `Bash(rm:*)` ask rule prompts (classifier as a -# backstop). +# PreToolUse guard for `rm`: auto-allow deletes whose every operand is a whitelisted target — +# under /tmp, or inside a git working tree located in $HOME (a version-controlled project dir). +# Any other command that runs `rm` gets an explicit `ask`, which is the ordinary permission +# prompt and the only one `rm` gets (see lib-guarded-verb.sh); a command that runs no `rm` at +# all makes no decision (exit 0). # -# The git-tree allowance trades on "this is a project under version control" being lower-stakes -# than a delete elsewhere — NOT on full recoverability: committed content is restorable via git, -# but untracked / .gitignore'd / uncommitted content, and an independent nested repo's history -# under a recursively-deleted parent, are NOT. Accepted as a deliberate convenience tradeoff. +# The command is read one segment at a time, so chaining and line breaks carry no weight of +# their own: `rm -f /tmp/a && rm -rf /tmp/b` is two deletes, each proved on its own operands. +# A decision covers the whole command line, so `allow` is emitted only when every segment is +# an `rm` this guard proved or a `cd` it could resolve. A line that mixes a proven `rm` with +# some other command makes no decision instead and leaves that line to the normal permission +# flow: the delete is not what needed a prompt, and waving the rest of the line through with +# it would turn a trailing `rm -f /tmp/x` into a way to auto-approve anything. # # Deny-by-default: every token must consist only of a safe character set (alphanumerics, # `. _ / -` and glob chars `* ? [ ]`). That set contains none of the characters bash uses for @@ -17,15 +20,16 @@ # and existing symlinks (so a symlink out of the allowed roots is caught), and a wildcard in a # non-final path segment is refused because it can expand through a symlink realpath can't see. # -# The git-repo allowance covers targets inside a git working tree under $HOME, and the tree's -# own root folder only when it is a linked worktree (`.git` is a pointer file, so history in -# the main repo survives); a primary checkout's root (`.git` is a history dir) and any `.git` -# path are never auto-allowed. Globs auto-allow only under /tmp — elsewhere their expansion +# Which targets those two roots cover, and the tradeoff they rest on, is `path_class` in +# lib-guarded-verb.sh. Globs auto-allow only under /tmp — elsewhere their expansion # could reach `.git` or a dotfile the literal checks never see. Relative operands resolve -# against the command's cwd (from the hook input). A PreToolUse `allow` overrides the ask rule. +# against the working directory the command runs from, which a `cd` in an earlier segment +# moves; once a `cd` is one this guard cannot resolve, that directory is unknown and a +# relative operand can no longer be proved. # # Assumes GNU `realpath` (-m) and `jq`, both present in this repo's Linux dev env. set -uo pipefail +. "${BASH_SOURCE[0]%/*}/lib-guarded-verb.sh" input=$(cat) command -v jq >/dev/null 2>&1 || exit 0 @@ -33,74 +37,104 @@ cmd=$(printf '%s' "$input" | jq -r '.tool_input.command // empty' 2>/dev/null) [ -z "$cmd" ] && exit 0 cwd=$(printf '%s' "$input" | jq -r '.cwd // empty' 2>/dev/null) -# A newline separates commands, and the tokenizer below only reads the first line — defer. -case "$cmd" in *$'\n'*) exit 0 ;; esac - -read -r -a toks <<< "$cmd" -# Bare leading `rm` only; wrappers (`timeout rm`), env prefixes, and `/bin/rm` defer. -[ "${toks[0]:-}" = "rm" ] || exit 0 - -# 0 (allow) iff the canonical path is an auto-allowable rm target: under /tmp, or strictly -# inside a git working tree located under $HOME. The walk stops at $HOME, so a dotfiles repo at -# ~ can't make all of $HOME deletable, and top-level ~ files stay protected. -allowed_target() { - local canon="$1" d root="" - case "$canon" in /tmp/?*) return 0 ;; esac - [ -n "${HOME:-}" ] || return 1 - case "$canon" in "$HOME"/?*) ;; *) return 1 ;; esac - case "$canon" in *"/.git" | *"/.git/"*) return 1 ;; esac # protect history, not recoverable - d="$canon" - while [ "$d" != "/" ] && [ "$d" != "$HOME" ]; do - [ -e "$d/.git" ] && { root="$d"; break; } - d=$(dirname "$d") - done - [ -n "$root" ] || return 1 # not inside a git working tree under $HOME - if [ "$canon" = "$root" ]; then - # Deleting the repo root folder itself: allow only for a linked worktree, whose `.git` is - # a file/pointer so the history lives in the main repo and survives. A primary checkout's - # `.git` is a directory holding the history, so deleting it is unrecoverable — defer. - [ -f "$root/.git" ] && return 0 - return 1 - fi - return 0 +# Every bail-out below goes through `defer`, so the forms this guard refuses to reason about — +# wrapped, quoted, expanded — still reach the user as a prompt whenever an `rm` runs among them. +runs_verb rm "$cmd" && guarded=1 || guarded=0 +defer() { + [ "$guarded" = 1 ] && decide ask "$1" + exit 0 } -had_operand=0 -end_opts=0 -i=1 -while [ "$i" -lt "${#toks[@]}" ]; do - t="${toks[$i]}" - i=$((i + 1)) - # Whitelist every token (flags included, so an operator hidden in a flag like `-rf;rm` - # can't slip past): any character outside the safe set makes it unsafe to reason about. - [ -n "$(printf '%s' "$t" | tr -d 'A-Za-z0-9._/*?[]-')" ] && exit 0 - # A glob in an option-looking token (`-[-]`) can expand to `--` and turn a later `-name` - # into an operand — never a real option, so defer. - case "$t" in -*[*?[]*) exit 0 ;; esac - if [ "$end_opts" = 0 ]; then - [ "$t" = "--" ] && { end_opts=1; continue; } - # Skip real options only before the first operand. A bare `-` is a filename, and under - # POSIXLY_CORRECT GNU rm stops option parsing at the first operand, so a later `-name` - # is a filename too — validate it rather than skipping it. - if [ "$had_operand" = 0 ]; then - case "$t" in -?*) continue ;; esac +has_substitution "$cmd" && defer "command substitution in the command line" + +# Proves one `rm` segment, whose tokens are in SEG_TOKS with `rm` at index 0, resolving relative +# operands against $seg_cwd. Returns only once every operand is an auto-allowable target; +# anything it cannot prove defers instead. +check_rm_segment() { + local i=1 t canon candidates had_operand=0 end_opts=0 + while [ "$i" -lt "${#SEG_TOKS[@]}" ]; do + t="${SEG_TOKS[$i]}" + i=$((i + 1)) + # Whitelist every token (flags included, so an operator hidden in a flag like `-rf;rm` + # can't slip past): any character outside the safe set makes it unsafe to reason about. + [ -n "$(printf '%s' "$t" | tr -d 'A-Za-z0-9._/*?[]-')" ] && defer "unsafe characters in \`$t\`" + # A glob in an option-looking token (`-[-]`) can expand to `--` and turn a later `-name` + # into an operand — never a real option, so defer. + case "$t" in -*[*?[]*) defer "glob inside the option \`$t\`" ;; esac + if [ "$end_opts" = 0 ]; then + [ "$t" = "--" ] && { end_opts=1; continue; } + # Skip real options only before the first operand. A bare `-` is a filename, and under + # POSIXLY_CORRECT GNU rm stops option parsing at the first operand, so a later `-name` + # is a filename too — validate it rather than skipping it. + if [ "$had_operand" = 0 ]; then + case "$t" in -?*) continue ;; esac + fi fi - fi - had_operand=1 - # No wildcard in a non-final path segment (`a/*/b`): it can expand through a symlink - # realpath can't see. A slashless glob (`*.rs`) is a final-segment match — fine. - case "$t" in */*) case "${t%/*}" in *[*?[]*) exit 0 ;; esac ;; esac - case "$t" in - /*) canon=$(realpath -m -- "$t" 2>/dev/null) ;; - *) canon=$(realpath -m -- "${cwd:-$PWD}/$t" 2>/dev/null) ;; + had_operand=1 + # No wildcard in a non-final path segment (`a/*/b`): it can expand through a symlink + # realpath can't see. A slashless glob (`*.rs`) is a final-segment match — fine. + case "$t" in */*) case "${t%/*}" in *[*?[]*) defer "glob in a non-final segment of \`$t\`" ;; esac ;; esac + # A relative operand has as many candidate paths as the command has candidate working + # directories, and every one of them has to be auto-allowable: a `cd` that fails at runtime + # leaves the delete running in the directory it started in. + case "$t" in + /*) candidates=$(realpath -m -- "$t" 2>/dev/null) ;; + *) [ -n "$seg_cwd" ] || defer "\`$t\` is relative to a working directory this guard cannot pin down" + candidates=$(realpath -m -- "$seg_cwd/$t" 2>/dev/null) + [ -n "$alt_cwd" ] && candidates="$candidates +$(realpath -m -- "$alt_cwd/$t" 2>/dev/null)" + ;; + esac + while IFS= read -r canon; do + [ -n "$canon" ] || defer "cannot resolve \`$t\`" + # A glob may auto-allow only under /tmp, where everything is deletable. Elsewhere its + # expansion could match `.git`, a dotfile like `.*`, or a nested checkout root that the + # literal-path checks never see — so require literal operands in git repos. + case "$t" in *[*?[]*) case "$canon" in /tmp/?*) ;; *) defer "glob \`$t\` is outside /tmp" ;; esac ;; esac + path_class "$canon" >/dev/null || defer "\`$canon\` is outside /tmp and not inside a git checkout in \$HOME" + done <<< "$candidates" + done + [ "$had_operand" = 1 ] || defer "no operand" +} + +split_segments "$cmd" +seg_cwd="${cwd:-$PWD}" +alt_cwd="" # where a `cd` that failed would have left the command +saw_cd=0 +proved=0 # at least one `rm` segment came out auto-allowable +only_ours=1 # ... and nothing else shares the command line + +for seg in "${SEGMENTS[@]}"; do + segment_tokens "$seg" + case "${SEG_TOKS[0]:-}" in + "") continue ;; + rm) + check_rm_segment + proved=1 + continue + ;; + cd) + # A `cd` writes nothing, so it never blocks an allow; it only moves where a later relative + # operand points, to one of the two candidates `apply_cd` describes. + if [ "$saw_cd" = 0 ] && new_cwd=$(apply_cd "$seg_cwd" "${SEG_TOKS[@]:1}"); then + alt_cwd="$seg_cwd" + seg_cwd="$new_cwd" + else + # Not the harmless segment an allow assumes: whatever this guard could not account for + # may be a redirect, and a redirect writes. Leave the line to the normal flow. + seg_cwd="" alt_cwd="" + only_ours=0 + fi + saw_cd=1 + continue + ;; esac - [ -n "$canon" ] || exit 0 - # A glob may auto-allow only under /tmp, where everything is deletable. Elsewhere its - # expansion could match `.git`, a dotfile like `.*`, or a nested checkout root that the - # literal-path checks never see — so require literal operands in git repos. - case "$t" in *[*?[]*) case "$canon" in /tmp/?*) ;; *) exit 0 ;; esac ;; esac - allowed_target "$canon" || exit 0 + # Some other command shares the line. If an `rm` runs inside it after all — behind a wrapper, + # an env prefix or a path — this guard cannot say what it deletes. + segment_runs_verb rm "$seg" && defer "rm is not the leading command word in \`$seg\`" + only_ours=0 done -[ "$had_operand" = 1 ] || exit 0 -jq -nc '{hookSpecificOutput:{hookEventName:"PreToolUse",permissionDecision:"allow",permissionDecisionReason:"rm operands are under /tmp or inside a git checkout in $HOME"}}' +[ "$proved" = 1 ] || exit 0 +[ "$only_ours" = 1 ] && decide allow 'rm operands are under /tmp or inside a git checkout in $HOME' +exit 0 diff --git a/.claude/hooks/lib-guarded-verb.sh b/.claude/hooks/lib-guarded-verb.sh new file mode 100644 index 0000000000..6ef76a5466 --- /dev/null +++ b/.claude/hooks/lib-guarded-verb.sh @@ -0,0 +1,268 @@ +#!/usr/bin/env bash +# Sourced by the PreToolUse guards; not a hook itself. +# +# A permission rule beats a hook: an `ask` rule prompts whatever a PreToolUse hook returns, which +# makes the hook's `allow` dead weight. So settings.json carries no `ask` rule for `rm`, `mv` or +# `chmod`, and the guards own both halves — `allow` what they can prove safe, `ask` for the rest. +# Removing a guard's `ask` path therefore removes that verb's prompt entirely. +# +# `set -f` is global to the sourcing script so that the unquoted word split in runs_verb cannot +# expand a glob operand against the filesystem. Neither guard relies on pathname expansion. +set -f + +# 0 iff ($1) starts with a command that only reads its input. An allowlist, because the +# opposite — naming the shells to avoid — would have to be complete: an unlisted one (`ash`, +# `rbash`, `busybox sh`) executes the body while the guard calls it data. Unrecognized here only +# costs a prompt. Text with no command word in it is not evidence of a reader either. +reads_only() { + local w + for w in $1; do + w="${w//[\"\'\\]/}" + w="${w%%<<*}" # a redirect needs no space: `cat<'* | '<'*) continue ;; esac + case "${w##*/}" in + cat | tee | head | tail | grep | sed | awk | sort | uniq | wc | cut | diff | tr \ + | jq | yq | gh | git | base64 | column | envsubst | python | python3 | node \ + | psql | mysql | sqlite3 | wmill) return 0 ;; + esac + return 1 + done + return 1 +} + +# A heredoc body is data rather than commands only when its delimiter is quoted and nothing +# executes it; a rule doesn't match a verb inside such a body, and a PR body would otherwise +# prompt for every `rm` in its text. Dropping one needs all of that, a delimiter that could +# really open a heredoc, and a terminator line — failing any part, nothing is dropped. +strip_heredoc_bodies() { + local -a lines=() + local line delim rest after trimmed piped quoted i j n + while IFS= read -r line; do lines+=("$line"); done <<< "$1" + n=${#lines[@]} + i=0 + while [ "$i" -lt "$n" ]; do + line="${lines[$i]}" + printf '%s\n' "$line" + i=$((i + 1)) + # A `#` opens a comment, and a comment opens no heredoc — including mid-line, as in + # `echo hi # cat < f`); prose after it means the `<<` sits + # inside a string (`echo "cat < f"` ends its redirect-looking text with the + # closing quote. That also refuses `cat < "f"`, a real heredoc, which only over-prompts. + after="${rest#"$delim"}" + after="${after#"${after%%[![:space:]]*}"}" + case "$after" in + *[\"\'\\]*) continue ;; + "" | '>'* | '<'* | '|'* | [0-9]'>'* | [0-9]'<'*) ;; + *) continue ;; + esac + # A real delimiter is a bare word or one wholly quoted (`<<'EOF'`, `<<\EOF`); a stray quote + # left in it means the `<<` was quoted prose. + quoted=0 + case "$delim" in + \'*\' | \"*\") delim="${delim:1:${#delim}-2}" quoted=1 ;; + \\?*) delim="${delim#\\}" quoted=1 ;; + esac + case "$delim" in + [A-Za-z_]*) ;; + *) continue ;; + esac + case "$delim" in *[!A-Za-z0-9_]*) continue ;; esac + # Only a quoted delimiter makes the body inert. Unquoted, the shell expands it before the + # consumer ever sees it, so a `$(rm -rf ~)` written in the body runs whatever reads it. + [ "$quoted" = 1 ] || continue + # Two commands can see this body: the one the `<<` belongs to, and anything it is then piped + # into. The first is whatever was started last before the `<<`, so splitting the text there + # on separators and substitution openers and taking the final piece finds `cat` in + # `--title "fix(agents): …" --body "$(cat <<`, without the title's parenthesis standing in + # for it. A line continuation (`bash \` then `<<'EOF'`) leaves that piece empty, which is + # not evidence of a reader and so keeps the body. + reads_only "$(printf '%s' "${line%%<<*}" | tr ';&|()`' '\n' | grep -v '^[[:space:]]*$' | tail -1)" || continue + piped="$after" + while :; do + case "$piped" in *'|'*) ;; *) break ;; esac + piped="${piped#*|}" + reads_only "${piped%%|*}" || continue 2 + done + j="$i" + while [ "$j" -lt "$n" ]; do + trimmed="${lines[$j]#"${lines[$j]%%[![:space:]]*}"}" + [ "$trimmed" = "$delim" ] && break + j=$((j + 1)) + done + [ "$j" -lt "$n" ] && i=$((j + 1)) + done +} + +# 0 iff ($1) runs as a command word in ($2), which must already be one +# segment (no separator left in it). Wrapper, env-prefix and `/bin/` forms all count. +segment_runs_verb() { + local verb="$1" w wrapped=0 + for w in $2; do + # The shell strips quotes and backslashes before it looks up the command, so `'rm'` and + # `r\m` run rm and have to compare equal to it. + w="${w//[\"\'\\]/}" + case "$w" in + "$verb" | */"$verb") return 0 ;; + *=*) ;; # leading env assignment + -* | *'>'* | *'<'*) ;; # a flag, or a leading redirect + [0-9]*) [ "$wrapped" = 1 ] || break ;; # a wrapper's duration, not `1:` in prose + '!' | '{' | '}' | if | then | elif | else | while | until | do) ;; # never the command + timeout | time | nice | nohup | stdbuf | command | builtin | noglob | xargs | sudo | env) + wrapped=1 ;; + # A wrapper's option value is indistinguishable from a command name (`stdbuf -o L rm`), + # so past a wrapper the scan runs to the end of the segment instead of stopping at the + # first ordinary word. Before one, that word is the command and the verb cannot follow + # it. Nothing bounds the scan: a wrapper takes unboundedly many operands + # (`env -u A -u B ...`), and any cutoff — a word count, or stopping at the first quoted + # word — drops the prompt for a real `sudo -u 'root' rm`. Prose after a wrapper is the + # price, and it only over-prompts. + *) [ "$wrapped" = 1 ] || break ;; + esac + done + return 1 +} + +# Splits ($1) into its command segments, into the global array SEGMENTS. Every guard +# reasons one segment at a time, so `a && b` is two commands here rather than one unparsable +# blob, and a newline is a separator like any other. +# +# The split set carries more than `; & |` and newlines: `$(`, backticks and `( )` open a nested +# command, and a separator that only ended statements would read `echo $(rm -rf ~)` as an +# `echo`. Braces are handled as words rather than separators, since splitting on them cuts +# `xargs -I {} … rm` in half and strands the `rm` in a segment that no longer knows a wrapper +# preceded it. +# +# `tr` and not `${1//[...]}`: a `}` inside the bracket expression closes the expansion itself, +# which silently leaves the command unsplit and every separator unseen. +split_segments() { + local seg + SEGMENTS=() + while IFS= read -r seg; do SEGMENTS+=("$seg"); done <<< "$(strip_heredoc_bodies "$1" | tr ';&|()`' '\n')" +} + +# 0 iff ($1) carries a command substitution outside a heredoc body. A substitution is +# concatenated into the word it sits in, and splitting on its opener cuts that word in half: +# `/tmp/a/`printf ../../etc`` would be proved as `/tmp/a/`, with the traversal validated as an +# unrelated segment. Nothing here can evaluate it, so a guard proves nothing about such a +# command. Heredoc bodies are excepted — those are data the split has already dropped. +has_substitution() { + case "$(strip_heredoc_bodies "$1")" in + *'$('* | *'`'*) return 0 ;; + esac + return 1 +} + +# Reads ($1) into the global array SEG_TOKS, dropping the shell keywords that can +# precede a command word so that `then rm -rf x` is analyzed as the `rm` it runs. Word +# splitting only: quotes are left in the token and fail the guards' charset check downstream, +# which is what keeps `rm -rf "$HOME/x"` unprovable. +segment_tokens() { + SEG_TOKS=() + read -r -a SEG_TOKS <<< "$1" + while [ "${#SEG_TOKS[@]}" -gt 0 ]; do + case "${SEG_TOKS[0]}" in + '!' | '{' | '}' | if | then | elif | else | while | until | do) SEG_TOKS=("${SEG_TOKS[@]:1}") ;; + *) break ;; + esac + done +} + +# Prints the directory a `cd` lands in, given the current one ($1) and the tokens after the +# `cd` ($2...). Fails, printing nothing, when the destination cannot be resolved — a variable, +# `-`, an option, a relative path, no operand at all (`cd` alone is $HOME), or more than one. +# +# Resolving says nothing about whether the `cd` will SUCCEED: the destination may not exist, and +# `;` runs the next command anyway, leaving it in the directory it started in. So a caller may +# never treat this as the working directory outright — it is one of two candidates, and a +# relative operand has to be provable against the one the command started in as well. That also +# makes a `cd` word splitting invented out of quoted text harmless: it can only add a candidate, +# never drop one. Past the first `cd` the branching outruns two candidates, so a caller that +# sees a second gives up on relative operands entirely. +apply_cd() { + local cwd="$1" t + shift + [ "$#" -eq 1 ] || return 1 + t="$1" + [ -n "$(printf '%s' "$t" | tr -d 'A-Za-z0-9._/-')" ] && return 1 + # Absolute only. A relative destination is not `$cwd/$t`: the shell searches $CDPATH first, + # so `cd ssh` may land in /etc/ssh, and this cannot see the caller's $CDPATH to rule it out. + case "$t" in /*) ;; *) return 1 ;; esac + realpath -m -- "$t" 2>/dev/null +} + +# Prints the class of a canonical path and returns 0: `tmp` for one strictly under /tmp, or +# `repo:` for one strictly inside the git working tree at , itself under $HOME. +# Fails, printing nothing, for anything else — those are the only roots the guards are willing +# to touch unprompted. The root is part of the class so that a caller pairing two operands can +# tell one checkout from another: sibling repos are separate permission boundaries, not one. +# +# The `repo` class trades on "this is a project under version control" being lower-stakes than +# the same act elsewhere — NOT on full recoverability: committed content is restorable via git, +# but untracked / .gitignore'd / uncommitted content, and an independent nested repo's history +# under a recursively-deleted parent, are NOT. Accepted as a deliberate convenience tradeoff. +# +# The walk stops at $HOME, so a dotfiles repo at ~ can't put all of $HOME in a class, and +# top-level ~ files stay out of one. A working tree's own root folder counts only when it is a +# linked worktree, whose `.git` is a pointer file so the history lives in the main repo and +# survives; a primary checkout's `.git` is a directory holding the history itself, so losing it +# is unrecoverable. +# +# Some paths are in no class in any root, /tmp included. Git history, and the agent's own guards +# and settings, because removing those is what removes the prompt on everything else. And every +# path `.claude/settings.json` refuses to read — `.env`, `secrets/`, `*.pem`, `*.key`, +# `credentials.json`, `.secret*` — because a `cp` or `mv` that is auto-allowed on both ends +# would rename one out of those globs and hand back through `Read` exactly what they deny. +path_class() { + local canon="$1" d root="" + case "$canon" in + *"/.git" | *"/.git/"* | *"/.claude" | *"/.claude/"*) return 1 ;; + *"/.env" | *"/.env."*) return 1 ;; + *"/secrets" | *"/secrets/"*) return 1 ;; + *.pem | *.key | *"/credentials.json") return 1 ;; + *"/.secret"* | *.secret | *.secrets) return 1 ;; + esac + case "$canon" in /tmp/?*) printf 'tmp'; return 0 ;; esac + [ -n "${HOME:-}" ] || return 1 + case "$canon" in "$HOME"/?*) ;; *) return 1 ;; esac + d="$canon" + while [ "$d" != "/" ] && [ "$d" != "$HOME" ]; do + [ -e "$d/.git" ] && { root="$d"; break; } + d=$(dirname "$d") + done + [ -n "$root" ] || return 1 # not inside a git working tree under $HOME + if [ "$canon" = "$root" ]; then + [ -f "$root/.git" ] || return 1 + fi + printf 'repo:%s' "$root" +} + +# 0 iff ($1) runs as a command word anywhere in ($2). Mirrors how a Bash +# permission rule matches, so that owning the prompt here doesn't narrow what used to prompt: +# a guard consults this before it starts proving segments, and every bail-out it then takes +# is a prompt for exactly the commands a rule would have caught. +runs_verb() { + local verb="$1" seg + split_segments "$2" + for seg in "${SEGMENTS[@]}"; do + segment_runs_verb "$verb" "$seg" && return 0 + done + return 1 +} + +# Emit a PreToolUse decision and exit. `ask` is the ordinary permission prompt. +decide() { + jq -nc --arg d "$1" --arg r "$2" \ + '{hookSpecificOutput:{hookEventName:"PreToolUse",permissionDecision:$d,permissionDecisionReason:$r}}' + exit 0 +} diff --git a/.claude/hooks/test-hooks.sh b/.claude/hooks/test-hooks.sh new file mode 100644 index 0000000000..01ed237baf --- /dev/null +++ b/.claude/hooks/test-hooks.sh @@ -0,0 +1,219 @@ +#!/usr/bin/env bash +# Decision table for the two scratch-dir PreToolUse guards. Run: bash .claude/hooks/test-hooks.sh +# +# What this pins is the `ask` column: a matcher change that turns one into a no-decision drops +# that command's only prompt (see lib-guarded-verb.sh). The wrapper, nested-command and quoted +# rows are the ones that catch it. +# +# The `allow` column carries its own weight, because a decision covers the whole command line: +# `allow` may only appear where every segment was proved here, and a line that also runs +# something unexamined has to come out `none` so the normal permission flow still sees it. +set -uo pipefail +H="$(cd "${BASH_SOURCE[0]%/*}" && pwd)" +CWD="$(git -C "$H" rev-parse --show-toplevel)" +OUT="$HOME/not-a-git-tree" # never written to; only the guards' path checks look at it +fails=0 + +# A tree's own root is auto-allowable only when it is a LINKED worktree, whose `.git` is a +# pointer file so the history lives in the main repo and survives; a primary checkout's `.git` +# is the history itself. The suite runs from either kind, so the rows that name the root follow +# the one it is run in — which is also what pins both halves of that rule. +if [ -f "$CWD/.git" ]; then + ROOT_SOLO=allow ROOT_CHAINED=none # linked worktree +else + ROOT_SOLO=ask ROOT_CHAINED=ask # primary checkout +fi + +run() { # run + local hook="$1" want="$2" cmd="$3" out got + out=$(jq -nc --arg c "$cmd" --arg w "$CWD" \ + '{tool_name:"Bash",tool_input:{command:$c},cwd:$w}' | "$H/$hook" 2>&1) + if [ -z "$out" ]; then + got=none + else + got=$(printf '%s' "$out" | jq -r '.hookSpecificOutput.permissionDecision // "PARSE-ERROR"' 2>/dev/null || echo PARSE-ERROR) + fi + local shown="${cmd//$'\n'/ ⏎ }" + if [ "$got" = "$want" ]; then + printf ' ok %-5s %s\n' "$got" "$shown" + else + printf 'FAIL want=%-5s got=%-5s %s\n %s\n' "$want" "$got" "$shown" "$out" + fails=$((fails + 1)) + fi +} + +echo "== guard-rm-outside-tmp.sh ==" +G=guard-rm-outside-tmp.sh +run $G allow "rm -rf /tmp/scratch/x" +run $G allow "rm -rf /tmp/scratch/*" +run $G allow "rm -rf $CWD/frontend/scratch" +run $G ask "rm -rf /tmp" +run $G ask "rm -rf $OUT" +run $G ask "rm -rf $CWD/.git" +run $G ask "rm -rf $CWD/.claude/hooks" # the guards may not delete themselves +run $G ask "rm $CWD/.claude/settings.json" +run $G ask "rm $CWD/.claude/settings.local.json" +run $G ask "rm -rf $CWD/backend/.env" +run $G ask "rm -rf $CWD/.env.local" +run $G $ROOT_SOLO "rm -rf $CWD" +run $G ask "rm -rf $CWD/*" +run $G ask "rm -rf /etc/passwd" +run $G ask 'rm -rf "$HOME/x"' +run $G ask "rm -rf /tmp/../$OUT" +run $G none "ls /tmp && rm -rf /tmp/x" # proved delete, unexamined neighbour +run $G ask 'echo $(rm -rf /etc)' +run $G ask 'echo `rm -rf /etc`' +run $G ask "{ rm -rf /etc; }" +run $G allow "{ rm -rf /tmp/scratch/x; }" # the keyword drops, the delete still proves +run $G ask "find . -name x | xargs rm" +run $G ask "timeout 5 rm -rf /tmp/x" +run $G ask "stdbuf -o L rm -rf /etc" +run $G ask "FOO=bar rm -rf /tmp/x" +run $G ask "/bin/rm -rf /tmp/x" +run $G ask "'rm' -rf /etc" +run $G ask 'r\m -rf /etc' +run $G ask "! rm -rf /etc" +run $G ask "if true; then rm -rf /etc; fi" +run $G ask ">/dev/null rm -rf $OUT" +# Data that merely mentions a verb is not a command. Both of these prompted in the field. +run $G none "$(printf 'gh pr create --body "$(cat <<%sEOF%s\ndrop `rm` and `mv` from the ask list\nrm is now guarded here\nEOF\n)"' "'" "'")" +run $G none "$(printf 'claude -p "run these in order:\n1: rm -rf /tmp/a\n2: mv /tmp/b /tmp/c"')" +# A wrapper's own flags and assignments are unbounded, so they may not be charged against the +# scan that looks past it — these run rm and must prompt. +run $G ask "env -i HOME=/tmp PATH=/usr/bin LANG=C USER=root SHELL=/bin/sh rm -rf /etc" +run $G ask "sudo -E -H -u root FOO=1 BAR=2 rm -rf $OUT" +run $G ask "xargs -a f -d d -E e -I {} -L 1 -n 1 rm /etc" +run $G ask "env -u A -u B -u C -u D -u E -u F -u G rm -rf /etc" +run $G ask "sudo -u 'root' rm -rf /etc" +run $G ask "$(printf 'echo hi # cat < f"\nrm -rf /etc\nEOF')" +run $G ask "$(printf 'echo "cat < /tmp/a"\nrm -rf /etc\ntrue')" +run $G ask "$(printf "echo 'cat < /tmp/a\nrm -rf /etc\nEOF' "'" "'")" +run $G none "$(printf 'cat <<%sEOF%s 2>&1 | tee /tmp/a\nrm -rf /etc\nEOF' "'" "'")" +# An unquoted body is expanded before its consumer sees it, so it is code. +run $G ask "$(printf 'cat < /tmp/a\n$(rm -rf /etc)\nEOF')" +run $G ask "$(printf 'cat < /tmp/a\nrm -rf /etc\nEOF')" +# ... but a real command after a heredoc still is one. +run $G ask "$(printf 'cat < /tmp/s.sh\nhello\nEOF\nrm -rf %s' "$OUT")" +run $G ask "$(printf 'echo "a << b"\nrm -rf %s' "$OUT")" +run $G none "git rm frontend/foo.ts" +run $G none 'echo $(ls /tmp)' +run $G none 'grep -rn "rm" backend/' +run $G none "cargo build --release" + +# Chaining and line breaks are not themselves a reason to prompt: each segment is proved on its +# own operands, and a `cd` moves where a relative one points. +run $G allow "rm -f /tmp/a; rm -rf /tmp/b" +run $G allow "$(printf 'rm -f /tmp/a\nrm -rf %s/frontend/scratch' "$CWD")" +run $G allow "cd /tmp/scratch && rm -rf sub" +run $G none "mkdir -p /tmp/x && rm -rf /tmp/x" +run $G ask "$(printf 'ls /tmp\nrm -rf /etc')" +# A `cd` this guard can resolve is where the relative operand lands; one it cannot leaves the +# working directory unknown, and an unknown one proves nothing. +run $G ask "cd /etc && rm -rf foo" +run $G ask 'cd "$D" && rm -rf foo' +run $G ask "cd $CWD && rm -rf .git" +run $G ask "cd /etc && cd /tmp/scratch && rm -rf sub" # a cd out is not walked back +# A `cd` can fail at runtime, and `;` runs the delete from where the command started, so a +# relative operand is proved from both directories. +run $G ask "cd /tmp/does-not-exist; rm -rf .git" +run $G ask "cd /tmp/does-not-exist; rm -rf backend/.env" +run $G ask "cd /tmp/a && cd /tmp/b && rm -rf sub" +run $G ask "rm -rf /tmp/clone/.git" # history is never in a class +run $G ask "rm -rf /tmp/scratch/id_rsa.key" +run $G none "cd /tmp >$OUT; rm -f /tmp/a" +# A substitution is concatenated into its word, so splitting on it would prove only the literal +# half; a relative `cd` is not $cwd/$t either, since the shell searches $CDPATH first. +run $G ask 'rm -rf /tmp/a/`printf ../../etc`' +run $G ask 'rm -rf /tmp/a/$(printf ../../etc)' +run $G ask "cd ssh && rm -rf moduli" + +echo +echo "== allow-fileops-in-tmp.sh ==" +A=allow-fileops-in-tmp.sh +run $A allow "mv /tmp/a /tmp/b" +run $A allow "chmod 755 /tmp/a" +run $A allow "cp -r /tmp/a /tmp/b" +run $A allow "tar -xzf /tmp/a.tar.gz -C /tmp/out" +run $A ask "mv /tmp/a $OUT" +run $A ask "mv $CWD/AGENTS.md /tmp/a" +run $A $ROOT_SOLO "chmod -R 777 $CWD" +run $A none "ls && mv /tmp/a /tmp/b" # proved move, unexamined neighbour +run $A ask 'echo $(mv /tmp/a /etc)' +run $A ask "timeout --signal KILL 5 mv /tmp/a /etc" +run $A ask "time -f FORMAT chmod 777 $OUT" +run $A ask "'mv' /tmp/a /etc" +run $A ask 'ch\mod 777 /etc' +run $A none "$(printf 'claude -p "run these in order:\n1: rm -rf /tmp/a\n2: mv /tmp/b /tmp/c"')" +run $A ask "env -i A=1 B=2 C=3 D=4 E=5 F=6 mv /tmp/a /etc" +run $A none "cp $CWD/AGENTS.md /tmp/a" +run $A none "tar -xzf /tmp/a.tar.gz -C $OUT" +run $A none "cargo build" + +run $A none "mkdir -p /tmp/x; mv /tmp/a /tmp/x; chmod 755 /tmp/x" # one write per line +run $A none "$(printf 'mv /tmp/a /tmp/b\nchmod 755 /tmp/b')" +run $A ask "ls && mv /tmp/a /etc" +run $A $ROOT_CHAINED "$(printf 'mkdir -p /tmp/x\nchmod -R 777 %s' "$CWD")" +run $A allow "cd /tmp/x && tar -xzf /tmp/a.tar.gz -C /tmp/out" +# The checkout is a root of its own, so an in-repo move or chmod is as auto-allowable as the +# in-repo delete already was — but one operation may not straddle it and /tmp. +run $A allow "chmod +x scripts/worktree-env" +run $A allow "mv backend/.sqlx backend/.sqlx.bad" +run $A allow "mv $CWD/frontend/a.ts $CWD/frontend/b.ts" +run $A ask "mv /tmp/a $CWD/frontend/a.ts" +run $A ask "chmod -R 777 $CWD/.git" +run $A ask "mv $CWD/backend/.env $CWD/backend/.env.bak" +run $A ask "mv $CWD/AGENTS.md $OUT" +run $A ask "cd /etc && mv a b" +# An auto-allowed rename may not carry a path out of the `Read` deny globs. +run $A ask "mv backend/server.pem backend/server.txt" +run $A none "cp backend/secrets/token frontend/token.txt" # cp has no prompt of its own, + # so what matters is it is not allowed +run $A ask "mv $CWD/backend/credentials.json /tmp/x" +run $A ask "cd /tmp/does-not-exist; mv .claude/settings.json settings.bak" +# A segment this hook cannot read whole may carry a redirect, and an earlier write can change +# what a later operand resolves to — neither may ride along on an allow. +run $A none "cd /tmp >$OUT; mv /tmp/a /tmp/b" +run $A none "cp -r /tmp/tree /tmp/live; cp /tmp/payload /tmp/live/link" +run $A ask 'mv /tmp/a/`printf ../../etc/x` /tmp/b' +# A sibling checkout is a different root: its files are outside what the Read tool is confined +# to, and copying them in would hand back what that confinement withholds. +EE="$(dirname "$CWD")/windmill-ee-private" # a sibling checkout; absent elsewhere, still not a root +run $A ask "mv $EE/backend/x.rs $CWD/backend/x.rs" +run $A none "cp $EE/README.md $CWD/README.copy" +# Directory form writes a path the command does not name — DEST/basename(SRC) — and `cp` +# follows that child when it is a symlink, as every `*_ee.rs` in this checkout is. +run $A ask "mv frontend/apps_ee.rs backend/windmill-api/src" +run $A none "cp frontend/apps_ee.rs backend/windmill-api/src" +run $A none "cp frontend/a.ts backend" +run $A ask "mv /tmp/a $CWD/backend" +# ... and a `cd` that fails at runtime may not hide that form: the destination is a directory +# in the directory the command actually ran in, whichever of the two that turns out to be. +run $A none "cd $CWD/AGENTS.md; cp frontend/apps_ee.rs backend/windmill-api/src" +run $A ask "cd $CWD/AGENTS.md; mv frontend/apps_ee.rs backend/windmill-api/src" +run $A none "cd /tmp/x && tar -xzf /tmp/a.tar.gz" # no -C, and the cwd is now two candidates +run $A allow "cp frontend/a.ts backend/a.ts" # ... naming the destination proves fine + +echo +[ "$fails" = 0 ] && echo "ALL PASS" || { echo "$fails FAILURES"; exit 1; } diff --git a/.claude/settings.json b/.claude/settings.json index 641ea70b53..a464ca3719 100644 --- a/.claude/settings.json +++ b/.claude/settings.json @@ -73,10 +73,7 @@ "Edit(**/.env.*)" ], "ask": [ - "Bash(rm:*)", "Bash(rmdir:*)", - "Bash(mv:*)", - "Bash(chmod:*)", "Bash(chown:*)", "Bash(truncate:*)", "Bash(shred:*)", diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000000..0e022ec801 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,14 @@ + + +## What does this PR do? + +## Related issue diff --git a/.github/workflows/ai-evals-test.yml b/.github/workflows/ai-evals-test.yml index 3ee7aed876..71e79395f3 100644 --- a/.github/workflows/ai-evals-test.yml +++ b/.github/workflows/ai-evals-test.yml @@ -22,6 +22,7 @@ on: - "frontend/src/lib/userDraft.svelte.ts" - "frontend/src/lib/userDraftDbSyncer.svelte.ts" - "frontend/src/lib/infer.ts" + - "frontend/src/lib/components/sessions/**" - ".github/workflows/ai-evals-test.yml" pull_request: types: [opened, reopened, ready_for_review] @@ -35,6 +36,7 @@ on: - "frontend/src/lib/userDraft.svelte.ts" - "frontend/src/lib/userDraftDbSyncer.svelte.ts" - "frontend/src/lib/infer.ts" + - "frontend/src/lib/components/sessions/**" - ".github/workflows/ai-evals-test.yml" concurrency: @@ -124,6 +126,11 @@ jobs: bun install bun test adapters/ + # Harness code that reaches into the frontend module graph; bun cannot load it. + - name: Run harness unit tests (frontend graph) + working-directory: ./ai_evals + run: bun run test:frontend-graph + - name: Run global AI evals timeout-minutes: 20 working-directory: ./ai_evals diff --git a/.github/workflows/cli-tests.yml b/.github/workflows/cli-tests.yml index 061165530d..e26231eda4 100644 --- a/.github/workflows/cli-tests.yml +++ b/.github/workflows/cli-tests.yml @@ -9,6 +9,13 @@ on: - "windmill-yaml-validator/**" - "backend/migrations/**" - ".github/workflows/cli-tests.yml" + # The bundles cli/ vendors from the frontend: their drift guards live in + # cli/test but the edits that break them land here. The policy bundle + # inlines its imports too, so those sources belong in the filter. + - "frontend/src/lib/components/raw_apps/**" + - "frontend/src/lib/components/recording/**" + - "frontend/src/lib/components/apps/editor/commonAppUtils.ts" + - "frontend/src/lib/components/apps/inputType.ts" pull_request: branches: [main] paths: @@ -16,6 +23,13 @@ on: - "windmill-yaml-validator/**" - "backend/migrations/**" - ".github/workflows/cli-tests.yml" + # The bundles cli/ vendors from the frontend: their drift guards live in + # cli/test but the edits that break them land here. The policy bundle + # inlines its imports too, so those sources belong in the filter. + - "frontend/src/lib/components/raw_apps/**" + - "frontend/src/lib/components/recording/**" + - "frontend/src/lib/components/apps/editor/commonAppUtils.ts" + - "frontend/src/lib/components/apps/inputType.ts" env: CARGO_TERM_COLOR: always diff --git a/.github/workflows/pr-ready-review.yml b/.github/workflows/pr-ready-review.yml index d2acde0dbe..4bfb5b4147 100644 --- a/.github/workflows/pr-ready-review.yml +++ b/.github/workflows/pr-ready-review.yml @@ -167,8 +167,14 @@ jobs: env: EXTRA_PROMPT: ${{ inputs.extra_prompt }} run: | + # prior-comments.md is PR comment text verbatim, and commenting needs no write access. + # With a fixed delimiter, a comment containing a bare `EOF` line closes the block early: + # the step dies, and whatever follows in that comment is read as further environment + # assignments for the rest of this job, which holds the review tokens. Hence a random + # delimiter, per GitHub's guidance for untrusted multiline values. + delimiter="REVIEW_PROMPT_EOF_$(openssl rand -hex 16)" { - echo 'REVIEW_PROMPT<> "$GITHUB_ENV" - name: Automatic PR Review diff --git a/.github/workflows/pr-review-commands.yml b/.github/workflows/pr-review-commands.yml index 3c04e4e5f2..d29ffe6561 100644 --- a/.github/workflows/pr-review-commands.yml +++ b/.github/workflows/pr-review-commands.yml @@ -25,16 +25,20 @@ jobs: REMAINDER_FIRST_LINE=${FIRST_LINE#"$FIRST_WORD"} REMAINDER_FIRST_LINE=${REMAINDER_FIRST_LINE# } REST=$(printf '%s' "$BODY" | tail -n +2) + # The value is the comment body, which anyone can write. A fixed delimiter lets a + # comment close the block early and have the rest of itself read as further step + # outputs, so the delimiter has to be unguessable. + delimiter="EXTRA_EOF_$(openssl rand -hex 16)" { echo "command=$COMMAND" - echo 'extra_prompt<> "$GITHUB_OUTPUT" ;; *) diff --git a/.github/workflows/sign-cla.yml b/.github/workflows/sign-cla.yml index 67822542a9..52329b6119 100644 --- a/.github/workflows/sign-cla.yml +++ b/.github/workflows/sign-cla.yml @@ -21,9 +21,15 @@ jobs: PERSONAL_ACCESS_TOKEN: ${{ secrets.CLA_PAT }} with: path-to-signatures: "signatures/cla.json" - path-to-document: "https://github.com/windmill-labs/windmill/blob/master/CLA.md" + path-to-document: "https://github.com/windmill-labs/windmill/blob/main/CLA.md" branch: "signatures" allowlist: rubenfiszel,bot* + custom-notsigned-prcomment: | + Thank you for taking the time to open this PR. + + Please note that **we are not seeking outside contribution at this time**. Small, trivially-verified PRs that fix a problem are still accepted, but low-value PRs (e.g. typo fixes) and PRs longer than a dozen or so lines will be closed. If you have a bigger idea, please open a [feature request](https://github.com/windmill-labs/windmill/issues/new?template=feature_request.md) instead. See [CONTRIBUTING.md](https://github.com/windmill-labs/windmill/blob/main/CONTRIBUTING.md) for the full policy. + + If your PR falls within that scope, we ask that you sign our [Contributor License Agreement](https://github.com/windmill-labs/windmill/blob/main/CLA.md) before we can accept it. You can sign the CLA by just posting a Pull Request Comment same as the below format. #below are the optional inputs - If the optional inputs are not given, then default values will be taken #remote-organization-name: enter the remote organization name where the signatures should be stored (Default is storing the signatures in the same repository) diff --git a/.release-please-manifest.json b/.release-please-manifest.json index 5b2dedb3d9..d66e79c5fa 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -1,3 +1,3 @@ { - ".": "1.789.0" + ".": "1.792.2" } diff --git a/AGENTS.md b/AGENTS.md index aab920746b..1da4be6c98 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -42,6 +42,7 @@ Open-source platform for internal tools, workflows, API integrations, background - **Domain vocabulary**: `CONTEXT.md` — the words this codebase uses for its own concepts (step, step setting, trigger step, …). Name things the way it does. - **CLI commands**: when adding/modifying/removing a command, subcommand, option, or description in `cli/src/commands/`, run `python system_prompts/generate.py` to refresh `system_prompts/auto-generated/` and `cli/src/guidance/skills.gen.ts`. The CLI docs the agents use to operate `wmill` are derived from the source — stale generated files give agents the wrong flags. - **Session recorder**: `frontend/src/lib/components/recording/` is also the recorder `wmill app dev --recording` serves, vendored into the CLI as `cli/src/commands/app/devRecorderBundle.gen.ts`. After changing `rawAppSnapshot.ts` or `rawAppRecording.svelte.ts`, run `bun run gen:dev-recorder` from `cli/` (`cli/test/dev_recorder_bundle_unit.test.ts` fails otherwise). +- **Raw-app policy**: `frontend/src/lib/components/raw_apps/rawAppPolicy.ts` also derives the policy the server's raw-app deploy stores, vendored into the bundle job as `backend/windmill-api/src/apps_raw_policy.gen.js`. After changing it or anything it imports, run `bun run gen:app-policy` from `cli/` (`cli/test/app_policy_bundle_unit.test.ts` fails otherwise). It rides in the job rather than being read from the CLI the job runs because the images install `windmill-cli` unpinned, so an image can carry one older than its server. ## Dev Environment @@ -146,10 +147,19 @@ $NAV --root backend callees "X" # what does X call? - **MUST `outline` before `Read`** on unfamiliar files — then `body` or `Read` with offset/limit for specifics - **Scratch stays outside the checkout.** Temp scripts, data dumps, cache backups and screenshots go in the session scratch directory or `/tmp`, so nothing temporary can end up - committed. Write `rm`/`mv`/`cp` as one plain unchained command: a PreToolUse hook - auto-allows those when every operand is under `/tmp` or inside this checkout, but it defers - on `&&`, `;`, redirects, quotes and `$VAR` — that deferral, not the delete itself, is what - turns a routine cleanup into a permission prompt. + committed. Write the paths in `rm`/`mv`/`cp` out literally: a PreToolUse hook proves each + operand, and auto-allows deletes, moves, copies and mode changes under `/tmp` or inside a git + checkout under `$HOME`, as long as one operation stays within a single root — a sibling + checkout is a root of its own (`tar` and `unzip` stay `/tmp`-only). Chain deletes freely, each + proved on its own operands, but keep writes to one per line, name the destination rather than + a directory to drop it in, and put anything else on its own line: a command the hook does not + prove drops the whole line back to the normal permission flow. A + quoted or `$VAR` operand, a `~`, a redirect, a `$(…)`, a relative `cd`, or a wrapper like + `xargs rm` cannot be proved, and that deferral is what turns a cleanup into a prompt. +- **Change files with Edit/Write, not the shell.** `sed -i`, `cat > file <<'EOF'` and inline + `python3 - <<'PY'` scripts put an edit through the PreToolUse guards and the permission + classifier, which match `Bash` and nothing else, so a routine edit arrives as a prompt. Bash + stays right for running things — tests, builds, git, one-off queries. - Search for existing code to reuse before writing new code - Follow established patterns in the codebase - Keep changes focused — don't refactor beyond what's asked diff --git a/CHANGELOG.md b/CHANGELOG.md index 9107cde620..0a729e40a1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,101 @@ # Changelog +## [1.792.2](https://github.com/windmill-labs/windmill/compare/v1.792.1...v1.792.2) (2026-08-19) + + +### Bug Fixes + +* check direct-deployment lock and superadmin in the deploy preflight ([#10748](https://github.com/windmill-labs/windmill/issues/10748)) ([ef8a8e8](https://github.com/windmill-labs/windmill/commit/ef8a8e821ca3a5f4a308c49226292565680bf90c)) +* make the listScripts parent_hash filter valid SQL ([#10752](https://github.com/windmill-labs/windmill/issues/10752)) ([f34b7fb](https://github.com/windmill-labs/windmill/commit/f34b7fbcfa104bdc00abbfc59ab4a3dd8cafe0e0)) +* **security:** validate ansible git repository URLs before invoking git ([#10759](https://github.com/windmill-labs/windmill/issues/10759)) ([fa7fbd3](https://github.com/windmill-labs/windmill/commit/fa7fbd348d4b184ea95cfc953ded7c72b6f04e5e)) + +## [1.792.1](https://github.com/windmill-labs/windmill/compare/v1.792.0...v1.792.1) (2026-08-18) + + +### Bug Fixes + +* route legacy AI entry points to sessions instead of the unmounted chat ([#10705](https://github.com/windmill-labs/windmill/issues/10705)) ([494e6f1](https://github.com/windmill-labs/windmill/commit/494e6f146e6a22bd498db58fa10c7866cd56dc4d)) + +## [1.792.0](https://github.com/windmill-labs/windmill/compare/v1.791.0...v1.792.0) (2026-08-18) + + +### Features + +* **frontend:** record the outcome of every AI chat tool call ([#10746](https://github.com/windmill-labs/windmill/issues/10746)) ([7b17e35](https://github.com/windmill-labs/windmill/commit/7b17e358b35bf4ef8c213ea13252a456e87acb32)) + + +### Bug Fixes + +* **api:** document cache_ignore_s3_path on the Script read schema ([#10742](https://github.com/windmill-labs/windmill/issues/10742)) ([6783a39](https://github.com/windmill-labs/windmill/commit/6783a396b144948fa60324eae888bc4a83917bc8)) +* audit the icon library against brand guidelines ([#10722](https://github.com/windmill-labs/windmill/issues/10722)) ([6749015](https://github.com/windmill-labs/windmill/commit/6749015fbf7afe0c6dcd53b1933b0152915afd32)) +* **cli:** keep script settings on push and repair the up-to-date check ([#10741](https://github.com/windmill-labs/windmill/issues/10741)) ([ef4dc46](https://github.com/windmill-labs/windmill/commit/ef4dc46d4bfd00e583a39e5c053c0022f1ad3abd)) +* show runtime-detected assets in a run's Assets tab ([#10738](https://github.com/windmill-labs/windmill/issues/10738)) ([1fa3bf3](https://github.com/windmill-labs/windmill/commit/1fa3bf3b291c32ffd62f77df59e24993afb7c78a)) + +## [1.791.0](https://github.com/windmill-labs/windmill/compare/v1.790.1...v1.791.0) (2026-08-17) + + +### Features + +* add empty state cards to list pages ([#10726](https://github.com/windmill-labs/windmill/issues/10726)) ([66bffaa](https://github.com/windmill-labs/windmill/commit/66bffaa60d48f992e56e459efb813b24e3942610)) +* **copilot:** let plan mode draw, but never write the plan ([#10725](https://github.com/windmill-labs/windmill/issues/10725)) ([fd9295a](https://github.com/windmill-labs/windmill/commit/fd9295a58e6868ccc6f371e35b875ae27196e99a)) + + +### Bug Fixes + +* compile resource types with no properties instead of throwing ([#10730](https://github.com/windmill-labs/windmill/issues/10730)) ([b17fdab](https://github.com/windmill-labs/windmill/commit/b17fdab8ff96aa7bbfc8b14389294bda1a9e0a07)) +* derive a raw app's policy on deploy, and default an omitted execution_mode ([#10733](https://github.com/windmill-labs/windmill/issues/10733)) ([343ce6e](https://github.com/windmill-labs/windmill/commit/343ce6e143343e65613d52d0f12c5264b4ab4c3a)) +* include delete_after_secs in script deploy payload ([#10731](https://github.com/windmill-labs/windmill/issues/10731)) ([05eba6c](https://github.com/windmill-labs/windmill/commit/05eba6c9ab078cdedc87f197549dbdbc4b360fe3)) +* support [@typechecked](https://github.com/typechecked) decorator in Python relative imports ([#8495](https://github.com/windmill-labs/windmill/issues/8495)) ([ab3c020](https://github.com/windmill-labs/windmill/commit/ab3c0206d7e9b32676d99ed0cd8c9d8939122584)) +* type s3-streamed columns that are all-null in the inference sample ([#10728](https://github.com/windmill-labs/windmill/issues/10728)) ([6b5b9f7](https://github.com/windmill-labs/windmill/commit/6b5b9f72d4f9ce86b21d8d21ae342b6c8dc14b93)) + +## [1.790.1](https://github.com/windmill-labs/windmill/compare/v1.790.0...v1.790.1) (2026-08-17) + + +### Bug Fixes + +* fall back to polling when a proxy mutes the job SSE stream ([#10716](https://github.com/windmill-labs/windmill/issues/10716)) ([64d78b4](https://github.com/windmill-labs/windmill/commit/64d78b4db1d7d939c598c86d1998218d52fbcc21)) + + +### Performance Improvements + +* cap resource content sent to the search modal ([#10714](https://github.com/windmill-labs/windmill/issues/10714)) ([529e960](https://github.com/windmill-labs/windmill/commit/529e9606297ee0b41456a66222f31409d7bc7669)) +* unblock workers before the API router is built ([#10711](https://github.com/windmill-labs/windmill/issues/10711)) ([0258f3f](https://github.com/windmill-labs/windmill/commit/0258f3f81b96bb8d4e343ba8aeba614f9c836579)) + +## [1.790.0](https://github.com/windmill-labs/windmill/compare/v1.789.0...v1.790.0) (2026-08-15) + + +### Features + +* add trigger_history table with source tracking ([#10696](https://github.com/windmill-labs/windmill/issues/10696)) ([633d7bc](https://github.com/windmill-labs/windmill/commit/633d7bcb2ea034c39b72f7a8f5109b4ccd71b0be)) +* advertise the pinned artifact version in get_preview_status ([#10691](https://github.com/windmill-labs/windmill/issues/10691)) ([850b028](https://github.com/windmill-labs/windmill/commit/850b028778afe0cda1dc357c9e647d066339f3ce)) +* **ai-sessions:** add plan mode ([#10057](https://github.com/windmill-labs/windmill/issues/10057)) ([caa1898](https://github.com/windmill-labs/windmill/commit/caa189868c6ec9ebc2b6311e07308a83fa25ad8d)) +* let the global AI chat call connected MCP servers as the user ([#10656](https://github.com/windmill-labs/windmill/issues/10656)) ([3f07a1a](https://github.com/windmill-labs/windmill/commit/3f07a1a803a3f8a176de754188f641bdfcaa6cec)) +* stream audit logs in batches when a page is slow to load ([#10695](https://github.com/windmill-labs/windmill/issues/10695)) ([9334727](https://github.com/windmill-labs/windmill/commit/9334727d99eac251b0a995916c7ea00bd9596cef)) +* **telemetry:** extend feature-usage tracking beyond AI features ([#10681](https://github.com/windmill-labs/windmill/issues/10681)) ([53eb946](https://github.com/windmill-labs/windmill/commit/53eb94659bd27e75ed4acf4ce414046ac8df4cc8)) + + +### Bug Fixes + +* **agents:** let the scratch-dir hooks own their permission prompt ([#10702](https://github.com/windmill-labs/windmill/issues/10702)) ([0a40b38](https://github.com/windmill-labs/windmill/commit/0a40b3806fc08c6d7b2f9fd9b7ade07fdf1841f1)) +* **agents:** stop the scratch-dir guards prompting on quoted text ([#10703](https://github.com/windmill-labs/windmill/issues/10703)) ([e6e2e53](https://github.com/windmill-labs/windmill/commit/e6e2e53e97bebd407d72819d6163c04b6fc0b6b0)) +* **ci:** use random delimiters for untrusted multiline workflow values ([#10706](https://github.com/windmill-labs/windmill/issues/10706)) ([0fc74de](https://github.com/windmill-labs/windmill/commit/0fc74dec5f9d9d6594f1bc4a85b895ee8c3bf17b)) +* confine jobs:run tokens to the jobs of the runnables they may start ([#10635](https://github.com/windmill-labs/windmill/issues/10635)) ([ee53327](https://github.com/windmill-labs/windmill/commit/ee533273dd2fa0dc70e45b9750f3556002b150b7)) +* drop sampling params on Claude models that reject them ([#10708](https://github.com/windmill-labs/windmill/issues/10708)) ([3468cb6](https://github.com/windmill-labs/windmill/commit/3468cb68b12c27f7f133e350b433fd9379fc8b07)) +* **groups:** replace instance-group delta-patching with a state-based reconciler ([#10686](https://github.com/windmill-labs/windmill/issues/10686)) ([b551033](https://github.com/windmill-labs/windmill/commit/b5510333eac99f575aa2251398ca58626e419968)) +* keep a resource's linked secret reference in sync while renaming ([#10693](https://github.com/windmill-labs/windmill/issues/10693)) ([60c5ad2](https://github.com/windmill-labs/windmill/commit/60c5ad252afe23642410743632be2f6eaf2fbffd)) +* keep non traffic-serving processes out of coordinated restarts ([#10694](https://github.com/windmill-labs/windmill/issues/10694)) ([6d03784](https://github.com/windmill-labs/windmill/commit/6d03784d4b15535666bd4afdc5bbde5af016e078)) +* recover from a refused mcp read assertion, drop stale discovery ([#10710](https://github.com/windmill-labs/windmill/issues/10710)) ([effdcd9](https://github.com/windmill-labs/windmill/commit/effdcd99155a9235856b245b7f49a37e0632db08)) +* refresh AI provider model defaults and capability metadata ([#10690](https://github.com/windmill-labs/windmill/issues/10690)) ([68fc782](https://github.com/windmill-labs/windmill/commit/68fc7825bb5cd04347debb1a30af227614b9d9f5)) +* send sage_intacct oauth client credentials in the request body ([#10685](https://github.com/windmill-labs/windmill/issues/10685)) ([bd5b3ea](https://github.com/windmill-labs/windmill/commit/bd5b3ea779fa6351e937fc3639f0bb985ffc1ce9)) + + +### Performance Improvements + +* back off the interactive worker shell under EXIT_AFTER_N_JOBS ([#10700](https://github.com/windmill-labs/windmill/issues/10700)) ([578d5e9](https://github.com/windmill-labs/windmill/commit/578d5e9a7d1016deaa81e5bf314029c5d7de9589)) +* cache resolved python interpreter path across worker restarts ([#10701](https://github.com/windmill-labs/windmill/issues/10701)) ([878b8ef](https://github.com/windmill-labs/windmill/commit/878b8ef4c47f650686d4a42fc9763d57cc1c62cf)) +* declare a settings pass instead of reading one setting at a time ([#10698](https://github.com/windmill-labs/windmill/issues/10698)) ([30f5d2e](https://github.com/windmill-labs/windmill/commit/30f5d2e7660ad5bfa335d76a69bd5c3ad8e70c77)) +* resolve the worker external IP in the background ([#10697](https://github.com/windmill-labs/windmill/issues/10697)) ([22eadab](https://github.com/windmill-labs/windmill/commit/22eadab67d52fe4cb6bf1e73d161a6a439770295)) + ## [1.789.0](https://github.com/windmill-labs/windmill/compare/v1.788.0...v1.789.0) (2026-08-13) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000000..e3bc91fd77 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,34 @@ +# Contributing to Windmill + +At this time, we are not seeking outside contribution. + +AI has made writing code easy. The hard part, today, is not writing the code, but reviewing it, +making sure quality stays high, and keeping the product coherent. In that light, unfortunately, +external code contributions are "donating" the easy part of the job, while creating more of the +hard work. + +With that said, we are happy to accept small, trivially-verified PRs that fix a problem. However, +we ask that you refrain from submitting low-value PRs (e.g. typo fixes) or PRs that are more than a +dozen or so lines. Such PRs will be closed with a reference to this guideline. + +If you have a big idea you'd like us to consider, feel free to open a +[feature request](https://github.com/windmill-labs/windmill/issues/new?template=feature_request.md) +about it. + +This policy may change in the future as the project matures. Until then, thank you for your +understanding. + +## What is still very welcome + +- [Bug reports](https://github.com/windmill-labs/windmill/issues/new?template=bug_report.yml), with + clear reproduction steps. +- [Feature requests](https://github.com/windmill-labs/windmill/issues/new?template=feature_request.md), + including for ideas too big to be a PR. +- Questions and feedback on [Discord](https://discord.gg/V7PM2YHsPB). +- Contributions to the [Windmill Hub](https://hub.windmill.dev), where scripts, flows and apps are + shared with the community. + +## If you do open a PR + +Small, self-contained fixes are still accepted. They require signing the +[CLA](./CLA.md), which the CLA bot will prompt for on your first PR. diff --git a/README.md b/README.md index 2d1d4d62ac..71d4d39a19 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ Scripts are turned into sharable UIs automatically, and can be composed together

- Try it - Website - Docs - Discord - Hub - Contributor's guide + Try it - Website - Docs - Discord - Hub - Contributing

# Windmill - Developer platform for APIs, background jobs, workflows and UIs @@ -62,6 +62,7 @@ https://github.com/user-attachments/assets/d80de1d9-64de-4d89-aacd-6df23fa81fc4 - [Run a local dev setup](#run-a-local-dev-setup) - [Frontend only](#frontend-only) - [Backend + Frontend](#backend--frontend) + - [Contributing](#contributing) - [Contributors](#contributors) - [Copyright](#copyright) @@ -260,7 +261,7 @@ On self-hosted instances, you might want to import all the approved resource typ | NATIVE_MODE | false | Enable native mode: sets NUM_WORKERS=8, rejects non-native jobs (nativets, postgresql, mysql, etc.) | Worker | | SLEEP_QUEUE | 50 | The number of ms to sleep in between the last check for new jobs in the DB. It is multiplied by NUM_WORKERS such that in average, for one worker instance, there is one pull every SLEEP_QUEUE ms. | Worker | | KEEP_JOB_DIR | false | Keep the job directory after the job is done. Useful for debugging. | Worker | -| EXIT_AFTER_N_JOBS | None | Exit the worker process after it has executed that many jobs, so that a supervisor restarts it and no process runs more than that many, bar the steps of a same-worker flow it has started, which it always finishes (set it to 1 for a process per job; jobs handed to a dedicated worker, and the worker's own init and periodic scripts, do not count). For deployments that isolate executions by process lifetime rather than with nsjail; note that a container restart resets the process, not the container filesystem, so caches and `/tmp` survive it. The worker name is then derived from the hostname instead of being random, so the restarted worker keeps its row in the workers list (an agent worker keeps the row but restarts its job count). Use one worker per process: workers of one process share its environment, so the first to reach the limit shuts the others down too. | Worker | +| EXIT_AFTER_N_JOBS | None | Exit the worker process after it has executed that many jobs, so that a supervisor restarts it and no process runs more than that many, bar the steps of a same-worker flow it has started, which it always finishes (set it to 1 for a process per job; jobs handed to a dedicated worker, and the worker's own init and periodic scripts, do not count). Not counting the init and periodic scripts means they run again on every restart: an init script's runtime is added to the latency of every batch of that many jobs, and a periodic script fires once per process start whatever its interval says. The worker's shell in the workers page also starts backed off rather than after the two minutes it otherwise takes, since a process due to be recycled cannot count on living that long: the first command of a session can wait up to 15s, later ones are immediate. For deployments that isolate executions by process lifetime rather than with nsjail; note that a container restart resets the process, not the container filesystem, so caches and `/tmp` survive it. The worker name is then derived from the hostname instead of being random, so the restarted worker keeps its row in the workers list (an agent worker keeps the row but restarts its job count). Use one worker per process: workers of one process share its environment, so the first to reach the limit shuts the others down too. | Worker | | WORKER_SUFFIX | None | Pins the last part of the worker name, which is otherwise random, so that a restarted worker keeps its row in the workers list. Only needed when several worker processes of the same worker group run on one host, since the name is derived from the hostname: give each of them a distinct value, as two processes sharing one must never happen. At most 64 letters, digits and underscores; anything else is refused at startup. | Worker | | LICENSE_KEY (EE only) | None | License key checked at startup for the Enterprise Edition of Windmill | Worker | | SLACK_SIGNING_SECRET | None | The signing secret of your Slack app. See [Slack documentation](https://api.slack.com/authentication/verifying-requests-from-slack) | Server | @@ -329,6 +330,12 @@ running options. 2. You can specify any feature flag you want to enable, for example `cargo run --features python` to enable the python executor. 7. Windmill should be available at `http://localhost:3000` +## Contributing + +At this time, we are not seeking outside contribution. Bug reports and feature requests remain very +welcome, and small, trivially-verified PRs that fix a problem are still accepted. See +[CONTRIBUTING.md](./CONTRIBUTING.md) for the full policy. + ## Contributors diff --git a/ai_evals/README.md b/ai_evals/README.md index 08725e8aa8..ee8f730fc3 100644 --- a/ai_evals/README.md +++ b/ai_evals/README.md @@ -150,6 +150,21 @@ Global initial fixtures can also seed `liveEditorDrafts` with `type`, currently open script, flow, or raw app editor so cases can test prompts that refer to "this" or the "current" item. +Global initial fixtures can seed the session's `artifacts` — `{ name, versions: [{ content, +note? }], role?, approvedVersion? }`, oldest version first, so the artifact starts with the +history `list_artifact_versions` reports — and the `previewTabs` open in its side panel, for +cases that run with `runtime.sessionChat: true`. A tab entry names one destination and may +be the `active` one: + +```json +"previewTabs": [{ "artifact": { "name": "Onboarding plan", "version": 2 }, "active": true }] +``` + +`page` (`{ href, label }`) and `item` (`{ kind, path }`) tabs work the same way. Tabs are +driven by the production tab model, so `open_preview`, `get_preview_status` and +`close_page` really open, report and close them, and a `version` is the pin a reader +chose in the artifact's version picker — which only `get_preview_status` reports. + Global initial fixtures can seed `workspace.variables` with `{ path, value, is_secret, description?, labels?, ws_specific? }` entries so cases can read and edit variables that already exist in the workspace. The mock mirrors the real @@ -265,6 +280,10 @@ Typical artifacts by mode: - `history/`: optional tracked pass-rate history written by `run --record`, one JSONL file per mode - `results/`: local benchmark output and artifacts +Harness unit tests run in two lanes: `bun test adapters/` for plain TypeScript, and +`bun run test:frontend-graph` for `*.vitest.ts` files, which exercise adapters built on +frontend code (Svelte runes, SvelteKit aliases) that bun cannot load. + ## Notes - Frontend modes reuse the production frontend chat code through the Vitest bridge. diff --git a/ai_evals/adapters/frontend/core/global/evalArtifactHelpers.test.ts b/ai_evals/adapters/frontend/core/global/evalArtifactHelpers.test.ts index c74f341735..99ac50ca98 100644 --- a/ai_evals/adapters/frontend/core/global/evalArtifactHelpers.test.ts +++ b/ai_evals/adapters/frontend/core/global/evalArtifactHelpers.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it } from "bun:test"; import { createEvalArtifactHelpers } from "./evalArtifactStore"; +import { planArtifactId } from "../../../../../frontend/src/lib/components/copilot/chat/artifacts/planIdentity"; // A hand-written stand-in for SessionArtifactsStore (bun has no IndexedDB), so nothing // makes it follow that class. A method missing from it surfaces as a tool throwing @@ -19,4 +20,20 @@ describe("eval artifact store", () => { expect(typeof (helpers.artifacts as any)[method]).toBe("function"); } }); + + it("files a plan under the id production derives, seeded or created", async () => { + const { helpers, sessionId } = createEvalArtifactHelpers([ + { name: "Seeded plan", role: "plan", versions: [{ content: "v1" }] }, + ]); + const seeded = await helpers.artifacts.listForSession(sessionId); + expect(seeded.map((a: any) => a.id)).toEqual([planArtifactId(sessionId)]); + + const other = createEvalArtifactHelpers(); + const created = await other.helpers.artifacts.create(other.sessionId, { + name: "Plan", + content: "v1", + role: "plan", + }); + expect(created.id).toBe(planArtifactId(other.sessionId)); + }); }); diff --git a/ai_evals/adapters/frontend/core/global/evalArtifactStore.ts b/ai_evals/adapters/frontend/core/global/evalArtifactStore.ts index ed67cce468..f2d77f64e0 100644 --- a/ai_evals/adapters/frontend/core/global/evalArtifactStore.ts +++ b/ai_evals/adapters/frontend/core/global/evalArtifactStore.ts @@ -1,11 +1,74 @@ +import { planArtifactId } from "../../../../../frontend/src/lib/components/copilot/chat/artifacts/planIdentity"; + // SessionArtifactsStore can't run here (bun has no IndexedDB, nor the compiled $state runes), // so mirror only the shape the artifact tools call, not its scoping or race handling. -export const EVAL_SESSION_ID = "eval-session"; -export function createEvalArtifactHelpers() { +// Cases run concurrently in one process and the preview handlers are registered +// process-wide, keyed by session id — so each run needs its own. +let sessionSeq = 0; + +/** An artifact the session already holds when the case starts: history has to predate the + * run, since one prompt cannot both build a past and reason about it. */ +export interface SeededArtifact { + name: string; + role?: "plan"; + /** Which version the user agreed to. Below the last one means the current text is a + * proposal they turned down, which is the state worth seeding. */ + approvedVersion?: number; + /** Oldest first; the last one is the artifact's current content. */ + versions: Array<{ content: string; note?: string }>; +} + +export function createEvalArtifactHelpers(seed: SeededArtifact[] = []) { + const sessionId = `eval-session-${sessionSeq++}`; const items = new Map>(); // Snapshots per artifact id, oldest first — the version tools read history from here. const history = new Map>>(); + // How a preview-tab fixture names the artifact its tab shows. + const seededIds = new Map(); let seq = 0; + for (const entry of seed) { + // Derived, not minted: the tools that must not touch the plan recognise it by this id, so + // an id of the harness's own would pass a case the real gate refuses. The counter advances + // either way, or seeding a plan would renumber the rows around it and collapse the update + // order they are sorted on. + const n = seq++; + const id = entry.role === "plan" ? planArtifactId(sessionId) : `eval-artifact-${n}`; + const current = entry.versions.at(-1); + if (!current) continue; + // A preview tab names the artifact it shows, so a shared name would open whichever + // one happened to be seeded last. + if (seededIds.has(entry.name)) { + throw new Error( + `Two seeded artifacts are named "${entry.name}" — a preview tab fixture could not tell them apart`, + ); + } + seededIds.set(entry.name, id); + items.set(id, { + id, + sessionId, + chatId: "eval-chat", + kind: "md", + name: entry.name, + content: current.content, + role: entry.role, + approvedVersion: entry.approvedVersion, + createdAt: 0, + updatedAt: seq, + version: entry.versions.length, + }); + history.set( + id, + entry.versions.map((v, i) => ({ + key: `${id}:${i + 1}`, + artifactId: id, + version: i + 1, + name: entry.name, + content: v.content, + savedAt: i, + note: v.note, + })), + ); + } const snapshotOf = ( artifact: Record, version: number, @@ -21,14 +84,31 @@ export function createEvalArtifactHelpers() { }); const store = { create: async (sessionId: string, input: Record) => { + // One plan per session, as SessionArtifactsStore enforces it — the tool refuses + // first, so reaching this means a case drove create_artifact past that message. + if ( + input.role === "plan" && + [...items.values()].some( + (a) => a.sessionId === sessionId && a.role === "plan", + ) + ) { + throw new Error(`Session ${sessionId} already has a plan document`); + } const now = seq++; const artifact = { - id: `eval-artifact-${now}`, + id: + input.role === "plan" + ? planArtifactId(sessionId) + : `eval-artifact-${now}`, sessionId, chatId: input.chatId, kind: input.kind ?? "md", name: input.name, content: input.content, + // The plan document is only distinguishable by these, both in the snapshot the + // judge reads and in what list_artifacts reports back to the model. + role: input.role, + approvedVersion: input.approvedVersion, createdAt: now, updatedAt: now, version: 1, @@ -58,6 +138,15 @@ export function createEvalArtifactHelpers() { ...existing, name: input.name ?? existing.name, content: input.content ?? existing.content, + // Carried only onto a version this write produced, as SessionArtifactsStore does: + // a rename cannot promote a proposal the user turned down. + approvedVersion: + input.approvedVersion ?? + (input.keepApproved && + existing.approvedVersion !== undefined && + contentChanged + ? version + : existing.approvedVersion), updatedAt: seq++, version, }; @@ -84,10 +173,12 @@ export function createEvalArtifactHelpers() { return { helpers: { artifacts: store, - sessionId: EVAL_SESSION_ID, + sessionId, getChatId: () => "eval-chat", - openArtifact: () => {}, + openArtifact: (_id: string, _name: string) => {}, }, + sessionId, + seededIds, snapshot: () => [...items.values()], }; } diff --git a/ai_evals/adapters/frontend/core/global/evalPreviewTabs.ts b/ai_evals/adapters/frontend/core/global/evalPreviewTabs.ts new file mode 100644 index 0000000000..f32bf1e22e --- /dev/null +++ b/ai_evals/adapters/frontend/core/global/evalPreviewTabs.ts @@ -0,0 +1,188 @@ +import { + setClosePreviewTabsHandler, + setGetPreviewStatusHandler, + setOpenPagePreviewHandler, + setOpenPreviewHandler, +} from "../../../../../frontend/src/lib/components/copilot/chat/global/core"; +import type { GlobalActivePreviewContext } from "../../../../../frontend/src/lib/components/copilot/chat/global/core"; +import { + describePreview, + previewTargetForSessionTarget, + selectPreviewTabsToClose, + SessionPreviewTabs, + whereIs, +} from "../../../../../frontend/src/lib/components/sessions/sessionPreviewTabs.svelte"; +import { + previewLocationContext, + previewLocationLabel, + promptSafe, + resolvePreviewTab, +} from "../../../../../frontend/src/lib/components/sessions/previewRouter"; +import type { ArtifactVersionTarget } from "../../../../../frontend/src/lib/components/sessions/previewRouter"; +import type { SessionTarget } from "../../../../../frontend/src/lib/components/sessions/sessionState.svelte"; + +// The side panel a session chat talks to, driven by the production tab model rather than +// by canned tool results — so a case measures what the real open_preview / get_preview_status +// / close_page report about the tabs the reader has. sessionRuntime.svelte.ts (the production +// owner of these handlers) can't run here: it reaches for IndexedDB, stores and live editors. + +export interface EvalPreviewTabFixture { + /** Artifact tab, named by the artifact fixture it shows. `version` pins it, as a reader does. */ + artifact?: { name: string; version?: number }; + /** Workspace page tab, e.g. `{ href: "/runs", label: "Runs" }`. */ + page?: { href: string; label: string }; + /** Editor tab for a workspace item. */ + item?: { kind: SessionTarget["kind"]; path: string }; + /** Tab the reader is looking at. Defaults to the last seeded one. */ + active?: boolean; +} + +// Registered once for the whole process, as production does at module load, and dispatched +// by session id: global cases run concurrently, so a per-run registration would have every +// case answering out of whichever run registered last. +const panels = new Map(); + +const NO_SESSION = "No active session; the preview panel is unavailable."; + +function panelFor(sessionId: string | undefined): SessionPreviewTabs | undefined { + return sessionId ? panels.get(sessionId) : undefined; +} + +setGetPreviewStatusHandler((sessionId) => { + const owner = panelFor(sessionId); + if (!owner) return NO_SESSION; + return describePreview(owner.tabs, owner.activeId, !!owner.displayedTab); +}); + +setOpenPreviewHandler(async ({ sessionId, kind, path }) => { + const owner = panelFor(sessionId); + if (!owner) return "Error: no active session to open the preview in."; + const target = previewTargetForSessionTarget(kind, path); + if (!target) { + return `Error: ${kind} targets cannot be shown in the preview panel.`; + } + // The pipeline branch of the production handler waits on an editor that only exists once + // a canvas mounts, which never happens here — a pipeline preview reports as any other. + const result = owner.open(target); + return result.status === "focused" + ? `A preview tab is already showing ${kind} "${path}" — focused it.` + : `Opened ${kind} preview for ${path} in a new tab in the side panel.`; +}); + +setOpenPagePreviewHandler(({ sessionId, href, label, newTab }) => { + const owner = panelFor(sessionId); + if (!owner) return undefined; + const result = owner.open({ type: "page", href, label }, { forceNewTab: newTab }); + if (result.status === "focused") { + return `A preview tab is already showing ${label} — focused it.`; + } + if (result.status === "retargeted") { + return `Updated the ${label} preview tab with the requested view.`; + } + return `Opened ${label} in a new preview tab in the side panel.`; +}); + +setClosePreviewTabsHandler(({ sessionId, all, match }) => { + const owner = panelFor(sessionId); + if (!owner) return NO_SESSION; + if (owner.tabs.length === 0) return "The preview panel has no open tabs."; + const labelFor = (t: (typeof owner.tabs)[number]) => + promptSafe(previewLocationLabel(whereIs(t))); + const doomed = selectPreviewTabsToClose(owner.tabs, { all, match }); + if (doomed.length === 0) { + return `No open tab matched "${match}". Open tabs: ${owner.tabs.map(labelFor).join(", ")}.`; + } + const closedLabels = doomed.map(labelFor); + for (const t of doomed) owner.close(t.id); + return `Closed ${closedLabels.length} preview tab${closedLabels.length === 1 ? "" : "s"} (${closedLabels.join(", ")}).`; +}); + +export interface EvalPreviewPanel { + /** Mirrors production: a written artifact is shown in the panel. `version` carries the + * caller's intent for the version picker — `latest` drops a pin the reader had set. */ + openArtifact: (id: string, name: string, version?: ArtifactVersionTarget) => void; + /** What the user message stamps as ACTIVE PREVIEW, as sessionRuntime's resolver reads it. */ + activePreview: () => GlobalActivePreviewContext | undefined; + dispose: () => void; +} + +export function createEvalPreviewPanel(input: { + sessionId: string; + tabs: EvalPreviewTabFixture[]; + /** Artifact ids by name, from the artifact fixture seeding. */ + artifactIds: Map; +}): EvalPreviewPanel { + // Nothing durable to write back to, and no debounce worth waiting on. + const owner = new SessionPreviewTabs( + { tabs: [], activeId: "", collapsed: false }, + { persist: () => {} }, + 0, + ); + // Opening a tab makes it the active one, so the fixture's pick can only be applied once + // every tab is seeded — selecting inside the loop would lose to the next open. + let requestedActive: string | undefined; + for (const fixture of input.tabs) { + const opened = seedTab(owner, fixture, input.artifactIds); + if (opened && fixture.active) requestedActive = opened; + } + if (requestedActive) owner.select(requestedActive); + // Registered last: seeding throws on a malformed fixture, and this map outlives the run. + panels.set(input.sessionId, owner); + + return { + openArtifact: (id, name, version) => { + owner.open({ type: "artifact", id, name, version }); + }, + activePreview: () => { + const tab = owner.displayedTab; + if (!tab) return undefined; + // Artifact and editor tabs are not iframes: they carry no page location, and an + // artifact's pinned version reaches the chat only through get_preview_status. + if (resolvePreviewTab(tab.url).kind !== "iframe") return undefined; + return previewLocationContext(whereIs(tab)); + }, + dispose: () => { + panels.delete(input.sessionId); + }, + }; +} + +// Seeds one tab through the production open path and returns its id, so a fixture cannot +// describe a tab the panel could not have reached on its own. +function seedTab( + owner: SessionPreviewTabs, + fixture: EvalPreviewTabFixture, + artifactIds: Map, +): string | undefined { + // A tab shows one destination; the branches below would silently keep the first. + const named = [fixture.artifact, fixture.page, fixture.item].filter(Boolean); + if (named.length > 1) { + throw new Error( + "Preview tab fixture sets more than one of artifact, page and item — a tab shows one of them", + ); + } + if (fixture.artifact) { + const id = artifactIds.get(fixture.artifact.name); + if (!id) { + throw new Error( + `Preview tab fixture references artifact "${fixture.artifact.name}", which no artifact fixture seeds`, + ); + } + owner.open({ type: "artifact", id, name: fixture.artifact.name }); + // A pin is the reader's own pick in the version picker, never a side effect of opening. + if (fixture.artifact.version !== undefined) { + owner.pinArtifactVersion(id, fixture.artifact.version); + } + } else if (fixture.page) { + owner.open({ type: "page", href: fixture.page.href, label: fixture.page.label }); + } else if (fixture.item) { + const target = previewTargetForSessionTarget(fixture.item.kind, fixture.item.path); + if (!target) { + throw new Error(`Preview tab fixture has an unpreviewable item kind: ${fixture.item.kind}`); + } + owner.open(target); + } else { + throw new Error("Preview tab fixture must set one of artifact, page or item"); + } + return owner.activeId; +} diff --git a/ai_evals/adapters/frontend/core/global/evalPreviewTabs.vitest.ts b/ai_evals/adapters/frontend/core/global/evalPreviewTabs.vitest.ts new file mode 100644 index 0000000000..b9a2990a0a --- /dev/null +++ b/ai_evals/adapters/frontend/core/global/evalPreviewTabs.vitest.ts @@ -0,0 +1,37 @@ +import { expect, it, vi } from 'vitest' + +// The panel pulls in the global tool module, which reaches the editor stack it never uses here. +vi.mock('monaco-editor', () => ({ + editor: {}, + languages: {}, + KeyCode: {}, + Uri: { parse: (value: string) => ({ toString: () => value }) }, + MarkerSeverity: { Error: 8, Warning: 4, Info: 2, Hint: 1 } +})) +vi.mock('@codingame/monaco-vscode-standalone-typescript-language-features', () => ({ + getTypeScriptWorker: async () => async () => ({}), + typescriptVersion: 'test' +})) +vi.mock('@codingame/monaco-vscode-languages-service-override', () => ({ default: () => ({}) })) +vi.mock('$lib/components/vscode', () => ({})) + +const { createEvalPreviewPanel } = await import('./evalPreviewTabs') + +// Every open makes its own tab active, so a fixture's `active` flag only means anything if +// it survives the tabs seeded after it. Lose that and a case still runs — against a panel +// state its author never described. +it('keeps the tab a fixture marks active, not the last one seeded', () => { + const panel = createEvalPreviewPanel({ + sessionId: 'eval-preview-tabs-unit-test', + tabs: [ + { page: { href: '/runs', label: 'Runs' }, active: true }, + { artifact: { name: 'Onboarding plan' } } + ], + artifactIds: new Map([['Onboarding plan', 'eval-artifact-0']]) + }) + try { + expect(panel.activePreview()?.location).toBe('/runs') + } finally { + panel.dispose() + } +}) diff --git a/ai_evals/adapters/frontend/core/global/globalEvalRunner.ts b/ai_evals/adapters/frontend/core/global/globalEvalRunner.ts index dcd771663a..d35d874f71 100644 --- a/ai_evals/adapters/frontend/core/global/globalEvalRunner.ts +++ b/ai_evals/adapters/frontend/core/global/globalEvalRunner.ts @@ -12,9 +12,19 @@ import { getGlobalDraft, listGlobalDrafts, } from "../../../../../frontend/src/lib/components/copilot/chat/global/userDraftAdapter"; +import { appendPlanModeInstructions } from "../../../../../frontend/src/lib/components/copilot/chat/planMode"; import type { Tool as ProductionTool } from "../../../../../frontend/src/lib/components/copilot/chat/shared"; +import { createEvalPlanTools } from "./planModeTools"; import { UserDraft } from "../../../../../frontend/src/lib/userDraft.svelte"; -import { createEvalArtifactHelpers } from "./evalArtifactStore"; +import { + createEvalArtifactHelpers, + type SeededArtifact, +} from "./evalArtifactStore"; +import { + createEvalPreviewPanel, + type EvalPreviewPanel, + type EvalPreviewTabFixture, +} from "./evalPreviewTabs"; import type { ModeRunContext } from "../../../../core/types"; import type { GlobalDraftState } from "../../../../core/validators"; import type { WindmillBackendSettings } from "../../../../core/windmillBackendSettings"; @@ -83,11 +93,18 @@ export interface GlobalEvalOptions { user?: GlobalUserFixture; // Emulate a session chat (preview tools + session prompt); default false = standalone baseline. sessionChat?: boolean; + // Start in plan mode: the gate refuses every tool without `planModeSafe`, and the two plan + // tools are offered. Needs sessionChat, which is what plan mode is gated on in production. + planMode?: boolean; model?: string; maxIterations?: number; provider?: AIProvider; backend: WindmillBackendSettings; workspaceRoot?: string; + // Artifacts the session already holds when the run starts. + artifacts?: SeededArtifact[]; + /** Tabs already open in the side panel, including any artifact version the reader pinned. */ + previewTabs?: EvalPreviewTabFixture[]; runContext?: ModeRunContext; } @@ -106,27 +123,67 @@ export async function runGlobalEval( options.workspaceFixtures ?? {}, ); seedLiveEditorDrafts(workspaceRoot, options.liveEditorDrafts ?? []); + // Declared out here only so `finally` can reach it; a malformed fixture throws while + // building it, and everything seeded above still has to be torn down. + let panel: EvalPreviewPanel | undefined; try { + const evalArtifacts = createEvalArtifactHelpers(options.artifacts); + // Only a session chat has a side panel, so only it gets one here. Seeded tabs would + // otherwise vanish without a word, and the case would measure an empty panel. + if (!options.sessionChat && options.previewTabs?.length) { + throw new Error( + "This fixture seeds previewTabs, which only a session chat has — set runtime.sessionChat: true on the case.", + ); + } + if (options.sessionChat) { + panel = createEvalPreviewPanel({ + sessionId: evalArtifacts.sessionId, + tabs: options.previewTabs ?? [], + artifactIds: evalArtifacts.seededIds, + }); + } const model = options.model ?? "claude-haiku-4-5-20251001"; const injectActiveEditorContext = process.env[DISABLE_ACTIVE_EDITOR_CONTEXT_ENV] !== "1"; + const planMode = options.planMode + ? createEvalPlanTools({ + create: evalArtifacts.helpers.artifacts.create, + sessionId: evalArtifacts.helpers.sessionId, + chatId: evalArtifacts.helpers.getChatId(), + }) + : undefined; // Pass the seeded identity straight to the prompt builder rather than mutating // the process-global `userStore`, so concurrent cases never race on it. - const evalArtifacts = createEvalArtifactHelpers(); + const baseSystemMessage = prepareGlobalSystemMessage(undefined, { + user: options.user, + previewTools: options.sessionChat ?? false, + }); const rawResult = await runEval({ userPrompt, - systemMessage: prepareGlobalSystemMessage(undefined, { - user: options.user, - previewTools: options.sessionChat ?? false, + systemMessage: baseSystemMessage, + // Re-derived per request, as production's getter is: the instructions have to leave + // the prompt when the plan is approved, or the model is still told it may not build + // while the gate has already opened. + getSystemMessage: planMode + ? () => + planMode.isPlanModeActive() + ? appendPlanModeInstructions(baseSystemMessage, 0) + : baseSystemMessage + : undefined, + isPlanModeActive: planMode?.isPlanModeActive, + isToolAvailable: planMode?.isToolAvailable, + userMessage: prepareGlobalUserMessage(userPrompt, [], { + ...(injectActiveEditorContext ? { workspace: workspaceRoot } : {}), + activePreview: panel?.activePreview(), }), - userMessage: prepareGlobalUserMessage( - userPrompt, - [], - injectActiveEditorContext ? { workspace: workspaceRoot } : {}, - ), - tools: getGlobalEvalTools(options.sessionChat ?? false), - helpers: evalArtifacts.helpers, + tools: [ + ...getGlobalEvalTools(options.sessionChat ?? false), + ...(planMode?.tools ?? []), + ], + helpers: panel + ? { ...evalArtifacts.helpers, openArtifact: panel.openArtifact } + : evalArtifacts.helpers, apiKey, getOutput: async () => ({ ...(await collectGlobalDraftState(workspaceRoot)), @@ -159,6 +216,7 @@ export async function runGlobalEval( finalContextTokens: rawResult.finalContextTokens, }; } finally { + panel?.dispose(); clearGlobalDrafts(workspaceRoot); clearLiveEditorDrafts(workspaceRoot, options.liveEditorDrafts ?? []); unregisterBenchmarkWorkspaceRunnables(workspaceRoot); diff --git a/ai_evals/adapters/frontend/core/global/planModeTools.ts b/ai_evals/adapters/frontend/core/global/planModeTools.ts new file mode 100644 index 0000000000..bea5538fc8 --- /dev/null +++ b/ai_evals/adapters/frontend/core/global/planModeTools.ts @@ -0,0 +1,69 @@ +import { + EXIT_PLAN_MODE_TOOL, + EXIT_PLAN_MODE_TOOL_DESCRIPTION, + derivePlanTitle, + exitPlanModeArgs, + planSummaryOf, +} from "../../../../../frontend/src/lib/components/copilot/chat/planMode"; +import { PLAN_MODE_MESSAGES } from "../../../../../frontend/src/lib/components/copilot/chat/planModeMessages"; +import { createToolDef } from "../../../../../frontend/src/lib/components/copilot/chat/shared"; +import type { Tool as ProductionTool } from "../../../../../frontend/src/lib/components/copilot/chat/shared"; + +/** + * `exit_plan_mode` built from the production schema, description and messages, so a case + * exercises the real gate and wording with the posture living here rather than on the + * manager. It resolves immediately — the runners define no `requestConfirmation`, so the + * plan is always approved and a refused one cannot be expressed. + */ +export function createEvalPlanTools(artifacts: { + create: ( + sessionId: string, + input: Record, + ) => Promise<{ id: string; name: string }>; + sessionId: string; + chatId: string; +}): { + tools: ProductionTool<{}>[]; + isPlanModeActive: () => boolean; + isToolAvailable: (name: string) => boolean; +} { + let planActive = true; + return { + isPlanModeActive: () => planActive, + // Withdrawn on approval, as production's tool getter does it: leaving it advertised + // invites a second hand-over of a plan already agreed, which would write a duplicate. + // Production would offer enter_plan_mode in its place; these cases stop at the first + // hand-over, so a fresh planning round belongs to a case of its own. + isToolAvailable: (name) => name !== EXIT_PLAN_MODE_TOOL || planActive, + // Production offers one plan tool at a time and these cases start in plan mode, so + // enter_plan_mode would only invite a turn spent entering a posture already held. + tools: [ + { + def: createToolDef( + exitPlanModeArgs, + EXIT_PLAN_MODE_TOOL, + EXIT_PLAN_MODE_TOOL_DESCRIPTION, + ), + // Carries the safety tag for the same reason production does: it is the only way out + // of the posture, so the gate must not refuse it. + planModeSafe: true, + fn: async ({ args }) => { + const summary = planSummaryOf(args); + if (!summary?.trim()) { + return PLAN_MODE_MESSAGES.missingSummary; + } + planActive = false; + await artifacts.create(artifacts.sessionId, { + name: derivePlanTitle(summary), + content: summary, + kind: "md", + role: "plan", + approvedVersion: 1, + chatId: artifacts.chatId, + }); + return PLAN_MODE_MESSAGES.approvedWithDoc; + }, + }, + ] as ProductionTool<{}>[], + }; +} diff --git a/ai_evals/adapters/frontend/core/shared/baseEvalRunner.ts b/ai_evals/adapters/frontend/core/shared/baseEvalRunner.ts index f787743dfa..4501f24810 100644 --- a/ai_evals/adapters/frontend/core/shared/baseEvalRunner.ts +++ b/ai_evals/adapters/frontend/core/shared/baseEvalRunner.ts @@ -43,6 +43,15 @@ export interface RunEvalParams { getOutput: () => TOutput | Promise; /** Model and Windmill backend configuration */ options: EvalRunnerOptions; + /** Drives the production plan-mode gate in processToolCall. Absent leaves it inert, + * which is what every mode but an opted-in global case wants. */ + isPlanModeActive?: () => boolean; + /** Which of `tools` the model is offered on this request. Absent offers all of them. */ + isToolAvailable?: (name: string) => boolean; + /** Re-read before every request, as production's systemMessage getter is. Needed when a + * tool changes what the prompt should say — plan mode's instructions have to come back + * out once the plan is approved. Falls back to the fixed `systemMessage`. */ + getSystemMessage?: () => ChatCompletionSystemMessageParam; onAssistantMessageStart?: () => void; onAssistantToken?: (token: string) => void; onAssistantMessageEnd?: () => void; @@ -68,6 +77,9 @@ export async function runEval( onAssistantToken, onAssistantMessageEnd, onToolCall, + isPlanModeActive, + isToolAvailable, + getSystemMessage, } = params; let shouldEmitMessageStart = true; @@ -119,6 +131,7 @@ export async function runEval( } = { setToolStatus: () => {}, removeToolStatus: () => {}, + isPlanModeActive, onNewToken: (token: string) => { if (shouldEmitMessageStart) { onAssistantMessageStart?.(); @@ -140,8 +153,17 @@ export async function runEval( try { const result = await runChatLoop({ messages, - systemMessage, - tools: wrappedTools, + get systemMessage() { + return getSystemMessage?.() ?? systemMessage; + }, + // Re-derived per request, as `systemMessage` is: a tool the posture has withdrawn + // must leave the schema too, or the model keeps being offered a call the run has + // moved past — and the token counts a case reports include a tool it cannot use. + get tools() { + return isToolAvailable + ? wrappedTools.filter((t) => isToolAvailable(t.def.function.name)) + : wrappedTools; + }, helpers, abortController, callbacks, diff --git a/ai_evals/adapters/frontend/vitest.unit.config.ts b/ai_evals/adapters/frontend/vitest.unit.config.ts new file mode 100644 index 0000000000..7c5a794508 --- /dev/null +++ b/ai_evals/adapters/frontend/vitest.unit.config.ts @@ -0,0 +1,31 @@ +import { fileURLToPath } from 'node:url' +import frontendConfig from '../../../frontend/vite.config.js' + +// Harness unit tests that reach into the frontend module graph. They can't run under +// `bun test` (Svelte runes and the SvelteKit aliases both need this build), so they are +// named `*.vitest.ts` — bun's `*.test.ts` sweep skips them and this config claims them. +const FRONTEND_VITE_CONFIG_PATH = fileURLToPath(new URL('../../../frontend/vite.config.js', import.meta.url)) +const FRONTEND_TEST_SETUP_PATH = fileURLToPath( + new URL('../../../frontend/src/lib/test-setup.ts', import.meta.url) +) +const UNIT_TESTS = fileURLToPath(new URL('./**/*.vitest.ts', import.meta.url)) + +const config = { + ...frontendConfig, + test: { + ...frontendConfig.test, + projects: [ + { + extends: FRONTEND_VITE_CONFIG_PATH, + test: { + name: 'server', + environment: 'node', + include: [UNIT_TESTS], + setupFiles: [FRONTEND_TEST_SETUP_PATH] + } + } + ] + } +} + +export default config diff --git a/ai_evals/cases/global.yaml b/ai_evals/cases/global.yaml index ec2260c0e2..54fae9b24f 100644 --- a/ai_evals/cases/global.yaml +++ b/ai_evals/cases/global.yaml @@ -1180,6 +1180,7 @@ - id: global-closepage1-close-runs-tab prompt: |- You just opened the runs page for me in the side panel. Close that tab, I'm done looking at it. + initial: ai_evals/fixtures/frontend/global/initial/preview_runs_tab.json runtime: maxTurns: 6 sessionChat: true @@ -1233,6 +1234,32 @@ - creates one artifact and revises it rather than creating a second artifact - each revision carries a short description of what changed +# A reader who pins an older version in the artifact's picker is looking at something the +# artifact tools never report: an artifact tab carries no ACTIVE PREVIEW section, so the pin +# reaches the chat through get_preview_status alone. Asked what is on screen, the model has +# to read the panel instead of answering from the artifact's own history. + +- id: global-artifact-pinned-version-question + prompt: |- + Which version of the onboarding plan am I looking at right now? + initial: ai_evals/fixtures/frontend/global/initial/artifact_onboarding_plan_pinned_v2.json + runtime: + maxTurns: 6 + sessionChat: true + validate: + draftCountExactly: 0 + toolExpect: + requiredToolsUsed: + - get_preview_status + forbiddenToolsUsed: + - create_artifact + - update_artifact + - deploy_workspace_item + skipJudge: true + judgeChecklist: + - answers that the panel is showing version 2, not the latest version 5 + - does not edit or re-create the artifact + # --- Documentation search (search_docs) --- # Pure product-knowledge questions: the assistant should consult the docs via # search_docs and answer conversationally, not draft or mutate anything. No @@ -1756,8 +1783,55 @@ judgeChecklist: - saves the plan as a markdown artifact via create_artifact rather than only replying inline - the artifact content has a title, a one-line summary, and three or four bullet steps for onboarding + - the artifact is registered as the session's plan (role "plan"), not as an ordinary note - the user asked for the plan they will come back to and revise - does not create a flow or script draft yet +- id: global-planmode1-hands-over-a-plan + prompt: |- + Our support inbox is a mess. I want incoming emails triaged by urgency and routed to the + right team, with anything urgent also posted to Slack. + Work out how you'd build this in Windmill. + initial: ai_evals/fixtures/frontend/global/initial/user_admin_evals_folder.json + runtime: + maxTurns: 10 + sessionChat: true + planMode: true + # No draft assertion: approving the plan opens the gate mid-run, and building from there is + # what production asks for, so a draft is not a failure. The gate itself is covered by + # shared.test.ts; what only a real model can show is whether it researches and hands over a + # usable plan instead of guessing at one. + toolExpect: + requiredToolsUsed: + - exit_plan_mode + # Not "saves the plan as an artifact": exit_plan_mode writes it, so the harness would + # satisfy that on every run the tool is called at all — it grades itself, not the model. + judgeChecklist: + - the plan covers classifying an incoming email by urgency, routing it to a team, and posting urgent ones to Slack + - the plan is specific about what would be built in Windmill (a flow and its steps, or the scripts involved) + +- id: global-planmode2-sketches-while-planning + prompt: |- + We're moving our nightly CSV export off a schedule and onto a webhook the vendor calls + when their file is ready. Work out how you'd rebuild it in Windmill — and draw me the + shape of it before you write anything, I find that easier to react to than prose. + initial: ai_evals/fixtures/frontend/global/initial/user_admin_evals_folder.json + runtime: + maxTurns: 10 + sessionChat: true + planMode: true + # Both tools, because either alone is a different behaviour: create_artifact without + # exit_plan_mode means the model filed the plan as the drawing, and exit_plan_mode without + # create_artifact means the posture blocked the drawing it was asked for. + toolExpect: + requiredToolsUsed: + - create_artifact + - exit_plan_mode + judgeChecklist: + - saves a diagram of the proposed design as an artifact rather than only describing it in chat + - the diagram covers the webhook that starts the run and the steps that replace the nightly schedule + - hands the plan over with exit_plan_mode instead of leaving it in the artifact + - does not register the diagram as the session's plan document + - id: global-npm1-script-search-package prompt: |- Find a good npm package for parsing RSS/Atom feeds and use it to create a draft Bun script diff --git a/ai_evals/core/types.ts b/ai_evals/core/types.ts index f1655bd18f..b5195443d0 100644 --- a/ai_evals/core/types.ts +++ b/ai_evals/core/types.ts @@ -33,6 +33,9 @@ export interface EvalCaseRuntimeSpec { appContext?: EvalCaseRuntimeAppContextSpec; // Global mode: run as a session chat (preview tools + session prompt) vs the standalone chat. sessionChat?: boolean; + // Global session chats: start the case in plan mode, so the workspace-changing tools are + // refused until the model hands over a plan with exit_plan_mode. + planMode?: boolean; } export interface FlowValidationSpec { diff --git a/ai_evals/fixtures/frontend/global/initial/artifact_onboarding_plan_pinned_v2.json b/ai_evals/fixtures/frontend/global/initial/artifact_onboarding_plan_pinned_v2.json new file mode 100644 index 0000000000..dd4ab37873 --- /dev/null +++ b/ai_evals/fixtures/frontend/global/initial/artifact_onboarding_plan_pinned_v2.json @@ -0,0 +1,38 @@ +{ + "user": { + "username": "admin", + "is_admin": true + }, + "artifacts": [ + { + "name": "Onboarding plan", + "versions": [ + { + "content": "# Onboarding plan\n\nA staged rollout of the customer onboarding flow.\n\n- Collect the signup form\n- Create the customer record\n- Send the welcome email\n" + }, + { + "content": "# Onboarding plan\n\nA staged rollout of the customer onboarding flow.\n\n- Collect the signup form\n- Verify the company domain\n- Create the customer record\n- Send the welcome email\n", + "note": "Added domain verification" + }, + { + "content": "# Onboarding plan\n\nA staged rollout of the customer onboarding flow.\n\n- Collect the signup form\n- Verify the company domain\n- Create the customer record\n- Send the welcome email\n- Schedule the 7-day check-in\n", + "note": "Added the 7-day check-in" + }, + { + "content": "# Onboarding plan\n\nA staged rollout of the customer onboarding flow.\n\n- Collect the signup form\n- Verify the company domain\n- Create the customer record in the CRM\n- Send the welcome email\n- Schedule the 7-day check-in\n", + "note": "Named the CRM as the record store" + }, + { + "content": "# Onboarding plan\n\nA staged rollout of the customer onboarding flow.\n\n- Collect the signup form\n- Verify the company domain\n- Create the customer record in the CRM\n- Send the welcome email\n- Schedule the 7-day check-in\n- Hand over to the account manager\n", + "note": "Added the account-manager handover" + } + ] + } + ], + "previewTabs": [ + { + "artifact": { "name": "Onboarding plan", "version": 2 }, + "active": true + } + ] +} diff --git a/ai_evals/fixtures/frontend/global/initial/preview_runs_tab.json b/ai_evals/fixtures/frontend/global/initial/preview_runs_tab.json new file mode 100644 index 0000000000..565e5ede37 --- /dev/null +++ b/ai_evals/fixtures/frontend/global/initial/preview_runs_tab.json @@ -0,0 +1,12 @@ +{ + "user": { + "username": "admin", + "is_admin": true + }, + "previewTabs": [ + { + "page": { "href": "/runs", "label": "Runs" }, + "active": true + } + ] +} diff --git a/ai_evals/modes/global.ts b/ai_evals/modes/global.ts index 5628c21d5d..ad93b09a55 100644 --- a/ai_evals/modes/global.ts +++ b/ai_evals/modes/global.ts @@ -6,9 +6,15 @@ import { type GlobalLiveEditorDraftFixture, type GlobalUserFixture, } from "../adapters/frontend/core/global/globalEvalRunner"; +import type { SeededArtifact } from "../adapters/frontend/core/global/evalArtifactStore"; +import type { EvalPreviewTabFixture } from "../adapters/frontend/core/global/evalPreviewTabs"; import type { BenchmarkWorkspaceRunnables } from "../adapters/frontend/mockBackend"; import type { FrontendEvalModelConfig } from "../core/models"; -import type { BenchmarkArtifactFile, GlobalValidationSpec, ModeRunner } from "../core/types"; +import type { + BenchmarkArtifactFile, + GlobalValidationSpec, + ModeRunner, +} from "../core/types"; import { validateGlobalState, type GlobalDraftState } from "../core/validators"; import type { WindmillBackendSettings } from "../core/windmillBackendSettings"; import { getFrontendApiKey } from "./frontendCommon"; @@ -17,6 +23,8 @@ export interface GlobalInitialFixture { workspace?: BenchmarkWorkspaceRunnables; liveEditorDrafts?: GlobalLiveEditorDraftFixture[]; user?: GlobalUserFixture; + artifacts?: SeededArtifact[]; + previewTabs?: EvalPreviewTabFixture[]; } export function createGlobalModeRunner( @@ -41,7 +49,10 @@ export function createGlobalModeRunner( workspaceFixtures: initial?.workspace, liveEditorDrafts: initial?.liveEditorDrafts, user: initial?.user, + artifacts: initial?.artifacts, + previewTabs: initial?.previewTabs, sessionChat: context.evalCase?.runtime?.sessionChat, + planMode: context.evalCase?.runtime?.planMode, maxIterations: context.evalCase?.runtime?.maxTurns, provider: modelConfig.provider, model: modelConfig.model, @@ -81,7 +92,9 @@ export function createGlobalModeRunner( }; } -async function loadGlobalInitialFixture(path: string): Promise { +async function loadGlobalInitialFixture( + path: string, +): Promise { if ((await stat(path)).isDirectory()) { const { initialFrontend, initialBackend, initialDatatables } = await loadAppFixtureForEval(path); @@ -104,14 +117,20 @@ async function loadGlobalInitialFixture(path: string): Promise { +async function loadGlobalExpectedFixture( + path: string, +): Promise { return JSON.parse(await readFile(path, "utf8")) as GlobalDraftState; } diff --git a/ai_evals/package.json b/ai_evals/package.json index b7fbeaa1a5..6720c910cb 100644 --- a/ai_evals/package.json +++ b/ai_evals/package.json @@ -4,7 +4,8 @@ "type": "module", "scripts": { "cli": "bun cli/index.ts", - "typecheck": "tsc -p tsconfig.json" + "typecheck": "tsc -p tsconfig.json", + "test:frontend-graph": "cd ../frontend && node_modules/.bin/vitest run --project server --config ../ai_evals/adapters/frontend/vitest.unit.config.ts" }, "dependencies": { "@anthropic-ai/claude-agent-sdk": "^0.2.25", diff --git a/ai_evals/tsconfig.json b/ai_evals/tsconfig.json index 7b06d5788a..50b2b1110d 100644 --- a/ai_evals/tsconfig.json +++ b/ai_evals/tsconfig.json @@ -14,6 +14,8 @@ ], "exclude": [ "./**/*.test.ts", - "./adapters/frontend/vitest.config.ts" + "./**/*.vitest.ts", + "./adapters/frontend/vitest.config.ts", + "./adapters/frontend/vitest.unit.config.ts" ] } diff --git a/backend/.sqlx/query-a41c4cbaffdb714e4a963557de5a4011744d684eb24e03cb4beae6a512613159.json b/backend/.sqlx/query-0c18351237816fe0c56e23801fcb8e70dbffcf08ed121e55c871f727c4ddf626.json similarity index 80% rename from backend/.sqlx/query-a41c4cbaffdb714e4a963557de5a4011744d684eb24e03cb4beae6a512613159.json rename to backend/.sqlx/query-0c18351237816fe0c56e23801fcb8e70dbffcf08ed121e55c871f727c4ddf626.json index 0141c379b4..ceb05001e4 100644 --- a/backend/.sqlx/query-a41c4cbaffdb714e4a963557de5a4011744d684eb24e03cb4beae6a512613159.json +++ b/backend/.sqlx/query-0c18351237816fe0c56e23801fcb8e70dbffcf08ed121e55c871f727c4ddf626.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "UPDATE worker_ping SET ping_at = now(), jobs_executed = $1, custom_tags = $2,\n occupancy_rate = $3, memory_usage = $4, wm_memory_usage = $5, vcpus = COALESCE($7, vcpus),\n memory = COALESCE($8, memory), occupancy_rate_15s = $9, occupancy_rate_5m = $10, occupancy_rate_30m = $11, native_mode = $12 WHERE worker = $6", + "query": "UPDATE worker_ping SET ping_at = now(), jobs_executed = $1, custom_tags = $2,\n occupancy_rate = $3, memory_usage = $4, wm_memory_usage = $5, vcpus = COALESCE($7, vcpus),\n memory = COALESCE($8, memory), occupancy_rate_15s = $9, occupancy_rate_5m = $10, occupancy_rate_30m = $11, native_mode = $12, ip = COALESCE($13, ip) WHERE worker = $6", "describe": { "columns": [], "parameters": { @@ -16,10 +16,11 @@ "Float4", "Float4", "Float4", - "Bool" + "Bool", + "Varchar" ] }, "nullable": [] }, - "hash": "a41c4cbaffdb714e4a963557de5a4011744d684eb24e03cb4beae6a512613159" + "hash": "0c18351237816fe0c56e23801fcb8e70dbffcf08ed121e55c871f727c4ddf626" } diff --git a/backend/.sqlx/query-0df1c23ac429b5807d42e1da5ee2e2176a8a774f107bece5804cc7b221365c89.json b/backend/.sqlx/query-0df1c23ac429b5807d42e1da5ee2e2176a8a774f107bece5804cc7b221365c89.json new file mode 100644 index 0000000000..cc5e9a8d45 --- /dev/null +++ b/backend/.sqlx/query-0df1c23ac429b5807d42e1da5ee2e2176a8a774f107bece5804cc7b221365c89.json @@ -0,0 +1,58 @@ +{ + "db_name": "PostgreSQL", + "query": "WITH RECURSIVE job_tree AS (\n SELECT id, tag FROM v2_job WHERE id = $2 AND workspace_id = $1\n UNION\n SELECT j.id, j.tag FROM v2_job j JOIN job_tree t ON j.parent_job = t.id\n WHERE j.workspace_id = $1\n )\n SELECT\n a.path,\n a.kind AS \"kind!: windmill_common::assets::AssetKind\",\n -- Several jobs of the tree touch one asset, each recording its own\n -- access. A job that recorded none contributes nothing rather than\n -- erasing a sibling's, so an all-null group is the only unknown one.\n -- Grouping here, not in Rust, is what makes LIMIT count assets: the\n -- retention keeps up to ten job rows per asset.\n COALESCE(bool_or(a.usage_access_type IN ('r', 'rw')), false) AS \"any_read!\",\n COALESCE(bool_or(a.usage_access_type IN ('w', 'rw')), false) AS \"any_write!\"\n FROM asset a JOIN job_tree t ON a.usage_path = t.id::text\n WHERE a.workspace_id = $1 AND a.usage_kind = 'job'\n AND ($3::text[] IS NULL OR t.tag = ANY($3))\n GROUP BY a.path, a.kind\n ORDER BY a.path, a.kind\n LIMIT $4", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "kind!: windmill_common::assets::AssetKind", + "type_info": { + "Custom": { + "name": "asset_kind", + "kind": { + "Enum": [ + "s3object", + "resource", + "variable", + "ducklake", + "datatable", + "volume", + "dbt" + ] + } + } + } + }, + { + "ordinal": 2, + "name": "any_read!", + "type_info": "Bool" + }, + { + "ordinal": 3, + "name": "any_write!", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text", + "Uuid", + "TextArray", + "Int8" + ] + }, + "nullable": [ + false, + false, + null, + null + ] + }, + "hash": "0df1c23ac429b5807d42e1da5ee2e2176a8a774f107bece5804cc7b221365c89" +} diff --git a/backend/.sqlx/query-1d346a14ad5586af347b8e7ac413500a39efa20e4915ffa56fd40537597db36e.json b/backend/.sqlx/query-1d346a14ad5586af347b8e7ac413500a39efa20e4915ffa56fd40537597db36e.json new file mode 100644 index 0000000000..e8ffcfb95b --- /dev/null +++ b/backend/.sqlx/query-1d346a14ad5586af347b8e7ac413500a39efa20e4915ffa56fd40537597db36e.json @@ -0,0 +1,26 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT 'schedule' AS \"kind!\", COUNT(*)::BIGINT AS \"count!\" FROM schedule\n UNION ALL SELECT 'http', COUNT(*)::BIGINT FROM http_trigger\n UNION ALL SELECT 'websocket', COUNT(*)::BIGINT FROM websocket_trigger\n UNION ALL SELECT 'kafka', COUNT(*)::BIGINT FROM kafka_trigger\n UNION ALL SELECT 'nats', COUNT(*)::BIGINT FROM nats_trigger\n UNION ALL SELECT 'postgres', COUNT(*)::BIGINT FROM postgres_trigger\n UNION ALL SELECT 'mqtt', COUNT(*)::BIGINT FROM mqtt_trigger\n UNION ALL SELECT 'sqs', COUNT(*)::BIGINT FROM sqs_trigger\n UNION ALL SELECT 'gcp', COUNT(*)::BIGINT FROM gcp_trigger\n UNION ALL SELECT 'azure', COUNT(*)::BIGINT FROM azure_trigger\n UNION ALL SELECT 'amqp', COUNT(*)::BIGINT FROM amqp_trigger\n UNION ALL SELECT 'email', COUNT(*)::BIGINT FROM email_trigger\n -- Grouped, not a single 'native' key: these fire as nextcloud/google/github,\n -- so a lone key would not line up with the `trigger`/`fired` series.\n UNION ALL SELECT service_name::text, COUNT(*)::BIGINT FROM native_trigger GROUP BY service_name\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "kind!", + "type_info": "Text" + }, + { + "ordinal": 1, + "name": "count!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + null, + null + ] + }, + "hash": "1d346a14ad5586af347b8e7ac413500a39efa20e4915ffa56fd40537597db36e" +} diff --git a/backend/.sqlx/query-2132a715995f2775917c59c01ba66f2e472a2347f25cc7109f36a805806ee6e2.json b/backend/.sqlx/query-2132a715995f2775917c59c01ba66f2e472a2347f25cc7109f36a805806ee6e2.json deleted file mode 100644 index 673b8f4574..0000000000 --- a/backend/.sqlx/query-2132a715995f2775917c59c01ba66f2e472a2347f25cc7109f36a805806ee6e2.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT username, added_via\n FROM usr\n WHERE workspace_id = $1 AND email = $2\n AND added_via->>'source' = 'instance_group'\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "username", - "type_info": "Varchar" - }, - { - "ordinal": 1, - "name": "added_via", - "type_info": "Jsonb" - } - ], - "parameters": { - "Left": [ - "Text", - "Text" - ] - }, - "nullable": [ - false, - true - ] - }, - "hash": "2132a715995f2775917c59c01ba66f2e472a2347f25cc7109f36a805806ee6e2" -} diff --git a/backend/.sqlx/query-9530b234b2ac8f7360552be5e6bb25270e6b4185ce5c7f3c6edd9fbaabf77544.json b/backend/.sqlx/query-236028886d13526daa184f9e6d0a4b2ae43fbaf7f0bbbc8246c5e2d73b6f6aee.json similarity index 50% rename from backend/.sqlx/query-9530b234b2ac8f7360552be5e6bb25270e6b4185ce5c7f3c6edd9fbaabf77544.json rename to backend/.sqlx/query-236028886d13526daa184f9e6d0a4b2ae43fbaf7f0bbbc8246c5e2d73b6f6aee.json index 25a898e2f1..6a43b44e46 100644 --- a/backend/.sqlx/query-9530b234b2ac8f7360552be5e6bb25270e6b4185ce5c7f3c6edd9fbaabf77544.json +++ b/backend/.sqlx/query-236028886d13526daa184f9e6d0a4b2ae43fbaf7f0bbbc8246c5e2d73b6f6aee.json @@ -1,11 +1,11 @@ { "db_name": "PostgreSQL", - "query": "SELECT DISTINCT email FROM usr WHERE added_via->>'source' = 'instance_group' AND added_via->>'group' = $1", + "query": "SELECT instance_role FROM instance_group WHERE name = $1 FOR UPDATE", "describe": { "columns": [ { "ordinal": 0, - "name": "email", + "name": "instance_role", "type_info": "Varchar" } ], @@ -15,8 +15,8 @@ ] }, "nullable": [ - false + true ] }, - "hash": "9530b234b2ac8f7360552be5e6bb25270e6b4185ce5c7f3c6edd9fbaabf77544" + "hash": "236028886d13526daa184f9e6d0a4b2ae43fbaf7f0bbbc8246c5e2d73b6f6aee" } diff --git a/backend/.sqlx/query-255d37bb63595ebfcc61582d0b5e265b861b8b4d650435533e90eeb1d5ee3a68.json b/backend/.sqlx/query-255d37bb63595ebfcc61582d0b5e265b861b8b4d650435533e90eeb1d5ee3a68.json new file mode 100644 index 0000000000..64b6d70542 --- /dev/null +++ b/backend/.sqlx/query-255d37bb63595ebfcc61582d0b5e265b861b8b4d650435533e90eeb1d5ee3a68.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT COUNT(*)::BIGINT AS \"count!\" FROM flow\n WHERE archived = false AND pg_column_size(value) >= $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Int4" + ] + }, + "nullable": [ + null + ] + }, + "hash": "255d37bb63595ebfcc61582d0b5e265b861b8b4d650435533e90eeb1d5ee3a68" +} diff --git a/backend/.sqlx/query-2d5272e17f5c96185c7f655a6580f1e849dca05d1418798f5ea11dd91c9477bc.json b/backend/.sqlx/query-2d5272e17f5c96185c7f655a6580f1e849dca05d1418798f5ea11dd91c9477bc.json new file mode 100644 index 0000000000..d3799b5475 --- /dev/null +++ b/backend/.sqlx/query-2d5272e17f5c96185c7f655a6580f1e849dca05d1418798f5ea11dd91c9477bc.json @@ -0,0 +1,36 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT resource.path,\n COALESCE(left(pretty.value, $3), '') as \"value!\",\n COALESCE(length(pretty.value) > $3, false) as \"truncated!\"\n FROM resource, LATERAL (SELECT jsonb_pretty(resource.value) as value OFFSET 0) pretty\n WHERE workspace_id = $1 LIMIT $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "value!", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "truncated!", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text", + "Int8", + "Int4" + ] + }, + "nullable": [ + false, + null, + null + ] + }, + "hash": "2d5272e17f5c96185c7f655a6580f1e849dca05d1418798f5ea11dd91c9477bc" +} diff --git a/backend/.sqlx/query-c7c0b7f760f9616ec4a18a8916226b65990f5055c1bc25eedefd303be75f553f.json b/backend/.sqlx/query-374863f06094e404523e700df832243894e40a081fd2e7f95b466612aed054de.json similarity index 65% rename from backend/.sqlx/query-c7c0b7f760f9616ec4a18a8916226b65990f5055c1bc25eedefd303be75f553f.json rename to backend/.sqlx/query-374863f06094e404523e700df832243894e40a081fd2e7f95b466612aed054de.json index 3b175bb9b8..f8b19be3c5 100644 --- a/backend/.sqlx/query-c7c0b7f760f9616ec4a18a8916226b65990f5055c1bc25eedefd303be75f553f.json +++ b/backend/.sqlx/query-374863f06094e404523e700df832243894e40a081fd2e7f95b466612aed054de.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT workspace_id FROM workspace_settings WHERE auto_invite->'instance_groups' IS NOT NULL AND auto_invite->'instance_groups' ? $1", + "query": "SELECT workspace_id FROM workspace_settings WHERE auto_invite->'instance_groups' ?| $1", "describe": { "columns": [ { @@ -11,12 +11,12 @@ ], "parameters": { "Left": [ - "Text" + "TextArray" ] }, "nullable": [ false ] }, - "hash": "c7c0b7f760f9616ec4a18a8916226b65990f5055c1bc25eedefd303be75f553f" + "hash": "374863f06094e404523e700df832243894e40a081fd2e7f95b466612aed054de" } diff --git a/backend/.sqlx/query-3b0eb0571f287eb64c84cf2c8303401af7b78d0fb6cbaa10ccb90769320c8c7a.json b/backend/.sqlx/query-3b0eb0571f287eb64c84cf2c8303401af7b78d0fb6cbaa10ccb90769320c8c7a.json new file mode 100644 index 0000000000..ca79568bb0 --- /dev/null +++ b/backend/.sqlx/query-3b0eb0571f287eb64c84cf2c8303401af7b78d0fb6cbaa10ccb90769320c8c7a.json @@ -0,0 +1,26 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n COUNT(*) FILTER (WHERE slack_command_script IS NOT NULL)::BIGINT AS \"slack!\",\n COUNT(*) FILTER (WHERE teams_command_script IS NOT NULL)::BIGINT AS \"teams!\"\n FROM workspace_settings", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "slack!", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "teams!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + null, + null + ] + }, + "hash": "3b0eb0571f287eb64c84cf2c8303401af7b78d0fb6cbaa10ccb90769320c8c7a" +} diff --git a/backend/.sqlx/query-3bd4f38a1629a69ddda622b6b436198b47c2fe1a507358d49f05031e7beedab6.json b/backend/.sqlx/query-3bd4f38a1629a69ddda622b6b436198b47c2fe1a507358d49f05031e7beedab6.json deleted file mode 100644 index 0c71ee8ad2..0000000000 --- a/backend/.sqlx/query-3bd4f38a1629a69ddda622b6b436198b47c2fe1a507358d49f05031e7beedab6.json +++ /dev/null @@ -1,34 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT workspace_id,\n auto_invite->'instance_groups_roles' as instance_groups_roles,\n auto_invite->'instance_groups' as instance_groups_json\n FROM workspace_settings\n WHERE auto_invite->'instance_groups' ? $1\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "workspace_id", - "type_info": "Varchar" - }, - { - "ordinal": 1, - "name": "instance_groups_roles", - "type_info": "Jsonb" - }, - { - "ordinal": 2, - "name": "instance_groups_json", - "type_info": "Jsonb" - } - ], - "parameters": { - "Left": [ - "Text" - ] - }, - "nullable": [ - false, - null, - null - ] - }, - "hash": "3bd4f38a1629a69ddda622b6b436198b47c2fe1a507358d49f05031e7beedab6" -} diff --git a/backend/.sqlx/query-5d160ba4958583f1ad42de846c544d8d8e81e1b54925a0c5f2cedc1817d99a1b.json b/backend/.sqlx/query-4348832b4b99021b19a752a0375f7728fb10035ff4a71b8e0242e1184d192a32.json similarity index 58% rename from backend/.sqlx/query-5d160ba4958583f1ad42de846c544d8d8e81e1b54925a0c5f2cedc1817d99a1b.json rename to backend/.sqlx/query-4348832b4b99021b19a752a0375f7728fb10035ff4a71b8e0242e1184d192a32.json index 8fcffe364a..b6342d9b36 100644 --- a/backend/.sqlx/query-5d160ba4958583f1ad42de846c544d8d8e81e1b54925a0c5f2cedc1817d99a1b.json +++ b/backend/.sqlx/query-4348832b4b99021b19a752a0375f7728fb10035ff4a71b8e0242e1184d192a32.json @@ -1,11 +1,11 @@ { "db_name": "PostgreSQL", - "query": "SELECT igroup FROM email_to_igroup WHERE email = $1", + "query": "SELECT name FROM instance_group WHERE name = $1 FOR UPDATE", "describe": { "columns": [ { "ordinal": 0, - "name": "igroup", + "name": "name", "type_info": "Varchar" } ], @@ -18,5 +18,5 @@ false ] }, - "hash": "5d160ba4958583f1ad42de846c544d8d8e81e1b54925a0c5f2cedc1817d99a1b" + "hash": "4348832b4b99021b19a752a0375f7728fb10035ff4a71b8e0242e1184d192a32" } diff --git a/backend/.sqlx/query-43a689277803e5e2204e10263a5749675652c23a231fce65257b053b6faad231.json b/backend/.sqlx/query-43a689277803e5e2204e10263a5749675652c23a231fce65257b053b6faad231.json new file mode 100644 index 0000000000..86af8f2b7b --- /dev/null +++ b/backend/.sqlx/query-43a689277803e5e2204e10263a5749675652c23a231fce65257b053b6faad231.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT name FROM instance_group WHERE name = ANY($1) ORDER BY name FOR UPDATE", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "name", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "TextArray" + ] + }, + "nullable": [ + false + ] + }, + "hash": "43a689277803e5e2204e10263a5749675652c23a231fce65257b053b6faad231" +} diff --git a/backend/.sqlx/query-472f224dc9e17d2c50cb9db13c34cc1fb9adb6d0ea36cf223541adb7cac17bdd.json b/backend/.sqlx/query-472f224dc9e17d2c50cb9db13c34cc1fb9adb6d0ea36cf223541adb7cac17bdd.json deleted file mode 100644 index b60ecae184..0000000000 --- a/backend/.sqlx/query-472f224dc9e17d2c50cb9db13c34cc1fb9adb6d0ea36cf223541adb7cac17bdd.json +++ /dev/null @@ -1,35 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT workspace_id, username, email\n FROM usr\n WHERE email = $1\n AND added_via->>'source' = 'instance_group'\n AND added_via->>'group' = $2\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "workspace_id", - "type_info": "Varchar" - }, - { - "ordinal": 1, - "name": "username", - "type_info": "Varchar" - }, - { - "ordinal": 2, - "name": "email", - "type_info": "Varchar" - } - ], - "parameters": { - "Left": [ - "Text", - "Text" - ] - }, - "nullable": [ - false, - false, - false - ] - }, - "hash": "472f224dc9e17d2c50cb9db13c34cc1fb9adb6d0ea36cf223541adb7cac17bdd" -} diff --git a/backend/.sqlx/query-49bf26ae4b7e3421507f9e7e42c59ad7e0f481a9550e0f70e1145d5e541bd6e5.json b/backend/.sqlx/query-49bf26ae4b7e3421507f9e7e42c59ad7e0f481a9550e0f70e1145d5e541bd6e5.json new file mode 100644 index 0000000000..005ac90b79 --- /dev/null +++ b/backend/.sqlx/query-49bf26ae4b7e3421507f9e7e42c59ad7e0f481a9550e0f70e1145d5e541bd6e5.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT auto_invite->'instance_groups' as \"groups: serde_json::Value\",\n auto_invite->'instance_groups_roles' as \"roles: serde_json::Value\"\n FROM workspace_settings WHERE workspace_id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "groups: serde_json::Value", + "type_info": "Jsonb" + }, + { + "ordinal": 1, + "name": "roles: serde_json::Value", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null, + null + ] + }, + "hash": "49bf26ae4b7e3421507f9e7e42c59ad7e0f481a9550e0f70e1145d5e541bd6e5" +} diff --git a/backend/.sqlx/query-5344f222417c28efd4f724cbd83382fc69a223dfbb91ab40df895ab60d0f6228.json b/backend/.sqlx/query-5344f222417c28efd4f724cbd83382fc69a223dfbb91ab40df895ab60d0f6228.json new file mode 100644 index 0000000000..08c08bbf75 --- /dev/null +++ b/backend/.sqlx/query-5344f222417c28efd4f724cbd83382fc69a223dfbb91ab40df895ab60d0f6228.json @@ -0,0 +1,26 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT trigger_kind::text AS \"kind!\", COUNT(*)::BIGINT AS \"count!\"\n FROM capture WHERE created_at > now() - interval '30 days' GROUP BY trigger_kind", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "kind!", + "type_info": "Text" + }, + { + "ordinal": 1, + "name": "count!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + null, + null + ] + }, + "hash": "5344f222417c28efd4f724cbd83382fc69a223dfbb91ab40df895ab60d0f6228" +} diff --git a/backend/.sqlx/query-66e2f8468ba64f22b7a7caa18639d7c833ac2ec573bd89d878b5c8b1afc74d3a.json b/backend/.sqlx/query-66e2f8468ba64f22b7a7caa18639d7c833ac2ec573bd89d878b5c8b1afc74d3a.json deleted file mode 100644 index 15681aab51..0000000000 --- a/backend/.sqlx/query-66e2f8468ba64f22b7a7caa18639d7c833ac2ec573bd89d878b5c8b1afc74d3a.json +++ /dev/null @@ -1,34 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT\n workspace_id,\n auto_invite->'instance_groups_roles' as instance_groups_roles,\n auto_invite->'instance_groups' as instance_groups_json\n FROM workspace_settings\n WHERE\n auto_invite->'instance_groups' IS NOT NULL\n AND auto_invite->'instance_groups' ? $1\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "workspace_id", - "type_info": "Varchar" - }, - { - "ordinal": 1, - "name": "instance_groups_roles", - "type_info": "Jsonb" - }, - { - "ordinal": 2, - "name": "instance_groups_json", - "type_info": "Jsonb" - } - ], - "parameters": { - "Left": [ - "Text" - ] - }, - "nullable": [ - false, - null, - null - ] - }, - "hash": "66e2f8468ba64f22b7a7caa18639d7c833ac2ec573bd89d878b5c8b1afc74d3a" -} diff --git a/backend/.sqlx/query-6c6b4bd4bd19878fce25d3a8a5ee02686b358b616c318e4bfca084d96b38f1c4.json b/backend/.sqlx/query-6c6b4bd4bd19878fce25d3a8a5ee02686b358b616c318e4bfca084d96b38f1c4.json new file mode 100644 index 0000000000..6576d06ffa --- /dev/null +++ b/backend/.sqlx/query-6c6b4bd4bd19878fce25d3a8a5ee02686b358b616c318e4bfca084d96b38f1c4.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT config AS \"config!\" FROM config WHERE name LIKE 'worker__%' AND config IS NOT NULL", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "config!", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + true + ] + }, + "hash": "6c6b4bd4bd19878fce25d3a8a5ee02686b358b616c318e4bfca084d96b38f1c4" +} diff --git a/backend/.sqlx/query-6fda4517a72b25b0eab47bc69127ee45467d4d45239fb556534429b03442b27b.json b/backend/.sqlx/query-6fda4517a72b25b0eab47bc69127ee45467d4d45239fb556534429b03442b27b.json new file mode 100644 index 0000000000..17641bc8f7 --- /dev/null +++ b/backend/.sqlx/query-6fda4517a72b25b0eab47bc69127ee45467d4d45239fb556534429b03442b27b.json @@ -0,0 +1,46 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT username, email, is_admin, operator,\n added_via->>'group' as granting_group\n FROM usr\n WHERE workspace_id = $1 AND added_via->>'source' = 'instance_group'\n ORDER BY email", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "username", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "email", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "is_admin", + "type_info": "Bool" + }, + { + "ordinal": 3, + "name": "operator", + "type_info": "Bool" + }, + { + "ordinal": 4, + "name": "granting_group", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + false, + false, + false, + null + ] + }, + "hash": "6fda4517a72b25b0eab47bc69127ee45467d4d45239fb556534429b03442b27b" +} diff --git a/backend/.sqlx/query-772dc28e57666282d8993268843d3a87e8899b968827251745f998e4dc25863a.json b/backend/.sqlx/query-772dc28e57666282d8993268843d3a87e8899b968827251745f998e4dc25863a.json new file mode 100644 index 0000000000..d5659d725b --- /dev/null +++ b/backend/.sqlx/query-772dc28e57666282d8993268843d3a87e8899b968827251745f998e4dc25863a.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO feature_usage (feature, kind, key, value)\n SELECT * FROM UNNEST($1::text[], $2::text[], $3::text[], $4::bigint[])\n ON CONFLICT (feature, kind, key, entity_id, day)\n DO UPDATE SET value = feature_usage.value + EXCLUDED.value, updated_at = now()", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "TextArray", + "TextArray", + "TextArray", + "Int8Array" + ] + }, + "nullable": [] + }, + "hash": "772dc28e57666282d8993268843d3a87e8899b968827251745f998e4dc25863a" +} diff --git a/backend/.sqlx/query-2ee6d24b95cdda151585dcff19f8e7c931785fc21f7bbe9c3a82671943ced0ea.json b/backend/.sqlx/query-7947ffe31b8e6f4a38fba9d9caf434fd409ea2806a7c720c2b1d76b2e7db6c32.json similarity index 65% rename from backend/.sqlx/query-2ee6d24b95cdda151585dcff19f8e7c931785fc21f7bbe9c3a82671943ced0ea.json rename to backend/.sqlx/query-7947ffe31b8e6f4a38fba9d9caf434fd409ea2806a7c720c2b1d76b2e7db6c32.json index 6acb31666c..e69a61e65d 100644 --- a/backend/.sqlx/query-2ee6d24b95cdda151585dcff19f8e7c931785fc21f7bbe9c3a82671943ced0ea.json +++ b/backend/.sqlx/query-7947ffe31b8e6f4a38fba9d9caf434fd409ea2806a7c720c2b1d76b2e7db6c32.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "UPDATE schedule SET enabled = false, error = $1 WHERE workspace_id = $2 AND path = $3", + "query": "UPDATE schedule SET enabled = false, error = $1 WHERE workspace_id = $2 AND path = $3 AND enabled = true", "describe": { "columns": [], "parameters": { @@ -12,5 +12,5 @@ }, "nullable": [] }, - "hash": "2ee6d24b95cdda151585dcff19f8e7c931785fc21f7bbe9c3a82671943ced0ea" + "hash": "7947ffe31b8e6f4a38fba9d9caf434fd409ea2806a7c720c2b1d76b2e7db6c32" } diff --git a/backend/.sqlx/query-7b3eadb62ddd07e5e12eb8b5150ddce33008b61bbdab4006194ca7ef5b754802.json b/backend/.sqlx/query-7b3eadb62ddd07e5e12eb8b5150ddce33008b61bbdab4006194ca7ef5b754802.json new file mode 100644 index 0000000000..6ca02417f3 --- /dev/null +++ b/backend/.sqlx/query-7b3eadb62ddd07e5e12eb8b5150ddce33008b61bbdab4006194ca7ef5b754802.json @@ -0,0 +1,26 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n (SELECT COUNT(*) FROM script\n WHERE dedicated_worker = true AND archived = false AND deleted = false)::BIGINT AS \"scripts!\",\n (SELECT COUNT(*) FROM flow WHERE dedicated_worker = true AND archived = false)::BIGINT AS \"flows!\"", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "scripts!", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "flows!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + null, + null + ] + }, + "hash": "7b3eadb62ddd07e5e12eb8b5150ddce33008b61bbdab4006194ca7ef5b754802" +} diff --git a/backend/.sqlx/query-f582cac90b4b7d732956b74eebc51323ef8acd3f627e7517451fcc72998d22bc.json b/backend/.sqlx/query-815c5e8fd91dc119a803f4f1f56b8016bdaabfa2ff61a1abe8f73f5a7336ab61.json similarity index 64% rename from backend/.sqlx/query-f582cac90b4b7d732956b74eebc51323ef8acd3f627e7517451fcc72998d22bc.json rename to backend/.sqlx/query-815c5e8fd91dc119a803f4f1f56b8016bdaabfa2ff61a1abe8f73f5a7336ab61.json index 5bfc4710c8..fc25b16733 100644 --- a/backend/.sqlx/query-f582cac90b4b7d732956b74eebc51323ef8acd3f627e7517451fcc72998d22bc.json +++ b/backend/.sqlx/query-815c5e8fd91dc119a803f4f1f56b8016bdaabfa2ff61a1abe8f73f5a7336ab61.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "UPDATE usr SET added_via = $1 WHERE workspace_id = $2 AND email = $3", + "query": "UPDATE usr SET added_via = $1 WHERE workspace_id = $2 AND email = $3 AND added_via->>'source' = 'instance_group'", "describe": { "columns": [], "parameters": { @@ -12,5 +12,5 @@ }, "nullable": [] }, - "hash": "f582cac90b4b7d732956b74eebc51323ef8acd3f627e7517451fcc72998d22bc" + "hash": "815c5e8fd91dc119a803f4f1f56b8016bdaabfa2ff61a1abe8f73f5a7336ab61" } diff --git a/backend/.sqlx/query-83ec97f6aad154e0e06ee05a3647dab8f89b1b2d7a569c7eac8c4169e37b9f8b.json b/backend/.sqlx/query-83ec97f6aad154e0e06ee05a3647dab8f89b1b2d7a569c7eac8c4169e37b9f8b.json deleted file mode 100644 index 4e0bc23b69..0000000000 --- a/backend/.sqlx/query-83ec97f6aad154e0e06ee05a3647dab8f89b1b2d7a569c7eac8c4169e37b9f8b.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT value FROM global_settings WHERE name = 'smtp_settings'", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "value", - "type_info": "Jsonb" - } - ], - "parameters": { - "Left": [] - }, - "nullable": [ - false - ] - }, - "hash": "83ec97f6aad154e0e06ee05a3647dab8f89b1b2d7a569c7eac8c4169e37b9f8b" -} diff --git a/backend/.sqlx/query-84fcddaf5bc61d607a6e6e5e31de7436b203a3baa7ef0509cb8e6c52270ae3a9.json b/backend/.sqlx/query-84fcddaf5bc61d607a6e6e5e31de7436b203a3baa7ef0509cb8e6c52270ae3a9.json new file mode 100644 index 0000000000..557466e8b7 --- /dev/null +++ b/backend/.sqlx/query-84fcddaf5bc61d607a6e6e5e31de7436b203a3baa7ef0509cb8e6c52270ae3a9.json @@ -0,0 +1,35 @@ +{ + "db_name": "PostgreSQL", + "query": "WITH RECURSIVE chain(id, parent_job) AS (\n SELECT id, parent_job FROM v2_job WHERE id = $1 AND workspace_id = $2\n UNION ALL\n SELECT j.id, j.parent_job FROM v2_job j\n JOIN chain c ON j.id = c.parent_job AND j.workspace_id = $2\n )\n SELECT j.runnable_path,\n CASE\n WHEN j.kind IN ('script', 'script_hub', 'unassigned_script') THEN 'scripts'\n WHEN j.kind IN ('flow', 'unassigned_flow') THEN 'flows'\n WHEN j.kind IN ('singlestepflow', 'unassigned_singlestepflow') THEN\n CASE WHEN COALESCE(\n (SELECT m->'value'->>'type'\n FROM jsonb_array_elements(j.raw_flow->'modules') m\n WHERE m->>'id' IN ('a', 'main')\n LIMIT 1),\n 'script'\n ) = 'flow' THEN 'flows' ELSE 'scripts' END\n END AS scope_kind,\n CASE WHEN j.trigger_kind = 'app' THEN j.trigger END AS launched_by_app\n FROM v2_job j JOIN chain c ON c.id = j.id\n WHERE j.workspace_id = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "runnable_path", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "scope_kind", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "launched_by_app", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text" + ] + }, + "nullable": [ + true, + null, + null + ] + }, + "hash": "84fcddaf5bc61d607a6e6e5e31de7436b203a3baa7ef0509cb8e6c52270ae3a9" +} diff --git a/backend/.sqlx/query-75e740531bf794a8568350348253612cfbdecb9fb9cf768f431d4dbd4cd56cfb.json b/backend/.sqlx/query-87873ad46b94e26f840d1710146e90c639beb2a5389432e64ecd94adbf154516.json similarity index 51% rename from backend/.sqlx/query-75e740531bf794a8568350348253612cfbdecb9fb9cf768f431d4dbd4cd56cfb.json rename to backend/.sqlx/query-87873ad46b94e26f840d1710146e90c639beb2a5389432e64ecd94adbf154516.json index 3445985466..c2e40c655e 100644 --- a/backend/.sqlx/query-75e740531bf794a8568350348253612cfbdecb9fb9cf768f431d4dbd4cd56cfb.json +++ b/backend/.sqlx/query-87873ad46b94e26f840d1710146e90c639beb2a5389432e64ecd94adbf154516.json @@ -1,22 +1,23 @@ { "db_name": "PostgreSQL", - "query": "SELECT auto_invite->'instance_groups' FROM workspace_settings WHERE workspace_id = $1", + "query": "SELECT policy FROM app WHERE path = $1 AND workspace_id = $2 FOR UPDATE", "describe": { "columns": [ { "ordinal": 0, - "name": "?column?", + "name": "policy", "type_info": "Jsonb" } ], "parameters": { "Left": [ + "Text", "Text" ] }, "nullable": [ - null + false ] }, - "hash": "75e740531bf794a8568350348253612cfbdecb9fb9cf768f431d4dbd4cd56cfb" + "hash": "87873ad46b94e26f840d1710146e90c639beb2a5389432e64ecd94adbf154516" } diff --git a/backend/.sqlx/query-928767710fb8b7dc0b1edc897d8ce9b6b59ae2f63e684d3db6eecbcffd767711.json b/backend/.sqlx/query-928767710fb8b7dc0b1edc897d8ce9b6b59ae2f63e684d3db6eecbcffd767711.json new file mode 100644 index 0000000000..36e067a79a --- /dev/null +++ b/backend/.sqlx/query-928767710fb8b7dc0b1edc897d8ce9b6b59ae2f63e684d3db6eecbcffd767711.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO trigger_history\n (workspace_id, trigger_kind, path, operation, source, username, changes)\n SELECT $1, $2, p, $3, $4, $5, $6 FROM unnest($7::text[]) p", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Varchar", + "Varchar", + "Varchar", + "Jsonb", + "TextArray" + ] + }, + "nullable": [] + }, + "hash": "928767710fb8b7dc0b1edc897d8ce9b6b59ae2f63e684d3db6eecbcffd767711" +} diff --git a/backend/.sqlx/query-9d9fbcb598c582a29d65be87a1e35baa410f93be3cac4b8dfd33ddbef446d3fe.json b/backend/.sqlx/query-9d9fbcb598c582a29d65be87a1e35baa410f93be3cac4b8dfd33ddbef446d3fe.json deleted file mode 100644 index 2c95e06391..0000000000 --- a/backend/.sqlx/query-9d9fbcb598c582a29d65be87a1e35baa410f93be3cac4b8dfd33ddbef446d3fe.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO worker_ping (worker_instance, worker, ip, custom_tags, worker_group, dedicated_worker, dedicated_workers, wm_version, vcpus, memory, job_isolation, native_mode) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) ON CONFLICT (worker)\n DO UPDATE set ping_at = now(), worker_instance = EXCLUDED.worker_instance, ip = EXCLUDED.ip, custom_tags = EXCLUDED.custom_tags, worker_group = EXCLUDED.worker_group, dedicated_worker = EXCLUDED.dedicated_worker, dedicated_workers = EXCLUDED.dedicated_workers, wm_version = EXCLUDED.wm_version, vcpus = COALESCE(EXCLUDED.vcpus, worker_ping.vcpus), memory = COALESCE(EXCLUDED.memory, worker_ping.memory), job_isolation = EXCLUDED.job_isolation, native_mode = EXCLUDED.native_mode, current_job_id = NULL, current_job_workspace_id = NULL\n RETURNING jobs_executed", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "jobs_executed", - "type_info": "Int4" - } - ], - "parameters": { - "Left": [ - "Varchar", - "Varchar", - "Varchar", - "TextArray", - "Varchar", - "Varchar", - "TextArray", - "Varchar", - "Int8", - "Int8", - "Text", - "Bool" - ] - }, - "nullable": [ - false - ] - }, - "hash": "9d9fbcb598c582a29d65be87a1e35baa410f93be3cac4b8dfd33ddbef446d3fe" -} diff --git a/backend/.sqlx/query-a2f047f9ca4b8a47c985fa092ba0d2dc54f7169af1c940b84345c477865ae82c.json b/backend/.sqlx/query-a2f047f9ca4b8a47c985fa092ba0d2dc54f7169af1c940b84345c477865ae82c.json new file mode 100644 index 0000000000..800d6c3305 --- /dev/null +++ b/backend/.sqlx/query-a2f047f9ca4b8a47c985fa092ba0d2dc54f7169af1c940b84345c477865ae82c.json @@ -0,0 +1,26 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT trigger_kind::text AS \"kind!\", COUNT(*)::BIGINT AS \"count!\"\n FROM capture_config GROUP BY trigger_kind", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "kind!", + "type_info": "Text" + }, + { + "ordinal": 1, + "name": "count!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + null, + null + ] + }, + "hash": "a2f047f9ca4b8a47c985fa092ba0d2dc54f7169af1c940b84345c477865ae82c" +} diff --git a/backend/.sqlx/query-ae1973cf7dda23c1583521c7edd1a7e3beb695edaa00adf875cfdbb81ebe96dc.json b/backend/.sqlx/query-ae1973cf7dda23c1583521c7edd1a7e3beb695edaa00adf875cfdbb81ebe96dc.json new file mode 100644 index 0000000000..e1a5206335 --- /dev/null +++ b/backend/.sqlx/query-ae1973cf7dda23c1583521c7edd1a7e3beb695edaa00adf875cfdbb81ebe96dc.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE workspace_settings SET\n auto_invite = jsonb_set(\n jsonb_set(\n COALESCE(auto_invite, '{}'::jsonb),\n '{instance_groups}',\n (SELECT COALESCE(jsonb_agg(\n CASE WHEN elem #>> '{}' = $1 THEN to_jsonb($2::text) ELSE elem END), '[]'::jsonb)\n FROM jsonb_array_elements(COALESCE(auto_invite->'instance_groups', '[]'::jsonb)) elem)\n ),\n '{instance_groups_roles}',\n CASE WHEN COALESCE(auto_invite->'instance_groups_roles', '{}'::jsonb) ? $1\n THEN (COALESCE(auto_invite->'instance_groups_roles', '{}'::jsonb) - $1)\n || jsonb_build_object($2::text, auto_invite->'instance_groups_roles'->$1)\n ELSE COALESCE(auto_invite->'instance_groups_roles', '{}'::jsonb)\n END\n )\n WHERE auto_invite->'instance_groups' ? $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "ae1973cf7dda23c1583521c7edd1a7e3beb695edaa00adf875cfdbb81ebe96dc" +} diff --git a/backend/.sqlx/query-bbc2638aae4fb3556c8d876e4efd402c7b7c93ff9fd89364afd41470a92a5e2d.json b/backend/.sqlx/query-bbc2638aae4fb3556c8d876e4efd402c7b7c93ff9fd89364afd41470a92a5e2d.json new file mode 100644 index 0000000000..e59183f51a --- /dev/null +++ b/backend/.sqlx/query-bbc2638aae4fb3556c8d876e4efd402c7b7c93ff9fd89364afd41470a92a5e2d.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT name FROM instance_group WHERE name <> ALL($1)", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "name", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "TextArray" + ] + }, + "nullable": [ + false + ] + }, + "hash": "bbc2638aae4fb3556c8d876e4efd402c7b7c93ff9fd89364afd41470a92a5e2d" +} diff --git a/backend/.sqlx/query-c748617e060bc41b5922df4b433f20b6971b993e0671b6ccb45db5ef028550bc.json b/backend/.sqlx/query-c748617e060bc41b5922df4b433f20b6971b993e0671b6ccb45db5ef028550bc.json new file mode 100644 index 0000000000..7f6fe7a81c --- /dev/null +++ b/backend/.sqlx/query-c748617e060bc41b5922df4b433f20b6971b993e0671b6ccb45db5ef028550bc.json @@ -0,0 +1,33 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO worker_ping (worker_instance, worker, ip, custom_tags, worker_group, dedicated_worker, dedicated_workers, wm_version, vcpus, memory, job_isolation, native_mode) VALUES ($1, $2, COALESCE($3, 'NO IP'), $4, $5, $6, $7, $8, $9, $10, $11, $12) ON CONFLICT (worker)\n DO UPDATE set ping_at = now(), worker_instance = EXCLUDED.worker_instance, ip = COALESCE($3, worker_ping.ip), custom_tags = EXCLUDED.custom_tags, worker_group = EXCLUDED.worker_group, dedicated_worker = EXCLUDED.dedicated_worker, dedicated_workers = EXCLUDED.dedicated_workers, wm_version = EXCLUDED.wm_version, vcpus = COALESCE(EXCLUDED.vcpus, worker_ping.vcpus), memory = COALESCE(EXCLUDED.memory, worker_ping.memory), job_isolation = EXCLUDED.job_isolation, native_mode = EXCLUDED.native_mode, current_job_id = NULL, current_job_workspace_id = NULL\n RETURNING jobs_executed", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "jobs_executed", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Text", + "TextArray", + "Varchar", + "Varchar", + "TextArray", + "Varchar", + "Int8", + "Int8", + "Text", + "Bool" + ] + }, + "nullable": [ + false + ] + }, + "hash": "c748617e060bc41b5922df4b433f20b6971b993e0671b6ccb45db5ef028550bc" +} diff --git a/backend/.sqlx/query-d069ad741996e3ea992bcfb33e290ba5b87be1c047c189c9b8eb13da97e2085d.json b/backend/.sqlx/query-d069ad741996e3ea992bcfb33e290ba5b87be1c047c189c9b8eb13da97e2085d.json new file mode 100644 index 0000000000..7e667694b9 --- /dev/null +++ b/backend/.sqlx/query-d069ad741996e3ea992bcfb33e290ba5b87be1c047c189c9b8eb13da97e2085d.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT pg_advisory_xact_lock(hashtext('reconcile_workspace_instance_groups'), hashtext($1))", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "pg_advisory_xact_lock", + "type_info": "Void" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "d069ad741996e3ea992bcfb33e290ba5b87be1c047c189c9b8eb13da97e2085d" +} diff --git a/backend/.sqlx/query-d1f701e81fc98933802356f292455da604367ef35893c2bb095bcd637567b114.json b/backend/.sqlx/query-d1f701e81fc98933802356f292455da604367ef35893c2bb095bcd637567b114.json new file mode 100644 index 0000000000..064e7c38aa --- /dev/null +++ b/backend/.sqlx/query-d1f701e81fc98933802356f292455da604367ef35893c2bb095bcd637567b114.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT email, igroup FROM email_to_igroup WHERE igroup = ANY($1)", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "email", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "igroup", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "TextArray" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "d1f701e81fc98933802356f292455da604367ef35893c2bb095bcd637567b114" +} diff --git a/backend/.sqlx/query-dab323eda1fcaff0435e98d77e544ed5d63dd6023b1dab77d5f188b299f03b9d.json b/backend/.sqlx/query-dab323eda1fcaff0435e98d77e544ed5d63dd6023b1dab77d5f188b299f03b9d.json deleted file mode 100644 index ee4c5ff1af..0000000000 --- a/backend/.sqlx/query-dab323eda1fcaff0435e98d77e544ed5d63dd6023b1dab77d5f188b299f03b9d.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE workspace_settings SET\n auto_invite = jsonb_set(\n jsonb_set(\n COALESCE(auto_invite, '{}'::jsonb),\n '{instance_groups}',\n (SELECT COALESCE(jsonb_agg(elem), '[]'::jsonb) FROM jsonb_array_elements(COALESCE(auto_invite->'instance_groups', '[]'::jsonb)) elem WHERE elem #>> '{}' != $1)\n ),\n '{instance_groups_roles}',\n COALESCE(auto_invite->'instance_groups_roles', '{}'::jsonb) - $1\n )\n WHERE auto_invite->'instance_groups' IS NOT NULL AND auto_invite->'instance_groups' ? $1", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text" - ] - }, - "nullable": [] - }, - "hash": "dab323eda1fcaff0435e98d77e544ed5d63dd6023b1dab77d5f188b299f03b9d" -} diff --git a/backend/.sqlx/query-db9b48f91a2387e08a2eaa5bda344edf83bfcb8c8c330d524ed4eaafedbdbc0e.json b/backend/.sqlx/query-db9b48f91a2387e08a2eaa5bda344edf83bfcb8c8c330d524ed4eaafedbdbc0e.json new file mode 100644 index 0000000000..05c496b552 --- /dev/null +++ b/backend/.sqlx/query-db9b48f91a2387e08a2eaa5bda344edf83bfcb8c8c330d524ed4eaafedbdbc0e.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO global_settings (name, value) VALUES ($1, $2)\n ON CONFLICT (name) DO UPDATE SET value = EXCLUDED.value\n WHERE jsonb_typeof(global_settings.value) <> 'string'\n RETURNING value", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "value", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Varchar", + "Jsonb" + ] + }, + "nullable": [ + false + ] + }, + "hash": "db9b48f91a2387e08a2eaa5bda344edf83bfcb8c8c330d524ed4eaafedbdbc0e" +} diff --git a/backend/.sqlx/query-e24252d48a1fcca73f20d62f37c9d7dc2071580be6d604979c95b3374cd4ad77.json b/backend/.sqlx/query-e24252d48a1fcca73f20d62f37c9d7dc2071580be6d604979c95b3374cd4ad77.json new file mode 100644 index 0000000000..d52ba8fb85 --- /dev/null +++ b/backend/.sqlx/query-e24252d48a1fcca73f20d62f37c9d7dc2071580be6d604979c95b3374cd4ad77.json @@ -0,0 +1,173 @@ +{ + "db_name": "PostgreSQL", + "query": "\n WITH scanned AS (\n SELECT a.* FROM (\n SELECT value FROM flow\n WHERE archived = false AND pg_column_size(value) < $1\n LIMIT $2\n ) f,\n LATERAL (\n SELECT\n bool_or(m->'value'->>'type' = 'forloopflow') AS forloop,\n bool_or(m->'value'->>'type' = 'whileloopflow') AS whileloop,\n bool_or(m->'value'->>'type' = 'branchall') AS branchall,\n bool_or(m->'value'->>'type' = 'branchall'\n AND m->'value'->>'parallel' = 'false') AS branchall_seq,\n bool_or(m->'value'->>'type' = 'branchone') AS branchone,\n bool_or(m->'value'->>'type' = 'aiagent') AS aiagent,\n bool_or(m->'value'->>'type' = 'flow') AS subflow,\n bool_or(m->'value'->>'type' = 'identity') AS identity,\n bool_or(m->'value'->>'is_trigger' = 'true') AS trigger_step,\n bool_or(m->'value'->>'squash' = 'true') AS squash,\n bool_or(m->'value'->>'type' IN ('forloopflow', 'whileloopflow')\n AND m->'value'->>'parallel' = 'true') AS parallel_loop,\n bool_or(m->'value'->>'type' IN ('forloopflow', 'whileloopflow')\n AND m->'value'->>'skip_failures' = 'false') AS keep_failures,\n bool_or(m->'value' ? 'parallelism') AS parallelism,\n bool_or(m ? 'sleep') AS sleep,\n bool_or(m ? 'cache_ttl') AS cache,\n bool_or(m->'mock'->>'enabled' = 'true') AS mock,\n bool_or(m ? 'suspend') AS suspend,\n bool_or(m ? 'retry') AS retry,\n bool_or(m ? 'timeout') AS timeout,\n bool_or(m ? 'priority') AS priority,\n bool_or(m ? 'debouncing') AS debounce,\n bool_or(m ? 'delete_after_secs') AS lifetime,\n bool_or(m->>'continue_on_error' = 'true') AS continue_on_error,\n bool_or(m ? 'stop_after_if' OR m ? 'stop_after_all_iters_if') AS early_stop,\n bool_or(m ? 'skip_if') AS skip\n FROM jsonb_path_query(f.value, '$.**.modules[*]') m\n ) a\n )\n SELECT\n COUNT(*)::BIGINT AS \"flows_scanned!\",\n COUNT(*) FILTER (WHERE forloop)::BIGINT AS \"forloopflow!\",\n COUNT(*) FILTER (WHERE whileloop)::BIGINT AS \"whileloopflow!\",\n COUNT(*) FILTER (WHERE branchall)::BIGINT AS \"branchall!\",\n COUNT(*) FILTER (WHERE branchall_seq)::BIGINT AS \"branchall_sequential!\",\n COUNT(*) FILTER (WHERE branchone)::BIGINT AS \"branchone!\",\n COUNT(*) FILTER (WHERE aiagent)::BIGINT AS \"aiagent!\",\n COUNT(*) FILTER (WHERE subflow)::BIGINT AS \"subflow!\",\n COUNT(*) FILTER (WHERE identity)::BIGINT AS \"identity!\",\n COUNT(*) FILTER (WHERE trigger_step)::BIGINT AS \"trigger_step!\",\n COUNT(*) FILTER (WHERE squash)::BIGINT AS \"squash!\",\n COUNT(*) FILTER (WHERE parallel_loop)::BIGINT AS \"parallel_loop!\",\n COUNT(*) FILTER (WHERE keep_failures)::BIGINT AS \"keep_failures!\",\n COUNT(*) FILTER (WHERE parallelism)::BIGINT AS \"parallelism!\",\n COUNT(*) FILTER (WHERE sleep)::BIGINT AS \"sleep!\",\n COUNT(*) FILTER (WHERE cache)::BIGINT AS \"cache!\",\n COUNT(*) FILTER (WHERE mock)::BIGINT AS \"mock!\",\n COUNT(*) FILTER (WHERE suspend)::BIGINT AS \"suspend!\",\n COUNT(*) FILTER (WHERE retry)::BIGINT AS \"retry!\",\n COUNT(*) FILTER (WHERE timeout)::BIGINT AS \"timeout!\",\n COUNT(*) FILTER (WHERE priority)::BIGINT AS \"priority!\",\n COUNT(*) FILTER (WHERE debounce)::BIGINT AS \"debounce!\",\n COUNT(*) FILTER (WHERE lifetime)::BIGINT AS \"lifetime!\",\n COUNT(*) FILTER (WHERE continue_on_error)::BIGINT AS \"continue_on_error!\",\n COUNT(*) FILTER (WHERE early_stop)::BIGINT AS \"early_stop!\",\n COUNT(*) FILTER (WHERE skip)::BIGINT AS \"skip!\"\n FROM scanned\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "flows_scanned!", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "forloopflow!", + "type_info": "Int8" + }, + { + "ordinal": 2, + "name": "whileloopflow!", + "type_info": "Int8" + }, + { + "ordinal": 3, + "name": "branchall!", + "type_info": "Int8" + }, + { + "ordinal": 4, + "name": "branchall_sequential!", + "type_info": "Int8" + }, + { + "ordinal": 5, + "name": "branchone!", + "type_info": "Int8" + }, + { + "ordinal": 6, + "name": "aiagent!", + "type_info": "Int8" + }, + { + "ordinal": 7, + "name": "subflow!", + "type_info": "Int8" + }, + { + "ordinal": 8, + "name": "identity!", + "type_info": "Int8" + }, + { + "ordinal": 9, + "name": "trigger_step!", + "type_info": "Int8" + }, + { + "ordinal": 10, + "name": "squash!", + "type_info": "Int8" + }, + { + "ordinal": 11, + "name": "parallel_loop!", + "type_info": "Int8" + }, + { + "ordinal": 12, + "name": "keep_failures!", + "type_info": "Int8" + }, + { + "ordinal": 13, + "name": "parallelism!", + "type_info": "Int8" + }, + { + "ordinal": 14, + "name": "sleep!", + "type_info": "Int8" + }, + { + "ordinal": 15, + "name": "cache!", + "type_info": "Int8" + }, + { + "ordinal": 16, + "name": "mock!", + "type_info": "Int8" + }, + { + "ordinal": 17, + "name": "suspend!", + "type_info": "Int8" + }, + { + "ordinal": 18, + "name": "retry!", + "type_info": "Int8" + }, + { + "ordinal": 19, + "name": "timeout!", + "type_info": "Int8" + }, + { + "ordinal": 20, + "name": "priority!", + "type_info": "Int8" + }, + { + "ordinal": 21, + "name": "debounce!", + "type_info": "Int8" + }, + { + "ordinal": 22, + "name": "lifetime!", + "type_info": "Int8" + }, + { + "ordinal": 23, + "name": "continue_on_error!", + "type_info": "Int8" + }, + { + "ordinal": 24, + "name": "early_stop!", + "type_info": "Int8" + }, + { + "ordinal": 25, + "name": "skip!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Int4", + "Int8" + ] + }, + "nullable": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ] + }, + "hash": "e24252d48a1fcca73f20d62f37c9d7dc2071580be6d604979c95b3374cd4ad77" +} diff --git a/backend/.sqlx/query-e242c733ca0accb5287e80c97abe3bbae0638612bd111b6b6196dd2808d4b6d0.json b/backend/.sqlx/query-e242c733ca0accb5287e80c97abe3bbae0638612bd111b6b6196dd2808d4b6d0.json new file mode 100644 index 0000000000..d28a19f599 --- /dev/null +++ b/backend/.sqlx/query-e242c733ca0accb5287e80c97abe3bbae0638612bd111b6b6196dd2808d4b6d0.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE workspace_settings SET\n auto_invite = jsonb_set(\n jsonb_set(\n COALESCE(auto_invite, '{}'::jsonb),\n '{instance_groups}',\n (SELECT COALESCE(jsonb_agg(elem), '[]'::jsonb)\n FROM jsonb_array_elements(COALESCE(auto_invite->'instance_groups', '[]'::jsonb)) elem\n WHERE elem #>> '{}' <> ALL($1))\n ),\n '{instance_groups_roles}',\n CASE WHEN jsonb_typeof(auto_invite->'instance_groups_roles') = 'object'\n THEN (auto_invite->'instance_groups_roles') - $1::text[]\n ELSE '{}'::jsonb\n END\n )\n WHERE auto_invite->'instance_groups' ?| $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "TextArray" + ] + }, + "nullable": [] + }, + "hash": "e242c733ca0accb5287e80c97abe3bbae0638612bd111b6b6196dd2808d4b6d0" +} diff --git a/backend/.sqlx/query-565db14b889f69dfbda5db400a33223fd20548fb106d2a5e5669c0e23ecaa0eb.json b/backend/.sqlx/query-e4e621724d830b318c06734683c8a48b0b692f7e390f2e95162d39ff3f6aeba7.json similarity index 64% rename from backend/.sqlx/query-565db14b889f69dfbda5db400a33223fd20548fb106d2a5e5669c0e23ecaa0eb.json rename to backend/.sqlx/query-e4e621724d830b318c06734683c8a48b0b692f7e390f2e95162d39ff3f6aeba7.json index af526daf8f..fee76e4be4 100644 --- a/backend/.sqlx/query-565db14b889f69dfbda5db400a33223fd20548fb106d2a5e5669c0e23ecaa0eb.json +++ b/backend/.sqlx/query-e4e621724d830b318c06734683c8a48b0b692f7e390f2e95162d39ff3f6aeba7.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT name FROM instance_group WHERE name = $1", + "query": "SELECT name FROM instance_group WHERE id = $1 FOR UPDATE", "describe": { "columns": [ { @@ -18,5 +18,5 @@ false ] }, - "hash": "565db14b889f69dfbda5db400a33223fd20548fb106d2a5e5669c0e23ecaa0eb" + "hash": "e4e621724d830b318c06734683c8a48b0b692f7e390f2e95162d39ff3f6aeba7" } diff --git a/backend/.sqlx/query-e52a80386b132e53a956458e2eb77d29bd6da4fc8511ed44f21d49f4c0965d36.json b/backend/.sqlx/query-e52a80386b132e53a956458e2eb77d29bd6da4fc8511ed44f21d49f4c0965d36.json new file mode 100644 index 0000000000..26a5b9f932 --- /dev/null +++ b/backend/.sqlx/query-e52a80386b132e53a956458e2eb77d29bd6da4fc8511ed44f21d49f4c0965d36.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO trigger_history\n (workspace_id, trigger_kind, path, operation, source, username, changes)\n VALUES ($1, $2, $3, $4, $5, $6, $7)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Varchar", + "Varchar", + "Varchar", + "Varchar", + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "e52a80386b132e53a956458e2eb77d29bd6da4fc8511ed44f21d49f4c0965d36" +} diff --git a/backend/.sqlx/query-e63e275a158040659c41ec8d1ef9107558b0003f495ba3f5fba63b77616d12c8.json b/backend/.sqlx/query-e63e275a158040659c41ec8d1ef9107558b0003f495ba3f5fba63b77616d12c8.json new file mode 100644 index 0000000000..d5a9d425da --- /dev/null +++ b/backend/.sqlx/query-e63e275a158040659c41ec8d1ef9107558b0003f495ba3f5fba63b77616d12c8.json @@ -0,0 +1,26 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT kind::text AS \"kind!\", COUNT(*)::BIGINT AS \"count!\"\n FROM script WHERE archived = false AND deleted = false GROUP BY kind", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "kind!", + "type_info": "Text" + }, + { + "ordinal": 1, + "name": "count!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + null, + null + ] + }, + "hash": "e63e275a158040659c41ec8d1ef9107558b0003f495ba3f5fba63b77616d12c8" +} diff --git a/backend/.sqlx/query-ee182378d9a760c3593b483703ec1682488ca9e6e402ec41abaa6ae3ca263854.json b/backend/.sqlx/query-ee182378d9a760c3593b483703ec1682488ca9e6e402ec41abaa6ae3ca263854.json deleted file mode 100644 index ec170d17c8..0000000000 --- a/backend/.sqlx/query-ee182378d9a760c3593b483703ec1682488ca9e6e402ec41abaa6ae3ca263854.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT email_to_igroup.email\n FROM email_to_igroup\n INNER JOIN instance_group ON instance_group.name = email_to_igroup.igroup\n WHERE instance_group.name = $1\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "email", - "type_info": "Varchar" - } - ], - "parameters": { - "Left": [ - "Text" - ] - }, - "nullable": [ - false - ] - }, - "hash": "ee182378d9a760c3593b483703ec1682488ca9e6e402ec41abaa6ae3ca263854" -} diff --git a/backend/.sqlx/query-fcbbc3b697249c6ee0ca542ea42fadecddead568110dd531fbace439c2263e4f.json b/backend/.sqlx/query-fcbbc3b697249c6ee0ca542ea42fadecddead568110dd531fbace439c2263e4f.json new file mode 100644 index 0000000000..b039aab41c --- /dev/null +++ b/backend/.sqlx/query-fcbbc3b697249c6ee0ca542ea42fadecddead568110dd531fbace439c2263e4f.json @@ -0,0 +1,71 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT id, trigger_kind, path, operation, source, username, created_at, changes\n FROM trigger_history\n WHERE workspace_id = $1\n AND ($2::TEXT IS NULL OR trigger_kind = $2)\n AND ($3::TEXT IS NULL OR path = $3)\n AND ( $6\n OR path = ANY($7)\n OR EXISTS ( SELECT 1 FROM unnest($8::text[]) AS pfx\n WHERE path = pfx\n OR left(path, length(pfx) + 1) = pfx || '/' ) )\n ORDER BY id DESC\n LIMIT $4 OFFSET $5", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "trigger_kind", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "path", + "type_info": "Varchar" + }, + { + "ordinal": 3, + "name": "operation", + "type_info": "Varchar" + }, + { + "ordinal": 4, + "name": "source", + "type_info": "Varchar" + }, + { + "ordinal": 5, + "name": "username", + "type_info": "Varchar" + }, + { + "ordinal": 6, + "name": "created_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 7, + "name": "changes", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text", + "Int8", + "Int8", + "Bool", + "TextArray", + "TextArray" + ] + }, + "nullable": [ + false, + false, + false, + false, + false, + true, + false, + true + ] + }, + "hash": "fcbbc3b697249c6ee0ca542ea42fadecddead568110dd531fbace439c2263e4f" +} diff --git a/backend/.sqlx/query-fed842c14aa37998da2b3cfafc71f7364132ea1e40e687aa84c3d02399e3bfb5.json b/backend/.sqlx/query-fed842c14aa37998da2b3cfafc71f7364132ea1e40e687aa84c3d02399e3bfb5.json deleted file mode 100644 index 78c0158757..0000000000 --- a/backend/.sqlx/query-fed842c14aa37998da2b3cfafc71f7364132ea1e40e687aa84c3d02399e3bfb5.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT path, value from resource WHERE workspace_id = $1 LIMIT $2", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "path", - "type_info": "Varchar" - }, - { - "ordinal": 1, - "name": "value", - "type_info": "Jsonb" - } - ], - "parameters": { - "Left": [ - "Text", - "Int8" - ] - }, - "nullable": [ - false, - true - ] - }, - "hash": "fed842c14aa37998da2b3cfafc71f7364132ea1e40e687aa84c3d02399e3bfb5" -} diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 3c965b2a9a..806966c2b1 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -1339,7 +1339,7 @@ dependencies = [ "aws-smithy-runtime-api", "aws-smithy-types", "h2 0.3.27", - "h2 0.4.15", + "h2 0.4.16", "http 0.2.12", "http 1.5.0", "http-body 0.4.6", @@ -2302,9 +2302,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.4.2" +version = "1.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d262e149917187838d5b42777c8253bcb64500067342904e7d429499a6f277e" +checksum = "509591b7bcd67f4ef775afad7662703b4935daaa6ec0e5605cfb1090b32a2b6d" dependencies = [ "find-msvc-tools", "jobserver", @@ -3958,7 +3958,7 @@ dependencies = [ "deno_tls", "dyn-clone", "error_reporter", - "h2 0.4.15", + "h2 0.4.16", "hickory-resolver", "http 1.5.0", "http-body-util", @@ -5011,9 +5011,9 @@ dependencies = [ [[package]] name = "find-msvc-tools" -version = "0.1.10" +version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26b73573e6edcd2af0cdf47bd6cb58f0b3839491263c314eaad1ccf24430e1de" +checksum = "d45db016d36b838f563236e9193d0ee6ce38f3f68b6c94e914b4929c96bbb890" [[package]] name = "fixedbitset" @@ -5750,9 +5750,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.4.15" +version = "0.4.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6cb093c84e8bd9b188d4c4a8cb6579fc016968d14c99882163cd3ff402a4f155" +checksum = "a9f37a958b41b3b19ee2707c06439c0e9e547e847223eb791ecb0cb821c65e27" dependencies = [ "atomic-waker", "bytes", @@ -6170,7 +6170,7 @@ dependencies = [ "bytes", "futures-channel", "futures-core", - "h2 0.4.15", + "h2 0.4.16", "http 1.5.0", "http-body 1.1.0", "httparse", @@ -6378,9 +6378,9 @@ dependencies = [ [[package]] name = "icu_collections" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" +checksum = "fa68d21081c4a05d5a901a1c62add574c77048b6a1c67be3b50ce0b60d4ca513" dependencies = [ "displaydoc", "potential_utf", @@ -6392,9 +6392,9 @@ dependencies = [ [[package]] name = "icu_locale_core" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" +checksum = "d56e28588da92eee5c3201a6eff33fabdd49b62269c8938d4ff050ce4d900deb" dependencies = [ "displaydoc", "litemap", @@ -6405,9 +6405,9 @@ dependencies = [ [[package]] name = "icu_normalizer" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" +checksum = "12f9cf5f235641ed274641dd81c3f28d870e276763d0797aeeab72317b1c646f" dependencies = [ "icu_collections", "icu_normalizer_data", @@ -6419,16 +6419,17 @@ dependencies = [ [[package]] name = "icu_normalizer_data" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" +checksum = "1563da1ed3e0b3bf3d74c9b85917ac9c56464d2f57242270c09c9e752f8021a0" [[package]] name = "icu_properties" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" +checksum = "7e7ca276ad3145661a65914e6daf131ca5120cd3dcee8f8f3214b8875184a148" dependencies = [ + "displaydoc", "icu_collections", "icu_locale_core", "icu_properties_data", @@ -6439,15 +6440,15 @@ dependencies = [ [[package]] name = "icu_properties_data" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" +checksum = "e590f038c1464a96894fd6d10127e90a8be4509f56ff7ecef851b15cee0b7caa" [[package]] name = "icu_provider" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" +checksum = "92a7ed671a6aad807a8651a2e1782a6598fda9ce5185dd8158549e95a91c6428" dependencies = [ "displaydoc", "icu_locale_core", @@ -7196,9 +7197,9 @@ dependencies = [ [[package]] name = "libredox" -version = "0.1.19" +version = "0.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2026a5056764a10b2bf5d56488cba40da507f5493a6a429340e2004d9ed085fa" +checksum = "28d0a00925a9f930d679b6789b721e3a7f9ed110f41b86d2497caa780c3a070a" dependencies = [ "bitflags 2.13.1", "libc", @@ -7275,9 +7276,9 @@ checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" [[package]] name = "litemap" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" +checksum = "47d9d19d1d6efa0109d2f65ff4c85cddd50bd572e5a00127ab10987290bcefae" [[package]] name = "lock_api" @@ -7684,9 +7685,9 @@ dependencies = [ [[package]] name = "minicov" -version = "0.3.8" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4869b6a491569605d66d3952bcdf03df789e5b536e5f0cf7758a7f08a55ae24d" +checksum = "c3aa3aa12b448ac225b3102217d1ac5cc717908f02722926524b0599c933c7a0" dependencies = [ "cc", "walkdir", @@ -9005,9 +9006,9 @@ checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" [[package]] name = "pest" -version = "2.8.8" +version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7df728be843c7070fab6ab7c328c4e9e9d78e23bf749c0669c86ee7ebfa050a2" +checksum = "5a07a60cc7a4d00c91f95c685609d1d2f79050e6804b70ebedd7650f0b839bcf" dependencies = [ "memchr", "ucd-trie", @@ -9015,9 +9016,9 @@ dependencies = [ [[package]] name = "pest_derive" -version = "2.8.8" +version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e2dd6fc3b26b3462ee188aac870f5a41d398f1cd5e2408d16531bd71c9591fd" +checksum = "b3a83744a5c8455b8b3e0dc5031362780a347c878bdd11584d1a8984228cc88d" dependencies = [ "pest", "pest_generator", @@ -9025,9 +9026,9 @@ dependencies = [ [[package]] name = "pest_generator" -version = "2.8.8" +version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a7a9205cfb6f596a9e8b689c0a15f9ceb7a1aafae7aaf788150ac65b29975b6" +checksum = "e0cd3451aa3de60d4b9a1e736885e4dea6b31617598026f12256ad566d63304a" dependencies = [ "pest", "pest_meta", @@ -9038,9 +9039,9 @@ dependencies = [ [[package]] name = "pest_meta" -version = "2.8.8" +version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85abd351c0de1e8384fc791a0737111a350394937e92b956b743dac12429f57c" +checksum = "e04d3a0849e241d7dfce834c83b1c5edc8622009e8dd51a12ba1927c32f05496" dependencies = [ "pest", ] @@ -9273,9 +9274,9 @@ dependencies = [ [[package]] name = "pkg-config" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" +checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" [[package]] name = "plain" @@ -9398,9 +9399,9 @@ dependencies = [ [[package]] name = "potential_utf" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" +checksum = "d83eb9bc6d8e5cf568e7a1101d60ee05e81ed50ea106026f3d18deeb046d7661" dependencies = [ "zerovec", ] @@ -9730,9 +9731,9 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.16" +version = "0.11.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f4bfc015262b9df63c8845072ce59068853ff5872180c2ce2f13038b970e560" +checksum = "04759210543be93709136e28212294a659ef5001836ff4eab4d663e4529bba83" dependencies = [ "aws-lc-rs", "bytes", @@ -10105,18 +10106,18 @@ dependencies = [ [[package]] name = "ref-cast" -version = "1.0.26" +version = "1.0.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "216e8f773d7923bcba9ceb86a86c93cabb3903a11872fc3f138c49630e50b96d" +checksum = "7e440fb4e4b4147295338efb76001ab9e4efc0e5839df2c47fc5ac2381d365c3" dependencies = [ "ref-cast-impl", ] [[package]] name = "ref-cast-impl" -version = "1.0.26" +version = "1.0.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c9283685feec7d69af75fb0e858d5e7378f33fe4fc699383b2916ab9273e03c" +checksum = "92ecd8964f8453721699a1ed72037b0db49ce2f5a5138486ee89bed6f67cdf3a" dependencies = [ "proc-macro2", "quote", @@ -10194,7 +10195,7 @@ dependencies = [ "futures-channel", "futures-core", "futures-util", - "h2 0.4.15", + "h2 0.4.16", "http 1.5.0", "http-body 1.1.0", "http-body-util", @@ -10242,7 +10243,7 @@ dependencies = [ "encoding_rs", "futures-core", "futures-util", - "h2 0.4.15", + "h2 0.4.16", "http 1.5.0", "http-body 1.1.0", "http-body-util", @@ -10431,9 +10432,9 @@ dependencies = [ [[package]] name = "rmcp-macros" -version = "3.1.2" +version = "3.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6898e24cd16342b59bfa8a53c2c04b9cf62fc8a2cfea57b9c038b09984bfc521" +checksum = "54817231d63a35330d5c3a5782a4a166fbe3b914a9faab440d3dd8bd37971890" dependencies = [ "darling 0.24.0", "proc-macro2", @@ -13115,9 +13116,9 @@ dependencies = [ [[package]] name = "tinystr" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" +checksum = "b1e27c91459209c2986af3dcf603a5a74a4368754ce37414f59acc971167f643" dependencies = [ "displaydoc", "zerovec", @@ -13513,7 +13514,7 @@ dependencies = [ "base64 0.22.1", "bytes", "flate2", - "h2 0.4.15", + "h2 0.4.16", "http 1.5.0", "http-body 1.1.0", "http-body-util", @@ -13545,7 +13546,7 @@ dependencies = [ "axum 0.8.9", "base64 0.22.1", "bytes", - "h2 0.4.15", + "h2 0.4.16", "http 1.5.0", "http-body 1.1.0", "http-body-util", @@ -14252,9 +14253,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.24.0" +version = "1.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf3923a6f5c4c6382e0b653c4117f48d631ea17f38ed86e2a828e6f7412f5239" +checksum = "2cefc03fd367c0c6d4305de1b312cf00248c4114f4a0418ce6a6af769e3b0bd9" dependencies = [ "getrandom 0.4.3", "js-sys", @@ -14664,7 +14665,7 @@ dependencies = [ [[package]] name = "windmill" -version = "1.789.0" +version = "1.792.2" dependencies = [ "anyhow", "async-nats", @@ -14749,7 +14750,7 @@ dependencies = [ [[package]] name = "windmill-ai" -version = "1.789.0" +version = "1.792.2" dependencies = [ "async-stream", "async-trait", @@ -14782,7 +14783,7 @@ dependencies = [ [[package]] name = "windmill-alerting" -version = "1.789.0" +version = "1.792.2" dependencies = [ "axum 0.8.9", "chrono", @@ -14795,7 +14796,7 @@ dependencies = [ [[package]] name = "windmill-api" -version = "1.789.0" +version = "1.792.2" dependencies = [ "anyhow", "argon2", @@ -14935,7 +14936,7 @@ dependencies = [ [[package]] name = "windmill-api-agent-workers" -version = "1.789.0" +version = "1.792.2" dependencies = [ "axum 0.8.9", "chrono", @@ -14958,7 +14959,7 @@ dependencies = [ [[package]] name = "windmill-api-assets" -version = "1.789.0" +version = "1.792.2" dependencies = [ "axum 0.8.9", "chrono", @@ -14975,7 +14976,7 @@ dependencies = [ [[package]] name = "windmill-api-auth" -version = "1.789.0" +version = "1.792.2" dependencies = [ "anyhow", "axum 0.8.9", @@ -15001,7 +15002,7 @@ dependencies = [ [[package]] name = "windmill-api-client" -version = "1.789.0" +version = "1.792.2" dependencies = [ "reqwest 0.12.28", "serde", @@ -15011,7 +15012,7 @@ dependencies = [ [[package]] name = "windmill-api-configs" -version = "1.789.0" +version = "1.792.2" dependencies = [ "axum 0.8.9", "chrono", @@ -15028,7 +15029,7 @@ dependencies = [ [[package]] name = "windmill-api-debug" -version = "1.789.0" +version = "1.792.2" dependencies = [ "axum 0.8.9", "base64 0.22.1", @@ -15050,7 +15051,7 @@ dependencies = [ [[package]] name = "windmill-api-embeddings" -version = "1.789.0" +version = "1.792.2" dependencies = [ "anyhow", "axum 0.8.9", @@ -15073,7 +15074,7 @@ dependencies = [ [[package]] name = "windmill-api-flow-conversations" -version = "1.789.0" +version = "1.792.2" dependencies = [ "axum 0.8.9", "chrono", @@ -15089,7 +15090,7 @@ dependencies = [ [[package]] name = "windmill-api-flows" -version = "1.789.0" +version = "1.792.2" dependencies = [ "axum 0.8.9", "chrono", @@ -15111,7 +15112,7 @@ dependencies = [ [[package]] name = "windmill-api-groups" -version = "1.789.0" +version = "1.792.2" dependencies = [ "axum 0.8.9", "chrono", @@ -15132,7 +15133,7 @@ dependencies = [ [[package]] name = "windmill-api-inputs" -version = "1.789.0" +version = "1.792.2" dependencies = [ "axum 0.8.9", "chrono", @@ -15146,7 +15147,7 @@ dependencies = [ [[package]] name = "windmill-api-integration-tests" -version = "1.789.0" +version = "1.792.2" dependencies = [ "anyhow", "async-nats", @@ -15181,7 +15182,7 @@ dependencies = [ [[package]] name = "windmill-api-jobs" -version = "1.789.0" +version = "1.792.2" dependencies = [ "anyhow", "axum 0.8.9", @@ -15206,7 +15207,7 @@ dependencies = [ [[package]] name = "windmill-api-npm-proxy" -version = "1.789.0" +version = "1.792.2" dependencies = [ "axum 0.8.9", "chrono", @@ -15234,7 +15235,7 @@ dependencies = [ [[package]] name = "windmill-api-openapi" -version = "1.789.0" +version = "1.792.2" dependencies = [ "anyhow", "axum 0.8.9", @@ -15256,7 +15257,7 @@ dependencies = [ [[package]] name = "windmill-api-schedule" -version = "1.789.0" +version = "1.792.2" dependencies = [ "axum 0.8.9", "chrono", @@ -15276,7 +15277,7 @@ dependencies = [ [[package]] name = "windmill-api-scripts" -version = "1.789.0" +version = "1.792.2" dependencies = [ "axum 0.8.9", "chrono", @@ -15314,7 +15315,7 @@ dependencies = [ [[package]] name = "windmill-api-settings" -version = "1.789.0" +version = "1.792.2" dependencies = [ "anyhow", "axum 0.8.9", @@ -15342,7 +15343,7 @@ dependencies = [ [[package]] name = "windmill-api-sse" -version = "1.789.0" +version = "1.792.2" dependencies = [ "lazy_static", "serde", @@ -15354,12 +15355,11 @@ dependencies = [ [[package]] name = "windmill-api-users" -version = "1.789.0" +version = "1.792.2" dependencies = [ "argon2", "axum 0.8.9", "chrono", - "dashmap", "http 1.5.0", "hyper 1.11.0", "lazy_static", @@ -15379,7 +15379,7 @@ dependencies = [ [[package]] name = "windmill-api-workers" -version = "1.789.0" +version = "1.792.2" dependencies = [ "axum 0.8.9", "chrono", @@ -15393,7 +15393,7 @@ dependencies = [ [[package]] name = "windmill-api-workspaces" -version = "1.789.0" +version = "1.792.2" dependencies = [ "axum 0.8.9", "chrono", @@ -15428,7 +15428,7 @@ dependencies = [ [[package]] name = "windmill-audit" -version = "1.789.0" +version = "1.792.2" dependencies = [ "chrono", "lazy_static", @@ -15442,7 +15442,7 @@ dependencies = [ [[package]] name = "windmill-autoscaling" -version = "1.789.0" +version = "1.792.2" dependencies = [ "anyhow", "axum 0.8.9", @@ -15461,7 +15461,7 @@ dependencies = [ [[package]] name = "windmill-common" -version = "1.789.0" +version = "1.792.2" dependencies = [ "aes-gcm", "aho-corasick", @@ -15565,7 +15565,7 @@ dependencies = [ [[package]] name = "windmill-dep-map" -version = "1.789.0" +version = "1.792.2" dependencies = [ "chrono", "itertools 0.14.0", @@ -15584,7 +15584,7 @@ dependencies = [ [[package]] name = "windmill-git-sync" -version = "1.789.0" +version = "1.792.2" dependencies = [ "regex", "serde", @@ -15599,7 +15599,7 @@ dependencies = [ [[package]] name = "windmill-indexer" -version = "1.789.0" +version = "1.792.2" dependencies = [ "anyhow", "astral-tokio-tar", @@ -15623,7 +15623,7 @@ dependencies = [ [[package]] name = "windmill-jseval" -version = "1.789.0" +version = "1.792.2" dependencies = [ "anyhow", "futures", @@ -15640,7 +15640,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.789.0" +version = "1.792.2" dependencies = [ "itertools 0.14.0", "lazy_static", @@ -15656,7 +15656,7 @@ dependencies = [ [[package]] name = "windmill-mcp" -version = "1.789.0" +version = "1.792.2" dependencies = [ "anyhow", "async-trait", @@ -15677,7 +15677,7 @@ dependencies = [ [[package]] name = "windmill-native-triggers" -version = "1.789.0" +version = "1.792.2" dependencies = [ "anyhow", "async-trait", @@ -15708,7 +15708,7 @@ dependencies = [ [[package]] name = "windmill-oauth" -version = "1.789.0" +version = "1.792.2" dependencies = [ "anyhow", "arc-swap", @@ -15733,7 +15733,7 @@ dependencies = [ [[package]] name = "windmill-object-store" -version = "1.789.0" +version = "1.792.2" dependencies = [ "anyhow", "async-stream", @@ -15767,7 +15767,7 @@ dependencies = [ [[package]] name = "windmill-operator" -version = "1.789.0" +version = "1.792.2" dependencies = [ "anyhow", "futures", @@ -15785,7 +15785,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.789.0" +version = "1.792.2" dependencies = [ "convert_case 0.6.0", "serde", @@ -15794,7 +15794,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.789.0" +version = "1.792.2" dependencies = [ "anyhow", "lazy_static", @@ -15806,7 +15806,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.789.0" +version = "1.792.2" dependencies = [ "anyhow", "serde_json", @@ -15818,7 +15818,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.789.0" +version = "1.792.2" dependencies = [ "anyhow", "gosyn", @@ -15830,7 +15830,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.789.0" +version = "1.792.2" dependencies = [ "anyhow", "lazy_static", @@ -15842,7 +15842,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.789.0" +version = "1.792.2" dependencies = [ "anyhow", "serde_json", @@ -15854,7 +15854,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.789.0" +version = "1.792.2" dependencies = [ "anyhow", "nu-parser", @@ -15865,7 +15865,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.789.0" +version = "1.792.2" dependencies = [ "anyhow", "itertools 0.14.0", @@ -15876,7 +15876,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.789.0" +version = "1.792.2" dependencies = [ "anyhow", "itertools 0.14.0", @@ -15888,7 +15888,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-asset" -version = "1.789.0" +version = "1.792.2" dependencies = [ "anyhow", "rustpython-ast", @@ -15899,7 +15899,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.789.0" +version = "1.792.2" dependencies = [ "anyhow", "async-recursion", @@ -15921,7 +15921,7 @@ dependencies = [ [[package]] name = "windmill-parser-r" -version = "1.789.0" +version = "1.792.2" dependencies = [ "anyhow", "serde_json", @@ -15933,7 +15933,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.789.0" +version = "1.792.2" dependencies = [ "anyhow", "lazy_static", @@ -15947,7 +15947,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.789.0" +version = "1.792.2" dependencies = [ "anyhow", "convert_case 0.6.0", @@ -15964,7 +15964,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.789.0" +version = "1.792.2" dependencies = [ "anyhow", "lazy_static", @@ -15977,7 +15977,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql-asset" -version = "1.789.0" +version = "1.792.2" dependencies = [ "anyhow", "serde", @@ -15989,7 +15989,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.789.0" +version = "1.792.2" dependencies = [ "anyhow", "lazy_static", @@ -16007,7 +16007,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts-asset" -version = "1.789.0" +version = "1.792.2" dependencies = [ "anyhow", "serde-wasm-bindgen", @@ -16023,7 +16023,7 @@ dependencies = [ [[package]] name = "windmill-parser-wac" -version = "1.789.0" +version = "1.792.2" dependencies = [ "anyhow", "rustpython-ast", @@ -16039,7 +16039,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.789.0" +version = "1.792.2" dependencies = [ "anyhow", "lazy_static", @@ -16053,7 +16053,7 @@ dependencies = [ [[package]] name = "windmill-queue" -version = "1.789.0" +version = "1.792.2" dependencies = [ "anyhow", "async-recursion", @@ -16092,7 +16092,7 @@ dependencies = [ [[package]] name = "windmill-runtime-nativets" -version = "1.789.0" +version = "1.792.2" dependencies = [ "anyhow", "const_format", @@ -16132,7 +16132,7 @@ dependencies = [ [[package]] name = "windmill-sql-datatype-parser-wasm" -version = "1.789.0" +version = "1.792.2" dependencies = [ "getrandom 0.3.4", "wasm-bindgen", @@ -16143,7 +16143,7 @@ dependencies = [ [[package]] name = "windmill-store" -version = "1.789.0" +version = "1.792.2" dependencies = [ "anyhow", "async-recursion", @@ -16178,7 +16178,7 @@ dependencies = [ [[package]] name = "windmill-test-utils" -version = "1.789.0" +version = "1.792.2" dependencies = [ "anyhow", "async-trait", @@ -16202,7 +16202,7 @@ dependencies = [ [[package]] name = "windmill-trigger" -version = "1.789.0" +version = "1.792.2" dependencies = [ "anyhow", "async-trait", @@ -16235,7 +16235,7 @@ dependencies = [ [[package]] name = "windmill-trigger-amqp" -version = "1.789.0" +version = "1.792.2" dependencies = [ "anyhow", "async-trait", @@ -16262,7 +16262,7 @@ dependencies = [ [[package]] name = "windmill-trigger-azure" -version = "1.789.0" +version = "1.792.2" dependencies = [ "anyhow", "async-trait", @@ -16295,7 +16295,7 @@ dependencies = [ [[package]] name = "windmill-trigger-email" -version = "1.789.0" +version = "1.792.2" dependencies = [ "anyhow", "async-trait", @@ -16315,7 +16315,7 @@ dependencies = [ [[package]] name = "windmill-trigger-gcp" -version = "1.789.0" +version = "1.792.2" dependencies = [ "anyhow", "async-trait", @@ -16349,7 +16349,7 @@ dependencies = [ [[package]] name = "windmill-trigger-http" -version = "1.789.0" +version = "1.792.2" dependencies = [ "anyhow", "async-trait", @@ -16385,7 +16385,7 @@ dependencies = [ [[package]] name = "windmill-trigger-kafka" -version = "1.789.0" +version = "1.792.2" dependencies = [ "anyhow", "async-trait", @@ -16408,7 +16408,7 @@ dependencies = [ [[package]] name = "windmill-trigger-mqtt" -version = "1.789.0" +version = "1.792.2" dependencies = [ "anyhow", "async-trait", @@ -16432,7 +16432,7 @@ dependencies = [ [[package]] name = "windmill-trigger-nats" -version = "1.789.0" +version = "1.792.2" dependencies = [ "anyhow", "async-nats", @@ -16456,7 +16456,7 @@ dependencies = [ [[package]] name = "windmill-trigger-postgres" -version = "1.789.0" +version = "1.792.2" dependencies = [ "anyhow", "async-trait", @@ -16491,7 +16491,7 @@ dependencies = [ [[package]] name = "windmill-trigger-sqs" -version = "1.789.0" +version = "1.792.2" dependencies = [ "anyhow", "async-trait", @@ -16519,7 +16519,7 @@ dependencies = [ [[package]] name = "windmill-trigger-websocket" -version = "1.789.0" +version = "1.792.2" dependencies = [ "anyhow", "async-trait", @@ -16544,7 +16544,7 @@ dependencies = [ [[package]] name = "windmill-types" -version = "1.789.0" +version = "1.792.2" dependencies = [ "anyhow", "bitflags 2.13.1", @@ -16563,7 +16563,7 @@ dependencies = [ [[package]] name = "windmill-worker" -version = "1.789.0" +version = "1.792.2" dependencies = [ "anyhow", "async-once-cell", @@ -16679,7 +16679,7 @@ dependencies = [ [[package]] name = "windmill-worker-volumes" -version = "1.789.0" +version = "1.792.2" dependencies = [ "bytes", "futures", @@ -17234,9 +17234,9 @@ checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" [[package]] name = "writeable" -version = "0.6.3" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" +checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc" [[package]] name = "wyz" @@ -17440,9 +17440,9 @@ dependencies = [ [[package]] name = "zerotrie" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" +checksum = "4ea269c3bd32f0a32c321907a2ae912ba6f4649bb0fc764a15627e99a7095a3f" dependencies = [ "displaydoc", "yoke", @@ -17451,9 +17451,9 @@ dependencies = [ [[package]] name = "zerovec" -version = "0.11.6" +version = "0.11.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" +checksum = "94b5c6b5976d66c1d703c4fd17d3f5e43c8cedaacf604961b171adc7130896d8" dependencies = [ "yoke", "zerofrom", @@ -17462,13 +17462,13 @@ dependencies = [ [[package]] name = "zerovec-derive" -version = "0.11.3" +version = "0.11.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" +checksum = "9f212a141d820099d57ffafb9569be9617a6f27d3dc881fbee8fb56642f917a9" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.3", ] [[package]] diff --git a/backend/Cargo.toml b/backend/Cargo.toml index f41ff1c2ab..4bf21a1495 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "windmill" -version = "1.789.0" +version = "1.792.2" authors.workspace = true edition.workspace = true @@ -88,7 +88,7 @@ members = [ exclude = ["./windmill-duckdb-ffi-internal", "./parsers/windmill-parser-wasm"] [workspace.package] -version = "1.789.0" +version = "1.792.2" authors = ["Ruben Fiszel "] edition = "2021" diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 6f6c280541..ec7c2ac03e 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -a65162b22b127b54c0686095ee1b16b04e3111f7 +bd4de74eb37b32a2b6c7c69f6dedac031ef8436b diff --git a/backend/migrations/20260813195023_preserve_orphaned_instance_group_members.down.sql b/backend/migrations/20260813195023_preserve_orphaned_instance_group_members.down.sql new file mode 100644 index 0000000000..c2eb12a15a --- /dev/null +++ b/backend/migrations/20260813195023_preserve_orphaned_instance_group_members.down.sql @@ -0,0 +1,9 @@ +-- Restore the instance_group source for members the up migration converted. The stripped +-- auto_invite references cannot be restored (the groups they named no longer exist). +UPDATE usr +SET added_via = jsonb_build_object( + 'source', 'instance_group', + 'group', added_via->>'migrated_from_instance_group' +) +WHERE added_via->>'source' = 'manual' + AND added_via ? 'migrated_from_instance_group'; diff --git a/backend/migrations/20260813195023_preserve_orphaned_instance_group_members.up.sql b/backend/migrations/20260813195023_preserve_orphaned_instance_group_members.up.sql new file mode 100644 index 0000000000..581401b854 --- /dev/null +++ b/backend/migrations/20260813195023_preserve_orphaned_instance_group_members.up.sql @@ -0,0 +1,54 @@ +-- Strip auto_invite references to groups that no longer exist, so a later group created +-- with the same name cannot silently re-acquire the mapping. +UPDATE workspace_settings +SET auto_invite = jsonb_set( + jsonb_set( + auto_invite, + '{instance_groups}', + COALESCE( + (SELECT jsonb_agg(elem) + FROM jsonb_array_elements(auto_invite->'instance_groups') elem + WHERE EXISTS (SELECT 1 FROM instance_group ig WHERE ig.name = elem #>> '{}')), + '[]'::jsonb + ) + ), + '{instance_groups_roles}', + CASE WHEN jsonb_typeof(auto_invite->'instance_groups_roles') = 'object' + THEN (SELECT COALESCE(jsonb_object_agg(key, value), '{}'::jsonb) + FROM jsonb_each(auto_invite->'instance_groups_roles') + WHERE EXISTS (SELECT 1 FROM instance_group ig WHERE ig.name = key)) + ELSE '{}'::jsonb + END +) +WHERE jsonb_typeof(auto_invite->'instance_groups') = 'array' + AND EXISTS ( + SELECT 1 FROM jsonb_array_elements(auto_invite->'instance_groups') elem + WHERE NOT EXISTS (SELECT 1 FROM instance_group ig WHERE ig.name = elem #>> '{}') + ); + +-- Workspace members whose instance-group grant can no longer be re-derived become manual +-- members. Group deletion, overwrite imports and some SCIM paths used to mutate groups +-- without carrying the change through to workspace membership, leaving members whose +-- granting group was deleted — or who were dropped from a group that still exists. Under +-- state-based reconciliation those members belong to zero configured groups, so the first +-- reconcile touching their workspace would otherwise remove them and destroy their drafts, +-- favorites, tokens and permissions. The original group name is kept under +-- 'migrated_from_instance_group' so admins can identify and prune them deliberately. +UPDATE usr +SET added_via = jsonb_build_object( + 'source', 'manual', + 'migrated_from_instance_group', added_via->>'group' +) +WHERE added_via->>'source' = 'instance_group' + AND NOT EXISTS ( + SELECT 1 + FROM workspace_settings ws + JOIN LATERAL jsonb_array_elements_text( + CASE WHEN jsonb_typeof(ws.auto_invite->'instance_groups') = 'array' + THEN ws.auto_invite->'instance_groups' + ELSE '[]'::jsonb + END + ) g ON true + JOIN email_to_igroup e ON e.igroup = g.value AND e.email = usr.email + WHERE ws.workspace_id = usr.workspace_id + ); diff --git a/backend/migrations/20260814090221_trigger_history.down.sql b/backend/migrations/20260814090221_trigger_history.down.sql new file mode 100644 index 0000000000..c4aad347b0 --- /dev/null +++ b/backend/migrations/20260814090221_trigger_history.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS trigger_history; diff --git a/backend/migrations/20260814090221_trigger_history.up.sql b/backend/migrations/20260814090221_trigger_history.up.sql new file mode 100644 index 0000000000..1eb4f778a2 --- /dev/null +++ b/backend/migrations/20260814090221_trigger_history.up.sql @@ -0,0 +1,65 @@ +-- Append-only record of every schedule/trigger mutation: who, what changed, and +-- from which kind of client. +CREATE TABLE IF NOT EXISTS trigger_history ( + id BIGSERIAL PRIMARY KEY, + workspace_id VARCHAR(50) NOT NULL REFERENCES workspace(id) ON DELETE CASCADE, + -- 'schedule' or a trigger's TRIGGER_TYPE ('http', 'kafka', ...). Not the + -- TRIGGER_KIND enum: that one is capture-oriented and misses 'schedule'. + trigger_kind VARCHAR(50) NOT NULL, + path VARCHAR(255) NOT NULL, + -- 'create' | 'update' | 'delete' | 'enable' | 'disable' | 'suspend' + operation VARCHAR(20) NOT NULL, + -- 'ui' | 'cli' | 'api' | 'worker' + source VARCHAR(20) NOT NULL, + -- NULL when the server acted on its own (worker auto-disable). + username VARCHAR(255), + created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(), + -- {field: {old, new}} for the fields that actually changed. `old` is + -- absent where it is not known: a create, and the workspace-wide handler + -- override that rewrites every schedule without reading them first. NULL + -- when the operation carries no field-level diff at all (delete). + changes JSONB +); + +CREATE INDEX IF NOT EXISTS idx_trigger_history_workspace_kind_path + ON trigger_history(workspace_id, trigger_kind, path, id DESC); + +CREATE INDEX IF NOT EXISTS idx_trigger_history_workspace_id + ON trigger_history(workspace_id, id DESC); + +GRANT ALL ON TABLE trigger_history TO windmill_user; +GRANT ALL ON TABLE trigger_history TO windmill_admin; +GRANT ALL ON SEQUENCE trigger_history_id_seq TO windmill_user; +GRANT ALL ON SEQUENCE trigger_history_id_seq TO windmill_admin; + +ALTER TABLE trigger_history ENABLE ROW LEVEL SECURITY; + +CREATE POLICY admin_all ON trigger_history FOR ALL TO windmill_admin USING (true) WITH CHECK (true); + +-- Every mutating trigger route writes through the RLS pool, so windmill_user +-- must be able to append. +CREATE POLICY allow_insert ON trigger_history FOR INSERT TO windmill_user WITH CHECK (true); + +-- Reads mirror the path half of the live trigger's own policies: a row can +-- quote a schedule's `args`, so it must not be readable by anyone who could not +-- read the trigger it describes. Deliberately narrower than the live row on one +-- point — the `extra_perms` grants have no counterpart here, since the history +-- does not carry the row's ACL and must survive its deletion. Narrower is the +-- safe direction. +CREATE POLICY see_own ON trigger_history FOR SELECT TO windmill_user +USING ( + SPLIT_PART(path::text, '/', 1) = 'u' + AND SPLIT_PART(path::text, '/', 2) = current_setting('session.user') +); + +CREATE POLICY see_member ON trigger_history FOR SELECT TO windmill_user +USING ( + SPLIT_PART(path::text, '/', 1) = 'g' + AND SPLIT_PART(path::text, '/', 2) = ANY(regexp_split_to_array(current_setting('session.groups'), ',')) +); + +CREATE POLICY see_folder_extra_perms_user ON trigger_history FOR SELECT TO windmill_user +USING ( + SPLIT_PART(path::text, '/', 1) = 'f' + AND SPLIT_PART(path::text, '/', 2) = ANY(regexp_split_to_array(current_setting('session.folders_read'), ',')) +); diff --git a/backend/oauth_connect.json b/backend/oauth_connect.json index 9e74822d98..cec49861cd 100644 --- a/backend/oauth_connect.json +++ b/backend/oauth_connect.json @@ -159,7 +159,8 @@ "sage_intacct": { "auth_url": "https://api.intacct.com/ia/api/v1/oauth2/authorize", "token_url": "https://api.intacct.com/ia/api/v1/oauth2/token", - "scopes": ["offline_access"] + "scopes": ["offline_access"], + "req_body_auth": true }, "spotify": { "auth_url": "https://accounts.spotify.com/authorize", diff --git a/backend/parsers/windmill-parser-wasm/Cargo.lock b/backend/parsers/windmill-parser-wasm/Cargo.lock index c26c76cda3..9f7870863e 100644 --- a/backend/parsers/windmill-parser-wasm/Cargo.lock +++ b/backend/parsers/windmill-parser-wasm/Cargo.lock @@ -6191,7 +6191,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windmill-common" -version = "1.789.0" +version = "1.792.2" dependencies = [ "aho-corasick", "anyhow", @@ -6274,7 +6274,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.789.0" +version = "1.792.2" dependencies = [ "proc-macro2", "quote", @@ -6286,7 +6286,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.789.0" +version = "1.792.2" dependencies = [ "convert_case", "serde", @@ -6295,7 +6295,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.789.0" +version = "1.792.2" dependencies = [ "anyhow", "lazy_static", @@ -6307,7 +6307,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.789.0" +version = "1.792.2" dependencies = [ "anyhow", "serde_json", @@ -6319,7 +6319,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.789.0" +version = "1.792.2" dependencies = [ "anyhow", "gosyn", @@ -6331,7 +6331,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.789.0" +version = "1.792.2" dependencies = [ "anyhow", "lazy_static", @@ -6343,7 +6343,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.789.0" +version = "1.792.2" dependencies = [ "anyhow", "serde_json", @@ -6355,7 +6355,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.789.0" +version = "1.792.2" dependencies = [ "anyhow", "nu-parser", @@ -6366,7 +6366,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.789.0" +version = "1.792.2" dependencies = [ "anyhow", "itertools 0.14.0", @@ -6377,7 +6377,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.789.0" +version = "1.792.2" dependencies = [ "anyhow", "itertools 0.14.0", @@ -6389,7 +6389,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-asset" -version = "1.789.0" +version = "1.792.2" dependencies = [ "anyhow", "rustpython-ast", @@ -6400,7 +6400,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.789.0" +version = "1.792.2" dependencies = [ "anyhow", "async-recursion", @@ -6422,7 +6422,7 @@ dependencies = [ [[package]] name = "windmill-parser-r" -version = "1.789.0" +version = "1.792.2" dependencies = [ "anyhow", "serde_json", @@ -6434,7 +6434,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.789.0" +version = "1.792.2" dependencies = [ "anyhow", "lazy_static", @@ -6448,7 +6448,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.789.0" +version = "1.792.2" dependencies = [ "anyhow", "convert_case", @@ -6465,7 +6465,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.789.0" +version = "1.792.2" dependencies = [ "anyhow", "lazy_static", @@ -6478,7 +6478,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql-asset" -version = "1.789.0" +version = "1.792.2" dependencies = [ "anyhow", "serde", @@ -6490,7 +6490,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.789.0" +version = "1.792.2" dependencies = [ "anyhow", "lazy_static", @@ -6508,7 +6508,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts-asset" -version = "1.789.0" +version = "1.792.2" dependencies = [ "anyhow", "serde-wasm-bindgen", @@ -6524,7 +6524,7 @@ dependencies = [ [[package]] name = "windmill-parser-wac" -version = "1.789.0" +version = "1.792.2" dependencies = [ "anyhow", "rustpython-ast", @@ -6540,7 +6540,7 @@ dependencies = [ [[package]] name = "windmill-parser-wasm" -version = "1.789.0" +version = "1.792.2" dependencies = [ "anyhow", "getrandom 0.2.17", @@ -6572,7 +6572,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.789.0" +version = "1.792.2" dependencies = [ "anyhow", "lazy_static", @@ -6586,7 +6586,7 @@ dependencies = [ [[package]] name = "windmill-types" -version = "1.789.0" +version = "1.792.2" dependencies = [ "anyhow", "bitflags", diff --git a/backend/parsers/windmill-parser-wasm/Cargo.toml b/backend/parsers/windmill-parser-wasm/Cargo.toml index 1a50255db5..03243d5824 100644 --- a/backend/parsers/windmill-parser-wasm/Cargo.toml +++ b/backend/parsers/windmill-parser-wasm/Cargo.toml @@ -12,7 +12,7 @@ resolver = "2" members = ["."] [workspace.package] -version = "1.789.0" +version = "1.792.2" edition = "2021" authors = ["Ruben Fiszel "] diff --git a/backend/src/main.rs b/backend/src/main.rs index 43a92ce4bf..2dff3209ac 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -1629,6 +1629,10 @@ Windmill Community Edition {GIT_VERSION} } } + // `workers_f` must stay ahead of `server_f`: these are polled on one task in + // declaration order, and `run_server` yields once after handing over the base + // internal url so the workers get past that oneshot before it builds its router. + // Ordering `server_f` first makes them wait out the whole build instead. if mcp_mode { futures::try_join!(workers_f, server_f)?; } else { @@ -1691,7 +1695,8 @@ async fn process_notify_event( "restart_worker_group" => { if worker_mode && payload == *WORKER_GROUP { tracing::info!("Restart requested for worker group '{payload}'"); - spawn_graceful_killpill(tx, db, 30, "worker group restart requested").await; + spawn_graceful_killpill(tx, db, 30, "worker group restart requested", server_mode) + .await; } } "notify_webhook_change" => { @@ -1997,8 +2002,14 @@ async fn process_notify_event( reload_otel_tracing_proxy_setting(conn).await; if worker_mode { tracing::info!("OTEL tracing proxy setting changed, restarting worker"); - spawn_graceful_killpill(tx, db, 30, "OTEL tracing proxy setting change") - .await; + spawn_graceful_killpill( + tx, + db, + 30, + "OTEL tracing proxy setting change", + server_mode, + ) + .await; } } REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING => { @@ -2009,12 +2020,20 @@ async fn process_notify_event( } EXPOSE_METRICS_SETTING => { tracing::info!("Metrics setting changed, restarting"); - spawn_graceful_killpill(tx, db, 30, "metrics setting change").await; + spawn_graceful_killpill(tx, db, 30, "metrics setting change", server_mode) + .await; } EMAIL_DOMAIN_SETTING => { tracing::info!("Email domain setting changed"); if server_mode { - spawn_graceful_killpill(tx, db, 30, "email domain setting change").await; + spawn_graceful_killpill( + tx, + db, + 30, + "email domain setting change", + server_mode, + ) + .await; } } EXPOSE_DEBUG_METRICS_SETTING => { @@ -2050,19 +2069,26 @@ async fn process_notify_event( } OTEL_SETTING => { tracing::info!("OTEL setting changed, restarting"); - spawn_graceful_killpill(tx, db, 30, "OTEL setting change").await; + spawn_graceful_killpill(tx, db, 30, "OTEL setting change", server_mode).await; } REQUEST_SIZE_LIMIT_SETTING => { if server_mode { tracing::info!("Request limit size change detected, killing server expecting to be restarted"); - spawn_graceful_killpill(tx, db, 30, "request size limit change").await; + spawn_graceful_killpill( + tx, + db, + 30, + "request size limit change", + server_mode, + ) + .await; } } SAML_METADATA_SETTING => { tracing::info!( "SAML metadata change detected, killing server expecting to be restarted" ); - spawn_graceful_killpill(tx, db, 30, "SAML metadata change").await; + spawn_graceful_killpill(tx, db, 30, "SAML metadata change", server_mode).await; } HUB_BASE_URL_SETTING => { if let Err(e) = reload_hub_base_url_setting(conn, server_mode).await { @@ -2183,12 +2209,7 @@ pub async fn run_workers( // #[cfg(tokio_unstable)] // let monitor = tokio_metrics::TaskMonitor::new(); - let ip = windmill_common::external_ip::get_ip() - .await - .unwrap_or_else(|e| { - tracing::warn!(error = e.to_string(), "failed to get external IP"); - "unretrievable IP".to_string() - }); + windmill_common::external_ip::resolve_ip_in_background(); let mut handles = Vec::with_capacity(num_workers as usize); @@ -2232,7 +2253,6 @@ pub async fn run_workers( let conn1 = wk_conf.conn.clone(); let worker_name = wk_conf.worker_name.clone(); WORKERS_NAMES.write().await.push(worker_name.clone()); - let ip = ip.clone(); let rx = killpill_rxs.pop().unwrap(); let tx = tx.clone(); let base_internal_url = base_internal_url.clone(); @@ -2249,7 +2269,6 @@ pub async fn run_workers( worker_name, i as u64, num_workers as u32, - &ip, rx, tx, &base_internal_url, @@ -2286,16 +2305,24 @@ pub async fn run_workers( /// then the sleep+kill is spawned in the background so the notification handler is not blocked. /// /// Falls back to drain-only delay if DB coordination fails. +/// +/// Only `server_mode` processes coordinate, on the strength of the worker case: a worker +/// group restarting costs queue latency rather than lost work, `v2_job_queue` being durable. +/// Were workers to take part, one could claim the `is_first` slot and leave every server +/// holding its shutdown open for a peer that serves no API traffic. async fn spawn_graceful_killpill( tx: &KillpillSender, db: &Pool, safety_margin_secs: u64, context: &str, + server_mode: bool, ) { // Minimum delay before any restart to let in-flight requests drain const DRAIN_DELAY_SECS: u64 = 3; - let (delay, is_first) = + let (delay, is_first) = if !server_mode { + (DRAIN_DELAY_SECS, true) + } else { match coordinate_restart_delay(db, safety_margin_secs, DRAIN_DELAY_SECS).await { Ok(r) => r, Err(e) => { @@ -2305,7 +2332,8 @@ async fn spawn_graceful_killpill( ); (DRAIN_DELAY_SECS, true) } - }; + } + }; tracing::info!( "Scheduling {context} graceful shutdown in {delay}s (first_to_restart={is_first})" diff --git a/backend/src/monitor.rs b/backend/src/monitor.rs index a9f747cfe5..93ad5945b8 100644 --- a/backend/src/monitor.rs +++ b/backend/src/monitor.rs @@ -12,7 +12,7 @@ use std::{ }; use chrono::{DateTime, NaiveDateTime, Utc}; -use futures::{stream::FuturesUnordered, StreamExt}; +use futures::{future::BoxFuture, stream::FuturesUnordered, StreamExt}; use serde::{de::DeserializeOwned, Deserialize}; use sqlx::{Pool, Postgres}; use tokio::{ @@ -37,7 +37,6 @@ use windmill_common::ee_oss::low_disk_alerts; #[cfg(feature = "enterprise")] use windmill_common::ee_oss::{jobs_waiting_alerts, worker_groups_alerts}; -#[cfg(feature = "oauth2")] use windmill_common::global_settings::OAUTH_SETTING; use windmill_common::otel_oss::{ otel_incr_zombie_delete_count, otel_incr_zombie_restart_count, otel_set_db_pool, @@ -52,13 +51,14 @@ use windmill_common::{ error, flow_status::{FlowStatus, FlowStatusModule}, global_settings::{ - AUDIT_LOG_RETENTION_DAYS_SETTING, BASE_URL_SETTING, BUNFIG_INSTALL_SCOPES_SETTING, - BUN_INSTALL_MIN_RELEASE_AGE_SETTING, CONCURRENCY_KEY_MAX_QUEUED_SETTING, - CRITICAL_ALERTS_ON_DB_OVERSIZE_SETTING, CRITICAL_ALERTS_ON_TOKEN_EXPIRY_SETTING, - CRITICAL_ALERT_MUTE_UI_SETTING, CRITICAL_ERROR_CHANNELS_SETTING, - DEFAULT_TAGS_PER_WORKSPACE_SETTING, DEFAULT_TAGS_WORKSPACES_SETTING, - DISABLE_PASSWORD_LOGIN, DISABLE_PASSWORD_LOGIN_SETTING, EXPOSE_DEBUG_METRICS_SETTING, - EXPOSE_METRICS_SETTING, EXTRA_PIP_INDEX_URL_SETTING, + get_or_create_jwt_secret, load_value_from_global_settings, + AUDIT_LOG_RETENTION_DAYS_SETTING, BASE_URL_SETTING, + BUNFIG_INSTALL_SCOPES_SETTING, BUN_INSTALL_MIN_RELEASE_AGE_SETTING, + CONCURRENCY_KEY_MAX_QUEUED_SETTING, CRITICAL_ALERTS_ON_DB_OVERSIZE_SETTING, + CRITICAL_ALERTS_ON_TOKEN_EXPIRY_SETTING, CRITICAL_ALERT_MUTE_UI_SETTING, + CRITICAL_ERROR_CHANNELS_SETTING, CUSTOM_TAGS_SETTING, DEFAULT_TAGS_PER_WORKSPACE_SETTING, + DEFAULT_TAGS_WORKSPACES_SETTING, DISABLE_PASSWORD_LOGIN, DISABLE_PASSWORD_LOGIN_SETTING, + EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING, EXTRA_PIP_INDEX_URL_SETTING, FORK_WORKSPACE_TAG_APPEND_FORK_SUFFIX_SETTING, HUB_API_SECRET_SETTING, HUB_BASE_URL_SETTING, INSTANCE_PYTHON_VERSION_SETTING, JOB_DEFAULT_TIMEOUT_SECS_SETTING, JOB_ISOLATION_SETTING, JWT_SECRET_SETTING, KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, @@ -70,8 +70,8 @@ use windmill_common::{ RETENTION_PERIOD_SECS_SETTING, SAML_METADATA_SETTING, SANDBOX_IMAGE_CACHE_MAX_MB_SETTING, SANDBOX_IMAGE_DEFAULT_REGISTRY_SETTING, SANDBOX_IMAGE_MAX_SIZE_MB_SETTING, SANDBOX_IMAGE_PULL_POLICY_SETTING, SANDBOX_REGISTRY_AUTH_SETTING, SCIM_TOKEN_SETTING, - STORE_AUDIT_LOGS_S3_SETTING, TIMEOUT_WAIT_RESULT_SETTING, UV_EXCLUDE_NEWER_SETTING, - UV_INDEX_STRATEGY_SETTING, UV_PYTHON_INSTALL_MIRROR_SETTING, + SMTP_SETTING, STORE_AUDIT_LOGS_S3_SETTING, TIMEOUT_WAIT_RESULT_SETTING, + UV_EXCLUDE_NEWER_SETTING, UV_INDEX_STRATEGY_SETTING, UV_PYTHON_INSTALL_MIRROR_SETTING, WORKSPACE_FAIRNESS_DURATION_SECS_SETTING, WORKSPACE_FAIRNESS_ENABLED_SETTING, WORKSPACE_FAIRNESS_MAX_PERCENT_SETTING, WORKSPACE_FAIRNESS_MIN_TOTAL_SETTING, WORKSPACE_MAX_QUEUED_JOBS_SETTING, @@ -83,11 +83,11 @@ use windmill_common::{ server::load_smtp_config, tracing_init::JSON_FMT, users::truncate_token, - utils::{empty_as_none, now_from_db, rd_string, report_critical_error, Mode, HUB_API_SECRET}, + utils::{empty_as_none, now_from_db, report_critical_error, Mode, HUB_API_SECRET}, worker::{ load_env_vars, load_init_bash_from_env, load_periodic_bash_script_from_env, load_periodic_bash_script_interval_from_env, load_whitelist_env_vars_from_env, - load_worker_config, reload_custom_tags_setting, store_pull_query, + load_worker_config, store_pull_query, store_suspended_pull_query, Connection, WorkerConfig, CLOUD_HOSTED, CONCURRENCY_KEY_MAX_QUEUED, CONCURRENCY_KEY_MAX_QUEUED_DEFAULT, DEFAULT_TAGS_PER_WORKSPACE, DEFAULT_TAGS_WORKSPACES, FORK_WORKSPACE_TAG_APPEND_FORK_SUFFIX, INDEXER_CONFIG, @@ -225,6 +225,11 @@ lazy_static::lazy_static! { .unwrap_or(20); } +/// Load every setting this process cares about, at startup and on every full-reload tick. +/// +/// Reads are declared into a [`SettingsPass`] rather than issued one at a time, so the dozens +/// this pass makes cost a single fetch. See [`SettingsPass`] for what declaring buys and what +/// it requires of the order things are declared in. pub async fn initial_load( conn: &Connection, tx: KillpillSender, @@ -232,102 +237,134 @@ pub async fn initial_load( server_mode: bool, #[cfg(feature = "parquet")] disable_s3_store: bool, ) { - if let Err(e) = reload_base_url_setting(&conn).await { - tracing::error!("Error loading base url: {:?}", e) - } + let mut pass = SettingsPass::new(); + + pass.settings( + &[OAUTH_SETTING, BASE_URL_SETTING], + false, + move |mut v| async move { + if let Err(e) = apply_base_url_setting( + conn, + v.remove(OAUTH_SETTING).flatten(), + v.remove(BASE_URL_SETTING).flatten(), + ) + .await + { + tracing::error!("Error loading base url: {:?}", e) + } + }, + ); + + pass.setting(CRITICAL_ERROR_CHANNELS_SETTING, false, |v| async move { + apply_critical_error_channels_setting(v) + }); + + pass.setting(EXPOSE_METRICS_SETTING, true, |v| async move { + apply_metrics_enabled(v) + }); + pass.setting(EXPOSE_DEBUG_METRICS_SETTING, true, |v| async move { + apply_metrics_debug_enabled(v) + }); + pass.setting(CRITICAL_ALERT_MUTE_UI_SETTING, true, |v| async move { + apply_critical_alert_mute_ui_setting(v) + }); + pass.setting( + CRITICAL_ALERTS_ON_TOKEN_EXPIRY_SETTING, + true, + |v| async move { apply_critical_alerts_on_token_expiry_setting(v) }, + ); if let Some(db) = conn.as_sql() { - if let Err(e) = reload_critical_error_channels_setting(&db).await { - tracing::error!("Could loading critical error emails setting: {:?}", e); - } - } - - if let Err(e) = load_metrics_enabled(conn).await { - tracing::error!("Error loading expose metrics: {e:#}"); - } - - if let Err(e) = load_metrics_debug_enabled(conn).await { - tracing::error!("Error loading expose debug metrics: {e:#}"); - } - - if let Err(e) = reload_critical_alert_mute_ui_setting(conn).await { - tracing::error!("Error loading critical alert mute ui setting: {e:#}"); - } - - if let Err(e) = reload_critical_alerts_on_token_expiry_setting(conn).await { - tracing::error!("Error loading critical alerts on token expiry setting: {e:#}"); - } - - if let Some(db) = conn.as_sql() { - if let Err(e) = load_tag_per_workspace_enabled(db).await { - tracing::error!("Error loading default tag per workpsace: {e:#}"); - } - - if let Err(e) = load_tag_per_workspace_workspaces(db).await { - tracing::error!("Error loading default tag per workpsace workspaces: {e:#}"); - } - - if let Err(e) = load_fork_workspace_tag_append_fork_suffix(db).await { - tracing::error!("Error loading fork workspace tag append fork suffix: {e:#}"); - } - - if let Err(e) = load_preview_tags_override(db).await { - tracing::error!("Error loading preview tags override: {e:#}"); - } + pass.setting(DEFAULT_TAGS_PER_WORKSPACE_SETTING, false, |v| async move { + apply_tag_per_workspace_enabled(v) + }); + pass.setting(DEFAULT_TAGS_WORKSPACES_SETTING, false, |v| async move { + apply_tag_per_workspace_workspaces(v) + }); + pass.setting( + FORK_WORKSPACE_TAG_APPEND_FORK_SUFFIX_SETTING, + false, + |v| async move { apply_fork_workspace_tag_append_fork_suffix(v) }, + ); + pass.setting(PREVIEW_TAGS_OVERRIDE_SETTING, false, |v| async move { + apply_preview_tags_override(v) + }); // Load per-workspace retention overrides before the first cleanup tick so a fresh server // never sweeps globally without honoring configured longer-retention workspaces. - if let Err(e) = load_retention_period_overrides(db).await { - tracing::error!("Error loading per-workspace retention overrides: {e:#}"); - } + pass.action(async move { + if let Err(e) = load_retention_period_overrides(db).await { + tracing::error!("Error loading per-workspace retention overrides: {e:#}"); + } + }); - // Workspace fairness (cloud-only). Load the percentage/duration/min knobs - // *before* the enabled flag so that `load_workspace_fairness_enabled` reads - // current values when re-storing the pull queries. - if let Err(e) = load_workspace_fairness_max_percent(db).await { - tracing::error!("Error loading workspace fairness max percent: {e:#}"); - } - if let Err(e) = load_workspace_fairness_duration_secs(db).await { - tracing::error!("Error loading workspace fairness duration secs: {e:#}"); - } - if let Err(e) = load_workspace_fairness_min_total(db).await { - tracing::error!("Error loading workspace fairness min total: {e:#}"); - } - if let Err(e) = load_workspace_fairness_enabled(db).await { - tracing::error!("Error loading workspace fairness enabled: {e:#}"); - } + // Workspace fairness (cloud-only). The percentage/duration/min knobs apply + // *before* the enabled flag so that `apply_workspace_fairness_enabled` reads + // current values when re-storing the pull queries, which declaration order gives us. + pass.setting( + WORKSPACE_FAIRNESS_MAX_PERCENT_SETTING, + false, + |v| async move { apply_workspace_fairness_max_percent(v) }, + ); + pass.setting( + WORKSPACE_FAIRNESS_DURATION_SECS_SETTING, + false, + |v| async move { apply_workspace_fairness_duration_secs(v) }, + ); + pass.setting( + WORKSPACE_FAIRNESS_MIN_TOTAL_SETTING, + false, + |v| async move { apply_workspace_fairness_min_total(v) }, + ); + pass.setting(WORKSPACE_FAIRNESS_ENABLED_SETTING, false, |v| { + apply_workspace_fairness_enabled(v) + }); - // Only the cloud reads this cap, so don't spend a query loading it anywhere else. + // Only the cloud reads these caps, so don't ask for them anywhere else. if *CLOUD_HOSTED { - if let Err(e) = load_concurrency_key_max_queued(db).await { - tracing::error!("Error loading concurrency key max queued: {e:#}"); - } - if let Err(e) = load_workspace_max_queued_jobs(db).await { - tracing::error!("Error loading workspace max queued jobs: {e:#}"); - } + pass.setting(CONCURRENCY_KEY_MAX_QUEUED_SETTING, false, |v| async move { + apply_concurrency_key_max_queued(v) + }); + pass.setting(WORKSPACE_MAX_QUEUED_JOBS_SETTING, false, |v| async move { + apply_workspace_max_queued_jobs(v) + }); } } if server_mode { if let Some(db) = conn.as_sql() { - load_require_preexisting_user(db).await; - load_disable_password_login(db).await; - if let Err(e) = reload_critical_alerts_on_db_oversize(db).await { - tracing::error!( - "Error reloading critical alerts on db oversize setting: {:?}", - e - ) - } - windmill_common::min_version::store_min_keep_alive_version(db).await; - reload_instance_events_webhook_setting(db).await; + pass.setting( + REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, + false, + |v| async move { apply_require_preexisting_user(v) }, + ); + pass.setting(DISABLE_PASSWORD_LOGIN_SETTING, false, |v| async move { + apply_disable_password_login(v) + }); + pass.action(async move { + if let Err(e) = reload_critical_alerts_on_db_oversize(db).await { + tracing::error!( + "Error reloading critical alerts on db oversize setting: {:?}", + e + ) + } + }); + pass.action(windmill_common::min_version::store_min_keep_alive_version(db)); + pass.setting( + windmill_common::global_settings::INSTANCE_EVENTS_WEBHOOK_SETTING, + false, + |v| async move { apply_instance_events_webhook_setting(v) }, + ); } } if worker_mode { - load_keep_job_dir(conn).await; + pass.setting(KEEP_JOB_DIR_SETTING, true, |v| async move { + apply_keep_job_dir(v) + }); match conn { Connection::Sql(db) => { - reload_worker_config(&db, tx, false).await; + pass.action(reload_worker_config(&db, tx, false)); } Connection::Http(_) => { // TODO: reload worker config from http @@ -359,61 +396,119 @@ pub async fn initial_load( } } - if let Err(e) = reload_hub_base_url_setting(conn, server_mode).await { - tracing::error!("Error reloading hub base url: {:?}", e) - } + pass.setting(HUB_BASE_URL_SETTING, true, move |v| async move { + if let Err(e) = apply_hub_base_url_setting(conn, server_mode, v).await { + tracing::error!("Error reloading hub base url: {:?}", e) + } + }); if let Some(db) = conn.as_sql() { - if let Err(e) = reload_jwt_secret_setting(db).await { - tracing::error!("Could not reload jwt secret setting: {:?}", e); - } + pass.setting(JWT_SECRET_SETTING, false, move |v| async move { + if let Err(e) = apply_jwt_secret_setting(db, v).await { + tracing::error!("Could not reload jwt secret setting: {:?}", e); + } + }); - if let Err(e) = reload_custom_tags_setting(db).await { - tracing::error!("Error reloading custom tags: {:?}", e) - } - - if let Err(e) = reload_app_workspaced_route_setting(db).await { - tracing::error!("Error reloading app workspaced route: {:?}", e) - } - - if let Err(e) = reload_http_route_workspaced_route_setting(db).await { - tracing::error!("Error reloading http route workspaced route: {:?}", e) - } + pass.setting(CUSTOM_TAGS_SETTING, false, |v| async move { + windmill_common::worker::apply_custom_tags_setting(v) + }); + pass.setting(APP_WORKSPACED_ROUTE_SETTING, false, |v| async move { + apply_app_workspaced_route_setting(v) + }); + pass.setting( + HTTP_ROUTE_WORKSPACED_ROUTE_SETTING, + false, + move |v| async move { + if let Err(e) = apply_http_route_workspaced_route_setting(db, v).await { + tracing::error!("Error reloading http route workspaced route: {:?}", e) + } + }, + ); } + // A step rather than a plain await: an AWS OIDC store mints its first token against an + // issuer built from `BASE_URL` (`oidc_ee.rs`), and with `OTEL_ENVIRONMENT` set nothing + // loads that before this pass does, so running ahead of the applier would sign with the + // unset default and fall back to the 10s retry. #[cfg(feature = "parquet")] if !disable_s3_store { if let Some(db) = conn.as_sql() { let db2 = db.clone(); - match reload_object_store_setting(db).await { - ObjectStoreReload::Later => { - tokio::spawn(async move { - tokio::time::sleep(Duration::from_secs(10)).await; - match reload_object_store_setting(&db2).await { - ObjectStoreReload::Later => { - tracing::error!("Giving up on loading object store setting"); + pass.action(async move { + match reload_object_store_setting(db).await { + ObjectStoreReload::Later => { + tokio::spawn(async move { + tokio::time::sleep(Duration::from_secs(10)).await; + match reload_object_store_setting(&db2).await { + ObjectStoreReload::Later => { + tracing::error!("Giving up on loading object store setting"); + } + ObjectStoreReload::Never => { + tracing::info!("Object store setting successfully loaded"); + } } - ObjectStoreReload::Never => { - tracing::info!("Object store setting successfully loaded"); - } - } - }); + }); + } + ObjectStoreReload::Never => (), } - ObjectStoreReload::Never => (), - } + }); } } if let Some(db) = conn.as_sql() { - reload_smtp_config(db).await; + let _ = db; + pass.setting(SMTP_SETTING, false, |v| async move { + tracing::info!("Reloading smtp config..."); + SMTP_CONFIG.store(std::sync::Arc::new( + windmill_common::server::parse_smtp_config(v), + )); + }); } - reload_hub_api_secret_setting(&conn).await; + pass.option_setting_with( + HUB_API_SECRET_SETTING, + "HUB_API_SECRET", + |v: Option| async move { HUB_API_SECRET.store(std::sync::Arc::new(v)) }, + ); if server_mode { - reload_retention_period_setting(&conn).await; - reload_audit_log_retention_days_setting(&conn).await; - reload_store_audit_logs_s3_setting(&conn).await; + pass.setting(RETENTION_PERIOD_SECS_SETTING, true, |v| async move { + JOB_RETENTION_SECS.store( + parse_setting_value::( + v, + RETENTION_PERIOD_SECS_SETTING, + "JOB_RETENTION_SECS", + 60 * 60 * 24 * 30, + |x| x, + ), + Ordering::Relaxed, + ) + }); + pass.setting(AUDIT_LOG_RETENTION_DAYS_SETTING, true, |v| async move { + AUDIT_LOG_RETENTION_DAYS.store( + // 0 means use default: 365 for EE, 14 for CE + parse_setting_value::( + v, + AUDIT_LOG_RETENTION_DAYS_SETTING, + "AUDIT_LOG_RETENTION_DAYS", + 0, + |x| x, + ), + Ordering::Relaxed, + ) + }); + pass.setting(STORE_AUDIT_LOGS_S3_SETTING, true, |v| async move { + STORE_AUDIT_LOGS_S3.store( + parse_setting_value::( + v, + STORE_AUDIT_LOGS_S3_SETTING, + "STORE_AUDIT_LOGS_S3", + false, + |x| x, + ), + Ordering::Relaxed, + ) + }); // Env-var enable has no settings-row xmin and no runtime enable event; // anchor the export cursor at startup so rows committed before the // first export tick are not skipped (no-op when a settings row exists @@ -421,66 +516,177 @@ pub async fn initial_load( // Enterprise feature; the core logic lives in `crate::ee` (OSS gets a // no-op), gated here on a valid Enterprise license. #[cfg(feature = "parquet")] - if STORE_AUDIT_LOGS_S3.load(std::sync::atomic::Ordering::Relaxed) - && matches!( - windmill_common::ee_oss::get_license_plan().await, - windmill_common::ee_oss::LicensePlan::Enterprise - ) - { - if let Some(db) = conn.as_sql() { - crate::ee_oss::anchor_audit_logs_s3_checkpoint_env_var(&db).await; - } - } - reload_request_size(&conn).await; - reload_saml_metadata_setting(&conn).await; - reload_scim_token_setting(&conn).await; - - // Ensure audit partitions exist before any requests arrive if let Some(db) = conn.as_sql() { - manage_audit_partitions(&db, audit_log_retention_days().await).await; + pass.action(async move { + if STORE_AUDIT_LOGS_S3.load(std::sync::atomic::Ordering::Relaxed) + && matches!( + windmill_common::ee_oss::get_license_plan().await, + windmill_common::ee_oss::LicensePlan::Enterprise + ) + { + crate::ee_oss::anchor_audit_logs_s3_checkpoint_env_var(&db).await; + } + }); + } + pass.required_setting( + REQUEST_SIZE_LIMIT_SETTING, + "REQUEST_SIZE_LIMIT", + DEFAULT_BODY_LIMIT, + REQUEST_SIZE_LIMIT.clone(), + |x| x.mul(1024 * 1024), + ); + pass.option_setting( + SAML_METADATA_SETTING, + "SAML_METADATA", + SAML_METADATA.clone(), + ); + pass.option_setting(SCIM_TOKEN_SETTING, "SCIM_TOKEN", SCIM_TOKEN.clone()); + + // Ensure audit partitions exist before any requests arrive. A step rather than a + // plain await: it drops partitions past `audit_log_retention_days`, so running it + // before that setting is applied would sweep with the compile-time default. + if let Some(db) = conn.as_sql() { + pass.action(async move { + manage_audit_partitions(&db, audit_log_retention_days().await).await + }); } } if worker_mode { - reload_job_default_timeout_setting(&conn).await; - reload_job_isolation_setting(&conn).await; - reload_nsjail_tmpfs_size_setting(&conn).await; - reload_nsjail_tmp_backing_setting(&conn).await; - reload_sandbox_image_max_size_setting(&conn).await; - reload_sandbox_image_cache_max_setting(&conn).await; - reload_sandbox_image_pull_policy_setting(&conn).await; - reload_sandbox_image_default_registry_setting(&conn).await; - reload_sandbox_registry_auth_setting(&conn).await; - reload_extra_pip_index_url_setting(&conn).await; - reload_pip_index_url_setting(&conn).await; - reload_uv_index_strategy_setting(&conn).await; - reload_uv_exclude_newer_setting(&conn).await; - reload_uv_python_install_mirror_setting(&conn).await; - reload_bun_install_min_release_age_setting(&conn).await; - reload_npm_config_registry_setting(&conn).await; - reload_bunfig_install_scopes_setting(&conn).await; - reload_npmrc_setting(&conn).await; - reload_instance_python_version_setting(&conn).await; - reload_nuget_config_setting(&conn).await; - reload_powershell_repo_url_setting(&conn).await; - reload_powershell_repo_pat_setting(&conn).await; - reload_maven_repos_setting(&conn).await; - reload_maven_settings_xml_setting(&conn).await; - reload_no_default_maven_setting(&conn).await; - reload_ruby_repos_setting(&conn).await; - reload_cargo_registries_setting(&conn).await; - reload_workspace_registries_setting(&conn).await; + use windmill_common::global_settings as gs; + pass.option_setting( + JOB_DEFAULT_TIMEOUT_SECS_SETTING, + "JOB_DEFAULT_TIMEOUT_SECS", + JOB_DEFAULT_TIMEOUT.clone(), + ); + pass.setting(JOB_ISOLATION_SETTING, true, apply_job_isolation_setting); + pass.option_setting( + NSJAIL_TMPFS_SIZE_MB_SETTING, + "NSJAIL_TMPFS_SIZE_MB", + NSJAIL_TMPFS_SIZE_MB.clone(), + ); + pass.option_setting( + NSJAIL_TMP_BACKING_SETTING, + "NSJAIL_TMP_BACKING", + NSJAIL_TMP_BACKING.clone(), + ); + pass.option_setting( + SANDBOX_IMAGE_MAX_SIZE_MB_SETTING, + "SANDBOX_IMAGE_MAX_SIZE_MB", + SANDBOX_IMAGE_MAX_SIZE_MB.clone(), + ); + pass.option_setting( + SANDBOX_IMAGE_CACHE_MAX_MB_SETTING, + "SANDBOX_IMAGE_CACHE_MAX_MB", + SANDBOX_IMAGE_CACHE_MAX_MB.clone(), + ); + pass.option_setting( + SANDBOX_IMAGE_PULL_POLICY_SETTING, + "SANDBOX_IMAGE_PULL_POLICY", + SANDBOX_IMAGE_PULL_POLICY.clone(), + ); + pass.option_setting( + SANDBOX_IMAGE_DEFAULT_REGISTRY_SETTING, + "SANDBOX_IMAGE_DEFAULT_REGISTRY", + SANDBOX_IMAGE_DEFAULT_REGISTRY.clone(), + ); + pass.setting( + SANDBOX_REGISTRY_AUTH_SETTING, + true, + apply_sandbox_registry_auth_setting, + ); + pass.option_setting( + EXTRA_PIP_INDEX_URL_SETTING, + "PIP_EXTRA_INDEX_URL", + PIP_EXTRA_INDEX_URL.clone(), + ); + pass.option_setting( + PIP_INDEX_URL_SETTING, + "PIP_INDEX_URL", + PIP_INDEX_URL.clone(), + ); + pass.option_setting( + UV_INDEX_STRATEGY_SETTING, + "UV_INDEX_STRATEGY", + UV_INDEX_STRATEGY.clone(), + ); + pass.option_setting( + UV_EXCLUDE_NEWER_SETTING, + "UV_EXCLUDE_NEWER", + UV_EXCLUDE_NEWER.clone(), + ); + pass.option_setting( + UV_PYTHON_INSTALL_MIRROR_SETTING, + "UV_PYTHON_INSTALL_MIRROR", + UV_PYTHON_INSTALL_MIRROR.clone(), + ); + pass.option_setting( + BUN_INSTALL_MIN_RELEASE_AGE_SETTING, + "BUN_INSTALL_MIN_RELEASE_AGE", + BUN_INSTALL_MIN_RELEASE_AGE.clone(), + ); + pass.option_setting( + NPM_CONFIG_REGISTRY_SETTING, + "NPM_CONFIG_REGISTRY", + NPM_CONFIG_REGISTRY.clone(), + ); + pass.option_setting( + BUNFIG_INSTALL_SCOPES_SETTING, + "BUNFIG_INSTALL_SCOPES", + BUNFIG_INSTALL_SCOPES.clone(), + ); + pass.option_setting(NPMRC_SETTING, "NPMRC", NPMRC.clone()); + pass.option_setting( + INSTANCE_PYTHON_VERSION_SETTING, + "INSTANCE_PYTHON_VERSION", + INSTANCE_PYTHON_VERSION.clone(), + ); + pass.option_setting(NUGET_CONFIG_SETTING, "NUGET_CONFIG", NUGET_CONFIG.clone()); + pass.option_setting( + POWERSHELL_REPO_URL_SETTING, + "POWERSHELL_REPO_URL", + POWERSHELL_REPO_URL.clone(), + ); + pass.option_setting( + POWERSHELL_REPO_PAT_SETTING, + "POWERSHELL_REPO_PAT", + POWERSHELL_REPO_PAT.clone(), + ); + pass.option_setting(gs::MAVEN_REPOS_SETTING, "MAVEN_REPOS", MAVEN_REPOS.clone()); + pass.option_setting( + gs::MAVEN_SETTINGS_XML_SETTING, + "MAVEN_SETTINGS_XML", + MAVEN_SETTINGS_XML.clone(), + ); + pass.action(write_maven_settings_xml()); + pass.setting(gs::NO_DEFAULT_MAVEN_SETTING, true, |v| async move { + apply_no_default_maven_setting(v) + }); + pass.url_list_setting( + gs::RUBY_REPOS_SETTING, + "RUBY_REPOS", + windmill_worker::RUBY_REPOS.clone(), + ); + pass.option_setting( + gs::CARGO_REGISTRIES_SETTING, + "CARGO_REGISTRIES", + CARGO_REGISTRIES.clone(), + ); + pass.setting( + gs::WORKSPACE_REGISTRIES_SETTING, + true, + apply_workspace_registries_setting, + ); } + + pass.run(conn).await; } -pub async fn load_metrics_enabled(conn: &Connection) -> error::Result<()> { - let metrics_enabled = - load_value_from_global_settings_with_conn(conn, EXPOSE_METRICS_SETTING, true).await; - match metrics_enabled { - Ok(Some(serde_json::Value::Bool(t))) => METRICS_ENABLED.store(t, Ordering::Relaxed), - _ => (), - }; - Ok(()) + +pub fn apply_metrics_enabled(value: Option) { + if let Some(serde_json::Value::Bool(t)) = value { + METRICS_ENABLED.store(t, Ordering::Relaxed) + } } #[derive(serde::Deserialize)] @@ -564,23 +770,26 @@ pub async fn load_otel(db: &DB) { } pub async fn load_tag_per_workspace_enabled(db: &DB) -> error::Result<()> { - let metrics_enabled = - load_value_from_global_settings(db, DEFAULT_TAGS_PER_WORKSPACE_SETTING).await; - - match metrics_enabled { - Ok(Some(serde_json::Value::Bool(t))) => { - DEFAULT_TAGS_PER_WORKSPACE.store(t, Ordering::Relaxed) - } - _ => (), - }; + let v = load_value_from_global_settings(db, DEFAULT_TAGS_PER_WORKSPACE_SETTING).await?; + apply_tag_per_workspace_enabled(v); Ok(()) } -pub async fn load_tag_per_workspace_workspaces(db: &DB) -> error::Result<()> { - let workspaces = load_value_from_global_settings(db, DEFAULT_TAGS_WORKSPACES_SETTING).await; +pub fn apply_tag_per_workspace_enabled(value: Option) { + if let Some(serde_json::Value::Bool(t)) = value { + DEFAULT_TAGS_PER_WORKSPACE.store(t, Ordering::Relaxed) + } +} - match workspaces { - Ok(Some(serde_json::Value::Array(t))) => { +pub async fn load_tag_per_workspace_workspaces(db: &DB) -> error::Result<()> { + let v = load_value_from_global_settings(db, DEFAULT_TAGS_WORKSPACES_SETTING).await?; + apply_tag_per_workspace_workspaces(v); + Ok(()) +} + +pub fn apply_tag_per_workspace_workspaces(value: Option) { + match value { + Some(serde_json::Value::Array(t)) => { let workspaces = t .iter() .filter_map(|x| x.as_str()) @@ -588,24 +797,25 @@ pub async fn load_tag_per_workspace_workspaces(db: &DB) -> error::Result<()> { .collect::>(); DEFAULT_TAGS_WORKSPACES.store(std::sync::Arc::new(Some(workspaces))); } - Ok(None) => { + None => { DEFAULT_TAGS_WORKSPACES.store(std::sync::Arc::new(None)); } _ => (), }; - Ok(()) } pub async fn load_preview_tags_override(db: &DB) -> error::Result<()> { - let value = load_value_from_global_settings(db, PREVIEW_TAGS_OVERRIDE_SETTING).await; - - match value { - Ok(Some(serde_json::Value::Bool(t))) => PREVIEW_TAGS_OVERRIDE.store(t, Ordering::Relaxed), - _ => (), - }; + let v = load_value_from_global_settings(db, PREVIEW_TAGS_OVERRIDE_SETTING).await?; + apply_preview_tags_override(v); Ok(()) } +pub fn apply_preview_tags_override(value: Option) { + if let Some(serde_json::Value::Bool(t)) = value { + PREVIEW_TAGS_OVERRIDE.store(t, Ordering::Relaxed) + } +} + // Upper bound on the duration window. Postgres `make_interval(secs => $1::int4)` is the consumer // downstream, so this stays comfortably below `i32::MAX` and the subsequent `u32 -> i32` cast in // `workspace_fairness::refresh_overloaded` cannot wrap into a negative interval (which would @@ -635,12 +845,17 @@ pub async fn load_workspace_fairness_enabled(db: &DB) -> error::Result<()> { // atomic untouched rather than silently toggling the feature off across the whole cluster // (which would also trigger an unnecessary `store_pull_query` rebuild — exactly when DB load // is probably highest). - let new_enabled = - match load_value_from_global_settings(db, WORKSPACE_FAIRNESS_ENABLED_SETTING).await? { - Some(serde_json::Value::Bool(t)) => t, - // Setting unset / non-bool → explicit off. - _ => false, - }; + let v = load_value_from_global_settings(db, WORKSPACE_FAIRNESS_ENABLED_SETTING).await?; + apply_workspace_fairness_enabled(v).await; + Ok(()) +} + +pub async fn apply_workspace_fairness_enabled(value: Option) { + let new_enabled = match value { + Some(serde_json::Value::Bool(t)) => t, + // Setting unset / non-bool → explicit off. + _ => false, + }; let prev = WORKSPACE_FAIRNESS_ENABLED.swap(new_enabled, Ordering::Relaxed); // Re-store the pull queries so the fairness variants appear/disappear in // lockstep with the toggle. @@ -648,18 +863,23 @@ pub async fn load_workspace_fairness_enabled(db: &DB) -> error::Result<()> { let wc = windmill_common::worker::WORKER_CONFIG.load_full(); store_pull_query(&wc).await; } - Ok(()) } pub async fn load_workspace_fairness_max_percent(db: &DB) -> error::Result<()> { - // Distinguish three outcomes: - // - `Err(_)`: transient DB issue. Leave the atomic alone (don't clobber a known-good value - // because of a network blip during a notify-event propagation). - // - `Ok(None)` or `Ok(Some(invalid))`: setting is unset / explicitly cleared / corrupt. - // Restore the default so a deletion via the admin UI actually takes effect at runtime - // instead of leaving the stale in-memory value pinned until restart. - // - `Ok(Some(valid))`: clamp and store. - match load_value_from_global_settings(db, WORKSPACE_FAIRNESS_MAX_PERCENT_SETTING).await? { + let v = load_value_from_global_settings(db, WORKSPACE_FAIRNESS_MAX_PERCENT_SETTING).await?; + apply_workspace_fairness_max_percent(v); + Ok(()) +} + +// Distinguish three outcomes: +// - `Err(_)`: transient DB issue. Leave the atomic alone (don't clobber a known-good value +// because of a network blip during a notify-event propagation). +// - `Ok(None)` or `Ok(Some(invalid))`: setting is unset / explicitly cleared / corrupt. +// Restore the default so a deletion via the admin UI actually takes effect at runtime +// instead of leaving the stale in-memory value pinned until restart. +// - `Ok(Some(valid))`: clamp and store. +pub fn apply_workspace_fairness_max_percent(value: Option) { + match value { Some(serde_json::Value::Number(n)) => { let v = n .as_u64() @@ -672,12 +892,17 @@ pub async fn load_workspace_fairness_max_percent(db: &DB) -> error::Result<()> { .store(WORKSPACE_FAIRNESS_MAX_PERCENT_DEFAULT, Ordering::Relaxed); } } - Ok(()) } pub async fn load_workspace_fairness_duration_secs(db: &DB) -> error::Result<()> { - // See `load_workspace_fairness_max_percent` for the Err / None / invalid policy. - match load_value_from_global_settings(db, WORKSPACE_FAIRNESS_DURATION_SECS_SETTING).await? { + let v = load_value_from_global_settings(db, WORKSPACE_FAIRNESS_DURATION_SECS_SETTING).await?; + apply_workspace_fairness_duration_secs(v); + Ok(()) +} + +// See `load_workspace_fairness_max_percent` for the Err / None / invalid policy. +pub fn apply_workspace_fairness_duration_secs(value: Option) { + match value { Some(serde_json::Value::Number(n)) => { // Clamp to the safe range before narrowing. The downstream `u32 -> i32` cast in // `workspace_fairness::refresh_overloaded` makes any value above `i32::MAX` toxic @@ -693,12 +918,17 @@ pub async fn load_workspace_fairness_duration_secs(db: &DB) -> error::Result<()> .store(WORKSPACE_FAIRNESS_DURATION_SECS_DEFAULT, Ordering::Relaxed); } } - Ok(()) } pub async fn load_workspace_fairness_min_total(db: &DB) -> error::Result<()> { - // See `load_workspace_fairness_max_percent` for the Err / None / invalid policy. - match load_value_from_global_settings(db, WORKSPACE_FAIRNESS_MIN_TOTAL_SETTING).await? { + let v = load_value_from_global_settings(db, WORKSPACE_FAIRNESS_MIN_TOTAL_SETTING).await?; + apply_workspace_fairness_min_total(v); + Ok(()) +} + +// See `load_workspace_fairness_max_percent` for the Err / None / invalid policy. +pub fn apply_workspace_fairness_min_total(value: Option) { + match value { Some(serde_json::Value::Number(n)) => { // Clamp before narrowing — same reasoning as `_duration_secs`, just for the // counting threshold rather than the interval. @@ -713,12 +943,17 @@ pub async fn load_workspace_fairness_min_total(db: &DB) -> error::Result<()> { .store(WORKSPACE_FAIRNESS_MIN_TOTAL_DEFAULT, Ordering::Relaxed); } } - Ok(()) } pub async fn load_concurrency_key_max_queued(db: &DB) -> error::Result<()> { - // See `load_workspace_fairness_max_percent` for the Err / None / invalid policy. - match load_value_from_global_settings(db, CONCURRENCY_KEY_MAX_QUEUED_SETTING).await? { + let v = load_value_from_global_settings(db, CONCURRENCY_KEY_MAX_QUEUED_SETTING).await?; + apply_concurrency_key_max_queued(v); + Ok(()) +} + +// See `load_workspace_fairness_max_percent` for the Err / None / invalid policy. +pub fn apply_concurrency_key_max_queued(value: Option) { + match value { Some(serde_json::Value::Number(n)) => { // `0` is a meaningful value here (disable the cap), so unlike the fairness knobs // the lower bound is 0 rather than 1. @@ -746,7 +981,6 @@ pub async fn load_concurrency_key_max_queued(db: &DB) -> error::Result<()> { CONCURRENCY_KEY_MAX_QUEUED.store(CONCURRENCY_KEY_MAX_QUEUED_DEFAULT, Ordering::Relaxed); } } - Ok(()) } pub async fn load_workspace_max_queued_jobs(db: &DB) -> error::Result<()> { @@ -754,8 +988,14 @@ pub async fn load_workspace_max_queued_jobs(db: &DB) -> error::Result<()> { if !*CLOUD_HOSTED { return Ok(()); } - // Same Err / None / invalid policy as load_concurrency_key_max_queued: 0 disables. - match load_value_from_global_settings(db, WORKSPACE_MAX_QUEUED_JOBS_SETTING).await? { + let v = load_value_from_global_settings(db, WORKSPACE_MAX_QUEUED_JOBS_SETTING).await?; + apply_workspace_max_queued_jobs(v); + Ok(()) +} + +/// Same Err / None / invalid policy as [`apply_concurrency_key_max_queued`]: 0 disables. +pub fn apply_workspace_max_queued_jobs(value: Option) { + match value { Some(serde_json::Value::Number(n)) => { let v = n .as_u64() @@ -779,52 +1019,67 @@ pub async fn load_workspace_max_queued_jobs(db: &DB) -> error::Result<()> { WORKSPACE_MAX_QUEUED_JOBS.store(WORKSPACE_MAX_QUEUED_JOBS_DEFAULT, Ordering::Relaxed); } } - Ok(()) } pub async fn load_fork_workspace_tag_append_fork_suffix(db: &DB) -> error::Result<()> { - let value = - load_value_from_global_settings(db, FORK_WORKSPACE_TAG_APPEND_FORK_SUFFIX_SETTING).await; - - match value { - Ok(Some(serde_json::Value::Bool(t))) => { - FORK_WORKSPACE_TAG_APPEND_FORK_SUFFIX.store(t, Ordering::Relaxed) - } - Ok(None) => FORK_WORKSPACE_TAG_APPEND_FORK_SUFFIX.store(false, Ordering::Relaxed), - _ => (), - }; + let v = + load_value_from_global_settings(db, FORK_WORKSPACE_TAG_APPEND_FORK_SUFFIX_SETTING).await?; + apply_fork_workspace_tag_append_fork_suffix(v); Ok(()) } +pub fn apply_fork_workspace_tag_append_fork_suffix(value: Option) { + match value { + Some(serde_json::Value::Bool(t)) => { + FORK_WORKSPACE_TAG_APPEND_FORK_SUFFIX.store(t, Ordering::Relaxed) + } + None => FORK_WORKSPACE_TAG_APPEND_FORK_SUFFIX.store(false, Ordering::Relaxed), + _ => (), + }; +} + pub async fn reload_critical_alert_mute_ui_setting(conn: &Connection) -> error::Result<()> { - if let Ok(Some(serde_json::Value::Bool(t))) = - load_value_from_global_settings_with_conn(conn, CRITICAL_ALERT_MUTE_UI_SETTING, true).await - { + let v = + load_value_from_global_settings_with_conn(conn, CRITICAL_ALERT_MUTE_UI_SETTING, true).await?; + apply_critical_alert_mute_ui_setting(v); + Ok(()) +} + +pub fn apply_critical_alert_mute_ui_setting(value: Option) { + if let Some(serde_json::Value::Bool(t)) = value { CRITICAL_ALERT_MUTE_UI_ENABLED.store(t, Ordering::Relaxed); } - Ok(()) } pub async fn reload_critical_alerts_on_token_expiry_setting( conn: &Connection, ) -> error::Result<()> { - if let Ok(Some(serde_json::Value::Bool(t))) = load_value_from_global_settings_with_conn( + let v = load_value_from_global_settings_with_conn( conn, CRITICAL_ALERTS_ON_TOKEN_EXPIRY_SETTING, true, ) - .await - { - CRITICAL_ALERTS_ON_TOKEN_EXPIRY.store(t, Ordering::Relaxed); - } + .await?; + apply_critical_alerts_on_token_expiry_setting(v); Ok(()) } +pub fn apply_critical_alerts_on_token_expiry_setting(value: Option) { + if let Some(serde_json::Value::Bool(t)) = value { + CRITICAL_ALERTS_ON_TOKEN_EXPIRY.store(t, Ordering::Relaxed); + } +} + pub async fn load_metrics_debug_enabled(conn: &Connection) -> error::Result<()> { - let metrics_enabled = - load_value_from_global_settings_with_conn(conn, EXPOSE_DEBUG_METRICS_SETTING, true).await; - match metrics_enabled { - Ok(Some(serde_json::Value::Bool(t))) => { + let v = + load_value_from_global_settings_with_conn(conn, EXPOSE_DEBUG_METRICS_SETTING, true).await?; + apply_metrics_debug_enabled(v); + Ok(()) +} + +pub fn apply_metrics_debug_enabled(value: Option) { + match value { + Some(serde_json::Value::Bool(t)) => { METRICS_DEBUG_ENABLED.store(t, Ordering::Relaxed); //_RJEM_MALLOC_CONF=prof:true,prof_active:false,lg_prof_interval:30,lg_prof_sample:21,prof_prefix:/tmp/jeprof #[cfg(all(not(target_env = "msvc"), feature = "jemalloc"))] @@ -836,7 +1091,6 @@ pub async fn load_metrics_debug_enabled(conn: &Connection) -> error::Result<()> } _ => (), }; - Ok(()) } #[cfg(all(not(target_env = "msvc"), feature = "jemalloc"))] @@ -1132,16 +1386,18 @@ fn read_log_counters(ts_str: String) -> (usize, usize) { } pub async fn load_keep_job_dir(conn: &Connection) { - let value = load_value_from_global_settings_with_conn(conn, KEEP_JOB_DIR_SETTING, true).await; - match value { - Ok(Some(serde_json::Value::Bool(t))) => KEEP_JOB_DIR.store(t, Ordering::Relaxed), - Err(e) => { - tracing::error!("Error loading keep job dir metrics: {e:#}"); - } - _ => (), + match load_value_from_global_settings_with_conn(conn, KEEP_JOB_DIR_SETTING, true).await { + Ok(v) => apply_keep_job_dir(v), + Err(e) => tracing::error!("Error loading keep job dir metrics: {e:#}"), }; } +pub fn apply_keep_job_dir(value: Option) { + if let Some(serde_json::Value::Bool(t)) = value { + KEEP_JOB_DIR.store(t, Ordering::Relaxed) + } +} + pub async fn reload_otel_tracing_proxy_setting(conn: &Connection) { match load_value_from_global_settings_with_conn(conn, OTEL_TRACING_PROXY_SETTING, true).await { Ok(Some(settings)) => match serde_json::from_value::(settings) { @@ -1176,27 +1432,29 @@ pub async fn reload_otel_tracing_proxy_setting(conn: &Connection) { } pub async fn load_require_preexisting_user(db: &DB) { - let value = - load_value_from_global_settings(db, REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING).await; - match value { - Ok(Some(serde_json::Value::Bool(t))) => { - REQUIRE_PREEXISTING_USER_FOR_OAUTH.store(t, Ordering::Relaxed) - } - Err(e) => { - tracing::error!("Error loading keep job dir metrics: {e:#}"); - } - _ => (), + match load_value_from_global_settings(db, REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING).await { + Ok(v) => apply_require_preexisting_user(v), + Err(e) => tracing::error!("Error loading require_preexisting_user setting: {e:#}"), }; } +pub fn apply_require_preexisting_user(value: Option) { + if let Some(serde_json::Value::Bool(t)) = value { + REQUIRE_PREEXISTING_USER_FOR_OAUTH.store(t, Ordering::Relaxed) + } +} + pub async fn load_disable_password_login(db: &DB) { - let value = load_value_from_global_settings(db, DISABLE_PASSWORD_LOGIN_SETTING).await; + match load_value_from_global_settings(db, DISABLE_PASSWORD_LOGIN_SETTING).await { + Ok(v) => apply_disable_password_login(v), + Err(e) => tracing::error!("Error loading disable_password_login setting: {e:#}"), + }; +} + +pub fn apply_disable_password_login(value: Option) { match value { - Ok(Some(serde_json::Value::Bool(t))) => DISABLE_PASSWORD_LOGIN.store(t, Ordering::Relaxed), - Ok(None) => DISABLE_PASSWORD_LOGIN.store(false, Ordering::Relaxed), - Err(e) => { - tracing::error!("Error loading disable_password_login setting: {e:#}"); - } + Some(serde_json::Value::Bool(t)) => DISABLE_PASSWORD_LOGIN.store(t, Ordering::Relaxed), + None => DISABLE_PASSWORD_LOGIN.store(false, Ordering::Relaxed), _ => (), }; } @@ -2203,22 +2461,25 @@ async fn delete_log_files_from_disk_and_store( pub async fn reload_instance_events_webhook_setting(db: &DB) { use windmill_common::global_settings::INSTANCE_EVENTS_WEBHOOK_SETTING; - use windmill_common::webhook::INSTANCE_EVENTS_WEBHOOK; - let value = load_value_from_global_settings(db, INSTANCE_EVENTS_WEBHOOK_SETTING).await; + match load_value_from_global_settings(db, INSTANCE_EVENTS_WEBHOOK_SETTING).await { + Ok(v) => apply_instance_events_webhook_setting(v), + Err(e) => tracing::error!("Error loading instance_events_webhook setting: {e:#}"), + } +} + +pub fn apply_instance_events_webhook_setting(value: Option) { + use windmill_common::webhook::INSTANCE_EVENTS_WEBHOOK; match value { - Ok(Some(serde_json::Value::String(s))) if !s.is_empty() => { + Some(serde_json::Value::String(s)) if !s.is_empty() => { INSTANCE_EVENTS_WEBHOOK.store(std::sync::Arc::new(Some(s))); } - Ok(None) | Ok(Some(serde_json::Value::Null)) | Ok(Some(serde_json::Value::String(_))) => { + None | Some(serde_json::Value::Null) | Some(serde_json::Value::String(_)) => { // Fall back to env var if DB has no value INSTANCE_EVENTS_WEBHOOK.store(std::sync::Arc::new( std::env::var("INSTANCE_EVENTS_WEBHOOK").ok(), )); } - Err(e) => { - tracing::error!("Error loading instance_events_webhook setting: {e:#}"); - } _ => (), }; } @@ -2238,15 +2499,6 @@ pub async fn reload_timeout_wait_result_setting(conn: &Connection) { .await; } -pub async fn reload_saml_metadata_setting(conn: &Connection) { - reload_option_setting_with_tracing( - conn, - SAML_METADATA_SETTING, - "SAML_METADATA", - SAML_METADATA.clone(), - ) - .await; -} pub async fn reload_extra_pip_index_url_setting(conn: &Connection) { reload_option_setting_with_tracing( @@ -2338,9 +2590,6 @@ pub async fn reload_bunfig_install_scopes_setting(conn: &Connection) { .await; } -pub async fn reload_npmrc_setting(conn: &Connection) { - reload_option_setting_with_tracing(conn, NPMRC_SETTING, "NPMRC", NPMRC.clone()).await; -} pub async fn reload_nuget_config_setting(conn: &Connection) { reload_option_setting_with_tracing( @@ -2390,7 +2639,12 @@ pub async fn reload_maven_settings_xml_setting(conn: &Connection) { MAVEN_SETTINGS_XML.clone(), ) .await; + write_maven_settings_xml().await; +} +/// The half of [`reload_maven_settings_xml_setting`] after the read: mirrors the loaded value +/// onto disk, where the Maven CLI looks for it. +pub async fn write_maven_settings_xml() { if !cfg!(feature = "enterprise") { return; } @@ -2416,21 +2670,24 @@ pub async fn reload_maven_settings_xml_setting(conn: &Connection) { } pub async fn reload_no_default_maven_setting(conn: &Connection) { - let value = load_value_from_global_settings_with_conn( + match load_value_from_global_settings_with_conn( conn, windmill_common::global_settings::NO_DEFAULT_MAVEN_SETTING, true, ) - .await; - match value { - Ok(Some(serde_json::Value::Bool(t))) => NO_DEFAULT_MAVEN.store(t, Ordering::Relaxed), - Err(e) => { - tracing::error!("Error loading no default maven repository: {e:#}"); - } - _ => (), + .await + { + Ok(v) => apply_no_default_maven_setting(v), + Err(e) => tracing::error!("Error loading no default maven repository: {e:#}"), }; } +pub fn apply_no_default_maven_setting(value: Option) { + if let Some(serde_json::Value::Bool(t)) = value { + NO_DEFAULT_MAVEN.store(t, Ordering::Relaxed) + } +} + pub async fn reload_ruby_repos_setting(conn: &Connection) { reload_url_list_setting_with_tracing( conn, @@ -2441,25 +2698,23 @@ pub async fn reload_ruby_repos_setting(conn: &Connection) { .await; } -pub async fn reload_cargo_registries_setting(conn: &Connection) { - reload_option_setting_with_tracing( - conn, - windmill_common::global_settings::CARGO_REGISTRIES_SETTING, - "CARGO_REGISTRIES", - CARGO_REGISTRIES.clone(), - ) - .await; -} pub async fn reload_workspace_registries_setting(conn: &Connection) { - let value = load_value_from_global_settings_with_conn( + match load_value_from_global_settings_with_conn( conn, windmill_common::global_settings::WORKSPACE_REGISTRIES_SETTING, true, ) - .await; + .await + { + Ok(v) => apply_workspace_registries_setting(v).await, + Err(e) => tracing::error!("Error loading workspace_registries setting: {e:#}"), + } +} + +pub async fn apply_workspace_registries_setting(value: Option) { match value { - Ok(Some(v)) => match serde_json::from_value::(v) { + Some(v) => match serde_json::from_value::(v) { Ok(parsed) => { tracing::info!( "Loaded workspace registries for {} workspaces", @@ -2471,12 +2726,9 @@ pub async fn reload_workspace_registries_setting(conn: &Connection) { tracing::error!("Error parsing workspace_registries setting: {e:#}"); } }, - Ok(None) => { + None => { *WORKSPACE_REGISTRIES.write().await = None; } - Err(e) => { - tracing::error!("Error loading workspace_registries setting: {e:#}"); - } } } @@ -2622,16 +2874,14 @@ pub async fn reload_sandbox_registry_auth_setting(conn: &Connection) { // Secret-aware: the value is a raw docker/podman auth.json with credentials, so // it must never be logged. Load directly (the generic reload_option_setting path // logs the value via load_option_setting_value) and only log a redacted message. - let q = - match load_value_from_global_settings_with_conn(conn, SANDBOX_REGISTRY_AUTH_SETTING, true) - .await - { - Ok(q) => q, - Err(e) => { - tracing::error!("Error reloading setting SANDBOX_REGISTRY_AUTH: {e:?}"); - return; - } - }; + match load_value_from_global_settings_with_conn(conn, SANDBOX_REGISTRY_AUTH_SETTING, true).await + { + Ok(q) => apply_sandbox_registry_auth_setting(q).await, + Err(e) => tracing::error!("Error reloading setting SANDBOX_REGISTRY_AUTH: {e:?}"), + } +} + +pub async fn apply_sandbox_registry_auth_setting(q: Option) { let value = q.and_then(|q| serde_json::from_value::(q).ok()); let configured = value.as_ref().is_some_and(|v| !v.trim().is_empty()); *SANDBOX_REGISTRY_AUTH.write().await = value; @@ -2639,15 +2889,17 @@ pub async fn reload_sandbox_registry_auth_setting(conn: &Connection) { } pub async fn reload_job_isolation_setting(conn: &Connection) { - let value = - match load_value_from_global_settings_with_conn(conn, JOB_ISOLATION_SETTING, true).await { - Ok(Some(v)) => JobIsolationLevel::from_str(v.as_str().unwrap_or("")), - Ok(None) => JobIsolationLevel::Undefined, - Err(e) => { - tracing::error!("Error reloading job_isolation setting: {:?}", e); - return; - } - }; + match load_value_from_global_settings_with_conn(conn, JOB_ISOLATION_SETTING, true).await { + Ok(v) => apply_job_isolation_setting(v).await, + Err(e) => tracing::error!("Error reloading job_isolation setting: {:?}", e), + } +} + +pub async fn apply_job_isolation_setting(value: Option) { + let value = match value { + Some(v) => JobIsolationLevel::from_str(v.as_str().unwrap_or("")), + None => JobIsolationLevel::Undefined, + }; let old_value = JobIsolationLevel::from_u8(JOB_ISOLATION.swap(value as u8, Ordering::Relaxed)); if old_value != value { tracing::info!( @@ -2670,20 +2922,6 @@ pub async fn reload_job_isolation_setting(conn: &Connection) { } } -pub async fn reload_request_size(conn: &Connection) { - if let Err(e) = reload_setting( - conn, - REQUEST_SIZE_LIMIT_SETTING, - "REQUEST_SIZE_LIMIT", - DEFAULT_BODY_LIMIT, - REQUEST_SIZE_LIMIT.clone(), - |x| x.mul(1024 * 1024), - ) - .await - { - tracing::error!("Error reloading retention period: {:?}", e) - } -} async fn resolve_license_key_value(conn: &Connection, quiet: bool) -> anyhow::Result { let q = load_value_from_global_settings_with_conn(conn, LICENSE_KEY_SETTING, true) @@ -2792,18 +3030,321 @@ pub async fn reload_option_setting_with_tracing( } } -pub async fn load_value_from_global_settings( - db: &DB, - setting_name: &str, -) -> error::Result> { - let r = sqlx::query!( - "SELECT value FROM global_settings WHERE name = $1", - setting_name - ) - .fetch_optional(db) - .await? - .map(|x| x.value); - Ok(r) +type SettingApplier<'a> = + Box) -> BoxFuture<'a, ()> + Send + 'a>; + +/// The values a [`PassStep::Settings`] step asked for, keyed by setting name. +pub type SettingValues = std::collections::HashMap<&'static str, Option>; + +type MultiSettingApplier<'a> = Box BoxFuture<'a, ()> + Send + 'a>; + +enum PassStep<'a> { + /// A setting to read, paired with what to do once its value is in hand. + Setting { name: &'static str, http: bool, apply: SettingApplier<'a> }, + /// Several settings one piece of work needs together. + Settings { names: &'static [&'static str], http: bool, apply: MultiSettingApplier<'a> }, + /// Work that is not a settings read but has to keep its place in the sequence. + Action(BoxFuture<'a, ()>), +} + +/// One settings-loading pass: every read it will make, declared up front, then fetched +/// together and applied in the order they were declared. +/// +/// A pass that reads several dozen settings costs one round trip instead of one each, which +/// is invisible against a local database and is the bulk of a worker's startup latency +/// against a real one. Declaring is what makes the batch exact: the same `if server_mode` / +/// `if *CLOUD_HOSTED` branches that used to guard a read now guard a [`Self::setting`] call, +/// so the fetch asks for what this process actually needs and nothing else. +/// +/// Order is preserved end to end, which is what lets non-setting work sit in the middle of the +/// sequence via [`Self::action`]: appliers run in declaration order, so a setting whose applier +/// depends on an earlier one having landed still sees it. +pub struct SettingsPass<'a> { + steps: Vec>, +} + +impl<'a> SettingsPass<'a> { + pub fn new() -> Self { + SettingsPass { steps: Vec::new() } + } + + /// Declare a read. `http` mirrors `load_from_http` in + /// [`load_value_from_global_settings_with_conn`]: `false` means an agent worker leaves the + /// setting unset rather than asking the server for it. + pub fn setting(&mut self, name: &'static str, http: bool, apply: F) + where + F: FnOnce(Option) -> Fut + Send + 'a, + Fut: std::future::Future + Send + 'a, + { + self.steps.push(PassStep::Setting { + name, + http, + apply: Box::new(move |v| Box::pin(apply(v))), + }); + } + + /// Declare a step that needs several settings at once. It is skipped, like a single + /// [`Self::setting`], if any of them could not be read. + pub fn settings(&mut self, names: &'static [&'static str], http: bool, apply: F) + where + F: FnOnce(SettingValues) -> Fut + Send + 'a, + Fut: std::future::Future + Send + 'a, + { + self.steps.push(PassStep::Settings { + names, + http, + apply: Box::new(move |v| Box::pin(apply(v))), + }); + } + + /// Declare work that is not a settings read, keeping its position in the sequence. + pub fn action(&mut self, fut: Fut) + where + Fut: std::future::Future + Send + 'a, + { + self.steps.push(PassStep::Action(Box::pin(fut))); + } + + /// The batched form of [`reload_option_setting_with_tracing`]. + pub fn option_setting( + &mut self, + name: &'static str, + std_env_var: &'static str, + lock: Arc>>, + ) { + self.option_setting_with(name, std_env_var, move |v| async move { + *lock.write().await = v; + }); + } + + /// [`Self::option_setting`] for a setting held in something other than an + /// `Arc>>`, such as an `ArcSwap`. Going through here rather than calling + /// [`parse_option_setting_value`] from a bare [`Self::setting`] is what applies the + /// `FORCE_` rule, which a hand-rolled declaration would silently miss. + pub fn option_setting_with( + &mut self, + name: &'static str, + std_env_var: &'static str, + store: F, + ) where + T: FromStr + DeserializeOwned + Send + Sync + 'a, + F: FnOnce(Option) -> Fut + Send + 'a, + Fut: std::future::Future + Send + 'a, + { + if forced_env_value::(std_env_var).is_some() { + self.forced(move || async move { + store(parse_option_setting_value(None, name, std_env_var)).await + }); + return; + } + self.setting(name, true, move |v| async move { + store(parse_option_setting_value(v, name, std_env_var)).await + }); + } + + /// A setting with a default, the batched counterpart of [`load_setting_value`]. + pub fn required_setting( + &mut self, + name: &'static str, + std_env_var: &'static str, + default: T, + lock: Arc>, + transformer: fn(T) -> T, + ) { + self.setting(name, true, move |v| async move { + *lock.write().await = parse_setting_value(v, name, std_env_var, default, transformer); + }); + } + + /// The batched form of [`reload_url_list_setting_with_tracing`]. + pub fn url_list_setting( + &mut self, + name: &'static str, + std_env_var: &'static str, + lock: Arc>>>, + ) { + if std::env::var(format!("FORCE_{}", std_env_var)).is_ok() { + self.forced(move || async move { + *lock.write().await = parse_url_list_setting_value(None, name, std_env_var); + }); + return; + } + self.setting(name, true, move |v| async move { + *lock.write().await = parse_url_list_setting_value(v, name, std_env_var); + }); + } + + /// Apply a setting whose value comes from a `FORCE_` override. + /// + /// Declared as a step with no read: the override outranks the database, so fetching is + /// pointless, and more importantly a failed fetch must not drop it. A skipped applier is + /// how an unreadable setting keeps its current value, which for a forced one would mean + /// silently running unforced. + fn forced(&mut self, apply: F) + where + F: FnOnce() -> Fut + Send + 'a, + Fut: std::future::Future + Send + 'a, + { + self.action(async move { apply().await }); + } + + /// Fetch every declared setting in one go, then run the steps in declaration order. + pub async fn run(self, conn: &Connection) { + let over_http = matches!(conn, Connection::Http(_)); + let declared: Vec<(&'static str, bool)> = self + .steps + .iter() + .flat_map(|s| match s { + PassStep::Setting { name, http, .. } => vec![(*name, *http)], + PassStep::Settings { names, http, .. } => { + names.iter().map(|n| (*n, *http)).collect() + } + PassStep::Action(_) => vec![], + }) + .collect(); + + // A setting an agent worker is not allowed to ask the server for reads as unset, which + // is what `load_value_from_global_settings_with_conn(.., false)` returned for it. That + // is not the same as a read that failed, which is left out and skips its applier. + let names: Vec<&str> = declared + .iter() + .filter_map(|(name, http)| (!over_http || *http).then_some(*name)) + .collect(); + + let mut values = fetch_settings_batch(conn, &names).await; + // One failed query took every setting with it. At startup there is no known-good + // in-memory state to preserve, so read them individually rather than leave the process + // on compile-time defaults until the next full reload. Only the single-query transport + // can fail this way; over HTTP the batch already is the per-setting read. + if matches!(conn, Connection::Sql(_)) && values.is_empty() && !names.is_empty() { + tracing::warn!("Falling back to per-setting reads for {} settings", names.len()); + values = fetch_settings_individually(conn, &names).await; + } + for (name, http) in &declared { + if over_http && !*http { + values.insert(name.to_string(), None); + } + } + + for step in self.steps { + match step { + // A name missing from `values` is one whose read failed, not one that is + // unset. Skipping its applier is what keeps a transient database error from + // looking like a cleared setting: several of them reset to a default on + // `None`, and would otherwise clobber a known-good value on a blip. + PassStep::Setting { name, apply, .. } => { + if let Some(value) = values.remove(name) { + apply(value).await + } else { + tracing::warn!( + "Setting {name} could not be read, leaving its in-memory value unchanged" + ); + } + } + PassStep::Settings { names, apply, .. } => { + let asked: SettingValues = names + .iter() + .filter_map(|n| values.get(*n).map(|v| (*n, v.clone()))) + .collect(); + if asked.len() == names.len() { + apply(asked).await + } else { + let missing = names + .iter() + .filter(|n| !asked.contains_key(*n)) + .copied() + .collect::>() + .join(", "); + tracing::warn!( + "Settings {missing} could not be read, leaving the in-memory values of {} unchanged", + names.join(", ") + ); + } + } + PassStep::Action(fut) => fut.await, + } + } + } +} + +/// The value of a `FORCE_` override, when it is set and parses. +/// +/// Mirrors the check [`load_option_setting_value`] makes before it reads, so the batched and +/// per-setting paths agree on when the database is consulted at all. +fn forced_env_value(std_env_var: &str) -> Option { + std::env::var(format!("FORCE_{}", std_env_var)) + .ok() + .and_then(|x| x.parse::().ok()) +} + +/// Parse whitespace-separated URLs, dropping and reporting the ones that do not parse. +fn parse_url_list(raw: &str, source: &str) -> Vec { + raw.trim() + .split_whitespace() + .filter_map(|url_str| match url::Url::parse(url_str) { + Ok(url) => Some(url), + Err(e) => { + tracing::error!("Invalid URL in {}: '{}': {}", source, url_str, e); + None + } + }) + .collect() +} + +/// One read per setting, concurrently. The fallback for a batch that failed as a whole: a +/// name whose own read also fails is left out, so its applier is skipped rather than told the +/// setting is unset. +async fn fetch_settings_individually( + conn: &Connection, + names: &[&str], +) -> std::collections::HashMap> { + futures::future::join_all(names.iter().map(|name| async move { + ( + name.to_string(), + load_value_from_global_settings_with_conn(conn, name, true).await, + ) + })) + .await + .into_iter() + .filter_map(|(name, value)| match value { + Ok(value) => Some((name, value)), + Err(e) => { + tracing::error!("Error loading setting {name}: {e:#}"); + None + } + }) + .collect() +} + +/// Read many settings at once: one query over a database connection, and for an agent worker +/// one concurrent round of the per-setting endpoint rather than a sequential walk of it. +/// +/// A name whose read failed is left out entirely, which the caller distinguishes from a name +/// that is present with no value, i.e. genuinely unset. +async fn fetch_settings_batch( + conn: &Connection, + names: &[&str], +) -> std::collections::HashMap> { + match conn { + Connection::Sql(db) => { + match windmill_common::global_settings::load_values_from_global_settings(db, names) + .await + { + // Every requested name is accounted for: the ones with no row read as unset. + Ok(mut rows) => names + .iter() + .map(|name| (name.to_string(), rows.remove(*name))) + .collect(), + Err(e) => { + tracing::error!("Could not load global settings: {e:#}"); + std::collections::HashMap::new() + } + } + } + // An agent worker has no batch endpoint, but issuing the reads together still costs + // one round instead of one per setting. + Connection::Http(_) => fetch_settings_individually(conn, names).await, + } } pub async fn load_value_from_global_settings_with_conn( @@ -2848,6 +3389,22 @@ pub async fn load_option_setting_value( } let q = load_value_from_global_settings_with_conn(conn, setting_name, true).await?; + Ok(parse_option_setting_value(q, setting_name, std_env_var)) +} + +/// The half of [`load_option_setting_value`] after the read, so [`SettingsPass`] can parse a +/// value it already fetched. +pub fn parse_option_setting_value( + q: Option, + setting_name: &str, + std_env_var: &str, +) -> Option { + if let Some(force_value) = std::env::var(format!("FORCE_{}", std_env_var)) + .ok() + .and_then(|x| x.parse::().ok()) + { + return Some(force_value); + } let mut value = std::env::var(std_env_var) .ok() @@ -2866,7 +3423,7 @@ pub async fn load_option_setting_value( tracing::info!("Loaded {setting_name} setting to None"); } - Ok(value) + value } pub async fn reload_option_setting( @@ -2922,6 +3479,23 @@ pub async fn load_url_list_setting_value( } let q = load_value_from_global_settings_with_conn(conn, setting_name, true).await?; + Ok(parse_url_list_setting_value(q, setting_name, std_env_var)) +} + +/// The half of [`load_url_list_setting_value`] after the read, so [`SettingsPass`] can parse a +/// value it already fetched. +pub fn parse_url_list_setting_value( + q: Option, + setting_name: &str, + std_env_var: &str, +) -> Option> { + // A FORCE_ override wins over both the database and the ordinary env var. Invalid URLs in + // it are dropped with an error here rather than failing the read, since a settings pass + // has nowhere to return the failure to. + if let Ok(force_value) = std::env::var(format!("FORCE_{}", std_env_var)) { + let urls = parse_url_list(&force_value, &format!("FORCE_{}", std_env_var)); + return if urls.is_empty() { None } else { Some(urls) }; + } // Check regular environment variable let mut value = if let Ok(env_value) = std::env::var(std_env_var) { @@ -2972,7 +3546,7 @@ pub async fn load_url_list_setting_value( tracing::info!("Loaded {} setting to None", setting_name); } - Ok(value) + value } pub async fn reload_url_list_setting( @@ -2989,11 +3563,9 @@ pub async fn reload_url_list_setting( Ok(()) } -/// Load a required setting value without writing it anywhere. -/// -/// Extracted from [`reload_setting`] so callers that store the value in -/// something other than `Arc>` (e.g. `AtomicI64`, `AtomicBool`, -/// `ArcSwap`) can reuse the load pipeline. +/// Load a required setting value without writing it anywhere, so callers that store it in +/// something other than `Arc>` (e.g. `AtomicI64`, `AtomicBool`, `ArcSwap`) can +/// reuse the load pipeline. pub async fn load_setting_value( conn: &Connection, setting_name: &str, @@ -3002,7 +3574,24 @@ pub async fn load_setting_value( transformer: fn(T) -> T, ) -> error::Result { let q = load_value_from_global_settings_with_conn(conn, setting_name, true).await?; + Ok(parse_setting_value( + q, + setting_name, + std_env_var, + default, + transformer, + )) +} +/// The half of [`load_setting_value`] after the read, so [`SettingsPass`] can parse a value it +/// already fetched. +pub fn parse_setting_value( + q: Option, + setting_name: &str, + std_env_var: &str, + default: T, + transformer: fn(T) -> T, +) -> T { let mut value = std::env::var(std_env_var) .ok() .and_then(|x| x.parse::().ok()) @@ -3017,24 +3606,9 @@ pub async fn load_setting_value( } }; - Ok(value) + value } -pub async fn reload_setting( - conn: &Connection, - setting_name: &str, - std_env_var: &str, - default: T, - lock: Arc>, - transformer: fn(T) -> T, -) -> error::Result<()> { - let value = load_setting_value(conn, setting_name, std_env_var, default, transformer).await?; - { - let mut l = lock.write().await; - *l = value; - } - Ok(()) -} #[cfg(feature = "prometheus")] pub async fn monitor_pool(db: &DB) { @@ -3242,6 +3816,19 @@ pub async fn monitor_db( } }; + // Not gated on server_mode: feature-usage counters accumulate wherever an + // instrumented call site runs, and a worker that never flushed would lose + // its counts on shutdown. + let feature_usage_f = async { + if !initial_load { + if let Some(db) = conn.as_sql() { + if let Err(e) = windmill_common::feature_usage::flush_feature_usage(db).await { + tracing::error!("Error flushing feature_usage counters: {e}"); + } + } + } + }; + let verify_license_key_f = async { #[cfg(feature = "enterprise")] if !initial_load { @@ -3491,6 +4078,7 @@ pub async fn monitor_db( join!( expired_items_f, + feature_usage_f, zombie_jobs_f, stale_jobs_f, trim_resource_versions_f, @@ -4328,7 +4916,12 @@ pub async fn reload_worker_config(db: &DB, tx: KillpillSender, kill_if_change: b pub async fn load_base_url(conn: &Connection) -> error::Result { let q_base_url = load_value_from_global_settings_with_conn(conn, BASE_URL_SETTING, false).await?; + Ok(parse_base_url(q_base_url)) +} +/// The half of [`load_base_url`] after the read, so [`SettingsPass`] can use a value it +/// already fetched. Stores into `BASE_URL` as well as returning it. +pub fn parse_base_url(q_base_url: Option) -> String { let std_base_url = std::env::var("BASE_URL") .ok() .unwrap_or_else(|| "http://localhost".to_string()); @@ -4350,14 +4943,32 @@ pub async fn load_base_url(conn: &Connection) -> error::Result { std_base_url }; BASE_URL.store(std::sync::Arc::new(base_url.clone())); - Ok(base_url) + base_url } pub async fn reload_base_url_setting(conn: &Connection) -> error::Result<()> { #[cfg(feature = "oauth2")] - let oauths = if let Some(db) = conn.as_sql() { - let q_oauth = load_value_from_global_settings(db, OAUTH_SETTING).await?; + let q_oauth = match conn.as_sql() { + Some(db) => load_value_from_global_settings(db, OAUTH_SETTING).await?, + None => None, + }; + #[cfg(not(feature = "oauth2"))] + let q_oauth = None; + let q_base_url = + load_value_from_global_settings_with_conn(conn, BASE_URL_SETTING, false).await?; + apply_base_url_setting(conn, q_oauth, q_base_url).await +} +/// The half of [`reload_base_url_setting`] after the reads. +pub async fn apply_base_url_setting( + conn: &Connection, + q_oauth: Option, + q_base_url: Option, +) -> error::Result<()> { + // Both only reach a use under a feature gate. + let (_, _) = (&conn, &q_oauth); + #[cfg(feature = "oauth2")] + let oauths = if conn.as_sql().is_some() { if let Some(q) = q_oauth { if let Ok(v) = serde_json::from_value::< Option>, @@ -4374,7 +4985,7 @@ pub async fn reload_base_url_setting(conn: &Connection) -> error::Result<()> { } else { None }; - let base_url = load_base_url(conn).await?; + let base_url = parse_base_url(q_base_url); let is_secure = base_url.starts_with("https://"); #[cfg(feature = "oauth2")] @@ -5540,9 +6151,18 @@ pub async fn reload_hub_base_url_setting( conn: &Connection, server_mode: bool, ) -> error::Result<()> { - let hub_base_url = - load_value_from_global_settings_with_conn(conn, HUB_BASE_URL_SETTING, true).await?; + let v = load_value_from_global_settings_with_conn(conn, HUB_BASE_URL_SETTING, true).await?; + apply_hub_base_url_setting(conn, server_mode, v).await +} +/// The half of [`reload_hub_base_url_setting`] after the read. +pub async fn apply_hub_base_url_setting( + conn: &Connection, + server_mode: bool, + hub_base_url: Option, +) -> error::Result<()> { + // Only reaches a use under the `embedding` feature. + let _ = &conn; let base_url = if let Some(q) = hub_base_url { if let Ok(v) = serde_json::from_value::(q.clone()) { if v != "" { @@ -5586,9 +6206,12 @@ pub async fn reload_hub_base_url_setting( } pub async fn reload_critical_error_channels_setting(conn: &DB) -> error::Result<()> { - let critical_error_channels = - load_value_from_global_settings(conn, CRITICAL_ERROR_CHANNELS_SETTING).await?; + let v = load_value_from_global_settings(conn, CRITICAL_ERROR_CHANNELS_SETTING).await?; + apply_critical_error_channels_setting(v); + Ok(()) +} +pub fn apply_critical_error_channels_setting(critical_error_channels: Option) { let critical_error_channels = if let Some(q) = critical_error_channels { if let Ok(v) = serde_json::from_value::>(q.clone()) { v @@ -5604,14 +6227,15 @@ pub async fn reload_critical_error_channels_setting(conn: &DB) -> error::Result< }; CRITICAL_ERROR_CHANNELS.store(std::sync::Arc::new(critical_error_channels)); - - Ok(()) } pub async fn reload_app_workspaced_route_setting(conn: &DB) -> error::Result<()> { - let app_workspaced_route = - load_value_from_global_settings(conn, APP_WORKSPACED_ROUTE_SETTING).await?; + let v = load_value_from_global_settings(conn, APP_WORKSPACED_ROUTE_SETTING).await?; + apply_app_workspaced_route_setting(v); + Ok(()) +} +pub fn apply_app_workspaced_route_setting(app_workspaced_route: Option) { let ws_route = match app_workspaced_route { Some(serde_json::Value::Bool(ws_route)) => ws_route, None => false, @@ -5626,13 +6250,17 @@ pub async fn reload_app_workspaced_route_setting(conn: &DB) -> error::Result<()> }; APP_WORKSPACED_ROUTE.store(ws_route, Ordering::Relaxed); - Ok(()) } pub async fn reload_http_route_workspaced_route_setting(conn: &DB) -> error::Result<()> { - let http_route_workspaced_route = - load_value_from_global_settings(conn, HTTP_ROUTE_WORKSPACED_ROUTE_SETTING).await?; + let v = load_value_from_global_settings(conn, HTTP_ROUTE_WORKSPACED_ROUTE_SETTING).await?; + apply_http_route_workspaced_route_setting(conn, v).await +} +pub async fn apply_http_route_workspaced_route_setting( + conn: &DB, + http_route_workspaced_route: Option, +) -> error::Result<()> { let ws_route = match http_route_workspaced_route { Some(serde_json::Value::Bool(ws_route)) => ws_route, None => false, @@ -5689,30 +6317,34 @@ pub async fn reload_critical_alerts_on_db_oversize(conn: &DB) -> error::Result<( Ok(()) } -async fn generate_and_save_jwt_secret(db: &DB) -> error::Result { - let secret = rd_string(32); - sqlx::query!( - "INSERT INTO global_settings (name, value) VALUES ($1, $2) ON CONFLICT (name) DO UPDATE SET value = EXCLUDED.value", - JWT_SECRET_SETTING, - serde_json::to_value(&secret).unwrap() - ).execute(db).await?; - - Ok(secret) -} pub async fn reload_jwt_secret_setting(db: &DB) -> error::Result<()> { - let jwt_secret = load_value_from_global_settings(db, JWT_SECRET_SETTING).await?; + let v = load_value_from_global_settings(db, JWT_SECRET_SETTING).await?; + apply_jwt_secret_setting(db, v).await +} - let jwt_secret = if let Some(q) = jwt_secret { - if let Ok(v) = serde_json::from_value::(q.clone()) { - v - } else { - tracing::error!("Could not parse jwt_secret setting, generating new one"); - generate_and_save_jwt_secret(db).await? +/// The half of [`reload_jwt_secret_setting`] after the read. +/// +/// `value` may be stale, which is why generating falls to +/// [`get_or_create_jwt_secret`]: that statement, not this read, decides whether a new secret +/// is stored, so a pass that batched an absent read cannot overwrite one another process +/// wrote in the meantime. +pub async fn apply_jwt_secret_setting( + db: &DB, + value: Option, +) -> error::Result<()> { + let jwt_secret = match value { + Some(q) => match serde_json::from_value::(q) { + Ok(v) => v, + Err(_) => { + tracing::error!("Could not parse jwt_secret setting, generating new one"); + get_or_create_jwt_secret(db).await? + } + }, + None => { + tracing::info!("No jwt secret found, generating one"); + get_or_create_jwt_secret(db).await? } - } else { - tracing::info!("Not jwt secret found, generating one"); - generate_and_save_jwt_secret(db).await? }; JWT_SECRET.store(std::sync::Arc::new(jwt_secret)); diff --git a/backend/summarized_schema.txt b/backend/summarized_schema.txt index 44b62ca51e..732f1ece52 100644 --- a/backend/summarized_schema.txt +++ b/backend/summarized_schema.txt @@ -176,6 +176,9 @@ token: token_hash(char), token_prefix(char), token(char), label(char), expiratio FK: (workspace_id) -> workspace(id) token_expiry_notification: token_hash(char), expiration(ts) INDEX: idx_token_expiry_notification_expiration (expiration) +trigger_history: id(bigint), workspace_id(char), trigger_kind(char), path(char), operation(char), source(char), username(char), created_at(ts), changes(jsonb) + FK: (workspace_id) -> workspace(id) + INDEX: idx_trigger_history_workspace_kind_path (workspace_id, trigger_kind, path, id), idx_trigger_history_workspace_id (workspace_id, id) tutorial_progress: email(char), progress(bit64), skipped_all(bool) unique_ext_jwt_token: jwt_hash(bigint), last_used_at(ts), email(text), username(text), is_admin(bool), is_operator(bool), workspace_id(text?), label(text?), scopes(text[]?) usage: id(char), is_workspace(bool), month_(int), usage(int) diff --git a/backend/tests/fixtures/jobs_read_auth.sql b/backend/tests/fixtures/jobs_read_auth.sql index 03ee69b1f4..d7d11228aa 100644 --- a/backend/tests/fixtures/jobs_read_auth.sql +++ b/backend/tests/fixtures/jobs_read_auth.sql @@ -19,6 +19,76 @@ INSERT INTO token(token_hash, token_prefix, token, email, label, super_admin, sc ARRAY['jobs:read', 'if_jobs:filter_tags:deno'] ); +-- A path-scoped run token for test-user-2, as the trigger UI mints per runnable for a +-- webhook caller. test-user-2 created every job this token is asserted against, so the +-- `created_by` grant would otherwise hand it all of them; it must reach only jobs of +-- `f/shared/flow1`. +INSERT INTO token(token_hash, token_prefix, token, email, label, super_admin, scopes) VALUES ( + encode(sha256('RUN_SCOPED_TOKEN'::bytea), 'hex'), 'RUN_SCOPE', 'RUN_SCOPED_TOKEN', + 'test2@windmill.dev', 'flow webhook token', false, + ARRAY['jobs:run:flows:f/shared/flow1'] +); + +-- Same, scoped to a script. The two jobs below both run through a `singlestepflow` +-- wrapper (native retry / scheduled runs produce these) — one wrapping a script, one +-- wrapping a flow — so the confinement has to project each onto the runnable it wraps +-- rather than onto the wrapper's own `kind`. +INSERT INTO token(token_hash, token_prefix, token, email, label, super_admin, scopes) VALUES ( + encode(sha256('RUN_SCOPED_SCRIPT_TOKEN'::bytea), 'hex'), 'RUN_SCRIP', 'RUN_SCOPED_SCRIPT_TOKEN', + 'test2@windmill.dev', 'script webhook token', false, + ARRAY['jobs:run:scripts:u/test-user-2/wrapped_script'] +); + +INSERT INTO public.v2_job ( + id, workspace_id, created_by, created_at, permissioned_as, permissioned_as_email, + kind, script_lang, runnable_path, tag, visible_to_owner, raw_flow +) VALUES ( + '14141414-1414-1414-1414-141414141414', 'test-workspace', 'test-user-2', + '2023-01-01 00:00:00', 'u/test-user-2', 'test2@windmill.dev', + 'singlestepflow', 'deno', 'u/test-user-2/wrapped_script', 'deno', true, + '{"modules": [{"id": "a", "value": {"type": "script", "path": "u/test-user-2/wrapped_script"}}]}' +); +INSERT INTO public.v2_job_completed (id, workspace_id, duration_ms, status, result) VALUES + ('14141414-1414-1414-1414-141414141414', 'test-workspace', 1000, 'success'::job_status, + '{"wrapped": "WRAPPED_RESULT"}'); + +INSERT INTO public.v2_job ( + id, workspace_id, created_by, created_at, permissioned_as, permissioned_as_email, + kind, script_lang, runnable_path, tag, visible_to_owner, raw_flow +) VALUES ( + '15151515-1515-1515-1515-151515151515', 'test-workspace', 'test-user-2', + '2023-01-01 00:00:00', 'u/test-user-2', 'test2@windmill.dev', + 'singlestepflow', 'deno', 'f/shared/flow1', 'flow', true, + '{"modules": [{"id": "a", "value": {"type": "flow", "path": "f/shared/flow1"}}]}' +); +INSERT INTO public.v2_job_completed (id, workspace_id, duration_ms, status, result) VALUES + ('15151515-1515-1515-1515-151515151515', 'test-workspace', 1000, 'success'::job_status, + '{"wrapped": "WRAPPED_FLOW_RESULT"}'); + +-- A token pairing an app scope with a run scope, as someone driving an app's components +-- programmatically would build. `APP_INLINE_JOB` is an inline-script component run: no +-- `jobs:run` scope can name its kind, so only the `apps:run` half puts it in reach. +INSERT INTO token(token_hash, token_prefix, token, email, label, super_admin, scopes) VALUES ( + encode(sha256('APP_RUNNER_TOKEN'::bytea), 'hex'), 'APP_RUNNE', 'APP_RUNNER_TOKEN', + 'test2@windmill.dev', 'app runner token', false, + ARRAY['apps:run:u/test-user-2/dash', 'jobs:run:scripts:u/test-user-2/wrapped_script'] +); + +-- An inline-script component run of app `u/test-user-2/dash`, stamped with the +-- app provenance `execute_component` sets (`trigger_kind = 'app'`). +INSERT INTO public.v2_job ( + id, workspace_id, created_by, created_at, permissioned_as, permissioned_as_email, + kind, script_lang, runnable_path, tag, visible_to_owner, trigger_kind, trigger, args +) VALUES ( + '16161616-1616-1616-1616-161616161616', 'test-workspace', 'test-user-2', + '2023-01-01 00:00:00', 'u/test-user-2', 'test2@windmill.dev', + 'appscript', 'deno', NULL, 'deno', false, 'app', 'u/test-user-2/dash', + '{"component": "arg"}' +); +INSERT INTO public.v2_job_completed (id, workspace_id, duration_ms, status, result) VALUES + ('16161616-1616-1616-1616-161616161616', 'test-workspace', 1000, 'success'::job_status, + '{"inline": "APP_INLINE_RESULT"}'); + -- App embed token for the admin viewer (test-user). Mirrors a minted sandboxed -- low-code app token: carries the `app_embed` sentinel plus the embed scope set. -- Used to assert the token is confined to jobs the viewer LAUNCHED, not every job diff --git a/backend/tests/fixtures/typechecked_python.sql b/backend/tests/fixtures/typechecked_python.sql new file mode 100644 index 0000000000..e6350c22f3 --- /dev/null +++ b/backend/tests/fixtures/typechecked_python.sql @@ -0,0 +1,20 @@ +INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES ( +'test-workspace', +'test-user', +' +import inspect +import sys + +def greet(name: str) -> str: + # Verify that __file__ is set on this module (same check typeguard does) + mod = sys.modules[__name__] + source_file = inspect.getfile(mod) + return f"Hello, {name}! from {source_file}" + +def main(): + return greet("World") +', +'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}', +'', +'', +'f/system/typechecked_helper', 12349, 'python3', ''); diff --git a/backend/tests/jobs_read_auth.rs b/backend/tests/jobs_read_auth.rs index 317b1c6866..c8c154c250 100644 --- a/backend/tests/jobs_read_auth.rs +++ b/backend/tests/jobs_read_auth.rs @@ -46,6 +46,12 @@ const RUNNING_JOB: &str = "77777777-7777-7777-7777-777777777777"; const EMBED_OWN_JOB: &str = "12121212-1212-1212-1212-121212121212"; // A QUEUED job launched by the embed viewer (created_by test-user) — cancelable by it. const EMBED_OWN_QUEUED: &str = "13131313-1313-1313-1313-131313131313"; +// `singlestepflow` wrappers (as native retry / scheduled runs produce), one around a +// SCRIPT and one around a FLOW. +const WRAPPED_JOB: &str = "14141414-1414-1414-1414-141414141414"; +const WRAPPED_FLOW_JOB: &str = "15151515-1515-1515-1515-151515151515"; +// An inline-script component run of app `u/test-user-2/dash` (`trigger_kind = 'app'`). +const APP_INLINE_JOB: &str = "16161616-1616-1616-1616-161616161616"; // Queued sub-flow test-user-3 can see (folder `shared`), whose parent top flow they // cannot. Force cancel walks up to that parent. const QUEUED_VISIBLE_MID: &str = "55555555-5555-5555-5555-555555555555"; @@ -380,6 +386,160 @@ async fn test_single_job_read_authorization(db: Pool) -> anyhow::Resul } } + // ---- PATH-SCOPED RUN TOKEN: confined to jobs of the runnable it may start. + // RUN_SCOPED_TOKEN is test-user-2's `jobs:run:flows:f/shared/flow1` webhook + // token, and test-user-2 created every job asserted on below — so `created_by` + // alone would hand it all of them. + // Its own flow run reads, and so do the steps beneath it: a step's `runnable_path` + // is the inner script's, so the scope has to be satisfied through the ancestor. + for (path, expected) in [ + ( + format!("completed/get_result/{FLOW_JOB}"), + r#""flow": "done""#, + ), + ( + format!("completed/get_result/{STEP_JOB}"), + "STEP_RESULT_INHERITED", + ), + ] { + let (status, body) = get(&base, &path, Some("RUN_SCOPED_TOKEN")).await; + assert!( + status.is_success(), + "run-scoped token must read its own flow run ({path}, got {status}): {body}" + ); + assert!( + body.contains(expected), + "run-scoped token should get {expected} for {path}: {body}" + ); + } + // A job of any other runnable is out of scope, even though the same user created it. + for path in [ + format!("completed/get_result/{VICTIM}"), + format!("get_args/{VICTIM}"), + format!("get_logs/{VICTIM}"), + format!("getupdate/{VICTIM}?only_result=true"), + ] { + let (status, body) = get(&base, &path, Some("RUN_SCOPED_TOKEN")).await; + assert_eq!( + status, + reqwest::StatusCode::NOT_FOUND, + "run-scoped token must not read a job outside its scope ({path}, got {status}): {body}" + ); + for secret in [RESULT_SECRET, ARGS_SECRET, LOGS_SECRET] { + assert!( + !body.contains(secret), + "run-scoped token response for {path} leaked `{secret}`: {body}" + ); + } + } + // A `singlestepflow` wrapper (native retry / scheduled run) belongs to the runnable + // it wraps, not to the flow domain its `kind` suggests. Each wrapper is readable by + // the token scoped to the wrapped kind, and only by that one. + for (job, reader, denied) in [ + (WRAPPED_JOB, "RUN_SCOPED_SCRIPT_TOKEN", "RUN_SCOPED_TOKEN"), + ( + WRAPPED_FLOW_JOB, + "RUN_SCOPED_TOKEN", + "RUN_SCOPED_SCRIPT_TOKEN", + ), + ] { + let (status, body) = get(&base, &format!("completed/get_result/{job}"), Some(reader)).await; + assert!( + status.is_success() && body.contains("WRAPPED"), + "{reader} must read the singlestepflow wrapping its runnable (got {status}): {body}" + ); + let (status, body) = get(&base, &format!("completed/get_result/{job}"), Some(denied)).await; + assert_eq!( + status, + reqwest::StatusCode::NOT_FOUND, + "{denied} must not read a wrapper around the other kind (got {status}): {body}" + ); + } + + // An `apps:run:` scope is a start grant too: the inline-script component run it + // launched — a kind no `jobs:run` scope can name — stays readable to a token scoped + // to that app, and stays out of reach for one that is only scoped to run jobs. + let (status, body) = get( + &base, + &format!("completed/get_result/{APP_INLINE_JOB}"), + Some("APP_RUNNER_TOKEN"), + ) + .await; + assert!( + status.is_success() && body.contains("APP_INLINE_RESULT"), + "app-scoped token must read the component run its app launched (got {status}): {body}" + ); + let (status, body) = get( + &base, + &format!("completed/get_result/{APP_INLINE_JOB}"), + Some("RUN_SCOPED_SCRIPT_TOKEN"), + ) + .await; + assert_eq!( + status, + reqwest::StatusCode::NOT_FOUND, + "a token with no scope on the app must not read its component run (got {status}): {body}" + ); + + // An approval link is a bypass of the read gate, so the confinement is re-applied on + // top of it: it must not become a way for a scoped token to read an out-of-scope job. + // The link itself is untouched — a logged-out approver still reads the same job. + let approval_token = + windmill_common::variables::generate_approval_token("test-workspace", VICTIM.parse()?, &db) + .await?; + let (status, body) = get( + &base, + &format!("get/{VICTIM}?approval_token={approval_token}"), + None, + ) + .await; + assert!( + status.is_success(), + "an approval link must still authorize a logged-out read (got {status}): {body}" + ); + let (status, body) = get( + &base, + &format!("get/{VICTIM}?approval_token={approval_token}"), + Some("RUN_SCOPED_TOKEN"), + ) + .await; + assert_eq!( + status, + reqwest::StatusCode::NOT_FOUND, + "an approval link must not lift the run-scope confinement (got {status}): {body}" + ); + + // Same for the resume-secret bypass on the result route, which the approval page uses. + let (status, secret) = get( + &authed_base, + &format!("job_signature/{STEP_JOB}/0"), + Some("SECRET_TOKEN_2"), + ) + .await; + assert!(status.is_success(), "owner must mint a resume secret: {secret}"); + let secret = secret.trim().trim_matches('"').to_string(); + let approval_result = + format!("completed/get_result/{STEP_JOB}?suspended_job={STEP_JOB}&resume_id=0&secret={secret}"); + let (status, body) = get(&base, &approval_result, None).await; + assert!( + status.is_success(), + "a resume secret must still authorize a logged-out result read (got {status}): {body}" + ); + let (status, body) = get(&base, &approval_result, Some("RUN_SCOPED_SCRIPT_TOKEN")).await; + assert_eq!( + status, + reqwest::StatusCode::NOT_FOUND, + "a resume secret must not lift the run-scope confinement (got {status}): {body}" + ); + + // And a run grant is not an enumeration grant: the whole listing surface is denied. + let (status, body) = get(&authed_base, "list", Some("RUN_SCOPED_TOKEN")).await; + assert_eq!( + status, + reqwest::StatusCode::FORBIDDEN, + "run-scoped token must not enumerate jobs (got {status}): {body}" + ); + // ---- APP EMBED TOKEN: cancellation confined to the app's own jobs. The token // may cancel a job it launched (created_by == viewer), but `cancel_job_api` // denies (NotFound) a job created by someone else, even one the (admin) diff --git a/backend/tests/list_scripts_parent_hash.rs b/backend/tests/list_scripts_parent_hash.rs new file mode 100644 index 0000000000..bfad39fff8 --- /dev/null +++ b/backend/tests/list_scripts_parent_hash.rs @@ -0,0 +1,118 @@ +//! Pins the `parent_hash` filter of `GET /w/{workspace}/scripts/list`: the +//! request must succeed and return exactly the scripts whose `parent_hashes` +//! array contains the given hash. The filter is assembled into a raw SQL string, +//! so a malformed predicate is only caught by executing the query. + +use serde_json::json; +use sqlx::{Pool, Postgres}; +use windmill_test_utils::*; + +fn client() -> reqwest::Client { + reqwest::Client::new() +} + +fn authed(builder: reqwest::RequestBuilder, token: &str) -> reqwest::RequestBuilder { + builder.header("Authorization", format!("Bearer {}", token)) +} + +fn new_script(path: &str, content: &str) -> serde_json::Value { + json!({ + "path": path, + "summary": "", + "description": "", + "content": content, + "language": "deno", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": {}, + "required": [] + } + }) +} + +async fn create(port: u16, script: serde_json::Value) -> anyhow::Result { + let resp = authed( + client().post(format!( + "http://localhost:{port}/api/w/test-workspace/scripts/create" + )), + "SECRET_TOKEN", + ) + .json(&script) + .send() + .await?; + let status = resp.status(); + let body = resp.text().await?; + assert_eq!(status, 201, "script create should succeed: {body}"); + Ok(body) +} + +/// Paths returned by `scripts/list` for the given query string. +async fn list_paths(port: u16, query: &str) -> anyhow::Result> { + let resp = authed( + client().get(format!( + "http://localhost:{port}/api/w/test-workspace/scripts/list?{query}" + )), + "SECRET_TOKEN", + ) + .send() + .await?; + let status = resp.status(); + let body = resp.text().await?; + assert_eq!(status, 200, "scripts/list?{query} should succeed: {body}"); + let items: Vec = serde_json::from_str(&body)?; + Ok(items + .iter() + .map(|it| it["path"].as_str().unwrap().to_string()) + .collect()) +} + +#[sqlx::test(fixtures("base"))] +async fn test_list_scripts_parent_hash_filter(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + let lineage_path = "u/test-user/parent_hash_lineage"; + let unrelated_path = "u/test-user/parent_hash_unrelated"; + + // A two-version lineage: v2's `parent_hashes` contains v1's hash. + let v1 = create( + port, + new_script(lineage_path, "export async function main() { return 1; }"), + ) + .await?; + let mut v2_body = new_script(lineage_path, "export async function main() { return 2; }"); + v2_body["parent_hash"] = json!(v1); + let v2 = create(port, v2_body).await?; + + // ...plus a script with no lineage at all, which must never match. + create( + port, + new_script(unrelated_path, "export async function main() { return 3; }"), + ) + .await?; + + let all = list_paths(port, "").await?; + assert!( + all.contains(&lineage_path.to_string()) && all.contains(&unrelated_path.to_string()), + "unfiltered listing should return both scripts, got {all:?}" + ); + + let descendants = list_paths(port, &format!("parent_hash={v1}")).await?; + assert_eq!( + descendants, + vec![lineage_path.to_string()], + "parent_hash should return only the scripts descending from that hash" + ); + + // v2 is the head, so nothing descends from it. + let none = list_paths(port, &format!("parent_hash={v2}")).await?; + assert!( + none.is_empty(), + "no script descends from the head version, got {none:?}" + ); + + Ok(()) +} diff --git a/backend/tests/mcp_token_exfil.rs b/backend/tests/mcp_token_exfil.rs index ce278f3c53..1d36ba9546 100644 --- a/backend/tests/mcp_token_exfil.rs +++ b/backend/tests/mcp_token_exfil.rs @@ -19,6 +19,11 @@ //! and the request only fails later at the connect/SSRF step — proving the //! legitimate path still resolves the token (no over-blocking). //! +//! `POST .../resources/mcp_call_tool/{path}` reaches the same MCP server through +//! the same resource, so it is pinned to the same property here — both handlers +//! share `connect_mcp_client`, and a future split of that helper must not let +//! one of them regress. +//! //! SSRF rejection of an author-controlled URL is covered by the unit test in //! `windmill-mcp` (`from_resource_rejects_ssrf_url`). #![cfg(feature = "mcp")] @@ -27,6 +32,7 @@ use sqlx::{Pool, Postgres}; use windmill_test_utils::*; const SECRET_VALUE: &str = "S3CRET-MCP-TOKEN-VALUE"; +const RESOURCE_PATH: &str = "u/test-user-3/evil_mcp"; fn client() -> reqwest::Client { reqwest::Client::new() @@ -44,13 +50,28 @@ async fn get(base: &str, path: &str, token: &str) -> (reqwest::StatusCode, Strin (status, body) } -#[sqlx::test(fixtures("base", "mcp_token_exfil"))] -async fn test_mcp_token_not_exfiltrated(db: Pool) -> anyhow::Result<()> { - initialize_tracing().await; +async fn post( + base: &str, + path: &str, + token: &str, + body: serde_json::Value, +) -> (reqwest::StatusCode, String) { + let resp = client() + .post(format!("{base}/{path}")) + .header("Authorization", format!("Bearer {token}")) + .json(&body) + .send() + .await + .expect("request"); + let status = resp.status(); + let body = resp.text().await.expect("body"); + (status, body) +} - // Insert the locked secret variable with a real, workspace-key-encrypted - // value so an authorized read genuinely decrypts it. - let mc = windmill_common::variables::build_crypt(&db, "test-workspace").await?; +/// Insert the locked secret variable with a real, workspace-key-encrypted value +/// so an authorized read genuinely decrypts it. +async fn insert_locked_secret(db: &Pool) -> anyhow::Result<()> { + let mc = windmill_common::variables::build_crypt(db, "test-workspace").await?; let encrypted = windmill_common::variables::encrypt(&mc, SECRET_VALUE); // Runtime-checked query (not the `query!` macro) so no offline `.sqlx` cache // entry is needed for this test-only insert. @@ -59,13 +80,21 @@ async fn test_mcp_token_not_exfiltrated(db: Pool) -> anyhow::Result<() VALUES ('test-workspace', 'f/locked/secret_token', $1, true, 'Locked secret', '{}')", ) .bind(&encrypted) - .execute(&db) + .execute(db) .await?; + Ok(()) +} + +#[sqlx::test(fixtures("base", "mcp_token_exfil"))] +async fn test_mcp_token_not_exfiltrated(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + + insert_locked_secret(&db).await?; let server = ApiServer::start(db.clone()).await?; let port = server.addr.port(); let base = format!("http://localhost:{port}/api/w/test-workspace/resources/mcp_tools"); - let path = "u/test-user-3/evil_mcp"; + let path = RESOURCE_PATH; // ---- CORE REGRESSION: the developer can read the resource but must NOT be // able to resolve the locked secret. They are denied (401) at the @@ -109,3 +138,47 @@ async fn test_mcp_token_not_exfiltrated(db: Pool) -> anyhow::Result<() Ok(()) } + +#[sqlx::test(fixtures("base", "mcp_token_exfil"))] +async fn test_mcp_call_tool_token_not_exfiltrated(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + + insert_locked_secret(&db).await?; + + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let base = format!("http://localhost:{port}/api/w/test-workspace/resources/mcp_call_tool"); + let body = serde_json::json!({ "tool": "whoami", "arguments": {} }); + + let (status, resp) = post(&base, RESOURCE_PATH, "SECRET_TOKEN_3", body.clone()).await; + assert_eq!( + status, + reqwest::StatusCode::UNAUTHORIZED, + "developer must be denied resolving a secret they can't read (got {status}): {resp}" + ); + assert!( + !resp.contains(SECRET_VALUE), + "the locked secret must never leak to the developer: {resp}" + ); + assert!( + resp.contains("don't have access"), + "denial should come from the variable-RLS gate, not a connection error: {resp}" + ); + assert!( + !resp.contains("Failed to connect to MCP server"), + "developer must be blocked before the connection step (would mean the token was resolved): {resp}" + ); + + let (status, resp) = post(&base, RESOURCE_PATH, "SECRET_TOKEN", body).await; + assert_ne!( + status, + reqwest::StatusCode::UNAUTHORIZED, + "admin must clear the variable-RLS gate (got {status}): {resp}" + ); + assert!( + resp.contains("Failed to connect to MCP server"), + "admin should resolve the token and only fail at the connect/SSRF step: {resp}" + ); + + Ok(()) +} diff --git a/backend/tests/nativets_stress.rs b/backend/tests/nativets_stress.rs index 32057b41d2..21d3911778 100644 --- a/backend/tests/nativets_stress.rs +++ b/backend/tests/nativets_stress.rs @@ -241,7 +241,6 @@ fn spawn_workers( worker_name, i as u64, n as u32, - "127.0.0.1", rx, tx2, &base_internal_url, diff --git a/backend/tests/python_jobs.rs b/backend/tests/python_jobs.rs index 0f45d09147..b7aa671b21 100644 --- a/backend/tests/python_jobs.rs +++ b/backend/tests/python_jobs.rs @@ -1132,7 +1132,8 @@ async def main(item: str, qty: int, email: str): port, ) .await; - }); + }) + .await; Ok(()) } @@ -1240,6 +1241,50 @@ async fn test_python_wac_v2_with_preprocessor(db: Pool) -> anyhow::Res Ok(()) } +#[cfg(feature = "python")] +#[sqlx::test(fixtures("base", "typechecked_python"))] +async fn test_typechecked_decorator_python(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + let content = r#" +from f.system.typechecked_helper import greet + +def main(): + return greet("World") +"# + .to_owned(); + + let job = JobPayload::Code(RawCode { + hash: None, + content, + path: Some("f/system/test_typechecked".to_string()), + language: ScriptLang::Python3, + lock: None, + concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default() + .into(), + debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + cache_ttl: None, + cache_ignore_s3_path: None, + dedicated_worker: None, + modules: None, + tag: None, + }); + + let result = run_job_in_new_worker_until_complete(&db, false, job, port) + .await + .json_result() + .unwrap(); + + let result_str = result.as_str().unwrap(); + assert!( + result_str.starts_with("Hello, World! from "), + "unexpected result: {result_str}" + ); + Ok(()) +} + /// End-to-end comparison between the legacy `step()` suspend-and-replay path /// and the new SDK inline-persist fast path, toggled per-job via the /// `WM_WAC_INLINE_FAST_PATH` env var which the Python script sets on its own diff --git a/backend/tests/worker_ping_ip.rs b/backend/tests/worker_ping_ip.rs new file mode 100644 index 0000000000..8f98398b40 --- /dev/null +++ b/backend/tests/worker_ping_ip.rs @@ -0,0 +1,44 @@ +use sqlx::{Pool, Postgres}; +use windmill_common::{external_ip::UNKNOWN_IP, worker::insert_ping_query}; + +async fn insert_ping(db: &Pool, worker: &str, ip: Option<&str>) -> anyhow::Result<()> { + insert_ping_query( + "test-instance", + worker, + "default", + ip, + &[], + None, + None, + "test", + None, + None, + None, + false, + db, + ) + .await?; + Ok(()) +} + +/// The external IP resolves in the background, so the initial ping often has none yet. That must +/// not blank the address a previous process wrote to the row this one reclaims — worker names are +/// stable across restarts under EXIT_AFTER_N_JOBS. +#[sqlx::test] +async fn unresolved_ip_keeps_the_reclaimed_rows_address(db: Pool) -> anyhow::Result<()> { + insert_ping(&db, "wk-reclaimed", Some("1.2.3.4")).await?; + insert_ping(&db, "wk-reclaimed", None).await?; + let ip: String = sqlx::query_scalar("SELECT ip FROM worker_ping WHERE worker = $1") + .bind("wk-reclaimed") + .fetch_one(&db) + .await?; + assert_eq!(ip, "1.2.3.4"); + + insert_ping(&db, "wk-fresh", None).await?; + let ip: String = sqlx::query_scalar("SELECT ip FROM worker_ping WHERE worker = $1") + .bind("wk-fresh") + .fetch_one(&db) + .await?; + assert_eq!(ip, UNKNOWN_IP); + Ok(()) +} diff --git a/backend/windmill-ai/src/ai_bedrock.rs b/backend/windmill-ai/src/ai_bedrock.rs index 02ab702d31..b223725af2 100644 --- a/backend/windmill-ai/src/ai_bedrock.rs +++ b/backend/windmill-ai/src/ai_bedrock.rs @@ -109,6 +109,19 @@ const BEDROCK_PROMPT_CACHING_SUPPORTED_MODEL_IDS: &[&str] = &[ "anthropic.claude-3-5-sonnet-20241022-v2:0", ]; +/// Claude 4.6 and later are published under several id spellings for the same +/// model (`anthropic.claude-sonnet-4-6`, `...-4-6-v1`, `...-4-6-v1:0`), so they +/// are matched by family prefix rather than by exact id. +const BEDROCK_PROMPT_CACHING_SUPPORTED_MODEL_PREFIXES: &[&str] = &[ + "anthropic.claude-fable-5", + "anthropic.claude-opus-4-6", + "anthropic.claude-opus-4-7", + "anthropic.claude-opus-4-8", + "anthropic.claude-opus-5", + "anthropic.claude-sonnet-4-6", + "anthropic.claude-sonnet-5", +]; + fn build_default_cache_point() -> aws_sdk_bedrockruntime::types::CachePointBlock { aws_sdk_bedrockruntime::types::CachePointBlock::builder() .r#type(aws_sdk_bedrockruntime::types::CachePointType::Default) @@ -123,7 +136,7 @@ fn normalize_bedrock_model_id(model: &str) -> String { .unwrap_or(model) .to_ascii_lowercase(); - for prefix in ["global.", "us.", "eu.", "apac."] { + for prefix in ["global.", "us.", "eu.", "apac.", "au."] { if let Some(normalized_model) = model.strip_prefix(prefix) { return normalized_model.to_string(); } @@ -135,6 +148,9 @@ fn normalize_bedrock_model_id(model: &str) -> String { pub fn bedrock_model_supports_prompt_caching(model: &str) -> bool { let normalized_model = normalize_bedrock_model_id(model); BEDROCK_PROMPT_CACHING_SUPPORTED_MODEL_IDS.contains(&normalized_model.as_str()) + || BEDROCK_PROMPT_CACHING_SUPPORTED_MODEL_PREFIXES + .iter() + .any(|prefix| normalized_model.starts_with(prefix)) } fn append_cache_point_to_system_prompts(system_prompts: &mut Vec) { @@ -1241,6 +1257,27 @@ mod tests { )); } + /// Claude 4.6+ ships under bare, `-v1` and `-v1:0` spellings of the same id, + /// so every one of them has to reach the prefix match. + #[test] + fn bedrock_prompt_caching_supports_claude_4_6_and_later_id_spellings() { + for model in [ + "anthropic.claude-sonnet-4-6", + "anthropic.claude-sonnet-4-6-v1:0", + "us.anthropic.claude-opus-4-6-v1", + "global.anthropic.claude-opus-4-8", + "anthropic.claude-opus-5", + "eu.anthropic.claude-sonnet-5-v1:0", + "au.anthropic.claude-sonnet-5", + "anthropic.claude-fable-5", + ] { + assert!( + bedrock_model_supports_prompt_caching(model), + "{model} must support prompt caching" + ); + } + } + #[test] fn bedrock_prompt_caching_rejects_unsupported_or_opaque_model_ids() { assert!(!bedrock_model_supports_prompt_caching( @@ -1249,5 +1286,9 @@ mod tests { assert!(!bedrock_model_supports_prompt_caching( "arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/my-profile" )); + // Opus 4.5 is dated-id only — the 4.6+ prefixes must not swallow it. + assert!(!bedrock_model_supports_prompt_caching( + "anthropic.claude-opus-4-5-20251101-v2:0" + )); } } diff --git a/backend/windmill-ai/src/providers/anthropic.rs b/backend/windmill-ai/src/providers/anthropic.rs index 6ebc555da6..f22f28de14 100644 --- a/backend/windmill-ai/src/providers/anthropic.rs +++ b/backend/windmill-ai/src/providers/anthropic.rs @@ -1,3 +1,4 @@ +use super::{anthropic_model_rejects_sampling_params, REASONING_OFF_SENTINEL}; use crate::{ ai_google::parse_data_url, ai_providers::{AIPlatform, AIProvider}, @@ -137,17 +138,23 @@ pub struct AnthropicMessage { pub content: Vec, } -/// Adaptive thinking config for Anthropic native API. `summarized` display -/// matches the chat proxy path (renders a summarized thinking stream). +/// Thinking config for the Anthropic native API. `summarized` display matches +/// the chat proxy path (renders a summarized thinking stream); the disable +/// carries no display. #[derive(Serialize, Debug)] pub struct AnthropicThinking { pub r#type: &'static str, - pub display: &'static str, + #[serde(skip_serializing_if = "Option::is_none")] + pub display: Option<&'static str>, } impl AnthropicThinking { fn adaptive() -> Self { - Self { r#type: "adaptive", display: "summarized" } + Self { r#type: "adaptive", display: Some("summarized") } + } + + fn disabled() -> Self { + Self { r#type: "disabled", display: None } } } @@ -157,6 +164,37 @@ pub struct AnthropicOutputConfig { pub effort: String, } +/// Resolve the thinking config, effort and sampling params for a reasoning +/// selection. Temperature is dropped both under adaptive thinking, which +/// rejects it, and on the models that removed the sampling params outright. +fn anthropic_thinking_config( + model: &str, + reasoning_effort: Option<&str>, + temperature: Option, +) -> ( + Option, + Option, + Option, +) { + let temperature = (!anthropic_model_rejects_sampling_params(model)) + .then_some(temperature) + .flatten(); + match reasoning_effort { + // The disable sentinel is not an effort token — Anthropic's vocabulary + // is low..max and rejects it. The disable carries no effort either: + // pairing it with xhigh or max is itself a 400 on Opus 5. + Some(effort) if effort == REASONING_OFF_SENTINEL => { + (Some(AnthropicThinking::disabled()), None, temperature) + } + Some(effort) => ( + Some(AnthropicThinking::adaptive()), + Some(AnthropicOutputConfig { effort: effort.to_string() }), + None, + ), + None => (None, None, temperature), + } +} + /// Anthropic-specific request structure for standard API #[derive(Serialize)] pub struct AnthropicRequest<'a> { @@ -652,16 +690,8 @@ impl AnthropicQueryBuilder { } } - // Adaptive thinking rejects sampling params, so drop temperature when - // reasoning is on (Anthropic returns a hard 400 otherwise). - let (thinking, output_config, temperature) = match args.reasoning_effort { - Some(effort) => ( - Some(AnthropicThinking::adaptive()), - Some(AnthropicOutputConfig { effort: effort.to_string() }), - None, - ), - None => (None, None, args.temperature), - }; + let (thinking, output_config, temperature) = + anthropic_thinking_config(args.model, args.reasoning_effort, args.temperature); // Build request based on platform if self.is_vertex() { @@ -1096,6 +1126,83 @@ mod tests { assert!(body.get("temperature").is_none()); } + /// An agent step stores the chat's off sentinel verbatim as its + /// `reasoning_effort`, so the disable has to be translated here rather than + /// forwarded as an effort token Anthropic would reject. + #[test] + fn anthropic_thinking_config_translates_the_off_sentinel() { + let (thinking, output_config, _) = + anthropic_thinking_config("claude-sonnet-4-6", Some("none"), Some(0.5)); + assert_eq!(thinking.as_ref().map(|t| t.r#type), Some("disabled")); + assert!(output_config.is_none()); + + let (thinking, output_config, temperature) = + anthropic_thinking_config("claude-sonnet-4-6", Some("xhigh"), Some(0.5)); + assert_eq!(thinking.as_ref().map(|t| t.r#type), Some("adaptive")); + assert_eq!(output_config.map(|c| c.effort), Some("xhigh".to_string())); + // Adaptive thinking rejects sampling params on every model. + assert!(temperature.is_none()); + + let (thinking, output_config, temperature) = + anthropic_thinking_config("claude-sonnet-4-6", None, Some(0.5)); + assert!(thinking.is_none()); + assert!(output_config.is_none()); + assert_eq!(temperature, Some(0.5)); + } + + /// Live-verified: Opus 4.8 and the 5 family 400 with `temperature is + /// deprecated for this model` whatever the thinking mode, so the disable and + /// no-reasoning paths have to drop it too. + #[test] + fn anthropic_thinking_config_drops_sampling_params_on_models_that_reject_them() { + for model in [ + "claude-opus-5", + "claude-sonnet-5", + "claude-opus-4-8", + "anthropic/claude-opus-4.7", + "claude-fable-5", + ] { + for effort in [Some("none"), None] { + let (_, _, temperature) = anthropic_thinking_config(model, effort, Some(0.5)); + assert!( + temperature.is_none(), + "{model} must not carry temperature (effort {effort:?})" + ); + } + } + // Sonnet 4.6 still accepts them, so an off selection keeps temperature. + let (_, _, temperature) = + anthropic_thinking_config("claude-sonnet-4-6", Some("none"), Some(0.5)); + assert_eq!(temperature, Some(0.5)); + } + + #[test] + fn anthropic_request_serializes_the_off_sentinel_as_a_thinking_disable() { + let (thinking, output_config, temperature) = + anthropic_thinking_config("claude-opus-5", Some("none"), Some(0.5)); + let request = AnthropicRequest { + model: "claude-opus-5", + system: None, + messages: vec![], + tools: None, + tool_choice: None, + temperature, + thinking, + output_config, + max_tokens: Some(64000), + stream: true, + }; + + let body: serde_json::Value = + serde_json::from_str(&serde_json::to_string(&request).unwrap()).unwrap(); + assert_eq!(body["thinking"]["type"], "disabled"); + // A disable paired with an effort is a 400 on Opus 5, and `display` + // only applies to a thinking mode that actually runs. + assert!(body["thinking"].get("display").is_none()); + assert!(body.get("output_config").is_none()); + assert!(body.get("temperature").is_none()); + } + #[test] fn anthropic_request_omits_thinking_when_reasoning_off() { let request = AnthropicRequest { diff --git a/backend/windmill-ai/src/providers/bedrock.rs b/backend/windmill-ai/src/providers/bedrock.rs index 4405a50dcb..453607eea5 100644 --- a/backend/windmill-ai/src/providers/bedrock.rs +++ b/backend/windmill-ai/src/providers/bedrock.rs @@ -6,6 +6,7 @@ //! - Stream event parsing //! - Helper utilities +use super::{anthropic_model_rejects_sampling_params, REASONING_OFF_SENTINEL}; use crate::{ ai_bedrock::{ bedrock_model_supports_prompt_caching, bedrock_stream_event_is_block_stop, @@ -357,12 +358,11 @@ async fn handle_bedrock_sdk_streaming( let enable_prompt_caching = bedrock_model_supports_prompt_caching(model); let (bedrock_messages, system_prompts) = openai_messages_to_bedrock(&openai_req.messages, enable_prompt_caching)?; - // Adaptive thinking rejects sampling params; drop temperature when reasoning is on. - let temperature = openai_req - .reasoning_effort - .is_none() - .then_some(openai_req.temperature) - .flatten(); + let temperature = bedrock_temperature( + model, + openai_req.reasoning_effort.as_deref(), + openai_req.temperature, + ); let inference_config = create_inference_config(temperature, openai_req.max_tokens); let tool_config = build_tool_config_from_request( openai_req.tools.as_deref(), @@ -410,11 +410,35 @@ async fn handle_bedrock_sdk_streaming( }) } -/// Build the Converse `additionalModelRequestFields` enabling Claude adaptive -/// thinking at the given effort. `display: summarized` is billing-neutral on -/// Anthropic models and matches the direct-Anthropic chat path, which renders -/// summarized thinking in the UI. +/// Whether an effort token turns adaptive thinking on. `"none"` is the disable +/// sentinel rather than a level. +fn effort_enables_thinking(effort: Option<&str>) -> bool { + matches!(effort, Some(effort) if effort != REASONING_OFF_SENTINEL) +} + +/// Temperature survives only when thinking is not adaptive — which rejects +/// sampling params — and the model still accepts them at all. Non-Anthropic +/// Bedrock models (Nova, Llama) are unaffected by the second check. +fn bedrock_temperature(model: &str, effort: Option<&str>, temperature: Option) -> Option { + if effort_enables_thinking(effort) || anthropic_model_rejects_sampling_params(model) { + return None; + } + temperature +} + +/// Build the Converse `additionalModelRequestFields` carrying Claude's thinking +/// config. `display: summarized` is billing-neutral on Anthropic models and +/// matches the direct-Anthropic chat path, which renders summarized thinking in +/// the UI. fn bedrock_thinking_fields(effort: &str) -> aws_smithy_types::Document { + if effort == REASONING_OFF_SENTINEL { + // The disable carries no effort: pairing it with xhigh or max is a 400 + // on Opus 5, and omitting it leaves the model at the effort where the + // disable is accepted. + return json_to_document(serde_json::json!({ + "thinking": { "type": "disabled" } + })); + } json_to_document(serde_json::json!({ "thinking": { "type": "adaptive", "display": "summarized" }, "output_config": { "effort": effort } @@ -663,12 +687,11 @@ async fn handle_bedrock_sdk_non_streaming( let enable_prompt_caching = bedrock_model_supports_prompt_caching(model); let (bedrock_messages, system_prompts) = openai_messages_to_bedrock(&openai_req.messages, enable_prompt_caching)?; - // Adaptive thinking rejects sampling params; drop temperature when reasoning is on. - let temperature = openai_req - .reasoning_effort - .is_none() - .then_some(openai_req.temperature) - .flatten(); + let temperature = bedrock_temperature( + model, + openai_req.reasoning_effort.as_deref(), + openai_req.temperature, + ); let inference_config = create_inference_config(temperature, openai_req.max_tokens); let tool_config = build_tool_config_from_request( openai_req.tools.as_deref(), @@ -934,8 +957,7 @@ impl BedrockQueryBuilder { let (bedrock_messages, system_prompts) = openai_messages_to_bedrock(&prepared_messages, enable_prompt_caching)?; - // Adaptive thinking rejects sampling params; drop temperature when reasoning is on. - let temperature = reasoning_effort.is_none().then_some(temperature).flatten(); + let temperature = bedrock_temperature(model, reasoning_effort, temperature); // Build inference configuration using shared helper let inference_config = create_inference_config(temperature, max_tokens.map(|t| t as i32)); @@ -1285,6 +1307,37 @@ mod tests { ); } + #[test] + fn bedrock_thinking_fields_translate_the_off_sentinel_to_a_disable() { + let fields = document_to_json(&bedrock_thinking_fields("none")); + assert_eq!(fields["thinking"]["type"], "disabled"); + // An effort alongside the disable is a 400 on Opus 5. + assert!(fields.get("output_config").is_none()); + assert!(effort_enables_thinking(Some("xhigh"))); + assert!(!effort_enables_thinking(Some("none"))); + assert!(!effort_enables_thinking(None)); + } + + #[test] + fn bedrock_temperature_drops_sampling_params_per_thinking_mode_and_model() { + // Adaptive thinking rejects sampling params on every model... + let adaptive = bedrock_temperature("anthropic.claude-sonnet-4-6", Some("xhigh"), Some(0.5)); + assert!(adaptive.is_none()); + // ...while a model that still accepts them keeps them when off. + let off = bedrock_temperature("anthropic.claude-sonnet-4-6", Some("none"), Some(0.5)); + assert_eq!(off, Some(0.5)); + // The models that removed them drop them on every mode. + for (model, effort) in [ + ("global.anthropic.claude-opus-5", Some("none")), + ("anthropic.claude-opus-4-8", None), + ] { + assert!(bedrock_temperature(model, effort, Some(0.5)).is_none()); + } + // A non-Anthropic Bedrock model keeps its sampling params. + let nova = bedrock_temperature("amazon.nova-pro-v1:0", None, Some(0.5)); + assert_eq!(nova, Some(0.5)); + } + #[test] fn bedrock_thinking_fields_carry_adaptive_thinking_and_effort() { let fields = document_to_json(&bedrock_thinking_fields("xhigh")); diff --git a/backend/windmill-ai/src/providers/mod.rs b/backend/windmill-ai/src/providers/mod.rs index 094cc71490..af9ba3ff9f 100644 --- a/backend/windmill-ai/src/providers/mod.rs +++ b/backend/windmill-ai/src/providers/mod.rs @@ -8,6 +8,38 @@ pub mod other; use std::time::{Duration, Instant}; +/// The effort token the chat and agent surfaces send to turn reasoning off. +/// It is not a provider-native level — each provider translates it to its own +/// disable (Anthropic and Bedrock to `thinking: {type: "disabled"}`, DeepSeek to +/// its `thinking` param, Gemini to a zero budget or the model's floor). +pub(crate) const REASONING_OFF_SENTINEL: &str = "none"; + +/// Whether a Claude model removed the sampling params (`temperature`, `top_p`, +/// `top_k`). On these, any value is a hard 400 — `temperature is deprecated for +/// this model` — whatever the thinking mode, so the param has to be dropped on +/// the reasoning-off and no-reasoning paths too, not only under adaptive +/// thinking. +/// +/// Probed against the Messages API: `claude-opus-5`, `claude-sonnet-5` and +/// `claude-opus-4-8` reject them; `claude-sonnet-4-6` still accepts them. Opus +/// 4.7, Fable and Mythos are included from Anthropic's migration guide, which +/// documents the same removal, rather than from a probe. +/// +/// Matching is on the model name, so a Bedrock *application* inference profile — +/// whose id is opaque (`k1c3lwu20lem`) rather than derived from the model — +/// cannot be classified and keeps its sampling params. Resolving the backing +/// model would need a per-request AWS lookup; `bedrock_model_supports_prompt_caching` +/// degrades on the same ids for the same reason. +pub(crate) fn anthropic_model_rejects_sampling_params(model: &str) -> bool { + let model = model.to_lowercase().replace('.', "-"); + model.contains("claude-opus-4-7") + || model.contains("claude-opus-4-8") + || model.contains("claude-opus-5") + || model.contains("claude-sonnet-5") + || model.contains("claude-fable") + || model.contains("claude-mythos") +} + use windmill_common::cache::Cache; use crate::{ diff --git a/backend/windmill-api-auth/src/lib.rs b/backend/windmill-api-auth/src/lib.rs index d92c3a5f2d..88eb11093a 100644 --- a/backend/windmill-api-auth/src/lib.rs +++ b/backend/windmill-api-auth/src/lib.rs @@ -67,8 +67,9 @@ pub struct ApiAuthed { /// `label-*` string. Only `username_override_from_label` sets it. pub username_override_is_token_label: bool, /// Whether the request authenticated with the session token minted at browser login. - /// Only `trigger_or_fallback` reads it — see `is_session_label` for why it attributes - /// rather than proves, and must not gate authority. + /// Read by `trigger_or_fallback` and by `TriggerSource::of_request` (which attributes a + /// trigger mutation to the UI) — see `is_session_label` for why it attributes rather than + /// proves, and must not gate authority. pub is_session_token: bool, pub token_prefix: Option, pub read_only: bool, diff --git a/backend/windmill-api-auth/src/scopes.rs b/backend/windmill-api-auth/src/scopes.rs index ed273c7d15..26dd523294 100644 --- a/backend/windmill-api-auth/src/scopes.rs +++ b/backend/windmill-api-auth/src/scopes.rs @@ -274,6 +274,7 @@ pub enum ScopeDomain { // Native trigger domains NativeTriggers, + TriggersHistory, // System domains Audit, @@ -335,6 +336,7 @@ impl ScopeDomain { Self::PostgresTriggers => "postgres_triggers", Self::EmailTriggers => "email_triggers", Self::NativeTriggers => "native_triggers", + Self::TriggersHistory => "triggers_history", Self::Audit => "audit", Self::Settings => "settings", Self::Workers => "workers", @@ -401,6 +403,7 @@ impl ScopeDomain { "indexer" | "srch" => Some(Self::Indexer), "teams" => Some(Self::Teams), "native_triggers" => Some(Self::NativeTriggers), + "triggers_history" => Some(Self::TriggersHistory), "git_sync" | "github_app" => Some(Self::GitSync), "capture" => Some(Self::Capture), "drafts" => Some(Self::Drafts), @@ -689,27 +692,49 @@ fn extract_domain_from_route( ))) } -const RUN_WHITELISTED_GET_PATHS: [&'static str; 20] = [ +/// The reads a `jobs:run` scope implies: following, by id, a run the token started. +/// Every entry is keyed by a job id and confines an authenticated caller to its own +/// runnable — through `require_job_read_access`, through its own `jobs:run:flows:` +/// check, or, where an approval token or resume secret bypasses that gate, through a +/// direct `require_job_within_run_scope`. The one exception is +/// `jobs_u/get_root_job_id/`, which has no check at all but discloses only flow lineage, +/// to anyone, authenticated or not. Workspace-wide enumeration (`jobs/list`, counts, +/// exports) and credential minting (`job_view_token`) are deliberately absent — those are +/// `jobs:read`. Keep by-id read routes here in sync as they are added, or a run token +/// loses the ability to follow its own run through them. +const RUN_WHITELISTED_GET_PATHS: [&'static str; 32] = [ "jobs_u/get_flow/", "jobs_u/get_root_job_id/", "jobs_u/get/", "jobs_u/get_logs/", + "jobs_u/get_completed_logs_tail/", "jobs_u/get_flow_all_logs/", + "jobs_u/get_flow_all_logs_structured/", + "jobs_u/get_flow_all_results/", "jobs_u/get_args/", "jobs_u/get_flow_debug_info/", "jobs_u/completed/get/", "jobs_u/completed/get_result/", "jobs_u/completed/get_result_maybe/", + "jobs_u/completed/get_timing/", + "jobs_u/dispatch_events/", "jobs_u/getupdate/", "jobs_u/getupdate_sse/", "jobs_u/get_log_file/", + "jobs/run_progress/", + "jobs/dbt_graph/", + "jobs/dbt_resumable/", + "jobs/dbt_resumable_script/p/", "jobs/result_by_id/", "jobs/resume_urls/", "jobs/flow/user_states/", "jobs/job_signature/", + "jobs/wac_approval_urls/", "jobs/completed/get/", "jobs/completed/get_result/", "jobs/completed/get_result_maybe/", + "jobs/completed/get_timing/", + "jobs/get_otel_traces/", ]; /// Sentinel scope in app embed tokens. Grants nothing itself; `check_route_access` @@ -826,6 +851,60 @@ fn resource_metadata_route_allowed(suffix: &str) -> bool { || suffix.starts_with("resources/type/") } +/// The `jobs:run` scopes a token's job reads are confined to, or `None` when they are +/// not confined to particular runnables. +/// +/// A run scope is what the trigger UI mints per script or flow and hands to a webhook +/// caller / CI job: it may start the runnables it names and follow those runs, so its +/// by-id job reads must stay within what it can start (enforced by +/// `require_job_read_access`). Both the path (`jobs:run:flows:f/team/etl`) and the +/// kind-only (`jobs:run:scripts`, which legacy `jobs:runscript` tokens carry) forms +/// confine, since `ScopeDefinition::includes` already matches a candidate +/// `jobs:run::` against either. +/// +/// Returns `None` — unconfined — when the token is effectively unscoped, or carries a +/// jobs scope that grants job reads in its own right: `jobs:read`/`jobs:write`, or a +/// bare `jobs:run` (it can start anything, so confining its reads to "what it may run" +/// would restrict nothing). +pub fn job_read_run_confinement(scopes: Option<&[String]>) -> Option> { + let mut confinement = Vec::new(); + for scope in scopes? + .iter() + .filter(|s| !s.starts_with("if_jobs:filter_tags:")) + { + let Ok(scope) = ScopeDefinition::from_scope_string(scope) else { + continue; + }; + if ScopeDomain::from_str(&scope.domain) != Some(ScopeDomain::Jobs) { + continue; + } + match ScopeAction::from_str(&scope.action) { + Some(ScopeAction::Run) if scope.kind.is_some() || scope.resource.is_some() => { + confinement.push(scope) + } + Some(_) => return None, + None => continue, + } + } + (!confinement.is_empty()).then_some(confinement) +} + +/// Whether a job that ran `runnable_path` as `kind` (`scripts` or `flows`) is inside a +/// [`job_read_run_confinement`] set. +pub fn run_confinement_admits( + confinement: &[ScopeDefinition], + kind: &str, + runnable_path: &str, +) -> bool { + let required = ScopeDefinition::new( + ScopeDomain::Jobs.as_str(), + ScopeAction::Run.as_str(), + Some(kind), + Some(vec![runnable_path.to_string()]), + ); + confinement.iter().any(|scope| scope.includes(&required)) +} + fn scope_grants_access( scope: &ScopeDefinition, required_domain: ScopeDomain, @@ -862,15 +941,26 @@ fn scope_grants_access( return Ok(true); } - if !scope_action.includes(&required_action) - && !(scope_domain == ScopeDomain::Jobs - && required_action == ScopeAction::Read - && route_path.is_some_and(|p| { - RUN_WHITELISTED_GET_PATHS - .iter() - .any(|path| p.starts_with(path)) - })) + // `jobs:run` is a grant to *start* a runnable. The only reads it implies are the + // by-id routes a caller needs to follow the run it started + // (`RUN_WHITELISTED_GET_PATHS`) — never workspace-wide enumeration (`jobs/list`, + // counts, exports), which is what `jobs:read` is for. Those by-id reads are in turn + // confined to the runnable a path-scoped token names, by `require_job_read_access`. + // `ScopeAction::Run.includes(&Read)` (which exists so `apps:run` can fetch the app + // it runs) must not reach this domain, so decide it here rather than falling + // through to the hierarchy below. + if scope_domain == ScopeDomain::Jobs + && scope_action == ScopeAction::Run + && required_action == ScopeAction::Read { + return Ok(route_path.is_some_and(|p| { + RUN_WHITELISTED_GET_PATHS + .iter() + .any(|path| p.starts_with(path)) + })); + } + + if !scope_action.includes(&required_action) { return Ok(false); } @@ -1099,6 +1189,65 @@ mod tests { .is_err()); } + #[test] + fn jobs_run_reads_are_limited_to_the_by_id_poll_routes() { + let job = "/api/w/test/jobs_u/completed/get_result/019ff012-6b1e-0d6b-fc0d-0c85d34d9cec"; + let list = "/api/w/test/jobs/list"; + for scope in ["jobs:run", "jobs:run:scripts:u/admin/script"] { + // Following the run it started stays available... + assert!( + check_route_access(&[scope.to_string()], job, "GET").is_ok(), + "{scope} must reach the by-id job poll routes" + ); + // ...but a run grant is not a licence to enumerate the workspace's jobs. + assert!( + check_route_access(&[scope.to_string()], list, "GET").is_err(), + "{scope} must not reach jobs/list" + ); + } + assert!(check_route_access(&["jobs:read".to_string()], list, "GET").is_ok()); + } + + #[test] + fn run_scopes_confine_job_reads_by_kind_and_path() { + let confinement = + job_read_run_confinement(Some(&["jobs:run:flows:f/team/*".to_string()])).unwrap(); + assert!(run_confinement_admits(&confinement, "flows", "f/team/etl")); + // Right path, wrong kind — a script named like the flow is not the flow. + assert!(!run_confinement_admits( + &confinement, + "scripts", + "f/team/etl" + )); + assert!(!run_confinement_admits( + &confinement, + "flows", + "f/other/etl" + )); + + // A kind-only scope confines to that kind, at any path. + let kind_only = job_read_run_confinement(Some(&["jobs:run:scripts".to_string()])).unwrap(); + assert!(run_confinement_admits(&kind_only, "scripts", "u/admin/anything")); + assert!(!run_confinement_admits(&kind_only, "flows", "f/team/etl")); + + // Scopes that grant job reads in their own right leave reads unconfined. + for scopes in [ + vec!["jobs:read".to_string()], + vec!["jobs:run".to_string()], + vec![ + "jobs:run:scripts:u/admin/script".to_string(), + "jobs:read".to_string(), + ], + vec!["if_jobs:filter_tags:deno".to_string()], + ] { + assert!( + job_read_run_confinement(Some(&scopes)).is_none(), + "{scopes:?} must not confine job reads" + ); + } + assert!(job_read_run_confinement(None).is_none()); + } + #[test] fn test_new_domain_parsing() { // Test that new domains are properly parsed diff --git a/backend/windmill-api-groups/src/groups.rs b/backend/windmill-api-groups/src/groups.rs index d2a6c7d16e..69b50e4b83 100644 --- a/backend/windmill-api-groups/src/groups.rs +++ b/backend/windmill-api-groups/src/groups.rs @@ -514,6 +514,143 @@ async fn update_igroup( Ok(format!("Updated group {}", name)) } +/// Workspaces whose auto-assignment config references any of `groups`. +/// +/// Reads `workspace_settings` across the whole instance without checking the caller's rights. +/// Callers must have established superadmin beforehand; the result leaks which workspaces are +/// configured with a given instance group. +/// +/// This and every reconcile call site are gated on `private` alone, NOT `enterprise`: CE +/// builds ship `private` without `enterprise`, and gating on `enterprise` would scrub +/// references while stranding the affected workspace members on CE. +#[cfg(feature = "private")] +pub async fn workspaces_referencing_instance_groups( + groups: &[String], + tx: &mut Transaction<'_, Postgres>, +) -> Result> { + if groups.is_empty() { + return Ok(vec![]); + } + + let workspaces = sqlx::query_scalar!( + "SELECT workspace_id FROM workspace_settings WHERE auto_invite->'instance_groups' ?| $1", + groups + ) + .fetch_all(&mut **tx) + .await?; + + Ok(workspaces) +} + +/// Compute and advisory-lock every workspace whose auto-assignment config references any of +/// `groups`. Mutation paths call this after locking their `instance_group` rows and before +/// any other row lock — the hierarchy is group rows → workspace advisory locks → all other +/// row locks (see `reconcile_workspace_instance_groups`). Same authorization contract as +/// `workspaces_referencing_instance_groups`. +#[cfg(feature = "private")] +pub async fn lock_workspaces_referencing_instance_groups( + groups: &[String], + tx: &mut Transaction<'_, Postgres>, +) -> Result> { + use windmill_api_workspaces::workspaces_ee::lock_instance_group_workspaces; + + let workspaces = workspaces_referencing_instance_groups(groups, tx).await?; + lock_instance_group_workspaces(&workspaces, tx).await?; + Ok(workspaces) +} + +/// Drop `groups` from every workspace's instance-group auto-assignment config. +/// +/// Workspaces reference instance groups by name in `workspace_settings.auto_invite`, and +/// nothing in the schema ties those references to `instance_group` rows. A deleted group whose +/// name is left behind here silently re-acquires its members if a group of the same name is +/// created later. +/// +/// Mutates every workspace's settings, so callers must have established superadmin first. +/// Deliberately not audited per workspace: the mutation is instance-scoped and recorded by +/// the caller's global igroup audit event. +pub async fn remove_instance_groups_from_workspace_settings( + groups: &[String], + tx: &mut Transaction<'_, Postgres>, +) -> Result<()> { + if groups.is_empty() { + return Ok(()); + } + + // Row filter must stay `?|`: it yields false on a JSON `null` instance_groups, where + // jsonb_array_elements_text would instead raise and abort the whole transaction; the + // jsonb_typeof guard rules out the same class of value for the roles object. The filter is + // not index-backed — the GIN index covers the auto_invite column, not this expression — + // which is acceptable since workspace_settings holds one row per workspace. + sqlx::query!( + r#"UPDATE workspace_settings SET + auto_invite = jsonb_set( + jsonb_set( + COALESCE(auto_invite, '{}'::jsonb), + '{instance_groups}', + (SELECT COALESCE(jsonb_agg(elem), '[]'::jsonb) + FROM jsonb_array_elements(COALESCE(auto_invite->'instance_groups', '[]'::jsonb)) elem + WHERE elem #>> '{}' <> ALL($1)) + ), + '{instance_groups_roles}', + CASE WHEN jsonb_typeof(auto_invite->'instance_groups_roles') = 'object' + THEN (auto_invite->'instance_groups_roles') - $1::text[] + ELSE '{}'::jsonb + END + ) + WHERE auto_invite->'instance_groups' ?| $1"#, + groups + ) + .execute(&mut **tx) + .await?; + + Ok(()) +} + +/// Follow an instance-group rename through every workspace's auto-assignment config. +/// +/// Workspaces reference instance groups by name, so a rename that leaves the old name behind +/// strands those references: the reconciler resolves membership from the groups a workspace +/// references, and a name that no longer matches any group reads as "no members", which would +/// evict everyone granted through it on the next reconcile. +/// +/// Mutates every workspace's settings, so callers must have established superadmin first. +/// Deliberately not audited per workspace: the mutation is instance-scoped and recorded by +/// the caller's global igroup audit event. +pub async fn rename_instance_group_in_workspace_settings( + old_name: &str, + new_name: &str, + tx: &mut Transaction<'_, Postgres>, +) -> Result<()> { + // Row filter must stay `?`: it yields false on a JSON `null` instance_groups, where + // jsonb_array_elements would instead raise and abort the whole transaction. + sqlx::query!( + r#"UPDATE workspace_settings SET + auto_invite = jsonb_set( + jsonb_set( + COALESCE(auto_invite, '{}'::jsonb), + '{instance_groups}', + (SELECT COALESCE(jsonb_agg( + CASE WHEN elem #>> '{}' = $1 THEN to_jsonb($2::text) ELSE elem END), '[]'::jsonb) + FROM jsonb_array_elements(COALESCE(auto_invite->'instance_groups', '[]'::jsonb)) elem) + ), + '{instance_groups_roles}', + CASE WHEN COALESCE(auto_invite->'instance_groups_roles', '{}'::jsonb) ? $1 + THEN (COALESCE(auto_invite->'instance_groups_roles', '{}'::jsonb) - $1) + || jsonb_build_object($2::text, auto_invite->'instance_groups_roles'->$1) + ELSE COALESCE(auto_invite->'instance_groups_roles', '{}'::jsonb) + END + ) + WHERE auto_invite->'instance_groups' ? $1"#, + old_name, + new_name + ) + .execute(&mut **tx) + .await?; + + Ok(()) +} + async fn delete_igroup( authed: ApiAuthed, Extension(db): Extension, @@ -522,9 +659,10 @@ async fn delete_igroup( require_super_admin(&db, &authed.email).await?; let mut tx: Transaction<'_, Postgres> = db.begin().await?; - // Fetch group's instance_role and members before deletion + // FOR UPDATE: the group row is the group-level mutex, taken before the workspace + // advisory locks (see reconcile_workspace_instance_groups). let group_role = sqlx::query_scalar!( - "SELECT instance_role FROM instance_group WHERE name = $1", + "SELECT instance_role FROM instance_group WHERE name = $1 FOR UPDATE", &name ) .fetch_optional(&mut *tx) @@ -539,6 +677,13 @@ async fn delete_igroup( vec![] }; + // Captured and advisory-locked before the settings update strips the group from them. + #[cfg(feature = "private")] + let affected_workspaces = + lock_workspaces_referencing_instance_groups(std::slice::from_ref(&name), &mut tx).await?; + + remove_instance_groups_from_workspace_settings(std::slice::from_ref(&name), &mut tx).await?; + sqlx::query!("DELETE FROM email_to_igroup WHERE igroup = $1", name) .execute(&mut *tx) .await?; @@ -553,6 +698,12 @@ async fn delete_igroup( apply_instance_role(email, effective_role.as_deref(), &mut tx).await?; } + #[cfg(feature = "private")] + { + use windmill_api_workspaces::workspaces_ee::reconcile_workspace_instance_groups; + reconcile_workspace_instance_groups(&affected_workspaces, &mut tx, &authed).await?; + } + audit_log( &mut *tx, &authed, @@ -823,12 +974,22 @@ async fn add_user_igroup( let mut tx: Transaction<'_, Postgres> = db.begin().await?; - let group_opt = sqlx::query_scalar!("SELECT name FROM instance_group WHERE name = $1", name) - .fetch_optional(&mut *tx) - .await?; + // FOR UPDATE: the group row is the group-level mutex, taken before the workspace + // advisory locks (see reconcile_workspace_instance_groups). + let group_opt = sqlx::query_scalar!( + "SELECT name FROM instance_group WHERE name = $1 FOR UPDATE", + name + ) + .fetch_optional(&mut *tx) + .await?; not_found_if_none(group_opt, "IGroup", &name)?; + // Before the membership insert's row lock. + #[cfg(feature = "private")] + let affected_workspaces = + lock_workspaces_referencing_instance_groups(std::slice::from_ref(&name), &mut tx).await?; + sqlx::query!( "INSERT INTO email_to_igroup (email, igroup) VALUES ($1, $2) ON CONFLICT DO NOTHING", email, @@ -848,86 +1009,17 @@ async fn add_user_igroup( ) .await?; - // Sync user to workspaces configured with this instance group - #[cfg(all(feature = "private", feature = "enterprise"))] - { - use windmill_api_workspaces::workspaces_ee::auto_add_user; - use windmill_common::users::compute_highest_workspace_role; - - // Find all instance groups this user belongs to (includes the newly added group) - let user_igroups: Vec = sqlx::query_scalar!( - "SELECT igroup FROM email_to_igroup WHERE email = $1", - &email - ) - .fetch_all(&mut *tx) - .await?; - - let workspaces = sqlx::query!( - r#" - SELECT workspace_id, - auto_invite->'instance_groups_roles' as instance_groups_roles, - auto_invite->'instance_groups' as instance_groups_json - FROM workspace_settings - WHERE auto_invite->'instance_groups' ? $1 - "#, - &name - ) - .fetch_all(&mut *tx) - .await?; - - for ws in workspaces { - let roles: std::collections::HashMap = ws - .instance_groups_roles - .and_then(|r| serde_json::from_value(r).ok()) - .unwrap_or_default(); - - let ws_configured_groups: Vec = ws - .instance_groups_json - .and_then(|ig| serde_json::from_value(ig).ok()) - .unwrap_or_default(); - - let (best_group, is_admin, is_operator) = - compute_highest_workspace_role(&user_igroups, &ws_configured_groups, &roles); - - let instance_group_source = serde_json::json!({ - "source": "instance_group", - "group": &best_group - }); - - // auto_add_user creates the user if they don't exist (ON CONFLICT DO NOTHING). - // The operator flag here doesn't matter for the final state — the UPDATE below - // always sets the correct is_admin/operator based on the highest-precedence role. - auto_add_user( - &email, - &ws.workspace_id, - &false, - &mut tx, - &authed, - Some(instance_group_source.clone()), - ) - .await?; - - // Set the correct role based on highest precedence across all groups. - // For new users, auto_add_user already stored added_via with source=instance_group, - // so this UPDATE will match. For existing instance_group users, it upgrades/corrects - // the role. Manually-added users (added_via is NULL or non-instance_group) are not affected. - sqlx::query!( - "UPDATE usr SET is_admin = $1, operator = $2, added_via = $3 WHERE workspace_id = $4 AND email = $5 AND added_via->>'source' = 'instance_group'", - is_admin, - is_operator, - &instance_group_source, - &ws.workspace_id, - &email - ) - .execute(&mut *tx) - .await?; - } - } - // Apply instance-level role from group membership let effective_role = compute_effective_instance_role(&email, &mut tx).await?; apply_instance_role(&email, effective_role.as_deref(), &mut tx).await?; + // Sync workspace membership derived from this instance group. + #[cfg(feature = "private")] + { + use windmill_api_workspaces::workspaces_ee::reconcile_workspace_instance_groups; + reconcile_workspace_instance_groups(&affected_workspaces, &mut tx, &authed).await?; + } + tx.commit().await?; Ok(format!("Added {} to igroup {}", email, name)) } @@ -1100,12 +1192,22 @@ async fn remove_user_igroup( require_super_admin(&db, &authed.email).await?; let mut tx = db.begin().await?; - let group_opt = sqlx::query_scalar!("SELECT name FROM instance_group WHERE name = $1", name,) - .fetch_optional(&mut *tx) - .await?; + // FOR UPDATE: the group row is the group-level mutex, taken before the workspace + // advisory locks (see reconcile_workspace_instance_groups). + let group_opt = sqlx::query_scalar!( + "SELECT name FROM instance_group WHERE name = $1 FOR UPDATE", + name, + ) + .fetch_optional(&mut *tx) + .await?; not_found_if_none(group_opt, "IGroup", &name)?; + // Before the membership delete's row lock. + #[cfg(feature = "private")] + let affected_workspaces = + lock_workspaces_referencing_instance_groups(std::slice::from_ref(&name), &mut tx).await?; + sqlx::query!( "DELETE FROM email_to_igroup WHERE email = $1 AND igroup = $2", email, @@ -1125,17 +1227,19 @@ async fn remove_user_igroup( ) .await?; - // Remove user from workspaces where they were added via this instance group - #[cfg(all(feature = "private", feature = "enterprise"))] - { - use windmill_api_workspaces::workspaces_ee::remove_users_from_instance_group_workspaces; - remove_users_from_instance_group_workspaces(&email, &name, &mut tx).await?; - } - // Recompute instance-level role after group removal let effective_role = compute_effective_instance_role(&email, &mut tx).await?; apply_instance_role(&email, effective_role.as_deref(), &mut tx).await?; + // Re-derive workspace membership now that the base tables reflect the removal: drops the + // user where this group was their only access source, or re-roles them from the groups + // they still belong to. + #[cfg(feature = "private")] + { + use windmill_api_workspaces::workspaces_ee::reconcile_workspace_instance_groups; + reconcile_workspace_instance_groups(&affected_workspaces, &mut tx, &authed).await?; + } + tx.commit().await?; Ok(format!("Removed {} from igroup {}", email, name)) } @@ -1265,6 +1369,37 @@ async fn overwrite_igroups( require_super_admin(&db, &authed.email).await?; let mut tx = db.begin().await?; + // The import replaces the whole group catalog, so the whole-table lock is its + // group-mutex phase, taken first like every path's group locks (see + // reconcile_workspace_instance_groups). Per-row FOR UPDATE would miss rows committed + // after the scan, which the unqualified deletes below would then lock after the + // workspace locks — the inverted order. EXCLUSIVE conflicts with the writes and the + // FOR UPDATE of every other mutation path while leaving plain reads unblocked. + sqlx::query("LOCK TABLE instance_group IN EXCLUSIVE MODE") + .execute(&mut *tx) + .await?; + + let imported_names: Vec = igroups.iter().map(|g| g.name.clone()).collect(); + // NULL-safe and correct for an empty import: `name <> ALL('{}')` is true for every row. + let previous_names: Vec = sqlx::query_scalar!( + "SELECT name FROM instance_group WHERE name <> ALL($1)", + &imported_names + ) + .fetch_all(&mut *tx) + .await?; + + // Membership of retained groups is wiped and re-imported below, so workspaces referencing + // either side of the import may see their projection change. Captured and advisory-locked + // before the settings update strips the dropped groups from them. + #[cfg(feature = "private")] + let affected_workspaces = { + let mut all_names = previous_names.clone(); + all_names.extend(imported_names.iter().cloned()); + lock_workspaces_referencing_instance_groups(&all_names, &mut tx).await? + }; + + remove_instance_groups_from_workspace_settings(&previous_names, &mut tx).await?; + sqlx::query!("DELETE FROM email_to_igroup") .execute(&mut *tx) .await?; @@ -1325,6 +1460,15 @@ async fn overwrite_igroups( apply_instance_role(email, None, &mut tx).await?; } + // Runs after the re-insert so the reconciler judges membership against the imported + // state: a member who moved from a dropped group to a retained one is re-roled in place + // instead of losing workspace access. + #[cfg(feature = "private")] + { + use windmill_api_workspaces::workspaces_ee::reconcile_workspace_instance_groups; + reconcile_workspace_instance_groups(&affected_workspaces, &mut tx, &authed).await?; + } + audit_log( &mut *tx, &authed, diff --git a/backend/windmill-api-integration-tests/tests/assets.rs b/backend/windmill-api-integration-tests/tests/assets.rs new file mode 100644 index 0000000000..e1e7645e52 --- /dev/null +++ b/backend/windmill-api-integration-tests/tests/assets.rs @@ -0,0 +1,248 @@ +use serde_json::{json, Value}; +use sqlx::{Pool, Postgres}; +use uuid::Uuid; +use windmill_test_utils::*; + +fn client() -> reqwest::Client { + reqwest::Client::new() +} + +fn bearer(builder: reqwest::RequestBuilder, token: &str) -> reqwest::RequestBuilder { + builder.header("Authorization", format!("Bearer {token}")) +} + +async fn insert_job(db: &Pool, parent: Option, tag: &str) -> anyhow::Result { + let id = Uuid::new_v4(); + sqlx::query( + "INSERT INTO v2_job (id, workspace_id, tag, created_by, permissioned_as, \ + permissioned_as_email, kind, parent_job, same_worker, visible_to_owner) \ + VALUES ($1, 'test-workspace', $3, 'test-user', 'u/test-user', \ + 'test@windmill.dev', 'script', $2, false, true)", + ) + .bind(id) + .bind(parent) + .bind(tag) + .execute(db) + .await?; + Ok(id) +} + +/// `access_type` is `None` for a detection that could not tell read from write +/// — how a resource passed in a job's arguments is recorded. +async fn insert_job_asset( + db: &Pool, + job: Uuid, + path: &str, + access_type: Option<&str>, +) -> anyhow::Result<()> { + sqlx::query( + "INSERT INTO asset (workspace_id, path, kind, usage_access_type, usage_path, usage_kind) \ + VALUES ('test-workspace', $1, 's3object', $2::text::asset_access_type, $3, 'job')", + ) + .bind(path) + .bind(access_type) + .bind(job.to_string()) + .execute(db) + .await?; + Ok(()) +} + +/// A run reports what its whole job tree touched: runtime detection records +/// against the job that did the operation, which for a flow step or a +/// workflow-as-code task is never the job the user opened. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_list_run_assets_covers_child_jobs(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let ws = format!("http://localhost:{port}/api/w/test-workspace"); + + let parent = insert_job(&db, None, "other").await?; + let child = insert_job(&db, Some(parent), "other").await?; + let unrelated = insert_job(&db, None, "other").await?; + + insert_job_asset(&db, parent, "/data/shared.json", Some("r")).await?; + insert_job_asset(&db, child, "/data/shared.json", Some("w")).await?; + insert_job_asset(&db, child, "/data/child_only.json", Some("w")).await?; + insert_job_asset(&db, parent, "/data/from_args.json", None).await?; + insert_job_asset(&db, child, "/data/from_args.json", Some("w")).await?; + insert_job_asset(&db, unrelated, "/data/unrelated.json", Some("w")).await?; + + let resp = bearer( + client().get(format!("{ws}/jobs/run_assets/{parent}")), + "SECRET_TOKEN", + ) + .send() + .await?; + assert_eq!(resp.status().as_u16(), 200); + let body: Value = resp.json().await?; + assert_eq!(body["truncated"], json!(false)); + assert_eq!( + body["assets"], + json!([ + { "path": "/data/child_only.json", "kind": "s3object", "access_type": "w" }, + // The parent recorded no access type for this one; that must not erase + // the child's. + { "path": "/data/from_args.json", "kind": "s3object", "access_type": "w" }, + { "path": "/data/shared.json", "kind": "s3object", "access_type": "rw" }, + ]), + "parent should report its own and its child's assets, with access types merged" + ); + + let resp = bearer( + client().get(format!("{ws}/jobs/run_assets/{child}")), + "SECRET_TOKEN", + ) + .send() + .await?; + assert_eq!(resp.status().as_u16(), 200); + let body: Value = resp.json().await?; + assert_eq!( + body["assets"], + json!([ + { "path": "/data/child_only.json", "kind": "s3object", "access_type": "w" }, + { "path": "/data/from_args.json", "kind": "s3object", "access_type": "w" }, + { "path": "/data/shared.json", "kind": "s3object", "access_type": "w" }, + ]), + "a child should report only what it touched itself" + ); + + // `asset` has no RLS of its own, so the job read gate is the only thing + // standing between another member and these paths. + let resp = bearer( + client().get(format!("{ws}/jobs/run_assets/{parent}")), + "SECRET_TOKEN_2", + ) + .send() + .await?; + assert_eq!( + resp.status().as_u16(), + 403, + "a member who cannot read the run must not read its assets" + ); + + // ...and a share link is what lets that same member in. The tree is walked + // outside the caller's RLS precisely so this works, since the token grants + // access their own permissions do not. + let view_token: String = bearer( + client().get(format!("{ws}/jobs/job_view_token/{parent}")), + "SECRET_TOKEN", + ) + .send() + .await? + .text() + .await?; + let resp = bearer( + client().get(format!("{ws}/jobs/run_assets/{parent}")), + "SECRET_TOKEN_2", + ) + .header("X-View-Token", view_token) + .send() + .await?; + assert_eq!(resp.status().as_u16(), 200); + let body: Value = resp.json().await?; + assert_eq!( + body["assets"].as_array().map(|a| a.len()), + Some(3), + "a share-link viewer should see the whole tree's assets" + ); + + Ok(()) +} + +/// The read gate only checks the root job's tag, so the walk has to keep a +/// tag-scoped token out of descendants outside its scope — without hiding the +/// ones below them, which the token could have asked for directly. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_list_run_assets_scopes_descendants_by_tag(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let ws = format!("http://localhost:{port}/api/w/test-workspace"); + + sqlx::query( + "INSERT INTO token (token_hash, token_prefix, token, email, label, super_admin, scopes) \ + VALUES (encode(sha256('TAG_TOKEN'::bytea), 'hex'), 'TAG_TOK', 'TAG_TOKEN', \ + 'test@windmill.dev', 'tag-only', true, ARRAY['if_jobs:filter_tags:other'])", + ) + .execute(&db) + .await?; + + let parent = insert_job(&db, None, "other").await?; + let in_scope = insert_job(&db, Some(parent), "other").await?; + let out_of_scope = insert_job(&db, Some(parent), "deno").await?; + let below_out_of_scope = insert_job(&db, Some(out_of_scope), "other").await?; + insert_job_asset(&db, in_scope, "/data/in_scope.json", Some("w")).await?; + insert_job_asset(&db, out_of_scope, "/data/out_of_scope.json", Some("w")).await?; + insert_job_asset(&db, below_out_of_scope, "/data/nested.json", Some("w")).await?; + + let resp = bearer( + client().get(format!("{ws}/jobs/run_assets/{parent}")), + "TAG_TOKEN", + ) + .send() + .await?; + assert_eq!(resp.status().as_u16(), 200); + let body: Value = resp.json().await?; + assert_eq!( + body["assets"], + json!([ + { "path": "/data/in_scope.json", "kind": "s3object", "access_type": "w" }, + { "path": "/data/nested.json", "kind": "s3object", "access_type": "w" }, + ]), + "a tag-scoped token must not read assets of descendants outside its tags, \ + but must still reach in-scope jobs below them" + ); + + Ok(()) +} + +/// A fan-out run can touch more assets than one response should carry, and the +/// cut must be reported rather than served as if it were the whole list. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_list_run_assets_caps_the_list(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let ws = format!("http://localhost:{port}/api/w/test-workspace"); + + // Three child jobs touching the same 1200 assets: the cap counts assets, and + // the retention allows ten job rows per asset, so a row-counted cap would cut + // this at a third of the list. + let parent = insert_job(&db, None, "other").await?; + for _ in 0..3 { + let child = insert_job(&db, Some(parent), "other").await?; + sqlx::query( + "INSERT INTO asset (workspace_id, path, kind, usage_access_type, usage_path, usage_kind) \ + SELECT 'test-workspace', '/out/' || lpad(g::text, 6, '0') || '.json', 's3object', 'w', \ + $1, 'job' FROM generate_series(1, 1200) g", + ) + .bind(child.to_string()) + .execute(&db) + .await?; + } + + let resp = bearer( + client().get(format!("{ws}/jobs/run_assets/{parent}")), + "SECRET_TOKEN", + ) + .send() + .await?; + assert_eq!(resp.status().as_u16(), 200); + let body: Value = resp.json().await?; + assert_eq!(body["truncated"], json!(true)); + let assets = body["assets"].as_array().expect("assets array"); + assert_eq!(assets.len(), 1000); + assert_eq!( + assets[0], + json!({ "path": "/out/000001.json", "kind": "s3object", "access_type": "w" }), + "the cap keeps the head of the ordered list, with its access type merged" + ); + // Three jobs touched each asset, so a cap counting rows rather than assets + // would fill the list with repeats and stop around /out/000334.json. + assert_eq!( + assets[999]["path"], "/out/001000.json", + "the cap counts assets, not asset rows" + ); + Ok(()) +} diff --git a/backend/windmill-api-integration-tests/tests/fixtures/resources_test.sql b/backend/windmill-api-integration-tests/tests/fixtures/resources_test.sql index 1e04195ae0..2f101662fe 100644 --- a/backend/windmill-api-integration-tests/tests/fixtures/resources_test.sql +++ b/backend/windmill-api-integration-tests/tests/fixtures/resources_test.sql @@ -69,6 +69,12 @@ VALUES ('test-workspace', 'u/test-user/fileset_resource', '{"config.yaml": "key: value", "data/input.json": "{\"items\": []}"}', 'A fileset resource', 'test_fileset', '{}', 'test-user'); +-- === list_search value cap test data === + +INSERT INTO resource (workspace_id, path, value, description, resource_type, extra_perms, created_by) +VALUES ('test-workspace', 'u/test-user/oversized_resource', jsonb_build_object('big', repeat('x', 5000)), + 'Larger than the list_search cap', 'object', '{}', 'test-user'); + -- === mcp_tools test data === INSERT INTO resource (workspace_id, path, value, description, resource_type, extra_perms, created_by) diff --git a/backend/windmill-api-integration-tests/tests/groups.rs b/backend/windmill-api-integration-tests/tests/groups.rs index 0fd8ee7e08..f86522aae3 100644 --- a/backend/windmill-api-integration-tests/tests/groups.rs +++ b/backend/windmill-api-integration-tests/tests/groups.rs @@ -159,12 +159,7 @@ async fn test_group_endpoints(db: Pool) -> anyhow::Result<()> { .send() .await .unwrap(); - assert_eq!( - resp.status(), - 200, - "create igroup: {}", - resp.text().await? - ); + assert_eq!(resp.status(), 200, "create igroup: {}", resp.text().await?); // --- list instance groups --- let resp = authed(client().get(format!("{global_base}/list"))) @@ -199,12 +194,7 @@ async fn test_group_endpoints(db: Pool) -> anyhow::Result<()> { .send() .await .unwrap(); - assert_eq!( - resp.status(), - 200, - "update igroup: {}", - resp.text().await? - ); + assert_eq!(resp.status(), 200, "update igroup: {}", resp.text().await?); // verify update let resp = authed(client().get(format!("{global_base}/get/test_igroup"))) @@ -220,12 +210,7 @@ async fn test_group_endpoints(db: Pool) -> anyhow::Result<()> { .send() .await .unwrap(); - assert_eq!( - resp.status(), - 200, - "adduser igroup: {}", - resp.text().await? - ); + assert_eq!(resp.status(), 200, "adduser igroup: {}", resp.text().await?); // verify membership let resp = authed(client().get(format!("{global_base}/get/test_igroup"))) @@ -243,13 +228,11 @@ async fn test_group_endpoints(db: Pool) -> anyhow::Result<()> { ); // --- removeuser from instance group --- - let resp = authed(client().post(format!( - "{global_base}/removeuser/test_igroup" - ))) - .json(&json!({"email": "test@windmill.dev"})) - .send() - .await - .unwrap(); + let resp = authed(client().post(format!("{global_base}/removeuser/test_igroup"))) + .json(&json!({"email": "test@windmill.dev"})) + .send() + .await + .unwrap(); assert_eq!(resp.status(), 200); // --- export (EE-gated) --- @@ -280,12 +263,7 @@ async fn test_group_endpoints(db: Pool) -> anyhow::Result<()> { .send() .await .unwrap(); - assert_eq!( - resp.status(), - 200, - "delete igroup: {}", - resp.text().await? - ); + assert_eq!(resp.status(), 200, "delete igroup: {}", resp.text().await?); // verify deleted let resp = authed(client().get(format!("{global_base}/list"))) @@ -297,3 +275,641 @@ async fn test_group_endpoints(db: Pool) -> anyhow::Result<()> { Ok(()) } + +/// Deleting an instance group must not revoke workspace access a member still holds through +/// another configured group. +/// +/// `added_via.group` records only the member's highest-precedence group, so any cleanup keyed +/// on that field alone evicts members who still qualify via a lower-precedence one. Membership +/// must be re-derived from all the groups the workspace still references. +#[cfg(feature = "private")] +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_delete_instance_group_preserves_access_via_other_group( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let global_base = format!("http://localhost:{port}/api/groups"); + let ws_base = format!("http://localhost:{port}/api/w/test-workspace/workspaces"); + + for g in ["igroup_a", "igroup_b"] { + let resp = authed(client().post(format!("{global_base}/create"))) + .json(&json!({ "name": g, "summary": g })) + .send() + .await?; + assert_eq!(resp.status(), 200, "create {g}"); + } + + // multi@ belongs to both groups; only_a@ only to the group that gets deleted. + for (g, email) in [ + ("igroup_a", "multi@example.com"), + ("igroup_b", "multi@example.com"), + ("igroup_a", "only_a@example.com"), + ] { + let resp = authed(client().post(format!("{global_base}/adduser/{g}"))) + .json(&json!({ "email": email })) + .send() + .await?; + assert_eq!(resp.status(), 200, "adduser {g}/{email}"); + } + + // igroup_a grants the higher-precedence role, so added_via lands on it. + let resp = authed(client().post(format!("{ws_base}/edit_instance_groups"))) + .json(&json!({ + "groups": ["igroup_a", "igroup_b"], + "roles": { "igroup_a": "admin", "igroup_b": "developer" } + })) + .send() + .await?; + assert_eq!( + resp.status(), + 200, + "edit_instance_groups: {}", + resp.text().await? + ); + + let (is_admin, via): (bool, Option) = sqlx::query_as( + "SELECT is_admin, added_via->>'group' FROM usr + WHERE workspace_id = 'test-workspace' AND email = 'multi@example.com'", + ) + .fetch_one(&db) + .await?; + assert!(is_admin, "multi@ should start as admin via igroup_a"); + assert_eq!(via.as_deref(), Some("igroup_a")); + + // Workspace state that must survive the group removal. `delete_workspace_user_internal` + // drops all of this, so a delete-and-re-add of a still-qualifying member loses it silently. + let username: String = sqlx::query_scalar( + "SELECT username FROM usr WHERE workspace_id = 'test-workspace' AND email = 'multi@example.com'", + ) + .fetch_one(&db) + .await?; + sqlx::query( + "INSERT INTO favorite (workspace_id, usr, path, favorite_kind) + VALUES ('test-workspace', $1, 'f/keep/me', 'script')", + ) + .bind(&username) + .execute(&db) + .await?; + sqlx::query( + "INSERT INTO draft (workspace_id, path, typ, value) + VALUES ('test-workspace', 'u/' || $1 || '/keep', 'script', '{}'::jsonb)", + ) + .bind(&username) + .execute(&db) + .await?; + + let resp = authed(client().delete(format!("{global_base}/delete/igroup_a"))) + .send() + .await?; + assert_eq!( + resp.status(), + 200, + "delete igroup_a: {}", + resp.text().await? + ); + + // Still a member, downgraded to igroup_b's role rather than evicted. + let (is_admin, is_operator, via): (bool, bool, Option) = sqlx::query_as( + "SELECT is_admin, operator, added_via->>'group' FROM usr + WHERE workspace_id = 'test-workspace' AND email = 'multi@example.com'", + ) + .fetch_one(&db) + .await?; + assert!(!is_admin, "multi@ should lose admin with igroup_a gone"); + assert!(!is_operator, "igroup_b grants developer, not operator"); + assert_eq!( + via.as_deref(), + Some("igroup_b"), + "added_via should re-point at the surviving group" + ); + + // Their workspace state is intact: they were never deleted and re-added. + let favorites: i64 = sqlx::query_scalar( + "SELECT count(*) FROM favorite WHERE workspace_id = 'test-workspace' AND path = 'f/keep/me'", + ) + .fetch_one(&db) + .await?; + assert_eq!( + favorites, 1, + "favorite must survive losing a non-sole group" + ); + let drafts: i64 = sqlx::query_scalar( + "SELECT count(*) FROM draft WHERE workspace_id = 'test-workspace' AND path LIKE 'u/%/keep'", + ) + .fetch_one(&db) + .await?; + assert_eq!(drafts, 1, "draft must survive losing a non-sole group"); + + // igroup_a was only_a@'s sole path in, so they are removed. + let remaining: i64 = sqlx::query_scalar( + "SELECT count(*) FROM usr + WHERE workspace_id = 'test-workspace' AND email = 'only_a@example.com'", + ) + .fetch_one(&db) + .await?; + assert_eq!(remaining, 0, "only_a@ should be removed with igroup_a"); + + // The deleted group leaves no dangling reference in either auto_invite field. + let (groups, roles): (serde_json::Value, serde_json::Value) = sqlx::query_as( + "SELECT auto_invite->'instance_groups', auto_invite->'instance_groups_roles' + FROM workspace_settings WHERE workspace_id = 'test-workspace'", + ) + .fetch_one(&db) + .await?; + assert_eq!(groups, json!(["igroup_b"]), "igroup_a should be stripped"); + assert_eq!( + roles, + json!({ "igroup_b": "developer" }), + "igroup_a's role entry should be stripped" + ); + + Ok(()) +} + +/// Removing a member from one instance group must re-derive their role from the groups they +/// still belong to, not leave the privileges the removed group granted. +/// +/// Still-qualifying members keep their `usr` row (deleting it would destroy their workspace +/// data), so the removal path must recompute that row's role — otherwise a member dropped +/// from an admin group keeps `is_admin` through the stale row. +#[cfg(feature = "private")] +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_remove_user_from_instance_group_rederives_role( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let global_base = format!("http://localhost:{port}/api/groups"); + let ws_base = format!("http://localhost:{port}/api/w/test-workspace/workspaces"); + + for g in ["role_a", "role_b"] { + let resp = authed(client().post(format!("{global_base}/create"))) + .json(&json!({ "name": g })) + .send() + .await?; + assert_eq!(resp.status(), 200, "create {g}"); + let resp = authed(client().post(format!("{global_base}/adduser/{g}"))) + .json(&json!({ "email": "demoted@example.com" })) + .send() + .await?; + assert_eq!(resp.status(), 200, "adduser {g}"); + } + + let resp = authed(client().post(format!("{ws_base}/edit_instance_groups"))) + .json(&json!({ + "groups": ["role_a", "role_b"], + "roles": { "role_a": "admin", "role_b": "developer" } + })) + .send() + .await?; + assert_eq!(resp.status(), 200, "edit: {}", resp.text().await?); + + let is_admin: bool = sqlx::query_scalar( + "SELECT is_admin FROM usr WHERE workspace_id = 'test-workspace' AND email = 'demoted@example.com'", + ) + .fetch_one(&db) + .await?; + assert!(is_admin, "should start admin via role_a"); + + // Drop them from the admin group only. + let resp = authed(client().post(format!("{global_base}/removeuser/role_a"))) + .json(&json!({ "email": "demoted@example.com" })) + .send() + .await?; + assert_eq!(resp.status(), 200, "removeuser: {}", resp.text().await?); + + let (is_admin, is_operator, via): (bool, bool, Option) = sqlx::query_as( + "SELECT is_admin, operator, added_via->>'group' FROM usr + WHERE workspace_id = 'test-workspace' AND email = 'demoted@example.com'", + ) + .fetch_one(&db) + .await?; + assert!( + !is_admin, + "admin granted by role_a must not survive removal from role_a" + ); + assert!(!is_operator, "role_b grants developer"); + assert_eq!(via.as_deref(), Some("role_b")); + + Ok(()) +} + +/// An overwrite import that moves a member from a dropped group to a retained one must keep +/// their workspace data. +/// +/// Qualification must be judged against the imported membership, not the pre-import state: +/// judged too early, the member's new group is not yet visible, they are deleted, and any +/// re-add creates a fresh row stripped of everything workspace-scoped. +#[cfg(all(feature = "private", feature = "enterprise"))] +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_overwrite_igroups_preserves_moved_member_data( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let global_base = format!("http://localhost:{port}/api/groups"); + let ws_base = format!("http://localhost:{port}/api/w/test-workspace/workspaces"); + + for g in ["move_from", "move_to"] { + let resp = authed(client().post(format!("{global_base}/create"))) + .json(&json!({ "name": g })) + .send() + .await?; + assert_eq!(resp.status(), 200, "create {g}"); + } + // Member starts only in move_from. + let resp = authed(client().post(format!("{global_base}/adduser/move_from"))) + .json(&json!({ "email": "mover@example.com" })) + .send() + .await?; + assert_eq!(resp.status(), 200); + + let resp = authed(client().post(format!("{ws_base}/edit_instance_groups"))) + .json(&json!({ + "groups": ["move_from", "move_to"], + "roles": { "move_from": "developer", "move_to": "developer" } + })) + .send() + .await?; + assert_eq!(resp.status(), 200, "edit: {}", resp.text().await?); + + let username: String = sqlx::query_scalar( + "SELECT username FROM usr WHERE workspace_id = 'test-workspace' AND email = 'mover@example.com'", + ) + .fetch_one(&db) + .await?; + sqlx::query( + "INSERT INTO favorite (workspace_id, usr, path, favorite_kind) + VALUES ('test-workspace', $1, 'f/moved/keep', 'script')", + ) + .bind(&username) + .execute(&db) + .await?; + + // Import drops move_from entirely and puts the member in move_to instead. + let resp = authed(client().post(format!("{global_base}/overwrite"))) + .json(&json!([ + { "name": "move_to", "emails": ["mover@example.com"] } + ])) + .send() + .await?; + assert_eq!(resp.status(), 200, "overwrite: {}", resp.text().await?); + + let remaining: i64 = sqlx::query_scalar( + "SELECT count(*) FROM usr WHERE workspace_id = 'test-workspace' AND email = 'mover@example.com'", + ) + .fetch_one(&db) + .await?; + assert_eq!( + remaining, 1, + "member should still be in the workspace via move_to" + ); + + let favorites: i64 = sqlx::query_scalar( + "SELECT count(*) FROM favorite WHERE workspace_id = 'test-workspace' AND path = 'f/moved/keep'", + ) + .fetch_one(&db) + .await?; + assert_eq!( + favorites, 1, + "moving between groups in one import must not destroy workspace data" + ); + + Ok(()) +} + +/// A full-import overwrite must reconcile the membership of retained groups too: a member +/// dropped from a retained group loses the access that group granted, and a member who only +/// lost their highest-precedence group is re-roled in place instead of keeping a stale +/// elevated role. +/// +/// Regression: the delta-based cleanup only acted on groups that disappeared from the import, +/// so an import that kept a group but dropped some of its members never cleaned those members +/// up. +#[cfg(all(feature = "private", feature = "enterprise"))] +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_overwrite_igroups_reconciles_retained_group_membership( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let global_base = format!("http://localhost:{port}/api/groups"); + let ws_base = format!("http://localhost:{port}/api/w/test-workspace/workspaces"); + + for g in ["top_admins", "base_devs"] { + let resp = authed(client().post(format!("{global_base}/create"))) + .json(&json!({ "name": g })) + .send() + .await?; + assert_eq!(resp.status(), 200, "create {g}"); + } + + // demoted@ holds admin via top_admins and developer via base_devs; dropped@ only has + // base_devs. + for (g, email) in [ + ("top_admins", "demoted@example.com"), + ("base_devs", "demoted@example.com"), + ("base_devs", "dropped@example.com"), + ] { + let resp = authed(client().post(format!("{global_base}/adduser/{g}"))) + .json(&json!({ "email": email })) + .send() + .await?; + assert_eq!(resp.status(), 200, "adduser {g}/{email}"); + } + + let resp = authed(client().post(format!("{ws_base}/edit_instance_groups"))) + .json(&json!({ + "groups": ["top_admins", "base_devs"], + "roles": { "top_admins": "admin", "base_devs": "developer" } + })) + .send() + .await?; + assert_eq!(resp.status(), 200, "edit: {}", resp.text().await?); + + let (is_admin, via): (bool, Option) = sqlx::query_as( + "SELECT is_admin, added_via->>'group' FROM usr + WHERE workspace_id = 'test-workspace' AND email = 'demoted@example.com'", + ) + .fetch_one(&db) + .await?; + assert!(is_admin, "demoted@ should start as admin via top_admins"); + assert_eq!(via.as_deref(), Some("top_admins")); + + // Workspace state that must survive the demotion. + let username: String = sqlx::query_scalar( + "SELECT username FROM usr WHERE workspace_id = 'test-workspace' AND email = 'demoted@example.com'", + ) + .fetch_one(&db) + .await?; + sqlx::query( + "INSERT INTO favorite (workspace_id, usr, path, favorite_kind) + VALUES ('test-workspace', $1, 'f/lifecycle/keep', 'script')", + ) + .bind(&username) + .execute(&db) + .await?; + + // The import retains both groups but drops demoted@ from top_admins and dropped@ from + // base_devs. + let resp = authed(client().post(format!("{global_base}/overwrite"))) + .json(&json!([ + { "name": "top_admins", "emails": [] }, + { "name": "base_devs", "emails": ["demoted@example.com"] } + ])) + .send() + .await?; + assert_eq!(resp.status(), 200, "overwrite: {}", resp.text().await?); + + // demoted@ stays, re-roled to base_devs' developer, with their data intact. + let (is_admin, is_operator, via): (bool, bool, Option) = sqlx::query_as( + "SELECT is_admin, operator, added_via->>'group' FROM usr + WHERE workspace_id = 'test-workspace' AND email = 'demoted@example.com'", + ) + .fetch_one(&db) + .await?; + assert!( + !is_admin, + "admin from top_admins must not survive being dropped from it" + ); + assert!(!is_operator, "base_devs grants developer"); + assert_eq!(via.as_deref(), Some("base_devs")); + + let favorites: i64 = sqlx::query_scalar( + "SELECT count(*) FROM favorite WHERE workspace_id = 'test-workspace' AND path = 'f/lifecycle/keep'", + ) + .fetch_one(&db) + .await?; + assert_eq!( + favorites, 1, + "re-roling in place must not destroy workspace data" + ); + + // dropped@ lost their only configured group even though the group itself was retained. + let remaining: i64 = sqlx::query_scalar( + "SELECT count(*) FROM usr + WHERE workspace_id = 'test-workspace' AND email = 'dropped@example.com'", + ) + .fetch_one(&db) + .await?; + assert_eq!( + remaining, 0, + "member dropped from a retained group must be removed" + ); + + // Both groups were retained, so the workspace config is untouched. + let (groups, roles): (serde_json::Value, serde_json::Value) = sqlx::query_as( + "SELECT auto_invite->'instance_groups', auto_invite->'instance_groups_roles' + FROM workspace_settings WHERE workspace_id = 'test-workspace'", + ) + .fetch_one(&db) + .await?; + assert_eq!(groups, json!(["top_admins", "base_devs"])); + assert_eq!( + roles, + json!({ "top_admins": "admin", "base_devs": "developer" }) + ); + + Ok(()) +} + +/// Members whose `added_via` source is not 'instance_group' — manually added users, and the +/// orphaned members the `preserve_orphaned_instance_group_members` migration converted to +/// manual — are invisible to reconciliation: never re-roled and never removed, even when they +/// also appear in a configured group's membership. +#[cfg(feature = "private")] +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_reconcile_ignores_non_instance_group_members( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let global_base = format!("http://localhost:{port}/api/groups"); + let ws_base = format!("http://localhost:{port}/api/w/test-workspace/workspaces"); + + let resp = authed(client().post(format!("{global_base}/create"))) + .json(&json!({ "name": "visible_grp" })) + .send() + .await?; + assert_eq!(resp.status(), 200, "create"); + for email in ["kept@example.com", "shielded@example.com"] { + let resp = authed(client().post(format!("{global_base}/adduser/visible_grp"))) + .json(&json!({ "email": email })) + .send() + .await?; + assert_eq!(resp.status(), 200, "adduser {email}"); + } + + // shielded@ is already in the workspace through a non-instance_group source (the shape + // the migration leaves behind), at a role the group config would not grant. The username + // deliberately differs from the instance-derived one ('shielded'): an unguarded + // auto_add_user would then insert a second usr row for the email instead of no-op'ing on + // a username conflict, so the count assertions below can catch it. + sqlx::query( + r#"INSERT INTO usr (workspace_id, username, email, is_admin, operator, added_via) + VALUES ('test-workspace', 'shielded_legacy', 'shielded@example.com', true, false, + '{"source": "manual", "migrated_from_instance_group": "gone_grp"}'::jsonb)"#, + ) + .execute(&db) + .await?; + + let resp = authed(client().post(format!("{ws_base}/edit_instance_groups"))) + .json(&json!({ + "groups": ["visible_grp"], + "roles": { "visible_grp": "developer" } + })) + .send() + .await?; + assert_eq!(resp.status(), 200, "edit: {}", resp.text().await?); + + // kept@ was auto-added via the group; shielded@ kept their single manual row untouched. + let kept: i64 = sqlx::query_scalar( + "SELECT count(*) FROM usr WHERE workspace_id = 'test-workspace' AND email = 'kept@example.com' + AND added_via->>'source' = 'instance_group'", + ) + .fetch_one(&db) + .await?; + assert_eq!(kept, 1, "group member should be auto-added"); + + let shielded_rows: i64 = sqlx::query_scalar( + "SELECT count(*) FROM usr WHERE workspace_id = 'test-workspace' AND email = 'shielded@example.com'", + ) + .fetch_one(&db) + .await?; + assert_eq!( + shielded_rows, 1, + "reconciliation must not create a second usr row for a member already present under a non-instance_group source" + ); + + // Dropping both users from the group removes the instance_group-sourced member but must + // leave the manual row alone. + for email in ["kept@example.com", "shielded@example.com"] { + let resp = authed(client().post(format!("{global_base}/removeuser/visible_grp"))) + .json(&json!({ "email": email })) + .send() + .await?; + assert_eq!(resp.status(), 200, "removeuser {email}"); + } + + let kept: i64 = sqlx::query_scalar( + "SELECT count(*) FROM usr WHERE workspace_id = 'test-workspace' AND email = 'kept@example.com'", + ) + .fetch_one(&db) + .await?; + assert_eq!( + kept, 0, + "instance_group-sourced member loses access with their only group" + ); + + let (username, is_admin, via_source): (String, bool, Option) = sqlx::query_as( + "SELECT username, is_admin, added_via->>'source' FROM usr + WHERE workspace_id = 'test-workspace' AND email = 'shielded@example.com'", + ) + .fetch_one(&db) + .await?; + assert_eq!( + username, "shielded_legacy", + "the original manual row must be the only one" + ); + assert!( + is_admin, + "manual member's role must not be touched by reconciliation" + ); + assert_eq!(via_source.as_deref(), Some("manual")); + + Ok(()) +} + +/// The upgrade migration converts every member the reconciler would evict — those whose +/// granting group was deleted and those dropped from a group that still exists — and leaves +/// still-qualifying members alone. The migration has already run against the empty test +/// database by the time this executes, so the test fabricates pre-fix state and re-executes +/// the migration's statements, which are idempotent plain UPDATEs. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_preserve_orphaned_members_migration(db: Pool) -> anyhow::Result<()> { + // ghost_grp pins the statement order: it is referenced by the workspace and still has a + // membership row, but no instance_group row. Only when the reference strip runs before + // the conversion does ghost@ read as unconverted-by-membership nowhere and get preserved; + // converting first would spare them on the doomed reference and then strand them. + sqlx::raw_sql( + r#" + INSERT INTO workspace (id, name, owner) VALUES ('mig-ws', 'mig-ws', 'admin@windmill.dev'); + INSERT INTO workspace_settings (workspace_id, auto_invite) VALUES + ('mig-ws', '{"instance_groups": ["gone_grp", "ghost_grp", "live_grp"], "instance_groups_roles": {"gone_grp": "admin", "ghost_grp": "developer", "live_grp": "developer"}}'::jsonb); + INSERT INTO instance_group (name) VALUES ('live_grp'); + INSERT INTO email_to_igroup (email, igroup) VALUES + ('live@example.com', 'live_grp'), + ('ghost@example.com', 'ghost_grp'); + INSERT INTO usr (workspace_id, username, email, is_admin, operator, added_via) VALUES + ('mig-ws', 'orphan', 'orphan@example.com', true, false, '{"source": "instance_group", "group": "gone_grp"}'::jsonb), + ('mig-ws', 'droppedu', 'dropped@example.com', false, false, '{"source": "instance_group", "group": "live_grp"}'::jsonb), + ('mig-ws', 'ghostmember', 'ghost@example.com', false, false, '{"source": "instance_group", "group": "ghost_grp"}'::jsonb), + ('mig-ws', 'livemember', 'live@example.com', false, false, '{"source": "instance_group", "group": "live_grp"}'::jsonb); + "#, + ) + .execute(&db) + .await?; + + sqlx::raw_sql(include_str!( + "../../migrations/20260813195023_preserve_orphaned_instance_group_members.up.sql" + )) + .execute(&db) + .await?; + + // Deleted-group orphan and retained-group-dropped orphan both become manual members + // with the original group recorded; the still-qualifying member is untouched. + for (email, expected_group) in [ + ("orphan@example.com", "gone_grp"), + ("dropped@example.com", "live_grp"), + ("ghost@example.com", "ghost_grp"), + ] { + let (source, migrated_from): (Option, Option) = sqlx::query_as( + "SELECT added_via->>'source', added_via->>'migrated_from_instance_group' + FROM usr WHERE workspace_id = 'mig-ws' AND email = $1", + ) + .bind(email) + .fetch_one(&db) + .await?; + assert_eq!( + source.as_deref(), + Some("manual"), + "{email} should be converted" + ); + assert_eq!( + migrated_from.as_deref(), + Some(expected_group), + "{email} marker" + ); + } + + let (source, group): (Option, Option) = sqlx::query_as( + "SELECT added_via->>'source', added_via->>'group' + FROM usr WHERE workspace_id = 'mig-ws' AND email = 'live@example.com'", + ) + .fetch_one(&db) + .await?; + assert_eq!( + source.as_deref(), + Some("instance_group"), + "still-qualifying member spared" + ); + assert_eq!(group.as_deref(), Some("live_grp")); + + // The dangling references are stripped from both auto_invite fields; the live one stays. + let (groups, roles): (serde_json::Value, serde_json::Value) = sqlx::query_as( + "SELECT auto_invite->'instance_groups', auto_invite->'instance_groups_roles' + FROM workspace_settings WHERE workspace_id = 'mig-ws'", + ) + .fetch_one(&db) + .await?; + assert_eq!(groups, json!(["live_grp"])); + assert_eq!(roles, json!({ "live_grp": "developer" })); + + Ok(()) +} diff --git a/backend/windmill-api-integration-tests/tests/resources.rs b/backend/windmill-api-integration-tests/tests/resources.rs index 07afbb3442..3f0b21b184 100644 --- a/backend/windmill-api-integration-tests/tests/resources.rs +++ b/backend/windmill-api-integration-tests/tests/resources.rs @@ -199,6 +199,25 @@ async fn test_resource_endpoints(db: Pool) -> anyhow::Result<()> { let list = resp.json::>().await?; assert!(!list.is_empty()); + // Values are capped so the search modal never has to hold a whole workspace of + // resource content in memory. + let find = |path: &str| { + list.iter() + .find(|r| r["path"] == path) + .unwrap_or_else(|| panic!("{path} missing from list_search")) + .clone() + }; + let oversized = find("u/test-user/oversized_resource"); + assert_eq!(oversized["value"].as_str().unwrap().chars().count(), 4000); + assert_eq!(oversized["truncated"], true); + + let simple = find("u/test-user/simple_resource"); + assert!(simple["value"].as_str().unwrap().contains("\"host\"")); + assert_eq!(simple["truncated"], false); + + // A null value must still come back as searchable text, not null. + assert_eq!(find("u/test-user/null_resource")["value"], ""); + // --- list_names --- let resp = authed(client().get(format!("{base}/list_names/object"))) .send() diff --git a/backend/windmill-api-schedule/src/lib.rs b/backend/windmill-api-schedule/src/lib.rs index b4e9bf269a..ec790ec060 100644 --- a/backend/windmill-api-schedule/src/lib.rs +++ b/backend/windmill-api-schedule/src/lib.rs @@ -27,6 +27,9 @@ use windmill_common::{ db::UserDB, error::{Error, JsonResult, Result}, schedule::Schedule, + trigger_history::{ + self, TriggerHistoryEvent, TriggerOperation, TriggerSource, SCHEDULE_TRIGGER_KIND, + }, user_drafts::{ delete_all_drafts_for_path, fetch_draft_only_list_rows, overlay_or_draft_only, UserDraftItemKind, WithDraftOverlay, WithDraftQuery, @@ -85,6 +88,40 @@ fn resolve_edited_by(authed: &ApiAuthed) -> String { authed.username.clone() } +/// Append this mutation to `trigger_history`, diffing the row against `before`. +/// +/// Call it on the transaction that made the change, after the change: the +/// snapshot it takes is the "after" side of the diff, and the two commit or roll +/// back together. +async fn record_schedule_history( + tx: &mut sqlx::PgConnection, + authed: &ApiAuthed, + w_id: &str, + path: &str, + operation: TriggerOperation, + before: Option, +) -> Result<()> { + let after = trigger_history::snapshot_row(&mut *tx, "schedule", w_id, path).await?; + // Nothing to describe when the row is not there after the write: the same + // guard the trigger side needs, kept here so the two read alike. + if after.is_none() { + return Ok(()); + } + trigger_history::record( + &mut *tx, + TriggerHistoryEvent { + workspace_id: w_id, + trigger_kind: SCHEDULE_TRIGGER_KIND, + path, + operation, + source: TriggerSource::of_request(authed.is_session_token), + username: Some(&authed.username), + changes: trigger_history::summarize_changes(before.as_ref(), after.as_ref()), + }, + ) + .await +} + pub fn workspaced_service() -> Router { Router::new() .route("/list", get(list_schedule)) @@ -417,6 +454,16 @@ async fn create_schedule( .await .map_err(|e| Error::internal_err(format!("inserting schedule in {w_id}: {e:#}")))?; + record_schedule_history( + &mut *tx, + &authed, + &w_id, + &ns.path, + TriggerOperation::Create, + None, + ) + .await?; + audit_log( &mut *tx, &authed, @@ -524,6 +571,8 @@ async fn edit_schedule( authed.email.clone() }; + let before = trigger_history::snapshot_row(&mut *tx, "schedule", &w_id, path).await?; + let schedule = sqlx::query_as!( Schedule, r#" @@ -632,6 +681,16 @@ async fn edit_schedule( // like set_enabled, flow updates, and worker job completions. clear_schedule(&mut tx, path, &w_id).await?; + record_schedule_history( + &mut *tx, + &authed, + &w_id, + path, + TriggerOperation::Update, + before, + ) + .await?; + audit_log( &mut *tx, &authed, @@ -1084,6 +1143,8 @@ pub async fn set_enabled( } } } + let before = trigger_history::snapshot_row(&mut *tx, "schedule", &w_id, path).await?; + // email is still written for backwards compat with old workers that don't know about permissioned_as let schedule_o = sqlx::query_as!( Schedule, @@ -1139,6 +1200,20 @@ pub async fn set_enabled( clear_schedule(&mut tx, path, &w_id).await?; + record_schedule_history( + &mut *tx, + &authed, + &w_id, + path, + if payload.enabled { + TriggerOperation::Enable + } else { + TriggerOperation::Disable + }, + before, + ) + .await?; + audit_log( &mut *tx, &authed, @@ -1285,6 +1360,22 @@ async fn delete_schedule( .await?; } + // No diff: the row is gone, and the trashbin above already keeps its full + // contents for a restore. + trigger_history::record( + &mut *tx, + TriggerHistoryEvent { + workspace_id: &w_id, + trigger_kind: SCHEDULE_TRIGGER_KIND, + path, + operation: TriggerOperation::Delete, + source: TriggerSource::of_request(authed.is_session_token), + username: Some(&authed.username), + changes: None, + }, + ) + .await?; + audit_log( &mut *tx, &authed, @@ -1373,6 +1464,11 @@ async fn set_default_error_handler( } if payload.override_existing { + // The rewrite and its history rows go in one transaction: on separate + // connections a concurrent edit could interleave, leaving the + // id-ordered drawer showing the wrong latest change, and a failed + // insert would leave the schedules rewritten with nothing recording it. + let mut tx = db.begin().await?; let updated_schedules: Vec; match payload.handler_type { HandlerType::Error => { @@ -1386,14 +1482,14 @@ async fn set_default_error_handler( payload.number_of_occurence_exact, w_id, ) - .fetch_all(&db) + .fetch_all(&mut *tx) .await?; } else { updated_schedules = sqlx::query_scalar!( "UPDATE schedule SET ws_error_handler_muted = false, on_failure = NULL, on_failure_extra_args = NULL, on_failure_times = NULL, on_failure_exact = NULL WHERE workspace_id = $1 RETURNING path", w_id, ) - .fetch_all(&db) + .fetch_all(&mut *tx) .await?; } } @@ -1406,14 +1502,14 @@ async fn set_default_error_handler( payload.number_of_occurence, w_id, ) - .fetch_all(&db) + .fetch_all(&mut *tx) .await?; } else { updated_schedules = sqlx::query_scalar!( "UPDATE schedule SET on_recovery = NULL, on_recovery_extra_args = NULL, on_recovery_times = NULL WHERE workspace_id = $1 RETURNING path", w_id, ) - .fetch_all(&db) + .fetch_all(&mut *tx) .await?; } } @@ -1425,18 +1521,70 @@ async fn set_default_error_handler( payload.extra_args, w_id, ) - .fetch_all(&db) + .fetch_all(&mut *tx) .await?; } else { updated_schedules = sqlx::query_scalar!( "UPDATE schedule SET on_success = NULL, on_success_extra_args = NULL WHERE workspace_id = $1 RETURNING path", w_id, ) - .fetch_all(&db) + .fetch_all(&mut *tx) .await?; } } } + // One row per schedule the workspace-wide override rewrote, so a handler + // that appeared on a schedule nobody edited is traceable. Every column + // the UPDATE above wrote, not just the handler path: the mute flag and + // the occurrence thresholds are what someone auditing a surprise + // notification change most needs. No `old` side and no + // already-had-this-value filter — the UPDATE rewrites the whole + // workspace unconditionally, so these rows record the write rather than + // a delta. + // Built from the same values the branch that ran actually bound: a reset + // (`payload.path` absent) hardcodes NULL / false in SQL while the request + // still carries the form's other fields, so reading them here would name + // values the write never produced. + let cleared = payload.path.is_none(); + let handler_path = payload.path.clone(); + let extra_args = (!cleared).then(|| payload.extra_args.clone()).flatten(); + let times = (!cleared).then_some(payload.number_of_occurence).flatten(); + let handler_fields = match payload.handler_type { + HandlerType::Error => serde_json::json!({ + "on_failure": { "new": handler_path }, + "on_failure_extra_args": { "new": extra_args }, + "on_failure_times": { "new": times }, + "on_failure_exact": { + "new": (!cleared).then_some(payload.number_of_occurence_exact).flatten() + }, + "ws_error_handler_muted": { + "new": !cleared && payload.workspace_handler_muted.unwrap_or(false) + }, + }), + HandlerType::Recovery => serde_json::json!({ + "on_recovery": { "new": handler_path }, + "on_recovery_extra_args": { "new": extra_args }, + "on_recovery_times": { "new": times }, + }), + HandlerType::Success => serde_json::json!({ + "on_success": { "new": handler_path }, + "on_success_extra_args": { "new": extra_args }, + }), + }; + trigger_history::record_bulk( + &mut tx, + &w_id, + SCHEDULE_TRIGGER_KIND, + &updated_schedules, + TriggerOperation::Update, + TriggerSource::of_request(authed.is_session_token), + Some(&authed.username), + Some(handler_fields), + ) + .await?; + + tx.commit().await?; + for updated_schedule_path in updated_schedules { // managed ducklake-maintenance rows get the handler update (their // failures should reach workspace handlers) but must not be diff --git a/backend/windmill-api-scripts/src/scripts.rs b/backend/windmill-api-scripts/src/scripts.rs index 3f23d0416f..3b66f7317b 100644 --- a/backend/windmill-api-scripts/src/scripts.rs +++ b/backend/windmill-api-scripts/src/scripts.rs @@ -289,7 +289,9 @@ async fn list_scripts( sqlb.and_where_eq("parent_hashes[array_upper(parent_hashes, 1)]", &ph.0); } if let Some(ph) = &lq.parent_hash { - sqlb.and_where_eq("any(parent_hashes)", &ph.0); + // ANY() only ever sits on the right of the comparison; `and_where_eq` would + // emit it on the left and Postgres rejects that as a syntax error. + sqlb.and_where("? = ANY(parent_hashes)".bind(&ph.0)); } if let Some(it) = &lq.is_template { sqlb.and_where_eq("is_template", it); diff --git a/backend/windmill-api-users/Cargo.toml b/backend/windmill-api-users/Cargo.toml index c322d6cfda..13ab8143d8 100644 --- a/backend/windmill-api-users/Cargo.toml +++ b/backend/windmill-api-users/Cargo.toml @@ -21,7 +21,6 @@ windmill-api-auth.workspace = true windmill-audit.workspace = true windmill-git-sync.workspace = true -dashmap.workspace = true argon2.workspace = true axum.workspace = true chrono.workspace = true diff --git a/backend/windmill-api-users/src/users.rs b/backend/windmill-api-users/src/users.rs index 96b89e322c..67e1721109 100644 --- a/backend/windmill-api-users/src/users.rs +++ b/backend/windmill-api-users/src/users.rs @@ -46,6 +46,7 @@ use windmill_common::audit::AuditAuthor; use windmill_common::auth::{safe_token_prefix, TOKEN_PREFIX_LEN}; use windmill_common::global_settings::AUTOMATE_USERNAME_CREATION_SETTING; use windmill_common::oauth2::InstanceEvent; +use windmill_common::per_minute_counter::PerMinuteCounter; use windmill_common::users::truncate_token; use windmill_common::users::COOKIE_NAME; use windmill_common::users::{ @@ -67,41 +68,24 @@ use windmill_git_sync::handle_deployment_metadata; pub const COOKIE_PATH: &str = "/"; -const TOKEN_CREATE_LIMIT_PER_MINUTE: i32 = 10; +const TOKEN_CREATE_LIMIT_PER_MINUTE: u32 = 10; -struct TokenRateLimitEntry { - count: i32, - minute_bucket: i64, -} - -static TOKEN_CREATE_RATE_LIMIT: LazyLock> = - LazyLock::new(dashmap::DashMap::new); +static TOKEN_CREATE_RATE_LIMIT: LazyLock> = + LazyLock::new(PerMinuteCounter::new); fn check_token_create_rate_limit(username: &str) -> Result<()> { if !*CLOUD_HOSTED { return Ok(()); } - let current_minute = chrono::Utc::now().timestamp() / 60; - - let mut entry = TOKEN_CREATE_RATE_LIMIT - .entry(username.to_string()) - .or_insert(TokenRateLimitEntry { count: 0, minute_bucket: current_minute }); - - if entry.minute_bucket != current_minute { - entry.count = 0; - entry.minute_bucket = current_minute; + if TOKEN_CREATE_RATE_LIMIT.try_increment(username.to_string(), TOKEN_CREATE_LIMIT_PER_MINUTE) { + return Ok(()); } - if entry.count >= TOKEN_CREATE_LIMIT_PER_MINUTE { - return Err(Error::Generic( - StatusCode::TOO_MANY_REQUESTS, - "Too many token creation requests. Please try again later.".to_string(), - )); - } - - entry.count += 1; - Ok(()) + Err(Error::Generic( + StatusCode::TOO_MANY_REQUESTS, + "Too many token creation requests. Please try again later.".to_string(), + )) } pub fn workspaced_service() -> Router { @@ -1386,7 +1370,7 @@ async fn convert_user_to_group( )); } - // Determine the group with highest precedence (same logic as process_instance_group_auto_adds) + // Determine the group with highest precedence (same logic as reconcile_workspace_instance_groups) let roles: std::collections::HashMap = if let Some(roles_json) = &eligible_groups[0].instance_groups_roles { serde_json::from_value(roles_json.clone()).unwrap_or_default() diff --git a/backend/windmill-api-workspaces/src/workspaces.rs b/backend/windmill-api-workspaces/src/workspaces.rs index eff742006e..9bc4890fe6 100644 --- a/backend/windmill-api-workspaces/src/workspaces.rs +++ b/backend/windmill-api-workspaces/src/workspaces.rs @@ -11489,41 +11489,6 @@ struct LogFeatureUsagePayload { events: Vec, } -// Only registered (feature, kind) actions are accepted, so telemetry stays -// limited to predefined feature actions. Keys are shape-checked (identifier-like, -// no spaces) rather than pinned to value sets: they come from our own frontend -// (modes, tab/draft kinds, tool names, provider:model) and pinning every value -// server-side was not worth the maintenance. -const FEATURE_USAGE_KINDS: &[(&str, &str)] = &[ - ("ai_session", "created"), - ("ai_session", "message"), - ("ai_session", "autonomy"), - ("ai_session", "tab"), - ("ai_session", "tokens"), - ("ai_session", "deployed"), - ("ai_session", "archived"), - ("ai_session", "deleted"), - ("ai_session", "beta_optout"), - ("ai_session", "beta_optin"), - ("ai_chat", "message"), - ("ai_chat", "model"), - ("ai_chat", "tool"), - ("flow_editor", "panel_placement"), -]; - -fn is_identifier_shaped(s: &str, max_len: usize) -> bool { - !s.is_empty() - && s.len() <= max_len - && s.chars() - .all(|c| c.is_ascii_alphanumeric() || matches!(c, '_' | '-' | ':' | '.' | '/')) -} - -fn valid_feature_usage_event(e: &FeatureUsageEvent) -> bool { - FEATURE_USAGE_KINDS.contains(&(e.feature.as_str(), e.kind.as_str())) - && (e.key.is_empty() || is_identifier_shaped(&e.key, 100)) - && (e.entity_id.is_empty() || is_identifier_shaped(&e.entity_id, 50)) -} - async fn log_feature_usage( Extension(db): Extension, Json(payload): Json, @@ -11532,7 +11497,15 @@ async fn log_feature_usage( // single INSERT error out ("cannot affect row a second time"). let mut agg: HashMap<(String, String, String, String), i64> = HashMap::new(); for e in payload.events.into_iter().take(MAX_FEATURE_USAGE_EVENTS) { - if !valid_feature_usage_event(&e) { + // Which actions may be recorded lives in + // `windmill_common::feature_usage`, shared with the in-process writer so + // both admit exactly the same events. + if !windmill_common::feature_usage::is_recordable_event( + &e.feature, + &e.kind, + &e.key, + &e.entity_id, + ) { continue; } let value = e.value.unwrap_or(1).clamp(1, 1_000_000); @@ -11542,12 +11515,18 @@ async fn log_feature_usage( if agg.is_empty() { return Ok(StatusCode::NO_CONTENT); } - let mut features = Vec::with_capacity(agg.len()); - let mut kinds = Vec::with_capacity(agg.len()); - let mut keys = Vec::with_capacity(agg.len()); - let mut entity_ids = Vec::with_capacity(agg.len()); - let mut values = Vec::with_capacity(agg.len()); - for ((feature, kind, key, entity_id), value) in agg { + // Sorted for the same reason as `flush_feature_usage`: this endpoint and the + // backend flusher upsert the same rows, and two batches touching them in + // opposite orders deadlock. + let mut rows: Vec<((String, String, String, String), i64)> = agg.into_iter().collect(); + rows.sort_unstable_by(|a, b| a.0.cmp(&b.0)); + + let mut features = Vec::with_capacity(rows.len()); + let mut kinds = Vec::with_capacity(rows.len()); + let mut keys = Vec::with_capacity(rows.len()); + let mut entity_ids = Vec::with_capacity(rows.len()); + let mut values = Vec::with_capacity(rows.len()); + for ((feature, kind, key, entity_id), value) in rows { features.push(feature); kinds.push(kind); keys.push(key); diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index c61c092c95..ec641ef4ff 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.789.0 + version: 1.792.2 title: Windmill API contact: @@ -268,6 +268,15 @@ paths: type: string - $ref: "#/components/parameters/ResourceName" - $ref: "#/components/parameters/ActionKind" + - name: before_id + in: query + description: > + only return logs with an id strictly lower than this one. Logs are ordered by + descending id, so this is a keyset cursor to stream a page in several batches + without paying a growing offset. + schema: + type: integer + format: int64 - name: all_workspaces in: query description: get audit logs for all workspaces @@ -8035,10 +8044,19 @@ paths: properties: path: type: string - value: {} + value: + type: string + description: >- + pretty-printed JSON rendering of the resource value, capped at + 4000 characters — a search preview, not the value itself (use + get_value for that) + truncated: + type: boolean + description: whether value was cut short by that cap required: - path - value + - truncated /w/{workspace}/resources/mcp_tools/{path}: get: @@ -8063,11 +8081,72 @@ paths: type: string description: type: string - parameters: + inputSchema: type: object + annotations: + type: object + properties: + title: + type: string + readOnlyHint: + type: boolean + destructiveHint: + type: boolean + idempotentHint: + type: boolean + openWorldHint: + type: boolean required: - name - - parameters + - inputSchema + + /w/{workspace}/resources/mcp_call_tool/{path}: + post: + summary: call a tool on the MCP server described by the resource + operationId: callMcpTool + tags: + - resource + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/Path" + requestBody: + description: tool name and arguments + required: true + content: + application/json: + schema: + type: object + properties: + tool: + type: string + arguments: + type: object + read_only: + type: boolean + description: | + set when the caller ran the tool without asking the user to + confirm it; the call is refused unless the server's live + listing marks the tool read-only + required: + - tool + responses: + "200": + description: | + the MCP tool result, forwarded verbatim. A tool that ran but failed + returns 200 with isError true. + content: + application/json: + schema: + type: object + properties: + content: + type: array + items: + type: object + structuredContent: + type: object + isError: + type: boolean /w/{workspace}/resources/list_names/{name}: get: @@ -20333,6 +20412,36 @@ paths: type: string nullable: true + /w/{workspace}/triggers_history/list: + get: + summary: list the history of schedule and trigger modifications + operationId: listTriggerHistory + tags: + - trigger + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/Page" + - $ref: "#/components/parameters/PerPage" + - name: trigger_kind + description: "'schedule' or a trigger type (http, kafka, ...)" + in: query + schema: + type: string + - name: path + description: only return the history of the trigger at this path + in: query + schema: + type: string + responses: + "200": + description: trigger history + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/TriggerHistoryEntry" + /w/{workspace}/folders/list: get: summary: list folders @@ -23738,6 +23847,55 @@ paths: items: $ref: "#/components/schemas/AssetProgress" + /w/{workspace}/jobs/run_assets/{id}: + get: + summary: List the assets a run touched at runtime + description: > + Assets detected while the run executed (SDK S3 calls, resources passed as + arguments), aggregated over the job and all of its child jobs so a flow or + workflow-as-code run reports what its steps and tasks touched. Authorized + through the job, the same gate as `run_progress`. Recording is asynchronous, + so an asset can take a few minutes after the run to appear, and only the most + recent runs that touched an asset keep that record. A run that fans out can + touch more assets than one response should carry, so the list is capped and + `truncated` says when it was cut. + operationId: listRunAssets + tags: + - job + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: id + in: path + required: true + description: The job whose runtime assets to read + schema: + type: string + format: uuid + responses: + "200": + description: assets this run and its child jobs touched + content: + application/json: + schema: + type: object + required: [assets, truncated] + properties: + truncated: + type: boolean + description: whether the run touched more assets than are listed + assets: + type: array + items: + type: object + required: [path, kind] + properties: + path: + type: string + kind: + $ref: "#/components/schemas/AssetKind" + access_type: + $ref: "#/components/schemas/AssetUsageAccessType" + /w/{workspace}/assets/partitions_in_range: get: summary: List expected partitions of a ducklake asset in a date range with their materialization status (enterprise) @@ -26263,6 +26421,8 @@ components: type: integer cache_ttl: type: number + cache_ignore_s3_path: + type: boolean dedicated_worker: type: boolean ws_error_handler_muted: @@ -28236,6 +28396,44 @@ components: is_fileset: type: boolean + TriggerHistoryEntry: + type: object + properties: + id: + type: integer + format: int64 + trigger_kind: + type: string + description: "'schedule' or a trigger type (http, kafka, ...)" + path: + type: string + operation: + type: string + enum: [create, update, delete, enable, disable, suspend] + source: + type: string + description: The kind of client the change came from. `worker` means the server disabled the trigger on its own after a failure. + enum: [ui, cli, api, worker] + username: + type: string + nullable: true + description: Unset when the server acted on its own. + created_at: + type: string + format: date-time + changes: + type: object + nullable: true + additionalProperties: true + description: "{field: {old, new}} for the fields that actually changed. Unset for a delete." + required: + - id + - trigger_kind + - path + - operation + - source + - created_at + Schedule: type: object properties: @@ -31779,6 +31977,13 @@ components: execution_mode: type: string enum: [viewer, publisher, anonymous] + description: >- + Who the app's runnables execute as. Optional, and what omitting it + means depends on the operation: creating an app defaults it to + `publisher` (runs on behalf of the app's publisher and requires an + authenticated viewer), while updating one keeps the mode the app is + already deployed under. Either way `anonymous`, which makes the app + publicly executable, is never assumed on_behalf_of: type: string on_behalf_of_email: diff --git a/backend/windmill-api/src/apps.rs b/backend/windmill-api/src/apps.rs index 037f701346..3b0fc60d79 100644 --- a/backend/windmill-api/src/apps.rs +++ b/backend/windmill-api/src/apps.rs @@ -287,8 +287,11 @@ pub type AllowUserResources = Vec; #[derive(Serialize, Deserialize, Debug, PartialEq, Copy, Clone, Default)] #[serde(rename_all = "lowercase")] pub enum ExecutionMode { - #[default] Anonymous, + /// Default for a policy that omits `execution_mode`. It MUST stay a mode + /// that requires an authenticated viewer: an omitted field must never be + /// able to publish an app anonymously (publicly executable). + #[default] Publisher, Viewer, } @@ -334,7 +337,13 @@ pub struct Policy { pub triggerables: Option>, #[serde(skip_serializing_if = "Option::is_none")] pub triggerables_v2: Option>, - pub execution_mode: ExecutionMode, + /// `None` when the policy states no mode, which the OpenAPI schema (and so + /// the MCP tools generated from it) allows. Create resolves that to + /// [`ExecutionMode::default`]; update keeps the deployed app's mode, so a + /// partial policy cannot silently re-permission an app. Read the effective + /// mode with [`Policy::execution_mode`], never this field. + #[serde(default, rename = "execution_mode")] + stated_execution_mode: Option, pub s3_inputs: Option>, pub allowed_s3_keys: Option>, // WIN-2006: publisher opt-in to iframe sandbox isolation (alpha). When true the @@ -351,6 +360,25 @@ pub struct Policy { pub frontend_sdk_scopes: Option>, } +impl Policy { + /// The mode this policy runs under. A policy that states none resolves to + /// [`ExecutionMode::default`], never to `anonymous`, so an app can only be + /// publicly executable because someone said so. + pub fn execution_mode(&self) -> ExecutionMode { + self.stated_execution_mode.unwrap_or_default() + } + + /// What the policy states, or `None` for "not stated". Only the write paths + /// need this: everything else wants [`Policy::execution_mode`]. + fn stated_execution_mode(&self) -> Option { + self.stated_execution_mode + } + + pub fn set_execution_mode(&mut self, execution_mode: ExecutionMode) { + self.stated_execution_mode = Some(execution_mode); + } +} + #[derive(Deserialize)] pub struct CreateApp { pub path: String, @@ -1189,7 +1217,7 @@ async fn get_public_app_by_secret( let policy = serde_json::from_str::(app.policy.0.get()).map_err(to_anyhow)?; - if !matches!(policy.execution_mode, ExecutionMode::Anonymous) { + if !matches!(policy.execution_mode(), ExecutionMode::Anonymous) { if opt_authed.is_none() { return Err(Error::NotAuthorized( "App visibility does not allow public access and you are not logged in".to_string(), @@ -2199,7 +2227,10 @@ async fn create_app_internal<'a>( )); } } - if matches!(app.policy.execution_mode, ExecutionMode::Anonymous) { + // Pin the mode the app is created under, so the stored policy states one + // even when the caller did not. + app.policy.set_execution_mode(app.policy.execution_mode()); + if matches!(app.policy.execution_mode(), ExecutionMode::Anonymous) { if let RuleCheckResult::Blocked(msg) = check_user_against_rule( w_id, &ProtectionRuleKind::RestrictAnonymousAppDeployment, @@ -2612,7 +2643,7 @@ async fn update_app( let opath = path.to_string(); let db2 = db.clone(); let (new_tx, npath, v_id) = - update_app_internal(authed, db, user_db, &w_id, path, false, ns).await?; + update_app_internal(authed, db, user_db, &w_id, path, false, ns, None).await?; new_tx.commit().await?; tally_app_rename(&db2, &w_id, &opath, &npath, v_id).await; @@ -2671,7 +2702,7 @@ async fn update_app_raw_source( "value with the app's `files` is required to deploy a raw app".to_string(), )); }; - let files: RawAppSourceFiles = serde_json::from_str(value.0.get()) + let value: RawAppSourceValue = serde_json::from_str(value.0.get()) .map_err(|e| Error::BadRequest(format!("app value is not a raw app source: {e}")))?; // All before the compile, which costs a job on a worker: it must not run for @@ -2690,17 +2721,41 @@ async fn update_app_raw_source( reject_kind_change(path, true, Some(deployed_raw_app))?; } - let (js, css) = - apps_raw_bundle::bundle_raw_app_sources(&db, &user_db, &authed, &w_id, &files.files) - .await?; + let bundled = apps_raw_bundle::bundle_raw_app_sources( + &db, + &user_db, + &authed, + &w_id, + &value.files, + &value.runnables, + ) + .await?; let opath = path.to_string(); let db2 = db.clone(); - let (mut tx, npath, v_id) = - update_app_internal(authed, db, user_db, &w_id, path, true, ns).await?; - store_raw_app_file(&w_id, &v_id, "js", bytes::Bytes::from(js), &mut tx).await?; - if !css.is_empty() { - store_raw_app_file(&w_id, &v_id, "css", bytes::Bytes::from(css), &mut tx).await?; + // The new sources bring new runnables, so the deployed triggerables no longer + // describe them. Merged inside, under the app-row lock. + let (mut tx, npath, v_id) = update_app_internal( + authed, + db, + user_db, + &w_id, + path, + true, + ns, + Some(bundled.triggerables_v2), + ) + .await?; + store_raw_app_file(&w_id, &v_id, "js", bytes::Bytes::from(bundled.js), &mut tx).await?; + if !bundled.css.is_empty() { + store_raw_app_file( + &w_id, + &v_id, + "css", + bytes::Bytes::from(bundled.css), + &mut tx, + ) + .await?; } tx.commit().await?; tally_app_rename(&db2, &w_id, &opath, &npath, v_id).await; @@ -2775,7 +2830,7 @@ async fn create_app_raw_source( Extension(db): Extension, Extension(webhook): Extension, Path(w_id): Path, - Json(app): Json, + Json(mut app): Json, ) -> Result<(StatusCode, String)> { if authed.is_operator { return Err(Error::NotAuthorized( @@ -2801,7 +2856,7 @@ async fn create_app_raw_source( return Err(Error::PermissionDenied(msg)); } - let files: RawAppSourceFiles = serde_json::from_str(app.value.0.get()) + let value: RawAppSourceValue = serde_json::from_str(app.value.0.get()) .map_err(|e| Error::BadRequest(format!("app value is not a raw app source: {e}")))?; // Before the compile, which costs a job on a worker: it must not run for a @@ -2816,14 +2871,31 @@ async fn create_app_raw_source( ))); } - let (js, css) = - apps_raw_bundle::bundle_raw_app_sources(&db, &user_db, &authed, &w_id, &files.files) - .await?; + let bundled = apps_raw_bundle::bundle_raw_app_sources( + &db, + &user_db, + &authed, + &w_id, + &value.files, + &value.runnables, + ) + .await?; + // The bundle carries the grants for the runnables it was built from, so the + // app cannot be deployed with a policy that does not describe them. + app.policy.triggerables = None; + app.policy.triggerables_v2 = Some(bundled.triggerables_v2); let (mut tx, npath, v_id) = create_app_internal(authed, db, user_db, &w_id, true, app).await?; - store_raw_app_file(&w_id, &v_id, "js", bytes::Bytes::from(js), &mut tx).await?; - if !css.is_empty() { - store_raw_app_file(&w_id, &v_id, "css", bytes::Bytes::from(css), &mut tx).await?; + store_raw_app_file(&w_id, &v_id, "js", bytes::Bytes::from(bundled.js), &mut tx).await?; + if !bundled.css.is_empty() { + store_raw_app_file( + &w_id, + &v_id, + "css", + bytes::Bytes::from(bundled.css), + &mut tx, + ) + .await?; } tx.commit().await?; @@ -2835,11 +2907,19 @@ async fn create_app_raw_source( Ok((StatusCode::CREATED, npath)) } -/// The part of a raw app's value the bundler needs. +/// The part of a raw app's value the source endpoints read: `files` to bundle, +/// and `runnables` to derive the policy from, passed through untouched because +/// the CLI is what reads them. #[derive(Deserialize)] -struct RawAppSourceFiles { +struct RawAppSourceValue { #[serde(default)] files: HashMap, + #[serde(default = "empty_runnables")] + runnables: Box, +} + +fn empty_runnables() -> Box { + RawValue::from_string("{}".to_string()).expect("valid json") } fn reject_kind_change(path: &str, raw_app: bool, deployed_raw_app: Option) -> Result<()> { @@ -2945,7 +3025,11 @@ async fn update_app_raw<'a>( &w_id, path, multipart, - update_app_internal + // `/apps/update_raw` carries a whole prebuilt app: its policy comes from + // the caller (the editor, the CLI), which derived the triggerables itself. + |authed, db, user_db, w_id, path, raw_app, app| update_app_internal( + authed, db, user_db, w_id, path, raw_app, app, None + ) ) .await?; tally_app_rename(&db2, &w_id, &opath, &npath, v_id).await; @@ -2976,7 +3060,11 @@ async fn update_app_internal<'a>( w_id: &str, path: &str, raw_app: bool, - ns: EditApp, + mut ns: EditApp, + // Raw-app source deploys derive these from the value on a worker. Merged + // into the policy here rather than by the caller so it happens under the + // app-row lock taken below. + derived_triggerables: Option>, ) -> Result<(sqlx::Transaction<'a, sqlx::Postgres>, String, i64)> { use sql_builder::prelude::*; @@ -3021,6 +3109,10 @@ async fn update_app_internal<'a>( || ns.summary.is_some() || ns.custom_path.is_some() || ns.labels.is_some() + // A source deploy may send nothing but the value, and its runnables are + // what the derived grants describe: skipping the UPDATE here would keep + // the grants keyed to the sources this deploy just replaced. + || derived_triggerables.is_some() { let mut sqlb = SqlBuilder::update_table("app"); sqlb.and_where_eq("path", "?".bind(&path)); @@ -3088,24 +3180,66 @@ async fn update_app_internal<'a>( } } - if let Some(mut npolicy) = ns.policy { + // A source deploy brings triggerables derived from its value but may send + // no policy at all, in which case the rest of the deployed one carries + // over. Either way a policy is about to be written wholesale. + let caller_sent_policy = ns.policy.is_some(); + if caller_sent_policy || derived_triggerables.is_some() { + // One locked read serving everything below: the mode an omitted one + // inherits, the already-anonymous check, and the policy a source + // deploy carries over. FOR UPDATE holds the row until this + // transaction's UPDATE commits, so a policy edit landing between the + // read and the write cannot be clobbered by this stale snapshot. + let deployed = sqlx::query_scalar!( + "SELECT policy FROM app WHERE path = $1 AND workspace_id = $2 FOR UPDATE", + path, + w_id + ) + .fetch_optional(&mut *tx) + .await?; + let deployed_policy = deployed + .as_ref() + .and_then(|p| serde_json::from_value::(p.clone()).ok()); + + let mut npolicy = match ns.policy.take() { + Some(npolicy) => npolicy, + // Carrying the deployed policy forward is only safe if all of it + // survived the round trip; a partial one would silently drop + // `on_behalf_of`, sandboxing or S3 rules. + None => deployed_policy.clone().ok_or_else(|| { + Error::internal_err(format!( + "app {path} has no readable policy to deploy its new sources under" + )) + })?, + }; + if let Some(triggerables) = derived_triggerables { + npolicy.triggerables = None; + npolicy.triggerables_v2 = Some(triggerables); + } validate_frontend_sdk_scopes(&npolicy)?; - if matches!(npolicy.execution_mode, ExecutionMode::Anonymous) && !authed.is_admin { + // The policy is written wholesale, so one that states no mode would + // otherwise re-permission the app to the create-time default: a + // `viewer` app would start running on the publisher's identity. Keep + // the deployed mode instead, and make the stored policy state it. + if npolicy.stated_execution_mode().is_none() { + npolicy.set_execution_mode( + deployed_policy + .as_ref() + .map(|d| d.execution_mode()) + .unwrap_or_default(), + ); + } + if matches!(npolicy.execution_mode(), ExecutionMode::Anonymous) && !authed.is_admin { // Restricted users may keep deploying an app that is already // public, but flipping an app to anonymous (public) access is // gated by the RestrictAnonymousAppDeployment protection rule. - // FOR UPDATE locks the row until this transaction's policy - // UPDATE commits, so a concurrent admin downgrade cannot be - // silently overwritten by a stale redeploy keeping anonymous. - let already_anonymous = sqlx::query_scalar!( - "SELECT policy->>'execution_mode' = 'anonymous' FROM app WHERE path = $1 AND workspace_id = $2 FOR UPDATE", - path, - w_id - ) - .fetch_optional(&mut *tx) - .await? - .flatten() - .unwrap_or(false); + // An unreadable deployed policy reads as not-anonymous, the + // strict direction. + let already_anonymous = deployed + .as_ref() + .and_then(|p| p.get("execution_mode")) + .and_then(|m| m.as_str()) + == Some("anonymous"); if !already_anonymous { if let RuleCheckResult::Blocked(msg) = check_user_against_rule( w_id, @@ -3131,7 +3265,10 @@ async fn update_app_internal<'a>( preserved_on_behalf_of = Some(obo_email.clone()); } } - } else { + } else if caller_sent_policy { + // Submitting a policy is how a deployer claims the app's + // execution identity. A source deploy that sent none is not + // claiming anything, so whoever the app already runs as stays. npolicy.on_behalf_of = Some(username_to_permissioned_as(&authed.username)); npolicy.on_behalf_of_email = Some(authed.email.clone()); } @@ -3345,7 +3482,7 @@ async fn get_on_behalf_details_from_policy_and_authed( policy: &Policy, opt_authed: &Option, ) -> Result<(String, String, String)> { - let (username, permissioned_as, email) = match policy.execution_mode { + let (username, permissioned_as, email) = match policy.execution_mode() { ExecutionMode::Anonymous => { let username = opt_authed .as_ref() @@ -3542,7 +3679,7 @@ async fn execute_component( force_viewer_delete_after_secs, .. } => ( - &Policy { execution_mode: ExecutionMode::Viewer, ..Default::default() }, + &Policy { stated_execution_mode: Some(ExecutionMode::Viewer), ..Default::default() }, &PolicyTriggerableInputs { static_inputs, one_of_inputs: force_viewer_one_of_fields.unwrap_or_default(), @@ -3639,7 +3776,7 @@ async fn execute_component( let policy_triggerables = triggerables_v2 .get(path) // start with `path` in case we can avoid the next` format!`. .or_else(|| triggerables_v2.get(&format!("{}:{}", payload.component, &path))) - .or(match policy.execution_mode { + .or(match policy.execution_mode() { // A Viewer app may invoke any deployed `script`/`flow` it // references (resolved as the caller), but caller-supplied // inline `raw_code` must match a publisher-pinned @@ -3658,7 +3795,7 @@ async fn execute_component( }; // Check rate limit for anonymous (public) executions - if matches!(policy.execution_mode, ExecutionMode::Anonymous) && opt_authed.is_none() { + if matches!(policy.execution_mode(), ExecutionMode::Anonymous) && opt_authed.is_none() { if let Some(limit) = crate::workspaces::get_public_app_rate_limit(&db, &w_id).await? { if limit > 0 { crate::public_app_rate_limit::check_and_increment(&w_id, limit)?; @@ -3668,7 +3805,8 @@ async fn execute_component( // Execution is publisher and an user is authenticated: check if the user is authorized to // execute the app. - if let (ExecutionMode::Publisher, Some(authed)) = (policy.execution_mode, opt_authed.as_ref()) { + if let (ExecutionMode::Publisher, Some(authed)) = (policy.execution_mode(), opt_authed.as_ref()) + { lazy_static! { /// Cache for the permit to execute an app component. static ref PERMIT_CACHE: cache::Cache<[u8; 32], bool> = cache::Cache::new(1000); @@ -3993,7 +4131,7 @@ async fn upload_s3_file_from_app( } check_scopes(authed, || format!("apps:write:{}", path.to_path()))?; Some(Policy { - execution_mode: ExecutionMode::Viewer, + stated_execution_mode: Some(ExecutionMode::Viewer), triggerables: None, triggerables_v2: None, on_behalf_of: None, @@ -4406,7 +4544,7 @@ async fn get_on_behalf_authed_from_app( ) -> Result<(ApiAuthed, Policy)> { let policy = if let Some(force_allowed_s3_keys) = force_allowed_s3_keys { Policy { - execution_mode: ExecutionMode::Viewer, + stated_execution_mode: Some(ExecutionMode::Viewer), triggerables: None, triggerables_v2: None, on_behalf_of: None, @@ -4430,7 +4568,7 @@ async fn get_on_behalf_authed_from_app( .map(|p| serde_json::from_value::(p).map_err(to_anyhow)) .transpose()? .unwrap_or_else(|| Policy { - execution_mode: ExecutionMode::Viewer, + stated_execution_mode: Some(ExecutionMode::Viewer), triggerables: None, triggerables_v2: None, on_behalf_of: None, @@ -4497,7 +4635,7 @@ async fn check_if_allowed_to_access_s3_file_from_app( } } - if matches!(policy.execution_mode, ExecutionMode::Viewer) && !is_app_embed { + if matches!(policy.execution_mode(), ExecutionMode::Viewer) && !is_app_embed { // Viewer mode: the on-behalf identity IS the viewer, so the downstream // get_workspace_s3_resource_and_check_paths already bounds the read by // their own perms — no provenance gate (it would over-restrict). Embed @@ -5507,3 +5645,25 @@ mod embed_token_tests { assert!(parse_embed_policy("not json").is_err()); } } + +#[cfg(test)] +mod policy_tests { + /// `execution_mode` is optional in the OpenAPI schema the API clients and the + /// MCP tools are generated from, so an omitted one must deserialize. It must + /// resolve to `publisher`, never `anonymous`, and must stay distinguishable + /// from a stated `publisher` so the update path can keep the deployed mode + /// instead of re-permissioning the app. + #[test] + fn policy_execution_mode_defaults_to_publisher() { + use super::{ExecutionMode, Policy}; + + let p: Policy = serde_json::from_str("{}").expect("empty policy must deserialize"); + assert_eq!(p.execution_mode(), ExecutionMode::Publisher); + assert_eq!(p.stated_execution_mode(), None); + + // An explicit mode still wins, and reads back as stated. + let p: Policy = serde_json::from_str(r#"{"execution_mode": "anonymous"}"#).unwrap(); + assert_eq!(p.execution_mode(), ExecutionMode::Anonymous); + assert_eq!(p.stated_execution_mode(), Some(ExecutionMode::Anonymous)); + } +} diff --git a/backend/windmill-api/src/apps_raw_bundle.rs b/backend/windmill-api/src/apps_raw_bundle.rs index 833e7c6b58..f50f94f8eb 100644 --- a/backend/windmill-api/src/apps_raw_bundle.rs +++ b/backend/windmill-api/src/apps_raw_bundle.rs @@ -28,11 +28,24 @@ use windmill_common::{ }; use windmill_queue::{push, PushArgs, PushIsolationLevel}; +use crate::apps::PolicyTriggerableInputs; use crate::db::ApiAuthed; /// The bundle job's script, as its own file so it stays readable TypeScript. const BUNDLER_TS: &str = include_str!("apps_raw_bundler.ts"); +/// The frontend's policy derivation, bundled by `cli/generate-app-policy.ts` and +/// prepended to the job so it runs there. Carried in the binary rather than +/// invoked through the job's `wmill`: the images install the CLI unpinned, so an +/// image can hold one older than its server, and reaching for a pinned release +/// off npm is what the installed-CLI branch below exists to avoid. +const POLICY_JS: &str = include_str!("apps_raw_policy.gen.js"); + +/// What the job runs: the derivation, then the bundler that calls it. +fn bundle_job_script() -> String { + format!("{POLICY_JS}\n{BUNDLER_TS}") +} + /// Cap the job so a pathological `package.json` can't sit on a worker forever. /// This bounds the *run*, not the wait: see `wait_for_bundle`. const BUNDLE_TIMEOUT_SECS: i32 = 300; @@ -41,6 +54,18 @@ const BUNDLE_TIMEOUT_SECS: i32 = 300; struct BundleResult { js_gz: String, css_gz: String, + /// Parsed rather than passed through: a policy the server cannot read is a + /// failed deploy, not an app whose runnables are refused at run time. + #[serde(default)] + triggerables_v2: HashMap, +} + +/// What a bundle produced: the js/css a deployed raw app serves, and the +/// `triggerables_v2` its policy grants. +pub(crate) struct BundledApp { + pub js: String, + pub css: String, + pub triggerables_v2: HashMap, } /// This server's release, without the git describe suffix an off-tag build @@ -55,7 +80,7 @@ fn release_version() -> String { /// installed: the CLI for this server's release, fetched on the spot. A dev /// server is off-tag and so asks for the last release; to build with an /// unreleased CLI set `WM_RAW_APP_BUNDLER_CLI` to the whole command, e.g. -/// `bun run /path/to/cli/src/main.ts app bundle` — which also stops the job from +/// `bun run /path/to/cli/src/main.ts app bundle`, which also stops the job from /// preferring an installed `wmill`. fn bundler_cli_command() -> Vec { match std::env::var("WM_RAW_APP_BUNDLER_CLI") { @@ -88,7 +113,8 @@ pub(crate) async fn bundle_raw_app_sources( authed: &ApiAuthed, w_id: &str, files: &HashMap, -) -> Result<(String, String)> { + runnables: &serde_json::value::RawValue, +) -> Result { crate::utils::check_scopes(authed, || "jobs:run".to_string())?; if files.is_empty() { @@ -119,6 +145,7 @@ pub(crate) async fn bundle_raw_app_sources( "prefer_installed_cli".to_string(), to_raw_value(&!overridden), ); + args.insert("runnables".to_string(), runnables.to_owned()); let tx = PushIsolationLevel::Isolated(user_db.clone(), authed.clone().into()); let (uuid, tx) = push( @@ -127,7 +154,7 @@ pub(crate) async fn bundle_raw_app_sources( w_id, JobPayload::Code(RawCode { hash: None, - content: BUNDLER_TS.to_string(), + content: bundle_job_script(), path: Some("bundle raw app".to_string()), language: ScriptLang::Bun, lock: None, @@ -179,7 +206,7 @@ async fn wait_for_bundle( w_id: &str, uuid: Uuid, authed: &ApiAuthed, -) -> Result<(String, String)> { +) -> Result { let (result, success) = windmill_api_jobs::execution::run_wait_result_internal( db, uuid, @@ -207,7 +234,7 @@ async fn wait_for_bundle( let limit = *crate::REQUEST_SIZE_LIMIT.read().await * 5; let js = gunzip_b64(&bundle.js_gz, limit)?; let css = gunzip_b64(&bundle.css_gz, limit - js.len())?; - Ok((js, css)) + Ok(BundledApp { js, css, triggerables_v2: bundle.triggerables_v2 }) } /// Bounded: what the job returns is compressed, so the result-size cap says diff --git a/backend/windmill-api/src/apps_raw_bundler.ts b/backend/windmill-api/src/apps_raw_bundler.ts index ceefc93207..6663a0f83e 100644 --- a/backend/windmill-api/src/apps_raw_bundler.ts +++ b/backend/windmill-api/src/apps_raw_bundler.ts @@ -9,13 +9,23 @@ * and the Svelte/Vue plugins included. Reimplementing any of that here would be * a third bundler to keep in step with the other two. */ +declare const __wmillAppPolicy: { + updateRawAppPolicy: ( + runnables: Record, + current: undefined + ) => Promise<{ triggerables_v2: Record }> +} + export async function main( files: Record, shared_ui: Record | undefined, cli_command: string[], // Set unless the server was told to build with a specific command. - prefer_installed_cli: boolean | undefined -): Promise<{ js_gz: string; css_gz: string }> { + prefer_installed_cli: boolean | undefined, + // The app's `value.runnables`, whose policy is derived here for the same + // reason the bundle is built here: it has to match what the editor writes. + runnables: Record | undefined +): Promise<{ js_gz: string; css_gz: string; triggerables_v2: Record }> { const fs = await import('node:fs/promises') const path = await import('node:path') @@ -114,8 +124,53 @@ export async function main( throw new Error('bundle produced no javascript:\n' + buildOutput) } + // A runnable the derivation can't fully classify still yields a key, just an + // unusable one (`r:undefined/undefined`, or the hash of an absent script), and + // the deploy would then succeed with grants no run can ever match. The tool + // schema describes `runnables` only as an object and the on-disk format has no + // discriminator at all (`wmill app push` adds it), so these shapes are all + // reachable: check them before deriving and name the ones at fault. An + // explicitly empty entry is a runnable nobody configured yet, and needs no + // grant. + const nonEmpty = (v: unknown) => typeof v === 'string' && v.length > 0 + // The prefixes `execute_component` resolves a run against; anything else is a + // grant no run can match. + const RUN_TYPES = ['script', 'flow', 'hubscript'] + const malformed = Object.entries(runnables ?? {}) + .filter(([, r]) => r != null) + .filter(([, r]) => { + if (typeof r !== 'object') return true + const run = r as Record + if (run.type === 'inline' || run.type === 'runnableByName') { + return !nonEmpty(run.inlineScript?.content) + } + if (run.type === 'path' || run.type === 'runnableByPath') { + return !RUN_TYPES.includes(run.runType) || !nonEmpty(run.path) + } + return true + }) + .map(([id]) => id) + if (malformed.length > 0) { + throw new Error( + `no policy could be derived for runnable(s) ${malformed.join(', ')}: each must be an ` + + `object with a \`type\` of "inline" (with \`inlineScript.content\`) or "path" (with ` + + `\`path\` and a \`runType\` of ${RUN_TYPES.join(', ')})` + ) + } + + // The policy's `triggerables_v2` is the allowlist the server matches every run + // against, keyed by a hash of each inline runnable's code. Derived by the + // frontend's own code, bundled into this script by cli/generate-app-policy.ts, + // so the keys are the ones the app editor writes: anything else leaves the + // app's runnables "forbidden by policy". Prepended above as a plain `var`, so + // it is in this module's scope (a module's top-level `var` is not a global). + const { triggerables_v2 } = await __wmillAppPolicy.updateRawAppPolicy( + runnables ?? {}, + undefined + ) + // Gzipped so a large app's bundle stays well inside MAX_RESULT_SIZE_MB, which // a deployment can set far below the 500MB default. const gz = (s: string) => Buffer.from(Bun.gzipSync(Buffer.from(s, 'utf8'))).toString('base64') - return { js_gz: gz(js), css_gz: gz(css) } + return { js_gz: gz(js), css_gz: gz(css), triggerables_v2 } } diff --git a/backend/windmill-api/src/apps_raw_policy.gen.js b/backend/windmill-api/src/apps_raw_policy.gen.js new file mode 100644 index 0000000000..1d7c5a48fc --- /dev/null +++ b/backend/windmill-api/src/apps_raw_policy.gen.js @@ -0,0 +1,7 @@ +// Generated by cli/generate-app-policy.ts. Do not edit. +// Run `bun run gen:app-policy` from cli/ to rebuild it from +// frontend/src/lib/components/raw_apps/rawAppPolicy.ts. +// +// Prepended to the raw-app bundle job (see apps_raw_bundle.rs), which calls +// `__wmillAppPolicy.updateRawAppPolicy` from its own module scope. +var __wmillAppPolicy=(()=>{var g=Object.create;var i=Object.defineProperty;var b=Object.getOwnPropertyDescriptor;var m=Object.getOwnPropertyNames;var A=Object.getPrototypeOf,S=Object.prototype.hasOwnProperty;var x=(e=>typeof require<"u"?require:typeof Proxy<"u"?new Proxy(e,{get:(t,p)=>(typeof require<"u"?require:t)[p]}):e)(function(e){if(typeof require<"u")return require.apply(this,arguments);throw Error('Dynamic require of "'+e+'" is not supported')});var f=(e,t)=>{for(var p in t)i(e,p,{get:t[p],enumerable:!0})},s=(e,t,p,a)=>{if(t&&typeof t=="object"||typeof t=="function")for(let r of m(t))!S.call(e,r)&&r!==p&&i(e,r,{get:()=>t[r],enumerable:!(a=b(t,r))||a.enumerable});return e};var h=(e,t,p)=>(p=e!=null?g(A(e)):{},s(t||!e||!e.__esModule?i(p,"default",{value:e,enumerable:!0}):p,e)),R=e=>s(i({},"__esModule",{value:!0}),e);var v={};f(v,{updateRawAppPolicy:()=>d});function u(e){return Object.fromEntries(Object.entries(e??{}).filter(([t,p])=>p.type=="static").map(([t,p])=>[t,p.value]))}async function c(e){try{let t=new TextEncoder().encode(e),p=await crypto.subtle.digest("SHA-256",t);return Array.from(new Uint8Array(p)).map(n=>n.toString(16).padStart(2,"0")).join("")}catch{let{Sha256:t}=await import("@aws-crypto/sha256-js"),p=new t;return p.update(e??""),Array.from(await p.digest()).map(n=>n.toString(16).padStart(2,"0")).join("")}}function l(e){return e?.type=="runnableByPath"||e?.type=="path"}function y(e){return e?.type=="runnableByName"||e?.type=="inline"}async function d(e,t){let p=(await Promise.all(Object.entries(e).map(async([n,o])=>await _(n,o,o?.fields??{})))).filter(n=>n!=null),a=Object.fromEntries(p);return{...t,triggerables_v2:a}}function I(e,t){let p={};typeof e.delete_after_secs=="number"&&e.delete_after_secs>=0&&(p.delete_after_secs=e.delete_after_secs);let a=Object.entries(t).map(([r,n])=>n.sensitive?r:void 0).filter(Boolean);return a.length>0&&(p.sensitive_inputs=a),e.inlineScript?.tag&&(p.tag=e.inlineScript.tag),p}async function _(e,t,p){let a=u(p),r=Object.entries(p).map(([n,o])=>o.allowUserResources?n:void 0).filter(Boolean);if(y(t)){let n=await c(t.inlineScript?.content);return[`${e}:rawscript/${n}`,{static_inputs:a,one_of_inputs:{},allow_user_resources:r,...I(t,p)}]}else if(l(t)){let n=t.runType!=="hubscript"?t.runType:"script";return[`${e}:${n}/${t.path}`,{static_inputs:a,one_of_inputs:{},allow_user_resources:r,...I(t,p)}]}}return R(v);})(); diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index acfdc6639b..584c44bbec 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -25,6 +25,7 @@ use std::time::Instant; use tokio::io::AsyncReadExt; use tower::ServiceBuilder; use url::Url; +use windmill_common::assets::AssetUsageAccessType; #[cfg(all(feature = "enterprise", feature = "instance_smtp"))] use windmill_common::auth::is_super_admin_email; use windmill_common::auth::TOKEN_PREFIX_LEN; @@ -78,7 +79,7 @@ use crate::{ users::{ get_scope_tags, require_owner_of_path, require_path_read_access_for_preview, OptAuthed, }, - utils::{check_scopes, content_plain, require_super_admin}, + utils::{build_scope_path_predicate, check_scopes, content_plain, require_super_admin}, }; use anyhow::Context; use axum::{ @@ -136,6 +137,7 @@ pub fn workspaced_service() -> Router { Router::new() .route("/run_progress/{id}", get(get_run_progress)) + .route("/run_assets/{id}", get(list_run_assets)) .route("/dbt_graph/{id}", get(get_dbt_run_graph)) .route("/dbt_resumable/{id}", get(get_dbt_resumable)) .route( @@ -1188,6 +1190,121 @@ async fn get_run_progress( Ok(Json(rows)) } +#[derive(Serialize)] +struct RunAsset { + path: String, + kind: windmill_common::assets::AssetKind, + #[serde(skip_serializing_if = "Option::is_none")] + access_type: Option, +} + +#[derive(Serialize)] +struct RunAssets { + assets: Vec, + truncated: bool, +} + +/// A fan-out run — a forloop writing one object per iteration — touches as many +/// assets as it has steps, and the whole list would land in one response and one +/// list in the browser. Cap it, and say so rather than serving a prefix that +/// reads like the whole answer. +const RUN_ASSETS_CAP: usize = 1000; + +/// Assets a run touched, as recorded by runtime detection, aggregated over the +/// whole job tree: the recorder attributes an asset to the job that performed +/// the operation, which for a flow step or a workflow-as-code task is not the +/// job the user opened. Lives with the job routes for the same reason +/// `run_progress` does — `asset` has no RLS, and `require_job_read_access` is +/// what applies the view token, a scoped token's tag filter and the app-embed +/// cutoff. +async fn list_run_assets( + authed: ApiAuthed, + OptViewToken(view_token): OptViewToken, + Extension(db): Extension, + Extension(user_db): Extension, + Path((w_id, job_id)): Path<(String, Uuid)>, +) -> error::JsonResult { + let created_by = sqlx::query_scalar!( + "SELECT created_by FROM v2_job WHERE id = $1 AND workspace_id = $2", + job_id, + &w_id + ) + .fetch_optional(&db) + .await?; + let Some(created_by) = created_by else { + return Ok(Json(RunAssets { assets: vec![], truncated: false })); + }; + require_job_read_access( + &db, + &user_db, + &authed, + &w_id, + &job_id, + &created_by, + view_token.as_deref(), + ) + .await?; + + // Walked on `db`, like the flow tree is: the gate above is what authorizes the + // run, and it grants access the caller's own RLS does not have — a view token, + // or a job the caller launched that runs as someone else. Re-filtering the tree + // through `user_db` would drop exactly those, and hand a share-link viewer the + // empty tab this endpoint exists to fix. The tag scope is the one restriction + // that must still hold per job, since the gate only checked the root. It gates + // which jobs' assets are read, not which are walked through: the gate admits a + // job on its own tag, so an out-of-scope job in the middle of the tree must not + // hide a descendant the caller could have asked for directly. + let scope_tags = get_scope_tags(&authed).map(|v| v.iter().map(|s| s.to_string()).collect_vec()); + let rows = sqlx::query!( + r#"WITH RECURSIVE job_tree AS ( + SELECT id, tag FROM v2_job WHERE id = $2 AND workspace_id = $1 + UNION + SELECT j.id, j.tag FROM v2_job j JOIN job_tree t ON j.parent_job = t.id + WHERE j.workspace_id = $1 + ) + SELECT + a.path, + a.kind AS "kind!: windmill_common::assets::AssetKind", + -- Several jobs of the tree touch one asset, each recording its own + -- access. A job that recorded none contributes nothing rather than + -- erasing a sibling's, so an all-null group is the only unknown one. + -- Grouping here, not in Rust, is what makes LIMIT count assets: the + -- retention keeps up to ten job rows per asset. + COALESCE(bool_or(a.usage_access_type IN ('r', 'rw')), false) AS "any_read!", + COALESCE(bool_or(a.usage_access_type IN ('w', 'rw')), false) AS "any_write!" + FROM asset a JOIN job_tree t ON a.usage_path = t.id::text + WHERE a.workspace_id = $1 AND a.usage_kind = 'job' + AND ($3::text[] IS NULL OR t.tag = ANY($3)) + GROUP BY a.path, a.kind + ORDER BY a.path, a.kind + LIMIT $4"#, + w_id, + job_id, + scope_tags.as_deref(), + // One asset past the cap is how the response learns it was cut. + RUN_ASSETS_CAP as i64 + 1 + ) + .fetch_all(&db) + .await?; + + let truncated = rows.len() > RUN_ASSETS_CAP; + let assets = rows + .into_iter() + .take(RUN_ASSETS_CAP) + .map(|row| RunAsset { + path: row.path, + kind: row.kind, + access_type: match (row.any_read, row.any_write) { + (true, true) => Some(AssetUsageAccessType::RW), + (true, false) => Some(AssetUsageAccessType::R), + (false, true) => Some(AssetUsageAccessType::W), + (false, false) => None, + }, + }) + .collect(); + Ok(Json(RunAssets { assets, truncated })) +} + async fn get_flow_job_debug_info( OptViewToken(view_token): OptViewToken, OptAuthed(opt_authed): OptAuthed, @@ -1458,6 +1575,11 @@ pub(crate) async fn require_job_read_access( } } + // A path-scoped `jobs:run` token is likewise hard-restricted to the runnables it + // may start, ahead of every grant below — the token is handed out to run one thing, + // so it must not read jobs of anything else merely because its owner could. + require_job_within_run_scope(db, authed, w_id, job_id).await?; + // Fast path: you can always read a job you launched. This is also load-bearing // for apps — a component job runs as the app policy's `permissioned_as`, but its // `created_by` is the launching viewer, so the RLS probe below would hide it. @@ -1583,6 +1705,95 @@ pub(crate) async fn require_job_read_access( } } +/// Confines a path-scoped `jobs:run::` token to jobs of the runnables it may +/// start. Such a token is minted per script/flow for a webhook or CI caller, which needs +/// to start that runnable and poll the resulting job — nothing more. Without this, the +/// by-id read routes it reaches for polling (`RUN_WHITELISTED_GET_PATHS`) would serve +/// it the args/result/logs of any job its owner's identity can see, defeating the path +/// confinement the scope exists to provide. +/// +/// The scope may be satisfied by the job itself or by any of its `parent_job` ancestors: +/// a flow step's `runnable_path` is the inner runnable's, so a `jobs:run:flows:` +/// token inspecting its own run must still reach the steps beneath it. +/// +/// An `apps:run|write:` scope is a start grant too, so a job an app launched +/// (`trigger_kind = 'app'`, an app-provenance stamp `/jobs/run` cannot forge) satisfies +/// the confinement for a token scoped to that app. Without this, a token holding both +/// could start an app's inline-script component but not read the run back — those jobs +/// are `AppScript`/`Preview` kinds that no `jobs:run` scope can name. +/// +/// No-op — and no query — for every caller whose job reads are not run-confined (see +/// `job_read_run_confinement`), which is all sessions, unscoped tokens and `jobs:read` +/// tokens. +async fn require_job_within_run_scope( + db: &DB, + authed: &ApiAuthed, + w_id: &str, + job_id: &Uuid, +) -> error::Result<()> { + let Some(confinement) = + windmill_api_auth::scopes::job_read_run_confinement(authed.scopes.as_deref()) + else { + return Ok(()); + }; + // `scope_kind` is the runnable kind a `jobs:run::` scope can name, or + // NULL for a job no such scope reaches directly (previews, dependency jobs, + // flow-inlined scripts) — those are still readable as a step of a matching flow, + // through their ancestors. A `singlestepflow` wraps either a script or a flow, so it + // projects onto the wrapped runnable the same way the batch-rerun query does. + let chain = sqlx::query!( + r#"WITH RECURSIVE chain(id, parent_job) AS ( + SELECT id, parent_job FROM v2_job WHERE id = $1 AND workspace_id = $2 + UNION ALL + SELECT j.id, j.parent_job FROM v2_job j + JOIN chain c ON j.id = c.parent_job AND j.workspace_id = $2 + ) + SELECT j.runnable_path, + CASE + WHEN j.kind IN ('script', 'script_hub', 'unassigned_script') THEN 'scripts' + WHEN j.kind IN ('flow', 'unassigned_flow') THEN 'flows' + WHEN j.kind IN ('singlestepflow', 'unassigned_singlestepflow') THEN + CASE WHEN COALESCE( + (SELECT m->'value'->>'type' + FROM jsonb_array_elements(j.raw_flow->'modules') m + WHERE m->>'id' IN ('a', 'main') + LIMIT 1), + 'script' + ) = 'flow' THEN 'flows' ELSE 'scripts' END + END AS scope_kind, + CASE WHEN j.trigger_kind = 'app' THEN j.trigger END AS launched_by_app + FROM v2_job j JOIN chain c ON c.id = j.id + WHERE j.workspace_id = $2"#, + job_id, + w_id, + ) + .fetch_all(db) + .await?; + + let runs_app = build_scope_path_predicate(authed, "apps", "run"); + let in_scope = + chain.iter().any( + |job| match (job.runnable_path.as_deref(), job.scope_kind.as_deref()) { + (Some(runnable_path), Some(kind)) + if windmill_api_auth::scopes::run_confinement_admits( + &confinement, + kind, + runnable_path, + ) => + { + true + } + _ => job.launched_by_app.as_deref().is_some_and(&runs_app), + }, + ); + + if in_scope { + Ok(()) + } else { + Err(Error::NotFound(format!("Job {job_id} not found"))) + } +} + /// Self + every `parent_job` ancestor (intermediate sub-flows up to the top-level /// root) of `job_id`, resolved via the root DB (flow lineage is not sensitive). /// Falls back to `[job_id]` if the row is absent so callers still run their probe. @@ -1902,7 +2113,15 @@ async fn get_job( // same visibility as `jobs/list` (see `require_job_read_access`), or hold a public // share link when logged out — which `public_view_grant` already established above, // so skip re-deriving it here: this handler is what the public run page polls. - if !has_valid_approval_token && !public_view_grant { + if has_valid_approval_token || public_view_grant { + // Both grants skip the gate below, and with it the run-scope confinement that + // gate carries. That confinement is a hard restriction, so re-apply it: holding + // an approval link for a job must not let a scoped token read one outside the + // runnables it may start. + if let Some(authed) = opt_authed.as_ref() { + require_job_within_run_scope(&db, authed, &w_id, &id).await?; + } + } else { require_opt_authed_job_read_access( &db, &user_db, @@ -5276,6 +5495,15 @@ pub async fn get_suspended_job_flow( .flatten() .ok_or_else(|| anyhow::anyhow!("parent flow job not found"))?; + // The resume secret is this route's gate, so it never reaches + // `require_job_read_access` and the run-scope confinement that gate carries. Re-apply + // it against the flow whose args and status are about to be returned: holding a + // resume secret must not let a scoped token read a flow it may not run. Anonymous + // approvers are unaffected. + if let Some(authed) = authed.as_ref() { + require_job_within_run_scope(&db, authed, &w_id, &flow_id).await?; + } + let flow = GetQuery::new() .without_logs() .without_code() @@ -5455,9 +5683,14 @@ pub async fn create_job_signature( pub async fn get_flow_user_state( authed: ApiAuthed, + Extension(db): Extension, Extension(user_db): Extension, Path((w_id, job_id, key)): Path<(String, Uuid, String)>, ) -> error::JsonResult> { + // Reachable by a `jobs:run` token (it is one of the by-id routes a run needs), so + // apply the same run-scope confinement as the other single-job reads. RLS below + // still governs which jobs the owner's identity can see at all. + require_job_within_run_scope(&db, &authed, &w_id, &job_id).await?; let mut tx = user_db.begin(&authed).await?; let r = sqlx::query_scalar!( r#" @@ -10597,7 +10830,14 @@ async fn get_completed_job_result( _ => false, }; - if !approval_secret_ok { + if approval_secret_ok { + // The approval secret skips the gate below, and with it the run-scope + // confinement that gate carries — re-apply it, as `get_job` does for the + // approval token. Anonymous approval access is untouched. + if let Some(authed) = opt_authed.as_ref() { + require_job_within_run_scope(&db, authed, &w_id, &id).await?; + } + } else { require_opt_authed_job_read_access( &db, &user_db, diff --git a/backend/windmill-api/src/lib.rs b/backend/windmill-api/src/lib.rs index e67420c53c..d2e0ea8e78 100644 --- a/backend/windmill-api/src/lib.rs +++ b/backend/windmill-api/src/lib.rs @@ -178,6 +178,7 @@ mod teams_oss; mod token; mod tracing_init; mod trash; +mod trigger_history; pub mod triggers; mod users; #[cfg(feature = "private")] @@ -280,6 +281,23 @@ async fn set_deploy_origin( windmill_common::deploy_origin::scope(origin, next.run(req)).await } +/// Scope the request in the client kind it declares, so a trigger mutation can +/// be attributed to the CLI rather than to a bare API call. Entered for every +/// request, undeclared ones included: `TriggerSource::of_request` reads the +/// scope's absence as "no request is being served", which is what separates a +/// caller from a worker disabling a trigger on its own. +async fn set_request_client( + req: axum::extract::Request, + next: axum::middleware::Next, +) -> axum::response::Response { + let client = req + .headers() + .get(windmill_common::trigger_history::CLIENT_HEADER) + .and_then(|v| v.to_str().ok()) + .and_then(windmill_common::trigger_history::client_from_header); + windmill_common::trigger_history::scope_client(client, next.run(req)).await +} + #[cfg(not(feature = "tantivy"))] type IndexReader = (); @@ -566,6 +584,16 @@ pub async fn run_server( (Router::new(), Router::new(), Option::<()>::None) }; + // Workers block on this before pulling their first job, so it is released ahead of + // the router tree below. `try_join!` polls this future and `workers_f` on one task, + // so the yield is what lets them proceed; without it they wait out the whole + // synchronous build. A request arriving first queues in the bound listener's backlog. + if let Err(e) = port_tx.send(format!("http://localhost:{}", port)) { + tracing::error!("Failed to send port: {e:#}"); + return Err(anyhow::anyhow!("Failed to send port, exiting early: {e:#}")); + } + tokio::task::yield_now().await; + let mcp_list_tools_service = { #[cfg(feature = "mcp")] { @@ -639,6 +667,7 @@ pub async fn run_server( .nest("/folders_history", folder_history::workspaced_service()) .nest("/groups", groups::workspaced_service()) .nest("/groups_history", group_history::workspaced_service()) + .nest("/triggers_history", trigger_history::workspaced_service()) .nest("/inputs", windmill_api_inputs::workspaced_service()) .nest("/internal_db", internal_db::workspaced_service()) .route("/labels/list", get(list_workspace_labels)) @@ -1136,6 +1165,8 @@ pub async fn run_server( let app = app.layer(axum::middleware::from_fn(set_deploy_origin)); + let app = app.layer(axum::middleware::from_fn(set_request_client)); + let app = app.layer(CatchPanicLayer::custom(|err| { tracing::error!("panic in handler, returning 500: {:?}", err); Response::builder() @@ -1160,14 +1191,11 @@ pub async fn run_server( name.map(|x| format!("name={x}")).unwrap_or_default() ); - if let Err(e) = port_tx.send(format!("http://localhost:{}", port)) { - tracing::error!("Failed to send port: {e:#}"); - return Err(anyhow::anyhow!("Failed to send port, exiting early: {e:#}")); - } - // Announce this server is ready so coordinated restarts can detect a healthy peer. - if let Err(e) = announce_server_started(&db).await { - tracing::warn!("Failed to announce server started: {e:#}"); + if server_mode { + if let Err(e) = announce_server_started(&db).await { + tracing::warn!("Failed to announce server started: {e:#}"); + } } let server = server.with_graceful_shutdown(async move { @@ -1314,25 +1342,35 @@ pub async fn wait_for_db_migrations( const SERVER_HEARTBEAT_TASK: &str = "server_heartbeat"; -/// Write a server-started heartbeat to `background_task_state` so that -/// other instances waiting to restart can detect this server is healthy. +/// Write a server-started heartbeat to `background_task_state` so that other +/// traffic-serving instances waiting to restart can detect this one is healthy. +/// +/// Only `server_mode` processes announce, since only they are peers worth waiting for: +/// `spawn_graceful_killpill` holds a shutdown open to keep the API answered, and a worker, +/// indexer or MCP process coming up is no evidence that it is. +/// +/// The row is keyed per host and `owner` per process, and both halves carry weight. +/// `INSTANCE_NAME` is random per start, so a row keyed on it never conflicts and +/// accumulates one row per start; `owner` is what tells a peer's start from its own +/// when processes share a host. async fn announce_server_started(db: &DB) -> anyhow::Result<()> { - use windmill_common::INSTANCE_NAME; + use windmill_common::{utils::HOSTNAME, INSTANCE_NAME}; let instance = INSTANCE_NAME.as_str(); + let host = HOSTNAME.as_str(); sqlx::query( "INSERT INTO background_task_state (name, value, running, owner, started_at, updated_at) VALUES ($1, '\"started\"'::jsonb, true, $2, NOW(), NOW()) ON CONFLICT (name) - DO UPDATE SET updated_at = NOW(), running = true, owner = $2", + DO UPDATE SET started_at = NOW(), updated_at = NOW(), running = true, owner = $2", ) - .bind(format!("{SERVER_HEARTBEAT_TASK}:{instance}")) + .bind(format!("{SERVER_HEARTBEAT_TASK}:{host}")) .bind(instance) .execute(db) .await?; - tracing::info!("Announced server started for instance {instance}"); + tracing::info!("Announced server started for instance {instance} on host {host}"); Ok(()) } @@ -1362,10 +1400,10 @@ pub async fn check_any_server_started(db: &DB, not_before: chrono::DateTime not_before` (the moment a restart was initiated), diff --git a/backend/windmill-api/src/mcp/auto_generated_endpoints.rs b/backend/windmill-api/src/mcp/auto_generated_endpoints.rs index 937ae32bf8..6971c4a715 100644 --- a/backend/windmill-api/src/mcp/auto_generated_endpoints.rs +++ b/backend/windmill-api/src/mcp/auto_generated_endpoints.rs @@ -1199,7 +1199,7 @@ Creates a new version of an existing script when called with the same path and t }, "execution_mode": { "type": "string", - "description": "Possible values: viewer, publisher, anonymous" + "description": "Who the app's runnables execute as. Optional, and what omitting it means depends on the operation: creating an app defaults it to `publisher` (runs on behalf of the app's publisher and requires an authenticated viewer), while updating one keeps the mode the app is already deployed under. Either way `anonymous`, which makes the app publicly executable, is never assumed. Possible values: viewer, publisher, anonymous" }, "on_behalf_of": { "type": "string" @@ -1313,7 +1313,7 @@ Creates a new version of an existing script when called with the same path and t }, "execution_mode": { "type": "string", - "description": "Possible values: viewer, publisher, anonymous" + "description": "Who the app's runnables execute as. Optional, and what omitting it means depends on the operation: creating an app defaults it to `publisher` (runs on behalf of the app's publisher and requires an authenticated viewer), while updating one keeps the mode the app is already deployed under. Either way `anonymous`, which makes the app publicly executable, is never assumed. Possible values: viewer, publisher, anonymous" }, "on_behalf_of": { "type": "string" diff --git a/backend/windmill-api/src/mcp_tools.rs b/backend/windmill-api/src/mcp_tools.rs index bba945ac54..e1931873a9 100644 --- a/backend/windmill-api/src/mcp_tools.rs +++ b/backend/windmill-api/src/mcp_tools.rs @@ -2,6 +2,7 @@ use axum::{ extract::{Extension, Path}, Json, }; +use serde::Deserialize; use serde_json::value::RawValue; use windmill_api_auth::{check_scopes, ApiAuthed}; use windmill_common::{ @@ -11,21 +12,46 @@ use windmill_common::{ }; use windmill_store::{resources::explain_resource_perm_error, variables::get_value_internal}; -pub(crate) async fn get_mcp_tools( - authed: ApiAuthed, - Extension(db): Extension, - Extension(user_db): Extension, - Path((w_id, path)): Path<(String, StripPath)>, -) -> JsonResult> { - let path = path.to_path(); - check_scopes(&authed, || format!("resources:read:{}", path))?; +/// A connected MCP server is a third party the user chose, reached over a +/// connection this request holds open: without a deadline one that never answers +/// pins an API worker and the chat turn behind it for as long as it likes. +const MCP_DEADLINE: std::time::Duration = std::time::Duration::from_secs(60); +/// Best-effort courtesy to the server, so it cannot extend the deadline above. +const MCP_SHUTDOWN_DEADLINE: std::time::Duration = std::time::Duration::from_secs(5); - let mut tx = user_db.clone().begin(&authed).await?; +async fn with_deadline( + what: &str, + fut: impl std::future::Future>, +) -> Result { + tokio::time::timeout(MCP_DEADLINE, fut) + .await + .map_err(|_| { + Error::ExecutionErr(format!( + "MCP server did not answer within {}s ({what})", + MCP_DEADLINE.as_secs() + )) + })? +} + +/// Connect to the MCP server described by the `mcp` resource at `path`. +/// +/// The caller is responsible for the scope check; everything else (resource +/// visibility, token resolution) goes through the caller's permissioned path so +/// the endpoint can never act as a confused deputy for a resource or secret the +/// caller cannot read. +async fn connect_mcp_client( + authed: &ApiAuthed, + db: &DB, + user_db: &UserDB, + w_id: &str, + path: &str, +) -> Result { + let mut tx = user_db.clone().begin(authed).await?; let resource_value_o = sqlx::query_scalar!( "SELECT value as \"value: sqlx::types::Json>\" FROM resource WHERE path = $1 AND workspace_id = $2", - &path, - &w_id + path, + w_id ) .fetch_optional(&mut *tx) .await?; @@ -33,7 +59,7 @@ pub(crate) async fn get_mcp_tools( tx.commit().await?; if resource_value_o.is_none() { - explain_resource_perm_error(&path, &w_id, &db, &authed).await?; + explain_resource_perm_error(path, w_id, db, authed).await?; } let resource_value = not_found_if_none(resource_value_o, "Resource", path)? @@ -58,20 +84,20 @@ pub(crate) async fn get_mcp_tools( WHERE variable.path = $1 AND variable.workspace_id = $2 "#, token_var_path, - &w_id + w_id ) - .fetch_optional(&db) + .fetch_optional(db) .await?; if let Some(info) = token_info { if let (Some(account_id), Some(true)) = (info.account_id, info.is_expired) { - let refresh_tx = user_db.clone().begin(&authed).await?; + let refresh_tx = user_db.clone().begin(authed).await?; if let Err(e) = crate::oauth2_oss::_refresh_token( refresh_tx, token_var_path, - &w_id, + w_id, account_id, - &db, + db, ) .await { @@ -93,17 +119,40 @@ pub(crate) async fn get_mcp_tools( if token_var_path.trim().is_empty() { None } else { - let db_authed = - DbWithOptAuthed::from_authed(&authed, db.clone(), Some(user_db.clone())); - Some(get_value_internal(&db_authed, &w_id, token_var_path, false).await?) + let db_authed = DbWithOptAuthed::from_authed(authed, db.clone(), Some(user_db.clone())); + Some(get_value_internal(&db_authed, w_id, token_var_path, false).await?) } } else { None }; - let client = windmill_mcp::McpClient::from_resource(mcp_resource, token) + windmill_mcp::McpClient::from_resource(mcp_resource, token) .await - .map_err(|e| Error::ExecutionErr(format!("Failed to connect to MCP server: {}", e)))?; + .map_err(|e| Error::ExecutionErr(format!("Failed to connect to MCP server: {}", e))) +} + +async fn shutdown_mcp_client(client: windmill_mcp::McpClient) { + match tokio::time::timeout(MCP_SHUTDOWN_DEADLINE, client.shutdown()).await { + Ok(Err(e)) => tracing::warn!("Failed to shutdown MCP client: {}", e), + Err(_) => tracing::warn!("MCP client shutdown timed out"), + Ok(Ok(())) => {} + } +} + +pub(crate) async fn get_mcp_tools( + authed: ApiAuthed, + Extension(db): Extension, + Extension(user_db): Extension, + Path((w_id, path)): Path<(String, StripPath)>, +) -> JsonResult> { + let path = path.to_path(); + check_scopes(&authed, || format!("resources:read:{}", path))?; + + let client = with_deadline( + "listing tools", + connect_mcp_client(&authed, &db, &user_db, &w_id, path), + ) + .await?; let tools: Vec = client .available_tools() @@ -114,9 +163,71 @@ pub(crate) async fn get_mcp_tools( }) .collect::>>()?; - if let Err(e) = client.shutdown().await { - tracing::warn!("Failed to shutdown MCP client: {}", e); - } + shutdown_mcp_client(client).await; Ok(Json(tools)) } + +#[derive(Deserialize)] +pub(crate) struct CallMcpToolRequest { + tool: String, + arguments: Option>, + /// Set by a caller that skipped the user's confirmation because it had + /// listed the tool as read-only. Verified below against the live listing. + read_only: Option, +} + +/// `readOnlyHint` is the server's own claim, so this cannot tell a hostile +/// server from an honest one; what it guarantees is that the claim comes from +/// the server about to be called, not from a listing of whatever the resource +/// pointed at when the caller cached it. +fn tool_is_read_only(client: &windmill_mcp::McpClient, tool: &str) -> bool { + client + .available_tools() + .iter() + .find(|t| t.name.as_ref() == tool) + .and_then(|t| t.annotations.as_ref()) + .and_then(|a| a.read_only_hint) + .unwrap_or(false) +} + +pub(crate) async fn call_mcp_tool( + authed: ApiAuthed, + Extension(db): Extension, + Extension(user_db): Extension, + Path((w_id, path)): Path<(String, StripPath)>, + Json(req): Json, +) -> JsonResult { + let path = path.to_path(); + check_scopes(&authed, || format!("resources:write:{}", path))?; + + let arguments = req.arguments.as_ref().map(|a| a.get()).unwrap_or("{}"); + // One deadline over the whole exchange (connect, then call), so a server that + // stalls after answering the handshake is bounded too. + let (client, result) = with_deadline(&format!("calling {}", req.tool), async { + let client = connect_mcp_client(&authed, &db, &user_db, &w_id, path).await?; + if req.read_only == Some(true) && !tool_is_read_only(&client, &req.tool) { + return Ok((client, None)); + } + let result = client.call_tool(&req.tool, arguments).await; + Ok((client, Some(result))) + }) + .await?; + + shutdown_mcp_client(client).await; + + let Some(result) = result else { + return Err(Error::BadRequest(format!( + "MCP tool {} is not marked read-only by the server, it must be called as a tool that modifies data", + req.tool + ))); + }; + + // A tool that ran but reported failure comes back as `Ok` with `isError: + // true` in the payload; forwarding it verbatim lets the caller show the + // server's own error text instead of a generic 500. + let result = result + .map_err(|e| Error::ExecutionErr(format!("Failed to call MCP tool {}: {}", req.tool, e)))?; + + Ok(Json(result)) +} diff --git a/backend/windmill-api/src/public_app_rate_limit.rs b/backend/windmill-api/src/public_app_rate_limit.rs index 5d49c17a71..baeb90ec0b 100644 --- a/backend/windmill-api/src/public_app_rate_limit.rs +++ b/backend/windmill-api/src/public_app_rate_limit.rs @@ -6,42 +6,27 @@ * LICENSE-AGPL for a copy of the license. */ -use chrono::Utc; -use dashmap::DashMap; use hyper::StatusCode; use std::sync::LazyLock; use windmill_common::error::{Error, Result}; +use windmill_common::per_minute_counter::PerMinuteCounter; -struct RateLimitEntry { - count: i32, - minute_bucket: i64, -} - -static RATE_LIMIT_COUNTER: LazyLock> = LazyLock::new(DashMap::new); +static RATE_LIMIT_COUNTER: LazyLock> = + LazyLock::new(PerMinuteCounter::new); pub fn check_and_increment(workspace_id: &str, limit: i32) -> Result<()> { - let current_minute = Utc::now().timestamp() / 60; - - let mut entry = RATE_LIMIT_COUNTER - .entry(workspace_id.to_string()) - .or_insert(RateLimitEntry { count: 0, minute_bucket: current_minute }); - - if entry.minute_bucket != current_minute { - entry.count = 0; - entry.minute_bucket = current_minute; + // Clamp before the cast: `as u32` on a negative limit wraps into an effectively unlimited + // allowance, where a non-positive limit must reject every execution. + if RATE_LIMIT_COUNTER.try_increment(workspace_id.to_string(), limit.max(0) as u32) { + return Ok(()); } - if entry.count >= limit { - return Err(Error::Generic( - StatusCode::TOO_MANY_REQUESTS, - format!( - "Rate limit exceeded for public app executions in workspace '{}'. \ - Limit: {} per minute per server.", - workspace_id, limit - ), - )); - } - - entry.count += 1; - Ok(()) + Err(Error::Generic( + StatusCode::TOO_MANY_REQUESTS, + format!( + "Rate limit exceeded for public app executions in workspace '{}'. \ + Limit: {} per minute per server.", + workspace_id, limit + ), + )) } diff --git a/backend/windmill-api/src/resources.rs b/backend/windmill-api/src/resources.rs index ee26ce758e..803927e43c 100644 --- a/backend/windmill-api/src/resources.rs +++ b/backend/windmill-api/src/resources.rs @@ -1,16 +1,18 @@ #[cfg(feature = "mcp")] -use axum::routing::get; +use axum::routing::{get, post}; use axum::Router; -/// Wraps the subcrate's workspaced_service with the mcp_tools route -/// that depends on windmill-api internals. +/// Wraps the subcrate's workspaced_service with the mcp_tools routes +/// that depend on windmill-api internals. pub fn workspaced_service() -> Router { let router = windmill_store::resources::workspaced_service(); #[cfg(feature = "mcp")] - use crate::mcp_tools::get_mcp_tools; + use crate::mcp_tools::{call_mcp_tool, get_mcp_tools}; #[cfg(feature = "mcp")] - let router = router.route("/mcp_tools/{*path}", get(get_mcp_tools)); + let router = router + .route("/mcp_tools/{*path}", get(get_mcp_tools)) + .route("/mcp_call_tool/{*path}", post(call_mcp_tool)); router } diff --git a/backend/windmill-api/src/token.rs b/backend/windmill-api/src/token.rs index acb65d6cf0..fc808767ec 100644 --- a/backend/windmill-api/src/token.rs +++ b/backend/windmill-api/src/token.rs @@ -236,6 +236,26 @@ lazy_static! { }], }); + // Read-only: `trigger_history` is append-only and written by the server + // alone, so there is no `triggers_history:write`. Its own domain rather + // than a `schedules`/`*_triggers` alias: one listing spans every kind, + // and a history row quotes the whole trigger row (a schedule's `args` + // included), so reading it is an explicit grant rather than a side + // effect of being able to read the trigger. Path-selectable because the + // route filters rows by the caller's path grants. + groups.push(ScopeDomain { + name: "Trigger History".to_string(), + description: Some( + "Read-only access to the modification history of schedules and triggers" + .to_string(), + ), + scopes: vec![ScopeOption { + value: "triggers_history:read".to_string(), + label: "Read".to_string(), + requires_resource_path: true, + }], + }); + groups.extend(build_standard_scope_domains()); groups.extend(build_trigger_scope_domains()); diff --git a/backend/windmill-api/src/trash.rs b/backend/windmill-api/src/trash.rs index 416546b850..53fc088582 100644 --- a/backend/windmill-api/src/trash.rs +++ b/backend/windmill-api/src/trash.rs @@ -9,6 +9,9 @@ use windmill_common::{ db::UserDB, error::{Error, Result}, trashbin::{self, TrashItem, TrashItemWithData}, + trigger_history::{ + self, TriggerHistoryEvent, TriggerOperation, TriggerSource, SCHEDULE_TRIGGER_KIND, + }, utils::require_admin, }; @@ -88,6 +91,31 @@ async fn restore_trash_item( .execute(&mut *tx) .await?; + // A restore puts the trigger back, so the history has to say so: otherwise + // the last thing it records for a live trigger is its own deletion. The + // trashed row is the snapshot, so this needs no extra read. + let restored_trigger_kind = match item.item_kind.as_str() { + SCHEDULE_TRIGGER_KIND => Some(SCHEDULE_TRIGGER_KIND), + // `_trigger` is what `delete_trigger` trashes it under, and the + // stem is the `TRIGGER_TYPE` the history records against. + kind => kind.strip_suffix("_trigger"), + }; + if let Some(trigger_kind) = restored_trigger_kind { + trigger_history::record( + &mut *tx, + TriggerHistoryEvent { + workspace_id: &w_id, + trigger_kind, + path: &item.item_path, + operation: TriggerOperation::Create, + source: TriggerSource::of_request(authed.is_session_token), + username: Some(&authed.username), + changes: trigger_history::summarize_changes(None, item.item_data.get("row")), + }, + ) + .await?; + } + audit_log( &mut *tx, &authed, diff --git a/backend/windmill-api/src/trigger_history.rs b/backend/windmill-api/src/trigger_history.rs new file mode 100644 index 0000000000..75ca03b90f --- /dev/null +++ b/backend/windmill-api/src/trigger_history.rs @@ -0,0 +1,105 @@ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2022 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +use axum::{ + extract::{Extension, Path, Query}, + routing::get, + Router, +}; +use serde::{Deserialize, Serialize}; +use windmill_api_auth::{build_scope_path_filter, check_scopes, ApiAuthed, ScopePathFilter}; +use windmill_common::{ + db::UserDB, + error::JsonResult, + utils::{paginate, Pagination}, +}; + +pub fn workspaced_service() -> Router { + Router::new().route("/list", get(list_trigger_history)) +} + +#[derive(Serialize)] +pub struct TriggerHistoryEntry { + pub id: i64, + pub trigger_kind: String, + pub path: String, + pub operation: String, + pub source: String, + pub username: Option, + pub created_at: chrono::DateTime, + pub changes: Option, +} + +#[derive(Deserialize)] +pub struct ListTriggerHistoryQuery { + pub page: Option, + pub per_page: Option, + /// `"schedule"` or a trigger type (`"http"`, `"kafka"`, …). + pub trigger_kind: Option, + pub path: Option, +} + +/// Two gates, because they answer different questions: the RLS policies on +/// `trigger_history` bound the rows to what the *user* may read, and +/// `triggers_history:read:` bounds them further to what this *token* may +/// read. Without the second, a token scoped to one path could read the diffs of +/// every trigger its user can see, and a `create` row quotes the whole trigger +/// row, a schedule's `args` included. +async fn list_trigger_history( + authed: ApiAuthed, + Extension(user_db): Extension, + Path(w_id): Path, + Query(query): Query, +) -> JsonResult> { + if let Some(path) = query.path.as_deref() { + check_scopes(&authed, || format!("triggers_history:read:{}", path))?; + } + + // In the WHERE, not a retain after the fetch: the result is paginated, and a + // post-fetch filter would let a page's size report how many rows the token + // may not read — and return short pages that read as "no history". + let (scope_all, scope_exact, scope_prefix) = + match build_scope_path_filter(&authed, "triggers_history", "read") { + ScopePathFilter::AllowAll => (true, Vec::new(), Vec::new()), + ScopePathFilter::Restricted { exact, prefix } => (false, exact, prefix), + }; + + let mut tx = user_db.begin(&authed).await?; + + let (per_page, offset) = paginate(Pagination { page: query.page, per_page: query.per_page }); + + let history = sqlx::query_as!( + TriggerHistoryEntry, + "SELECT id, trigger_kind, path, operation, source, username, created_at, changes + FROM trigger_history + WHERE workspace_id = $1 + AND ($2::TEXT IS NULL OR trigger_kind = $2) + AND ($3::TEXT IS NULL OR path = $3) + AND ( $6 + OR path = ANY($7) + OR EXISTS ( SELECT 1 FROM unnest($8::text[]) AS pfx + WHERE path = pfx + OR left(path, length(pfx) + 1) = pfx || '/' ) ) + ORDER BY id DESC + LIMIT $4 OFFSET $5", + w_id, + query.trigger_kind, + query.path, + per_page as i64, + offset as i64, + scope_all, + &scope_exact[..], + &scope_prefix[..], + ) + .fetch_all(&mut *tx) + .await?; + + tx.commit().await?; + + Ok(axum::Json(history)) +} diff --git a/backend/windmill-api/src/workspaces_export.rs b/backend/windmill-api/src/workspaces_export.rs index 037971b19e..4e7aaf0f48 100644 --- a/backend/windmill-api/src/workspaces_export.rs +++ b/backend/windmill-api/src/workspaces_export.rs @@ -78,6 +78,8 @@ struct ScriptMetadata { #[serde(skip_serializing_if = "Option::is_none")] cache_ttl: Option, #[serde(skip_serializing_if = "Option::is_none")] + cache_ignore_s3_path: Option, + #[serde(skip_serializing_if = "Option::is_none")] dedicated_worker: Option, #[serde(skip_serializing_if = "is_none_or_false")] ws_error_handler_muted: Option, @@ -826,6 +828,7 @@ pub(crate) async fn tarball_workspace( concurrency_settings: script.runnable_settings.concurrency_settings, debouncing_settings: script.runnable_settings.debouncing_settings, cache_ttl: script.cache_ttl, + cache_ignore_s3_path: script.cache_ignore_s3_path, dedicated_worker: script.dedicated_worker, ws_error_handler_muted: script.ws_error_handler_muted, priority: script.priority, diff --git a/backend/windmill-audit/src/lib.rs b/backend/windmill-audit/src/lib.rs index 8e74edfbf0..f1443fdb18 100644 --- a/backend/windmill-audit/src/lib.rs +++ b/backend/windmill-audit/src/lib.rs @@ -37,5 +37,8 @@ pub struct ListAuditLogQuery { pub resource: Option, pub before: Option>, pub after: Option>, + // Keyset cursor on the `id DESC` ordering. Lets a client stream a page in small batches + // without paying a growing OFFSET on every batch. + pub before_id: Option, pub all_workspaces: Option, } diff --git a/backend/windmill-common/src/external_ip.rs b/backend/windmill-common/src/external_ip.rs index b9713f67fd..5ffd713f5b 100644 --- a/backend/windmill-common/src/external_ip.rs +++ b/backend/windmill-common/src/external_ip.rs @@ -12,8 +12,58 @@ //! connections to be from whitelisted IP addresses. use crate::utils::configure_client; +use std::sync::OnceLock; use std::time::Duration; +/// No address has ever been established for the row. Matches the `worker_ping.ip` column default, +/// and doubles as what an agent sends while its lookup is in flight, since a server that predates +/// the lookup being asynchronous rejects an initial ping carrying nothing. +pub const UNKNOWN_IP: &str = "NO IP"; + +/// The lookup ran and could not produce an address. Distinct from [`UNKNOWN_IP`] because it tells +/// an operator the difference between "never asked" and "asked, and this instance cannot reach the +/// hub", which is the actionable one. Both are filtered out of the addresses the frontend offers +/// for whitelisting. +pub const UNRETRIEVABLE_IP: &str = "unretrievable IP"; + +/// `worker_ping.ip` is `VARCHAR(50)`, and a failed initial ping takes the worker down, so an +/// overlong value must not reach the insert. +const MAX_IP_LEN: usize = 50; + +static EXTERNAL_IP: OnceLock = OnceLock::new(); + +/// The external IP of this process, [`UNRETRIEVABLE_IP`] once the lookup has failed, or `None` +/// while it is still in flight. +pub fn cached_ip() -> Option<&'static str> { + EXTERNAL_IP.get().map(String::as_str) +} + +/// Resolves the external IP into the process-wide cache without blocking the caller. The value is +/// informational, and behind a firewall the lookup burns its whole 5s connect timeout on every +/// process start, so nothing on the worker startup path may wait on it. +pub fn resolve_ip_in_background() { + tokio::spawn(async { + let ip = get_ip() + .await + .map(|ip| { + if ip.len() > MAX_IP_LEN { + tracing::error!("external IP lookup returned an overlong value, ignoring it"); + UNRETRIEVABLE_IP.to_string() + } else { + ip + } + }) + .unwrap_or_else(|e| { + tracing::warn!( + error = e.to_string(), + "failed to get external IP, workers of this process will report it as unretrievable" + ); + UNRETRIEVABLE_IP.to_string() + }); + let _ = EXTERNAL_IP.set(ip); + }); +} + pub async fn get_ip() -> anyhow::Result { tokio::select! { biased; diff --git a/backend/windmill-common/src/feature_usage_oss.rs b/backend/windmill-common/src/feature_usage_oss.rs new file mode 100644 index 0000000000..3219e9d0ac --- /dev/null +++ b/backend/windmill-common/src/feature_usage_oss.rs @@ -0,0 +1,26 @@ +//! OSS fallback for anonymous feature-usage collection. +//! +//! Collection is a `private` feature (see `feature_usage_ee`). The public build +//! never sends a stats payload (`stats_oss`), so counting anything would only +//! write rows nothing reads: every entry point here is inert, and the +//! `log_feature_usage` endpoint accepts its posts without recording them. + +use sqlx::{Pool, Postgres}; + +/// No action is recordable in the public build. +pub fn is_recordable_event( + _feature: &str, + _kind: &str, + _key: &str, + _entity_id: &str, +) -> bool { + false +} + +/// No-op: nothing is counted in the public build. +pub fn log_feature_usage(_feature: &'static str, _kind: &'static str, _key: &str) {} + +/// Nothing accumulates, so there is nothing to flush. +pub async fn flush_feature_usage(_db: &Pool) -> Result<(), sqlx::Error> { + Ok(()) +} diff --git a/backend/windmill-common/src/git_sync_oss.rs b/backend/windmill-common/src/git_sync_oss.rs index 1e17532ec7..5f8ba771fb 100644 --- a/backend/windmill-common/src/git_sync_oss.rs +++ b/backend/windmill-common/src/git_sync_oss.rs @@ -38,6 +38,73 @@ pub fn sanitize_git_url(url: &str) -> String { GIT_URL_USERINFO_RE.replace(url, "://***@").into_owned() } +/// Validate a user-supplied git remote URL before it is handed to `git` (`clone`, +/// `ls-remote`, `remote add`, `fetch`, ...). Two classes of abuse are rejected: +/// - Argument injection: a URL that git parses as a command-line option (e.g. +/// `--upload-pack=`) turns `git ls-remote HEAD` into arbitrary command +/// execution on the worker host, outside any job sandbox. +/// - Dangerous transports: git's remote-helper syntax (`ext::sh -c ...`, `fd::...`) runs +/// arbitrary programs, and `file://` / local paths read host files — both escape the +/// intended network-only fetch. +/// +/// Only the standard network transports are allowed: `http(s)`, `ssh`, `git`, and the +/// scp-like `[user@]host:path` shorthand. Validation is transport-syntax based (not git +/// version dependent) so it holds regardless of git's own option/protocol handling. +pub fn validate_git_repo_url(url: &str) -> crate::error::Result<()> { + let reject = + |msg: &str| crate::error::Error::BadRequest(format!("Invalid git repository URL: {msg}")); + + let trimmed = url.trim(); + if trimmed.is_empty() { + return Err(reject("the URL is empty")); + } + // Leading '-' makes git parse the URL as an option (argument injection). + if trimmed.starts_with('-') { + return Err(reject("the URL must not start with '-'")); + } + // `::
` remote-helper transports execute arbitrary programs. + if trimmed.contains("::") { + return Err(reject("remote-helper transports (`::`) are not allowed")); + } + + if let Some((scheme, _rest)) = trimmed.split_once("://") { + // A real scheme is ASCII-alnum plus `+ - .` and holds no slash (a slash means the + // `://` came from the path, so there is no scheme and this is not a valid URL). + let is_scheme = !scheme.is_empty() + && !scheme.contains('/') + && scheme + .chars() + .all(|c| c.is_ascii_alphanumeric() || matches!(c, '+' | '-' | '.')); + if !is_scheme { + return Err(reject("malformed URL scheme")); + } + match scheme.to_ascii_lowercase().as_str() { + "http" | "https" | "ssh" | "git" => Ok(()), + other => Err(reject(&format!( + "scheme `{other}` is not allowed (use http(s), ssh, or git)" + ))), + } + } else { + // No scheme: accept only the scp-like `[user@]host:path` shorthand. The host (the + // part before the first `:`) must be non-empty and slash-free; a slash there means a + // local path (`./repo`, `/abs/repo`), and a single-letter host is a Windows drive. + let Some((host, _path)) = trimmed.split_once(':') else { + return Err(reject( + "local paths are not allowed; use an http(s), ssh, or git URL", + )); + }; + let bad_host = host.is_empty() + || host.contains('/') + || (host.len() == 1 && host.chars().all(|c| c.is_ascii_alphabetic())); + if bad_host { + return Err(reject( + "local paths are not allowed; use an http(s), ssh, or git URL", + )); + } + Ok(()) + } +} + pub fn prepend_token_to_github_url( github_url: &str, installation_token: &str, @@ -92,4 +159,51 @@ mod tests { "not a url://***@host/repo" ); } + + use super::validate_git_repo_url; + + #[test] + fn accepts_standard_transports() { + for url in [ + "https://github.com/org/repo.git", + "http://internal.example/org/repo.git", + "https://user:token@github.com/org/repo.git", + "ssh://git@github.com/org/repo.git", + "ssh://git@github.com:2222/org/repo.git", + "git://github.com/org/repo.git", + "git@github.com:org/repo.git", + "user@host.example:path/to/repo", + ] { + assert!(validate_git_repo_url(url).is_ok(), "should accept {url}"); + } + } + + #[test] + fn rejects_argument_injection() { + for url in [ + "--upload-pack=touch /tmp/pwned", + "-oProxyCommand=touch /tmp/pwned", + "--config=core.fsmonitor=touch /tmp/pwned", + ] { + assert!(validate_git_repo_url(url).is_err(), "should reject {url}"); + } + } + + #[test] + fn rejects_remote_helpers_and_local_transports() { + for url in [ + "ext::sh -c 'id > /tmp/pwned'", + "fd::17/foo", + "file:///etc/passwd", + "/etc/passwd", + "./local/repo", + "../local/repo", + "ftp://host/repo", + "C:\\path\\to\\repo", + "", + " ", + ] { + assert!(validate_git_repo_url(url).is_err(), "should reject {url:?}"); + } + } } diff --git a/backend/windmill-common/src/global_settings.rs b/backend/windmill-common/src/global_settings.rs index 1a5cb3f1e6..37b4b60da4 100644 --- a/backend/windmill-common/src/global_settings.rs +++ b/backend/windmill-common/src/global_settings.rs @@ -331,6 +331,60 @@ use crate::error; use sqlx::postgres::Postgres; use sqlx::Pool; +/// Read several settings in one round trip. Names with no row are simply absent from the +/// result, exactly as [`load_value_from_global_settings`] returns `None` for them. +pub async fn load_values_from_global_settings( + db: &Pool, + names: &[&str], +) -> error::Result> { + // Listing the names keeps this on the primary key. `global_settings` also holds + // `workspace_dependencies_map_rebuilt:`, one row per workspace with no + // cleanup path, so a predicate that scanned the table would grow with workspace count. + let rows = sqlx::query!( + "SELECT name, value FROM global_settings WHERE name = ANY($1)", + names as &[&str] + ) + .fetch_all(db) + .await?; + Ok(rows.into_iter().map(|r| (r.name, r.value)).collect()) +} + +/// Return the instance's JWT secret, generating one only if the row holds nothing usable. +/// +/// The write has to be conditional rather than a plain upsert, for two reasons. A usable +/// secret must never be overwritten: replicas booting together would each install their own +/// and reject each other's tokens. And `notify_global_setting_change` fires on every write to +/// this table, so an unconditional upsert would make each startup trigger a cluster-wide +/// settings reload. An empty `RETURNING` is how a caller learns another process's secret +/// stands, and reads that one instead. +/// +/// Safe to call with a value read earlier: the statement, not the caller's read, decides. +pub async fn get_or_create_jwt_secret(db: &Pool) -> error::Result { + let candidate = crate::utils::rd_string(32); + let stored = sqlx::query_scalar!( + "INSERT INTO global_settings (name, value) VALUES ($1, $2) + ON CONFLICT (name) DO UPDATE SET value = EXCLUDED.value + WHERE jsonb_typeof(global_settings.value) <> 'string' + RETURNING value", + JWT_SECRET_SETTING, + serde_json::to_value(&candidate)? + ) + .fetch_optional(db) + .await?; + + match stored { + Some(_) => Ok(candidate), + None => load_value_from_global_settings(db, JWT_SECRET_SETTING) + .await? + .and_then(|v| serde_json::from_value::(v).ok()) + .ok_or_else(|| { + error::Error::InternalErr( + "jwt_secret conflicted but holds no usable value".to_string(), + ) + }), + } +} + pub async fn load_value_from_global_settings( db: &Pool, setting_name: &str, diff --git a/backend/windmill-common/src/lib.rs b/backend/windmill-common/src/lib.rs index a8de6d74c5..b72d0ae1d9 100644 --- a/backend/windmill-common/src/lib.rs +++ b/backend/windmill-common/src/lib.rs @@ -56,6 +56,13 @@ pub mod email_ee; pub mod email_oss; pub mod error; pub mod external_ip; +#[cfg(feature = "private")] +pub mod feature_usage_ee; +pub mod feature_usage_oss; +#[cfg(feature = "private")] +pub use feature_usage_ee as feature_usage; +#[cfg(not(feature = "private"))] +pub use feature_usage_oss as feature_usage; pub mod flow_conversations; pub mod flow_status; pub mod flows; @@ -90,6 +97,7 @@ pub mod otel_oss; #[cfg(feature = "private")] pub mod partition_ee; pub mod partition_oss; +pub mod per_minute_counter; #[cfg(feature = "private")] pub use partition_ee as partition; #[cfg(not(feature = "private"))] @@ -121,6 +129,7 @@ pub mod teams_ee; pub mod teams_oss; pub mod tracing_init; pub mod trashbin; +pub mod trigger_history; pub mod triggers; pub mod user_drafts; pub mod usernames; diff --git a/backend/windmill-common/src/login_rate_limit.rs b/backend/windmill-common/src/login_rate_limit.rs index aede8df2e4..8c60c30b88 100644 --- a/backend/windmill-common/src/login_rate_limit.rs +++ b/backend/windmill-common/src/login_rate_limit.rs @@ -1,31 +1,24 @@ use chrono::Utc; -use dashmap::DashMap; use hyper::StatusCode; -use std::sync::atomic::{AtomicI32, AtomicI64, AtomicU64, Ordering}; +use std::sync::atomic::{AtomicI32, AtomicI64, Ordering}; use std::sync::LazyLock; use crate::error::{Error, Result}; +use crate::per_minute_counter::PerMinuteCounter; use crate::worker::CLOUD_HOSTED; -const DEFAULT_PER_IP_LIMIT: i32 = 120; -const DEFAULT_PER_ACCOUNT_LIMIT: i32 = 30; +const DEFAULT_PER_IP_LIMIT: u32 = 120; +const DEFAULT_PER_ACCOUNT_LIMIT: u32 = 30; const DEFAULT_GLOBAL_LIMIT: i32 = 10000; -const EVICTION_INTERVAL: u64 = 256; -struct RateLimitEntry { - count: i32, - minute_bucket: i64, -} - -static IP_RATE_LIMIT: LazyLock> = LazyLock::new(DashMap::new); -static ACCOUNT_RATE_LIMIT: LazyLock> = LazyLock::new(DashMap::new); +static IP_RATE_LIMIT: LazyLock> = LazyLock::new(PerMinuteCounter::new); +static ACCOUNT_RATE_LIMIT: LazyLock> = + LazyLock::new(PerMinuteCounter::new); static GLOBAL_COUNT: AtomicI32 = AtomicI32::new(0); static GLOBAL_MINUTE: AtomicI64 = AtomicI64::new(0); -static EVICTION_COUNTER: AtomicU64 = AtomicU64::new(0); - -static PER_IP_LIMIT: LazyLock = LazyLock::new(|| { +static PER_IP_LIMIT: LazyLock = LazyLock::new(|| { std::env::var("LOGIN_RATE_LIMIT_PER_IP") .ok() .and_then(|v| v.parse().ok()) @@ -35,11 +28,11 @@ static PER_IP_LIMIT: LazyLock = LazyLock::new(|| { static PER_IP_LIMIT_EXPLICIT: LazyLock = LazyLock::new(|| { std::env::var("LOGIN_RATE_LIMIT_PER_IP") .ok() - .and_then(|v| v.parse::().ok()) + .and_then(|v| v.parse::().ok()) .is_some() }); -static PER_ACCOUNT_LIMIT: LazyLock = LazyLock::new(|| { +static PER_ACCOUNT_LIMIT: LazyLock = LazyLock::new(|| { std::env::var("LOGIN_RATE_LIMIT_PER_ACCOUNT") .ok() .and_then(|v| v.parse().ok()) @@ -49,7 +42,7 @@ static PER_ACCOUNT_LIMIT: LazyLock = LazyLock::new(|| { static PER_ACCOUNT_LIMIT_EXPLICIT: LazyLock = LazyLock::new(|| { std::env::var("LOGIN_RATE_LIMIT_PER_ACCOUNT") .ok() - .and_then(|v| v.parse::().ok()) + .and_then(|v| v.parse::().ok()) .is_some() }); @@ -86,57 +79,11 @@ pub fn extract_client_ip(headers: &axum::http::HeaderMap) -> Option { None } -fn maybe_evict(maps: &[&DashMap], current_minute: i64) { - let count = EVICTION_COUNTER.fetch_add(1, Ordering::Relaxed); - if count % EVICTION_INTERVAL == 0 { - for map in maps { - map.retain(|_, v| v.minute_bucket >= current_minute - 1); - } - } -} - -/// Atomically check the rate limit and increment the counter. Follows the -/// `public_app_rate_limit.rs` pattern — the DashMap entry lock is held across -/// both the check and the increment, preventing TOCTOU races. -fn check_and_increment( - map: &DashMap, - key: &str, - limit: i32, - current_minute: i64, -) -> Result<()> { - let mut entry = map - .entry(key.to_string()) - .or_insert(RateLimitEntry { count: 0, minute_bucket: current_minute }); - - if entry.minute_bucket != current_minute { - entry.count = 0; - entry.minute_bucket = current_minute; - } - - if entry.count >= limit { - return Err(Error::Generic( - StatusCode::TOO_MANY_REQUESTS, - "Too many login attempts. Please try again later.".to_string(), - )); - } - - entry.count += 1; - Ok(()) -} - -fn record_failure(map: &DashMap, key: &str) { - let current_minute = Utc::now().timestamp() / 60; - - let mut entry = map - .entry(key.to_string()) - .or_insert(RateLimitEntry { count: 0, minute_bucket: current_minute }); - - if entry.minute_bucket != current_minute { - entry.count = 1; - entry.minute_bucket = current_minute; - } else { - entry.count += 1; - } +fn too_many_attempts() -> Error { + Error::Generic( + StatusCode::TOO_MANY_REQUESTS, + "Too many login attempts. Please try again later.".to_string(), + ) } /// Called BEFORE authentication. Checks and increments global + per-IP counters. @@ -147,36 +94,30 @@ pub fn check_and_increment_login_attempt( headers: &axum::http::HeaderMap, email: &str, ) -> Result<()> { - let current_minute = Utc::now().timestamp() / 60; - maybe_evict(&[&IP_RATE_LIMIT, &ACCOUNT_RATE_LIMIT], current_minute); - - // Global limit: always on, uses atomics (single key, no need for DashMap) - check_and_increment_global(current_minute)?; + // Global limit: always on, uses atomics (single key, no need for a map) + check_and_increment_global()?; // Per-IP limit: CLOUD_HOSTED or explicit opt-in if *CLOUD_HOSTED || *PER_IP_LIMIT_EXPLICIT { if let Some(ip) = extract_client_ip(headers) { - check_and_increment(&IP_RATE_LIMIT, &ip, *PER_IP_LIMIT, current_minute)?; + if !IP_RATE_LIMIT.try_increment(ip, *PER_IP_LIMIT) { + return Err(too_many_attempts()); + } } } // Per-account check (read-only, does not increment — failures are recorded separately) if *CLOUD_HOSTED || *PER_ACCOUNT_LIMIT_EXPLICIT { - let entry = ACCOUNT_RATE_LIMIT.get(email); - if let Some(entry) = entry { - if entry.minute_bucket == current_minute && entry.count >= *PER_ACCOUNT_LIMIT { - return Err(Error::Generic( - StatusCode::TOO_MANY_REQUESTS, - "Too many login attempts. Please try again later.".to_string(), - )); - } + if ACCOUNT_RATE_LIMIT.count(email) >= *PER_ACCOUNT_LIMIT { + return Err(too_many_attempts()); } } Ok(()) } -fn check_and_increment_global(current_minute: i64) -> Result<()> { +fn check_and_increment_global() -> Result<()> { + let current_minute = Utc::now().timestamp() / 60; let stored_minute = GLOBAL_MINUTE.load(Ordering::Relaxed); if stored_minute != current_minute { // Minute rolled over — reset. Race here is benign: worst case two threads @@ -188,10 +129,7 @@ fn check_and_increment_global(current_minute: i64) -> Result<()> { let count = GLOBAL_COUNT.fetch_add(1, Ordering::Relaxed); if count >= *GLOBAL_LIMIT { - return Err(Error::Generic( - StatusCode::TOO_MANY_REQUESTS, - "Too many login attempts. Please try again later.".to_string(), - )); + return Err(too_many_attempts()); } Ok(()) @@ -201,6 +139,6 @@ fn check_and_increment_global(current_minute: i64) -> Result<()> { /// Per-account is only active on CLOUD_HOSTED or when LOGIN_RATE_LIMIT_PER_ACCOUNT is explicitly set. pub fn record_login_failure(email: &str) { if *CLOUD_HOSTED || *PER_ACCOUNT_LIMIT_EXPLICIT { - record_failure(&ACCOUNT_RATE_LIMIT, email); + ACCOUNT_RATE_LIMIT.increment(email.to_string()); } } diff --git a/backend/windmill-common/src/per_minute_counter.rs b/backend/windmill-common/src/per_minute_counter.rs new file mode 100644 index 0000000000..f52426c4d7 --- /dev/null +++ b/backend/windmill-common/src/per_minute_counter.rs @@ -0,0 +1,127 @@ +use dashmap::DashMap; +use std::borrow::Borrow; +use std::hash::Hash; +use std::sync::atomic::{AtomicU64, Ordering}; + +/// Calls between eviction sweeps. +const EVICTION_INTERVAL: u64 = 256; + +struct Bucket { + count: u32, + minute: i64, +} + +/// Events recorded per key within the current wall-clock minute, held in process memory. +/// +/// Counts are per process and reset on restart, so N servers raise any threshold built on +/// this to N times its configured value. Stale keys are swept as calls come in, which keeps the +/// map bounded without a background task. +pub struct PerMinuteCounter { + buckets: DashMap, + calls: AtomicU64, +} + +impl PerMinuteCounter { + pub fn new() -> Self { + Self { buckets: DashMap::new(), calls: AtomicU64::new(0) } + } + + /// Events recorded for `key` in the current minute, without recording one. A key never + /// seen, or last seen in an earlier minute, counts as zero. + pub fn count(&self, key: &Q) -> u32 + where + K: Borrow, + Q: Hash + Eq + ?Sized, + { + let minute = current_minute(); + self.buckets + .get(key) + .filter(|bucket| bucket.minute == minute) + .map_or(0, |bucket| bucket.count) + } + + /// Record one event and return the new count for the current minute. + pub fn increment(&self, key: K) -> u32 { + self.bump_at(key, None, current_minute()).0 + } + + /// Record one event unless `key` has already reached `limit` this minute. Returns false + /// when the limit was already reached, in which case nothing was recorded. + pub fn try_increment(&self, key: K, limit: u32) -> bool { + self.bump_at(key, Some(limit), current_minute()).1 + } + + /// Returns the count for `minute` and whether this call recorded an event. Takes the + /// minute rather than reading the clock so the rollover and eviction paths are testable. + fn bump_at(&self, key: K, limit: Option, minute: i64) -> (u32, bool) { + // The entry guard holds a lock on its DashMap shard, and the `retain` below takes + // every shard. Keeping the guard alive across that call deadlocks the caller, so this + // block is load-bearing: it must end before the sweep, not be flattened into the body. + let outcome = { + let mut bucket = self + .buckets + .entry(key) + .or_insert(Bucket { count: 0, minute }); + if bucket.minute != minute { + bucket.minute = minute; + bucket.count = 0; + } + if limit.is_some_and(|limit| bucket.count >= limit) { + (bucket.count, false) + } else { + bucket.count += 1; + (bucket.count, true) + } + }; + // Periodically drop what neither the current nor the previous minute can still need. + // Counts every call, refusals included: a key pinned at its limit must still drive + // sweeps, or a sustained burst of refusals would leave the map unbounded. + if self.calls.fetch_add(1, Ordering::Relaxed) % EVICTION_INTERVAL == 0 { + self.buckets.retain(|_, bucket| bucket.minute >= minute - 1); + } + outcome + } +} + +fn current_minute() -> i64 { + chrono::Utc::now().timestamp() / 60 +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn refuses_at_the_limit_without_recording() { + let counter = PerMinuteCounter::new(); + assert_eq!(counter.bump_at("a".to_string(), Some(2), 100), (1, true)); + assert_eq!(counter.bump_at("a".to_string(), Some(2), 100), (2, true)); + // A refused call must not record, or a caller held at the limit would keep inflating + // its own count and never recover within the minute. + assert_eq!(counter.bump_at("a".to_string(), Some(2), 100), (2, false)); + assert_eq!(counter.bump_at("a".to_string(), Some(2), 100), (2, false)); + } + + #[test] + fn resets_on_the_next_minute() { + let counter = PerMinuteCounter::new(); + counter.bump_at("a".to_string(), None, 100); + counter.bump_at("a".to_string(), None, 100); + assert_eq!(counter.bump_at("a".to_string(), None, 101), (1, true)); + } + + #[test] + fn sweep_drops_keys_older_than_the_previous_minute() { + let counter = PerMinuteCounter::new(); + counter.bump_at("stale".to_string(), None, 100); + counter.bump_at("previous".to_string(), None, 199); + // Sweeps fire every EVICTION_INTERVAL calls; drive the counter to the next one with + // filler events that all land in minute 200. + while counter.calls.load(Ordering::Relaxed) <= EVICTION_INTERVAL { + counter.bump_at("filler".to_string(), None, 200); + } + assert!(!counter.buckets.contains_key("stale")); + assert!(counter.buckets.contains_key("previous")); + assert!(counter.buckets.contains_key("filler")); + } +} diff --git a/backend/windmill-common/src/server.rs b/backend/windmill-common/src/server.rs index bd948a4f96..19da3222e9 100644 --- a/backend/windmill-common/src/server.rs +++ b/backend/windmill-common/src/server.rs @@ -1,6 +1,6 @@ use serde::{Deserialize, Serialize}; -use crate::{error, DB}; +use crate::{error, global_settings::SMTP_SETTING, DB}; #[derive(Serialize, Deserialize, PartialEq, Clone, Debug)] pub struct Smtp { @@ -27,13 +27,17 @@ pub struct SmtpConfigOpt { } pub async fn load_smtp_config(db: &DB) -> error::Result> { - let config: SmtpConfigOpt = - sqlx::query_scalar!("SELECT value FROM global_settings WHERE name = 'smtp_settings'",) - .fetch_optional(db) - .await? - .map(|x| serde_json::from_value(x).ok()) - .flatten() - .unwrap_or_default(); + let value = crate::global_settings::load_value_from_global_settings(db, SMTP_SETTING).await?; + Ok(parse_smtp_config(value)) +} + +/// The half of [`load_smtp_config`] after the read, so a batched settings pass can parse a +/// value it already fetched. +pub fn parse_smtp_config(value: Option) -> Option { + let config: SmtpConfigOpt = value + .map(|x| serde_json::from_value(x).ok()) + .flatten() + .unwrap_or_default(); let config_smtp = if let (Some(host), username, password) = (config.smtp_host, config.smtp_username, config.smtp_password) @@ -87,7 +91,7 @@ pub async fn load_smtp_config(db: &DB) -> error::Result> { tracing::warn!("SMTP not configured"); } - Ok(smtp) + smtp } impl Default for SmtpConfigOpt { diff --git a/backend/windmill-common/src/trigger_history.rs b/backend/windmill-common/src/trigger_history.rs new file mode 100644 index 0000000000..70c86b9ffd --- /dev/null +++ b/backend/windmill-common/src/trigger_history.rs @@ -0,0 +1,551 @@ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2022 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +//! Append-only history of schedule and trigger mutations (`trigger_history`). +//! +//! Every field of a row is derived by the server at write time: the caller +//! passes what it is doing, never who it claims to be or where it claims to +//! come from. **Who** (the authed username, or nobody for a server-initiated +//! change) and **what** (a field-level diff computed from the row before and +//! after the write) are derived by the server and cannot be forged. **From what +//! kind of client** ([`TriggerSource`]) is weaker on purpose: a first-party +//! client declares itself in a header, so it attributes rather than proves — +//! see [`TriggerSource::of_request`]. +//! +//! # What is recorded +//! +//! Authoring a single trigger through its own surface — create, update, delete, +//! enable/disable/suspend, restore from the trashbin, and the workspace-wide +//! default-handler override — plus the server disabling one after a failure. +//! **Adding a route that authors a trigger means adding a `record` call to it**; +//! nothing enforces that, because the alternative (a database trigger) cannot +//! see who or which client asked, and would fire on every listener ping. +//! +//! Deliberately outside that line, and not a gap to be closed one call site at a +//! time: +//! +//! - **Cascades of renaming or deleting something else** — a script/flow rename +//! rewriting `script_path` (`triggers::update_triggers_script_path`), a user +//! being removed rewriting ownership. The event belongs to the runnable or the +//! user, not to the trigger. +//! - **Workspace-level bulk operations** — archive, fork clone, cross-workspace +//! deploy. They move whole workspaces; a per-trigger row per path would say +//! nothing the workspace event does not. +//! - **Runtime housekeeping** — clearing `paused_until` / `error` after a run, +//! consumer-offset state (`reset_offset`, `server_id`), the managed +//! ducklake-maintenance schedule. The same category as the `server_id` and +//! `last_server_ping` columns the diff already drops. +//! +//! # The server-initiated disables: the disable wins +//! +//! When the server disables a trigger it could not run, two things want to be +//! true and cannot both be guaranteed: the trigger ends up disabled, and the +//! history says who disabled it. The disable wins, every time. +//! +//! A trigger left enabled reads as healthy while never firing again, and for a +//! flow schedule nothing comes back to retry — it arms its next occurrence when +//! the flow *starts*, so once the runnable is gone that code is never reached +//! again. Enabled-and-dead is silent; disabled-without-an-audit-row is not, and +//! the trigger's own `error` column still says why. +//! +//! So each writer puts the disabling `UPDATE` and the record in one +//! transaction, with only the insert inside a savepoint +//! ([`record_in_disable_tx`]). Both land on the same commit, and the trigger's +//! row lock is held across the pair — so the row cannot end up describing a +//! trigger deleted and recreated at that path in between. If the insert alone +//! fails it rolls back to the savepoint, the disable still commits, and the lost +//! row is reported to the workspace error handler and the critical alert +//! channel — loud, never silent. +//! +//! # Authorization contract +//! +//! None of the helpers here authorize anything: they take a connection and +//! write what they are given, exactly like `audit_log`. A caller must already +//! have authorized the mutation *and* performed it, and must derive `username` +//! from the request's `ApiAuthed` and `source` from +//! [`TriggerSource::of_request`] — never from anything the request body +//! carries. Reads are gated separately, by the RLS policies on the table and by +//! the token scopes the listing route checks. + +use sqlx::{Acquire, PgConnection}; + +use crate::error::Result; + +/// Header a first-party client sets to name itself. Only `cli`, `ui` and `api` +/// mean anything; any other value, and the header being absent, falls back to +/// what the credentials say. +pub const CLIENT_HEADER: &str = "x-windmill-client"; + +/// `trigger_kind` a schedule is recorded under. Triggers use their own +/// `TriggerCrud::TRIGGER_TYPE`. +pub const SCHEDULE_TRIGGER_KIND: &str = "schedule"; + +/// The kind of client a trigger mutation came from. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum TriggerSource { + /// A browser session in the Windmill app. + Ui, + /// The `wmill` CLI (including the git-sync pull that shells out to it). + Cli, + /// A direct API call with a token: user scripts, CI, third-party clients. + Api, + /// No request at all: a worker or a trigger listener disabling something + /// after a failure. + Worker, +} + +impl TriggerSource { + pub fn as_str(&self) -> &'static str { + match self { + TriggerSource::Ui => "ui", + TriggerSource::Cli => "cli", + TriggerSource::Api => "api", + TriggerSource::Worker => "worker", + } + } + + fn from_client_header(value: &str) -> Option { + match value.trim().to_ascii_lowercase().as_str() { + "cli" => Some(TriggerSource::Cli), + "ui" => Some(TriggerSource::Ui), + "api" => Some(TriggerSource::Api), + _ => None, + } + } + + /// The source of the request currently being served. + /// + /// The declared client wins when it is one we know; otherwise the token + /// decides, and only the session token minted at browser login attributes + /// to the UI. Both inputs are attribution, never authority — nothing reads + /// a history row to make an access decision, so a caller lying about either + /// only mislabels its own row. + pub fn of_request(is_session_token: bool) -> Self { + match REQUEST_CLIENT.try_with(|client| *client) { + Ok(Some(source)) => source, + Ok(None) if is_session_token => TriggerSource::Ui, + Ok(None) => TriggerSource::Api, + // Outside a request there is no caller to attribute to. The + // server-initiated paths pass `Worker` themselves; this is what + // keeps a stray call from inventing one. + Err(_) => TriggerSource::Worker, + } + } +} + +/// What a mutation did to the trigger it is recorded against. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum TriggerOperation { + Create, + Update, + Delete, + Enable, + Disable, + Suspend, +} + +impl TriggerOperation { + pub fn as_str(&self) -> &'static str { + match self { + TriggerOperation::Create => "create", + TriggerOperation::Update => "update", + TriggerOperation::Delete => "delete", + TriggerOperation::Enable => "enable", + TriggerOperation::Disable => "disable", + TriggerOperation::Suspend => "suspend", + } + } +} + +tokio::task_local! { + static REQUEST_CLIENT: Option; +} + +/// Run `f` with `client` as the declared client of every trigger mutation it +/// causes. Entered for every request, unmarked ones included, so that having no +/// scope at all means "not serving a request" — which is what +/// [`TriggerSource::Worker`] records. +pub async fn scope_client( + client: Option, + f: F, +) -> F::Output { + REQUEST_CLIENT.scope(client, f).await +} + +/// Parse the declared client of the request being served, if any. +pub fn client_from_header(value: &str) -> Option { + TriggerSource::from_client_header(value) +} + +/// Row fields that say nothing about the change itself: bookkeeping the history +/// row already carries, and listener runtime state that moves on its own. +const IGNORED_FIELDS: &[&str] = &[ + "workspace_id", + "edited_at", + "edited_by", + "extra_perms", + "last_server_ping", + "server_id", + // Listener runtime state like the two above: every trigger update clears it, + // so keeping it here would tag an ordinary edit with the failure it had + // before. The server-initiated disables put the error in `changes` + // themselves, so nothing is lost. + "error", + // Written from the requester on every schedule mutation, purely for workers + // that predate `permissioned_as`; it tracks the editor, not the schedule. + "email", +]; + +/// A `changes` payload bigger than this is replaced by the list of field names +/// it would have held. A schedule's `args` is caller-supplied and bounded only +/// by the API's request-size limit, and a history row is not worth a +/// multi-megabyte write. +const MAX_CHANGES_BYTES: usize = 32 * 1024; + +/// The row at `path` as JSON, or `None` when there is none — which, on an RLS +/// connection, also covers a row the caller cannot see. +/// +/// `FOR UPDATE`, so the preimage and the mutation that follows it see the same +/// row: without the lock another request can commit between the two, and its +/// change then lands in this caller's diff under this caller's name. +/// +/// Two things follow from taking the lock here rather than at the write: +/// +/// - The only row locked is the one the caller is about to write, and the +/// schedule paths reach the job queue only afterwards, so their documented +/// schedule-then-queue order is unchanged. +/// - The lock is held for whatever the caller does before its own `UPDATE`. For +/// `TriggerCrud::update_trigger` that includes the impl's external work — the +/// postgres impl opens a replication slot on a user-supplied host, the gcp and +/// azure impls call their subscription APIs — so a concurrent `setmode`, a +/// listener error write, or a script rename's bulk `script_path` update waits +/// on that call. Bounded by those APIs, not by us; the alternative is a +/// preimage inside each impl next to its own `UPDATE`. +/// +/// `table` is interpolated: pass a compile-time constant, never anything a +/// caller can reach. +pub async fn snapshot_row( + conn: &mut PgConnection, + table: &'static str, + workspace_id: &str, + path: &str, +) -> Result> { + // SAFETY: `table` is a compile-time constant. + let snapshot: Option = sqlx::query_scalar(&format!( + "SELECT to_jsonb(t) FROM {table} t WHERE workspace_id = $1 AND path = $2 FOR UPDATE" + )) + .bind(workspace_id) + .bind(path) + .fetch_optional(&mut *conn) + .await?; + Ok(snapshot) +} + +/// A field-level diff of two row snapshots, as `{field: {"old": …, "new": …}}`, +/// with `"old"` omitted where there is none to report. +/// +/// A create (`before` absent) keeps every non-null column of the new row, which +/// is its initial shape including whatever the column defaults supplied — +/// `to_jsonb` cannot tell a caller-set column from a defaulted one. Returns +/// `None` when nothing meaningful changed. +pub fn summarize_changes( + before: Option<&serde_json::Value>, + after: Option<&serde_json::Value>, +) -> Option { + let empty = serde_json::Map::new(); + let before = before.and_then(|v| v.as_object()).unwrap_or(&empty); + let after = after.and_then(|v| v.as_object())?; + + // Names of the changed fields, and the running size of what has been cloned + // so far. Measured as it goes rather than by serializing the finished map: + // a caller-sized `args` would otherwise be cloned in full and then copied + // again just to learn it was too big. + let mut fields = Vec::new(); + let mut changes = serde_json::Map::new(); + let mut bytes = 0usize; + for (field, new_value) in after { + if IGNORED_FIELDS.contains(&field.as_str()) { + continue; + } + let old_value = before.get(field); + match old_value { + Some(old_value) if old_value == new_value => continue, + None if new_value.is_null() => continue, + _ => {} + } + fields.push(field.clone()); + if bytes <= MAX_CHANGES_BYTES { + bytes += json_len(new_value) + old_value.map_or(0, json_len) + field.len(); + } + if bytes > MAX_CHANGES_BYTES { + continue; + } + let mut entry = serde_json::Map::new(); + if let Some(old_value) = old_value { + entry.insert("old".to_string(), old_value.clone()); + } + entry.insert("new".to_string(), new_value.clone()); + changes.insert(field.clone(), serde_json::Value::Object(entry)); + } + + if fields.is_empty() { + return None; + } + if bytes > MAX_CHANGES_BYTES { + return Some(serde_json::json!({ "truncated_fields": fields })); + } + Some(serde_json::Value::Object(changes)) +} + +/// Serialized size of `value` without building the string for it. +fn json_len(value: &serde_json::Value) -> usize { + struct Counter(usize); + impl std::io::Write for Counter { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + self.0 += buf.len(); + Ok(buf.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + let mut counter = Counter(0); + let _ = serde_json::to_writer(&mut counter, value); + counter.0 +} + +/// jsonb rejects `\u0000` inside a string, and `changes` quotes caller-supplied +/// text — a schedule's `args`, a worker's error message. One NUL anywhere in +/// there would fail the insert and cost the row. +fn strip_nuls(value: &mut serde_json::Value) { + match value { + serde_json::Value::String(s) if s.contains('\0') => *s = s.replace('\0', ""), + serde_json::Value::Array(items) => items.iter_mut().for_each(strip_nuls), + serde_json::Value::Object(map) => map.values_mut().for_each(strip_nuls), + _ => {} + } +} + +/// The last word on what reaches the column, applied at the write itself so a +/// hand-built `changes` (the server-initiated disables carry an error string of +/// unknown length and origin) gets it too, not just a computed diff. +fn cap_changes(changes: Option) -> Option { + let mut changes = changes?; + strip_nuls(&mut changes); + if json_len(&changes) <= MAX_CHANGES_BYTES { + return Some(changes); + } + let fields = changes + .as_object() + .map(|o| o.keys().cloned().collect::>()) + .unwrap_or_default(); + Some(serde_json::json!({ "truncated_fields": fields })) +} + +/// One trigger mutation, as it is about to be recorded. +#[derive(Clone)] +pub struct TriggerHistoryEvent<'a> { + pub workspace_id: &'a str, + /// `"schedule"`, or the trigger's `TRIGGER_TYPE` (`"http"`, `"kafka"`, …). + pub trigger_kind: &'a str, + pub path: &'a str, + pub operation: TriggerOperation, + pub source: TriggerSource, + /// `None` when the server acted on its own. + pub username: Option<&'a str>, + pub changes: Option, +} + +impl<'a> TriggerHistoryEvent<'a> { + /// The event for a trigger the server disabled on its own after a failure. + /// + /// `forced_state` is the column the disable wrote, in the same + /// `{field: {old, new}}` shape as a diff — the two disable paths write + /// different columns (`enabled` for a schedule, `mode` for a trigger). + /// + /// Record this only when the disabling `UPDATE` reported an affected row, + /// and only when that `UPDATE` was itself predicated on the trigger still + /// being enabled. The server reads the trigger long before it writes, so + /// without both the row describes a transition a user had already made. + pub fn server_disable( + workspace_id: &'a str, + trigger_kind: &'a str, + path: &'a str, + mut forced_state: serde_json::Value, + error: &str, + ) -> Self { + if let Some(obj) = forced_state.as_object_mut() { + obj.insert("error".to_string(), serde_json::json!({ "new": error })); + } + Self { + workspace_id, + trigger_kind, + path, + operation: TriggerOperation::Disable, + source: TriggerSource::Worker, + username: None, + changes: Some(forced_state), + } + } +} + +/// Record a disable inside the transaction that made it, without letting a +/// failed insert take the disable down with it. +/// +/// The caller's `UPDATE` holds the trigger's row lock until that transaction +/// commits, and this runs inside that window — so the row cannot end up +/// describing a trigger that was deleted and recreated at the same path in +/// between, which is the whole point of doing it here rather than on a second +/// connection afterwards. +/// +/// The insert itself goes in a savepoint. If it fails it rolls back alone, the +/// caller still commits the disable, and the reason comes back here so the +/// caller can alert: a trigger left enabled reads as healthy while never firing +/// again, which is worse than a missing audit row. +pub async fn record_in_disable_tx( + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + event: TriggerHistoryEvent<'_>, +) -> Option { + let mut savepoint = match tx.begin().await { + Ok(savepoint) => savepoint, + Err(e) => return Some(e.to_string()), + }; + match record(&mut savepoint, event).await { + Ok(()) => savepoint.commit().await.err().map(|e| e.to_string()), + Err(e) => { + savepoint.rollback().await.ok(); + Some(e.to_string()) + } + } +} + +/// Append `event` to the history. +/// +/// Pass the same connection as the mutation for the two to commit together. +/// Does not authorize — see the module docs. +pub async fn record(conn: &mut PgConnection, event: TriggerHistoryEvent<'_>) -> Result<()> { + sqlx::query!( + "INSERT INTO trigger_history + (workspace_id, trigger_kind, path, operation, source, username, changes) + VALUES ($1, $2, $3, $4, $5, $6, $7)", + event.workspace_id, + event.trigger_kind, + event.path, + event.operation.as_str(), + event.source.as_str(), + event.username, + cap_changes(event.changes) as _, + ) + .execute(&mut *conn) + .await?; + Ok(()) +} + +/// Append one row per path, all describing the same change. +/// +/// For the workspace-wide operations that rewrite every schedule at once, where +/// a per-path diff would cost a snapshot per row and say the same thing each +/// time. Does not authorize — see the module docs. +pub async fn record_bulk( + conn: &mut PgConnection, + workspace_id: &str, + trigger_kind: &str, + paths: &[String], + operation: TriggerOperation, + source: TriggerSource, + username: Option<&str>, + changes: Option, +) -> Result<()> { + if paths.is_empty() { + return Ok(()); + } + sqlx::query!( + "INSERT INTO trigger_history + (workspace_id, trigger_kind, path, operation, source, username, changes) + SELECT $1, $2, p, $3, $4, $5, $6 FROM unnest($7::text[]) p", + workspace_id, + trigger_kind, + operation.as_str(), + source.as_str(), + username, + cap_changes(changes) as _, + paths, + ) + .execute(&mut *conn) + .await?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + /// The whole worker side of the attribution rests on this: a mutation made + /// outside a request records `worker` without each call site saying so. + #[tokio::test] + async fn client_is_absent_outside_a_request() { + assert_eq!(TriggerSource::of_request(false), TriggerSource::Worker); + assert_eq!( + scope_client(None, async { TriggerSource::of_request(true) }).await, + TriggerSource::Ui + ); + assert_eq!( + scope_client(None, async { TriggerSource::of_request(false) }).await, + TriggerSource::Api + ); + assert_eq!( + scope_client(Some(TriggerSource::Cli), async { + TriggerSource::of_request(true) + }) + .await, + TriggerSource::Cli + ); + } + + /// `error` and `edited_at` stand in for the whole ignore list: every trigger + /// update clears `error`, so without it an ordinary edit would carry the + /// failure the trigger had before it. + #[test] + fn diff_keeps_only_what_changed() { + let before = json!({"schedule": "0 0 * * *", "enabled": true, "edited_at": "a", "error": "boom"}); + let after = json!({"schedule": "0 1 * * *", "enabled": true, "edited_at": "b", "error": null}); + assert_eq!( + summarize_changes(Some(&before), Some(&after)), + Some(json!({"schedule": {"old": "0 0 * * *", "new": "0 1 * * *"}})) + ); + assert_eq!(summarize_changes(Some(&before), Some(&before)), None); + } + + #[test] + fn create_drops_null_columns_and_bookkeeping() { + let after = json!({"schedule": "0 0 * * *", "summary": null, "workspace_id": "w"}); + assert_eq!( + summarize_changes(None, Some(&after)), + Some(json!({"schedule": {"new": "0 0 * * *"}})) + ); + } + + /// A NUL reaching the column fails the insert, and `changes` quotes + /// caller-supplied text — so this is the difference between a recorded + /// disable and a lost one. + #[test] + fn nul_bytes_never_reach_the_column() { + let changes = cap_changes(Some(json!({ "error": { "new": "boom\u{0}tail" } }))); + assert_eq!(changes, Some(json!({ "error": { "new": "boomtail" } }))); + } + + #[test] + fn oversized_changes_keep_the_field_names() { + let after = json!({ "args": "x".repeat(MAX_CHANGES_BYTES + 1) }); + assert_eq!( + summarize_changes(None, Some(&after)), + Some(json!({"truncated_fields": ["args"]})) + ); + } +} diff --git a/backend/windmill-common/src/worker.rs b/backend/windmill-common/src/worker.rs index a94160be33..deff9c0588 100644 --- a/backend/windmill-common/src/worker.rs +++ b/backend/windmill-common/src/worker.rs @@ -30,6 +30,7 @@ use crate::{ agent_workers::PingJobStatusResponse, cache::{unwrap_or_error, RawNode, RawScript}, error::{self, to_anyhow}, + external_ip::UNKNOWN_IP, global_settings::CUSTOM_TAGS_SETTING, indexer::TantivyIndexerSettings, server::Smtp, @@ -948,20 +949,22 @@ pub fn write_file_at_user_defined_location( } pub async fn reload_custom_tags_setting(db: &DB) -> error::Result<()> { - let q = sqlx::query!( - "SELECT value FROM global_settings WHERE name = $1", - CUSTOM_TAGS_SETTING - ) - .fetch_optional(db) - .await?; + let q = + crate::global_settings::load_value_from_global_settings(db, CUSTOM_TAGS_SETTING).await?; + apply_custom_tags_setting(q); + Ok(()) +} +/// The half of [`reload_custom_tags_setting`] after the read, so a batched settings pass can +/// apply a value it already fetched. +pub fn apply_custom_tags_setting(q: Option) { let tags = if let Some(q) = q { - if let Ok(v) = serde_json::from_value::>(q.value.clone()) { + if let Ok(v) = serde_json::from_value::>(q.clone()) { v } else { tracing::error!( "Could not parse custom tags setting as vec of strings, found: {:#?}", - &q.value + &q ); vec![] } @@ -989,7 +992,6 @@ pub async fn reload_custom_tags_setting(db: &DB) -> error::Result<()> { ] .concat(), )); - Ok(()) } #[cfg(not(windows))] @@ -1749,25 +1751,23 @@ pub async fn update_ping_http( insert_ping.occupancy_rate_5m, insert_ping.occupancy_rate_30m, insert_ping.native_mode.unwrap_or(false), + insert_ping.ip.as_deref(), db, ) .await? } PingType::Initial => { - if insert_ping.worker_instance.is_none() - || insert_ping.version.is_none() - || insert_ping.ip.is_none() - { - return Err(anyhow::anyhow!( - "Worker instance, version and ip are required" - )); + if insert_ping.worker_instance.is_none() || insert_ping.version.is_none() { + return Err(anyhow::anyhow!("Worker instance and version are required")); } insert_ping_query( &insert_ping.worker_instance.unwrap(), &worker_name, worker_group, - &insert_ping.ip.unwrap(), + // An agent worker sends the sentinel rather than nothing, to stay acceptable to + // servers that still require an IP here; both mean "not resolved yet". + insert_ping.ip.as_deref().filter(|ip| *ip != UNKNOWN_IP), insert_ping.tags.unwrap_or_default().as_slice(), insert_ping.dw, insert_ping.dws.as_deref(), @@ -1904,12 +1904,13 @@ pub async fn fetch_raw_script_from_app_query( /// `wm_version`, hold the instance-wide `MIN_VERSION` back forever, and one still naming the /// job that process was killed mid-way through skews the zombie/OOM diagnostics that read it. /// `started_at` and `jobs_executed` are the only two columns carried over, being the -/// continuity itself. +/// continuity itself — plus `ip` for as long as `ip` is `None`, which means the external IP +/// lookup has not resolved yet and the predecessor's address is still the best guess. pub async fn insert_ping_query( worker_instance: &str, worker_name: &str, worker_group: &str, - ip: &str, + ip: Option<&str>, tags: &[String], dw: Option, dws: Option<&[String]>, @@ -1920,9 +1921,13 @@ pub async fn insert_ping_query( native_mode: bool, db: &DB, ) -> anyhow::Result { + // A NULL `ip` means the external IP lookup is still in flight; a later ping fills it in, and + // meanwhile the value a previous process wrote to a reclaimed row is the best guess we have. A + // lookup that has failed reports `external_ip::UNRETRIEVABLE_IP`, which does overwrite it. The + // literal below must stay equal to `external_ip::UNKNOWN_IP`. let previous_jobs_executed = sqlx::query_scalar!( - "INSERT INTO worker_ping (worker_instance, worker, ip, custom_tags, worker_group, dedicated_worker, dedicated_workers, wm_version, vcpus, memory, job_isolation, native_mode) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) ON CONFLICT (worker) - DO UPDATE set ping_at = now(), worker_instance = EXCLUDED.worker_instance, ip = EXCLUDED.ip, custom_tags = EXCLUDED.custom_tags, worker_group = EXCLUDED.worker_group, dedicated_worker = EXCLUDED.dedicated_worker, dedicated_workers = EXCLUDED.dedicated_workers, wm_version = EXCLUDED.wm_version, vcpus = COALESCE(EXCLUDED.vcpus, worker_ping.vcpus), memory = COALESCE(EXCLUDED.memory, worker_ping.memory), job_isolation = EXCLUDED.job_isolation, native_mode = EXCLUDED.native_mode, current_job_id = NULL, current_job_workspace_id = NULL + "INSERT INTO worker_ping (worker_instance, worker, ip, custom_tags, worker_group, dedicated_worker, dedicated_workers, wm_version, vcpus, memory, job_isolation, native_mode) VALUES ($1, $2, COALESCE($3, 'NO IP'), $4, $5, $6, $7, $8, $9, $10, $11, $12) ON CONFLICT (worker) + DO UPDATE set ping_at = now(), worker_instance = EXCLUDED.worker_instance, ip = COALESCE($3, worker_ping.ip), custom_tags = EXCLUDED.custom_tags, worker_group = EXCLUDED.worker_group, dedicated_worker = EXCLUDED.dedicated_worker, dedicated_workers = EXCLUDED.dedicated_workers, wm_version = EXCLUDED.wm_version, vcpus = COALESCE(EXCLUDED.vcpus, worker_ping.vcpus), memory = COALESCE(EXCLUDED.memory, worker_ping.memory), job_isolation = EXCLUDED.job_isolation, native_mode = EXCLUDED.native_mode, current_job_id = NULL, current_job_workspace_id = NULL RETURNING jobs_executed", worker_instance, worker_name, @@ -2027,12 +2032,13 @@ pub async fn update_worker_ping_main_loop_query( occupancy_rate_5m: Option, occupancy_rate_30m: Option, native_mode: bool, + ip: Option<&str>, db: &DB, ) -> anyhow::Result<()> { timeout(Duration::from_secs(10), sqlx::query!( "UPDATE worker_ping SET ping_at = now(), jobs_executed = $1, custom_tags = $2, occupancy_rate = $3, memory_usage = $4, wm_memory_usage = $5, vcpus = COALESCE($7, vcpus), - memory = COALESCE($8, memory), occupancy_rate_15s = $9, occupancy_rate_5m = $10, occupancy_rate_30m = $11, native_mode = $12 WHERE worker = $6", + memory = COALESCE($8, memory), occupancy_rate_15s = $9, occupancy_rate_5m = $10, occupancy_rate_30m = $11, native_mode = $12, ip = COALESCE($13, ip) WHERE worker = $6", jobs_executed, tags, occupancy_rate, @@ -2045,6 +2051,7 @@ pub async fn update_worker_ping_main_loop_query( occupancy_rate_5m, occupancy_rate_30m, native_mode, + ip, ) .execute(db)) .await??; diff --git a/backend/windmill-common/tests/global_settings_batch.rs b/backend/windmill-common/tests/global_settings_batch.rs new file mode 100644 index 0000000000..7701a9505a --- /dev/null +++ b/backend/windmill-common/tests/global_settings_batch.rs @@ -0,0 +1,94 @@ +//! `load_values_from_global_settings` is what a settings pass fetches with, so the difference +//! between "no row" and "the read failed" has to survive it: several settings reset to a +//! default when they read as unset, and would clobber a known-good value on a transient error. + +use serde_json::json; +use sqlx::{Pool, Postgres}; +use windmill_common::global_settings::{ + load_values_from_global_settings, BASE_URL_SETTING, SCIM_TOKEN_SETTING, +}; + +async fn set_setting(db: &Pool, name: &str, value: serde_json::Value) { + sqlx::query( + "INSERT INTO global_settings (name, value) VALUES ($1, $2) \ + ON CONFLICT (name) DO UPDATE SET value = EXCLUDED.value", + ) + .bind(name) + .bind(value) + .execute(db) + .await + .expect("failed to write global setting"); +} + +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn batch_returns_only_rows_that_exist(db: Pool) { + set_setting(&db, BASE_URL_SETTING, json!("set")).await; + // A dynamically named row, of which the table holds one per workspace with no cleanup + // path. Asking by name is what keeps a pass from scaling with how many of them exist. + set_setting(&db, "wm_test_dynamic:some_workspace", json!({})).await; + + let values = load_values_from_global_settings(&db, &[BASE_URL_SETTING, SCIM_TOKEN_SETTING]) + .await + .unwrap(); + + assert_eq!(values.get(BASE_URL_SETTING), Some(&json!("set"))); + assert_eq!( + values.get(SCIM_TOKEN_SETTING), + None, + "a name with no row must be absent, which the caller reads as unset" + ); + assert_eq!(values.len(), 1, "unrequested names must not come back"); +} + +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn batch_reports_failure_rather_than_an_empty_result(db: Pool) { + set_setting(&db, BASE_URL_SETTING, json!("set")).await; + + // Closing a pool makes every query on it fail. It has to be a separate pool built from the + // same options — `Pool` is a handle, so closing a clone of `db` would take `db` down too. + let unusable = sqlx::postgres::PgPoolOptions::new() + .connect_with((*db.connect_options()).clone()) + .await + .expect("failed to open second pool"); + unusable.close().await; + + assert!( + load_values_from_global_settings(&unusable, &[BASE_URL_SETTING]) + .await + .is_err(), + "a failed read must be an error, not an empty map that reads as every setting unset" + ); +} + +/// `get_or_create_jwt_secret` decides in SQL rather than from the caller's read, so that +/// replicas booting together cannot each install their own secret and reject each other's +/// tokens. Reverting it to a plain upsert would pass every other test in this file. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn jwt_secret_is_created_once_and_never_overwritten(db: Pool) { + let first = windmill_common::global_settings::get_or_create_jwt_secret(&db) + .await + .unwrap(); + assert!(!first.is_empty()); + + // Concurrent callers must converge on the one secret that landed, not clobber it. + let (a, b) = tokio::join!( + windmill_common::global_settings::get_or_create_jwt_secret(&db), + windmill_common::global_settings::get_or_create_jwt_secret(&db), + ); + assert_eq!(a.unwrap(), first); + assert_eq!(b.unwrap(), first); + + // A value that is not a usable secret is replaced rather than left in place. + set_setting(&db, "jwt_secret", json!(12345)).await; + let repaired = windmill_common::global_settings::get_or_create_jwt_secret(&db) + .await + .unwrap(); + assert_ne!(repaired, first); + assert_eq!( + windmill_common::global_settings::get_or_create_jwt_secret(&db) + .await + .unwrap(), + repaired, + "once repaired it must be stable again" + ); +} diff --git a/backend/windmill-oauth/src/lib.rs b/backend/windmill-oauth/src/lib.rs index 32648a2115..2650b68b4d 100644 --- a/backend/windmill-oauth/src/lib.rs +++ b/backend/windmill-oauth/src/lib.rs @@ -1179,6 +1179,18 @@ mod tests { assert!(verifier.verify("123", "body", "wrong_sig").is_err()); } + // Sage Intacct's token endpoint rejects HTTP Basic client authentication on the + // refresh_token grant (`invalid_client`), so its credentials must go in the form body. + #[test] + fn sage_intacct_registry_entry_uses_request_body_client_auth() { + let registry: HashMap = + serde_json::from_str(include_str!("../../oauth_connect.json")).unwrap(); + assert_eq!( + registry.get("sage_intacct").unwrap().req_body_auth, + Some(true) + ); + } + #[test] fn canonical_provider_name_strips_sandbox_suffix() { assert_eq!(canonical_provider_name("docusign_sandbox"), "docusign"); diff --git a/backend/windmill-object-store/src/lib.rs b/backend/windmill-object-store/src/lib.rs index 2482cf14dd..1481830f2a 100644 --- a/backend/windmill-object-store/src/lib.rs +++ b/backend/windmill-object-store/src/lib.rs @@ -1214,6 +1214,45 @@ impl RecordBatchWriter for RecordBatchWriterEnum { } } +/// Infer the Arrow schema of a newline-delimited JSON file. +/// +/// Inference only looks at the first `DEFAULT_SCHEMA_INFER_MAX_RECORD` rows, and a column +/// that holds nothing but JSON `null` across that sample is typed `DataType::Null`, which +/// makes the reader reject the first real value further down the file. When a longer file +/// leaves such a column behind, re-infer over all of it so the column's type comes from +/// wherever its first non-null value is. +#[cfg(feature = "parquet")] +fn infer_ndjson_schema( + path: &std::path::Path, + row_count: u64, +) -> anyhow::Result { + use datafusion::arrow::datatypes::{DataType, Schema}; + use datafusion::arrow::json::reader::infer_json_schema; + use datafusion::datasource::file_format::DEFAULT_SCHEMA_INFER_MAX_RECORD; + + fn is_untyped(data_type: &DataType) -> bool { + match data_type { + DataType::Null => true, + DataType::Struct(fields) => fields.iter().any(|f| is_untyped(f.data_type())), + DataType::List(field) | DataType::LargeList(field) => is_untyped(field.data_type()), + _ => false, + } + } + + let infer = |max_records: Option| -> anyhow::Result { + let reader = std::io::BufReader::new(std::fs::File::open(path)?); + Ok(infer_json_schema(reader, max_records).map_err(to_anyhow)?.0) + }; + + let schema = infer(Some(DEFAULT_SCHEMA_INFER_MAX_RECORD))?; + if row_count > DEFAULT_SCHEMA_INFER_MAX_RECORD as u64 + && schema.fields().iter().any(|f| is_untyped(f.data_type())) + { + return infer(None); + } + Ok(schema) +} + #[cfg(feature = "parquet")] struct ChannelWriter { sender: tokio::sync::mpsc::Sender>, @@ -1380,10 +1419,21 @@ where ingest_start.elapsed(), ); + let inferred_schema = { + let path = path.clone(); + task::spawn_blocking(move || infer_ndjson_schema(&path, row_count)) + .await + .map_err(to_anyhow)?? + }; + let ctx = SessionContext::new(); - ctx.register_json("my_table", path_str, NdJsonReadOptions::default()) - .await - .map_err(to_anyhow)?; + ctx.register_json( + "my_table", + path_str, + NdJsonReadOptions::default().schema(&inferred_schema), + ) + .await + .map_err(to_anyhow)?; let df = ctx.sql("SELECT * FROM my_table").await.map_err(to_anyhow)?; let schema = df.schema().clone().into(); @@ -2329,4 +2379,34 @@ mod tests { let result = get_logs_from_store(1, "logs", &None).await; assert!(result.is_none()); } + + #[cfg(feature = "parquet")] + #[tokio::test] + async fn test_convert_json_line_stream_value_after_null_only_sample() { + use datafusion::datasource::file_format::DEFAULT_SCHEMA_INFER_MAX_RECORD as SAMPLE; + use futures::StreamExt; + + // One more all-null row than the inference sample can see, so the column's type has + // to come from the row that follows it. + let total = SAMPLE + 1; + let rows = (0..total).map(|i| { + Ok::<_, anyhow::Error>(serde_json::json!({ + "col": if i < SAMPLE { serde_json::Value::Null } else { serde_json::json!("2023-11-30") } + })) + }); + + let (mut out, stats) = + convert_json_line_stream(futures::stream::iter(rows), S3ModeFormat::Json, None) + .await + .unwrap(); + assert_eq!(stats.rows, total as u64); + + let mut bytes = Vec::new(); + while let Some(chunk) = out.next().await { + bytes.extend_from_slice(&chunk.expect("row after the null-only sample must decode")); + } + let parsed: Vec = serde_json::from_slice(&bytes).unwrap(); + assert_eq!(parsed.len(), total); + assert_eq!(parsed[total - 1]["col"], serde_json::json!("2023-11-30")); + } } diff --git a/backend/windmill-queue/src/jobs.rs b/backend/windmill-queue/src/jobs.rs index 2b4ebe8bfd..b286082b1c 100644 --- a/backend/windmill-queue/src/jobs.rs +++ b/backend/windmill-queue/src/jobs.rs @@ -2503,6 +2503,68 @@ pub async fn send_success_to_workspace_handler<'a, 'c, T: Serialize + Send + Syn Ok(()) } +/// The event for a schedule the server disabled on its own. +pub fn schedule_auto_disable_event<'a>( + workspace_id: &'a str, + path: &'a str, + error: &str, +) -> windmill_common::trigger_history::TriggerHistoryEvent<'a> { + windmill_common::trigger_history::TriggerHistoryEvent::server_disable( + workspace_id, + windmill_common::trigger_history::SCHEDULE_TRIGGER_KIND, + path, + serde_json::json!({ "enabled": { "old": true, "new": false } }), + error, + ) +} + +/// Disable a schedule the server can no longer arm, and record that it did. +/// +/// Contract on `record_in_disable_tx`. Here `tx` is the job-completion +/// transaction, so the savepoint also keeps a failed insert from poisoning it. +/// +/// Returns `Err` only when the disable itself failed; a lost history row comes +/// back through `history_lost` for the caller to report. +async fn disable_schedule_and_record( + tx: &mut Transaction<'_, Postgres>, + schedule: &Schedule, + err: &Error, + history_lost: &mut Option, +) -> Result { + let disable_result = sqlx::query!( + "UPDATE schedule SET enabled = false, error = $1 WHERE workspace_id = $2 AND path = $3 AND enabled = true", + err.to_string(), + &schedule.workspace_id, + &schedule.path + ) + .execute(&mut **tx) + .await; + + #[cfg(feature = "failpoints")] + let disable_result = if schedule_failpoints::is_active( + schedule_failpoints::ScheduleFailPoint::ScheduleDisable, + ) { + Err(sqlx::Error::Protocol( + "failpoint: schedule disable".to_string(), + )) + } else { + disable_result + }; + + let rows = disable_result?.rows_affected(); + // Zero rows means a user disabled the schedule first: no transition of ours + // to record. + if rows == 0 { + return Ok(0); + } + + let event = + schedule_auto_disable_event(&schedule.workspace_id, &schedule.path, &err.to_string()); + *history_lost = windmill_common::trigger_history::record_in_disable_tx(tx, event).await; + + Ok(rows) +} + pub async fn try_schedule_next_job<'c>( db: &Pool, mut tx: Transaction<'c, Postgres>, @@ -2657,36 +2719,31 @@ pub async fn try_schedule_next_job<'c>( "Could not push next scheduled job for {}: {err}. Disabling schedule.", schedule.path ); - let disable_result = sqlx::query!( - "UPDATE schedule SET enabled = false, error = $1 WHERE workspace_id = $2 AND path = $3", - err.to_string(), - &schedule.workspace_id, - &schedule.path - ) - .execute(&mut *tx) - .await; - #[cfg(feature = "failpoints")] - let disable_result = if schedule_failpoints::is_active( - schedule_failpoints::ScheduleFailPoint::ScheduleDisable, - ) { - Err(sqlx::Error::Protocol( - "failpoint: schedule disable".to_string(), - )) - } else { - disable_result - }; - if let Err(disable_err) = disable_result { + let mut history_lost = None; + match disable_schedule_and_record(&mut tx, schedule, err, &mut history_lost).await { + Err(disable_err) => { + report_error_to_workspace_handler_or_critical_side_channel( + job, + db, + format!( + "Could not push next scheduled job for {} and could not disable schedule: {disable_err}", + schedule.path, + ), + ) + .await; + } + Ok(_) => push_err = None, + } + if let Some(history_err) = history_lost { report_error_to_workspace_handler_or_critical_side_channel( job, db, format!( - "Could not push next scheduled job for {} and could not disable schedule: {disable_err}", + "Disabled schedule {} but could not record it in the trigger history: {history_err}", schedule.path, ), ) .await; - } else { - push_err = None; } } } @@ -6583,6 +6640,20 @@ async fn push_inner<'c, 'd>( ) .unzip(); + // Which trigger kinds an instance actually fires. Counted here rather than + // aggregated from `v2_job` later: that table's only usable index is + // (workspace_id, created_at), so a windowed GROUP BY over it is a full scan. + // + // Root jobs only. A scheduled flow hands every step push its own + // `schedule_path` (see `FlowJob::schedule_path`), so counting per push would + // score one run as a fire per step job — a loop pushes two of those per + // iteration — burying every other kind, and would sit on the per-step path. + if flow_step_id.is_none() { + if let Some(kind) = trigger_kind.as_ref() { + windmill_common::feature_usage::log_feature_usage("trigger", "fired", kind.as_str()); + } + } + #[cfg(feature = "cloud")] if *CLOUD_HOSTED { check_workspace_queue_cap(&mut *tx, workspace_id).await?; diff --git a/backend/windmill-store/src/resources.rs b/backend/windmill-store/src/resources.rs index a3b5d2a1a7..421d147bdf 100644 --- a/backend/windmill-store/src/resources.rs +++ b/backend/windmill-store/src/resources.rs @@ -9,7 +9,6 @@ use dashmap::DashMap; use std::collections::HashMap; use std::net::IpAddr; -use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::LazyLock; use windmill_api_auth::{ @@ -17,6 +16,7 @@ use windmill_api_auth::{ require_super_admin, ApiAuthed, Tokened, }; use windmill_common::db::DB; +use windmill_common::per_minute_counter::PerMinuteCounter; use windmill_common::workspaces::{check_deploy_rules, RuleCheckResult}; use crate::secret_backend_ext::rename_vault_secret; @@ -250,8 +250,17 @@ async fn list_names( #[derive(Serialize, FromRow)] pub struct SearchResource { path: String, - value: serde_json::Value, + /// Pretty-printed JSON, capped at `SEARCH_RESOURCE_VALUE_MAX_CHARS`. + value: String, + truncated: bool, } + +/// This route hands the browser every readable resource's value at once and the client keeps +/// them all in memory, so without a cap a workspace of large JSON resources sends tens of MB +/// and freezes the tab. Content search only fuzzy-matches and previews a few lines of each. +/// The value is spelled out in `listSearchResource`'s openapi.yaml description; change both. +const SEARCH_RESOURCE_VALUE_MAX_CHARS: i32 = 4000; + async fn list_search_resources( authed: ApiAuthed, Path(w_id): Path, @@ -263,9 +272,17 @@ async fn list_search_resources( let allowed = build_scope_path_predicate(&authed, "resources", "read"); let rows = sqlx::query_as!( SearchResource, - "SELECT path, value from resource WHERE workspace_id = $1 LIMIT $2", + // `OFFSET 0` fences the subquery so the planner cannot pull it up: without it + // jsonb_pretty is inlined into both the left() and the length(), serializing + // every value twice. + r#"SELECT resource.path, + COALESCE(left(pretty.value, $3), '') as "value!", + COALESCE(length(pretty.value) > $3, false) as "truncated!" + FROM resource, LATERAL (SELECT jsonb_pretty(resource.value) as value OFFSET 0) pretty + WHERE workspace_id = $1 LIMIT $2"#, &w_id, - n + n, + SEARCH_RESOURCE_VALUE_MAX_CHARS ) .fetch_all(&mut *tx) .await? @@ -1041,17 +1058,11 @@ pub const MAX_RESOURCE_VERSIONS: i64 = 100; /// low is the safe direction for something that only ever logs. const RESOURCE_WRITE_ADVISORY_PER_MIN: u32 = 20; -struct ResourceWriteRate { - count: u32, - minute_bucket: i64, -} - /// Writes seen per (workspace, path) per minute. Purely advisory, and deliberately so: nothing /// is throttled, the count is per process and resets on restart, so it undercounts across /// servers. That is affordable for a log line and is what keeps this off the write path proper. -static RESOURCE_WRITE_RATES: LazyLock> = - LazyLock::new(DashMap::new); -static RESOURCE_WRITES_SEEN: AtomicU64 = AtomicU64::new(0); +static RESOURCE_WRITE_RATES: LazyLock> = + LazyLock::new(PerMinuteCounter::new); /// Notice a caller rewriting one resource in a loop and point them at a store meant for it. /// Counts writes rather than versions: an unchanged value records nothing, but it still costs a @@ -1063,29 +1074,10 @@ fn note_resource_write(w_id: &str, path: &str, resource_type: &str) { if INTERNAL_RESOURCE_TYPES.contains(&resource_type) { return; } - let minute_bucket = chrono::Utc::now().timestamp() / 60; - // The entry guard holds a lock on its DashMap shard, and `retain` below takes every shard. - // Keeping the guard alive across that call deadlocks the request handler, so this block is - // load-bearing: it must end before the eviction, not be flattened into the function body. - let reached_cap = { - let mut rate = RESOURCE_WRITE_RATES - .entry((w_id.to_string(), path.to_string())) - .or_insert(ResourceWriteRate { count: 0, minute_bucket }); - if rate.minute_bucket != minute_bucket { - rate.minute_bucket = minute_bucket; - rate.count = 0; - } - rate.count += 1; - rate.count == RESOURCE_WRITE_ADVISORY_PER_MIN - }; - // Bounded without a background task: periodically drop what neither the current nor the - // previous minute can still need. - if RESOURCE_WRITES_SEEN.fetch_add(1, Ordering::Relaxed) % 256 == 0 { - RESOURCE_WRITE_RATES.retain(|_, rate| rate.minute_bucket >= minute_bucket - 1); - } + let writes = RESOURCE_WRITE_RATES.increment((w_id.to_string(), path.to_string())); // Once per minute per path: `==` rather than `>=` so a sustained loop logs at the crossing // and then stays quiet until the bucket rolls over. - if reached_cap { + if writes == RESOURCE_WRITE_ADVISORY_PER_MIN { tracing::warn!( workspace_id = %w_id, path = %path, diff --git a/backend/windmill-test-utils/src/lib.rs b/backend/windmill-test-utils/src/lib.rs index 060e8af983..3766a6050d 100644 --- a/backend/windmill-test-utils/src/lib.rs +++ b/backend/windmill-test-utils/src/lib.rs @@ -407,24 +407,41 @@ pub async fn in_test_worker( /// /// The chain contains `?Send` futures (trait objects), so `tokio::spawn` /// is not viable; `std::thread::spawn` sidesteps the shared-stack issue. -pub fn run_in_isolated_thread(f: F) -> R +/// +/// Await the thread, never `join()` it: the caller's runtime owns the `sqlx` +/// pool, and sqlx hands a connection's permit back from a task it spawns when +/// the connection drops. Blocking that runtime holds those permits for as long +/// as the body runs, so the body's own `push` dies on `PoolTimedOut`. +/// +/// The thread is detached, so dropping this future leaves the body — including +/// its worker and its pool handles — running past the end of the test. Don't +/// wrap the call in `timeout`/`select!`. +pub async fn run_in_isolated_thread(f: F) -> R where F: FnOnce() -> Fut + Send + 'static, Fut: std::future::Future, R: Send + 'static, { + let (tx, rx) = tokio::sync::oneshot::channel(); std::thread::Builder::new() .stack_size(8 * 1024 * 1024) .spawn(move || { - let rt = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - .expect("build current-thread runtime"); - rt.block_on(f()) + let res = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + let rt = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("build current-thread runtime"); + rt.block_on(f()) + })); + let _ = tx.send(res); }) - .expect("spawn isolated test thread") - .join() - .expect("isolated test thread panicked") + .expect("spawn isolated test thread"); + + match rx.await { + Ok(Ok(res)) => res, + Ok(Err(panic)) => std::panic::resume_unwind(panic), + Err(_) => panic!("isolated test thread died without returning"), + } } pub fn spawn_test_worker( @@ -442,7 +459,6 @@ pub fn spawn_test_worker( let (tx, rx) = KillpillSender::new(1); let worker_instance: &str = "test worker instance"; let worker_name: String = next_worker_name(); - let ip: &str = Default::default(); let conn = conn.to_owned(); let tx2 = tx.clone(); @@ -465,7 +481,6 @@ pub fn spawn_test_worker( worker_name, 1, 1, - ip, rx, tx2, &base_internal_url, @@ -494,7 +509,6 @@ pub fn spawn_test_worker_dedicated( let (tx, rx) = KillpillSender::new(1); let worker_instance: &str = "test worker instance"; let worker_name: String = next_worker_name(); - let ip: &str = Default::default(); let conn = conn.to_owned(); let tx2 = tx.clone(); @@ -548,7 +562,6 @@ pub fn spawn_test_worker_dedicated( worker_name, 1, 1, - ip, rx, tx2, &base_internal_url, diff --git a/backend/windmill-trigger-http/src/handler.rs b/backend/windmill-trigger-http/src/handler.rs index 17321b7775..14e1b9a4d2 100644 --- a/backend/windmill-trigger-http/src/handler.rs +++ b/backend/windmill-trigger-http/src/handler.rs @@ -307,6 +307,37 @@ pub async fn create_many_http_triggers( .map_err(|err| error_wrapper(&new_http_trigger.config.route_path, err.into()))?; } + // Bulk create is still authoring, so it records like the single-create + // route rather than being the one way to make a trigger appear with no + // history behind it. + let created = windmill_common::trigger_history::snapshot_row( + &mut *tx, + "http_trigger", + &w_id, + &new_http_trigger.base.path, + ) + .await + .map_err(|err| error_wrapper(&new_http_trigger.config.route_path, err))?; + windmill_common::trigger_history::record( + &mut *tx, + windmill_common::trigger_history::TriggerHistoryEvent { + workspace_id: &w_id, + trigger_kind: HttpTrigger::TRIGGER_TYPE, + path: &new_http_trigger.base.path, + operation: windmill_common::trigger_history::TriggerOperation::Create, + source: windmill_common::trigger_history::TriggerSource::of_request( + authed.is_session_token, + ), + username: Some(&authed.username), + changes: windmill_common::trigger_history::summarize_changes( + None, + created.as_ref(), + ), + }, + ) + .await + .map_err(|err| error_wrapper(&new_http_trigger.config.route_path, err))?; + audit_log( &mut *tx, &authed, diff --git a/backend/windmill-trigger/src/handler.rs b/backend/windmill-trigger/src/handler.rs index ad3a6b097a..00d8cbf828 100644 --- a/backend/windmill-trigger/src/handler.rs +++ b/backend/windmill-trigger/src/handler.rs @@ -16,6 +16,7 @@ use windmill_api_auth::{build_scope_path_predicate, check_scopes, ApiAuthed}; use windmill_common::{ db::UserDB, error::{Error, JsonResult, Result}, + trigger_history::{self, TriggerHistoryEvent, TriggerOperation, TriggerSource}, user_drafts::{ delete_all_drafts_for_path, delete_own_draft_for_path, fetch_draft_only_list_rows, overlay_or_draft_only, UserDraftItemKind, WithDraftOverlay, WithDraftQuery, @@ -458,6 +459,45 @@ pub trait TriggerCrud: Send + Sync + 'static { } } +/// Append this mutation to `trigger_history`, diffing the row at `path` against +/// `before`. +/// +/// Call it on the transaction that made the change, after the change: the +/// snapshot it takes is the "after" side of the diff, and the two commit or roll +/// back together. +/// +/// Records nothing when the snapshots say no row was written. `TriggerCrud::update_trigger` +/// returns `Result<()>` and several impls do not check `rows_affected`, so an +/// update aimed at a path that does not exist — or that RLS hides from the +/// caller — reaches here having changed nothing; without this the caller could +/// forge history rows at any path, since the insert policy is `WITH CHECK (true)`. +async fn record_trigger_history( + tx: &mut PgConnection, + authed: &ApiAuthed, + workspace_id: &str, + path: &str, + operation: TriggerOperation, + before: Option, +) -> Result<()> { + let after = trigger_history::snapshot_row(&mut *tx, T::TABLE_NAME, workspace_id, path).await?; + if after.is_none() || (operation == TriggerOperation::Update && before.is_none()) { + return Ok(()); + } + trigger_history::record( + &mut *tx, + TriggerHistoryEvent { + workspace_id, + trigger_kind: T::TRIGGER_TYPE, + path, + operation, + source: TriggerSource::of_request(authed.is_session_token), + username: Some(&authed.username), + changes: trigger_history::summarize_changes(before.as_ref(), after.as_ref()), + }, + ) + .await +} + pub fn trigger_routes() -> Router { let mut router = Router::new() .route("/create", post(create_trigger::)) @@ -556,6 +596,16 @@ async fn create_trigger( .await?; } + record_trigger_history::( + &mut *tx, + &authed, + &workspace_id, + &new_path, + TriggerOperation::Create, + None, + ) + .await?; + audit_log( &mut *tx, &authed, @@ -782,6 +832,9 @@ async fn update_trigger( &authed.username, ); + let before = + trigger_history::snapshot_row(&mut *tx, T::TABLE_NAME, &workspace_id, path).await?; + handler .update_trigger(&db, &mut *tx, &authed, &workspace_id, path, edit_trigger) .await?; @@ -799,6 +852,18 @@ async fn update_trigger( .await?; } + // Recorded at the new path, so a rename reads as one event there with + // `path` among the changed fields rather than a delete plus a create. + record_trigger_history::( + &mut *tx, + &authed, + &workspace_id, + &new_path, + TriggerOperation::Update, + before, + ) + .await?; + audit_log( &mut *tx, &authed, @@ -913,6 +978,22 @@ async fn delete_trigger( .await?; } + // No diff: the row is gone, and the trashbin above already keeps its full + // contents for a restore. + trigger_history::record( + &mut *tx, + TriggerHistoryEvent { + workspace_id: &workspace_id, + trigger_kind: T::TRIGGER_TYPE, + path, + operation: TriggerOperation::Delete, + source: TriggerSource::of_request(authed.is_session_token), + username: Some(&authed.username), + changes: None, + }, + ) + .await?; + audit_log( &mut *tx, &authed, @@ -1052,6 +1133,9 @@ async fn set_trigger_mode( } } + let before = + trigger_history::snapshot_row(&mut *tx, T::TABLE_NAME, &workspace_id, path).await?; + let updated = handler .set_trigger_mode(&authed, &mut *tx, &workspace_id, path, &payload.mode) .await?; @@ -1063,6 +1147,20 @@ async fn set_trigger_mode( ))); } + record_trigger_history::( + &mut *tx, + &authed, + &workspace_id, + path, + match payload.mode { + TriggerMode::Enabled => TriggerOperation::Enable, + TriggerMode::Disabled => TriggerOperation::Disable, + TriggerMode::Suspended => TriggerOperation::Suspend, + }, + before, + ) + .await?; + tx.commit().await?; handle_deployment_metadata( diff --git a/backend/windmill-trigger/src/listener.rs b/backend/windmill-trigger/src/listener.rs index e327ed3155..d2c80b4d65 100644 --- a/backend/windmill-trigger/src/listener.rs +++ b/backend/windmill-trigger/src/listener.rs @@ -386,9 +386,14 @@ pub trait Listener: TriggerCrud + TriggerJobArgs { error: String, ) { if listening_trigger.trigger_mode { - // SAFETY: Self::TABLE_NAME is a compile-time constant. - let report_status = sqlx::query(&format!( - r#" + // Contract on `record_in_disable_tx`: one transaction so the row + // lock spans both writes. + let mut history_err = None; + let report_status = async { + let mut tx = db.begin().await?; + // SAFETY: Self::TABLE_NAME is a compile-time constant. + let rows = sqlx::query(&format!( + r#" UPDATE {} SET @@ -398,18 +403,60 @@ pub trait Listener: TriggerCrud + TriggerJobArgs { last_server_ping = NULL WHERE workspace_id = $2 AND - path = $3 + path = $3 AND + mode <> 'disabled'::TRIGGER_MODE "#, - Self::TABLE_NAME - )) - .bind(&error) - .bind(&listening_trigger.workspace_id) - .bind(&listening_trigger.path) - .execute(db) + Self::TABLE_NAME + )) + .bind(&error) + .bind(&listening_trigger.workspace_id) + .bind(&listening_trigger.path) + .execute(&mut *tx) + .await? + .rows_affected(); + + // Zero rows: deleted, or a user disabled it first — no + // transition of ours to record. + if rows > 0 { + // `to_key`, not `Display`: it is what lines up with the + // `TRIGGER_TYPE` the API records under. + let trigger_kind = Self::TRIGGER_KIND.to_key(); + history_err = windmill_common::trigger_history::record_in_disable_tx( + &mut tx, + windmill_common::trigger_history::TriggerHistoryEvent::server_disable( + &listening_trigger.workspace_id, + &trigger_kind, + &listening_trigger.path, + serde_json::json!({ "mode": { "new": "disabled" } }), + &error, + ), + ) + .await; + } + tx.commit().await?; + Ok::<(), Error>(()) + } .await; + if let Some(history_err) = history_err { + // Spawned: the commit above made the cleared `server_id` visible, + // so the ping branch of the enclosing `select!` is about to + // finish and drop everything left in this future. Awaiting the + // alert here would lose the one signal that the row is missing. + let message = format!( + "Disabled {} trigger {} but could not record it in the trigger history: {}", + Self::TRIGGER_KIND, + listening_trigger.path, + history_err + ); + let (db, workspace_id) = (db.clone(), listening_trigger.workspace_id.clone()); + tokio::spawn(async move { + report_critical_error(message, db, Some(&workspace_id), None).await; + }); + } + match report_status { - Ok(_) => { + Ok(()) => { report_critical_error( format!( "Disabling {} trigger {} because of error: {}", diff --git a/backend/windmill-worker/loader.py b/backend/windmill-worker/loader.py index d3dc7b8a66..ab10c36b86 100644 --- a/backend/windmill-worker/loader.py +++ b/backend/windmill-worker/loader.py @@ -2,6 +2,7 @@ import sys import os from importlib.abc import MetaPathFinder, Loader from importlib.machinery import ModuleSpec, SourceFileLoader +from importlib.util import spec_from_file_location import time # Injected by backend: maps script path -> temp storage hash so preview jobs @@ -38,7 +39,7 @@ class WindmillFinder(MetaPathFinder): fullpath = folder + "/" + splitted[-1] + ".py" if os.path.exists(fullpath): - return ModuleSpec(name, SourceFileLoader(name, fullpath)) + return spec_from_file_location(name, fullpath) import urllib.parse @@ -70,7 +71,7 @@ class WindmillFinder(MetaPathFinder): return ModuleSpec(name, WindmillLoader(name)) with open(fullpath, "w+") as f: f.write(r) - return ModuleSpec(name, SourceFileLoader(name, fullpath)) + return spec_from_file_location(name, fullpath) except urllib.error.HTTPError as e: duration = time.time() - req_start if e.code != 404: diff --git a/backend/windmill-worker/src/ansible_executor.rs b/backend/windmill-worker/src/ansible_executor.rs index 05bcf930fb..783f472922 100644 --- a/backend/windmill-worker/src/ansible_executor.rs +++ b/backend/windmill-worker/src/ansible_executor.rs @@ -13,7 +13,7 @@ use tokio::process::Command; use uuid::Uuid; use windmill_common::{ error, - git_sync_oss::{prepend_token_to_github_url, sanitize_git_url}, + git_sync_oss::{prepend_token_to_github_url, sanitize_git_url, validate_git_repo_url}, worker::{ is_allowed_file_location, split_python_requirements, to_raw_value, write_file, write_file_at_user_defined_location, Connection, PyVAlias, WORKER_CONFIG, @@ -247,6 +247,24 @@ async fn prepare_socket_root(root: &str, stale_after: std::time::Duration) { } } +/// Validate every user-controlled field of a `GitRepo` before it reaches `git`. The `url` goes +/// through the transport allowlist (`validate_git_repo_url`); `branch` and `commit` are passed as +/// positional/option arguments, so a leading `-` would let git parse them as options (argument +/// injection). Called at each entry point that spawns `git` with these fields. +fn validate_git_repo(repo: &GitRepo) -> error::Result<()> { + validate_git_repo_url(&repo.url)?; + for (field, value) in [("branch", &repo.branch), ("commit", &repo.commit)] { + if let Some(value) = value { + if value.trim_start().starts_with('-') { + return Err(error::Error::BadRequest(format!( + "Invalid git repository `{field}`: must not start with '-'" + ))); + } + } + } + Ok(()) +} + async fn clone_repo( repo: &GitRepo, job_dir: &str, @@ -259,6 +277,7 @@ async fn clone_repo( occupancy_metrics: &mut OccupancyMetrics, git_ssh_cmd: &str, ) -> error::Result { + validate_git_repo(repo)?; let target_path = is_allowed_file_location(job_dir, &repo.target_path)?; let mut clone_cmd = Command::new(GIT_PATH.as_str()); @@ -400,6 +419,7 @@ async fn clone_repo_without_history( occupancy_metrics: &mut OccupancyMetrics, git_ssh_cmd: &str, ) -> error::Result<()> { + validate_git_repo(repo)?; let target_path = is_allowed_file_location(job_dir, &repo.target_path)?; create_empty_dir(&target_path)?; @@ -926,6 +946,7 @@ pub async fn get_git_repo_full_head_commit_hash( repo: &GitRepo, git_ssh_cmd: &str, ) -> anyhow::Result { + validate_git_repo(repo)?; let mut git_cmd = Command::new(GIT_PATH.as_str()); git_cmd diff --git a/backend/windmill-worker/src/python_versions.rs b/backend/windmill-worker/src/python_versions.rs index 11f673e8f4..11674efcd6 100644 --- a/backend/windmill-worker/src/python_versions.rs +++ b/backend/windmill-worker/src/python_versions.rs @@ -3,10 +3,12 @@ use std::{ process::Stdio, str::FromStr, sync::Arc, + time::UNIX_EPOCH, }; use chrono::{DateTime, Duration, Utc}; use itertools::Itertools; +use serde::{Deserialize, Serialize}; use serde_json::Value; use tokio::{fs::DirBuilder, process::Command, sync::RwLock}; use uuid::Uuid; @@ -465,7 +467,7 @@ impl PyV { w_id: &str, occupancy_metrics: &mut Option<&mut OccupancyMetrics>, ) -> error::Result> { - let py_path = self.find_python().await; + let py_path = self.find_python_cached().await; // Runtime is not installed if let Err(py_err) = py_path { @@ -480,7 +482,7 @@ impl PyV { return Err(err); } else { // Try to find one more time - let py_path = self.find_python().await; + let py_path = self.find_python_cached().await; if let Err(err) = py_path { tracing::error!( @@ -489,7 +491,6 @@ impl PyV { return Err(err); } - // TODO: Cache the result py_path } } else { @@ -600,7 +601,37 @@ impl PyV { .await?; Ok(()) } + /// Same as [`Self::find_python`] but backed by [`PY_PATH_CACHE_DIR`], which outlives the + /// worker process. The subprocess is only spawned when there is nothing usable on disk. + async fn find_python_cached(&self) -> error::Result> { + // Keyed on the requested version, not on the resolved patch: uv answers a minor-only + // request with its own minor-version link, which it re-points when a newer patch is + // installed, so an entry follows patch upgrades without being invalidated. + let version = self.to_string(); + // Without an identity for uv an upgrade would go unnoticed, so the cache is skipped. + let uv = uv_identity().await; + + if let Some(ref uv) = uv { + if let Some(py_path) = read_cached_python_path(&PY_PATH_CACHE_DIR, uv, &version).await { + // Serving a path that no longer exists is far worse than the spawn it saves, so + // the interpreter is checked instead of trusted (the install dir may have been + // wiped, or uv may have moved it). + if tokio::fs::try_exists(&py_path).await.unwrap_or(false) { + return Ok(Some(py_path)); + } + } + } + + let py_path = self.find_python().await; + if let (Some(uv), Ok(Some(py_path))) = (&uv, &py_path) { + write_cached_python_path(&PY_PATH_CACHE_DIR, uv, &version, py_path).await; + } + py_path + } + async fn find_python(&self) -> error::Result> { + tracing::debug!("Resolving python {} with uv python find", self.to_string()); + #[cfg(windows)] let uv_cmd = "uv"; @@ -670,6 +701,81 @@ impl PyV { } } +lazy_static::lazy_static! { + /// Sits next to `PY_INSTALL_DIR` rather than inside it, so uv never sees these entries while + /// scanning that directory for managed interpreters. + static ref PY_PATH_CACHE_DIR: String = format!("{}_paths", *PY_INSTALL_DIR); +} + +#[cfg(windows)] +lazy_static::lazy_static! { + /// uv is invoked as a bare `uv` on windows, hence resolved through PATH, which + /// [`tokio::fs::metadata`] does not search. PATH does not change under us, so the lookup is + /// done once. + static ref UV_PATH: Option = std::env::split_paths(PATH_ENV.as_str()) + .map(|dir| dir.join("uv.exe")) + .find(|path| path.is_file()) + .map(|path| path.to_string_lossy().into_owned()); +} + +/// Interpreter path resolved by `uv python find` for one requested version. +#[derive(Serialize, Deserialize)] +struct CachedPythonPath { + /// Identity of the uv that resolved `path`. An upgraded uv may pick a different interpreter + /// for the same request, so an entry left by another uv is ignored. + uv: String, + path: String, +} + +/// `None` disables the cache: an upgrade of a uv we cannot stat would go unnoticed. +async fn uv_identity() -> Option { + #[cfg(unix)] + let uv_cmd = UV_PATH.clone(); + + // Initializing the static probes PATH synchronously, which must not happen on the runtime. + #[cfg(windows)] + let uv_cmd = tokio::task::spawn_blocking(|| UV_PATH.clone()) + .await + .ok() + .flatten()?; + + let metadata = tokio::fs::metadata(&uv_cmd).await.ok()?; + let mtime = metadata.modified().ok()?.duration_since(UNIX_EPOCH).ok()?; + Some(format!("{uv_cmd}:{}:{}", metadata.len(), mtime.as_secs())) +} + +/// One file per version, so that workers resolving different versions concurrently cannot drop +/// each other's entry the way a shared map would. +fn cached_python_path_file(dir: &str, version: &str) -> String { + format!("{dir}/{version}.json") +} + +async fn read_cached_python_path(dir: &str, uv: &str, version: &str) -> Option { + let content = tokio::fs::read(cached_python_path_file(dir, version)) + .await + .ok()?; + let cached = serde_json::from_slice::(&content).ok()?; + (cached.uv == uv).then_some(cached.path) +} + +async fn write_cached_python_path(dir: &str, uv: &str, version: &str, py_path: &str) { + let cached = CachedPythonPath { uv: uv.to_owned(), path: py_path.to_owned() }; + + // Written aside and renamed so that a concurrent worker never reads a half-written entry. + let tmp_file = format!("{dir}/{}.tmp", Uuid::new_v4()); + let write = async { + tokio::fs::create_dir_all(dir).await?; + tokio::fs::write(&tmp_file, serde_json::to_vec(&cached)?).await?; + tokio::fs::rename(&tmp_file, cached_python_path_file(dir, version)).await?; + Ok::<_, anyhow::Error>(()) + }; + + if let Err(e) = write.await { + tracing::warn!("Could not cache resolved python path ({py_path}): {e}"); + let _ = tokio::fs::remove_file(&tmp_file).await; + } +} + #[cfg(test)] mod tests { use super::*; @@ -965,4 +1071,25 @@ mod tests { ) .await; } + + #[tokio::test] + async fn test_cached_python_path_is_scoped_to_uv() { + let dir = std::env::temp_dir() + .join(format!("wm_py_path_cache_{}", Uuid::new_v4())) + .to_string_lossy() + .into_owned(); + + write_cached_python_path(&dir, "uv-a", "3.12", "/py/3.12/bin/python3.12").await; + assert_eq!( + read_cached_python_path(&dir, "uv-a", "3.12") + .await + .as_deref(), + Some("/py/3.12/bin/python3.12") + ); + // An upgraded uv may pick a different interpreter, so its entries cannot be reused + assert_eq!(read_cached_python_path(&dir, "uv-b", "3.12").await, None); + assert_eq!(read_cached_python_path(&dir, "uv-a", "3.13").await, None); + + tokio::fs::remove_dir_all(&dir).await.unwrap(); + } } diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index eb35b38d4c..090d37bcab 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -41,6 +41,7 @@ use windmill_common::{ agent_workers::DECODED_AGENT_TOKEN, apps::AppScriptId, cache::{future::FutureCachedExt, ScriptData, ScriptMetadata}, + external_ip::cached_ip, schema::{should_validate_schema, SchemaValidator}, utils::{create_directory_async, WarnAfterExt}, worker::{ @@ -261,6 +262,7 @@ const NUM_SECS_READINGS: u64 = 60; const INCLUDE_DEPS_PY_SH_CONTENT: &str = include_str!("../nsjail/download_deps.py.sh"); const WORKER_SHELL_NAP_TIME_DURATION: u64 = 15; +const WORKER_SHELL_INITIAL_NAP_TIME_DURATION: u64 = 5; const TIMEOUT_TO_RESET_WORKER_SHELL_NAP_TIME_DURATION: u64 = 2 * 60; pub const DEFAULT_SLEEP_QUEUE: u64 = 50; @@ -2201,6 +2203,71 @@ pub async fn handle_all_job_kind_error( } } +/// How long the interactive shell loop waits before polling its tag again, when it found no +/// job. The sub-second cadence only pays off while somebody is typing into the shell, so it +/// is reserved for a session that has run a command: before the first one this process serves +/// there is nothing to keep responsive, only the next session to notice. +/// +/// - a live session, last command under `TIMEOUT_TO_RESET_WORKER_SHELL_NAP_TIME_DURATION` +/// ago: `sleep_queue() * 10` +/// - no command yet this process: `WORKER_SHELL_INITIAL_NAP_TIME_DURATION`, which bounds how +/// long the first command of a session waits +/// - nothing for `TIMEOUT_TO_RESET_WORKER_SHELL_NAP_TIME_DURATION`, counted from the last +/// command or from process start: `WORKER_SHELL_NAP_TIME_DURATION` +/// +/// A worker whose process is recycled after N jobs cannot count on living long enough to +/// reach that last state, and at N=1 never does, so it starts there instead. That holds for +/// any N, since N says nothing about how long a process lasts. +fn interactive_shell_nap( + now: Instant, + started_at: Instant, + last_executed_job: Option, + recycles_after_n_jobs: bool, +) -> Duration { + let quiet_since = last_executed_job.unwrap_or(started_at); + if now.duration_since(quiet_since).as_secs() > TIMEOUT_TO_RESET_WORKER_SHELL_NAP_TIME_DURATION { + return Duration::from_secs(WORKER_SHELL_NAP_TIME_DURATION); + } + match last_executed_job { + Some(_) => Duration::from_millis(sleep_queue() * 10), + None if recycles_after_n_jobs => Duration::from_secs(WORKER_SHELL_NAP_TIME_DURATION), + None => Duration::from_secs(WORKER_SHELL_INITIAL_NAP_TIME_DURATION), + } +} + +#[cfg(test)] +mod interactive_shell_nap_tests { + use super::*; + + const LONG: Duration = Duration::from_secs(WORKER_SHELL_NAP_TIME_DURATION); + const INITIAL: Duration = Duration::from_secs(WORKER_SHELL_INITIAL_NAP_TIME_DURATION); + + #[test] + fn only_a_live_shell_session_gets_the_sub_second_cadence() { + let start = Instant::now(); + let quiet = + start + Duration::from_secs(TIMEOUT_TO_RESET_WORKER_SHELL_NAP_TIME_DURATION + 1); + let fast = Duration::from_millis(sleep_queue() * 10); + // Nobody has opened a shell on this worker yet, so there is no session to keep + // responsive: only the first command of the next one to notice. + assert_eq!(interactive_shell_nap(start, start, None, false), INITIAL); + assert_eq!(interactive_shell_nap(quiet, start, None, false), LONG); + // A worker recycled after N jobs may never live to back off, so it starts backed off. + assert_eq!(interactive_shell_nap(start, start, None, true), LONG); + // Either way, a served shell job is a live session and gets the fast cadence. + assert_eq!(interactive_shell_nap(quiet, start, Some(quiet), true), fast); + assert_eq!( + interactive_shell_nap( + quiet + Duration::from_secs(TIMEOUT_TO_RESET_WORKER_SHELL_NAP_TIME_DURATION + 1), + start, + Some(quiet), + true + ), + LONG + ); + } +} + fn start_interactive_worker_shell( conn: Connection, hostname: String, @@ -2211,10 +2278,12 @@ fn start_interactive_worker_shell( worker_dir: String, ) -> JoinHandle<()> { tokio::spawn(async move { - let mut occupancy_metrics = OccupancyMetrics::new(Instant::now()); + let started_at = Instant::now(); + let mut occupancy_metrics = OccupancyMetrics::new(started_at); - let mut last_executed_job: Option = - Instant::now().checked_sub(Duration::from_millis(2500)); + // `None` means no shell job has been served yet, which the nap distinguishes from a + // shell session that has gone quiet. + let mut last_executed_job: Option = None; loop { if let Ok(_) = killpill_rx.try_recv() { @@ -2329,16 +2398,12 @@ fn start_interactive_worker_shell( last_executed_job = Some(Instant::now()); } Ok(None) => { - let now = Instant::now(); - let nap_time = match last_executed_job { - Some(last) - if now.duration_since(last).as_secs() - > TIMEOUT_TO_RESET_WORKER_SHELL_NAP_TIME_DURATION => - { - Duration::from_secs(WORKER_SHELL_NAP_TIME_DURATION) - } - _ => Duration::from_millis(sleep_queue() * 10), - }; + let nap_time = interactive_shell_nap( + Instant::now(), + started_at, + last_executed_job, + EXIT_AFTER_N_JOBS.is_some(), + ); tokio::select! { _ = tokio::time::sleep(nap_time) => { } @@ -2487,7 +2552,6 @@ pub async fn run_worker( worker_name: String, i_worker: u64, num_workers: u32, - ip: &str, mut killpill_rx: tokio::sync::broadcast::Receiver<()>, killpill_tx: KillpillSender, base_internal_url: &str, @@ -2583,7 +2647,8 @@ pub async fn run_worker( let mut last_ping = Instant::now() - Duration::from_secs(NUM_SECS_PING + 1); - let previous_jobs_executed = insert_ping(hostname, &worker_name, ip, conn) + let mut reported_ip = cached_ip(); + let previous_jobs_executed = insert_ping(hostname, &worker_name, reported_ip, conn) .await .expect("initial ping could be sent"); @@ -2799,7 +2864,7 @@ pub async fn run_worker( .is_some_and(|dws| !dws.is_empty()) }; - if EXIT_AFTER_N_JOBS.is_some() && i_worker == 1 { + if let Some(max_jobs) = (*EXIT_AFTER_N_JOBS).filter(|_| i_worker == 1) { if num_workers > 1 { tracing::warn!( worker = %worker_name, hostname = %hostname, @@ -2816,6 +2881,27 @@ pub async fn run_worker( workers: those run outside its main loop and are never counted." ); } + let config = WORKER_CONFIG.load(); + if config.init_bash.is_some() { + tracing::warn!( + worker = %worker_name, hostname = %hostname, + "EXIT_AFTER_N_JOBS is set and this worker group has an init script: the init \ + script prepares the environment the limit recycles, so it is not counted and runs \ + again on every restart. Every {max_jobs} job(s) therefore pushes and executes an \ + init job of its own first, and waits for it." + ); + } + // No interval check: loading a worker config whose periodic script has no interval, or + // one below MIN_PERIODIC_SCRIPT_INTERVAL_SECONDS, fails and kills the worker, so a + // script that reaches here is one the periodic task runs. + if config.periodic_script_bash.is_some() { + tracing::warn!( + worker = %worker_name, hostname = %hostname, + "EXIT_AFTER_N_JOBS is set and this worker group has a periodic script: it runs \ + once when the worker starts, so it runs every {max_jobs} job(s) whatever its \ + interval says." + ); + } } #[cfg(feature = "benchmark")] @@ -3066,7 +3152,26 @@ pub async fn run_worker( otel_set_worker_uptime(&worker_name, start_time.elapsed().as_secs_f64()); - if last_ping.elapsed().as_secs() > NUM_SECS_PING { + // The external IP resolves in the background, after the initial ping. Pinging on the very + // next iteration rather than the next periodic one is what gets it into the row of a worker + // whose process is short-lived (EXIT_AFTER_N_JOBS). + let ip = cached_ip(); + let ip_just_resolved = reported_ip.is_none() && ip.is_some(); + if ip_just_resolved || last_ping.elapsed().as_secs() > NUM_SECS_PING { + // Servers older than the background lookup take an IP from the initial ping only, so an + // agent has to register a second time to deliver whatever the lookup settled on, an + // address or the unretrievable marker. Registering also clears the row's job columns, + // which costs at most the last job's id here: no job of this worker is in flight at this + // point in the loop, and the next one refills them. + if ip_just_resolved && conn.as_sql().is_none() { + if let Err(e) = insert_ping(hostname, &worker_name, ip, &conn).await { + tracing::warn!( + worker = %worker_name, hostname = %hostname, + "failed to re-register with the resolved external IP: {e}" + ); + } + } + let read_cgroups = *REFRESH_CGROUP_READINGS && last_reading.elapsed().as_secs() > NUM_SECS_READINGS; update_worker_ping_full( @@ -3077,6 +3182,7 @@ pub async fn run_worker( &hostname, &mut occupancy_metrics, &killpill_tx, + ip, ) .await; @@ -3084,6 +3190,7 @@ pub async fn run_worker( last_reading = Instant::now(); } last_ping = Instant::now(); + reported_ip = ip; } if (jobs_executed as u32 + vacuum_shift) % VACUUM_PERIOD == 0 { diff --git a/backend/windmill-worker/src/worker_flow.rs b/backend/windmill-worker/src/worker_flow.rs index 7066b35945..42246c6358 100644 --- a/backend/windmill-worker/src/worker_flow.rs +++ b/backend/windmill-worker/src/worker_flow.rs @@ -2923,15 +2923,39 @@ pub async fn handle_flow( // its own disable write failed. Retry it: rearm_schedule turns // these into NoOp, so without disabling here the schedule would // stay enabled yet never run. - if let Err(disable_err) = sqlx::query!( - "UPDATE schedule SET enabled = false, error = $1 WHERE workspace_id = $2 AND path = $3", - err.to_string(), - &flow_job.workspace_id, - &schedule.path - ) - .execute(db) - .await - { + // Contract on `record_in_disable_tx`. Worth knowing here: + // this is the last chance to disable, because a flow + // schedule arms its next occurrence when the flow *starts*, + // so once the flow is gone nothing reaches this code again. + let mut history_lost = None; + let disable_result = async { + let mut tx = db.begin().await?; + let rows = sqlx::query!( + "UPDATE schedule SET enabled = false, error = $1 WHERE workspace_id = $2 AND path = $3 AND enabled = true", + err.to_string(), + &flow_job.workspace_id, + &schedule.path + ) + .execute(&mut *tx) + .await? + .rows_affected(); + if rows > 0 { + history_lost = windmill_common::trigger_history::record_in_disable_tx( + &mut tx, + windmill_queue::jobs::schedule_auto_disable_event( + &flow_job.workspace_id, + &schedule.path, + &err.to_string(), + ), + ) + .await; + } + tx.commit().await?; + Ok::<(), Error>(()) + } + .await; + + if let Err(disable_err) = disable_result { report_error_to_workspace_handler_or_critical_side_channel( &mini_job, db, @@ -2942,6 +2966,17 @@ pub async fn handle_flow( ) .await; } + if let Some(history_err) = history_lost { + report_error_to_workspace_handler_or_critical_side_channel( + &mini_job, + db, + format!( + "Disabled schedule {} but could not record it in the trigger history: {history_err}", + schedule.path, + ), + ) + .await; + } } else { // Transient error (DB contention, timeout) after retry exhaustion: // not the schedule's fault. Report it but leave the schedule diff --git a/backend/windmill-worker/src/worker_utils.rs b/backend/windmill-worker/src/worker_utils.rs index 6f47b2d863..67e9831481 100644 --- a/backend/windmill-worker/src/worker_utils.rs +++ b/backend/windmill-worker/src/worker_utils.rs @@ -4,6 +4,7 @@ use uuid::Uuid; use windmill_common::{ agent_workers::{PingJobStatus, PingJobStatusResponse}, cache, + external_ip::UNKNOWN_IP, worker::{ get_memory, get_vcpus, get_windmill_memory_usage, get_worker_memory_usage, insert_ping_query, update_job_ping_query, update_worker_ping_from_job_query, @@ -26,6 +27,7 @@ pub(crate) async fn update_worker_ping_full( hostname: &str, occupancy_metrics: &mut OccupancyMetrics, killpill_tx: &KillpillSender, + ip: Option<&str>, ) { let wc = WORKER_CONFIG.load(); let tags = wc.worker_tags.clone(); @@ -64,6 +66,7 @@ pub(crate) async fn update_worker_ping_full( occupancy_rate_5m, occupancy_rate_30m, native_mode, + ip, ) }) .retry( @@ -110,6 +113,7 @@ async fn update_worker_ping_full_inner( occupancy_rate_5m: Option, occupancy_rate_30m: Option, native_mode: bool, + ip: Option<&str>, ) -> anyhow::Result<()> { match conn { Connection::Sql(db) => { @@ -126,6 +130,7 @@ async fn update_worker_ping_full_inner( occupancy_rate_5m, occupancy_rate_30m, native_mode, + ip, db, ) .await?; @@ -139,7 +144,7 @@ async fn update_worker_ping_full_inner( last_job_executed: None, last_job_workspace_id: None, worker_instance: None, - ip: None, + ip: ip.map(str::to_string), tags: Some(tags.to_vec()), dw: None, dws: None, @@ -169,7 +174,7 @@ async fn update_worker_ping_full_inner( pub async fn insert_ping( worker_instance: &str, worker_name: &str, - ip: &str, + ip: Option<&str>, db: &Connection, ) -> anyhow::Result { let (tags, dw, dws, native_mode) = { @@ -228,7 +233,10 @@ pub async fn insert_ping( last_job_executed: None, last_job_workspace_id: None, worker_instance: Some(worker_instance.to_string()), - ip: Some(ip.to_string()), + // Servers older than the background lookup reject an initial ping with + // no IP, and an agent worker routinely runs against one, so the + // not-resolved-yet case goes over the wire as the sentinel. + ip: Some(ip.unwrap_or(UNKNOWN_IP).to_string()), tags: Some(tags.to_vec()), dw: dw, dws: dws, diff --git a/benchmarks/lib.ts b/benchmarks/lib.ts index b24823d3cb..f5dffe015f 100644 --- a/benchmarks/lib.ts +++ b/benchmarks/lib.ts @@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts"; import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts"; import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts"; -export const VERSION = "v1.789.0"; +export const VERSION = "v1.792.2"; export async function login(email: string, password: string): Promise { return await windmill.UserService.login({ diff --git a/cli/bun.lock b/cli/bun.lock index fb536a5eee..cf3a87bf41 100644 --- a/cli/bun.lock +++ b/cli/bun.lock @@ -9,7 +9,7 @@ "@cliffy/command": "npm:@jsr/cliffy__command@1.0.0", "@cliffy/prompt": "npm:@jsr/cliffy__prompt@1.0.0", "@cliffy/table": "npm:@jsr/cliffy__table@1.0.0", - "@windmill-labs/shared-utils": "^1.0.12", + "@windmill-labs/shared-utils": "^1.0.13", "diff": "^5.2.0", "esbuild": "0.28.0", "get-port": "7.1.0", @@ -175,7 +175,7 @@ "@types/ws": ["@types/ws@8.18.1", "", { "dependencies": { "@types/node": "*" } }, "sha512-ThVF6DCVhA8kUGy+aazFQ4kXQ7E1Ty7A3ypFOe0IcJV8O/M511G99AW24irKrW56Wt44yG9+ij8FaqoBGkuBXg=="], - "@windmill-labs/shared-utils": ["@jsr/windmill-labs__shared-utils@1.0.12", "https://npm.jsr.io/~/11/@jsr/windmill-labs__shared-utils/1.0.12.tgz", {}, "sha512-bJOacyfxxNPwNTzA4AxCB5iGFop0h3mCgs+E9j3ZaJYDo1soblY16CebnQ56EPy/M3V344X/QoOFBORyRo1Mnw=="], + "@windmill-labs/shared-utils": ["@windmill-labs/shared-utils@1.0.13", "", {}, "sha512-LB3bFq8i0bS2fwM8EqGRODrR1CXLXXyiNa766By1ZbSJYCmTzK8NVXqcluJP1ne7is80LRVZdOgWTw6YX+61Dg=="], "acorn": ["acorn@8.16.0", "", { "bin": "bin/acorn" }, "sha512-UVJyE9MttOsBQIDKw1skb9nAwQuR5wuGD3+82K6JgJlm/Y+KI92oNsMNGZCYdDsVtRHSak0pcV5Dno5+4jh9sw=="], diff --git a/cli/generate-app-policy.ts b/cli/generate-app-policy.ts new file mode 100644 index 0000000000..76dd35efb2 --- /dev/null +++ b/cli/generate-app-policy.ts @@ -0,0 +1,93 @@ +/** + * Bundles the frontend's raw-app policy derivation into a script the server's + * bundle job carries, and writes it next to that job's source. + * + * The derivation is not re-implemented in Rust, and not shelled out to `wmill` + * either. `triggerables_v2` is the allowlist every component run is matched + * against, keyed by `:rawscript/` — a key + * derived any other way leaves a deployed app's runnables "forbidden by policy", + * which reads as a broken app rather than a failed deploy. So the app editor, + * `wmill app push` and the server all have to agree, and the way to guarantee + * that is one source. + * + * It rides in the job script rather than in the CLI the job already runs, + * because the images install `windmill-cli` unpinned: an image can carry a CLI + * older than its server, and falling back to `bun x windmill-cli@` + * needs npm reachable at deploy time, which is exactly what the bundler's + * installed-CLI branch exists to avoid. The job script is `include_str!`d into + * the server binary, so it always matches the server. + * + * Run `bun run gen:app-policy` after touching + * frontend/src/lib/components/raw_apps/rawAppPolicy.ts or anything it imports; + * test/app_policy_bundle_unit.test.ts fails when the committed bundle no longer + * matches those sources. + */ +import * as esbuild from "esbuild"; +import { readFileSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; + +const REPO_ROOT = join(import.meta.dir, ".."); +const RAW_APPS_DIR = join( + REPO_ROOT, + "frontend", + "src", + "lib", + "components", + "raw_apps", +); +export const OUT_FILE = join( + REPO_ROOT, + "backend", + "windmill-api", + "src", + "apps_raw_policy.gen.js", +); + +const HEADER = `// Generated by cli/generate-app-policy.ts. Do not edit. +// Run \`bun run gen:app-policy\` from cli/ to rebuild it from +// frontend/src/lib/components/raw_apps/rawAppPolicy.ts. +// +// Prepended to the raw-app bundle job (see apps_raw_bundle.rs), which calls +// \`__wmillAppPolicy.updateRawAppPolicy\` from its own module scope. +`; + +/** The bundled IIFE, header included. Exported so the staleness test can build + * it and compare, rather than keep a hash in step by hand. */ +export async function buildAppPolicyBundle(): Promise { + const result = await esbuild.build({ + stdin: { + contents: `export { updateRawAppPolicy } from './rawAppPolicy'`, + resolveDir: RAW_APPS_DIR, + loader: "ts", + }, + bundle: true, + format: "iife", + globalName: "__wmillAppPolicy", + target: "es2022", + // `hash()` falls back to this only when Web Crypto is missing, which it + // never is on a worker's bun. Left unresolved so the bundle needs nothing + // installed where it runs. + external: ["@aws-crypto/sha256-js"], + minify: true, + write: false, + legalComments: "none", + }); + return HEADER + result.outputFiles[0].text; +} + +if (import.meta.main) { + const js = await buildAppPolicyBundle(); + const before = (() => { + try { + return readFileSync(OUT_FILE, "utf-8"); + } catch { + return ""; + } + })(); + writeFileSync(OUT_FILE, js, "utf-8"); + console.log( + `${before === js ? "Unchanged" : "Wrote"} ${OUT_FILE} (${ + Math.round(js.length / 1024) + } KB)`, + ); +} diff --git a/cli/package.json b/cli/package.json index 34544c2abb..03497dc887 100644 --- a/cli/package.json +++ b/cli/package.json @@ -10,6 +10,7 @@ "dev": "bun run src/main.ts", "build": "./build.sh", "gen:dev-recorder": "bun run generate-dev-recorder.ts", + "gen:app-policy": "bun run generate-app-policy.ts", "test": "bun test test/", "test:unit": "UNIT_ONLY=1 bun test test/*_unit*", "check": "bunx tsc --noEmit", @@ -20,7 +21,7 @@ "@cliffy/command": "npm:@jsr/cliffy__command@1.0.0", "@cliffy/prompt": "npm:@jsr/cliffy__prompt@1.0.0", "@cliffy/table": "npm:@jsr/cliffy__table@1.0.0", - "@windmill-labs/shared-utils": "^1.0.12", + "@windmill-labs/shared-utils": "^1.0.13", "diff": "^5.2.0", "esbuild": "0.28.0", "get-port": "7.1.0", diff --git a/cli/src/commands/script/script.ts b/cli/src/commands/script/script.ts index a30c8d70d4..8c93bdb963 100644 --- a/cli/src/commands/script/script.ts +++ b/cli/src/commands/script/script.ts @@ -592,6 +592,7 @@ export async function handleFile( ws_error_handler_muted: typed?.ws_error_handler_muted, dedicated_worker: typed?.dedicated_worker, cache_ttl: typed?.cache_ttl, + cache_ignore_s3_path: typed?.cache_ignore_s3_path, concurrency_time_window_s: normConcurrencyTimeWindowS, concurrent_limit: normConcurrentLimit, deployment_message: message, @@ -602,8 +603,14 @@ export async function handleFile( concurrency_key: typed?.concurrency_key, debounce_key: typed?.debounce_key, debounce_delay_s: typed?.debounce_delay_s, + debounce_args_to_accumulate: typed?.debounce_args_to_accumulate, + max_total_debouncing_time: typed?.max_total_debouncing_time, + max_total_debounces_amount: typed?.max_total_debounces_amount, codebase: await codebase?.getDigest(forceTar), timeout: nonePositiveInt(typed?.timeout), + // 0 means "delete immediately after completion", so it must survive as 0 + // rather than being folded into "unset" the way the positive-only settings are. + delete_after_secs: typed?.delete_after_secs, on_behalf_of_email: typed?.on_behalf_of_email, envs: typed?.envs, modules: modules, @@ -635,6 +642,12 @@ export async function handleFile( (typed.description === remote.description && typed.summary === remote.summary && typed.kind == remote.kind && + // A `.ts` file changes language when defaultTs flips, content untouched. + // bun and bunnative share that extension, so the inferred language is always + // bun; the server derives bunnative back from the `//native` annotation in + // the content, which is compared above. + language == + (remote.language === "bunnative" ? "bun" : remote.language) && !remote.archived && (Array.isArray(remote?.lock) ? remote?.lock?.join("\n") @@ -646,6 +659,8 @@ export async function handleFile( remote.ws_error_handler_muted && typed.dedicated_worker == remote.dedicated_worker && typed.cache_ttl == remote.cache_ttl && + Boolean(typed.cache_ignore_s3_path) == + Boolean(remote.cache_ignore_s3_path) && normConcurrencyTimeWindowS == normalizeConcurrency( remote.concurrent_limit, @@ -659,15 +674,23 @@ export async function handleFile( Boolean(remote.visible_to_runner_only) && Boolean(typed.has_preprocessor) == Boolean(remote.has_preprocessor) && - typed.priority == Boolean(remote.priority) && + typed.priority == remote.priority && nonePositiveInt(typed.timeout) == nonePositiveInt(remote.timeout) && + typed.delete_after_secs == remote.delete_after_secs && //@ts-ignore typed.concurrency_key == remote["concurrency_key"] && typed.debounce_key == remote["debounce_key"] && typed.debounce_delay_s == remote["debounce_delay_s"] && + deepEqual( + typed.debounce_args_to_accumulate ?? null, + remote.debounce_args_to_accumulate ?? null + ) && + typed.max_total_debouncing_time == remote.max_total_debouncing_time && + typed.max_total_debounces_amount == remote.max_total_debounces_amount && typed.codebase == remote.codebase && (hasOnBehalfOf ? true : typed.on_behalf_of_email == remote.on_behalf_of_email) && deepEqual(typed.envs, remote.envs) && + deepEqual(typed.labels ?? null, remote.labels ?? null) && deepEqual(modules ?? null, remote.modules ?? null)) ) { log.info(colors.green(`Script ${remotePath} is up to date`)); diff --git a/cli/src/core/client.ts b/cli/src/core/client.ts index f7073fe501..32776bd498 100644 --- a/cli/src/core/client.ts +++ b/cli/src/core/client.ts @@ -14,6 +14,17 @@ export function markRequestsAsSyncOrigin() { OpenAPI.HEADERS = { ...existing, "X-Windmill-Deploy-Origin": "sync" }; } +/** + * Name this process as the CLI on every subsequent request, so a trigger the + * CLI created or disabled is attributed to `cli` rather than to a bare API + * call in `trigger_history`. Attribution only — nothing on the server grants + * anything on the strength of it. + */ +export function markRequestsAsCliClient() { + const existing = typeof OpenAPI.HEADERS === "object" ? OpenAPI.HEADERS : {}; + OpenAPI.HEADERS = { ...existing, "X-Windmill-Client": "cli" }; +} + export function setClient(token?: string, baseUrl?: string) { if (baseUrl === undefined) { baseUrl = process.env["BASE_INTERNAL_URL"] ?? diff --git a/cli/src/core/constants.ts b/cli/src/core/constants.ts index c2c8423402..11cf3f2430 100644 --- a/cli/src/core/constants.ts +++ b/cli/src/core/constants.ts @@ -10,4 +10,4 @@ export const WM_FORK_PREFIX = "wm-fork"; // (e.g. utils.ts) can read it without importing main.ts and creating a circular // dependency (main → workspace → utils → main) that triggers a TDZ. // Re-exported from main.ts for backwards compatibility. -export const VERSION = "1.789.0"; +export const VERSION = "1.792.2"; diff --git a/cli/src/main.ts b/cli/src/main.ts index ff8c576ee9..ccce20da30 100755 --- a/cli/src/main.ts +++ b/cli/src/main.ts @@ -32,6 +32,7 @@ import { OpenAPI } from "../gen/index.ts"; import { getHeaders } from "./utils/utils.ts"; import { detectAuthGatewayChallenge } from "./utils/http_guards.ts"; import { setShowDiffs } from "./core/conf.ts"; +import { markRequestsAsCliClient } from "./core/client.ts"; import { NpmProvider } from "./utils/upgrade.ts"; import { pull as hubPull } from "./commands/hub/hub.ts"; import { pull, push } from "./commands/sync/sync.ts"; @@ -300,6 +301,7 @@ async function main() { if (extraHeaders) { OpenAPI.HEADERS = extraHeaders; } + markRequestsAsCliClient(); OpenAPI.interceptors.response.use(async (response) => { await detectAuthGatewayChallenge(response); return response; diff --git a/cli/src/utils/resource_types.ts b/cli/src/utils/resource_types.ts index 5bd56c9a8e..8317ee7ab5 100644 --- a/cli/src/utils/resource_types.ts +++ b/cli/src/utils/resource_types.ts @@ -4,18 +4,29 @@ function quotePropName(name: string): string { return /^[a-zA-Z_$][a-zA-Z0-9_$]*$/.test(name) ? name : JSON.stringify(name); } -export function compileResourceTypeToTsType(schema: Schema) { - function rec(x: { [name: string]: SchemaProperty }, root = false) { - let res = "{\n"; +function isPropertyMap(x: unknown): x is { [name: string]: SchemaProperty } { + return typeof x === "object" && x !== null && !Array.isArray(x); +} + +// Schemas are free-form jsonb: the column is nullable and hub types such as +// `record` or `dbt_profile` carry `{}` / `{"type":"object"}` with no +// `properties`. Anything that is not a property map compiles to `any`, since a +// throw here aborts the whole rt.d.ts generation. +export function compileResourceTypeToTsType(schema: Schema | undefined | null) { + function rec(x: unknown): string { + if (!isPropertyMap(x)) { + return "any"; + } const entries = Object.entries(x); if (entries.length == 0) { return "any"; } + let res = "{\n"; let i = 0; for (let [name, prop] of entries) { - if (prop.type == "object") { - res += ` ${quotePropName(name)}: ${rec(prop.properties ?? {})}`; - } else if (prop.type == "array") { + if (prop?.type == "object") { + res += ` ${quotePropName(name)}: ${rec(prop.properties)}`; + } else if (prop?.type == "array") { res += ` ${quotePropName(name)}: ${prop?.items?.type ?? "any"}[]`; } else { let typ = prop?.type ?? "any"; @@ -33,5 +44,5 @@ export function compileResourceTypeToTsType(schema: Schema) { return res; } - return rec(schema.properties, true); + return rec(schema?.properties); } diff --git a/cli/test/app_policy_bundle_unit.test.ts b/cli/test/app_policy_bundle_unit.test.ts new file mode 100644 index 0000000000..4da05462ad --- /dev/null +++ b/cli/test/app_policy_bundle_unit.test.ts @@ -0,0 +1,69 @@ +/** + * The raw-app bundle job carries the frontend's policy derivation, vendored by + * cli/generate-app-policy.ts into backend/windmill-api/src/apps_raw_policy.gen.js + * and prepended to the job script. + * + * If that copy drifts from the frontend source, deployed apps get policy keys + * the app editor would not have written, and every runnable is refused at run + * time with "forbidden by policy" — an app that deploys and then does nothing. + * So rebuild the bundle here and fail when the committed one no longer matches. + * Fix by running `bun run gen:app-policy` from cli/. + * + * No backend required. + */ + +import { expect, test, describe } from "bun:test"; +import { readFileSync } from "node:fs"; +import { buildAppPolicyBundle, OUT_FILE } from "../generate-app-policy.ts"; + +describe("raw app policy bundle", () => { + test("the committed bundle matches the frontend source", async () => { + // Line endings normalized: a CRLF checkout is the same bundle, and must not + // read as drift (the committed file's header arrives as CRLF on Windows). + const lf = (s: string) => s.replace(/\r\n/g, "\n"); + expect(lf(readFileSync(OUT_FILE, "utf-8"))).toBe( + lf(await buildAppPolicyBundle()), + ); + }); + + test("derives the keys the app editor writes", async () => { + // Exercise the committed artifact itself, not the frontend module: it is + // what actually runs on the worker. + // A module's top-level `var` is not a global, and the job prepends this + // bundle into its own module, so reach the binding the same way it does. + const { updateRawAppPolicy } = new Function( + `${readFileSync(OUT_FILE, "utf-8")}\nreturn __wmillAppPolicy`, + )(); + + const content = "export async function main(a: string) { return a }\n"; + const sha = new Bun.CryptoHasher("sha256").update(content).digest("hex"); + + const policy = await updateRawAppPolicy( + { + inline: { + type: "inline", + inlineScript: { content, language: "bun" }, + fields: { + pinned: { type: "static", value: "by-the-publisher" }, + secret: { type: "static", value: "shh", sensitive: true }, + }, + }, + by_flow: { type: "path", runType: "flow", path: "u/admin/f", fields: {} }, + }, + undefined, + ); + + expect(Object.keys(policy.triggerables_v2).sort()).toEqual([ + "by_flow:flow/u/admin/f", + `inline:rawscript/${sha}`, + ]); + // `sensitive_inputs` is what makes the server encrypt the arg before it + // reaches the job, so losing it would silently store the value in plaintext. + const inline = policy.triggerables_v2[`inline:rawscript/${sha}`]; + expect(inline.static_inputs).toEqual({ + pinned: "by-the-publisher", + secret: "shh", + }); + expect(inline.sensitive_inputs).toEqual(["secret"]); + }); +}); diff --git a/cli/test/resource_types_unit.test.ts b/cli/test/resource_types_unit.test.ts index 0966ae25bf..c8bebb1baa 100644 --- a/cli/test/resource_types_unit.test.ts +++ b/cli/test/resource_types_unit.test.ts @@ -52,6 +52,26 @@ test("non-identifier property names are double-quoted", () => { expect(out).toContain(' "3leading": boolean'); }); +// WIN-2392: hub types like `record` (schema `{}`) or `dbt_profile` +// (`{"type":"object"}`) have no `properties`, and the schema column itself is +// nullable. A type whose property map is missing must still compile, otherwise +// it aborts the whole rt.d.ts generation. +test("schemas without a usable property map compile to any", () => { + expect(compileResourceTypeToTsType(undefined)).toBe("any"); + expect(compileResourceTypeToTsType(null)).toBe("any"); + expect(compileResourceTypeToTsType({ type: "object" } as any)).toBe("any"); + expect(compileResourceTypeToTsType({ properties: null } as any)).toBe("any"); + expect(compileResourceTypeToTsType(schema({}))).toBe("any"); +}); + +test("a null property compiles to any instead of throwing", () => { + const out = compileResourceTypeToTsType( + schema({ host: null, port: { type: "integer" } } as any) + ); + expect(out).toContain(" host: any"); + expect(out).toContain(" port: number"); +}); + test("nested object and array property names are quoted too", () => { const out = compileResourceTypeToTsType( schema({ diff --git a/cli/test/script_push_up_to_date.test.ts b/cli/test/script_push_up_to_date.test.ts new file mode 100644 index 0000000000..e4a3e91268 --- /dev/null +++ b/cli/test/script_push_up_to_date.test.ts @@ -0,0 +1,145 @@ +/** + * `wmill script push` short-circuits when the local script already matches the + * remote. The comparison has to hold in both directions: an untouched script + * deploys nothing, and every field the push body carries (labels and the language + * inferred from defaultTs included) still counts as a change. + */ + +import { expect, test } from "bun:test"; +import { writeFile, readFile, mkdir } from "node:fs/promises"; +import { withTestBackend } from "./test_backend.ts"; +import { waitForDeploymentJobs } from "./new_commands_helpers.ts"; + +test("Integration: script push skips an unchanged script and deploys a changed one", async () => { + await withTestBackend(async (backend, tempDir) => { + const uniqueId = Date.now(); + const scriptPath = `f/test/uptodate_${uniqueId}`; + const getScript = async () => + await ( + await backend.apiRequest!( + `/api/w/${backend.workspace}/scripts/get/p/${scriptPath}`, + ) + ).json(); + const push = async () => + await backend.runCLICommand(["script", "push", `${scriptPath}.ts`], tempDir); + const wmillYaml = (defaultTs: string) => + `defaultTs: ${defaultTs}\nincludes:\n - "${scriptPath}**"\nexcludes: []\n`; + + await mkdir(`${tempDir}/f/test`, { recursive: true }); + await backend.apiRequest!(`/api/w/${backend.workspace}/folders/create`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ name: "test" }), + }); + const createResp = await backend.apiRequest!( + `/api/w/${backend.workspace}/scripts/create`, + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + path: scriptPath, + content: `export async function main() {\n return "Hello world";\n}`, + summary: "Test up to date", + description: "", + language: "bun", + kind: "script", + schema: { + $schema: "https://json-schema.org/draft/2020-12/schema", + type: "object", + properties: {}, + required: [], + }, + labels: ["l1"], + }), + }, + ); + expect(createResp.ok).toEqual(true); + + await writeFile(`${tempDir}/wmill.yaml`, wmillYaml("bun"), "utf-8"); + // The lock a deploy's dependency job writes is part of the comparison, so every + // pull has to happen after that job has landed or the skip races it. + await waitForDeploymentJobs(backend); + expect((await backend.runCLICommand(["sync", "pull", "--yes"], tempDir)).code).toEqual(0); + + const hashBefore = (await getScript()).hash; + expect((await push()).stdout).toContain("is up to date"); + expect((await getScript()).hash).toEqual(hashBefore); + + const metadataPath = `${tempDir}/${scriptPath}.script.yaml`; + await writeFile( + metadataPath, + (await readFile(metadataPath, "utf-8")).replace("- l1", "- l2"), + "utf-8", + ); + expect((await push()).stdout).not.toContain("is up to date"); + expect((await getScript()).labels).toEqual(["l2"]); + + // 2, not 0 or 1: those two are the values a truthiness comparison would also + // call equal, so they cannot pin that priority is compared by value. + await waitForDeploymentJobs(backend); + expect((await backend.runCLICommand(["sync", "pull", "--yes"], tempDir)).code).toEqual(0); + await writeFile(metadataPath, (await readFile(metadataPath, "utf-8")) + "priority: 2\n", "utf-8"); + expect((await push()).stdout).not.toContain("is up to date"); + expect((await getScript()).priority).toEqual(2); + await waitForDeploymentJobs(backend); + expect((await backend.runCLICommand(["sync", "pull", "--yes"], tempDir)).code).toEqual(0); + expect((await push()).stdout).toContain("is up to date"); + + await writeFile(`${tempDir}/wmill.yaml`, wmillYaml("deno"), "utf-8"); + expect((await push()).stdout).not.toContain("is up to date"); + expect((await getScript()).language).toEqual("deno"); + }); +}); + +test("Integration: an unchanged bunnative script is not redeployed", async () => { + await withTestBackend(async (backend, tempDir) => { + const uniqueId = Date.now(); + const scriptPath = `f/test/native_${uniqueId}`; + + await mkdir(`${tempDir}/f/test`, { recursive: true }); + await backend.apiRequest!(`/api/w/${backend.workspace}/folders/create`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ name: "test" }), + }); + const createResp = await backend.apiRequest!( + `/api/w/${backend.workspace}/scripts/create`, + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + path: scriptPath, + // The server stores this as bunnative: the language is derived from the + // annotation, and no file extension can express it back. + content: `//native\nexport async function main() {\n return "Hello world";\n}`, + summary: "Test bunnative", + description: "", + language: "bun", + kind: "script", + }), + }, + ); + expect(createResp.ok).toEqual(true); + + await writeFile( + `${tempDir}/wmill.yaml`, + `defaultTs: bun\nincludes:\n - "${scriptPath}**"\nexcludes: []\n`, + "utf-8", + ); + await waitForDeploymentJobs(backend); + expect((await backend.runCLICommand(["sync", "pull", "--yes"], tempDir)).code).toEqual(0); + + const remote = await ( + await backend.apiRequest!( + `/api/w/${backend.workspace}/scripts/get/p/${scriptPath}`, + ) + ).json(); + expect(remote.language).toEqual("bunnative"); + + const result = await backend.runCLICommand( + ["script", "push", `${scriptPath}.ts`], + tempDir, + ); + expect(result.stdout).toContain("is up to date"); + }); +}); diff --git a/cli/test/script_runtime_settings_sync.test.ts b/cli/test/script_runtime_settings_sync.test.ts new file mode 100644 index 0000000000..2b3272b20a --- /dev/null +++ b/cli/test/script_runtime_settings_sync.test.ts @@ -0,0 +1,126 @@ +/** + * Runtime settings that live only in the script metadata file (the retention + * delay, the debouncing bounds, the cache s3-path flag) must survive a sync + * pull/push cycle. A field missing from the create_script body the CLI builds + * lands as NULL on the deployed version; one missing from its up-to-date + * comparison makes a change to it alone report as up to date and never deploy. + */ + +import { expect, test } from "bun:test"; +import { writeFile, readFile, mkdir } from "node:fs/promises"; +import { withTestBackend } from "./test_backend.ts"; +import { waitForDeploymentJobs } from "./new_commands_helpers.ts"; + +// The debounce bounds this PR also restores cannot be asserted here: a build without +// git tags reports a bare commit as its version, GIT_SEM_VERSION then falls back to +// 0.1.0, and every version-gated feature (debouncing wants 1.566.0) is refused. That is +// what CI builds, so a fixture that sets any debounce field fails at create. +const SETTINGS = { + delete_after_secs: 900, + cache_ignore_s3_path: true, +}; + +test("Integration: script runtime settings survive a sync pull/push cycle", async () => { + await withTestBackend(async (backend, tempDir) => { + const uniqueId = Date.now(); + const scriptPath = `f/test/settings_${uniqueId}`; + const getScript = async () => { + const resp = await backend.apiRequest!( + `/api/w/${backend.workspace}/scripts/get/p/${scriptPath}`, + ); + expect(resp.ok).toEqual(true); + return await resp.json(); + }; + + await mkdir(`${tempDir}/f/test`, { recursive: true }); + const folderResp = await backend.apiRequest!( + `/api/w/${backend.workspace}/folders/create`, + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ name: "test" }), + }, + ); + const folderStatus = `${folderResp.status} ${await folderResp.text()}`; + + const createResp = await backend.apiRequest!( + `/api/w/${backend.workspace}/scripts/create`, + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + path: scriptPath, + content: `export async function main() {\n return "Hello world";\n}`, + summary: "Test runtime settings", + description: "", + language: "bun", + kind: "script", + schema: { + $schema: "https://json-schema.org/draft/2020-12/schema", + type: "object", + properties: {}, + required: [], + }, + ...SETTINGS, + }), + }, + ); + if (!createResp.ok) { + throw new Error( + `scripts/create failed: ${createResp.status} ${await createResp.text()} ` + + `(folders/create: ${folderStatus})`, + ); + } + + await writeFile( + `${tempDir}/wmill.yaml`, + `defaultTs: bun\nincludes:\n - "${scriptPath}**"\nexcludes: []\n`, + "utf-8", + ); + + // The lock a deploy's dependency job writes is compared before the settings are, + // so a pull taken before that job lands makes the next push deploy over the lock + // instead of over the setting under test. + await waitForDeploymentJobs(backend); + const pullResult = await backend.runCLICommand(["sync", "pull", "--yes"], tempDir); + expect(pullResult.code).toEqual(0); + + const metadataPath = `${tempDir}/${scriptPath}.script.yaml`; + const pulledMetadata = await readFile(metadataPath, "utf-8"); + for (const key of Object.keys(SETTINGS)) { + expect(pulledMetadata).toContain(key); + } + + // A content-only edit must carry the settings through to the new version. + const scriptFilePath = `${tempDir}/${scriptPath}.ts`; + const originalContent = await readFile(scriptFilePath, "utf-8"); + await writeFile( + scriptFilePath, + originalContent.replace("Hello world", "Hello world modified"), + "utf-8", + ); + expect((await backend.runCLICommand(["sync", "push", "--yes"], tempDir)).code).toEqual(0); + + const afterContentPush = await getScript(); + expect(afterContentPush.content).toContain("Hello world modified"); + for (const [key, value] of Object.entries(SETTINGS)) { + expect(afterContentPush[key]).toEqual(value); + } + + // A settings-only edit must reach the remote rather than be skipped as up to + // date. 0 is "delete immediately after completion", not "unset". + await waitForDeploymentJobs(backend); + expect((await backend.runCLICommand(["sync", "pull", "--yes"], tempDir)).code).toEqual(0); + await writeFile( + metadataPath, + (await readFile(metadataPath, "utf-8")).replace( + `delete_after_secs: ${SETTINGS.delete_after_secs}`, + "delete_after_secs: 0", + ), + "utf-8", + ); + expect((await backend.runCLICommand(["sync", "push", "--yes"], tempDir)).code).toEqual(0); + + expect((await getScript()).delete_after_secs).toEqual(0); + }); +}); diff --git a/docs/feature-telemetry.md b/docs/feature-telemetry.md index fed78cc064..947329fc45 100644 --- a/docs/feature-telemetry.md +++ b/docs/feature-telemetry.md @@ -4,9 +4,9 @@ anonymous usage-stats payload. It answers "does anyone use this, and which variant do they pick" without any identifying data leaving the instance. -It currently carries 14 registered actions across three features (`ai_session`, `ai_chat`, -`flow_editor`). Nearly all of the product is uninstrumented, so new user-facing work is the -opportunity to change that. +It currently carries 20 registered actions across eight features (`ai_session`, `ai_chat`, +`flow_editor`, `flow_run`, `flow_step`, `trigger`, `command_script`, `hub_script`). Nearly all of +the product is uninstrumented, so new user-facing work is the opportunity to change that. ## When to instrument @@ -49,9 +49,10 @@ vocabulary closed and small — enumerate the values in a TS union next to the c Four steps. Skipping step 1 or 3 fails quietly. **1. Register the pair** in `FEATURE_USAGE_KINDS` -(`backend/windmill-api-workspaces/src/workspaces.rs`). An unregistered `(feature, kind)` is -dropped by `valid_feature_usage_event` with a bare `continue` — no error, no log, still a 204 to -the browser. Frontend-only instrumentation records **nothing** and looks like it worked. +(`backend/windmill-common/src/feature_usage_ee.rs`, tracked in `windmill-ee-private`). An +unregistered `(feature, kind)` is dropped by `is_recordable_event` with a bare `continue` — no +error, no log, still a 204 to the browser. Frontend-only instrumentation records **nothing** and +looks like it worked. **2. Log from the frontend:** @@ -75,6 +76,10 @@ under-discloses what it sends. This has already drifted once. SELECT feature, kind, key, entity_id, day, value FROM feature_usage ORDER BY updated_at DESC LIMIT 10; ``` +Collection sits behind the `private` feature, so a public build records nothing from either the +HTTP route or the Rust helper. Run the backend with `--features enterprise,private` or this query +stays empty however correct the instrumentation is. + ## Privacy rules Only aggregated counts ever leave the instance, and only when telemetry is enabled and minimal @@ -85,8 +90,21 @@ cannot be collected — drop it. Counters aggregate over the last 30 days; rows are pruned after 60. -## Backend-only features +## Logging from the backend -Ingestion is frontend-only: `log_feature_usage` is an HTTP route the browser posts to, and there -is no Rust-side helper. A feature with no UI cannot be instrumented today without adding one. -Scope the default to user-facing work, and say so rather than implying backend coverage exists. +A feature with no UI is instrumented the same way, from Rust: + +```rust +windmill_common::feature_usage::log_feature_usage("trigger", "fired", kind.as_str()); +``` + +Same registry, same key rules, and the same silent drop when the pair is unregistered. `feature` +and `kind` are `&'static str` so a call site cannot pass a computed pair. The call increments an +in-memory counter and returns; the monitor loop flushes the accumulator, so it is cheap enough for +hot paths — but only cheap per call, not free: a key with unbounded cardinality would grow the map +until it hits the per-action cap and starts dropping new keys. + +There is no `entity_id` and no explicit `value` on this path: it counts occurrences. + +`feature_usage_ee` holds the registry and the writer; the public build gets the inert +`feature_usage_oss`, since a CE instance never sends a stats payload. diff --git a/docs/validation.md b/docs/validation.md index 3bfd1325ce..050eacd75b 100644 --- a/docs/validation.md +++ b/docs/validation.md @@ -39,6 +39,7 @@ After all code changes are done, run `./update_sqlx.sh` from `backend/` to regen | Modified Flow structures | Also update `openflow.openapi.yaml` | | Changed DB schema | Update `backend/summarized_schema.txt` if needed | | Enterprise file changes | Companion PR in `windmill-ee-private` (see `docs/enterprise.md`) | +| Changed a hook in `.claude/hooks/` | `bash .claude/hooks/test-hooks.sh` — pins which commands prompt | ## When to Write Tests diff --git a/frontend/package-lock.json b/frontend/package-lock.json index f99e965ea5..4bfeac0b46 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "@windmill-labs/components", - "version": "1.789.0", + "version": "1.792.2", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@windmill-labs/components", - "version": "1.789.0", + "version": "1.792.2", "hasInstallScript": true, "license": "AGPL-3.0", "dependencies": { diff --git a/frontend/package.json b/frontend/package.json index b95774a03d..0e16aeb90a 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "@windmill-labs/components", - "version": "1.789.0", + "version": "1.792.2", "scripts": { "dev": "vite dev", "dev:ui-builder": "mv static/ui_builder static/ui_builder.dev-disabled 2>/dev/null || true ; trap 'mv static/ui_builder.dev-disabled static/ui_builder 2>/dev/null || true' EXIT ; vite dev", diff --git a/frontend/sharedUtils/vite.sharedUtils.config.js b/frontend/sharedUtils/vite.sharedUtils.config.js index 98410f737d..4073f1be16 100644 --- a/frontend/sharedUtils/vite.sharedUtils.config.js +++ b/frontend/sharedUtils/vite.sharedUtils.config.js @@ -5,7 +5,7 @@ import { exec } from 'child_process' import { promisify } from 'util' const execAsync = promisify(exec) -const VERSION = '1.0.12' +const VERSION = '1.0.13' export default defineConfig({ build: { diff --git a/frontend/src/lib/components/AppConnectDrawer.svelte b/frontend/src/lib/components/AppConnectDrawer.svelte index 39df15721e..67c19b0fe7 100644 --- a/frontend/src/lib/components/AppConnectDrawer.svelte +++ b/frontend/src/lib/components/AppConnectDrawer.svelte @@ -7,6 +7,8 @@ import AppConnectInner from './AppConnectInner.svelte' import DarkModeObserver from './DarkModeObserver.svelte' + import IconedResourceType from './IconedResourceType.svelte' + import { addResourceTitle } from './resourceTypeDisplay' interface Props { expressOAuthSetup?: boolean @@ -14,7 +16,11 @@ disableChatOffset?: boolean } - let { expressOAuthSetup = false, workspace = undefined, disableChatOffset = false }: Props = $props() + let { + expressOAuthSetup = false, + workspace = undefined, + disableChatOffset = false + }: Props = $props() let drawer: Drawer | undefined = $state() let resourceType = $state('') @@ -51,16 +57,21 @@ step = 1 dispatch('close') }} - size="800px" + size="700px" {disableChatOffset} > 1 ? resourceType : undefined)} id="add-resource-drawer" on:close={drawer?.closeDrawer} tooltip="Resources represent connections to third party systems. Learn more on how to integrate external APIs." documentationLink="https://www.windmill.dev/docs/integrations/integrations_on_windmill" > + {#snippet titleExtra()} + {#if step > 1 && resourceType} + + {/if} + {/snippet} void } | undefined = $state(undefined) + let filter = $state('') let value: string = $state('') let valueToken: TokenResponse | undefined = undefined @@ -98,6 +109,21 @@ let connectClient: string = $state('') let connectsManual: { key: string; img?: string; instructions: string[] }[] | undefined = $state(undefined) + let resourceTypeDescriptions: Record = $state({}) + // Types made in this workspace, by the `c_` prefix the resources page adds or by the + // workspace they live in — the hub sync writes its own into `admins`, which every + // workspace reads from. `created_by` looks like the same signal but isn't: seeded hub + // types carry a username too. + let customResourceTypes: Set = $state(new Set()) + + // Hub descriptions are markdown; a row shows one line of it, where fenced blocks and + // backticks read as noise. + const plainDescription = (d: string) => + d + .replace(/```[\s\S]*?```/g, '') + .replace(/`/g, '') + .replace(/\s+/g, ' ') + .trim() let args: any = $state({}) let renderDescription = $state(true) @@ -274,6 +300,14 @@ * credentials form with the user's own credentials — even when the instance * has shared ones (the "Instance-configured OAuth APIs" section is the entry * point for those). Every other type opens the raw manual form. */ + function connectOauth(key: string) { + manual = false + connectClient = key + resourceType = stripSandboxSuffix(key) + resetClientCredentialsState() + next() + } + function selectFromOthers(key: string) { connectClient = key resourceType = key @@ -298,6 +332,8 @@ loadResourceTypes() } step = 1 //express && !manual ? 3 : 1 + // The list is keyboard-driven from the search field, so it takes focus on open. + tick().then(() => searchInput?.focus()) value = '' description = '' labels = undefined @@ -356,9 +392,13 @@ } } + // Google's terms require its own button on the control that starts the sign-in, which is + // the step-2 Connect: step 1 only picks a type, and a manual step 2 saves a resource + // without ever reaching Google. run(() => { isGoogleSignin = - step == 1 && + step == 2 && + !manual && (resourceType == 'google' || resourceType == 'gmail' || resourceType == 'gcal' || @@ -395,6 +435,30 @@ const availableRts = await ResourceService.listResourceTypeNames({ workspace: effectiveWorkspace }) + // The prefix alone identifies a workspace-made type, and it rides on the names call the + // list already needs — so the custom section survives the full list below 403ing. + customResourceTypes = new Set(availableRts.filter(isCustomResourceTypeName)) + + // Descriptions only feed search, so they are fetched off the critical path and + // allowed to fail: `resources/type/list` is not on the public app domain's route + // allow-list (`listnames` is), and it carries every type's full schema. Awaiting it + // would hold the list behind a request nothing on screen needs -- in a published + // app, behind one that is guaranteed to 403. resourceTypeDescriptions feeds a + // $derived, so search re-ranks when they land. + ResourceService.listResourceType({ workspace: effectiveWorkspace }) + .then((types) => { + resourceTypeDescriptions = Object.fromEntries( + types.filter((t) => t.description).map((t) => [t.name, t.description!]) + ) + // A type sitting in this workspace was made here too, but only the full list carries + // `workspace_id`. Inside `admins` the two are indistinguishable — every type lives + // there — so the prefix is all there is to go on. + customResourceTypes = new Set([ + ...customResourceTypes, + ...types.filter((t) => t.workspace_id && t.workspace_id !== 'admins').map((t) => t.name) + ]) + }) + .catch(() => {}) // "Others" lists every resource type — including instance-configured OAuth // providers — so any of them can also be connected with the user's own @@ -561,7 +625,9 @@ args = {} } else { getResourceTypeInfo() - getScopesAndParams() + // Awaited: the popup is built from `scopes`, so advancing before this + // resolves sends the user to an authorize url with no scope at all. + await getScopesAndParams() } step += 1 } else if (step == 2 && !manual) { @@ -868,6 +934,129 @@ let filteredConnects: { key: string }[] = $state([]) let filteredConnectsManual: { key: string; img?: string; instructions: string[] }[] = $state([]) + // uFuzzy scores the name and the description as one string, so searching "google" ranks + // every type whose description mentions Google alongside the ones named after it. Re-sort + // on which field matched, keeping uFuzzy's order within a tier. + const rank = (items: { key: string }[] | undefined) => + items && + sortResourceTypesByMatch( + items, + filter, + (x) => x.key, + (x) => resourceTypeDescriptions[x.key] + ) + let rankedConnects = $derived(rank(filteredConnects)) + let rankedConnectsManual = $derived( + rank(filteredConnectsManual) as typeof filteredConnectsManual | undefined + ) + + let searching = $derived(filter.trim() !== '') + + // Browsing, the "Others" list leads with the native database types. Searching, that + // grouping would outrank the search itself — `ms_sql_server` sorting under `mysql` on + // "sql" — so the ranked order stands on its own. + let manualOrderedKeys = $derived( + !searching + ? [ + ...(rankedConnectsManual ?? []) + .filter((x) => nativeLanguagesCategory.includes(x.key)) + .map((x) => x.key), + ...(rankedConnectsManual ?? []) + .filter((x) => !nativeLanguagesCategory.includes(x.key)) + .map((x) => x.key) + ] + : (rankedConnectsManual ?? []).map((x) => x.key) + ) + + let customKeys = $derived(manualOrderedKeys.filter((key) => customResourceTypes.has(key))) + let otherKeys = $derived(manualOrderedKeys.filter((key) => !customResourceTypes.has(key))) + + // Every row in the order it is rendered, so arrow keys walk the sections as one list. + // A provider appears in more than one, so rows are addressed by index, not by name. + let navItems = $derived([ + ...customKeys.map((key) => ({ key, oauth: false })), + ...(rankedConnects ?? []).map((x) => ({ key: x.key, oauth: true })), + ...otherKeys.map((key) => ({ key, oauth: false })) + ]) + // Both lists start undefined and render skeletons; "nothing found" only means something + // once they have landed. + let listsLoaded = $derived(rankedConnectsManual !== undefined && rankedConnects !== undefined) + let highlightedIndex = $state(-1) + const rowDomId = (index: number) => `resource-type-row-${index}` + + // Set at hover time rather than up front, so only the descriptions the row actually cut + // off carry a tooltip. + function titleIfTruncated(e: MouseEvent & { currentTarget: HTMLElement }) { + const el = e.currentTarget + el.title = el.scrollWidth > el.clientWidth ? (el.textContent?.trim() ?? '') : '' + } + const oauthRowOffset = $derived(customKeys.length) + const otherRowOffset = $derived(customKeys.length + (rankedConnects?.length ?? 0)) + + // Sections are rendered in a fixed order, so the best match is not necessarily the first + // row: rank the rows against the query to find it. + function bestMatchIndex(): number { + let best = navItems.length > 0 ? 0 : -1 + let bestRank = Infinity + navItems.forEach((item, index) => { + const rank = resourceTypeMatchRank(item.key, resourceTypeDescriptions[item.key], filter) + if (rank < bestRank) { + bestRank = rank + best = index + } + }) + return best + } + + // Filtering reshuffles the rows under the highlight: point it at the best match so Enter + // takes the top hit, and drop it entirely once the filter is cleared. + $effect(() => { + navItems + filter + untrack(() => (highlightedIndex = searching ? bestMatchIndex() : -1)) + }) + + // Scrolling rows under a resting pointer makes the browser fire `mouseenter` on each one, + // which would drag the highlight back under the cursor as the arrow keys move it. Only a + // real pointer move hands the highlight back to the mouse. + let pointerOwnsHighlight = $state(true) + + function highlightHovered(index: number) { + if (pointerOwnsHighlight) highlightedIndex = index + } + + function moveHighlight(delta: number) { + const count = navItems.length + if (count === 0) return + pointerOwnsHighlight = false + // Rows are tabbable buttons, so focus can sit on one. Enter then activates whatever is + // focused, which has to stay the highlighted row. + const rowWasFocused = document.activeElement?.id?.startsWith('resource-type-row-') ?? false + highlightedIndex = + highlightedIndex < 0 + ? delta > 0 + ? 0 + : count - 1 + : (highlightedIndex + delta + count) % count + const row = document.getElementById(rowDomId(highlightedIndex)) + row?.scrollIntoView({ block: 'nearest' }) + if (rowWasFocused) row?.focus() + } + + function onListKeydown(e: KeyboardEvent) { + if (step !== 1) return + if (e.key === 'ArrowDown' || e.key === 'ArrowUp') { + e.preventDefault() + moveHighlight(e.key === 'ArrowDown' ? 1 : -1) + } else if (e.key === 'Enter' && (e.target as HTMLElement)?.id === SEARCH_INPUT_ID) { + // A focused row activates itself on Enter; this covers Enter typed in the search field. + const item = navItems[highlightedIndex] + if (!item) return + e.preventDefault() + item.oauth ? connectOauth(item.key) : selectFromOthers(item.key) + } + } + let editScopes = $state(false) @@ -880,118 +1069,184 @@ })) : undefined} bind:filteredItems={filteredConnects} - f={(x) => x.key} + f={(x) => resourceTypeSearchText(x.key, resourceTypeDescriptions[x.key])} /> x.key} + f={(x) => resourceTypeSearchText(x.key, resourceTypeDescriptions[x.key])} /> {#if step == 1} -
-
- - + + +
(pointerOwnsHighlight = true)} + > +
+
+ + +
+
+ + {#snippet resourceRow(key: string)} +
+
+ +
+
+
+ {resourceTypeDisplayName(key)} + {key} +
+ {#if resourceTypeDescriptions[key]} + + {plainDescription(resourceTypeDescriptions[key])} + + {/if} +
+
+ {/snippet} + + {#snippet sectionHeading(title: string, count: number)} +

+ {title}{#if searching}{count}{/if} +

+ {/snippet} + + {#snippet resourceButton(key: string, index: number, oauth: boolean)} + + {/snippet} + +
+ {#if searching && listsLoaded && navItems.length === 0} +
+ No resource type matches “{filter.trim()}” + + Search on the name, the product or what the resource holds — or sync resource types + with the hub for more. + +
+ {:else} + +
+ {#if customKeys.length > 0} +
+ {@render sectionHeading('Custom resource types', customKeys.length)} +
+ {#each customKeys as key, i} + {@render resourceButton(key, i, false)} + {/each} +
+
+ {/if} + + {#if !searching || (rankedConnects?.length ?? 0) > 0} +
+ {@render sectionHeading( + 'Instance-configured OAuth APIs', + rankedConnects?.length ?? 0 + )} +
+ {#if rankedConnects} + {#each rankedConnects as { key }, i} + {@render resourceButton(key, oauthRowOffset + i, true)} + {/each} + {:else} + {#each new Array(3) as _} + + {/each} + {/if} +
+ {#if !searching && connects && connects.filter(isSharedConnect).length == 0} +
No OAuth APIs have been set up on this instance. To add OAuth APIs, first sync + the resource types with the hub, then add OAuth configuration. See documentation +
+ {/if} +
+ {/if} + + {#if !searching || otherKeys.length > 0} +
+ {@render sectionHeading('Others', otherKeys.length)} + + {#if !searching && connectsManual && connectsManual?.length < 10} +
+ Resource types have not been synced with the hub +
+ {/if} + +
+ {#if rankedConnectsManual} + {#each otherKeys as key, i} + {@render resourceButton(key, otherRowOffset + i, false)} + {/each} + {:else} + {#each new Array(9) as _} + + {/each} + {/if} +
+
+ {/if} +
+ {/if} +
+
+ { + connectsManual = undefined + await loadResourceTypes() + connects = undefined + await loadConnects() + }} />
- -

Instance-configured OAuth APIs

-
- {#if filteredConnects} - {#each filteredConnects as { key }} - - {/each} - {:else} - {#each new Array(3) as _} - - {/each} - {/if} -
- {#if connects && connects.filter(isSharedConnect).length == 0} -
No OAuth APIs have been set up on this instance. To add OAuth APIs, first sync the resource - types with the hub, then add OAuth configuration. See documentation -
- {/if} - -

Others

- - {#if connectsManual && connectsManual?.length < 10} -
- Resource types have not been synced with the hub -
- {/if} - -
- {#if filteredConnectsManual} - {#each filteredConnectsManual as { key }} - {#if nativeLanguagesCategory.includes(key)} - - {/if} - {/each} - {/if} - {#if filteredConnectsManual} - {#each filteredConnectsManual as { key }} - {#if !nativeLanguagesCategory.includes(key)} - - - {/if} - {/each} - {:else} - {#each new Array(9) as _} - - {/each} - {/if} -
-
- { - connectsManual = undefined - await loadResourceTypes() - connects = undefined - await loadConnects() - }} - /> -
{:else if step == 2 && manual}
+ {#if !emptyString(resourceTypeInfo?.description)} + + {/if}
{#if resourceTypeInfo?.description} -
-

Description

-
- -
-
+ {/if} diff --git a/frontend/src/lib/components/AppConnectLightweightResourcePicker.svelte b/frontend/src/lib/components/AppConnectLightweightResourcePicker.svelte index 53671974b2..73cc0c6633 100644 --- a/frontend/src/lib/components/AppConnectLightweightResourcePicker.svelte +++ b/frontend/src/lib/components/AppConnectLightweightResourcePicker.svelte @@ -34,9 +34,11 @@ -
+ +
{#if !express} -
+
{#if step > 2} @@ -54,14 +56,16 @@
{/if} - +
+ +
diff --git a/frontend/src/lib/components/AuthSettings.svelte b/frontend/src/lib/components/AuthSettings.svelte index 2ad6ab917a..c5f0a2040a 100644 --- a/frontend/src/lib/components/AuthSettings.svelte +++ b/frontend/src/lib/components/AuthSettings.svelte @@ -53,6 +53,13 @@ hideTabs = false }: Props = $props() + // The callback lands on a frontend route, so a base url that is not the origin + // the admin is browsing is almost always a misconfiguration. + let browserOrigin = typeof window !== 'undefined' ? window.location.origin : '' + let baseUrlMismatch = $derived( + !!baseUrl && !!browserOrigin && baseUrl.replace(/\/$/, '') !== browserOrigin + ) + $effect(() => { if (oauths == undefined) { oauths = {} @@ -522,6 +529,27 @@ bind:password={oauths[k]['secret']} /> +
+ Redirect URL + {#if !baseUrl} + + Set it in Core settings. The redirect url is built from it, and {k} needs the exact + value. + + {:else} + + {/if} + {#if baseUrlMismatch} + + This is built from the instance base url. Update it in Core settings if it is + wrong, or {k} will reject the callback. + + {/if} +
These credentials are for {#if !windmillBuiltins.includes(k) || (registryCcCapable(k) && registryAuthCodeCapable(k))} diff --git a/frontend/src/lib/components/BatchLoadProgress.svelte b/frontend/src/lib/components/BatchLoadProgress.svelte new file mode 100644 index 0000000000..16cf2d7212 --- /dev/null +++ b/frontend/src/lib/components/BatchLoadProgress.svelte @@ -0,0 +1,62 @@ + + +
+ Loading {itemsLabel}: {loaded} of {total}... +
+
+
+ {#if batchSize != null} + Batch size: + { + const v = parseInt(e.currentTarget.value) + if (v >= 1 && v <= maxBatchSize) { + onBatchSizeChange?.(v) + } else { + e.currentTarget.value = String(batchSize) + } + } + }} + /> + {/if} + +
diff --git a/frontend/src/lib/components/CompareDrafts.svelte b/frontend/src/lib/components/CompareDrafts.svelte index 88107f8bba..a05511ea42 100644 --- a/frontend/src/lib/components/CompareDrafts.svelte +++ b/frontend/src/lib/components/CompareDrafts.svelte @@ -22,7 +22,11 @@ discardDraft, draftBaseIsStale } from '$lib/utils_draft_deploy' - import { checkDeployPermission, type DeployPermission } from '$lib/utils_workspace_deploy' + import { + checkDeployPermission, + deployPermissionForKinds, + type DeployPermission + } from '$lib/utils_workspace_deploy' import { type DraftItem, invalidateWorkspaceDrafts, @@ -304,20 +308,30 @@ let selectedItems = $state([]) let deploying = $state(false) - // Whether the user may deploy drafts into this workspace — fills the - // `RestrictDeployToDeployers` (+ operator) gap via the shared util, same as the - // fork compare page and the session review drawer. Fail-open while resolving. - let deployPerm = $state({ ok: true }) + // Whether the user may deploy drafts into this workspace, via the shared util — + // same as the fork compare page and the session review drawer. Fail-open while + // resolving. + let workspaceDeployPerm = $state({ ok: true }) $effect(() => { const ws = currentWorkspaceId // Reset to fail-open on workspace change, and drop a stale resolution — // otherwise the previous workspace's verdict lingers (or lands last) and // gates the wrong workspace. - deployPerm = { ok: true } + workspaceDeployPerm = { ok: true } void checkDeployPermission(ws).then((p) => { - if (ws === currentWorkspaceId) deployPerm = p + if (ws === currentWorkspaceId) workspaceDeployPerm = p }) }) + // A direct-deployment lock never reaches trigger or schedule drafts server-side, so it must + // not disable a selection made only of those. One refused kind still blocks the whole action. + let deployPerm = $derived( + deployPermissionForKinds( + workspaceDeployPerm, + visibleItems + .filter((i) => selectedItems.includes(i.key) && isDeployable(i)) + .map((i) => i.draftKind) + ) + ) // Select all on the first non-empty load (acting on everything is the common // intent); only once, so a refetch after a deploy doesn't re-select the // leftovers. diff --git a/frontend/src/lib/components/CompareWorkspaces.svelte b/frontend/src/lib/components/CompareWorkspaces.svelte index 875eda25ce..375e73480e 100644 --- a/frontend/src/lib/components/CompareWorkspaces.svelte +++ b/frontend/src/lib/components/CompareWorkspaces.svelte @@ -41,6 +41,7 @@ import type { Kind } from '$lib/utils_deployable' import { checkDeployPermission, + deployPermissionForKinds, deployItem, deleteItemInWorkspace, diffActionableInDirection, @@ -950,21 +951,28 @@ toggleDeploymentDirection(v) } - // Fetch user permissions for both workspaces + // Fetch user permissions for both workspaces. The server's `can_preserve_on_behalf_of` reads + // the merged `is_admin || super_admin`, which `whoami` reports as two fields. $effect(() => { ;[currentWorkspaceId, parentWorkspaceId] async function fetchPermissions() { try { const parentUser = await UserService.whoami({ workspace: parentWorkspaceId }) canPreserveInParent = - parentUser.is_admin || parentUser.groups?.includes('wm_deployers') || false + parentUser.is_admin || + parentUser.is_super_admin || + parentUser.groups?.includes('wm_deployers') || + false } catch { canPreserveInParent = false } try { const currentUser = await UserService.whoami({ workspace: currentWorkspaceId }) canPreserveInCurrent = - currentUser.is_admin || currentUser.groups?.includes('wm_deployers') || false + currentUser.is_admin || + currentUser.is_super_admin || + currentUser.groups?.includes('wm_deployers') || + false } catch { canPreserveInCurrent = false } @@ -972,10 +980,9 @@ fetchPermissions() }) - // Can the user actually deploy into the target workspace? Fills the frontend - // gap for the `RestrictDeployToDeployers` rule (+ operator), shared with the - // session review drawer via the same checkDeployPermission util. Cached per - // workspace; `deployPerm` tracks whichever side the current direction targets. + // Can the user actually deploy into the target workspace? Shared with the session + // review drawer via the same checkDeployPermission util. Cached per workspace; + // `workspaceDeployPerm` tracks whichever side the current direction targets. let deployPerms = $state>({}) const deployPermFetched = new Set() $effect(() => { @@ -985,7 +992,17 @@ void checkDeployPermission(ws).then((p) => (deployPerms = { ...deployPerms, [ws]: p })) } }) - let deployPerm = $derived(deployPerms[deployTargetWorkspace] ?? { ok: true }) + let workspaceDeployPerm = $derived(deployPerms[deployTargetWorkspace] ?? { ok: true }) + // A direct-deployment lock never reaches schedules or triggers server-side, so it must not + // disable a selection made only of those. One refused kind still blocks the whole action. + let deployPerm = $derived( + deployPermissionForKinds( + workspaceDeployPerm, + (comparison?.diffs ?? []) + .filter((d) => selectedItems.includes(getItemKey(d))) + .map((d) => d.kind) + ) + ) // Fetch summaries and on_behalf_of_email when comparison data loads $effect(() => { diff --git a/frontend/src/lib/components/ContentSearchInner.svelte b/frontend/src/lib/components/ContentSearchInner.svelte index 34f3815a47..6690e0c0bf 100644 --- a/frontend/src/lib/components/ContentSearchInner.svelte +++ b/frontend/src/lib/components/ContentSearchInner.svelte @@ -14,7 +14,8 @@ import ToggleButtonGroup from './common/toggleButton-v2/ToggleButtonGroup.svelte' import ToggleButton from './common/toggleButton-v2/ToggleButton.svelte' import FlowIcon from './home/FlowIcon.svelte' - import { Button } from './common' + import Tooltip from './meltComponents/Tooltip.svelte' + import { Badge, Button } from './common' import YAML from 'yaml' import { twMerge } from 'tailwind-merge' import ContentSearchInnerItem from './ContentSearchInnerItem.svelte' @@ -55,8 +56,12 @@ let scripts: undefined | { path: string; content: string }[] = $state(undefined) let filteredScriptItems: { path: string; content: string; marked: any }[] = $state([]) - let resources: undefined | { path: string; value: any }[] = $state(undefined) - let filteredResourceItems: { path: string; value: any; marked: any }[] = $state([]) + // Resource values are arbitrary user JSON and can be huge, so the API sends them already + // rendered and length-capped. Keep them as text — re-serializing here blocks the main + // thread for seconds on workspaces with many large resources. + type ResourceHit = { path: string; value: string; truncated: boolean } + let resources: undefined | ResourceHit[] = $state(undefined) + let filteredResourceItems: (ResourceHit & { marked: any })[] = $state([]) let flows: undefined | { path: string; value: any }[] = $state(undefined) let filteredFlowItems: { path: string; value: any; marked: any }[] = $state([]) @@ -127,7 +132,7 @@ filter={search} items={resources} f={(s) => { - return YAML.stringify(s.value) + return s.value }} bind:filteredItems={filteredResourceItems} /> @@ -217,6 +222,17 @@
apps
+ {#if resources} + {@const nTruncated = resources.filter((r) => r.truncated).length} + {#if nTruncated > 0} + +
+ {nTruncated} of those resources {nTruncated === 1 ? 'is' : 'are'} too large to search in full + — only {nTruncated === 1 ? 'its' : 'their'} beginning is matched. +
+ {/if} + {/if}
@@ -269,6 +285,15 @@ on:close > {#snippet actions()} + {#if item.truncated} + + Truncated + {#snippet text()} + This resource is too large to search in full: only its beginning is matched + and shown. + {/snippet} + + {/if} diff --git a/frontend/src/lib/components/DropdownSubmenuItem.svelte b/frontend/src/lib/components/DropdownSubmenuItem.svelte index ed08fd7abd..631f6374bd 100644 --- a/frontend/src/lib/components/DropdownSubmenuItem.svelte +++ b/frontend/src/lib/components/DropdownSubmenuItem.svelte @@ -3,7 +3,8 @@ import MenuItem from '$lib/components/meltComponents/MenuItem.svelte' import { melt } from '@melt-ui/svelte' import { twMerge } from 'tailwind-merge' - import { ChevronRight } from 'lucide-svelte' + import { Check, ChevronRight } from 'lucide-svelte' + import Toggle from '$lib/components/Toggle.svelte' import type { Item } from '$lib/utils' import type { MenubarMenuElements, createDropdownMenu } from '@melt-ui/svelte' import { Tooltip } from './meltComponents' @@ -65,12 +66,29 @@ item={meltItem} > {#if subItem.icon} - + {/if}

{subItem.displayName}

{@render subItem.extra?.()} + {#if subItem.shortcut || subItem.selected || subItem.toggle !== undefined} +
+ {#if subItem.shortcut} + {subItem.shortcut} + {/if} + {#if subItem.selected} + + {/if} + {#if subItem.toggle !== undefined} + + + {/if} +
+ {/if} {#if subItem.tooltip} {#snippet text()} diff --git a/frontend/src/lib/components/DropdownV2Inner.svelte b/frontend/src/lib/components/DropdownV2Inner.svelte index 1196589474..425c41d8f5 100644 --- a/frontend/src/lib/components/DropdownV2Inner.svelte +++ b/frontend/src/lib/components/DropdownV2Inner.svelte @@ -1,5 +1,6 @@ -
+
{#if asPlainText}

{md}

{:else} diff --git a/frontend/src/lib/components/InstanceSettings.svelte b/frontend/src/lib/components/InstanceSettings.svelte index 0a21e8897e..1f89829c41 100644 --- a/frontend/src/lib/components/InstanceSettings.svelte +++ b/frontend/src/lib/components/InstanceSettings.svelte @@ -1063,8 +1063,12 @@
  • job usage (language, total duration, count)
  • git sync repo count (sync vs promotion mode)
  • feature usage telemetry: aggregated AI chat and AI session usage counts, including AI - provider and model identifiers (last 30 days)
    • feature usage (counts of which product features are used, including AI provider and + model identifiers and the names of public hub scripts used, last 30 days) +
  • feature adoption (counts of which flow, script, trigger and worker features your + deployed items use)
  • resource counts (workspaces, scripts per language, flows, workflows as code, low-code @@ -1110,8 +1114,12 @@
  • user usage (author count, operator count)
  • development instance status
  • feature usage telemetry: aggregated AI chat and AI session usage counts, including AI - provider and model identifiers (last 30 days)
    • feature usage (counts of which product features are used, including AI provider and + model identifiers and the names of public hub scripts used, last 30 days) +
  • feature adoption (counts of which flow, script, trigger and worker features your + deployed items use)
  • resource counts (workspaces, scripts per language, flows, workflows as code, low-code diff --git a/frontend/src/lib/components/JobLoader.svelte b/frontend/src/lib/components/JobLoader.svelte index ec9e4b693c..71bd2fbb51 100644 --- a/frontend/src/lib/components/JobLoader.svelte +++ b/frontend/src/lib/components/JobLoader.svelte @@ -87,6 +87,7 @@ let finished: string[] = [] let ITERATIONS_BEFORE_SLOW_REFRESH = 10 let ITERATIONS_BEFORE_SUPER_SLOW_REFRESH = 100 + const MAX_SSE_ATTEMPTS = 3 let lastStartedAt: number = Date.now() let currentId: string | undefined = $state(undefined) @@ -179,7 +180,7 @@ lastCompletedJobId = undefined clearCurrentJob() lastCallbacks = callbacks - noPingTimeout = undefined + clearNoPingTimeout() const startedAt = Date.now() const testId = await fn() @@ -669,16 +670,30 @@ } } - function setNoPingTimeout(id: string, attempt: number, callbacks?: Callbacks) { + function clearNoPingTimeout() { if (noPingTimeout) { clearTimeout(noPingTimeout) + noPingTimeout = undefined } + } + + function setNoPingTimeout(id: string, attempt: number, callbacks?: Callbacks) { + clearNoPingTimeout() if (isCurrentJob(id)) { noPingTimeout = setTimeout(() => { + noPingTimeout = undefined if (isCurrentJob(id)) { currentEventSource?.close() currentEventSource = undefined - loadTestJobWithSSE(id, attempt + 1, callbacks) + // A proxy that buffers the response rather than cutting it keeps the + // connection open and error-free, so this watchdog is the only signal that + // no event is getting through. It has to share the retry budget: otherwise + // it reopens an equally mute stream forever and polling is never reached. + if (attempt < MAX_SSE_ATTEMPTS) { + loadTestJobWithSSE(id, attempt + 1, callbacks) + } else { + syncer(id, callbacks) + } } }, 10000) } @@ -841,10 +856,7 @@ if (previewJobUpdates.completed) { currentEventSource?.close() currentEventSource = undefined - if (noPingTimeout) { - clearTimeout(noPingTimeout) - noPingTimeout = undefined - } + clearNoPingTimeout() isCompleted = true if (onlyResult) { callbacks?.doneResult?.({ @@ -869,16 +881,26 @@ console.warn('SSE error:', error) currentEventSource?.close() currentEventSource = undefined + clearNoPingTimeout() let delay = 1000 let isNoLogsChange = error.type == noLogsChangeRestartEvent if (isNoLogsChange) { delay = 0 } - if (attempt < 3 || isNoLogsChange) { + if (attempt < MAX_SSE_ATTEMPTS || isNoLogsChange) { if (!isNoLogsChange) { - console.log(`SSE error (1), retrying ... attempt: ${attempt + 1}/3`) + console.log( + `SSE error (1), retrying ... attempt: ${attempt + 1}/${MAX_SSE_ATTEMPTS}` + ) } - setTimeout(() => loadTestJobWithSSE(id, attempt + 1, callbacks), delay) + // A no-logs restart is deliberate (the caller wants a stream with different + // query args), not a failure, so it must not consume the retry budget: + // toggling the flow graph tab would otherwise exhaust it in a few clicks + // and strand a healthy stream on polling. + setTimeout( + () => loadTestJobWithSSE(id, isNoLogsChange ? attempt : attempt + 1, callbacks), + delay + ) } else { // Fall back to polling on error setTimeout(() => syncer(id, callbacks), 1000) @@ -901,9 +923,10 @@ // Fall back to polling on error currentEventSource?.close() currentEventSource = undefined + clearNoPingTimeout() - if (attempt < 3) { - console.log(`SSE error (2), retrying ... attempt: ${attempt}/3`) + if (attempt < MAX_SSE_ATTEMPTS) { + console.log(`SSE error (2), retrying ... attempt: ${attempt}/${MAX_SSE_ATTEMPTS}`) attempt++ loadTestJobWithSSE(id, attempt, callbacks) } else { @@ -942,6 +965,7 @@ clearCurrentId() currentEventSource?.close() currentEventSource = undefined + clearNoPingTimeout() replayTimeouts.forEach(clearTimeout) replayTimeouts = [] }) diff --git a/frontend/src/lib/components/LightweightResourcePicker.svelte b/frontend/src/lib/components/LightweightResourcePicker.svelte index a7fe9fd0d1..bb0c21b4c1 100644 --- a/frontend/src/lib/components/LightweightResourcePicker.svelte +++ b/frontend/src/lib/components/LightweightResourcePicker.svelte @@ -8,6 +8,8 @@ import type { AppViewerContext } from './apps/types' import { sendUserToast } from '$lib/toast' import Select from './select/Select.svelte' + import IconedResourceType from './IconedResourceType.svelte' + import { addResourceTitle } from './resourceTypeDisplay' interface Props { value: string | undefined @@ -103,11 +105,16 @@ {:else} + {#snippet titleExtra()} + {#if resourceType} + + {/if} + {/snippet} {#await import('./AppConnectLightweightResourcePicker.svelte')} {:then Module} diff --git a/frontend/src/lib/components/ModulePreviewResultViewer.svelte b/frontend/src/lib/components/ModulePreviewResultViewer.svelte index 6a0fef6f79..3f39531b0a 100644 --- a/frontend/src/lib/components/ModulePreviewResultViewer.svelte +++ b/frontend/src/lib/components/ModulePreviewResultViewer.svelte @@ -85,8 +85,12 @@ bind:this={outputPickerInner} > {#snippet copilot_fix()} - {#if lang && editor && diffEditor && stepsInputArgs.getStepArgs(mod.id) && selectedJob?.type === 'CompletedJob' && !selectedJob.success && getStringError(selectedJob.result)} - + {@const stepError = + selectedJob?.type === 'CompletedJob' && !selectedJob.success + ? getStringError(selectedJob.result) + : undefined} + {#if lang && editor && diffEditor && stepsInputArgs.getStepArgs(mod.id) && stepError} + {/if} {/snippet} diff --git a/frontend/src/lib/components/ResourceEditor.svelte b/frontend/src/lib/components/ResourceEditor.svelte index 8b10841512..367888655f 100644 --- a/frontend/src/lib/components/ResourceEditor.svelte +++ b/frontend/src/lib/components/ResourceEditor.svelte @@ -154,12 +154,6 @@ ) }) - let linkedVars = $derived( - Object.entries(current?.args ?? {}) - .filter(([_, v]) => typeof v == 'string' && v == `$var:${initialPath}`) - .map(([k, _]) => k) - ) - const dirtyWorkspaces = $derived( Object.keys(states).filter((ws) => !draftValuesEqual(states[ws].draft, initialStates[ws])) ) @@ -312,16 +306,19 @@ }) }) - $effect(() => { + /** Sole writer of `current.path` — an arg still holding `$var:` is the resource's own linked secret, which the backend renames + * along with the resource, so the reference moves with it. An arg pointing at + * any other variable was set by the user and is left alone. */ + function setPath(npath: string): void { if (!current) return - if (linkedVars.length > 0 && current.path) { - untrack(() => { - linkedVars.forEach((k) => { - current!.args[k] = `$var:${current!.path}` - }) - }) + const prev = current.path + // `args` is whatever the raw JSON editor parsed — `null` included. + for (const [k, v] of Object.entries(current.args ?? {})) { + if (v === `$var:${prev}`) current.args[k] = `$var:${npath}` } - }) + current.path = npath + } export async function save(): Promise { const dirty = dirtyWorkspaces @@ -388,7 +385,7 @@ {#if current} {#key current} current!.path, setPath} bind:labels={current.labels} bind:description={current.description} bind:args={current.args} diff --git a/frontend/src/lib/components/ResourceEditorDrawer.svelte b/frontend/src/lib/components/ResourceEditorDrawer.svelte index ecde6e2f1a..d247654968 100644 --- a/frontend/src/lib/components/ResourceEditorDrawer.svelte +++ b/frontend/src/lib/components/ResourceEditorDrawer.svelte @@ -16,6 +16,8 @@ } from './sessions/pageDrawerSession' import { RESOURCES_PATH } from './sessions/previewPaths' import ResourceVersionHistory from './ResourceVersionHistory.svelte' + import IconedResourceType from './IconedResourceType.svelte' + import { addResourceTitle } from './resourceTypeDisplay' let { workspace = undefined, @@ -98,10 +100,15 @@ on:close={() => clearPageDrawerAnchor(RESOURCES_PATH)} > + {#snippet titleExtra()} + {#if mode == 'new' && resource_type} + + {/if} + {/snippet} {#await import('./ResourceEditor.svelte')} {:then Module} diff --git a/frontend/src/lib/components/ResourceForm.svelte b/frontend/src/lib/components/ResourceForm.svelte index 7c49364b0a..cddd83aae7 100644 --- a/frontend/src/lib/components/ResourceForm.svelte +++ b/frontend/src/lib/components/ResourceForm.svelte @@ -15,7 +15,6 @@ import Toggle from './Toggle.svelte' import TestConnection from './TestConnection.svelte' import { Pen } from 'lucide-svelte' - import Markdown from 'svelte-exmarkdown' import autosize from '$lib/autosize' import GfmMarkdown from './GfmMarkdown.svelte' import TestTriggerConnection from './triggers/TestTriggerConnection.svelte' @@ -24,6 +23,7 @@ import ResourceGen from './copilot/ResourceGen.svelte' import SyncResourceTypes from './SyncResourceTypes.svelte' import Label from './Label.svelte' + import ResourcePathHint from './ResourcePathHint.svelte' interface Props { path: string @@ -142,6 +142,10 @@ }) +{#if !emptyString(resourceTypeInfo?.description)} + +{/if} + {#if !hidePath}
    {#if !can_write} @@ -152,6 +156,7 @@
    {/if}