fix(perf): harden release benchmark gate

AGENTS.md

@@ -59,9 +59,9 @@ Inside pane-scaled render and layout loops:
 Prefer deterministic operation or architecture tests to wall-clock CI limits.
 Performance benchmarks are supporting evidence, not substitutes for behavioral
 coverage. Before a stable release, `just bench-release-smoke` must compare the
-candidate with the current stable binary under hidden and visible output; use
-the longer release matrix only when the smoke test moves materially or when
-validating performance work.
+candidate with the current stable binary under hidden and visible output. When
+the result moves materially or when validating performance work, repeat it with
+`HERDR_PERF_SAMPLE_SECONDS=60` and investigate the affected scenario.
 
 ### Runtime/client boundary guardrail
 

justfile

@@ -73,7 +73,7 @@ build:
 bench-render-scale:
     cargo test --release --locked --bin herdr render_scale_profile -- --ignored --nocapture --test-threads=1
 
-# Fast end-to-end CPU comparison against the current stable release
+# ~3-5 minute CPU comparison; downloads stable unless HERDR_PERF_BASELINE_BIN is set
 bench-release-smoke:
     cargo build --release --locked
     scripts/release_perf_smoke.sh "${CARGO_TARGET_DIR:-target}/release/herdr"
@@ -135,7 +135,7 @@ release-docs-check:
     just website-build
     cd website && bun run build:draft
 
-# Validate release docs and review full-render scaling before release preparation
+# Validate release docs, render scaling, and end-to-end CPU before release preparation
 pre-release-check:
     just release-docs-check
     just bench-render-scale

scripts/release_perf_case.sh

@@ -50,7 +50,9 @@ cleanup() {
   tmux kill-session -t "$name" >/dev/null 2>&1 || true
   rm -rf "$state"
 }
-trap cleanup EXIT INT TERM
+trap cleanup EXIT
+trap 'exit 130' INT
+trap 'exit 143' TERM
 
 printf -v launch 'exec '
 printf -v quoted '%q ' "${launch_env[@]}" "$bin" --session "$name"
@@ -65,6 +67,8 @@ done
 [[ -n "$panes_json" ]] || { echo "session API did not become ready" >&2; exit 1; }
 root_pane=$(printf '%s\n' "$panes_json" | jq -r '.result.panes[0].pane_id')
 workspace_id=$("${control_env[@]}" "$bin" workspace list | jq -r '.result.workspaces[0].workspace_id')
+[[ -n "$root_pane" && "$root_pane" != null ]] || { echo "session did not report a root pane" >&2; exit 1; }
+[[ -n "$workspace_id" && "$workspace_id" != null ]] || { echo "session did not report a workspace" >&2; exit 1; }
 pane_file="$state/pane-ids.txt"
 printf '%s\n' "$root_pane" > "$pane_file"
 
@@ -94,24 +98,23 @@ done
 [[ -n "$server_pid" ]] || { echo "could not find server pid" >&2; exit 1; }
 client_pid=$(tmux list-panes -s -t "$name" -F '#{pane_pid}')
 [[ -n "$client_pid" ]] || { echo "could not find client pid" >&2; exit 1; }
-all_pids="$server_pid $client_pid"
+all_pids=("$server_pid" "$client_pid")
 
 sleep "$warmup"
 raw="$out/cpu-raw.txt"
 if [[ $platform == linux ]]; then
-  pid_csv=$(printf '%s\n' $all_pids | paste -sd, -)
+  pid_csv=$(IFS=,; echo "${all_pids[*]}")
   LC_ALL=C pidstat -h -u -p "$pid_csv" 1 "$seconds" > "$raw"
 else
   top_args=(top -l $((seconds + 1)) -s 1 -stats pid,cpu,time -n 2)
-  for pid in $all_pids; do top_args+=(-pid "$pid"); done
+  for pid in "${all_pids[@]}"; do top_args+=(-pid "$pid"); done
   LC_ALL=C "${top_args[@]}" > "$raw"
 fi
 
 mean_linux() {
   awk -v target="$2" '
     /^Linux/ || /^#/ || NF < 5 { next }
-    { found=0; for (i=1; i<=NF; i++) if ($i == target) { found=1; break }
-      if (found && $(NF-2) ~ /^[0-9]+([.][0-9]+)?$/) { sum += $(NF-2); count++ } }
+    $3 == target && $(NF-2) ~ /^[0-9]+([.][0-9]+)?$/ { sum += $(NF-2); count++ }
     END { if (!count) exit 1; printf "%.6f,%d", sum/count, count }
   ' "$1"
 }
@@ -124,7 +127,7 @@ mean_macos() {
 }
 
 total=0
-for pid in $all_pids; do
+for pid in "${all_pids[@]}"; do
   if [[ $platform == linux ]]; then parsed=$(mean_linux "$raw" "$pid"); else parsed=$(mean_macos "$raw" "$pid"); fi
   mean=${parsed%,*}
   samples=${parsed#*,}

scripts/release_perf_producer.pl

@@ -4,7 +4,11 @@ use warnings;
 use Time::HiRes qw(clock_gettime sleep CLOCK_MONOTONIC);
 
 my ($rate, $gate, $label) = @ARGV;
-die "usage: $0 <rate-hz> <gate-file> <label>\n" unless $rate && $gate && $label;
+die "usage: $0 <rate-hz> <gate-file> <label>\n"
+    unless defined $rate && defined $gate && defined $label
+    && length $gate && length $label;
+die "rate must be a positive number\n"
+    unless $rate =~ /\A(?:[0-9]+(?:\.[0-9]*)?|\.[0-9]+)\z/ && $rate > 0;
 
 sleep 0.01 until -e $gate;
 $| = 1;
@@ -15,6 +19,8 @@ while (1) {
     $sequence++;
     printf "\rbench-output-%08d-%s", $sequence, $label;
     $next += $period;
-    my $remaining = $next - clock_gettime(CLOCK_MONOTONIC);
+    my $now = clock_gettime(CLOCK_MONOTONIC);
+    $next = $now + $period if $next < $now - $period;
+    my $remaining = $next - $now;
     sleep $remaining if $remaining > 0;
 }

scripts/release_perf_smoke.sh

@@ -8,9 +8,15 @@ fi
 
 candidate=$(cd "$(dirname "$1")" && pwd)/$(basename "$1")
 [[ -x "$candidate" ]] || { echo "candidate binary is not executable: $candidate" >&2; exit 1; }
-for command in curl jq lsof perl tmux; do
+script_dir=$(cd "$(dirname "$0")" && pwd)
+repo_root=$(cd "$script_dir/.." && pwd)
+baseline=${HERDR_PERF_BASELINE_BIN:-}
+for command in jq lsof perl tmux; do
   command -v "$command" >/dev/null || { echo "required command not found: $command" >&2; exit 1; }
 done
+if [[ -z "$baseline" ]]; then
+  command -v curl >/dev/null || { echo "required command not found: curl" >&2; exit 1; }
+fi
 
 case "$(uname -s)" in
   Linux) platform=linux; command -v pidstat >/dev/null || { echo "required command not found: pidstat" >&2; exit 1; } ;;
@@ -25,12 +31,13 @@ esac
 
 root=$(mktemp -d /var/tmp/herdr-release-perf-smoke.XXXXXX)
 cleanup() { rm -rf "$root"; }
-trap cleanup EXIT INT TERM
+trap cleanup EXIT
+trap 'exit 130' INT
+trap 'exit 143' TERM
 mkdir -p "$root/results"
 
-baseline=${HERDR_PERF_BASELINE_BIN:-}
 if [[ -z "$baseline" ]]; then
-  baseline_version=$(jq -er '.version' website/latest.json)
+  baseline_version=$(jq -er '.version' "$repo_root/website/latest.json")
   baseline="$root/herdr-baseline"
   curl -fL --retry 3 \
     "https://github.com/herdrdev/herdr/releases/download/v${baseline_version}/herdr-${platform}-${arch}" \
@@ -41,7 +48,6 @@ else
 fi
 [[ -x "$baseline" ]] || { echo "baseline binary is not executable: $baseline" >&2; exit 1; }
 
-script_dir=$(cd "$(dirname "$0")" && pwd)
 case_script="$script_dir/release_perf_case.sh"
 seconds=${HERDR_PERF_SAMPLE_SECONDS:-10}
 warmup=${HERDR_PERF_WARMUP_SECONDS:-3}
@@ -68,6 +74,10 @@ printf '%-12s %12s %12s %12s\n' scenario baseline candidate change
 for scenario in hidden50 visible30; do
   baseline_total=$(mean_total baseline "$scenario")
   candidate_total=$(mean_total candidate "$scenario")
+  if awk -v before="$baseline_total" -v after="$candidate_total" 'BEGIN { exit !(before <= 0 || after <= 0) }'; then
+    echo "error: $scenario measured no CPU usage; the benchmark did not exercise the binaries" >&2
+    failed=1
+  fi
   change=$(awk -v before="$baseline_total" -v after="$candidate_total" 'BEGIN { if (before == 0) print "n/a"; else printf "%+.1f%%", (after-before)/before*100 }')
   printf '%-12s %12s %12s %12s\n' "$scenario" "$baseline_total" "$candidate_total" "$change"
   if awk -v before="$baseline_total" -v after="$candidate_total" 'BEGIN { exit !(after > before * 1.25 && after - before > 0.5) }'; then

scripts/test_ui_hot_path_architecture.py

@@ -161,6 +161,7 @@ class UiHotPathArchitectureTests(unittest.TestCase):
         )
 
     def test_app_and_server_avoid_aggregate_terminal_state(self) -> None:
+        self.assertTrue(APP_SERVER_SOURCES, "No app/server Rust sources were discovered")
         violations = find_violations(APP_SERVER_SOURCES, AGGREGATE_STATE_CALLS)
 
         self.assertEqual(

src/server/headless.rs

@@ -9784,15 +9784,17 @@ next_tab = ""
     }
 
     #[test]
-    fn direct_terminal_observer_keeps_hidden_pty_source_renderable() {
-        let (mut server, background_pane) = hidden_pty_visibility_test_server(&[]);
+    fn direct_terminal_observer_keeps_hidden_pty_source_renderable_with_app_client() {
+        let (mut server, background_pane) = hidden_pty_visibility_test_server(&[(120, 40)]);
+        assert!(!server.pty_sources_visible_to_any_render_target(&HashSet::from([background_pane])));
+
         let terminal_id = server.app.state.workspaces[0]
             .terminal_id(background_pane)
             .expect("background terminal id")
             .to_string();
         let (client_tx, _client_control_rx, _client_rx) = test_client_writer();
         server.clients.insert(
-            1,
+            2,
             ClientConnection::new_with_mode(
                 ClientConnectionMode::TerminalObserve { terminal_id },
                 None,
@@ -9800,7 +9802,7 @@ next_tab = ""
                 crate::kitty_graphics::HostCellSize::default(),
                 crate::terminal_theme::TerminalTheme::default(),
                 None,
-                1,
+                2,
                 RenderEncoding::SemanticFrame,
                 false,
                 Some(client_tx),