From a793c39e1898d0a02ae22ba31a4591c9f9f7bb1a Mon Sep 17 00:00:00 2001 From: Jinwoo-H Date: Thu, 2 Jul 2026 21:16:47 -0400 Subject: [PATCH] Add cross-terminal baseline results (Orca 1.4.91 prod vs Terminal.app vs Ghostty) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Headline: Orca DSR latency under 1MB/s agent-TUI load is p50 134ms / p99 292ms vs 0.45ms (Terminal.app) and 0.21ms (Ghostty). Idle latency is fine (0.69ms p50) — the problem is queueing under load, not the pipeline hop. agent-tui fenced throughput: Orca 2.0 MB/s vs Terminal.app 37 MB/s, Ghostty 78 MB/s. Co-authored-by: Orca --- ...seline-jul02-2026-07-03T01-13-35-758Z.json | 116 ++++++++++++++++++ ...seline-jul02-2026-07-03T01-14-56-619Z.json | 116 ++++++++++++++++++ ...seline-jul02-2026-07-03T01-15-36-499Z.json | 116 ++++++++++++++++++ 3 files changed, 348 insertions(+) create mode 100644 tools/benchmarks/results/terminal-pipeline-baseline-jul02-2026-07-03T01-13-35-758Z.json create mode 100644 tools/benchmarks/results/terminal-pipeline-baseline-jul02-2026-07-03T01-14-56-619Z.json create mode 100644 tools/benchmarks/results/terminal-pipeline-baseline-jul02-2026-07-03T01-15-36-499Z.json diff --git a/tools/benchmarks/results/terminal-pipeline-baseline-jul02-2026-07-03T01-13-35-758Z.json b/tools/benchmarks/results/terminal-pipeline-baseline-jul02-2026-07-03T01-13-35-758Z.json new file mode 100644 index 00000000000..67efa785e5b --- /dev/null +++ b/tools/benchmarks/results/terminal-pipeline-baseline-jul02-2026-07-03T01-13-35-758Z.json @@ -0,0 +1,116 @@ +{ + "benchmark": "terminal-pipeline", + "label": "baseline-jul02", + "timestamp": "2026-07-03T01:13:35.758Z", + "terminal": { + "program": "Orca", + "version": "1.4.91", + "term": "xterm-256color", + "cols": 114, + "rows": 85, + "tty": true + }, + "machine": { + "platform": "darwin", + "arch": "arm64", + "cpu": "Apple M4", + "release": "25.2.0", + "node": "v26.0.0" + }, + "config": { + "sizeMb": 10, + "iterations": 5, + "dsrCount": 200 + }, + "dsrIdle": { + "count": 200, + "p50": 0.6944579999999974, + "p90": 5.172749999999951, + "p99": 22.682000000000016, + "max": 29.694584000000077, + "timeouts": 0 + }, + "throughput": [ + { + "fixture": "ascii-log", + "fixtureBytes": 10485767, + "wallMs": { + "count": 5, + "p50": 769.1521660000003, + "p90": 930.617459, + "p99": 930.617459, + "max": 930.617459 + }, + "mbPerSec": { + "count": 5, + "p50": 13.001337209677974, + "p90": 13.263896170152437, + "p99": 13.263896170152437, + "max": 13.263896170152437 + } + }, + { + "fixture": "cjk-emoji", + "fixtureBytes": 10485782, + "wallMs": { + "count": 5, + "p50": 666.630083, + "p90": 769.2970410000007, + "p99": 769.2970410000007, + "max": 769.2970410000007 + }, + "mbPerSec": { + "count": 5, + "p50": 15.000854650651823, + "p90": 17.00796526725481, + "p99": 17.00796526725481, + "max": 17.00796526725481 + } + }, + { + "fixture": "agent-tui", + "fixtureBytes": 10486459, + "wallMs": { + "count": 5, + "p50": 4935.6103330000005, + "p90": 5991.0535, + "p99": 5991.0535, + "max": 5991.0535 + }, + "mbPerSec": { + "count": 5, + "p50": 2.0262269392463335, + "p90": 2.1317889684101723, + "p99": 2.1317889684101723, + "max": 2.1317889684101723 + } + }, + { + "fixture": "styles-stress", + "fixtureBytes": 10485774, + "wallMs": { + "count": 5, + "p50": 1277.6687080000047, + "p90": 2020.588208000001, + "p99": 2020.588208000001, + "max": 2020.588208000001 + }, + "mbPerSec": { + "count": 5, + "p50": 7.826765490010258, + "p90": 8.91418468880451, + "p99": 8.91418468880451, + "max": 8.91418468880451 + } + } + ], + "dsrUnderLoad": { + "paceBytesPerSec": 1048576, + "count": 50, + "p50": 133.92545800000516, + "p90": 184.93379200000345, + "p99": 292.1940420000028, + "max": 292.1940420000028, + "timeouts": 0 + } +} diff --git a/tools/benchmarks/results/terminal-pipeline-baseline-jul02-2026-07-03T01-14-56-619Z.json b/tools/benchmarks/results/terminal-pipeline-baseline-jul02-2026-07-03T01-14-56-619Z.json new file mode 100644 index 00000000000..b260417b760 --- /dev/null +++ b/tools/benchmarks/results/terminal-pipeline-baseline-jul02-2026-07-03T01-14-56-619Z.json @@ -0,0 +1,116 @@ +{ + "benchmark": "terminal-pipeline", + "label": "baseline-jul02", + "timestamp": "2026-07-03T01:14:56.619Z", + "terminal": { + "program": "Apple_Terminal", + "version": "466", + "term": "xterm-256color", + "cols": 80, + "rows": 24, + "tty": true + }, + "machine": { + "platform": "darwin", + "arch": "arm64", + "cpu": "Apple M4", + "release": "25.2.0", + "node": "v26.0.0" + }, + "config": { + "sizeMb": 10, + "iterations": 5, + "dsrCount": 200 + }, + "dsrIdle": { + "count": 200, + "p50": 0.35495800000001054, + "p90": 0.4861670000000231, + "p99": 0.6817499999999654, + "max": 0.8802500000000464, + "timeouts": 0 + }, + "throughput": [ + { + "fixture": "ascii-log", + "fixtureBytes": 10485767, + "wallMs": { + "count": 5, + "p50": 255.169625, + "p90": 261.48012500000004, + "p99": 261.48012500000004, + "max": 261.48012500000004 + }, + "mbPerSec": { + "count": 5, + "p50": 39.18964365652931, + "p90": 40.31244151336906, + "p99": 40.31244151336906, + "max": 40.31244151336906 + } + }, + { + "fixture": "cjk-emoji", + "fixtureBytes": 10485782, + "wallMs": { + "count": 5, + "p50": 268.68208300000015, + "p90": 271.7308750000002, + "p99": 271.7308750000002, + "max": 271.7308750000002 + }, + "mbPerSec": { + "count": 5, + "p50": 37.218786117699395, + "p90": 43.7027084951767, + "p99": 43.7027084951767, + "max": 43.7027084951767 + } + }, + { + "fixture": "agent-tui", + "fixtureBytes": 10486147, + "wallMs": { + "count": 5, + "p50": 268.55675000000065, + "p90": 272.7946670000001, + "p99": 272.7946670000001, + "max": 272.7946670000001 + }, + "mbPerSec": { + "count": 5, + "p50": 37.23745194250536, + "p90": 37.74582959818335, + "p99": 37.74582959818335, + "max": 37.74582959818335 + } + }, + { + "fixture": "styles-stress", + "fixtureBytes": 10485769, + "wallMs": { + "count": 5, + "p50": 456.44829200000004, + "p90": 465.0257079999992, + "p99": 465.0257079999992, + "max": 465.0257079999992 + }, + "mbPerSec": { + "count": 5, + "p50": 21.908305405749765, + "p90": 22.363718752676004, + "p99": 22.363718752676004, + "max": 22.363718752676004 + } + } + ], + "dsrUnderLoad": { + "paceBytesPerSec": 1048576, + "count": 50, + "p50": 0.44887500000004366, + "p90": 3.032791999999972, + "p99": 7.881083000000217, + "max": 7.881083000000217, + "timeouts": 0 + } +} diff --git a/tools/benchmarks/results/terminal-pipeline-baseline-jul02-2026-07-03T01-15-36-499Z.json b/tools/benchmarks/results/terminal-pipeline-baseline-jul02-2026-07-03T01-15-36-499Z.json new file mode 100644 index 00000000000..9b6132816b9 --- /dev/null +++ b/tools/benchmarks/results/terminal-pipeline-baseline-jul02-2026-07-03T01-15-36-499Z.json @@ -0,0 +1,116 @@ +{ + "benchmark": "terminal-pipeline", + "label": "baseline-jul02", + "timestamp": "2026-07-03T01:15:36.499Z", + "terminal": { + "program": "ghostty", + "version": "1.3.1", + "term": "xterm-ghostty", + "cols": 195, + "rows": 50, + "tty": true + }, + "machine": { + "platform": "darwin", + "arch": "arm64", + "cpu": "Apple M4", + "release": "25.2.0", + "node": "v26.0.0" + }, + "config": { + "sizeMb": 10, + "iterations": 5, + "dsrCount": 200 + }, + "dsrIdle": { + "count": 200, + "p50": 0.19437500000003638, + "p90": 0.28274999999996453, + "p99": 0.7157499999999573, + "max": 1.139250000000004, + "timeouts": 0 + }, + "throughput": [ + { + "fixture": "ascii-log", + "fixtureBytes": 10485767, + "wallMs": { + "count": 5, + "p50": 107.54324999999994, + "p90": 110.16970800000013, + "p99": 110.16970800000013, + "max": 110.16970800000013 + }, + "mbPerSec": { + "count": 5, + "p50": 92.98590730445864, + "p90": 94.26480023982661, + "p99": 94.26480023982661, + "max": 94.26480023982661 + } + }, + { + "fixture": "cjk-emoji", + "fixtureBytes": 10485782, + "wallMs": { + "count": 5, + "p50": 94.36416700000018, + "p90": 95.08550000000014, + "p99": 95.08550000000014, + "max": 95.08550000000014 + }, + "mbPerSec": { + "count": 5, + "p50": 105.97265147092267, + "p90": 121.20894788314553, + "p99": 121.20894788314553, + "max": 121.20894788314553 + } + }, + { + "fixture": "agent-tui", + "fixtureBytes": 10486087, + "wallMs": { + "count": 5, + "p50": 128.88804200000004, + "p90": 157.06174999999985, + "p99": 157.06174999999985, + "max": 157.06174999999985 + }, + "mbPerSec": { + "count": 5, + "p50": 77.58913624819797, + "p90": 83.67685741331184, + "p99": 83.67685741331184, + "max": 83.67685741331184 + } + }, + { + "fixture": "styles-stress", + "fixtureBytes": 10485775, + "wallMs": { + "count": 5, + "p50": 93.27766700000029, + "p90": 94.88462499999969, + "p99": 94.88462499999969, + "max": 94.88462499999969 + }, + "mbPerSec": { + "count": 5, + "p50": 107.20695131788315, + "p90": 108.70407862679501, + "p99": 108.70407862679501, + "max": 108.70407862679501 + } + } + ], + "dsrUnderLoad": { + "paceBytesPerSec": 1048576, + "count": 50, + "p50": 0.20724999999947613, + "p90": 0.31195800000023155, + "p99": 6.07287499999984, + "max": 6.07287499999984, + "timeouts": 0 + } +}