mirror of
https://github.com/herdrdev/herdr.git
synced 2026-09-21 08:01:05 +00:00
test: focus agent detection coverage on engine contracts (#4338)
This commit is contained in:
Can Celik
@@ -198,7 +198,9 @@ manual testing, reset `C:\work\repo` back to a clean checkout before finishing.
|
||||
|
||||
Agent detection changes should use the manifest hot-reload loop. Use the project-local `herdr-throwaway-repro` skill to create a disposable named session and drive the real agent UI through Herdr's CLI/API into the target state. Read the pane with `herdr agent read <pane> --source detection --format text` and inspect matching with `herdr agent explain <pane> --json`. Update the bundled manifest in `src/detect/manifests/<agent>.toml`, copy that manifest to the local override path at `~/.config/herdr/agent-detection/<agent>.toml`, then run `herdr server reload-agent-manifests` against the session under test. Before writing the override, check whether one already exists; never overwrite or remove a pre-existing override without alignment. Once the rule is correct, remove the temporary override or restore the previous one exactly so the committed bundled manifest remains the source of truth.
|
||||
|
||||
Do not add large agent-specific full-screen fixture suites for routine manifest tuning. Keep Rust tests focused on manifest parsing, rule semantics, skip-state semantics, source precedence, cache reload behavior, and update flow. Use live pane reads for agent-specific screen evidence.
|
||||
Unit-test Herdr's detection engine, not individual CLI agents' screen or title conventions. Use synthetic manifests and minimal input strings to test parsing, regions, matching, AND/OR/NOT gates, rule priority, skip-state semantics, source precedence, cache reload behavior, and update flow. Keep bundled-manifest schema validation, process identification, and integration hook/protocol tests. Do not add tests that classify captured or invented CLI screens against bundled agent rules, or freeze an agent's specific detection rule IDs and priorities.
|
||||
|
||||
Validate agent-specific detection behavior with live smoke tests through the manifest hot-reload loop above. Exercise the changed state and nearby transitions (idle, working, blocked, and background work where supported), including relevant optional OSC settings. Record the CLI version, observed signals, and outcomes. Passing engine tests proves the rules execute as written; it does not prove compatibility with the current CLI.
|
||||
|
||||
`distribution/agent-detection/` is the remotely published catalog for released clients. Keep changes for already released agents aligned with their bundled manifests unless the validator records an exact compatibility exception. A newly bundled agent that current stable clients cannot identify may remain unpublished behind an exact version-and-digest exception, but it must be added to the catalog and the exception removed before the first stable release that ships it. `just release-docs-check` enforces that no unpublished exceptions remain.
|
||||
|
||||
|
||||
+209
-970
File diff suppressed because it is too large
Load Diff
@@ -893,14 +893,6 @@ mod tests {
|
||||
std::env::temp_dir().join(unique)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn moved_agent_detection_routes_through_production_dispatch() {
|
||||
let detection = detect_agent(Some(Agent::Pi), "Working...");
|
||||
|
||||
assert_eq!(detection.state, AgentState::Working);
|
||||
assert!(detection.visible_working);
|
||||
}
|
||||
|
||||
// ---- Agent identification ----
|
||||
|
||||
#[test]
|
||||
@@ -1278,101 +1270,6 @@ mod tests {
|
||||
assert_eq!(identify_agent_in_job(&source_checkout), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn letta_manifest_detects_observed_working_and_idle_chrome() {
|
||||
let empty = manifest::explain(Agent::Letta, "");
|
||||
assert_eq!(empty.state, AgentState::Unknown);
|
||||
assert_eq!(
|
||||
empty.matched_rule.as_ref().map(|rule| rule.id.as_str()),
|
||||
Some("no_live_state_evidence")
|
||||
);
|
||||
for screen in [
|
||||
"✻ Thinking…\nTutor is reflecting… (esc to interrupt · 2m 3s)",
|
||||
"My Tutor is thinking about thinking… (esc to interrupt · 3s)",
|
||||
"Tutor is calibrating… (interrupting)",
|
||||
"• Run Compile the integration\n└ Running... (1s)",
|
||||
] {
|
||||
assert_eq!(
|
||||
detect_state(Some(Agent::Letta), screen),
|
||||
AgentState::Working
|
||||
);
|
||||
}
|
||||
assert_eq!(
|
||||
detect_state(
|
||||
Some(Agent::Letta),
|
||||
"────────────────\n› Try \"debug this error\"\n────────────────\nTutor · No model selected"
|
||||
),
|
||||
AgentState::Idle
|
||||
);
|
||||
assert_eq!(
|
||||
detect_state(
|
||||
Some(Agent::Letta),
|
||||
"────────────────\n› explain this code\n────────────────\nTutor · No model selected"
|
||||
),
|
||||
AgentState::Unknown
|
||||
);
|
||||
let selector = manifest::explain(
|
||||
Agent::Letta,
|
||||
"8 pinned agents available.\n\n> Resume Bob (pinned)\n View all 8 profiles\n Create a new agent (--new)\n\n ↑↓ navigate · Enter select · Esc exit",
|
||||
);
|
||||
assert_eq!(selector.state, AgentState::Unknown);
|
||||
assert_eq!(
|
||||
selector.matched_rule.as_ref().map(|rule| rule.id.as_str()),
|
||||
Some("profile_selector")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn letta_manifest_uses_osc_activity_and_approval_signals() {
|
||||
let idle_screen =
|
||||
"────────────────\n› Try \"debug this error\"\n────────────────\nTutor · GPT-5.5";
|
||||
|
||||
for title in ["⠋ Tutor", "project | ⠏ Tutor"] {
|
||||
let detection = detect_agent_with_osc(Some(Agent::Letta), idle_screen, title, "");
|
||||
assert_eq!(detection.state, AgentState::Working);
|
||||
assert!(detection.visible_working);
|
||||
}
|
||||
|
||||
for title in [
|
||||
"[ ! ] Action Required | Tutor",
|
||||
"[ . ] Action Required | Tutor",
|
||||
] {
|
||||
let detection = detect_agent_with_osc(Some(Agent::Letta), idle_screen, title, "");
|
||||
assert_eq!(detection.state, AgentState::Blocked);
|
||||
assert!(detection.visible_blocker);
|
||||
}
|
||||
|
||||
let detection = detect_agent_with_osc(Some(Agent::Letta), idle_screen, "Tutor", "4;3;0");
|
||||
assert_eq!(detection.state, AgentState::Blocked);
|
||||
assert!(detection.visible_blocker);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn letta_manifest_detects_observed_command_approval() {
|
||||
let approval = r#"✻ Thinking…
|
||||
|
||||
────────────────────────────────────────────────────────────────
|
||||
Run this command?
|
||||
|
||||
$ rm -f /var/tmp/herdr-blocked-capture-never-created
|
||||
|
||||
❯ 1. Yes
|
||||
2. No, and tell Letta Code what to do differently
|
||||
|
||||
Enter to select · Esc to cancel"#;
|
||||
assert_eq!(
|
||||
detect_state(Some(Agent::Letta), approval),
|
||||
AgentState::Blocked
|
||||
);
|
||||
assert_eq!(
|
||||
detect_state(
|
||||
Some(Agent::Letta),
|
||||
"The user asked: Run this command?\n›\n────\nTutor · No model selected"
|
||||
),
|
||||
AgentState::Idle
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn identify_agent_in_job_detects_windows_cursor_install() {
|
||||
let job = crate::platform::ForegroundJob {
|
||||
|
||||
Reference in New Issue
Block a user