mirror of
https://github.com/stablyai/orca.git
synced 2026-10-01 16:02:03 +00:00
Jinwoo-H
1c06ea3aa1
The matrix picked its driven request from a hardcoded prefix list and `continue`d
past any family the list did not name. That was 10 of 23 families — every one the
source-control migration added — with no red test to say so, which is why that
migration's mutation evidence came down to single hand-written scenarios.
`replyMatrixSites` now takes every completion step in a family's base scenario:
61 sites instead of 13, one golden per site, no judgement about which request is
the "real" one and nothing to edit when a domain is added. A family that scripts
no reply throws, a repeated request name throws, and a census test asserts every
family in the manifest has a matrix. A variant's downstream replies are marked
`optional` and answered only if the request is outstanding, so a diverged reply
that ends the chain records the truth instead of failing on an unsent request.
The `normal` partition replays the first fulfilled reply the family records for
that request, rather than a payload the test file invented per family. Absent and
null do not count — each is already a partition — so four sites with no other
recorded success are inventoried in REPLY_MATRIX_NORMAL_RESULT_INVENTORY with a
reason each, and an entry whose family later records a success fails.
Two partitions added: a refusal and a transport rejection with no message. That
is the axis that separates a refusal falling back to the screen's copy from a
transport drop surfacing its empty message verbatim; without it the two paths
produce the same text and collapsing them is invisible. Every source-control
family carried a hand-written `*-empty-message` scenario for exactly that.
13 hand-written scenarios the matrix now covers are deleted: 8 `*-empty-message`
cases plus sc-history-rejected, sc-commit-message-null-result, sc-eligibility-
refused, sc-create-stops-on-push-refusal and sc-base-ref-rejected. Kept, with
reasons, are the ones the matrix cannot reach: a different action or action args
(sc-review-commit-*, sc-prefill-*, sc-create-{refused,rejected}-empty-message,
sc-prerequisite-{publish,force-with-lease,skipped}), a payload shape rather than
an envelope shape (sc-review-status-entries-not-array, sc-create-existing-review),
and multi-request combinations (sc-base-ref-{unavailable,repo-fallback},
sc-reveal-timeout).
Goldens: 118 -> 153. All 92 survivors changed by their `recorderSha256` line only;
no recorded observation moved. Re-recorded from the pinned baseline in a separate
tree so the goldens stay attributable to pre-migration product source.
`matrix-hostedreview.create-intent-git.commit-1` fails on this branch, and it is
a true positive: `hostReplyErrorTextOrFallback` stringifies a non-string in-band
host error where main returned `result?.error || fallback` and passed the object
through. Left failing — the fix is a product change, documented in the README.
Claude-Session: https://claude.ai/code/session_01JNnE9qzUZMMnqpZWCqM3nb
679 lines
16 KiB
JSON
679 lines
16 KiB
JSON
{
|
|
"operation": "source-control.commit-message",
|
|
"family": "git.commit-message-ai",
|
|
"namedDeltas": [],
|
|
"runnerVersion": 1,
|
|
"baseline": "7ce8e18d07666955a386fb349c09b1f1e65d1a97",
|
|
"lockfileSha256": "788c1234b38a61fc882fca292341fce6690deee9cc6695673b0575cf2c4cf571",
|
|
"recorderSha256": "674202f169a3eda3555b4cf2a9fe8822b3f9bf6816f90cc0b1aadc9f9334a596",
|
|
"platform": "darwin",
|
|
"scenarioVersion": 1,
|
|
"projectionVersion": 2,
|
|
"goldenFormatVersion": 2,
|
|
"values": {
|
|
"08550c229bdf": {
|
|
"generate": {
|
|
"status": "fulfilled",
|
|
"startedAt": 0,
|
|
"settledAt": 0,
|
|
"value": {
|
|
"error": "No commit message generated",
|
|
"success": false
|
|
}
|
|
}
|
|
},
|
|
"1400db9aa00b": {
|
|
"generate": {
|
|
"status": "fulfilled",
|
|
"startedAt": 0,
|
|
"settledAt": 0,
|
|
"value": {
|
|
"error": "outer refused",
|
|
"success": false
|
|
}
|
|
}
|
|
},
|
|
"19fcb56780ba": [
|
|
{
|
|
"name": "git.generateCommitMessage#1",
|
|
"args": [
|
|
{
|
|
"name": "method",
|
|
"value": "git.generateCommitMessage"
|
|
},
|
|
{
|
|
"name": "params",
|
|
"value": {
|
|
"worktree": "id:repo42::/p"
|
|
}
|
|
},
|
|
{
|
|
"name": "options",
|
|
"value": {
|
|
"$rpc": "absent"
|
|
}
|
|
}
|
|
],
|
|
"settlement": {
|
|
"status": "fulfilled",
|
|
"startedAt": 0,
|
|
"settledAt": 0,
|
|
"value": {
|
|
"id": "frame-1",
|
|
"ok": true,
|
|
"result": {
|
|
"error": {
|
|
"message": "inner refused"
|
|
},
|
|
"ok": false
|
|
}
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"2a3bf1815766": [
|
|
{
|
|
"name": "git.generateCommitMessage#1",
|
|
"args": [
|
|
{
|
|
"name": "method",
|
|
"value": "git.generateCommitMessage"
|
|
},
|
|
{
|
|
"name": "params",
|
|
"value": {
|
|
"worktree": "id:repo42::/p"
|
|
}
|
|
},
|
|
{
|
|
"name": "options",
|
|
"value": {
|
|
"$rpc": "absent"
|
|
}
|
|
}
|
|
],
|
|
"settlement": {
|
|
"status": "fulfilled",
|
|
"startedAt": 0,
|
|
"settledAt": 0,
|
|
"value": {
|
|
"id": "frame-1",
|
|
"ok": true,
|
|
"result": {
|
|
"error": "inner refused",
|
|
"ok": false
|
|
}
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"2abe7c0c3763": [
|
|
{
|
|
"name": "git.generateCommitMessage#1",
|
|
"args": [
|
|
{
|
|
"name": "method",
|
|
"value": "git.generateCommitMessage"
|
|
},
|
|
{
|
|
"name": "params",
|
|
"value": {
|
|
"worktree": "id:repo42::/p"
|
|
}
|
|
},
|
|
{
|
|
"name": "options",
|
|
"value": {
|
|
"$rpc": "absent"
|
|
}
|
|
}
|
|
],
|
|
"settlement": {
|
|
"status": "pending",
|
|
"startedAt": 0
|
|
}
|
|
}
|
|
],
|
|
"2f20bda3b3f5": [
|
|
{
|
|
"name": "git.generateCommitMessage#1",
|
|
"args": [
|
|
{
|
|
"name": "method",
|
|
"value": "git.generateCommitMessage"
|
|
},
|
|
{
|
|
"name": "params",
|
|
"value": {
|
|
"worktree": "id:repo42::/p"
|
|
}
|
|
},
|
|
{
|
|
"name": "options",
|
|
"value": {
|
|
"$rpc": "absent"
|
|
}
|
|
}
|
|
],
|
|
"settlement": {
|
|
"status": "fulfilled",
|
|
"startedAt": 0,
|
|
"settledAt": 0,
|
|
"value": {
|
|
"error": {
|
|
"code": "refused",
|
|
"message": "outer refused"
|
|
},
|
|
"id": "frame-1",
|
|
"ok": false
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"33bfe371e5b0": {
|
|
"generate": {
|
|
"status": "fulfilled",
|
|
"startedAt": 0,
|
|
"settledAt": 0,
|
|
"value": {
|
|
"message": "feat: recorded",
|
|
"success": true
|
|
}
|
|
}
|
|
},
|
|
"3ef8a5f65bc8": {
|
|
"generated": {
|
|
"message": "feat: recorded",
|
|
"success": true
|
|
}
|
|
},
|
|
"3f0453e3c8f9": {
|
|
"generate": {
|
|
"status": "rejected",
|
|
"startedAt": 0,
|
|
"settledAt": 0,
|
|
"error": {
|
|
"category": "Error",
|
|
"message": "transport failure",
|
|
"isRpcDeliveryUnknown": true
|
|
}
|
|
}
|
|
},
|
|
"4f53cda18c2b": [],
|
|
"5246b9fd2d12": {
|
|
"generated": {
|
|
"error": "outer refused",
|
|
"success": false
|
|
}
|
|
},
|
|
"55963844f040": [
|
|
{
|
|
"name": "git.generateCommitMessage#1",
|
|
"args": [
|
|
{
|
|
"name": "method",
|
|
"value": "git.generateCommitMessage"
|
|
},
|
|
{
|
|
"name": "params",
|
|
"value": {
|
|
"worktree": "id:repo42::/p"
|
|
}
|
|
},
|
|
{
|
|
"name": "options",
|
|
"value": {
|
|
"$rpc": "absent"
|
|
}
|
|
}
|
|
],
|
|
"settlement": {
|
|
"status": "rejected",
|
|
"startedAt": 0,
|
|
"settledAt": 0,
|
|
"error": {
|
|
"category": "Error",
|
|
"message": "",
|
|
"isRpcDeliveryUnknown": true
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"5becddaebf57": [
|
|
{
|
|
"name": "git.generateCommitMessage#1",
|
|
"json": "{\"id\":\"frame-1\",\"deviceToken\":\"recording-device\",\"method\":\"git.generateCommitMessage\",\"params\":{\"worktree\":\"id:repo42::/p\"}}"
|
|
}
|
|
],
|
|
"613b061103e5": [
|
|
{
|
|
"name": "git.generateCommitMessage#1",
|
|
"args": [
|
|
{
|
|
"name": "method",
|
|
"value": "git.generateCommitMessage"
|
|
},
|
|
{
|
|
"name": "params",
|
|
"value": {
|
|
"worktree": "id:repo42::/p"
|
|
}
|
|
},
|
|
{
|
|
"name": "options",
|
|
"value": {
|
|
"$rpc": "absent"
|
|
}
|
|
}
|
|
],
|
|
"settlement": {
|
|
"status": "fulfilled",
|
|
"startedAt": 0,
|
|
"settledAt": 0,
|
|
"value": {
|
|
"id": "frame-1",
|
|
"ok": true,
|
|
"result": {
|
|
"message": "feat: recorded",
|
|
"success": true
|
|
}
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"61c7ca32577a": {
|
|
"generate": {
|
|
"status": "rejected",
|
|
"startedAt": 0,
|
|
"settledAt": 0,
|
|
"error": {
|
|
"category": "Error",
|
|
"message": "",
|
|
"isRpcDeliveryUnknown": true
|
|
}
|
|
}
|
|
},
|
|
"6453ff669a2f": {
|
|
"generated": {
|
|
"error": "Failed to generate commit message",
|
|
"success": false
|
|
}
|
|
},
|
|
"647e7870fa6f": {
|
|
"generate": {
|
|
"status": "fulfilled",
|
|
"startedAt": 0,
|
|
"settledAt": 0,
|
|
"value": {
|
|
"error": "Unknown method",
|
|
"success": false
|
|
}
|
|
}
|
|
},
|
|
"8be8f41478d8": [
|
|
{
|
|
"name": "git.generateCommitMessage#1",
|
|
"args": [
|
|
{
|
|
"name": "method",
|
|
"value": "git.generateCommitMessage"
|
|
},
|
|
{
|
|
"name": "params",
|
|
"value": {
|
|
"worktree": "id:repo42::/p"
|
|
}
|
|
},
|
|
{
|
|
"name": "options",
|
|
"value": {
|
|
"$rpc": "absent"
|
|
}
|
|
}
|
|
],
|
|
"settlement": {
|
|
"status": "rejected",
|
|
"startedAt": 0,
|
|
"settledAt": 0,
|
|
"error": {
|
|
"category": "Error",
|
|
"message": "transport failure",
|
|
"isRpcDeliveryUnknown": true
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"91471f1c9738": [
|
|
{
|
|
"name": "git.generateCommitMessage#1",
|
|
"args": [
|
|
{
|
|
"name": "method",
|
|
"value": "git.generateCommitMessage"
|
|
},
|
|
{
|
|
"name": "params",
|
|
"value": {
|
|
"worktree": "id:repo42::/p"
|
|
}
|
|
},
|
|
{
|
|
"name": "options",
|
|
"value": {
|
|
"$rpc": "absent"
|
|
}
|
|
}
|
|
],
|
|
"settlement": {
|
|
"status": "fulfilled",
|
|
"startedAt": 0,
|
|
"settledAt": 0,
|
|
"value": {
|
|
"id": "frame-1",
|
|
"ok": true,
|
|
"result": {
|
|
"$rpc": "null"
|
|
}
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"915059a8a000": {
|
|
"generated": {
|
|
"error": "Unknown method",
|
|
"success": false
|
|
}
|
|
},
|
|
"927b68030eb2": [
|
|
{
|
|
"name": "git.generateCommitMessage#1",
|
|
"args": [
|
|
{
|
|
"name": "method",
|
|
"value": "git.generateCommitMessage"
|
|
},
|
|
{
|
|
"name": "params",
|
|
"value": {
|
|
"worktree": "id:repo42::/p"
|
|
}
|
|
},
|
|
{
|
|
"name": "options",
|
|
"value": {
|
|
"$rpc": "absent"
|
|
}
|
|
}
|
|
],
|
|
"settlement": {
|
|
"status": "fulfilled",
|
|
"startedAt": 0,
|
|
"settledAt": 0,
|
|
"value": {
|
|
"id": "frame-1",
|
|
"ok": true
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"93f07b82cf9c": [
|
|
{
|
|
"name": "git.generateCommitMessage#1",
|
|
"args": [
|
|
{
|
|
"name": "method",
|
|
"value": "git.generateCommitMessage"
|
|
},
|
|
{
|
|
"name": "params",
|
|
"value": {
|
|
"worktree": "id:repo42::/p"
|
|
}
|
|
},
|
|
{
|
|
"name": "options",
|
|
"value": {
|
|
"$rpc": "absent"
|
|
}
|
|
}
|
|
],
|
|
"settlement": {
|
|
"status": "fulfilled",
|
|
"startedAt": 0,
|
|
"settledAt": 0,
|
|
"value": {
|
|
"error": {
|
|
"code": "refused",
|
|
"message": ""
|
|
},
|
|
"id": "frame-1",
|
|
"ok": false
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"adb40821f3e2": {
|
|
"generated": "ungenerated"
|
|
},
|
|
"b308d9d641d6": {
|
|
"generate": {
|
|
"status": "pending",
|
|
"startedAt": 0
|
|
}
|
|
},
|
|
"c44fce5322f2": {
|
|
"generate": {
|
|
"status": "fulfilled",
|
|
"startedAt": 0,
|
|
"settledAt": 0,
|
|
"value": {
|
|
"error": "Failed to generate commit message",
|
|
"success": false
|
|
}
|
|
}
|
|
},
|
|
"ea1e8dc75c9d": [
|
|
{
|
|
"name": "git.generateCommitMessage#1",
|
|
"args": [
|
|
{
|
|
"name": "method",
|
|
"value": "git.generateCommitMessage"
|
|
},
|
|
{
|
|
"name": "params",
|
|
"value": {
|
|
"worktree": "id:repo42::/p"
|
|
}
|
|
},
|
|
{
|
|
"name": "options",
|
|
"value": {
|
|
"$rpc": "absent"
|
|
}
|
|
}
|
|
],
|
|
"settlement": {
|
|
"status": "fulfilled",
|
|
"startedAt": 0,
|
|
"settledAt": 0,
|
|
"value": {
|
|
"id": "frame-1",
|
|
"ok": true,
|
|
"result": {
|
|
"error": "refused"
|
|
}
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"eb5bf7333dc4": [
|
|
{
|
|
"name": "git.generateCommitMessage#1",
|
|
"args": [
|
|
{
|
|
"name": "method",
|
|
"value": "git.generateCommitMessage"
|
|
},
|
|
{
|
|
"name": "params",
|
|
"value": {
|
|
"worktree": "id:repo42::/p"
|
|
}
|
|
},
|
|
{
|
|
"name": "options",
|
|
"value": {
|
|
"$rpc": "absent"
|
|
}
|
|
}
|
|
],
|
|
"settlement": {
|
|
"status": "fulfilled",
|
|
"startedAt": 0,
|
|
"settledAt": 0,
|
|
"value": {
|
|
"error": {
|
|
"code": "method_not_found",
|
|
"message": "Unknown method"
|
|
},
|
|
"id": "frame-1",
|
|
"ok": false
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"faad14b9f95d": {
|
|
"generated": {
|
|
"error": "No commit message generated",
|
|
"success": false
|
|
}
|
|
}
|
|
},
|
|
"recording": {
|
|
"scenario": "matrix-git.commit-message-ai-git.generatecommitmessage-1",
|
|
"checkpoints": [
|
|
{
|
|
"id": "sc-commit-message-generated.prelude:pending",
|
|
"observation": {
|
|
"sender": "2abe7c0c3763",
|
|
"payloads": "5becddaebf57",
|
|
"settlements": "b308d9d641d6",
|
|
"state": "adb40821f3e2",
|
|
"effects": "4f53cda18c2b"
|
|
}
|
|
},
|
|
{
|
|
"id": "sc-commit-message-generated.normal:settled",
|
|
"observation": {
|
|
"sender": "613b061103e5",
|
|
"payloads": "5becddaebf57",
|
|
"settlements": "33bfe371e5b0",
|
|
"state": "3ef8a5f65bc8",
|
|
"effects": "4f53cda18c2b"
|
|
}
|
|
},
|
|
{
|
|
"id": "sc-commit-message-generated.result-absent:settled",
|
|
"observation": {
|
|
"sender": "927b68030eb2",
|
|
"payloads": "5becddaebf57",
|
|
"settlements": "c44fce5322f2",
|
|
"state": "6453ff669a2f",
|
|
"effects": "4f53cda18c2b"
|
|
}
|
|
},
|
|
{
|
|
"id": "sc-commit-message-generated.result-null:settled",
|
|
"observation": {
|
|
"sender": "91471f1c9738",
|
|
"payloads": "5becddaebf57",
|
|
"settlements": "c44fce5322f2",
|
|
"state": "6453ff669a2f",
|
|
"effects": "4f53cda18c2b"
|
|
}
|
|
},
|
|
{
|
|
"id": "sc-commit-message-generated.inner-ok-missing:settled",
|
|
"observation": {
|
|
"sender": "ea1e8dc75c9d",
|
|
"payloads": "5becddaebf57",
|
|
"settlements": "08550c229bdf",
|
|
"state": "faad14b9f95d",
|
|
"effects": "4f53cda18c2b"
|
|
}
|
|
},
|
|
{
|
|
"id": "sc-commit-message-generated.inner-false-string-error:settled",
|
|
"observation": {
|
|
"sender": "2a3bf1815766",
|
|
"payloads": "5becddaebf57",
|
|
"settlements": "08550c229bdf",
|
|
"state": "faad14b9f95d",
|
|
"effects": "4f53cda18c2b"
|
|
}
|
|
},
|
|
{
|
|
"id": "sc-commit-message-generated.inner-false-object-error:settled",
|
|
"observation": {
|
|
"sender": "19fcb56780ba",
|
|
"payloads": "5becddaebf57",
|
|
"settlements": "08550c229bdf",
|
|
"state": "faad14b9f95d",
|
|
"effects": "4f53cda18c2b"
|
|
}
|
|
},
|
|
{
|
|
"id": "sc-commit-message-generated.outer-refused:settled",
|
|
"observation": {
|
|
"sender": "2f20bda3b3f5",
|
|
"payloads": "5becddaebf57",
|
|
"settlements": "1400db9aa00b",
|
|
"state": "5246b9fd2d12",
|
|
"effects": "4f53cda18c2b"
|
|
}
|
|
},
|
|
{
|
|
"id": "sc-commit-message-generated.outer-refused-no-message:settled",
|
|
"observation": {
|
|
"sender": "93f07b82cf9c",
|
|
"payloads": "5becddaebf57",
|
|
"settlements": "c44fce5322f2",
|
|
"state": "6453ff669a2f",
|
|
"effects": "4f53cda18c2b"
|
|
}
|
|
},
|
|
{
|
|
"id": "sc-commit-message-generated.method-not-found:settled",
|
|
"observation": {
|
|
"sender": "eb5bf7333dc4",
|
|
"payloads": "5becddaebf57",
|
|
"settlements": "647e7870fa6f",
|
|
"state": "915059a8a000",
|
|
"effects": "4f53cda18c2b"
|
|
}
|
|
},
|
|
{
|
|
"id": "sc-commit-message-generated.transport-rejection:settled",
|
|
"observation": {
|
|
"sender": "8be8f41478d8",
|
|
"payloads": "5becddaebf57",
|
|
"settlements": "3f0453e3c8f9",
|
|
"state": "adb40821f3e2",
|
|
"effects": "4f53cda18c2b"
|
|
}
|
|
},
|
|
{
|
|
"id": "sc-commit-message-generated.transport-rejection-no-message:settled",
|
|
"observation": {
|
|
"sender": "55963844f040",
|
|
"payloads": "5becddaebf57",
|
|
"settlements": "61c7ca32577a",
|
|
"state": "adb40821f3e2",
|
|
"effects": "4f53cda18c2b"
|
|
}
|
|
}
|
|
]
|
|
}
|
|
}
|