starred/openobserve
1
0
Fork 0
mirror of https://github.com/openobserve/openobserve.git synced 2026-11-04 16:09:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/main' into fix/synthetics-not-measured

Browse Source 
# Conflicts:
#	src/config/src/config.rs
#	src/infra/src/table/migration/mod.rs
This commit is contained in:
Loakesh Indiran
2026-11-03 19:52:59 +05:30
parent 8e0ce98317 1d2a6b70d0
commit d69c89892e
235 changed files with 23090 additions and 1500 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/ai-code-review.yml
+42 -3
View File
@@ -68,7 +68,6 @@ jobs:
- name: anthropic
provider_id: anthropic-review
model_id: claude-sonnet-5-5
model_variant: high
api_key_env: ANTHROPIC_REVIEW_API_KEY
marker: "<!-- ai-code-review -->"
steps:
@@ -109,6 +108,11 @@ jobs:
fi
merge_base="$(gh api "repos/$REPO/compare/$base_sha...$head_sha" --jq '.merge_base_commit.sha' || true)"
echo "merge_base=$merge_base" >> "$GITHUB_OUTPUT"
{
echo "author=$(jq -r '.user.login // ""' <<<"$pr_json")"
echo "changed_files=$(jq -r '.changed_files // ""' <<<"$pr_json")"
echo "commit_count=$(jq -r '.commits // ""' <<<"$pr_json")"
} >> "$GITHUB_OUTPUT"
echo "REVIEW_RUN=true" >> "$GITHUB_ENV"
# If minting fails (app not installed, bad key) the steps below fall back to github.token.
@@ -149,6 +153,37 @@ jobs:
.claude/skills/ui-architect
.claude/skills/eslint-error-handling
# Bills nothing and posts nothing; a hand-started run, a failed listing or a failed check always reviews.
- name: Skip PRs with nothing to review
if: env.REVIEW_RUN == 'true' && github.event_name == 'pull_request_target'
env:
GH_TOKEN: ${{ github.token }}
REPO: ${{ github.repository }}
PR_NUMBER: ${{ steps.pr.outputs.number }}
PR_AUTHOR: ${{ steps.pr.outputs.author }}
# The listings are checked against these, as both endpoints stop at a cap however far they are paged.
EXPECTED_FILES: ${{ steps.pr.outputs.changed_files }}
EXPECTED_COMMITS: ${{ steps.pr.outputs.commit_count }}
run: |
set -uo pipefail
list="$RUNNER_TEMP/pr-files.tsv"
commits="$RUNNER_TEMP/pr-commits.tsv"
# One line per file with its rename source, so code moved under docs/ is not taken for docs and the count can be checked.
if ! gh api --paginate "repos/$REPO/pulls/$PR_NUMBER/files?per_page=100" --jq '.[] | [.filename, (.previous_filename // "")] | @tsv' > "$list"; then
echo "::warning::Could not list the PR's files; reviewing."
: > "$list"
fi
# One line per commit: author login, committer login, signature verified; a missing field lists as empty and reviews.
if ! gh api --paginate "repos/$REPO/pulls/$PR_NUMBER/commits?per_page=100" --jq '.[] | [(.author.login // ""), (.committer.login // ""), ((.commit.verification.verified // "") | tostring)] | @tsv' > "$commits"; then
echo "::warning::Could not list the PR's commits; reviewing."
: > "$commits"
fi
reason="$(node engine/scripts/ai-review/skip-check.mjs "$list" "$commits")" || reason=""
if [ -n "$reason" ]; then
echo "::notice::PR #$PR_NUMBER needs no review ($reason); skipped."
echo "REVIEW_RUN=false" >> "$GITHUB_ENV"
fi
# Paired OSS+ENT changes are reviewed once, by the o2-enterprise workflow, which pulls this PR's diff in.
- name: Defer to a paired o2-enterprise PR
if: env.REVIEW_RUN == 'true' && !inputs.ignore_paired
@@ -291,10 +326,12 @@ jobs:
FORCE_FULL: ${{ inputs.force_full || 'false' }}
REVIEW_PROVIDER_ID: ${{ matrix.provider.provider_id }}
REVIEW_MODEL_ID: ${{ matrix.provider.model_id }}
REVIEW_MODEL_VARIANT: ${{ matrix.provider.model_variant }}
# The effort is set here only, so the variable is the single source; each value must be a variant in opencode.ci.jsonc.
REVIEW_MODEL_VARIANT: ${{ vars.AI_REVIEW_EFFORT || 'high' }}
REVIEW_COORDINATOR_VARIANT: ${{ vars.AI_REVIEW_COORDINATOR_EFFORT || vars.AI_REVIEW_EFFORT || 'high' }}
REVIEW_API_KEY_ENV: ${{ matrix.provider.api_key_env }}
REVIEW_MARKER: ${{ matrix.provider.marker }}
# opencode.ci.jsonc: every tool (shell, edit, webfetch) allowed with no prompts; "default" is the read-only opencode.jsonc.
# opencode.ci.jsonc: file tools plus a read-only shell allowlist, no web or sub-agent tools, no prompts; "default" is the read-only opencode.jsonc.
REVIEW_OPENCODE_CONFIG: ci-full
# Empty under the github.token fallback: any workflow a same-repo PR adds can also post as github-actions[bot].
REVIEW_BOT_LOGIN: ${{ steps.app_token.outputs.token && format('{0}[bot]', steps.app_token.outputs.app-slug) || '' }}
@@ -313,6 +350,8 @@ jobs:
# Optional ceiling on automatic reviews per PR; unset means no cap, so each new commit gets one review. The manual input bypasses it for one run.
REVIEW_MAX_PASSES: ${{ vars.AI_REVIEW_MAX_PASSES }}
REVIEW_IGNORE_PASS_LIMIT: ${{ inputs.ignore_pass_limit || 'false' }}
# A hand-started run reviews even a diff the last review already read.
REVIEW_MANUAL_RUN: ${{ github.event_name == 'workflow_dispatch' }}
# Independent reviewer sessions per area (1-3), merged by the coordinator; one cold pass misses findings a second catches.
REVIEW_COLD_PASSES: ${{ vars.AI_REVIEW_COLD_PASSES || '2' }}
# An unknown value means off, so a typo can never start approving.
opencode.ci.jsonc
+875 -16
View File
@@ -1,7 +1,5 @@
{
// Loaded from the trusted base-branch checkout via OPENCODE_CONFIG when REVIEW_OPENCODE_CONFIG=ci-full.
// Same provider, model and agents as opencode.jsonc, but every tool (shell, edit, webfetch, ...) is allowed
// and nothing asks for permission. Keep the two files in step apart from the agent permissions.
// ci-full, loaded from the trusted base checkout: keep in step with opencode.jsonc except agent permissions (file tools, shell allowlist, no prompts).
"$schema": "https://opencode.ai/config.json",
// Nothing here may reach into the PR tree: no `instructions`, no `skills`, no plugins.
@@ -28,8 +26,14 @@
"models": {
"claude-sonnet-5-5": {
"name": "Claude Sonnet 5.5",
// Selected with REVIEW_MODEL_VARIANT=high; opencode sends adaptive thinking and "output_config": { "effort": "high" }.
"variants": [{ "id": "high", "settings": { "thinking": { "type": "adaptive" }, "effort": "high" } }],
// REVIEW_MODEL_VARIANT and REVIEW_COORDINATOR_VARIANT pick one; opencode sends adaptive thinking and "output_config": { "effort": "<id>" }.
"variants": [
{ "id": "low", "settings": { "thinking": { "type": "adaptive" }, "effort": "low" } },
{ "id": "medium", "settings": { "thinking": { "type": "adaptive" }, "effort": "medium" } },
{ "id": "high", "settings": { "thinking": { "type": "adaptive" }, "effort": "high" } },
{ "id": "xhigh", "settings": { "thinking": { "type": "adaptive" }, "effort": "xhigh" } },
{ "id": "max", "settings": { "thinking": { "type": "adaptive" }, "effort": "max" } }
],
"limit": { "context": 1000000, "output": 128000 }
}
}
@@ -46,7 +50,7 @@
"models": {
"deepseek-flash": {
"name": "DeepSeek-V4.1-Flash",
// Selected with REVIEW_MODEL_VARIANT=max (the workflow matrix); opencode sends "reasoning_effort": "max".
// Selected with REVIEW_MODEL_VARIANT=max; opencode sends "reasoning_effort": "max".
"variants": [{ "id": "max", "settings": { "reasoningEffort": "max" } }],
"limit": { "context": 1048576, "output": 393216 }
}
@@ -56,52 +60,907 @@
"model": "anthropic-review/claude-sonnet-5-5",
// One allow-all rule: it comes after opencode's built-in defaults (last match wins), so no tool ever asks.
// Last match wins after opencode's built-in defaults, so nothing asks; the shell allowlist guards cost, not secrets (see README).
"agents": {
"ai-review-security": {
"mode": "primary",
"model": "anthropic-review/claude-sonnet-5-5",
"permissions": [{ "action": "*", "resource": "*", "effect": "allow" }]
"permissions": [
{ "action": "*", "resource": "*", "effect": "allow" },
{ "action": "webfetch", "resource": "*", "effect": "deny" },
{ "action": "websearch", "resource": "*", "effect": "deny" },
{ "action": "subagent", "resource": "*", "effect": "deny" },
{ "action": "question", "resource": "*", "effect": "deny" },
{ "action": "execute", "resource": "*", "effect": "deny" },
{ "action": "skill", "resource": "*", "effect": "deny" },
{ "action": "shell", "resource": "*", "effect": "deny" },
{ "action": "shell", "resource": "git diff *", "effect": "allow" },
{ "action": "shell", "resource": "git log *", "effect": "allow" },
{ "action": "shell", "resource": "git show *", "effect": "allow" },
{ "action": "shell", "resource": "git blame *", "effect": "allow" },
{ "action": "shell", "resource": "git ls-files *", "effect": "allow" },
{ "action": "shell", "resource": "git status *", "effect": "allow" },
{ "action": "shell", "resource": "git rev-parse *", "effect": "allow" },
{ "action": "shell", "resource": "git merge-base *", "effect": "allow" },
{ "action": "shell", "resource": "git cat-file -p *", "effect": "allow" },
{ "action": "shell", "resource": "git cat-file -t *", "effect": "allow" },
{ "action": "shell", "resource": "git cat-file -s *", "effect": "allow" },
{ "action": "shell", "resource": "git --no-pager diff *", "effect": "allow" },
{ "action": "shell", "resource": "git --no-pager log *", "effect": "allow" },
{ "action": "shell", "resource": "git --no-pager show *", "effect": "allow" },
{ "action": "shell", "resource": "git --no-pager blame *", "effect": "allow" },
{ "action": "shell", "resource": "git --no-pager ls-files *", "effect": "allow" },
{ "action": "shell", "resource": "git --no-pager status *", "effect": "allow" },
{ "action": "shell", "resource": "rg *", "effect": "allow" },
{ "action": "shell", "resource": "grep *", "effect": "allow" },
{ "action": "shell", "resource": "cat *", "effect": "allow" },
{ "action": "shell", "resource": "ls *", "effect": "allow" },
{ "action": "shell", "resource": "head *", "effect": "allow" },
{ "action": "shell", "resource": "tail *", "effect": "allow" },
{ "action": "shell", "resource": "find *", "effect": "allow" },
{ "action": "shell", "resource": "sed -n *", "effect": "allow" },
{ "action": "shell", "resource": "wc *", "effect": "allow" },
{ "action": "shell", "resource": "tail -f *", "effect": "deny" },
{ "action": "shell", "resource": "tail * -f *", "effect": "deny" },
{ "action": "shell", "resource": "tail -F *", "effect": "deny" },
{ "action": "shell", "resource": "tail * -F *", "effect": "deny" },
{ "action": "shell", "resource": "tail --follow*", "effect": "deny" },
{ "action": "shell", "resource": "tail * --follow*", "effect": "deny" },
{ "action": "shell", "resource": "find -exec *", "effect": "deny" },
{ "action": "shell", "resource": "find * -exec *", "effect": "deny" },
{ "action": "shell", "resource": "find -execdir *", "effect": "deny" },
{ "action": "shell", "resource": "find * -execdir *", "effect": "deny" },
{ "action": "shell", "resource": "find -ok *", "effect": "deny" },
{ "action": "shell", "resource": "find * -ok *", "effect": "deny" },
{ "action": "shell", "resource": "find -okdir *", "effect": "deny" },
{ "action": "shell", "resource": "find * -okdir *", "effect": "deny" },
{ "action": "shell", "resource": "find -delete *", "effect": "deny" },
{ "action": "shell", "resource": "find * -delete *", "effect": "deny" },
{ "action": "shell", "resource": "find -fprint *", "effect": "deny" },
{ "action": "shell", "resource": "find * -fprint *", "effect": "deny" },
{ "action": "shell", "resource": "find -fprint0 *", "effect": "deny" },
{ "action": "shell", "resource": "find * -fprint0 *", "effect": "deny" },
{ "action": "shell", "resource": "find -fprintf *", "effect": "deny" },
{ "action": "shell", "resource": "find * -fprintf *", "effect": "deny" },
{ "action": "shell", "resource": "find -fls *", "effect": "deny" },
{ "action": "shell", "resource": "find * -fls *", "effect": "deny" },
{ "action": "shell", "resource": "cat /*", "effect": "deny" },
{ "action": "shell", "resource": "cat * /*", "effect": "deny" },
{ "action": "shell", "resource": "cat *../*", "effect": "deny" },
{ "action": "shell", "resource": "cat *=/*", "effect": "deny" },
{ "action": "shell", "resource": "cat *environ*", "effect": "deny" },
{ "action": "shell", "resource": "head /*", "effect": "deny" },
{ "action": "shell", "resource": "head * /*", "effect": "deny" },
{ "action": "shell", "resource": "head *../*", "effect": "deny" },
{ "action": "shell", "resource": "head *=/*", "effect": "deny" },
{ "action": "shell", "resource": "head *environ*", "effect": "deny" },
{ "action": "shell", "resource": "tail /*", "effect": "deny" },
{ "action": "shell", "resource": "tail * /*", "effect": "deny" },
{ "action": "shell", "resource": "tail *../*", "effect": "deny" },
{ "action": "shell", "resource": "tail *=/*", "effect": "deny" },
{ "action": "shell", "resource": "tail *environ*", "effect": "deny" },
{ "action": "shell", "resource": "grep /*", "effect": "deny" },
{ "action": "shell", "resource": "grep * /*", "effect": "deny" },
{ "action": "shell", "resource": "grep *../*", "effect": "deny" },
{ "action": "shell", "resource": "grep *=/*", "effect": "deny" },
{ "action": "shell", "resource": "grep *environ*", "effect": "deny" },
{ "action": "shell", "resource": "rg /*", "effect": "deny" },
{ "action": "shell", "resource": "rg * /*", "effect": "deny" },
{ "action": "shell", "resource": "rg *../*", "effect": "deny" },
{ "action": "shell", "resource": "rg *=/*", "effect": "deny" },
{ "action": "shell", "resource": "rg *environ*", "effect": "deny" },
{ "action": "shell", "resource": "sed /*", "effect": "deny" },
{ "action": "shell", "resource": "sed * /*", "effect": "deny" },
{ "action": "shell", "resource": "sed *../*", "effect": "deny" },
{ "action": "shell", "resource": "sed *=/*", "effect": "deny" },
{ "action": "shell", "resource": "sed *environ*", "effect": "deny" },
{ "action": "shell", "resource": "wc /*", "effect": "deny" },
{ "action": "shell", "resource": "wc * /*", "effect": "deny" },
{ "action": "shell", "resource": "wc *../*", "effect": "deny" },
{ "action": "shell", "resource": "wc *=/*", "effect": "deny" },
{ "action": "shell", "resource": "wc *environ*", "effect": "deny" },
{ "action": "shell", "resource": "*>*", "effect": "deny" }
]
},
"ai-review-correctness": {
"mode": "primary",
"model": "anthropic-review/claude-sonnet-5-5",
"permissions": [{ "action": "*", "resource": "*", "effect": "allow" }]
"permissions": [
{ "action": "*", "resource": "*", "effect": "allow" },
{ "action": "webfetch", "resource": "*", "effect": "deny" },
{ "action": "websearch", "resource": "*", "effect": "deny" },
{ "action": "subagent", "resource": "*", "effect": "deny" },
{ "action": "question", "resource": "*", "effect": "deny" },
{ "action": "execute", "resource": "*", "effect": "deny" },
{ "action": "skill", "resource": "*", "effect": "deny" },
{ "action": "shell", "resource": "*", "effect": "deny" },
{ "action": "shell", "resource": "git diff *", "effect": "allow" },
{ "action": "shell", "resource": "git log *", "effect": "allow" },
{ "action": "shell", "resource": "git show *", "effect": "allow" },
{ "action": "shell", "resource": "git blame *", "effect": "allow" },
{ "action": "shell", "resource": "git ls-files *", "effect": "allow" },
{ "action": "shell", "resource": "git status *", "effect": "allow" },
{ "action": "shell", "resource": "git rev-parse *", "effect": "allow" },
{ "action": "shell", "resource": "git merge-base *", "effect": "allow" },
{ "action": "shell", "resource": "git cat-file -p *", "effect": "allow" },
{ "action": "shell", "resource": "git cat-file -t *", "effect": "allow" },
{ "action": "shell", "resource": "git cat-file -s *", "effect": "allow" },
{ "action": "shell", "resource": "git --no-pager diff *", "effect": "allow" },
{ "action": "shell", "resource": "git --no-pager log *", "effect": "allow" },
{ "action": "shell", "resource": "git --no-pager show *", "effect": "allow" },
{ "action": "shell", "resource": "git --no-pager blame *", "effect": "allow" },
{ "action": "shell", "resource": "git --no-pager ls-files *", "effect": "allow" },
{ "action": "shell", "resource": "git --no-pager status *", "effect": "allow" },
{ "action": "shell", "resource": "rg *", "effect": "allow" },
{ "action": "shell", "resource": "grep *", "effect": "allow" },
{ "action": "shell", "resource": "cat *", "effect": "allow" },
{ "action": "shell", "resource": "ls *", "effect": "allow" },
{ "action": "shell", "resource": "head *", "effect": "allow" },
{ "action": "shell", "resource": "tail *", "effect": "allow" },
{ "action": "shell", "resource": "find *", "effect": "allow" },
{ "action": "shell", "resource": "sed -n *", "effect": "allow" },
{ "action": "shell", "resource": "wc *", "effect": "allow" },
{ "action": "shell", "resource": "tail -f *", "effect": "deny" },
{ "action": "shell", "resource": "tail * -f *", "effect": "deny" },
{ "action": "shell", "resource": "tail -F *", "effect": "deny" },
{ "action": "shell", "resource": "tail * -F *", "effect": "deny" },
{ "action": "shell", "resource": "tail --follow*", "effect": "deny" },
{ "action": "shell", "resource": "tail * --follow*", "effect": "deny" },
{ "action": "shell", "resource": "find -exec *", "effect": "deny" },
{ "action": "shell", "resource": "find * -exec *", "effect": "deny" },
{ "action": "shell", "resource": "find -execdir *", "effect": "deny" },
{ "action": "shell", "resource": "find * -execdir *", "effect": "deny" },
{ "action": "shell", "resource": "find -ok *", "effect": "deny" },
{ "action": "shell", "resource": "find * -ok *", "effect": "deny" },
{ "action": "shell", "resource": "find -okdir *", "effect": "deny" },
{ "action": "shell", "resource": "find * -okdir *", "effect": "deny" },
{ "action": "shell", "resource": "find -delete *", "effect": "deny" },
{ "action": "shell", "resource": "find * -delete *", "effect": "deny" },
{ "action": "shell", "resource": "find -fprint *", "effect": "deny" },
{ "action": "shell", "resource": "find * -fprint *", "effect": "deny" },
{ "action": "shell", "resource": "find -fprint0 *", "effect": "deny" },
{ "action": "shell", "resource": "find * -fprint0 *", "effect": "deny" },
{ "action": "shell", "resource": "find -fprintf *", "effect": "deny" },
{ "action": "shell", "resource": "find * -fprintf *", "effect": "deny" },
{ "action": "shell", "resource": "find -fls *", "effect": "deny" },
{ "action": "shell", "resource": "find * -fls *", "effect": "deny" },
{ "action": "shell", "resource": "cat /*", "effect": "deny" },
{ "action": "shell", "resource": "cat * /*", "effect": "deny" },
{ "action": "shell", "resource": "cat *../*", "effect": "deny" },
{ "action": "shell", "resource": "cat *=/*", "effect": "deny" },
{ "action": "shell", "resource": "cat *environ*", "effect": "deny" },
{ "action": "shell", "resource": "head /*", "effect": "deny" },
{ "action": "shell", "resource": "head * /*", "effect": "deny" },
{ "action": "shell", "resource": "head *../*", "effect": "deny" },
{ "action": "shell", "resource": "head *=/*", "effect": "deny" },
{ "action": "shell", "resource": "head *environ*", "effect": "deny" },
{ "action": "shell", "resource": "tail /*", "effect": "deny" },
{ "action": "shell", "resource": "tail * /*", "effect": "deny" },
{ "action": "shell", "resource": "tail *../*", "effect": "deny" },
{ "action": "shell", "resource": "tail *=/*", "effect": "deny" },
{ "action": "shell", "resource": "tail *environ*", "effect": "deny" },
{ "action": "shell", "resource": "grep /*", "effect": "deny" },
{ "action": "shell", "resource": "grep * /*", "effect": "deny" },
{ "action": "shell", "resource": "grep *../*", "effect": "deny" },
{ "action": "shell", "resource": "grep *=/*", "effect": "deny" },
{ "action": "shell", "resource": "grep *environ*", "effect": "deny" },
{ "action": "shell", "resource": "rg /*", "effect": "deny" },
{ "action": "shell", "resource": "rg * /*", "effect": "deny" },
{ "action": "shell", "resource": "rg *../*", "effect": "deny" },
{ "action": "shell", "resource": "rg *=/*", "effect": "deny" },
{ "action": "shell", "resource": "rg *environ*", "effect": "deny" },
{ "action": "shell", "resource": "sed /*", "effect": "deny" },
{ "action": "shell", "resource": "sed * /*", "effect": "deny" },
{ "action": "shell", "resource": "sed *../*", "effect": "deny" },
{ "action": "shell", "resource": "sed *=/*", "effect": "deny" },
{ "action": "shell", "resource": "sed *environ*", "effect": "deny" },
{ "action": "shell", "resource": "wc /*", "effect": "deny" },
{ "action": "shell", "resource": "wc * /*", "effect": "deny" },
{ "action": "shell", "resource": "wc *../*", "effect": "deny" },
{ "action": "shell", "resource": "wc *=/*", "effect": "deny" },
{ "action": "shell", "resource": "wc *environ*", "effect": "deny" },
{ "action": "shell", "resource": "*>*", "effect": "deny" }
]
},
"ai-review-code-quality": {
"mode": "primary",
"model": "anthropic-review/claude-sonnet-5-5",
"permissions": [{ "action": "*", "resource": "*", "effect": "allow" }]
"permissions": [
{ "action": "*", "resource": "*", "effect": "allow" },
{ "action": "webfetch", "resource": "*", "effect": "deny" },
{ "action": "websearch", "resource": "*", "effect": "deny" },
{ "action": "subagent", "resource": "*", "effect": "deny" },
{ "action": "question", "resource": "*", "effect": "deny" },
{ "action": "execute", "resource": "*", "effect": "deny" },
{ "action": "skill", "resource": "*", "effect": "deny" },
{ "action": "shell", "resource": "*", "effect": "deny" },
{ "action": "shell", "resource": "git diff *", "effect": "allow" },
{ "action": "shell", "resource": "git log *", "effect": "allow" },
{ "action": "shell", "resource": "git show *", "effect": "allow" },
{ "action": "shell", "resource": "git blame *", "effect": "allow" },
{ "action": "shell", "resource": "git ls-files *", "effect": "allow" },
{ "action": "shell", "resource": "git status *", "effect": "allow" },
{ "action": "shell", "resource": "git rev-parse *", "effect": "allow" },
{ "action": "shell", "resource": "git merge-base *", "effect": "allow" },
{ "action": "shell", "resource": "git cat-file -p *", "effect": "allow" },
{ "action": "shell", "resource": "git cat-file -t *", "effect": "allow" },
{ "action": "shell", "resource": "git cat-file -s *", "effect": "allow" },
{ "action": "shell", "resource": "git --no-pager diff *", "effect": "allow" },
{ "action": "shell", "resource": "git --no-pager log *", "effect": "allow" },
{ "action": "shell", "resource": "git --no-pager show *", "effect": "allow" },
{ "action": "shell", "resource": "git --no-pager blame *", "effect": "allow" },
{ "action": "shell", "resource": "git --no-pager ls-files *", "effect": "allow" },
{ "action": "shell", "resource": "git --no-pager status *", "effect": "allow" },
{ "action": "shell", "resource": "rg *", "effect": "allow" },
{ "action": "shell", "resource": "grep *", "effect": "allow" },
{ "action": "shell", "resource": "cat *", "effect": "allow" },
{ "action": "shell", "resource": "ls *", "effect": "allow" },
{ "action": "shell", "resource": "head *", "effect": "allow" },
{ "action": "shell", "resource": "tail *", "effect": "allow" },
{ "action": "shell", "resource": "find *", "effect": "allow" },
{ "action": "shell", "resource": "sed -n *", "effect": "allow" },
{ "action": "shell", "resource": "wc *", "effect": "allow" },
{ "action": "shell", "resource": "tail -f *", "effect": "deny" },
{ "action": "shell", "resource": "tail * -f *", "effect": "deny" },
{ "action": "shell", "resource": "tail -F *", "effect": "deny" },
{ "action": "shell", "resource": "tail * -F *", "effect": "deny" },
{ "action": "shell", "resource": "tail --follow*", "effect": "deny" },
{ "action": "shell", "resource": "tail * --follow*", "effect": "deny" },
{ "action": "shell", "resource": "find -exec *", "effect": "deny" },
{ "action": "shell", "resource": "find * -exec *", "effect": "deny" },
{ "action": "shell", "resource": "find -execdir *", "effect": "deny" },
{ "action": "shell", "resource": "find * -execdir *", "effect": "deny" },
{ "action": "shell", "resource": "find -ok *", "effect": "deny" },
{ "action": "shell", "resource": "find * -ok *", "effect": "deny" },
{ "action": "shell", "resource": "find -okdir *", "effect": "deny" },
{ "action": "shell", "resource": "find * -okdir *", "effect": "deny" },
{ "action": "shell", "resource": "find -delete *", "effect": "deny" },
{ "action": "shell", "resource": "find * -delete *", "effect": "deny" },
{ "action": "shell", "resource": "find -fprint *", "effect": "deny" },
{ "action": "shell", "resource": "find * -fprint *", "effect": "deny" },
{ "action": "shell", "resource": "find -fprint0 *", "effect": "deny" },
{ "action": "shell", "resource": "find * -fprint0 *", "effect": "deny" },
{ "action": "shell", "resource": "find -fprintf *", "effect": "deny" },
{ "action": "shell", "resource": "find * -fprintf *", "effect": "deny" },
{ "action": "shell", "resource": "find -fls *", "effect": "deny" },
{ "action": "shell", "resource": "find * -fls *", "effect": "deny" },
{ "action": "shell", "resource": "cat /*", "effect": "deny" },
{ "action": "shell", "resource": "cat * /*", "effect": "deny" },
{ "action": "shell", "resource": "cat *../*", "effect": "deny" },
{ "action": "shell", "resource": "cat *=/*", "effect": "deny" },
{ "action": "shell", "resource": "cat *environ*", "effect": "deny" },
{ "action": "shell", "resource": "head /*", "effect": "deny" },
{ "action": "shell", "resource": "head * /*", "effect": "deny" },
{ "action": "shell", "resource": "head *../*", "effect": "deny" },
{ "action": "shell", "resource": "head *=/*", "effect": "deny" },
{ "action": "shell", "resource": "head *environ*", "effect": "deny" },
{ "action": "shell", "resource": "tail /*", "effect": "deny" },
{ "action": "shell", "resource": "tail * /*", "effect": "deny" },
{ "action": "shell", "resource": "tail *../*", "effect": "deny" },
{ "action": "shell", "resource": "tail *=/*", "effect": "deny" },
{ "action": "shell", "resource": "tail *environ*", "effect": "deny" },
{ "action": "shell", "resource": "grep /*", "effect": "deny" },
{ "action": "shell", "resource": "grep * /*", "effect": "deny" },
{ "action": "shell", "resource": "grep *../*", "effect": "deny" },
{ "action": "shell", "resource": "grep *=/*", "effect": "deny" },
{ "action": "shell", "resource": "grep *environ*", "effect": "deny" },
{ "action": "shell", "resource": "rg /*", "effect": "deny" },
{ "action": "shell", "resource": "rg * /*", "effect": "deny" },
{ "action": "shell", "resource": "rg *../*", "effect": "deny" },
{ "action": "shell", "resource": "rg *=/*", "effect": "deny" },
{ "action": "shell", "resource": "rg *environ*", "effect": "deny" },
{ "action": "shell", "resource": "sed /*", "effect": "deny" },
{ "action": "shell", "resource": "sed * /*", "effect": "deny" },
{ "action": "shell", "resource": "sed *../*", "effect": "deny" },
{ "action": "shell", "resource": "sed *=/*", "effect": "deny" },
{ "action": "shell", "resource": "sed *environ*", "effect": "deny" },
{ "action": "shell", "resource": "wc /*", "effect": "deny" },
{ "action": "shell", "resource": "wc * /*", "effect": "deny" },
{ "action": "shell", "resource": "wc *../*", "effect": "deny" },
{ "action": "shell", "resource": "wc *=/*", "effect": "deny" },
{ "action": "shell", "resource": "wc *environ*", "effect": "deny" },
{ "action": "shell", "resource": "*>*", "effect": "deny" }
]
},
"ai-review-performance": {
"mode": "primary",
"model": "anthropic-review/claude-sonnet-5-5",
"permissions": [{ "action": "*", "resource": "*", "effect": "allow" }]
"permissions": [
{ "action": "*", "resource": "*", "effect": "allow" },
{ "action": "webfetch", "resource": "*", "effect": "deny" },
{ "action": "websearch", "resource": "*", "effect": "deny" },
{ "action": "subagent", "resource": "*", "effect": "deny" },
{ "action": "question", "resource": "*", "effect": "deny" },
{ "action": "execute", "resource": "*", "effect": "deny" },
{ "action": "skill", "resource": "*", "effect": "deny" },
{ "action": "shell", "resource": "*", "effect": "deny" },
{ "action": "shell", "resource": "git diff *", "effect": "allow" },
{ "action": "shell", "resource": "git log *", "effect": "allow" },
{ "action": "shell", "resource": "git show *", "effect": "allow" },
{ "action": "shell", "resource": "git blame *", "effect": "allow" },
{ "action": "shell", "resource": "git ls-files *", "effect": "allow" },
{ "action": "shell", "resource": "git status *", "effect": "allow" },
{ "action": "shell", "resource": "git rev-parse *", "effect": "allow" },
{ "action": "shell", "resource": "git merge-base *", "effect": "allow" },
{ "action": "shell", "resource": "git cat-file -p *", "effect": "allow" },
{ "action": "shell", "resource": "git cat-file -t *", "effect": "allow" },
{ "action": "shell", "resource": "git cat-file -s *", "effect": "allow" },
{ "action": "shell", "resource": "git --no-pager diff *", "effect": "allow" },
{ "action": "shell", "resource": "git --no-pager log *", "effect": "allow" },
{ "action": "shell", "resource": "git --no-pager show *", "effect": "allow" },
{ "action": "shell", "resource": "git --no-pager blame *", "effect": "allow" },
{ "action": "shell", "resource": "git --no-pager ls-files *", "effect": "allow" },
{ "action": "shell", "resource": "git --no-pager status *", "effect": "allow" },
{ "action": "shell", "resource": "rg *", "effect": "allow" },
{ "action": "shell", "resource": "grep *", "effect": "allow" },
{ "action": "shell", "resource": "cat *", "effect": "allow" },
{ "action": "shell", "resource": "ls *", "effect": "allow" },
{ "action": "shell", "resource": "head *", "effect": "allow" },
{ "action": "shell", "resource": "tail *", "effect": "allow" },
{ "action": "shell", "resource": "find *", "effect": "allow" },
{ "action": "shell", "resource": "sed -n *", "effect": "allow" },
{ "action": "shell", "resource": "wc *", "effect": "allow" },
{ "action": "shell", "resource": "tail -f *", "effect": "deny" },
{ "action": "shell", "resource": "tail * -f *", "effect": "deny" },
{ "action": "shell", "resource": "tail -F *", "effect": "deny" },
{ "action": "shell", "resource": "tail * -F *", "effect": "deny" },
{ "action": "shell", "resource": "tail --follow*", "effect": "deny" },
{ "action": "shell", "resource": "tail * --follow*", "effect": "deny" },
{ "action": "shell", "resource": "find -exec *", "effect": "deny" },
{ "action": "shell", "resource": "find * -exec *", "effect": "deny" },
{ "action": "shell", "resource": "find -execdir *", "effect": "deny" },
{ "action": "shell", "resource": "find * -execdir *", "effect": "deny" },
{ "action": "shell", "resource": "find -ok *", "effect": "deny" },
{ "action": "shell", "resource": "find * -ok *", "effect": "deny" },
{ "action": "shell", "resource": "find -okdir *", "effect": "deny" },
{ "action": "shell", "resource": "find * -okdir *", "effect": "deny" },
{ "action": "shell", "resource": "find -delete *", "effect": "deny" },
{ "action": "shell", "resource": "find * -delete *", "effect": "deny" },
{ "action": "shell", "resource": "find -fprint *", "effect": "deny" },
{ "action": "shell", "resource": "find * -fprint *", "effect": "deny" },
{ "action": "shell", "resource": "find -fprint0 *", "effect": "deny" },
{ "action": "shell", "resource": "find * -fprint0 *", "effect": "deny" },
{ "action": "shell", "resource": "find -fprintf *", "effect": "deny" },
{ "action": "shell", "resource": "find * -fprintf *", "effect": "deny" },
{ "action": "shell", "resource": "find -fls *", "effect": "deny" },
{ "action": "shell", "resource": "find * -fls *", "effect": "deny" },
{ "action": "shell", "resource": "cat /*", "effect": "deny" },
{ "action": "shell", "resource": "cat * /*", "effect": "deny" },
{ "action": "shell", "resource": "cat *../*", "effect": "deny" },
{ "action": "shell", "resource": "cat *=/*", "effect": "deny" },
{ "action": "shell", "resource": "cat *environ*", "effect": "deny" },
{ "action": "shell", "resource": "head /*", "effect": "deny" },
{ "action": "shell", "resource": "head * /*", "effect": "deny" },
{ "action": "shell", "resource": "head *../*", "effect": "deny" },
{ "action": "shell", "resource": "head *=/*", "effect": "deny" },
{ "action": "shell", "resource": "head *environ*", "effect": "deny" },
{ "action": "shell", "resource": "tail /*", "effect": "deny" },
{ "action": "shell", "resource": "tail * /*", "effect": "deny" },
{ "action": "shell", "resource": "tail *../*", "effect": "deny" },
{ "action": "shell", "resource": "tail *=/*", "effect": "deny" },
{ "action": "shell", "resource": "tail *environ*", "effect": "deny" },
{ "action": "shell", "resource": "grep /*", "effect": "deny" },
{ "action": "shell", "resource": "grep * /*", "effect": "deny" },
{ "action": "shell", "resource": "grep *../*", "effect": "deny" },
{ "action": "shell", "resource": "grep *=/*", "effect": "deny" },
{ "action": "shell", "resource": "grep *environ*", "effect": "deny" },
{ "action": "shell", "resource": "rg /*", "effect": "deny" },
{ "action": "shell", "resource": "rg * /*", "effect": "deny" },
{ "action": "shell", "resource": "rg *../*", "effect": "deny" },
{ "action": "shell", "resource": "rg *=/*", "effect": "deny" },
{ "action": "shell", "resource": "rg *environ*", "effect": "deny" },
{ "action": "shell", "resource": "sed /*", "effect": "deny" },
{ "action": "shell", "resource": "sed * /*", "effect": "deny" },
{ "action": "shell", "resource": "sed *../*", "effect": "deny" },
{ "action": "shell", "resource": "sed *=/*", "effect": "deny" },
{ "action": "shell", "resource": "sed *environ*", "effect": "deny" },
{ "action": "shell", "resource": "wc /*", "effect": "deny" },
{ "action": "shell", "resource": "wc * /*", "effect": "deny" },
{ "action": "shell", "resource": "wc *../*", "effect": "deny" },
{ "action": "shell", "resource": "wc *=/*", "effect": "deny" },
{ "action": "shell", "resource": "wc *environ*", "effect": "deny" },
{ "action": "shell", "resource": "*>*", "effect": "deny" }
]
},
"ai-review-documentation": {
"mode": "primary",
"model": "anthropic-review/claude-sonnet-5-5",
"permissions": [{ "action": "*", "resource": "*", "effect": "allow" }]
"permissions": [
{ "action": "*", "resource": "*", "effect": "allow" },
{ "action": "webfetch", "resource": "*", "effect": "deny" },
{ "action": "websearch", "resource": "*", "effect": "deny" },
{ "action": "subagent", "resource": "*", "effect": "deny" },
{ "action": "question", "resource": "*", "effect": "deny" },
{ "action": "execute", "resource": "*", "effect": "deny" },
{ "action": "skill", "resource": "*", "effect": "deny" },
{ "action": "shell", "resource": "*", "effect": "deny" },
{ "action": "shell", "resource": "git diff *", "effect": "allow" },
{ "action": "shell", "resource": "git log *", "effect": "allow" },
{ "action": "shell", "resource": "git show *", "effect": "allow" },
{ "action": "shell", "resource": "git blame *", "effect": "allow" },
{ "action": "shell", "resource": "git ls-files *", "effect": "allow" },
{ "action": "shell", "resource": "git status *", "effect": "allow" },
{ "action": "shell", "resource": "git rev-parse *", "effect": "allow" },
{ "action": "shell", "resource": "git merge-base *", "effect": "allow" },
{ "action": "shell", "resource": "git cat-file -p *", "effect": "allow" },
{ "action": "shell", "resource": "git cat-file -t *", "effect": "allow" },
{ "action": "shell", "resource": "git cat-file -s *", "effect": "allow" },
{ "action": "shell", "resource": "git --no-pager diff *", "effect": "allow" },
{ "action": "shell", "resource": "git --no-pager log *", "effect": "allow" },
{ "action": "shell", "resource": "git --no-pager show *", "effect": "allow" },
{ "action": "shell", "resource": "git --no-pager blame *", "effect": "allow" },
{ "action": "shell", "resource": "git --no-pager ls-files *", "effect": "allow" },
{ "action": "shell", "resource": "git --no-pager status *", "effect": "allow" },
{ "action": "shell", "resource": "rg *", "effect": "allow" },
{ "action": "shell", "resource": "grep *", "effect": "allow" },
{ "action": "shell", "resource": "cat *", "effect": "allow" },
{ "action": "shell", "resource": "ls *", "effect": "allow" },
{ "action": "shell", "resource": "head *", "effect": "allow" },
{ "action": "shell", "resource": "tail *", "effect": "allow" },
{ "action": "shell", "resource": "find *", "effect": "allow" },
{ "action": "shell", "resource": "sed -n *", "effect": "allow" },
{ "action": "shell", "resource": "wc *", "effect": "allow" },
{ "action": "shell", "resource": "tail -f *", "effect": "deny" },
{ "action": "shell", "resource": "tail * -f *", "effect": "deny" },
{ "action": "shell", "resource": "tail -F *", "effect": "deny" },
{ "action": "shell", "resource": "tail * -F *", "effect": "deny" },
{ "action": "shell", "resource": "tail --follow*", "effect": "deny" },
{ "action": "shell", "resource": "tail * --follow*", "effect": "deny" },
{ "action": "shell", "resource": "find -exec *", "effect": "deny" },
{ "action": "shell", "resource": "find * -exec *", "effect": "deny" },
{ "action": "shell", "resource": "find -execdir *", "effect": "deny" },
{ "action": "shell", "resource": "find * -execdir *", "effect": "deny" },
{ "action": "shell", "resource": "find -ok *", "effect": "deny" },
{ "action": "shell", "resource": "find * -ok *", "effect": "deny" },
{ "action": "shell", "resource": "find -okdir *", "effect": "deny" },
{ "action": "shell", "resource": "find * -okdir *", "effect": "deny" },
{ "action": "shell", "resource": "find -delete *", "effect": "deny" },
{ "action": "shell", "resource": "find * -delete *", "effect": "deny" },
{ "action": "shell", "resource": "find -fprint *", "effect": "deny" },
{ "action": "shell", "resource": "find * -fprint *", "effect": "deny" },
{ "action": "shell", "resource": "find -fprint0 *", "effect": "deny" },
{ "action": "shell", "resource": "find * -fprint0 *", "effect": "deny" },
{ "action": "shell", "resource": "find -fprintf *", "effect": "deny" },
{ "action": "shell", "resource": "find * -fprintf *", "effect": "deny" },
{ "action": "shell", "resource": "find -fls *", "effect": "deny" },
{ "action": "shell", "resource": "find * -fls *", "effect": "deny" },
{ "action": "shell", "resource": "cat /*", "effect": "deny" },
{ "action": "shell", "resource": "cat * /*", "effect": "deny" },
{ "action": "shell", "resource": "cat *../*", "effect": "deny" },
{ "action": "shell", "resource": "cat *=/*", "effect": "deny" },
{ "action": "shell", "resource": "cat *environ*", "effect": "deny" },
{ "action": "shell", "resource": "head /*", "effect": "deny" },
{ "action": "shell", "resource": "head * /*", "effect": "deny" },
{ "action": "shell", "resource": "head *../*", "effect": "deny" },
{ "action": "shell", "resource": "head *=/*", "effect": "deny" },
{ "action": "shell", "resource": "head *environ*", "effect": "deny" },
{ "action": "shell", "resource": "tail /*", "effect": "deny" },
{ "action": "shell", "resource": "tail * /*", "effect": "deny" },
{ "action": "shell", "resource": "tail *../*", "effect": "deny" },
{ "action": "shell", "resource": "tail *=/*", "effect": "deny" },
{ "action": "shell", "resource": "tail *environ*", "effect": "deny" },
{ "action": "shell", "resource": "grep /*", "effect": "deny" },
{ "action": "shell", "resource": "grep * /*", "effect": "deny" },
{ "action": "shell", "resource": "grep *../*", "effect": "deny" },
{ "action": "shell", "resource": "grep *=/*", "effect": "deny" },
{ "action": "shell", "resource": "grep *environ*", "effect": "deny" },
{ "action": "shell", "resource": "rg /*", "effect": "deny" },
{ "action": "shell", "resource": "rg * /*", "effect": "deny" },
{ "action": "shell", "resource": "rg *../*", "effect": "deny" },
{ "action": "shell", "resource": "rg *=/*", "effect": "deny" },
{ "action": "shell", "resource": "rg *environ*", "effect": "deny" },
{ "action": "shell", "resource": "sed /*", "effect": "deny" },
{ "action": "shell", "resource": "sed * /*", "effect": "deny" },
{ "action": "shell", "resource": "sed *../*", "effect": "deny" },
{ "action": "shell", "resource": "sed *=/*", "effect": "deny" },
{ "action": "shell", "resource": "sed *environ*", "effect": "deny" },
{ "action": "shell", "resource": "wc /*", "effect": "deny" },
{ "action": "shell", "resource": "wc * /*", "effect": "deny" },
{ "action": "shell", "resource": "wc *../*", "effect": "deny" },
{ "action": "shell", "resource": "wc *=/*", "effect": "deny" },
{ "action": "shell", "resource": "wc *environ*", "effect": "deny" },
{ "action": "shell", "resource": "*>*", "effect": "deny" }
]
},
"ai-review-release": {
"mode": "primary",
"model": "anthropic-review/claude-sonnet-5-5",
"permissions": [{ "action": "*", "resource": "*", "effect": "allow" }]
"permissions": [
{ "action": "*", "resource": "*", "effect": "allow" },
{ "action": "webfetch", "resource": "*", "effect": "deny" },
{ "action": "websearch", "resource": "*", "effect": "deny" },
{ "action": "subagent", "resource": "*", "effect": "deny" },
{ "action": "question", "resource": "*", "effect": "deny" },
{ "action": "execute", "resource": "*", "effect": "deny" },
{ "action": "skill", "resource": "*", "effect": "deny" },
{ "action": "shell", "resource": "*", "effect": "deny" },
{ "action": "shell", "resource": "git diff *", "effect": "allow" },
{ "action": "shell", "resource": "git log *", "effect": "allow" },
{ "action": "shell", "resource": "git show *", "effect": "allow" },
{ "action": "shell", "resource": "git blame *", "effect": "allow" },
{ "action": "shell", "resource": "git ls-files *", "effect": "allow" },
{ "action": "shell", "resource": "git status *", "effect": "allow" },
{ "action": "shell", "resource": "git rev-parse *", "effect": "allow" },
{ "action": "shell", "resource": "git merge-base *", "effect": "allow" },
{ "action": "shell", "resource": "git cat-file -p *", "effect": "allow" },
{ "action": "shell", "resource": "git cat-file -t *", "effect": "allow" },
{ "action": "shell", "resource": "git cat-file -s *", "effect": "allow" },
{ "action": "shell", "resource": "git --no-pager diff *", "effect": "allow" },
{ "action": "shell", "resource": "git --no-pager log *", "effect": "allow" },
{ "action": "shell", "resource": "git --no-pager show *", "effect": "allow" },
{ "action": "shell", "resource": "git --no-pager blame *", "effect": "allow" },
{ "action": "shell", "resource": "git --no-pager ls-files *", "effect": "allow" },
{ "action": "shell", "resource": "git --no-pager status *", "effect": "allow" },
{ "action": "shell", "resource": "rg *", "effect": "allow" },
{ "action": "shell", "resource": "grep *", "effect": "allow" },
{ "action": "shell", "resource": "cat *", "effect": "allow" },
{ "action": "shell", "resource": "ls *", "effect": "allow" },
{ "action": "shell", "resource": "head *", "effect": "allow" },
{ "action": "shell", "resource": "tail *", "effect": "allow" },
{ "action": "shell", "resource": "find *", "effect": "allow" },
{ "action": "shell", "resource": "sed -n *", "effect": "allow" },
{ "action": "shell", "resource": "wc *", "effect": "allow" },
{ "action": "shell", "resource": "tail -f *", "effect": "deny" },
{ "action": "shell", "resource": "tail * -f *", "effect": "deny" },
{ "action": "shell", "resource": "tail -F *", "effect": "deny" },
{ "action": "shell", "resource": "tail * -F *", "effect": "deny" },
{ "action": "shell", "resource": "tail --follow*", "effect": "deny" },
{ "action": "shell", "resource": "tail * --follow*", "effect": "deny" },
{ "action": "shell", "resource": "find -exec *", "effect": "deny" },
{ "action": "shell", "resource": "find * -exec *", "effect": "deny" },
{ "action": "shell", "resource": "find -execdir *", "effect": "deny" },
{ "action": "shell", "resource": "find * -execdir *", "effect": "deny" },
{ "action": "shell", "resource": "find -ok *", "effect": "deny" },
{ "action": "shell", "resource": "find * -ok *", "effect": "deny" },
{ "action": "shell", "resource": "find -okdir *", "effect": "deny" },
{ "action": "shell", "resource": "find * -okdir *", "effect": "deny" },
{ "action": "shell", "resource": "find -delete *", "effect": "deny" },
{ "action": "shell", "resource": "find * -delete *", "effect": "deny" },
{ "action": "shell", "resource": "find -fprint *", "effect": "deny" },
{ "action": "shell", "resource": "find * -fprint *", "effect": "deny" },
{ "action": "shell", "resource": "find -fprint0 *", "effect": "deny" },
{ "action": "shell", "resource": "find * -fprint0 *", "effect": "deny" },
{ "action": "shell", "resource": "find -fprintf *", "effect": "deny" },
{ "action": "shell", "resource": "find * -fprintf *", "effect": "deny" },
{ "action": "shell", "resource": "find -fls *", "effect": "deny" },
{ "action": "shell", "resource": "find * -fls *", "effect": "deny" },
{ "action": "shell", "resource": "cat /*", "effect": "deny" },
{ "action": "shell", "resource": "cat * /*", "effect": "deny" },
{ "action": "shell", "resource": "cat *../*", "effect": "deny" },
{ "action": "shell", "resource": "cat *=/*", "effect": "deny" },
{ "action": "shell", "resource": "cat *environ*", "effect": "deny" },
{ "action": "shell", "resource": "head /*", "effect": "deny" },
{ "action": "shell", "resource": "head * /*", "effect": "deny" },
{ "action": "shell", "resource": "head *../*", "effect": "deny" },
{ "action": "shell", "resource": "head *=/*", "effect": "deny" },
{ "action": "shell", "resource": "head *environ*", "effect": "deny" },
{ "action": "shell", "resource": "tail /*", "effect": "deny" },
{ "action": "shell", "resource": "tail * /*", "effect": "deny" },
{ "action": "shell", "resource": "tail *../*", "effect": "deny" },
{ "action": "shell", "resource": "tail *=/*", "effect": "deny" },
{ "action": "shell", "resource": "tail *environ*", "effect": "deny" },
{ "action": "shell", "resource": "grep /*", "effect": "deny" },
{ "action": "shell", "resource": "grep * /*", "effect": "deny" },
{ "action": "shell", "resource": "grep *../*", "effect": "deny" },
{ "action": "shell", "resource": "grep *=/*", "effect": "deny" },
{ "action": "shell", "resource": "grep *environ*", "effect": "deny" },
{ "action": "shell", "resource": "rg /*", "effect": "deny" },
{ "action": "shell", "resource": "rg * /*", "effect": "deny" },
{ "action": "shell", "resource": "rg *../*", "effect": "deny" },
{ "action": "shell", "resource": "rg *=/*", "effect": "deny" },
{ "action": "shell", "resource": "rg *environ*", "effect": "deny" },
{ "action": "shell", "resource": "sed /*", "effect": "deny" },
{ "action": "shell", "resource": "sed * /*", "effect": "deny" },
{ "action": "shell", "resource": "sed *../*", "effect": "deny" },
{ "action": "shell", "resource": "sed *=/*", "effect": "deny" },
{ "action": "shell", "resource": "sed *environ*", "effect": "deny" },
{ "action": "shell", "resource": "wc /*", "effect": "deny" },
{ "action": "shell", "resource": "wc * /*", "effect": "deny" },
{ "action": "shell", "resource": "wc *../*", "effect": "deny" },
{ "action": "shell", "resource": "wc *=/*", "effect": "deny" },
{ "action": "shell", "resource": "wc *environ*", "effect": "deny" },
{ "action": "shell", "resource": "*>*", "effect": "deny" }
]
},
"ai-review-frontend": {
"mode": "primary",
"model": "anthropic-review/claude-sonnet-5-5",
"permissions": [{ "action": "*", "resource": "*", "effect": "allow" }]
"permissions": [
{ "action": "*", "resource": "*", "effect": "allow" },
{ "action": "webfetch", "resource": "*", "effect": "deny" },
{ "action": "websearch", "resource": "*", "effect": "deny" },
{ "action": "subagent", "resource": "*", "effect": "deny" },
{ "action": "question", "resource": "*", "effect": "deny" },
{ "action": "execute", "resource": "*", "effect": "deny" },
{ "action": "skill", "resource": "*", "effect": "deny" },
{ "action": "shell", "resource": "*", "effect": "deny" },
{ "action": "shell", "resource": "git diff *", "effect": "allow" },
{ "action": "shell", "resource": "git log *", "effect": "allow" },
{ "action": "shell", "resource": "git show *", "effect": "allow" },
{ "action": "shell", "resource": "git blame *", "effect": "allow" },
{ "action": "shell", "resource": "git ls-files *", "effect": "allow" },
{ "action": "shell", "resource": "git status *", "effect": "allow" },
{ "action": "shell", "resource": "git rev-parse *", "effect": "allow" },
{ "action": "shell", "resource": "git merge-base *", "effect": "allow" },
{ "action": "shell", "resource": "git cat-file -p *", "effect": "allow" },
{ "action": "shell", "resource": "git cat-file -t *", "effect": "allow" },
{ "action": "shell", "resource": "git cat-file -s *", "effect": "allow" },
{ "action": "shell", "resource": "git --no-pager diff *", "effect": "allow" },
{ "action": "shell", "resource": "git --no-pager log *", "effect": "allow" },
{ "action": "shell", "resource": "git --no-pager show *", "effect": "allow" },
{ "action": "shell", "resource": "git --no-pager blame *", "effect": "allow" },
{ "action": "shell", "resource": "git --no-pager ls-files *", "effect": "allow" },
{ "action": "shell", "resource": "git --no-pager status *", "effect": "allow" },
{ "action": "shell", "resource": "rg *", "effect": "allow" },
{ "action": "shell", "resource": "grep *", "effect": "allow" },
{ "action": "shell", "resource": "cat *", "effect": "allow" },
{ "action": "shell", "resource": "ls *", "effect": "allow" },
{ "action": "shell", "resource": "head *", "effect": "allow" },
{ "action": "shell", "resource": "tail *", "effect": "allow" },
{ "action": "shell", "resource": "find *", "effect": "allow" },
{ "action": "shell", "resource": "sed -n *", "effect": "allow" },
{ "action": "shell", "resource": "wc *", "effect": "allow" },
{ "action": "shell", "resource": "tail -f *", "effect": "deny" },
{ "action": "shell", "resource": "tail * -f *", "effect": "deny" },
{ "action": "shell", "resource": "tail -F *", "effect": "deny" },
{ "action": "shell", "resource": "tail * -F *", "effect": "deny" },
{ "action": "shell", "resource": "tail --follow*", "effect": "deny" },
{ "action": "shell", "resource": "tail * --follow*", "effect": "deny" },
{ "action": "shell", "resource": "find -exec *", "effect": "deny" },
{ "action": "shell", "resource": "find * -exec *", "effect": "deny" },
{ "action": "shell", "resource": "find -execdir *", "effect": "deny" },
{ "action": "shell", "resource": "find * -execdir *", "effect": "deny" },
{ "action": "shell", "resource": "find -ok *", "effect": "deny" },
{ "action": "shell", "resource": "find * -ok *", "effect": "deny" },
{ "action": "shell", "resource": "find -okdir *", "effect": "deny" },
{ "action": "shell", "resource": "find * -okdir *", "effect": "deny" },
{ "action": "shell", "resource": "find -delete *", "effect": "deny" },
{ "action": "shell", "resource": "find * -delete *", "effect": "deny" },
{ "action": "shell", "resource": "find -fprint *", "effect": "deny" },
{ "action": "shell", "resource": "find * -fprint *", "effect": "deny" },
{ "action": "shell", "resource": "find -fprint0 *", "effect": "deny" },
{ "action": "shell", "resource": "find * -fprint0 *", "effect": "deny" },
{ "action": "shell", "resource": "find -fprintf *", "effect": "deny" },
{ "action": "shell", "resource": "find * -fprintf *", "effect": "deny" },
{ "action": "shell", "resource": "find -fls *", "effect": "deny" },
{ "action": "shell", "resource": "find * -fls *", "effect": "deny" },
{ "action": "shell", "resource": "cat /*", "effect": "deny" },
{ "action": "shell", "resource": "cat * /*", "effect": "deny" },
{ "action": "shell", "resource": "cat *../*", "effect": "deny" },
{ "action": "shell", "resource": "cat *=/*", "effect": "deny" },
{ "action": "shell", "resource": "cat *environ*", "effect": "deny" },
{ "action": "shell", "resource": "head /*", "effect": "deny" },
{ "action": "shell", "resource": "head * /*", "effect": "deny" },
{ "action": "shell", "resource": "head *../*", "effect": "deny" },
{ "action": "shell", "resource": "head *=/*", "effect": "deny" },
{ "action": "shell", "resource": "head *environ*", "effect": "deny" },
{ "action": "shell", "resource": "tail /*", "effect": "deny" },
{ "action": "shell", "resource": "tail * /*", "effect": "deny" },
{ "action": "shell", "resource": "tail *../*", "effect": "deny" },
{ "action": "shell", "resource": "tail *=/*", "effect": "deny" },
{ "action": "shell", "resource": "tail *environ*", "effect": "deny" },
{ "action": "shell", "resource": "grep /*", "effect": "deny" },
{ "action": "shell", "resource": "grep * /*", "effect": "deny" },
{ "action": "shell", "resource": "grep *../*", "effect": "deny" },
{ "action": "shell", "resource": "grep *=/*", "effect": "deny" },
{ "action": "shell", "resource": "grep *environ*", "effect": "deny" },
{ "action": "shell", "resource": "rg /*", "effect": "deny" },
{ "action": "shell", "resource": "rg * /*", "effect": "deny" },
{ "action": "shell", "resource": "rg *../*", "effect": "deny" },
{ "action": "shell", "resource": "rg *=/*", "effect": "deny" },
{ "action": "shell", "resource": "rg *environ*", "effect": "deny" },
{ "action": "shell", "resource": "sed /*", "effect": "deny" },
{ "action": "shell", "resource": "sed * /*", "effect": "deny" },
{ "action": "shell", "resource": "sed *../*", "effect": "deny" },
{ "action": "shell", "resource": "sed *=/*", "effect": "deny" },
{ "action": "shell", "resource": "sed *environ*", "effect": "deny" },
{ "action": "shell", "resource": "wc /*", "effect": "deny" },
{ "action": "shell", "resource": "wc * /*", "effect": "deny" },
{ "action": "shell", "resource": "wc *../*", "effect": "deny" },
{ "action": "shell", "resource": "wc *=/*", "effect": "deny" },
{ "action": "shell", "resource": "wc *environ*", "effect": "deny" },
{ "action": "shell", "resource": "*>*", "effect": "deny" }
]
},
"ai-review-qa": {
"mode": "primary",
"model": "anthropic-review/claude-sonnet-5-5",
"permissions": [{ "action": "*", "resource": "*", "effect": "allow" }]
"permissions": [
{ "action": "*", "resource": "*", "effect": "allow" },
{ "action": "webfetch", "resource": "*", "effect": "deny" },
{ "action": "websearch", "resource": "*", "effect": "deny" },
{ "action": "subagent", "resource": "*", "effect": "deny" },
{ "action": "question", "resource": "*", "effect": "deny" },
{ "action": "execute", "resource": "*", "effect": "deny" },
{ "action": "skill", "resource": "*", "effect": "deny" },
{ "action": "shell", "resource": "*", "effect": "deny" },
{ "action": "shell", "resource": "git diff *", "effect": "allow" },
{ "action": "shell", "resource": "git log *", "effect": "allow" },
{ "action": "shell", "resource": "git show *", "effect": "allow" },
{ "action": "shell", "resource": "git blame *", "effect": "allow" },
{ "action": "shell", "resource": "git ls-files *", "effect": "allow" },
{ "action": "shell", "resource": "git status *", "effect": "allow" },
{ "action": "shell", "resource": "git rev-parse *", "effect": "allow" },
{ "action": "shell", "resource": "git merge-base *", "effect": "allow" },
{ "action": "shell", "resource": "git cat-file -p *", "effect": "allow" },
{ "action": "shell", "resource": "git cat-file -t *", "effect": "allow" },
{ "action": "shell", "resource": "git cat-file -s *", "effect": "allow" },
{ "action": "shell", "resource": "git --no-pager diff *", "effect": "allow" },
{ "action": "shell", "resource": "git --no-pager log *", "effect": "allow" },
{ "action": "shell", "resource": "git --no-pager show *", "effect": "allow" },
{ "action": "shell", "resource": "git --no-pager blame *", "effect": "allow" },
{ "action": "shell", "resource": "git --no-pager ls-files *", "effect": "allow" },
{ "action": "shell", "resource": "git --no-pager status *", "effect": "allow" },
{ "action": "shell", "resource": "rg *", "effect": "allow" },
{ "action": "shell", "resource": "grep *", "effect": "allow" },
{ "action": "shell", "resource": "cat *", "effect": "allow" },
{ "action": "shell", "resource": "ls *", "effect": "allow" },
{ "action": "shell", "resource": "head *", "effect": "allow" },
{ "action": "shell", "resource": "tail *", "effect": "allow" },
{ "action": "shell", "resource": "find *", "effect": "allow" },
{ "action": "shell", "resource": "sed -n *", "effect": "allow" },
{ "action": "shell", "resource": "wc *", "effect": "allow" },
{ "action": "shell", "resource": "tail -f *", "effect": "deny" },
{ "action": "shell", "resource": "tail * -f *", "effect": "deny" },
{ "action": "shell", "resource": "tail -F *", "effect": "deny" },
{ "action": "shell", "resource": "tail * -F *", "effect": "deny" },
{ "action": "shell", "resource": "tail --follow*", "effect": "deny" },
{ "action": "shell", "resource": "tail * --follow*", "effect": "deny" },
{ "action": "shell", "resource": "find -exec *", "effect": "deny" },
{ "action": "shell", "resource": "find * -exec *", "effect": "deny" },
{ "action": "shell", "resource": "find -execdir *", "effect": "deny" },
{ "action": "shell", "resource": "find * -execdir *", "effect": "deny" },
{ "action": "shell", "resource": "find -ok *", "effect": "deny" },
{ "action": "shell", "resource": "find * -ok *", "effect": "deny" },
{ "action": "shell", "resource": "find -okdir *", "effect": "deny" },
{ "action": "shell", "resource": "find * -okdir *", "effect": "deny" },
{ "action": "shell", "resource": "find -delete *", "effect": "deny" },
{ "action": "shell", "resource": "find * -delete *", "effect": "deny" },
{ "action": "shell", "resource": "find -fprint *", "effect": "deny" },
{ "action": "shell", "resource": "find * -fprint *", "effect": "deny" },
{ "action": "shell", "resource": "find -fprint0 *", "effect": "deny" },
{ "action": "shell", "resource": "find * -fprint0 *", "effect": "deny" },
{ "action": "shell", "resource": "find -fprintf *", "effect": "deny" },
{ "action": "shell", "resource": "find * -fprintf *", "effect": "deny" },
{ "action": "shell", "resource": "find -fls *", "effect": "deny" },
{ "action": "shell", "resource": "find * -fls *", "effect": "deny" },
{ "action": "shell", "resource": "cat /*", "effect": "deny" },
{ "action": "shell", "resource": "cat * /*", "effect": "deny" },
{ "action": "shell", "resource": "cat *../*", "effect": "deny" },
{ "action": "shell", "resource": "cat *=/*", "effect": "deny" },
{ "action": "shell", "resource": "cat *environ*", "effect": "deny" },
{ "action": "shell", "resource": "head /*", "effect": "deny" },
{ "action": "shell", "resource": "head * /*", "effect": "deny" },
{ "action": "shell", "resource": "head *../*", "effect": "deny" },
{ "action": "shell", "resource": "head *=/*", "effect": "deny" },
{ "action": "shell", "resource": "head *environ*", "effect": "deny" },
{ "action": "shell", "resource": "tail /*", "effect": "deny" },
{ "action": "shell", "resource": "tail * /*", "effect": "deny" },
{ "action": "shell", "resource": "tail *../*", "effect": "deny" },
{ "action": "shell", "resource": "tail *=/*", "effect": "deny" },
{ "action": "shell", "resource": "tail *environ*", "effect": "deny" },
{ "action": "shell", "resource": "grep /*", "effect": "deny" },
{ "action": "shell", "resource": "grep * /*", "effect": "deny" },
{ "action": "shell", "resource": "grep *../*", "effect": "deny" },
{ "action": "shell", "resource": "grep *=/*", "effect": "deny" },
{ "action": "shell", "resource": "grep *environ*", "effect": "deny" },
{ "action": "shell", "resource": "rg /*", "effect": "deny" },
{ "action": "shell", "resource": "rg * /*", "effect": "deny" },
{ "action": "shell", "resource": "rg *../*", "effect": "deny" },
{ "action": "shell", "resource": "rg *=/*", "effect": "deny" },
{ "action": "shell", "resource": "rg *environ*", "effect": "deny" },
{ "action": "shell", "resource": "sed /*", "effect": "deny" },
{ "action": "shell", "resource": "sed * /*", "effect": "deny" },
{ "action": "shell", "resource": "sed *../*", "effect": "deny" },
{ "action": "shell", "resource": "sed *=/*", "effect": "deny" },
{ "action": "shell", "resource": "sed *environ*", "effect": "deny" },
{ "action": "shell", "resource": "wc /*", "effect": "deny" },
{ "action": "shell", "resource": "wc * /*", "effect": "deny" },
{ "action": "shell", "resource": "wc *../*", "effect": "deny" },
{ "action": "shell", "resource": "wc *=/*", "effect": "deny" },
{ "action": "shell", "resource": "wc *environ*", "effect": "deny" },
{ "action": "shell", "resource": "*>*", "effect": "deny" }
]
},
"ai-review-coordinator": {
"mode": "primary",
"model": "anthropic-review/claude-sonnet-5-5",
"permissions": [{ "action": "*", "resource": "*", "effect": "allow" }]
"permissions": [
{ "action": "*", "resource": "*", "effect": "allow" },
{ "action": "webfetch", "resource": "*", "effect": "deny" },
{ "action": "websearch", "resource": "*", "effect": "deny" },
{ "action": "subagent", "resource": "*", "effect": "deny" },
{ "action": "question", "resource": "*", "effect": "deny" },
{ "action": "execute", "resource": "*", "effect": "deny" },
{ "action": "skill", "resource": "*", "effect": "deny" },
{ "action": "shell", "resource": "*", "effect": "deny" },
{ "action": "shell", "resource": "git diff *", "effect": "allow" },
{ "action": "shell", "resource": "git log *", "effect": "allow" },
{ "action": "shell", "resource": "git show *", "effect": "allow" },
{ "action": "shell", "resource": "git blame *", "effect": "allow" },
{ "action": "shell", "resource": "git ls-files *", "effect": "allow" },
{ "action": "shell", "resource": "git status *", "effect": "allow" },
{ "action": "shell", "resource": "git rev-parse *", "effect": "allow" },
{ "action": "shell", "resource": "git merge-base *", "effect": "allow" },
{ "action": "shell", "resource": "git cat-file -p *", "effect": "allow" },
{ "action": "shell", "resource": "git cat-file -t *", "effect": "allow" },
{ "action": "shell", "resource": "git cat-file -s *", "effect": "allow" },
{ "action": "shell", "resource": "git --no-pager diff *", "effect": "allow" },
{ "action": "shell", "resource": "git --no-pager log *", "effect": "allow" },
{ "action": "shell", "resource": "git --no-pager show *", "effect": "allow" },
{ "action": "shell", "resource": "git --no-pager blame *", "effect": "allow" },
{ "action": "shell", "resource": "git --no-pager ls-files *", "effect": "allow" },
{ "action": "shell", "resource": "git --no-pager status *", "effect": "allow" },
{ "action": "shell", "resource": "rg *", "effect": "allow" },
{ "action": "shell", "resource": "grep *", "effect": "allow" },
{ "action": "shell", "resource": "cat *", "effect": "allow" },
{ "action": "shell", "resource": "ls *", "effect": "allow" },
{ "action": "shell", "resource": "head *", "effect": "allow" },
{ "action": "shell", "resource": "tail *", "effect": "allow" },
{ "action": "shell", "resource": "find *", "effect": "allow" },
{ "action": "shell", "resource": "sed -n *", "effect": "allow" },
{ "action": "shell", "resource": "wc *", "effect": "allow" },
{ "action": "shell", "resource": "tail -f *", "effect": "deny" },
{ "action": "shell", "resource": "tail * -f *", "effect": "deny" },
{ "action": "shell", "resource": "tail -F *", "effect": "deny" },
{ "action": "shell", "resource": "tail * -F *", "effect": "deny" },
{ "action": "shell", "resource": "tail --follow*", "effect": "deny" },
{ "action": "shell", "resource": "tail * --follow*", "effect": "deny" },
{ "action": "shell", "resource": "find -exec *", "effect": "deny" },
{ "action": "shell", "resource": "find * -exec *", "effect": "deny" },
{ "action": "shell", "resource": "find -execdir *", "effect": "deny" },
{ "action": "shell", "resource": "find * -execdir *", "effect": "deny" },
{ "action": "shell", "resource": "find -ok *", "effect": "deny" },
{ "action": "shell", "resource": "find * -ok *", "effect": "deny" },
{ "action": "shell", "resource": "find -okdir *", "effect": "deny" },
{ "action": "shell", "resource": "find * -okdir *", "effect": "deny" },
{ "action": "shell", "resource": "find -delete *", "effect": "deny" },
{ "action": "shell", "resource": "find * -delete *", "effect": "deny" },
{ "action": "shell", "resource": "find -fprint *", "effect": "deny" },
{ "action": "shell", "resource": "find * -fprint *", "effect": "deny" },
{ "action": "shell", "resource": "find -fprint0 *", "effect": "deny" },
{ "action": "shell", "resource": "find * -fprint0 *", "effect": "deny" },
{ "action": "shell", "resource": "find -fprintf *", "effect": "deny" },
{ "action": "shell", "resource": "find * -fprintf *", "effect": "deny" },
{ "action": "shell", "resource": "find -fls *", "effect": "deny" },
{ "action": "shell", "resource": "find * -fls *", "effect": "deny" },
{ "action": "shell", "resource": "cat /*", "effect": "deny" },
{ "action": "shell", "resource": "cat * /*", "effect": "deny" },
{ "action": "shell", "resource": "cat *../*", "effect": "deny" },
{ "action": "shell", "resource": "cat *=/*", "effect": "deny" },
{ "action": "shell", "resource": "cat *environ*", "effect": "deny" },
{ "action": "shell", "resource": "head /*", "effect": "deny" },
{ "action": "shell", "resource": "head * /*", "effect": "deny" },
{ "action": "shell", "resource": "head *../*", "effect": "deny" },
{ "action": "shell", "resource": "head *=/*", "effect": "deny" },
{ "action": "shell", "resource": "head *environ*", "effect": "deny" },
{ "action": "shell", "resource": "tail /*", "effect": "deny" },
{ "action": "shell", "resource": "tail * /*", "effect": "deny" },
{ "action": "shell", "resource": "tail *../*", "effect": "deny" },
{ "action": "shell", "resource": "tail *=/*", "effect": "deny" },
{ "action": "shell", "resource": "tail *environ*", "effect": "deny" },
{ "action": "shell", "resource": "grep /*", "effect": "deny" },
{ "action": "shell", "resource": "grep * /*", "effect": "deny" },
{ "action": "shell", "resource": "grep *../*", "effect": "deny" },
{ "action": "shell", "resource": "grep *=/*", "effect": "deny" },
{ "action": "shell", "resource": "grep *environ*", "effect": "deny" },
{ "action": "shell", "resource": "rg /*", "effect": "deny" },
{ "action": "shell", "resource": "rg * /*", "effect": "deny" },
{ "action": "shell", "resource": "rg *../*", "effect": "deny" },
{ "action": "shell", "resource": "rg *=/*", "effect": "deny" },
{ "action": "shell", "resource": "rg *environ*", "effect": "deny" },
{ "action": "shell", "resource": "sed /*", "effect": "deny" },
{ "action": "shell", "resource": "sed * /*", "effect": "deny" },
{ "action": "shell", "resource": "sed *../*", "effect": "deny" },
{ "action": "shell", "resource": "sed *=/*", "effect": "deny" },
{ "action": "shell", "resource": "sed *environ*", "effect": "deny" },
{ "action": "shell", "resource": "wc /*", "effect": "deny" },
{ "action": "shell", "resource": "wc * /*", "effect": "deny" },
{ "action": "shell", "resource": "wc *../*", "effect": "deny" },
{ "action": "shell", "resource": "wc *=/*", "effect": "deny" },
{ "action": "shell", "resource": "wc *environ*", "effect": "deny" },
{ "action": "shell", "resource": "*>*", "effect": "deny" }
]
}
}
}
opencode.jsonc
+9 -3
View File
@@ -27,8 +27,14 @@
"models": {
"claude-sonnet-5-5": {
"name": "Claude Sonnet 5.5",
// Selected with REVIEW_MODEL_VARIANT=high; opencode sends adaptive thinking and "output_config": { "effort": "high" }.
"variants": [{ "id": "high", "settings": { "thinking": { "type": "adaptive" }, "effort": "high" } }],
// REVIEW_MODEL_VARIANT and REVIEW_COORDINATOR_VARIANT pick one; opencode sends adaptive thinking and "output_config": { "effort": "<id>" }.
"variants": [
{ "id": "low", "settings": { "thinking": { "type": "adaptive" }, "effort": "low" } },
{ "id": "medium", "settings": { "thinking": { "type": "adaptive" }, "effort": "medium" } },
{ "id": "high", "settings": { "thinking": { "type": "adaptive" }, "effort": "high" } },
{ "id": "xhigh", "settings": { "thinking": { "type": "adaptive" }, "effort": "xhigh" } },
{ "id": "max", "settings": { "thinking": { "type": "adaptive" }, "effort": "max" } }
],
"limit": { "context": 1000000, "output": 128000 }
}
}
@@ -45,7 +51,7 @@
"models": {
"deepseek-flash": {
"name": "DeepSeek-V4.1-Flash",
// Selected with REVIEW_MODEL_VARIANT=max (the workflow matrix); opencode sends "reasoning_effort": "max".
// Selected with REVIEW_MODEL_VARIANT=max; opencode sends "reasoning_effort": "max".
"variants": [{ "id": "max", "settings": { "reasoningEffort": "max" } }],
"limit": { "context": 1048576, "output": 393216 }
}
scripts/ai-review/README.md
+90 -10
View File
@@ -1,7 +1,7 @@
# AI Code Review (`scripts/ai-review`)
Automated PR review that posts a single consolidated comment on every pull request. Driven by
`.github/workflows/ai-code-review.yml` → `run-review.mjs`, using Claude Sonnet 5.5 (`anthropic-review/claude-sonnet-5-5`) at high effort with adaptive thinking via `opencode`.
`.github/workflows/ai-code-review.yml` → `run-review.mjs`, using Claude Sonnet 5.5 (`anthropic-review/claude-sonnet-5-5`) with adaptive thinking via `opencode`, at `high` effort unless the `AI_REVIEW_EFFORT` variable says otherwise (see Effort below).
The posted comment is branded **OpenObserve Code Reviewer**. Branding is *cosmetic + identity
only* — it does not change what the reviewer finds or decides.
@@ -18,12 +18,12 @@ whose marker still names the provider, such as o2-enterprise's older one, keeps
## How it works (one paragraph)
On each PR the workflow fetches the diff, filters noise, picks a risk tier (`trivial` / `lite` /
`full`), runs the tier's specialist agents (security, correctness, code-quality, performance,
`full` / `test-only` / `ci-only`), runs the tier's specialist agents (security, correctness, code-quality, performance,
documentation) in parallel — plus any *conditional* agent whose domain the diff touches (frontend,
qa, and release for migrations, `Cargo.toml`/`package.json`, `src/config/src/config.rs`, `.github/`,
`deploy/` and Docker files) — and a coordinator consolidates their findings into one comment with
a decision. Correctness owns the PR's behaviour end to end (callers, sibling
paths, state, the tests that prove the change); every tier runs it. Before the agents start, the
paths, state, the tests that prove the change); every tier but `ci-only` runs it. Before the agents start, the
runner greps the PR tree for where each symbol the diff changes is used outside the diff, and for
other definitions of the same name (`src/impact.mjs`), and gives that map to every reviewer.
Security and correctness also get `openobserve-checklist.md`, the defect classes that recur in
@@ -40,6 +40,50 @@ and folds findings at the same file and nearby line when one title's words conta
reviewed when any of its passes finished, and the comment notes a lost pass. Each pass is billed,
so the review costs about M times one pass; reviewer spans carry `review.pass`.
### Risk tiers
`assessRiskTier` in `run-review.mjs` picks the tier. More than 50 files, or any path that looks
security-sensitive (`auth`, `token`, `secret`, `crypto`, `password`, `oauth`, `unsafe`, `acme`,
`cert`), is `full`. Otherwise a PR whose every file is test code (`tests/`, `test/`, `e2e/`,
`fixtures/`, `*.spec.*`, `*.test.*`, `*_test.rs`; `isTestOnlyPath` in `src/selection.mjs`) is
`test-only` and runs correctness and code-quality, whatever its size (qa joins when Playwright files change); a rename counts by both its source and
its destination, so code moved from `src/` into `tests/` is not test-only. A PR whose every file is CI
definition (`.github/workflows/**`, `.github/actions/**` or a top-level `.github/*.yml`;
`isCiOnlyPath`) is `ci-only` and runs security and release. Scripts, including this engine and the
two opencode configs, are code and take the size tiers below. Everything else is `trivial` (at most 10
lines), `lite` (at most 100 lines) or `full`. Conditional agents are added on top as before, and
`force_full` still forces `full`.
### Effort
`AI_REVIEW_EFFORT` (`low`, `medium`, `high`, `xhigh` or `max`, default `high`) sets the reviewers' effort, and
`AI_REVIEW_COORDINATOR_EFFORT` the coordinator's (default: the reviewers' value). The workflow
passes them as `REVIEW_MODEL_VARIANT` and `REVIEW_COORDINATOR_VARIANT`, and the matrix carries no
variant of its own, so the variables are the only source. Each value is a variant of
`claude-sonnet-5-5` in both opencode configs, which opencode sends as adaptive thinking plus
`"output_config": { "effort": "<value>" }`. A value that is not a declared variant stops the run
before it claims a pass or calls the model, and the comment's earlier verdict and `Auto-approved`
line give way to a 🔴 Low line saying the review did not run. Spans carry the effort in
`gen_ai.request.reasoning_effort`.
### PRs that are never reviewed
The workflow's `Skip PRs with nothing to review` step runs right after the engine checkout and
before anything is installed. It lists the PR's files (a rename lists its old path too) and the
author login, committer login and signature state of each commit, and runs `skip-check.mjs` (`nothingToReviewReason` in
`src/selection.mjs`). It skips nothing unless both listings match the PR's `changed_files` and
`commits` counts and stay under the endpoints' caps (3000 files, 250 commits; `listingComplete`),
since a listing cut short could hide a code file or a human commit. A PR is skipped, with nothing posted and no model called, when its author is
`dependabot[bot]` or `renovate[bot]` and every commit is authored by that same bot, committed by
it or by `web-flow` (how GitHub commits a bot's API-made commits) and carries a signature GitHub
verified, when every file is docs (`*.md` or `docs/**`, but not markdown under `scripts/ai-review/` or
`.github/`, nor any instruction loader the sanitizer strips, at any depth and in any case: `AGENTS.md`, `CLAUDE.md`,
`CONTEXT.md` or a file under `.claude/`, `.agents/` or `.opencode/`; `isLoaderPath` in `src/sanitize.mjs`), or when every file is one the
translation workflow commits (`web/src/locales/languages/*.json` other than `en-US.json`, which a
person writes, and `scripts/translations/.translation_state.json`). Every other bot or App author
is reviewed, and so is a dependency-bot PR someone else pushed to. A manual dispatch always
reviews, and so does a run where the file or commit listing, or `skip-check.mjs` itself, fails.
The coordinator also lists the findings it kept in a hidden JSON block, which the runner strips
from the comment. Kept P0, P1 and P2 findings whose `file:line` is an added or context line of the
diff are posted as one `COMMENT` review with inline threads (at most 10, never approve or request
@@ -58,13 +102,34 @@ latest review is recorded, so a force-push back to an older reviewed head is rev
finished review records what it covered in a hidden `o2-review-sha` tag (the head SHA, plus the paired head in a
paired review, so a new commit on either side is new) and shows it in a visible `Reviewed commit`
line with the review number. A run whose head matches the tag skips before any model call, which
covers a queued run, a re-label and a plain dispatch, including the paired dispatch from
openobserve. The tag is only written when it is true: the diff the reviewers read is rebuilt locally
covers a queued run, a re-label and the paired dispatch from openobserve (o2-enterprise does not
set `REVIEW_MANUAL_RUN`); a manual dispatch here, `force_full` or `ignore_pass_limit` reviews it again. The tag is only written when it is true: the diff the reviewers read is rebuilt locally
from `MERGE_BASE_SHA` and `HEAD_SHA` (both sides of a paired review), because the API serves the
branch as it is now. If that cannot be done, or no reviewer finished (each one failed or returned
nothing parseable), the run writes no tag and names no commit, so that commit can be reviewed again.
A run where no reviewer finished also reports `Not reviewed` and fails the check.
A stamped review that was complete (every reviewer pass finished without a failure, timeout or
wrap-up, and the coordinator merged them without falling back; `reviewComplete` in
`src/passes.mjs`) also records a fingerprint of the unfiltered diff, in a hidden
`o2-review-diff` tag (`diffFingerprint` in `src/pr-diff.mjs`: a SHA-256 of the diff with blob ids
and hunk line numbers removed but each hunk's function context kept, so the same lines changed in another function differ, while a rebase or a merge from main that leaves the PR's own changed
and context lines alone keeps it). Lockfiles, assets and generated files the reviewers never see are
in it, so adding one reviews again instead of carrying the old merge confidence over. Equal text proves
the same change only where it sits once in the base, so both the write and the skip also need every
hunk's old lines (context and removed lines) to occur exactly once in its base file (`hunksLocatable`);
an edit to one of two identical blocks, or a base file that cannot be read, reviews. When a new head's pinned diff has the same fingerprint, the run
skips with `unchanged_diff` before any model call: it uses no pass, keeps the review and its merge
confidence, and moves the `o2-review-sha` tag to the new head; the `Reviewed commit` line keeps the
commit the findings' line numbers refer to. Binary files keep their blob ids and trailing spaces count, so
both still change the fingerprint. The
check runs ahead of the review cap, so such a push is skipped, not refused, after the cap. A manual
dispatch (`REVIEW_MANUAL_RUN`), `force_full` or `ignore_pass_limit` always reviews, and so does a
diff that is not pinned to the head. In `approve` mode the skip still dismisses the earlier
approval, and the kept review's `Auto-approved` line becomes `Needs human review`, because a
skip does not rerun the approval gate's checks. A partial review records no fingerprint, so the next push
reviews again even when its diff is the same.
There is no cap on reviews per PR unless the variable `AI_REVIEW_MAX_PASSES` is a positive integer;
unset, empty, `0` or an invalid value means no cap. Reviews are counted either way, in a hidden
`o2-review-pass` tag written before the reviewers start, so a run stopped by removing the label or
@@ -76,9 +141,9 @@ the list call fails) the run fails instead of reviewing, since neither the revie
count can be checked.
A push does not cancel a review in progress: the paid run finishes and only the newest push waits
in the concurrency queue. To review a commit again, dispatch the workflow with `ignore_pass_limit`
(it bypasses both the dedupe and the cap, and is forwarded to a paired o2-enterprise review) or with
`force_full` (it bypasses the dedupe only), or delete the review comment to reset the count.
in the concurrency queue. To review a commit again, dispatch the workflow (a plain dispatch or
`force_full` bypasses the dedupe only; `ignore_pass_limit` bypasses both the dedupe and the cap; the
two inputs are forwarded to a paired o2-enterprise review), or delete the review comment to reset the count.
The comment marker is matched as a pattern, so a retired provider leg's comment is adopted and
rewritten in place, including older comments whose marker was redacted to
@@ -204,8 +269,8 @@ A PR must never be able to run code next to the review secrets or weaken its own
| `src/opencode.mjs` env allowlist | `opencode serve` runs in `./pr` with only `PATH`, a fresh temp `HOME`/XDG dirs, `OPENCODE_CONFIG` (this repo's `opencode.jsonc` or `opencode.ci.jsonc` from `./engine`), the `OPENCODE_*` disable flags, a random server password and the one provider key. No `GH_*`, `GITHUB_*`, `ACTIONS_*`, `RUNNER_*` or `O2_*` variables. |
| Config selection | `REVIEW_OPENCODE_CONFIG` = `default` (`opencode.jsonc`) or `ci-full` (`opencode.ci.jsonc`); any other value stops the run. The workflow sets `ci-full`, which keeps `approve` ineligible (see Auto-approve). |
| `opencode.jsonc` permissions | Each `ai-review-*` agent: deny `*`, then allow `read`, `glob`, `grep`; `external_directory` denied. A `provider.use` policy allows only `anthropic-review`. LSP, formatters, snapshots and sharing are off. |
| `opencode.ci.jsonc` permissions | Same provider, model and agents, but each agent allows `*`: shell, edit, write and webfetch run with no prompt. The PR checkout is untrusted input to an agent that can run commands, and the provider key is in the opencode process environment, so a shell command can read it. |
| Startup self-check | Refuses to review if any agent is missing, any non-built-in plugin or MCP server loaded, or a granted tool's plugin is not active. Under `default` it also refuses any agent that resolves to allow a tool other than `read`/`glob`/`grep` or any outside directory; under `ci-full` it refuses an agent that does not allow every tool without asking. Permission prompts are rejected under `default` and approved once under `ci-full`. |
| `opencode.ci.jsonc` permissions | Same provider, model, variants and agents. Each agent allows `*` (read, glob, grep, edit, write run with no prompt), then denies `webfetch`, `websearch`, `subagent`, `question` (which would wait on an answer nobody in CI gives), `execute` (opencode 2.2.0's Code Mode: it runs JavaScript with `fetch`, so it would reach the network past the `webfetch` deny) and `skill` (reviewers read the skill copies with `read`). `shell` is kept to an allowlist: `git diff`, `git log`, `git show`, `git blame`, `git ls-files`, `git status` (each also after `git --no-pager`), `git rev-parse`, `git merge-base`, `git cat-file -p`/`-t`/`-s`, `rg`, `grep`, `cat`, `ls`, `head`, `tail`, `find`, `sed -n` and `wc`. Later deny rules refuse `find` with `-exec`, `-execdir`, `-ok`, `-okdir`, `-delete`, `-fprint`, `-fprint0`, `-fprintf` or `-fls` as a separate word; `tail -f`, `-F` and `--follow`; any absolute path, `../` path, `=/` argument or `environ` in `cat`, `head`, `tail`, `grep`, `rg`, `sed` and `wc` (so `/proc/*/environ` and `/dev/*` are not read through them); and any command containing `>` (an output redirect). `git grep` is not allowed (its `-O` runs a pager command). opencode checks every command in a pipeline, `&&` or `;` chain and inside `$(...)` or backticks, so the command runs only when every part is allowed, and a denied command fails outright, without a prompt. The allowlist stops builds, installs and long exploration. It is not a sandbox: `rg --pre` or a GNU `sed` `e` command can still run a program, a glob or a relative path inside the checkout is not inspected, and edit/write stay allowed in the disposable checkout. The PR checkout is untrusted input, and the provider key is in the opencode process environment, so a command that gets past these rules can read it. |
| Startup self-check | Refuses to review if any agent is missing, any non-built-in plugin or MCP server loaded, or a granted tool's plugin is not active. Under `default` it also refuses any agent that resolves to allow a tool other than `read`/`glob`/`grep` or any outside directory; under `ci-full` it refuses an agent that does not allow read, glob, grep, edit and write without asking, that leaves webfetch, websearch, subagent, question, execute or skill enabled, or whose shell is denied or not restricted to an allowlist. Permission prompts are rejected under `default` and approved once under `ci-full`. |
| `src/comments.mjs` | The previous review is only ever a marker comment by the reviewer app (`REVIEW_BOT_LOGIN` = `<app-slug>[bot]`), read across every page; anyone else's comment carrying the marker is never fed to the reviewers or edited. When the app token cannot be minted the review posts as `github-actions[bot]`, which any same-repo PR workflow can also post as, so that run uses no previous review and posts a fresh comment. |
| `src/scrub.mjs` | Before posting: redacts values of secret-named env vars and token-shaped strings (inside code too), breaks forged `<!-- o2-review` markers, and neutralises `@mentions` outside code. |
@@ -230,6 +295,21 @@ blocking request that waits for the run dies at undici's 300 s `headersTimeout`
A timed-out agent is interrupted server-side; transient provider errors are retried at most twice,
each in a fresh session.
A wrap-up keeps a long session's findings instead of discarding them at the 20-minute timeout. A
reviewer still running 3 minutes before its timeout is interrupted, and once opencode confirms the
interrupt the same session gets one more prompt asking for its verified findings in the required
format with no further tools. The interrupt keeps the steps already run, so the answer comes from
what the agent read, mostly from cache. If the interrupt is not confirmed, or the answer does not
arrive in time, the session ends as a timeout, as before; a run that finished on its own just
before the interrupt keeps its own answer. No agent sets opencode's `steps` cap: on 2.2.0 the capped
turn ends the request on an assistant message (its "maximum steps reached" text), a prefill the
model rejects with a 400 when thinking is on; `test/integration/opencode-sandbox.test.mjs` pins
that. Reviewer
spans carry `review.wrapped_up`. A wrapped-up answer counts as findings but not as a finished
reviewer, so an area with no uninterrupted pass lowers merge confidence like an unfinished one and
blocks auto-approval. Sessions shorter than 6 minutes, and the coordinator, get no
wrap-up.
The binary is pinned in `opencode-pin.env` and installed by `install-opencode.sh`, which verifies
the release asset's SHA-256. Bump both lines together, to a release at least 14 days old (waived
once, for 2.2.0).
scripts/ai-review/agents/coordinator.md
+2 -2
View File
@@ -119,7 +119,7 @@ If findings exist:
| | |
|---|---|
| **Risk tier** | [trivial \| lite \| full] |
| **Risk tier** | [trivial \| lite \| full \| test-only \| ci-only] |
| **Reviewers** | [exactly the agents listed in `<reviewers_ran>`, each with its glyph] |
</details>
@@ -142,7 +142,7 @@ If NO issues across all reviewers:
| | |
|---|---|
| **Risk tier** | [trivial \| lite \| full] |
| **Risk tier** | [trivial \| lite \| full \| test-only \| ci-only] |
| **Reviewers** | [exactly the agents listed in `<reviewers_ran>`, each with its glyph] |
</details>
scripts/ai-review/run-review.mjs
+118 -15
View File
@@ -26,14 +26,17 @@ import {
parseRawDiff,
reviewerFacts,
sizeReasons,
withoutCarriedApproval,
} from "./src/auto-approve.mjs";
import {
ANY_REVIEW_MARKER,
NOTICE_TAG,
REVIEW_HEADING,
REVIEW_TITLE,
bodyAtFailedRun,
bodyAtPassLimit,
bodyAtReviewedHead,
bodyAtUnchangedDiff,
bodyAtUnreviewedHead,
claimedReviewBody,
finishReviewBody,
@@ -44,8 +47,10 @@ import {
reviewKey,
reviewPassCount,
reviewSkipReason,
reviewedDiffFingerprint,
sanitizeReviewBody,
selectOwnReviewComment,
shouldSkipAsUnchangedDiff,
withPassTag,
} from "./src/comments.mjs";
import {
@@ -56,10 +61,11 @@ import {
selectInlineComments,
} from "./src/inline.mjs";
import { changedCodeEvidence, mergeConfidence, reviewerFinished, severityCounts, unfinishedReviewers } from "./src/confidence.mjs";
import { OpencodeServer, resolveOpencodeConfig, selfCheck, shouldRetry } from "./src/opencode.mjs";
import { lostPasses, lostPassesNote, mergePassResults, parseColdPasses, passLabel, passPlan, runPooled } from "./src/passes.mjs";
import { OpencodeServer, modelVariantIds, resolveOpencodeConfig, selfCheck, shouldRetry } from "./src/opencode.mjs";
import { stripJsonc } from "./src/jsonc.mjs";
import { lostPasses, lostPassesNote, mergePassResults, parseColdPasses, passLabel, passPlan, reviewComplete, runPooled } from "./src/passes.mjs";
import { MAX_USER_PROMPT_CHARS, fitCoordinatorPrompt } from "./src/coordinator-input.mjs";
import { fetchPinnedDiff, prefixDiffPaths, reviewerDiff } from "./src/pr-diff.mjs";
import { diffFingerprint, fetchPinnedDiff, hunksLocatable, prefixDiffPaths, reviewerDiff, runFile } from "./src/pr-diff.mjs";
import {
buildFallbackReview,
partialFindingsBlock,
@@ -73,7 +79,7 @@ import {
} from "./src/report.mjs";
import { sanitizePrTree } from "./src/sanitize.mjs";
import { scrubReviewBody, scrubSecrets } from "./src/scrub.mjs";
import { isReleaseFile, resolveRulesFiles } from "./src/selection.mjs";
import { isReleaseFile, narrowTier, resolveRulesFiles } from "./src/selection.mjs";
import { buildImpactContext } from "./src/impact.mjs";
const __dirname = dirname(fileURLToPath(import.meta.url));
@@ -81,7 +87,7 @@ const __dirname = dirname(fileURLToPath(import.meta.url));
// ENGINE_DIR is the trusted base-branch checkout; PR_DIR is the PR head, which is data only and never executed.
const ENGINE_DIR = resolve(process.env.ENGINE_DIR || resolve(__dirname, "../.."));
const PR_DIR = process.env.PR_DIR ? resolve(process.env.PR_DIR) : "";
// REVIEW_OPENCODE_CONFIG: "default" (opencode.jsonc, read-only tools) or "ci-full" (opencode.ci.jsonc, every tool allowed).
// REVIEW_OPENCODE_CONFIG: "default" (opencode.jsonc, read-only tools) or "ci-full" (opencode.ci.jsonc, file tools and a shell allowlist).
const OPENCODE_CONFIG = resolveOpencodeConfig(process.env.REVIEW_OPENCODE_CONFIG);
const OPENCODE_CONFIG_PATH = resolve(ENGINE_DIR, OPENCODE_CONFIG.file);
// A same-named PR in another repo reviewed together with this one (the OSS side of an ENT pair).
@@ -102,13 +108,14 @@ const PAIRED = {
//
// - REVIEW_PROVIDER_ID / REVIEW_MODEL_ID: opencode provider+model IDs (must match a
// provider/model registered in opencode.jsonc).
// - REVIEW_MODEL_VARIANT: opencode `variant` on session.prompt (empty ⇒ omit).
// - REVIEW_MODEL_VARIANT / REVIEW_COORDINATOR_VARIANT: opencode `variant` (effort) of reviewers / coordinator (coordinator defaults to reviewers').
// - REVIEW_API_KEY_ENV: name of the env var holding the provider API key (checked for presence).
// - REVIEW_MARKER: HTML comment marker that identifies this provider's comment on the PR, so
// providers post/update independent comments and never clobber each other.
const PROVIDER_ID = process.env.REVIEW_PROVIDER_ID || "anthropic-review";
const MODEL_ID = process.env.REVIEW_MODEL_ID || "claude-sonnet-5-5";
const MODEL_VARIANT = process.env.REVIEW_MODEL_VARIANT ?? "";
const COORDINATOR_VARIANT = process.env.REVIEW_COORDINATOR_VARIANT || MODEL_VARIANT;
const GEN_AI_PROVIDER_NAME = process.env.REVIEW_GEN_AI_PROVIDER_NAME || genAiProviderName(PROVIDER_ID);
// REVIEW_API_KEY_ENV holds the NAME of the env var carrying the key (e.g. "ANTHROPIC_REVIEW_API_KEY"),
// never the key itself — the value is read only via apiKey() below and is never logged or
@@ -135,6 +142,8 @@ const REVIEW_INLINE = (process.env.REVIEW_INLINE || "false").toLowerCase() === "
// Optional ceiling on paid reviews per PR, 0 for none; the count lives in the review comment, so deleting that comment resets it.
const REVIEW_MAX_PASSES = parseMaxPasses(process.env.REVIEW_MAX_PASSES);
const REVIEW_IGNORE_PASS_LIMIT = process.env.REVIEW_IGNORE_PASS_LIMIT === "true";
// A run someone started by hand always reviews, even a diff the last review already read.
const REVIEW_MANUAL_RUN = process.env.REVIEW_MANUAL_RUN === "true";
// Anything but an exact known mode is "off": the gate must be opted into.
const AUTO_APPROVE_MODE = parseAutoApproveMode(process.env.REVIEW_AUTO_APPROVE);
// Only the workflow's "no open PR on this branch" path sets this; any failed or skipped pairing lookup leaves it unset.
@@ -151,6 +160,11 @@ const OVERALL_TIMEOUT_MS = 40 * 60 * 1000;
// Kept free at the end of the run for rendering and posting the comment.
const POST_RESERVE_MS = 2 * 60 * 1000;
const MIN_COORDINATOR_MS = 60 * 1000;
// A reviewer still running this long before its timeout is interrupted and asked for its findings, which a hard timeout throws away.
const REVIEWER_WRAP_UP = {
ms: 3 * 60 * 1000,
text: "Time is up for this review. Do not call any more tools. Reply now with your final output in the exact format your instructions require: a <findings> block with every issue you have already verified, or the no-issues marker if you verified none. Do not report anything you have not checked.",
};
// Covers every area of two passes at once (8 reviewers at most); a third pass queues instead of adding to provider rate limits.
const REVIEWER_CONCURRENCY = 16;
const TRACE_SERVICE_NAME = "github-actions.ai-code-review";
@@ -255,6 +269,15 @@ const RISK_TIERS = {
agents: ["security", "correctness", "code-quality", "performance", "documentation"],
coordinatorAgent: COORDINATOR_AGENT,
},
"test-only": {
// qa reviews Playwright only, and is added by CONDITIONAL_AGENTS when those files change.
agents: ["correctness", "code-quality"],
coordinatorAgent: COORDINATOR_AGENT,
},
"ci-only": {
agents: ["security", "release"],
coordinatorAgent: COORDINATOR_AGENT,
},
};
// Agents added on top of the tier's set when the diff touches their domain, regardless of size.
@@ -301,6 +324,20 @@ function execErrorText(err) {
return [err?.message, err?.stderr?.toString(), err?.stdout?.toString()].filter(Boolean).join(" ");
}
// cat-file reads the raw blob, so the PR's own attributes cannot reshape the base text a hunk is matched against.
function readBaseFile(path) {
const paired = Boolean(PAIRED.dir) && path.startsWith(`${PAIRED.dir}/`);
const rel = paired ? path.slice(PAIRED.dir.length + 1) : path;
const dir = paired ? resolve(PR_DIR, PAIRED.dir) : PR_DIR;
const base = paired ? PAIRED.mergeBase : process.env.MERGE_BASE_SHA;
if (!dir || !base) return null;
try {
return runFile("git", ["-C", dir, "cat-file", "blob", `${base}:${rel}`]);
} catch {
return null;
}
}
// The API refuses diffs over 20k lines; the workflow fetches the merge base into PR_DIR so the same three-dot diff can be built locally.
function fetchPrDiff(prNumber) {
const log = message => console.log(`[${isoNow()}] ${message}`);
@@ -739,9 +776,13 @@ function filterDiff(diff) {
function assessRiskTier(filteredDiff) {
const totalLines = filteredDiff.totalAdded + filteredDiff.totalRemoved;
const fileCount = filteredDiff.files.length;
const hasSecurityFiles = filteredDiff.files.some(f => isSecuritySensitiveFile(f.newPath));
// A rename's source counts too: moving code out of production or CI removes it from there.
const paths = filteredDiff.files.flatMap(f => [f.oldPath, f.newPath]);
const hasSecurityFiles = paths.some(isSecuritySensitiveFile);
if (fileCount > 50 || hasSecurityFiles) return "full";
const narrow = narrowTier(paths.map(logicalPath));
if (narrow) return narrow;
if (totalLines <= 10 && fileCount <= 20) return "trivial";
if (totalLines <= 100 && fileCount <= 20) return "lite";
return "full";
@@ -782,7 +823,7 @@ async function opencodeClient() {
const client = await OPENCODE.ensureStarted();
if (!selfChecks.has(client)) {
const check = selfCheck(client, reviewOpencodeAgents(), { readOnly: OPENCODE_CONFIG.readOnly }).then(result => {
const grant = OPENCODE_CONFIG.readOnly ? "only read/glob/grep allowed" : "every tool allowed (ci-full)";
const grant = OPENCODE_CONFIG.readOnly ? "only read/glob/grep allowed" : "file tools and a shell allowlist (ci-full)";
console.log(`[${isoNow()}] opencode self-check passed: ${result.plugins} built-in plugins, ${grant} for ${result.agents} agents`);
return result;
});
@@ -797,11 +838,12 @@ function remainingRunMs() {
}
async function callOpencode(agentName, systemPrompt, userPrompt, timeoutMs = AGENT_TIMEOUT_MS, traceOptions = {}) {
const variant = traceOptions.agentKey === "coordinator" ? COORDINATOR_VARIANT : MODEL_VARIANT;
const llmSpan = TRACE.startSpan("gen_ai.chat", {
"gen_ai.operation.name": "chat",
"gen_ai.provider.name": GEN_AI_PROVIDER_NAME,
"gen_ai.request.model": MODEL_ID,
"gen_ai.request.reasoning_effort": MODEL_VARIANT || "default",
"gen_ai.request.reasoning_effort": variant || "default",
"ai.review.operation.name": traceOptions.operationName || "chat",
"ai.agent.key": traceOptions.agentKey,
"ai.agent.name": traceOptions.agentName,
@@ -826,12 +868,14 @@ async function callOpencode(agentName, systemPrompt, userPrompt, timeoutMs = AGE
const result = await client.prompt({
agent: agentName,
model: { providerID: PROVIDER_ID, modelID: MODEL_ID },
variant: MODEL_VARIANT,
variant,
system: truncatedSystem,
text: truncatedUser,
title: `ai-review:${traceOptions.agentKey || agentName}${traceOptions.pass ? `:pass-${traceOptions.pass}` : ""}`,
timeoutMs,
wrapUp: traceOptions.wrapUp,
});
if (result.wrappedUp) console.warn(`[${isoNow()}] ${agentName}: interrupted near its timeout and asked for its findings`);
if (result.rejectedPermissions > 0) {
console.warn(`[${isoNow()}] ${agentName}: rejected ${result.rejectedPermissions} permission request(s)`);
}
@@ -844,8 +888,9 @@ async function callOpencode(agentName, systemPrompt, userPrompt, timeoutMs = AGE
"ai.review.output.chars": result.text.length,
"ai.review.attempts": attempt,
"opencode.permission_rejections": result.rejectedPermissions,
"ai.review.wrapped_up": Boolean(result.wrappedUp),
});
return { text: result.text, responseId: result.responseId };
return { text: result.text, responseId: result.responseId, wrappedUp: Boolean(result.wrappedUp) };
} catch (err) {
failedUsage = addUsage(failedUsage, err?.usage);
const retry = shouldRetry({
@@ -945,6 +990,7 @@ async function runReviewer(agentKey, agentDef, diff, prContext, existingReview,
agentName: agentDef.name,
pass,
operationName: "code_review",
wrapUp: REVIEWER_WRAP_UP,
},
);
const text = completion.text;
@@ -964,9 +1010,10 @@ async function runReviewer(agentKey, agentDef, diff, prContext, existingReview,
"review.elapsed_seconds": Number(elapsed),
"review.findings": findings.length,
"review.raw_text_length": text.length,
"review.wrapped_up": completion.wrappedUp,
"gen_ai.response.id": completion.responseId,
});
return { agentKey, agentName: agentDef.name, pass, findings, rawText: text, error: null, genAIResponseId: completion.responseId, fullDiff, parsed };
return { agentKey, agentName: agentDef.name, pass, findings, rawText: text, error: null, genAIResponseId: completion.responseId, fullDiff, parsed, wrappedUp: completion.wrappedUp };
} catch (err) {
const elapsed = ((Date.now() - start) / 1000).toFixed(1);
console.error(`[${isoNow()}] ${label} FAILED after ${elapsed}s: ${err.message}`);
@@ -1146,6 +1193,10 @@ function noteReviewedHead(prNumber, comment) {
noteOnComment(prNumber, comment, bodyAtReviewedHead(comment.body, REVIEW_MAX_PASSES));
}
function noteFailedRun(prNumber, comment, key) {
if (comment) noteOnComment(prNumber, comment, bodyAtFailedRun(comment.body, key));
}
function noteSkippedPush(prNumber, comment, key) {
noteOnComment(prNumber, comment, bodyAtPassLimit(comment.body, REVIEW_MAX_PASSES, key));
}
@@ -1262,6 +1313,34 @@ function writeFindingsArtifact(prNumber, tier, results, coordinatorResult, inlin
// ─── Main ──────────────────────────────────────────────────────────────────
// A variant the config does not declare would fail every session, so the run stops before it claims a pass.
function variantProblem() {
let config;
try {
config = JSON.parse(stripJsonc(readFileSync(OPENCODE_CONFIG_PATH, "utf-8")));
} catch (err) {
return `cannot read ${OPENCODE_CONFIG.file}: ${err.message}`;
}
const known = modelVariantIds(config, PROVIDER_ID, MODEL_ID);
const unknown = [...new Set([MODEL_VARIANT, COORDINATOR_VARIANT])].filter(v => v && !known.includes(v));
return unknown.length > 0 ? `variant "${unknown.join('", "')}" is not declared for ${MODEL_SLUG} in ${OPENCODE_CONFIG.file} (declared: ${known.join(", ") || "none"})` : "";
}
// Fetched ahead of the cap check, and only when the comment holds a fingerprint, so a push that changed nothing is skipped even past the cap.
function unchangedDiffCheck(prNumber, comment) {
const force = REVIEW_MANUAL_RUN || REVIEW_IGNORE_PASS_LIMIT || process.env.FORCE_FULL === "true";
if (force || !reviewedDiffFingerprint(comment?.body)) return null;
try {
const fetched = fetchPrDiff(prNumber);
const fingerprint = diffFingerprint(fetched.diff);
const unchanged = shouldSkipAsUnchangedDiff(comment.body, fingerprint, { pinned: fetched.pinned, force }) && hunksLocatable(fetched.diff, readBaseFile);
return { fetched, unchanged };
} catch (err) {
console.warn(`[${isoNow()}] Could not fetch the diff to compare with the last review, so it is reviewed: ${execErrorText(err).slice(0, 300)}`);
return null;
}
}
function prepareReviewTree(rootSpan) {
if (!PR_DIR) {
if (process.env.GITHUB_ACTIONS === "true") throw new Error("PR_DIR is required in CI: the PR must be reviewed from its own sanitized checkout");
@@ -1323,6 +1402,7 @@ async function main() {
maxPasses: REVIEW_MAX_PASSES,
ignorePassLimit: REVIEW_IGNORE_PASS_LIMIT,
forceFull: process.env.FORCE_FULL === "true",
manual: REVIEW_MANUAL_RUN,
});
const gate = createAutoApproveGate({
mode: AUTO_APPROVE_MODE,
@@ -1333,6 +1413,15 @@ async function main() {
});
// Ahead of the cap skip too: a push the cap leaves unreviewed must still lose the earlier approval.
if (skipReason !== "already_reviewed") gate.start();
const early = skipReason === "already_reviewed" ? null : unchangedDiffCheck(prNumber, existingComment);
if (early?.unchanged) {
console.log(`[${isoNow()}] PR #${prNumber} skipped (unchanged_diff): ${headKey} changes the same lines the last review read. Dispatch the workflow to review it again.`);
const kept = bodyAtUnchangedDiff(existingComment.body, headKey, REVIEW_MAX_PASSES);
noteOnComment(prNumber, existingComment, AUTO_APPROVE_MODE === "approve" ? withoutCarriedApproval(kept) : kept);
outcome = "skipped";
TRACE.setSpanAttributes(rootSpan, { "review.skipped": true, "review.skip_reason": "unchanged_diff" });
return;
}
if (skipReason) {
console.log(`[${isoNow()}] PR #${prNumber} skipped (${skipReason}): ${headKey || "no head"}, ${passesDone} review(s) so far. Dispatch the workflow with ignore_pass_limit to review again.`);
if (skipReason === "pass_limit_reached") noteSkippedPush(prNumber, existingComment, headKey);
@@ -1366,7 +1455,14 @@ async function main() {
throw new Error(message);
}
console.log(`[${isoNow()}] ${MODEL_SLUG} (via opencode serve): configured`);
const variantError = variantProblem();
if (variantError) {
outcome = "misconfigured";
TRACE.setSpanAttributes(rootSpan, { "review.skipped": true, "review.skip_reason": "unknown_variant" });
noteFailedRun(prNumber, existingComment, headKey);
throw new Error(`${variantError}; fix AI_REVIEW_EFFORT or AI_REVIEW_COORDINATOR_EFFORT.`);
}
console.log(`[${isoNow()}] ${MODEL_SLUG} (via opencode serve): configured, reviewers at "${MODEL_VARIANT || "default"}", coordinator at "${COORDINATOR_VARIANT || "default"}"`);
// 1. Get PR diff
console.log(`[${isoNow()}] Fetching PR diff...`);
@@ -1376,7 +1472,7 @@ async function main() {
let diffPinned = false;
const diffSpan = TRACE.startSpan("github.pr.diff", { "github.pr.number": prNumber }, rootSpan);
try {
({ diff, own: ownDiff, pinned: diffPinned } = fetchPrDiff(prNumber));
({ diff, own: ownDiff, pinned: diffPinned } = early?.fetched ?? fetchPrDiff(prNumber));
const diffStats = diff.split("\n").length;
TRACE.endSpan(diffSpan, { "diff.raw_lines": diffStats });
console.log(`[${isoNow()}] Raw diff: ${diffStats} lines`);
@@ -1393,7 +1489,10 @@ async function main() {
const rawFiles = parseDiffFiles(diff);
const filtered = filterDiff(diff);
const changedLines = filtered.totalAdded + filtered.totalRemoved;
// The unfiltered diff: a lockfile or asset the filter hides still changes what the verdict covers.
const fingerprint = diffFingerprint(diff);
TRACE.endSpan(filterSpan, {
"diff.fingerprint": fingerprint,
"diff.changed_lines": changedLines,
"diff.filtered_files": filtered.files.length,
"diff.skipped_files": rawFiles.length - filtered.files.length,
@@ -1651,8 +1750,12 @@ async function main() {
// A run where no reviewer finished reviewed nothing, so it does not use up a pass.
const reviewed = completedAgents.length > 0;
const passes = reviewed ? passesDone + 1 : passesDone;
const complete = reviewComplete(passResults, coordinatorResult);
// A reviewer reads at most MAX_DIFF_TOKENS chars of diff, so a longer one was never fully reviewed.
const fullyRead = diff.length <= MAX_DIFF_TOKENS;
if (reviewed && !complete) console.warn(`[${isoNow()}] The review was partial, so the next push re-reviews even an unchanged diff`);
if (reviewed && !diffPinned) console.warn(`[${isoNow()}] The reviewed diff is not tied to ${headKey}, so the commit stays reviewable`);
finalReview = finishReviewBody(sanitizedReview, { key: headKey, passes, maxPasses: REVIEW_MAX_PASSES, reviewed: reviewed && diffPinned, confidence, gateLine: autoApproveLine(autoApprove) });
finalReview = finishReviewBody(sanitizedReview, { key: headKey, passes, maxPasses: REVIEW_MAX_PASSES, reviewed: reviewed && diffPinned, confidence, gateLine: autoApproveLine(autoApprove), diffFingerprint: complete && fullyRead && diffPinned && hunksLocatable(diff, readBaseFile) ? fingerprint : "" });
TRACE.setSpanAttributes(rootSpan, { "review.final_length": finalReview.length });
console.log(`[${isoNow()}] Final review length: ${finalReview.length} chars`);
scripts/ai-review/skip-check.mjs
+21
View File
@@ -0,0 +1,21 @@
// Runs before npm ci and opencode are installed, so it may import node built-ins and src/ modules that import only built-ins.
import { readFileSync } from "node:fs";
import { listingComplete, nothingToReviewReason } from "./src/selection.mjs";
const TSV_ESCAPES = { "\\t": "\t", "\\n": "\n", "\\r": "\r", "\\\\": "\\" };
const lines = file => {
const out = file ? readFileSync(file, "utf-8").split("\n").map(line => line.replace(/\r$/, "")) : [];
if (out.at(-1) === "") out.pop();
return out;
};
const unescapeTsv = field => field.replace(/\\[tnr\\]/g, seq => TSV_ESCAPES[seq]);
const [listFile, commitsFile] = process.argv.slice(2);
const fileLines = lines(listFile).filter(line => line.trim());
const paths = fileLines.flatMap(line => line.split("\t").map(unescapeTsv).filter(Boolean));
const commits = lines(commitsFile).map(line => {
const [author = "", committer = "", verified = ""] = line.split("\t");
return { author, committer, verified: verified === "true" };
});
const complete = listingComplete({ files: fileLines.length, commits: commits.length, expectedFiles: process.env.EXPECTED_FILES, expectedCommits: process.env.EXPECTED_COMMITS });
process.stdout.write(complete ? nothingToReviewReason({ author: process.env.PR_AUTHOR || "", commits, paths }) : "");
scripts/ai-review/src/auto-approve.mjs
+8 -1
View File
@@ -1,4 +1,5 @@
import { removeRepeatedly } from "./comments.mjs";
import { reviewerCompleted } from "./confidence.mjs";
import { LOADER_DIRS, LOADER_FILES } from "./sanitize.mjs";
export const MAX_APPROVE_LINES = 100;
@@ -62,6 +63,7 @@ const SUBMODULE_MODE = "160000";
const KNOWN_HEADER_RE = /^(?:index |--- |\+\+\+ |similarity index |dissimilarity index |rename |copy |old mode |new mode |new file mode |deleted file mode |Binary files |GIT binary patch|\\ |$)/;
const HUNK_RE = /^@@ -\d+(?:,(\d+))? \+\d+(?:,(\d+))? @@/;
const APPROVED_LINE = "**Auto-approved** — small change, all gates passed.";
const UNCHANGED_DIFF_APPROVAL_LINE = "**Needs human review** — an approval does not carry over to a new commit, even one with the same reviewed diff";
const APPROVAL_BODY = "Auto-approved by the review gate: small change, all gates passed. CI must still pass before merge.";
const DISMISSAL_MESSAGE = "New commits arrived; an automatic approval does not carry over.";
@@ -234,7 +236,7 @@ function reviewerReasons(reviewers) {
// Parsed reviewer output is the only proof a reviewer looked; a clean exit with empty text is not.
export function reviewerFacts(results) {
return results.map(r => ({ key: r.agentKey, ran: !r.skipped, finished: !r.skipped && !r.error && r.parsed === true, fullDiff: r.fullDiff === true }));
return results.map(r => ({ key: r.agentKey, ran: !r.skipped, finished: reviewerCompleted(r), fullDiff: r.fullDiff === true }));
}
function pairingReasons({ paired, pairClear }) {
@@ -386,3 +388,8 @@ export function autoApproveLine(result) {
if (result?.mode !== "approve") return "";
return result.approved ? APPROVED_LINE : `**Needs human review** — ${result.reasons.join("; ")}`;
}
// The unchanged-diff skip still dismisses the approval: the fingerprint ignores lockfiles and other noise the approval gate checks.
export function withoutCarriedApproval(body) {
return String(body ?? "").split(APPROVED_LINE).join(UNCHANGED_DIFF_APPROVAL_LINE);
}
scripts/ai-review/src/comments.mjs
+39 -10
View File
@@ -1,4 +1,4 @@
import { isSettledConfidenceLine, mergeConfidenceLine, reviewingConfidenceLine, staleConfidenceLine } from "./confidence.mjs";
import { isSettledConfidenceLine, mergeConfidenceLine, notRunConfidenceLine, reviewingConfidenceLine, staleConfidenceLine } from "./confidence.mjs";
import { scrubReviewBody } from "./scrub.mjs";
export const REVIEW_PASS_TAG = "<!--\\s*o2-review-pass:(\\d+)\\s*-->";
@@ -6,6 +6,8 @@ export const REVIEW_PASS_TAG = "<!--\\s*o2-review-pass:(\\d+)\\s*-->";
export const REVIEW_SHA_TAG = "<!--\\s*o2-review-sha:([^\\s<>]+)\\s*-->";
// The verdict a later head's line replaced, kept in case the head returns to the reviewed commit.
export const REVIEW_HELD_TAG = "<!--\\s*o2-review-held:([^\\s<>]+)\\s*-->";
// Fingerprint of the diff the sha tag's review read, so a later head with the same changes (rebase, merge from main) needs no new review.
export const REVIEW_DIFF_TAG = "<!--\\s*o2-review-diff:([0-9a-f]{64})\\s*-->";
// Matched as a pattern so a retired leg's comment is still adopted; `[redacted]` is what redaction left in comments whose marker named the provider.
export const ANY_REVIEW_MARKER = "<!--\\s*ai-code-review[a-z0-9-]*(?:\\[redacted\\])?\\s*-->";
// Tags a comment that stands in for a review (post-paired-pointer.sh, the "key not set" notice, the in-progress claim); it is replaced in place but is not a previous review.
@@ -26,7 +28,7 @@ const AUTO_APPROVE_LINE_RE = /^(?:[^\w\n`]|<\/?[a-z]+>)*(?:auto[- ]?approved|nee
const FORGED_LINE_RE = /mergeconfidence|autoapproved|needshumanreview|reviewedcommit/;
// The fixed head of a finding line, glyph and location then category, which leaves no room for a status phrase ahead of the category.
const FINDING_LINE_RE = /^\s*- (?:[^\w\s`*<]+\s+)?(?:`[^`\n]+`\s+)?\*\*\[[^\]\n]+\]\*\*/;
const TRAILING_TAGS_RE = /(?:\s*<!--\s*o2-review-(?:(?:sha|held):[^\s<>]+|pass:\d+)\s*-->)+\s*$/i;
const TRAILING_TAGS_RE = /(?:\s*<!--\s*o2-review-(?:(?:sha|held|diff):[^\s<>]+|pass:\d+)\s*-->)+\s*$/i;
const HTML_COMMENT_RE = /<!--[\s\S]*?--!?>/g;
const MARKUP_RE = /<!--[\s\S]*?--!?>|<[^>]*>|&[a-z]+;/gi;
// "summary" stays out: it would strip the <summary> of the findings fold, and a leaked finding field sits inside a stripped <finding> block.
@@ -87,8 +89,30 @@ export function withReviewShaTag(body, key) {
return `${withoutTag(body, REVIEW_SHA_TAG)}\n${reviewShaTag(key)}`;
}
export function shouldSkipAsReviewed(body, key, { ignorePassLimit = false, forceFull = false } = {}) {
if (!key || ignorePassLimit || forceFull) return false;
// Only a body whose sha tag names a reviewed head can vouch for a diff.
export function reviewedDiffFingerprint(body) {
if (!reviewedKey(body)) return "";
return (body || "").match(new RegExp(REVIEW_DIFF_TAG, "i"))?.[1] ?? "";
}
export function withReviewDiffTag(body, fingerprint) {
const bare = withoutTag(body, REVIEW_DIFF_TAG);
return fingerprint ? `${bare}\n<!-- o2-review-diff:${fingerprint} -->` : bare;
}
// A forced run (dispatch, force_full, ignore_pass_limit) always reviews; otherwise a pinned diff equal to the reviewed one is skipped.
export function shouldSkipAsUnchangedDiff(body, fingerprint, { pinned = false, force = false } = {}) {
if (force || !pinned || !fingerprint) return false;
return reviewedDiffFingerprint(body) === fingerprint;
}
// Costs no pass; the visible commit stays the reviewed one because finding line numbers refer to it, while the tag moves so the next push compares against this head.
export function bodyAtUnchangedDiff(body, key, maxPasses) {
return withReviewShaTag(bodyAtReviewedHead(body, maxPasses), key);
}
export function shouldSkipAsReviewed(body, key, { ignorePassLimit = false, forceFull = false, manual = false } = {}) {
if (!key || ignorePassLimit || forceFull || manual) return false;
return reviewedKey(body) === key;
}
@@ -103,8 +127,8 @@ export function passLimitReached(passes, maxPasses) {
}
// Dedupe comes first so a queued run on a reviewed head stays silent instead of rewriting the limit note.
export function reviewSkipReason({ body, key, passesDone, maxPasses, ignorePassLimit = false, forceFull = false }) {
if (shouldSkipAsReviewed(body, key, { ignorePassLimit, forceFull })) return "already_reviewed";
export function reviewSkipReason({ body, key, passesDone, maxPasses, ignorePassLimit = false, forceFull = false, manual = false }) {
if (shouldSkipAsReviewed(body, key, { ignorePassLimit, forceFull, manual })) return "already_reviewed";
if (!ignorePassLimit && passLimitReached(passesDone, maxPasses)) return "pass_limit_reached";
return "";
}
@@ -207,20 +231,25 @@ export function bodyAtPassLimit(body, maxPasses, head) {
return bodyAtUnreviewedHead(withPassLimitNote(body, passLimitNote(maxPasses, reviewedKey(body), head)), head);
}
// Even on the covered commit: the run already dismissed the approval, so the old verdict and gate line must not read as current.
export function bodyAtFailedRun(body, head) {
return head ? withConfidenceLine(withHeldVerdict(body), notRunConfidenceLine(shortKey(head))) : body;
}
// The sha tag says that exact commit was read and stops it being reviewed again, so a run that cannot prove that leaves it off.
export function finishReviewBody(body, { key, passes, maxPasses, reviewed, confidence, gateLine = "" }) {
export function finishReviewBody(body, { key, passes, maxPasses, reviewed, confidence, gateLine = "", diffFingerprint = "" }) {
const stamped = reviewed && key;
const lines = [confidence ? mergeConfidenceLine(confidence) : "", gateLine].filter(Boolean);
let out = insertAfterCallout(body, lines.join("\n\n"));
if (stamped) out += `\n\n${reviewedCommitLine(key, passes, maxPasses)}`;
if (passLimitReached(passes, maxPasses)) out += `\n\n${passLimitNote(maxPasses)}`;
if (stamped) out = withReviewShaTag(out, key);
if (stamped) out = withReviewDiffTag(withReviewShaTag(out, key), diffFingerprint);
return withPassTag(out, passes);
}
// What the reviewers see as <previous_review>: the review alone, none of the runner's bookkeeping.
export function previousReviewText(body) {
return [REVIEW_SHA_TAG, REVIEW_PASS_TAG, REVIEW_HELD_TAG].reduce(withoutTag, stripRunnerLines(String(body ?? "")));
return [REVIEW_SHA_TAG, REVIEW_PASS_TAG, REVIEW_HELD_TAG, REVIEW_DIFF_TAG].reduce(withoutTag, stripRunnerLines(String(body ?? "")));
}
// Yields the text after each pass, last the first that changed nothing: a removal can splice a new match out of what surrounds it.
@@ -303,7 +332,7 @@ function unwrapJsonBody(body) {
export function sanitizeReviewBody(body, { marker, providerId, modelId, env }) {
if (!body) return { text: "", secretHits: 0 };
const anyMarker = new RegExp(ANY_REVIEW_MARKER, "gi");
const runnerTags = new RegExp(`${ANY_REVIEW_MARKER}|${REVIEW_PASS_TAG}|${REVIEW_SHA_TAG}|${REVIEW_HELD_TAG}`, "gi");
const runnerTags = new RegExp(`${ANY_REVIEW_MARKER}|${REVIEW_PASS_TAG}|${REVIEW_SHA_TAG}|${REVIEW_HELD_TAG}|${REVIEW_DIFF_TAG}`, "gi");
let cleaned = unwrapJsonBody(body)
.replace(/\\n/g, "\n")
scripts/ai-review/src/confidence.mjs
+10 -1
View File
@@ -25,8 +25,13 @@ export function reviewerFinished(r) {
return !r.skipped && !r.error && r.parsed === true;
}
// A reviewer interrupted at its soft deadline answered in haste, so its output counts as findings but not as a finished review.
export function reviewerCompleted(r) {
return reviewerFinished(r) && !r.wrappedUp;
}
export function unfinishedReviewers(ranResults) {
return ranResults.filter(r => !reviewerFinished(r)).map(r => r.agentName);
return ranResults.filter(r => !reviewerCompleted(r)).map(r => r.agentName);
}
function plural(count, noun) {
@@ -78,6 +83,10 @@ export function staleConfidenceLine(shortHead) {
return `**Merge confidence: ${CONFIDENCE_LABEL.low}** — \`${shortHead}\` has not been reviewed`;
}
export function notRunConfidenceLine(shortHead) {
return `**Merge confidence: ${CONFIDENCE_LABEL.low}** — the review did not run because of a CI configuration error; \`${shortHead}\` has not been reviewed`;
}
export function reviewingConfidenceLine(shortHead) {
return `**Merge confidence: 🟡 Reviewing** — \`${shortHead}\` is being reviewed`;
}
scripts/ai-review/src/jsonc.mjs
+18
View File
@@ -0,0 +1,18 @@
// Comments and trailing commas out, so the opencode configs parse as JSON.
export function stripJsonc(text) {
let out = "";
for (let i = 0; i < text.length; i++) {
const c = text[i];
if (c === '"') {
const start = i;
for (i++; i < text.length && text[i] !== '"'; i++) if (text[i] === "\\") i++;
out += text.slice(start, i + 1);
} else if (c === "/" && text[i + 1] === "/") {
while (i < text.length && text[i] !== "\n") i++;
out += "\n";
} else {
out += c;
}
}
return out.replace(/,(\s*[}\]])/g, "$1");
}
scripts/ai-review/src/opencode.mjs
+59 -10
View File
@@ -23,7 +23,11 @@ export const BUILTIN_TOOL_IDS = new Set([
const ALLOWED_TOOL_IDS = new Set(["read", "glob", "grep"]);
const EDIT_TOOL_IDS = new Set(["edit", "write"]);
// What ci-full must actually grant without asking; a config that fell back to the defaults fails here.
const CI_FULL_TOOL_IDS = ["read", "glob", "grep", "edit", "write", "shell", "webfetch"];
const CI_FULL_TOOL_IDS = ["read", "glob", "grep", "edit", "write"];
// question waits on an answer nobody in CI gives; execute runs JavaScript with fetch, so it would bypass the webfetch deny.
const CI_FULL_DENIED_TOOL_IDS = ["webfetch", "websearch", "subagent", "question", "execute", "skill"];
// Offered, but only for the commands the config allowlists.
const CI_FULL_RESTRICTED_TOOL_IDS = ["shell"];
// 2.x dropped OPENCODE_PURE and the plugin, skill and LSP-download flags; sanitizePrTree covers what they did.
const SANDBOX_FLAGS = {
@@ -42,6 +46,7 @@ const INTERRUPTED_ERROR = { type: "interrupted", message: "opencode interrupted
const DEFAULT_REQUEST_HEADERS_TIMEOUT_MS = 30_000;
const STATUS_POLL_MS = 15_000;
const STREAM_RETRY_MS = 1_000;
const INTERRUPT_SETTLE_MS = 20_000;
const TRANSIENT_ERROR_RE = /ECONNRESET|ECONNREFUSED|ETIMEDOUT|EPIPE|socket hang up|overloaded|rate limit|too many requests|temporarily unavailable|bad gateway|service unavailable|gateway timeout/i;
// opencode 2.x StructuredError types; the first set is worth a fresh session, the second never is, whatever the message says.
const TRANSIENT_ERROR_TYPES = new Set(["provider.rate-limit", "provider.transport", "provider.timeout", "provider.internal"]);
@@ -217,7 +222,7 @@ export class OpencodeClient {
}
// Sends the prompt (2.x admits it and returns at once) and waits on the event stream, so no request ever blocks for the length of an agent run (undici's 300 s headersTimeout killed every long review).
async prompt({ agent, model, variant, system, text, title, timeoutMs }) {
async prompt({ agent, model, variant, system, text, title, timeoutMs, wrapUp = null }) {
await this.ensureEventStream();
// 2.x binds agent, model and variant to the session; the prompt itself carries only text.
const session = (await this.json("POST", "/api/session", {
@@ -233,17 +238,20 @@ export class OpencodeClient {
if (system) {
await this.json("PUT", `/api/experimental/session/${session.id}/instructions/entries/${SYSTEM_ENTRY_KEY}`, { value: system });
}
const softMs = wrapUpStartMs(timeoutMs, wrapUp);
// Armed before the POST: the run can fail before the prompt call returns, and nothing would replay it.
const finished = waiter.wait(timeoutMs);
const finished = waiter.wait(softMs);
await this.json("POST", `/api/session/${session.id}/prompt`, { text });
const outcome = await finished;
let outcome = await finished;
let wrappedUp = false;
if (outcome.timedOut && softMs < timeoutMs) ({ outcome, wrappedUp } = await this.wrapUpSession(session.id, waiter, wrapUp.text, timeoutMs - softMs));
if (outcome.timedOut) {
await this.request("POST", `/api/session/${session.id}/interrupt`).catch(() => {});
throw new OpencodeError(`opencode agent ${agent} timed out after ${timeoutMs}ms`, { name: "TimeoutError" });
}
const messages = await this.json("GET", `/api/session/${session.id}/message?type=assistant&order=desc&limit=1`);
totals = await this.json("GET", `/api/session/${session.id}`).catch(() => null);
return finalAssistantResult(messages?.data, totals?.data, outcome.error, waiter.rejectedPermissions);
return { ...finalAssistantResult(messages?.data, totals?.data, outcome.error, waiter.rejectedPermissions), wrappedUp };
} catch (err) {
// A failed or timed-out session was still billed; its totals are gone once the session is deleted.
totals ??= await this.json("GET", `/api/session/${session.id}`).catch(() => null);
@@ -256,6 +264,23 @@ export class OpencodeClient {
}
}
// The interrupt keeps the finished steps in the session, so the follow-up prompt answers from everything the agent already read.
async wrapUpSession(sessionID, waiter, text, budgetMs) {
const deadline = Date.now() + budgetMs;
const stopped = waiter.wait(Math.min(INTERRUPT_SETTLE_MS, budgetMs));
await this.request("POST", `/api/session/${sessionID}/interrupt`).catch(() => {});
const stop = await stopped;
// An unconfirmed interrupt ends as the hard timeout; a run that ended on its own keeps its own outcome.
if (stop.timedOut || stop.error?.type !== "interrupted") return { outcome: stop, wrappedUp: false };
// Nothing interrupts the follow-up before its wait ends, so a late interrupted event or idle record is the first run's.
waiter.ignoreInterrupted = true;
const remainingMs = deadline - Date.now();
if (remainingMs <= 0) return { outcome: { timedOut: true, error: null }, wrappedUp: true };
const finished = waiter.wait(remainingMs);
await this.json("POST", `/api/session/${sessionID}/prompt`, { text });
return { outcome: await finished, wrappedUp: true };
}
close() {
this.closed = true;
this.stream?.close();
@@ -270,6 +295,7 @@ class SessionWaiter {
this.sessionID = sessionID;
this.rejectedPermissions = 0;
this.settle = null;
this.ignoreInterrupted = false;
}
wait(timeoutMs) {
@@ -291,6 +317,7 @@ class SessionWaiter {
}
onError(error) {
if (this.ignoreInterrupted && error?.type === "interrupted") return;
this.settle?.({ timedOut: false, error: error || { type: "unknown", message: "opencode run failed" } });
}
@@ -353,11 +380,13 @@ function permissionAction(toolId) {
return EDIT_TOOL_IDS.has(toolId) ? "edit" : toolId;
}
// Same as opencode 2.2.0: a trailing " *" also matches the bare command, so "git diff *" covers "git diff".
export function wildcardMatch(input, pattern) {
const escaped = pattern
let escaped = pattern
.replace(/[.+^${}()|[\]\\]/g, "\\$&")
.replace(/\*/g, ".*")
.replace(/\?/g, ".");
if (escaped.endsWith(" .*")) escaped = `${escaped.slice(0, -3)}( .*)?`;
return new RegExp(`^${escaped}$`, "s").test(input);
}
@@ -375,7 +404,7 @@ export async function selfCheck(client, expectedAgents, { readOnly = true } = {}
const broken = plugins.filter(p => p.state?.status !== "active");
if (broken.length > 0) throw new OpencodeError(`opencode plugins failed to load: ${broken.map(p => `${p.id}: ${p.state?.error || "unknown"}`).join("; ")}`);
const active = new Set(plugins.map(p => p.id));
const missing = (readOnly ? [...ALLOWED_TOOL_IDS] : CI_FULL_TOOL_IDS).filter(id => !active.has(`opencode.tool.${id}`));
const missing = (readOnly ? [...ALLOWED_TOOL_IDS] : [...CI_FULL_TOOL_IDS, ...CI_FULL_RESTRICTED_TOOL_IDS]).filter(id => !active.has(`opencode.tool.${id}`));
if (missing.length > 0) throw new OpencodeError(`opencode tools did not load: ${missing.join(", ")}`);
const mcp = (await client.json("GET", "/api/mcp"))?.data || [];
if (mcp.length > 0) throw new OpencodeError(`opencode loaded MCP servers; refusing to review`);
@@ -397,9 +426,7 @@ export async function selfCheck(client, expectedAgents, { readOnly = true } = {}
const problem = externalDirectoryProblem(ruleset);
if (problem) problems.push(`${name}: ${problem}`);
} else {
for (const toolId of CI_FULL_TOOL_IDS) {
if (!toolAllowedWithoutAsking(toolId, ruleset)) problems.push(`${name}: ${toolId} not fully allowed`);
}
problems.push(...ciFullProblems(ruleset).map(problem => `${name}: ${problem}`));
}
}
if (problems.length > 0) throw new OpencodeError(`opencode permission self-check failed: ${problems.join("; ")}`);
@@ -415,6 +442,28 @@ export function externalDirectoryProblem(ruleset) {
return extra.length > 0 ? `external_directory allows ${extra.map(r => r.resource).join(", ")}` : "";
}
// ci-full grants the file tools outright, denies the tools that only cost time, and keeps the shell on an allowlist.
export function ciFullProblems(ruleset) {
const problems = CI_FULL_TOOL_IDS.filter(id => !toolAllowedWithoutAsking(id, ruleset)).map(id => `${id} not fully allowed`);
problems.push(...CI_FULL_DENIED_TOOL_IDS.filter(id => toolEnabled(id, ruleset)).map(id => `${id} allowed`));
for (const id of CI_FULL_RESTRICTED_TOOL_IDS) {
if (!toolEnabled(id, ruleset)) problems.push(`${id} denied`);
else if (toolAllowedWithoutAsking(id, ruleset)) problems.push(`${id} allows every command`);
}
return problems;
}
// The follow-up only gets a slot when the first run keeps at least as long as the follow-up; shorter sessions run to the hard timeout.
export function wrapUpStartMs(timeoutMs, wrapUp) {
return wrapUp?.ms > 0 && wrapUp.text && timeoutMs >= 2 * wrapUp.ms ? timeoutMs - wrapUp.ms : timeoutMs;
}
// The variant ids a model declares in an opencode config; a variant opencode does not know must fail the run before any session starts.
export function modelVariantIds(config, providerId, modelId) {
const variants = config?.providers?.[providerId]?.models?.[modelId]?.variants;
return Array.isArray(variants) ? variants.map(v => v?.id).filter(Boolean) : [];
}
// A retry must leave room for its own full timeout plus whatever still has to run after it (the coordinator).
export function shouldRetry({ err, attempt, maxRetries, remainingMs, timeoutMs, reserveMs = 0 }) {
return attempt <= maxRetries && isTransientError(err) && remainingMs > timeoutMs + reserveMs;
scripts/ai-review/src/passes.mjs
+11 -2
View File
@@ -1,4 +1,4 @@
import { reviewerFinished, shortName } from "./confidence.mjs";
import { reviewerCompleted, reviewerFinished, shortName } from "./confidence.mjs";
export const DEFAULT_COLD_PASSES = 2;
export const MAX_COLD_PASSES = 3;
@@ -43,8 +43,9 @@ export async function runPooled(items, limit, worker) {
function mergeArea(group) {
const finished = group.filter(reviewerFinished);
const completed = finished.filter(reviewerCompleted);
const ran = group.filter(r => !r.skipped);
const base = finished[0] ?? ran[0] ?? group[0];
const base = completed[0] ?? finished[0] ?? ran[0] ?? group[0];
const covered = finished.length > 0;
return {
agentKey: base.agentKey,
@@ -58,6 +59,8 @@ function mergeArea(group) {
skipped: group.every(r => r.skipped),
fullDiff: base.fullDiff,
parsed: covered,
// Only an uninterrupted pass completes an area; wrapped-up output still counts as findings.
wrappedUp: covered && completed.length === 0,
passes: group.length,
passesFinished: finished.length,
passResults: group,
@@ -78,6 +81,12 @@ export function lostPasses(results) {
return results.filter(r => !r.skipped && r.passesFinished > 0 && r.passesFinished < r.passes);
}
// Only a review with every pass finished in time and a real coordinator merge may let the next push with the same diff skip.
export function reviewComplete(passResults, coordinatorResult) {
if (!coordinatorResult || coordinatorResult.fallback) return false;
return passResults.every(r => r.skipped || reviewerCompleted(r));
}
// Written by code like the failed-reviewers line, so a reader learns an area got less scrutiny than the rest.
export function lostPassesNote(results) {
const lost = lostPasses(results);
scripts/ai-review/src/pr-diff.mjs
+43
View File
@@ -1,4 +1,5 @@
import { execFileSync } from "node:child_process";
import { createHash } from "node:crypto";
const MAX_BUFFER = 200 * 1024 * 1024;
const TOO_LARGE_RE = /exceeded the maximum number of lines|too_large|HTTP 406|HTTP 422/i;
@@ -55,6 +56,39 @@ export function fetchPinnedDiff(params) {
return pinDiff({ ...params, diff: fetchDiff(params) });
}
// Blob ids and hunk line numbers move with the base, so a rebase keeps the fingerprint; the function context after `@@` stays, as the same lines in another function are another change.
export function diffFingerprint(diff) {
const text = String(diff ?? "");
// A binary file has no hunks, so its blob ids are the only record of its content.
const keepBlobIds = /^(Binary files .* differ|GIT binary patch)$/m.test(text);
const normalized = text
.split("\n")
.map(line => (keepBlobIds ? line : line.replace(/^index [0-9a-f]+\.\.[0-9a-f]+/, "index")))
.map(line => line.replace(/^@@ -\d+(?:,\d+)? \+\d+(?:,\d+)? @@/, "@@"))
.join("\n")
.replace(/\n+$/, "");
return createHash("sha256").update(normalized).digest("hex");
}
// Equal text names one change only where each hunk's old lines sit once in the base, so an edit to one of two identical blocks cannot pass for the other.
export function hunksLocatable(diff, readBase) {
for (const section of String(diff ?? "").split(/^diff --git /m).slice(1)) {
const firstHunk = section.search(/^@@ /m);
if (firstHunk === -1) continue;
const header = section.slice(0, firstHunk);
if (/^--- \/dev\/null$/m.test(header)) continue;
const path = header.match(/^--- a\/(.*?)\t?$/m)?.[1];
const base = path ? readBase(path) : null;
if (typeof base !== "string") return false;
const baseLines = base.split("\n");
for (const hunk of section.slice(firstHunk).split(/^@@ [^\n]*(?:\n|$)/m).slice(1)) {
const old = hunk.split("\n").filter(line => line.startsWith(" ") || line.startsWith("-")).map(line => line.slice(1));
if (old.length === 0 ? base !== "" : occurrences(baseLines, old) !== 1) return false;
}
}
return true;
}
// `unfocused` keeps the whole diff for a gate candidate, whose approval must rest on reviewers that saw every file; requiresFocus agents still opt out.
export function reviewerDiff(diff, { fileFocus, requiresFocus }, { unfocused = false, pathOf = path => path } = {}) {
if (!fileFocus) return { diff, skipped: false, fullDiff: true };
@@ -107,3 +141,12 @@ function prefixGitHeader(line, p) {
function prefixBinaryLine(line, p) {
return line.replace(/(^Binary files |and )("?)([ab])\//g, `$1$2$3/${p}/`);
}
// Stops at two: the caller only needs to know whether a block is unique.
function occurrences(lines, block) {
let found = 0;
for (let i = 0; i + block.length <= lines.length && found < 2; i += 1) {
if (lines[i] === block[0] && block.every((line, j) => lines[i + j] === line)) found += 1;
}
return found;
}
scripts/ai-review/src/sanitize.mjs
+6
View File
@@ -38,6 +38,12 @@ export function sanitizePrTree(prDir, contextSourceDir) {
return { removed, contextFiles: copied };
}
// The same names the walk strips, so a PR touching only these still steers a reviewer and is never docs-only.
export function isLoaderPath(filePath) {
const parts = String(filePath ?? "").toLowerCase().split("/");
return LOADER_FILES.has(parts.at(-1)) || parts.some(part => LOADER_DIRS.has(part));
}
function walk(dir, root, removed) {
for (const entry of readdirSync(dir, { withFileTypes: true })) {
const path = join(dir, entry.name);
scripts/ai-review/src/selection.mjs
+61
View File
@@ -1,5 +1,12 @@
import { existsSync } from "node:fs";
import { basename, dirname, join } from "node:path";
import { isLoaderPath } from "./sanitize.mjs";
// Dependency bots whose PRs only bump versions; any other bot or App can write code and is reviewed.
const DEPENDENCY_BOTS = new Set(["dependabot[bot]", "renovate[bot]"]);
// GitHub's files and commits endpoints stop at these counts however far they are paged.
const MAX_LISTED_FILES = 3000;
const MAX_LISTED_COMMITS = 250;
// Diffs that can affect a release: migrations, dependency manifests, server config, CI and deployment files.
export function isReleaseFile(filePath) {
@@ -22,3 +29,57 @@ export function resolveRulesFiles(files, exists = existsSync) {
}
return out;
}
// Test code only: test trees, spec/test files and fixtures, in any language.
export function isTestOnlyPath(filePath) {
if (isCiOnlyPath(filePath)) return false;
if (/(^|\/)(tests?|__tests__|e2e|fixtures?|testdata)\//.test(filePath)) return true;
if (/\.(spec|test)\.[cm]?[jt]sx?$/.test(filePath)) return true;
return /(^|\/)[^/]+_test\.(rs|go|py)$/.test(filePath) || /(^|\/)test_[^/]+\.py$/.test(filePath);
}
// Workflows, actions and .github config only; scripts, the review engine and its opencode configs are code and take the normal tiers.
export function isCiOnlyPath(filePath) {
return /^\.github\/((workflows|actions)\/|[^/]+\.ya?ml$)/.test(filePath);
}
export function narrowTier(paths) {
if (paths.length === 0) return "";
if (paths.every(isTestOnlyPath)) return "test-only";
if (paths.every(isCiOnlyPath)) return "ci-only";
return "";
}
// Prompt, agent-rule and workflow markdown steers the AI or CI, so it is reviewed like code.
export function isDocsPath(filePath) {
if (/^(scripts\/ai-review\/|\.github\/)/.test(filePath) || isLoaderPath(filePath)) return false;
return /\.md$/i.test(filePath) || /^docs\//.test(filePath);
}
// What update-translations.yml commits; en-US.json is the English source a person writes, so it is still reviewed.
export function isTranslationPath(filePath) {
if (filePath === "scripts/translations/.translation_state.json") return true;
return /^web\/src\/locales\/languages\/[^/]+\.json$/.test(filePath) && !filePath.endsWith("/en-US.json");
}
// A listing is whole only when it matches the PR's own counts below each endpoint's cap; anything else may hide a file or commit.
export function listingComplete({ files, commits, expectedFiles, expectedCommits }) {
const count = value => (/^\d+$/.test(String(value ?? "").trim()) ? Number(value) : Number.NaN);
const wantFiles = count(expectedFiles);
const wantCommits = count(expectedCommits);
return files === wantFiles && commits === wantCommits && wantFiles < MAX_LISTED_FILES && wantCommits < MAX_LISTED_COMMITS;
}
// GitHub commits a bot's API-made commits as web-flow, so the committer is either the bot or web-flow, and GitHub must have verified the signature.
export function botCommit(commit, bot) {
return commit?.author === bot && (commit.committer === bot || commit.committer === "web-flow") && commit.verified === true;
}
// Why a PR needs no review at all, or "" when it does; a PR with no listed files is reviewed so a failed listing never skips one.
export function nothingToReviewReason({ author = "", commits = [], paths = [] }) {
if (DEPENDENCY_BOTS.has(author) && commits.length > 0 && commits.every(c => botCommit(c, author))) return "bot_author";
if (paths.length === 0) return "";
if (paths.every(isDocsPath)) return "docs_only";
if (paths.every(isTranslationPath)) return "translation_only";
return "";
}
scripts/ai-review/test/comments.test.mjs
+77 -1
View File
@@ -1,12 +1,14 @@
import test from "node:test";
import assert from "node:assert/strict";
import { spawnSync } from "node:child_process";
import { readFileSync } from "node:fs";
import {
ANY_REVIEW_MARKER,
REVIEW_HEADING,
bodyAtPassLimit,
bodyAtReviewedHead,
bodyAtUnchangedDiff,
bodyAtUnreviewedHead,
claimedReviewBody,
finishReviewBody,
@@ -20,16 +22,18 @@ import {
reviewShaTag,
reviewSkipReason,
reviewedCommitLine,
reviewedDiffFingerprint,
reviewedKey,
sanitizeReviewBody,
selectOwnReviewComment,
shouldSkipAsReviewed,
shouldSkipAsUnchangedDiff,
withPassLimitNote,
withPassTag,
withReviewShaTag,
withoutAutoApproveLine,
} from "../src/comments.mjs";
import { autoApproveLine } from "../src/auto-approve.mjs";
import { autoApproveLine, withoutCarriedApproval } from "../src/auto-approve.mjs";
import { mergeConfidence } from "../src/confidence.mjs";
const MARKER = ANY_REVIEW_MARKER;
@@ -637,3 +641,75 @@ test("a reviewed head drops a limit note once the cap no longer applies", () =>
}
assert.equal(bodyAtReviewedHead(body, 2), body);
});
const FP_A = "a".repeat(64);
const FP_B = "b".repeat(64);
const HEAD_1 = "1".repeat(40);
const HEAD_2 = "2".repeat(40);
const stampedReview = (fingerprint = FP_A, extra = {}) => finishReviewBody("## 🔎 OpenObserve Code Review\n\n> [!TIP]\n> ### ✅ Approved", {
key: HEAD_1, passes: 1, maxPasses: 0, reviewed: true, confidence: { level: "high", reasons: ["no blockers"] }, diffFingerprint: fingerprint, ...extra,
});
test("a stamped review carries the fingerprint of the diff it read; an unstamped one vouches for none", () => {
assert.equal(reviewedDiffFingerprint(stampedReview()), FP_A);
assert.equal(reviewedDiffFingerprint(stampedReview(FP_A, { reviewed: false })), "");
assert.equal(reviewedDiffFingerprint(stampedReview("")), "");
assert.equal(reviewedDiffFingerprint(`plain text <!-- o2-review-diff:${FP_A} -->`), "", "without a sha tag no head was reviewed");
});
test("an unchanged diff is skipped unless the run is forced or the diff is not pinned to the head", () => {
const body = stampedReview();
assert.equal(shouldSkipAsUnchangedDiff(body, FP_A, { pinned: true }), true);
assert.equal(shouldSkipAsUnchangedDiff(body, FP_B, { pinned: true }), false);
assert.equal(shouldSkipAsUnchangedDiff(body, FP_A, { pinned: false }), false);
assert.equal(shouldSkipAsUnchangedDiff(body, FP_A, { pinned: true, force: true }), false);
assert.equal(shouldSkipAsUnchangedDiff(body, "", { pinned: true }), false);
});
test("an unchanged-diff skip in approve mode keeps the review but not the auto-approval it can no longer back", () => {
const approved = autoApproveLine({ mode: "approve", approved: true, reasons: [] });
const body = stampedReview(FP_A, { gateLine: approved });
assert.ok(body.includes(approved));
const next = withoutCarriedApproval(bodyAtUnchangedDiff(body, HEAD_2, 0));
assert.equal(next.includes("Auto-approved"), false);
assert.match(next, /Needs human review/);
assert.equal(reviewedKey(next), HEAD_2);
assert.equal(reviewedDiffFingerprint(next), FP_A);
assert.equal(withoutCarriedApproval(stampedReview()), stampedReview(), "a body with no approval line is unchanged");
const source = readFileSync(new URL("../run-review.mjs", import.meta.url), "utf-8");
assert.match(source, /AUTO_APPROVE_MODE === "approve" \? withoutCarriedApproval\(kept\) : kept/);
});
test("a partial review leaves no fingerprint, so the next push with the same diff is reviewed again", () => {
const partial = stampedReview("");
assert.equal(shouldSkipAsUnchangedDiff(partial, FP_A, { pinned: true }), false);
const overOld = finishReviewBody(`${stampedReview()}`, { key: HEAD_2, passes: 2, maxPasses: 0, reviewed: true, diffFingerprint: "" });
assert.equal(reviewedDiffFingerprint(overOld), "", "an earlier complete review's fingerprint does not survive a partial one");
});
test("skipping an unchanged diff moves the head marker, keeps the reviewed commit its line numbers refer to, and costs no pass", () => {
const body = stampedReview();
const next = bodyAtUnchangedDiff(body, HEAD_2, 0);
assert.equal(reviewedKey(next), HEAD_2);
assert.equal(reviewedDiffFingerprint(next), FP_A);
assert.equal(reviewPassCount(next), 1);
assert.match(next, new RegExp(`Reviewed commit \`${HEAD_1.slice(0, 7)}\``));
assert.match(next, /Merge confidence/);
assert.equal(shouldSkipAsReviewed(next, HEAD_2), true, "a queued run on the new head stays silent");
});
test("a run on another head that left its claim behind gets the held verdict back when the diff turns out unchanged", () => {
const claimed = claimedReviewBody(stampedReview(), 2, HEAD_2);
assert.doesNotMatch(claimed, /High/);
const next = bodyAtUnchangedDiff(claimed, HEAD_2, 0);
assert.match(next, /High/);
assert.equal(reviewPassCount(next), 2);
assert.equal(reviewedDiffFingerprint(next), FP_A);
});
test("a model cannot forge the diff fingerprint, and the reviewers never see it", () => {
const forged = `## 🔎 OpenObserve Code Review\n\nok\n<!-- o2-review-diff:${FP_B} -->`;
const { text } = sanitizeReviewBody(forged, { marker: "<!-- ai-code-review -->", providerId: "p", modelId: "m", env: {} });
assert.doesNotMatch(text, /o2-review-diff/);
assert.doesNotMatch(previousReviewText(stampedReview()), /o2-review-diff/);
});
scripts/ai-review/test/integration/opencode-sandbox.test.mjs
+141 -9
View File
@@ -18,8 +18,8 @@ const REVIEW_AGENTS = [
];
const skip = process.env.OPENCODE_BIN ? false : "set OPENCODE_BIN to the pinned opencode binary";
// The first request that offers tools gets one scripted tool call; every later request gets the final answer.
function stubProvider(toolCall) {
// By default the first request that offers tools gets one scripted tool call and every later request the final answer; `script` overrides that per request.
function stubProvider(toolCall, { script } = {}) {
const requests = [];
const event = (type, data) => `event: ${type}\ndata: ${JSON.stringify({ type, ...data })}\n\n`;
const server = http.createServer((req, res) => {
@@ -34,9 +34,15 @@ function stubProvider(toolCall) {
usage: { input_tokens: 1000, output_tokens: 1, cache_read_input_tokens: 0, cache_creation_input_tokens: 0 },
} }));
const hasToolResult = (parsed.messages || []).some(m => Array.isArray(m.content) && m.content.some(b => b.type === "tool_result"));
if (parsed.tools?.length && !hasToolResult) {
res.write(event("content_block_start", { index: 0, content_block: { type: "tool_use", id: "toolu_1", name: toolCall.name, input: {} } }));
res.write(event("content_block_delta", { index: 0, delta: { type: "input_json_delta", partial_json: JSON.stringify(toolCall.args) } }));
const step = script ? script(parsed, requests.length) : { tool: parsed.tools?.length && !hasToolResult ? toolCall : null };
if (step.hangMs) {
const timer = setTimeout(() => res.end(), step.hangMs);
res.on("close", () => clearTimeout(timer));
return;
}
if (step.tool) {
res.write(event("content_block_start", { index: 0, content_block: { type: "tool_use", id: `toolu_${requests.length}`, name: step.tool.name, input: {} } }));
res.write(event("content_block_delta", { index: 0, delta: { type: "input_json_delta", partial_json: JSON.stringify(step.tool.args) } }));
res.write(event("content_block_stop", { index: 0 }));
res.write(event("message_delta", { delta: { stop_reason: "tool_use" }, usage: { output_tokens: 100 } }));
} else {
@@ -102,9 +108,10 @@ async function withServer(prDir, configPath, run, { readOnly = true } = {}) {
}
}
function stubConfig(file, stub, configDir) {
function stubConfig(file, stub, configDir, edit = () => {}) {
const config = JSON.parse(stripJsonc(readFileSync(join(ENGINE_DIR, file), "utf-8")));
config.providers["anthropic-review"].settings = { baseURL: stub.url };
edit(config);
const configPath = join(configDir, "opencode.json");
writeFileSync(configPath, JSON.stringify(config));
return configPath;
@@ -189,12 +196,17 @@ test("without sanitize, the env flags still keep project config, tools and agent
assert.deepEqual(toolNames, ["glob", "grep", "read", "skill"]);
});
test("ci-full: every tool is offered and runs without a prompt, and the shell sees no GitHub token", { skip, timeout: 120_000 }, async t => {
const allowShell = (...commands) => config => {
config.agents["ai-review-correctness"].permissions.push(...commands.map(resource => ({ action: "shell", resource, effect: "allow" })));
};
test("ci-full: file tools and the shell are offered without a prompt, and the shell sees no GitHub token", { skip, timeout: 120_000 }, async t => {
const pr = plantedPr();
const stub = await stubProvider({ name: "shell", args: { command: "echo ran-$((6*7)); env", description: "probe" } });
const configDir = mkdtempSync(join(tmpdir(), "ai-review-config-"));
cleanup(t, stub, pr, configDir);
const configPath = stubConfig(OPENCODE_CONFIGS["ci-full"].file, stub, configDir);
// The allowlist refuses echo and env, so this probe of the shell's environment adds them for the test alone.
const configPath = stubConfig(OPENCODE_CONFIGS["ci-full"].file, stub, configDir, allowShell("echo *", "env *"));
sanitizePrTree(pr.root, ENGINE_DIR);
await withServer(pr.root, configPath, async client => {
@@ -206,8 +218,128 @@ test("ci-full: every tool is offered and runs without a prompt, and the shell se
}, { readOnly: false });
const toolNames = (stub.requests[0].body.tools || []).map(tool => tool.name);
for (const name of ["read", "glob", "grep", "edit", "write", "shell", "webfetch"]) assert.ok(toolNames.includes(name), name);
for (const name of ["read", "glob", "grep", "edit", "write", "shell"]) assert.ok(toolNames.includes(name), name);
for (const name of ["webfetch", "websearch", "subagent", "question", "execute"]) assert.equal(toolNames.includes(name), false, `${name} is offered`);
const content = toolResultOf(stub.requests.at(-1));
assert.match(content, /ran-42/);
assert.doesNotMatch(content, /ghs_must_not_leak/);
});
test("the real self-check passes the shipped ci-full config and refuses it with execute or the whole shell re-allowed", { skip, timeout: 120_000 }, async t => {
for (const [extra, problem] of [[null, null], [{ action: "execute", resource: "*", effect: "allow" }, /ai-review-qa: execute allowed/], [{ action: "shell", resource: "*", effect: "allow" }, /ai-review-qa: shell allows every command/]]) {
const pr = plantedPr();
const stub = await stubProvider(null);
const configDir = mkdtempSync(join(tmpdir(), "ai-review-config-"));
cleanup(t, stub, pr, configDir);
const configPath = stubConfig(OPENCODE_CONFIGS["ci-full"].file, stub, configDir, config => { if (extra) config.agents["ai-review-qa"].permissions.push(extra); });
sanitizePrTree(pr.root, ENGINE_DIR);
await withServer(pr.root, configPath, async client => {
const check = selfCheck(client, REVIEW_AGENTS, { readOnly: false });
if (problem) await assert.rejects(check, problem);
else assert.equal((await check).agents, REVIEW_AGENTS.length);
}, { readOnly: false });
}
});
test("ci-full offers skill only because opencode appends a narrower rule, and refuses every skill", { skip, timeout: 120_000 }, async t => {
const pr = plantedPr();
const stub = await stubProvider({ name: "skill", args: { id: "browser" } });
const configDir = mkdtempSync(join(tmpdir(), "ai-review-config-"));
cleanup(t, stub, pr, configDir);
const configPath = stubConfig(OPENCODE_CONFIGS["ci-full"].file, stub, configDir);
sanitizePrTree(pr.root, ENGINE_DIR);
await withServer(pr.root, configPath, async client => {
const result = await client.prompt({ ...PROMPT, title: "skill" });
assert.equal(result.rejectedPermissions, 0);
}, { readOnly: false });
assert.match(toolResultOf(stub.requests.at(-1)), /Permission denied: skill/);
});
test("ci-full shell runs an allowlisted command and refuses a build outright, without asking", { skip, timeout: 120_000 }, async t => {
// A RegExp is the output an allowed command must print; null means opencode must refuse it.
const ONE = /\b1\b/;
const cases = [
["cat src/main.rs | wc -l", ONE], ["grep -c main src/main.rs", ONE], ["find src -name '*.rs' | wc -l", ONE], ["tail -n 1 src/main.rs | wc -l", ONE],
["ls src && wc -l src/main.rs", ONE], ["find src -name '*-executor*' | wc -l", /\b0\b/], ["cargo build", null], ["ls && touch built.txt", null],
["find . -name main.rs -exec touch built.txt ;", null], ["find src -fprint built.txt", null], ["git grep -nOsh main", null],
["cat $(touch built.txt)", null], ["cat `touch built.txt`", null], ["cat src/main.rs > built.txt", null], ["cat /proc/self/environ", null],
];
for (const [command, output] of cases) {
const pr = plantedPr();
const stub = await stubProvider({ name: "shell", args: { command, description: "probe" } });
const configDir = mkdtempSync(join(tmpdir(), "ai-review-config-"));
cleanup(t, stub, pr, configDir);
const configPath = stubConfig(OPENCODE_CONFIGS["ci-full"].file, stub, configDir);
sanitizePrTree(pr.root, ENGINE_DIR);
await withServer(pr.root, configPath, async client => {
const result = await client.prompt({ ...PROMPT, title: "shell" });
assert.equal(result.rejectedPermissions, 0, "a denied command must not reach a prompt that ci-full would approve");
}, { readOnly: false });
const content = toolResultOf(stub.requests.at(-1));
assert.match(content, output ?? /Permission denied: shell/, command);
assert.equal(existsSync(join(pr.root, "built.txt")), false, command);
}
});
// Why neither config sets `steps`: if opencode ever stops ending the capped turn on an assistant message, this fails and the cap can come back.
test("opencode 2.2.0's step cap ends the capped turn on an assistant prefill, which the model rejects with thinking on", { skip, timeout: 120_000 }, async t => {
const pr = plantedPr();
// Asks for another tool call on every turn that allows one, so only the cap can end the loop.
const stub = await stubProvider(null, { script: parsed => ({ tool: parsed.tool_choice?.type === "none" ? null : { name: "glob", args: { pattern: "*.rs" } } }) });
const configDir = mkdtempSync(join(tmpdir(), "ai-review-config-"));
cleanup(t, stub, pr, configDir);
const configPath = stubConfig(OPENCODE_CONFIGS["ci-full"].file, stub, configDir, config => { config.agents["ai-review-correctness"].steps = 3; });
sanitizePrTree(pr.root, ENGINE_DIR);
await withServer(pr.root, configPath, client => client.prompt({ ...PROMPT, title: "steps" }), { readOnly: false });
assert.equal(stub.requests.length, 3);
const capped = stub.requests.at(-1).body;
assert.deepEqual(capped.tool_choice, { type: "none" });
assert.equal(capped.messages.at(-1).role, "assistant");
assert.match(JSON.stringify(capped.messages.at(-1).content), /MAXIMUM STEPS REACHED/);
for (const request of stub.requests.slice(0, -1)) assert.equal(request.body.messages.at(-1).role, "user");
});
test("each effort variant reaches the provider as its own effort level", { skip, timeout: 120_000 }, async t => {
for (const variant of ["low", "medium", "xhigh", "max"]) {
const pr = plantedPr();
const stub = await stubProvider({ name: "glob", args: { pattern: "*.rs" } });
const configDir = mkdtempSync(join(tmpdir(), "ai-review-config-"));
cleanup(t, stub, pr, configDir);
const configPath = stubConfig(OPENCODE_CONFIGS["ci-full"].file, stub, configDir);
sanitizePrTree(pr.root, ENGINE_DIR);
await withServer(pr.root, configPath, client => client.prompt({ ...PROMPT, variant, title: variant }), { readOnly: false });
for (const request of stub.requests) {
assert.deepEqual(request.body.output_config, { effort: variant });
assert.equal(request.body.thinking?.type, "adaptive");
for (const field of ["temperature", "top_p", "top_k", "tool_choice"]) assert.equal(request.body[field], undefined, field);
}
}
});
test("a session still running at its soft deadline is interrupted and answers the wrap-up prompt from the same context", { skip, timeout: 120_000 }, async t => {
const pr = plantedPr();
const wrapText = "WRAP-UP-NOW-5d1e";
// A tool step, then a model turn that never returns; only the wrap-up prompt gets an answer.
const stub = await stubProvider(null, {
script: (parsed, n) => {
if (parsed.messages?.some(m => JSON.stringify(m.content).includes(wrapText))) return { tool: null };
return n === 1 ? { tool: { name: "glob", args: { pattern: "*.rs" } } } : { hangMs: 60_000 };
},
});
const configDir = mkdtempSync(join(tmpdir(), "ai-review-config-"));
cleanup(t, stub, pr, configDir);
const configPath = stubConfig(OPENCODE_CONFIGS["ci-full"].file, stub, configDir);
sanitizePrTree(pr.root, ENGINE_DIR);
await withServer(pr.root, configPath, async client => {
const result = await client.prompt({ ...PROMPT, title: "wrap-up", timeoutMs: 20_000, wrapUp: { ms: 10_000, text: wrapText } });
assert.equal(result.wrappedUp, true);
assert.equal(result.text, "<no-issues/>");
}, { readOnly: false });
const last = stub.requests.at(-1);
assert.match(last.raw, new RegExp(wrapText));
assert.ok(last.body.messages.some(m => Array.isArray(m.content) && m.content.some(b => b.type === "tool_result")), "the wrap-up lost the steps the agent had already run");
for (const request of stub.requests) {
for (const field of ["temperature", "top_p", "top_k", "tool_choice"]) assert.equal(request.body[field], undefined, field);
assert.equal(request.body.messages.at(-1).role, "user", "the wrap-up must not end on an assistant prefill");
}
});
scripts/ai-review/test/jsonc.mjs
+1 -18
View File
@@ -1,18 +1 @@
// Comments and trailing commas out, so the opencode configs parse as JSON in tests.
export function stripJsonc(text) {
let out = "";
for (let i = 0; i < text.length; i++) {
const c = text[i];
if (c === '"') {
const start = i;
for (i++; i < text.length && text[i] !== '"'; i++) if (text[i] === "\\") i++;
out += text.slice(start, i + 1);
} else if (c === "/" && text[i + 1] === "/") {
while (i < text.length && text[i] !== "\n") i++;
out += "\n";
} else {
out += c;
}
}
return out.replace(/,(\s*[}\]])/g, "$1");
}
export { stripJsonc } from "../src/jsonc.mjs";
scripts/ai-review/test/opencode.test.mjs
+132 -11
View File
@@ -11,15 +11,19 @@ import {
OPENCODE_CONFIGS,
OpencodeClient,
buildServerEnv,
ciFullProblems,
createSseParser,
externalDirectoryProblem,
httpRequest,
isTransientError,
modelVariantIds,
resolveOpencodeConfig,
selfCheck,
shouldRetry,
toolAllowedWithoutAsking,
toolEnabled,
wildcardMatch,
wrapUpStartMs,
} from "../src/opencode.mjs";
import { stripJsonc } from "./jsonc.mjs";
@@ -28,15 +32,16 @@ const REPO_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), "../../..");
const AGENT_RUN_MS = 1500;
const HEADERS_TIMEOUT_MS = 1000;
function fakeOpencode({ idleAfterMs = AGENT_RUN_MS, finalEvent = "session.execution.succeeded", error = null, errorBeforeAck = null, eventStream = true } = {}) {
function fakeOpencode({ idleAfterMs = AGENT_RUN_MS, finalEvent = "session.execution.succeeded", error = null, errorBeforeAck = null, eventStream = true, wrapUpAfterMs = null, interruptEventDelayMs = 50 } = {}) {
const calls = [];
let prompts = 0;
const streams = new Set();
let idle = null;
const emit = (type, data) => {
for (const res of streams) res.write(`data: ${JSON.stringify({ id: "evt", type, data })}\n\n`);
};
const finishRun = () => {
idle = { type: "idle", outcome: finalEvent === "session.execution.succeeded" ? "succeeded" : "failed", ...(error ? { error } : {}) };
idle = { id: `idle_${prompts}`, type: "idle", outcome: finalEvent === "session.execution.succeeded" ? "succeeded" : "failed", ...(error ? { error } : {}) };
if (eventStream) emit(finalEvent, { sessionID: "ses_1", ...(error ? { error } : {}) });
};
const server = http.createServer((req, res) => {
@@ -73,7 +78,13 @@ function fakeOpencode({ idleAfterMs = AGENT_RUN_MS, finalEvent = "session.execut
setTimeout(() => send(200, { data: { id: "msg_u1" } }), 200);
return;
}
send(200, { data: { id: "msg_u1" } });
prompts += 1;
send(200, { data: { id: `msg_u${prompts}` } });
// The second prompt is the wrap-up: it answers, where the first ran on until interrupted.
if (wrapUpAfterMs !== null && prompts > 1) {
setTimeout(finishRun, wrapUpAfterMs);
return;
}
setTimeout(() => emit("permission.asked", { id: "per_1", sessionID: "ses_1", action: "shell", resources: ["*"] }), 50);
if (idleAfterMs !== null) setTimeout(finishRun, idleAfterMs);
return;
@@ -103,7 +114,11 @@ function fakeOpencode({ idleAfterMs = AGENT_RUN_MS, finalEvent = "session.execut
res.end();
return;
}
if (req.method === "POST" && req.url === "/api/session/ses_1/interrupt") return send(200, { interrupted: true });
if (req.method === "POST" && req.url === "/api/session/ses_1/interrupt") {
idle = { id: `idle_${prompts}`, type: "idle", outcome: "interrupted" };
setTimeout(() => emit("session.execution.interrupted", { sessionID: "ses_1" }), interruptEventDelayMs);
return send(200, { interrupted: true });
}
if (req.method === "DELETE" && req.url === "/api/session/ses_1") {
res.writeHead(204);
res.end();
@@ -216,6 +231,61 @@ test("a timed-out agent is interrupted server-side and not reported as transient
}
});
test("a reviewer still running near its timeout is interrupted and asked for its findings in the same session", async () => {
const fake = await fakeOpencode({ idleAfterMs: null, wrapUpAfterMs: 100 });
try {
const client = new OpencodeClient({ baseUrl: fake.baseUrl, password: "pw", statusPollMs: 100 });
const started = Date.now();
const result = await client.prompt({ ...PROMPT, timeoutMs: 3000, wrapUp: { ms: 1000, text: "wrap up now" } });
client.close();
assert.ok(Date.now() - started < 3000, "waited for the hard timeout instead of the wrap-up answer");
assert.equal(result.wrappedUp, true);
assert.equal(result.text, "<no-issues/>");
assert.equal(result.info.tokens.input, 2000, "tokens are the whole session's, both prompts included");
const prompts = fake.calls.filter(c => c.url === "/api/session/ses_1/prompt");
assert.deepEqual(prompts.map(c => c.body.text), ["", "wrap up now"]);
assert.equal(fake.calls.filter(c => c.url === "/api/session/ses_1/interrupt").length, 1);
} finally {
await fake.close();
}
});
test("an interrupted event that arrives after the wrap-up prompt went out does not end the wrap-up", async () => {
// The poll confirms the interrupt first; the stream's interrupted event lands while the follow-up is running.
const fake = await fakeOpencode({ idleAfterMs: null, wrapUpAfterMs: 600, interruptEventDelayMs: 400 });
try {
const client = new OpencodeClient({ baseUrl: fake.baseUrl, password: "pw", statusPollMs: 100 });
const result = await client.prompt({ ...PROMPT, timeoutMs: 4000, wrapUp: { ms: 2000, text: "wrap up now" } });
client.close();
assert.equal(result.wrappedUp, true);
assert.equal(result.text, "<no-issues/>");
assert.equal(fake.calls.filter(c => c.url === "/api/session/ses_1/prompt").length, 2);
} finally {
await fake.close();
}
});
test("a wrap-up that does not answer in its slot still ends as a hard timeout", async () => {
const fake = await fakeOpencode({ idleAfterMs: null });
try {
const client = new OpencodeClient({ baseUrl: fake.baseUrl, password: "pw", statusPollMs: 100 });
await assert.rejects(client.prompt({ ...PROMPT, timeoutMs: 1600, wrapUp: { ms: 800, text: "wrap up now" } }), err => err.name === "TimeoutError");
client.close();
assert.equal(fake.calls.filter(c => c.url === "/api/session/ses_1/prompt").length, 2);
} finally {
await fake.close();
}
});
test("the wrap-up slot is taken only from a session at least twice its length", () => {
const wrapUp = { ms: 3 * 60_000, text: "wrap" };
assert.equal(wrapUpStartMs(20 * 60_000, wrapUp), 17 * 60_000);
assert.equal(wrapUpStartMs(6 * 60_000, wrapUp), 3 * 60_000);
assert.equal(wrapUpStartMs(5 * 60_000, wrapUp), 5 * 60_000);
assert.equal(wrapUpStartMs(20 * 60_000, null), 20 * 60_000);
assert.equal(wrapUpStartMs(20 * 60_000, { ms: 60_000, text: "" }), 20 * 60_000);
});
test("a provider rate limit surfaced as session.execution.failed is transient", async () => {
const error = { type: "provider.rate-limit", message: "Too Many Requests", status: 429 };
const fake = await fakeOpencode({ idleAfterMs: 100, finalEvent: "session.execution.failed", error });
@@ -354,6 +424,14 @@ const READ_ONLY = [
rule("skill", "browser", "deny"),
];
const FULL = [...BASE, rule("*", "*", "allow"), rule("browser", "*", "deny")];
const CI_FULL = [
...BASE,
rule("*", "*", "allow"),
...["webfetch", "websearch", "subagent", "question", "execute", "skill"].map(id => rule(id, "*", "deny")),
rule("shell", "*", "deny"),
rule("shell", "git diff *", "allow"),
rule("browser", "*", "deny"),
];
const agent = (permissions, id = "ai-review-security") => ({ id, name: id, mode: "primary", permissions });
test("self-check passes only for read-only agents and built-in plugins", async () => {
@@ -379,17 +457,21 @@ test("self-check passes only for read-only agents and built-in plugins", async (
await assert.rejects(selfCheck(fakeClient({ agents: [agent(FULL)] }), names), /shell allowed/, "the default config must refuse a full grant");
});
test("ci-full self-check wants every agent registered with the full grant, and still refuses foreign plugins", async () => {
test("ci-full self-check wants file tools granted, web and sub-agent tools denied and the shell allowlisted", async () => {
const names = ["ai-review-security", "ai-review-correctness"];
const agents = [agent(FULL), agent(FULL, "ai-review-correctness")];
const agents = [agent(CI_FULL), agent(CI_FULL, "ai-review-correctness")];
const result = await selfCheck(fakeClient({ agents }), names, { readOnly: false });
assert.equal(result.agents, 2);
await assert.rejects(selfCheck(fakeClient({ agents: [agent(FULL)] }), names, { readOnly: false }), /ai-review-correctness: not registered/);
await assert.rejects(selfCheck(fakeClient({ agents: [agent(CI_FULL)] }), names, { readOnly: false }), /ai-review-correctness: not registered/);
await assert.rejects(
selfCheck(fakeClient({ agents: [agent(READ_ONLY), agent(FULL, "ai-review-correctness")] }), names, { readOnly: false }),
selfCheck(fakeClient({ agents: [agent(READ_ONLY), agent(CI_FULL, "ai-review-correctness")] }), names, { readOnly: false }),
/ai-review-security: edit not fully allowed/,
);
assert.deepEqual(ciFullProblems(FULL), ["webfetch allowed", "websearch allowed", "subagent allowed", "question allowed", "execute allowed", "skill allowed", "shell allows every command"]);
assert.deepEqual(ciFullProblems([...CI_FULL, rule("execute", "*", "allow")]), ["execute allowed"]);
assert.deepEqual(ciFullProblems([...CI_FULL, rule("shell", "*", "deny")]), ["shell denied"]);
assert.deepEqual(ciFullProblems(CI_FULL), []);
await assert.rejects(
selfCheck(fakeClient({ agents, plugins: [...BUILTIN_PLUGINS, { id: "p", source: { type: "package", target: "evil" }, state: { status: "active" } }] }), names, { readOnly: false }),
/non-built-in plugins/,
@@ -427,10 +509,44 @@ test("both configs register the same agents, provider and model; only the defaul
for (const toolId of BUILTIN_TOOL_IDS) assert.equal(toolEnabled(toolId, readOnly), ["read", "glob", "grep"].includes(toolId), `${name} ${toolId}`);
assert.equal(externalDirectoryProblem(readOnly), "", name);
const full = [...BASE, ...ci.agents[name].permissions];
for (const toolId of BUILTIN_TOOL_IDS) assert.equal(toolAllowedWithoutAsking(toolId, full), true, `${name} ${toolId}`);
assert.deepEqual(ciFullProblems(full), [], name);
// opencode 2.2.0 ends the capped turn on an assistant message, a prefill the model rejects, so no agent sets a step cap.
for (const config of [base, ci]) for (const key of ["steps", "maxSteps"]) assert.equal(config.agents[name][key], undefined, `${name} ${key}`);
}
});
// Evaluated as opencode 2.2.0 does: the last rule whose action and resource both match decides.
function shellDecision(rules, command) {
return [...BASE, ...rules].findLast(r => wildcardMatch("shell", r.action) && wildcardMatch(command, r.resource))?.effect;
}
test("the ci-full shell runs the read-only allowlist and nothing that builds, installs or reaches the network", () => {
const ci = JSON.parse(stripJsonc(readFileSync(join(REPO_ROOT, OPENCODE_CONFIGS["ci-full"].file), "utf-8")));
for (const [name, agentConfig] of Object.entries(ci.agents)) {
for (const command of ["git diff", "git diff origin/main -- src", "git log -5 --oneline", "git show HEAD:src/a.rs", "git blame -L 1,5 a.rs", "rg -n foo src", "cat a.rs", "ls", "ls -la web", "head -50 a.rs", "sed -n 1,40p a.rs", "wc -l a.rs",
"grep -rn foo src", "tail -n 20 a.rs", "find src -name '*.rs'", "find . -type f -newer a.rs", "find src -name '*-executor*'", "find . -name ok-delete.rs",
"git ls-files src", "git status", "git status --short", "git rev-parse HEAD", "git merge-base HEAD origin/main", "git cat-file -p HEAD:a.rs", "git cat-file -t HEAD", "git cat-file -s HEAD",
"git --no-pager diff --stat", "git --no-pager log -3", "git --no-pager show HEAD", "rg -n foo src/env_util.rs"]) {
assert.equal(shellDecision(agentConfig.permissions, command), "allow", `${name}: ${command}`);
}
for (const command of ["cargo build", "cargo test -p config", "npm ci", "npx vitest", "curl https://example.com", "git push", "git fetch origin", "git difftool", "sed -i s/a/b/ a.rs", "make", "python3 x.py", "bash -c ls",
"find . -exec rm {} ;", "find -execdir sh -c x ;", "find . -name x -ok rm {} ;", "find . -delete", "find . -fprint out.txt", "find . -fls out.txt",
"tail -f a.log", "tail -n 5 -F a.log", "tail --follow=name a.log", "git grep foo", "git grep -nOsh foo", "git grep --open-files=sh foo", "git --no-pager grep foo",
"git --no-pager push", "git --no-pager -c core.pager=sh log", "git cat-file --batch", "git cat-file blob HEAD",
"cat /proc/self/environ", "head -c 100 /proc/1/environ", "tail /dev/stdin", "grep -a KEY /proc/self/environ", "rg KEY /proc", "sed -n p /proc/self/environ",
"cat ../../proc/self/environ", "cat src/environ", "rg --file=/proc/self/environ x", "wc -c /p*/self/env*", "cat /etc/passwd", "cat a.rs > b.rs", "rg foo src>>out.txt", "cat a.rs 2>/dev/null"]) {
assert.equal(shellDecision(agentConfig.permissions, command), "deny", `${name}: ${command}`);
}
}
});
test("a trailing ' *' also matches the bare command, as in opencode 2.2.0", () => {
assert.equal(wildcardMatch("git diff", "git diff *"), true);
assert.equal(wildcardMatch("git diff --stat", "git diff *"), true);
assert.equal(wildcardMatch("git difftool", "git diff *"), false);
assert.equal(wildcardMatch("read", "*"), true);
});
test("the review runs on the anthropic-review provider, the only one the policies allow, with the workflow's variant", () => {
const base = JSON.parse(stripJsonc(readFileSync(join(REPO_ROOT, OPENCODE_CONFIGS.default.file), "utf-8")));
const allowed = base.policies.filter(p => p.action === "provider.use" && p.effect === "allow").map(p => p.resource);
@@ -441,9 +557,14 @@ test("the review runs on the anthropic-review provider, the only one the policie
assert.equal(provider.package, "@opencode/ai/providers/anthropic");
assert.deepEqual(provider.env, ["ANTHROPIC_REVIEW_API_KEY"]);
const model = provider.models["claude-sonnet-5-5"];
assert.deepEqual(model.variants, [{ id: "high", settings: { thinking: { type: "adaptive" }, effort: "high" } }]);
assert.deepEqual(model.variants, ["low", "medium", "high", "xhigh", "max"].map(id => ({ id, settings: { thinking: { type: "adaptive" }, effort: id } })));
assert.deepEqual(modelVariantIds(base, "anthropic-review", "claude-sonnet-5-5"), ["low", "medium", "high", "xhigh", "max"]);
assert.deepEqual(modelVariantIds(base, "anthropic-review", "no-such-model"), []);
const workflow = readFileSync(join(REPO_ROOT, ".github/workflows/ai-code-review.yml"), "utf-8");
assert.match(workflow, /provider_id: anthropic-review\n\s+model_id: claude-sonnet-5-5\n\s+model_variant: high\n\s+api_key_env: ANTHROPIC_REVIEW_API_KEY\n/);
assert.match(workflow, /provider_id: anthropic-review\n\s+model_id: claude-sonnet-5-5\n\s+api_key_env: ANTHROPIC_REVIEW_API_KEY\n/);
assert.doesNotMatch(workflow, /model_variant/, "the effort variable is the only source of the variant");
assert.match(workflow, /REVIEW_MODEL_VARIANT: \$\{\{ vars\.AI_REVIEW_EFFORT \|\| 'high' \}\}/);
assert.match(workflow, /REVIEW_COORDINATOR_VARIANT: \$\{\{ vars\.AI_REVIEW_COORDINATOR_EFFORT \|\| vars\.AI_REVIEW_EFFORT \|\| 'high' \}\}/);
assert.match(workflow, /ANTHROPIC_REVIEW_API_KEY: \$\{\{ secrets\.USAGE_CREDIT_API_KEY \}\}/);
});
scripts/ai-review/test/passes.test.mjs
+21
View File
@@ -1,5 +1,6 @@
import test from "node:test";
import assert from "node:assert/strict";
import { readFileSync } from "node:fs";
import { mergeConfidence, unfinishedReviewers } from "../src/confidence.mjs";
import { reviewerFacts } from "../src/auto-approve.mjs";
@@ -10,6 +11,7 @@ import {
parseColdPasses,
passLabel,
passPlan,
reviewComplete,
runPooled,
} from "../src/passes.mjs";
import { buildFallbackReview, partialFindingsBlock, selectFallbackFindings } from "../src/report.mjs";
@@ -137,3 +139,22 @@ test("the fallback never merges two different blockers that share most of their
]);
assert.equal(selectFallbackFindings(same).length, 1);
});
test("only a review whose every pass finished in time and whose coordinator merged may skip an unchanged diff next time", () => {
const coordinator = { fallback: false };
const skippedPass = { agentKey: "frontend", agentName: "frontend Reviewer", pass: 1, findings: [], error: null, skipped: true };
assert.equal(reviewComplete([done("security", 1), done("security", 2), skippedPass], coordinator), true);
assert.equal(reviewComplete([done("security", 1), failed("security", 2)], coordinator), false, "a lost cold pass is a partial review");
assert.equal(reviewComplete([done("security", 1), failed("security", 2, { timedOut: true })], coordinator), false);
assert.equal(reviewComplete([done("security", 1), { ...done("qa", 1), parsed: false }], coordinator), false, "unparseable output is unfinished");
assert.equal(reviewComplete([done("security", 1), { ...done("qa", 1), wrappedUp: true }], coordinator), false, "a reviewer cut short at its soft deadline");
assert.equal(reviewComplete([done("security", 1)], { fallback: true }), false, "the coordinator fell back");
assert.equal(reviewComplete([done("security", 1)], null), false, "the coordinator never ran");
});
test("run-review writes the diff fingerprint only for a complete review", () => {
const source = readFileSync(new URL("../run-review.mjs", import.meta.url), "utf-8");
assert.match(source, /const complete = reviewComplete\(passResults, coordinatorResult\);/);
assert.match(source, /diffFingerprint: complete && fullyRead && diffPinned && hunksLocatable\(diff, readBaseFile\) \? fingerprint : ""/);
assert.equal((source.match(/diffFingerprint:/g) || []).length, 1, "every finishReviewBody call goes through the completeness check");
});
scripts/ai-review/test/pr-diff.test.mjs
+112 -1
View File
@@ -6,7 +6,7 @@ import { tmpdir } from "node:os";
import { join } from "node:path";
import { parseRawDiff, sizeReasons } from "../src/auto-approve.mjs";
import { fetchDiff, fetchPinnedDiff, localDiff, pinDiff, prefixDiffPaths } from "../src/pr-diff.mjs";
import { diffFingerprint, fetchDiff, fetchPinnedDiff, hunksLocatable, localDiff, pinDiff, prefixDiffPaths } from "../src/pr-diff.mjs";
const DIFF = [
"diff --git a/src/lib.rs b/src/lib.rs",
@@ -199,3 +199,114 @@ test("a renamed symlink or submodule whose header carries no mode cannot pass as
assert.match(edited, /^index \S+ 100644$/m);
assert.deepEqual(parseRawDiff(edited).problems, []);
});
test("a rebase that moves the PR's lines keeps its diff fingerprint, a changed line does not", () => {
const dir = mkdtempSync(join(tmpdir(), "ai-review-fingerprint-"));
const git = (...args) => execFileSync("git", ["-C", dir, "-c", "user.name=t", "-c", "user.email=t@t", "-c", "commit.gpgsign=false", ...args], { encoding: "utf-8" });
try {
git("init", "-q", "-b", "main");
const lines = Array.from({ length: 40 }, (_, i) => `line ${i}`);
writeFileSync(join(dir, "a.rs"), `${lines.join("\n")}\n`);
git("add", ".");
git("commit", "-qm", "base");
git("checkout", "-qb", "pr");
writeFileSync(join(dir, "a.rs"), `${lines.map((l, i) => (i === 30 ? "line 30 changed" : l)).join("\n")}\n`);
git("commit", "-qam", "pr change");
const before = localDiff(dir, "main", "pr");
git("checkout", "-q", "main");
writeFileSync(join(dir, "a.rs"), `head 1\nhead 2\nhead 3\n${lines.join("\n")}\n`);
git("commit", "-qam", "main moves on");
git("checkout", "-q", "pr");
git("rebase", "-q", "main");
const after = localDiff(dir, "main", "pr");
assert.notEqual(before, after, "the rebase must change hunk numbers and blob ids, or this test proves nothing");
assert.equal(diffFingerprint(after), diffFingerprint(before));
writeFileSync(join(dir, "a.rs"), `head 1\nhead 2\nhead 3\n${lines.map((l, i) => (i === 30 ? "line 30 changed again" : l)).join("\n")}\n`);
git("commit", "-qam", "another change");
assert.notEqual(diffFingerprint(localDiff(dir, "main", "pr")), diffFingerprint(before));
} finally {
rmSync(dir, { recursive: true, force: true });
}
});
test("the fingerprint ignores only blob ids and hunk numbers", () => {
const base = "diff --git a/a b/a\nindex 1111111..2222222 100644\n--- a/a\n+++ b/a\n@@ -1,2 +1,2 @@ fn x\n ctx\n-old\n+new\n";
assert.equal(diffFingerprint(base.replace("1111111..2222222", "3333333..4444444").replace("-1,2 +1,2 @@ fn x", "-9,2 +12,2 @@ fn x")), diffFingerprint(base));
assert.notEqual(diffFingerprint(base.replace("@@ fn x", "@@ fn y")), diffFingerprint(base), "the function a hunk sits in is part of what was reviewed");
assert.notEqual(diffFingerprint(base.replace("+new", "+newer")), diffFingerprint(base));
assert.notEqual(diffFingerprint(base.replace("diff --git a/a b/a", "diff --git a/b b/b")), diffFingerprint(base));
assert.match(diffFingerprint(base), /^[0-9a-f]{64}$/);
});
test("the same lines changed inside a different function get a different fingerprint", () => {
const dir = mkdtempSync(join(tmpdir(), "ai-review-fingerprint-"));
const git = (...args) => execFileSync("git", ["-C", dir, "-c", "user.name=t", "-c", "user.email=t@t", "-c", "commit.gpgsign=false", ...args], { encoding: "utf-8" });
const fn = (name, allowed = "user.active") => `function ${name}(user) {\n const id = user.id;\n const name = user.name;\n const role = user.role;\n const group = user.group;\n const allowed = ${allowed};\n const details = { id, name, role, group };\n const present = Boolean(details);\n const result = Boolean(allowed);\n return present && result;\n}\n`;
try {
git("init", "-q", "-b", "main");
writeFileSync(join(dir, "rules.js"), `${fn("preview")}\n${fn("authorize")}`);
git("add", ".");
git("commit", "-qm", "base");
git("checkout", "-qb", "safe");
writeFileSync(join(dir, "rules.js"), `${fn("preview", "true")}\n${fn("authorize")}`);
git("commit", "-qam", "safe");
git("checkout", "-qb", "unsafe", "main");
writeFileSync(join(dir, "rules.js"), `${fn("preview")}\n${fn("authorize", "true")}`);
git("commit", "-qam", "unsafe");
const safe = localDiff(dir, "main", "safe");
const unsafe = localDiff(dir, "main", "unsafe");
assert.match(safe, /^@@ .* @@ function preview\(user\) \{$/m);
assert.match(unsafe, /^@@ .* @@ function authorize\(user\) \{$/m);
assert.notEqual(diffFingerprint(unsafe), diffFingerprint(safe));
} finally {
rmSync(dir, { recursive: true, force: true });
}
});
test("an edit to one of two identical blocks in a function cannot be located, so it is never taken for the reviewed one", () => {
const dir = mkdtempSync(join(tmpdir(), "ai-review-fingerprint-"));
const git = (...args) => execFileSync("git", ["-C", dir, "-c", "user.name=t", "-c", "user.email=t@t", "-c", "commit.gpgsign=false", ...args], { encoding: "utf-8" });
const block = (flag, allowed) => [` if (${flag}) {`, " const a = 1;", " const b = 2;", " const c = 3;", ` ok = ${allowed};`, " const d = 4;", " const e = 5;", " const f = 6;", " }"];
const source = (first, second) => ["function check(user, internal) {", " let ok = false;", ...block("internal", first), ...block("!internal", second), " return ok;", "}", ""].join("\n");
const readBase = ref => path => { try { return execFileSync("git", ["-C", dir, "cat-file", "blob", `${ref}:${path}`], { encoding: "utf-8" }); } catch { return null; } };
try {
git("init", "-q", "-b", "main");
writeFileSync(join(dir, "check.js"), source("user.admin", "user.admin"));
git("add", ".");
git("commit", "-qm", "base");
git("checkout", "-qb", "internal");
writeFileSync(join(dir, "check.js"), source("true", "user.admin"));
git("commit", "-qam", "internal");
git("checkout", "-qb", "external", "main");
writeFileSync(join(dir, "check.js"), source("user.admin", "true"));
git("commit", "-qam", "external");
const internal = localDiff(dir, "main", "internal");
const external = localDiff(dir, "main", "external");
assert.notEqual(internal, external);
assert.equal(hunksLocatable(external, readBase("main")), false, "the edited block's old text sits twice in the base");
assert.equal(hunksLocatable(internal, readBase("main")), false);
writeFileSync(join(dir, "other.js"), "a\nb\nc\nd\ne\nf\ng\n");
git("add", "other.js");
git("commit", "-qm", "unique");
writeFileSync(join(dir, "other.js"), "a\nb\nc\nD\ne\nf\ng\n");
git("commit", "-qam", "edit unique");
const unique = localDiff(dir, "HEAD~1", "HEAD");
assert.equal(hunksLocatable(unique, readBase("HEAD~1")), true);
assert.equal(hunksLocatable(unique, () => null), false, "an unreadable base file cannot locate the edit");
assert.equal(hunksLocatable("diff --git a/n b/n\nnew file mode 100644\n--- /dev/null\n+++ b/n\n@@ -0,0 +1 @@\n+x\n", () => null), true, "a new file has nothing to locate");
} finally {
rmSync(dir, { recursive: true, force: true });
}
});
test("a changed binary file or trailing space on the last line changes the fingerprint", () => {
const bin = id => `diff --git a/m.wasm b/m.wasm\nindex 1111111..${id} 100644\nBinary files a/m.wasm and b/m.wasm differ\n`;
assert.notEqual(diffFingerprint(bin("2222222")), diffFingerprint(bin("3333333")));
const csv = tail => `diff --git a/a.csv b/a.csv\nindex 1111111..2222222 100644\n--- a/a.csv\n+++ b/a.csv\n@@ -1 +1 @@\n-x\n+secret${tail}\n`;
assert.notEqual(diffFingerprint(csv(" ")), diffFingerprint(csv(" ")));
});
test("a file mode change keeps a different fingerprint", () => {
const d = mode => `diff --git a/l b/l\nindex 1111111..2222222 ${mode}\n--- a/l\n+++ b/l\n@@ -1 +1 @@\n-old-target\n\\ No newline at end of file\n+new-target\n\\ No newline at end of file\n`;
assert.notEqual(diffFingerprint(d("100644")), diffFingerprint(d("120000")));
});
scripts/ai-review/test/report.test.mjs
+153 -6
View File
@@ -37,7 +37,8 @@ import {
} from "../src/report.mjs";
import { extractInlineBlock } from "../src/inline.mjs";
import { LOADER_DIRS, LOADER_FILES } from "../src/sanitize.mjs";
import { reviewerDiff } from "../src/pr-diff.mjs";
import { diffFingerprint, hunksLocatable, reviewerDiff } from "../src/pr-diff.mjs";
import { narrowTier } from "../src/selection.mjs";
import * as approvalModule from "../src/auto-approve.mjs";
import * as commentsModule from "../src/comments.mjs";
import * as confidenceModule from "../src/confidence.mjs";
@@ -779,7 +780,8 @@ const mainSource = () => {
const source = readFileSync(new URL("../run-review.mjs", import.meta.url), "utf-8");
const fetcher = source.slice(source.indexOf("function fetchPrDiff("), source.indexOf("function pairedPrContext("));
const notes = source.slice(source.indexOf("function noteOnComment("), source.indexOf("function noteSkippedPush("));
return `${fetcher}\n${notes}\n${source.slice(source.indexOf("async function main() {"), source.indexOf("// ─── Entry point"))}\nreturn main;`;
const unchanged = source.slice(source.indexOf("function unchangedDiffCheck("), source.indexOf("function prepareReviewTree("));
return `${fetcher}\n${notes}\n${unchanged}\n${source.slice(source.indexOf("async function main() {"), source.indexOf("// ─── Entry point"))}\nreturn main;`;
};
const cleanCoordinatorOutput = `${commentsModule.REVIEW_HEADING}\n\n> [!TIP]\n> ### ✅ Approved\n> Looks right.\n\n<!-- ai-review-inline [] -->`;
const finishedReviewer = key => ({ agentKey: key, agentName: `${key} Reviewer`, findings: [], parsed: true, fullDiff: true, error: null });
@@ -787,19 +789,19 @@ const silentReviewer = key => ({ ...finishedReviewer(key), parsed: false, rawTex
const smallRawDiff = "diff --git a/docs/a.md b/docs/a.md\n--- a/docs/a.md\n+++ b/docs/a.md\n@@ -1 +1 @@\n-old\n+new\n";
// Every default makes the run an approvable one, so a test changes the single fact it is about.
async function runMain({ results = [finishedReviewer("security"), finishedReviewer("correctness")], byPass = null, pinned = {}, mode = "approve", existing = null, reviewedBody = "plain words", prNow = {}, paired = {}, contextFails = false, noiseOnly = false, diffFails = false, coordinatorOutput = cleanCoordinatorOutput, rawDiff = smallRawDiff, keptDiff = null } = {}) {
async function runMain({ results = [finishedReviewer("security"), finishedReviewer("correctness")], byPass = null, pinned = {}, mode = "approve", existing = null, reviewedBody = "plain words", prNow = {}, paired = {}, contextFails = false, noiseOnly = false, diffFails = false, manual = false, head = MAIN_HEAD, coordinatorOutput = cleanCoordinatorOutput, rawDiff = smallRawDiff, keptDiff = null, variantError = "", ignorePassLimit = false, forceFull = false, baseFile = "old\n" } = {}) {
const run = { posted: "", approvals: [], logs: [], coordinatorCalls: 0, coordinatorArgs: null, reviewerCalls: [], artifact: null, updates: 0, creates: 0, claims: 0, sanitized: 0 };
const pr = { ...eligibleInput().pr, title: "fix a bug", body: "plain words", head: { sha: MAIN_HEAD }, changed_files: 1, additions: 1, deletions: 1, ...prNow };
const pr = { ...eligibleInput().pr, title: "fix a bug", body: "plain words", head: { sha: head }, changed_files: 1, additions: 1, deletions: 1, ...prNow };
const noop = () => {};
const scope = {
...commentsModule, ...confidenceModule, ...approvalModule, ...reportModule, ...passesModule,
REVIEW_COLD_PASSES: 2, REVIEWER_CONCURRENCY: 16,
process: { env: { PR_NUMBER: "1", GITHUB_REPOSITORY: "test/repo", HEAD_SHA: MAIN_HEAD, MERGE_BASE_SHA: "c".repeat(40) } },
process: { env: { PR_NUMBER: "1", GITHUB_REPOSITORY: "test/repo", HEAD_SHA: head, MERGE_BASE_SHA: "c".repeat(40), ...(forceFull ? { FORCE_FULL: "true" } : {}) } },
console: { log: (...args) => run.logs.push(args.join(" ")), warn: noop, error: noop },
TRACE: { startSpan: () => ({}), endSpan: noop, setSpanAttributes: noop }, activeRootSpan: null, isoNow: () => "now", prepareReviewTree: noop,
findExistingReviewComment: () => ({ comment: existing, ok: true }),
passesDoneIn: comment => (comment ? commentsModule.reviewPassCount(comment.body) : 0),
REVIEW_IGNORE_PASS_LIMIT: false, REVIEW_MAX_PASSES: 0, REVIEW_BOT_LOGIN: "review[bot]", REVIEW_PAIR_CLEAR: true, AUTO_APPROVE_MODE: mode,
REVIEW_IGNORE_PASS_LIMIT: ignorePassLimit, REVIEW_MANUAL_RUN: manual, MAX_DIFF_TOKENS: 150_000, diffFingerprint, hunksLocatable, readBaseFile: () => baseFile, variantProblem: () => variantError, MODEL_VARIANT: "high", COORDINATOR_VARIANT: "high", REVIEW_MAX_PASSES: 0, REVIEW_BOT_LOGIN: "review[bot]", REVIEW_PAIR_CLEAR: true, AUTO_APPROVE_MODE: mode,
REVIEW_MARKER: "<!-- ai-code-review -->", PAIRED: { repo: "", prNumber: "", dir: "", mergeBase: "", head: "", ...paired }, PR_DIR: "/pr", resolve: (...parts) => parts.join("/"),
githubReviewApi: () => ({ readPr: () => pr, listReviews: () => [], dismissReview: noop, approve: payload => run.approvals.push(payload) }),
noteSkippedPush: noop, apiKey: () => "configured", API_KEY_VAR_NAME: "TEST_KEY", MODEL_SLUG: "test/model",
@@ -922,6 +924,109 @@ test("a push the run stops on before claiming a pass loses the old verdict and k
}
});
test("a new head whose diff matches the reviewed one is skipped without a pass, a model call or a claim", async () => {
const first = await runMain({ mode: "off" });
assert.equal(commentsModule.reviewedDiffFingerprint(first.posted), diffFingerprint(smallRawDiff));
const rebased = await runMain({ mode: "off", existing: { id: 1, body: first.posted }, head: PAIRED_MAIN_HEAD });
assert.equal(rebased.error, undefined);
assert.ok(rebased.logs.some(line => line.includes("unchanged_diff")));
assert.deepEqual(rebased.reviewerCalls, []);
assert.equal(rebased.coordinatorCalls, 0);
assert.equal(rebased.claims, 0);
assert.equal(rebased.updates, 1);
assert.equal(commentsModule.reviewedKey(rebased.posted), PAIRED_MAIN_HEAD);
assert.equal(commentsModule.reviewPassCount(rebased.posted), 1);
assert.match(rebased.posted, /Merge confidence: 🟢 High/);
});
test("a hand-started run, a changed diff or an unpinned diff is reviewed even after a matching review", async () => {
const first = await runMain({ mode: "off" });
const existing = { id: 1, body: first.posted };
const changed = `${smallRawDiff}+more\n`;
for (const run of [
await runMain({ mode: "off", existing, head: PAIRED_MAIN_HEAD, manual: true }),
await runMain({ mode: "off", existing, head: PAIRED_MAIN_HEAD, rawDiff: changed }),
await runMain({ mode: "off", existing, head: PAIRED_MAIN_HEAD, pinned: { "test/repo": false } }),
]) {
assert.equal(run.logs.some(line => line.includes("unchanged_diff")), false);
assert.equal(run.claims, 1);
assert.ok(run.reviewerCalls.length > 0);
assert.equal(commentsModule.reviewPassCount(run.posted), 2);
}
});
test("a hand-started, force-full or pass-limit-ignoring run reviews the head the last review already covered", async () => {
const first = await runMain({ mode: "off" });
const existing = { id: 1, body: first.posted };
for (const bypass of [{ manual: true }, { forceFull: true }, { ignorePassLimit: true }]) {
const run = await runMain({ mode: "off", existing, ...bypass });
assert.equal(run.error, undefined, JSON.stringify(bypass));
assert.equal(run.logs.some(line => /already_reviewed|unchanged_diff/.test(line)), false, JSON.stringify(bypass));
assert.ok(run.reviewerCalls.length > 0, JSON.stringify(bypass));
assert.equal(run.claims, 1, JSON.stringify(bypass));
}
const automatic = await runMain({ mode: "off", existing });
assert.ok(automatic.logs.some(line => line.includes("already_reviewed")));
assert.deepEqual(automatic.reviewerCalls, []);
});
test("reviewers cut short at their soft deadline are not finished, so the verdict is not High and the gate does not approve", async () => {
const wrapped = key => ({ ...finishedReviewer(key), wrappedUp: true });
for (const results of [[wrapped("security"), wrapped("correctness")], [finishedReviewer("security"), wrapped("correctness")]]) {
const run = await runMain({ results });
assert.equal(run.error, undefined);
assert.equal(run.approvals.length, 0);
assert.doesNotMatch(run.posted, /Merge confidence: 🟢 High/);
assert.doesNotMatch(run.posted, /all reviewers finished/);
assert.doesNotMatch(run.posted, /Auto-approved/);
assert.equal(commentsModule.reviewedDiffFingerprint(run.posted), "");
}
const byPass = (key, pass) => ({ ...finishedReviewer(key), wrappedUp: pass === 2 });
const oneCleanPass = await runMain({ byPass });
assert.equal(oneCleanPass.approvals.length, 1, "an area with one uninterrupted pass is still reviewed");
});
test("an invalid effort setting retracts the old verdict and approval line without claiming a pass", async () => {
const first = await runMain();
assert.match(first.posted, /Merge confidence: 🟢 High/);
assert.match(first.posted, /Auto-approved/);
const variantError = 'variant "typo" is not declared for test/model in opencode.ci.jsonc (declared: high)';
for (const stop of [{ head: PAIRED_MAIN_HEAD, rawDiff: `${smallRawDiff}+more\n` }, { manual: true }]) {
const run = await runMain({ existing: { id: 1, body: first.posted }, variantError, ...stop });
assert.match(run.error, /variant "typo" is not declared/);
assert.equal(run.claims, 0);
assert.deepEqual(run.reviewerCalls, []);
assert.equal(run.updates, 1);
assert.doesNotMatch(run.posted, /High/);
assert.doesNotMatch(run.posted, /Auto-approved/);
assert.match(run.posted, /the review did not run/);
assert.doesNotMatch(run.posted, /typo|test\/model/, "the configuration detail stays in the log");
assert.equal(commentsModule.reviewPassCount(run.posted), 1);
assert.equal(commentsModule.reviewedKey(run.posted), MAIN_HEAD);
}
});
test("an unchanged diff whose hunk text appears twice in the base is reviewed, as the edit cannot be located", async () => {
const first = await runMain({ mode: "off" });
const twice = await runMain({ mode: "off", existing: { id: 1, body: first.posted }, head: PAIRED_MAIN_HEAD, baseFile: "old\nx\nold\n" });
assert.equal(twice.logs.some(line => line.includes("unchanged_diff")), false);
assert.equal(twice.claims, 1);
assert.equal(commentsModule.reviewedDiffFingerprint(twice.posted), "", "a review whose edit cannot be located records no fingerprint");
const unreadable = await runMain({ mode: "off", existing: { id: 1, body: first.posted }, head: PAIRED_MAIN_HEAD, baseFile: null });
assert.equal(unreadable.claims, 1);
assert.ok((await runMain({ mode: "off", existing: { id: 1, body: first.posted }, head: PAIRED_MAIN_HEAD })).logs.some(line => line.includes("unchanged_diff")));
});
test("a push that only adds a production asset the noise filter hides is reviewed, not skipped with the old High", async () => {
const first = await runMain({ mode: "off" });
assert.match(first.posted, /Merge confidence: 🟢 High/);
const svg = "diff --git a/web/src/assets/chart.svg b/web/src/assets/chart.svg\nnew file mode 100644\n--- /dev/null\n+++ b/web/src/assets/chart.svg\n@@ -0,0 +1 @@\n+<svg/>\n";
const next = await runMain({ mode: "off", existing: { id: 1, body: first.posted }, head: PAIRED_MAIN_HEAD, rawDiff: `${smallRawDiff}${svg}`, keptDiff: smallRawDiff });
assert.equal(next.logs.some(line => line.includes("unchanged_diff")), false);
assert.equal(next.claims, 1);
assert.doesNotMatch(next.posted, /Merge confidence: 🟢 High/);
});
test("a dangling second block cannot read as a clean count, so the gate refuses", () => {
const output = `${cleanCoordinatorOutput}\n<!-- ai-review-inline [{"file":"docs/a.md","line":1,"severity":"p1","title":"broken"}]`;
const inline = extractInlineBlock(output);
@@ -1343,3 +1448,45 @@ test("a tilde-fenced or blockquoted report example is not a finished review", ()
];
for (const [text, parsed] of table) assert.equal(reviewerReported(text, parseFindings(text)), parsed, text);
});
// The tier table and its chooser are lifted from the runner, which starts a review on import.
function riskTierFns() {
const source = readFileSync(new URL("../run-review.mjs", import.meta.url), "utf-8");
const tiers = source.slice(source.indexOf("const COORDINATOR_AGENT"), source.indexOf("// Agents added on top"));
const security = source.slice(source.indexOf("function isSecuritySensitiveFile("), source.indexOf("// Scoped to web/src on purpose"));
const assess = source.slice(source.indexOf("function assessRiskTier("), source.indexOf("function selectAgents("));
return new Function("narrowTier", "logicalPath", `${tiers}\n${security}\n${assess}\nreturn { RISK_TIERS, assessRiskTier };`)(narrowTier, p => p);
}
test("test-only and CI-only PRs of any size get their narrow reviewer set, unless a path or the file count escalates to full", () => {
const { RISK_TIERS, assessRiskTier } = riskTierFns();
assert.deepEqual(RISK_TIERS["test-only"].agents, ["correctness", "code-quality"]);
assert.deepEqual(RISK_TIERS["ci-only"].agents, ["security", "release"]);
const diff = (paths, lines = 500) => ({ files: paths.map(newPath => ({ oldPath: newPath, newPath })), totalAdded: lines, totalRemoved: 0 });
assert.equal(assessRiskTier(diff(["tests/ui-testing/playwright-tests/a.spec.js"])), "test-only");
assert.equal(assessRiskTier(diff([".github/workflows/ci.yml"], 3)), "ci-only");
assert.equal(assessRiskTier(diff(["src/handler/http/auth/tests/login.rs"])), "full", "a security-looking path still escalates");
assert.equal(assessRiskTier(diff(Array.from({ length: 51 }, (_, i) => `tests/api-testing/test_${i}.py`))), "full");
assert.equal(assessRiskTier(diff(["src/service/search.rs"], 5)), "trivial");
assert.equal(assessRiskTier(diff(["src/service/search.rs"], 50)), "lite");
});
test("an action or workflow under a test-named directory stays ci-only, not test-only", () => {
assert.equal(narrowTier([".github/actions/test/action.yml"]), "ci-only");
assert.equal(narrowTier([".github/workflows/tests.yml"]), "ci-only");
});
test("a narrow tier needs both a rename's source and its destination to qualify", () => {
const { assessRiskTier } = riskTierFns();
const moved = (oldPath, newPath) => ({ files: [{ oldPath, newPath }], totalAdded: 120, totalRemoved: 120 });
assert.equal(assessRiskTier(moved("src/service/search.rs", "tests/search.rs")), "full", "moving production code into tests removes it from production");
assert.equal(assessRiskTier(moved("scripts/deploy.sh", ".github/workflows/deploy.yml")), "full");
assert.equal(assessRiskTier(moved("src/handler/http/auth/login.rs", "src/service/login.rs")), "full", "a security-looking source still escalates");
assert.equal(assessRiskTier(moved("tests/a_test.py", "tests/b_test.py")), "test-only");
});
test("a diff longer than a reviewer reads records no fingerprint, so the next push reviews it", async () => {
const big = `diff --git a/docs/a.md b/docs/a.md\n--- a/docs/a.md\n+++ b/docs/a.md\n@@ -1 +1,2 @@\n-old\n+new\n+${"x".repeat(160_000)}\n`;
const run = await runMain({ mode: "off", rawDiff: big });
assert.equal(commentsModule.reviewedDiffFingerprint(run.posted), "");
});
scripts/ai-review/test/selection.test.mjs
+156 -1
View File
@@ -1,7 +1,14 @@
import test from "node:test";
import assert from "node:assert/strict";
import { isReleaseFile, resolveRulesFiles } from "../src/selection.mjs";
import { spawnSync } from "node:child_process";
import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { isCiOnlyPath, isDocsPath, isReleaseFile, isTestOnlyPath, listingComplete, narrowTier, nothingToReviewReason, resolveRulesFiles } from "../src/selection.mjs";
import { isSensitivePath } from "../src/auto-approve.mjs";
import { LOADER_DIRS, LOADER_FILES } from "../src/sanitize.mjs";
test("release review runs only for release-relevant files", () => {
for (const path of [
@@ -40,3 +47,151 @@ test("rules fall back to CLAUDE.md only where AGENTS.md is absent", () => {
);
assert.deepEqual(resolveRulesFiles(["/x/AGENTS.md"], p => p === "/x/CLAUDE.md"), ["/x/CLAUDE.md"]);
});
test("a PR of test code only gets the test-only tier, of CI files only the ci-only tier, and anything else neither", () => {
assert.equal(narrowTier(["tests/ui-testing/playwright-tests/a.spec.js", "src/service/tests/query.rs", "web/src/utils/date.spec.ts"]), "test-only");
assert.equal(narrowTier(["src/config/src/lib_test.rs", "tests/api-testing/test_streams.py", "web/src/test/fixtures/x.json"]), "test-only");
assert.equal(narrowTier(["scripts/ai-review/test/comments.test.mjs"]), "test-only", "the engine's own tests want a correctness review");
assert.equal(narrowTier([".github/workflows/ai-code-review.yml", ".github/actions/setup/action.yml", ".github/dependabot.yml"]), "ci-only");
for (const path of ["scripts/ai-review/run-review.mjs", "scripts/ai-review/agents/qa.md", "opencode.jsonc", "opencode.ci.jsonc", "scripts/test-assist-ci-batch.sh", "scripts/ci-telemetry/o2_llm_trace.py", "scripts/x.js", ".github/CODEOWNERS"]) {
assert.equal(isCiOnlyPath(path), false, `${path} is code, not CI config`);
}
assert.equal(narrowTier([".github/workflows/ai-code-review.yml", "scripts/ai-review/run-review.mjs"]), "", "the review engine takes the normal tiers");
assert.equal(narrowTier(["tests/ui-testing/a.spec.js", "src/service/search.rs"]), "");
assert.equal(narrowTier([".github/workflows/x.yml", "web/src/App.vue"]), "");
assert.equal(narrowTier([".github/workflows/x.yml", "tests/api-testing/test_x.py"]), "", "a mix is neither narrow set");
assert.equal(narrowTier([]), "");
for (const path of ["src/service/search.rs", "web/src/App.vue", "scripts/translations/main.py", "contest/a.rs", "latest/x.ts"]) {
assert.equal(isTestOnlyPath(path), false, path);
assert.equal(isCiOnlyPath(path), false, path);
}
});
test("dependency-bot, docs-only and translation-only PRs need no review; anything else, or an empty file list, does", () => {
const bot = (login, extra = {}) => ({ author: login, committer: "web-flow", verified: true, ...extra });
assert.equal(nothingToReviewReason({ author: "dependabot[bot]", commits: [bot("dependabot[bot]"), bot("dependabot[bot]")], paths: ["Cargo.toml"] }), "bot_author");
assert.equal(nothingToReviewReason({ author: "renovate[bot]", commits: [bot("renovate[bot]", { committer: "renovate[bot]" })], paths: ["web/package.json"] }), "bot_author");
const reviewed = {
"a person's commit": bot("alice"),
"an unlinked commit author": bot(""),
"a person as committer": bot("dependabot[bot]", { committer: "alice" }),
"a missing committer": bot("dependabot[bot]", { committer: "" }),
"an unverified signature": bot("dependabot[bot]", { verified: false }),
"another bot's commit": bot("renovate[bot]"),
};
for (const [why, commit] of Object.entries(reviewed)) {
assert.equal(nothingToReviewReason({ author: "dependabot[bot]", commits: [bot("dependabot[bot]"), commit], paths: ["Cargo.toml"] }), "", why);
}
assert.equal(nothingToReviewReason({ author: "dependabot[bot]", commits: [], paths: ["Cargo.toml"] }), "", "a failed commit listing reviews");
for (const author of ["github-actions[bot]", "Copilot", "o2-ai-bot[bot]", "Dependabot[bot]"]) {
assert.equal(nothingToReviewReason({ author, commits: [bot(author)], paths: ["src/a.rs"] }), "", `${author} is reviewed`);
}
assert.equal(nothingToReviewReason({ author: "alice", paths: ["README.md", "docs/setup.txt", "web/README.MD"] }), "docs_only");
assert.equal(nothingToReviewReason({ author: "alice", paths: ["web/src/locales/languages/fr.json", "web/src/locales/languages/de.json", "scripts/translations/.translation_state.json"] }), "translation_only");
assert.equal(nothingToReviewReason({ author: "alice", paths: ["web/src/locales/languages/en-US.json"] }), "", "the English source is written by a person");
assert.equal(nothingToReviewReason({ author: "alice", paths: ["web/src/locales/languages/fr.json", "web/src/App.vue"] }), "");
assert.equal(nothingToReviewReason({ author: "alice", paths: ["README.md", "src/main.rs"] }), "");
assert.equal(nothingToReviewReason({ author: "alice", paths: ["web/src/locales/dbmMessages.spec.ts"] }), "");
assert.equal(nothingToReviewReason({ author: "alice", paths: [] }), "", "a failed file listing must never skip a review");
});
test("skip-check.mjs prints the reason the workflow step reads, and nothing when the PR is reviewed", () => {
const dir = mkdtempSync(join(tmpdir(), "ai-review-skip-"));
try {
const list = join(dir, "files.txt");
const commits = join(dir, "commits.txt");
const run = (files, env = {}, authors = null) => {
writeFileSync(list, files.join("\n"));
env = { EXPECTED_FILES: String(files.length), EXPECTED_COMMITS: String(authors?.length ?? 0), ...env };
// The workflow's jq output: one login per line, newline-terminated.
if (authors) writeFileSync(commits, authors.map(a => `${a}\n`).join(""));
const args = authors ? [list, commits] : [list];
const out = spawnSync(process.execPath, [new URL("../skip-check.mjs", import.meta.url).pathname, ...args], { encoding: "utf-8", env: { PATH: process.env.PATH, ...env } });
assert.equal(out.status, 0, out.stderr);
return out.stdout;
};
assert.equal(run(["docs/a.md"], { PR_AUTHOR: "alice" }), "docs_only");
const ok = "renovate[bot]\tweb-flow\ttrue";
assert.equal(run(["src/a.rs"], { PR_AUTHOR: "renovate[bot]" }, [ok, "renovate[bot]\trenovate[bot]\ttrue"]), "bot_author");
assert.equal(run(["src/a.rs"], { PR_AUTHOR: "renovate[bot]" }, [ok, "\tweb-flow\ttrue"]), "", "an unlinked commit author is not the bot");
assert.equal(run(["src/a.rs"], { PR_AUTHOR: "renovate[bot]" }, [ok, "renovate[bot]\tweb-flow\tfalse"]), "", "an unverified commit reviews");
assert.equal(run(["src/a.rs"], { PR_AUTHOR: "renovate[bot]" }, [ok, "renovate[bot]\tweb-flow\t"]), "", "a missing verification reviews");
assert.equal(run(["src/a.rs"], { PR_AUTHOR: "renovate[bot]" }, [ok, "renovate[bot]"]), "", "a short line reviews");
assert.equal(run(["src/a.rs"], { PR_AUTHOR: "renovate[bot]" }, []), "", "a failed commit listing reviews");
assert.equal(run(["src/a.rs"], { PR_AUTHOR: "renovate[bot]" }), "", "no commit file reviews");
assert.equal(run(["src/a.rs", "docs/a.md"], { PR_AUTHOR: "alice" }), "");
assert.equal(run([], { PR_AUTHOR: "alice" }), "");
assert.equal(run(["docs/new.md\tdocs/old.md"], { PR_AUTHOR: "alice" }), "docs_only", "a rename is one listed file");
assert.equal(run(["docs/new.md\tsrc/old.rs"], { PR_AUTHOR: "alice" }), "", "a rename source outside docs reviews");
} finally {
rmSync(dir, { recursive: true, force: true });
}
});
test("a file or commit listing that may be cut short never skips a review", () => {
const dir = mkdtempSync(join(tmpdir(), "ai-review-skip-"));
try {
const list = join(dir, "files.tsv");
const commits = join(dir, "commits.tsv");
const run = (fileCount, commitCount, env) => {
writeFileSync(list, Array.from({ length: fileCount }, (_, i) => `docs/a${i}.md`).join("\n"));
writeFileSync(commits, Array.from({ length: commitCount }, () => "renovate[bot]\tweb-flow\ttrue\n").join(""));
const out = spawnSync(process.execPath, [new URL("../skip-check.mjs", import.meta.url).pathname, list, commits], { encoding: "utf-8", env: { PATH: process.env.PATH, PR_AUTHOR: "alice", ...env } });
assert.equal(out.status, 0, out.stderr);
return out.stdout;
};
assert.equal(run(2, 1, { EXPECTED_FILES: "2", EXPECTED_COMMITS: "1" }), "docs_only");
assert.equal(run(3000, 1, { EXPECTED_FILES: "3000", EXPECTED_COMMITS: "1" }), "", "the files endpoint stops at 3000");
assert.equal(run(3000, 1, { EXPECTED_FILES: "3200", EXPECTED_COMMITS: "1" }), "");
assert.equal(run(2, 1, { EXPECTED_FILES: "3", EXPECTED_COMMITS: "1" }), "", "a listing shorter than the PR's own count");
assert.equal(run(2, 1, { EXPECTED_FILES: "2", EXPECTED_COMMITS: "2" }), "");
assert.equal(run(2, 250, { EXPECTED_FILES: "2", EXPECTED_COMMITS: "250" }), "", "the commits endpoint stops at 250");
assert.equal(run(2, 1, {}), "", "unknown PR counts review");
assert.equal(run(2, 1, { EXPECTED_FILES: "two", EXPECTED_COMMITS: "1" }), "");
assert.equal(listingComplete({ files: 2, commits: 1, expectedFiles: "2", expectedCommits: "1" }), true);
assert.equal(listingComplete({ files: 2999, commits: 249, expectedFiles: "2999", expectedCommits: "249" }), true);
assert.equal(listingComplete({ files: 250, commits: 250, expectedFiles: "250", expectedCommits: "250" }), false);
} finally {
rmSync(dir, { recursive: true, force: true });
}
});
test("the workflow skips nothing-to-review PRs before installing anything, and lets a hand-started run force a review", () => {
const workflow = readFileSync(new URL("../../../.github/workflows/ai-code-review.yml", import.meta.url), "utf-8");
const skipAt = workflow.indexOf("- name: Skip PRs with nothing to review");
assert.ok(skipAt > workflow.indexOf("- name: Check out review engine"), "the check runs from the engine checkout");
for (const later of ["- name: Defer to a paired o2-enterprise PR", "- name: Install engine dependencies", "- name: Install opencode", "- name: Run AI Code Review"]) {
assert.ok(skipAt < workflow.indexOf(later), later);
}
const step = workflow.slice(skipAt, workflow.indexOf("\n\n", skipAt));
assert.match(step, /if: env\.REVIEW_RUN == 'true' && github\.event_name == 'pull_request_target'/);
assert.match(step, /reason="\$\(node engine\/scripts\/ai-review\/skip-check\.mjs "\$list" "\$commits"\)" \|\| reason=""/, "a failed check must review, not skip");
assert.match(step, /pulls\/\$PR_NUMBER\/commits/);
assert.match(step, /\.committer\.login \/\/ ""/);
assert.match(step, /\.commit\.verification\.verified \/\/ ""/);
assert.match(step, /EXPECTED_FILES: \$\{\{ steps\.pr\.outputs\.changed_files \}\}/);
assert.match(step, /EXPECTED_COMMITS: \$\{\{ steps\.pr\.outputs\.commit_count \}\}/);
assert.match(step, /\[\.filename, \(\.previous_filename \/\/ ""\)\] \| @tsv/, "one line per listed file, so the count can be checked");
assert.match(workflow, /echo "changed_files=\$\(jq -r '\.changed_files \/\/ ""'/);
assert.match(workflow, /echo "commit_count=\$\(jq -r '\.commits \/\/ ""'/);
assert.doesNotMatch(step, /set -[a-z]*e/, "set -e would end the step on a failed listing instead of reviewing");
assert.match(step, /REVIEW_RUN=false/);
assert.match(workflow, /REVIEW_MANUAL_RUN: \$\{\{ github\.event_name == 'workflow_dispatch' \}\}/);
});
test("markdown that steers the AI or CI is not docs-only", () => {
for (const p of ["README.md", "docs/guide/a.md"]) assert.equal(isDocsPath(p), true, p);
for (const p of ["scripts/ai-review/agents/coordinator.md", "AGENTS.md", "web/CLAUDE.md", ".claude/skills/x/SKILL.md", ".agents/skills/x/SKILL.md", ".agents/README.md", ".github/PULL_REQUEST_TEMPLATE.md"]) assert.equal(isDocsPath(p), false, p);
});
test("every instruction loader the sanitizer removes, at any depth and case, is reviewed rather than skipped as docs", () => {
const paths = ["CONTEXT.md", "src/Context.md", "web/agents.md", "web/.claude/skills/x/SKILL.md", "web/.agents/skills/x/SKILL.md", "web/.opencode/agents/reviewer.md", "docs/.Claude/notes.md"];
for (const name of LOADER_FILES) if (name.endsWith(".md")) paths.push(`docs/deep/${name.toUpperCase()}`);
for (const dir of LOADER_DIRS) paths.push(`docs/${dir}/guide.md`);
for (const p of paths) {
assert.equal(isSensitivePath(p), true, `the approval gate already treats ${p} as an instruction loader`);
assert.equal(isDocsPath(p), false, p);
assert.equal(nothingToReviewReason({ author: "alice", paths: [p] }), "", p);
}
for (const p of ["docs/context-menu.md", "web/agents-guide.md", "docs/claude/notes.md"]) assert.equal(isDocsPath(p), true, p);
});
skills/openobserve-instrument/SKILL.md
+313
View File
@@ -0,0 +1,313 @@
---
name: openobserve-instrument
description: Instrument an LLM or AI-agent app to send OpenTelemetry traces to OpenObserve, run one sample request, print the link to the first trace, and run the OpenObserve AI data health checks. Use when the user asks to instrument, trace, or connect an AI app to OpenObserve, or to check that its AI telemetry is complete.
---
# Instrument an AI app for OpenObserve and verify the data
You edit the user's project so every LLM call is traced to OpenObserve, prove it with one real
request, and report what is still missing. Work in the project root. Print what you change at each
step; leave the diff for the user to review and commit.
Helper script: `scripts/verify.py` next to this file (Python 3 standard library only). If you
fetched this file by URL, fetch the script from the same directory:
`https://raw.githubusercontent.com/openobserve/openobserve/main/skills/openobserve-instrument/scripts/verify.py`
and save it outside the project (for example `/tmp/openobserve-verify.py`).
## Rules
- Never print, echo, log or paste a secret value. Mention variable **names** only. Do not `cat` the
`.env`; read it with code that prints names, or let `verify.py` read it.
- Never put the token in source code, the chat, or a command line you show. The app reads it from
the environment (`load_dotenv()`).
- Do not commit. Do not change unrelated code.
- The org and stream named in the user's prompt win over values in `.env`.
## 1. Read the configuration
The user downloads a `.env` from the OpenObserve AI integration card into the project root. It holds:
| Variable | Required | Meaning |
|---|---|---|
| `OPENOBSERVE_URL` | yes | OpenObserve API base URL, e.g. `https://api.openobserve.ai` |
| `OPENOBSERVE_ORG` | yes | Organization identifier |
| `OPENOBSERVE_AUTH_TOKEN` | yes | `Basic <base64>` (or `Bearer <token>`) |
| `OPENOBSERVE_TRACES_STREAM_NAME` | no | Traces stream; `OPENOBSERVE_STREAM` is accepted too. Default `default` |
| `OPENOBSERVE_UI_URL` | no | Web UI root when it differs from `OPENOBSERVE_URL`, used for links only (links are `<root>/web/…`) |
Check which names exist, for example:
```bash
python3 -c "import re;print(', '.join(m.group(1) for l in open('.env') if (m:=re.match(r'\s*(?:export\s+)?(OPENOBSERVE_\w+)\s*=',l))))"
```
Print `✓ .env found: <names>`. If a required name is missing, stop and tell the user to download
the `.env` from the integration card (Data Sources → AI Integrations → *their SDK* → *With a coding
agent*). If the user named an org or a stream, set `OPENOBSERVE_ORG=<org>` or
`OPENOBSERVE_TRACES_STREAM_NAME=<stream>` in `.env` (the SDK reads these names). Make sure `.env`
is listed in `.gitignore`; add it if not.
## 2. Read the project
Find and print one line, e.g. `Python 3.12 · openai 1.54 · entry point app/main.py · agent name "research-agent"`:
- **Language and runtime**: `pyproject.toml`, `requirements*.txt`, `setup.py`, `package.json`, `go.mod`, …
- **LLM SDK / framework** in use: imports and dependencies (`openai`, `anthropic`, `google-genai`,
`langchain`, `crewai`, `google-adk`, `openai-agents`, `litellm`, `claude-agent-sdk`, OpenRouter
via the `openai` client, …).
- **Entry point**: the module that starts the app (`__main__`, `main.py`, CLI script in
`pyproject.toml`, server startup file).
- **Agent name**: the project or service name (`[project].name`, package name, directory name).
- **Version source**: `[project].version`, `__version__`, `package.json` `version`, or the git
short SHA (`git rev-parse --short HEAD`) when nothing else exists.
- **Session id**: whatever identifies one conversation or request in the app (chat/thread/session
id, request id). If the app has none, generate one per run with `uuid.uuid4()`.
## 3. Install
### Python, supported SDK — use the OpenObserve installer
Run the installer the integration card uses, pinned to a reviewed commit: download it to a temp
directory, check its SHA-256 against the one below, and run it from there only if they match;
never pipe a URL into a shell. A mismatch means the file is not the reviewed one: stop and tell
the user instead of running it or retrying another way. It installs the packages into the
interpreter it finds and verifies the imports. It takes the token only as a command-line argument,
which other local processes can read, so it never gets the real one: the project's `.env` already
holds the token. Give it a placeholder and point its env file into the temp directory, so it
cannot overwrite the project's `.env`. The URL is not a secret; read it from `.env` inside the
command. Do not `source` the `.env`: the token value `Basic <base64>` contains an unquoted space,
which a shell reads as a command.
```bash
env_get() { python3 -c 'import sys
for l in open(".env"):
k, _, v = l.strip().removeprefix("export ").partition("=")
if k.strip() == sys.argv[1]: print(v.strip().strip("\"'\''")); break' "$1"; }
O2_INSTALLER_RAW=https://raw.githubusercontent.com/openobserve/o2-datasource/9fdf2ca60b03a43273677a875140cfd19e29262b/ai
O2_INSTALLER_SHA256=307c0d275c8b91f270d557d41524d438f799934fe8502b513e3a22e6b2fcb0af
O2_INSTALLER_DIR="$(mktemp -d)"
mkdir "$O2_INSTALLER_DIR/frameworks"
curl -fsSL "$O2_INSTALLER_RAW/frameworks/setup.sh" -o "$O2_INSTALLER_DIR/frameworks/setup.sh"
if command -v sha256sum >/dev/null; then o2_sha256() { sha256sum "$@"; }
else o2_sha256() { shasum -a 256 "$@"; }; fi
if echo "$O2_INSTALLER_SHA256 $O2_INSTALLER_DIR/frameworks/setup.sh" | o2_sha256 -c -; then
REPO_RAW="$O2_INSTALLER_RAW" bash "$O2_INSTALLER_DIR/frameworks/setup.sh" \
--integration=<integration> \
--url="$(env_get OPENOBSERVE_URL)" \
--org=<org> \
--token="Bearer placeholder" \
--env-file="$O2_INSTALLER_DIR/.env" \
--quiet
else
echo "setup.sh does not match the reviewed SHA-256; not running it" >&2
fi
```
The SHA-256 is that of `ai/frameworks/setup.sh` at the pinned commit. Moving the pin to a newer
reviewed commit means updating both lines: `O2_INSTALLER_RAW`, and `O2_INSTALLER_SHA256` from
`gh api -H "Accept: application/vnd.github.raw"
"repos/openobserve/o2-datasource/contents/ai/frameworks/setup.sh?ref=<commit>" | shasum -a 256`.
`REPO_RAW` makes the installer fetch its helper library and integration list from the same
commit. `<integration>` is one of `openai`, `anthropic`, `gemini`, `langchain`, `crewai`,
`google-adk`, `claude-agent-sdk`, `openai-agents`, `openrouter`, `litellm`. If the project uses a
virtualenv, uv, poetry or pipenv, install into that environment instead and add the packages to
the project's dependency file. Packages and init lines per integration:
| Integration | Packages (plus `openobserve-telemetry-sdk python-dotenv`) | Instrumentor |
|---|---|---|
| openai | `opentelemetry-instrumentation-openai` | `from opentelemetry.instrumentation.openai import OpenAIInstrumentor` |
| anthropic | `opentelemetry-instrumentation-anthropic` | `from opentelemetry.instrumentation.anthropic import AnthropicInstrumentor` |
| gemini | `opentelemetry-instrumentation-google-generativeai` | `from opentelemetry.instrumentation.google_generativeai import GoogleGenerativeAiInstrumentor` |
| langchain | `opentelemetry-instrumentation-langchain` | `from opentelemetry.instrumentation.langchain import LangchainInstrumentor` |
| crewai | `openinference-instrumentation-crewai` | `from openinference.instrumentation.crewai import CrewAIInstrumentor` (set `CREWAI_TELEMETRY_OPT_OUT=true`; call `openobserve_init()` first) |
| google-adk | `openinference-instrumentation-google-adk` | `from openinference.instrumentation.google_adk import GoogleADKInstrumentor` |
| openai-agents | `openinference-instrumentation-openai-agents` | `from openinference.instrumentation.openai_agents import OpenAIAgentsInstrumentor` |
| openrouter | `openinference-instrumentation-openai` | `from openinference.instrumentation.openai import OpenAIInstrumentor` |
| litellm | `openinference-instrumentation-litellm` | `from openinference.instrumentation.litellm import LiteLLMInstrumentor` |
| claude-agent-sdk | — | wrap each `query()` in a span (see the snippet the installer saves, below) |
The installer's env file holds the placeholder token; never copy its `OPENOBSERVE_*` lines.
Merge only the integration's extra variables (for example `CREWAI_TELEMETRY_OPT_OUT`) into the
project's `.env`, keep the snippet it saved (`paste_me_<integration>.py`) if you need it, then
remove the temp directory:
```bash
python3 - "$O2_INSTALLER_DIR/.env" <<'PY'
import sys
def name(line):
return line.strip().removeprefix("export ").partition("=")[0].strip()
text = open(".env").read()
have = {name(l) for l in text.splitlines() if "=" in l}
extra = [l.strip() for l in open(sys.argv[1])
if "=" in l and not l.lstrip().startswith("#")
and not name(l).startswith("OPENOBSERVE_") and name(l) not in have]
if extra:
with open(".env", "a") as f:
f.write(("" if not text or text.endswith("\n") else "\n") + "\n".join(extra) + "\n")
print("extra env:", ", ".join(name(x) for x in extra) or "none")
PY
cat "$O2_INSTALLER_DIR/paste_me_<integration>.py"
rm -rf "$O2_INSTALLER_DIR"
```
Print `✓ setup.sh@9fdf2ca --integration=<integration>` and the extra env names (no values).
### Anything else — plain OTLP/HTTP
For other languages or SDKs without an instrumentor, use the standard OpenTelemetry SDK with the
OTLP/HTTP exporter and an OpenInference or OpenTelemetry GenAI instrumentor where one exists:
- Endpoint: `<OPENOBSERVE_URL>/api/<org>/v1/traces` (protobuf or JSON over HTTP).
- Headers: `Authorization: <OPENOBSERVE_AUTH_TOKEN>` and `stream-name: <stream>`.
- Environment form, read by every OpenTelemetry SDK:
`OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=<url>/api/<org>/v1/traces`,
`OTEL_EXPORTER_OTLP_TRACES_HEADERS=Authorization=Basic%20<base64>,stream-name=<stream>`
(write these into `.env` from the existing values; do not print them).
- If no instrumentor exists, wrap each LLM call in a span named `chat <model>` with
`gen_ai.operation.name=chat`, `gen_ai.provider.name`, `gen_ai.request.model`,
`gen_ai.response.model`, `gen_ai.usage.input_tokens` and `gen_ai.usage.output_tokens`.
## 4. Edit the entry point
Put the init lines at the very top of the entry point, **before** any import of the LLM client
(instrumentors patch the client at import time). Python with the OpenAI SDK:
```python
import os
import uuid
from contextlib import contextmanager
from typing import Optional # Python 3.9: no `str | None`
from dotenv import load_dotenv
load_dotenv() # openobserve_init() reads OPENOBSERVE_* from the environment
from opentelemetry import baggage, context
from opentelemetry.sdk.trace import SpanProcessor
from opentelemetry.instrumentation.openai import OpenAIInstrumentor
from openobserve import get_tracer_provider, openobserve_init
AGENT_NAME = "research-agent" # from step 2
AGENT_VERSION = "1.4.0" # from step 2 (read it at runtime when you can)
class ConversationIdProcessor(SpanProcessor):
"""Copy the request's conversation id onto every span started inside it."""
def on_start(self, span, parent_context=None):
conversation_id = baggage.get_baggage("gen_ai.conversation.id", parent_context)
if conversation_id:
span.set_attribute("gen_ai.conversation.id", str(conversation_id))
OpenAIInstrumentor().instrument()
openobserve_init(
agent_name=AGENT_NAME,
resource_attributes={
"service.name": AGENT_NAME,
"gen_ai.agent.name": AGENT_NAME,
"gen_ai.agent.version": AGENT_VERSION,
},
)
get_tracer_provider().add_span_processor(ConversationIdProcessor())
@contextmanager
def conversation(conversation_id: Optional[str] = None):
"""Wrap one request / chat turn: `with conversation(session_id): ...`."""
token = context.attach(
baggage.set_baggage("gen_ai.conversation.id", conversation_id or str(uuid.uuid4()))
)
try:
yield
finally:
context.detach(token)
from openai import OpenAI # client imports come after the init lines
```
Then wrap the code that handles one request or conversation turn in `with conversation(<the app's
session id>):`. These three attributes are what two of the data health checks look for:
`gen_ai.agent.name` (missing agent identity), `gen_ai.agent.version`, and
`gen_ai.conversation.id` (spans without session id). Short-lived scripts must exit normally (the
SDK flushes at exit) or call `openobserve_flush()` before exiting.
Print each edit, e.g.:
```
✓ OpenAIInstrumentor().instrument() and openobserve_init() above "from openai import"
✓ gen_ai.agent.name = research-agent, version from pyproject.toml
✓ gen_ai.conversation.id = the request's session id
```
## 5. Run one sample request
Record the start time, then run the smallest real request the app supports (a CLI flag, a test
script, or one HTTP call to the running server). Use the app's own credentials (`OPENAI_API_KEY`
etc.); never ask for them in chat.
```bash
START=$(date +%s)
python -m app.main --once "Summarize the EU AI Act disclosure rules"
```
If the app cannot make a real call (no API key), tell the user and stop; do not fake spans.
`load_dotenv()` never overrides a variable that is already set, even to an empty string; if the
app reports a missing key that `.env` has, `unset` the empty variable in the shell first.
## 6. Verify: first trace and data health
```bash
python3 <skill dir>/scripts/verify.py --env-file .env --stream <stream> --since "$START" --service <agent name> --agent <agent name>
```
`--agent` scopes the data-health checks to this app; without it they cover every app on the stream.
The script waits up to 120 s for GenAI spans (those with `gen_ai.operation.name`; HTTP or database
spans alone do not count), prints the span count and the link to the first trace, then calls
`GET <url>/api/<org>/gen_ai/data_health?stream_name=…&start_time=…&end_time=…&refresh=true&include_samples=true`
and prints each check with the page that fixes it. Show its output to the user as is:
```
✓ 14 spans arrived in 6.2 s
First trace
https://<openobserve>/web/traces/trace-details?stream=default&trace_id=4e0c91d7…&from=…&to=…&org_identifier=default
Data health
✓ Missing agent identity 0 of 14 spans
✓ Spans without session id 0 of 14 spans
✓ Zero-duration spans 0 of 14 spans
✗ Unpriced models acme-internal-llm · 3,420 tokens counted as $0
Fix in LLM Model Pricing: https://<openobserve>/web/settings/model_pricing/gaps?stream=default&org_identifier=default
3 of 4 checks pass. Fix the items above, then run the checks again.
Exit code: 1 (data-health checks found an error).
```
Exit codes: `0` GenAI spans arrived and no check is in error, `1` a check is in error, `2`
configuration problem, a server without the data health API (upgrade it), or no GenAI span before
the timeout (every check covering 0 spans counts as
none). On `2` with no GenAI spans: confirm the OpenAI/Anthropic instrumentor is installed, the init
lines run before the client import, the process exits cleanly, and the stream name matches.
What each failed check means and where to fix it:
| Check | Fix |
|---|---|
| Missing agent identity | Set `gen_ai.agent.name` (step 4) or map an existing attribute in Settings → GenAI Agent Mapping |
| Spans without session id | Wrap each request in `conversation(...)` (step 4) |
| Zero-duration spans | Spans ended before the work did — usually a span closed before a streamed response was consumed. See the AI integration docs |
| Unpriced models | Add the model's price in Settings → LLM Model Pricing (the link opens the unpriced list) |
When the user fixes something and asks to check again, rerun only the checks:
```bash
python3 <skill dir>/scripts/verify.py --env-file .env --stream <stream> --since "$START" --agent <agent name> --health-only
```
## 7. Report
End with: the files you changed, the first trace link, the health summary line, and the fixes left
for the user.
skills/openobserve-instrument/scripts/verify.py
Executable
+338
View File
@@ -0,0 +1,338 @@
#!/usr/bin/env python3
# Copyright 2026 OpenObserve Inc.
# SPDX-License-Identifier: AGPL-3.0-or-later
"""Wait for an instrumented app's first GenAI spans in OpenObserve, print the
first trace link, then run the OpenObserve AI data health checks.
Only spans with gen_ai.operation.name count: HTTP or database spans alone mean
the LLM client is not instrumented.
Python 3 standard library only. Credentials come from the project's .env (or
the process environment); the script prints variable names, never values.
Exit codes:
0 GenAI spans arrived and no health check is in error
1 GenAI spans arrived but at least one health check is in error
2 configuration problem, no GenAI span arrived before the timeout, or the server
has no data health API (upgrade OpenObserve)
"""
import argparse
import json
import os
import sys
import time
import urllib.error
import urllib.parse
import urllib.request
REQUIRED_VARS = ("OPENOBSERVE_URL", "OPENOBSERVE_ORG", "OPENOBSERVE_AUTH_TOKEN")
STREAM_VARS = ("OPENOBSERVE_TRACES_STREAM_NAME", "OPENOBSERVE_STREAM")
DOCS_URL = "https://openobserve.ai/docs/integration/ai/"
CHECK_ORDER = (
"missing_agent_identity",
"missing_session_id",
"zero_duration_spans",
"unpriced_models",
)
CHECK_LABELS = {
"missing_agent_identity": "Missing agent identity",
"missing_session_id": "Spans without session id",
"zero_duration_spans": "Zero-duration spans",
"unpriced_models": "Unpriced models",
}
STATUS_MARKS = {"pass": "✓", "warning": "!", "error": "✗"}
def parse_env_file(path):
"""Parse KEY=VALUE lines (optional `export`, quotes and comments)."""
values = {}
with open(path, encoding="utf-8") as fh:
for raw_line in fh:
line = raw_line.strip()
if not line or line.startswith("#") or "=" not in line:
continue
if line.startswith("export "):
line = line[len("export ") :].lstrip()
key, _, value = line.partition("=")
key = key.strip()
value = value.strip()
if len(value) >= 2 and value[0] == value[-1] and value[0] in ("'", '"'):
value = value[1:-1]
elif " #" in value:
value = value.split(" #", 1)[0].rstrip()
values[key] = value
return values
def load_config(env_file):
file_values = {}
if os.path.isfile(env_file):
file_values = parse_env_file(env_file)
found = [k for k in REQUIRED_VARS + STREAM_VARS + ("OPENOBSERVE_UI_URL",) if k in file_values]
print(f" .env found ({env_file}): {', '.join(found) if found else 'no OPENOBSERVE_* variables'}")
else:
print(f" No .env at {env_file}; using the process environment")
def get(name):
return os.environ.get(name) or file_values.get(name) or ""
missing = [name for name in REQUIRED_VARS if not get(name)]
if missing:
print(f"✗ Missing {', '.join(missing)}. Download the .env from the AI integration card.")
sys.exit(2)
return get
def auth_header(token):
token = token.strip()
if token.split(" ", 1)[0] in ("Basic", "Bearer"):
return token
return f"Basic {token}"
class Client:
def __init__(self, base_url, org, token):
self.base_url = base_url.rstrip("/")
self.org = org
self.auth = auth_header(token)
def request(self, method, path, query=None, body=None):
url = f"{self.base_url}/api/{urllib.parse.quote(self.org, safe='')}/{path}"
if query:
url += "?" + urllib.parse.urlencode(query)
data = json.dumps(body).encode("utf-8") if body is not None else None
req = urllib.request.Request(url, data=data, method=method)
req.add_header("Authorization", self.auth)
req.add_header("Accept", "application/json")
if data is not None:
req.add_header("Content-Type", "application/json")
with urllib.request.urlopen(req, timeout=30) as resp:
return json.loads(resp.read().decode("utf-8") or "{}")
def search_traces(self, sql, start_us, end_us, size):
body = {
"query": {
"sql": sql,
"start_time": start_us,
"end_time": end_us,
"from": 0,
"size": size,
}
}
return self.request("POST", "_search", {"type": "traces"}, body).get("hits", [])
def sql_string(value):
return "'" + value.replace("'", "''") + "'"
def first_trace(client, stream, service, start_us, end_us):
"""Return (span_count, first_trace_id) for GenAI spans in the window, or (0, None)."""
where = " WHERE gen_ai_operation_name IS NOT NULL"
if service:
where += f" AND service_name = {sql_string(service)}"
table = '"' + stream.replace('"', '""') + '"'
count_hits = client.search_traces(
f"SELECT count(*) AS spans FROM {table}{where}", start_us, end_us, 1
)
spans = int(count_hits[0].get("spans", 0)) if count_hits else 0
if spans == 0:
return 0, None
trace_hits = client.search_traces(
f"SELECT trace_id, min(_timestamp) AS first_ts FROM {table}{where} "
"GROUP BY trace_id ORDER BY first_ts ASC LIMIT 1",
start_us,
end_us,
1,
)
return spans, (trace_hits[0].get("trace_id") if trace_hits else None)
def wait_for_spans(client, stream, service, start_us, timeout_s):
started = time.monotonic()
last_error = None
while True:
end_us = int(time.time() * 1_000_000)
try:
spans, trace_id = first_trace(client, stream, service, start_us, end_us)
if spans and trace_id:
return spans, trace_id, time.monotonic() - started
except urllib.error.HTTPError as err:
# 401/403 never heal; anything else (e.g. the stream does not exist yet) may.
if err.code in (401, 403):
print(f"✗ OpenObserve rejected the credentials (HTTP {err.code}). Check OPENOBSERVE_AUTH_TOKEN.")
sys.exit(2)
last_error = f"HTTP {err.code}"
except urllib.error.URLError as err:
last_error = str(err.reason)
if time.monotonic() - started >= timeout_s:
reason = f" (last error: {last_error})" if last_error else ""
print(f"✗ No GenAI spans on stream '{stream}' after {timeout_s:.0f} s{reason}.")
print(" Only spans with gen_ai.operation.name count. Check that the OpenAI/Anthropic")
print(" instrumentor is installed, that the init lines run before the LLM client is")
print(" imported, and that the process exits cleanly or calls openobserve_flush().")
sys.exit(2)
time.sleep(3)
def trace_url(ui_url, org, stream, trace_id, start_us, end_us):
query = urllib.parse.urlencode(
{
"stream": stream,
"trace_id": trace_id,
"from": start_us,
"to": end_us,
"org_identifier": org,
}
)
return f"{ui_url}/web/traces/trace-details?{query}"
def fix_link(check_id, ui_url, org, stream):
org_q = urllib.parse.quote(org, safe="")
if check_id == "unpriced_models":
stream_q = urllib.parse.quote(stream, safe="")
return (
"Fix in LLM Model Pricing",
f"{ui_url}/web/settings/model_pricing/gaps?stream={stream_q}&org_identifier={org_q}",
)
if check_id == "missing_agent_identity":
return (
"Fix in GenAI Agent Mapping",
f"{ui_url}/web/settings/gen_ai_agent_mapping?org_identifier={org_q}",
)
return ("See", DOCS_URL)
def describe(check):
check_id = check.get("id", "")
affected = int(check.get("affected", 0))
total = int(check.get("total", 0))
if check_id == "unpriced_models" and check.get("models"):
parts = [
f"{m.get('model') or '(no model name)'} · {int(m.get('tokens', 0)):,} tokens counted as $0"
for m in check["models"]
]
if check.get("models_truncated"):
# The server lists the models with the most tokens; the totals cover all of them.
parts.append(
f"top {len(check['models'])} of {int(check.get('model_count', 0)):,} models, "
f"{int(check.get('unpriced_tokens', 0)):,} tokens in all"
)
return ", ".join(parts)
return f"{affected:,} of {plural(total, 'span')}"
def plural(count, noun):
return f"{count:,} {noun}" + ("" if count == 1 else "s")
def run_health(client, ui_url, org, stream, start_us, end_us, agent):
query = {
"stream_name": stream,
"start_time": start_us,
"end_time": end_us,
"refresh": "true",
# The script prints a sample trace per failing check.
"include_samples": "true",
}
if agent:
query["agent"] = agent
try:
result = client.request("GET", "gen_ai/data_health", query)
except urllib.error.HTTPError as err:
if err.code == 404:
print("✗ This OpenObserve server has no AI data health API yet; upgrade it to run the checks.")
return 2
print(f"✗ Data health request failed (HTTP {err.code}).")
return 2
except urllib.error.URLError as err:
print(f"✗ Data health request failed ({err.reason}).")
return 2
checks = {c.get("id"): c for c in result.get("checks", [])}
if not any(int(c.get("total", 0)) for c in checks.values()):
scope = f" for agent '{agent}'" if agent else ""
print(f"✗ No GenAI spans on stream '{stream}'{scope} in this window: every check covered 0 spans.")
if agent:
print(" Check the agent name and time window, or omit --agent to inspect the whole stream.")
else:
print(" No span carries gen_ai.operation.name, so the LLM client is not instrumented.")
print(f" Install the OpenAI/Anthropic instrumentor: {DOCS_URL}")
return 2
ordered = [checks[i] for i in CHECK_ORDER if i in checks]
ordered += [c for i, c in checks.items() if i not in CHECK_ORDER]
width = max(len(CHECK_LABELS.get(c.get("id"), c.get("id", ""))) for c in ordered) if ordered else 0
passed = 0
has_error = False
for check in ordered:
check_id = check.get("id", "")
status = check.get("status", "pass")
label = CHECK_LABELS.get(check_id, check_id)
print(f" {STATUS_MARKS.get(status, '?')} {label.ljust(width)} {describe(check)}")
if status == "pass":
passed += 1
continue
has_error = has_error or status == "error"
text, link = fix_link(check_id, ui_url, org, stream)
print(f" {' ' * width} {text}: {link}")
samples = check.get("samples") or []
if samples:
print(f" {' ' * width} Sample trace: {trace_url(ui_url, org, stream, samples[0], start_us, end_us)}")
print()
if passed == len(ordered):
print(f" All {len(ordered)} checks pass.")
else:
print(f" {passed} of {len(ordered)} checks pass. Fix the items above, then run the checks again.")
return 1 if has_error else 0
def main():
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument("--env-file", default=".env", help="path to the project .env (default: ./.env)")
parser.add_argument("--stream", help="traces stream (default: OPENOBSERVE_TRACES_STREAM_NAME, OPENOBSERVE_STREAM, then 'default')")
parser.add_argument("--since", type=float, help="Unix seconds the sample run started (default: 15 minutes ago)")
parser.add_argument("--timeout", type=float, default=120, help="seconds to wait for spans (default: 120)")
parser.add_argument("--service", help="only count spans whose service.name is this value")
parser.add_argument("--agent", help="scope the health checks to this agent name")
parser.add_argument("--ui-url", help="OpenObserve web URL when it differs from OPENOBSERVE_URL (or set OPENOBSERVE_UI_URL)")
parser.add_argument("--health-only", action="store_true", help="skip waiting for spans and rerun the health checks")
args = parser.parse_args()
get = load_config(args.env_file)
base_url = get("OPENOBSERVE_URL").rstrip("/")
org = get("OPENOBSERVE_ORG")
stream = args.stream or next((get(v) for v in STREAM_VARS if get(v)), "") or "default"
ui_url = (args.ui_url or get("OPENOBSERVE_UI_URL") or base_url).rstrip("/")
client = Client(base_url, org, get("OPENOBSERVE_AUTH_TOKEN"))
since = args.since if args.since is not None else time.time() - 15 * 60
# One minute of slack covers clock skew between this machine and the server.
start_us = int((since - 60) * 1_000_000)
print(f" Org {org} · stream {stream}")
if not args.health_only:
print(f" Waiting for GenAI spans on '{stream}' (up to {args.timeout:.0f} s)…")
spans, trace_id, elapsed = wait_for_spans(client, stream, args.service, start_us, args.timeout)
# Give the exporter's trailing batch a moment, then recount so the
# printed total and the checks cover the whole run.
time.sleep(3)
end_us = int((time.time() + 60) * 1_000_000)
spans = first_trace(client, stream, args.service, start_us, end_us)[0] or spans
print(f" ✓ {plural(spans, 'span')} arrived in {elapsed:.1f} s")
print()
print(" First trace")
print(f" {trace_url(ui_url, org, stream, trace_id, start_us, end_us)}")
print()
else:
end_us = int((time.time() + 60) * 1_000_000)
print(" Data health")
return run_health(client, ui_url, org, stream, start_us, end_us, args.agent)
if __name__ == "__main__":
sys.exit(main())
src/api/http/src/handler/http/router/mod.rs
+7 -4
View File
@@ -31,10 +31,10 @@ use openobserve_api_management::request::org_domains;
#[cfg(feature = "cpu-profiling")]
use openobserve_api_management::request::profiling;
use openobserve_api_management::request::{
alerts, announcements, authz, dashboards, db_monitoring, downtimes, folders, ingest_rejections,
kv, metrics_usage, model_pricing, organization, public_dashboards, query_history,
rum_analytics, service_accounts, short_url, slack, slos, sourcemaps, status, status_pages,
stream, synthetics, users,
alerts, announcements, authz, dashboards, db_monitoring, downtimes, folders, gen_ai_health,
ingest_rejections, kv, metrics_usage, model_pricing, organization, public_dashboards,
query_history, rum_analytics, service_accounts, short_url, slack, slos, sourcemaps, status,
status_pages, stream, synthetics, users,
};
#[cfg(feature = "cloud")]
use openobserve_api_management::request::{cloud, organization::billing_group};
@@ -1346,6 +1346,9 @@ pub fn service_routes() -> Router {
// NOTE: named routes MUST be registered before {model_id} or they match as a model ID
.route("/{org_id}/llm/models/built-in", get(model_pricing::get_built_in))
.route("/{org_id}/llm/models/refresh-built-in", post(model_pricing::refresh_built_in))
.route("/{org_id}/llm/models/unpriced", get(model_pricing::list_unpriced))
// AI data health checks (LLM Insights strip, instrument skill)
.route("/{org_id}/gen_ai/data_health", get(gen_ai_health::get_data_health))
.route("/{org_id}/llm/models/test", post(model_pricing::test_model_match))
.route("/{org_id}/llm/models/{model_id}", get(model_pricing::get).put(model_pricing::update).delete(model_pricing::delete))
src/api/http/src/handler/http/router/one_ack_tests.rs
+324 -33
View File
@@ -1488,40 +1488,331 @@ async fn openfga_edit_on_any_alert_of_an_incident_admits() {
assert!(!ack::may_edit_any_alert_with(&problem, only("none".into())).await);
}
/// Presses Resolve as an admin on the page route (UI, bulk, Slack), incident PATCH or phone key 6.
async fn resolve_on(surface: Surface, f: &IncidentFixture) {
let (status, body) = match surface {
Surface::IncidentRoute => {
let uri = format!(
"/api/v2/{}/alerts/incidents/{}/update",
org(),
f.incident_id
);
let (status, body) = call_as(
admin(),
Method::PATCH,
&uri,
Some(json!({"status": "resolved"})),
)
.await;
(status, body.to_string())
}
Surface::PageRoute => {
let uri = format!("/api/{}/oncall/responses/{}/resolve", org(), f.owner);
let (status, body) = post(admin(), &uri, json!({})).await;
(status, body.to_string())
}
Surface::PhoneKey => {
let tok = token_for(&f.owner, admin(), token::TokenPurpose::Voice).await;
post_form(
&format!("/api/v2/{}/oncall/ack/voice/{tok}", org()),
"Digits=6",
)
.await
}
Surface::AlertRoute | Surface::EmailLink => unreachable!("no Resolve on {surface:?}"),
};
assert_eq!(status, StatusCode::OK, "{surface:?}: {body}");
}
/// R7: a person's Resolve on every surface ends the problem's ack and restarts no page.
async fn every_resolve_ends_the_acknowledgement() {
for surface in [
Surface::PageRoute,
Surface::IncidentRoute,
Surface::PhoneKey,
] {
let f = incident_with_owner_and_impacted_pages(&format!("r7-{surface:?}")).await;
let ack = format!("/api/v2/{}/alerts/{}/ack", org(), f.alert_id);
let (status, body) = post(member(), &ack, json!({})).await;
assert_eq!(status, StatusCode::OK, "{surface:?}: {body}");
resolve_on(surface, &f).await;
assert_eq!(row_holder(&f.alert_id).await, None, "{surface:?}");
assert_eq!(
incident_holder(&f.incident_id).await.0,
match surface {
Surface::IncidentRoute => "resolved",
_ => "open",
},
"{surface:?}"
);
if !matches!(surface, Surface::IncidentRoute) {
let kinds: Vec<String> = infra::table::incident_events::get(org(), &f.incident_id)
.await
.unwrap()
.iter()
.map(|e| serde_json::to_value(e).unwrap()["type"].to_string())
.collect();
assert!(
kinds.contains(&"\"AckEndedByPageResolve\"".to_string())
&& !kinds.contains(&"\"Reopened\"".to_string()),
"{surface:?}: the timeline says why the ack ended: {kinds:?}"
);
assert_eq!(page(&f.owner).await.state, ResponseState::Resolved);
let left = page(&f.impacted).await;
assert_eq!(
(left.state, left.acked_by.as_deref()),
(ResponseState::Acknowledged, Some(member())),
"{surface:?}: the problem's other pages are left as they are"
);
}
}
}
/// R7 [weak]: the page closes first; a failed clear is an error and a retried Resolve clears it.
async fn a_failed_clear_is_an_error_and_a_retried_resolve_clears_it() {
for retry in [Surface::PageRoute, Surface::PhoneKey] {
let name = format!("r7-clear-fails-{retry:?}");
let alert_id = firing_alert(&name).await;
let response = alert_page(&alert_id, Some(&team(&name, &[admin()]).await)).await;
let (status, body) = post(
admin(),
&format!("/api/v2/{}/alerts/{alert_id}/ack", org()),
json!({}),
)
.await;
assert_eq!(status, StatusCode::OK, "{body}");
let uri = format!("/api/{}/oncall/responses/{}/resolve", org(), response.id);
let stuck = "r7_clear_fails";
fail_updates(stuck, "alert_acks", "alert_id", &alert_id).await;
let (status, body) = post(admin(), &uri, json!({})).await;
unfail(stuck).await;
assert_eq!(status, StatusCode::INTERNAL_SERVER_ERROR, "{body}");
assert_eq!(page(&response.id).await.state, ResponseState::Resolved);
assert_eq!(
row_holder(&alert_id).await.map(|(who, _)| who).as_deref(),
Some(admin())
);
let (status, body) = match retry {
Surface::PhoneKey => {
let tok = token_for(&response.id, admin(), token::TokenPurpose::Voice).await;
post_form(
&format!("/api/v2/{}/oncall/ack/voice/{tok}", org()),
"Digits=6",
)
.await
}
_ => {
let (status, body) = post(admin(), &uri, json!({})).await;
(status, body.to_string())
}
};
assert_eq!(status, StatusCode::OK, "{retry:?}: {body}");
assert_eq!(row_holder(&alert_id).await, None, "{retry:?}");
}
}
/// R7 [weak] on the incident: a re-sent Resolve finishes a failed clear with no second event.
async fn a_failed_incident_clear_is_finished_by_resolving_again() {
let f = incident_with_owner_and_impacted_pages("r7-incident-clear-fails").await;
let (status, body) = post(
member(),
&format!("/api/v2/{}/alerts/{}/ack", org(), f.alert_id),
json!({}),
)
.await;
assert_eq!(status, StatusCode::OK, "{body}");
let patch = format!(
"/api/v2/{}/alerts/incidents/{}/update",
org(),
f.incident_id
);
let resolve = || {
call_as(
admin(),
Method::PATCH,
&patch,
Some(json!({"status": "resolved"})),
)
};
let stuck = "r7_incident_clear_fails";
fail_updates(stuck, "alert_acks", "alert_id", &f.alert_id).await;
let (status, body) = resolve().await;
unfail(stuck).await;
assert_eq!(status, StatusCode::INTERNAL_SERVER_ERROR, "{body}");
assert_eq!(incident_holder(&f.incident_id).await.0, "resolved");
assert!(row_holder(&f.alert_id).await.is_some());
let (status, body) = resolve().await;
assert_eq!(status, StatusCode::OK, "{body}");
assert_eq!(row_holder(&f.alert_id).await, None);
let resolved = infra::table::incident_events::get(org(), &f.incident_id)
.await
.unwrap()
.iter()
.filter(|e| serde_json::to_value(e).unwrap()["type"] == "Resolved")
.count();
assert_eq!(resolved, 1, "the retry writes no second resolved event");
}
/// Re-fires the alert's one row at `at`, as a firing after a recovery does.
async fn refire(alert_id: &str, at: i64) {
let state = AlertState {
last_outcome: Some(RunOutcome::Firing),
last_outcome_at: Some(at),
since: Some(at),
firing_started_at: Some(at),
..AlertState::empty(alert_id, "")
};
alert_states::persist(
&StateUpdate {
state: Some(state),
transition: None,
},
None,
)
.await
.unwrap();
}
/// Who holds the ack of the alert's firing that started at `firing`.
async fn holder_of(alert_id: &str, firing: i64) -> Option<String> {
alert_acks::get(org(), alert_id, "")
.await
.unwrap()
.filter(|a| a.in_force_for(Some(firing)))
.map(|a| a.acked_by)
}
/// N1: a page covers firings up to its close, so a gate-held open page covers a re-fire.
async fn a_resolve_ends_the_firings_its_page_covered_up_to_its_close() {
let alert_id = firing_alert("n1-cover").await;
let page_of = alert_page(&alert_id, Some(&team("n1-cover", &[admin()]).await)).await;
let ack = format!("/api/v2/{}/alerts/{alert_id}/ack", org());
let uri = format!("/api/{}/oncall/responses/{}/resolve", org(), page_of.id);
let during_gate = page_of.opened_at + 1;
refire(&alert_id, during_gate).await;
let (status, body) = post(admin(), &ack, json!({})).await;
assert_eq!(status, StatusCode::OK, "{body}");
let (status, body) = post(admin(), &uri, json!({})).await;
assert_eq!(status, StatusCode::OK, "{body}");
assert_eq!(
holder_of(&alert_id, during_gate).await,
None,
"the open page was the re-fire's page"
);
let after_close = page(&page_of.id).await.closed_at.unwrap() + 1_000_000;
refire(&alert_id, after_close).await;
let (status, body) = post(admin(), &ack, json!({})).await;
assert_eq!(status, StatusCode::OK, "{body}");
let (status, body) = post(admin(), &uri, json!({})).await;
assert_eq!(status, StatusCode::OK, "{body}");
assert_eq!(
holder_of(&alert_id, after_close).await.as_deref(),
Some(admin()),
"a closed page does not cover a later firing"
);
}
/// N2: a re-sent incident Resolve re-runs the on-call repair a first failure only logged.
async fn a_resent_incident_resolve_repairs_its_pages() {
use sea_orm::ConnectionTrait;
let f = incident_with_owner_and_impacted_pages("n2-repair").await;
let patch = format!(
"/api/v2/{}/alerts/incidents/{}/update",
org(),
f.incident_id
);
let resolve = || {
call_as(
admin(),
Method::PATCH,
&patch,
Some(json!({"status": "resolved"})),
)
};
let recovered = || async {
oncall_responses::list_events(&f.owner)
.await
.unwrap()
.iter()
.any(|e| e.kind == ResponseEventKind::Recovery)
};
let stuck = "n2_recovery_fails";
unfail(stuck).await;
infra::db::get_orm_client_rw()
.await
.execute_unprepared(&format!(
"CREATE TRIGGER {stuck} BEFORE INSERT ON oncall_response_events \
WHEN NEW.response_id = '{}' BEGIN SELECT RAISE(ABORT, 'insert fails'); END",
f.owner
))
.await
.expect("fault");
let (status, body) = resolve().await;
unfail(stuck).await;
assert_eq!(status, StatusCode::OK, "{body}");
assert!(!recovered().await, "the first recovery failed");
let (status, body) = resolve().await;
assert_eq!(status, StatusCode::OK, "{body}");
assert!(recovered().await, "the re-sent Resolve repaired the page");
let resolved = infra::table::incident_events::get(org(), &f.incident_id)
.await
.unwrap()
.iter()
.filter(|e| serde_json::to_value(e).unwrap()["type"] == "Resolved")
.count();
assert_eq!(resolved, 1);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn one_ack_end_to_end() {
// Boxed: inline, debug builds give each step its own slot here and overflow a 2 MB thread.
let _oncall = oncall(true).await;
meta_store().await;
seed().await;
the_problem_is_the_incident_when_there_is_one_else_the_firing().await;
every_page_action_runs_the_one_rule().await;
a_teamless_page_refuses_escalate_and_snooze_and_can_be_handed_off().await;
let f = incident_with_owner_and_impacted_pages("ak8").await;
an_alert_ack_acknowledges_its_incident_and_stops_every_page(&f).await;
a_take_over_moves_every_record(&f).await;
an_unack_clears_every_record(&f).await;
the_incident_routes_admit_a_paged_member_and_refuse_a_stranger(&f).await;
a_firing_joining_an_acknowledged_incident_is_acknowledged(&f).await;
a_refused_hand_off_keeps_the_ack_and_the_page().await;
a_hand_off_clears_the_ack_and_moves_the_page().await;
a_mute_snoozes_the_open_page().await;
an_impacted_page_acknowledges_the_whole_problem().await;
a_page_whose_alert_stopped_firing_still_stops_and_restarts().await;
promote_runs_the_one_rule().await;
a_failed_page_step_is_finished_by_the_next_press().await;
a_mute_says_why_a_page_was_not_snoozed().await;
a_failed_release_stops_the_hand_off().await;
a_hand_off_to_a_person_clears_the_ack().await;
every_surface_reaches_the_one_ack().await;
the_phone_and_the_link_refuse_someone_the_rule_refuses().await;
openfga_edit_on_any_alert_of_an_incident_admits().await;
a_page_whose_alert_is_gone_can_be_taken_over().await;
a_failed_incident_read_is_an_error().await;
a_press_on_a_closed_page_answers_from_the_page().await;
a_press_on_an_older_open_page_stops_it().await;
a_leftover_page_stays_stopped_on_unack_and_take_over().await;
a_leftover_pages_team_is_a_responder().await;
a_press_that_changes_nothing_says_so().await;
an_unack_whose_restart_lost_its_trigger_still_reports_it().await;
a_failed_page_read_is_an_error().await;
Box::pin(meta_store()).await;
Box::pin(seed()).await;
Box::pin(the_problem_is_the_incident_when_there_is_one_else_the_firing()).await;
Box::pin(every_page_action_runs_the_one_rule()).await;
Box::pin(a_teamless_page_refuses_escalate_and_snooze_and_can_be_handed_off()).await;
let f = Box::pin(incident_with_owner_and_impacted_pages("ak8")).await;
Box::pin(an_alert_ack_acknowledges_its_incident_and_stops_every_page(
&f,
))
.await;
Box::pin(a_take_over_moves_every_record(&f)).await;
Box::pin(an_unack_clears_every_record(&f)).await;
Box::pin(the_incident_routes_admit_a_paged_member_and_refuse_a_stranger(&f)).await;
Box::pin(a_firing_joining_an_acknowledged_incident_is_acknowledged(
&f,
))
.await;
Box::pin(a_refused_hand_off_keeps_the_ack_and_the_page()).await;
Box::pin(a_hand_off_clears_the_ack_and_moves_the_page()).await;
Box::pin(a_mute_snoozes_the_open_page()).await;
Box::pin(an_impacted_page_acknowledges_the_whole_problem()).await;
Box::pin(a_page_whose_alert_stopped_firing_still_stops_and_restarts()).await;
Box::pin(promote_runs_the_one_rule()).await;
Box::pin(a_failed_page_step_is_finished_by_the_next_press()).await;
Box::pin(a_mute_says_why_a_page_was_not_snoozed()).await;
Box::pin(a_failed_release_stops_the_hand_off()).await;
Box::pin(a_hand_off_to_a_person_clears_the_ack()).await;
Box::pin(every_surface_reaches_the_one_ack()).await;
Box::pin(the_phone_and_the_link_refuse_someone_the_rule_refuses()).await;
Box::pin(openfga_edit_on_any_alert_of_an_incident_admits()).await;
Box::pin(a_page_whose_alert_is_gone_can_be_taken_over()).await;
Box::pin(a_failed_incident_read_is_an_error()).await;
Box::pin(a_press_on_a_closed_page_answers_from_the_page()).await;
Box::pin(a_press_on_an_older_open_page_stops_it()).await;
Box::pin(a_leftover_page_stays_stopped_on_unack_and_take_over()).await;
Box::pin(a_leftover_pages_team_is_a_responder()).await;
Box::pin(a_press_that_changes_nothing_says_so()).await;
Box::pin(an_unack_whose_restart_lost_its_trigger_still_reports_it()).await;
Box::pin(a_failed_page_read_is_an_error()).await;
Box::pin(every_resolve_ends_the_acknowledgement()).await;
Box::pin(a_failed_clear_is_an_error_and_a_retried_resolve_clears_it()).await;
Box::pin(a_failed_incident_clear_is_finished_by_resolving_again()).await;
Box::pin(a_resolve_ends_the_firings_its_page_covered_up_to_its_close()).await;
Box::pin(a_resent_incident_resolve_repairs_its_pages()).await;
}
src/api/http/src/handler/http/router/openapi.rs
+43
View File
@@ -315,6 +315,8 @@ use crate::{
gen_ai::get_agent_mapping,
gen_ai::list_scored_agents,
gen_ai::save_agent_mapping,
openobserve_api_management::request::gen_ai_health::get_data_health,
openobserve_api_management::request::model_pricing::list_unpriced,
openobserve_api_management::request::alerts::deduplication::get_semantic_groups,
openobserve_api_management::request::alerts::deduplication::preview_semantic_groups_diff,
openobserve_api_management::request::alerts::deduplication::save_semantic_groups,
@@ -607,6 +609,18 @@ use crate::{
config::meta::gen_ai::GenAiAgentMappingConfig,
gen_ai::GenAiAgentListItem,
gen_ai::GenAiAgentListResponse,
openobserve_api_management::request::gen_ai_health::DataHealthResponse,
openobserve_api_management::request::gen_ai_health::HealthCheck,
openobserve_api_management::request::gen_ai_health::ModelUsage,
openobserve_api_management::request::gen_ai_health::CheckId,
openobserve_api_management::request::gen_ai_health::CheckStatus,
openobserve_api_management::request::model_pricing::UnpricedModelsResponse,
openobserve_api_management::request::model_pricing::UnpricedModel,
openobserve_api_management::request::model_pricing::UnpricedModelMatch,
openobserve_api_management::request::gen_ai_health::UnpricedModelRow,
config::meta::model_pricing::CostStatus,
config::meta::model_pricing::PricingSource,
config::meta::model_pricing::PricingTierDefinition,
config::meta::dashboards::Dashboard,
config::meta::dashboards::v1::AxisItem,
config::meta::dashboards::v1::Dashboard,
@@ -1050,6 +1064,7 @@ use crate::{
(name = "Domain Management", description = "Allowed email domains for sign-in (enterprise)"),
(name = "EvalJobs", description = "Online evaluation jobs (enterprise)"),
(name = "Experiments", description = "LLM evaluation experiments (enterprise)"),
(name = "GenAI Settings", description = "GenAI agent mapping, the agent list and AI data health"),
(name = "Groups", description = "User group management"),
(name = "Keys", description = "Cipher key management (enterprise)"),
(name = "LLM", description = "LLM model pricing definitions"),
@@ -2156,6 +2171,34 @@ mod tests {
}
}
#[test]
fn unpriced_models_path_and_schemas_are_registered() {
let spec = serde_json::to_value(ApiDoc::openapi()).unwrap();
let op = &spec["paths"]["/api/{org_id}/llm/models/unpriced"]["get"];
assert_eq!(op["operationId"], "ListUnpricedModels", "{op}");
assert_eq!(
op["responses"]["200"]["content"]["application/json"]["schema"]["$ref"],
"#/components/schemas/UnpricedModelsResponse"
);
for code in ["403", "429"] {
assert!(op["responses"][code].is_object(), "{code}");
assert!(
spec["paths"]["/api/{org_id}/gen_ai/data_health"]["get"]["responses"][code]
.is_object(),
"data_health {code}"
);
}
let schemas = spec["components"]["schemas"].as_object().unwrap();
for name in [
"UnpricedModelsResponse",
"UnpricedModel",
"UnpricedModelMatch",
"UnpricedModelRow",
] {
assert!(schemas.contains_key(name), "schema {name} missing");
}
}
#[test]
fn fleet_reads_are_documented_with_their_403() {
let spec = serde_json::to_value(ApiDoc::openapi()).unwrap();
src/api/http/src/handler/http/router/openapi_coverage/baseline.txt
-4
View File
@@ -203,7 +203,6 @@ all ratelimit POST /api/{org_id}/functions/test
all mcp GET /api/{org_id}/gen_ai/agents
all ratelimit GET /api/{org_id}/gen_ai/agents
all summary GET /api/{org_id}/gen_ai/agents
all tag GET /api/{org_id}/gen_ai/agents
all mcp GET /api/{org_id}/groups
all ratelimit GET /api/{org_id}/groups
all mcp POST /api/{org_id}/groups
@@ -280,15 +279,12 @@ all summary PUT /api/{org_id}/service_streams/config/identity
all mcp GET /api/{org_id}/settings/gen_ai/agent_mapping
all ratelimit GET /api/{org_id}/settings/gen_ai/agent_mapping
all summary GET /api/{org_id}/settings/gen_ai/agent_mapping
all tag GET /api/{org_id}/settings/gen_ai/agent_mapping
all mcp PUT /api/{org_id}/settings/gen_ai/agent_mapping
all ratelimit PUT /api/{org_id}/settings/gen_ai/agent_mapping
all summary PUT /api/{org_id}/settings/gen_ai/agent_mapping
all tag PUT /api/{org_id}/settings/gen_ai/agent_mapping
all mcp DELETE /api/{org_id}/settings/gen_ai/agent_registry
all ratelimit DELETE /api/{org_id}/settings/gen_ai/agent_registry
all summary DELETE /api/{org_id}/settings/gen_ai/agent_registry
all tag DELETE /api/{org_id}/settings/gen_ai/agent_registry
all security POST /api/{org_id}/short
all security GET /api/{org_id}/short/{short_id}
all response POST /api/{org_id}/slack/link-requests/{request_id}/resend
src/api/http/src/router/http/mod.rs
+5
View File
@@ -897,6 +897,11 @@ mod tests {
assert!(is_querier_route("/api/default/config"));
assert!(!is_querier_route("/api/default/config/_json"));
assert!(is_querier_route("/api/default/ai/chat_stream"));
assert!(is_querier_route(
"/api/default/gen_ai/data_health?stream_name=default&start_time=1&end_time=2"
));
assert!(is_querier_route("/api/default/llm/models/unpriced"));
assert!(!is_querier_route("/api/default/llm/models"));
}
#[test]
src/api/management/src/models/datasets/mod.rs
+15 -1
View File
@@ -69,7 +69,10 @@ pub struct DatasetResponseBody {
pub name: String,
pub description: Option<String>,
pub tags: Vec<String>,
/// Per-row MVCC counter: the value an Experiment pins and a Snapshot reads.
pub global_version: i64,
/// The Dataset version to display: one per write request.
pub write_version: i64,
/// Live (non-deleted) Items, counted once per logical Item rather than once
/// per stored revision.
pub item_count: i64,
@@ -115,8 +118,14 @@ impl From<ListDatasetItemsQuery> for ListDatasetItems {
#[into_params(parameter_in = Query)]
#[serde(deny_unknown_fields)]
pub struct DatasetSnapshotRowsQuery {
/// Snapshot version to resolve. Defaults to the Dataset's current version.
/// Row-level Snapshot/pin key. When both this and `write_version` are
/// omitted, reads the Dataset's current row version.
pub version: Option<i64>,
/// Displayed write version W to pin. Resolves to the highest row version
/// among rows whose write version is at or below W (labels and row versions
/// are separate counters); cannot be combined with `version`. Use the
/// returned `snapshotVersion` as the Experiment's `datasetVersion`.
pub write_version: Option<i64>,
/// Zero-based result offset. Defaults to 0.
pub from: Option<usize>,
/// Page size from 1 through 100. Defaults to 20.
@@ -128,6 +137,7 @@ impl From<DatasetSnapshotRowsQuery> for ReadSnapshotRows {
let defaults = ReadSnapshotRows::default();
Self {
snapshot_version: value.version,
write_version: value.write_version,
from: value.from.unwrap_or(defaults.from),
size: value.size.unwrap_or(defaults.size),
}
@@ -251,6 +261,8 @@ pub struct DatasetItemResponseBody {
pub input: Value,
pub expected_output: Option<Value>,
pub global_version: i64,
/// The Dataset version of the write request that stored this row.
pub write_version: i64,
pub is_deleted: bool,
pub source: DatasetItemSourceResponseBody,
pub source_ref: Option<String>,
@@ -442,6 +454,7 @@ impl From<Dataset> for DatasetResponseBody {
description: value.description,
tags: value.tags,
global_version: value.global_version,
write_version: value.write_version,
item_count: value.item_count,
sources: value.sources.into(),
created_by: value.created_by,
@@ -558,6 +571,7 @@ impl From<DatasetItem> for DatasetItemResponseBody {
input: value.input,
expected_output: value.expected_output,
global_version: value.global_version,
write_version: value.write_version,
is_deleted: value.is_deleted,
source: value.source.into(),
source_ref: value.source_ref,
src/api/management/src/models/experiment_comparison.rs
+2 -1
View File
@@ -32,7 +32,8 @@ pub struct ExperimentComparisonQuery {
pub baseline_id: String,
pub candidate_id: String,
pub threshold: Option<f64>,
/// Comma-separated dimension IDs; omitted selects all eligible columns, empty selects none.
/// Comma-separated dimension IDs; omitted selects every eligible Score column (cost and
/// latency gate only when listed), empty selects none.
pub outcome_dimensions: Option<String>,
}
src/api/management/src/models/experiments.rs
+79 -41
View File
@@ -30,11 +30,11 @@ use openobserve_core::llm_evaluations::{
RecordBatch, RecordBatchResult,
},
results::{
ExperimentAggregateSummary, ExperimentClientScoreSummary, ExperimentProgress,
ExperimentResultScore, ExperimentResultScoreStatus, ExperimentResultSlot,
ExperimentResultTaskStatus, ExperimentScoreOutcomes, ExperimentScoreSummary,
ExperimentSkipSummary, ExperimentSlotStatus, ExperimentSummaryStatus,
ExperimentTaskOutcomes, ScoringStatus,
ExperimentAggregateSummary, ExperimentClientScoreSummary, ExperimentFailureCause,
ExperimentProgress, ExperimentResultScore, ExperimentResultScoreStatus,
ExperimentResultSlot, ExperimentResultTaskStatus, ExperimentScoreOutcomes,
ExperimentScoreSummary, ExperimentSkipSummary, ExperimentSlotStatus,
ExperimentSummaryStatus, ExperimentTaskOutcomes, ScoringStatus,
},
},
};
@@ -78,6 +78,8 @@ pub struct ExperimentResultRowBody {
pub output: Option<Value>,
pub score_summaries: Vec<ExperimentScoreSummaryBody>,
pub p50_latency_ms: Option<u64>,
/// Sample message of the row's largest failure cause (grouped by kind, else by first line).
pub error_message: Option<String>,
pub dispersion: Option<ExperimentRowDispersionBody>,
}
@@ -1022,7 +1024,10 @@ pub struct ExperimentResponseBody {
/// Resolved from the Dataset the experiment pinned, so callers do not have
/// to fetch the Dataset list just to label a run. None when it was deleted.
pub dataset_name: Option<String>,
/// The pinned `global_version`: the row cut every Slot reads.
pub dataset_version: i64,
/// The Dataset version to display for that pin: one per write request.
pub dataset_write_version: i64,
pub dataset_filter: Option<DatasetSnapshotFilterBody>,
pub task: ExperimentTaskBody,
pub scorers: Vec<PinnedExperimentScorerBody>,
@@ -1053,10 +1058,10 @@ impl ExperimentResponseBody {
self.dataset_name = dataset_name;
self
}
}
impl From<Experiment> for ExperimentResponseBody {
fn from(value: Experiment) -> Self {
/// `dataset_write_version` is what the pin shows as, resolved by the
/// handler from the Dataset's rows.
pub fn new(value: Experiment, dataset_write_version: i64) -> Self {
Self {
id: value.id,
org_id: value.org_id,
@@ -1065,6 +1070,7 @@ impl From<Experiment> for ExperimentResponseBody {
dataset_id: value.dataset_id,
dataset_name: None,
dataset_version: value.dataset_version,
dataset_write_version,
dataset_filter: value.dataset_filter.map(Into::into),
task: value.task.into(),
scorers: value.scorers.into_iter().map(Into::into).collect(),
@@ -1149,6 +1155,7 @@ pub struct ExperimentRowDetailResponseBody {
pub struct ExperimentRowSnapshotBody {
pub dataset_id: String,
pub dataset_version: i64,
pub dataset_write_version: i64,
}
#[derive(Debug, Clone, Serialize, ToSchema)]
@@ -1404,6 +1411,31 @@ pub struct ExperimentAggregateSummaryBody {
pub incomplete_task_slots: u64,
pub incomplete_score_dimensions: u64,
pub error_task_slots: u64,
/// Top 3 failure causes, largest first: errored slots grouped by kind, by first line if none.
pub failure_causes: Vec<ExperimentFailureCauseBody>,
/// Distinct failure causes across every errored slot.
pub failure_cause_count: u64,
}
#[derive(Debug, Clone, Serialize, ToSchema)]
#[serde(rename_all = "camelCase")]
pub struct ExperimentFailureCauseBody {
/// Shared failure taxonomy (`provider_auth`, `timeout`, `execution_error`, …); absent
/// when the producer did not classify the failure, including current SDK/client paths.
pub kind: Option<String>,
/// The group's most common first error line.
pub message: String,
pub slot_count: u64,
}
impl From<ExperimentFailureCause> for ExperimentFailureCauseBody {
fn from(value: ExperimentFailureCause) -> Self {
Self {
kind: value.kind,
message: value.message,
slot_count: value.slot_count,
}
}
}
impl From<ExperimentSlotStatus> for ExperimentSlotStatusBody {
@@ -1573,6 +1605,8 @@ impl From<ExperimentAggregateSummary> for ExperimentAggregateSummaryBody {
incomplete_task_slots: value.incomplete_task_slots,
incomplete_score_dimensions: value.incomplete_score_dimensions,
error_task_slots: value.error_task_slots,
failure_causes: value.failure_causes.into_iter().map(Into::into).collect(),
failure_cause_count: value.failure_cause_count,
}
}
}
@@ -1585,10 +1619,10 @@ pub struct CreateExperimentResponseBody {
pub created: bool,
}
impl From<CreateExperimentResult> for CreateExperimentResponseBody {
fn from(value: CreateExperimentResult) -> Self {
impl CreateExperimentResponseBody {
pub fn new(value: CreateExperimentResult, dataset_write_version: i64) -> Self {
Self {
experiment: value.experiment.into(),
experiment: ExperimentResponseBody::new(value.experiment, dataset_write_version),
preview: value.preview.into(),
created: value.created,
}
@@ -1769,37 +1803,40 @@ mod tests {
#[test]
fn execution_only_response_uses_explicit_execution_field_names() {
let body = ExperimentResponseBody::from(Experiment {
id: "experiment-1".to_string(),
org_id: "org-1".to_string(),
name: "Contract".to_string(),
description: None,
dataset_id: "dataset-1".to_string(),
dataset_version: 1,
dataset_filter: None,
task: ExperimentTaskConfig::Sdk {
task_fingerprint: "sha256:test".to_string(),
config: serde_json::json!({}),
let body = ExperimentResponseBody::new(
Experiment {
id: "experiment-1".to_string(),
org_id: "org-1".to_string(),
name: "Contract".to_string(),
description: None,
dataset_id: "dataset-1".to_string(),
dataset_version: 1,
dataset_filter: None,
task: ExperimentTaskConfig::Sdk {
task_fingerprint: "sha256:test".to_string(),
config: serde_json::json!({}),
},
scorers: Vec::new(),
trial_count: 1,
metadata: None,
status: ExperimentStatus::Completed,
status_reason: Some("done".to_string()),
deadline_at: 1,
completed_at: Some(2),
lifecycle_version: 1,
retry_count: 0,
scores_settled_at: Some(2),
idempotency_key: None,
is_baseline: false,
prompt_id: Some("prompt-1".to_string()),
prompt_name: Some("Support answer".to_string()),
prompt_version: Some(3),
prompt_content_hash: Some("sha256:content".to_string()),
created_by: "owner@example.com".to_string(),
created_at: 1,
},
scorers: Vec::new(),
trial_count: 1,
metadata: None,
status: ExperimentStatus::Completed,
status_reason: Some("done".to_string()),
deadline_at: 1,
completed_at: Some(2),
lifecycle_version: 1,
retry_count: 0,
scores_settled_at: Some(2),
idempotency_key: None,
is_baseline: false,
prompt_id: Some("prompt-1".to_string()),
prompt_name: Some("Support answer".to_string()),
prompt_version: Some(3),
prompt_content_hash: Some("sha256:content".to_string()),
created_by: "owner@example.com".to_string(),
created_at: 1,
});
1,
);
let value = serde_json::to_value(body).unwrap();
@@ -1831,6 +1868,7 @@ mod tests {
dataset_id: "dataset-1".to_string(),
dataset_name: Some("Dataset".to_string()),
dataset_version: 1,
dataset_write_version: 1,
dataset_filter: None,
task: ExperimentTaskBody::Sdk {
task_fingerprint: "sha256:test".to_string(),
src/api/management/src/models/scorers/mod.rs
+72 -1
View File
@@ -32,7 +32,11 @@ pub struct LlmJudgeScorerParams {
pub provider_id: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub model: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
#[serde(
default,
skip_serializing_if = "Option::is_none",
deserialize_with = "deserialize_optional_f64"
)]
pub temperature: Option<f64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub max_tokens: Option<u32>,
@@ -46,6 +50,25 @@ pub struct LlmJudgeScorerParams {
pub extra_metadata_fields: Option<Vec<ExtraMetadataField>>,
}
/// Deserializes an optional float through `serde_json::Number`.
///
/// The workspace enables `serde_json/arbitrary_precision`, under which a
/// non-integer number is handed to visitors as a private map. Inside an
/// internally tagged enum (`#[serde(tag = "type")]`) that map is buffered and
/// a plain `f64` field rejects it with `invalid type: map, expected f64`.
/// `Number` understands both the buffered map and a plain number.
fn deserialize_optional_f64<'de, D>(deserializer: D) -> Result<Option<f64>, D::Error>
where
D: serde::Deserializer<'de>,
{
Option::<serde_json::Number>::deserialize(deserializer)?
.map(|n| {
n.as_f64()
.ok_or_else(|| serde::de::Error::custom(format!("{n} is not a valid f64")))
})
.transpose()
}
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, ToSchema)]
#[serde(rename_all = "snake_case", deny_unknown_fields)]
pub struct RemoteScorerParams {
@@ -748,6 +771,54 @@ mod tests {
);
}
/// Issue #3143: with `serde_json/arbitrary_precision`, a float inside the
/// internally tagged scorer config used to fail with
/// `invalid type: map, expected f64`. Parse from raw JSON, as the HTTP
/// extractor does.
#[test]
fn test_llm_judge_scorer_accepts_float_temperature() {
let scorer_json = r#"{
"type": "llm_judge",
"producesScoreConfigVersion": 2,
"template": "Judge {{input}}",
"params": {
"provider_id": "p1",
"temperature": 0.1,
"max_tokens": 256,
"timeout_ms": 30000
}
}"#;
let create: ScorerRequestBody =
serde_json::from_str(&format!(r#"{{"name": "judge", "scorer": {scorer_json}}}"#))
.unwrap();
let scorer = infra::table::scorers::Scorer::from(create);
assert_eq!(scorer.params["temperature"].as_f64(), Some(0.1));
assert_eq!(scorer.params["max_tokens"].as_u64(), Some(256));
let stored: LlmJudgeScorerParams = serde_json::from_value(scorer.params).unwrap();
assert_eq!(stored.temperature, Some(0.1));
let update: ScorerUpdateRequestBody =
serde_json::from_str(&format!(r#"{{"scorer": {scorer_json}}}"#)).unwrap();
let ScorerUpdateConfig::LlmJudge(update) = update.scorer else {
panic!("expected llm_judge");
};
assert_eq!(update.params.temperature, Some(0.1));
let test: ScorerTestRequestBody = serde_json::from_str(&format!(
r#"{{"name": "judge", "scorer": {scorer_json}, "inputVariables": {{}}}}"#
))
.unwrap();
let ScorerCreateConfig::LlmJudge(test) = test.scorer else {
panic!("expected llm_judge");
};
assert_eq!(test.params.temperature, Some(0.1));
let omitted: LlmJudgeScorerParams =
serde_json::from_str(r#"{"provider_id": "p1"}"#).unwrap();
assert_eq!(omitted.temperature, None);
}
#[test]
fn test_remote_scorer_request_deserialize_rejects_response_paths() {
let result: Result<ScorerRequestBody, _> = serde_json::from_value(serde_json::json!({
src/api/management/src/request/alerts/ack.rs
+2 -1
View File
@@ -300,7 +300,7 @@ pub(crate) async fn act_on(
}
}
/// The verb's answer as JSON: an `AckResponse`, or an `UnackResponse` for an unack.
/// The verb's answer as JSON: an `AckResponse`, or an `UnackResponse` for a verb that clears.
pub(crate) async fn act(
problem: &Problem,
actor: &str,
@@ -314,6 +314,7 @@ pub(crate) async fn act(
Verb::HandOff => {
serde_json::to_value(ack::release_for_hand_off(problem, actor, source).await?)
}
Verb::Resolve => serde_json::to_value(ack::resolve(problem, actor, source, None).await?),
};
body.map_err(|e| AckError::Internal(e.to_string()))
}
src/api/management/src/request/alerts/incidents.rs
+30
View File
@@ -285,6 +285,9 @@ pub async fn update_incident(
)
.await
{
Ok(incident) if status == IncidentStatus::Resolved => {
end_ack_after_resolve(org_id, incident_id, &user_id, incident).await
}
Ok(incident) => MetaHttpResponse::json(incident),
Err(e) => {
if e.to_string().contains("not found") {
@@ -755,6 +758,33 @@ fn json_status_response(status: axum::http::StatusCode, body: serde_json::Value)
.unwrap()
}
/// A person resolved the incident: end the ack its alerts carry, then answer with the incident.
#[cfg(feature = "enterprise")]
async fn end_ack_after_resolve(
org_id: String,
incident_id: String,
actor: &str,
incident: impl Serialize,
) -> Response {
use openobserve_core::alerts::ack::{self, Problem};
let problem = Problem::Incident {
org_id,
incident_id,
};
match ack::resolve(
&problem,
actor,
config::meta::alerts::ack::AckSource::Incident,
None,
)
.await
{
Ok(_) => MetaHttpResponse::json(incident),
Err(e) => crate::request::alerts::ack::error_response(e),
}
}
#[cfg(feature = "enterprise")]
/// CancelIncidentRca
#[utoipa::path(
src/api/management/src/request/datasets/mod.rs
+1 -1
View File
@@ -134,7 +134,7 @@ pub async fn list_dataset_items(
tag = "Datasets",
operation_id = "GetDatasetSnapshotRows",
summary = "Read the row set of a pinned Dataset Snapshot",
description = "Resolves the Dataset's MVCC history at `version` and returns one page of the live rows that Snapshot contains. Omitting `version` reads the current Snapshot and answers with the version it resolved. This is the read an Experiment pins at creation, so it is also how a client inspects the exact rows a historical Experiment ran against.",
description = "Resolves the Dataset's MVCC history at row-level `version` and returns one page of live rows. Alternatively, `write_version` W resolves to the highest row version among rows whose write version is at or below W; write versions and row versions are separate counters. Omitting both reads the current Snapshot. The response's `snapshotVersion` is always a row-level pin key: use it as the Experiment's `datasetVersion`. Existing pins and `version` reads retain their original semantics.",
security(("Authorization" = [])),
params(
("org_id" = String, Path, description = "Organization name"),
src/api/management/src/request/experiments.rs
+65 -18
View File
@@ -90,6 +90,29 @@ impl ScorerSelection<'_> {
}
}
/// What an Experiment's pinned `global_version` shows as. When the Dataset's
/// rows cannot be read the pin shows as itself, as it does for a deleted
/// Dataset; the response describes an action that already happened.
async fn pinned_write_version(org_id: &str, dataset_id: &str, pin: i64) -> i64 {
datasets::pinned_write_version(org_id, dataset_id, pin)
.await
.unwrap_or_else(|error| {
log::error!(
"[Experiment] failed to resolve the write version of Dataset {dataset_id} at {pin}: {error}"
);
pin
})
}
async fn experiment_body(
org_id: &str,
experiment: experiments::Experiment,
) -> ExperimentResponseBody {
let shown =
pinned_write_version(org_id, &experiment.dataset_id, experiment.dataset_version).await;
ExperimentResponseBody::new(experiment, shown)
}
fn experiment_error_response(error: ExperimentError) -> Response {
match error {
ExperimentError::Database(error) => {
@@ -333,6 +356,11 @@ fn experiment_result_rows(
score_configs,
),
p50_latency_ms: aggregate.p50_latency_ms,
error_message: aggregate
.failure_causes
.into_iter()
.next()
.map(|cause| cause.message),
dispersion: dispersions.remove(&first.row_id).map(Into::into),
})
})
@@ -602,7 +630,13 @@ pub async fn create_experiment(
match experiments::create(&org_id, &user.user_id, input, &canonical_request).await {
Ok(result) => {
set_ownership(&org_id, "experiments", Authz::new(&result.experiment.id)).await;
MetaHttpResponse::json(CreateExperimentResponseBody::from(result))
let shown = pinned_write_version(
&org_id,
&result.experiment.dataset_id,
result.experiment.dataset_version,
)
.await;
MetaHttpResponse::json(CreateExperimentResponseBody::new(result, shown))
}
Err(error) => experiment_error_response(error),
}
@@ -636,15 +670,8 @@ pub async fn list_experiments(
Ok(list) => list,
Err(error) => return MetaHttpResponse::forbidden(error.to_string()),
};
// One Dataset query for the whole list — a lookup per experiment would be N+1.
let dataset_names: std::collections::HashMap<String, String> = datasets::list(&org_id)
.await
.map(|list| {
list.into_iter()
.map(|dataset| (dataset.id, dataset.name))
.collect()
})
.unwrap_or_default();
// One names-only Dataset query for the whole list — no per-experiment lookups or aggregates.
let dataset_names = datasets::list_names(&org_id).await.unwrap_or_default();
let dataset_id = query
.dataset_id
.as_deref()
@@ -700,14 +727,27 @@ pub async fn list_experiments(
} else {
Vec::new()
};
// One query for every pin in the list, not one per experiment.
let pins: Vec<(String, i64)> = experiments
.iter()
.map(|experiment| (experiment.dataset_id.clone(), experiment.dataset_version))
.collect();
let shown_versions = datasets::pinned_write_versions(&org_id, &pins)
.await
.unwrap_or_else(|error| {
log::error!("[Experiment] failed to resolve pinned Dataset write versions: {error}");
HashMap::new()
});
MetaHttpResponse::json(ListExperimentsResponseBody {
list: experiments
.into_iter()
.map(|experiment| {
let id = experiment.id.clone();
let dataset_name = dataset_names.get(&experiment.dataset_id).cloned();
let pin = (experiment.dataset_id.clone(), experiment.dataset_version);
let shown = shown_versions.get(&pin).copied().unwrap_or(pin.1);
ExperimentViewResponseBody {
experiment: ExperimentResponseBody::from(experiment)
experiment: ExperimentResponseBody::new(experiment, shown)
.with_dataset_name(dataset_name),
summary: include_summary.then(|| {
experiment_summary_body(
@@ -877,9 +917,12 @@ pub async fn get_experiment(
summaries.remove(&experiment.id).unwrap_or_default(),
&score_configs,
);
let shown =
pinned_write_version(&org_id, &experiment.dataset_id, experiment.dataset_version).await;
MetaHttpResponse::json(ExperimentDetailResponseBody {
experiment: ExperimentViewResponseBody {
experiment: ExperimentResponseBody::from(experiment).with_dataset_name(dataset_name),
experiment: ExperimentResponseBody::new(experiment, shown)
.with_dataset_name(dataset_name),
summary: Some(summary),
},
preview: preview.into(),
@@ -1231,11 +1274,14 @@ pub async fn get_experiment_row(
.slots
.first()
.expect("a resolved Experiment row always contains at least one trial");
let dataset_write_version =
pinned_write_version(&org_id, &experiment.dataset_id, experiment.dataset_version).await;
let response = ExperimentRowDetailResponseBody {
experiment_id: experiment.id,
snapshot: ExperimentRowSnapshotBody {
dataset_id: experiment.dataset_id,
dataset_version: experiment.dataset_version,
dataset_write_version,
},
navigation: ExperimentRowNavigationBody {
row_index: row.row_index,
@@ -1279,7 +1325,7 @@ pub async fn get_experiment_row(
)]
pub async fn cancel_experiment(Path((org_id, experiment_id)): Path<(String, String)>) -> Response {
match experiments::cancel(&org_id, &experiment_id).await {
Ok(experiment) => MetaHttpResponse::json(ExperimentResponseBody::from(experiment)),
Ok(experiment) => MetaHttpResponse::json(experiment_body(&org_id, experiment).await),
Err(error) => experiment_error_response(error),
}
}
@@ -1314,7 +1360,7 @@ pub async fn set_experiment_baseline(
}
match baseline::set_baseline(&org_id, &experiment_id, &user.user_id).await {
Ok(change) => MetaHttpResponse::json(ExperimentBaselineResponseBody {
experiment: ExperimentResponseBody::from(change.experiment),
experiment: experiment_body(&org_id, change.experiment).await,
previous_baseline_id: change.previous_baseline_id,
}),
Err(error) => experiment_error_response(error),
@@ -1349,7 +1395,7 @@ pub async fn clear_experiment_baseline(
return response;
}
match baseline::clear_baseline(&org_id, &experiment_id, &user.user_id).await {
Ok(experiment) => MetaHttpResponse::json(ExperimentResponseBody::from(experiment)),
Ok(experiment) => MetaHttpResponse::json(experiment_body(&org_id, experiment).await),
Err(error) => experiment_error_response(error),
}
}
@@ -1433,7 +1479,7 @@ pub async fn retry_experiment(
)
.await
{
Ok(experiment) => MetaHttpResponse::json(ExperimentResponseBody::from(experiment)),
Ok(experiment) => MetaHttpResponse::json(experiment_body(&org_id, experiment).await),
Err(error) => experiment_error_response(error),
}
}
@@ -1563,7 +1609,7 @@ pub async fn clone_experiment(
match experiments::clone_sealed(&org_id, &experiment_id, &user.user_id, overrides).await {
Ok(experiment) => {
set_ownership(&org_id, "experiments", Authz::new(&experiment.id)).await;
MetaHttpResponse::json(ExperimentResponseBody::from(experiment))
MetaHttpResponse::json(experiment_body(&org_id, experiment).await)
}
Err(error) => experiment_error_response(error),
}
@@ -1705,7 +1751,7 @@ pub async fn finalize_experiment(
return response;
}
match ingest::finalize(&org_id, &experiment_id).await {
Ok(experiment) => MetaHttpResponse::json(ExperimentResponseBody::from(experiment)),
Ok(experiment) => MetaHttpResponse::json(experiment_body(&org_id, experiment).await),
Err(error) => ingest_error_response(error),
}
}
@@ -1749,6 +1795,7 @@ mod tests {
output: None,
score_summaries: Vec::new(),
p50_latency_ms: None,
error_message: None,
dispersion: Some(crate::models::experiments::ExperimentRowDispersionBody {
row_id,
logical_id: format!("case-{row_index}"),
src/api/management/src/request/gen_ai_health.rs
+1716
View File
File diff suppressed because it is too large Load Diff
src/api/management/src/request/mod.rs
+1
View File
@@ -45,6 +45,7 @@ pub mod fleet;
#[allow(deprecated)]
pub mod folders;
pub mod gen_ai;
pub mod gen_ai_health;
pub mod ingest_rejections;
pub mod keys;
pub mod kv;
src/api/management/src/request/model_pricing.rs
+277 -2
View File
@@ -13,7 +13,7 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
use std::collections::{BTreeSet, HashMap};
use std::collections::{BTreeSet, HashMap, HashSet};
use axum::{
Json,
@@ -33,7 +33,9 @@ use {
openobserve_core::auth::{UserEmail, check_permissions},
};
use crate::common::meta::http::HttpResponse as MetaHttpResponse;
use crate::{
common::meta::http::HttpResponse as MetaHttpResponse, request::gen_ai_health::MAX_AGENT_NAMES,
};
fn source_priority(source: &PricingSource) -> u8 {
match source {
@@ -449,6 +451,8 @@ pub async fn delete(
pub struct BuiltInQuery {
#[serde(default)]
pub search: String,
#[serde(default)]
pub metadata_only: bool,
}
/// A model pricing entry from the community/built-in source.
@@ -468,7 +472,10 @@ pub struct BuiltInModelPricingEntry {
pub struct BuiltInModelPricingResponse {
pub models: Vec<BuiltInModelPricingEntry>,
pub source_url: String,
/// Unix seconds of this server's last successful online sync; 0 = none since startup.
pub last_updated: i64,
/// Date of the seed price list compiled into this binary.
pub seed_date: &'static str,
}
/// GetBuiltInModelPricing
@@ -486,6 +493,7 @@ pub struct BuiltInModelPricingResponse {
params(
("org_id" = String, Path, description = "Organization name"),
("search" = Option<String>, Query, description = "Filter by name"),
("metadata_only" = Option<bool>, Query, description = "Return seed/sync metadata without loading the catalogue"),
),
responses(
(status = 200, description = "Success", content_type = "application/json", body = Object),
@@ -519,6 +527,14 @@ pub async fn get_built_in(
}
let source_url = config::get_config().common.model_pricing_source_url.clone();
if query.metadata_only {
return MetaHttpResponse::json(BuiltInModelPricingResponse {
models: Vec::new(),
source_url,
last_updated: openobserve_builtins::model_pricing::last_sync_timestamp(),
seed_date: openobserve_builtins::model_pricing::SEED_DATE,
});
}
// Read from DB (synced by background job) instead of making a live HTTP call.
let all_built_in = match model_pricing::list(BUILT_IN_ORG).await {
@@ -553,6 +569,7 @@ pub async fn get_built_in(
models,
source_url,
last_updated,
seed_date: openobserve_builtins::model_pricing::SEED_DATE,
})
}
@@ -623,6 +640,180 @@ pub async fn refresh_built_in(
}
}
/// The pricing definition a listed model resolves to today.
#[derive(Debug, Serialize, utoipa::ToSchema)]
pub struct UnpricedModelMatch {
pub name: String,
pub match_pattern: String,
pub source: PricingSource,
pub tiers: Vec<PricingTierDefinition>,
}
#[derive(Debug, Serialize, utoipa::ToSchema)]
pub struct UnpricedModel {
#[serde(flatten)]
pub row: crate::request::gen_ai_health::UnpricedModelRow,
/// Null when no definition matches the model today.
pub matched: Option<UnpricedModelMatch>,
/// An exact definition covers new spans; these older spans retain their recorded cost.
pub priced_now: bool,
}
#[derive(Debug, Serialize, utoipa::ToSchema)]
pub struct UnpricedModelsResponse {
/// Models with recorded pricing gaps, including those priced since ingestion.
pub models: Vec<UnpricedModel>,
/// Distinct models ingest marked in the window, uncapped and before today's prices are
/// applied; spans without a model name count as one.
pub model_count: i64,
/// Tokens on every span ingest marked in the window.
pub tokens: i64,
/// The server listed only the models with the most tokens; `model_count` has them all.
pub models_truncated: bool,
}
/// Joins recorded pricing gaps with today's definitions without hiding historical gaps.
fn resolve_unpriced(
rows: Vec<crate::request::gen_ai_health::UnpricedModelRow>,
entries: &[CachedModelPricing],
now_micros: i64,
) -> Vec<UnpricedModel> {
let mut by_model: HashMap<Option<String>, UnpricedModel> = HashMap::new();
let mut order = Vec::new();
for row in rows {
let resolved = row
.model
.as_deref()
.and_then(|m| model_pricing::find_pricing_sync_at(entries, m, Some(now_micros)));
let priced_now = crate::request::gen_ai_health::priced_now(resolved.as_ref());
// A model seen both unpriced and approximate (a price was added mid-window) is one row.
if let Some(existing) = by_model.get_mut(&row.model) {
existing.row.spans += row.spans;
existing.row.tokens += row.tokens;
existing.row.cost += row.cost;
// Counts per status overlap, so the larger is a floor for the merged distinct count.
existing.row.agent_count = existing.row.agent_count.max(row.agent_count);
let mut seen: HashSet<String> = existing.row.agents.iter().cloned().collect();
for agent in row.agents {
if existing.row.agents.len() >= MAX_AGENT_NAMES {
break;
}
if seen.insert(agent.clone()) {
existing.row.agents.push(agent);
}
}
existing.row.agent_count = existing
.row
.agent_count
.max(existing.row.agents.len() as i64);
continue;
}
let matched = resolved.map(|r| UnpricedModelMatch {
name: r.definition.name,
match_pattern: r.definition.match_pattern,
source: r.definition.source,
tiers: r.definition.tiers,
});
let mut row = row;
if !priced_now {
row.status = if matched.is_some() {
config::meta::model_pricing::CostStatus::Approximate
} else {
config::meta::model_pricing::CostStatus::Unpriced
};
}
order.push(row.model.clone());
by_model.insert(
row.model.clone(),
UnpricedModel {
row,
matched,
priced_now,
},
);
}
order
.into_iter()
.filter_map(|m| by_model.remove(&m))
.collect()
}
/// ListUnpricedModels
///
/// #{"ratelimit_module":"ModelPricing", "ratelimit_module_operation":"list"}#
#[utoipa::path(
get,
path = "/{org_id}/llm/models/unpriced",
context_path = "/api",
tag = "LLM",
operation_id = "ListUnpricedModels",
summary = "List models seen in traces with no price or an approximate one",
description = "Models in the stream's spans that ingest marked unpriced or approximately priced during the window, with spans, tokens and recorded cost. Historical gaps remain after adding an exact price and are marked priced_now; old costs are not backfilled. Spans with tokens but no model name are listed as one row with a null model.",
security(("Authorization" = [])),
params(
("org_id" = String, Path, description = "Organization name"),
crate::request::gen_ai_health::UnpricedModelsQuery,
),
responses(
(status = 200, description = "Success", body = UnpricedModelsResponse),
(status = 400, description = "Invalid query"),
(status = 403, description = "Forbidden"),
(status = 429, description = "Search not allowed for the organization"),
(status = 500, description = "Internal server error"),
),
extensions(
("x-o2-ratelimit" = json!({"module": "LLM", "operation": "list"})),
("x-o2-mcp" = json!({"enabled": false}))
),
)]
pub async fn list_unpriced(
Path(org_id): Path<String>,
openobserve_api_common::extractors::Headers(user_email): openobserve_api_common::extractors::Headers<
openobserve_core::auth::UserEmail,
>,
Query(query): Query<crate::request::gen_ai_health::UnpricedModelsQuery>,
) -> Response {
let user_id = &user_email.user_id;
#[cfg(feature = "enterprise")]
if !check_permissions(
&org_id,
&org_id,
user_id,
"model_pricing",
"LIST",
None,
true,
false,
false,
)
.await
{
return MetaHttpResponse::forbidden("Unauthorized Access");
}
if let Err(resp) =
crate::request::gen_ai_health::authorize_unpriced(&org_id, user_id, &query).await
{
return resp;
}
match crate::request::gen_ai_health::unpriced_models(&org_id, user_id, &query).await {
Ok(listing) => {
let entries = model_pricing::get_org_pricing_entries(&org_id);
let models =
resolve_unpriced(listing.rows, &entries, config::utils::time::now_micros());
MetaHttpResponse::json(UnpricedModelsResponse {
models,
model_count: listing.model_count,
tokens: listing.tokens,
models_truncated: listing.truncated,
})
}
Err(e) => {
log::error!("[model_pricing] unpriced models for {org_id}: {e}");
MetaHttpResponse::internal_error("Failed to list unpriced models")
}
}
}
/// Strip leading inline flag groups like `(?i)`, `(?ms)`, `(?-i)` from a regex pattern.
fn strip_leading_flags(s: &str) -> &str {
let mut rest = s;
@@ -1489,6 +1680,90 @@ mod tests {
]
}
fn unpriced_row(
model: &str,
status: config::meta::model_pricing::CostStatus,
tokens: i64,
) -> crate::request::gen_ai_health::UnpricedModelRow {
crate::request::gen_ai_health::UnpricedModelRow {
model: Some(model.to_string()),
status,
spans: 1,
tokens,
cost: 0.0,
agents: vec!["a".to_string()],
agent_count: 1,
}
}
#[test]
fn test_resolve_unpriced_keeps_historical_gaps_and_merges_statuses() {
use config::meta::model_pricing::CostStatus;
let mut entries = built_in_gpt_4o();
// Approximate today: `gpt-5.4` covers only part of `gpt-5.4-mini`.
entries.push(entry("GPT-5.4", r"(?i)gpt-5\.4", PricingSource::BuiltIn));
let rows = vec![
unpriced_row("acme-internal-llm", CostStatus::Unpriced, 100),
// Now exactly priced, but the historical gap must remain visible.
unpriced_row("gpt-4o", CostStatus::Unpriced, 50),
unpriced_row("gpt-5.4-mini", CostStatus::Approximate, 30),
// Seen unpriced earlier in the window, approximate since: one row.
unpriced_row("gpt-5.4-mini", CostStatus::Unpriced, 20),
];
let out = resolve_unpriced(rows, &entries, 0);
let models: Vec<_> = out.iter().map(|m| m.row.model.as_deref()).collect();
assert_eq!(
models,
[
Some("acme-internal-llm"),
Some("gpt-4o"),
Some("gpt-5.4-mini")
]
);
assert!(out[0].matched.is_none());
assert!(!out[0].priced_now);
assert!(out[1].priced_now);
assert_eq!(out[2].row.status, CostStatus::Approximate);
assert!(!out[2].priced_now);
assert_eq!(out[2].row.tokens, 50);
assert_eq!(out[2].matched.as_ref().unwrap().name, "GPT-5.4");
}
#[test]
fn test_resolve_unpriced_keeps_spans_without_a_model_name() {
use config::meta::model_pricing::CostStatus;
let mut no_model = unpriced_row("x", CostStatus::Unpriced, 70);
no_model.model = None;
let mut no_model_again = no_model.clone();
no_model_again.tokens = 30;
let out = resolve_unpriced(vec![no_model, no_model_again], &built_in_gpt_4o(), 0);
assert_eq!(out.len(), 1);
assert_eq!(out[0].row.model, None);
assert_eq!(out[0].row.status, CostStatus::Unpriced);
assert_eq!(out[0].row.tokens, 100);
assert!(out[0].matched.is_none());
// The model is null on the wire, not an empty name.
let wire = config::utils::json::to_value(&out[0]).unwrap();
assert_eq!(wire.get("model"), Some(&config::utils::json::Value::Null));
}
#[test]
fn test_resolve_unpriced_caps_merged_agent_names_and_keeps_the_count() {
use config::meta::model_pricing::CostStatus;
let names = |range: std::ops::Range<usize>| range.map(|i| format!("agent-{i}")).collect();
let mut unpriced = unpriced_row("m", CostStatus::Unpriced, 10);
unpriced.agents = names(0..8);
unpriced.agent_count = 25;
let mut approximate = unpriced_row("m", CostStatus::Approximate, 10);
approximate.agents = names(4..10);
approximate.agent_count = 6;
let out = resolve_unpriced(vec![unpriced, approximate], &[], 0);
let agents = &out[0].row.agents;
assert_eq!(agents.len(), MAX_AGENT_NAMES);
assert_eq!(agents.iter().collect::<HashSet<_>>().len(), MAX_AGENT_NAMES);
assert_eq!(out[0].row.agent_count, 25);
}
#[test]
fn test_captured_models_prefix_rule_also_catches_other_models() {
let captured = captured_models(&built_in_gpt_4o(), &org_candidate(r"(?i)^gpt-4o"));
src/api/management/src/request/oncall/mod.rs
+69 -8
View File
@@ -2277,7 +2277,12 @@ async fn act_by_phone(
PhoneAction::Acknowledge => {
service::phone_ack_outcome(&org_id, &response_id, &actor).await
}
PhoneAction::Resolve | PhoneAction::Escalate => {
PhoneAction::Resolve => {
// A retried press 6 finishes a clear the first press could not.
end_ack_by_phone(&org_id, &response_id, problem.as_ref(), &actor).await?;
Ok(o2_enterprise::enterprise::oncall::notify::PressOutcome::Closed)
}
PhoneAction::Escalate => {
Ok(o2_enterprise::enterprise::oncall::notify::PressOutcome::Closed)
}
};
@@ -2301,11 +2306,46 @@ async fn act_by_phone(
}
service::phone_ack_outcome(&org_id, &response_id, &actor).await
}
PhoneAction::Resolve => service::phone_resolve(&org_id, &response_id, &actor).await,
PhoneAction::Resolve => {
let outcome = service::phone_resolve(&org_id, &response_id, &actor).await?;
end_ack_by_phone(&org_id, &response_id, problem.as_ref(), &actor).await?;
Ok(outcome)
}
PhoneAction::Escalate => service::phone_escalate(&org_id, &response_id, &actor).await,
}
}
/// Press 6 ends the problem's ack once the page is closed; a page whose alert is gone has none.
#[cfg(feature = "enterprise")]
async fn end_ack_by_phone(
org_id: &str,
response_id: &str,
problem: Option<&openobserve_core::alerts::ack::Problem>,
actor: &str,
) -> Result<(), anyhow::Error> {
use openobserve_core::alerts::ack;
let Some(p) = problem else {
return Ok(());
};
let Some(page) = infra::table::oncall_responses::get(org_id, response_id).await? else {
return Ok(());
};
let closed_at = page
.closed_at
.unwrap_or_else(config::utils::time::now_micros);
let cover = ack::page_cover(&page.subject, closed_at);
ack::resolve(
p,
actor,
config::meta::alerts::ack::AckSource::Oncall,
cover,
)
.await
.map(|_| ())
.map_err(|e| anyhow::anyhow!("end the ack after a resolve by phone: {e:?}"))
}
/// TwiML refusal for an inactive or unreadable org, as `inactive_org_response` refuses email acks.
#[cfg(feature = "enterprise")]
fn voice_status_refusal(
@@ -2575,18 +2615,25 @@ pub async fn get_response(
pub async fn resolve_response(
Path((org_id, response_id)): Path<(String, String)>,
#[cfg(feature = "enterprise")] Headers(user_email): Headers<UserEmail>,
#[cfg(feature = "enterprise")] delegated: Option<
axum::Extension<openobserve_api_common::auth::delegation::DelegatedCaller>,
>,
#[cfg(feature = "enterprise")] headers: axum::http::HeaderMap,
ValidatedJson(body): ValidatedJson<Option<ResolveRequest>>,
) -> Response {
#[cfg(feature = "enterprise")]
{
use crate::request::alerts::ack::{ack_source, error_response};
if !allowed(&org_id, &user_email.user_id, RESPONSES, "POST").await {
return MetaHttpResponse::forbidden("Forbidden");
}
if let Err(resp) = page_problem(&org_id, &response_id, &user_email.user_id).await {
return resp;
}
let problem = match page_problem(&org_id, &response_id, &user_email.user_id).await {
Ok(p) => p,
Err(resp) => return resp,
};
let body = body.unwrap_or_default();
match o2_enterprise::enterprise::oncall::escalation::resolve(
let record = match o2_enterprise::enterprise::oncall::escalation::resolve(
&org_id,
&response_id,
&user_email.user_id,
@@ -2595,9 +2642,23 @@ pub async fn resolve_response(
)
.await
{
Ok(record) => MetaHttpResponse::json(record),
Err(e) => to_response(e),
Ok(record) => record,
Err(e) => return to_response(e),
};
// Closed first, so a failed clear is retried by resolving the closed page again.
if let Some(p) = problem.as_ref() {
let source = ack_source(delegated.as_deref(), &headers);
let closed_at = record
.closed_at
.unwrap_or_else(config::utils::time::now_micros);
let cover = openobserve_core::alerts::ack::page_cover(&record.subject, closed_at);
if let Err(e) =
openobserve_core::alerts::ack::resolve(p, &user_email.user_id, source, cover).await
{
return error_response(e);
}
}
MetaHttpResponse::json(record)
}
#[cfg(not(feature = "enterprise"))]
{
src/api/search/src/traces/list/query.rs
+3 -1
View File
@@ -212,10 +212,12 @@ pub fn build_llm_trace_query(
sum(gen_ai_usage_output_tokens) as gen_ai_usage_details_output, \
{total_tokens_expr}, \
sum(gen_ai_usage_cost) as gen_ai_usage_cost_details, \
{}, \
array_agg(DISTINCT gen_ai_response_model) FILTER (WHERE gen_ai_response_model IS NOT NULL AND gen_ai_response_model != '') as gen_ai_response_models, \
{first_msg_clause} as gen_ai_input_messages, \
{extra_selects} \
FROM {stream_ident}"
FROM {stream_ident}",
validated.unpriced_selects()
)
} else {
let total_tokens_expr = if validated.has_total_tokens {
src/api/search/src/traces/list/response.rs
+12 -2
View File
@@ -34,6 +34,10 @@ pub struct TraceResponseItem {
gen_ai_usage_cost: f64,
gen_ai_input_messages: Option<serde_json::Value>,
models: Vec<String>,
/// Tokens of this trace's spans that no price matched; their cost is missing.
gen_ai_unpriced_tokens: i64,
/// Models of those unpriced spans.
gen_ai_unpriced_models: Vec<String>,
}
#[derive(Debug, Default, Serialize)]
@@ -189,6 +193,8 @@ fn insert_summary(
max_duration
};
let (gen_ai_unpriced_tokens, gen_ai_unpriced_models) =
crate::traces::schema_compat::parse_unpriced(item);
batch.traces.insert(
trace_id.clone(),
TraceResponseItem {
@@ -227,6 +233,8 @@ fn insert_summary(
.collect()
})
.unwrap_or_default(),
gen_ai_unpriced_tokens,
gen_ai_unpriced_models,
},
);
}
@@ -290,7 +298,8 @@ mod tests {
"first_service_name": "api", "first_operation_name": "request",
"gen_ai_usage_details_input": 10, "gen_ai_usage_details_output": 20,
"gen_ai_usage_details_total": 30, "gen_ai_usage_cost_details": 0.5,
"gen_ai_input_messages": "hello", "gen_ai_response_models": ["model", null]
"gen_ai_input_messages": "hello", "gen_ai_response_models": ["model", null],
"gen_ai_unpriced_tokens": 12, "gen_ai_unpriced_models": ["acme-internal-llm"]
})
}
@@ -307,7 +316,8 @@ mod tests {
"first_event": {"service_name": "gateway", "operation_name": "request"},
"gen_ai_usage_input_tokens": 10, "gen_ai_usage_output_tokens": 20,
"gen_ai_usage_total_tokens": 30, "gen_ai_usage_cost": 0.5,
"gen_ai_input_messages": "hello", "models": ["model"]
"gen_ai_input_messages": "hello", "models": ["model"],
"gen_ai_unpriced_tokens": 12, "gen_ai_unpriced_models": ["acme-internal-llm"]
})
);
}
src/api/search/src/traces/schema_compat.rs
+76
View File
@@ -133,6 +133,45 @@ pub(super) struct ValidatedLlmSchema {
pub(super) has_cost_estimated_without_cache: bool,
pub(super) has_cost_cache_read_savings: bool,
pub(super) has_cost_net_cache_impact: bool,
/// `gen_ai_usage_cost_status` exists (ingest marks unpriced / approximate spans).
pub(super) has_cost_status: bool,
}
impl ValidatedLlmSchema {
/// Per-row unpriced token sum and unpriced model list; zero / absent when the stream
/// predates the cost marker.
pub(super) fn unpriced_selects(&self) -> String {
if !self.has_gen_ai || !self.has_cost_status {
return "0 as gen_ai_unpriced_tokens".to_string();
}
let tokens = if self.has_total_tokens {
"gen_ai_usage_total_tokens"
} else {
"COALESCE(gen_ai_usage_input_tokens, 0) + COALESCE(gen_ai_usage_output_tokens, 0)"
};
format!(
"COALESCE(sum(CASE WHEN gen_ai_usage_cost_status = 'unpriced' THEN {tokens} ELSE 0 END), 0) as gen_ai_unpriced_tokens, \
array_agg(DISTINCT gen_ai_response_model) FILTER (WHERE gen_ai_usage_cost_status = 'unpriced' \
AND gen_ai_response_model IS NOT NULL AND gen_ai_response_model != '') as gen_ai_unpriced_models"
)
}
}
/// Reads the per-row unpriced fields [`ValidatedLlmSchema::unpriced_selects`] adds.
pub(super) fn parse_unpriced(item: &config::utils::json::Value) -> (i64, Vec<String>) {
let tokens = item
.get("gen_ai_unpriced_tokens")
.map_or(0, config::utils::json::get_int_value);
let models = item
.get("gen_ai_unpriced_models")
.and_then(|v| v.as_array())
.map(|a| {
a.iter()
.filter_map(|m| m.as_str().map(str::to_string))
.collect()
})
.unwrap_or_default();
(tokens, models)
}
impl ValidatedLlmSchema {
@@ -158,6 +197,7 @@ impl ValidatedLlmSchema {
has_cost_estimated_without_cache: false,
has_cost_cache_read_savings: false,
has_cost_net_cache_impact: false,
has_cost_status: false,
}
}
}
@@ -239,6 +279,10 @@ pub(super) fn validate_llm_schema(
has_cost_estimated_without_cache,
has_cost_cache_read_savings,
has_cost_net_cache_impact,
has_cost_status: has_gen_ai
&& schema
.field_with_name(config::meta::model_pricing::COST_STATUS_COLUMN)
.is_ok(),
})
}
@@ -433,4 +477,36 @@ mod tests {
assert!(!fallback.has_gen_ai);
assert_eq!(fallback.columns, LlmColumns::legacy());
}
#[test]
fn unpriced_selects_follow_the_cost_status_column() {
let base = [
"gen_ai_usage_input_tokens",
"gen_ai_usage_output_tokens",
"gen_ai_usage_cost",
"gen_ai_response_model",
"gen_ai_conversation_id",
];
// A stream written before the marker: constant zero, no reference to the column.
let old = validate_llm_schema(&schema_with(&base), "s").unwrap();
assert!(!old.has_cost_status);
assert_eq!(old.unpriced_selects(), "0 as gen_ai_unpriced_tokens");
let mut fields = base.to_vec();
fields.push("gen_ai_usage_cost_status");
let new = validate_llm_schema(&schema_with(&fields), "s").unwrap();
assert!(new.has_cost_status);
let sql = new.unpriced_selects();
assert!(sql.contains("gen_ai_usage_cost_status = 'unpriced'"));
assert!(sql.contains("as gen_ai_unpriced_models"));
let (tokens, models) = parse_unpriced(&config::utils::json::json!({
"gen_ai_unpriced_tokens": 7, "gen_ai_unpriced_models": ["a", null]
}));
assert_eq!((tokens, models), (7, vec!["a".to_string()]));
assert_eq!(
parse_unpriced(&config::utils::json::json!({})),
(0, Vec::new())
);
}
}
src/api/search/src/traces/session.rs
+11 -1
View File
@@ -1027,6 +1027,7 @@ fn build_session_trace_details_sql(
"gen_ai_usage_cost_net_cache_impact",
"gen_ai_usage_cost_net_cache_impact",
);
let unpriced_selects = validated.unpriced_selects();
format!(
"SELECT trace_id, min({TIMESTAMP_COL_NAME}) as zo_sql_timestamp, \
min(start_time) as trace_start_time, max(end_time) as trace_end_time, \
@@ -1035,6 +1036,7 @@ fn build_session_trace_details_sql(
sum(gen_ai_usage_output_tokens) as gen_ai_usage_details_output, \
{total_tokens_expr}, \
sum(gen_ai_usage_cost) as gen_ai_usage_cost_details, \
{unpriced_selects}, \
{cache_read_tokens_expr}, \
{cache_creation_tokens_expr}, \
{cost_cache_read_expr}, \
@@ -1149,7 +1151,8 @@ fn build_session_trace_response_item(
} else {
max_duration
};
let (gen_ai_unpriced_tokens, gen_ai_unpriced_models) =
super::schema_compat::parse_unpriced(item);
let hit = SessionTraceResponseItem {
trace_id: tid.clone(),
user_ids,
@@ -1215,6 +1218,8 @@ fn build_session_trace_response_item(
.collect()
})
.unwrap_or_default(),
gen_ai_unpriced_tokens,
gen_ai_unpriced_models,
};
Some((tid, service_count, hit))
@@ -1626,6 +1631,7 @@ fn build_latest_sessions_sql(
"gen_ai_usage_cost_net_cache_impact",
"gen_ai_usage_cost_net_cache_impact",
);
let unpriced_selects = validated.unpriced_selects();
let stream_ident = traces::session::quote_identifier(stream_name);
format!(
@@ -1639,6 +1645,7 @@ fn build_latest_sessions_sql(
sum({output_tokens_col}) as gen_ai_usage_output_tokens, \
{total_tokens_expr}, \
sum({cost_col}) as gen_ai_usage_cost, \
{unpriced_selects}, \
{cache_read_tokens_expr}, \
{cache_creation_tokens_expr}, \
{cost_cache_read_expr}, \
@@ -1722,6 +1729,9 @@ struct SessionTraceResponseItem {
gen_ai_usage_cost_net_cache_impact: f64,
gen_ai_input_messages: Option<json::Value>,
models: Vec<String>,
/// Tokens of this trace's spans that no price matched; their cost is missing.
gen_ai_unpriced_tokens: i64,
gen_ai_unpriced_models: Vec<String>,
}
#[derive(Debug, Default, Serialize)]
src/builtins/src/model_pricing.rs
+128 -50
View File
@@ -17,6 +17,8 @@
//!
//! Syncs model pricing definitions from the configured GitHub source into the database
//! under the `_openobserve` org. Runs on startup and periodically (every 6h by default).
//! Until a sync succeeds, or forever on an offline install, the seed list compiled into the
//! binary ([`SEED_PRICING_JSON`]) supplies the built-in prices.
use std::sync::{
LazyLock,
@@ -37,12 +39,49 @@ pub static GITHUB_SERVICE: LazyLock<GitHubDataService> = LazyLock::new(GitHubDat
/// Timestamp (Unix seconds) of the last successful sync from GitHub. 0 means never synced.
static LAST_SYNC_TIMESTAMP: AtomicI64 = AtomicI64::new(0);
/// The built-in price list as of [`SEED_DATE`], shipped in the binary so prices work offline.
/// Same format as the GitHub source (`llm_pricing.json` in openobserve/sdr_patterns).
pub const SEED_PRICING_JSON: &str = include_str!("model_pricing_seed.json");
/// Date (UTC) of the upstream `llm_pricing.json` commit that [`SEED_PRICING_JSON`] copies.
pub const SEED_DATE: &str = "2026-10-28";
/// Returns the Unix timestamp (seconds) of the last successful built-in pricing sync.
/// Returns 0 if no sync has completed yet.
pub fn last_sync_timestamp() -> i64 {
LAST_SYNC_TIMESTAMP.load(Ordering::Relaxed)
}
/// Parses the compiled-in seed list.
pub fn seed_entries() -> Result<Vec<BuiltInModelPricingEntry>, anyhow::Error> {
let raw: Vec<serde_json::Value> = serde_json::from_str(SEED_PRICING_JSON)
.map_err(|e| anyhow::anyhow!("Failed to parse seed pricing JSON: {e}"))?;
Ok(parse_entries(raw))
}
/// Fill missing compiled-in prices, retrying incomplete seeds without overwriting synced rows.
pub async fn load_seed() -> Result<usize, anyhow::Error> {
let names = table::model_pricing::list_built_in_names().await?;
let existing: std::collections::HashSet<&str> = names.iter().map(String::as_str).collect();
let mut missing = Vec::new();
for entry in seed_entries()? {
if existing.contains(entry.name.as_str()) {
continue;
}
let definition = entry.into_definition();
definition.validate_tiers(true).map_err(|error| {
anyhow::anyhow!("Invalid seed price '{}': {error}", definition.name)
})?;
missing.push(definition);
}
let created = usize::try_from(table::model_pricing::insert_missing_built_ins(missing).await?)?;
if created > 0 {
emit_sync_event().await;
}
log::info!("[model_pricing_sync] loaded {created} seed prices dated {SEED_DATE}");
Ok(created)
}
/// A model pricing entry from the community/built-in GitHub JSON source.
#[derive(Debug, Deserialize)]
pub struct BuiltInModelPricingEntry {
@@ -95,51 +134,7 @@ pub async fn sync_built_in_from_github(force_refresh: bool) -> Result<SyncResult
.into_iter()
.collect();
let mut created = 0usize;
let mut updated = 0usize;
// Upsert each entry
for entry in entries {
let is_new = !existing_names.contains(&entry.name);
let definition = ModelPricingDefinition {
id: None, // will be looked up by (org, name) on upsert
org_id: BUILT_IN_ORG.to_string(),
name: entry.name,
match_pattern: entry.match_pattern,
enabled: true,
tiers: entry.tiers,
valid_from: None,
sort_order: 0,
source: PricingSource::BuiltIn,
provider: entry.provider,
description: entry.description,
created_at: 0, // set by DB layer
updated_at: 0,
children: Vec::new(),
prepared_rule_keys: Default::default(),
};
// An invalid entry could overprice every span; keep the previously stored row.
if let Err(e) = definition.validate_tiers(true) {
log::warn!(
"[model_pricing_sync] skipping invalid entry '{}': {e}",
definition.name
);
continue;
}
match table::model_pricing::put(definition).await {
Ok(_) => {
if is_new {
created += 1;
} else {
updated += 1;
}
}
Err(e) => {
log::warn!("[model_pricing_sync] failed to upsert built-in model: {e}");
}
}
}
let (created, updated) = upsert_entries(entries, &existing_names).await;
// Disable models that were removed from the upstream source
let mut disabled = 0usize;
@@ -162,11 +157,7 @@ pub async fn sync_built_in_from_github(force_refresh: bool) -> Result<SyncResult
}
}
// Emit coordinator event so all nodes refresh the _openobserve cache
let event_key = format!("/model_pricing/{BUILT_IN_ORG}/_sync");
if let Err(e) = infra::coordinator::emit_put(&event_key).await {
log::error!("[model_pricing_sync] failed to emit sync event: {e}");
}
emit_sync_event().await;
LAST_SYNC_TIMESTAMP.store(chrono::Utc::now().timestamp(), Ordering::Relaxed);
@@ -181,6 +172,72 @@ pub async fn sync_built_in_from_github(force_refresh: bool) -> Result<SyncResult
})
}
/// Upserts catalogue entries as built-in definitions; returns (created, updated).
/// An entry whose name is in `existing_names` counts as updated.
async fn upsert_entries(
entries: Vec<BuiltInModelPricingEntry>,
existing_names: &std::collections::HashSet<String>,
) -> (usize, usize) {
let mut created = 0usize;
let mut updated = 0usize;
for entry in entries {
let is_new = !existing_names.contains(&entry.name);
let definition = entry.into_definition();
// An invalid entry could overprice every span; keep the previously stored row.
if let Err(e) = definition.validate_tiers(true) {
log::warn!(
"[model_pricing_sync] skipping invalid entry '{}': {e}",
definition.name
);
continue;
}
match table::model_pricing::put(definition).await {
Ok(_) => {
if is_new {
created += 1;
} else {
updated += 1;
}
}
Err(e) => {
log::warn!("[model_pricing_sync] failed to upsert built-in model: {e}");
}
}
}
(created, updated)
}
/// Tells every node to refresh its `_openobserve` pricing cache.
async fn emit_sync_event() {
let event_key = format!("/model_pricing/{BUILT_IN_ORG}/_sync");
if let Err(e) = infra::coordinator::emit_put(&event_key).await {
log::error!("[model_pricing_sync] failed to emit sync event: {e}");
}
}
impl BuiltInModelPricingEntry {
fn into_definition(self) -> ModelPricingDefinition {
ModelPricingDefinition {
id: None, // will be looked up by (org, name) on upsert
org_id: BUILT_IN_ORG.to_string(),
name: self.name,
match_pattern: self.match_pattern,
enabled: true,
tiers: self.tiers,
valid_from: None,
sort_order: 0,
source: PricingSource::BuiltIn,
provider: self.provider,
description: self.description,
created_at: 0, // set by DB layer
updated_at: 0,
children: Vec::new(),
prepared_rule_keys: Default::default(),
}
}
}
/// Parses each catalogue entry on its own so one malformed entry does not stop the whole sync.
fn parse_entries(raw: Vec<serde_json::Value>) -> Vec<BuiltInModelPricingEntry> {
raw.into_iter()
@@ -201,6 +258,27 @@ fn parse_entries(raw: Vec<serde_json::Value>) -> Vec<BuiltInModelPricingEntry> {
mod tests {
use super::*;
/// The seed list is all an offline install has, so every entry must parse and validate
/// without any network access.
#[test]
fn test_seed_list_loads_offline() {
let raw: Vec<serde_json::Value> = serde_json::from_str(SEED_PRICING_JSON).unwrap();
let entries = seed_entries().unwrap();
assert!(
entries.len() > 50,
"seed list has {} entries",
entries.len()
);
assert_eq!(entries.len(), raw.len(), "an entry failed to parse");
for entry in entries {
let definition = entry.into_definition();
definition
.validate_tiers(true)
.unwrap_or_else(|e| panic!("{}: {e}", definition.name));
}
assert!(chrono::NaiveDate::parse_from_str(SEED_DATE, "%Y-%m-%d").is_ok());
}
#[test]
fn test_last_sync_timestamp_initial_value() {
// Initial value is 0 (never synced), unless tests run after sync
src/builtins/src/model_pricing_seed.json
+5622
View File
File diff suppressed because it is too large Load Diff
src/config/src/config.rs
+3 -1
View File
@@ -131,7 +131,9 @@ pub type RwBTreeMap<K, V> = tokio::sync::RwLock<BTreeMap<K, V>>;
// 131: org domain mapping migrations
// 132: create oncall_chat_messages; add oncall_teams slack columns.
// 133: add status page run streaks to synthetics.
pub const DB_SCHEMA_VERSION: u64 = 134;
// 134: add write_version to llm_dataset_items, so one write request is one Dataset version.
// 135: add last_run_class and last_run_class_at to synthetics.
pub const DB_SCHEMA_VERSION: u64 = 135;
pub const DB_SCHEMA_KEY: &str = "/db_schema_version/";
// global version variables
src/config/src/meta/alerts/incidents.rs
+15
View File
@@ -638,6 +638,11 @@ pub enum IncidentEventType {
reason: String,
},
/// A person resolved one of the incident's pages, which ended its acknowledgement.
AckEndedByPageResolve {
user_id: String,
},
DimensionsUpgraded {
from_key: String,
to_key: String,
@@ -748,6 +753,7 @@ impl IncidentEventType {
"Acknowledged",
"Resolved",
"Reopened",
"AckEndedByPageResolve",
"DimensionsUpgraded",
"TitleChanged",
"AssignmentChanged",
@@ -832,6 +838,12 @@ impl IncidentEvent {
Self::now(IncidentEventType::Resolved { user_id })
}
pub fn ack_ended_by_page_resolve(user_id: impl Into<String>) -> Self {
Self::now(IncidentEventType::AckEndedByPageResolve {
user_id: user_id.into(),
})
}
pub fn reopened(user_id: impl Into<String>, reason: impl Into<String>) -> Self {
Self::now(IncidentEventType::Reopened {
user_id: user_id.into(),
@@ -1091,6 +1103,9 @@ mod tests {
user_id: "u".into(),
reason: "r".into(),
},
IncidentEventType::AckEndedByPageResolve {
user_id: "u".into(),
},
IncidentEventType::DimensionsUpgraded {
from_key: "a".into(),
to_key: "b".into(),
src/config/src/meta/eval_job_health.rs
+3 -2
View File
@@ -22,11 +22,12 @@ use serde::{Deserialize, Serialize};
#[serde(rename_all = "snake_case")]
pub enum EvalJobHealth {
Healthy,
/// More than 20% of at least 5 recent scorer runs failed.
/// More than 20% of at least 5 recent scorer runs failed, or nothing ran recently and the
/// lookback's newest outcome is an error.
Degraded,
/// The newest scorer outcome is a `provider_auth` error.
WaitingForCredentials,
/// No scorer ran in the recent window.
/// No scorer ran in the 24-hour lookback.
NoRecentRuns,
}
src/config/src/meta/model_pricing.rs
+29
View File
@@ -117,6 +117,35 @@ impl MatchKind {
}
}
/// Stream column that marks how ingest priced an LLM span. Exact non-zero prices and
/// client-reported costs leave it unset, so streams written before it existed have no such
/// column.
pub const COST_STATUS_COLUMN: &str = "gen_ai_usage_cost_status";
/// How ingest priced an LLM span whose cost is not an exact non-zero price; stored in
/// [`COST_STATUS_COLUMN`].
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, ToSchema)]
#[serde(rename_all = "snake_case")]
pub enum CostStatus {
/// No pricing definition matched the model: the span keeps its tokens and has no cost.
Unpriced,
/// A pattern matched only part of the model name; the cost uses that pattern's prices.
Approximate,
/// The model matched a price exactly and the span costs nothing (a free or muted model).
/// The zero cost is a real price, unlike the zero legacy spans stored for no price.
Free,
}
impl CostStatus {
pub fn as_str(self) -> &'static str {
match self {
CostStatus::Unpriced => "unpriced",
CostStatus::Approximate => "approximate",
CostStatus::Free => "free",
}
}
}
/// A model pricing definition that maps model names (via regex) to pricing tiers.
#[derive(Clone, Debug, Serialize, Deserialize, Default, ToSchema)]
#[serde(default)]
src/config/src/meta/oncall/preset.rs
+51 -4
View File
@@ -60,6 +60,9 @@ const MINUTES_PER_DAY: u32 = 1440;
const DEFAULT_BUSINESS_START_MINUTE: u32 = 9 * 60;
const DEFAULT_BUSINESS_END_MINUTE: u32 = 17 * 60;
/// Thursday 00:00: the boundary the split-the-week summary describes before anything is chosen.
const DEFAULT_SPLIT_BOUNDARY_DAY: u8 = 3;
const DEFAULT_SPLIT_BOUNDARY_MINUTE: u32 = 0;
fn default_business_days() -> Vec<u8> {
vec![0, 1, 2, 3, 4]
@@ -73,6 +76,14 @@ fn default_business_end() -> u32 {
DEFAULT_BUSINESS_END_MINUTE
}
fn default_split_boundary_day() -> u8 {
DEFAULT_SPLIT_BOUNDARY_DAY
}
fn default_split_boundary_minute() -> u32 {
DEFAULT_SPLIT_BOUNDARY_MINUTE
}
/// Which preset. Serialises to the id the catalogue publishes.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize, ToSchema)]
#[serde(rename_all = "snake_case")]
@@ -156,9 +167,11 @@ pub enum PresetSpec {
first: Group,
/// Covers from the boundary to the end of the week.
second: Group,
/// 0 = Monday … 6 = Sunday.
/// 0 = Monday … 6 = Sunday. Defaults to Thursday.
#[serde(default = "default_split_boundary_day")]
boundary_day: u8,
/// Minutes from midnight in the team's timezone.
/// Minutes from midnight in the team's timezone. Defaults to 00:00.
#[serde(default = "default_split_boundary_minute")]
boundary_minute: u32,
},
/// A working-hours layer over an out-of-hours catch-all.
@@ -1018,8 +1031,10 @@ pub fn catalogue() -> Vec<PresetDescriptor> {
"Hand over on",
"0 = Monday … 6 = Sunday. Monday 00:00 is refused: it would give the first group none of the week.",
)
.bounded(0, 6),
minute_input("boundary_minute", "Hand over at"),
.bounded(0, 6)
.defaulting(serde_json::json!(DEFAULT_SPLIT_BOUNDARY_DAY)),
minute_input("boundary_minute", "Hand over at")
.defaulting(serde_json::json!(DEFAULT_SPLIT_BOUNDARY_MINUTE)),
]),
},
PresetDescriptor {
@@ -1795,6 +1810,38 @@ mod tests {
);
}
/// The split-the-week form opens on the boundary its summary shows, so Save works untouched.
#[test]
fn test_split_the_week_defaults_to_thursday_midnight() {
let entries = catalogue();
let split = entries
.iter()
.find(|e| e.id == PresetId::SplitTheWeek)
.unwrap();
let default_of = |field: &str| {
let input = split.inputs.iter().find(|i| i.field == field).unwrap();
assert!(!input.required, "{field} is still required");
input.default.clone()
};
assert_eq!(default_of("boundary_day"), Some(serde_json::json!(3)));
assert_eq!(default_of("boundary_minute"), Some(serde_json::json!(0)));
let spec: PresetSpec = serde_json::from_str(
r#"{"preset":"split_the_week",
"first":{"members":["ana@o2.ai"]},
"second":{"members":["bo@o2.ai"]}}"#,
)
.unwrap();
let rotations = build(&spec, chrono_tz::UTC, MONDAY, DEFAULT_HANDOVER_MICROS).unwrap();
assert_eq!(
rotations[0].restrictions,
vec![TimeWindow {
days: vec![0, 1, 2],
start_minute: 0,
end_minute: MINUTES_PER_DAY,
}]
);
}
/// A region whose hours straddle local midnight is a normal thing to
/// describe, and it must not be mistaken for an empty window.
#[test]
src/config/src/meta/self_reporting/llm_experiments.rs
+8
View File
@@ -103,6 +103,11 @@ pub struct ExperimentExecutionRecord {
pub output: Option<serde_json::Value>,
#[serde(skip_serializing_if = "Option::is_none")]
pub error_message: Option<String>,
/// Failure class from the shared evaluation taxonomy (`provider_auth`,
/// `timeout`, `execution_error`, …). Absent on records written before it
/// existed, which group by message alone.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub error_kind: Option<String>,
/// Permanent reason an intentionally ineligible slot was not executed.
#[serde(skip_serializing_if = "Option::is_none")]
pub skip_reason: Option<ExperimentSkipReason>,
@@ -148,6 +153,7 @@ impl ExperimentExecutionRecord {
Self {
output: Some(serde_json::Value::String(String::new())),
error_message: Some(String::new()),
error_kind: Some(String::new()),
skip_reason: Some(ExperimentSkipReason::NoReference),
error_attempt_count: Some(0),
latency_ms: Some(0),
@@ -334,6 +340,7 @@ mod tests {
status: ExperimentExecutionStatus::Ok,
output: Some(json!("It ships tomorrow")),
error_message: None,
error_kind: None,
skip_reason: None,
error_attempt_count: None,
latency_ms: Some(42),
@@ -367,6 +374,7 @@ mod tests {
"status",
"output",
"error_message",
"error_kind",
"skip_reason",
"error_attempt_count",
"latency_ms",
src/config/src/router.rs
+17 -6
View File
@@ -14,7 +14,7 @@
// along with this program. If not, see <http://www.gnu.org/licenses/>.
/// usize indicates the number of parts to skip based on their actual paths.
const QUERIER_ROUTES: [(&str, usize); 43] = [
const QUERIER_ROUTES: [(&str, usize); 47] = [
("config", 0), // /config (unauthenticated bootstrap)
("config", 2), // /api/{org_id}/config (authenticated full)
("summary", 2), // /api/{org_id}/summary
@@ -56,11 +56,16 @@ const QUERIER_ROUTES: [(&str, usize); 43] = [
("chat_stream", 3), /* /api/{org_id}/ai/chat_stream
* {label_name}/
* values */
("discovery", 2), // /api/{org_id}/discovery
("annotation_queues", 2), // /api/{org_id}/annotation_queues/...
("playground", 2), // /api/{org_id}/playground/...
("service_streams", 2), // /api/{org_id}/service_streams/...
("node/list", 2), // /api/_meta/node/list
("discovery", 2), // /api/{org_id}/discovery
("annotation_queues", 2), // /api/{org_id}/annotation_queues/...
("playground", 2), // /api/{org_id}/playground/...
("service_streams", 2), // /api/{org_id}/service_streams/...
("node/list", 2), // /api/_meta/node/list
("gen_ai/data_health", 2), // /api/{org_id}/gen_ai/data_health
("llm/models/unpriced", 2), // /api/{org_id}/llm/models/unpriced
// Queriers own the built-in GitHub sync, so its node-local sync time lives there.
("llm/models/built-in", 2), // /api/{org_id}/llm/models/built-in
("llm/models/refresh-built-in", 2), // /api/{org_id}/llm/models/refresh-built-in
];
const QUERIER_ROUTES_BY_BODY: [&str; 9] = [
"/_search",
@@ -240,6 +245,12 @@ mod tests {
assert!(is_querier_route(
"/api/org1/traces/time_range?trace_id=a,b,c"
));
// Built-in catalogue status and refresh read the querier's sync state.
assert!(is_querier_route(
"/api/org1/llm/models/built-in?metadata_only=true"
));
assert!(is_querier_route("/api/org1/llm/models/refresh-built-in"));
assert!(!is_querier_route("/api/org1/llm/models"));
// the org-level entry must not swallow the OTLP ingest route
assert!(!is_querier_route("/api/org1/traces"));
for leaf in ["meta", "tag_values", "series", "merge", "diff"] {
src/core/src/alerts/ack.rs
+183 -22
View File
@@ -111,6 +111,8 @@ pub enum Verb {
Unack,
/// An unack for a hand-off: the hand-off, not the unack, restarts the page it moves.
HandOff,
/// A person fixed the problem: its ack ends and no page restarts.
Resolve,
}
/// Which firing groups of an alert an ack names.
@@ -181,6 +183,15 @@ pub struct PlanFacts {
/// Who holds the firing groups of the alert the caller did not name.
pub other_holders: Vec<String>,
pub pages: Vec<OpenPage>,
/// What a page's Resolve may end; `None` ends the whole problem.
pub cover: Option<PageCover>,
}
/// A page covers the firings of its alert that started by the time it closed.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct PageCover {
pub alert_id: String,
pub closed_at: i64,
}
#[derive(Clone, Debug, PartialEq, Eq)]
@@ -384,7 +395,7 @@ pub async fn resolve_target(org_id: &str, alert_id: &str) -> Result<AckTarget, A
pub fn plan(f: &PlanFacts) -> AckPlan {
match f.verb {
Verb::Acknowledge | Verb::TakeOver => plan_hold(f),
Verb::Unack | Verb::HandOff => plan_release(f),
Verb::Unack | Verb::HandOff | Verb::Resolve => plan_release(f),
}
}
@@ -535,7 +546,7 @@ pub async fn take_over(
/// [weak: the ack is cleared first; a page that fails to restart stays acknowledged until the next
/// unack; one restarted with no ladder armed is logged and left to `reconcile_abandoned`]
pub async fn unack(p: &Problem, actor: &str, source: AckSource) -> Result<UnackResponse, AckError> {
release(p, Verb::Unack, actor, source).await
release(p, Verb::Unack, actor, source, None).await
}
/// Clears the problem's ack for a hand-off, which restarts the page it moves.
@@ -546,7 +557,28 @@ pub async fn release_for_hand_off(
actor: &str,
source: AckSource,
) -> Result<UnackResponse, AckError> {
release(p, Verb::HandOff, actor, source).await
release(p, Verb::HandOff, actor, source, None).await
}
/// Ends the covered ack after a person's Resolve. [weak: a retried Resolve clears the rest]
pub async fn resolve(
p: &Problem,
actor: &str,
source: AckSource,
cover: Option<PageCover>,
) -> Result<UnackResponse, AckError> {
release(p, Verb::Resolve, actor, source, cover).await
}
/// An alert page covers its alert's firings up to its first close; an incident page all. [pure]
pub fn page_cover(subject: &config::meta::oncall::SubjectRef, closed_at: i64) -> Option<PageCover> {
match subject.subject_type {
config::meta::oncall::SubjectType::Alert => Some(PageCover {
alert_id: source_of_page(&subject.source_id).to_string(),
closed_at,
}),
config::meta::oncall::SubjectType::Incident => None,
}
}
/// A firing that joins an acknowledged incident takes the incident's ack, so nobody is re-paged.
@@ -1044,6 +1076,26 @@ async fn org_editors(org_id: &str) -> Vec<(String, UserRole, Option<String>)> {
out
}
/// The records a release ends; a page's Resolve leaves its alert's firings newer than its close.
fn covered(f: &PlanFacts) -> Vec<&HeldRecord> {
let Some(c) = f.cover.as_ref().filter(|_| f.verb == Verb::Resolve) else {
return f.records.iter().collect();
};
let newer = |r: &HeldRecord| {
matches!(&r.record, AckRecord::Firing { alert_id, firing_started_at, .. }
if *alert_id == c.alert_id && *firing_started_at > c.closed_at)
};
let incident = f
.records
.iter()
.any(|r| matches!(r.record, AckRecord::Incident { .. }));
// An incident is one problem: a page whose alert fired again after it closed covers none of it.
if incident && f.records.iter().any(newer) {
return vec![];
}
f.records.iter().filter(|r| !newer(r)).collect()
}
fn same(a: &str, b: &str) -> bool {
a.trim().eq_ignore_ascii_case(b.trim())
}
@@ -1137,13 +1189,13 @@ fn hold_step(p: &OpenPage, f: &PlanFacts, by: &str, take_over: bool) -> PageStep
}
}
/// An unack restarts the current pages an ack stopped (D34); a hand-off leaves them be.
/// An unack restarts the current pages an ack stopped (D34); a hand-off and a resolve do not.
fn plan_release(f: &PlanFacts) -> AckPlan {
let ends = covered(f);
// Any held record, carried ones too: a retry after a partial clear must finish the job.
let held = f.records.iter().any(|r| r.holder.is_some());
let held = ends.iter().any(|r| r.holder.is_some());
let writes = if held {
f.records
.iter()
ends.iter()
.filter_map(|r| {
r.holder
.as_ref()
@@ -1203,8 +1255,10 @@ async fn release(
verb: Verb,
actor: &str,
source: AckSource,
cover: Option<PageCover>,
) -> Result<UnackResponse, AckError> {
let g = gather(p, verb, actor).await?;
let mut g = gather(p, verb, actor).await?;
g.facts.cover = cover;
let plan = plan(&g.facts);
let out = carry_out(p, &g, &plan, actor, source).await?;
Ok(UnackResponse {
@@ -1235,6 +1289,7 @@ async fn gather(p: &Problem, verb: Verb, actor: &str) -> Result<Gathered, AckErr
other_groups_unacked: 0,
other_holders: vec![],
pages,
cover: None,
},
targets: HashMap::new(),
held: HashMap::new(),
@@ -1293,7 +1348,7 @@ async fn one_row(
firing_started_at: Option<i64>,
verb: Verb,
) -> Result<Vec<(String, i64)>, AckError> {
if matches!(verb, Verb::Unack | Verb::HandOff) {
if matches!(verb, Verb::Unack | Verb::HandOff | Verb::Resolve) {
let firing = match firing_started_at {
Some(f) => Some(f),
None => current_firing(target, group_key).await?,
@@ -1354,19 +1409,22 @@ async fn gather_incident(
};
let status: IncidentStatus = model.status.parse().unwrap_or_default();
g.incident_status = Some(status);
// A resolved incident has nothing to acknowledge or clear; reopening it is its own verb.
if status == IncidentStatus::Resolved {
let resolved = status == IncidentStatus::Resolved;
// A resolved incident has nothing to acknowledge; only a resolve still clears its alerts' rows.
if resolved && g.facts.verb != Verb::Resolve {
return Ok(());
}
g.facts.records.push(HeldRecord {
record: AckRecord::Incident {
incident_id: incident_id.to_string(),
},
holder: (status == IncidentStatus::Acknowledged)
.then_some(model.acknowledged_by)
.flatten(),
named: true,
});
if !resolved {
g.facts.records.push(HeldRecord {
record: AckRecord::Incident {
incident_id: incident_id.to_string(),
},
holder: (status == IncidentStatus::Acknowledged)
.then_some(model.acknowledged_by)
.flatten(),
named: true,
});
}
for alert_id in incident_alert_ids(incident_id).await? {
let target = match resolve_target(org_id, &alert_id).await {
Ok(t) => t,
@@ -1425,7 +1483,8 @@ async fn carry_out(
.any(|r| &r.record == record && r.named);
let won = match record {
AckRecord::Incident { incident_id } => {
let won = write_incident(p.org_id(), incident_id, write, actor).await?;
let won =
write_incident(p.org_id(), incident_id, write, actor, g.facts.verb).await?;
if won {
incident_status = Some(status_after(write));
} else if *write == RecordWrite::Claim {
@@ -1586,6 +1645,7 @@ async fn write_incident(
incident_id: &str,
write: &RecordWrite,
actor: &str,
verb: Verb,
) -> Result<bool, AckError> {
use config::meta::alerts::incidents::IncidentEvent;
use infra::table::alert_incidents;
@@ -1605,7 +1665,12 @@ async fn write_incident(
),
RecordWrite::Clear { holder } => (
alert_incidents::unacknowledge(org_id, incident_id, holder).await?,
IncidentEvent::reopened(actor, "Unacknowledged"),
match verb {
Verb::Resolve => IncidentEvent::ack_ended_by_page_resolve(actor),
Verb::Acknowledge | Verb::TakeOver | Verb::Unack | Verb::HandOff => {
IncidentEvent::reopened(actor, "Unacknowledged")
}
},
),
};
if won {
@@ -1621,6 +1686,7 @@ async fn write_incident(
_incident_id: &str,
_write: &RecordWrite,
_actor: &str,
_verb: Verb,
) -> Result<bool, AckError> {
Err(AckError::NotFound)
}
@@ -2059,6 +2125,14 @@ mod tests {
other_groups_unacked: 0,
other_holders: vec![],
pages,
cover: None,
}
}
fn cover(closed_at: i64) -> PageCover {
PageCover {
alert_id: "a1".into(),
closed_at,
}
}
@@ -2225,6 +2299,82 @@ mod tests {
vec![],
Answer::Changed,
),
(
"a resolve clears every record and restarts no page",
facts_for(
Verb::Resolve,
vec![incident(ana), firing("", ana, false)],
vec![page("owner", ana), page("paging", None)],
),
vec![cleared(&incident(ana)), cleared(&firing("", ana, false))],
vec![("owner", Leave), ("paging", Leave)],
vec![],
Answer::Changed,
),
(
"a resolve with nothing held changes nothing",
facts_for(
Verb::Resolve,
vec![firing("", None, true)],
vec![page("p1", ana)],
),
vec![],
vec![("p1", Leave)],
vec![],
Answer::NotHeld,
),
(
"a page's resolve leaves a firing that started after the page closed",
PlanFacts {
cover: Some(cover(50)),
..facts_for(
Verb::Resolve,
vec![firing("", ana, true)],
vec![page("p1", ana)],
)
},
vec![],
vec![("p1", Leave)],
vec![],
Answer::NotHeld,
),
(
"a page's resolve ends a firing that started while it was open",
PlanFacts {
cover: Some(cover(100)),
..facts_for(Verb::Resolve, vec![firing("", ana, true)], vec![])
},
vec![cleared(&firing("", ana, true))],
vec![],
vec![],
Answer::Changed,
),
(
"a page's resolve covers none of an incident its alert re-fired into after the close",
PlanFacts {
cover: Some(cover(50)),
..facts_for(
Verb::Resolve,
vec![incident(ana), firing("", ana, false)],
vec![],
)
},
vec![],
vec![],
vec![],
Answer::NotHeld,
),
(
"a hand-off ignores a cover",
PlanFacts {
cover: Some(cover(50)),
..facts_for(Verb::HandOff, vec![firing("", ana, true)], vec![])
},
vec![cleared(&firing("", ana, true))],
vec![],
vec![],
Answer::Changed,
),
(
"an unack with nothing held still restarts the page an ack stopped",
facts_for(
@@ -2333,6 +2483,17 @@ mod tests {
}
}
#[test]
fn an_alert_page_covers_its_alerts_firings_up_to_its_close_and_an_incident_page_all() {
use config::meta::oncall::{SubjectRef, SubjectType};
let subject = |kind, source: &str| SubjectRef::new(kind, source.to_string(), 1);
assert_eq!(
page_cover(&subject(SubjectType::Alert, "a1:group:g1"), 70),
Some(cover(70))
);
assert_eq!(page_cover(&subject(SubjectType::Incident, "i1"), 70), None);
}
#[test]
fn a_page_names_its_source_without_the_group_or_impacted_suffix() {
for (page_source, source) in [
src/core/src/alerts/incidents.rs
+10 -4
View File
@@ -3183,15 +3183,21 @@ pub async fn update_status_as(
return through_the_one_ack(org_id, incident_id, status, user_id).await;
}
// Read before the resolve clears it, so resolving a muted incident still notifies nobody.
let before = infra::table::alert_incidents::get(org_id, incident_id).await?;
let muted_before = reads_mute_before(
status,
o2_enterprise::enterprise::common::config::get_config()
.downtimes
.enabled,
) && infra::table::alert_incidents::get(org_id, incident_id)
.await?
) && before
.as_ref()
.is_some_and(|m| m.muted_by_downtime_id.is_some());
let updated = infra::table::alert_incidents::update_status(org_id, incident_id, status).await?;
// A re-sent Resolve re-runs the idempotent repairs but keeps the first row and event.
let again = status == "resolved" && before.as_ref().is_some_and(|m| m.status == "resolved");
let updated = match before {
Some(m) if again => m,
_ => infra::table::alert_incidents::update_status(org_id, incident_id, status).await?,
};
let muted = status_change_is_muted(
quiet || muted_before,
@@ -3216,7 +3222,7 @@ pub async fn update_status_as(
"open" => Some(IncidentEvent::reopened(user_id, "Manually reopened")),
_ => None,
};
if let Some(evt) = event
if let Some(evt) = event.filter(|_| !again)
&& let Err(e) = append_event_unless_muted(org_id, incident_id, evt, muted).await
{
log::error!("[Incidents] Failed to record status event: org_id: {org_id}, error: {e}");
src/core/src/incidents.rs
+5
View File
@@ -109,6 +109,11 @@ async fn get_event_metadata(
metadata.push(("reason", reason.into()));
}
IncidentEventType::AckEndedByPageResolve { user_id } => {
metadata.push(("event_type", "ack_ended_by_page_resolve".into()));
metadata.push(("user_id", user_id.into()));
}
IncidentEventType::DimensionsUpgraded { from_key, to_key } => {
metadata.push(("event_type", "dimensions_upgraded".into()));
metadata.push(("from_key", from_key.into()));
src/core/src/traces/mod.rs
+50 -15
View File
@@ -88,7 +88,7 @@ const SERVICE: &str = "service";
const PARENT_SPAN_ID: &str = "reference.parent_span_id";
const PARENT_TRACE_ID: &str = "reference.parent_trace_id";
const REF_TYPE: &str = "reference.ref_type";
const RESERVED_SPAN_FIELDS: [&str; 33] = [
const RESERVED_SPAN_FIELDS: [&str; 34] = [
"trace_id",
"span_id",
"flags",
@@ -122,6 +122,10 @@ const RESERVED_SPAN_FIELDS: [&str; 33] = [
crate::db_monitoring::O2_DB_ENV,
crate::db_monitoring::O2_DB_STMT_CLASS,
crate::db_monitoring::O2_DB_BATCH_MULTIPLIER,
// The ingest pricing marker: any spelling a client sends (`gen_ai.usage.cost_status`,
// `gen_ai_usage_cost_status`, …) on any span gets the attr_ prefix, so only the
// processor's own value reaches the column.
config::meta::model_pricing::COST_STATUS_COLUMN,
"events",
"links",
TIMESTAMP_COL_NAME,
@@ -359,15 +363,18 @@ async fn queue_gen_ai_agent_observations(org_id: &str, mut observations: AgentOb
.await;
}
/// Strip client-supplied derived-identity keys (`o2_db_*`, `infer_service_*`)
/// from a JSON-path record BEFORE re-derivation (design D1 condition 1):
/// Strip client-supplied derived keys (`o2_db_*`, `infer_service_*`, the ingest pricing
/// marker) from a JSON-path record BEFORE re-derivation (design D1 condition 1):
/// `RESERVED_SPAN_FIELDS` protects the OTLP path only — the JSON ingest path
/// flattens the caller's record keys directly, so a spoofed `o2_db_fingerprint`
/// (or `infer_service_name`, fixed in the same PR per the design) would land as
/// a trusted aggregation key. Legitimate values written by a first-pass OTLP
/// enrichment re-derive identically from the raw `db_*` / peer attributes that
/// are still on the record (the derivations are deterministic).
/// are still on the record (the derivations are deterministic). This path does not
/// price spans, so it never sets `gen_ai_usage_cost_status`; a value on the record is
/// not ingest's.
fn strip_client_supplied_derived_fields(record_val: &mut Map<String, json::Value>) {
record_val.remove(config::meta::model_pricing::COST_STATUS_COLUMN);
// one key scan instead of 20 map removals: a record almost never carries a derived key
if !record_val
.keys()
@@ -873,15 +880,12 @@ pub async fn handle_otlp_request(
let otel_processor = OtelIngestionProcessor::new();
// Pre-load user-defined model pricing entries for this org (in-memory cache, no I/O).
// When ZO_MODEL_PRICING_ENABLED=false, no pricing applies and spans keep only reported costs.
static EMPTY_PRICING: std::sync::OnceLock<
std::sync::Arc<Vec<crate::db::model_pricing::CachedModelPricing>>,
> = std::sync::OnceLock::new();
let org_pricing_entries = if config::get_config().common.model_pricing_enabled {
crate::db::model_pricing::get_org_pricing_entries(org_id)
} else {
std::sync::Arc::clone(EMPTY_PRICING.get_or_init(|| std::sync::Arc::new(vec![])))
};
// When ZO_MODEL_PRICING_ENABLED=false, nothing is priced or marked and spans keep only
// reported costs.
let org_pricing_entries = config::get_config()
.common
.model_pricing_enabled
.then(|| crate::db::model_pricing::get_org_pricing_entries(org_id));
let gen_ai_agent_mapping_config =
crate::db::system_settings::get_gen_ai_agent_mapping_config(org_id).await;
@@ -967,7 +971,7 @@ pub async fn handle_otlp_request(
&service_att_map,
scope_name,
&events,
&org_pricing_entries,
org_pricing_entries.as_deref().map(Vec::as_slice),
start_time,
&gen_ai_agent_mapping_config,
);
@@ -2554,7 +2558,7 @@ mod tests {
#[test]
fn test_reserved_span_fields() {
let reserved_span_fields = &super::RESERVED_SPAN_FIELDS;
assert_eq!(reserved_span_fields.len(), 33);
assert_eq!(reserved_span_fields.len(), 34);
// All eleven DBM identity columns are reserved (design D1 condition 1).
for field in crate::db_monitoring::ALL_DB_FIELDS {
assert!(
@@ -2980,6 +2984,37 @@ mod tests {
}
}
#[test]
fn test_span_attribute_key_blocks_cost_status_spoofing() {
// Only the processor's own marker may reach gen_ai_usage_cost_status; a client's
// value in any spelling, on any span, keeps its data under an attr_ key.
let service_att_map = std::collections::HashMap::new();
for raw in [
"gen_ai.usage.cost_status",
"gen_ai_usage_cost_status",
"Gen_AI.Usage.Cost-Status",
] {
assert_eq!(attr_key(raw, &service_att_map), format!("attr_{raw}"));
}
}
#[test]
fn test_strip_client_cost_status_on_json_path() {
use config::utils::json;
let mut record_val: json::Map<String, json::Value> = json::Map::new();
record_val.insert(
config::meta::model_pricing::COST_STATUS_COLUMN.to_string(),
json::json!("unpriced"),
);
record_val.insert("gen_ai_usage_cost".to_string(), json::json!(0.25));
super::strip_client_supplied_derived_fields(&mut record_val);
assert!(!record_val.contains_key(config::meta::model_pricing::COST_STATUS_COLUMN));
assert!(record_val.contains_key("gen_ai_usage_cost"));
}
#[test]
fn test_strip_client_supplied_derived_fields_on_json_path() {
// The JSON ingest path flattens client keys directly — RESERVED_SPAN_FIELDS
src/core/src/traces/otel/attributes.rs
+3
View File
@@ -151,6 +151,9 @@ impl GenAiExtensions {
pub const USAGE_COST_CACHE_READ_SAVINGS: &'static str = "gen_ai.usage.cost.cache_read_savings";
/// Net cost impact after cache reads and writes (Float64). OO extension.
pub const USAGE_COST_NET_CACHE_IMPACT: &'static str = "gen_ai.usage.cost.net_cache_impact";
/// How ingest priced the span (`unpriced` / `approximate`); unset for exact prices and
/// client-reported costs. Flattens to `config::meta::model_pricing::COST_STATUS_COLUMN`.
pub const USAGE_COST_STATUS: &'static str = "gen_ai.usage.cost_status";
}
pub struct LLMAttributes;
src/core/src/traces/otel/extractors/input_output.rs
+132 -9
View File
@@ -29,6 +29,8 @@ use crate::{
};
const MAX_INDEX: usize = 100;
const TRACELOOP_PROMPT_PREFIX: &str = "gen_ai.prompt";
const TRACELOOP_COMPLETION_PREFIX: &str = "gen_ai.completion";
pub struct InputOutputExtractor;
@@ -218,11 +220,28 @@ impl InputOutputExtractor {
return true;
}
// Check prefixes for nested attributes
key.starts_with("gen_ai.prompt")
|| key.starts_with("gen_ai.completion")
|| key.starts_with("llm.input_messages")
|| key.starts_with("llm.output_messages")
// OpenInference nested messages
if key.starts_with("llm.input_messages") || key.starts_with("llm.output_messages") {
return true;
}
// TraceLoop nested prompt/completion (`gen_ai.prompt.0.content`, …) and
// other `gen_ai.prompt.*` / `gen_ai.completion.*` keys such as prompt
// template variables are not stored as columns. The prompt reference
// (name/version/label) is not a message; the processor overwrites it
// with the normalised value, or removes it when the value is invalid.
if matches!(
key,
GenAiAttributes::PROMPT_NAME
| GenAiAttributes::PROMPT_VERSION
| GenAiAttributes::PROMPT_LABEL
) {
return false;
}
key == TRACELOOP_PROMPT_PREFIX
|| key == TRACELOOP_COMPLETION_PREFIX
|| key.starts_with("gen_ai.prompt.")
|| key.starts_with("gen_ai.completion.")
}
fn extract_vercel_ai_sdk(
@@ -404,13 +423,13 @@ impl InputOutputExtractor {
) -> (Option<json::Value>, Option<json::Value>) {
let input_attrs: HashMap<String, json::Value> = attributes
.iter()
.filter(|(k, _)| k.starts_with("gen_ai.prompt"))
.filter(|(k, _)| is_traceloop_nested_key(k, TRACELOOP_PROMPT_PREFIX))
.map(|(k, v)| (k.clone(), v.clone()))
.collect();
let output_attrs: HashMap<String, json::Value> = attributes
.iter()
.filter(|(k, _)| k.starts_with("gen_ai.completion"))
.filter(|(k, _)| is_traceloop_nested_key(k, TRACELOOP_COMPLETION_PREFIX))
.map(|(k, v)| (k.clone(), v.clone()))
.collect();
@@ -421,7 +440,7 @@ impl InputOutputExtractor {
let input = if !input_attrs.is_empty() {
Some(convert_key_path_to_nested_object(
&input_attrs,
"gen_ai.prompt",
TRACELOOP_PROMPT_PREFIX,
))
} else {
None
@@ -430,7 +449,7 @@ impl InputOutputExtractor {
let output = if !output_attrs.is_empty() {
Some(convert_key_path_to_nested_object(
&output_attrs,
"gen_ai.completion",
TRACELOOP_COMPLETION_PREFIX,
))
} else {
None
@@ -482,6 +501,24 @@ impl InputOutputExtractor {
}
}
/// True for TraceLoop's legacy indexed message keys: `<prefix>` itself or
/// `<prefix>.<N>[.…]` (e.g. `gen_ai.prompt.0.content`). Named keys under the
/// same prefix, such as the prompt reference `gen_ai.prompt.name`, are not
/// TraceLoop messages.
fn is_traceloop_nested_key(key: &str, prefix: &str) -> bool {
let Some(rest) = key.strip_prefix(prefix) else {
return false;
};
if rest.is_empty() {
return true;
}
let Some(rest) = rest.strip_prefix('.') else {
return false;
};
let index = rest.split('.').next().unwrap_or_default();
!index.is_empty() && index.bytes().all(|b| b.is_ascii_digit())
}
/// Converts nested key paths (e.g., "gen_ai.prompt.0.role") to nested objects/arrays
/// This is critical for handling TraceLoop and OpenInference attributes
fn convert_key_path_to_nested_object(
@@ -650,6 +687,92 @@ mod tests {
assert!(!extractor.is_input_output_attribute("gen_ai.request.model"));
}
/// Issue #3138: two chat spans, identical except span B carries the prompt
/// reference (`gen_ai.prompt.name|version|label`). Both must keep their
/// messages, and B must keep the reference attributes.
#[test]
fn test_prompt_reference_does_not_swallow_messages() {
let extractor = InputOutputExtractor;
let input_messages =
json::json!(r#"[{"role":"user","parts":[{"type":"text","content":"What is 2+2?"}]}]"#);
let output_messages =
json::json!(r#"[{"role":"assistant","parts":[{"type":"text","content":"4"}]}]"#);
let span_a: HashMap<String, json::Value> = HashMap::from([
("gen_ai.operation.name".to_string(), json::json!("chat")),
(
GenAiAttributes::INPUT_MESSAGES.to_string(),
input_messages.clone(),
),
(
GenAiAttributes::OUTPUT_MESSAGES.to_string(),
output_messages.clone(),
),
]);
let mut span_b = span_a.clone();
span_b.insert(
GenAiAttributes::PROMPT_NAME.to_string(),
json::json!("support-answer"),
);
span_b.insert(
GenAiAttributes::PROMPT_VERSION.to_string(),
json::json!("3"),
);
span_b.insert(
GenAiAttributes::PROMPT_LABEL.to_string(),
json::json!("production"),
);
for attrs in [&span_a, &span_b] {
let (input, output) = extractor.extract(&[], attrs, "");
assert_eq!(input.as_ref(), Some(&input_messages));
assert_eq!(output.as_ref(), Some(&output_messages));
}
let kept: Vec<&str> = span_b
.keys()
.map(String::as_str)
.filter(|k| !extractor.is_input_output_attribute(k))
.collect();
for key in [
GenAiAttributes::PROMPT_NAME,
GenAiAttributes::PROMPT_VERSION,
GenAiAttributes::PROMPT_LABEL,
] {
assert!(kept.contains(&key), "{key} must be kept");
}
assert!(!kept.contains(&GenAiAttributes::INPUT_MESSAGES));
}
#[test]
fn test_traceloop_nested_ignores_prompt_reference() {
let extractor = InputOutputExtractor;
let attrs = HashMap::from([
("gen_ai.prompt.0.role".to_string(), json::json!("user")),
("gen_ai.prompt.0.content".to_string(), json::json!("Hello")),
("gen_ai.completion.0.content".to_string(), json::json!("Hi")),
(
GenAiAttributes::PROMPT_NAME.to_string(),
json::json!("support-answer"),
),
(
"gen_ai.prompt.variable.customer".to_string(),
json::json!("acme"),
),
]);
let (input, output) = extractor.extract(&[], &attrs, "");
assert_eq!(
input,
Some(json::json!([{"role": "user", "content": "Hello"}]))
);
assert_eq!(output, Some(json::json!([{"content": "Hi"}])));
assert!(extractor.is_input_output_attribute("gen_ai.prompt.0.content"));
assert!(extractor.is_input_output_attribute("gen_ai.completion.0.content"));
assert!(extractor.is_input_output_attribute("gen_ai.prompt.variable.customer"));
assert!(!extractor.is_input_output_attribute(GenAiAttributes::PROMPT_NAME));
}
#[test]
fn test_extract_traceloop_nested() {
let extractor = InputOutputExtractor;
src/core/src/traces/otel/mod.rs
+2 -2
View File
@@ -55,7 +55,7 @@
//! &resource_attributes,
//! Some("openai"),
//! &events,
//! &[], // pass org pricing entries here in production
//! Some(&[]), // org pricing entries in production; None when pricing is disabled
//! span_start_nanos, // span.start_time_unix_nano from OTLP payload
//! &GenAiAgentMappingConfig::default(),
//! );
@@ -84,7 +84,7 @@
//! &resource_attrs,
//! Some("openai"),
//! &events,
//! &[],
//! Some(&[]),
//! 0,
//! &GenAiAgentMappingConfig::default(),
//! );
src/core/src/traces/otel/processor.rs
+393 -78
View File
@@ -23,7 +23,9 @@ use std::collections::HashMap;
use config::{
meta::{
gen_ai::GenAiAgentMappingConfig,
model_pricing::{ModelPricingDefinition, PricingContext},
model_pricing::{
COST_STATUS_COLUMN, CostStatus, MatchKind, ModelPricingDefinition, PricingContext,
},
},
utils::{json, time::parse_timestamp_micro_from_value},
};
@@ -136,12 +138,14 @@ impl OtelIngestionProcessor {
resource_attributes,
scope_name,
events,
org_pricing_entries,
Some(org_pricing_entries),
span_start_nanos,
&GenAiAgentMappingConfig::default(),
);
}
/// Enriches an LLM span. `org_pricing_entries` is `None` when model pricing is disabled
/// (`ZO_MODEL_PRICING_ENABLED=false`): no cost is computed and no cost status is written.
#[allow(clippy::too_many_arguments)]
pub fn process_span_with_pricing_and_agent_mapping(
&self,
@@ -149,7 +153,7 @@ impl OtelIngestionProcessor {
resource_attributes: &HashMap<String, json::Value>,
scope_name: Option<&str>,
events: &[Event],
org_pricing_entries: &[CachedModelPricing],
org_pricing_entries: Option<&[CachedModelPricing]>,
span_start_nanos: u64,
agent_mapping_config: &GenAiAgentMappingConfig,
) {
@@ -167,7 +171,7 @@ impl OtelIngestionProcessor {
.retain(|key, _| !self.input_output_extractor.is_input_output_attribute(key));
// Phase 2: Compute derived values — token counts, cost estimates, defaults.
let (usage, cost) = self.compute_usage_and_cost(
let (usage, cost, cost_status) = self.compute_usage_and_cost(
&extracted,
span_attributes,
resource_attributes,
@@ -176,7 +180,7 @@ impl OtelIngestionProcessor {
);
// Phase 3: Write enriched attributes back to the span.
self.emit_enriched_attributes(span_attributes, &extracted, &usage, &cost);
self.emit_enriched_attributes(span_attributes, &extracted, &usage, &cost, cost_status);
}
// ── Phase 1: Extract raw data ──────────────────────────────────────────
@@ -260,17 +264,27 @@ impl OtelIngestionProcessor {
// ── Phase 2: Compute usage, cost, and defaults ─────────────────────────
/// Returns token usage, cost components and, for a model not priced exactly at a non-zero
/// cost, its [`CostStatus`]. An unpriced span gets no cost entries, not a zero total.
/// `org_pricing_entries` is `None` when model pricing is disabled: nothing is priced and
/// no span is marked.
fn compute_usage_and_cost(
&self,
extracted: &SpanExtractions,
span_attributes: &HashMap<String, json::Value>,
resource_attributes: &HashMap<String, json::Value>,
org_pricing_entries: &[CachedModelPricing],
org_pricing_entries: Option<&[CachedModelPricing]>,
span_start_nanos: u64,
) -> (HashMap<String, i64>, HashMap<String, f64>) {
) -> (
HashMap<String, i64>,
HashMap<String, f64>,
Option<CostStatus>,
) {
let mut usage = extracted.usage.clone();
let mut cost = extracted.cost.clone();
let mut input_includes_cache = extracted.input_includes_cache;
let mut cost_status = None;
let mut zero_priced = false;
if is_generation_or_embedding(extracted.op_name) {
// Zero means the payload carried no start time; treat it as unknown rather
@@ -305,42 +319,69 @@ impl OtelIngestionProcessor {
let (billable_usage, tier_usage) = build_pricing_usage(&usage, input_includes_cache);
// Resolve only for spans without a client-reported cost; the lookup runs regexes.
// A span with tokens but no model name has nothing to price: it is unpriced, not
// free. One without tokens stays unmarked. With pricing disabled nothing is priced
// or marked: the span keeps only a reported cost.
if cost.is_empty()
&& let Some(resolved) = extracted.model_name.as_ref().and_then(|mn| {
crate::db::model_pricing::find_pricing_sync_at(
org_pricing_entries,
mn,
span_ts_micros,
)
})
&& org_pricing_entries.is_some()
&& extracted.model_name.is_none()
&& usage.values().any(|&v| v > 0)
{
let pricing_def = &resolved.definition;
let ctx = pricing_context(
pricing_def,
span_attributes,
resource_attributes,
&extracted.model_params,
cost_status = Some(CostStatus::Unpriced);
} else if cost.is_empty()
&& let Some(org_pricing_entries) = org_pricing_entries
&& let Some(model_name) = extracted.model_name.as_deref()
{
let resolved = crate::db::model_pricing::find_pricing_sync_at(
org_pricing_entries,
model_name,
span_ts_micros,
);
let result =
crate::db::model_pricing::calculate_cost_from_definition_with_tier_usage(
let mut priced = false;
if let Some(resolved) = resolved {
let pricing_def = &resolved.definition;
let ctx = pricing_context(
pricing_def,
&billable_usage,
&tier_usage,
span_ts_micros,
&ctx,
span_attributes,
resource_attributes,
&extracted.model_params,
);
if !result.cost.is_empty() {
log::debug!(
"[model_pricing] model='{}' pattern='{}' match={:?} tier='{}' total_cost={:.8}",
extracted.model_name.as_deref().unwrap_or(""),
pricing_def.match_pattern,
resolved.match_kind,
result.tier_name,
result.cost.get("total").copied().unwrap_or(0.0),
);
cost = result.cost;
add_cache_derived_costs(&mut cost, &tier_usage, &result.prices);
cost.values_mut().for_each(|v| *v = round_cost(*v));
let result =
crate::db::model_pricing::calculate_cost_from_definition_with_tier_usage(
pricing_def,
&billable_usage,
&tier_usage,
span_ts_micros,
&ctx,
);
// A selected tier prices the span even when its cost is zero (no tokens,
// or a free model); only "no definition / no tier" is unpriced.
priced = result.tier_index.is_some();
if priced && resolved.match_kind == MatchKind::Approximate {
cost_status = Some(CostStatus::Approximate);
}
// Free means the exact match's selected tier prices every unit at $0, not
// that this span happened to cost $0 (a priced model's zero-token span).
zero_priced = priced
&& resolved.match_kind == MatchKind::Exact
&& !result.prices.is_empty()
&& result.prices.values().all(|&price| price == 0.0);
if !result.cost.is_empty() {
log::debug!(
"[model_pricing] model='{}' pattern='{}' match={:?} tier='{}' total_cost={:.8}",
model_name,
pricing_def.match_pattern,
resolved.match_kind,
result.tier_name,
result.cost.get("total").copied().unwrap_or(0.0),
);
cost = result.cost;
add_cache_derived_costs(&mut cost, &tier_usage, &result.prices);
cost.values_mut().for_each(|v| *v = round_cost(*v));
}
}
if !priced && usage.values().any(|&tokens| tokens > 0) {
cost_status = Some(CostStatus::Unpriced);
}
}
}
@@ -363,13 +404,20 @@ impl OtelIngestionProcessor {
usage.insert("total".to_string(), input + cache_input + output);
}
// Ensure cost total (sum all components, not just input+output).
if !cost.contains_key("total") {
// Ensure cost total (sum all components, not just input+output). An unpriced span keeps
// no cost at all so readers can tell "no price" from "free".
if cost_status != Some(CostStatus::Unpriced) && !cost.contains_key("total") {
let total = actual_cost_total(&cost);
cost.insert("total".to_string(), total);
}
// An exact all-zero price (a free or muted model) is still a price; mark it so readers
// do not take the zero for a legacy unpriced span. A priced model's zero-token span
// stays unmarked.
if zero_priced && cost.get("total").copied() == Some(0.0) {
cost_status = Some(CostStatus::Free);
}
(usage, cost)
(usage, cost, cost_status)
}
// ── Phase 3: Emit enriched attributes ──────────────────────────────────
@@ -380,6 +428,7 @@ impl OtelIngestionProcessor {
extracted: &SpanExtractions,
usage: &HashMap<String, i64>,
cost: &HashMap<String, f64>,
cost_status: Option<CostStatus>,
) {
span_attributes.insert(
GenAiAttributes::OPERATION_NAME.to_string(),
@@ -512,6 +561,20 @@ impl OtelIngestionProcessor {
if let Some(&v) = cost.get("total") {
insert_if_absent(span_attributes, GenAiAttributes::USAGE_COST, json::json!(v));
}
// Only ingest sets the marker; a client-sent value in either spelling (both flatten to
// the same column) would fake one or collide with the ingest value.
span_attributes.remove(GenAiExtensions::USAGE_COST_STATUS);
span_attributes.remove(COST_STATUS_COLUMN);
if let Some(status) = cost_status {
if status == CostStatus::Unpriced {
// A client-sent zero is not a price; leave the cost null.
span_attributes.remove(GenAiAttributes::USAGE_COST);
}
span_attributes.insert(
GenAiExtensions::USAGE_COST_STATUS.to_string(),
json::json!(status.as_str()),
);
}
if let Some(uid) = &extracted.user_id {
span_attributes.insert(OtelAttributes::USER_ID.to_string(), json::json!(uid));
@@ -524,20 +587,27 @@ impl OtelIngestionProcessor {
);
}
if let Some(pname) = &extracted.prompt_name {
span_attributes.insert(GenAiAttributes::PROMPT_NAME.to_string(), json::json!(pname));
}
if let Some(version) = extracted.prompt_version {
span_attributes.insert(
GenAiAttributes::PROMPT_VERSION.to_string(),
json::json!(version),
);
}
if let Some(label) = &extracted.prompt_label {
span_attributes.insert(
GenAiAttributes::PROMPT_LABEL.to_string(),
json::json!(label),
);
// The raw prompt reference survives extraction; replace it with the normalised value,
// or drop it when the extractor rejected it (a non-string name, a version that is not a
// positive integer), so `gen_ai_prompt_version` stays an Int64 column.
for (key, value) in [
(
GenAiAttributes::PROMPT_NAME,
extracted.prompt_name.as_deref().map(|v| json::json!(v)),
),
(
GenAiAttributes::PROMPT_VERSION,
extracted.prompt_version.map(|v| json::json!(v)),
),
(
GenAiAttributes::PROMPT_LABEL,
extracted.prompt_label.as_deref().map(|v| json::json!(v)),
),
] {
match value {
Some(value) => span_attributes.insert(key.to_string(), value),
None => span_attributes.remove(key),
};
}
// TTFT: convert microseconds → Float64 seconds.
@@ -1011,7 +1081,7 @@ mod tests {
&HashMap::new(),
None,
&[],
&[],
Some(&[]),
0,
&config,
);
@@ -1233,6 +1303,51 @@ mod tests {
assert!(!span_attrs.contains_key("gen_ai_prompt_variable_customer"));
}
#[test]
fn process_span_replaces_or_drops_raw_prompt_reference() {
let processor = OtelIngestionProcessor::new();
// Rejected by the extractor: a non-numeric version and a non-string name.
let mut rejected = HashMap::from([
(GenAiAttributes::PROMPT_NAME.to_string(), json::json!(42)),
(
GenAiAttributes::PROMPT_VERSION.to_string(),
json::json!("latest"),
),
(
GenAiAttributes::PROMPT_LABEL.to_string(),
json::json!("production"),
),
]);
processor.process_span(&mut rejected, &HashMap::new(), None, &[]);
assert!(!rejected.contains_key(GenAiAttributes::PROMPT_VERSION));
assert!(!rejected.contains_key(GenAiAttributes::PROMPT_NAME));
assert_eq!(
rejected.get(GenAiAttributes::PROMPT_LABEL),
Some(&json::json!("production"))
);
// A numeric string version is stored as the integer it names.
let mut accepted = HashMap::from([
(
GenAiAttributes::PROMPT_NAME.to_string(),
json::json!("support-answer"),
),
(
GenAiAttributes::PROMPT_VERSION.to_string(),
json::json!("3"),
),
]);
processor.process_span(&mut accepted, &HashMap::new(), None, &[]);
assert_eq!(
accepted.get(GenAiAttributes::PROMPT_VERSION),
Some(&json::json!(3))
);
assert_eq!(
accepted.get(GenAiAttributes::PROMPT_NAME),
Some(&json::json!("support-answer"))
);
}
#[test]
fn test_process_span_calculates_cost_from_tokens() {
let processor = OtelIngestionProcessor::new();
@@ -1508,39 +1623,93 @@ mod tests {
);
}
#[test]
fn test_process_span_no_cost_calculation_for_unknown_model() {
let processor = OtelIngestionProcessor::new();
fn chat_span(model: &str) -> HashMap<String, json::Value> {
let mut span_attrs = HashMap::new();
span_attrs.insert("gen_ai.operation.name".to_string(), json::json!("chat"));
span_attrs.insert(
"gen_ai.request.model".to_string(),
json::json!("unknown-model-xyz-2024"),
);
span_attrs.insert(
"gen_ai.input.messages".to_string(),
json::json!("[{\"role\":\"user\"}]"),
);
span_attrs.insert(
"gen_ai.output.messages".to_string(),
json::json!("[{\"role\":\"assistant\"}]"),
);
span_attrs.insert("gen_ai.request.model".to_string(), json::json!(model));
span_attrs.insert("gen_ai.usage.input_tokens".to_string(), json::json!(1000));
span_attrs.insert("gen_ai.usage.output_tokens".to_string(), json::json!(500));
span_attrs
}
let resource_attrs = HashMap::new();
let events = vec![];
fn cost_status(span_attrs: &HashMap<String, json::Value>) -> Option<&str> {
span_attrs
.get(GenAiExtensions::USAGE_COST_STATUS)
.and_then(|v| v.as_str())
}
processor.process_span(&mut span_attrs, &resource_attrs, None, &events);
#[test]
fn test_unpriced_model_keeps_tokens_and_has_no_cost() {
let processor = OtelIngestionProcessor::new();
let mut span_attrs = chat_span("unknown-model-xyz-2024");
// A client-sent zero is not a price either.
span_attrs.insert("gen_ai.usage.cost".to_string(), json::json!(0));
// No pricing definition matched for unknown model — total cost should be zero.
processor.process_span(&mut span_attrs, &HashMap::new(), None, &[]);
assert_eq!(cost_status(&span_attrs), Some("unpriced"));
assert!(!span_attrs.contains_key(GenAiAttributes::USAGE_COST));
assert!(!span_attrs.contains_key(GenAiExtensions::USAGE_COST_INPUT));
assert!(!span_attrs.contains_key(GenAiExtensions::USAGE_COST_OUTPUT));
assert_eq!(
span_attrs
span_attrs.get(GenAiAttributes::USAGE_TOTAL_TOKENS),
Some(&json::json!(1500))
);
}
#[test]
fn test_approximate_match_keeps_cost_and_is_marked() {
let processor = OtelIngestionProcessor::new();
// `(?i)claude-sonnet-4-6` covers only the start of the dated name.
let mut span_attrs = chat_span("claude-sonnet-4-6-20260101");
processor.process_span(&mut span_attrs, &HashMap::new(), None, &[]);
assert_eq!(cost_status(&span_attrs), Some("approximate"));
let total = span_attrs
.get(GenAiAttributes::USAGE_COST)
.and_then(|v| v.as_f64())
.unwrap();
assert!((total - (1000.0 * 3e-06 + 500.0 * 1.5e-05)).abs() < 1e-12);
}
#[test]
fn test_exact_match_and_client_cost_write_no_marker() {
let processor = OtelIngestionProcessor::new();
let mut exact = chat_span("gpt-4o");
exact.insert(
"gen_ai.usage.cost_status".to_string(),
json::json!("unpriced"),
);
processor.process_span(&mut exact, &HashMap::new(), None, &[]);
assert_eq!(cost_status(&exact), None);
assert!(exact.contains_key(GenAiAttributes::USAGE_COST));
let mut client_cost = chat_span("unknown-model-xyz-2024");
client_cost.insert("gen_ai.usage.cost".to_string(), json::json!(0.25));
processor.process_span(&mut client_cost, &HashMap::new(), None, &[]);
assert_eq!(cost_status(&client_cost), None);
assert_eq!(
client_cost
.get(GenAiAttributes::USAGE_COST)
.and_then(|v| v.as_f64()),
Some(0.0)
Some(0.25)
);
// The flattened spelling lands in the same column and must not survive either.
let mut flattened = chat_span("gpt-4o");
flattened.insert(COST_STATUS_COLUMN.to_string(), json::json!("unpriced"));
processor.process_span(&mut flattened, &HashMap::new(), None, &[]);
assert_eq!(cost_status(&flattened), None);
assert!(!flattened.contains_key(COST_STATUS_COLUMN));
assert!(flattened.contains_key(GenAiAttributes::USAGE_COST));
// An ingest-set marker replaces a client-sent flattened one instead of colliding.
let mut unpriced = chat_span("unknown-model-xyz-2024");
unpriced.insert(COST_STATUS_COLUMN.to_string(), json::json!("approximate"));
processor.process_span(&mut unpriced, &HashMap::new(), None, &[]);
assert_eq!(cost_status(&unpriced), Some("unpriced"));
assert!(!unpriced.contains_key(COST_STATUS_COLUMN));
}
#[test]
@@ -1594,6 +1763,152 @@ mod tests {
);
}
#[test]
fn test_no_model_with_tokens_is_unpriced_without_tokens_unmarked() {
let processor = OtelIngestionProcessor::new();
let mut with_tokens = chat_span("x");
with_tokens.remove("gen_ai.request.model");
processor.process_span(&mut with_tokens, &HashMap::new(), None, &[]);
assert_eq!(cost_status(&with_tokens), Some("unpriced"));
assert!(!with_tokens.contains_key(GenAiAttributes::USAGE_COST));
assert_eq!(
with_tokens.get(GenAiAttributes::USAGE_TOTAL_TOKENS),
Some(&json::json!(1500))
);
let mut no_tokens = HashMap::new();
no_tokens.insert("gen_ai.operation.name".to_string(), json::json!("chat"));
processor.process_span(&mut no_tokens, &HashMap::new(), None, &[]);
assert_eq!(cost_status(&no_tokens), None);
no_tokens.insert(
"gen_ai.request.model".to_string(),
json::json!("unknown-zero-usage"),
);
processor.process_span(&mut no_tokens, &HashMap::new(), None, &[]);
assert_eq!(cost_status(&no_tokens), None);
// A client-reported cost still wins over the missing model.
let mut client_cost = chat_span("x");
client_cost.remove("gen_ai.request.model");
client_cost.insert("gen_ai.usage.cost".to_string(), json::json!(0.25));
processor.process_span(&mut client_cost, &HashMap::new(), None, &[]);
assert_eq!(cost_status(&client_cost), None);
}
fn zero_price_entry(model: &str) -> CachedModelPricing {
use config::meta::model_pricing::PricingTierDefinition;
let pattern = format!("^{model}$");
CachedModelPricing {
definition: ModelPricingDefinition {
name: format!("{model} (muted)"),
match_pattern: pattern.clone(),
enabled: true,
tiers: vec![PricingTierDefinition {
name: "Default".to_string(),
condition: None,
prices: HashMap::from([
("input".to_string(), 0.0),
("output".to_string(), 0.0),
]),
utc_windows: Vec::new(),
rules: Vec::new(),
priority: 0,
}],
..Default::default()
},
compiled_regex: regex::Regex::new(&pattern).unwrap(),
}
}
#[test]
fn test_exact_zero_price_with_tokens_is_marked_free() {
let processor = OtelIngestionProcessor::new();
let entries = vec![zero_price_entry("acme-internal-llm")];
let mut muted = chat_span("acme-internal-llm");
processor.process_span_with_pricing(&mut muted, &HashMap::new(), None, &[], &entries, 0);
assert_eq!(cost_status(&muted), Some("free"));
assert_eq!(
muted
.get(GenAiAttributes::USAGE_COST)
.and_then(|v| v.as_f64()),
Some(0.0)
);
// An exactly matched zero price is free even when no tokens were used.
let mut no_tokens = HashMap::new();
no_tokens.insert("gen_ai.operation.name".to_string(), json::json!("chat"));
no_tokens.insert(
"gen_ai.request.model".to_string(),
json::json!("acme-internal-llm"),
);
processor.process_span_with_pricing(
&mut no_tokens,
&HashMap::new(),
None,
&[],
&entries,
0,
);
assert_eq!(cost_status(&no_tokens), Some("free"));
}
#[test]
fn test_zero_token_span_of_priced_model_is_not_free() {
let processor = OtelIngestionProcessor::new();
let mut entry = zero_price_entry("acme-paid-llm");
entry.definition.tiers[0].prices = HashMap::from([
("input".to_string(), 0.000001),
("output".to_string(), 0.000002),
]);
let entries = vec![entry];
let mut no_tokens = HashMap::new();
no_tokens.insert("gen_ai.operation.name".to_string(), json::json!("chat"));
no_tokens.insert(
"gen_ai.request.model".to_string(),
json::json!("acme-paid-llm"),
);
processor.process_span_with_pricing(
&mut no_tokens,
&HashMap::new(),
None,
&[],
&entries,
0,
);
assert_eq!(cost_status(&no_tokens), None);
}
#[test]
fn test_pricing_disabled_writes_no_marker() {
let processor = OtelIngestionProcessor::new();
for model in [Some("unknown-model-xyz-2024"), Some("gpt-4o"), None] {
let mut span_attrs = chat_span(model.unwrap_or("x"));
if model.is_none() {
span_attrs.remove("gen_ai.request.model");
}
processor.process_span_with_pricing_and_agent_mapping(
&mut span_attrs,
&HashMap::new(),
None,
&[],
None,
0,
&GenAiAgentMappingConfig::default(),
);
assert_eq!(cost_status(&span_attrs), None, "{model:?}");
assert!(!span_attrs.contains_key(COST_STATUS_COLUMN), "{model:?}");
// Tokens are still counted; nothing is priced.
assert_eq!(
span_attrs.get(GenAiAttributes::USAGE_TOTAL_TOKENS),
Some(&json::json!(1500))
);
assert!(!span_attrs.contains_key(GenAiExtensions::USAGE_COST_INPUT));
}
}
#[test]
fn test_process_span_extracts_evaluation_scores() {
let processor = OtelIngestionProcessor::new();
src/db/src/schema.rs
+6
View File
@@ -232,6 +232,12 @@ static GEN_AI_SCHEMA_FIELDS: std::sync::LazyLock<Vec<Field>> = std::sync::LazyLo
// written for label-resolved calls, so it must be provisioned up front.
Field::new("gen_ai_prompt_name", DataType::Utf8, true),
Field::new("gen_ai_prompt_label", DataType::Utf8, true),
// Written only for unpriced/approximate spans, so provision it for the readers.
Field::new(
config::meta::model_pricing::COST_STATUS_COLUMN,
DataType::Utf8,
true,
),
// Integer fields
Field::new("gen_ai_prompt_version", DataType::Int64, true),
Field::new("gen_ai_usage_input_tokens", DataType::Int64, true),
src/infra/src/table/entity/llm_dataset_items.rs
+35
View File
@@ -19,6 +19,12 @@ pub struct Model {
pub input: Json,
pub expected_output: Json,
pub global_version: i64,
/// The Dataset version shown for the write request that stored this row; every row of one
/// request shares it, while `global_version` stays unique per row. NULL for rows written
/// before the column existed or by an older node: such a row is its own write, numbered by
/// `global_version`. Display only: two regions can label different writes with one value.
#[serde(default)]
pub write_version: Option<i64>,
pub is_deleted: bool,
pub source: String,
pub source_ref: Option<String>,
@@ -36,3 +42,32 @@ pub struct Model {
pub enum Relation {}
impl ActiveModelBehavior for ActiveModel {}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn replicated_rows_preserve_write_versions_and_accept_older_payloads() {
// EvalDatasetMessage carries this row as JSON; old nodes omit the new column.
let legacy = serde_json::json!({
"row_id": "row-1",
"logical_id": "item-1",
"org_id": "org-1",
"dataset_id": "dataset-1",
"input": "question",
"expected_output": "answer",
"global_version": 59,
"is_deleted": false,
"source": "manual",
"updated_by": "owner@example.com",
"updated_at": 100
});
let mut row: Model = serde_json::from_value(legacy).unwrap();
assert_eq!(row.write_version, None);
row.write_version = Some(57);
let payload = serde_json::to_value(&row).unwrap();
assert_eq!(payload["write_version"], 57);
assert_eq!(serde_json::from_value::<Model>(payload).unwrap(), row);
}
}
src/infra/src/table/migration/m20261028_000001_add_llm_dataset_item_write_version.rs
+222
View File
@@ -0,0 +1,222 @@
// Copyright 2026 OpenObserve Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
//! Adds the nullable `write_version` to `llm_dataset_items`: the Dataset version a row's write
//! request was shown as. NULL marks a row written before this column or by an older node.
//! The existing row-version indexes are untouched; one new index lets a write find the Dataset's
//! highest label (and the highest legacy row version) without scanning its history.
use sea_orm_migration::prelude::*;
const LLM_DATASET_ITEMS: &str = "llm_dataset_items";
const WRITE_VERSION: &str = "write_version";
const WRITE_VERSION_IDX: &str = "idx_llm_dataset_items_write_version";
#[derive(DeriveMigrationName)]
pub struct Migration;
#[async_trait::async_trait]
impl MigrationTrait for Migration {
async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> {
if !manager.has_column(LLM_DATASET_ITEMS, WRITE_VERSION).await? {
manager
.alter_table(
Table::alter()
.table(LlmDatasetItems::Table)
.add_column(
ColumnDef::new(LlmDatasetItems::WriteVersion)
.big_integer()
.null(),
)
.to_owned(),
)
.await?;
}
if !manager
.has_index(LLM_DATASET_ITEMS, WRITE_VERSION_IDX)
.await?
{
manager
.create_index(
Index::create()
.name(WRITE_VERSION_IDX)
.table(LlmDatasetItems::Table)
.col(LlmDatasetItems::DatasetId)
.col(LlmDatasetItems::WriteVersion)
.col(LlmDatasetItems::GlobalVersion)
.to_owned(),
)
.await?;
}
Ok(())
}
async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> {
if manager
.has_index(LLM_DATASET_ITEMS, WRITE_VERSION_IDX)
.await?
{
manager
.drop_index(
Index::drop()
.name(WRITE_VERSION_IDX)
.table(LlmDatasetItems::Table)
.to_owned(),
)
.await?;
}
if !manager.has_column(LLM_DATASET_ITEMS, WRITE_VERSION).await? {
return Ok(());
}
manager
.alter_table(
Table::alter()
.table(LlmDatasetItems::Table)
.drop_column(LlmDatasetItems::WriteVersion)
.to_owned(),
)
.await
}
}
#[derive(DeriveIden)]
enum LlmDatasetItems {
Table,
DatasetId,
WriteVersion,
GlobalVersion,
}
#[cfg(test)]
mod tests {
use sea_orm::{ConnectionTrait, Database, DatabaseConnection, Statement};
use sea_orm_migration::MigrationName;
use super::*;
#[test]
fn test_migration_name() {
assert_eq!(
Migration.name(),
"m20261028_000001_add_llm_dataset_item_write_version"
);
}
async fn items_db() -> DatabaseConnection {
let db = Database::connect("sqlite::memory:").await.unwrap();
for sql in [
"CREATE TABLE llm_dataset_items (row_id TEXT PRIMARY KEY, dataset_id TEXT NOT NULL, \
logical_id TEXT NOT NULL, global_version BIGINT NOT NULL)",
"CREATE UNIQUE INDEX uq_llm_dataset_items_version ON llm_dataset_items (dataset_id, \
global_version)",
"CREATE INDEX idx_llm_dataset_items_latest ON llm_dataset_items (dataset_id, \
logical_id, global_version)",
"INSERT INTO llm_dataset_items VALUES ('row-1', 'dataset-1', 'item-1', 1)",
] {
db.execute_unprepared(sql).await.unwrap();
}
db
}
async fn index_names(db: &DatabaseConnection) -> Vec<String> {
let rows = db
.query_all(Statement::from_string(
db.get_database_backend(),
"SELECT name FROM sqlite_master WHERE type = 'index' AND tbl_name = \
'llm_dataset_items' AND sql IS NOT NULL ORDER BY name",
))
.await
.unwrap();
rows.iter()
.map(|row| row.try_get::<String>("", "name").unwrap())
.collect()
}
async fn insert(db: &DatabaseConnection, row_id: &str, version: i64) -> bool {
db.execute(Statement::from_sql_and_values(
db.get_database_backend(),
"INSERT INTO llm_dataset_items (row_id, dataset_id, logical_id, global_version, \
write_version) VALUES (?, 'dataset-1', ?, ?, 2)",
[row_id.into(), row_id.into(), version.into()],
))
.await
.is_ok()
}
#[tokio::test]
async fn up_adds_a_nullable_column_and_one_head_index_keeping_the_version_indexes() {
let db = items_db().await;
let manager = SchemaManager::new(&db);
let mut indexes = index_names(&db).await;
Migration.up(&manager).await.unwrap();
Migration.up(&manager).await.unwrap();
assert!(
manager
.has_column(LLM_DATASET_ITEMS, WRITE_VERSION)
.await
.unwrap()
);
indexes.push(WRITE_VERSION_IDX.to_string());
indexes.sort();
assert_eq!(index_names(&db).await, indexes);
// The pre-existing row reads as NULL: a legacy write.
let legacy = db
.query_one(Statement::from_string(
db.get_database_backend(),
"SELECT write_version FROM llm_dataset_items WHERE row_id = 'row-1'",
))
.await
.unwrap()
.unwrap();
assert_eq!(
legacy.try_get::<Option<i64>>("", WRITE_VERSION).unwrap(),
None
);
// Rows of one write share a write version but never a row version.
assert!(insert(&db, "row-2", 2).await);
assert!(insert(&db, "row-3", 3).await);
assert!(!insert(&db, "row-4", 3).await);
}
#[tokio::test]
async fn down_drops_the_column_and_its_index() {
let db = items_db().await;
let manager = SchemaManager::new(&db);
let indexes = index_names(&db).await;
Migration.up(&manager).await.unwrap();
assert!(insert(&db, "row-2", 2).await);
Migration.down(&manager).await.unwrap();
Migration.down(&manager).await.unwrap();
assert!(
!manager
.has_column(LLM_DATASET_ITEMS, WRITE_VERSION)
.await
.unwrap()
);
assert_eq!(index_names(&db).await, indexes);
let rows = db
.query_all(Statement::from_string(
db.get_database_backend(),
"SELECT row_id FROM llm_dataset_items",
))
.await
.unwrap();
assert_eq!(rows.len(), 2);
}
}
src/infra/src/table/migration/mod.rs
+4 -1
View File
@@ -243,6 +243,7 @@ mod m20261025_000005_add_held_action_to_slack_link_requests;
mod m20261026_000001_backfill_v1_rca_pointer;
mod m20261027_000001_alter_query_history;
mod m20261028_000001_add_group_label_set_to_alert_states;
mod m20261028_000001_add_llm_dataset_item_write_version;
mod m20261029_000001_widen_alert_dedup_fingerprint;
mod m20261030_000001_add_suppress_repeats_to_alerts;
mod m20261030_000001_create_org_domain_ownership_table;
@@ -770,6 +771,7 @@ impl MigratorTrait for Migrator {
Box::new(m20261030_000002_create_org_domain_ownership_req_table::Migration),
Box::new(m20261101_000001_create_oncall_slack::Migration),
Box::new(m20261103_000001_add_status_streaks_to_synthetics::Migration),
Box::new(m20261028_000001_add_llm_dataset_item_write_version::Migration),
Box::new(m20261103_000002_add_run_class_to_synthetics::Migration),
]
}
@@ -872,7 +874,8 @@ mod tests {
),
(132, "m20261101_000001_create_oncall_slack"),
(133, "m20261103_000001_add_status_streaks_to_synthetics"),
(134, "m20261103_000002_add_run_class_to_synthetics"),
(134, "m20261028_000001_add_llm_dataset_item_write_version"),
(135, "m20261103_000002_add_run_class_to_synthetics"),
];
// Two branches that each claim the next version merge cleanly, so a reused number must fail
src/infra/src/table/model_pricing.rs
+98 -7
View File
@@ -21,8 +21,8 @@ use config::{
utils::json,
};
use sea_orm::{
ActiveModelTrait, ColumnTrait, DatabaseConnection, EntityTrait, ModelTrait, QueryFilter,
QueryOrder, QuerySelect, Set, TryIntoModel,
ActiveModelTrait, ColumnTrait, ConnectionTrait, DatabaseConnection, EntityTrait, ModelTrait,
QueryFilter, QueryOrder, QuerySelect, Set, TryIntoModel, sea_query::OnConflict,
};
use crate::{
@@ -136,6 +136,49 @@ pub async fn put(item: ModelPricingDefinition) -> Result<ModelPricingDefinition,
}
}
/// Fill missing built-ins atomically without replacing synced or disabled prices.
pub async fn insert_missing_built_ins(items: Vec<ModelPricingDefinition>) -> Result<u64, Error> {
insert_missing_built_ins_with_db(get_orm_client_rw().await, items).await
}
async fn insert_missing_built_ins_with_db<C: ConnectionTrait>(
db: &C,
items: Vec<ModelPricingDefinition>,
) -> Result<u64, Error> {
if items.is_empty() {
return Ok(0);
}
let now = chrono::Utc::now().timestamp_micros();
let rows = items
.into_iter()
.map(|item| {
Ok(ActiveModel {
id: Set(ider::uuid()),
org: Set(config::meta::model_pricing::BUILT_IN_ORG.to_string()),
name: Set(item.name),
match_pattern: Set(item.match_pattern),
enabled: Set(item.enabled),
tiers: Set(json::to_value(&item.tiers)?),
valid_from: Set(item.valid_from),
sort_order: Set(item.sort_order),
source: Set(PricingSource::BuiltIn.as_str().to_string()),
provider: Set((!item.provider.is_empty()).then_some(item.provider)),
description: Set((!item.description.is_empty()).then_some(item.description)),
created_at: Set(now),
updated_at: Set(now),
})
})
.collect::<Result<Vec<_>, Error>>()?;
Ok(Entity::insert_many(rows)
.on_conflict(
OnConflict::columns([Column::Org, Column::Name])
.do_nothing()
.to_owned(),
)
.exec_without_returning(db)
.await?)
}
pub async fn get(org_id: &str, name: &str) -> Result<Option<ModelPricingDefinition>, Error> {
let client = get_orm_client_ro().await;
match get_by_org_and_name(client, org_id, name).await? {
@@ -153,9 +196,9 @@ pub async fn get_by_id(id: &str) -> Result<Option<ModelPricingDefinition>, Error
}
/// Return the distinct org IDs that have at least one model pricing definition.
/// Used for startup cache warming.
/// Startup and coordinator-driven cache loads must see committed prices, not replica lag.
pub async fn list_orgs() -> Result<Vec<String>, Error> {
let client = get_orm_client_ro().await;
let client = get_orm_client_rw().await;
let rows: Vec<(String,)> = Entity::find()
.select_only()
.column(Column::Org)
@@ -167,7 +210,7 @@ pub async fn list_orgs() -> Result<Vec<String>, Error> {
}
pub async fn list(org_id: &str) -> Result<Vec<ModelPricingDefinition>, Error> {
let client = get_orm_client_ro().await;
let client = get_orm_client_rw().await;
let rows = Entity::find()
.filter(Column::Org.eq(org_id))
.order_by(Column::Name, sea_orm::Order::Asc)
@@ -217,9 +260,10 @@ pub async fn delete_by_org(org_id: &str) -> Result<(), Error> {
}
/// Return all model names belonging to the built-in org.
/// Used by the sync job to detect models removed from the upstream source.
/// Used for synchronization and seed initialization; read the primary so a
/// lagging replica cannot make a stored built-in list appear empty.
pub async fn list_built_in_names() -> Result<Vec<String>, Error> {
let client = get_orm_client_ro().await;
let client = get_orm_client_rw().await;
let rows: Vec<(String,)> = Entity::find()
.select_only()
.column(Column::Name)
@@ -266,6 +310,53 @@ mod tests {
}
}
#[tokio::test]
async fn seed_retries_missing_names_without_overwriting_synced_prices() {
let db = sea_orm::Database::connect("sqlite::memory:").await.unwrap();
let backend = db.get_database_backend();
let schema = sea_orm::Schema::new(backend);
db.execute(backend.build(&schema.create_table_from_entity(Entity)))
.await
.unwrap();
db.execute_unprepared("CREATE UNIQUE INDEX seed_model_names ON model_pricing (org, name)")
.await
.unwrap();
let id = svix_ksuid::Ksuid::new(None, None).to_string();
let mut stored: ModelPricingDefinition = make_model(&id).try_into().unwrap();
stored.enabled = false;
stored.match_pattern = "^newer-synced-pattern$".to_string();
assert_eq!(
insert_missing_built_ins_with_db(&db, vec![stored.clone()])
.await
.unwrap(),
1
);
let mut old_seed = stored.clone();
old_seed.enabled = true;
old_seed.match_pattern = "^old-seed-pattern$".to_string();
let mut missing = old_seed.clone();
missing.name = "missing-model".to_string();
assert_eq!(
insert_missing_built_ins_with_db(&db, vec![old_seed.clone(), missing.clone()])
.await
.unwrap(),
1
);
let saved =
get_by_org_and_name(&db, config::meta::model_pricing::BUILT_IN_ORG, &stored.name)
.await
.unwrap()
.unwrap();
assert!(!saved.enabled);
assert_eq!(saved.match_pattern, "^newer-synced-pattern$");
assert_eq!(
insert_missing_built_ins_with_db(&db, vec![old_seed, missing])
.await
.unwrap(),
0
);
}
#[test]
fn test_try_from_model_valid_ksuid() {
let id = svix_ksuid::Ksuid::new(None, None).to_string();
src/infra/src/table/score_configs.rs
+8 -9
View File
@@ -245,27 +245,26 @@ pub async fn get_versions(
Ok(records)
}
/// Updating a score config deactivates its previous versions, so each entity
/// has one active row; the dedupe only guards legacy rows written before that
/// rule. Reads the primary: the web refetches this list right after a save, and a
/// lagging read-only replica would return the pre-save rows.
pub async fn get_all_by_org(org_id: &str) -> Result<Vec<ScoreConfig>, errors::Error> {
let client = get_orm_client_rw().await;
// Get the latest version per logical entity.
let records: Vec<Model> = Entity::find()
.filter(Column::OrgId.eq(org_id))
.filter(Column::IsActive.eq(true))
.order_by(Column::Name, Order::Asc)
.order_by(Column::Version, Order::Desc)
.all(client)
.all(get_orm_client_rw().await)
.await?;
// Deduplicate: keep only the highest version per entity.
// Keep the highest version per logical entity.
let mut seen = std::collections::HashSet::new();
let records: Vec<_> = records
Ok(records
.into_iter()
.map(ScoreConfig::from)
.filter(|c| seen.insert(c.entity_id.clone()))
.collect();
Ok(records)
.collect())
}
pub async fn delete(entity_id: &str, org_id: &str) -> Result<(), errors::Error> {
src/infra/src/table/scorers.rs
+13 -20
View File
@@ -236,52 +236,45 @@ pub async fn get_versions(org_id: &str, entity_id: &str) -> Result<Vec<Scorer>,
Ok(records)
}
/// Updating a scorer deactivates its previous versions, so each entity has one
/// active row; the dedupe only guards legacy rows written before that rule.
/// Use the primary: provider deletion also uses this list to reject dependencies.
pub async fn get_all_by_org(org_id: &str) -> Result<Vec<Scorer>, errors::Error> {
let client = get_orm_client_rw().await;
let records: Vec<Model> = Entity::find()
.filter(Column::OrgId.eq(org_id))
.filter(Column::IsActive.eq(true))
.order_by(Column::Name, Order::Asc)
.order_by(Column::Version, Order::Desc)
.all(client)
.all(get_orm_client_rw().await)
.await?;
// Deduplicate: keep highest version per logical entity.
let mut seen = std::collections::HashSet::new();
let records: Vec<_> = records
.into_iter()
.map(Scorer::from)
.filter(|s| seen.insert(s.entity_id.clone()))
.collect();
Ok(records)
Ok(latest_per_entity(records))
}
pub async fn get_by_type(
org_id: &str,
scorer_type: &ScorerType,
) -> Result<Vec<Scorer>, errors::Error> {
let client = get_orm_client_rw().await;
let records: Vec<Model> = Entity::find()
.filter(Column::OrgId.eq(org_id))
.filter(Column::ScorerType.eq(*scorer_type))
.filter(Column::IsActive.eq(true))
.order_by(Column::Name, Order::Asc)
.order_by(Column::Version, Order::Desc)
.all(client)
.all(get_orm_client_ro().await)
.await?;
// Deduplicate by logical entity.
Ok(latest_per_entity(records))
}
/// Keep the highest version per logical entity (rows arrive version-descending).
fn latest_per_entity(records: Vec<Model>) -> Vec<Scorer> {
let mut seen = std::collections::HashSet::new();
let records: Vec<_> = records
records
.into_iter()
.map(Scorer::from)
.filter(|s| seen.insert(s.entity_id.clone()))
.collect();
Ok(records)
.collect()
}
pub async fn has_active_by_score_config_id(
src/jobs/src/job/mod.rs
+5 -1
View File
@@ -869,11 +869,15 @@ pub async fn init() -> Result<(), anyhow::Error> {
o2_enterprise::enterprise::common::remote_defaults::spawn_refresher();
if config::get_config().common.model_pricing_enabled {
// Warm from committed seed rows even offline or when coordinator notification fails.
if let Err(e) = openobserve_builtins::model_pricing::load_seed().await {
log::error!("[model_pricing] loading seed prices failed: {e}");
}
db::model_pricing::cache()
.await
.expect("model pricing cache failed");
// Sync built-in model pricing from GitHub (initial + periodic)
// Only queriers perform the initial and periodic GitHub sync.
if LOCAL_NODE.is_querier() {
tokio::task::spawn(async {
if let Err(e) =
src/super_cluster_queue/src/eval_datasets.rs
+149 -5
View File
@@ -11,13 +11,16 @@ use infra::{
table::entity::{llm_dataset_items as dataset_items, llm_datasets as datasets},
};
use o2_enterprise::enterprise::super_cluster::queue::{EvalDatasetMessage, Message, MessageType};
use sea_orm::{ActiveModelTrait, ColumnTrait, EntityTrait, QueryFilter, Set, TransactionTrait};
use sea_orm::{
ActiveModelTrait, ColumnTrait, ConnectionTrait, EntityTrait, QueryFilter, Set,
TransactionTrait, sea_query::Expr,
};
pub(crate) async fn process(msg: Message) -> Result<()> {
match msg.message_type {
MessageType::EvalDatasetPut => {
let EvalDatasetMessage::Put { dataset, items } = msg.try_into()?;
apply_put(dataset, items).await?;
apply_put(get_orm_client_rw().await, dataset, items).await?;
}
MessageType::EvalDatasetDelete => {
let (org_id, dataset_id) =
@@ -36,7 +39,11 @@ pub(crate) async fn process(msg: Message) -> Result<()> {
Ok(())
}
async fn apply_put(dataset: datasets::Model, items: Vec<dataset_items::Model>) -> Result<()> {
async fn apply_put<C: ConnectionTrait + TransactionTrait>(
db: &C,
dataset: datasets::Model,
items: Vec<dataset_items::Model>,
) -> Result<()> {
// Item messages carry the parent only as a version watermark. Dataset
// metadata is authoritative only in a metadata-only snapshot, preventing
// an item written from stale regional state from reverting a rename.
@@ -50,7 +57,6 @@ async fn apply_put(dataset: datasets::Model, items: Vec<dataset_items::Model>) -
));
}
let db = get_orm_client_rw().await;
let txn = db.begin().await?;
match datasets::Entity::find_by_id(&dataset.id).one(&txn).await? {
Some(current) if current.org_id != dataset.org_id => {
@@ -81,11 +87,25 @@ async fn apply_put(dataset: datasets::Model, items: Vec<dataset_items::Model>) -
.one(&txn)
.await?
{
if current != item {
// This region may hold the row without its display label (an older node's echo,
// or a row from before the column). Fill in that one column so MAX(write_version)
// and the snapshot pins agree with the origin region; a stored label stays.
let missing_label = item
.write_version
.filter(|_| current.write_version.is_none());
if !compatible_item_rows(current, &item) {
return Err(Error::Message(
"Dataset Item row ID contains conflicting data".to_string(),
));
}
if let Some(label) = missing_label {
dataset_items::Entity::update_many()
.col_expr(dataset_items::Column::WriteVersion, Expr::value(label))
.filter(dataset_items::Column::RowId.eq(&item.row_id))
.filter(dataset_items::Column::WriteVersion.is_null())
.exec(&txn)
.await?;
}
continue;
}
if dataset_items::Entity::find()
@@ -107,6 +127,18 @@ async fn apply_put(dataset: datasets::Model, items: Vec<dataset_items::Model>) -
Ok(())
}
/// Older nodes omit the display label when echoing an otherwise identical immutable row, so a
/// missing label on either side is not a conflict; two different labels still are.
fn compatible_item_rows(
mut current: dataset_items::Model,
incoming: &dataset_items::Model,
) -> bool {
if current.write_version.is_none() || incoming.write_version.is_none() {
current.write_version = incoming.write_version;
}
current == *incoming
}
async fn apply_delete(org_id: &str, dataset_id: &str) -> Result<()> {
let db = get_orm_client_rw().await;
let txn = db.begin().await?;
@@ -123,3 +155,115 @@ async fn apply_delete(org_id: &str, dataset_id: &str) -> Result<()> {
txn.commit().await?;
Ok(())
}
#[cfg(test)]
mod tests {
use sea_orm::{ConnectOptions, Database, DatabaseConnection, Schema};
use super::*;
fn row(write_version: Option<i64>) -> dataset_items::Model {
serde_json::from_value(serde_json::json!({
"row_id": "row-1",
"logical_id": "item-1",
"org_id": "org-1",
"dataset_id": "dataset-1",
"input": "question",
"expected_output": "answer",
"global_version": 59,
"write_version": write_version,
"is_deleted": false,
"source": "telemetry",
"updated_by": "owner@example.com",
"updated_at": 100
}))
.unwrap()
}
#[test]
fn an_older_node_can_echo_a_row_without_its_display_label() {
assert!(compatible_item_rows(row(Some(57)), &row(None)));
assert!(compatible_item_rows(row(None), &row(Some(57))));
assert!(compatible_item_rows(row(Some(57)), &row(Some(57))));
assert!(!compatible_item_rows(row(Some(57)), &row(Some(58))));
}
#[test]
fn missing_display_labels_do_not_hide_immutable_content_conflicts() {
let mut changed = row(None);
changed.input = serde_json::json!("different question");
assert!(!compatible_item_rows(row(Some(57)), &changed));
let mut changed = row(None);
changed.global_version = 60;
assert!(!compatible_item_rows(row(Some(57)), &changed));
}
// One connection: separate connections to `sqlite::memory:` get separate databases.
async fn db() -> DatabaseConnection {
let mut opts = ConnectOptions::new("sqlite::memory:".to_string());
opts.max_connections(1);
let db = Database::connect(opts).await.unwrap();
let backend = db.get_database_backend();
let schema = Schema::new(backend);
for stmt in [
schema.create_table_from_entity(datasets::Entity),
schema.create_table_from_entity(dataset_items::Entity),
] {
db.execute(backend.build(&stmt)).await.unwrap();
}
db
}
fn dataset() -> datasets::Model {
datasets::Model {
id: "dataset-1".to_string(),
org_id: "org-1".to_string(),
name: "Golden set".to_string(),
description: None,
tags: None,
global_version: 59,
created_by: "owner@example.com".to_string(),
created_at: 1,
updated_by: "owner@example.com".to_string(),
updated_at: 1,
}
}
async fn stored(db: &DatabaseConnection) -> dataset_items::Model {
dataset_items::Entity::find_by_id("row-1")
.one(db)
.await
.unwrap()
.unwrap()
}
#[tokio::test]
async fn a_replayed_label_reaches_a_row_stored_without_one() {
let db = db().await;
apply_put(&db, dataset(), vec![row(None)]).await.unwrap();
apply_put(&db, dataset(), vec![row(Some(57))])
.await
.unwrap();
assert_eq!(stored(&db).await, row(Some(57)));
}
#[tokio::test]
async fn an_unlabelled_echo_never_erases_a_stored_label() {
let db = db().await;
apply_put(&db, dataset(), vec![row(Some(57))])
.await
.unwrap();
apply_put(&db, dataset(), vec![row(None)]).await.unwrap();
assert_eq!(stored(&db).await.write_version, Some(57));
}
#[tokio::test]
async fn a_label_is_not_filled_in_beside_conflicting_content() {
let db = db().await;
apply_put(&db, dataset(), vec![row(None)]).await.unwrap();
let mut changed = row(Some(57));
changed.input = serde_json::json!("different question");
assert!(apply_put(&db, dataset(), vec![changed]).await.is_err());
assert_eq!(stored(&db).await, row(None));
}
}
tests/ui-testing/pages/alertsPages/alertSaveQueryModePage.js
+2 -2
View File
@@ -280,9 +280,9 @@ export class AlertSaveQueryModePage {
).toBeVisible({ timeout: 15000 });
}
async expectDetailConditionEmpty() {
async expectDetailConditionCountOnly() {
const condition = this.page.locator(this.locators.detailCondition);
await expect(condition).toContainText('—');
await expect(condition).toContainText('count >=');
await expect(condition).not.toContainText('SELECT');
}
tests/ui-testing/pages/tracesPages/tracesPage.js
+43
View File
@@ -6,6 +6,7 @@ import { confirmDraftSignOut } from '../generalPages/draftSignOut.js';
import { dateTimeButtonLocator, relative30SecondsButtonLocator, absoluteTabLocator, Past30SecondsValue } from '../commonActions.js';
const testLogger = require('../../playwright-tests/utils/test-logger.js');
const { isCloudEnvironment } = require('../cloudPages/cloud-env.js');
const { ingestTraces } = require('../../playwright-tests/utils/trace-ingestion.js');
// Rate and Errors come from web/src/plugins/traces/metrics/metrics.json; Duration is the latency heatmap.
const TRACES_METRICS_PANELS = ['Rate', 'Errors', 'Duration'];
@@ -13,8 +14,50 @@ const CHART_ZOOM_START_RATIO = 0.35;
const CHART_ZOOM_END_RATIO = 0.75;
const CHART_DRAG_STEPS = 20;
const QUERY_CLEAR_ATTEMPTS = 10;
const DURATION_UNIT_US = { us: 1, ms: 1000, s: 1000000 };
export function decodedSqlOf(request) {
if (request.method() !== 'POST' || !/_search/.test(request.url())) return null;
const body = request.postDataJSON?.() ?? null;
const query = body?.query ?? {};
if (typeof query.sql !== 'string') return null;
const base64 = body.encoding === 'base64' || query.encoding === 'base64';
return {
sql: base64 ? Buffer.from(query.sql, 'base64').toString('utf8') : query.sql,
start: query.start_time,
end: query.end_time,
};
}
// The comparison sends its own counts and LIMIT 4000 samples; nothing else does.
export const isComparison = (q) => /^SELECT count\(\*\) AS n\b/.test(q.sql) || /LIMIT 4000$/.test(q.sql);
export const isCount = (q) => /^SELECT count\(\*\) AS n\b/.test(q.sql);
// The table search: spans mode posts a base64 `SELECT *` SQL.
export const isTableSearch = (q) => /^SELECT \* FROM/.test(q.sql);
// The band's bounds in µs as the SQL writes them; a missing side is absent.
export function bandSql(editorBand) {
const toUs = (m) => (m ? Number(m[1]) * DURATION_UNIT_US[m[2]] : null);
const lo = toUs(editorBand.match(/duration >= '(\d+)(us|ms|s)'/));
const hi = toUs(editorBand.match(/duration < '(\d+)(us|ms|s)'/));
return [lo ? `duration >= ${lo}` : '', hi !== null ? `duration < ${hi}` : ''].filter(Boolean).join(' AND ');
}
export class TracesPage {
// Static so a beforeAll, which has no page fixture, can seed through its own context.
static async seedTraces(browser, ageMs = 0) {
const context = await browser.newContext({ storageState: 'playwright-tests/utils/auth/user.json' });
const page = await context.newPage();
try {
for (const scenario of [{ count: 6, forceScenario: 'error' }, { count: 10, forceScenario: 'success' }]) {
const seeded = await ingestTraces(page, scenario.count, { forceScenario: scenario.forceScenario, ageMs });
expect(seeded.failed, `Seeding ${scenario.forceScenario} traces must succeed`).toBe(0);
}
} finally {
await context.close();
}
}
constructor(page) {
this.page = page;
tests/ui-testing/playwright-tests/Alerts/alerts-save-query-mode.spec.js
+3 -4
View File
@@ -317,14 +317,13 @@ test.describe('Alerts — Choose Query Mode on save', {
expect(saved.query_condition.multi_time_range).toEqual([]);
});
test('the detail page shows no condition for a Builder alert that still stores SQL', async ({ page }) => {
test('the detail page shows the Builder count, not the SQL a Builder alert still stores', async ({ page }) => {
const builderId = await seed(page, builderEmptyWithSql(uniq('auto_qm_detbuilder')));
const sqlId = await seed(page, sqlWithBuilderCondition(uniq('auto_qm_detsql')));
// The stored SQL never runs on a Builder alert, so showing it here would
// claim the alert watches something it does not.
// The stored SQL never runs on a Builder alert; with no filters it counts every row.
await pm.alertSaveQueryModePage.openDetailConfiguration(builderId);
await pm.alertSaveQueryModePage.expectDetailConditionEmpty();
await pm.alertSaveQueryModePage.expectDetailConditionCountOnly();
// A SQL alert still shows the query it actually evaluates.
await pm.alertSaveQueryModePage.openDetailConfiguration(sqlId);
tests/ui-testing/playwright-tests/Traces/tracesTriageLoop.spec.js
+17 -41
View File
@@ -3,39 +3,21 @@
const { test, expect, navigateToBase } = require('../utils/enhanced-baseFixtures.js');
const testLogger = require('../utils/test-logger.js');
const PageManager = require('../../pages/page-manager.js');
const { ingestTraces } = require('../utils/trace-ingestion.js');
const {
TracesPage,
decodedSqlOf,
isComparison,
isCount,
isTableSearch,
bandSql,
} = require('../../pages/tracesPages/tracesPage.js');
const USER_FILTER = "duration >= '1ms'";
const DURATION_UNIT_US = { us: 1, ms: 1000, s: 1000000 };
const MINUTE_US = 60 * 1000000;
const HOUR_US = 60 * MINUTE_US;
function decodedSqlOf(request) {
if (request.method() !== 'POST' || !/_search/.test(request.url())) return null;
const body = request.postDataJSON?.() ?? null;
const query = body?.query ?? {};
if (typeof query.sql !== 'string') return null;
const base64 = body.encoding === 'base64' || query.encoding === 'base64';
return {
sql: base64 ? Buffer.from(query.sql, 'base64').toString('utf8') : query.sql,
start: query.start_time,
end: query.end_time,
};
}
// The comparison sends its own counts and LIMIT 4000 samples; nothing else does.
const isComparison = (q) => /^SELECT count\(\*\) AS n\b/.test(q.sql) || /LIMIT 4000$/.test(q.sql);
const isCount = (q) => /^SELECT count\(\*\) AS n\b/.test(q.sql);
// The table search: spans mode posts a base64 `SELECT *` SQL.
const isTableSearch = (q) => /^SELECT \* FROM/.test(q.sql);
// The band's bounds in µs as the SQL writes them; a missing side is absent.
function bandSql(editorBand) {
const toUs = (m) => (m ? Number(m[1]) * DURATION_UNIT_US[m[2]] : null);
const lo = toUs(editorBand.match(/duration >= '(\d+)(us|ms|s)'/));
const hi = toUs(editorBand.match(/duration < '(\d+)(us|ms|s)'/));
return [lo ? `duration >= ${lo}` : '', hi !== null ? `duration < ${hi}` : ''].filter(Boolean).join(' AND ');
}
// The default brush spans 35-75% of a 15m range (~10-4 min ago); spans stamped now sit right of it.
const BRUSH_WINDOW_AGE_MS = 6 * 60 * 1000;
test.describe('Traces triage loop', () => {
let pm;
@@ -43,16 +25,7 @@ test.describe('Traces triage loop', () => {
test.beforeAll(async ({ browser }) => {
test.setTimeout(120000);
const context = await browser.newContext({ storageState: 'playwright-tests/utils/auth/user.json' });
const page = await context.newPage();
try {
for (const scenario of [{ count: 6, forceScenario: 'error' }, { count: 10, forceScenario: 'success' }]) {
const seeded = await ingestTraces(page, scenario.count, { forceScenario: scenario.forceScenario });
expect(seeded.failed, `Seeding ${scenario.forceScenario} traces must succeed`).toBe(0);
}
} finally {
await context.close();
}
await TracesPage.seedTraces(browser);
});
test.beforeEach(async ({ page }, testInfo) => {
@@ -246,8 +219,9 @@ test.describe('Traces triage loop', () => {
test('P1: an Errors brush compares error spans with the rest of its window', {
tag: ['@tracesTriageLoop', '@traces', '@functional', '@P1', '@all'],
}, async ({ page }) => {
}, async ({ page, browser }) => {
test.setTimeout(240000);
await TracesPage.seedTraces(browser, BRUSH_WINDOW_AGE_MS);
await searchWith('');
const preBox = queries.filter(isTableSearch).at(-1);
expect(await pm.tracesPage.zoomOnMetricsPanel('Errors')).toBeTruthy();
@@ -267,8 +241,9 @@ test.describe('Traces triage loop', () => {
test('P1: a Rate brush compares its window with the rest of the range', {
tag: ['@tracesTriageLoop', '@traces', '@functional', '@P1', '@all'],
}, async ({ page }) => {
}, async ({ page, browser }) => {
test.setTimeout(240000);
await TracesPage.seedTraces(browser, BRUSH_WINDOW_AGE_MS);
await searchWith('');
expect(await pm.tracesPage.zoomOnMetricsPanel('Rate')).toBeTruthy();
await page.waitForTimeout(2000);
@@ -296,8 +271,9 @@ test.describe('Traces triage loop across time zones', () => {
test('P2: an Errors brush under an Asia/Kolkata app zone keeps the brushed window', {
tag: ['@tracesTriageLoop', '@traces', '@functional', '@P2', '@all'],
}, async ({ page }) => {
}, async ({ page, browser }) => {
test.setTimeout(240000);
await TracesPage.seedTraces(browser, BRUSH_WINDOW_AGE_MS);
const queries = [];
page.on('request', (r) => {
const q = decodedSqlOf(r);
web/src/components/DateTime.spec.ts
+38
View File
@@ -322,6 +322,26 @@ describe("DateTime Component", () => {
expect(result2).toBe(false);
});
// C8: a forward-looking caller lifts the cap; every other caller keeps today.
it("caps the calendar at today unless a later max date is passed", () => {
// "Today" is the mocked timestampToTimezoneDate above.
wrapper = createWrapper({ disableRelative: true });
expect(wrapper.vm.calendarMaxDate).toBe("2025/01/01");
wrapper.unmount();
wrapper = createWrapper({ disableRelative: true, maxDate: "2099/01/01" });
expect(wrapper.vm.calendarMaxDate).toBe("2099/01/01");
wrapper.unmount();
wrapper = createWrapper({
disableRelative: true,
minDate: "2023/01/01",
maxDate: "2099/01/01",
});
expect(wrapper.vm.optionsFn("2098/06/15")).toBe(true);
expect(wrapper.vm.optionsFn("2099/01/02")).toBe(false);
});
it("should test computeRelativePeriod function", () => {
wrapper = createWrapper({
queryRangeRestrictionInHour: 48,
@@ -661,6 +681,24 @@ describe("DateTime Component", () => {
).toBeUndefined();
});
// F2: a caller that lifts the calendar cap lifts the › cap with it, up to the end of maxDate.
it("should shift forward past now up to the end of maxDate", () => {
wrapper = createWrapper({ disableRelative: true, maxDate: "2026/09/11" });
store.state.savedViewFlag = false;
wrapper.vm.selectedType = "absolute";
wrapper.vm.selectedDate = { from: "2026/09/10", to: "2026/09/10" };
wrapper.vm.selectedTime = { startTime: "11:00:00", endTime: "12:00:00" };
expect(wrapper.vm.isNextShiftDisabled()).toBe(false);
wrapper.vm.shiftTimeRange("next");
expect(wrapper.vm.selectedTime).toEqual({ startTime: "12:00:00", endTime: "13:00:00" });
wrapper.vm.selectedDate = { from: "2026/09/11", to: "2026/09/11" };
wrapper.vm.selectedTime = { startTime: "23:00:00", endTime: "23:59:59" };
expect(wrapper.vm.isNextShiftDisabled()).toBe(true);
});
it("should still shift backward from a window that ends at now", () => {
wrapper = createWrapper();
store.state.savedViewFlag = false;
web/src/components/DateTime.vue
+15 -8
View File
@@ -557,6 +557,11 @@ export default defineComponent({
type: String,
default: null,
},
/** The last pickable calendar day (yyyy/MM/dd); absent means today, for a caller that schedules ahead. */
maxDate: {
type: String,
default: null,
},
menuAlign: {
type: String as PropType<"center" | "start" | "end">,
default: "end",
@@ -837,6 +842,7 @@ export default defineComponent({
});
const calendarMaxDate = computed(() => {
if (props.maxDate) return props.maxDate;
return timestampToTimezoneDate(new Date().getTime(), store.state.timezone, "yyyy/MM/dd");
});
@@ -1566,11 +1572,7 @@ export default defineComponent({
};
const optionsFn = (date: string) => {
const formattedDate = timestampToTimezoneDate(
new Date().getTime(),
store.state.timezone,
"yyyy/MM/dd",
);
const formattedDate = calendarMaxDate.value;
if (props.disableRelative) {
return date >= props.minDate && date <= formattedDate;
}
@@ -1643,8 +1645,8 @@ export default defineComponent({
let delta = (direction === "prev" ? -1 : 1) * duration;
if (direction === "next") {
// The calendar rejects future dates, so a forward shift stops at now.
delta = Math.min(delta, Date.now() * 1000 - endTime);
// A forward shift stops where the calendar does: now, or the end of `maxDate`.
delta = Math.min(delta, forwardCeilingMicros() - endTime);
if (delta < MICROS_PER_SECOND) return;
}
const startDateTime = convertUnixTime(startTime + delta);
@@ -1660,12 +1662,17 @@ export default defineComponent({
if (!props.autoApply) saveDate("absolute");
};
const forwardCeilingMicros = () =>
props.maxDate
? convertToUtcTimestamp(`${props.maxDate} 23:59:59`, store.state.timezone)
: Date.now() * 1000;
const isNextShiftDisabled = () => {
if (selectedType.value === "relative") {
return (parseCalendarToken(calendarToken.value)?.offset ?? 0) === 0;
}
const { endUTC } = getUTCTimeStamp();
return !(endUTC + MICROS_PER_SECOND <= Date.now() * 1000);
return !(endUTC + MICROS_PER_SECOND <= forwardCeilingMicros());
};
// Only the calendar section is a 2-column grid of this/last pairs; every other row is 1-item.
web/src/components/alerts/IncidentTimeline.spec.ts
+9
View File
@@ -353,6 +353,15 @@ describe("IncidentTimeline - getInlineEventText", () => {
expect(text).toContain("5 times");
});
/// O1: a page Resolve that ended the incident's ack says so, not "Reopened".
it("says a page Resolve ended the acknowledgement", async () => {
const w = await mountComp();
await flushPromises();
const event = { type: "AckEndedByPageResolve", data: { user_id: "ana@o2.ai" } };
expect((w.vm as any).getEventBadgeText(event)).toBe("Acknowledgement ended");
expect((w.vm as any).getInlineEventText(event)).toBe("by resolving one of its pages");
});
it("returns text for ai_analysis_begin", async () => {
const w = await mountComp();
await flushPromises();
web/src/components/alerts/IncidentTimeline.vue
+9
View File
@@ -160,6 +160,8 @@ const getEventIcon = (event: any): string => {
return "check";
case "Reopened":
return "replay";
case "AckEndedByPageResolve":
return "check";
case "DimensionsUpgraded":
return "arrow-upward";
case "TitleChanged":
@@ -201,6 +203,8 @@ const getEventBadgeColor = (event: any): string => {
return "var(--color-success-600)";
case "Reopened":
return "var(--color-orange-500)";
case "AckEndedByPageResolve":
return "var(--color-blue-500)";
case "DimensionsUpgraded":
return "var(--color-ai-accent)";
case "TitleChanged":
@@ -239,6 +243,8 @@ const getEventBadgeText = (event: any): string => {
return t("alerts.incidents.timeline.badgeResolved");
case "Reopened":
return t("alerts.incidents.timeline.badgeReopened");
case "AckEndedByPageResolve":
return t("alerts.incidents.timeline.badgeAckEnded");
case "DimensionsUpgraded":
return t("alerts.incidents.timeline.badgeDimensionsUpgraded");
case "TitleChanged":
@@ -340,6 +346,9 @@ const getInlineEventText = (event: any): string => {
? t("alerts.incidents.timeline.incidentWas")
: t("alerts.incidents.timeline.theIncident");
case "AckEndedByPageResolve":
return t("alerts.incidents.timeline.ackEndedByPageResolve");
case "Alert": {
const alertName = bold(data.alert_name || t("alerts.incidents.timeline.unnamedAlert"));
const count = Number(data.count ?? 0);
web/src/components/common/KpiCard.vue
+3 -6
View File
@@ -30,15 +30,15 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
<template>
<component
:is="as"
class="bg-card-glass-bg rounded-default border-border-default flex flex-col gap-1 border px-3.5 py-2.5 transition-shadow duration-200 ease-in-out hover:shadow-md max-lg:shrink-0 max-lg:basis-auto max-lg:flex-row-reverse max-lg:items-center max-lg:gap-1.5 max-lg:px-1.5 max-lg:py-1"
class="bg-card-glass-bg rounded-default border-border-default flex min-w-0 flex-col gap-1 border px-3.5 py-2.5 transition-shadow duration-200 ease-in-out hover:shadow-md max-lg:shrink-0 max-lg:basis-auto max-lg:gap-0.5 max-lg:px-2 max-lg:py-1.5"
:data-test="dataTest"
:title="label && !lgUp ? String(label) : undefined"
>
<!-- The label stays visible at every width: a compact tile without its
label is a bare number nobody can read. -->
<div class="mb-1 flex items-center justify-between gap-2 max-lg:mb-0">
<OTruncatedText
as="div"
:tooltip="$slots.label ? false : undefined"
class="max-lg:hidden"
:class="labelClass || 'text-2xs text-text-secondary leading-normal font-semibold'"
>
<slot name="label">{{ label }}</slot>
@@ -80,9 +80,6 @@ import { raw, type I18nText } from "@/types/i18n";
import OIcon from "@/lib/core/Icon/OIcon.vue";
import OTruncatedText from "@/lib/core/Typography/OTruncatedText.vue";
import type { IconName } from "@/lib/core/Icon/OIcon.icons";
import useBreakpoint from "@/composables/useBreakpoint";
const { lgUp } = useBreakpoint();
withDefaults(
defineProps<{
web/src/components/dbm/DbmCoverageLine.spec.ts
+16
View File
@@ -62,6 +62,22 @@ describe("DbmCoverageLine", () => {
expect(lineText(wrapper)).toContain("they're exact");
});
it("says a several-schema database shows its slowest schema's speeds", () => {
const wrapper = mountLine({ exactPercentiles: true, slowestSchemaSpeeds: true });
expect(lineText(wrapper)).toBe(
"All databases reporting. Speeds are exact per schema; a database with several schemas shows its slowest one.",
);
});
it("keeps the estimated copy when the server fused windows, several schemas or not", () => {
const wrapper = mountLine({
exactPercentiles: true,
slowestSchemaSpeeds: true,
freshness: freshness({ percentiles_estimated: true }),
});
expect(lineText(wrapper)).toContain("close but not exact");
});
it("never claims exactness when the server fused windows", () => {
const wrapper = mountLine({
exactPercentiles: false,
web/src/components/dbm/DbmCoverageLine.vue
+6 -2
View File
@@ -155,6 +155,8 @@ const props = withDefaults(
* separate question the response answers via `percentiles_estimated`.
*/
exactPercentiles?: boolean;
/** Some row folds several schemas, so its speeds are its slowest schema's, not its own. */
slowestSchemaSpeeds?: boolean;
/**
* What these rows ARE. `list` (default) is a ranked table, where the share
* of total time is the headline claim. `query` is one statement, where that
@@ -172,6 +174,7 @@ const props = withDefaults(
other: () => [],
topNSubset: false,
exactPercentiles: false,
slowestSchemaSpeeds: false,
filterLabel: null,
subject: "list",
inline: false,
@@ -411,8 +414,9 @@ const summary = computed<I18nText>(() => {
// The Overview grain measures every call, so it states exactness plainly
// rather than borrowing the per-query "heaviest N" sentence.
if (props.exactPercentiles) {
return estimatedSpeeds.value
? t("dbm.coverage.lineExactEstimated")
if (estimatedSpeeds.value) return t("dbm.coverage.lineExactEstimated");
return props.slowestSchemaSpeeds
? t("dbm.coverage.lineExactSlowestSchema")
: t("dbm.coverage.lineExact");
}
if (otherTime.value <= 0) {
web/src/components/ingestion/ai/AIIntegrationDetail.spec.ts
+51 -2
View File
@@ -1,6 +1,6 @@
// Copyright 2026 OpenObserve Inc.
import { mount, VueWrapper } from "@vue/test-utils";
import { flushPromises, mount, VueWrapper } from "@vue/test-utils";
import { describe, expect, it, beforeEach, afterEach, vi } from "vitest";
vi.mock("@/composables/useIngestion", () => ({
@@ -26,8 +26,15 @@ vi.mock("./content", () => ({
},
}));
import { createMemoryHistory, createRouter } from "vue-router";
import AIIntegrationDetail from "./AIIntegrationDetail.vue";
import store from "@/test/unit/helpers/store";
import { INSTRUMENT_SKILL_URL } from "./codingAgentPrompt";
const router = createRouter({
history: createMemoryHistory(),
routes: [{ path: "/", component: { template: "<div />" } }],
});
const CARD_STUB = {
name: "AIIntegrationCard",
@@ -52,11 +59,15 @@ describe("AIIntegrationDetail", () => {
mount(AIIntegrationDetail, {
props,
global: {
plugins: [store],
plugins: [store, router],
stubs: { AIIntegrationCard: CARD_STUB, CredentialCodeBlock: COPY_STUB },
},
});
beforeEach(async () => {
await router.push({ path: "/", query: {} });
});
afterEach(() => {
wrapper?.unmount();
vi.clearAllMocks();
@@ -119,4 +130,42 @@ describe("AIIntegrationDetail", () => {
expect(wrapper.text()).toContain("Select an integration to view details.");
});
});
describe("setup tabs", () => {
const agentTab = '[data-test="ai-integration-tab-coding-agent"]';
const agentPanel = '[data-test="ai-coding-agent-setup"]';
it("opens on the step-by-step card by default", () => {
wrapper = mountDetail({ categorySlug: "frameworks", integrationSlug: "langchain" });
expect(wrapper.find('[data-test="ai-integration-card-stub"]').exists()).toBe(true);
expect(wrapper.find(agentPanel).exists()).toBe(false);
});
it("switches to the coding-agent prompt when its tab is clicked", async () => {
wrapper = mountDetail({ categorySlug: "frameworks", integrationSlug: "langchain" });
await wrapper.find(agentTab).trigger("mousedown", { button: 0 });
await wrapper.find(agentTab).trigger("click");
await flushPromises();
expect(wrapper.find(agentPanel).exists()).toBe(true);
expect(wrapper.find('[data-test="ai-integration-card-stub"]').exists()).toBe(false);
});
it("preselects the coding-agent tab from ?tab=agent and names the org and stream", async () => {
await router.push({ path: "/", query: { tab: "agent" } });
wrapper = mountDetail({ categorySlug: "frameworks", integrationSlug: "langchain" });
await flushPromises();
const prompt = wrapper.findComponent({ name: "OCodeBlock" }).props("code") as string;
const org = store.state.selectedOrganization.identifier;
expect(prompt).toContain(INSTRUMENT_SKILL_URL);
expect(prompt).toContain(`org ${org}, stream default.`);
});
it("follows a later ?tab change on the same integration", async () => {
wrapper = mountDetail({ categorySlug: "frameworks", integrationSlug: "langchain" });
expect(wrapper.find(agentPanel).exists()).toBe(false);
await router.push({ path: "/", query: { tab: "agent" } });
await flushPromises();
expect(wrapper.find(agentPanel).exists()).toBe(true);
});
});
});
web/src/components/ingestion/ai/AIIntegrationDetail.vue
+52 -2
View File
@@ -15,7 +15,8 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
-->
<script setup lang="ts">
import { computed } from "vue";
import { computed, ref, watch } from "vue";
import { useRoute } from "vue-router";
import { raw, useI18nTyped } from "@/types/i18n";
import { useStore } from "vuex";
import CredentialCodeBlock from "@/components/ingestion/CredentialCodeBlock.vue";
@@ -29,6 +30,10 @@ import { getRichCardContent } from "./content/richCard/registry";
import AIIntegrationCard from "./content/AIIntegrationCard.vue";
import AIRichSetupCard from "@/components/ingestion/setupCard/SetupCardRenderer.vue";
import OBanner from "@/lib/feedback/Banner/OBanner.vue";
import OTabs from "@/lib/navigation/Tabs/OTabs.vue";
import OTab from "@/lib/navigation/Tabs/OTab.vue";
import CodingAgentSetup from "./CodingAgentSetup.vue";
import { CODING_AGENT_TAB, STEP_BY_STEP_TAB } from "./codingAgentPrompt";
const props = defineProps<{
categorySlug: string;
@@ -38,6 +43,7 @@ const props = defineProps<{
const { t } = useI18nTyped();
const store = useStore();
const { aiContent, endpoint } = useIngestion();
const route = useRoute();
const category = computed<AICategory | undefined>(() =>
aiCategories.find((c) => c.slug === props.categorySlug),
@@ -84,12 +90,56 @@ const richContentNeedsPasscode = computed(() =>
const richContent = computed(() =>
getRichCardContent(integration.value?.contentSlug ?? integration.value?.slug, subs.value, t),
);
// "With a coding agent" / "Step by step". `?tab=agent` (e.g. from the LLM
// Insights empty state) opens the coding-agent tab; anything else opens the
// stepped card, which stays the default.
const tabFromQuery = (tab: unknown) =>
tab === CODING_AGENT_TAB ? CODING_AGENT_TAB : STEP_BY_STEP_TAB;
const activeTab = ref<string | number>(tabFromQuery(route?.query?.tab));
watch(
() => route?.query?.tab,
(tab) => {
activeTab.value = tabFromQuery(tab);
},
);
// The stream the prompt names: the card's own stream default, else "default".
const agentDefaultStream = computed(
() =>
richContent.value?.streamInput?.default || richContent.value?.detect?.streamName || "default",
);
const agentGuideName = computed(
() => richContent.value?.provider.name ?? String(integration.value?.name ?? ""),
);
</script>
<template>
<div v-if="integration" class="p-2">
<OTabs v-model="activeTab" dense bordered data-test="ai-integration-setup-tabs">
<OTab
:name="CODING_AGENT_TAB"
:label="t('ingestion.ai.codingAgent.tabAgent')"
data-test="ai-integration-tab-coding-agent"
/>
<OTab
:name="STEP_BY_STEP_TAB"
:label="t('ingestion.ai.codingAgent.tabSteps')"
data-test="ai-integration-tab-step-by-step"
/>
</OTabs>
<CodingAgentSetup
v-if="activeTab === CODING_AGENT_TAB"
:key="`agent-${integrationSlug}`"
:subs="subs"
:default-stream="agentDefaultStream"
:detect="richContent?.detect"
:guide-name="agentGuideName"
:doc-url="richContent?.docUrl ?? docURL"
:passcode-forbidden="passcodeForbidden"
/>
<OBanner
v-if="passcodeForbidden && richContent && richContentNeedsPasscode"
v-else-if="passcodeForbidden && richContent && richContentNeedsPasscode"
variant="warning"
data-test="ai-integration-detail-passcode-forbidden"
:content="t('ingestion.passcodeForbiddenMessage')"
web/src/components/ingestion/ai/CodingAgentSetup.spec.ts
+97
View File
@@ -0,0 +1,97 @@
// Copyright 2026 OpenObserve Inc.
import { flushPromises, mount, VueWrapper } from "@vue/test-utils";
import { afterEach, describe, expect, it, vi } from "vitest";
const downloadEnvFile = vi.fn();
vi.mock("@/components/ingestion/setupCard/envFile", () => ({
downloadEnvFile: (...args: unknown[]) => downloadEnvFile(...args),
}));
import CodingAgentSetup from "./CodingAgentSetup.vue";
import store from "@/test/unit/helpers/store";
const SUBS = { url: "https://api.example.com", org: "team1", token: "dG9rZW4=" };
const startCheck = vi.fn();
const FIRST_EVENT_STUB = {
name: "FirstEventStatus",
template: '<div data-test="first-event-status-stub" />',
props: ["org", "signal", "targetStream", "match", "filter", "guideName", "docUrl"],
emits: ["copy-command"],
methods: { start: startCheck, probeNow: vi.fn() },
};
describe("CodingAgentSetup", () => {
let wrapper: VueWrapper;
const mountSetup = (props: Partial<InstanceType<typeof CodingAgentSetup>["$props"]> = {}) =>
mount(CodingAgentSetup, {
props: {
subs: SUBS,
defaultStream: "default",
guideName: "OpenAI",
passcodeForbidden: false,
...props,
},
global: { plugins: [store], stubs: { FirstEventStatus: FIRST_EVENT_STUB } },
});
const promptCode = () => wrapper.findComponent({ name: "OCodeBlock" }).props("code") as string;
afterEach(() => {
wrapper?.unmount();
vi.clearAllMocks();
});
it("puts the edited stream into the prompt and the first-span check", async () => {
wrapper = mountSetup({
detect: { streamType: "traces", filter: "gen_ai_provider_name = 'openai'" },
});
expect(promptCode()).toContain("org team1, stream default.");
await wrapper.find('[data-test="ai-coding-agent-stream-input"] input').setValue("ai_traces");
await flushPromises();
expect(promptCode()).toContain("org team1, stream ai_traces.");
expect(wrapper.findComponent(FIRST_EVENT_STUB).props("targetStream")).toBe("ai_traces");
});
it("starts the first-span check when the prompt is copied, as the status row says", async () => {
wrapper = mountSetup({ detect: { streamType: "traces" } });
wrapper.findComponent({ name: "OCodeBlock" }).vm.$emit("copy", { partial: false });
expect(startCheck).toHaveBeenCalledWith("copy");
});
it("re-copies through the prompt block when the status bar asks for the command", async () => {
wrapper = mountSetup({ detect: { streamType: "traces" } });
const click = vi.fn();
wrapper
.find('[data-test="ai-coding-agent-prompt-copy-btn"]')
.element.addEventListener("click", click);
wrapper.findComponent(FIRST_EVENT_STUB).vm.$emit("copy-command");
expect(click).toHaveBeenCalled();
});
it("downloads the card's .env plus the chosen stream and the web UI root", async () => {
wrapper = mountSetup();
await wrapper.find('[data-test="ai-coding-agent-stream-input"] input').setValue("ai_traces");
await wrapper.find('[data-test="ai-coding-agent-env-btn"]').trigger("click");
expect(downloadEnvFile).toHaveBeenCalledWith(SUBS, {
OPENOBSERVE_TRACES_STREAM_NAME: "ai_traces",
OPENOBSERVE_UI_URL: window.location.origin,
});
});
it("offers no .env when the org forbids its passcode", () => {
wrapper = mountSetup({ passcodeForbidden: true });
expect(wrapper.find('[data-test="ai-coding-agent-passcode-forbidden"]').exists()).toBe(true);
expect(wrapper.find('[data-test="ai-coding-agent-env-btn"]').exists()).toBe(false);
});
it("names the coding agents in the prompt step title from code, not the translation", () => {
wrapper = mountSetup();
const title = wrapper.find('[data-test="ai-coding-agent-prompt-step"]').text();
expect(title).toContain("Claude Code, Cursor or Codex");
expect(title).not.toContain("{client");
});
});
web/src/components/ingestion/ai/CodingAgentSetup.vue
+164
View File
@@ -0,0 +1,164 @@
<!-- Copyright 2026 OpenObserve Inc.
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
-->
<!--
"With a coding agent" setup for an AI integration: download the per-org .env,
then paste one prompt into Claude Code / Cursor / Codex. The prompt points the
agent at the openobserve-instrument skill by URL, so nothing has to be
installed and the token stays in the .env, never in the chat.
-->
<script setup lang="ts">
import { computed, ref } from "vue";
import { raw, useI18nTyped } from "@/types/i18n";
import OButton from "@/lib/core/Button/OButton.vue";
import OCodeBlock from "@/lib/core/Code/OCodeBlock.vue";
import OBanner from "@/lib/feedback/Banner/OBanner.vue";
import OInput from "@/lib/forms/Input/OInput.vue";
import OStepper from "@/lib/navigation/Stepper/OStepper.vue";
import OStep from "@/lib/navigation/Stepper/OStep.vue";
import FirstEventStatus from "@/components/ingestion/FirstEventStatus.vue";
import { downloadEnvFile } from "@/components/ingestion/setupCard/envFile";
import { getPath } from "@/utils/zincutils";
import { safeHttpUrl } from "@/components/ingestion/setupCard/subs";
import type { CardSubstitutions, RichCardDetect } from "@/components/ingestion/setupCard/types";
import { STREAM_NAME_RE, buildCodingAgentPrompt } from "./codingAgentPrompt";
const props = defineProps<{
/** Per-org url/org/token — the same values the stepped card's .env carries. */
subs: CardSubstitutions;
/** Stream the prompt names before the user edits it. */
defaultStream: string;
/** The integration's first-span check; omitted → no live status bar. */
detect?: RichCardDetect;
guideName: string;
docUrl?: string;
/** The org forbids showing its ingestion passcode, so there is no .env to hand out. */
passcodeForbidden: boolean;
}>();
const { t } = useI18nTyped();
const streamName = ref(props.defaultStream);
const streamNameError = computed(() =>
streamName.value.trim() && !STREAM_NAME_RE.test(streamName.value.trim())
? t("ingestion.setupCard.streamNameError")
: raw(""),
);
const effectiveStream = computed(() => streamName.value.trim() || props.defaultStream);
const prompt = computed(() => buildCodingAgentPrompt(props.subs.org, effectiveStream.value));
const promptTitle = computed(() =>
t("ingestion.ai.codingAgent.promptTitle", {
client1: raw("Claude Code"),
client2: raw("Cursor"),
client3: raw("Codex"),
}),
);
const root = ref<HTMLElement | null>(null);
const statusBar = ref<InstanceType<typeof FirstEventStatus> | null>(null);
// Copying the prompt is the go signal, so the first-span check starts then, as on the stepped card.
const onPromptCopy = () => statusBar.value?.start("copy");
// The status bar's copy button re-copies through the prompt block, so the clipboard and toast match.
const copyPrompt = () =>
root.value?.querySelector<HTMLElement>('[data-test="ai-coding-agent-prompt-copy-btn"]')?.click();
// Beyond the card's three keys: the stream the prompt names (the SDK reads this
// name) and the web UI root (the skill appends /web/…), so its trace and fix
// links open the UI even where the API lives on another host or port.
const downloadEnv = () =>
downloadEnvFile(props.subs, {
OPENOBSERVE_TRACES_STREAM_NAME: effectiveStream.value,
OPENOBSERVE_UI_URL: window.location.origin + getPath().replace(/\/?web\/$/, ""),
});
</script>
<template>
<div
ref="root"
class="text-text-heading max-w-[61.25rem] px-1 py-4 text-sm"
data-test="ai-coding-agent-setup"
>
<div class="mb-7 [&_.text-input-hint]:whitespace-nowrap [&_label]:mb-0.5">
<OInput
v-model="streamName"
:label="t('ingestion.ai.codingAgent.streamLabel')"
:placeholder="raw(defaultStream)"
:help-text="!streamNameError ? t('ingestion.ai.codingAgent.streamHelp') : undefined"
:error="!!streamNameError"
:error-message="streamNameError"
size="sm"
width="md"
data-test="ai-coding-agent-stream-input"
/>
</div>
<OStepper :model-value="1" orientation="vertical" expanded :animated="false">
<OStep
:name="1"
:title="t('ingestion.ai.codingAgent.envTitle')"
data-test="ai-coding-agent-env-step"
>
<OBanner
v-if="passcodeForbidden"
variant="warning"
data-test="ai-coding-agent-passcode-forbidden"
:content="t('ingestion.passcodeForbiddenMessage')"
/>
<template v-else>
<p class="text-text-secondary text-compact mt-0 mb-2.5 leading-[1.45]">
{{ t("ingestion.ai.codingAgent.envHelp") }}
</p>
<OButton
variant="outline"
size="sm"
icon-left="download"
data-test="ai-coding-agent-env-btn"
@click="downloadEnv"
>
{{ t("ingestion.ai.codingAgent.envButton") }}
</OButton>
</template>
</OStep>
<OStep :name="2" :title="promptTitle" data-test="ai-coding-agent-prompt-step">
<OCodeBlock
:code="prompt"
lang="text"
chrome="editor"
:filename="t('ingestion.ai.codingAgent.promptLabel')"
wrap
:copy-message="t('ingestion.ai.codingAgent.promptCopied')"
data-test="ai-coding-agent-prompt"
@copy="onPromptCopy"
/>
<FirstEventStatus
v-if="detect"
ref="statusBar"
class="mt-3"
:org="subs.org"
:signal="detect.streamType"
:target-stream="effectiveStream"
:match="detect.match"
:filter="detect.filter || undefined"
:guide-name="guideName"
:doc-url="docUrl ? safeHttpUrl(docUrl) : undefined"
@copy-command="copyPrompt"
/>
</OStep>
</OStepper>
</div>
</template>
web/src/components/ingestion/ai/codingAgentPrompt.spec.ts
+22
View File
@@ -0,0 +1,22 @@
// Copyright 2026 OpenObserve Inc.
import { describe, expect, it } from "vitest";
import { INSTRUMENT_SKILL_URL, buildCodingAgentPrompt } from "./codingAgentPrompt";
describe("buildCodingAgentPrompt", () => {
it("names the skill URL, the org and the stream", () => {
expect(buildCodingAgentPrompt("team1", "ai_traces")).toBe(
`Instrument this app for OpenObserve with the openobserve-instrument skill (${INSTRUMENT_SKILL_URL}).\n` +
"Use the .env in the project root, org team1, stream ai_traces.\n" +
"Run one sample request, then check the data.",
);
});
it("substitutes each org and stream independently", () => {
const prompt = buildCodingAgentPrompt("default", "default");
expect(prompt).toContain("org default, stream default.");
expect(buildCodingAgentPrompt("aiobs_ux_test", "llm")).toContain(
"org aiobs_ux_test, stream llm.",
);
});
});
web/src/components/ingestion/ai/codingAgentPrompt.ts
+42
View File
@@ -0,0 +1,42 @@
// Copyright 2026 OpenObserve Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
/**
* The openobserve-instrument skill, served from the openobserve repo so any
* coding agent can fetch and follow it without installing a plugin.
*/
export const INSTRUMENT_SKILL_URL =
"https://raw.githubusercontent.com/openobserve/openobserve/main/skills/openobserve-instrument/SKILL.md";
/** `?tab=` value that opens an AI integration on its "With a coding agent" tab. */
export const CODING_AGENT_TAB = "agent";
/** The default tab: the integration's existing stepped card. */
export const STEP_BY_STEP_TAB = "steps";
/** Stream names the installer accepts (letters, digits, underscore). */
export const STREAM_NAME_RE = /^[a-zA-Z0-9_]+$/;
/**
* The prompt the user pastes into their coding agent. It carries the org and
* stream but never the token: the skill reads that from the downloaded .env.
* Agent-facing text, so it stays English regardless of the UI locale.
*/
export function buildCodingAgentPrompt(org: string, stream: string): string {
return (
`Instrument this app for OpenObserve with the openobserve-instrument skill (${INSTRUMENT_SKILL_URL}).\n` +
`Use the .env in the project root, org ${org}, stream ${stream}.\n` +
`Run one sample request, then check the data.`
);
}
web/src/components/ingestion/setupCard/SetupCardRenderer.vue
+2 -12
View File
@@ -50,6 +50,7 @@ import { OPEN_TOKEN_PICKER } from "@/composables/useCredentialSnippet";
import type { FirstEventState, StreamSignal } from "@/composables/firstEvent/useFirstEventWatch";
import FirstEventStatus from "../FirstEventStatus.vue";
import { cliFlagArg, safeHttpUrl, sanitizeInput, shellQuote } from "./subs";
import { downloadEnvFile } from "./envFile";
import type {
CardSubstitutions,
IngestTokenState,
@@ -412,18 +413,7 @@ const onStepCopy = (step: RichCardStep, index: number, payload?: CodeBlockCopyPa
};
// ── .env download (from the per-org subs) ────────────────────────────────────
const downloadEnv = () => {
const body =
`OPENOBSERVE_URL=${props.subs.url}\n` +
`OPENOBSERVE_ORG=${props.subs.org}\n` +
`OPENOBSERVE_AUTH_TOKEN=Basic ${props.subs.token}\n`;
const blob = new Blob([body], { type: "text/plain" });
const a = document.createElement("a");
a.href = URL.createObjectURL(blob);
a.download = ".env";
a.click();
setTimeout(() => URL.revokeObjectURL(a.href), 1000);
};
const downloadEnv = () => downloadEnvFile(props.subs);
// ── presentation helpers ─────────────────────────────────────────────────────
const escapeHtml = (s: string) =>
web/src/components/ingestion/setupCard/envFile.ts
+41
View File
@@ -0,0 +1,41 @@
// Copyright 2026 OpenObserve Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
import type { CardSubstitutions } from "./types";
/**
* The `.env` body a setup card hands out: the per-org url, org and Basic-auth
* token, then any `extra` entries the caller adds (e.g. the web URL for links).
*/
export function buildEnvFile(subs: CardSubstitutions, extra: Record<string, string> = {}): string {
return (
`OPENOBSERVE_URL=${subs.url}\n` +
`OPENOBSERVE_ORG=${subs.org}\n` +
`OPENOBSERVE_AUTH_TOKEN=Basic ${subs.token}\n` +
Object.entries(extra)
.map(([key, value]) => `${key}=${value}\n`)
.join("")
);
}
/** Save the per-org `.env` as a browser download. */
export function downloadEnvFile(subs: CardSubstitutions, extra: Record<string, string> = {}): void {
const blob = new Blob([buildEnvFile(subs, extra)], { type: "text/plain" });
const a = document.createElement("a");
a.href = URL.createObjectURL(blob);
a.download = ".env";
a.click();
setTimeout(() => URL.revokeObjectURL(a.href), 1000);
}
web/src/components/oncall/OnCallCoverForm.schema.ts
+1 -1
View File
@@ -25,7 +25,7 @@ const MAX_SPAN_MICROS = 90 * MICROS_PER_DAY;
* catch a milliseconds-for-microseconds mistake, which lands roughly 55,000
* years out — so the message says the limit rather than guessing at the cause.
*/
const MAX_AHEAD_MICROS = 730 * MICROS_PER_DAY;
export const MAX_AHEAD_MICROS = 730 * MICROS_PER_DAY;
export const makeOnCallCoverSchema = (
t: TranslateFn,
web/src/components/oncall/OnCallCoverForm.spec.ts
+12
View File
@@ -247,6 +247,18 @@ describe("OnCallCoverForm — which clock the times are on", () => {
expect(teamLine.text()).toContain("UTC");
});
// C8: a cover looks ahead, so the calendar runs from today to the 730-day limit, on the picker's clock.
it("offers today up to 730 days ahead, and says a refusal under the picker", () => {
const wrapper = renderCover({ viewerTimezone: IST });
const day = (micros: number) => toZonedInputValue(micros, IST).slice(0, 10).replace(/-/g, "/");
const now = Date.now() * 1000;
const picker = wrapper.findComponent({ name: "OFormDateTimeRange" });
expect(picker.attributes("min-date")).toBe(day(now));
expect(picker.attributes("max-date")).toBe(day(now + 730 * MICROS_PER_DAY));
expect(picker.attributes()).toHaveProperty("show-error");
});
/// One clock, said once. A reader on the team's own timezone has nothing to
/// reconcile, and repeating the same range twice reads as two windows.
it("says it once when the reader is already on the team's clock", async () => {
web/src/components/oncall/OnCallCoverForm.vue
+10 -1
View File
@@ -130,6 +130,9 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
:label="t('oncall.coverWhen')"
required
disable-relative
show-error
:min-date="calendarDay(0)"
:max-date="calendarDay(MAX_AHEAD_MICROS)"
data-test="oncall-cover-window"
/>
</div>
@@ -181,7 +184,7 @@ export interface UpcomingShift extends Shift {
rotationName: string;
}
import { formatInZone, fromZonedInputValue, toZonedInputValue } from "@/utils/oncall";
import { makeOnCallCoverSchema } from "./OnCallCoverForm.schema";
import { MAX_AHEAD_MICROS, makeOnCallCoverSchema } from "./OnCallCoverForm.schema";
/**
* What the cover half of this dialog holds.
@@ -440,6 +443,12 @@ function applyPreset(key: (typeof PRESETS)[number]["key"]) {
form.setFieldValue("window", { type: "absolute", ...ranges[key] });
}
/// The picker's calendar day `aheadMicros` from now, on the clock the picker shows.
function calendarDay(aheadMicros: number): string {
const at = Date.now() * 1000 + aheadMicros;
return toZonedInputValue(at, props.viewerTimezone).slice(0, 10).replace(/-/g, "/");
}
/// `timeZoneName` is the whole point: the two ends of a handover mean nothing
/// without the clock they are on, and a UTC team read from IST is off by five
/// and a half hours with no way to tell from the numbers.
web/src/components/oncall/OnCallEscalationCell.vue
+7 -1
View File
@@ -72,8 +72,10 @@ const props = withDefaults(
totalRungs?: number | null;
/** The step the record says it reached, for a list with no progress per row. */
reachedStep?: number | null;
/** The page names a team that no longer exists, so its ladder pages nobody. */
teamDeleted?: boolean;
}>(),
{ progress: null, progressLoaded: true, totalRungs: null, reachedStep: null },
{ progress: null, progressLoaded: true, totalRungs: null, reachedStep: null, teamDeleted: false },
);
const { t } = useI18nTyped();
@@ -87,6 +89,7 @@ const phrase = computed(() =>
progressLoaded: props.progressLoaded,
totalRungs: props.totalRungs,
reachedStep: props.reachedStep,
teamDeleted: props.teamDeleted,
nowMicros: nowMicros.value,
}),
);
@@ -102,6 +105,7 @@ const toneClass = computed(() => {
return "text-text-body";
case "snoozed":
case "reachedNobody":
case "nobodyPaged":
return "text-status-warning-text";
case "nobodyAnswered":
case "held":
@@ -141,6 +145,8 @@ const headline = computed<I18nText>(() => {
});
case "nobodyAnswered":
return t("oncall.state_exhausted");
case "nobodyPaged":
return t("oncall.escalationNobodyPaged");
case "held":
return t("oncall.escalationHeld", { n: p.secondsLeft });
case "pagesNobody":
web/src/components/oncall/OnCallMembers.spec.ts
+29
View File
@@ -396,6 +396,35 @@ describe("OnCallMembers", () => {
);
});
/// R9: a following rotation shares the roster, so the dialog says its person can change.
it("says a rotation whose order the remove changes can change who is on call", async () => {
const rule = (members: string[]) => ({
name: "Base",
members,
shift_micros: 604_800_000_000,
anchor_micros: 0,
});
const wrapper = render({
members: [member("ana@o2.ai"), member("bob@o2.ai")],
onCallNow: [
...now,
{ rotation_id: "r2", rotation_name: "Secondary", rule: "Base", user_email: "bob@o2.ai" },
],
rotations: [
{ id: "r1", name: "Primary", shift_rules: [rule(["ana@o2.ai", "bob@o2.ai"])] },
{ id: "r2", name: "Secondary", shift_rules: [rule(["ana@o2.ai", "bob@o2.ai"])] },
{ id: "r3", name: "Weekend", shift_rules: [rule(["bob@o2.ai"])] },
],
} as any);
await flushPromises();
await wrapper.find('[data-test="oncall-members-remove-ana@o2.ai"]').trigger("click");
await flushPromises();
expect(wrapper.find('[data-test="dialog-box"]').text()).toBe(
"They will no longer be paged for this team. Bob takes over Ana Sharma's current shift in Primary. They also leave Secondary, so who is on call there can change too.",
);
});
// Before the team name loads, the title names the person alone rather than "from ?".
it("names only the person while the team name is unknown", async () => {
const wrapper = render({ members: [member("ana@o2.ai")], onCallNow: now });
web/src/components/oncall/OnCallMembers.vue
+25 -12
View File
@@ -911,20 +911,33 @@ const removeTitle = computed<I18nText>(() => {
: t("oncall.removeMemberTitlePerson", { name });
});
/// Who takes the shift the removed person holds now. A cover is skipped: it is not the roster handing over.
/// Who takes the removed person's current shifts (a cover is not the roster handing over), and which rotations reorder.
const removeMessage = computed<I18nText>(() => {
const email = memberToRemove.value?.user_email.toLowerCase();
const email = memberToRemove.value?.user_email.toLowerCase() ?? "";
const leaving = raw(nameOf(memberToRemove.value?.user_email ?? ""));
const handovers = props.onCallNow
.filter((position) => position.user_email.toLowerCase() === email && !position.override_id)
.map((position) => {
const next = position.next_user_email;
const rotation = raw(position.rotation_name);
return next && next.toLowerCase() !== email
? t("oncall.removeMemberHandsOver", { name: raw(nameOf(next)), leaving, rotation })
: t("oncall.removeMemberLeavesGap", { leaving, rotation });
});
return raw([t("oncall.removeMemberBody"), ...handovers].join(" "));
const held = props.onCallNow.filter(
(position) => position.user_email.toLowerCase() === email && !position.override_id,
);
const handovers = held.map((position) => {
const next = position.next_user_email;
const rotation = raw(position.rotation_name);
return next && next.toLowerCase() !== email
? t("oncall.removeMemberHandsOver", { name: raw(nameOf(next)), leaving, rotation })
: t("oncall.removeMemberLeavesGap", { leaving, rotation });
});
// Who these rotations put on call next is the server's shift maths, so the dialog names them without guessing.
const reordered = (props.rotations ?? [])
.filter((rotation) => !held.some((position) => position.rotation_id === rotation.id))
.filter((rotation) => rotationMembers(rotation).some((m) => m.toLowerCase() === email))
.map((rotation) => rotation.name);
const reshuffle = reordered.length
? [
t("oncall.removeMemberReorders", {
rotations: raw(listInLocale(reordered, locale.value)),
}),
]
: [];
return raw([t("oncall.removeMemberBody"), ...handovers, ...reshuffle].join(" "));
});
async function confirmRemoveMember() {
web/src/components/oncall/OnCallPolicyEditor.spec.ts
+43
View File
@@ -1172,6 +1172,49 @@ describe("targets that name a rotation", () => {
expect(savedTargets().filter((t: any) => t.rotation_id === "rot_primary")).toHaveLength(2);
});
/// R8: a rotation lands on the open priority alone until "All priorities" is ticked.
it("adds a rotation to the open priority only, unless All priorities is ticked", async () => {
const twoPriorities: OnCallPolicy = {
...policy,
rungs: [1, 2].map((priority) => ({
priority,
steps: [{ after_micros: 0, targets: [{ kind: "user", email: "ana@o2.ai" }] }],
channels: ["email"],
})),
};
const wrapper = mount(OnCallPolicyEditor, {
props: { teamId: "team_1", policy: twoPriorities, open: true, rotations: TWO },
global: { plugins: [i18n, store], stubs },
});
await flushPromises();
const pick = async (rotationId: string, allPriorities?: boolean) => {
targetPicker(wrapper).vm.$emit("update:modelValue", "rotation");
await flushPromises();
const box = wrapper.findComponent('[data-test="oncall-policy-rotation-all-priorities"]');
if (allPriorities === undefined) expect(box.props("modelValue")).toBe(false);
else box.vm.$emit("update:modelValue", allPriorities);
await flushPromises();
wrapper
.findComponent('[data-test="oncall-policy-pick-rotation-1-0"]')
.vm.$emit("update:modelValue", rotationId);
await flushPromises();
};
await pick("rot_primary");
await pick("rot_secondary", true);
await wrapper.find('[data-test="oncall-policy-save"]').trigger("click");
await flushPromises();
const { rungs } = (service.setPolicy.mock.calls.at(-1)![0] as any).data;
const rotationsOf = (priority: number) =>
rungs
.find((r: any) => r.priority === priority)
.steps[0].targets.filter((t: any) => t.kind === "rotation")
.map((t: any) => t.rotation_id);
expect(rotationsOf(1)).toEqual(["rot_primary", "rot_secondary"]);
expect(rotationsOf(2)).toEqual(["rot_secondary"]);
});
/// Found by this suite leaking between tests: `reset` shallow-copied each
/// step, so the draft shared `targets` with `props.policy` and every add or
/// remove edited the policy in place. Cancel throws the draft away, which
web/src/components/oncall/OnCallPolicyEditor.vue
+2 -8
View File
@@ -233,9 +233,6 @@
@update:model-value="(v: any) => addRotation(String(v))"
/>
</span>
<!-- On by default. A rotation is a position on the team, and
one that exists at P1 but not P2 is a decision almost
nobody means to make. -->
<OCheckbox
v-model="applyToAllPriorities"
size="sm"
@@ -716,11 +713,8 @@ const pendingUserStep = ref<LadderStep | null>(null);
/// person on call in it or everyone on it, and which was asked for decides
/// what the picked rotation MEANS.
const pendingRotation = ref<{ step: LadderStep; mode: RotationMode } | null>(null);
/// Whether the next rotation lands on every priority at once. Sticky for the
/// life of the dialog: somebody adding a position to the ladder is doing it for
/// the ladder, not for one priority, and re-ticking per priority would be the
/// same chore in a different shape.
const applyToAllPriorities = ref(true);
/// Off until ticked, so an edit on one priority tab changes only that priority.
const applyToAllPriorities = ref(false);
const memberOptions = ref<{ label: I18nText; value: string }[]>([]);
const teamMembers = computed(() => memberOptions.value.map((m) => m.value));
/// The rotation in force, so the preview names people rather than target

Some files were not shown because too many files have changed in this diff Show More