openobserve/scripts/ai-review/test/telemetry.test.mjs

import test from "node:test";
import assert from "node:assert/strict";
import { readFileSync } from "node:fs";

import {
  addUsage,
  cleanAttributeValue,
  genAiRequestAttributes,
  genAiResponseAttributes,
  sha256,
} from "../telemetry.mjs";

test("request content uses standard GenAI attributes without TraceLoop prompt keys", () => {
  const attributes = genAiRequestAttributes("Follow the review policy.", "Review this diff.");

  assert.equal(
    Object.keys(attributes).some(key => key.startsWith("gen_ai.prompt.")),
    false,
  );
  assert.deepEqual(JSON.parse(attributes["gen_ai.system_instructions"]), [
    { type: "text", content: "Follow the review policy." },
  ]);
  assert.deepEqual(JSON.parse(attributes["gen_ai.input.messages"]), [
    {
      role: "user",
      parts: [{ type: "text", content: "Review this diff." }],
    },
  ]);
  assert.equal(
    attributes["ai.review.system_instructions.sha256"],
    sha256("Follow the review policy."),
  );
});

test("response content and provider-reported usage use standard attributes", () => {
  const attributes = genAiResponseAttributes(
    {
      info: {
        id: "msg-123",
        modelID: "deepseek-flash",
        finish: "stop",
        cost: 0.42,
        tokens: {
          input: 1234,
          output: 56,
          reasoning: 12,
          cache: { read: 78, write: 9 },
        },
      },
    },
    "Looks good.",
    "fallback-model",
  );

  assert.deepEqual(JSON.parse(attributes["gen_ai.output.messages"]), [
    {
      role: "assistant",
      parts: [{ type: "text", content: "Looks good." }],
    },
  ]);
  assert.deepEqual(attributes["gen_ai.response.finish_reasons"], ["stop"]);
  assert.equal(attributes["gen_ai.usage.input_tokens"], 1234 + 78 + 9);
  assert.equal(attributes["gen_ai.usage.output_tokens"], 56 + 12);
  assert.equal(attributes["gen_ai.usage.total_tokens"], 1234 + 78 + 9 + 56 + 12);
  assert.equal(attributes["gen_ai.usage.cache_read.input_tokens"], 78);
  assert.equal(attributes["gen_ai.usage.cache_creation.input_tokens"], 9);
  assert.equal(attributes["gen_ai.usage.cache_read_tokens"], undefined);
  assert.equal(attributes["gen_ai.usage.cost"], undefined, "the server prices the span from its token counts");
  assert.equal(Object.keys(attributes).some(key => key.includes("cost")), false);
  assert.equal(attributes["ai.usage.reasoning_tokens"], 12);
  assert.equal(
    Object.keys(attributes).some(key => key.startsWith("gen_ai.completion.")),
    false,
  );
});

test("usage without cache or reasoning breakdown passes through unchanged", () => {
  const attributes = genAiResponseAttributes(
    { info: { tokens: { input: 100, output: 20 } } },
    "ok",
    "deepseek-flash",
  );

  assert.equal(attributes["gen_ai.usage.input_tokens"], 100);
  assert.equal(attributes["gen_ai.usage.output_tokens"], 20);
  assert.equal(attributes["gen_ai.usage.cache_read.input_tokens"], undefined);
});

test("missing provider usage is omitted instead of replaced with synthetic counts", () => {
  const attributes = genAiResponseAttributes(
    { info: { id: "msg-456" } },
    "No usage available.",
    "requested-model",
  );

  assert.equal(attributes["gen_ai.usage.input_tokens"], undefined);
  assert.equal(attributes["gen_ai.usage.output_tokens"], undefined);
  assert.equal(attributes["gen_ai.usage.cost"], undefined);
  assert.equal(attributes["gen_ai.response.model"], "requested-model");
});

test("homogeneous primitive arrays remain native OTel attribute arrays", () => {
  assert.deepEqual(cleanAttributeValue(["security", "performance"]), [
    "security",
    "performance",
  ]);
  assert.equal(cleanAttributeValue([{ name: "security" }]), '[{"name":"security"}]');
});

test("review instrumentation does not emit legacy TraceLoop content attributes", () => {
  const reviewSource = readFileSync(new URL("../run-review.mjs", import.meta.url), "utf8");

  assert.doesNotMatch(reviewSource, /["']gen_ai\.prompt\./);
  assert.doesNotMatch(reviewSource, /["']gen_ai\.completion\./);
});

test("usage of retried attempts is summed, and a missing side is skipped", () => {
  const a = { tokens: { input: 10, output: 2, reasoning: 5, cache: { read: 100, write: 0 } } };
  const b = { tokens: { input: 20, output: 3, reasoning: 7, cache: { read: 50, write: 1 } } };
  const sum = addUsage(a, b);
  assert.deepEqual(sum.tokens, { input: 30, output: 5, reasoning: 12, cache: { read: 150, write: 1 } });
  assert.equal(addUsage(null, b), b);
  assert.equal(addUsage(a, undefined), a);
});

test("neither config carries model prices, so the server's pricing table is the only price list", () => {
  for (const file of ["opencode.jsonc", "opencode.ci.jsonc"]) {
    const config = readFileSync(new URL(`../../../${file}`, import.meta.url), "utf8");
    assert.doesNotMatch(config, /"cost"\s*:/, file);
  }
});