windmill/ai_evals/adapters/cli/runtime.test.ts

import { describe, expect, it } from "bun:test";
import {
  anthropicUsageToBenchmarkTokenUsage,
  extractCliResultTokenUsage,
  endpointEnv,
  extractProposedWmillCommands,
  getFirstMutationToolIndex,
  parseWmillInvocationLog,
} from "./runtime";

describe("anthropicUsageToBenchmarkTokenUsage", () => {
  it("includes cache tokens in prompt usage", () => {
    expect(
      anthropicUsageToBenchmarkTokenUsage({
        input_tokens: 120,
        output_tokens: 45,
        cache_creation_input_tokens: 30,
        cache_read_input_tokens: 5,
      })
    ).toEqual({
      prompt: 155,
      completion: 45,
      total: 200,
    });
  });

  it("returns null when usage is absent", () => {
    expect(anthropicUsageToBenchmarkTokenUsage(null)).toBeNull();
  });
});

describe("extractCliResultTokenUsage", () => {
  it("reads aggregate usage from the SDK result event", () => {
    expect(
      extractCliResultTokenUsage({
        type: "result",
        usage: {
          input_tokens: 400,
          output_tokens: 120,
          cache_creation_input_tokens: 50,
          cache_read_input_tokens: 25,
        },
      })
    ).toEqual({
      prompt: 475,
      completion: 120,
      total: 595,
    });
  });

  it("falls back to modelUsage when aggregate usage is unavailable", () => {
    expect(
      extractCliResultTokenUsage({
        type: "result",
        modelUsage: {
          opus: {
            inputTokens: 200,
            outputTokens: 60,
            cacheCreationInputTokens: 10,
            cacheReadInputTokens: 5,
          },
          haiku: {
            inputTokens: 80,
            outputTokens: 20,
            cacheCreationInputTokens: 0,
            cacheReadInputTokens: 15,
          },
        },
      })
    ).toEqual({
      prompt: 310,
      completion: 80,
      total: 390,
    });
  });
});

describe("extractProposedWmillCommands", () => {
  it("extracts proposed commands from bullets, code blocks, and inline code", () => {
    expect(
      extractProposedWmillCommands(`
Next:
- \`wmill generate-metadata --yes\`
- wmill sync push

You can inspect failures with \`wmill job logs 123\`.
`)
    ).toEqual([
      "wmill generate-metadata --yes",
      "wmill sync push",
      "wmill job logs 123",
    ]);
  });

  it("extracts inline prose commands that are not wrapped in backticks", () => {
    expect(
      extractProposedWmillCommands(
        "The first command is wmill sync pull before you edit locally."
      )
    ).toEqual(["wmill sync pull"]);
  });

  it("extracts multiple inline prose commands from a single sentence", () => {
    expect(
      extractProposedWmillCommands(
        "Run wmill generate-metadata and then wmill sync push when you are ready."
      )
    ).toEqual(["wmill generate-metadata", "wmill sync push"]);
  });

  it("ignores negated command mentions", () => {
    expect(
      extractProposedWmillCommands(
        "Do not run `wmill sync push`. Instead run `wmill sync pull` first."
      )
    ).toEqual(["wmill sync pull"]);
  });
});

describe("parseWmillInvocationLog", () => {
  it("parses stubbed wmill invocations into structured records", () => {
    expect(
      parseWmillInvocationLog(`noise
__WMILL_BENCHMARK__
2026-04-21T12:00:00+00:00
/tmp/workspace
2
generate-metadata
--yes
__WMILL_BENCHMARK__
2026-04-21T12:00:05+00:00
/tmp/workspace
3
sync
push
--dry-run
`)
    ).toEqual([
      {
        argv: ["generate-metadata", "--yes"],
        cwd: "/tmp/workspace",
        timestamp: "2026-04-21T12:00:00+00:00",
      },
      {
        argv: ["sync", "push", "--dry-run"],
        cwd: "/tmp/workspace",
        timestamp: "2026-04-21T12:00:05+00:00",
      },
    ]);
  });
});

describe("getFirstMutationToolIndex", () => {
  it("counts wmill only as a command, not in a file name", () => {
    const bash = (command: string) => ({ tool: "Bash", input: { command }, timestamp: 0 });
    expect(
      getFirstMutationToolIndex([
        bash("cat AGENTS.wmill.md .agents/skills/x/SKILL.md"),
        bash("ls && wmill generate-metadata"),
      ])
    ).toBe(1);
    expect(getFirstMutationToolIndex([bash("cd f\nwmill sync push")])).toBe(0);
  });
});

describe("endpointEnv", () => {
  it("passes the endpoint only its own key", () => {
    const env = endpointEnv(
      { baseUrl: "https://example.test", apiKeyEnv: "DEEPSEEK_API_KEY" },
      "m",
      { PATH: "/bin", DEEPSEEK_API_KEY: "ds", OPENAI_API_KEY: "oa", ANTHROPIC_AUTH_TOKEN: "t" }
    );
    expect(env).toEqual({
      PATH: "/bin",
      ANTHROPIC_BASE_URL: "https://example.test",
      ANTHROPIC_API_KEY: "ds",
      ANTHROPIC_DEFAULT_HAIKU_MODEL: "m",
    });
  });
});