mirror of
https://github.com/windmill-labs/windmill.git
synced 2026-08-25 16:02:11 +00:00
Merge remote-tracking branch 'origin/main' into glm/install-workspace-picker
# Conflicts: # frontend/src/lib/components/apps/components/display/dbtable/queries/dbQueriesUtils.ts # frontend/src/lib/components/datatableSchemaSql.ts # frontend/src/lib/components/workspaceSettings/projectMigrations.test.ts
This commit is contained in:
Guilhem Lemouel
@@ -61,7 +61,13 @@ else, go back to the barrel and grep.
|
||||
<Button startIcon={{ icon: ChevronLeft }} iconOnly onclick={prev} />
|
||||
```
|
||||
|
||||
Props: `variant?: 'accent' | 'accent-secondary' | 'default' | 'subtle'`, `unifiedSize?: 'sm' | 'md' | 'lg'`, `startIcon?: { icon: SvelteComponent }`, `iconOnly?: boolean`, `disabled?: boolean`
|
||||
Props: `variant?: 'accent' | 'accent-secondary' | 'default' | 'subtle'`, `unifiedSize?: '2xs' | 'xs' | 'sm' | 'md' | 'lg'`, `startIcon?: { icon: SvelteComponent }`, `iconOnly?: boolean`, `disabled?: boolean`
|
||||
|
||||
**`size` on `<Button>` is banned** — it, `spacingSize` and `extendedSize` are the legacy sizing
|
||||
system (`xs3`/`xs2`/`xs`/…, marked `@deprecated` in `Button.svelte`). Size every button with
|
||||
`unifiedSize`, the small ones included: `2xs` and `xs` are `h-5`, `sm` is `h-7`, `md` is `h-8`,
|
||||
`lg` is `h-10`. Existing `size="xs2"` call sites are legacy, not a precedent to copy. Same for
|
||||
`variant`: `contained`/`border`/`divider` are deprecated — use the four listed above.
|
||||
|
||||
### Text inputs — `<TextInput>`
|
||||
|
||||
|
||||
@@ -42,7 +42,7 @@ RUN wget https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VER
|
||||
RUN /usr/local/bin/python3 -m pip install pip-tools
|
||||
|
||||
# Bun
|
||||
COPY --from=oven/bun:1.3.10 /usr/local/bin/bun /usr/bin/bun
|
||||
COPY --from=oven/bun:1.4.0 /usr/local/bin/bun /usr/bin/bun
|
||||
|
||||
# Install windmill CLI
|
||||
RUN bun install -g windmill-cli \
|
||||
|
||||
@@ -61,7 +61,7 @@ jobs:
|
||||
|
||||
- uses: oven-sh/setup-bun@v2
|
||||
with:
|
||||
bun-version: 1.3.10
|
||||
bun-version: 1.4.0
|
||||
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
|
||||
@@ -75,7 +75,7 @@ jobs:
|
||||
|
||||
- uses: oven-sh/setup-bun@v2
|
||||
with:
|
||||
bun-version: 1.3.10
|
||||
bun-version: 1.4.0
|
||||
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
|
||||
@@ -73,7 +73,7 @@ jobs:
|
||||
|
||||
- uses: oven-sh/setup-bun@v2
|
||||
with:
|
||||
bun-version: 1.3.10
|
||||
bun-version: 1.4.0
|
||||
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
|
||||
@@ -58,7 +58,7 @@ jobs:
|
||||
go-version: 1.21.5
|
||||
- uses: oven-sh/setup-bun@v2
|
||||
with:
|
||||
bun-version: 1.3.10
|
||||
bun-version: 1.4.0
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: "20"
|
||||
|
||||
@@ -133,7 +133,7 @@ jobs:
|
||||
|
||||
- uses: oven-sh/setup-bun@v2
|
||||
with:
|
||||
bun-version: 1.3.10
|
||||
bun-version: 1.4.0
|
||||
|
||||
- uses: denoland/setup-deno@v2
|
||||
with:
|
||||
|
||||
@@ -150,13 +150,21 @@ jobs:
|
||||
COMMENT_URL: ${{ inputs.COMMENT_URL }}
|
||||
COMMENT_IS_EDIT: ${{ inputs.COMMENT_IS_EDIT }}
|
||||
run: |
|
||||
# 1) Find the thread by PR number
|
||||
# 1) Find the thread by PR number. A rate-limited or unauthorized
|
||||
# response carries no thread list at all, which under `bash -e` aborts
|
||||
# the step (jq cannot iterate null, nor parse an HTML error page)
|
||||
# rather than reaching the skip below. It is also not the same thing as
|
||||
# this PR having no thread, so it is reported rather than swallowed.
|
||||
threads=$(curl -s -H "Authorization: Bot $BOT_TOKEN" \
|
||||
"https://discord.com/api/v10/guilds/${GUILD_ID}/threads/active")
|
||||
if ! echo "$threads" | jq -e 'has("threads")' >/dev/null 2>&1; then
|
||||
echo "::warning::Discord returned no thread list; the comment on PR #${PR_NUMBER} was not relayed: ${threads:0:200}"
|
||||
exit 0
|
||||
fi
|
||||
thread_id=$(echo "$threads" | jq -r \
|
||||
--arg cid "$CHANNEL_ID" \
|
||||
--arg pref "#${PR_NUMBER}:" \
|
||||
'.threads[] | select(.parent_id == $cid and (.name | startswith($pref))) | .id')
|
||||
'(.threads // [])[] | select(.parent_id == $cid and (.name | startswith($pref))) | .id')
|
||||
|
||||
if [ -z "$thread_id" ]; then
|
||||
echo "Thread not found for PR #${PR_NUMBER}, skipping"
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
{
|
||||
".": "1.792.2"
|
||||
".": "1.796.0"
|
||||
}
|
||||
|
||||
@@ -1,5 +1,94 @@
|
||||
# Changelog
|
||||
|
||||
## [1.796.0](https://github.com/windmill-labs/windmill/compare/v1.795.0...v1.796.0) (2026-08-24)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* add instance setting to mute zombie job restart alerts ([#10813](https://github.com/windmill-labs/windmill/issues/10813)) ([2906504](https://github.com/windmill-labs/windmill/commit/2906504125136afa280884b55e2f877a29a466a9))
|
||||
* AI agent evals: datasets, scored runs and comparison ([#10633](https://github.com/windmill-labs/windmill/issues/10633)) ([9c55785](https://github.com/windmill-labs/windmill/commit/9c557859c5ffede921690cd3d224239b9305c9b8))
|
||||
* **datatables:** add a down migration from the migration viewer ([#10812](https://github.com/windmill-labs/windmill/issues/10812)) ([3b2a6d7](https://github.com/windmill-labs/windmill/commit/3b2a6d76045cf5ae48ddfb76b871e24fd889298a))
|
||||
* **frontend:** warn when COEP blocks cross-origin resources in raw app editor preview ([#10328](https://github.com/windmill-labs/windmill/issues/10328)) ([7751d3e](https://github.com/windmill-labs/windmill/commit/7751d3e43ee1abbba9a6cca026be78504f0c5dff))
|
||||
* track token cost in AI sessions and chats ([#10688](https://github.com/windmill-labs/windmill/issues/10688)) ([b6e0591](https://github.com/windmill-labs/windmill/commit/b6e059116aa55fa5aa1226f5b3300bb2c8683f1a))
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* **ai-chat:** keep the composer usable while a question is pending ([#10816](https://github.com/windmill-labs/windmill/issues/10816)) ([25a3e6e](https://github.com/windmill-labs/windmill/commit/25a3e6ea7a7efb29ea3868ac0d1453d074325717))
|
||||
* **frontend:** mint string password secrets in the operating workspace ([#10815](https://github.com/windmill-labs/windmill/issues/10815)) ([93081e2](https://github.com/windmill-labs/windmill/commit/93081e255f06386c3e21a838a752e5302ad6e6fe))
|
||||
* keep ai chat messages when leaving the page mid-generation ([#10809](https://github.com/windmill-labs/windmill/issues/10809)) ([541b6c8](https://github.com/windmill-labs/windmill/commit/541b6c849657d13fed3580407a00a996e891ad9e))
|
||||
* patch sqlx so a cancelled BEGIN cannot poison a pooled connection ([#10823](https://github.com/windmill-labs/windmill/issues/10823)) ([8dbd12e](https://github.com/windmill-labs/windmill/commit/8dbd12ecc1a8d0c03ba32b797c5a8cd9ee2d57b4))
|
||||
* qualify foreign key targets in generated datatable migrations ([#10821](https://github.com/windmill-labs/windmill/issues/10821)) ([29c311a](https://github.com/windmill-labs/windmill/commit/29c311ab318f7e443d696baf871b8a0558fa8524))
|
||||
|
||||
## [1.795.0](https://github.com/windmill-labs/windmill/compare/v1.794.1...v1.795.0) (2026-08-22)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* show the date on the runs dashboard chart axes ([#10808](https://github.com/windmill-labs/windmill/issues/10808)) ([a350f7c](https://github.com/windmill-labs/windmill/commit/a350f7c68e14909746427e716c5fdc3144d1df71))
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* keep raw-app files within their app folder on sync pull ([#10796](https://github.com/windmill-labs/windmill/issues/10796)) ([5b885ae](https://github.com/windmill-labs/windmill/commit/5b885ae311f079a7852e0ecd8fb94e57e79c707f))
|
||||
* keep workflow-as-code scripts off dedicated workers ([#10805](https://github.com/windmill-labs/windmill/issues/10805)) ([01891cd](https://github.com/windmill-labs/windmill/commit/01891cd73207e290a614cf7da506909fc4993646))
|
||||
* name the requested storage when a workspace storage lookup finds nothing ([#10803](https://github.com/windmill-labs/windmill/issues/10803)) ([01fc4f1](https://github.com/windmill-labs/windmill/commit/01fc4f1568c2010af7c23929ff50108b5e1fb635))
|
||||
* require an unscoped token to reach the workspace encryption key ([#10798](https://github.com/windmill-labs/windmill/issues/10798)) ([25d9a20](https://github.com/windmill-labs/windmill/commit/25d9a206304268326505ceef8c20cdb7941b6558))
|
||||
* require item read scope on workspace tarball export ([#10797](https://github.com/windmill-labs/windmill/issues/10797)) ([dc27db6](https://github.com/windmill-labs/windmill/commit/dc27db68de21c4d13033222cdaadbc4e2e732fe8))
|
||||
* scope capture deletion to the workspace in the request path ([#10795](https://github.com/windmill-labs/windmill/issues/10795)) ([40f0cab](https://github.com/windmill-labs/windmill/commit/40f0cab2adbdfbf1bfb12b7fbc3e419951fc8179))
|
||||
* size the ephemeral job token to the job timeout it must serve ([#10804](https://github.com/windmill-labs/windmill/issues/10804)) ([4b406e3](https://github.com/windmill-labs/windmill/commit/4b406e37c05a63ae89bc007ac0e6e669e7f84125))
|
||||
|
||||
## [1.794.1](https://github.com/windmill-labs/windmill/compare/v1.794.0...v1.794.1) (2026-08-21)
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* **ci:** unbreak the windows test jobs and the discord comment relay ([#10799](https://github.com/windmill-labs/windmill/issues/10799)) ([5088e13](https://github.com/windmill-labs/windmill/commit/5088e1370537641a83ad86959c586945f6033414))
|
||||
* keep every value of a repeated multipart field ([#10800](https://github.com/windmill-labs/windmill/issues/10800)) ([e0510fe](https://github.com/windmill-labs/windmill/commit/e0510fea21006a06eee7eecd4587161970d7f4d5))
|
||||
|
||||
## [1.794.0](https://github.com/windmill-labs/windmill/compare/v1.793.0...v1.794.0) (2026-08-21)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* inline login errors and a narrower single-column login card ([#10777](https://github.com/windmill-labs/windmill/issues/10777)) ([28b2ca6](https://github.com/windmill-labs/windmill/commit/28b2ca63672c916e07bd028eab07728d1aa4f0fe))
|
||||
* support application default credentials for gcp pub/sub triggers ([#10778](https://github.com/windmill-labs/windmill/issues/10778)) ([8e508ea](https://github.com/windmill-labs/windmill/commit/8e508ea01a1b41bd48b2ec3db29123430938f444))
|
||||
* upgrade bun to 1.4.0 and demote deno in the language picker ([#10784](https://github.com/windmill-labs/windmill/issues/10784)) ([d85050f](https://github.com/windmill-labs/windmill/commit/d85050f505b3ddc3f3c82f43dd3a9e4c32a1ee34))
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* apply the first script kind selection in the script editor ([#10789](https://github.com/windmill-labs/windmill/issues/10789)) ([75d0c29](https://github.com/windmill-labs/windmill/commit/75d0c29586a617f2cbfbf66720bdd0e47d6f92b4))
|
||||
* build the global chat's prompt identity from the operating workspace ([#10793](https://github.com/windmill-labs/windmill/issues/10793)) ([0b3dc3e](https://github.com/windmill-labs/windmill/commit/0b3dc3e5c9dd45847a26969f24ad33825445ea6a))
|
||||
* confine job tokens to workspace-scoped API routes ([#10631](https://github.com/windmill-labs/windmill/issues/10631)) ([9022dc9](https://github.com/windmill-labs/windmill/commit/9022dc9d440b95a4c45d22675f009acaf78daab7))
|
||||
* ground the chat's AI agent provider in the workspace's models ([#10774](https://github.com/windmill-labs/windmill/issues/10774)) ([449b1a6](https://github.com/windmill-labs/windmill/commit/449b1a69338479fb654308c83273613929a74fda))
|
||||
* make workspace preprocessor scripts selectable in flow preprocessor steps ([#10786](https://github.com/windmill-labs/windmill/issues/10786)) ([a9112b7](https://github.com/windmill-labs/windmill/commit/a9112b72a527af06a204827fbdff5ff9cb451f5d))
|
||||
* resolve a script path to its new version as soon as the lock lands ([#10794](https://github.com/windmill-labs/windmill/issues/10794)) ([3c8e4b4](https://github.com/windmill-labs/windmill/commit/3c8e4b43fd005db405f60794afca0e389445e350))
|
||||
* split the MCP script tools into createScript and updateScript ([#10783](https://github.com/windmill-labs/windmill/issues/10783)) ([92a454b](https://github.com/windmill-labs/windmill/commit/92a454b7a81cb1ecb98954387cbb8a361932775d))
|
||||
|
||||
## [1.793.0](https://github.com/windmill-labs/windmill/compare/v1.792.2...v1.793.0) (2026-08-20)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* add WM_ROOT_WORKSPACE, the closest dev or prod workspace of a job ([#10776](https://github.com/windmill-labs/windmill/issues/10776)) ([1f59841](https://github.com/windmill-labs/windmill/commit/1f59841a67766582b9b810ef7ff29a9f6d2bdced))
|
||||
* **cli:** deduplicate identical script lockfiles (dedupeLockfiles) ([#10769](https://github.com/windmill-labs/windmill/issues/10769)) ([05abf6d](https://github.com/windmill-labs/windmill/commit/05abf6d5aa0e4326c79a8cfabda721f8c06ec8c5))
|
||||
* guided setup wizard for data tables on Cloud ([#10584](https://github.com/windmill-labs/windmill/issues/10584)) ([5fb145c](https://github.com/windmill-labs/windmill/commit/5fb145c79ff74e7447a699c6c70c876ee69fad43))
|
||||
* make the Git Repo Viewer work with GitHub App repositories ([#10765](https://github.com/windmill-labs/windmill/issues/10765)) ([5099f40](https://github.com/windmill-labs/windmill/commit/5099f405d4f49b4f7231e7a327392221aabf4f64))
|
||||
* rework the resource type list in the add-resource drawer ([#10757](https://github.com/windmill-labs/windmill/issues/10757)) ([c7ec33c](https://github.com/windmill-labs/windmill/commit/c7ec33cc9f01aa7c616bf009f974dccb4396a5b4))
|
||||
* **sessions:** batch edit, filters and grouping in the session sidebar ([#10772](https://github.com/windmill-labs/windmill/issues/10772)) ([ac27d02](https://github.com/windmill-labs/windmill/commit/ac27d0200de8234bcf55d23915a87879ae1dd47b))
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* explain the 6-field cron format when a schedule is rejected ([#10768](https://github.com/windmill-labs/windmill/issues/10768)) ([f6645af](https://github.com/windmill-labs/windmill/commit/f6645af77e09df669f28a3e4a6c13ae630ebf85e))
|
||||
* gate the chat's open_page on the operating workspace's role ([#10779](https://github.com/windmill-labs/windmill/issues/10779)) ([ee1f981](https://github.com/windmill-labs/windmill/commit/ee1f9814c2c30e96b3e140ae0810080afc4a73d9))
|
||||
* refuse an MCP endpoint call whose required request body is empty ([#10771](https://github.com/windmill-labs/windmill/issues/10771)) ([dad8fed](https://github.com/windmill-labs/windmill/commit/dad8fed6477aaaca97e9b6cc348d8be53a76a43c))
|
||||
* reject invalid AI agent tool names when the chat writes a flow ([#10756](https://github.com/windmill-labs/windmill/issues/10756)) ([2b4369d](https://github.com/windmill-labs/windmill/commit/2b4369d7cb5c026c75a1c8c8fcd5d35dbf7ee5a8))
|
||||
* scope git-sync concurrency key per repository ([#10767](https://github.com/windmill-labs/windmill/issues/10767)) ([ed2ff6c](https://github.com/windmill-labs/windmill/commit/ed2ff6c5e7755fb32bb7c6fdb015456d8088c701))
|
||||
* **security:** a WM_TOKEN job token can never be a global superadmin (GHSA-hfh4-cx4h-3fcr) ([#10124](https://github.com/windmill-labs/windmill/issues/10124)) ([c2deea1](https://github.com/windmill-labs/windmill/commit/c2deea13b7d5d98e3fc2e0c624b14fd87f2f3341))
|
||||
* teach the AI the raw-app job bindings, the SDK reference and the draft/deployed split ([#10754](https://github.com/windmill-labs/windmill/issues/10754)) ([574775d](https://github.com/windmill-labs/windmill/commit/574775d50cbb34d114275463e1a4258dbabeb47c))
|
||||
|
||||
## [1.792.2](https://github.com/windmill-labs/windmill/compare/v1.792.1...v1.792.2) (2026-08-19)
|
||||
|
||||
|
||||
|
||||
+1
-1
@@ -287,7 +287,7 @@ COPY --from=windmill_duckdb_ffi_internal_builder /windmill-duckdb-ffi-internal/t
|
||||
|
||||
COPY --from=denoland/deno:2.2.1 --chmod=755 /usr/bin/deno /usr/bin/deno
|
||||
|
||||
COPY --from=oven/bun:1.3.10 /usr/local/bin/bun /usr/bin/bun
|
||||
COPY --from=oven/bun:1.4.0 /usr/local/bin/bun /usr/bin/bun
|
||||
|
||||
# Install windmill CLI
|
||||
RUN bun install -g windmill-cli \
|
||||
|
||||
@@ -285,6 +285,7 @@ On self-hosted instances, you might want to import all the approved resource typ
|
||||
| MIN_FREE_DISK_SPACE_MB | 15000 | Minimum amount of free space on worker. Sends critical alert if worker has less free space. | Worker |
|
||||
| RUN_UPDATE_CA_CERTIFICATE_AT_START | false | If true, runs CA certificate update command at startup before other initialization | All |
|
||||
| RUN_UPDATE_CA_CERTIFICATE_PATH | /usr/sbin/update-ca-certificates | Path to the CA certificate update command/script to run when RUN_UPDATE_CA_CERTIFICATE_AT_START is true | All |
|
||||
| GOOGLE_APPLICATION_CREDENTIALS | None | (ee only) Credentials file for GCP Pub/Sub triggers that authenticate as the instance rather than through a `gcloud` resource (workspace admins only). Application default credentials also resolve the gcloud well-known file and the GCE metadata server. Workload Identity Federation files work with the `file`, `url` and `aws` credential sources; the `executable` source is not supported. | Server |
|
||||
|
||||
## Run a local dev setup
|
||||
|
||||
|
||||
@@ -0,0 +1,36 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import {
|
||||
handleBenchmarkApiFetch,
|
||||
hasBenchmarkApiHandler,
|
||||
registerBenchmarkWorkspaceRunnables,
|
||||
unregisterBenchmarkWorkspace
|
||||
} from './mockBackend'
|
||||
|
||||
// A global eval run registers its workspace under a mkdtemp path, so the workspace id the
|
||||
// frontend interpolates into the models URL carries slashes. A handler that assumed a single
|
||||
// path segment silently fell through to the network, and every run took the offline fallback.
|
||||
const WORKSPACE = '/tmp/wmill-frontend-global-benchmark-abc123'
|
||||
const RESOURCE = 'f/evals/global/anthropic_main'
|
||||
const URL_FOR = (workspace: string) =>
|
||||
`http://benchmark.local/api/w/${workspace}/ai/proxy/models`
|
||||
|
||||
describe('benchmark /ai/proxy/models', () => {
|
||||
it('serves the seeded listing for a workspace id that is a path', async () => {
|
||||
registerBenchmarkWorkspaceRunnables(WORKSPACE, {
|
||||
aiProviders: [
|
||||
{ path: RESOURCE, kind: 'anthropic', models: ['claude-sonnet-5', 'claude-opus-5'] }
|
||||
]
|
||||
})
|
||||
try {
|
||||
expect(hasBenchmarkApiHandler(URL_FOR(WORKSPACE))).toBe(true)
|
||||
const response = handleBenchmarkApiFetch(URL_FOR(WORKSPACE), {
|
||||
headers: { 'X-Resource-Path': RESOURCE, 'X-Provider': 'anthropic' }
|
||||
})
|
||||
await expect(response.json()).resolves.toEqual({
|
||||
data: [{ id: 'claude-sonnet-5' }, { id: 'claude-opus-5' }]
|
||||
})
|
||||
} finally {
|
||||
unregisterBenchmarkWorkspace(WORKSPACE)
|
||||
}
|
||||
})
|
||||
})
|
||||
@@ -1,6 +1,7 @@
|
||||
import { mkdtemp, rm } from "fs/promises";
|
||||
import { tmpdir } from "os";
|
||||
import { join } from "path";
|
||||
import type { ChatCompletionMessageParam } from "openai/resources/chat/completions.mjs";
|
||||
import type { AIProvider } from "$lib/gen/types.gen";
|
||||
import {
|
||||
globalToolsFor,
|
||||
@@ -75,11 +76,35 @@ export interface GlobalUserFixture {
|
||||
folders_read?: string[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Flatten every assistant turn's text. Tool calls are excluded — only what the
|
||||
* user would actually read counts as having been said to them.
|
||||
*/
|
||||
function assistantTextOf(messages: ChatCompletionMessageParam[]): string {
|
||||
const parts: string[] = [];
|
||||
for (const message of messages) {
|
||||
if (message.role !== "assistant") continue;
|
||||
const content = message.content;
|
||||
if (typeof content === "string") {
|
||||
parts.push(content);
|
||||
} else if (Array.isArray(content)) {
|
||||
for (const part of content) {
|
||||
if (part && typeof part === "object" && "text" in part) {
|
||||
parts.push(String((part as { text?: unknown }).text ?? ""));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return parts.join("\n");
|
||||
}
|
||||
|
||||
export interface GlobalEvalResult {
|
||||
success: boolean;
|
||||
state: GlobalDraftState;
|
||||
error?: string;
|
||||
assistantMessageCount: number;
|
||||
/** Everything the assistant said to the user, for `assistantExpect` checks. */
|
||||
assistantText: string;
|
||||
toolCallCount: number;
|
||||
toolsUsed: string[];
|
||||
toolCallDetails: ToolCallDetail[];
|
||||
@@ -209,6 +234,7 @@ export async function runGlobalEval(
|
||||
success: rawResult.success,
|
||||
error: rawResult.error,
|
||||
assistantMessageCount: rawResult.iterations,
|
||||
assistantText: assistantTextOf(rawResult.messages),
|
||||
toolCallCount: rawResult.toolCallsCount,
|
||||
toolsUsed: rawResult.toolsCalled,
|
||||
toolCallDetails: rawResult.toolCallDetails,
|
||||
|
||||
@@ -5,6 +5,7 @@ import type {
|
||||
Flow,
|
||||
Job,
|
||||
ListableApp,
|
||||
ListableResource,
|
||||
ListableVariable,
|
||||
Script
|
||||
} from '../../../frontend/src/lib/gen'
|
||||
@@ -64,6 +65,22 @@ export interface BenchmarkWorkspaceVariable {
|
||||
ws_specific?: boolean
|
||||
}
|
||||
|
||||
/** An AI provider resource of the benchmark workspace, as an AI agent step would reference it.
|
||||
* `models` stands in for the provider's model listing, which no eval run can reach. */
|
||||
export interface BenchmarkWorkspaceAiProvider {
|
||||
path: string
|
||||
/** Resource type, which for AI resources is the provider kind (`anthropic`, `openai`, ...). */
|
||||
kind: string
|
||||
/** What this resource's `/ai/proxy/models` listing returns. */
|
||||
models?: string[]
|
||||
/** Set to point the resource at a gateway rather than the provider's own API. */
|
||||
base_url?: string
|
||||
/** Models the workspace AI settings selected for this provider. */
|
||||
configuredModels?: string[]
|
||||
/** Marks this provider's first configured model as the workspace default. */
|
||||
isDefault?: boolean
|
||||
}
|
||||
|
||||
export interface BenchmarkWorkspaceJob {
|
||||
/** Stable id so a case prompt can reference a specific run (e.g. for get_job_logs). */
|
||||
id?: string
|
||||
@@ -80,6 +97,7 @@ export interface BenchmarkWorkspaceRunnables {
|
||||
flows?: BenchmarkWorkspaceFlow[]
|
||||
apps?: BenchmarkWorkspaceApp[]
|
||||
variables?: BenchmarkWorkspaceVariable[]
|
||||
aiProviders?: BenchmarkWorkspaceAiProvider[]
|
||||
datatables?: BenchmarkDatatableSeed[]
|
||||
jobs?: BenchmarkWorkspaceJob[]
|
||||
}
|
||||
@@ -246,6 +264,59 @@ export function listBenchmarkVariables(workspace: string): ListableVariable[] |
|
||||
return (runnables.variables ?? []).map((seed) => buildBenchmarkVariable(workspace, seed, false))
|
||||
}
|
||||
|
||||
/** AI provider resources of a benchmark workspace, shaped like `ResourceService.listResource`
|
||||
* rows (which carry no value). Null when the workspace is not a benchmark one. */
|
||||
export function listBenchmarkAiProviderResources(workspace: string): ListableResource[] | null {
|
||||
const runnables = benchmarkWorkspaceRunnables.get(workspace)
|
||||
if (!runnables) {
|
||||
return null
|
||||
}
|
||||
return (runnables.aiProviders ?? []).map((seed) => ({
|
||||
workspace_id: workspace,
|
||||
path: seed.path,
|
||||
resource_type: seed.kind,
|
||||
value: null,
|
||||
is_oauth: false,
|
||||
is_linked: false,
|
||||
is_refreshed: false,
|
||||
extra_perms: {},
|
||||
edited_at: BENCHMARK_TIMESTAMP
|
||||
}))
|
||||
}
|
||||
|
||||
/** The value of a seeded AI provider resource. Only the endpoint fields are modelled — a key is
|
||||
* never needed, because no eval run calls the provider through this resource. */
|
||||
export function getBenchmarkResourceValue(
|
||||
workspace: string,
|
||||
path: string
|
||||
): Record<string, unknown> | null {
|
||||
const seed = benchmarkWorkspaceRunnables
|
||||
.get(workspace)
|
||||
?.aiProviders?.find((entry) => entry.path === path)
|
||||
if (!seed) {
|
||||
return null
|
||||
}
|
||||
return seed.base_url ? { base_url: seed.base_url } : {}
|
||||
}
|
||||
|
||||
/** The AI settings of a benchmark workspace, as `WorkspaceService.getCopilotInfo` returns them. */
|
||||
export function getBenchmarkAiConfig(workspace: string): Record<string, unknown> | null {
|
||||
const seeds = benchmarkWorkspaceRunnables.get(workspace)?.aiProviders
|
||||
if (!seeds) {
|
||||
return null
|
||||
}
|
||||
const providers: Record<string, unknown> = {}
|
||||
let defaultModel: { model: string; provider: string } | undefined
|
||||
for (const seed of seeds) {
|
||||
const models = seed.configuredModels ?? seed.models ?? []
|
||||
providers[seed.kind] = { resource_path: seed.path, models }
|
||||
if (seed.isDefault && models[0]) {
|
||||
defaultModel = { model: models[0], provider: seed.kind }
|
||||
}
|
||||
}
|
||||
return { providers, ...(defaultModel ? { default_model: defaultModel } : {}) }
|
||||
}
|
||||
|
||||
export function getBenchmarkVariableByPath(
|
||||
workspace: string,
|
||||
path: string,
|
||||
@@ -1094,6 +1165,10 @@ function parseBenchmarkRequestBody(
|
||||
/** True when `handleBenchmarkApiFetch` has an answer for this `/api/...` url.
|
||||
* Any other relative fetch must keep its normal (non-benchmark) behavior —
|
||||
* intercepting it with a synthetic 404 sends the model into retry loops. */
|
||||
// Not anchored: the frontend builds this URL from location.origin, so it arrives absolute. The
|
||||
// workspace id is greedy because an eval workspace is a temp directory path, slashes and all.
|
||||
const BENCHMARK_AI_MODELS_PATH = /\/api\/w\/(.+)\/ai\/proxy\/models$/
|
||||
|
||||
export function hasBenchmarkApiHandler(url: string): boolean {
|
||||
const path = url.split('?')[0]
|
||||
return (
|
||||
@@ -1102,7 +1177,8 @@ export function hasBenchmarkApiHandler(url: string): boolean {
|
||||
BENCHMARK_RUN_BY_PATH.test(path) ||
|
||||
/^\/api\/w\/[^/]+\/jobs\/queue\/list$/.test(path) ||
|
||||
path === '/api/embeddings/query_hub_scripts' ||
|
||||
path.startsWith('/api/scripts/hub/get_full/')
|
||||
path.startsWith('/api/scripts/hub/get_full/') ||
|
||||
BENCHMARK_AI_MODELS_PATH.test(path)
|
||||
)
|
||||
}
|
||||
|
||||
@@ -1113,6 +1189,17 @@ export function handleBenchmarkApiFetch(url: string, init?: RequestInit): Respon
|
||||
if (path === '/api/workers/list') {
|
||||
return Response.json(BENCHMARK_WORKERS)
|
||||
}
|
||||
// The provider's own model listing, which grounds an AI agent step's model id. Keyed by the
|
||||
// resource the caller names, so two seeded providers can serve different models.
|
||||
const aiModels = BENCHMARK_AI_MODELS_PATH.exec(path)
|
||||
if (aiModels) {
|
||||
const headers = new Headers(init?.headers)
|
||||
const resourcePath = headers.get('X-Resource-Path') ?? ''
|
||||
const seed = benchmarkWorkspaceRunnables
|
||||
.get(decodeURIComponent(aiModels[1]))
|
||||
?.aiProviders?.find((entry) => entry.path === resourcePath)
|
||||
return Response.json({ data: (seed?.models ?? []).map((id) => ({ id })) })
|
||||
}
|
||||
if (/^\/api\/w\/[^/]+\/jobs\/queue\/list$/.test(path)) {
|
||||
return Response.json([])
|
||||
}
|
||||
|
||||
@@ -9,6 +9,15 @@ import { handleBenchmarkApiFetch, hasBenchmarkApiHandler } from './mockBackend'
|
||||
// no meaning in the vitest environment — serve the ones the benchmark handles.
|
||||
// Every other relative fetch keeps its normal behavior (it fails the same way
|
||||
// it does without this stub) so unrelated tools see an unchanged environment.
|
||||
// The frontend builds API URLs from location.origin (fetchAvailableModels does), and node has no
|
||||
// location — without one those calls throw before the stub below ever sees them.
|
||||
if (typeof (globalThis as { location?: unknown }).location === 'undefined') {
|
||||
Object.defineProperty(globalThis, 'location', {
|
||||
value: new URL('http://benchmark.local/'),
|
||||
configurable: true
|
||||
})
|
||||
}
|
||||
|
||||
const ORIGINAL_FETCH = globalThis.fetch
|
||||
globalThis.fetch = (async (input: unknown, init?: RequestInit) => {
|
||||
const url = typeof input === 'string' ? input : ((input as Request | URL | null)?.url ?? '')
|
||||
@@ -57,8 +66,11 @@ vi.mock('$lib/gen', async () => {
|
||||
getBenchmarkOwnDraft,
|
||||
getBenchmarkScriptByHash,
|
||||
getBenchmarkScriptByPath,
|
||||
getBenchmarkAiConfig,
|
||||
getBenchmarkResourceValue,
|
||||
getBenchmarkVariableByPath,
|
||||
hasBenchmarkWorkspace,
|
||||
listBenchmarkAiProviderResources,
|
||||
listBenchmarkApps,
|
||||
listBenchmarkDatatables,
|
||||
listBenchmarkDrafts,
|
||||
@@ -301,6 +313,10 @@ vi.mock('$lib/gen', async () => {
|
||||
: actual.JobService.getJobLogs(data)
|
||||
}),
|
||||
WorkspaceService: wrapService(actual.WorkspaceService, {
|
||||
getCopilotInfo: async (data: { workspace: string }) =>
|
||||
hasBenchmarkWorkspace(data.workspace)
|
||||
? (getBenchmarkAiConfig(data.workspace) ?? {})
|
||||
: actual.WorkspaceService.getCopilotInfo(data),
|
||||
listDataTableTables: async (data: { workspace: string }) =>
|
||||
hasBenchmarkWorkspace(data.workspace)
|
||||
? (listBenchmarkDatatables(data.workspace) ?? [])
|
||||
@@ -340,15 +356,34 @@ vi.mock('$lib/gen', async () => {
|
||||
}),
|
||||
ResourceService: wrapService(actual.ResourceService, {
|
||||
existsResource: async (data: { workspace: string; path: string }) =>
|
||||
hasBenchmarkWorkspace(data.workspace) ? false : actual.ResourceService.existsResource(data),
|
||||
listResource: async (data: { workspace: string }) =>
|
||||
hasBenchmarkWorkspace(data.workspace) ? [] : actual.ResourceService.listResource(data),
|
||||
hasBenchmarkWorkspace(data.workspace)
|
||||
? Boolean(getBenchmarkResourceValue(data.workspace, data.path))
|
||||
: actual.ResourceService.existsResource(data),
|
||||
// Only AI provider resources are modelled: they are what an AI agent step references.
|
||||
listResource: async (data: { workspace: string; resourceType?: string }) => {
|
||||
if (!hasBenchmarkWorkspace(data.workspace)) {
|
||||
return actual.ResourceService.listResource(data)
|
||||
}
|
||||
const seeded = listBenchmarkAiProviderResources(data.workspace) ?? []
|
||||
const wanted = data.resourceType?.split(',')
|
||||
return wanted ? seeded.filter((r) => wanted.includes(r.resource_type)) : seeded
|
||||
},
|
||||
getResource: async (data: { workspace: string; path: string }) => {
|
||||
if (hasBenchmarkWorkspace(data.workspace)) {
|
||||
throw new Error(`Resource "${data.path}" not found in benchmark workspace`)
|
||||
}
|
||||
return actual.ResourceService.getResource(data)
|
||||
},
|
||||
getResourceValue: async (data: { workspace: string; path: string }) => {
|
||||
if (!hasBenchmarkWorkspace(data.workspace)) {
|
||||
return actual.ResourceService.getResourceValue(data)
|
||||
}
|
||||
const value = getBenchmarkResourceValue(data.workspace, data.path)
|
||||
if (!value) {
|
||||
throw new Error(`Resource "${data.path}" not found in benchmark workspace`)
|
||||
}
|
||||
return value
|
||||
},
|
||||
queryResourceTypes: async (data: { workspace: string }) =>
|
||||
hasBenchmarkWorkspace(data.workspace) ? [] : actual.ResourceService.queryResourceTypes(data)
|
||||
}),
|
||||
|
||||
@@ -324,3 +324,32 @@
|
||||
- reuses the existing datatable configuration rather than creating new tables
|
||||
- presents a read-only dashboard or summary of available analytics data
|
||||
- keeps the configured datatable references available in the app artifact
|
||||
|
||||
# GIT-967, app mode: asked to track a long-running job, the agent hand-wrote a
|
||||
# runnable that fetched the jobs REST API — guessing at WM_TOKEN and a base URL
|
||||
# until it fell back to localhost — instead of using backendAsync + getJob/waitJob,
|
||||
# which the generated ./wmill bindings already provide.
|
||||
- id: app-long-job-progress
|
||||
prompt: |-
|
||||
Add a "Generate report" button. Building the report takes a few minutes, so as soon
|
||||
as the user clicks it the app should show the run's job id and keep updating its
|
||||
status until it finishes, then display the result.
|
||||
runtime:
|
||||
maxTurns: 22
|
||||
validate:
|
||||
forbiddenAppContent:
|
||||
- BASE_INTERNAL_URL
|
||||
- WM_BASE_URL
|
||||
- WM_TOKEN
|
||||
- localhost:8000
|
||||
- getResultMaybe
|
||||
- jobs/list
|
||||
- getWorkspaceToken
|
||||
- getBaseUrl
|
||||
judgeChecklist:
|
||||
- adds a Generate report button that starts the report
|
||||
- shows the run's job id as soon as the run starts
|
||||
- keeps the status updating while the run is in flight and shows the result when it completes
|
||||
- starts the run with backendAsync and tracks it with getJob, waitJob or streamJob — all three are real exports of the generated ./wmill module, alongside backend and backendAsync
|
||||
- does not write a backend runnable that polls job status or lists jobs itself
|
||||
- does not call the Windmill API with fetch from the frontend, and does not read WM_TOKEN, BASE_INTERNAL_URL or WM_BASE_URL anywhere
|
||||
|
||||
@@ -2191,3 +2191,177 @@
|
||||
# one — the judge has to be told, or it reads the "" as the value having been cleared.
|
||||
- 'the draft''s empty value string is expected and correct: a secret variable draft stores "" when no new value is staged, which is exactly right for a description-only edit, so it does NOT mean the value was cleared or changed'
|
||||
- leaves the result as an AI draft and does not deploy it
|
||||
|
||||
# Reproduces GIT-967: a hello-world flow with a React app in front of it. The agent
|
||||
# built a backend runnable that fetched the Windmill REST API directly, guessing at
|
||||
# WM_TOKEN / BASE_INTERNAL_URL / WM_BASE_URL until it fell back to localhost:8000, and
|
||||
# wired the app to a flow that was never deployed. Both are pinned here.
|
||||
- id: global-app-triggers-flow
|
||||
prompt: |-
|
||||
Create a hello world workflow at `f/evals/global/hello_world_flow` that just returns a
|
||||
greeting, and a React app at `f/evals/global/hello_world_app` with a button that runs
|
||||
that workflow and shows the result on the page.
|
||||
initial: ai_evals/fixtures/frontend/global/initial/user_admin_evals_folder.json
|
||||
runtime:
|
||||
maxTurns: 24
|
||||
validate:
|
||||
draftCountAtLeast: 2
|
||||
requiredDrafts:
|
||||
- type: flow
|
||||
path: f/evals/global/hello_world_flow
|
||||
- type: app
|
||||
path: f/evals/global/hello_world_app
|
||||
valueIncludes:
|
||||
# The frontend must go through the generated bindings, which is the only
|
||||
# credentialed way it can reach anything server-side.
|
||||
- './wmill'
|
||||
valueExcludes:
|
||||
# Every artifact of hand-rolling HTTP against the Windmill API from a
|
||||
# runnable instead of using the client or a path runnable.
|
||||
- BASE_INTERNAL_URL
|
||||
- WM_BASE_URL
|
||||
- WM_TOKEN
|
||||
- localhost:8000
|
||||
- getResultMaybe
|
||||
- jobs/list
|
||||
- getWorkspaceToken
|
||||
- getBaseUrl
|
||||
toolExpect:
|
||||
requiredToolsUsed:
|
||||
- write_flow
|
||||
- init_app
|
||||
- write_app_runnable
|
||||
judgeChecklist:
|
||||
- creates a flow at f/evals/global/hello_world_flow that returns a greeting
|
||||
- creates a React raw app at f/evals/global/hello_world_app with a button that runs the flow
|
||||
- the app actually invokes the flow rather than reimplementing its logic in an inline runnable
|
||||
# Both items staying drafts is the CORRECT outcome — the chat must not deploy without
|
||||
# being asked. What is judged is that the flow is named as the one item needing a deploy.
|
||||
- 'leaving both the flow and the app as drafts is expected and correct: the chat deploys nothing unless asked. Judge only whether the flow is identified as the single item that will need deploying, and that the app is NOT presented as needing deployment to be tried'
|
||||
# The judge has repeatedly flagged a correct app as broken over this.
|
||||
- '`/wmill.d.ts` is generated by Windmill from the app''s runnables and is deliberately absent from the app''s files — its absence is correct and is NOT a missing-module bug'
|
||||
- the app reaches the flow through a backend runnable, not through hand-written HTTP calls to the Windmill API
|
||||
- the app's frontend calls the runnable via the generated ./wmill bindings — backend, or backendAsync together with waitJob/getJob/streamJob, all of which are real exports of that module
|
||||
- does not read WM_TOKEN, BASE_INTERNAL_URL or WM_BASE_URL, and does not construct a Windmill API URL anywhere
|
||||
- does not call windmill-client functions that do not exist, such as getBaseUrl or getWorkspaceToken
|
||||
# The point of the case: a path runnable (and wmill.runFlow*) resolves the deployed
|
||||
# item, so a flow left as a draft makes the app dead on arrival and the user has to be
|
||||
# told. This can't live in judgeChecklist — the global judge only ever sees the drafts,
|
||||
# never what the assistant said.
|
||||
assistantExpect:
|
||||
# Plain substring test: an alternative must name the FLOW and read as an outstanding
|
||||
# obligation. Flow-agnostic wording is satisfied by "the app must be deployed"; tense-neutral
|
||||
# wording by a deploy the agent only claims to have made. The mirror expectation "don't ask
|
||||
# to deploy the app" can't be a forbiddenMentions entry, since correct answers negate it.
|
||||
requiredMentionsAnyOf:
|
||||
- - deploy the flow
|
||||
- deploy that flow
|
||||
- deploy this flow
|
||||
- deploy just the flow
|
||||
- deploy the workflow
|
||||
- deploy that workflow
|
||||
- deploy hello_world_flow
|
||||
- flow must be deployed
|
||||
- flow needs to be deployed
|
||||
- flow has to be deployed
|
||||
- flow needs deploying
|
||||
- flow will need to be deployed
|
||||
- flow will have to be deployed
|
||||
- once the flow is deployed
|
||||
- until the flow is deployed
|
||||
- workflow must be deployed
|
||||
- workflow needs to be deployed
|
||||
- workflow has to be deployed
|
||||
|
||||
# --- AI agent steps: the provider config must name a model the referenced resource serves ---
|
||||
# The workspace AI settings are not a runtime gate, so the model id can only come from the
|
||||
# resources themselves. Both cases seed AI provider resources; without them the assistant has
|
||||
# nothing to write but a guessed id, which is the defect these pin.
|
||||
|
||||
- id: global-ai-agent-step-uses-workspace-model
|
||||
prompt: |-
|
||||
Create a draft flow at `f/evals/global/support_answer` with a single AI agent step that answers
|
||||
the user's question. The question comes in as a flow input called `query`. No tools.
|
||||
Leave it as an AI draft only; do not deploy or save it.
|
||||
initial: ai_evals/fixtures/frontend/global/initial/ai_provider_anthropic.json
|
||||
runtime:
|
||||
maxTurns: 10
|
||||
validate:
|
||||
draftCountExactly: 1
|
||||
requiredDrafts:
|
||||
- type: flow
|
||||
path: f/evals/global/support_answer
|
||||
valueIncludes:
|
||||
- aiagent
|
||||
- $res:f/evals/global/anthropic_main
|
||||
- claude-sonnet-5
|
||||
toolExpect:
|
||||
forbiddenToolsUsed:
|
||||
- deploy_workspace_item
|
||||
- delete_workspace_item
|
||||
judgeChecklist:
|
||||
- creates a draft flow at f/evals/global/support_answer with one AI agent step
|
||||
- the step's provider is an object carrying kind, resource and model, not a bare resource string
|
||||
- the provider references the workspace's own AI provider resource f/evals/global/anthropic_main
|
||||
- the model is one the workspace configured for that resource, not an id invented from memory
|
||||
- the user's question reaches the agent from the query flow input
|
||||
- the result stays as an AI draft and is not deployed
|
||||
|
||||
- id: global-ai-agent-step-asks-which-provider
|
||||
prompt: |-
|
||||
Create a draft flow at `f/evals/global/triage_ticket` with a single AI agent step that summarises
|
||||
an incoming ticket. The ticket text comes in as a flow input called `ticket`. No tools.
|
||||
Leave it as an AI draft only; do not deploy or save it.
|
||||
initial: ai_evals/fixtures/frontend/global/initial/ai_providers_two.json
|
||||
runtime:
|
||||
maxTurns: 10
|
||||
toolExpect:
|
||||
requiredToolsUsed:
|
||||
- askUserQuestion
|
||||
forbiddenToolsUsed:
|
||||
- deploy_workspace_item
|
||||
- delete_workspace_item
|
||||
# Two configured providers make the choice the user's: the step may only be written once the
|
||||
# question is answered, and it must use one of the workspace's own resources. Which one depends
|
||||
# on the answer, so the assertion covers the shared path prefix rather than a fixed resource.
|
||||
validate:
|
||||
draftCountExactly: 1
|
||||
requiredDrafts:
|
||||
- type: flow
|
||||
path: f/evals/global/triage_ticket
|
||||
valueIncludes:
|
||||
- aiagent
|
||||
- $res:f/evals/global/
|
||||
# Whether the assistant asked before writing lives in the tool record, not in the draft the
|
||||
# global judge sees.
|
||||
skipJudge: true
|
||||
|
||||
# Only satisfiable from the resource's own model listing: the workspace AI settings configure just
|
||||
# claude-sonnet-5, so an assistant working off the settings has no Opus id to write.
|
||||
- id: global-ai-agent-step-uses-listed-model
|
||||
prompt: |-
|
||||
Create a draft flow at `f/evals/global/deep_review` with a single AI agent step that reviews a
|
||||
pull request diff. The diff comes in as a flow input called `diff`. Use the Opus model — this
|
||||
one needs the strongest model available. No tools.
|
||||
Leave it as an AI draft only; do not deploy or save it.
|
||||
initial: ai_evals/fixtures/frontend/global/initial/ai_provider_anthropic.json
|
||||
runtime:
|
||||
maxTurns: 10
|
||||
validate:
|
||||
draftCountExactly: 1
|
||||
requiredDrafts:
|
||||
- type: flow
|
||||
path: f/evals/global/deep_review
|
||||
valueIncludes:
|
||||
- aiagent
|
||||
- $res:f/evals/global/anthropic_main
|
||||
- claude-opus-5
|
||||
toolExpect:
|
||||
forbiddenToolsUsed:
|
||||
- deploy_workspace_item
|
||||
- delete_workspace_item
|
||||
judgeChecklist:
|
||||
- creates a draft flow at f/evals/global/deep_review with one AI agent step
|
||||
- the step uses the workspace's anthropic resource f/evals/global/anthropic_main
|
||||
- the model is the Opus one the user asked for, taken from the models that resource serves
|
||||
- the diff flow input reaches the agent
|
||||
|
||||
@@ -21,6 +21,7 @@ interface RawEvalCase {
|
||||
validate?: EvalValidationSpec;
|
||||
toolExpect?: EvalCase["toolExpect"];
|
||||
cliExpect?: CliValidationSpec;
|
||||
assistantExpect?: EvalCase["assistantExpect"];
|
||||
judgeChecklist?: string[];
|
||||
skipJudge?: boolean;
|
||||
runtime?: EvalCaseRuntimeSpec;
|
||||
@@ -50,6 +51,7 @@ export async function loadCases(mode: EvalMode): Promise<EvalCase[]> {
|
||||
validate: entry.validate,
|
||||
toolExpect: entry.toolExpect,
|
||||
cliExpect: entry.cliExpect,
|
||||
assistantExpect: entry.assistantExpect,
|
||||
judgeChecklist: entry.judgeChecklist,
|
||||
skipJudge: entry.skipJudge,
|
||||
runtime: entry.runtime,
|
||||
|
||||
@@ -7,7 +7,10 @@ import type {
|
||||
FrontendBenchmarkProgressEvent,
|
||||
ModeRunner,
|
||||
} from "./types";
|
||||
import { validateToolExpectations } from "./validators";
|
||||
import {
|
||||
validateAssistantExpectations,
|
||||
validateToolExpectations,
|
||||
} from "./validators";
|
||||
|
||||
export async function runSuite<TInitial, TExpected, TActual>(input: {
|
||||
modeRunner: ModeRunner<TInitial, TExpected, TActual>;
|
||||
@@ -182,6 +185,10 @@ async function runCaseAttempts<TInitial, TExpected, TActual>(input: {
|
||||
...validateToolExpectations({
|
||||
run,
|
||||
toolExpect: input.evalCase.toolExpect,
|
||||
}),
|
||||
...validateAssistantExpectations({
|
||||
run,
|
||||
assistantExpect: input.evalCase.assistantExpect,
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
@@ -213,6 +213,24 @@ export type EvalValidationSpec =
|
||||
| AppValidationSpec
|
||||
| GlobalValidationSpec;
|
||||
|
||||
/**
|
||||
* Expectations on what the assistant SAID, for cases where the deliverable is
|
||||
* partly a warning to the user. The `global` judge only ever sees the resulting
|
||||
* drafts, so "tells the user X" is invisible to it and has to be checked here.
|
||||
* Needs a mode whose runner reports `assistantText`.
|
||||
*/
|
||||
export interface AssistantValidationSpec {
|
||||
/** Each entry: at least one of its phrases appears somewhere in the assistant's text. */
|
||||
requiredMentionsAnyOf?: string[][];
|
||||
/**
|
||||
* Plain case-insensitive substring test, so it cannot see negation: a phrase the correct
|
||||
* answer might use in the negative ("you don't need to deploy the app") is not a valid
|
||||
* entry. Use it for tokens that never legitimately appear, and leave nuanced "did the
|
||||
* assistant say the right thing" expectations to the judge checklist.
|
||||
*/
|
||||
forbiddenMentions?: string[];
|
||||
}
|
||||
|
||||
export interface EvalCase {
|
||||
id: string;
|
||||
prompt: string;
|
||||
@@ -221,6 +239,7 @@ export interface EvalCase {
|
||||
validate?: EvalValidationSpec;
|
||||
toolExpect?: ToolValidationSpec;
|
||||
cliExpect?: CliValidationSpec;
|
||||
assistantExpect?: AssistantValidationSpec;
|
||||
judgeChecklist?: string[];
|
||||
skipJudge?: boolean;
|
||||
runtime?: EvalCaseRuntimeSpec;
|
||||
@@ -287,6 +306,8 @@ export interface ModeRunOutput<TActual> {
|
||||
toolsUsed: string[];
|
||||
toolCallDetails?: ToolCallDetail[];
|
||||
skillsInvoked: string[];
|
||||
/** Concatenated assistant-visible text of the run, when the mode reports it. */
|
||||
assistantText?: string;
|
||||
tokenUsage?: BenchmarkTokenUsage | null;
|
||||
/**
|
||||
* Total input tokens occupying the context window on the LAST model request
|
||||
|
||||
@@ -1,9 +1,11 @@
|
||||
import { describe, expect, it } from "bun:test";
|
||||
import { loadCases } from "./cases";
|
||||
import {
|
||||
validateAppState,
|
||||
validateCliWorkspace,
|
||||
validateGlobalState,
|
||||
validateScriptState,
|
||||
validateAssistantExpectations,
|
||||
validateToolExpectations,
|
||||
} from "./validators";
|
||||
|
||||
@@ -41,6 +43,113 @@ describe("validateScriptState", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("validateAssistantExpectations", () => {
|
||||
it("checks assistant mentions across the whole run, not just the last turn", () => {
|
||||
const run = {
|
||||
success: true,
|
||||
actual: {},
|
||||
assistantMessageCount: 2,
|
||||
toolCallCount: 0,
|
||||
toolsUsed: [],
|
||||
skillsInvoked: [],
|
||||
assistantText: "Wired the app up.\nThe flow HAS TO BE DEPLOYED before the app works.",
|
||||
};
|
||||
|
||||
const checks = validateAssistantExpectations({
|
||||
run,
|
||||
assistantExpect: {
|
||||
requiredMentionsAnyOf: [["must be deployed", "has to be deployed"], ["never said"]],
|
||||
forbiddenMentions: ["WM_TOKEN"],
|
||||
},
|
||||
});
|
||||
|
||||
expect(checks.map((c) => c.passed)).toEqual([true, false, true]);
|
||||
});
|
||||
|
||||
it("rejects a deploy claim that names the app instead of the flow", () => {
|
||||
const checks = validateAssistantExpectations({
|
||||
run: {
|
||||
success: true,
|
||||
actual: {},
|
||||
assistantMessageCount: 1,
|
||||
toolCallCount: 0,
|
||||
toolsUsed: [],
|
||||
skillsInvoked: [],
|
||||
assistantText: "Built both. The app must be deployed before the button works.",
|
||||
},
|
||||
assistantExpect: {
|
||||
requiredMentionsAnyOf: [["deploy the flow", "flow must be deployed"]],
|
||||
},
|
||||
});
|
||||
|
||||
expect(checks.map((c) => c.passed)).toEqual([false]);
|
||||
});
|
||||
|
||||
it("fails instead of passing green when the mode reports no assistant text", () => {
|
||||
const checks = validateAssistantExpectations({
|
||||
run: {
|
||||
success: true,
|
||||
actual: {},
|
||||
assistantMessageCount: 1,
|
||||
toolCallCount: 0,
|
||||
toolsUsed: [],
|
||||
skillsInvoked: [],
|
||||
},
|
||||
assistantExpect: { forbiddenMentions: ["WM_TOKEN"] },
|
||||
});
|
||||
|
||||
expect(checks.map((c) => c.passed)).toEqual([false]);
|
||||
});
|
||||
});
|
||||
|
||||
// The matcher is a plain substring test, so its failure mode is accepting an answer it should
|
||||
// reject. The real alternatives are therefore exercised against wrong answers rather than
|
||||
// eyeballed, and read out of global.yaml so an edit there cannot silently loosen them.
|
||||
describe("global-app-triggers-flow deploy expectation", () => {
|
||||
const run = (assistantText: string) => ({
|
||||
success: true,
|
||||
actual: {},
|
||||
assistantMessageCount: 1,
|
||||
toolCallCount: 0,
|
||||
toolsUsed: [],
|
||||
skillsInvoked: [],
|
||||
assistantText,
|
||||
});
|
||||
|
||||
const passes = async (assistantText: string) => {
|
||||
const cases = await loadCases("global");
|
||||
const target = cases.find((c) => c.id === "global-app-triggers-flow");
|
||||
if (!target?.assistantExpect) throw new Error("case or its assistantExpect is missing");
|
||||
const checks = validateAssistantExpectations({
|
||||
run: run(assistantText),
|
||||
assistantExpect: target.assistantExpect,
|
||||
});
|
||||
return checks.every((c) => c.passed);
|
||||
};
|
||||
|
||||
// Deploying is impossible in eval mode and the judge only sees drafts, so a claim of
|
||||
// having deployed is a hallucination this case has to reject, not evidence of success.
|
||||
it.each([
|
||||
["names the app as what needs deploying", "Built both. The app must be deployed before the button works."],
|
||||
["claims the deploy is already done", "All set — done deploying the flow, everything works now."],
|
||||
["claims it deployed the flow itself", "I deployed the flow for you, so the button works."],
|
||||
["reports a completed deploy after the fact", "After deploying the flow, I clicked the button and it returns the greeting."],
|
||||
["reports a completed deploy instrumentally", "I fixed it by deploying the flow; everything works now."],
|
||||
["says nothing about deploying", "Built the flow and the app. The button calls the flow."],
|
||||
])("rejects an answer that %s", async (_label, text) => {
|
||||
expect(await passes(text)).toBe(false);
|
||||
});
|
||||
|
||||
it.each([
|
||||
["you'll need to deploy the flow before the app's button will work"],
|
||||
["the flow has to be deployed first; the app can stay a draft"],
|
||||
["once the flow is deployed, the button will work in the preview"],
|
||||
["want me to deploy just the flow? the app stays a draft"],
|
||||
])("accepts a correct answer: %s", async (text) => {
|
||||
expect(await passes(text)).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe("validateToolExpectations", () => {
|
||||
it("accepts Windmill-prefixed schedule paths", () => {
|
||||
const checks = validateToolExpectations({
|
||||
|
||||
@@ -2,6 +2,7 @@ import path from "node:path";
|
||||
import ts from "typescript";
|
||||
import type {
|
||||
AppValidationSpec,
|
||||
AssistantValidationSpec,
|
||||
BenchmarkCheck,
|
||||
CliTrace,
|
||||
CliValidationSpec,
|
||||
@@ -160,6 +161,52 @@ function valueIncludesAnyOf(value: unknown, lowercaseNeedles: string[]): boolean
|
||||
);
|
||||
}
|
||||
|
||||
export function validateAssistantExpectations(input: {
|
||||
run: ModeRunOutput<unknown>;
|
||||
assistantExpect?: AssistantValidationSpec;
|
||||
}): BenchmarkCheck[] {
|
||||
const expect = input.assistantExpect;
|
||||
if (!expect) {
|
||||
return [];
|
||||
}
|
||||
// Only some mode runners report assistantText. Defaulting a missing one to "" would pass
|
||||
// every forbiddenMentions entry forever, so a case that expects to inspect what the
|
||||
// assistant said fails on the mode that cannot show it.
|
||||
if (input.run.assistantText === undefined) {
|
||||
return [
|
||||
check(
|
||||
"assistant text is available to check",
|
||||
false,
|
||||
"this mode's runner does not report assistantText, so assistantExpect cannot be evaluated"
|
||||
),
|
||||
];
|
||||
}
|
||||
const text = input.run.assistantText;
|
||||
const checks: BenchmarkCheck[] = [];
|
||||
|
||||
for (const phrases of expect.requiredMentionsAnyOf ?? []) {
|
||||
checks.push(
|
||||
check(
|
||||
`assistant mentions one of: ${phrases.join(" / ")}`,
|
||||
phrases.some((phrase) => assistantMentions(text, phrase)),
|
||||
truncateForDetails(text)
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
for (const phrase of expect.forbiddenMentions ?? []) {
|
||||
checks.push(
|
||||
check(
|
||||
`assistant does not mention '${phrase}'`,
|
||||
!assistantMentions(text, phrase),
|
||||
truncateForDetails(text)
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
return checks;
|
||||
}
|
||||
|
||||
export function validateToolExpectations(input: {
|
||||
run: ModeRunOutput<unknown>;
|
||||
toolExpect?: ToolValidationSpec;
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
{
|
||||
"workspace": {
|
||||
"aiProviders": [
|
||||
{
|
||||
"path": "f/evals/global/anthropic_main",
|
||||
"kind": "anthropic",
|
||||
"models": ["claude-sonnet-5", "claude-opus-5", "claude-haiku-4-5"],
|
||||
"configuredModels": ["claude-sonnet-5"],
|
||||
"isDefault": true
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
{
|
||||
"workspace": {
|
||||
"aiProviders": [
|
||||
{
|
||||
"path": "f/evals/global/anthropic_main",
|
||||
"kind": "anthropic",
|
||||
"models": ["claude-sonnet-5", "claude-opus-5", "claude-haiku-4-5"],
|
||||
"configuredModels": ["claude-sonnet-5"],
|
||||
"isDefault": true
|
||||
},
|
||||
{
|
||||
"path": "f/evals/global/openai_main",
|
||||
"kind": "openai",
|
||||
"models": ["gpt-5.6-sol", "gpt-5.6-terra"],
|
||||
"configuredModels": ["gpt-5.6-sol"]
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -70,6 +70,7 @@ export function createGlobalModeRunner(
|
||||
toolsUsed: result.toolsUsed,
|
||||
toolCallDetails: result.toolCallDetails,
|
||||
skillsInvoked: [],
|
||||
assistantText: result.assistantText,
|
||||
tokenUsage: result.tokenUsage,
|
||||
finalContextTokens: result.finalContextTokens,
|
||||
};
|
||||
|
||||
+17
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "UPDATE eval_experiment\n SET subject = jsonb_set(\n jsonb_set(subject, '{kind}', '\"agent\"'),\n '{version}', to_jsonb($4::bigint))\n WHERE workspace_id = $1 AND dataset_path = $2 AND id = $3\n AND subject ->> 'kind' = 'agent_draft'",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Text",
|
||||
"Text",
|
||||
"Uuid",
|
||||
"Int8"
|
||||
]
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "01bce88dd622f314d1a09c24cd12df5e7d3ff6a15a93e1c0c95e262f7b3d0ef1"
|
||||
}
|
||||
+35
@@ -0,0 +1,35 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "SELECT path, value, version FROM resource_version WHERE workspace_id = $1 AND id = $2",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "path",
|
||||
"type_info": "Varchar"
|
||||
},
|
||||
{
|
||||
"ordinal": 1,
|
||||
"name": "value",
|
||||
"type_info": "Jsonb"
|
||||
},
|
||||
{
|
||||
"ordinal": 2,
|
||||
"name": "version",
|
||||
"type_info": "Int8"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Text",
|
||||
"Int8"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
false,
|
||||
true,
|
||||
false
|
||||
]
|
||||
},
|
||||
"hash": "0276e6030abb2eb00a68c568a9cc60f3e7c2af0331388c4b358035de865a121a"
|
||||
}
|
||||
+24
@@ -0,0 +1,24 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "SELECT coalesce(max(run_number), 0) + 1 FROM eval_experiment\n WHERE workspace_id = $1 AND dataset_path = $2 AND subject ->> 'path' = $3",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "?column?",
|
||||
"type_info": "Int4"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Text",
|
||||
"Text",
|
||||
"Text"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
null
|
||||
]
|
||||
},
|
||||
"hash": "0335de6713de6678b9bf266121af23abc46d5db95da095bb15726c5a2db7ad2f"
|
||||
}
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "UPDATE eval_dataset SET scorers = COALESCE((\n SELECT jsonb_agg(\n CASE WHEN elem->>'path' LIKE ('u/' || $2 || '/%')\n THEN jsonb_set(elem, '{path}', to_jsonb(REGEXP_REPLACE(elem->>'path', 'u/' || $2 || '/(.*)', $1 || '/\\1')))\n ELSE elem END)\n FROM jsonb_array_elements(scorers) elem), '[]'::jsonb)\n WHERE workspace_id = $3\n AND EXISTS (SELECT 1 FROM jsonb_array_elements(scorers) e WHERE e->>'path' LIKE ('u/' || $2 || '/%'))",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Text",
|
||||
"Text",
|
||||
"Text"
|
||||
]
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "0aae275d9196e742b5783df4e67c72459d45e275bfeafa2952349cae259ac9f0"
|
||||
}
|
||||
+24
@@ -0,0 +1,24 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "SELECT pg_advisory_xact_lock(hashtext('ai_eval_open:' || $1 || '/' || $2 || '/' || $3))",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "pg_advisory_xact_lock",
|
||||
"type_info": "Void"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Text",
|
||||
"Text",
|
||||
"Text"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
null
|
||||
]
|
||||
},
|
||||
"hash": "0d6700ccffb8179e365bbc1f03398e474f23b013f642ba29ad6f68e6f047c1e5"
|
||||
}
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "UPDATE eval_experiment SET subject = jsonb_set(subject, '{path}', to_jsonb(REGEXP_REPLACE(subject->>'path', 'u/' || $2 || '/(.*)', $1 || '/\\1'))) WHERE subject->>'path' LIKE ('u/' || $2 || '/%') AND workspace_id = $3",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Text",
|
||||
"Text",
|
||||
"Text"
|
||||
]
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "1815730982dcaf7239ddcb22f88ae5c79794213cf6278167f8afdbca30b1b15c"
|
||||
}
|
||||
+22
@@ -0,0 +1,22 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "SELECT count(*) AS \"count!\" FROM eval_experiment_case\n WHERE experiment_id = $1 AND status IS NOT NULL",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "count!",
|
||||
"type_info": "Int8"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Uuid"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
null
|
||||
]
|
||||
},
|
||||
"hash": "186c663249ffada82abf61ce214f52e2730501774a0ca4dc855380e6c6487917"
|
||||
}
|
||||
+59
@@ -0,0 +1,59 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "SELECT path, summary, scorers, created_at, created_by,\n edited_at, edited_by\n FROM eval_dataset WHERE workspace_id = $1 AND path = $2",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "path",
|
||||
"type_info": "Varchar"
|
||||
},
|
||||
{
|
||||
"ordinal": 1,
|
||||
"name": "summary",
|
||||
"type_info": "Varchar"
|
||||
},
|
||||
{
|
||||
"ordinal": 2,
|
||||
"name": "scorers",
|
||||
"type_info": "Jsonb"
|
||||
},
|
||||
{
|
||||
"ordinal": 3,
|
||||
"name": "created_at",
|
||||
"type_info": "Timestamptz"
|
||||
},
|
||||
{
|
||||
"ordinal": 4,
|
||||
"name": "created_by",
|
||||
"type_info": "Varchar"
|
||||
},
|
||||
{
|
||||
"ordinal": 5,
|
||||
"name": "edited_at",
|
||||
"type_info": "Timestamptz"
|
||||
},
|
||||
{
|
||||
"ordinal": 6,
|
||||
"name": "edited_by",
|
||||
"type_info": "Varchar"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Text",
|
||||
"Text"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
false,
|
||||
true,
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
false
|
||||
]
|
||||
},
|
||||
"hash": "196939257a334f7d37aa6d66153b251446d7701a893cb7852b81cf842c0fa228"
|
||||
}
|
||||
+28
@@ -0,0 +1,28 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "SELECT case_id, ordinal FROM eval_experiment_case WHERE experiment_id = $1",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "case_id",
|
||||
"type_info": "Uuid"
|
||||
},
|
||||
{
|
||||
"ordinal": 1,
|
||||
"name": "ordinal",
|
||||
"type_info": "Int4"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Uuid"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
false,
|
||||
false
|
||||
]
|
||||
},
|
||||
"hash": "1b6e229545f6b877e72d21728257d1bddaba15ef0fbe72bb4f43b45140f184ce"
|
||||
}
|
||||
+23
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "SELECT DISTINCT path FROM script\n WHERE workspace_id = $1 AND path = ANY($2)\n AND deleted = false AND lock IS NOT NULL AND lock_error_logs IS NULL",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "path",
|
||||
"type_info": "Varchar"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Text",
|
||||
"TextArray"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
false
|
||||
]
|
||||
},
|
||||
"hash": "1db80f3ba2c6c769a98424ebf9aaf168a4fa2c64e446a824038cf267236fe979"
|
||||
}
|
||||
+15
@@ -0,0 +1,15 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "UPDATE eval_score SET error = 'The case did not run'\n WHERE experiment_id = $1 AND ordinal = ANY($2)\n AND score IS NULL AND error IS NULL AND NOT not_applicable",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Uuid",
|
||||
"Int4Array"
|
||||
]
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "242845c86084e010ab33c2197d44af9aeb181672a2f2330dbe65bfe586376450"
|
||||
}
|
||||
+24
@@ -0,0 +1,24 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "INSERT INTO ai_token_usage (workspace_id, email, provider, model, session_id, input_tokens, cache_read_tokens, cache_write_tokens, output_tokens, reported_cost_nano_usd, requests)\n SELECT $1, $2, * FROM UNNEST($3::text[], $4::text[], $5::text[], $6::bigint[], $7::bigint[], $8::bigint[], $9::bigint[], $10::bigint[], $11::bigint[])\n ON CONFLICT (workspace_id, day, email, provider, model, session_id)\n DO UPDATE SET\n input_tokens = ai_token_usage.input_tokens + EXCLUDED.input_tokens,\n cache_read_tokens = ai_token_usage.cache_read_tokens + EXCLUDED.cache_read_tokens,\n cache_write_tokens = ai_token_usage.cache_write_tokens + EXCLUDED.cache_write_tokens,\n output_tokens = ai_token_usage.output_tokens + EXCLUDED.output_tokens,\n reported_cost_nano_usd = CASE\n WHEN EXCLUDED.reported_cost_nano_usd IS NULL\n THEN ai_token_usage.reported_cost_nano_usd\n ELSE COALESCE(ai_token_usage.reported_cost_nano_usd, 0)\n + EXCLUDED.reported_cost_nano_usd\n END,\n requests = ai_token_usage.requests + EXCLUDED.requests,\n updated_at = now()",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Varchar",
|
||||
"Varchar",
|
||||
"TextArray",
|
||||
"TextArray",
|
||||
"TextArray",
|
||||
"Int8Array",
|
||||
"Int8Array",
|
||||
"Int8Array",
|
||||
"Int8Array",
|
||||
"Int8Array",
|
||||
"Int8Array"
|
||||
]
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "24fcc2b69f30953915f0cbf246e1c19b2310075a644d81c21784c991e52b4001"
|
||||
}
|
||||
+28
@@ -0,0 +1,28 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "SELECT path, scorers FROM eval_dataset\n WHERE workspace_id = $1 ORDER BY edited_at DESC LIMIT 100",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "path",
|
||||
"type_info": "Varchar"
|
||||
},
|
||||
{
|
||||
"ordinal": 1,
|
||||
"name": "scorers",
|
||||
"type_info": "Jsonb"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Text"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
false,
|
||||
false
|
||||
]
|
||||
},
|
||||
"hash": "2b41dc4d872af0e230c31bef1a496604c1562cdfb0bdb54bd3b0c6579d73e46d"
|
||||
}
|
||||
+6
-2
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "INSERT INTO datatable_migrations (workspace_id, datatable, timestamp, name, code_up, code_down) VALUES ($1, $2, $3, $4, $5, $6) ON CONFLICT (workspace_id, datatable, timestamp) DO UPDATE SET name = EXCLUDED.name, code_up = EXCLUDED.code_up, code_down = EXCLUDED.code_down",
|
||||
"query": "INSERT INTO datatable_migrations (workspace_id, datatable, timestamp, name, code_up, code_down) VALUES ($1, $2, $3, $4, $5, $6) ON CONFLICT (workspace_id, datatable, timestamp) DO UPDATE SET name = EXCLUDED.name, code_up = EXCLUDED.code_up, code_down = EXCLUDED.code_down WHERE NOT $7 OR (datatable_migrations.name = $8::text AND datatable_migrations.code_up = $9::text AND datatable_migrations.code_down IS NOT DISTINCT FROM $10::text)",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
@@ -10,10 +10,14 @@
|
||||
"Int8",
|
||||
"Varchar",
|
||||
"Text",
|
||||
"Text",
|
||||
"Bool",
|
||||
"Text",
|
||||
"Text",
|
||||
"Text"
|
||||
]
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "c5639d48c92d6863f16f97de91eae590d0ed2f4b7831956bb429a47c478f7c1f"
|
||||
"hash": "2b93351b9c2c91272c3f69c9c8eba192d2e4d087c7d7db8d3b257bb864c8265d"
|
||||
}
|
||||
+23
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "SELECT path FROM resource WHERE workspace_id = $1 AND path = ANY($2) AND resource_type = 'ai_agent'",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "path",
|
||||
"type_info": "Varchar"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Text",
|
||||
"TextArray"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
false
|
||||
]
|
||||
},
|
||||
"hash": "307d5b797e51122dbf087e5dc9f33235a349731828d500c74a28b577469e2624"
|
||||
}
|
||||
+15
@@ -0,0 +1,15 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "INSERT INTO gcp_trigger (\n gcp_resource_path, project_id, topic_id, subscription_id, delivery_type,\n delivery_config, path, script_path, is_flow, workspace_id, edited_by,\n edited_at, extra_perms, server_id, last_server_ping, error,\n subscription_mode, error_handler_path, error_handler_args, retry,\n auto_acknowledge_msg, ack_deadline, mode, permissioned_as, labels\n )\n SELECT\n gcp_resource_path, project_id, topic_id, subscription_id, delivery_type,\n delivery_config, path, script_path, is_flow, $1, edited_by,\n edited_at, extra_perms, NULL, NULL, NULL,\n subscription_mode, error_handler_path, error_handler_args, retry,\n auto_acknowledge_msg, ack_deadline, 'disabled'::TRIGGER_MODE, permissioned_as, labels\n FROM gcp_trigger WHERE workspace_id = $2",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Varchar",
|
||||
"Text"
|
||||
]
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "30aa4958c8ed0459c9f83e3963bb413248cbb3bab75b0249a9fc4c2fd02da0bc"
|
||||
}
|
||||
+23
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "SELECT path FROM eval_dataset WHERE path LIKE $1 AND workspace_id = $2",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "path",
|
||||
"type_info": "Varchar"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Text",
|
||||
"Text"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
false
|
||||
]
|
||||
},
|
||||
"hash": "30b8590939bf3d6770cabca9f4ab3527e750d85d3b8cc0eadef55461de4a2687"
|
||||
}
|
||||
+23
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "SELECT version FROM resource_version WHERE workspace_id = $1 AND path = $2\n ORDER BY version DESC LIMIT 1",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "version",
|
||||
"type_info": "Int8"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Text",
|
||||
"Text"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
false
|
||||
]
|
||||
},
|
||||
"hash": "316e7c86082b6ee2864b88674cfac5b5b34c5de851f1c6f832b868101edcb052"
|
||||
}
|
||||
+52
@@ -0,0 +1,52 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "\n INSERT INTO gcp_trigger (\n gcp_resource_path,\n project_id,\n subscription_id,\n topic_id,\n delivery_type,\n delivery_config,\n workspace_id,\n path,\n script_path,\n is_flow,\n permissioned_as,\n mode,\n edited_by,\n error_handler_path,\n error_handler_args,\n retry,\n auto_acknowledge_msg,\n ack_deadline\n )\n VALUES (\n $1,\n $2,\n $3,\n $4,\n $5,\n $6,\n $7,\n $8,\n $9,\n $10,\n $11,\n $12,\n $13,\n $14,\n $15,\n $16,\n $17,\n $18\n )",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Varchar",
|
||||
"Varchar",
|
||||
"Varchar",
|
||||
"Varchar",
|
||||
{
|
||||
"Custom": {
|
||||
"name": "delivery_mode",
|
||||
"kind": {
|
||||
"Enum": [
|
||||
"push",
|
||||
"pull"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"Jsonb",
|
||||
"Varchar",
|
||||
"Varchar",
|
||||
"Varchar",
|
||||
"Bool",
|
||||
"Varchar",
|
||||
{
|
||||
"Custom": {
|
||||
"name": "trigger_mode",
|
||||
"kind": {
|
||||
"Enum": [
|
||||
"enabled",
|
||||
"disabled",
|
||||
"suspended"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"Varchar",
|
||||
"Varchar",
|
||||
"Jsonb",
|
||||
"Jsonb",
|
||||
"Bool",
|
||||
"Int4"
|
||||
]
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "3382ba7f9f437d0def24f8cee2a58402f2e627baa957e0fed3ca09ea9ca869b3"
|
||||
}
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "DELETE FROM eval_case\n WHERE workspace_id = $1 AND dataset_path = $2 AND NOT (id = ANY($3))",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Text",
|
||||
"Text",
|
||||
"UuidArray"
|
||||
]
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "34fbb2b141ad691e0cdc55bf2ebd6974e02a7c81d91217d3f74e03113edc9b0a"
|
||||
}
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "UPDATE eval_experiment SET created_by = $1 WHERE created_by = $2 AND workspace_id = $3",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Varchar",
|
||||
"Text",
|
||||
"Text"
|
||||
]
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "3ca2f72d2917f48644cb79daba9858f472be9f1bb8e504cc3dc2eb998e3f7b91"
|
||||
}
|
||||
+23
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "SELECT status::text AS \"status!\" FROM v2_job_completed WHERE id = $1 AND workspace_id = $2",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "status!",
|
||||
"type_info": "Text"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Uuid",
|
||||
"Text"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
null
|
||||
]
|
||||
},
|
||||
"hash": "44a39475ba202bd5852b666335308360569aae717eff3fe9cc9253a261ae2fa7"
|
||||
}
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "UPDATE eval_dataset SET path = REGEXP_REPLACE(path,'u/' || $2 || '/(.*)','u/' || $1 || '/\\1') WHERE path LIKE ('u/' || $2 || '/%') AND workspace_id = $3",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Text",
|
||||
"Text",
|
||||
"Text"
|
||||
]
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "4550ae568abf23045259f95195bf1a35f5dc8f6828c2f3df51520683af48ebbd"
|
||||
}
|
||||
+24
@@ -0,0 +1,24 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "WITH updated AS (\n UPDATE eval_dataset SET path = REGEXP_REPLACE(path, 'u/' || $2 || '/(.*)', $1 || '/\\1')\n WHERE path LIKE ('u/' || $2 || '/%') AND workspace_id = $3\n RETURNING 1\n ) SELECT COUNT(*) FROM updated",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "count",
|
||||
"type_info": "Int8"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Text",
|
||||
"Text",
|
||||
"Text"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
null
|
||||
]
|
||||
},
|
||||
"hash": "467ed4d282af003dd3b0d9542caec73feba3141f7345ffd1a6554fe44a6a9171"
|
||||
}
|
||||
+15
@@ -0,0 +1,15 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "INSERT INTO eval_dataset (workspace_id, path, summary, scorers, extra_perms, created_at, created_by, edited_at, edited_by)\n SELECT $2, path, summary, scorers, extra_perms, created_at, created_by, edited_at, edited_by\n FROM eval_dataset WHERE workspace_id = $1",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Text",
|
||||
"Varchar"
|
||||
]
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "479427dc09ebacb80cb20d553e93a577bb85b389ab056909a326212565a338bc"
|
||||
}
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "UPDATE eval_case SET created_by = $1 WHERE created_by = $2 AND workspace_id = $3",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Varchar",
|
||||
"Text",
|
||||
"Text"
|
||||
]
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "4be42f447e10e420f2a909579398b01b5175e14eee47ca39fe63242a78da767e"
|
||||
}
|
||||
+59
@@ -0,0 +1,59 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "SELECT path, summary, scorers, created_at, created_by, edited_at, edited_by\n FROM eval_dataset WHERE workspace_id = $1 AND path = $2 FOR UPDATE",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "path",
|
||||
"type_info": "Varchar"
|
||||
},
|
||||
{
|
||||
"ordinal": 1,
|
||||
"name": "summary",
|
||||
"type_info": "Varchar"
|
||||
},
|
||||
{
|
||||
"ordinal": 2,
|
||||
"name": "scorers",
|
||||
"type_info": "Jsonb"
|
||||
},
|
||||
{
|
||||
"ordinal": 3,
|
||||
"name": "created_at",
|
||||
"type_info": "Timestamptz"
|
||||
},
|
||||
{
|
||||
"ordinal": 4,
|
||||
"name": "created_by",
|
||||
"type_info": "Varchar"
|
||||
},
|
||||
{
|
||||
"ordinal": 5,
|
||||
"name": "edited_at",
|
||||
"type_info": "Timestamptz"
|
||||
},
|
||||
{
|
||||
"ordinal": 6,
|
||||
"name": "edited_by",
|
||||
"type_info": "Varchar"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Text",
|
||||
"Text"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
false,
|
||||
true,
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
false
|
||||
]
|
||||
},
|
||||
"hash": "4bf7f1a0fd87e79bb789cee06a5da68e465ce7ce67e80770f22a36a9e8320657"
|
||||
}
|
||||
+5
-5
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "SELECT path, value FROM resource_version WHERE workspace_id = $1 AND id = $2",
|
||||
"query": "SELECT path, scorers FROM eval_dataset WHERE workspace_id = $1 AND path = ANY($2)",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
@@ -10,20 +10,20 @@
|
||||
},
|
||||
{
|
||||
"ordinal": 1,
|
||||
"name": "value",
|
||||
"name": "scorers",
|
||||
"type_info": "Jsonb"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Text",
|
||||
"Int8"
|
||||
"TextArray"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
false,
|
||||
true
|
||||
false
|
||||
]
|
||||
},
|
||||
"hash": "e74e283951aa87627a46aa8286819cf7aa4fecbde17bc7a67ee6f49c427cee9e"
|
||||
"hash": "4e4a9c7b6e95f81101a68a6cc6d29fe4424eaed36d5ac9e4cd0a2a9d8b878eeb"
|
||||
}
|
||||
+47
@@ -0,0 +1,47 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "SELECT id, input, expected, created_at, created_by\n FROM eval_case\n WHERE workspace_id = $1 AND dataset_path = $2\n ORDER BY created_at, id",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "id",
|
||||
"type_info": "Uuid"
|
||||
},
|
||||
{
|
||||
"ordinal": 1,
|
||||
"name": "input",
|
||||
"type_info": "Jsonb"
|
||||
},
|
||||
{
|
||||
"ordinal": 2,
|
||||
"name": "expected",
|
||||
"type_info": "Jsonb"
|
||||
},
|
||||
{
|
||||
"ordinal": 3,
|
||||
"name": "created_at",
|
||||
"type_info": "Timestamptz"
|
||||
},
|
||||
{
|
||||
"ordinal": 4,
|
||||
"name": "created_by",
|
||||
"type_info": "Varchar"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Text",
|
||||
"Text"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
false,
|
||||
false,
|
||||
true,
|
||||
false,
|
||||
false
|
||||
]
|
||||
},
|
||||
"hash": "5329ce41bbdc36698ea059fec5f05ded7dfe1c64517289ad67028001417d30a3"
|
||||
}
|
||||
+1
-1
@@ -15,7 +15,7 @@
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
true
|
||||
null
|
||||
]
|
||||
},
|
||||
"hash": "5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55"
|
||||
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "UPDATE eval_experiment_case c SET job_id = j.id\n FROM v2_job j\n WHERE j.parent_job = $3 AND j.workspace_id = $2\n AND (j.args -> 'iter' -> 'value' ->> 'case_id')::uuid = c.case_id\n AND c.experiment_id = $1 AND c.job_id IS NULL",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Uuid",
|
||||
"Text",
|
||||
"Uuid"
|
||||
]
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "5d3560d7a42f86436fec362a790cea189be47e344d11871a90faa9220d509646"
|
||||
}
|
||||
+76
@@ -0,0 +1,76 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "SELECT ordinal, case_id, input, expected, job_id, subject_version,\n subject_draft_hash, output, answered, status\n FROM eval_experiment_case\n WHERE experiment_id = $1 ORDER BY ordinal",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "ordinal",
|
||||
"type_info": "Int4"
|
||||
},
|
||||
{
|
||||
"ordinal": 1,
|
||||
"name": "case_id",
|
||||
"type_info": "Uuid"
|
||||
},
|
||||
{
|
||||
"ordinal": 2,
|
||||
"name": "input",
|
||||
"type_info": "Jsonb"
|
||||
},
|
||||
{
|
||||
"ordinal": 3,
|
||||
"name": "expected",
|
||||
"type_info": "Jsonb"
|
||||
},
|
||||
{
|
||||
"ordinal": 4,
|
||||
"name": "job_id",
|
||||
"type_info": "Uuid"
|
||||
},
|
||||
{
|
||||
"ordinal": 5,
|
||||
"name": "subject_version",
|
||||
"type_info": "Int8"
|
||||
},
|
||||
{
|
||||
"ordinal": 6,
|
||||
"name": "subject_draft_hash",
|
||||
"type_info": "Varchar"
|
||||
},
|
||||
{
|
||||
"ordinal": 7,
|
||||
"name": "output",
|
||||
"type_info": "Text"
|
||||
},
|
||||
{
|
||||
"ordinal": 8,
|
||||
"name": "answered",
|
||||
"type_info": "Bool"
|
||||
},
|
||||
{
|
||||
"ordinal": 9,
|
||||
"name": "status",
|
||||
"type_info": "Varchar"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Uuid"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
true,
|
||||
true,
|
||||
true,
|
||||
true,
|
||||
true,
|
||||
true,
|
||||
true
|
||||
]
|
||||
},
|
||||
"hash": "5d5186bb17092425664d8c4f92ca6bfb8b08420cc2b2a52e90064d6a12f86113"
|
||||
}
|
||||
+22
@@ -0,0 +1,22 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "SELECT DISTINCT experiment_id FROM eval_score\n WHERE experiment_id = ANY($1) AND score IS NULL AND error IS NULL\n AND NOT not_applicable",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "experiment_id",
|
||||
"type_info": "Uuid"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"UuidArray"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
false
|
||||
]
|
||||
},
|
||||
"hash": "5dcaea907b1ebb2854becc44c00eac5838995dce3d2a696769e00fb74c34288f"
|
||||
}
|
||||
+26
@@ -0,0 +1,26 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "UPDATE eval_case SET input = $4, expected = $5\n WHERE workspace_id = $1 AND dataset_path = $2 AND id = $3\n RETURNING id",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "id",
|
||||
"type_info": "Uuid"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Text",
|
||||
"Text",
|
||||
"Uuid",
|
||||
"Jsonb",
|
||||
"Jsonb"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
false
|
||||
]
|
||||
},
|
||||
"hash": "61773dd5d5952607eddfacb1717126a87ee5dc4bd6664eaf31c2c0096fea08db"
|
||||
}
|
||||
+74
@@ -0,0 +1,74 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "SELECT\n (CASE $3::text\n WHEN 'day' THEN day::text\n WHEN 'user' THEN email\n ELSE ''\n END) AS \"key!\",\n provider AS \"provider!\",\n model AS \"model!\",\n SUM(input_tokens)::bigint AS \"input_tokens!\",\n SUM(cache_read_tokens)::bigint AS \"cache_read_tokens!\",\n SUM(cache_write_tokens)::bigint AS \"cache_write_tokens!\",\n SUM(output_tokens)::bigint AS \"output_tokens!\",\n SUM(reported_cost_nano_usd)::bigint AS \"reported_cost_nano_usd\",\n SUM(requests)::bigint AS \"requests!\"\n FROM ai_token_usage\n WHERE workspace_id = $1 AND day > CURRENT_DATE - $2::int\n AND ($5::text IS NULL OR email = $5)\n GROUP BY 1, provider, model\n ORDER BY SUM(input_tokens + cache_read_tokens + cache_write_tokens + output_tokens) DESC\n LIMIT $4",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "key!",
|
||||
"type_info": "Text"
|
||||
},
|
||||
{
|
||||
"ordinal": 1,
|
||||
"name": "provider!",
|
||||
"type_info": "Varchar"
|
||||
},
|
||||
{
|
||||
"ordinal": 2,
|
||||
"name": "model!",
|
||||
"type_info": "Varchar"
|
||||
},
|
||||
{
|
||||
"ordinal": 3,
|
||||
"name": "input_tokens!",
|
||||
"type_info": "Int8"
|
||||
},
|
||||
{
|
||||
"ordinal": 4,
|
||||
"name": "cache_read_tokens!",
|
||||
"type_info": "Int8"
|
||||
},
|
||||
{
|
||||
"ordinal": 5,
|
||||
"name": "cache_write_tokens!",
|
||||
"type_info": "Int8"
|
||||
},
|
||||
{
|
||||
"ordinal": 6,
|
||||
"name": "output_tokens!",
|
||||
"type_info": "Int8"
|
||||
},
|
||||
{
|
||||
"ordinal": 7,
|
||||
"name": "reported_cost_nano_usd",
|
||||
"type_info": "Int8"
|
||||
},
|
||||
{
|
||||
"ordinal": 8,
|
||||
"name": "requests!",
|
||||
"type_info": "Int8"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Text",
|
||||
"Int4",
|
||||
"Text",
|
||||
"Int8",
|
||||
"Text"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
null,
|
||||
false,
|
||||
false,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null
|
||||
]
|
||||
},
|
||||
"hash": "6a17a1dfeb75808e5d0726f1f8bd573168161abe350b047402cda4ac4a4d13e4"
|
||||
}
|
||||
+23
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "SELECT scorers FROM eval_dataset WHERE workspace_id = $1 AND path = $2 FOR UPDATE",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "scorers",
|
||||
"type_info": "Jsonb"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Text",
|
||||
"Text"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
false
|
||||
]
|
||||
},
|
||||
"hash": "6bc9d682aabdf8e79beb693e0090d07f3514628a4970b5f25ddf3920ba5e9c53"
|
||||
}
|
||||
+14
@@ -0,0 +1,14 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "DELETE FROM eval_experiment WHERE id = $1",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Uuid"
|
||||
]
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "77db9fcef0d3f9c9eb9edba6c0a54469259621218616ef64a4923f3396b183a2"
|
||||
}
|
||||
+20
@@ -0,0 +1,20 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "INSERT INTO eval_experiment_case\n (experiment_id, ordinal, case_id, input, expected, subject_version,\n subject_draft_hash)\n SELECT $1, ordinal, case_id, input, expected, subject_version, subject_draft_hash\n FROM UNNEST($2::int[], $3::uuid[], $4::jsonb[], $5::jsonb[], $6::bigint[], $7::text[])\n AS t(ordinal, case_id, input, expected, subject_version, subject_draft_hash)",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Uuid",
|
||||
"Int4Array",
|
||||
"UuidArray",
|
||||
"JsonbArray",
|
||||
"JsonbArray",
|
||||
"Int8Array",
|
||||
"TextArray"
|
||||
]
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "790e90a3aca284fd060aac049fcf27d2e39781fccc4dd8419f934085d5453fc7"
|
||||
}
|
||||
+26
@@ -0,0 +1,26 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "INSERT INTO eval_case\n (workspace_id, dataset_path, input, expected, created_by, created_at)\n VALUES ($1, $2, $3, $4, $5, clock_timestamp())\n RETURNING id",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "id",
|
||||
"type_info": "Uuid"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Varchar",
|
||||
"Varchar",
|
||||
"Jsonb",
|
||||
"Jsonb",
|
||||
"Varchar"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
false
|
||||
]
|
||||
},
|
||||
"hash": "7a14c6815a7acc912fd8836191263a373b5c427a0b2f48ca775fb8f5a0096e18"
|
||||
}
|
||||
+60
@@ -0,0 +1,60 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "SELECT s.experiment_id AS \"experiment_id!\", s.scorer_id AS \"scorer_id!\",\n avg(s.score) AS mean,\n count(s.score) AS \"scored!\",\n count(*) FILTER (WHERE s.error IS NOT NULL) AS \"failed!\",\n count(*) FILTER (WHERE t.pass_if IS NOT NULL AND s.score >= t.pass_if)\n AS \"passed!\",\n bool_or(t.pass_if IS NOT NULL) AS \"has_threshold!\"\n FROM eval_score s\n JOIN unnest($1::uuid[], $2::text[], $3::float8[])\n AS t(experiment_id, scorer_id, pass_if)\n ON t.experiment_id = s.experiment_id AND t.scorer_id = s.scorer_id\n GROUP BY s.experiment_id, s.scorer_id",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "experiment_id!",
|
||||
"type_info": "Uuid"
|
||||
},
|
||||
{
|
||||
"ordinal": 1,
|
||||
"name": "scorer_id!",
|
||||
"type_info": "Varchar"
|
||||
},
|
||||
{
|
||||
"ordinal": 2,
|
||||
"name": "mean",
|
||||
"type_info": "Float8"
|
||||
},
|
||||
{
|
||||
"ordinal": 3,
|
||||
"name": "scored!",
|
||||
"type_info": "Int8"
|
||||
},
|
||||
{
|
||||
"ordinal": 4,
|
||||
"name": "failed!",
|
||||
"type_info": "Int8"
|
||||
},
|
||||
{
|
||||
"ordinal": 5,
|
||||
"name": "passed!",
|
||||
"type_info": "Int8"
|
||||
},
|
||||
{
|
||||
"ordinal": 6,
|
||||
"name": "has_threshold!",
|
||||
"type_info": "Bool"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"UuidArray",
|
||||
"TextArray",
|
||||
"Float8Array"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
false,
|
||||
false,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null
|
||||
]
|
||||
},
|
||||
"hash": "7b040feaa84e85fff1a5ad1ddfc3d57b0f69b05f96ffafae0325cf3ff04f0f40"
|
||||
}
|
||||
-14
@@ -1,14 +0,0 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "\n DELETE FROM \n capture\n WHERE \n id = $1\n ",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Int8"
|
||||
]
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "7d5a161d916cb8d1485f8d72e6b4044f505f49edcd2ebeee4a77eae737f48795"
|
||||
}
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "UPDATE eval_experiment_case\n SET subject_version = $3, subject_draft_hash = NULL\n WHERE experiment_id = $1 AND subject_draft_hash = $2",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Uuid",
|
||||
"Text",
|
||||
"Int8"
|
||||
]
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "7f373cf063907999580d0541a1a4a216e3887fdc0a9bab0747a447c01a4b37a5"
|
||||
}
|
||||
+64
@@ -0,0 +1,64 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "SELECT ordinal, scorer_id, score, reason, checks, error, not_applicable, definition\n FROM eval_score WHERE experiment_id = $1",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "ordinal",
|
||||
"type_info": "Int4"
|
||||
},
|
||||
{
|
||||
"ordinal": 1,
|
||||
"name": "scorer_id",
|
||||
"type_info": "Varchar"
|
||||
},
|
||||
{
|
||||
"ordinal": 2,
|
||||
"name": "score",
|
||||
"type_info": "Float8"
|
||||
},
|
||||
{
|
||||
"ordinal": 3,
|
||||
"name": "reason",
|
||||
"type_info": "Text"
|
||||
},
|
||||
{
|
||||
"ordinal": 4,
|
||||
"name": "checks",
|
||||
"type_info": "Jsonb"
|
||||
},
|
||||
{
|
||||
"ordinal": 5,
|
||||
"name": "error",
|
||||
"type_info": "Text"
|
||||
},
|
||||
{
|
||||
"ordinal": 6,
|
||||
"name": "not_applicable",
|
||||
"type_info": "Bool"
|
||||
},
|
||||
{
|
||||
"ordinal": 7,
|
||||
"name": "definition",
|
||||
"type_info": "Varchar"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Uuid"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
false,
|
||||
false,
|
||||
true,
|
||||
true,
|
||||
true,
|
||||
true,
|
||||
false,
|
||||
false
|
||||
]
|
||||
},
|
||||
"hash": "80a5dd06cc5b9f7cb0bcd3b8cdd92c62d40a4ab97fab0f2dd330bd7629c0a632"
|
||||
}
|
||||
+9
-3
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "SELECT id, created_at, created_by FROM resource_version\n WHERE workspace_id = $1 AND path = $2 ORDER BY id DESC LIMIT $3",
|
||||
"query": "SELECT id, version, created_at, created_by FROM resource_version\n WHERE workspace_id = $1 AND path = $2 ORDER BY id DESC LIMIT $3",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
@@ -10,11 +10,16 @@
|
||||
},
|
||||
{
|
||||
"ordinal": 1,
|
||||
"name": "version",
|
||||
"type_info": "Int8"
|
||||
},
|
||||
{
|
||||
"ordinal": 2,
|
||||
"name": "created_at",
|
||||
"type_info": "Timestamptz"
|
||||
},
|
||||
{
|
||||
"ordinal": 2,
|
||||
"ordinal": 3,
|
||||
"name": "created_by",
|
||||
"type_info": "Varchar"
|
||||
}
|
||||
@@ -27,10 +32,11 @@
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
true
|
||||
]
|
||||
},
|
||||
"hash": "e27ed86394e6568afdf7a1dd72edd7943c87fdf1d05fba193cbe575cb7281db5"
|
||||
"hash": "82b2e8383ae7e345e45fb8a1b2bb362439f5b12a3dd3290a8dab17ccf9c391f3"
|
||||
}
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "UPDATE eval_dataset SET edited_by = $1 WHERE edited_by = $2 AND workspace_id = $3",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Varchar",
|
||||
"Text",
|
||||
"Text"
|
||||
]
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "858b5fe344d79913921d8c05e9e6219919f438e9a3ce0c3e996ea61d9b4f2ef8"
|
||||
}
|
||||
+24
@@ -0,0 +1,24 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "SELECT value FROM resource_version\n WHERE version = $1 AND workspace_id = $2 AND path = $3",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "value",
|
||||
"type_info": "Jsonb"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Int8",
|
||||
"Text",
|
||||
"Text"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
true
|
||||
]
|
||||
},
|
||||
"hash": "864184467477e73a45935bf9439ae60da7f9ecc12f8f98b926345a70f941845b"
|
||||
}
|
||||
+29
@@ -0,0 +1,29 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "SELECT r.value AS \"value: sqlx::types::Json<serde_json::Value>\",\n (SELECT version FROM resource_version v\n WHERE v.workspace_id = r.workspace_id AND v.path = r.path\n ORDER BY v.version DESC LIMIT 1) AS version\n FROM resource r\n WHERE r.workspace_id = $1 AND r.path = $2 AND r.resource_type = 'ai_agent'",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "value: sqlx::types::Json<serde_json::Value>",
|
||||
"type_info": "Jsonb"
|
||||
},
|
||||
{
|
||||
"ordinal": 1,
|
||||
"name": "version",
|
||||
"type_info": "Int8"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Text",
|
||||
"Text"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
true,
|
||||
null
|
||||
]
|
||||
},
|
||||
"hash": "897e8da49e4a5c3efdfb2c36fc7aff3d53432da9ef57f8a775850efcd50a965c"
|
||||
}
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "UPDATE eval_dataset SET extra_perms = extra_perms - ('u/' || $2) || jsonb_build_object(('u/' || $1), extra_perms->('u/' || $2)) WHERE extra_perms ? ('u/' || $2) AND workspace_id = $3",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Text",
|
||||
"Text",
|
||||
"Text"
|
||||
]
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "8c319ac3eb2a289a6709ea1ea1c164f1a56e328902dd46fd9b68d801c16f4ffa"
|
||||
}
|
||||
+58
@@ -0,0 +1,58 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "SELECT path, summary, scorers, created_at, created_by,\n edited_at, edited_by\n FROM eval_dataset WHERE workspace_id = $1 ORDER BY path",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "path",
|
||||
"type_info": "Varchar"
|
||||
},
|
||||
{
|
||||
"ordinal": 1,
|
||||
"name": "summary",
|
||||
"type_info": "Varchar"
|
||||
},
|
||||
{
|
||||
"ordinal": 2,
|
||||
"name": "scorers",
|
||||
"type_info": "Jsonb"
|
||||
},
|
||||
{
|
||||
"ordinal": 3,
|
||||
"name": "created_at",
|
||||
"type_info": "Timestamptz"
|
||||
},
|
||||
{
|
||||
"ordinal": 4,
|
||||
"name": "created_by",
|
||||
"type_info": "Varchar"
|
||||
},
|
||||
{
|
||||
"ordinal": 5,
|
||||
"name": "edited_at",
|
||||
"type_info": "Timestamptz"
|
||||
},
|
||||
{
|
||||
"ordinal": 6,
|
||||
"name": "edited_by",
|
||||
"type_info": "Varchar"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Text"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
false,
|
||||
true,
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
false
|
||||
]
|
||||
},
|
||||
"hash": "9071eff54395ca39809a5d55d1e3fedb64cfafa9fe74cd9446eb6ee526c2d9a0"
|
||||
}
|
||||
+23
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "SELECT COALESCE(s.flow_status, c.flow_status) -> 'modules' AS modules\n FROM v2_job j\n LEFT JOIN v2_job_status s ON s.id = j.id\n LEFT JOIN v2_job_completed c ON c.id = j.id\n WHERE j.id = $1 AND j.workspace_id = $2",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "modules",
|
||||
"type_info": "Jsonb"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Uuid",
|
||||
"Text"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
null
|
||||
]
|
||||
},
|
||||
"hash": "97a701e16c0ce4b8c6a1394c2a71b8ae45e16c11c8003c5f64c9f53d0e7226aa"
|
||||
}
|
||||
+23
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "SELECT gcp_resource_path IS NULL FROM gcp_trigger WHERE workspace_id = $1 AND path = $2 FOR UPDATE",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "?column?",
|
||||
"type_info": "Bool"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Text",
|
||||
"Text"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
null
|
||||
]
|
||||
},
|
||||
"hash": "9900abe4112310ccdace809aecf1bf6f5d3d350db5e6cd98557d0280c56ff0fa"
|
||||
}
|
||||
+23
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "SELECT j.id AS \"id!\" FROM v2_job j\n LEFT JOIN v2_job_completed c ON c.id = j.id AND c.workspace_id = $2\n WHERE j.id = ANY($1) AND j.workspace_id = $2 AND c.id IS NULL",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "id!",
|
||||
"type_info": "Uuid"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"UuidArray",
|
||||
"Text"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
false
|
||||
]
|
||||
},
|
||||
"hash": "9b1ad1bbf0c2dca3ce1cc9433c35db87e3101bcbcb0fa359885ca3807c1c9ca2"
|
||||
}
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "UPDATE eval_dataset SET scorers = COALESCE((\n SELECT jsonb_agg(\n CASE WHEN elem->>'path' LIKE ('u/' || $2 || '/%')\n THEN jsonb_set(elem, '{path}', to_jsonb(REGEXP_REPLACE(elem->>'path','u/' || $2 || '/(.*)','u/' || $1 || '/\\1')))\n ELSE elem END)\n FROM jsonb_array_elements(scorers) elem), '[]'::jsonb)\n WHERE workspace_id = $3\n AND EXISTS (SELECT 1 FROM jsonb_array_elements(scorers) e WHERE e->>'path' LIKE ('u/' || $2 || '/%'))",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Text",
|
||||
"Text",
|
||||
"Text"
|
||||
]
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "9d3ca755b323330033eb891ac7162a53440a5c713b58e6c4a467f6f0d5d7e37c"
|
||||
}
|
||||
+22
@@ -0,0 +1,22 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "\n WITH RECURSIVE chain AS (\n SELECT id, parent_workspace_id, is_dev_workspace, 0 AS depth\n FROM workspace WHERE id = $1\n UNION ALL\n SELECT w.id, w.parent_workspace_id, w.is_dev_workspace, chain.depth + 1\n FROM workspace w\n JOIN chain ON w.id = chain.parent_workspace_id\n WHERE chain.depth < 20\n )\n SELECT id AS \"id!\" FROM chain\n WHERE parent_workspace_id IS NULL OR is_dev_workspace\n ORDER BY depth LIMIT 1\n ",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "id!",
|
||||
"type_info": "Varchar"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Text"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
null
|
||||
]
|
||||
},
|
||||
"hash": "a09eb290f2fc7c2de0e4b009e7ba7046de30385983ea39bf203aa8dc20b7aca1"
|
||||
}
|
||||
+15
@@ -0,0 +1,15 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "INSERT INTO eval_case (workspace_id, dataset_path, input, expected, created_at, created_by)\n SELECT $2, dataset_path, input, expected, created_at, created_by\n FROM eval_case WHERE workspace_id = $1",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Text",
|
||||
"Varchar"
|
||||
]
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "a4c842e395714346d5178190793d8ae66f891421b72fc6adfa8a38060ca2f3fa"
|
||||
}
|
||||
+53
@@ -0,0 +1,53 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "SELECT s.ordinal, s.scorer_id, c.job_id AS \"job_id!\", d.status::text AS status,\n c.answered, (j.id IS NOT NULL) AS \"job_exists!\"\n FROM eval_score s\n JOIN eval_experiment_case c\n ON c.experiment_id = s.experiment_id AND c.ordinal = s.ordinal\n LEFT JOIN v2_job j ON j.id = c.job_id AND j.workspace_id = $2\n LEFT JOIN v2_job_completed d ON d.id = c.job_id AND d.workspace_id = $2\n WHERE s.experiment_id = $1 AND s.score IS NULL AND s.error IS NULL\n AND NOT s.not_applicable AND c.job_id IS NOT NULL",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "ordinal",
|
||||
"type_info": "Int4"
|
||||
},
|
||||
{
|
||||
"ordinal": 1,
|
||||
"name": "scorer_id",
|
||||
"type_info": "Varchar"
|
||||
},
|
||||
{
|
||||
"ordinal": 2,
|
||||
"name": "job_id!",
|
||||
"type_info": "Uuid"
|
||||
},
|
||||
{
|
||||
"ordinal": 3,
|
||||
"name": "status",
|
||||
"type_info": "Text"
|
||||
},
|
||||
{
|
||||
"ordinal": 4,
|
||||
"name": "answered",
|
||||
"type_info": "Bool"
|
||||
},
|
||||
{
|
||||
"ordinal": 5,
|
||||
"name": "job_exists!",
|
||||
"type_info": "Bool"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Uuid",
|
||||
"Text"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
false,
|
||||
false,
|
||||
true,
|
||||
null,
|
||||
true,
|
||||
null
|
||||
]
|
||||
},
|
||||
"hash": "a5e8cf0e559742330d67d36d9ddb75ef7de6e0ad74e0cc6c4056b782155b86c7"
|
||||
}
|
||||
+27
@@ -0,0 +1,27 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "UPDATE eval_dataset\n SET path = COALESCE($6, path), summary = COALESCE($3, summary),\n scorers = COALESCE($4, scorers), edited_at = now(), edited_by = $5\n WHERE workspace_id = $1 AND path = $2\n RETURNING path",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "path",
|
||||
"type_info": "Varchar"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Text",
|
||||
"Text",
|
||||
"Varchar",
|
||||
"Jsonb",
|
||||
"Varchar",
|
||||
"Varchar"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
false
|
||||
]
|
||||
},
|
||||
"hash": "a7b589b8d5cded97905bc24412ad8f3f0339a2a088243f045a212f9357205407"
|
||||
}
|
||||
+1
-1
@@ -72,7 +72,7 @@
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
false,
|
||||
true,
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
|
||||
+29
@@ -0,0 +1,29 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "SELECT id, run_job_id FROM eval_experiment WHERE workspace_id = $1 AND id = $2",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "id",
|
||||
"type_info": "Uuid"
|
||||
},
|
||||
{
|
||||
"ordinal": 1,
|
||||
"name": "run_job_id",
|
||||
"type_info": "Uuid"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Text",
|
||||
"Uuid"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
false,
|
||||
false
|
||||
]
|
||||
},
|
||||
"hash": "b2db4f32c615a99db7af23729682340328da39ac763bb26b6bcf82027fad48e5"
|
||||
}
|
||||
+3
-3
@@ -98,12 +98,12 @@
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
false,
|
||||
true,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
false,
|
||||
false
|
||||
true,
|
||||
true
|
||||
]
|
||||
},
|
||||
"hash": "b8e732a03969666444f73397ac153d23ec2af084b2f93da24c920532c1916384"
|
||||
|
||||
+29
@@ -0,0 +1,29 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "SELECT status::text AS \"status!\", duration_ms FROM v2_job_completed\n WHERE id = $1 AND workspace_id = $2",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "status!",
|
||||
"type_info": "Text"
|
||||
},
|
||||
{
|
||||
"ordinal": 1,
|
||||
"name": "duration_ms",
|
||||
"type_info": "Int8"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Uuid",
|
||||
"Text"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
null,
|
||||
false
|
||||
]
|
||||
},
|
||||
"hash": "bac36542b16b687a823067c013e3728f1cabe3774f581401514f522ea7522de5"
|
||||
}
|
||||
+54
@@ -0,0 +1,54 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "SELECT e.subject, e.run_number, e.run_job_id, e.created_at,\n e.created_by,\n (SELECT count(*) FROM eval_experiment_case c WHERE c.experiment_id = e.id)\n AS \"case_count!\"\n FROM eval_experiment e\n WHERE e.workspace_id = $1 AND e.dataset_path = $2 AND e.id = $3",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "subject",
|
||||
"type_info": "Jsonb"
|
||||
},
|
||||
{
|
||||
"ordinal": 1,
|
||||
"name": "run_number",
|
||||
"type_info": "Int4"
|
||||
},
|
||||
{
|
||||
"ordinal": 2,
|
||||
"name": "run_job_id",
|
||||
"type_info": "Uuid"
|
||||
},
|
||||
{
|
||||
"ordinal": 3,
|
||||
"name": "created_at",
|
||||
"type_info": "Timestamptz"
|
||||
},
|
||||
{
|
||||
"ordinal": 4,
|
||||
"name": "created_by",
|
||||
"type_info": "Varchar"
|
||||
},
|
||||
{
|
||||
"ordinal": 5,
|
||||
"name": "case_count!",
|
||||
"type_info": "Int8"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Text",
|
||||
"Text",
|
||||
"Uuid"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
null
|
||||
]
|
||||
},
|
||||
"hash": "bbce2221f5724016543a5ac1db7fa74b8b35375dfb0f5d39b398369468ad0774"
|
||||
}
|
||||
+66
@@ -0,0 +1,66 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "SELECT e.id, e.dataset_path, e.subject, e.run_number, e.run_job_id, e.created_at,\n e.created_by,\n (SELECT count(*) FROM eval_experiment_case c WHERE c.experiment_id = e.id)\n AS \"case_count!\"\n FROM eval_experiment e\n JOIN eval_dataset d ON d.workspace_id = e.workspace_id AND d.path = e.dataset_path\n WHERE e.workspace_id = $1\n AND ($3::text IS NULL OR e.subject ->> 'path' = $3)\n ORDER BY e.created_at DESC\n LIMIT $2",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "id",
|
||||
"type_info": "Uuid"
|
||||
},
|
||||
{
|
||||
"ordinal": 1,
|
||||
"name": "dataset_path",
|
||||
"type_info": "Varchar"
|
||||
},
|
||||
{
|
||||
"ordinal": 2,
|
||||
"name": "subject",
|
||||
"type_info": "Jsonb"
|
||||
},
|
||||
{
|
||||
"ordinal": 3,
|
||||
"name": "run_number",
|
||||
"type_info": "Int4"
|
||||
},
|
||||
{
|
||||
"ordinal": 4,
|
||||
"name": "run_job_id",
|
||||
"type_info": "Uuid"
|
||||
},
|
||||
{
|
||||
"ordinal": 5,
|
||||
"name": "created_at",
|
||||
"type_info": "Timestamptz"
|
||||
},
|
||||
{
|
||||
"ordinal": 6,
|
||||
"name": "created_by",
|
||||
"type_info": "Varchar"
|
||||
},
|
||||
{
|
||||
"ordinal": 7,
|
||||
"name": "case_count!",
|
||||
"type_info": "Int8"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Text",
|
||||
"Int8",
|
||||
"Text"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
null
|
||||
]
|
||||
},
|
||||
"hash": "bdb7185233941d1472a55c8ade3f61c82248f3f2c9a69c458bff3978ffd17379"
|
||||
}
|
||||
+54
@@ -0,0 +1,54 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "SELECT j.id, j.args AS \"args: sqlx::types::Json<Box<RawValue>>\",\n c.result AS \"result: sqlx::types::Json<Box<RawValue>>\",\n c.status::text AS status, c.duration_ms,\n s.schema AS \"schema: sqlx::types::Json<Box<RawValue>>\"\n FROM v2_job j\n LEFT JOIN v2_job_completed c ON c.id = j.id\n LEFT JOIN script s ON s.workspace_id = j.workspace_id AND s.hash = j.runnable_id\n WHERE j.id = ANY($1) AND j.workspace_id = $2 AND j.parent_job = $3",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "id",
|
||||
"type_info": "Uuid"
|
||||
},
|
||||
{
|
||||
"ordinal": 1,
|
||||
"name": "args: sqlx::types::Json<Box<RawValue>>",
|
||||
"type_info": "Jsonb"
|
||||
},
|
||||
{
|
||||
"ordinal": 2,
|
||||
"name": "result: sqlx::types::Json<Box<RawValue>>",
|
||||
"type_info": "Jsonb"
|
||||
},
|
||||
{
|
||||
"ordinal": 3,
|
||||
"name": "status",
|
||||
"type_info": "Text"
|
||||
},
|
||||
{
|
||||
"ordinal": 4,
|
||||
"name": "duration_ms",
|
||||
"type_info": "Int8"
|
||||
},
|
||||
{
|
||||
"ordinal": 5,
|
||||
"name": "schema: sqlx::types::Json<Box<RawValue>>",
|
||||
"type_info": "Json"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"UuidArray",
|
||||
"Text",
|
||||
"Uuid"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
false,
|
||||
true,
|
||||
true,
|
||||
null,
|
||||
false,
|
||||
true
|
||||
]
|
||||
},
|
||||
"hash": "be17a65f144cc21e849c7f8cfbf9d7271b953dda42e4ba09864da7bda731e752"
|
||||
}
|
||||
+30
@@ -0,0 +1,30 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "SELECT\n (SELECT hash FROM script\n WHERE path = $1 AND workspace_id = $2 AND deleted = false\n AND lock IS NOT NULL AND lock_error_logs IS NULL\n ORDER BY created_at DESC LIMIT 1) AS hash,\n (SELECT lock IS NULL AND lock_error_logs IS NULL\n AND created_at > now() - make_interval(secs => $3) FROM script\n WHERE path = $1 AND workspace_id = $2 AND deleted = false\n ORDER BY created_at DESC LIMIT 1) AS pending_lock",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "hash",
|
||||
"type_info": "Int8"
|
||||
},
|
||||
{
|
||||
"ordinal": 1,
|
||||
"name": "pending_lock",
|
||||
"type_info": "Bool"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Text",
|
||||
"Text",
|
||||
"Float8"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
null,
|
||||
null
|
||||
]
|
||||
},
|
||||
"hash": "bf4e451d2748c880ce5bd189037dda592a71717644c9c4206721de16cff59392"
|
||||
}
|
||||
+53
@@ -0,0 +1,53 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "\n UPDATE\n gcp_trigger\n SET\n gcp_resource_path = $1,\n subscription_id = $2,\n topic_id = $3,\n delivery_type = $4,\n delivery_config = $5,\n is_flow = $6,\n edited_by = $7,\n permissioned_as = $8,\n script_path = $9,\n path = $10,\n mode = $11,\n edited_at = now(),\n error = NULL,\n server_id = NULL,\n error_handler_path = $14,\n error_handler_args = $15,\n retry = $16,\n auto_acknowledge_msg = $17,\n ack_deadline = $18,\n project_id = $19\n WHERE\n workspace_id = $12 AND\n path = $13\n ",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Varchar",
|
||||
"Varchar",
|
||||
"Varchar",
|
||||
{
|
||||
"Custom": {
|
||||
"name": "delivery_mode",
|
||||
"kind": {
|
||||
"Enum": [
|
||||
"push",
|
||||
"pull"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"Jsonb",
|
||||
"Bool",
|
||||
"Varchar",
|
||||
"Varchar",
|
||||
"Varchar",
|
||||
"Varchar",
|
||||
{
|
||||
"Custom": {
|
||||
"name": "trigger_mode",
|
||||
"kind": {
|
||||
"Enum": [
|
||||
"enabled",
|
||||
"disabled",
|
||||
"suspended"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"Text",
|
||||
"Text",
|
||||
"Varchar",
|
||||
"Jsonb",
|
||||
"Jsonb",
|
||||
"Bool",
|
||||
"Int4",
|
||||
"Varchar"
|
||||
]
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "c7de1fe77fa2be4701b7185a3b3c91e2db92d2d2f45fd07a354f30e6248f21e6"
|
||||
}
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "UPDATE eval_experiment SET subject = jsonb_set(subject, '{path}', to_jsonb(REGEXP_REPLACE(subject->>'path','u/' || $2 || '/(.*)','u/' || $1 || '/\\1'))) WHERE subject->>'path' LIKE ('u/' || $2 || '/%') AND workspace_id = $3",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Text",
|
||||
"Text",
|
||||
"Text"
|
||||
]
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "d10efb37765ac9a7f2e15f71dbbc02ec917b6ecb014f938e68ced3cc7fe9dc86"
|
||||
}
|
||||
+23
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "UPDATE eval_experiment_case SET status = $2, answered = false\n WHERE experiment_id = $1 AND job_id IS NULL AND status IS NULL\n RETURNING ordinal",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "ordinal",
|
||||
"type_info": "Int4"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Uuid",
|
||||
"Varchar"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
false
|
||||
]
|
||||
},
|
||||
"hash": "d5cdb1121f2a414c0a70f4f7e4ae630bad0715bbbb3597cb1b14e41abe9fec14"
|
||||
}
|
||||
+100
@@ -0,0 +1,100 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "\n SELECT\n gcp_resource_path AS \"gcp_resource_path: _\",\n script_path,\n is_flow,\n mode as \"mode: _\",\n workspace_id,\n path,\n edited_by,\n permissioned_as,\n delivery_config AS \"delivery_config: _\",\n retry as \"retry: _\",\n error_handler_path,\n error_handler_args as \"error_handler_args: _\"\n FROM\n gcp_trigger\n WHERE\n workspace_id = $1 AND\n path = $2 AND\n delivery_type = 'push'::DELIVERY_MODE\n ",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "gcp_resource_path: _",
|
||||
"type_info": "Varchar"
|
||||
},
|
||||
{
|
||||
"ordinal": 1,
|
||||
"name": "script_path",
|
||||
"type_info": "Varchar"
|
||||
},
|
||||
{
|
||||
"ordinal": 2,
|
||||
"name": "is_flow",
|
||||
"type_info": "Bool"
|
||||
},
|
||||
{
|
||||
"ordinal": 3,
|
||||
"name": "mode: _",
|
||||
"type_info": {
|
||||
"Custom": {
|
||||
"name": "trigger_mode",
|
||||
"kind": {
|
||||
"Enum": [
|
||||
"enabled",
|
||||
"disabled",
|
||||
"suspended"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"ordinal": 4,
|
||||
"name": "workspace_id",
|
||||
"type_info": "Varchar"
|
||||
},
|
||||
{
|
||||
"ordinal": 5,
|
||||
"name": "path",
|
||||
"type_info": "Varchar"
|
||||
},
|
||||
{
|
||||
"ordinal": 6,
|
||||
"name": "edited_by",
|
||||
"type_info": "Varchar"
|
||||
},
|
||||
{
|
||||
"ordinal": 7,
|
||||
"name": "permissioned_as",
|
||||
"type_info": "Varchar"
|
||||
},
|
||||
{
|
||||
"ordinal": 8,
|
||||
"name": "delivery_config: _",
|
||||
"type_info": "Jsonb"
|
||||
},
|
||||
{
|
||||
"ordinal": 9,
|
||||
"name": "retry: _",
|
||||
"type_info": "Jsonb"
|
||||
},
|
||||
{
|
||||
"ordinal": 10,
|
||||
"name": "error_handler_path",
|
||||
"type_info": "Varchar"
|
||||
},
|
||||
{
|
||||
"ordinal": 11,
|
||||
"name": "error_handler_args: _",
|
||||
"type_info": "Jsonb"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Text",
|
||||
"Text"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
true,
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
true,
|
||||
true,
|
||||
true,
|
||||
true
|
||||
]
|
||||
},
|
||||
"hash": "d7a8660d977b1cb516871dff0f7e376d2cbb9a35ba68a5641ebce31aad10f513"
|
||||
}
|
||||
+18
@@ -0,0 +1,18 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "INSERT INTO eval_case\n (workspace_id, dataset_path, input, expected, created_by, created_at)\n VALUES ($1, $2, $3, $4, $5, clock_timestamp())",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Varchar",
|
||||
"Varchar",
|
||||
"Jsonb",
|
||||
"Jsonb",
|
||||
"Varchar"
|
||||
]
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "db19932d940b2467147eb2c13b059f2ec33b0c83b5c4a332a4b6e54d2591767c"
|
||||
}
|
||||
+23
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "SELECT path FROM eval_dataset WHERE workspace_id = $1 AND path = $2",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "path",
|
||||
"type_info": "Varchar"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Text",
|
||||
"Text"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
false
|
||||
]
|
||||
},
|
||||
"hash": "db63d41718c50e264a949885d2d7c13719ae650e9c788ebf125627b785fb9ee0"
|
||||
}
|
||||
+49
@@ -0,0 +1,49 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "SELECT id, input, expected, created_at, created_by\n FROM eval_case\n WHERE workspace_id = $1 AND dataset_path = $2\n ORDER BY created_at, id\n LIMIT $3 OFFSET $4",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "id",
|
||||
"type_info": "Uuid"
|
||||
},
|
||||
{
|
||||
"ordinal": 1,
|
||||
"name": "input",
|
||||
"type_info": "Jsonb"
|
||||
},
|
||||
{
|
||||
"ordinal": 2,
|
||||
"name": "expected",
|
||||
"type_info": "Jsonb"
|
||||
},
|
||||
{
|
||||
"ordinal": 3,
|
||||
"name": "created_at",
|
||||
"type_info": "Timestamptz"
|
||||
},
|
||||
{
|
||||
"ordinal": 4,
|
||||
"name": "created_by",
|
||||
"type_info": "Varchar"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Text",
|
||||
"Text",
|
||||
"Int8",
|
||||
"Int8"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
false,
|
||||
false,
|
||||
true,
|
||||
false,
|
||||
false
|
||||
]
|
||||
},
|
||||
"hash": "e3c080b84f50622e0a74524111ae292c958e71293695ce18f9905af4dd940495"
|
||||
}
|
||||
+23
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "SELECT args AS \"args: sqlx::types::Json<Box<RawValue>>\" FROM v2_job\n WHERE id = $1 AND workspace_id = $2",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "args: sqlx::types::Json<Box<RawValue>>",
|
||||
"type_info": "Jsonb"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Uuid",
|
||||
"Text"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
true
|
||||
]
|
||||
},
|
||||
"hash": "e6d7e9779eaa6e584b613675ae8602c405fe620937a4563a17db80219a2930d2"
|
||||
}
|
||||
+17
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "INSERT INTO eval_score (experiment_id, ordinal, scorer_id, definition)\n SELECT $1, ordinal, scorer_id, definition\n FROM UNNEST($2::int[], $3::text[], $4::text[]) AS t(ordinal, scorer_id, definition)\n ON CONFLICT (experiment_id, ordinal, scorer_id)\n DO UPDATE SET definition = EXCLUDED.definition, score = NULL, reason = NULL,\n checks = NULL, error = NULL, not_applicable = false",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Uuid",
|
||||
"Int4Array",
|
||||
"TextArray",
|
||||
"TextArray"
|
||||
]
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "e719e98cceef1383f882632c9398dd950ad0b1f30cde9c0392384c9a490290b9"
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user